This is an automated email from the ASF dual-hosted git repository. spmallette pushed a commit to branch TINKERPOP-2279 in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 3ba60a584d5b733fb18e5b099214e5a2b2440155 Author: Stephen Mallette <[email protected]> AuthorDate: Fri Aug 9 14:11:13 2019 -0400 Major refactoring for nullable writes. Added more serializer support with metrics, class, and strategies. Still a bit rough but more tests are passing. --- gremlin-python/pom.xml | 9 +- .../jython/gremlin_python/driver/serializer.py | 2 +- .../jython/gremlin_python/process/strategies.py | 5 +- .../jython/gremlin_python/process/traversal.py | 3 +- .../src/main/jython/gremlin_python/statics.py | 15 +- .../gremlin_python/structure/io/graphbinaryV1.py | 272 ++++++++++++++------- .../gremlin_python/structure/io/graphsonV3d0.py | 1 + gremlin-python/src/main/jython/radish/terrain.py | 2 + gremlin-python/src/main/jython/tests/conftest.py | 6 +- .../tests/driver/test_driver_remote_connection.py | 129 +--------- 10 files changed, 211 insertions(+), 233 deletions(-) diff --git a/gremlin-python/pom.xml b/gremlin-python/pom.xml index f3c3c85..48a45b0 100644 --- a/gremlin-python/pom.xml +++ b/gremlin-python/pom.xml @@ -419,13 +419,20 @@ limitations under the License. <env key="PYTHONPATH" value=""/> <arg line="setup.py install"/> </exec> - <!-- run for graphson 3.0 --> + <!-- run for graphson 2.0 --> <exec executable="env/bin/radish" dir="${project.build.directory}/python2" failonerror="true"> <env key="PYTHONPATH" value=""/> <env key="PYTHONIOENCODING" value="utf-8:surrogateescape"/> <arg line="-f dots -e -t -b ${project.build.directory}/python2/radish ${project.basedir}/../gremlin-test/features/ --user-data="serializer=application/vnd.gremlin-v3.0+json""/> <!-- -no-line-jump --> </exec> + <!-- run for graphbinary 1.0 --> + <exec executable="env/bin/radish" dir="${project.build.directory}/python2" + failonerror="true"> + <env key="PYTHONPATH" value=""/> + <env key="PYTHONIOENCODING" value="utf-8:surrogateescape"/> + <arg line="-f dots -e -t -b ${project.build.directory}/python2/radish ${project.basedir}/../gremlin-test/features/ --user-data="serializer=application/vnd.graphbinary-v1.0""/> <!-- -no-line-jump --> + </exec> </target> </configuration> </execution> diff --git a/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py b/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py index f805220..a95d292 100644 --- a/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py +++ b/gremlin-python/src/main/jython/gremlin_python/driver/serializer.py @@ -170,7 +170,7 @@ class GraphSONSerializersV3d0(GraphSONMessageSerializer): super(GraphSONSerializersV3d0, self).__init__(reader, writer, version) -class GraphBinaryMessageSerializerV1(object): +class GraphBinarySerializersV1(object): DEFAULT_READER_CLASS = graphbinaryV1.GraphBinaryReader DEFAULT_WRITER_CLASS = graphbinaryV1.GraphBinaryWriter DEFAULT_VERSION = b"application/vnd.graphbinary-v1.0" diff --git a/gremlin-python/src/main/jython/gremlin_python/process/strategies.py b/gremlin-python/src/main/jython/gremlin_python/process/strategies.py index cbd5e08..186e303 100644 --- a/gremlin-python/src/main/jython/gremlin_python/process/strategies.py +++ b/gremlin-python/src/main/jython/gremlin_python/process/strategies.py @@ -64,8 +64,9 @@ class PartitionStrategy(TraversalStrategy): class SubgraphStrategy(TraversalStrategy): + def __init__(self, vertices=None, edges=None, vertex_properties=None): - TraversalStrategy.__init__(self) + TraversalStrategy.__init__(self, fqcn="org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy") if vertices is not None: self.configuration["vertices"] = vertices if edges is not None: @@ -77,7 +78,7 @@ class SubgraphStrategy(TraversalStrategy): class VertexProgramStrategy(TraversalStrategy): def __init__(self, graph_computer=None, workers=None, persist=None, result=None, vertices=None, edges=None, configuration=None): - TraversalStrategy.__init__(self) + TraversalStrategy.__init__(self, fqcn="org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy") if graph_computer is not None: self.configuration["graphComputer"] = graph_computer if workers is not None: diff --git a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py index 4fb9f8c..5a73702 100644 --- a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py +++ b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py @@ -581,8 +581,9 @@ class TraversalStrategies(object): class TraversalStrategy(object): - def __init__(self, strategy_name=None, configuration=None): + def __init__(self, strategy_name=None, configuration=None, fqcn=None): self.strategy_name = type(self).__name__ if strategy_name is None else strategy_name + self.fqcn = fqcn self.configuration = {} if configuration is None else configuration def apply(self, traversal): diff --git a/gremlin-python/src/main/jython/gremlin_python/statics.py b/gremlin-python/src/main/jython/gremlin_python/statics.py index 2f06b97..012f52f 100644 --- a/gremlin-python/src/main/jython/gremlin_python/statics.py +++ b/gremlin-python/src/main/jython/gremlin_python/statics.py @@ -43,19 +43,20 @@ else: from types import TypeType from types import ListType from types import DictType + from types import TypeType class timestamp(float): """ In Python a timestamp is simply a float. This dummy class (similar to long), allows users to wrap a float - in a GLV script to make sure the value is serialized as a GraphSON timestamp. + in a GLV script to make sure the value is serialized as a Gremlin timestamp. """ pass class SingleByte(int): """ - Provides a way to pass a single byte via GraphSON. + Provides a way to pass a single byte via Gremlin. """ def __new__(cls, b): if -128 <= b < 128: @@ -66,7 +67,7 @@ class SingleByte(int): class SingleChar(str): """ - Provides a way to pass a single character via GraphSON. + Provides a way to pass a single character via Gremlin. """ def __new__(cls, c): if len(b) == 1: @@ -75,6 +76,14 @@ class SingleChar(str): raise ValueError("string must contain a single character") +class GremlinType(object): + """ + Provides a way to represent a "Java class" for Gremlin. + """ + def __init__(self, gremlin_type): + self.gremlin_type = gremlin_type + + staticMethods = {} staticEnums = {} default_lambda_language = "gremlin-python" diff --git a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py index f34a3d4..dd829cf 100644 --- a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py +++ b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphbinaryV1.py @@ -36,7 +36,7 @@ from isodate import parse_duration, duration_isoformat from gremlin_python import statics from gremlin_python.statics import FloatType, FunctionType, IntType, LongType, TypeType, DictType, ListType, SetType, \ - SingleByte, ByteBufferType, SingleChar + SingleByte, ByteBufferType, SingleChar, GremlinType from gremlin_python.process.traversal import Barrier, Binding, Bytecode, Cardinality, Column, Direction, Operator, \ Order, Pick, Pop, P, Scope, TextP, Traversal, Traverser, \ TraversalStrategy, T @@ -58,7 +58,7 @@ class DataType(Enum): string = 0x03 date = 0x04 timestamp = 0x05 - clazz = 0x06 #todo + clazz = 0x06 double = 0x07 float = 0x08 list = 0x09 @@ -68,7 +68,7 @@ class DataType(Enum): edge = 0x0d path = 0x0e property = 0x0f - graph = 0x10 + graph = 0x10 # no graph object in python yet vertex = 0x11 vertexproperty = 0x12 barrier = 0x13 @@ -93,14 +93,17 @@ class DataType(Enum): short = 0x26 #todo? boolean = 0x27 textp = 0x28 - traversalstrategy = 0x29 #todo + traversalstrategy = 0x29 bulkset = 0x2a - tree = 0x2b #todo - metrics = 0x2c #todo - traversalmetrics = 0x2d #todo + tree = 0x2b # no tree object in Python yet + metrics = 0x2c + traversalmetrics = 0x2d custom = 0x00 #todo +NULL_BYTES = [DataType.null.value, 0x01] + + class GraphBinaryTypeType(type): def __new__(mcs, name, bases, dct): cls = super(GraphBinaryTypeType, mcs).__new__(mcs, name, bases, dct) @@ -172,11 +175,11 @@ class _GraphBinaryTypeIO(object): "filter_": "filter", "id_": "id", "max_": "max", "min_": "min", "sum_": "sum"} @classmethod - def as_bytes(cls, graphbin_type=None, size=None, *args): + def as_bytes(cls, graphbin_type=None, size=None, nullable=True, *args): ba = bytearray() if graphbin_type is None else bytearray([graphbin_type.value]) - # todo: empty value flag just hardcoded in - ba.extend(struct.pack(">b", 0)) + if nullable: + ba.extend(struct.pack(">b", 0)) if size is not None: ba.extend(struct.pack(">i", size)) @@ -215,7 +218,7 @@ class _GraphBinaryTypeIO(object): def is_null(cls, buff, reader, else_opt, nullable=True): return None if nullable and buff.read(1)[0] == 0x01 else else_opt(buff, reader) - def dictify(self, obj, writer, as_value=False): + def dictify(self, obj, writer, as_value=False, nullable=True): raise NotImplementedError() def objectify(self, d, reader, nullable=True): @@ -229,12 +232,12 @@ class LongIO(_GraphBinaryTypeIO): byte_format = ">q" @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): if obj < -9223372036854775808 or obj > 9223372036854775807: raise Exception("TODO: don't forget bigint") else: return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, struct.pack(cls.byte_format, obj)) + None, nullable, struct.pack(cls.byte_format, obj)) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -258,7 +261,7 @@ class DateIO(_GraphBinaryTypeIO): graphbinary_type = DataType.date @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): try: timestamp_seconds = calendar.timegm(obj.utctimetuple()) pts = timestamp_seconds * 1e3 + getattr(obj, 'microsecond', 0) / 1e3 @@ -267,7 +270,7 @@ class DateIO(_GraphBinaryTypeIO): ts = int(round(pts)) return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, struct.pack(">q", ts)) + None, nullable, struct.pack(">q", ts)) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -283,18 +286,18 @@ class TimestampIO(_GraphBinaryTypeIO): graphbinary_type = DataType.timestamp @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): # Java timestamp expects milliseconds integer - Have to use int because of legacy Python ts = int(round(obj * 1000)) return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, struct.pack(">q", ts)) + None, nullable, struct.pack(">q", ts)) @classmethod def objectify(cls, buff, reader, nullable=True): # Python timestamp expects seconds return cls.is_null(buff, reader, lambda b, r: statics.timestamp(struct.unpack(">q", b.read(8))[0] / 1000.0), nullable) - + def _long_bits_to_double(bits): return struct.unpack('d', struct.pack('Q', bits))[0] @@ -313,19 +316,19 @@ class FloatIO(LongIO): byte_format = ">f" @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): if math.isnan(obj): return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, struct.pack(cls.byte_format, NAN)) + None, nullable, struct.pack(cls.byte_format, NAN)) elif math.isinf(obj) and obj > 0: return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, struct.pack(cls.byte_format, POSITIVE_INFINITY)) + None, nullable, struct.pack(cls.byte_format, POSITIVE_INFINITY)) elif math.isinf(obj) and obj < 0: return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, struct.pack(cls.byte_format, NEGATIVE_INFINITY)) + None, nullable, struct.pack(cls.byte_format, NEGATIVE_INFINITY)) else: return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, struct.pack(cls.byte_format, obj)) + None, nullable, struct.pack(cls.byte_format, obj)) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -346,23 +349,15 @@ class DoubleIO(FloatIO): return cls.is_null(buff, reader, lambda b, r: struct.unpack(cls.byte_format, b.read(8))[0], nullable) -class TypeSerializer(_GraphBinaryTypeIO): - python_type = TypeType - - @classmethod - def dictify(cls, typ, writer, as_value=False): - return writer.toDict(typ()) - - class StringIO(_GraphBinaryTypeIO): python_type = str graphbinary_type = DataType.string @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - len(obj), obj.encode("utf-8")) + len(obj), nullable, obj.encode("utf-8")) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -375,13 +370,13 @@ class ListIO(_GraphBinaryTypeIO): graphbinary_type = DataType.list @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): list_data = bytearray() for item in obj: list_data.extend(writer.writeObject(item)) return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - len(obj), list_data) + len(obj), nullable, list_data) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -414,14 +409,14 @@ class MapIO(_GraphBinaryTypeIO): graphbinary_type = DataType.map @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): map_data = bytearray() for k, v in obj.items(): map_data.extend(writer.writeObject(k)) map_data.extend(writer.writeObject(v)) return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - len(obj), map_data) + len(obj), nullable, map_data) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -446,9 +441,9 @@ class UuidIO(_GraphBinaryTypeIO): graphbinary_type = DataType.uuid @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), - None, obj.bytes) + None, nullable, obj.bytes) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -461,7 +456,7 @@ class EdgeIO(_GraphBinaryTypeIO): graphbinary_type = DataType.edge @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(writer.writeObject(obj.id)) ba.extend(cls.string_as_bytes(obj.label)) @@ -469,9 +464,9 @@ class EdgeIO(_GraphBinaryTypeIO): ba.extend(cls.string_as_bytes(obj.inV.label)) ba.extend(writer.writeObject(obj.outV.id)) ba.extend(cls.string_as_bytes(obj.outV.label)) - ba.extend([DataType.null.value]) - ba.extend([DataType.null.value]) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + ba.extend(NULL_BYTES) + ba.extend(NULL_BYTES) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -483,7 +478,7 @@ class EdgeIO(_GraphBinaryTypeIO): edgelbl = cls.read_string(b) edge = Edge(edgeid, Vertex(r.readObject(b), cls.read_string(b)), edgelbl, Vertex(r.readObject(b), cls.read_string(b))) - b.read(2) + b.read(4) return edge @@ -493,11 +488,11 @@ class PathIO(_GraphBinaryTypeIO): graphbinary_type = DataType.path @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(writer.writeObject(obj.labels)) ba.extend(writer.writeObject(obj.objects)) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -510,12 +505,12 @@ class PropertyIO(_GraphBinaryTypeIO): graphbinary_type = DataType.property @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(cls.string_as_bytes(obj.key)) ba.extend(writer.writeObject(obj.value)) - ba.extend([DataType.null.value]) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + ba.extend(NULL_BYTES) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -524,7 +519,7 @@ class PropertyIO(_GraphBinaryTypeIO): @classmethod def _read_property(cls, b, r): p = Property(cls.read_string(b), r.readObject(b), None) - b.read(1) + b.read(2) return p @@ -534,7 +529,7 @@ class TinkerGraphIO(_GraphBinaryTypeIO): graphbinary_type = DataType.graph @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): raise AttributeError("TinkerGraph serialization is not currently supported by gremlin-python") @classmethod @@ -548,12 +543,12 @@ class VertexIO(_GraphBinaryTypeIO): graphbinary_type = DataType.vertex @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(writer.writeObject(obj.id)) ba.extend(cls.string_as_bytes(obj.label)) - ba.extend([DataType.null.value]) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + ba.extend(NULL_BYTES) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -562,7 +557,7 @@ class VertexIO(_GraphBinaryTypeIO): @classmethod def _read_vertex(cls, b, r): vertex = Vertex(r.readObject(b), cls.read_string(b)) - b.read(1) + b.read(2) return vertex @@ -572,14 +567,14 @@ class VertexPropertyIO(_GraphBinaryTypeIO): graphbinary_type = DataType.vertexproperty @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(writer.writeObject(obj.id)) ba.extend(cls.string_as_bytes(obj.label)) ba.extend(writer.writeObject(obj.value)) - ba.extend([DataType.null.value]) - ba.extend([DataType.null.value]) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + ba.extend(NULL_BYTES) + ba.extend(NULL_BYTES) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -588,18 +583,17 @@ class VertexPropertyIO(_GraphBinaryTypeIO): @classmethod def _read_vertexproperty(cls, b, r): vp = VertexProperty(r.readObject(b), cls.read_string(b), r.readObject(b), None) - b.read(1) - b.read(1) + b.read(4) return vp class _EnumIO(_GraphBinaryTypeIO): @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(cls.string_as_bytes(cls.unmangleKeyword(str(obj.name)))) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -652,11 +646,11 @@ class BindingIO(_GraphBinaryTypeIO): graphbinary_type = DataType.binding @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(cls.string_as_bytes(obj.key)) ba.extend(writer.writeObject(obj.value)) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -668,7 +662,7 @@ class BytecodeIO(_GraphBinaryTypeIO): graphbinary_type = DataType.bytecode @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(struct.pack(">i", len(obj.step_instructions))) for inst in obj.step_instructions: @@ -684,9 +678,12 @@ class BytecodeIO(_GraphBinaryTypeIO): ba.extend(cls.string_as_bytes(inst_name)) ba.extend(struct.pack(">i", len(inst_args))) for arg in inst_args: - ba.extend(writer.writeObject(arg)) + if isinstance(arg, TypeType): + ba.extend(writer.writeObject(GremlinType(arg().fqcn))) + else: + ba.extend(writer.writeObject(arg)) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -729,7 +726,7 @@ class LambdaIO(_GraphBinaryTypeIO): graphbinary_type = DataType.lambda_ @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() if as_value else bytearray([cls.graphbinary_type.value]) lambda_result = obj() script = lambda_result if isinstance(lambda_result, str) else lambda_result[0] @@ -748,7 +745,7 @@ class LambdaIO(_GraphBinaryTypeIO): ba.extend(cls.string_as_bytes(script_cleaned)) ba.extend(struct.pack(">i", script_args)) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) class PIO(_GraphBinaryTypeIO): @@ -756,16 +753,25 @@ class PIO(_GraphBinaryTypeIO): python_type = P @classmethod - def dictify(cls, obj, writer, as_value=False): - ba = bytearray() if as_value else bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False, nullable=True): + ba = bytearray() ba.extend(cls.string_as_bytes(obj.operator)) - additional = [writer.writeObject(obj.value), writer.writeObject(obj.other)] \ - if obj.other is not None else [writer.writeObject(obj.value)] - ba.extend(struct.pack(">i", len(additional))) - for a in additional: - ba.extend(a) + + args = [] + if obj.other is None: + if isinstance(obj.value, ListType): + args = obj.value + else: + args.append(obj.value) + else: + args.append(obj.value) + args.append(obj.other) + + ba.extend(struct.pack(">i", len(args))) + for a in args: + ba.extend(writer.writeObject(a)) - return ba + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) class ScopeIO(_EnumIO): @@ -783,11 +789,11 @@ class TraverserIO(_GraphBinaryTypeIO): python_type = Traverser @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): ba = bytearray() ba.extend(struct.pack(">q", obj.bulk)) ba.extend(writer.writeObject(obj.object)) - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, ba) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -805,8 +811,8 @@ class ByteIO(_GraphBinaryTypeIO): graphbinary_type = DataType.byte @classmethod - def dictify(cls, obj, writer, as_value=False): - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, struct.pack(">b", obj)) + def dictify(cls, obj, writer, as_value=False, nullable=True): + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, struct.pack(">b", obj)) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -820,8 +826,8 @@ class ByteBufferIO(_GraphBinaryTypeIO): graphbinary_type = DataType.bytebuffer @classmethod - def dictify(cls, obj, writer, as_value=False): - return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), len(obj), obj) + def dictify(cls, obj, writer, as_value=False, nullable=True): + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), len(obj), nullable, obj) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -838,9 +844,9 @@ class BooleanIO(_GraphBinaryTypeIO): graphbinary_type = DataType.boolean @classmethod - def dictify(cls, obj, writer, as_value=False): + def dictify(cls, obj, writer, as_value=False, nullable=True): return cls.as_bytes(cls.write_as_value( - cls.graphbinary_type, as_value), None, struct.pack(">b", 0x01 if obj else 0x00)) + cls.graphbinary_type, as_value), None, nullable, struct.pack(">b", 0x01 if obj else 0x00)) @classmethod def objectify(cls, buff, reader, nullable=True): @@ -854,16 +860,25 @@ class TextPIO(_GraphBinaryTypeIO): python_type = TextP @classmethod - def dictify(cls, obj, writer, as_value=False): - ba = bytearray() if as_value else bytearray([cls.graphbinary_type.value]) + def dictify(cls, obj, writer, as_value=False, nullable=True): + ba = bytearray() ba.extend(cls.string_as_bytes(obj.operator)) - additional = [writer.writeObject(obj.value), writer.writeObject(obj.other)] \ - if obj.other is not None else [writer.writeObject(obj.value)] - ba.extend(struct.pack(">i", len(additional))) - for a in additional: - ba.extend(a) - return ba + args = [] + if obj.other is None: + if isinstance(obj.value, ListType): + args = obj.value + else: + args.append(obj.value) + else: + args.append(obj.value) + args.append(obj.other) + + ba.extend(struct.pack(">i", len(args))) + for a in args: + ba.extend(writer.writeObject(a)) + + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) class BulkSetIO(_GraphBinaryTypeIO): @@ -886,3 +901,72 @@ class BulkSetIO(_GraphBinaryTypeIO): size = size - 1 return the_list + + +class MetricsIO(_GraphBinaryTypeIO): + + graphbinary_type = DataType.metrics + + @classmethod + def objectify(cls, buff, reader, nullable=True): + return cls.is_null(buff, reader, cls._read_metrics, nullable) + + @classmethod + def _read_metrics(cls, b, r): + metricid = cls.read_string(b) + name = cls.read_string(b) + duration = r.toObject(b, DataType.long, nullable=False) + counts = r.toObject(b, DataType.map, nullable=False) + annotations = r.toObject(b, DataType.map, nullable=False) + metrics = r.toObject(b, DataType.list, nullable=False) + + return {"id": metricid, + "name": name, + "dur": duration, + "counts": counts, + "annotations": annotations, + "metrics": metrics} + + +class TraversalMetricsIO(_GraphBinaryTypeIO): + + graphbinary_type = DataType.traversalmetrics + + @classmethod + def objectify(cls, buff, reader, nullable=True): + return cls.is_null(buff, reader, cls._read_traversalmetrics, nullable) + + @classmethod + def _read_traversalmetrics(cls, b, r): + duration = r.toObject(b, DataType.long, nullable=False) + metrics = r.toObject(b, DataType.list, nullable=False) + + return {"dur": duration, + "metrics": metrics} + + +class ClassIO(_GraphBinaryTypeIO): + graphbinary_type = DataType.clazz + python_type = GremlinType + + @classmethod + def dictify(cls, obj, writer, as_value=False, nullable=True): + return cls.as_bytes(cls.write_as_value( + cls.graphbinary_type, as_value), None, nullable, StringIO.dictify(obj.gremlin_type, writer, True, False)) + + +class TraversalStrategyIO(_GraphBinaryTypeIO): + graphbinary_type = DataType.traversalstrategy + python_type = TraversalStrategy + + @classmethod + def dictify(cls, obj, writer, as_value=False, nullable=True): + ba = bytearray() + ba.extend(ClassIO.dictify(GremlinType(obj.fqcn), writer, True, False)) + conf = {k: cls._convert(v) for k, v in obj.configuration.items()} + ba.extend(MapIO.dictify(conf, writer, True, False)) + return cls.as_bytes(cls.write_as_value(cls.graphbinary_type, as_value), None, nullable, ba) + + @classmethod + def _convert(cls, v): + return v.bytecode if isinstance(v, Traversal) else v diff --git a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py index 8c3cd9f..10a74da 100644 --- a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py +++ b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py @@ -711,6 +711,7 @@ class TDeserializer(_GraphSONTypeIO): def objectify(cls, d, reader): return T[d] + class TraversalMetricsDeserializer(_GraphSONTypeIO): graphson_type = "g:TraversalMetrics" diff --git a/gremlin-python/src/main/jython/radish/terrain.py b/gremlin-python/src/main/jython/radish/terrain.py index 9122eac..245fd9b 100644 --- a/gremlin-python/src/main/jython/radish/terrain.py +++ b/gremlin-python/src/main/jython/radish/terrain.py @@ -89,6 +89,8 @@ def __create_remote(server_graph_name): if world.config.user_data["serializer"] == "application/vnd.gremlin-v3.0+json": s = serializer.GraphSONSerializersV3d0() + elif world.config.user_data["serializer"] == "application/vnd.graphbinary-v1.0": + s = serializer.GraphBinarySerializersV1() else: raise ValueError('serializer not found - ' + world.config.user_data["serializer"]) diff --git a/gremlin-python/src/main/jython/tests/conftest.py b/gremlin-python/src/main/jython/tests/conftest.py index fb31c31..9597f4d 100644 --- a/gremlin-python/src/main/jython/tests/conftest.py +++ b/gremlin-python/src/main/jython/tests/conftest.py @@ -29,7 +29,7 @@ from gremlin_python.driver.driver_remote_connection import ( from gremlin_python.driver.protocol import GremlinServerWSProtocol from gremlin_python.driver.serializer import ( GraphSONMessageSerializer, GraphSONSerializersV2d0, GraphSONSerializersV3d0, - GraphBinaryMessageSerializerV1) + GraphBinarySerializersV1) from gremlin_python.driver.tornado.transport import TornadoTransport gremlin_server_host = "localhost" @@ -88,7 +88,7 @@ def remote_connection(request): try: if request.param == 'graphbinaryv1': remote_conn = DriverRemoteConnection(gremlin_server_url, 'gmodern', - message_serializer=serializer.GraphBinaryMessageSerializerV1()) + message_serializer=serializer.GraphBinarySerializersV1()) elif request.param == 'graphsonv2': remote_conn = DriverRemoteConnection(gremlin_server_url, 'gmodern', message_serializer=serializer.GraphSONSerializersV2d0()) @@ -129,4 +129,4 @@ def graphson_serializer_v3(request): @pytest.fixture def graphbinary_serializer_v1(request): - return GraphBinaryMessageSerializerV1() + return GraphBinarySerializersV1() diff --git a/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py b/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py index 2ecd113..b2b4a7d 100644 --- a/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py +++ b/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py @@ -16,7 +16,6 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' -import pytest from tornado import ioloop, gen @@ -25,7 +24,6 @@ from gremlin_python.statics import long from gremlin_python.driver.driver_remote_connection import ( DriverRemoteConnection) from gremlin_python.process.traversal import Traverser -from gremlin_python.process.traversal import TraversalStrategy from gremlin_python.process.traversal import P from gremlin_python.process.graph_traversal import __ from gremlin_python.process.anonymous_traversal import traversal @@ -38,7 +36,6 @@ __author__ = 'Marko A. Rodriguez (http://markorodriguez.com)' class TestDriverRemoteConnection(object): def test_traversals(self, remote_connection): statics.load_statics(globals()) - assert "remoteconnection[ws://localhost:45940/gremlin,gmodern]" == str(remote_connection) g = traversal().withRemote(remote_connection) assert long(6) == g.V().count().toList()[0] @@ -60,8 +57,7 @@ class TestDriverRemoteConnection(object): assert 4 == g.V()[2:].count().next() assert 2 == g.V()[:2].count().next() # # - results = g.withSideEffect('a', ['josh', 'peter']).V(1).out('created').in_('created').values('name').where( - within('a')).toList() + results = g.withSideEffect('a', ['josh', 'peter']).V(1).out('created').in_('created').values('name').where(P.within('a')).toList() assert 2 == len(results) assert 'josh' in results assert 'peter' in results @@ -94,7 +90,6 @@ class TestDriverRemoteConnection(object): def test_iteration(self, remote_connection): statics.load_statics(globals()) - assert "remoteconnection[ws://localhost:45940/gremlin,gmodern]" == str(remote_connection) g = traversal().withRemote(remote_connection) t = g.V().count() @@ -134,16 +129,6 @@ class TestDriverRemoteConnection(object): statics.load_statics(globals()) # g = traversal().withRemote(remote_connection). \ - withStrategies(TraversalStrategy("SubgraphStrategy", - {"vertices": __.hasLabel("person"), - "edges": __.hasLabel("created")})) - assert 4 == g.V().count().next() - assert 0 == g.E().count().next() - assert 1 == g.V().label().dedup().count().next() - assert 4 == g.V().filter(lambda: ("lambda x: True", "gremlin-python")).count().next() - assert "person" == g.V().label().dedup().next() - # - g = traversal().withRemote(remote_connection). \ withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created"))) assert 4 == g.V().count().next() assert 0 == g.E().count().next() @@ -168,118 +153,6 @@ class TestDriverRemoteConnection(object): assert 6 == g.V().count().next() assert 6 == g.E().count().next() - def test_side_effects(self, remote_connection): - statics.load_statics(globals()) - # - g = traversal().withRemote(remote_connection) - ### - t = g.V().hasLabel("project").name.iterate() - assert 0 == len(t.side_effects.keys()) - with pytest.raises(Exception): - m = t.side_effects["m"] - ### - t = g.V().out("created").groupCount("m").by("name") - results = t.toSet() - assert 2 == len(results) - assert Vertex(3) in results - assert Vertex(5) in results - assert 1 == len(t.side_effects.keys()) - assert "m" in t.side_effects.keys() - m = t.side_effects["m"] - assert isinstance(m, dict) - assert 2 == len(m) - assert 3 == m["lop"] - assert 1 == m["ripple"] - assert isinstance(m["lop"], long) - assert isinstance(m["ripple"], long) - - # check status attributes - assert "host" in t.side_effects.status_attributes - - ## - t = g.V().out("created").groupCount("m").by("name").name.aggregate("n") - results = t.toSet() - assert 2 == len(results) - assert "lop" in results - assert "ripple" in results - assert 2 == len(t.side_effects.keys()) - assert "m" in t.side_effects.keys() - assert "n" in t.side_effects.keys() - n = t.side_effects.get("n") - assert isinstance(n, dict) - assert 2 == len(n) - assert "lop" in n.keys() - assert "ripple" in n.keys() - assert 3 == n["lop"] - assert 1 == n["ripple"] - - t = g.withSideEffect('m', 32).V().map(lambda: "x: x.sideEffects('m')") - results = t.toSet() - assert 1 == len(results) - assert 32 == list(results)[0] - assert 32 == t.side_effects['m'] - assert 1 == len(t.side_effects.keys()) - with pytest.raises(Exception): - x = t.side_effects["x"] - - a = g.V().has("name", "marko").next() - b = g.V().has("name", "peter").next() - edge = g.withSideEffect("b", b).V(a).addE("knows").to("b").next() - assert "knows" == edge.label - assert a == edge.outV - assert b == edge.inV - g.V().has("name", "marko").outE("knows").where(__.inV().has("name", "peter")).drop().iterate() - ## - edge = g.withSideEffect("a", a).withSideEffect("b", b).V().limit(1).addE("knows").from_("a").to("b").next() - assert "knows" == edge.label - assert a == edge.outV - assert b == edge.inV - g.V().has("name", "marko").outE("knows").where(__.inV().has("name", "peter")).drop().iterate() - - def test_side_effect_close(self, remote_connection): - g = traversal().withRemote(remote_connection) - t = g.V().aggregate('a').aggregate('b') - t.toList() - - # The 'a' key should return some side effects - results = t.side_effects.get('a') - assert results - - # Close result is None - results = t.side_effects.close() - assert not results - - # Shouldn't get any new info from server - # 'b' isn't in local cache - results = t.side_effects.get('b') - assert not results - - # But 'a' should still be cached locally - results = t.side_effects.get('a') - assert results - - # 'a' should have been added to local keys cache, but not 'b' - results = t.side_effects.keys() - assert len(results) == 1 - a, = results - assert a == 'a' - - # Try to get 'b' directly from server, should throw error - with pytest.raises(Exception): - t.side_effects.value_lambda('b') - - def test_promise(self, remote_connection): - g = traversal().withRemote(remote_connection) - future = g.V().aggregate('a').promise() - t = future.result() - assert len(t.toList()) == 6 - a, = t.side_effects.keys() - assert a == 'a' - results = t.side_effects.get('a') - assert results - results = t.side_effects.close() - assert not results - def test_in_tornado_app(remote_connection): # Make sure nothing weird with loops
