I just did Gods work. TranslationStrategy is now smart about removing itself from the Bytecode so all the Translators no longer have to have IS_TESTING in them. GraphTraversalTest has a sweet method that uses reflection to generate fake arguments to ensure that the Bytecode instruction arguments are identical to the method arguments. Found lots of redirects in GraphTraversal that have since been fixed. Step->Instruction are in one-to-one correspondence. More work on GraphSON and added more tests around serialization in both Python and Java. Added an IS_TESTING extension to PythonGraphSONTranslator that ensures that the GraphSON representaiton of the java Bytecode is the same as the Python bytecode. TaDa. Added some more cool helper methods to BytecodeHelper. Crazy cool stuff.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/47058017 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/47058017 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/47058017 Branch: refs/heads/master Commit: 47058017d9c61686f222d5a3495aef6e0cdf8302 Parents: eed6745 Author: Marko A. Rodriguez <[email protected]> Authored: Thu Nov 17 17:58:32 2016 -0700 Committer: Marko A. Rodriguez <[email protected]> Committed: Tue Nov 22 11:20:01 2016 -0700 ---------------------------------------------------------------------- .../gremlin/jsr223/JavaTranslator.java | 6 -- .../remote/traversal/step/map/RemoteStep.java | 6 +- .../traversal/dsl/graph/GraphTraversal.java | 11 ++- .../process/traversal/util/BytecodeHelper.java | 35 ++++++++ .../io/graphson/GraphSONSerializersV2d0.java | 9 +- .../traversal/dsl/graph/GraphTraversalTest.java | 94 ++++++++++++++++++++ .../gremlin/groovy/jsr223/GroovyTranslator.java | 6 -- .../python/GraphTraversalSourceGenerator.groovy | 3 - .../gremlin/python/jsr223/PythonTranslator.java | 7 +- .../gremlin_python/process/graph_traversal.py | 6 +- .../jsr223/PythonGraphSONJavaTranslator.java | 29 +++++- .../RemoteGraphGroovyTranslatorProvider.java | 4 +- .../decoration/TranslationStrategy.java | 28 +++--- .../gremlin/structure/io/IoPropertyTest.java | 17 ++++ .../io/graphson/GraphSONTranslator.java | 5 +- .../structure/io/gryo/GryoTranslator.java | 6 +- 16 files changed, 211 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/JavaTranslator.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/JavaTranslator.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/JavaTranslator.java index 29c6c46..ed09830 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/JavaTranslator.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/JavaTranslator.java @@ -47,8 +47,6 @@ import java.util.concurrent.ConcurrentHashMap; */ public final class JavaTranslator<S extends TraversalSource, T extends Traversal.Admin<?, ?>> implements Translator.StepTranslator<S, T> { - private static final boolean IS_TESTING = Boolean.valueOf(System.getProperty("is.testing", "false")); - private final S traversalSource; private final Class anonymousTraversal; private static final Map<Class<?>, Map<String, List<Method>>> GLOBAL_METHOD_CACHE = new ConcurrentHashMap<>(); @@ -73,10 +71,6 @@ public final class JavaTranslator<S extends TraversalSource, T extends Traversal TraversalSource dynamicSource = this.traversalSource; Traversal.Admin<?, ?> traversal = null; for (final Bytecode.Instruction instruction : bytecode.getSourceInstructions()) { - if (IS_TESTING && - instruction.getOperator().equals(TraversalSource.Symbols.withStrategies) && - instruction.getArguments()[0].toString().contains("TranslationStrategy")) - continue; dynamicSource = (TraversalSource) invokeMethod(dynamicSource, TraversalSource.class, instruction.getOperator(), instruction.getArguments()); } boolean spawned = false; http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/traversal/step/map/RemoteStep.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/traversal/step/map/RemoteStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/traversal/step/map/RemoteStep.java index 7db8e00..9800009 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/traversal/step/map/RemoteStep.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/remote/traversal/step/map/RemoteStep.java @@ -38,8 +38,6 @@ import java.util.NoSuchElementException; */ public final class RemoteStep<S, E> extends AbstractStep<S, E> { - private static final boolean IS_TESTING = Boolean.valueOf(System.getProperty("is.testing", "false")); - private transient RemoteConnection remoteConnection; private RemoteTraversal<?, E> remoteTraversal; @@ -57,9 +55,7 @@ public final class RemoteStep<S, E> extends AbstractStep<S, E> { protected Traverser.Admin<E> processNextStart() throws NoSuchElementException { if (null == this.remoteTraversal) { try { - this.remoteTraversal = this.remoteConnection.submit(IS_TESTING ? BytecodeHelper.filterInstructions(this.traversal.getBytecode(), - instruction -> !(instruction.getOperator().equals(TraversalSource.Symbols.withStrategies) && - instruction.getArguments()[0].toString().contains("TranslationStrategy"))) : this.traversal.getBytecode()); + this.remoteTraversal = this.remoteConnection.submit(this.traversal.getBytecode()); this.traversal.setSideEffects(this.remoteTraversal.getSideEffects()); } catch (final RemoteConnectionException sce) { throw new IllegalStateException(sce); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java index 0674e48..db1aa4a 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.java @@ -159,6 +159,8 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; +import static javafx.scene.input.KeyCode.M; + /** * @author Marko A. Rodriguez (http://markorodriguez.com) */ @@ -1454,7 +1456,8 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> { /////////////////// VERTEX PROGRAM STEPS //////////////// public default GraphTraversal<S, E> pageRank() { - return this.pageRank(0.85d); + this.asAdmin().getBytecode().addStep(Symbols.pageRank); + return this.asAdmin().addStep((Step<E, E>) new PageRankVertexProgramStep(this.asAdmin(), 0.85d)); } public default GraphTraversal<S, E> pageRank(final double alpha) { @@ -1485,7 +1488,8 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> { } public default GraphTraversal<S, E> barrier() { - return this.barrier(Integer.MAX_VALUE); + this.asAdmin().getBytecode().addStep(Symbols.barrier); + return this.asAdmin().addStep(new NoOpBarrierStep<>(this.asAdmin(), Integer.MAX_VALUE)); } public default GraphTraversal<S, E> barrier(final int maxBarrierSize) { @@ -1583,7 +1587,8 @@ public interface GraphTraversal<S, E> extends Traversal<S, E> { public default <E2> GraphTraversal<S, E> option(final Traversal<E, E2> traversalOption) { this.asAdmin().getBytecode().addStep(Symbols.option, traversalOption); - return this.option(TraversalOptionParent.Pick.any, traversalOption.asAdmin()); + ((TraversalOptionParent<Object, E, E2>) this.asAdmin().getEndStep()).addGlobalChildOption(TraversalOptionParent.Pick.any, traversalOption.asAdmin()); + return this; } //// http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/BytecodeHelper.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/BytecodeHelper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/BytecodeHelper.java index dc611d6..f378394 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/BytecodeHelper.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/BytecodeHelper.java @@ -20,10 +20,14 @@ package org.apache.tinkerpop.gremlin.process.traversal.util; import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory; import org.apache.tinkerpop.gremlin.util.function.Lambda; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.function.Predicate; +import java.util.function.Supplier; /** * @author Marko A. Rodriguez (http://markorodriguez.com) @@ -61,4 +65,35 @@ public final class BytecodeHelper { } return Optional.empty(); } + + public static void removeBindings(final Bytecode bytecode) { + for (final Bytecode.Instruction instruction : bytecode.getInstructions()) { + final Object[] arguments = instruction.getArguments(); + for (int i = 0; i < arguments.length; i++) { + if (arguments[i] instanceof Bytecode.Binding) + arguments[i] = ((Bytecode.Binding) arguments[i]).value(); + else if (arguments[i] instanceof Bytecode) + removeBindings((Bytecode) arguments[i]); + } + } + } + + public static void detachElements(final Bytecode bytecode) { + for (final Bytecode.Instruction instruction : bytecode.getInstructions()) { + final Object[] arguments = instruction.getArguments(); + for (int i = 0; i < arguments.length; i++) { + if (arguments[i] instanceof Bytecode) + detachElements((Bytecode) arguments[i]); + else if(arguments[i] instanceof List) { + final List<Object> list = new ArrayList<>(); + for(final Object object : (List)arguments[i]) { + list.add( DetachedFactory.detach(object, false)); + } + arguments[i] = list; + } + else + arguments[i] = DetachedFactory.detach(arguments[i], false); + } + } + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONSerializersV2d0.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONSerializersV2d0.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONSerializersV2d0.java index 4d99f49..1afbc0b 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONSerializersV2d0.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/graphson/GraphSONSerializersV2d0.java @@ -353,7 +353,7 @@ class GraphSONSerializersV2d0 { @Override public void serialize(final Integer integer, final JsonGenerator jsonGenerator, final SerializerProvider serializerProvider) throws IOException { - jsonGenerator.writeNumber(((Integer) integer).intValue()); + jsonGenerator.writeNumber(integer.intValue()); } } @@ -486,7 +486,10 @@ class GraphSONSerializersV2d0 { @Override public Property createObject(final Map<String, Object> propData) { - return new DetachedProperty((String) propData.get(GraphSONTokens.KEY), propData.get(GraphSONTokens.VALUE)); + final Object element = propData.get(GraphSONTokens.ELEMENT); + return element instanceof Element ? // graphson-non-embedded is treated differently, but since this is a hard coded embedding... + new DetachedProperty<>((String) propData.get(GraphSONTokens.KEY), propData.get(GraphSONTokens.VALUE), (Element) element) : + new DetachedProperty<>((String) propData.get(GraphSONTokens.KEY), propData.get(GraphSONTokens.VALUE)); } } @@ -503,7 +506,7 @@ class GraphSONSerializersV2d0 { propData.get(GraphSONTokens.ID), (String) propData.get(GraphSONTokens.LABEL), propData.get(GraphSONTokens.VALUE), (Map<String, Object>) propData.get(GraphSONTokens.PROPERTIES), - new DetachedVertex(propData.get(GraphSONTokens.VERTEX), Vertex.DEFAULT_LABEL, Collections.emptyMap())) : + new DetachedVertex(propData.get(GraphSONTokens.VERTEX), Vertex.DEFAULT_LABEL, null)) : new DetachedVertexProperty<>( propData.get(GraphSONTokens.ID), (String) propData.get(GraphSONTokens.LABEL), http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java index 98d590b..2c3c6c1 100644 --- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java +++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversalTest.java @@ -18,14 +18,22 @@ */ package org.apache.tinkerpop.gremlin.process.traversal.dsl.graph; +import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Scope; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.junit.Test; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; +import java.util.Random; import java.util.Set; +import static org.junit.Assert.assertEquals; + /** * @author Marko A. Rodriguez (http://markorodriguez.com) */ @@ -33,6 +41,7 @@ public class GraphTraversalTest { private static Set<String> NO_GRAPH = new HashSet<>(Arrays.asList("asAdmin", "by", "option", "iterate", "to", "from", "profile", "pageRank", "peerPressure", "program")); private static Set<String> NO_ANONYMOUS = new HashSet<>(Arrays.asList("start", "__")); + private static Set<String> IGNORES_BYTECODE = new HashSet<>(Arrays.asList("asAdmin", "iterate", "mapValues", "mapKeys")); @Test public void anonymousGraphTraversalShouldHaveMethodsOfGraphTraversal() { @@ -71,4 +80,89 @@ public class GraphTraversalTest { } } } + + @Test + public void graphTraversalMethodsShouldGenerateCorrespondingBytecode() throws Exception { + final Random random = new Random(); + for (Method stepMethod : GraphTraversal.class.getMethods()) { + if (Traversal.class.isAssignableFrom(stepMethod.getReturnType()) && !IGNORES_BYTECODE.contains(stepMethod.getName())) { + final GraphTraversal.Admin<?, ?> traversal = new DefaultGraphTraversal<>(); + Object[] arguments = new Object[stepMethod.getParameterCount()]; + final List<Object> list = new ArrayList<>(); + boolean doTest = true; + /// + if (stepMethod.getName().equals("by")) + traversal.order(); + else if (stepMethod.getName().equals("option")) + traversal.branch(__.identity().out(randomString(random))); + else if (stepMethod.getName().equals("to") || stepMethod.getName().equals("from")) + traversal.addE(randomString(random)); + if (stepMethod.getName().equals("range")) { + if (Scope.class.isAssignableFrom(stepMethod.getParameterTypes()[0])) { + list.add(arguments[0] = Scope.local); + list.add(arguments[1] = (long) (Math.abs(random.nextInt(10)))); + list.add(arguments[2] = (long) (Math.abs(random.nextInt(10) + 100))); + } else { + list.add(arguments[0] = (long) (Math.abs(random.nextInt(10)))); + list.add(arguments[1] = (long) (Math.abs(random.nextInt(10) + 100))); + } + } else { + for (int i = 0; i < stepMethod.getParameterTypes().length; i++) { + final Class<?> type = stepMethod.getParameterTypes()[i]; + if (int.class.isAssignableFrom(type)) + list.add(arguments[i] = Math.abs(random.nextInt(100))); + else if (long.class.isAssignableFrom(type)) + list.add(arguments[i] = (long) (Math.abs(random.nextInt(100)))); + else if (double.class.isAssignableFrom(type)) + list.add(arguments[i] = Math.abs(random.nextDouble())); + else if (String.class.isAssignableFrom(type)) + list.add(arguments[i] = randomString(random)); + else if (boolean.class.isAssignableFrom(type)) + list.add(arguments[i] = random.nextBoolean()); + else if (String[].class.isAssignableFrom(type)) { + arguments[i] = new String[random.nextInt(10) + 1]; + for (int j = 0; j < ((String[]) arguments[i]).length; j++) { + list.add(((String[]) arguments[i])[j] = randomString(random)); + } + } else if (Traversal.class.isAssignableFrom(type)) + list.add(arguments[i] = __.out(randomString(random)).in(randomString(random)).groupCount(randomString(random))); + else if (Traversal[].class.isAssignableFrom(type)) { + arguments[i] = new Traversal[random.nextInt(5) + 1]; + for (int j = 0; j < ((Traversal[]) arguments[i]).length; j++) { + list.add(((Traversal[]) arguments[i])[j] = __.as(randomString(random)).out(randomString(random)).both(randomString(random)).has(randomString(random)).store(randomString(random))); + } + } else if (P.class.isAssignableFrom(type)) + list.add(arguments[i] = P.gte(stepMethod.getName().contains("where") ? randomString(random) : random.nextInt())); + else if (Enum.class.isAssignableFrom(type)) + list.add(arguments[i] = type.getEnumConstants()[0]); + else { + // System.out.println(type); + doTest = false; + break; + } + } + } + if (doTest) { + stepMethod.invoke(traversal, arguments); + // System.out.print(stepMethod.getName() + "---"); + final Bytecode.Instruction instruction = traversal.getBytecode().getStepInstructions().get(traversal.getBytecode().getStepInstructions().size() - 1); + // System.out.println(instruction); + assertEquals(stepMethod.getName(), instruction.getOperator()); + assertEquals(list.size(), instruction.getArguments().length); + for (int i = 0; i < list.size(); i++) { + assertEquals(list.get(i) instanceof Traversal ? ((Traversal) list.get(i)).asAdmin().getBytecode() : list.get(i), instruction.getArguments()[i]); + } + } + } + } + } + + private final static String randomString(final Random random) { + String s = ""; + for (int i = 0; i < random.nextInt(10) + 1; i++) { + s = (s + (char) (random.nextInt(100) + 1)).trim(); + } + final String temp = "" + random.nextBoolean(); + return temp.substring(temp.length() - (random.nextInt(2) + 1)) + s.replace("\"", "x").replace(",", "y").replace("'", "z"); + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java ---------------------------------------------------------------------- diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java index a92d920..2e88197 100644 --- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java +++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java @@ -49,8 +49,6 @@ import java.util.Set; */ public final class GroovyTranslator implements Translator.ScriptTranslator { - private static final boolean IS_TESTING = Boolean.valueOf(System.getProperty("is.testing", "false")); - private final String traversalSource; private GroovyTranslator(final String traversalSource) { @@ -89,10 +87,6 @@ public final class GroovyTranslator implements Translator.ScriptTranslator { final StringBuilder traversalScript = new StringBuilder(start); for (final Bytecode.Instruction instruction : bytecode.getInstructions()) { final String methodName = instruction.getOperator(); - if (IS_TESTING && - instruction.getOperator().equals(TraversalSource.Symbols.withStrategies) && - instruction.getArguments()[0].toString().contains("TranslationStrategy")) - continue; if (0 == instruction.getArguments().length) traversalScript.append(".").append(methodName).append("()"); else { http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-python/src/main/groovy/org/apache/tinkerpop/gremlin/python/GraphTraversalSourceGenerator.groovy ---------------------------------------------------------------------- diff --git a/gremlin-python/src/main/groovy/org/apache/tinkerpop/gremlin/python/GraphTraversalSourceGenerator.groovy b/gremlin-python/src/main/groovy/org/apache/tinkerpop/gremlin/python/GraphTraversalSourceGenerator.groovy index 1470757..f4de2c1 100644 --- a/gremlin-python/src/main/groovy/org/apache/tinkerpop/gremlin/python/GraphTraversalSourceGenerator.groovy +++ b/gremlin-python/src/main/groovy/org/apache/tinkerpop/gremlin/python/GraphTraversalSourceGenerator.groovy @@ -82,7 +82,6 @@ under the License. findAll { GraphTraversalSource.class.equals(it.returnType) }. findAll { !it.name.equals("clone") && - !it.name.equals(TraversalSource.Symbols.withBindings) && !it.name.equals(TraversalSource.Symbols.withRemote) && !it.name.equals(TraversalSource.Symbols.withComputer) }. @@ -102,8 +101,6 @@ under the License. source = GraphTraversalSource(self.graph, TraversalStrategies(self.traversal_strategies), Bytecode(self.bytecode)) source.traversal_strategies.add_strategies([RemoteStrategy(remote_connection)]) return source - def withBindings(self, bindings): - return self def withComputer(self,graph_computer=None, workers=None, result=None, persist=None, vertices=None, edges=None, configuration=None): return self.withStrategies(VertexProgramStrategy(graph_computer,workers,result,persist,vertices,edges,configuration)) """) http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java ---------------------------------------------------------------------- diff --git a/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java b/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java index 71e610f..d4b9202 100644 --- a/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java +++ b/gremlin-python/src/main/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonTranslator.java @@ -59,7 +59,6 @@ import java.util.stream.Stream; */ public class PythonTranslator implements Translator.ScriptTranslator { - private static final boolean IS_TESTING = Boolean.valueOf(System.getProperty("is.testing", "false")); private static final Set<String> STEP_NAMES = Stream.of(GraphTraversal.class.getMethods()).filter(method -> Traversal.class.isAssignableFrom(method.getReturnType())).map(Method::getName).collect(Collectors.toSet()); private static final Set<String> NO_STATIC = Stream.of(T.values(), Operator.values()) .flatMap(arg -> IteratorUtils.stream(new ArrayIterator<>(arg))) @@ -109,11 +108,7 @@ public class PythonTranslator implements Translator.ScriptTranslator { for (final Bytecode.Instruction instruction : bytecode.getInstructions()) { final String methodName = instruction.getOperator(); final Object[] arguments = instruction.getArguments(); - if (IS_TESTING && - instruction.getOperator().equals(TraversalSource.Symbols.withStrategies) && - instruction.getArguments()[0].toString().contains("TranslationStrategy")) - continue; - else if (0 == arguments.length) + if (0 == arguments.length) traversalScript.append(".").append(SymbolHelper.toPython(methodName)).append("()"); else if (methodName.equals("range") && 2 == arguments.length && ((Number) arguments[0]).intValue() != 0) { if (((Number) arguments[0]).longValue() + 1 == ((Number) arguments[1]).longValue()) http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py ---------------------------------------------------------------------- diff --git a/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py index 740eec7..b712613 100644 --- a/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py +++ b/gremlin-python/src/main/jython/gremlin_python/process/graph_traversal.py @@ -34,6 +34,10 @@ class GraphTraversalSource(object): self.bytecode = bytecode def __repr__(self): return "graphtraversalsource[" + str(self.graph) + "]" + def withBindings(self, *args): + source = GraphTraversalSource(self.graph, TraversalStrategies(self.traversal_strategies), Bytecode(self.bytecode)) + source.bytecode.add_source("withBindings", *args) + return source def withBulk(self, *args): source = GraphTraversalSource(self.graph, TraversalStrategies(self.traversal_strategies), Bytecode(self.bytecode)) source.bytecode.add_source("withBulk", *args) @@ -62,8 +66,6 @@ class GraphTraversalSource(object): source = GraphTraversalSource(self.graph, TraversalStrategies(self.traversal_strategies), Bytecode(self.bytecode)) source.traversal_strategies.add_strategies([RemoteStrategy(remote_connection)]) return source - def withBindings(self, bindings): - return self def withComputer(self,graph_computer=None, workers=None, result=None, persist=None, vertices=None, edges=None, configuration=None): return self.withStrategies(VertexProgramStrategy(graph_computer,workers,result,persist,vertices,edges,configuration)) def E(self, *args): http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-python/src/test/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonGraphSONJavaTranslator.java ---------------------------------------------------------------------- diff --git a/gremlin-python/src/test/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonGraphSONJavaTranslator.java b/gremlin-python/src/test/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonGraphSONJavaTranslator.java index cdeb407..5144e31 100644 --- a/gremlin-python/src/test/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonGraphSONJavaTranslator.java +++ b/gremlin-python/src/test/java/org/apache/tinkerpop/gremlin/python/jsr223/PythonGraphSONJavaTranslator.java @@ -24,9 +24,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; import org.apache.tinkerpop.gremlin.process.traversal.Translator; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.util.BytecodeHelper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONXModuleV2d0; import org.apache.tinkerpop.gremlin.util.ScriptEngineCache; @@ -34,17 +36,24 @@ import javax.script.Bindings; import javax.script.ScriptContext; import javax.script.ScriptEngine; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import static org.junit.Assert.fail; /** * @author Marko A. Rodriguez (http://markorodriguez.com) */ final class PythonGraphSONJavaTranslator<S extends TraversalSource, T extends Traversal.Admin<?, ?>> implements Translator.StepTranslator<S, T> { + private final boolean IS_TESTING = Boolean.valueOf(System.getProperty("is.testing", "false")); private final PythonTranslator pythonTranslator; private final JavaTranslator<S, T> javaTranslator; private final GraphSONReader reader = GraphSONReader.build().mapper( GraphSONMapper.build().addCustomModule(GraphSONXModuleV2d0.build().create(false)) - .version(GraphSONVersion.V2_0).create()).create(); + .version(GraphSONVersion.V2_0).create()).create(); + private final GraphSONWriter writer = GraphSONWriter.build().mapper( + GraphSONMapper.build().addCustomModule(GraphSONXModuleV2d0.build().create(false)) + .version(GraphSONVersion.V2_0).create()).create(); public PythonGraphSONJavaTranslator(final PythonTranslator pythonTranslator, final JavaTranslator<S, T> javaTranslator) { this.pythonTranslator = pythonTranslator; @@ -69,9 +78,21 @@ final class PythonGraphSONJavaTranslator<S extends TraversalSource, T extends Tr bindings.putAll(jythonEngine.getBindings(ScriptContext.ENGINE_SCOPE)); bindings.put(this.pythonTranslator.getTraversalSource(), jythonEngine.eval("Graph().traversal()")); bindings.putAll(bytecode.getBindings()); - final String graphsonBytecode = jythonEngine.eval("graphson_writer.writeObject(" + this.pythonTranslator.translate(bytecode) + ")", bindings).toString(); - // System.out.println(graphsonBytecode); - return this.javaTranslator.translate(this.reader.readObject(new ByteArrayInputStream(graphsonBytecode.getBytes()), Bytecode.class)); + final String translatedGraphSONBytecode = jythonEngine.eval("graphson_writer.writeObject(" + this.pythonTranslator.translate(bytecode) + ")", bindings).toString(); + if (IS_TESTING) { + // verify that the GraphSON sent to Python is the same as the GraphSON returned by Python + final ByteArrayOutputStream output = new ByteArrayOutputStream(); + BytecodeHelper.removeBindings(bytecode); // this is because bindings are variables that get converted to values at translation + BytecodeHelper.detachElements(bytecode); // this is to get the minimal necessary representation + this.writer.writeObject(output, bytecode); + final String originalGraphSONBytecode = new String(output.toByteArray()); + // sometimes the Java Map and Python dict are sorted differently. (equals() is too strict -- only a few tests cases require this) + if (originalGraphSONBytecode.length() != translatedGraphSONBytecode.length()) + fail(originalGraphSONBytecode + "\n does not equal\n" + translatedGraphSONBytecode); + } + return this.javaTranslator.translate(this.reader.readObject(new ByteArrayInputStream(translatedGraphSONBytecode.getBytes()), Bytecode.class)); + + } catch (final Exception e) { throw new IllegalArgumentException(e.getMessage(), e); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/RemoteGraphGroovyTranslatorProvider.java ---------------------------------------------------------------------- diff --git a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/RemoteGraphGroovyTranslatorProvider.java b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/RemoteGraphGroovyTranslatorProvider.java index 7f4d58e..6949426 100644 --- a/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/RemoteGraphGroovyTranslatorProvider.java +++ b/gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/RemoteGraphGroovyTranslatorProvider.java @@ -53,9 +53,9 @@ public class RemoteGraphGroovyTranslatorProvider extends RemoteGraphProvider { @Override public GraphTraversalSource traversal(final Graph graph) { if (SKIP) - return graph.traversal(); + return super.traversal(graph); else { - final GraphTraversalSource g = graph.traversal(); + final GraphTraversalSource g = super.traversal(graph); return g.withStrategies(new TranslationStrategy(g, GroovyTranslator.of("g"))); } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/TranslationStrategy.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/TranslationStrategy.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/TranslationStrategy.java index cf93acf..0736c02 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/TranslationStrategy.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/TranslationStrategy.java @@ -78,16 +78,16 @@ public final class TranslationStrategy extends AbstractTraversalStrategy<Travers // verifications to ensure unsupported steps do not exist in the traversal if (IS_TESTING) { - if (traversal.getBytecode().toString().contains("$") || traversal.getBytecode().toString().contains("HashSetSupplier")) + if (traversal.getBytecode().toString().contains("$$Lambda$") || traversal.getBytecode().toString().contains("Supplier@")) throw new VerificationException("Test suite does not support lambdas", traversal); if (TraversalHelper.hasStepOfAssignableClassRecursively(ProgramVertexProgramStep.class, traversal)) throw new VerificationException("Test suite does not support embedded vertex programs", traversal); } final Traversal.Admin<?, ?> translatedTraversal; - final Bytecode bytecode = IS_TESTING ? + final Bytecode bytecode = removeTranslationStrategy(IS_TESTING ? insertBindingsForTesting(traversal.getBytecode()) : - traversal.getBytecode(); + traversal.getBytecode()); //////////////// if (this.translator instanceof Translator.StepTranslator) { // reflection based translation @@ -107,19 +107,15 @@ public final class TranslationStrategy extends AbstractTraversalStrategy<Travers throw new IllegalArgumentException("TranslationStrategy does not know how to process the provided translator type: " + this.translator.getClass().getSimpleName()); } //////////////// - if (IS_TESTING && // this tests to ensure that the bytecode going in is the same as the bytecode coming out - !BytecodeHelper.getLambdaLanguage(traversal.getBytecode()).isPresent()) - assertEquals(BytecodeHelper.filterInstructions(traversal.getBytecode(), - instruction -> - !(instruction.getOperator().equals(TraversalSource.Symbols.withStrategies) && - instruction.getArguments()[0] instanceof TranslationStrategy)), - translatedTraversal.getBytecode()); - //////////////// assert !translatedTraversal.isLocked(); assert !traversal.isLocked(); traversal.setSideEffects(translatedTraversal.getSideEffects()); TraversalHelper.removeAllSteps(traversal); TraversalHelper.removeToTraversal((Step) translatedTraversal.getStartStep(), EmptyStep.instance(), traversal); + //////////////// + if (IS_TESTING && !BytecodeHelper.getLambdaLanguage(bytecode).isPresent()) + // this tests to ensure that the bytecode being translated is the same as the bytecode of the generated traversal + assertEquals(removeTranslationStrategy(traversal.getBytecode()), translatedTraversal.getBytecode()); } @@ -128,9 +124,17 @@ public final class TranslationStrategy extends AbstractTraversalStrategy<Travers return POSTS; } + private static final Bytecode removeTranslationStrategy(final Bytecode bytecode) { + if (bytecode.getSourceInstructions().size() > 0) + bytecode.getSourceInstructions().remove(0); + return bytecode; + } + private static final Bytecode insertBindingsForTesting(final Bytecode bytecode) { final Bytecode newBytecode = new Bytecode(); - bytecode.getSourceInstructions().forEach(instruction -> newBytecode.addSource(instruction.getOperator(), instruction.getArguments())); + for (final Bytecode.Instruction instruction : bytecode.getSourceInstructions()) { + newBytecode.addSource(instruction.getOperator(), instruction.getArguments()); + } for (final Bytecode.Instruction instruction : bytecode.getStepInstructions()) { final Object[] args = instruction.getArguments(); final Object[] newArgs = new Object[args.length]; http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoPropertyTest.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoPropertyTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoPropertyTest.java index 3a3d6f0..13758f9 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoPropertyTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/IoPropertyTest.java @@ -20,8 +20,10 @@ package org.apache.tinkerpop.gremlin.structure.io; import org.apache.tinkerpop.gremlin.AbstractGremlinTest; import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion; @@ -90,6 +92,7 @@ public class IoPropertyTest extends AbstractGremlinTest { // select any vertexproperty that has both start/end time final VertexProperty p = (VertexProperty) g.V(convertToVertexId("marko")).properties("location").as("p").has("endTime").select("p").next(); + final Vertex v = p.element(); writer.writeVertexProperty(os, p); final AtomicBoolean called = new AtomicBoolean(false); @@ -101,6 +104,10 @@ public class IoPropertyTest extends AbstractGremlinTest { assertEquals(IteratorUtils.count(p.properties()), IteratorUtils.count(propertyAttachable.get().properties())); assertEquals(p.property("startTime").value(), ((Property) propertyAttachable.get().properties("startTime").next()).value()); assertEquals(p.property("endTime").value(), ((Property) propertyAttachable.get().properties("endTime").next()).value()); + if (ioType.equals("graphson-v2-embedded") || ioType.equals("graphson-v2")) { // TODO: make this work with Gryo + assertEquals(p.element(), propertyAttachable.get().element()); + assertEquals(p.element(), v); + } called.set(true); return propertyAttachable.get(); }); @@ -116,6 +123,7 @@ public class IoPropertyTest extends AbstractGremlinTest { try (final ByteArrayOutputStream os = new ByteArrayOutputStream()) { final GraphWriter writer = writerMaker.apply(graph); final VertexProperty p = g.V(convertToVertexId("marko")).next().property("name"); + final Vertex v = p.element(); writer.writeVertexProperty(os, p); final AtomicBoolean called = new AtomicBoolean(false); @@ -125,6 +133,10 @@ public class IoPropertyTest extends AbstractGremlinTest { assertEquals(p.value(), propertyAttachable.get().value()); assertEquals(p.key(), propertyAttachable.get().key()); assertEquals(0, IteratorUtils.count(propertyAttachable.get().properties())); + if (ioType.equals("graphson-v2-embedded") || ioType.equals("graphson-v2")) { // TODO: make this work with Gryo + assertEquals(p.element(), propertyAttachable.get().element()); + assertEquals(p.element(), v); + } called.set(true); return propertyAttachable.get(); }); @@ -140,6 +152,7 @@ public class IoPropertyTest extends AbstractGremlinTest { try (final ByteArrayOutputStream os = new ByteArrayOutputStream()) { final GraphWriter writer = writerMaker.apply(graph); final Property p = g.E(convertToEdgeId("marko", "knows", "vadas")).next().property("weight"); + final Edge e = (Edge) p.element(); writer.writeProperty(os, p); final AtomicBoolean called = new AtomicBoolean(false); @@ -148,6 +161,10 @@ public class IoPropertyTest extends AbstractGremlinTest { reader.readProperty(bais, propertyAttachable -> { assertEquals(p.value(), propertyAttachable.get().value()); assertEquals(p.key(), propertyAttachable.get().key()); + if (ioType.equals("graphson-v2-embedded")) { // TODO: make this work with Gryo + assertEquals(p.element(), propertyAttachable.get().element()); + assertEquals(p.element(), e); + } called.set(true); return propertyAttachable.get(); }); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/graphson/GraphSONTranslator.java ---------------------------------------------------------------------- diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/graphson/GraphSONTranslator.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/graphson/GraphSONTranslator.java index 4f50329..612c811 100644 --- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/graphson/GraphSONTranslator.java +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/graphson/GraphSONTranslator.java @@ -24,7 +24,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; import org.apache.tinkerpop.gremlin.process.traversal.Translator; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource; -import org.apache.tinkerpop.gremlin.process.traversal.util.BytecodeHelper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion; @@ -58,9 +57,7 @@ final class GraphSONTranslator<S extends TraversalSource, T extends Traversal.Ad public T translate(final Bytecode bytecode) { try { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - this.writer.writeObject(outputStream, BytecodeHelper.filterInstructions(bytecode, - instruction -> !(instruction.getOperator().equals(TraversalSource.Symbols.withStrategies) && - instruction.getArguments()[0].toString().contains("TranslationStrategy")))); + this.writer.writeObject(outputStream, bytecode); // System.out.println(new String(outputStream.toByteArray())); return this.wrappedTranslator.translate(this.reader.readObject(new ByteArrayInputStream(outputStream.toByteArray()), Bytecode.class)); } catch (final Exception e) { http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/47058017/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/gryo/GryoTranslator.java ---------------------------------------------------------------------- diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/gryo/GryoTranslator.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/gryo/GryoTranslator.java index 3a213d7..90e9eba 100644 --- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/gryo/GryoTranslator.java +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/io/gryo/GryoTranslator.java @@ -24,7 +24,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; import org.apache.tinkerpop.gremlin.process.traversal.Translator; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource; -import org.apache.tinkerpop.gremlin.process.traversal.util.BytecodeHelper; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; @@ -55,10 +54,7 @@ final class GryoTranslator<S extends TraversalSource, T extends Traversal.Admin< public T translate(final Bytecode bytecode) { try { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - this.writer.writeObject(outputStream, BytecodeHelper.filterInstructions(bytecode, - instruction -> !(instruction.getOperator().equals(TraversalSource.Symbols.withStrategies) && - instruction.getArguments()[0].toString().contains("TranslationStrategy")))); - //System.out.println(new String(outputStream.toByteArray())); + this.writer.writeObject(outputStream, bytecode); return this.wrappedTranslator.translate(this.reader.readObject(new ByteArrayInputStream(outputStream.toByteArray()), Bytecode.class)); } catch (final Exception e) { throw new IllegalStateException(e.getMessage(), e);
