[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user asfgit closed the pull request at: https://github.com/apache/tinkerpop/pull/893 ---
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user spmallette commented on a diff in the pull request: https://github.com/apache/tinkerpop/pull/893#discussion_r206269042 --- Diff: gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java --- @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect; + +import org.apache.tinkerpop.gremlin.process.traversal.IO; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.EmptyTraverser; +import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Handles read and write operations into the {@link Graph}. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class IoStep extends AbstractStep implements ReadWriting { + +private Parameters parameters = new Parameters(); +private boolean first = true; +private String file; +private Mode mode = Mode.UNSET; + +public IoStep(final Traversal.Admin traversal, final String file) { +super(traversal); + +if (null == file || file.isEmpty()) +throw new IllegalArgumentException("file cannot be null or empty"); + +this.file = file; +} + +@Override +public void setMode(final Mode mode) { +this.mode = mode; +} + +@Override +public Mode getMode() { +return mode; +} + +@Override +public String getFile() { +return file; +} + +@Override +public Parameters getParameters() { +return this.parameters; +} + +@Override +public void configure(final Object... keyValues) { +this.parameters.set(null, keyValues); +} + +@Override +protected Traverser.Admin processNextStart() { +if (mode == Mode.UNSET) throw new IllegalStateException("IO mode was not set to read() or write()"); +if (!this.first) throw FastNoSuchElementException.instance(); + +this.first = false; +final File file = new File(this.file); + +if (mode == Mode.READING) { +if (!file.exists()) throw new IllegalStateException(this.file + " does not exist"); +return read(fil
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user spmallette commented on a diff in the pull request: https://github.com/apache/tinkerpop/pull/893#discussion_r205939963 --- Diff: gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs --- @@ -1704,11 +1713,11 @@ public GraphTraversal(ICollection traversalStrategies, Bytec } /// -/// Adds the with step to this . --- End diff -- It's above in the diff - expand the section above. ---
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user spmallette commented on a diff in the pull request: https://github.com/apache/tinkerpop/pull/893#discussion_r205939944 --- Diff: gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java --- @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect; + +import org.apache.tinkerpop.gremlin.process.traversal.IO; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.EmptyTraverser; +import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Handles read and write operations into the {@link Graph}. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class IoStep extends AbstractStep implements ReadWriting { + +private Parameters parameters = new Parameters(); +private boolean first = true; +private String file; +private Mode mode = Mode.UNSET; + +public IoStep(final Traversal.Admin traversal, final String file) { +super(traversal); + +if (null == file || file.isEmpty()) +throw new IllegalArgumentException("file cannot be null or empty"); + +this.file = file; +} + +@Override +public void setMode(final Mode mode) { +this.mode = mode; +} + +@Override +public Mode getMode() { +return mode; +} + +@Override +public String getFile() { +return file; +} + +@Override +public Parameters getParameters() { +return this.parameters; +} + +@Override +public void configure(final Object... keyValues) { +this.parameters.set(null, keyValues); +} + +@Override +protected Traverser.Admin processNextStart() { +if (mode == Mode.UNSET) throw new IllegalStateException("IO mode was not set to read() or write()"); +if (!this.first) throw FastNoSuchElementException.instance(); + +this.first = false; +final File file = new File(this.file); + +if (mode == Mode.READING) { +if (!file.exists()) throw new IllegalStateException(this.file + " does not exist"); +return read(fil
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user dkuppitz commented on a diff in the pull request: https://github.com/apache/tinkerpop/pull/893#discussion_r205930660 --- Diff: gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs --- @@ -1704,11 +1713,11 @@ public GraphTraversal(ICollection traversalStrategies, Bytec } /// -/// Adds the with step to this . --- End diff -- This is a bit confusing. Why did `With` disappear? ---
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user dkuppitz commented on a diff in the pull request: https://github.com/apache/tinkerpop/pull/893#discussion_r205930559 --- Diff: gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java --- @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect; + +import org.apache.tinkerpop.gremlin.process.traversal.IO; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.Traverser; +import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.EmptyTraverser; +import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Handles read and write operations into the {@link Graph}. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class IoStep extends AbstractStep implements ReadWriting { + +private Parameters parameters = new Parameters(); +private boolean first = true; +private String file; +private Mode mode = Mode.UNSET; + +public IoStep(final Traversal.Admin traversal, final String file) { +super(traversal); + +if (null == file || file.isEmpty()) +throw new IllegalArgumentException("file cannot be null or empty"); + +this.file = file; +} + +@Override +public void setMode(final Mode mode) { +this.mode = mode; +} + +@Override +public Mode getMode() { +return mode; +} + +@Override +public String getFile() { +return file; +} + +@Override +public Parameters getParameters() { +return this.parameters; +} + +@Override +public void configure(final Object... keyValues) { +this.parameters.set(null, keyValues); +} + +@Override +protected Traverser.Admin processNextStart() { +if (mode == Mode.UNSET) throw new IllegalStateException("IO mode was not set to read() or write()"); +if (!this.first) throw FastNoSuchElementException.instance(); + +this.first = false; +final File file = new File(this.file); + +if (mode == Mode.READING) { +if (!file.exists()) throw new IllegalStateException(this.file + " does not exist"); +return read(file)
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user twilmes commented on a diff in the pull request: https://github.com/apache/tinkerpop/pull/893#discussion_r205649402 --- Diff: docs/src/upgrade/release-3.4.x.asciidoc --- @@ -65,6 +65,33 @@ release where breaking changes are allowed. See: link:https://issues.apache.org/jira/browse/TINKERPOP-1975[TINKERPOP-1975], link:http://tinkerpop.apache.org/docs/3.4.0/reference/#with-step[Reference Documentation] + io() Step + +There have been some important changes to IO operations for reading and writing graph data. The use of `Graph.io()` +has been deprecated to further remove dependence on the Graph (Structure) API for users and to extend these basic +operations to GLV users by making these features available as part of the Gremlin language. + +It is now possible to simply use Gremlin: + +[source,groovy] + +graph = ... +g = graph.traversal() +g.io(someInputFile).read().iterate() +g.io(someOutputFile).write().iterate() + + +While `io()` step is still single-threaded for OLTP style loading, it can be utilized in conjunction with OLAP which --- End diff -- Small nit, in spots I think it's referenced as `io()` step in others `io()`-step. Might be good to stick with one or another. It looks like the `-` style is prevalent in the docs. ---
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
Github user twilmes commented on a diff in the pull request: https://github.com/apache/tinkerpop/pull/893#discussion_r205648775 --- Diff: docs/src/reference/the-traversal.asciidoc --- @@ -1049,6 +1050,179 @@ inject(1,2).map {it.get() + 1}.map {g.V(it.get()).next()}.values('name') link:++http://tinkerpop.apache.org/javadocs/x.y.z/core/org/apache/tinkerpop/gremlin/process/traversal/dsl/graph/GraphTraversal.html#inject-E...-++[`inject(Object)`] +[[_gremlin_i_o]] +[[io-step]] +=== IO Step + +image:gremlin-io.png[width=250,float=left] The task of importing and exporting the data of `Graph` instances is the +job of the `io()`-step. By default, TinkerPop supports three formats for importing and exporting graph data in +<>, <>, and <>. + +NOTE: Additional documentation for TinkerPop IO formats can be found in the link:http://tinkerpop.apache.org/docs/x.y.z/dev/io/[IO Reference]. + +By itself the `io()` step merely configures the kind of importing and exporting that is going +to occur and it is the follow-on call to the `read()` or `write()` step that determines which of those actions will +execute. Therefore, a typical usage of the `io()` step would look like this: + +[source,java] + +g.io(someInputFile).read().iterate() +g.io(someOutputFile).write().iterate() + + +IMPORTANT: The commands above are still traversals and therefore require iteration to be executed, hence the use of +`iterate()` as a termination step. + +By default, the `io()` step will try to detect the right file format using the file name extension. To gain greater +control of the format use the `with()` step modulator to provide further information to `io()`. For example: + +[source,java] + +g.io(someInputFile). +with(IO.reader, IO.graphson). + read().iterate() +g.io(someOutputFile). +with(IO.writer,IO.graphml). + write().iterate() + + +The `IO` class is a helper for the `io()` step that provides expressions that can be used to help configure it +and in this case it allows direct specification of the "reader" or "writer" to use. The "reader" actually refers to +a `GraphReader` implementation and the `writer" refers to a `GraphWriter` implementation. The implementations of --- End diff -- Small typo here, a quote at the end of `writer"` instead of a tick. ---
[GitHub] tinkerpop pull request #893: TINKERPOP-1996 io()
GitHub user spmallette opened a pull request: https://github.com/apache/tinkerpop/pull/893 TINKERPOP-1996 io() https://issues.apache.org/jira/browse/TINKERPOP-1996 Introduces `io()` with full support for GLVs and both OLAP and OLTP. There's a bit too much to type here to fully describe this - please take a look at the documentation in the PR as it goes into the details. All tests pass with `docker/build.sh -t -n -i` VOTE +1 You can merge this pull request into a Git repository by running: $ git pull https://github.com/apache/tinkerpop TINKERPOP-1996 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/tinkerpop/pull/893.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #893 commit ec1d05f74fa875e4a07699dc89c3d1956aab586f Author: Stephen Mallette Date: 2018-06-29T19:04:17Z TINKERPOP-1996 Introduce read() and write() steps commit 0785090f67ff88a834f77c7181a69594049c65ea Author: Stephen Mallette Date: 2018-07-10T16:21:52Z TINKERPOP-1996 Have the basics of OLAP read()/write() steps working This is still fairly skeletal at this point. Just trying to make sure things work properly before building read()/write() out fully. commit ff2773a5ef1eff379cb5f3b809d4677ff64a076d Author: Stephen Mallette Date: 2018-07-10T16:55:06Z TINKERPOP-1996 Minor refactoring of Reading/Writing and javadoc commit d99909c4e19fcb3ec3b8a8c94eb054a4332ad219 Author: Stephen Mallette Date: 2018-07-10T18:53:52Z TINKERPOP-1996 read()/write() api changes for return type No more weird Map status return for read() and write(). Both just work like a terminator and self iterate to return nothing. commit 767d65b9d54f1774b13cc5a61db711268f213ff1 Author: Stephen Mallette Date: 2018-07-11T14:20:39Z TINKERPOP-1996 Made read()/write() terminator steps Without this approach the with() operator couldn't be used because the traversal would already be iterated on the call to read() and write(). In this way read() and write() are both terminators and modulators at the same time. commit d181563d24a401ac6550e06d86d78b12a8f16f51 Author: Stephen Mallette Date: 2018-07-11T14:36:11Z TINKERPOP-1996 Added some javadoc and code formatting commit 13e552b289b5266fcb418baaf2ae90626f6055c4 Author: Stephen Mallette Date: 2018-07-11T19:40:39Z TINKERPOP-1996 Added with() options for io() Included GraphReader and GraphWriter detection and added tests commit be9db8de6a9d2abb5222b7ab5b9326049060e85a Author: Stephen Mallette Date: 2018-07-12T12:17:56Z TINKERPOP-1996 none() doesn't need to be removed in HadoopIoStrategy Not sure why this was there in the first place. Removing it not allows Hadoop integration tests to pass, but seems to have no real effect on existing operations. commit 328737a371f8a2040d02f9c2dbb06d049ce3c881 Author: Stephen Mallette Date: 2018-07-12T15:19:10Z TINKERPOP-1996 Added IO to imports and javadoc fixes commit ae796378e07925f9385f3ec65c10022b59aab8b5 Author: Stephen Mallette Date: 2018-07-12T18:33:03Z TINKERPOP-1996 Deprecated Graph.io() and related infrastructure. commit 9e4da0149247a50277e2a468b0becf892426ce2e Author: Stephen Mallette Date: 2018-07-13T15:37:02Z TINKERPOP-1996 Fixed a bad method call for Configuring steps commit bd275a7ffa4f0d04634c830aa4f7577375c7944c Author: Stephen Mallette Date: 2018-07-13T15:37:21Z TINKERPOP-1996 Removed OptOuts for read()/write() tests Not necessary because existing checks ignore these. For read() you can't write to a HadoopGraph directly (i.e. create vertices/edges) and for write() (and technically read()) it is ignored as it requires a GraphComputer to work. commit f148e9331a945e0f4f707ea937b610e5902701c7 Author: Stephen Mallette Date: 2018-07-13T19:17:17Z TINKERPOP-1996 Got read/write() tests running for OLAP Introduced new Graph.Features to provider better separation between graph mutation features and graph loading features - they are two different things as demonstrated by io(). Fixed HadoopIoStep/Strategy so that they properly handle the different input/output format types expected. commit 576649fd5456f6390bf9481d01438a7e78db083e Author: Stephen Mallette Date: 2018-07-13T19:22:43Z TINKERPOP-1996 Updated changelog commit 62175c228b77bdbda96c11015f2974828df8f3aa Author: Stephen Mallette Date: 2018-07-13T21:31:46Z TINKERPOP-1996 Added docs for io() Killed all the old IO documentation that utilized the GraphReader/Writer classes directly as well as the Graph.io() method that is now deprecated. commit 6d05805ada657bcb3f50a60aa0c313c29d4611bb Author: Stephen Mallette Date: 2018-07-14T10:23:54Z TINKERPOP-1996 Moved IoStep implementations to sideEffect package These steps really a