Repository: arrow Updated Branches: refs/heads/master a8bf0fbc8 -> a9a570139
http://git-wip-us.apache.org/repos/asf/arrow/blob/a9a57013/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java ---------------------------------------------------------------------- diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java index 706f8e2..5970c57 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java @@ -6,28 +6,17 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * <p> * http://www.apache.org/licenses/LICENSE-2.0 - * + * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.arrow.tools; -import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.net.Socket; -import java.net.UnknownHostException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +package org.apache.arrow.tools; import com.google.common.collect.ImmutableList; @@ -57,6 +46,18 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.net.Socket; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class EchoServerTest { private static EchoServer server; @@ -94,8 +95,8 @@ public class EchoServerTest { BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE); VectorSchemaRoot root = new VectorSchemaRoot(asList(field), asList((FieldVector) vector), 0); try (Socket socket = new Socket("localhost", serverPort); - ArrowStreamWriter writer = new ArrowStreamWriter(root, null, socket.getOutputStream()); - ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), alloc)) { + ArrowStreamWriter writer = new ArrowStreamWriter(root, null, socket.getOutputStream()); + ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), alloc)) { writer.start(); for (int i = 0; i < batches; i++) { vector.allocateNew(16); @@ -111,7 +112,8 @@ public class EchoServerTest { assertEquals(new Schema(asList(field)), reader.getVectorSchemaRoot().getSchema()); - NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot().getFieldVectors().get(0); + NullableTinyIntVector readVector = (NullableTinyIntVector) reader.getVectorSchemaRoot() + .getFieldVectors().get(0); for (int i = 0; i < batches; i++) { reader.loadNextBatch(); assertEquals(16, reader.getVectorSchemaRoot().getRowCount()); @@ -131,7 +133,8 @@ public class EchoServerTest { public void basicTest() throws InterruptedException, IOException { BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE); - Field field = new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList()); + Field field = new Field("testField", true, new ArrowType.Int(8, true), Collections + .<Field>emptyList()); NullableTinyIntVector vector = new NullableTinyIntVector("testField", alloc, null); Schema schema = new Schema(asList(field)); @@ -150,7 +153,8 @@ public class EchoServerTest { DictionaryEncoding writeEncoding = new DictionaryEncoding(1L, false, null); try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); NullableIntVector writeVector = new NullableIntVector("varchar", allocator, writeEncoding); - NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dict", allocator, null)) { + NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dict", + allocator, null)) { writeVector.allocateNewSafe(); NullableIntVector.Mutator mutator = writeVector.getMutator(); mutator.set(0, 0); @@ -171,10 +175,12 @@ public class EchoServerTest { List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector); VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 6); - DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary(writeDictionaryVector, writeEncoding)); + DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary + (writeDictionaryVector, writeEncoding)); try (Socket socket = new Socket("localhost", serverPort); - ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket.getOutputStream()); + ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket + .getOutputStream()); ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) { writer.start(); writer.writeBatch(); @@ -202,7 +208,8 @@ public class EchoServerTest { Dictionary dictionary = reader.lookup(1L); Assert.assertNotNull(dictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary.getVector()).getAccessor(); + NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) dictionary + .getVector()).getAccessor(); Assert.assertEquals(3, dictionaryAccessor.getValueCount()); Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); @@ -215,7 +222,8 @@ public class EchoServerTest { public void testNestedDictionary() throws IOException { DictionaryEncoding writeEncoding = new DictionaryEncoding(2L, false, null); try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE); - NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dictionary", allocator, null); + NullableVarCharVector writeDictionaryVector = new NullableVarCharVector("dictionary", + allocator, null); ListVector writeVector = new ListVector("list", allocator, null, null)) { // data being written: @@ -245,10 +253,12 @@ public class EchoServerTest { List<FieldVector> vectors = ImmutableList.of((FieldVector) writeVector); VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 3); - DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary(writeDictionaryVector, writeEncoding)); + DictionaryProvider writeProvider = new MapDictionaryProvider(new Dictionary + (writeDictionaryVector, writeEncoding)); try (Socket socket = new Socket("localhost", serverPort); - ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket.getOutputStream()); + ArrowStreamWriter writer = new ArrowStreamWriter(root, writeProvider, socket + .getOutputStream()); ArrowStreamReader reader = new ArrowStreamReader(socket.getInputStream(), allocator)) { writer.start(); writer.writeBatch(); @@ -262,7 +272,8 @@ public class EchoServerTest { Assert.assertNotNull(readVector); Assert.assertNull(readVector.getField().getDictionary()); - DictionaryEncoding readEncoding = readVector.getField().getChildren().get(0).getDictionary(); + DictionaryEncoding readEncoding = readVector.getField().getChildren().get(0) + .getDictionary(); Assert.assertNotNull(readEncoding); Assert.assertEquals(2L, readEncoding.getId()); @@ -281,7 +292,8 @@ public class EchoServerTest { Dictionary readDictionary = reader.lookup(2L); Assert.assertNotNull(readDictionary); - NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) readDictionary.getVector()).getAccessor(); + NullableVarCharVector.Accessor dictionaryAccessor = ((NullableVarCharVector) + readDictionary.getVector()).getAccessor(); Assert.assertEquals(2, dictionaryAccessor.getValueCount()); Assert.assertEquals(new Text("foo"), dictionaryAccessor.getObject(0)); Assert.assertEquals(new Text("bar"), dictionaryAccessor.getObject(1)); http://git-wip-us.apache.org/repos/asf/arrow/blob/a9a57013/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java ---------------------------------------------------------------------- diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java index ee39f5e..78021f8 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/TestFileRoundtrip.java @@ -16,13 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.arrow.tools; -import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput; -import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput; -import static org.junit.Assert.assertEquals; - -import java.io.File; +package org.apache.arrow.tools; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -32,6 +27,12 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.File; + +import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput; +import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput; +import static org.junit.Assert.assertEquals; + public class TestFileRoundtrip { @Rule @@ -56,7 +57,7 @@ public class TestFileRoundtrip { writeInput(testInFile, allocator); - String[] args = { "-i", testInFile.getAbsolutePath(), "-o", testOutFile.getAbsolutePath()}; + String[] args = {"-i", testInFile.getAbsolutePath(), "-o", testOutFile.getAbsolutePath()}; int result = new FileRoundtrip(System.out, System.err).run(args); assertEquals(0, result); http://git-wip-us.apache.org/repos/asf/arrow/blob/a9a57013/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java ---------------------------------------------------------------------- diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java index 9d4ef5c..7d9a419 100644 --- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java +++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java @@ -16,22 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.arrow.tools; - -import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput; -import static org.apache.arrow.tools.ArrowFileTestFixtures.write; -import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData; -import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.StringReader; -import java.util.Map; +package org.apache.arrow.tools; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter; @@ -54,12 +40,75 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; +import java.util.Map; + +import static org.apache.arrow.tools.ArrowFileTestFixtures.validateOutput; +import static org.apache.arrow.tools.ArrowFileTestFixtures.write; +import static org.apache.arrow.tools.ArrowFileTestFixtures.writeData; +import static org.apache.arrow.tools.ArrowFileTestFixtures.writeInput; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + public class TestIntegration { @Rule public TemporaryFolder testFolder = new TemporaryFolder(); private BufferAllocator allocator; + private ObjectMapper om = new ObjectMapper(); + + { + DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter(); + prettyPrinter.indentArraysWith(NopIndenter.instance); + om.setDefaultPrettyPrinter(prettyPrinter); + om.enable(SerializationFeature.INDENT_OUTPUT); + om.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + } + + static void writeInputFloat(File testInFile, BufferAllocator allocator, double... f) throws + FileNotFoundException, IOException { + try ( + BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, + Integer.MAX_VALUE); + MapVector parent = new MapVector("parent", vectorAllocator, null)) { + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + Float8Writer floatWriter = rootWriter.float8("float"); + for (int i = 0; i < f.length; i++) { + floatWriter.setPosition(i); + floatWriter.writeFloat8(f[i]); + } + writer.setValueCount(f.length); + write(parent.getChild("root"), testInFile); + } + } + + static void writeInput2(File testInFile, BufferAllocator allocator) throws + FileNotFoundException, IOException { + int count = ArrowFileTestFixtures.COUNT; + try ( + BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, + Integer.MAX_VALUE); + MapVector parent = new MapVector("parent", vectorAllocator, null)) { + writeData(count, parent); + ComplexWriter writer = new ComplexWriterImpl("root", parent); + MapWriter rootWriter = writer.rootAsMap(); + IntWriter intWriter = rootWriter.integer("int"); + BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt"); + intWriter.setPosition(5); + intWriter.writeInt(999); + bigIntWriter.setPosition(4); + bigIntWriter.writeBigInt(777L); + writer.setValueCount(count); + write(parent.getChild("root"), testInFile); + } + } @Before public void init() { @@ -85,18 +134,21 @@ public class TestIntegration { Integration integration = new Integration(); // convert it to json - String[] args1 = { "-arrow", testInFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; + String[] args1 = {"-arrow", testInFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; integration.run(args1); // convert back to arrow - String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()}; + String[] args2 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()}; integration.run(args2); // check it is the same validateOutput(testOutFile, allocator); // validate arrow against json - String[] args3 = { "-arrow", testInFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()}; + String[] args3 = {"-arrow", testInFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.VALIDATE.name()}; integration.run(args3); } @@ -111,11 +163,13 @@ public class TestIntegration { Integration integration = new Integration(); // convert to arrow - String[] args1 = { "-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()}; + String[] args1 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()}; integration.run(args1); // convert back to json - String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json", testRoundTripJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; + String[] args2 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testRoundTripJSONFile + .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; integration.run(args2); BufferedReader orig = readNormalized(testJSONFile); @@ -139,11 +193,13 @@ public class TestIntegration { Integration integration = new Integration(); // convert to arrow - String[] args1 = { "-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()}; + String[] args1 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.JSON_TO_ARROW.name()}; integration.run(args1); // convert back to json - String[] args2 = { "-arrow", testOutFile.getAbsolutePath(), "-json", testRoundTripJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; + String[] args2 = {"-arrow", testOutFile.getAbsolutePath(), "-json", testRoundTripJSONFile + .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; integration.run(args2); BufferedReader orig = readNormalized(testJSONFile); @@ -156,22 +212,12 @@ public class TestIntegration { } } - private ObjectMapper om = new ObjectMapper(); - { - DefaultPrettyPrinter prettyPrinter = new DefaultPrettyPrinter(); - prettyPrinter.indentArraysWith(NopIndenter.instance); - om.setDefaultPrettyPrinter(prettyPrinter); - om.enable(SerializationFeature.INDENT_OUTPUT); - om.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); - } - private BufferedReader readNormalized(File f) throws IOException { - Map<?,?> tree = om.readValue(f, Map.class); + Map<?, ?> tree = om.readValue(f, Map.class); String normalized = om.writeValueAsString(tree); return new BufferedReader(new StringReader(normalized)); } - /** * the test should not be sensitive to small variations in float representation */ @@ -190,11 +236,13 @@ public class TestIntegration { Integration integration = new Integration(); // convert the "valid" file to json - String[] args1 = { "-arrow", testValidInFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; + String[] args1 = {"-arrow", testValidInFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; integration.run(args1); // compare the "invalid" file to the "valid" json - String[] args3 = { "-arrow", testInvalidInFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()}; + String[] args3 = {"-arrow", testInvalidInFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.VALIDATE.name()}; // this should fail integration.run(args3); } @@ -214,11 +262,13 @@ public class TestIntegration { Integration integration = new Integration(); // convert the "valid" file to json - String[] args1 = { "-arrow", testValidInFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; + String[] args1 = {"-arrow", testValidInFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.ARROW_TO_JSON.name()}; integration.run(args1); // compare the "invalid" file to the "valid" json - String[] args3 = { "-arrow", testInvalidInFile.getAbsolutePath(), "-json", testJSONFile.getAbsolutePath(), "-command", Command.VALIDATE.name()}; + String[] args3 = {"-arrow", testInvalidInFile.getAbsolutePath(), "-json", testJSONFile + .getAbsolutePath(), "-command", Command.VALIDATE.name()}; // this should fail try { integration.run(args3); @@ -229,39 +279,4 @@ public class TestIntegration { } } - - static void writeInputFloat(File testInFile, BufferAllocator allocator, double... f) throws FileNotFoundException, IOException { - try ( - BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", vectorAllocator, null)) { - ComplexWriter writer = new ComplexWriterImpl("root", parent); - MapWriter rootWriter = writer.rootAsMap(); - Float8Writer floatWriter = rootWriter.float8("float"); - for (int i = 0; i < f.length; i++) { - floatWriter.setPosition(i); - floatWriter.writeFloat8(f[i]); - } - writer.setValueCount(f.length); - write(parent.getChild("root"), testInFile); - } - } - - static void writeInput2(File testInFile, BufferAllocator allocator) throws FileNotFoundException, IOException { - int count = ArrowFileTestFixtures.COUNT; - try ( - BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE); - MapVector parent = new MapVector("parent", vectorAllocator, null)) { - writeData(count, parent); - ComplexWriter writer = new ComplexWriterImpl("root", parent); - MapWriter rootWriter = writer.rootAsMap(); - IntWriter intWriter = rootWriter.integer("int"); - BigIntWriter bigIntWriter = rootWriter.bigInt("bigInt"); - intWriter.setPosition(5); - intWriter.writeInt(999); - bigIntWriter.setPosition(4); - bigIntWriter.writeBigInt(777L); - writer.setValueCount(count); - write(parent.getChild("root"), testInFile); - } - } }
