http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java index fed2914..0785da6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java @@ -40,7 +40,7 @@ import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.record.WritableBatch; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -306,7 +306,7 @@ public class TestLoad extends ExecTest { BatchSchema schema1 = new SchemaBuilder() .add("a", MinorType.INT) .addMap("m") - .buildMap() + .resumeSchema() .build(); { assertTrue(loadBatch(allocator, batchLoader, schema1)); @@ -330,7 +330,7 @@ public class TestLoad extends ExecTest { .add("a", MinorType.INT) .addMap("m") .add("b", MinorType.VARCHAR) - .buildMap() + .resumeSchema() .build(); { assertTrue(loadBatch(allocator, batchLoader, schema2)); @@ -356,7 +356,7 @@ public class TestLoad extends ExecTest { .addMap("m") .add("b", MinorType.VARCHAR) .add("c", MinorType.INT) - .buildMap() + .resumeSchema() .build(); assertTrue(loadBatch(allocator, batchLoader, schema)); assertTrue(schema.isEquivalent(batchLoader.getSchema())); @@ -371,7 +371,7 @@ public class TestLoad extends ExecTest { .add("a", MinorType.INT) .addMap("m") .add("b", MinorType.VARCHAR) - .buildMap() + .resumeSchema() .build(); assertTrue(loadBatch(allocator, batchLoader, schema)); assertTrue(schema.isEquivalent(batchLoader.getSchema())); @@ -386,7 +386,7 @@ public class TestLoad extends ExecTest { .add("a", MinorType.INT) .addMap("m") .add("b", MinorType.INT) - .buildMap() + .resumeSchema() .build(); assertTrue(loadBatch(allocator, batchLoader, schema)); assertTrue(schema.isEquivalent(batchLoader.getSchema()));
http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java index 324fc73..1629a83 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java @@ -20,11 +20,11 @@ package org.apache.drill.exec.store; import com.google.common.base.Charsets; import com.google.common.io.Files; import org.apache.drill.test.BaseTestQuery; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.util.Text; -import org.apache.drill.test.rowSet.SchemaBuilder; import org.junit.BeforeClass; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java index 5b055af..3d22b26 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java @@ -33,7 +33,7 @@ import org.apache.drill.test.ClusterTest; import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSetBuilder; import org.apache.drill.test.rowSet.RowSetComparison; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.BeforeClass; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java index a56c8c6..7bccb33 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java @@ -24,8 +24,8 @@ import org.apache.drill.PlanTestBase; import org.apache.drill.categories.UnlikelyTest; import org.apache.commons.io.FileUtils; import org.apache.drill.exec.record.BatchSchema; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.exec.planner.physical.PlannerSettings; -import org.apache.drill.test.rowSet.SchemaBuilder; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java index 69667a8..8bd7af6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ExampleTest.java @@ -35,8 +35,8 @@ import org.apache.drill.test.LogFixture.LogFixtureBuilder; import org.apache.drill.test.QueryBuilder.QuerySummary; import org.apache.drill.test.rowSet.RowSet; import org.apache.drill.test.rowSet.RowSetBuilder; -import org.apache.drill.test.rowSet.SchemaBuilder; import org.apache.drill.test.rowSet.file.JsonFileBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java index c1a98ac..dfeaad1 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetBuilder.java @@ -74,13 +74,13 @@ public final class RowSetBuilder { */ public RowSetBuilder addRow(Object...values) { - writer.setRow(values); + writer.addRow(values); return this; } public RowSetBuilder addSelection(boolean selected, Object...values) { final int index = writer.rowIndex(); - writer.setRow(values); + writer.addRow(values); if (!selected) { skipIndices.add(index); http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java index 32b61ca..01c49ad 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetUtilities.java @@ -84,11 +84,11 @@ public class RowSetUtilities { case BIT: return value & 0x01; case SMALLINT: - return value % 32768; + return value & 0x7FFF; case UINT2: return value & 0xFFFF; case TINYINT: - return value % 128; + return value & 0x7F; case UINT1: return value & 0xFF; default: @@ -170,4 +170,73 @@ public class RowSetUtilities { throw new IllegalStateException( "Unexpected type: " + type); } } + + public static Object[] mapValue(Object... members) { + return members; + } + + public static Object[] singleMap(Object member) { + return new Object[] { member }; + } + + public static byte[] byteArray(Integer... elements) { + byte array[] = new byte[elements.length]; + for (int i = 0; i < elements.length; i++) { + array[i] = (byte) (int) elements[i]; + } + return array; + } + + public static Long[] longArray(Long... elements) { + return elements; + } + + public static double[] doubleArray(Double... elements) { + double array[] = new double[elements.length]; + for (int i = 0; i < elements.length; i++) { + array[i] = elements[i]; + } + return array; + } + + public static String[] strArray(String... elements) { + return elements; + } + + public static int[] intArray(Integer... elements) { + int array[] = new int[elements.length]; + for (int i = 0; i < elements.length; i++) { + array[i] = elements[i]; + } + return array; + } + + public static Object[][] mapArray(Object[]... elements) { + return elements; + } + + public static Object[] variantArray(Object... elements) { + return elements; + } + + public static Object[] listValue(Object... elements) { + return elements; + } + + public static Object[] singleList(Object element) { + return new Object[] { element }; + } + + public static Object[] objArray(Object... elements) { + return elements; + } + + public static Object[] singleObjArray(Object element) { + return new Object[] {element}; + } + + public static void verify(RowSet expected, RowSet actual) { + new RowSetComparison(expected).verifyAndClearAll(actual); + } + } http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriter.java index 874c0e1..f7abe35 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriter.java @@ -82,7 +82,8 @@ public interface RowSetWriter extends TupleWriter { * @param values variable-length argument list of column values */ - void setRow(Object...values); + RowSetWriter addRow(Object...values); + RowSetWriter addSingleCol(Object value); /** * Indicates if the current row position is valid for http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java index b649b11..aa7fb2f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetWriterImpl.java @@ -117,9 +117,15 @@ public class RowSetWriterImpl extends AbstractTupleWriter implements RowSetWrite } @Override - public void setRow(Object...values) { + public RowSetWriter addRow(Object...values) { setObject(values); save(); + return this; + } + + @Override + public RowSetWriter addSingleCol(Object value) { + return addRow(new Object[] {value}); } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java deleted file mode 100644 index 3d4df05..0000000 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/SchemaBuilder.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.test.rowSet; - -import org.apache.drill.common.types.TypeProtos.DataMode; -import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.record.BatchSchema; -import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.record.metadata.ColumnMetadata; -import org.apache.drill.exec.record.metadata.MapColumnMetadata; -import org.apache.drill.exec.record.metadata.MetadataUtils; -import org.apache.drill.exec.record.metadata.TupleMetadata; -import org.apache.drill.exec.record.metadata.TupleSchema; - -/** - * Builder of a row set schema expressed as a list of materialized - * fields. Optimized for use when creating schemas by hand in tests. - * <p> - * Example usage to create the following schema: <br> - * <tt>(c: INT, a: MAP(b: VARCHAR, d: INT, e: MAP(f: VARCHAR), g: INT), h: BIGINT)</tt> - * <p> - * Code:<pre><code> - * BatchSchema batchSchema = new SchemaBuilder() - * .add("c", MinorType.INT) - * .addMap("a") - * .addNullable("b", MinorType.VARCHAR) - * .add("d", MinorType.INT) - * .addMap("e") - * .add("f", MinorType.VARCHAR) - * .buildMap() - * .add("g", MinorType.INT) - * .buildMap() - * .addArray("h", MinorType.BIGINT) - * .build(); - * </code</pre> - */ - -public class SchemaBuilder { - - /** - * Build a column schema (AKA "materialized field") based on name and a - * variety of schema options. Every column needs a name and (minor) type, - * some may need a mode other than required, may need a width, may - * need scale and precision, and so on. - */ - - public static class ColumnBuilder { - private final String name; - private final MajorType.Builder typeBuilder; - - public ColumnBuilder(String name, MinorType type) { - this.name = name; - typeBuilder = MajorType.newBuilder() - .setMinorType(type) - .setMode(DataMode.REQUIRED); - } - - public ColumnBuilder setMode(DataMode mode) { - typeBuilder.setMode(mode); - return this; - } - - public ColumnBuilder setWidth(int width) { - return setPrecision(width); - } - - public ColumnBuilder setPrecision(int precision) { - typeBuilder.setPrecision(precision); - return this; - } - - public ColumnBuilder setScale(int scale, int precision) { - typeBuilder.setScale(scale); - typeBuilder.setPrecision(precision); - return this; - } - - public MaterializedField build() { - return MaterializedField.create(name, typeBuilder.build()); - } - } - - /** - * Internal structure for building a map. A map is just a schema, - * but one that is part of a parent column. - */ - - public static class MapBuilder extends SchemaBuilder { - private final SchemaBuilder parent; - private final String memberName; - private final DataMode mode; - - public MapBuilder(SchemaBuilder parent, String memberName, DataMode mode) { - this.parent = parent; - this.memberName = memberName; - // Optional maps not supported in Drill - assert mode != DataMode.OPTIONAL; - this.mode = mode; - } - - @Override - public BatchSchema build() { - throw new IllegalStateException("Cannot build for a nested schema"); - } - - @Override - public SchemaBuilder buildMap() { - // TODO: Use the map schema directly rather than - // rebuilding it as is done here. - - MaterializedField col = columnSchema(memberName, MinorType.MAP, mode); - for (ColumnMetadata md : schema) { - col.addChild(md.schema()); - } - parent.finishMap(MetadataUtils.newMap(col, schema)); - return parent; - } - - @Override - public SchemaBuilder withSVMode(SelectionVectorMode svMode) { - throw new IllegalStateException("Cannot set SVMode for a nested schema"); - } - } - - protected TupleSchema schema = new TupleSchema(); - private SelectionVectorMode svMode = SelectionVectorMode.NONE; - - /** - * Create a new empty schema. Allows appending columns to it. - */ - public SchemaBuilder() {} - - /** - * Create a new schema starting with the base schema. Allows appending additional columns to the actual schema. - */ - public SchemaBuilder(BatchSchema baseSchema) { - for (MaterializedField field : baseSchema) { - add(field); - } - } - - public SchemaBuilder add(String name, MajorType type) { - return add(MaterializedField.create(name, type)); - } - - public SchemaBuilder add(MaterializedField col) { - schema.add(col); - return this; - } - - /** - * Create a column schema using the "basic three" properties of name, type and - * cardinality (AKA "data mode.") Use the {@link ColumnBuilder} for to set - * other schema attributes. Name is relative to the enclosing map or tuple; - * it is not the fully qualified path name. - */ - - public static MaterializedField columnSchema(String name, MinorType type, DataMode mode) { - return MaterializedField.create(name, - MajorType.newBuilder() - .setMinorType(type) - .setMode(mode) - .build()); - } - - public SchemaBuilder add(String name, MinorType type, DataMode mode) { - return add(columnSchema(name, type, mode)); - } - - public SchemaBuilder add(String name, MinorType type) { - return add(name, type, DataMode.REQUIRED); - } - - public SchemaBuilder add(String name, MinorType type, int width) { - MaterializedField field = new SchemaBuilder.ColumnBuilder(name, type) - .setMode(DataMode.REQUIRED) - .setWidth(width) - .build(); - return add(field); - } - - public SchemaBuilder addNullable(String name, MinorType type) { - return add(name, type, DataMode.OPTIONAL); - } - - public SchemaBuilder addNullable(String name, MinorType type, int width) { - MaterializedField field = new SchemaBuilder.ColumnBuilder(name, type) - .setMode(DataMode.OPTIONAL) - .setWidth(width) - .build(); - return add(field); - } - - public SchemaBuilder addArray(String name, MinorType type) { - return add(name, type, DataMode.REPEATED); - } - - /** - * Add a map column. The returned schema builder is for the nested - * map. Building that map, using {@link MapBuilder#buildMap()}, - * will return the original schema builder. - * - * @param pathName the name of the map column - * @return a builder for the map - */ - - public MapBuilder addMap(String pathName) { - return new MapBuilder(this, pathName, DataMode.REQUIRED); - } - - public MapBuilder addMapArray(String pathName) { - return new MapBuilder(this, pathName, DataMode.REPEATED); - } - - public SchemaBuilder withSVMode(SelectionVectorMode svMode) { - this.svMode = svMode; - return this; - } - - public BatchSchema build() { - return schema.toBatchSchema(svMode); - } - - void finishMap(MapColumnMetadata map) { - schema.add(map); - } - - public SchemaBuilder buildMap() { - throw new IllegalStateException("Cannot build map for a top-level schema"); - } - - public TupleMetadata buildSchema() { - return schema; - } -} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/ColumnBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/ColumnBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/ColumnBuilder.java new file mode 100644 index 0000000..276b4c0 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/ColumnBuilder.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.schema; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.MaterializedField; + +/** + * Build a column schema (AKA "materialized field") based on name and a + * variety of schema options. Every column needs a name and (minor) type, + * some may need a mode other than required, may need a width, may + * need scale and precision, and so on. + */ + +public class ColumnBuilder { + private final String name; + private final MajorType.Builder typeBuilder; + + public ColumnBuilder(String name, MinorType type) { + this.name = name; + typeBuilder = MajorType.newBuilder() + .setMinorType(type) + .setMode(DataMode.REQUIRED); + } + + public ColumnBuilder setMode(DataMode mode) { + typeBuilder.setMode(mode); + return this; + } + + public ColumnBuilder setWidth(int width) { + return setPrecision(width); + } + + public ColumnBuilder setPrecision(int precision) { + typeBuilder.setPrecision(precision); + return this; + } + + public ColumnBuilder setScale(int scale, int precision) { + typeBuilder.setScale(scale); + typeBuilder.setPrecision(precision); + return this; + } + + public MaterializedField build() { + return MaterializedField.create(name, typeBuilder.build()); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/MapBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/MapBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/MapBuilder.java new file mode 100644 index 0000000..0c5ef97 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/MapBuilder.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.schema; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; +import org.apache.drill.exec.record.metadata.MapColumnMetadata; + +/** + * Internal structure for building a map. A map is just a schema, + * but one that is part of a parent column. + */ + +public class MapBuilder implements SchemaContainer { + private final SchemaContainer parent; + private final TupleBuilder tupleBuilder = new TupleBuilder(); + private final String memberName; + private final DataMode mode; + + public MapBuilder(SchemaContainer parent, String memberName, DataMode mode) { + this.parent = parent; + this.memberName = memberName; + this.mode = mode; + } + + @Override + public void addColumn(AbstractColumnMetadata column) { + tupleBuilder.addColumn(column); + } + + public MapBuilder add(String name, MajorType type) { + return add(MaterializedField.create(name, type)); + } + + public MapBuilder add(MaterializedField col) { + tupleBuilder.add(col); + return this; + } + + public MapBuilder add(String name, MinorType type, DataMode mode) { + tupleBuilder.add(name, type, mode); + return this; + } + + public MapBuilder add(String name, MinorType type) { + tupleBuilder.add(name, type); + return this; + } + + public MapBuilder add(String name, MinorType type, int width) { + tupleBuilder.add(name, type, width); + return this; + } + + public MapBuilder addNullable(String name, MinorType type) { + tupleBuilder.addNullable(name, type); + return this; + } + + public MapBuilder addNullable(String name, MinorType type, int width) { + tupleBuilder.addNullable(name, type, width); + return this; + } + + public MapBuilder addArray(String name, MinorType type) { + tupleBuilder.addArray(name, type); + return this; + } + + public MapBuilder addArray(String name, MinorType type, int dims) { + tupleBuilder.addArray(name, type, dims); + return this; + } + + public MapBuilder addDecimal(String name, MinorType type, + DataMode mode, int precision, int scale) { + tupleBuilder.addDecimal(name, type, mode, precision, scale); + return this; + } + + /** + * Add a map column. The returned schema builder is for the nested + * map. Building that map, using {@link MapBuilder#resumeSchema()}, + * will return the original schema builder. + * + * @param pathName the name of the map column + * @return a builder for the map + */ + + public MapBuilder addMap(String name) { + return tupleBuilder.addMap(this, name); + } + + public MapBuilder addMapArray(String name) { + return tupleBuilder.addMapArray(this, name); + } + + public UnionBuilder addUnion(String name) { + return tupleBuilder.addUnion(this, name); + } + + public UnionBuilder addList(String name) { + return tupleBuilder.addList(this, name); + } + + public RepeatedListBuilder addRepeatedList(String name) { + return tupleBuilder.addRepeatedList(this, name); + } + + private MapColumnMetadata buildCol() { + return new MapColumnMetadata(memberName, mode, tupleBuilder.schema()); + } + + public SchemaBuilder resumeSchema() { + parent.addColumn(buildCol()); + return (SchemaBuilder) parent; + } + + public MapBuilder resumeMap() { + parent.addColumn(buildCol()); + return (MapBuilder) parent; + } + + public RepeatedListBuilder resumeList() { + parent.addColumn(buildCol()); + return (RepeatedListBuilder) parent; + } + + public UnionBuilder resumeUnion() { + // TODO: Use the map schema directly rather than + // rebuilding it as is done here. + + parent.addColumn(buildCol()); + return (UnionBuilder) parent; + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/RepeatedListBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/RepeatedListBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/RepeatedListBuilder.java new file mode 100644 index 0000000..25a0825 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/RepeatedListBuilder.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.schema; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.RepeatedListColumnMetadata; + +/** + * Builder for a repeated list. Drill's metadata represents a repeated + * list as a chain of materialized fields and that is the pattern used + * here. It would certainly be cleaner to have a single field, with the + * number of dimensions as a property, but that is not how Drill evolved. + */ + +public class RepeatedListBuilder implements SchemaContainer { + + private final SchemaContainer parent; + private final String name; + private AbstractColumnMetadata child; + + public RepeatedListBuilder(SchemaContainer parent, String name) { + this.parent = parent; + this.name = name; + } + + public RepeatedListBuilder addDimension() { + return new RepeatedListBuilder(this, name); + } + + public MapBuilder addMapArray() { + // Existing code uses the repeated list name as the name of + // the vector within the list. + + return new MapBuilder(this, name, DataMode.REPEATED); + } + + public RepeatedListBuilder addArray(MinorType type) { + // Existing code uses the repeated list name as the name of + // the vector within the list. + + addColumn(MetadataUtils.newScalar(name, type, DataMode.REPEATED)); + return this; + } + + private RepeatedListColumnMetadata buildCol() { + return MetadataUtils.newRepeatedList(name, child); + } + + public void build() { + parent.addColumn(buildCol()); + } + + public RepeatedListBuilder resumeList() { + build(); + return (RepeatedListBuilder) parent; + } + + public SchemaBuilder resumeSchema() { + build(); + return (SchemaBuilder) parent; + } + + public UnionBuilder resumeUnion() { + build(); + return (UnionBuilder) parent; + } + + public MapBuilder resumeMap() { + build(); + return (MapBuilder) parent; + } + + @Override + public void addColumn(AbstractColumnMetadata column) { + assert child == null; + child = column; + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaBuilder.java new file mode 100644 index 0000000..2cc82a5 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaBuilder.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.schema; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.BatchSchema; +import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleMetadata; + +/** + * Builder of a row set schema expressed as a list of materialized + * fields. Optimized for use when creating schemas by hand in tests. + * <p> + * Example usage to create the following schema: <br> + * <tt>(c: INT, a: MAP(b: VARCHAR, d: INT, e: MAP(f: VARCHAR), g: INT), + * h: UNION(INT, MAP(h1: INT), LIST(BIGINT)), + * i: BIGINT[], j: VARCHAR[][][])</tt> + * <p> + * Code:<pre><code> + * BatchSchema batchSchema = new SchemaBuilder() + * .add("c", MinorType.INT) + * .addMap("a") + * .addNullable("b", MinorType.VARCHAR) + * .add("d", MinorType.INT) + * .addMap("e") // or .addMapArray("e") + * .add("f", MinorType.VARCHAR) + * .buildMap() + * .add("g", MinorType.INT) + * .buildMap() + * .addUnion("h") // or .addList("h") + * .addType(MinorType.INT) + * .addMap() + * .add("h1", MinorType.INT) + * .buildNested() + * .addList() + * .addType(MinorType.BIGINT) + * .buildNested() + * .build() + * .addArray("i", MinorType.BIGINT) + * .addRepeatedList("j") + * .addDimension() + * .addArray(MinorType.VARCHAR) + * .endDimension() + * .build() + * .build(); + * </code</pre> + */ + +public class SchemaBuilder implements SchemaContainer { + + /** + * Actual tuple schema builder. The odd layered structure is needed + * so that the return value of each method is the builder + * itself. We have two: one for the top-level schema, another for + * maps. (The list, repeated list, and union builders are similar, + * but they are each unique, so don't share "guts".) + */ + + private TupleBuilder tupleBuilder = new TupleBuilder(); + private SelectionVectorMode svMode = SelectionVectorMode.NONE; + + public SchemaBuilder() { } + + /** + * Create a new schema starting with the base schema. Allows appending + * additional columns to an additional schema. + */ + + public SchemaBuilder(BatchSchema baseSchema) { + for (MaterializedField field : baseSchema) { + add(field); + } + } + + /** + * Create a column schema using the "basic three" properties of name, type and + * cardinality (AKA "data mode.") Use the {@link ColumnBuilder} for to set + * other schema attributes. Name is relative to the enclosing map or tuple; + * it is not the fully qualified path name. + */ + + public static MaterializedField columnSchema(String name, MinorType type, DataMode mode) { + return MaterializedField.create(name, + MajorType.newBuilder() + .setMinorType(type) + .setMode(mode) + .build()); + } + + @Override + public void addColumn(AbstractColumnMetadata column) { + tupleBuilder.addColumn(column); + } + + public SchemaBuilder add(String name, MajorType type) { + return add(MaterializedField.create(name, type)); + } + + public SchemaBuilder add(MaterializedField col) { + tupleBuilder.add(col); + return this; + } + + public SchemaBuilder add(String name, MinorType type, DataMode mode) { + tupleBuilder.add(name, type, mode); + return this; + } + + public SchemaBuilder add(String name, MinorType type) { + tupleBuilder.add(name, type); + return this; + } + + public SchemaBuilder add(String name, MinorType type, int width) { + tupleBuilder.add(name, type, width); + return this; + } + + public SchemaBuilder addNullable(String name, MinorType type) { + tupleBuilder.addNullable(name, type); + return this; + } + + public SchemaBuilder addNullable(String name, MinorType type, int width) { + tupleBuilder.addNullable(name, type, width); + return this; + } + + public SchemaBuilder addArray(String name, MinorType type) { + tupleBuilder.addArray(name, type); + return this; + } + + public SchemaBuilder addDecimal(String name, MinorType type, DataMode mode, int precision, int scale) { + tupleBuilder.addDecimal(name, type, mode, precision, scale); + return this; + } + + /** + * Add a multi-dimensional array, implemented as a repeated vector + * along with 0 or more repeated list vectors. + * + * @param name column name + * @param type base data type + * @param dims number of dimensions, 1 or more + * @return this builder + */ + + public SchemaBuilder addArray(String name, MinorType type, int dims) { + tupleBuilder.addArray(name, type, dims); + return this; + } + + /** + * Add a map column. The returned schema builder is for the nested + * map. Building that map, using {@link MapBuilder#resumeSchema()}, + * will return the original schema builder. + * + * @param pathName the name of the map column + * @return a builder for the map + */ + + public MapBuilder addMap(String name) { + return tupleBuilder.addMap(this, name); + } + + public MapBuilder addMapArray(String name) { + return tupleBuilder.addMapArray(this, name); + } + + public UnionBuilder addUnion(String name) { + return tupleBuilder.addUnion(this, name); + } + + public UnionBuilder addList(String name) { + return tupleBuilder.addList(this, name); + } + + public RepeatedListBuilder addRepeatedList(String name) { + return tupleBuilder.addRepeatedList(this, name); + } + + public SchemaBuilder withSVMode(SelectionVectorMode svMode) { + this.svMode = svMode; + return this; + } + + public BatchSchema build() { + return tupleBuilder.batchSchema(svMode); + } + + public TupleMetadata buildSchema() { + return tupleBuilder.schema(); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaContainer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaContainer.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaContainer.java new file mode 100644 index 0000000..f7b3483 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/SchemaContainer.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.schema; + +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; + +/** + * Magic that allows one schema builder to nest inside another + * without needing to know the type of the parent. + */ + +interface SchemaContainer { + void addColumn(AbstractColumnMetadata column); +} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/TupleBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/TupleBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/TupleBuilder.java new file mode 100644 index 0000000..b88392f --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/TupleBuilder.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.schema; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MajorType; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.BatchSchema; +import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; +import org.apache.drill.exec.record.metadata.TupleSchema; + +/** + * Internal tuple builder shared by the schema and map builders. + * Those two classes can't inherit from this class because their + * versions of the "add" methods return themselves to allow fluent + * construction. + */ + +class TupleBuilder implements SchemaContainer { + + protected TupleSchema schema = new TupleSchema(); + + @Override + public void addColumn(AbstractColumnMetadata column) { + schema.add(column); + } + + public void add(String name, MajorType type) { + add(MaterializedField.create(name, type)); + } + + public void add(MaterializedField col) { + schema.add(col); + } + + public void add(String name, MinorType type, DataMode mode) { + add(SchemaBuilder.columnSchema(name, type, mode)); + } + + public void add(String name, MinorType type) { + add(name, type, DataMode.REQUIRED); + } + + public void add(String name, MinorType type, int width) { + MaterializedField field = new ColumnBuilder(name, type) + .setMode(DataMode.REQUIRED) + .setWidth(width) + .build(); + add(field); + } + + public void addNullable(String name, MinorType type) { + add(name, type, DataMode.OPTIONAL); + } + + public void addNullable(String name, MinorType type, int width) { + MaterializedField field = new ColumnBuilder(name, type) + .setMode(DataMode.OPTIONAL) + .setWidth(width) + .build(); + add(field); + } + + public void addArray(String name, MinorType type) { + add(name, type, DataMode.REPEATED); + } + + public void addDecimal(String name, MinorType type, DataMode mode, + int precision, int scale) { + MaterializedField field = new ColumnBuilder(name, type) + .setMode(mode) + .setScale(scale, precision) + .build(); + add(field); + } + + /** + * Add a multi-dimensional array, implemented as a repeated vector + * along with 0 or more repeated list vectors. + * + * @param name column name + * @param type base data type + * @param dims number of dimensions, 1 or more + * @return this builder + */ + + public void addArray(String name, MinorType type, int dims) { + assert dims >= 1; + if (dims == 1) { + addArray(name, type); + return; + } + RepeatedListBuilder listBuilder = addRepeatedList(this, name); + buildMultiDimArray(listBuilder, type, dims - 1); + listBuilder.build(); + } + + private void buildMultiDimArray(RepeatedListBuilder listBuilder, + MinorType type, int dims) { + if (dims == 1) { + listBuilder.addArray(type); + } else { + RepeatedListBuilder childBuilder = listBuilder.addDimension(); + buildMultiDimArray(childBuilder, type, dims - 1); + childBuilder.build(); + } + } + + /** + * Add a map column. The returned schema builder is for the nested + * map. Building that map, using {@link MapBuilder#resumeSchema()}, + * will return the original schema builder. + * + * @param pathName the name of the map column + * @return a builder for the map + */ + + public MapBuilder addMap(SchemaContainer parent, String name) { + return new MapBuilder(parent, name, DataMode.REQUIRED); + } + + public MapBuilder addMapArray(SchemaContainer parent, String name) { + return new MapBuilder(parent, name, DataMode.REPEATED); + } + + public UnionBuilder addUnion(SchemaContainer parent, String name) { + return new UnionBuilder(parent, name, MinorType.UNION, DataMode.OPTIONAL); + } + + public UnionBuilder addList(SchemaContainer parent, String name) { + return new UnionBuilder(parent, name, MinorType.LIST, DataMode.REPEATED); + } + + public RepeatedListBuilder addRepeatedList(SchemaContainer parent, String name) { + return new RepeatedListBuilder(parent, name); + } + + void finish(AbstractColumnMetadata col) { + schema.add(col); + } + + public BatchSchema batchSchema(SelectionVectorMode svMode) { + return schema.toBatchSchema(svMode); + } + + public TupleSchema schema() { + return schema; + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java new file mode 100644 index 0000000..e47e552 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/UnionBuilder.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.schema; + +import org.apache.drill.common.types.Types; +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; +import org.apache.drill.exec.record.metadata.VariantColumnMetadata; +import org.apache.drill.exec.record.metadata.VariantSchema; + +/** + * Builds unions or (non-repeated) lists (which implicitly contain + * unions.) + */ + +public class UnionBuilder implements SchemaContainer { + private final SchemaContainer parent; + private final String name; + private final MinorType type; + private final DataMode mode; + private final VariantSchema union; + + public UnionBuilder(SchemaContainer parent, String name, + MinorType type, DataMode mode) { + this.parent = parent; + this.name = name; + this.type = type; + this.mode = mode; + union = new VariantSchema(); + } + + private void checkType(MinorType type) { + if (union.hasType(type)) { + throw new IllegalArgumentException("Duplicate type: " + type); + } + } + + @Override + public void addColumn(AbstractColumnMetadata column) { + assert column.name().equals(Types.typeKey(column.type())); + union.addType(column); + } + + public UnionBuilder addType(MinorType type) { + checkType(type); + union.addType(type); + return this; + } + + public MapBuilder addMap() { + checkType(MinorType.MAP); + return new MapBuilder(this, Types.typeKey(MinorType.MAP), DataMode.OPTIONAL); + } + + public UnionBuilder addList() { + checkType(MinorType.LIST); + return new UnionBuilder(this, Types.typeKey(MinorType.LIST), + MinorType.LIST, DataMode.OPTIONAL); + } + + public RepeatedListBuilder addRepeatedList() { + checkType(MinorType.LIST); + return new RepeatedListBuilder(this, Types.typeKey(MinorType.LIST)); + } + + private VariantColumnMetadata buildCol() { + return new VariantColumnMetadata(name, type, union); + } + + public SchemaBuilder resumeSchema() { + parent.addColumn(buildCol()); + return (SchemaBuilder) parent; + } + + public UnionBuilder buildNested() { + parent.addColumn(buildCol()); + return (UnionBuilder) parent; + } + + public MapBuilder resumeMap() { + parent.addColumn(buildCol()); + return (MapBuilder) parent; + } + + public UnionBuilder resumeUnion() { + parent.addColumn(buildCol()); + return (UnionBuilder) parent; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/package-info.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/package-info.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/package-info.java new file mode 100644 index 0000000..d9ef775 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/schema/package-info.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Provides a fluent schema builder for use in tests. Handles all + * forms of Drill schemas, with emphasis on ease of use for the typical + * cases (flat schema or nested maps.) Enables construction of unions, + * union lists (AKA "list vector") repeated lists and combinations of + * the above structures. + */ + +package org.apache.drill.test.rowSet.schema; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java index 46f1cc3..c5277f8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/DummyWriterTest.java @@ -28,7 +28,7 @@ import org.apache.drill.exec.vector.accessor.writer.AbstractObjectWriter; import org.apache.drill.exec.vector.accessor.writer.AbstractTupleWriter; import org.apache.drill.exec.vector.accessor.writer.ColumnWriterFactory; import org.apache.drill.test.SubOperatorTest; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; public class DummyWriterTest extends SubOperatorTest { @@ -117,10 +117,10 @@ public class DummyWriterTest extends SubOperatorTest { .addMap("m1") .add("a", MinorType.INT) .addArray("b", MinorType.VARCHAR) - .buildMap() + .resumeSchema() .addMapArray("m2") .add("c", MinorType.INT) - .buildMap() + .resumeSchema() .buildSchema(); List<AbstractObjectWriter> writers = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java index 4819253..a2caeed 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/PerformanceTool.java @@ -34,7 +34,7 @@ import org.apache.drill.exec.vector.accessor.writer.AbstractArrayWriter.ArrayObj import org.apache.drill.exec.vector.accessor.writer.NullableScalarWriter; import org.apache.drill.exec.vector.accessor.writer.ScalarArrayWriter; import org.apache.drill.test.OperatorFixture; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import com.google.common.base.Stopwatch; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java index 5ba1c54..9ad4133 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java @@ -17,6 +17,8 @@ */ package org.apache.drill.test.rowSet.test; +import static org.apache.drill.test.rowSet.RowSetUtilities.intArray; +import static org.apache.drill.test.rowSet.RowSetUtilities.objArray; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -30,6 +32,7 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.VectorOverflowException; import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ArrayWriter; import org.apache.drill.exec.vector.accessor.ObjectType; @@ -47,7 +50,7 @@ import org.apache.drill.test.rowSet.RowSet.SingleRowSet; import org.apache.drill.test.rowSet.RowSetComparison; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetWriter; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; /** @@ -262,10 +265,10 @@ public class RowSetTest extends SubOperatorTest { // utility classes. SingleRowSet expected = fixture.rowSetBuilder(schema) - .addSingleCol(new int[] {10, 11}) - .addSingleCol(new int[] {20, 21, 22}) - .addSingleCol(new int[] {30}) - .addSingleCol(new int[] {40, 41}) + .addSingleCol(intArray(10, 11)) + .addSingleCol(intArray(20, 21, 22)) + .addSingleCol(intArray(30)) + .addSingleCol(intArray(40, 41)) .build(); new RowSetComparison(expected) .verifyAndClearAll(actual); @@ -283,7 +286,7 @@ public class RowSetTest extends SubOperatorTest { .add("a", MinorType.INT) .addMap("m") .addArray("b", MinorType.INT) - .buildMap() + .resumeSchema() .buildSchema(); ExtendableRowSet rowSet = fixture.rowSet(schema); RowSetWriter writer = rowSet.writer(); @@ -374,9 +377,9 @@ public class RowSetTest extends SubOperatorTest { assertEquals(actual.rowCount(), mapVector.getAccessor().getValueCount()); SingleRowSet expected = fixture.rowSetBuilder(schema) - .addRow(10, new Object[] {new int[] {11, 12}}) - .addRow(20, new Object[] {new int[] {21, 22}}) - .addRow(30, new Object[] {new int[] {31, 32}}) + .addRow(10, objArray(intArray(11, 12))) + .addRow(20, objArray(intArray(21, 22))) + .addRow(30, objArray(intArray(31, 32))) .build(); new RowSetComparison(expected) .verifyAndClearAll(actual); @@ -389,7 +392,7 @@ public class RowSetTest extends SubOperatorTest { .addMapArray("m") .add("b", MinorType.INT) .add("c", MinorType.INT) - .buildMap() + .resumeSchema() .buildSchema(); ExtendableRowSet rowSet = fixture.rowSet(schema); RowSetWriter writer = rowSet.writer(); @@ -514,9 +517,9 @@ public class RowSetTest extends SubOperatorTest { // Verify the readers and writers again using the testing tools. SingleRowSet expected = fixture.rowSetBuilder(schema) - .addRow(10, new Object[] {new Object[] {101, 102}, new Object[] {111, 112}}) - .addRow(20, new Object[] {new Object[] {201, 202}, new Object[] {211, 212}}) - .addRow(30, new Object[] {new Object[] {301, 302}, new Object[] {311, 312}}) + .addRow(10, objArray(objArray(101, 102), objArray(111, 112))) + .addRow(20, objArray(objArray(201, 202), objArray(211, 212))) + .addRow(30, objArray(objArray(301, 302), objArray(311, 312))) .build(); new RowSetComparison(expected) .verifyAndClearAll(actual); @@ -570,8 +573,8 @@ public class RowSetTest extends SubOperatorTest { assertFalse(reader.next()); SingleRowSet rs2 = fixture.rowSetBuilder(batchSchema) - .addRow(10, new int[] {100, 110}) - .addRow(20, new int[] {200, 120, 220}) + .addRow(10, intArray(100, 110)) + .addRow(20, intArray(200, 120, 220)) .addRow(30, null) .build(); http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java index f9f1b8a..490a9fd 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFillEmpties.java @@ -31,10 +31,10 @@ import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSet.ExtendableRowSet; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.test.rowSet.RowSetReader; import org.apache.drill.test.rowSet.RowSetUtilities; import org.apache.drill.test.rowSet.RowSetWriter; -import org.apache.drill.test.rowSet.SchemaBuilder; import org.junit.Test; /** http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java index a27fdf4..a30ba6b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestFixedWidthWriter.java @@ -31,7 +31,7 @@ import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.test.SubOperatorTest; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; /** http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java index 82d4d08..d2a99c4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestOffsetVectorWriter.java @@ -31,7 +31,7 @@ import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriter; import org.apache.drill.test.SubOperatorTest; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex; import org.junit.BeforeClass; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java index 939377a..1daeef4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestScalarAccessors.java @@ -31,9 +31,9 @@ import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.test.SubOperatorTest; import org.apache.drill.test.rowSet.RowSetReader; -import org.apache.drill.test.rowSet.SchemaBuilder; import org.joda.time.Period; import org.apache.drill.test.rowSet.RowSet.SingleRowSet; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; /** http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java new file mode 100644 index 0000000..1c48117 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestSchemaBuilder.java @@ -0,0 +1,598 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.test.rowSet.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.common.types.Types; +import org.apache.drill.exec.record.MaterializedField; +import org.apache.drill.exec.record.metadata.AbstractColumnMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.ColumnMetadata.StructureType; +import org.apache.drill.exec.record.metadata.MetadataUtils; +import org.apache.drill.exec.record.metadata.TupleMetadata; +import org.apache.drill.exec.record.metadata.VariantMetadata; +import org.apache.drill.test.DrillTest; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; +import org.junit.Test; + +/** + * The schema builder for tests has grown complex to handle maps, unions, + * lists and repeated lists. This test verifies that it assembles the various + * pieces correctly for the various nesting combinations. + */ + +public class TestSchemaBuilder extends DrillTest { + + @Test + public void testRowBasics() { + TupleMetadata schema = new SchemaBuilder() + .add("a", MinorType.VARCHAR, DataMode.OPTIONAL) // Generic + .add("b", MinorType.INT) // Required + .addNullable("c", MinorType.FLOAT8) // Convenience + .addArray("d", MinorType.BIGINT) // Convenience + .buildSchema(); + + assertEquals(4, schema.size()); + + ColumnMetadata a = schema.metadata(0); + assertEquals("a", a.name()); + assertEquals(MinorType.VARCHAR, a.type()); + assertEquals(DataMode.OPTIONAL, a.mode()); + + ColumnMetadata b = schema.metadata(1); + assertEquals("b", b.name()); + assertEquals(MinorType.INT, b.type()); + assertEquals(DataMode.REQUIRED, b.mode()); + + ColumnMetadata c = schema.metadata(2); + assertEquals("c", c.name()); + assertEquals(MinorType.FLOAT8, c.type()); + assertEquals(DataMode.OPTIONAL, c.mode()); + + ColumnMetadata d = schema.metadata(3); + assertEquals("d", d.name()); + assertEquals(MinorType.BIGINT, d.type()); + assertEquals(DataMode.REPEATED, d.mode()); + } + + @Test + public void testRowPreBuilt() { + + MaterializedField aField = MaterializedField.create("a", + Types.optional(MinorType.VARCHAR)); + AbstractColumnMetadata bCol = MetadataUtils.newScalar("b", + MinorType.INT, DataMode.REQUIRED); + + SchemaBuilder builder = new SchemaBuilder() + .add(aField); + + // Internal method, does not return builder itself. + + builder.addColumn(bCol); + + TupleMetadata schema = builder.buildSchema(); + + assertEquals(2, schema.size()); + + ColumnMetadata a = schema.metadata(0); + assertEquals("a", a.name()); + assertEquals(MinorType.VARCHAR, a.type()); + assertEquals(DataMode.OPTIONAL, a.mode()); + + ColumnMetadata b = schema.metadata(1); + assertEquals("b", b.name()); + assertEquals(MinorType.INT, b.type()); + assertEquals(DataMode.REQUIRED, b.mode()); + } + + /** + * Tests creating a map within a row. + * Also the basic map add column methods. + */ + + @Test + public void testMapInRow() { + TupleMetadata schema = new SchemaBuilder() + .addMap("m") + .add("a", MinorType.VARCHAR, DataMode.OPTIONAL) // Generic + .add("b", MinorType.INT) // Required + .addNullable("c", MinorType.FLOAT8) // Convenience + .addArray("d", MinorType.BIGINT) // Convenience + .resumeSchema() + .buildSchema(); + + assertEquals(1, schema.size()); + + ColumnMetadata m = schema.metadata(0); + assertEquals("m", m.name()); + assertTrue(m.isMap()); + assertEquals(DataMode.REQUIRED, m.mode()); + + TupleMetadata mapSchema = m.mapSchema(); + assertNotNull(mapSchema); + assertEquals(4, mapSchema.size()); + + ColumnMetadata a = mapSchema.metadata(0); + assertEquals("a", a.name()); + assertEquals(MinorType.VARCHAR, a.type()); + assertEquals(DataMode.OPTIONAL, a.mode()); + + ColumnMetadata b = mapSchema.metadata(1); + assertEquals("b", b.name()); + assertEquals(MinorType.INT, b.type()); + assertEquals(DataMode.REQUIRED, b.mode()); + + ColumnMetadata c = mapSchema.metadata(2); + assertEquals("c", c.name()); + assertEquals(MinorType.FLOAT8, c.type()); + assertEquals(DataMode.OPTIONAL, c.mode()); + + ColumnMetadata d = mapSchema.metadata(3); + assertEquals("d", d.name()); + assertEquals(MinorType.BIGINT, d.type()); + assertEquals(DataMode.REPEATED, d.mode()); + } + + /** + * Test building a union in the top-level schema. + * Also tests the basic union add type methods. + */ + + @Test + public void testUnionInRow() { + TupleMetadata schema = new SchemaBuilder() + .addUnion("u") + .addType(MinorType.VARCHAR) + .addType(MinorType.INT) + .resumeSchema() + .buildSchema(); + + assertEquals(1, schema.size()); + + ColumnMetadata u = schema.metadata(0); + assertEquals("u", u.name()); + assertEquals(StructureType.VARIANT, u.structureType()); + assertTrue(u.isVariant()); + assertEquals(MinorType.UNION, u.type()); + assertEquals(DataMode.OPTIONAL, u.mode()); + + VariantMetadata variant = u.variantSchema(); + assertNotNull(variant); + assertEquals(2, variant.size()); + + assertTrue(variant.hasType(MinorType.VARCHAR)); + ColumnMetadata vMember = variant.member(MinorType.VARCHAR); + assertNotNull(vMember); + assertEquals(Types.typeKey(MinorType.VARCHAR), vMember.name()); + assertEquals(MinorType.VARCHAR, vMember.type()); + assertEquals(DataMode.OPTIONAL, vMember.mode()); + + assertTrue(variant.hasType(MinorType.INT)); + ColumnMetadata iMember = variant.member(MinorType.INT); + assertNotNull(iMember); + assertEquals(Types.typeKey(MinorType.INT), iMember.name()); + assertEquals(MinorType.INT, iMember.type()); + assertEquals(DataMode.OPTIONAL, iMember.mode()); + } + + /** + * Test building a list (of unions) in the top-level schema. + */ + + @Test + public void testListInRow() { + TupleMetadata schema = new SchemaBuilder() + .addList("list") + .addType(MinorType.VARCHAR) + .addType(MinorType.INT) + .resumeSchema() + .buildSchema(); + + assertEquals(1, schema.size()); + + ColumnMetadata list = schema.metadata(0); + assertEquals("list", list.name()); + assertEquals(StructureType.VARIANT, list.structureType()); + assertTrue(list.isVariant()); + assertEquals(MinorType.LIST, list.type()); + + // Yes, strange. Though a list is, essentially, an array, an + // optional list has one set of semantics (in ListVector, not + // really supported), while a repeated list has entirely different + // semantics (in the RepeatedListVector) and is supported. + + assertEquals(DataMode.OPTIONAL, list.mode()); + + VariantMetadata variant = list.variantSchema(); + assertNotNull(variant); + assertEquals(2, variant.size()); + + assertTrue(variant.hasType(MinorType.VARCHAR)); + ColumnMetadata vMember = variant.member(MinorType.VARCHAR); + assertNotNull(vMember); + assertEquals(Types.typeKey(MinorType.VARCHAR), vMember.name()); + assertEquals(MinorType.VARCHAR, vMember.type()); + assertEquals(DataMode.OPTIONAL, vMember.mode()); + + assertTrue(variant.hasType(MinorType.INT)); + ColumnMetadata iMember = variant.member(MinorType.INT); + assertNotNull(iMember); + assertEquals(Types.typeKey(MinorType.INT), iMember.name()); + assertEquals(MinorType.INT, iMember.type()); + assertEquals(DataMode.OPTIONAL, iMember.mode()); + } + + /** + * Test building a repeated list in the top-level schema. + */ + + @Test + public void testRepeatedListInRow() { + TupleMetadata schema = new SchemaBuilder() + .addRepeatedList("list") + .addArray(MinorType.VARCHAR) + .resumeSchema() + .buildSchema(); + + assertEquals(1, schema.size()); + + ColumnMetadata list = schema.metadata(0); + assertEquals("list", list.name()); + assertFalse(list.isVariant()); + assertEquals(StructureType.MULTI_ARRAY, list.structureType()); + assertEquals(MinorType.LIST, list.type()); + + // See note above for the (non-repeated) list. + + assertEquals(DataMode.REPEATED, list.mode()); + + ColumnMetadata child = list.childSchema(); + assertNotNull(child); + assertEquals(list.name(), child.name()); + assertEquals(MinorType.VARCHAR, child.type()); + assertEquals(DataMode.REPEATED, child.mode()); + } + /** + * Test methods to provide a width (precision) for VarChar + * columns. The schema builder does not provide shortcuts for + * VarChar in lists, unions or repeated lists because these + * cases are obscure and seldom (never?) used. + */ + + @Test + public void testVarCharPrecision() { + TupleMetadata schema = new SchemaBuilder() + .add("a", MinorType.VARCHAR, 21) + .addNullable("b", MinorType.VARCHAR, 22) + .addMap("m") + .add("c", MinorType.VARCHAR, 23) + .addNullable("d", MinorType.VARCHAR, 24) + .resumeSchema() + .buildSchema(); + + assertEquals(3, schema.size()); + + // Use name methods, just for variety + + assertEquals(21, schema.metadata("a").precision()); + assertEquals(22, schema.metadata("b").precision()); + TupleMetadata mapSchema = schema.metadata("m").mapSchema(); + assertEquals(23, mapSchema.metadata("c").precision()); + assertEquals(24, mapSchema.metadata("d").precision()); + } + + /** + * Test the ability to specify decimal precision and scale. Decimal is + * broken in Drill, so we don't bother about decimals in unions, + * lists or repeated lists, though those methods could be added. + */ + + @Test + public void testDecimal() { + TupleMetadata schema = new SchemaBuilder() + .addDecimal("a", MinorType.DECIMAL18, DataMode.OPTIONAL, 5, 2) + .addDecimal("b", MinorType.DECIMAL18, DataMode.REQUIRED, 6, 3) + .addDecimal("c", MinorType.DECIMAL18, DataMode.REPEATED, 7, 4) + .addMap("m") + .addDecimal("d", MinorType.DECIMAL18, DataMode.OPTIONAL, 8, 1) + .resumeSchema() + .buildSchema(); + + // Use name methods, just for variety + + ColumnMetadata a = schema.metadata("a"); + assertEquals(DataMode.OPTIONAL, a.mode()); + assertEquals(5, a.precision()); + assertEquals(2, a.scale()); + + ColumnMetadata b = schema.metadata("b"); + assertEquals(DataMode.REQUIRED, b.mode()); + assertEquals(6, b.precision()); + assertEquals(3, b.scale()); + + ColumnMetadata c = schema.metadata("c"); + assertEquals(DataMode.REPEATED, c.mode()); + assertEquals(7, c.precision()); + assertEquals(4, c.scale()); + + ColumnMetadata d = schema.metadata("m").mapSchema().metadata("d"); + assertEquals(DataMode.OPTIONAL, d.mode()); + assertEquals(8, d.precision()); + assertEquals(1, d.scale()); + } + + /** + * Verify that the map-in-map plumbing works. + */ + + @Test + public void testMapInMap() { + TupleMetadata schema = new SchemaBuilder() + .addMap("m1") + .addMap("m2") + .add("a", MinorType.INT) + .resumeMap() + .add("b", MinorType.VARCHAR) + .resumeSchema() + .buildSchema(); + + TupleMetadata m1Schema = schema.metadata("m1").mapSchema(); + TupleMetadata m2Schema = m1Schema.metadata("m2").mapSchema(); + + ColumnMetadata a = m2Schema.metadata(0); + assertEquals("a", a.name()); + assertEquals(MinorType.INT, a.type()); + + ColumnMetadata b = m1Schema.metadata(1); + assertEquals("b", b.name()); + assertEquals(MinorType.VARCHAR, b.type()); + } + + /** + * Verify that the union-in-map plumbing works. + */ + + @Test + public void testUnionInMap() { + TupleMetadata schema = new SchemaBuilder() + .addMap("m1") + .addUnion("u") + .addType(MinorType.INT) + .resumeMap() + .add("b", MinorType.VARCHAR) + .resumeSchema() + .buildSchema(); + + TupleMetadata m1Schema = schema.metadata("m1").mapSchema(); + VariantMetadata uSchema = m1Schema.metadata("u").variantSchema(); + + assertTrue(uSchema.hasType(MinorType.INT)); + assertFalse(uSchema.hasType(MinorType.VARCHAR)); + + ColumnMetadata b = m1Schema.metadata(1); + assertEquals("b", b.name()); + assertEquals(MinorType.VARCHAR, b.type()); + } + + /** + * Verify that the repeated list-in-map plumbing works. + */ + + @Test + public void testRepeatedListInMap() { + TupleMetadata schema = new SchemaBuilder() + .addMap("m1") + .addRepeatedList("r") + .addArray(MinorType.INT) + .resumeMap() + .add("b", MinorType.VARCHAR) + .resumeSchema() + .buildSchema(); + + TupleMetadata m1Schema = schema.metadata("m1").mapSchema(); + + ColumnMetadata r = m1Schema.metadata(0); + assertEquals("r", r.name()); + assertEquals(MinorType.LIST, r.type()); + assertEquals(DataMode.REPEATED, r.mode()); + + ColumnMetadata child = r.childSchema(); + assertEquals(r.name(), child.name()); + assertEquals(MinorType.INT, child.type()); + + ColumnMetadata b = m1Schema.metadata(1); + assertEquals("b", b.name()); + assertEquals(MinorType.VARCHAR, b.type()); + } + + @Test + public void testMapInUnion() { + TupleMetadata schema = new SchemaBuilder() + .addUnion("u") + .addMap() + .add("a", MinorType.INT) + .add("b", MinorType.VARCHAR) + .resumeUnion() + .addType(MinorType.FLOAT8) + .resumeSchema() + .buildSchema(); + + ColumnMetadata u = schema.metadata("u"); + VariantMetadata variant = u.variantSchema(); + + ColumnMetadata mapType = variant.member(MinorType.MAP); + assertNotNull(mapType); + + TupleMetadata mapSchema = mapType.mapSchema(); + assertEquals(2, mapSchema.size()); + + assertTrue(variant.hasType(MinorType.FLOAT8)); + assertFalse(variant.hasType(MinorType.VARCHAR)); + } + + @Test + public void testRepeatedListInUnion() { + TupleMetadata schema = new SchemaBuilder() + .addUnion("u") + .addRepeatedList() + .addArray(MinorType.INT) + .resumeUnion() + .addType(MinorType.FLOAT8) + .resumeSchema() + .buildSchema(); + + ColumnMetadata u = schema.metadata("u"); + VariantMetadata variant = u.variantSchema(); + + ColumnMetadata listType = variant.member(MinorType.LIST); + assertNotNull(listType); + + ColumnMetadata child = listType.childSchema(); + assertEquals(MinorType.INT, child.type()); + + assertTrue(variant.hasType(MinorType.FLOAT8)); + assertFalse(variant.hasType(MinorType.VARCHAR)); + } + + // Note: list-in-union may be supported, but this area of the code is obscure + // and not a priority to maintain. The problem will be that both lists + // and repeated lists key off of the same type code: LIST, so it is + // ambiguous which is supported. The schema builder muddles through this + // case, but the rest of the code might not. + + @Test + public void testListInUnion() { + TupleMetadata schema = new SchemaBuilder() + .addUnion("u") + .addList() + .addType(MinorType.INT) + .resumeUnion() + .addType(MinorType.FLOAT8) + .resumeSchema() + .buildSchema(); + + ColumnMetadata u = schema.metadata("u"); + VariantMetadata variant = u.variantSchema(); + + ColumnMetadata listType = variant.member(MinorType.LIST); + assertNotNull(listType); + VariantMetadata listSchema = listType.variantSchema(); + assertTrue(listSchema.hasType(MinorType.INT)); + + assertTrue(variant.hasType(MinorType.FLOAT8)); + assertFalse(variant.hasType(MinorType.VARCHAR)); + } + + // Note: union-in-union not supported in Drill + + @Test + public void testMapInRepeatedList() { + TupleMetadata schema = new SchemaBuilder() + .addRepeatedList("x") + .addMapArray() + .add("a", MinorType.INT) + .addNullable("b", MinorType.VARCHAR) + .resumeList() + .resumeSchema() + .buildSchema(); + + ColumnMetadata list = schema.metadata("x"); + ColumnMetadata mapCol = list.childSchema(); + assertTrue(mapCol.isMap()); + TupleMetadata mapSchema = mapCol.mapSchema(); + + ColumnMetadata a = mapSchema.metadata("a"); + assertEquals(MinorType.INT, a.type()); + assertEquals(DataMode.REQUIRED, a.mode()); + + ColumnMetadata b = mapSchema.metadata("b"); + assertEquals(MinorType.VARCHAR, b.type()); + assertEquals(DataMode.OPTIONAL, b.mode()); + } + + /** + * Test that repeated lists can be nested to provide 3D or + * higher dimensions. + */ + + @Test + public void testRepeatedListInRepeatedList() { + TupleMetadata schema = new SchemaBuilder() + .addRepeatedList("x") + .addDimension() + .addArray(MinorType.VARCHAR) + .resumeList() + .resumeSchema() + .buildSchema(); + + assertEquals(1, schema.size()); + + ColumnMetadata outerList = schema.metadata(0); + assertEquals("x", outerList.name()); + assertEquals(StructureType.MULTI_ARRAY, outerList.structureType()); + assertEquals(MinorType.LIST, outerList.type()); + assertEquals(DataMode.REPEATED, outerList.mode()); + + ColumnMetadata innerList = outerList.childSchema(); + assertNotNull(innerList); + assertEquals(outerList.name(), innerList.name()); + assertEquals(StructureType.MULTI_ARRAY, innerList.structureType()); + assertEquals(MinorType.LIST, innerList.type()); + assertEquals(DataMode.REPEATED, innerList.mode()); + + ColumnMetadata child = innerList.childSchema(); + assertNotNull(child); + assertEquals(outerList.name(), child.name()); + assertEquals(MinorType.VARCHAR, child.type()); + assertEquals(DataMode.REPEATED, child.mode()); + } + + @Test + public void testRepeatedListShortcut() { + TupleMetadata schema = new SchemaBuilder() + .addArray("x", MinorType.VARCHAR, 3) + .buildSchema(); + + assertEquals(1, schema.size()); + + ColumnMetadata outerList = schema.metadata(0); + assertEquals("x", outerList.name()); + assertEquals(StructureType.MULTI_ARRAY, outerList.structureType()); + assertEquals(MinorType.LIST, outerList.type()); + assertEquals(DataMode.REPEATED, outerList.mode()); + + ColumnMetadata innerList = outerList.childSchema(); + assertNotNull(innerList); + assertEquals(outerList.name(), innerList.name()); + assertEquals(StructureType.MULTI_ARRAY, innerList.structureType()); + assertEquals(MinorType.LIST, innerList.type()); + assertEquals(DataMode.REPEATED, innerList.mode()); + + ColumnMetadata child = innerList.childSchema(); + assertNotNull(child); + assertEquals(outerList.name(), child.name()); + assertEquals(MinorType.VARCHAR, child.type()); + assertEquals(DataMode.REPEATED, child.mode()); + } +} http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java index 912f362..6f0eb66 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/TestVariableWidthWriter.java @@ -30,7 +30,7 @@ import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.ScalarWriter.ColumnWriterListener; import org.apache.drill.exec.vector.accessor.ValueType; import org.apache.drill.test.SubOperatorTest; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex; import org.bouncycastle.util.Arrays; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java index f3390d3..51a1ba4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java +++ b/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java @@ -30,7 +30,7 @@ import org.apache.drill.exec.vector.RepeatedVarCharVector; import org.apache.drill.exec.vector.UInt4Vector; import org.apache.drill.exec.vector.VarCharVector; import org.apache.drill.test.SubOperatorTest; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.junit.Test; import io.netty.buffer.DrillBuf; http://git-wip-us.apache.org/repos/asf/drill/blob/f653359c/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java index 234ad88..f120faf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java +++ b/exec/java-exec/src/test/java/org/apache/drill/vector/TestToNullable.java @@ -27,7 +27,7 @@ import org.apache.drill.exec.vector.NullableIntVector; import org.apache.drill.exec.vector.NullableVarCharVector; import org.apache.drill.exec.vector.VarCharVector; import org.apache.drill.test.SubOperatorTest; -import org.apache.drill.test.rowSet.SchemaBuilder; +import org.apache.drill.test.rowSet.schema.SchemaBuilder; import org.bouncycastle.util.Arrays; import org.junit.Test;
