Repository: arrow Updated Branches: refs/heads/master 6811d3fcf -> 512bc160e
ARROW-386: [Java] Respect case of struct / map field names Changes include: - Remove all toLowerCase() calls on field names in MapWriters.java template file, so that the writers can respect case of the field names. - Use lower-case keys for internalMap in UnionVector instead of camel-case (e.g. bigInt -> bigint). p.s. I don't know what is the original purpose of using camel case here. It did not conflict because all field names are converted to lower cases in the past. - Add a simple test case of MapWriter with mixed-case field names. Author: Jingyuan Wang <[email protected]> Closes #261 from alphalfalfa/arrow-386 and squashes the following commits: cd08145 [Jingyuan Wang] Remove unnecessary handleCase() call 7b28bfc [Jingyuan Wang] Pass caseSensitive Attribute down to nested MapWriters 2fe7bcf [Jingyuan Wang] Separate MapWriters with CaseSensitiveMapWriters d269e21 [Jingyuan Wang] Configure case sensitivity when constructing ComplexWriterImpl cba60d1 [Jingyuan Wang] Add option to MapWriters to configure the case sensitivity (defaulted as case-insensitive) 51da2a1 [Jingyuan Wang] Arrow-386: [Java] Respect case of struct / map field names Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/512bc160 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/512bc160 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/512bc160 Branch: refs/heads/master Commit: 512bc160ebaf8d6775ea67994262709e10a72795 Parents: 6811d3f Author: Jingyuan Wang <[email protected]> Authored: Fri Jan 20 12:43:20 2017 -0500 Committer: Wes McKinney <[email protected]> Committed: Fri Jan 20 12:43:20 2017 -0500 ---------------------------------------------------------------------- .../templates/CaseSensitiveMapWriters.java | 54 ++++++++++++++ .../src/main/codegen/templates/MapWriters.java | 35 +++++---- .../main/codegen/templates/UnionListWriter.java | 6 +- .../src/main/codegen/templates/UnionVector.java | 3 +- .../src/main/codegen/templates/UnionWriter.java | 12 +++- .../arrow/vector/complex/AbstractMapVector.java | 6 +- .../vector/complex/impl/ComplexWriterImpl.java | 17 +++-- .../complex/impl/NullableMapWriterFactory.java | 42 +++++++++++ .../vector/complex/impl/PromotableWriter.java | 31 +++++++- .../complex/writer/TestComplexWriter.java | 76 ++++++++++++++++++++ 10 files changed, 253 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java new file mode 100644 index 0000000..5357f9b --- /dev/null +++ b/java/vector/src/main/codegen/templates/CaseSensitiveMapWriters.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +<@pp.dropOutputFile /> +<#list ["Nullable", "Single"] as mode> +<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}CaseSensitiveMapWriter.java" /> +<#assign index = "idx()"> +<#if mode == "Single"> +<#assign containerClass = "MapVector" /> +<#else> +<#assign containerClass = "NullableMapVector" /> +</#if> + +<#include "/@includes/license.ftl" /> + +package org.apache.arrow.vector.complex.impl; + +<#include "/@includes/vv_imports.ftl" /> +/* + * This class is generated using FreeMarker and the ${.template_name} template. + */ +@SuppressWarnings("unused") +public class ${mode}CaseSensitiveMapWriter extends ${mode}MapWriter { + public ${mode}CaseSensitiveMapWriter(${containerClass} container) { + super(container); + } + + @Override + protected String handleCase(final String input){ + return input; + } + + @Override + protected NullableMapWriterFactory getNullableMapWriterFactory() { + return NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance(); + } + +} +</#list> http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/codegen/templates/MapWriters.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java index f41b600..4af6eee 100644 --- a/java/vector/src/main/codegen/templates/MapWriters.java +++ b/java/vector/src/main/codegen/templates/MapWriters.java @@ -48,7 +48,6 @@ public class ${mode}MapWriter extends AbstractFieldWriter { protected final ${containerClass} container; private final Map<String, FieldWriter> fields = Maps.newHashMap(); - public ${mode}MapWriter(${containerClass} container) { <#if mode == "Single"> if (container instanceof NullableMapVector) { @@ -65,8 +64,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter { list(child.getName()); break; case UNION: - UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class)); - fields.put(child.getName().toLowerCase(), writer); + UnionWriter writer = new UnionWriter(container.addOrGet(child.getName(), MinorType.UNION, UnionVector.class), getNullableMapWriterFactory()); + fields.put(handleCase(child.getName()), writer); break; <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> @@ -85,6 +84,14 @@ public class ${mode}MapWriter extends AbstractFieldWriter { } } + protected String handleCase(final String input) { + return input.toLowerCase(); + } + + protected NullableMapWriterFactory getNullableMapWriterFactory() { + return NullableMapWriterFactory.getNullableMapWriterFactoryInstance(); + } + @Override public int getValueCapacity() { return container.getValueCapacity(); @@ -102,16 +109,17 @@ public class ${mode}MapWriter extends AbstractFieldWriter { @Override public MapWriter map(String name) { - FieldWriter writer = fields.get(name.toLowerCase()); + String finalName = handleCase(name); + FieldWriter writer = fields.get(finalName); if(writer == null){ int vectorCount=container.size(); NullableMapVector vector = container.addOrGet(name, MinorType.MAP, NullableMapVector.class); - writer = new PromotableWriter(vector, container); + writer = new PromotableWriter(vector, container, getNullableMapWriterFactory()); if(vectorCount != container.size()) { writer.allocate(); } writer.setPosition(idx()); - fields.put(name.toLowerCase(), writer); + fields.put(finalName, writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized @@ -145,15 +153,16 @@ public class ${mode}MapWriter extends AbstractFieldWriter { @Override public ListWriter list(String name) { - FieldWriter writer = fields.get(name.toLowerCase()); + String finalName = handleCase(name); + FieldWriter writer = fields.get(finalName); int vectorCount = container.size(); if(writer == null) { - writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class), container); + writer = new PromotableWriter(container.addOrGet(name, MinorType.LIST, ListVector.class), container, getNullableMapWriterFactory()); if (container.size() > vectorCount) { writer.allocate(); } writer.setPosition(idx()); - fields.put(name.toLowerCase(), writer); + fields.put(finalName, writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized @@ -199,7 +208,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter { <#if minor.class?starts_with("Decimal") > public ${minor.class}Writer ${lowerName}(String name) { // returns existing writer - final FieldWriter writer = fields.get(name.toLowerCase()); + final FieldWriter writer = fields.get(handleCase(name)); assert writer != null; return writer; } @@ -209,18 +218,18 @@ public class ${mode}MapWriter extends AbstractFieldWriter { @Override public ${minor.class}Writer ${lowerName}(String name) { </#if> - FieldWriter writer = fields.get(name.toLowerCase()); + FieldWriter writer = fields.get(handleCase(name)); if(writer == null) { ValueVector vector; ValueVector currentVector = container.getChild(name); ${vectName}Vector v = container.addOrGet(name, MinorType.${upperName}, ${vectName}Vector.class<#if minor.class == "Decimal"> , new int[] {precision, scale}</#if>); - writer = new PromotableWriter(v, container); + writer = new PromotableWriter(v, container, getNullableMapWriterFactory()); vector = v; if (currentVector == null || currentVector != vector) { vector.allocateNewSafe(); } writer.setPosition(idx()); - fields.put(name.toLowerCase(), writer); + fields.put(handleCase(name), writer); } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/codegen/templates/UnionListWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index bb39fe8..d980830 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -43,8 +43,12 @@ public class UnionListWriter extends AbstractFieldWriter { private int lastIndex = 0; public UnionListWriter(ListVector vector) { + this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance()); + } + + public UnionListWriter(ListVector vector, NullableMapWriterFactory nullableMapWriterFactory) { this.vector = vector; - this.writer = new PromotableWriter(vector.getDataVector(), vector); + this.writer = new PromotableWriter(vector.getDataVector(), vector, nullableMapWriterFactory); this.offsets = vector.getOffsetVector(); } http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/codegen/templates/UnionVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index 18acdf4..1a6908d 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -136,6 +136,7 @@ public class UnionVector implements FieldVector { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> + <#assign lowerCaseName = name?lower_case/> <#if !minor.class?starts_with("Decimal")> private Nullable${name}Vector ${uncappedName}Vector; @@ -143,7 +144,7 @@ public class UnionVector implements FieldVector { public Nullable${name}Vector get${name}Vector() { if (${uncappedName}Vector == null) { int vectorCount = internalMap.size(); - ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", MinorType.${name?upper_case}, Nullable${name}Vector.class); + ${uncappedName}Vector = internalMap.addOrGet("${lowerCaseName}", MinorType.${name?upper_case}, Nullable${name}Vector.class); if (internalMap.size() > vectorCount) { ${uncappedName}Vector.allocateNew(); if (callBack != null) { http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/codegen/templates/UnionWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index efb66f1..880f537 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -16,6 +16,8 @@ * limitations under the License. */ +import org.apache.arrow.vector.complex.impl.NullableMapWriterFactory; + <@pp.dropOutputFile /> <@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionWriter.java" /> @@ -38,9 +40,15 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { private MapWriter mapWriter; private UnionListWriter listWriter; private List<BaseWriter> writers = Lists.newArrayList(); + private final NullableMapWriterFactory nullableMapWriterFactory; public UnionWriter(UnionVector vector) { + this(vector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance()); + } + + public UnionWriter(UnionVector vector, NullableMapWriterFactory nullableMapWriterFactory) { data = vector; + this.nullableMapWriterFactory = nullableMapWriterFactory; } @Override @@ -76,7 +84,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { private MapWriter getMapWriter() { if (mapWriter == null) { - mapWriter = new NullableMapWriter(data.getMap()); + mapWriter = nullableMapWriterFactory.build(data.getMap()); mapWriter.setPosition(idx()); writers.add(mapWriter); } @@ -90,7 +98,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { private ListWriter getListWriter() { if (listWriter == null) { - listWriter = new UnionListWriter(data.getList()); + listWriter = new UnionListWriter(data.getList(), nullableMapWriterFactory); listWriter.setPosition(idx()); writers.add(listWriter); } http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java index 23b4997..f030d16 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java @@ -155,7 +155,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { */ @Override public <T extends FieldVector> T getChild(String name, Class<T> clazz) { - final ValueVector v = vectors.get(name.toLowerCase()); + final ValueVector v = vectors.get(name); if (v == null) { return null; } @@ -191,7 +191,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { */ protected void putVector(String name, FieldVector vector) { final ValueVector old = vectors.put( - Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(), + Preconditions.checkNotNull(name, "field name cannot be null"), Preconditions.checkNotNull(vector, "vector cannot be null") ); if (old != null && old != vector) { @@ -254,7 +254,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector { */ @Override public VectorWithOrdinal getChildVectorWithOrdinal(String name) { - final int ordinal = vectors.getOrdinal(name.toLowerCase()); + final int ordinal = vectors.getOrdinal(name); if (ordinal < 0) { return null; } http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java index 761b1b4..dbdd205 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java @@ -37,13 +37,20 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri Mode mode = Mode.INIT; private final String name; private final boolean unionEnabled; + private final NullableMapWriterFactory nullableMapWriterFactory; private enum Mode { INIT, MAP, LIST }; - public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){ + public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled, boolean caseSensitive){ this.name = name; this.container = container; this.unionEnabled = unionEnabled; + nullableMapWriterFactory = caseSensitive? NullableMapWriterFactory.getNullableCaseSensitiveMapWriterFactoryInstance() : + NullableMapWriterFactory.getNullableMapWriterFactoryInstance(); + } + + public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled) { + this(name, container, unionEnabled, false); } public ComplexWriterImpl(String name, MapVector container){ @@ -122,8 +129,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri switch(mode){ case INIT: - NullableMapVector map = (NullableMapVector) container; - mapRoot = new NullableMapWriter(map); + mapRoot = nullableMapWriterFactory.build((NullableMapVector) container); mapRoot.setPosition(idx()); mode = Mode.MAP; break; @@ -144,7 +150,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri case INIT: NullableMapVector map = container.addOrGet(name, MinorType.MAP, NullableMapVector.class); - mapRoot = new NullableMapWriter(map); + mapRoot = nullableMapWriterFactory.build(map); mapRoot.setPosition(idx()); mode = Mode.MAP; break; @@ -159,7 +165,6 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri return mapRoot; } - @Override public void allocate() { if(mapRoot != null) { @@ -179,7 +184,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri if (container.size() > vectorCount) { listVector.allocateNew(); } - listRoot = new UnionListWriter(listVector); + listRoot = new UnionListWriter(listVector, nullableMapWriterFactory); listRoot.setPosition(idx()); mode = Mode.LIST; break; http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java new file mode 100644 index 0000000..d932cfb --- /dev/null +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/NullableMapWriterFactory.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.complex.impl; + +import org.apache.arrow.vector.complex.NullableMapVector; + +public class NullableMapWriterFactory { + private final boolean caseSensitive; + private static final NullableMapWriterFactory nullableMapWriterFactory = new NullableMapWriterFactory(false); + private static final NullableMapWriterFactory nullableCaseSensitiveWriterFactory = new NullableMapWriterFactory(true); + + public NullableMapWriterFactory(boolean caseSensitive) { + this.caseSensitive = caseSensitive; + } + + public NullableMapWriter build(NullableMapVector container) { + return this.caseSensitive? new NullableCaseSensitiveMapWriter(container) : new NullableMapWriter(container); + } + + public static NullableMapWriterFactory getNullableMapWriterFactoryInstance() { + return nullableMapWriterFactory; + } + + public static NullableMapWriterFactory getNullableCaseSensitiveMapWriterFactoryInstance() { + return nullableCaseSensitiveWriterFactory; + } +} http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 94ff82c..1f7253b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -22,6 +22,7 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.ZeroVector; import org.apache.arrow.vector.complex.AbstractMapVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.NullableMapVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.FieldWriter; import org.apache.arrow.vector.types.Types.MinorType; @@ -38,6 +39,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { private final AbstractMapVector parentContainer; private final ListVector listVector; + private final NullableMapWriterFactory nullableMapWriterFactory; private int position; private enum State { @@ -51,14 +53,24 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { private FieldWriter writer; public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) { + this(v, parentContainer, NullableMapWriterFactory.getNullableMapWriterFactoryInstance()); + } + + public PromotableWriter(ValueVector v, AbstractMapVector parentContainer, NullableMapWriterFactory nullableMapWriterFactory) { this.parentContainer = parentContainer; this.listVector = null; + this.nullableMapWriterFactory = nullableMapWriterFactory; init(v); } public PromotableWriter(ValueVector v, ListVector listVector) { + this(v, listVector, NullableMapWriterFactory.getNullableMapWriterFactoryInstance()); + } + + public PromotableWriter(ValueVector v, ListVector listVector, NullableMapWriterFactory nullableMapWriterFactory) { this.listVector = listVector; this.parentContainer = null; + this.nullableMapWriterFactory = nullableMapWriterFactory; init(v); } @@ -66,7 +78,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { if (v instanceof UnionVector) { state = State.UNION; unionVector = (UnionVector) v; - writer = new UnionWriter(unionVector); + writer = new UnionWriter(unionVector, nullableMapWriterFactory); } else if (v instanceof ZeroVector) { state = State.UNTYPED; } else { @@ -78,7 +90,20 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { state = State.SINGLE; vector = v; type = v.getMinorType(); - writer = type.getNewFieldWriter(vector); + switch (type) { + case MAP: + writer = nullableMapWriterFactory.build((NullableMapVector) vector); + break; + case LIST: + writer = new UnionListWriter((ListVector) vector, nullableMapWriterFactory); + break; + case UNION: + writer = new UnionWriter((UnionVector) vector, nullableMapWriterFactory); + break; + default: + writer = type.getNewFieldWriter(vector); + break; + } } @Override @@ -131,7 +156,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { unionVector = listVector.promoteToUnion(); } unionVector.addVector((FieldVector)tp.getTo()); - writer = new UnionWriter(unionVector); + writer = new UnionWriter(unionVector, nullableMapWriterFactory); writer.setPosition(idx()); for (int i = 0; i <= idx(); i++) { unionVector.getMutator().setType(i, vector.getMinorType()); http://git-wip-us.apache.org/repos/asf/arrow/blob/512bc160/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java ---------------------------------------------------------------------- diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index caa438a..2c0c853 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -23,7 +23,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -485,4 +487,78 @@ public class TestComplexWriter { Assert.assertTrue(intType.getIsSigned()); Assert.assertEquals(ArrowTypeID.Utf8, field.getChildren().get(1).getType().getTypeID()); } + + private Set<String> getFieldNames(List<Field> fields) { + Set<String> fieldNames = new HashSet<>(); + for (Field field: fields) { + fieldNames.add(field.getName()); + if (!field.getChildren().isEmpty()) { + for (String name: getFieldNames(field.getChildren())) { + fieldNames.add(field.getName() + "::" + name); + } + } + } + return fieldNames; + } + + @Test + public void mapWriterMixedCaseFieldNames() { + // test case-sensitive MapWriter + MapVector parent = new MapVector("parent", allocator, null); + ComplexWriter writer = new ComplexWriterImpl("rootCaseSensitive", parent, false, true); + MapWriter rootWriterCaseSensitive = writer.rootAsMap(); + rootWriterCaseSensitive.bigInt("int_field"); + rootWriterCaseSensitive.bigInt("Int_Field"); + rootWriterCaseSensitive.float4("float_field"); + rootWriterCaseSensitive.float4("Float_Field"); + MapWriter mapFieldWriterCaseSensitive = rootWriterCaseSensitive.map("map_field"); + mapFieldWriterCaseSensitive.varChar("char_field"); + mapFieldWriterCaseSensitive.varChar("Char_Field"); + ListWriter listFieldWriterCaseSensitive = rootWriterCaseSensitive.list("list_field"); + MapWriter listMapFieldWriterCaseSensitive = listFieldWriterCaseSensitive.map(); + listMapFieldWriterCaseSensitive.bit("bit_field"); + listMapFieldWriterCaseSensitive.bit("Bit_Field"); + + List<Field> fieldsCaseSensitive = parent.getField().getChildren().get(0).getChildren(); + Set<String> fieldNamesCaseSensitive = getFieldNames(fieldsCaseSensitive); + Assert.assertEquals(11, fieldNamesCaseSensitive.size()); + Assert.assertTrue(fieldNamesCaseSensitive.contains("int_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("Int_Field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("float_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("Float_Field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field::char_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("map_field::Char_Field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::Bit_Field")); + + // test case-insensitive MapWriter + ComplexWriter writerCaseInsensitive = new ComplexWriterImpl("rootCaseInsensitive", parent, false, false); + MapWriter rootWriterCaseInsensitive = writerCaseInsensitive.rootAsMap(); + + rootWriterCaseInsensitive.bigInt("int_field"); + rootWriterCaseInsensitive.bigInt("Int_Field"); + rootWriterCaseInsensitive.float4("float_field"); + rootWriterCaseInsensitive.float4("Float_Field"); + MapWriter mapFieldWriterCaseInsensitive = rootWriterCaseInsensitive.map("map_field"); + mapFieldWriterCaseInsensitive.varChar("char_field"); + mapFieldWriterCaseInsensitive.varChar("Char_Field"); + ListWriter listFieldWriterCaseInsensitive = rootWriterCaseInsensitive.list("list_field"); + MapWriter listMapFieldWriterCaseInsensitive = listFieldWriterCaseInsensitive.map(); + listMapFieldWriterCaseInsensitive.bit("bit_field"); + listMapFieldWriterCaseInsensitive.bit("Bit_Field"); + + List<Field> fieldsCaseInsensitive = parent.getField().getChildren().get(1).getChildren(); + Set<String> fieldNamesCaseInsensitive = getFieldNames(fieldsCaseInsensitive); + Assert.assertEquals(7, fieldNamesCaseInsensitive.size()); + Assert.assertTrue(fieldNamesCaseInsensitive.contains("int_field")); + Assert.assertTrue(fieldNamesCaseInsensitive.contains("float_field")); + Assert.assertTrue(fieldNamesCaseInsensitive.contains("map_field")); + Assert.assertTrue(fieldNamesCaseInsensitive.contains("map_field::char_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$")); + Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field")); + } } \ No newline at end of file
