jbonofre commented on code in PR #1414:
URL: https://github.com/apache/arrow-java/pull/1414#discussion_r4566262486
##########
vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java:
##########
@@ -954,4 +959,125 @@ public void testCopyStructVectorWithExtensionType() {
assertTrue(VectorEqualsVisitor.vectorEquals(from, to));
}
}
+
+ @Test
+ public void testCopyListOfTimeStampNanoTZVector() {
+ try (ListVector from = ListVector.empty("v", allocator);
+ ListVector to = ListVector.empty("v", allocator)) {
+ from.addOrGetVector(FieldType.nullable(new
ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")));
+ to.addOrGetVector(FieldType.nullable(new
ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC")));
+
+ UnionListWriter listWriter = from.getWriter();
+ listWriter.allocate();
+ for (int i = 0; i < COUNT; i++) {
+ listWriter.setPosition(i);
+ listWriter.startList();
+ listWriter.timeStampNanoTZ().writeTimeStampNanoTZ(i * 1_000_000L);
+ listWriter.timeStampNanoTZ().writeTimeStampNanoTZ(i * 2_000_000L);
+ listWriter.endList();
+ }
+ from.setValueCount(COUNT);
+
+ // copy values
+ FieldReader in = from.getReader();
+ FieldWriter out = to.getWriter();
+ for (int i = 0; i < COUNT; i++) {
+ in.setPosition(i);
+ out.setPosition(i);
+ ComplexCopier.copy(in, out);
+ }
+ to.setValueCount(COUNT);
+
+ // validate equals
+ assertTrue(VectorEqualsVisitor.vectorEquals(from, to));
+ }
+ }
+
+ @Test
+ public void testCopyStructOfTimeStampNanoTZVector() {
+ try (final StructVector from = StructVector.empty("v", allocator);
+ final StructVector to = StructVector.empty("v", allocator)) {
+ from.allocateNewSafe();
+
+ NullableStructWriter structWriter = from.getWriter();
+ for (int i = 0; i < COUNT; i++) {
+ structWriter.setPosition(i);
+ structWriter.start();
+ structWriter.timeStampNanoTZ("ts", "UTC").writeTimeStampNanoTZ(i *
1_000_000L);
+ structWriter.end();
+ }
+ from.setValueCount(COUNT);
+
+ // copy values
+ FieldReader in = from.getReader();
+ FieldWriter out = to.getWriter();
+ for (int i = 0; i < COUNT; i++) {
+ in.setPosition(i);
+ out.setPosition(i);
+ ComplexCopier.copy(in, out);
+ }
+ to.setValueCount(COUNT);
+
+ // validate equals
+ assertTrue(VectorEqualsVisitor.vectorEquals(from, to));
+ }
+ }
+
+ @Test
+ public void testCopyMapOfTimeStampNanoTZVector() {
+ // Before the fix ComplexCopier.getMapWriterForReader threw
+ // UnsupportedOperationException for a TimeStamp*TZ map value. The map
value
+ // child is pre-typed so the reader reports TIMESTAMPNANOTZ and the map
case
+ // is actually exercised. vectorEquals is not asserted because the
MapWriter
+ // scalar API has no timezone overload (unlike StructWriter), so the copied
+ // value child does not round-trip to the same timezone; this test locks in
+ // that copying the map no longer throws.
+ Field key = new Field("key", FieldType.notNullable(new ArrowType.Int(32,
true)), null);
+ Field value =
+ new Field(
+ "value",
+ FieldType.nullable(new ArrowType.Timestamp(TimeUnit.NANOSECOND,
"UTC")),
+ null);
+ Field entries =
+ new Field(
+ MapVector.DATA_VECTOR_NAME,
+ FieldType.notNullable(ArrowType.Struct.INSTANCE),
+ Arrays.asList(key, value));
+ Field mapField =
+ new Field("v", FieldType.nullable(new ArrowType.Map(false)),
Arrays.asList(entries));
+
+ try (final MapVector from = (MapVector) mapField.createVector(allocator);
+ final MapVector to = MapVector.empty("v", allocator, false)) {
+
+ from.allocateNew();
+
+ UnionMapWriter mapWriter = from.getWriter();
+ for (int i = 0; i < COUNT; i++) {
+ mapWriter.setPosition(i);
+ mapWriter.startMap();
+ mapWriter.startEntry();
+ mapWriter.key().integer().writeInt(i);
+ mapWriter.value().timeStampNanoTZ().writeTimeStampNanoTZ(i *
1_000_000L);
Review Comment:
This test can't catch the problem mentioned above, because the source map is
already malformed. `mapWriter.value().timeStampNanoTZ()` hits the same missing
`UnionMapWriter` override, so `from` already has its entries promoted to a
union and no `value` data. The comment at the top of the test blames the
mismatch on timezones, but I think both sides are UTC here, the vectors differ
in structure, not timezone.
Could we build the source directly on the vectors
(`startNewValue`/`endValue` on the map, set `key`/`value` on the entries
`StructVector`)? Then use a non-UTC zone and assert
`VectorEqualsVisitor.vectorEquals(from, to)` instead of `assertDoesNotThrow`
plus the value count. If we drop the map case instead, an
`assertThrows(UnsupportedOperationException.class, ...)` would document that
maps are not support yet 😄
##########
vector/src/main/codegen/templates/ComplexCopier.java:
##########
@@ -209,7 +222,7 @@ private static FieldWriter
getMapWriterForReader(FieldReader reader, MapWriter w
<#list vv.types as type><#list type.minor as minor><#assign name =
minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
- <#if !minor.typeParams?? || minor.class?starts_with("Decimal") >
+ <#if !minor.typeParams?? || minor.class?starts_with("Decimal") ||
is_timestamp_tz(minor.class) >
Review Comment:
Adding the TZ types to the map lookup turns an
`UnsupportedOperationException` into silently wrong data. `UnionMapWriter` only
overrides the key/value getters for types without `typeParams` (plus `decimal`,
`decimal256` and `fixedSizeBinary`). So `writer.value().timeStampNanoTZ()`
falls through to `UnionListWriter.timeStampNanoTZ()`, which returns the map
writer itself. The value is then written into the map entries vector instead of
the `value` field, and the entries vector gets promoted to a union.
Copying into `MapVector.empty(...)` would give the same result.
We could either:
- drop the TZ case from `getMapWriterForReader`, so maps keep throwing and
the list/struct support can land on its own, or
- add TZ overrides to `UnionMapWriter` that route to the `key`/`value`
fields of `entryWriter` and carry the timezone, as the struct path does?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]