This is an automated email from the ASF dual-hosted git repository.
cvandermerwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new aad58648737 Support picosecond tiemstamps when writing GenericRecord
and Beam Rows. (#37294)
aad58648737 is described below
commit aad58648737f889ba761418ae64512fbbdf017ff
Author: claudevdm <[email protected]>
AuthorDate: Wed Jan 14 10:34:45 2026 -0500
Support picosecond tiemstamps when writing GenericRecord and Beam Rows.
(#37294)
Co-authored-by: Claude <[email protected]>
---
.../AvroGenericRecordToStorageApiProto.java | 3 ++
.../io/gcp/bigquery/BeamRowToStorageApiProto.java | 3 ++
.../AvroGenericRecordToStorageApiProtoTest.java | 34 ++++++++++++++++++++++
.../gcp/bigquery/BeamRowToStorageApiProtoTest.java | 17 +++++++++++
4 files changed, 57 insertions(+)
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java
index 76174ac3d04..35751e2758e 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java
@@ -348,6 +348,9 @@ public class AvroGenericRecordToStorageApiProto {
fieldDescriptorFromAvroField(
new Schema.Field(field.name(), elementType, field.doc(),
field.defaultVal()));
builder = builder.setType(elementFieldSchema.getType());
+ if (elementFieldSchema.hasTimestampPrecision()) {
+
builder.setTimestampPrecision(elementFieldSchema.getTimestampPrecision());
+ }
builder.addAllFields(elementFieldSchema.getFieldsList());
builder = builder.setMode(TableFieldSchema.Mode.REPEATED);
break;
diff --git
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java
index adb8e4468c0..d940ff8dd7f 100644
---
a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java
+++
b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProto.java
@@ -237,6 +237,9 @@ public class BeamRowToStorageApiProto {
TableFieldSchema elementFieldSchema =
fieldDescriptorFromBeamField(Field.of(field.getName(),
elementType));
builder = builder.setType(elementFieldSchema.getType());
+ if (elementFieldSchema.hasTimestampPrecision()) {
+ builder =
builder.setTimestampPrecision(elementFieldSchema.getTimestampPrecision());
+ }
builder.addAllFields(elementFieldSchema.getFieldsList());
builder = builder.setMode(TableFieldSchema.Mode.REPEATED);
break;
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java
index deabb1dd05f..9698aaff1d7 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java
@@ -348,7 +348,23 @@ public class AvroGenericRecordToStorageApiProtoTest {
.endRecord();
}
+ private static Schema createRepeatedTimestampNanosSchema() {
+ Schema longSchema = Schema.create(Schema.Type.LONG);
+ longSchema.addProp("logicalType", "timestamp-nanos");
+
+ Schema arraySchema = Schema.createArray(longSchema);
+
+ return SchemaBuilder.record("RepeatedTimestampNanosRecord")
+ .fields()
+ .name("timestampNanosArray")
+ .type(arraySchema)
+ .noDefault()
+ .endRecord();
+ }
+
private static final Schema TIMESTAMP_NANOS_SCHEMA =
createTimestampNanosSchema();
+ private static final Schema REPEATED_TIMESTAMP_NANOS_SCHEMA =
+ createRepeatedTimestampNanosSchema();
private static GenericRecord baseRecord;
private static GenericRecord rawLogicalTypesRecord;
@@ -885,4 +901,22 @@ public class AvroGenericRecordToStorageApiProtoTest {
assertTrue(field.hasTimestampPrecision());
assertEquals(12L, field.getTimestampPrecision().getValue());
}
+
+ @Test
+ public void testProtoTableSchemaFromAvroSchemaRepeatedTimestampNanos() {
+ com.google.cloud.bigquery.storage.v1.TableSchema protoSchema =
+ AvroGenericRecordToStorageApiProto.protoTableSchemaFromAvroSchema(
+ REPEATED_TIMESTAMP_NANOS_SCHEMA);
+
+ assertEquals(1, protoSchema.getFieldsCount());
+ com.google.cloud.bigquery.storage.v1.TableFieldSchema field =
protoSchema.getFields(0);
+
+ assertEquals("timestampnanosarray", field.getName());
+ assertEquals(
+ com.google.cloud.bigquery.storage.v1.TableFieldSchema.Type.TIMESTAMP,
field.getType());
+ assertEquals(
+ com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode.REPEATED,
field.getMode());
+
+ assertEquals(12L, field.getTimestampPrecision().getValue());
+ }
}
diff --git
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java
index d7a88615a50..c546a7ca5d7 100644
---
a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java
+++
b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BeamRowToStorageApiProtoTest.java
@@ -69,6 +69,10 @@ public class BeamRowToStorageApiProtoTest {
Schema.builder()
.addField("timestampNanos",
FieldType.logicalType(Timestamp.NANOS).withNullable(true))
.build();
+ private static final Schema TIMESTAMP_NANOS_ARRAY_SCHEMA =
+ Schema.builder()
+ .addField("timestampNanosArray",
FieldType.array(FieldType.logicalType(Timestamp.NANOS)))
+ .build();
private static final EnumerationType TEST_ENUM =
EnumerationType.create("ONE", "TWO", "RED", "BLUE");
private static final Schema BASE_SCHEMA =
@@ -614,6 +618,19 @@ public class BeamRowToStorageApiProtoTest {
assertEquals(12L, field.getTimestampPrecision().getValue());
}
+ @Test
+ public void testTimestampNanosArraySchema() {
+ com.google.cloud.bigquery.storage.v1.TableSchema protoSchema =
+
BeamRowToStorageApiProto.protoTableSchemaFromBeamSchema(TIMESTAMP_NANOS_ARRAY_SCHEMA);
+
+ assertEquals(1, protoSchema.getFieldsCount());
+ TableFieldSchema field = protoSchema.getFields(0);
+ assertEquals(TableFieldSchema.Type.TIMESTAMP, field.getType());
+ assertEquals(
+ com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode.REPEATED,
field.getMode());
+ assertEquals(12L, field.getTimestampPrecision().getValue());
+ }
+
@Test
public void testTimestampNanosDescriptor() throws Exception {
DescriptorProto descriptor =
