sshkvar commented on a change in pull request #2757:
URL: https://github.com/apache/iceberg/pull/2757#discussion_r662034839
##########
File path: spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java
##########
@@ -100,4 +117,55 @@ public static void
validatePartitionTransforms(PartitionSpec spec) {
}
}
}
+
+ /**
+ * Responsible for checking if the table schema has a timestamp without
timezone column
+ * @param schema table schema to check if it contains a timestamp without
timezone column
+ * @return boolean indicating if the schema passed in has a timestamp field
without a timezone
+ */
+ public static boolean hasTimestampWithoutZone(Schema schema) {
+ return TypeUtil.find(schema, t ->
Types.TimestampType.withoutZone().equals(t)) != null;
+ }
+
+ /**
+ * Allow reading timestamp without time zone as timestamp with time zone.
Generally, this is not safe as timestamp
+ * without time zone is supposed to represent wall clock time semantics,
i.e. no matter the reader/writer timezone
+ * 3PM should always be read as 3PM, but timestamp with time zone represents
instant semantics, i.e the timestamp
+ * is adjusted so that the corresponding time in the reader timezone is
displayed.
+ * When set to false (default), we throw an exception at runtime
+ * "Spark does not support timestamp without time zone fields" if reading
timestamp without time zone fields
+ *
+ * @param readerConfig table read options
+ * @param sessionConf spark session configurations
+ * @return boolean indicating if reading timestamps without timezone is
allowed
+ */
+ public static boolean canHandleTimestampWithoutZone(Map<String, String>
readerConfig, RuntimeConfig sessionConf) {
+ String readerOption =
readerConfig.get(HANDLE_TIMESTAMP_WITHOUT_TIMEZONE_SESSION_PROPERTY);
+ if (readerOption != null) {
+ return Boolean.parseBoolean(readerOption);
+ }
+ String sessionConfValue =
sessionConf.get(HANDLE_TIMESTAMP_WITHOUT_TIMEZONE_SESSION_PROPERTY, null);
+ if (sessionConfValue != null) {
+ return Boolean.parseBoolean(sessionConfValue);
+ }
+ return false;
+ }
+
+ /**
+ * Check whether the spark session config contains a
'spark.sql.iceberg.read-timestamp-as-timestamp-without-timezone'
+ * property.
+ * Default value - false
+ *
+ * @param sessionConf a spark runtime config
+ * @return true if the session config has
spark.sql.iceberg.read-timestamp-as-timestamp-without-timezone property
+ * and this property is set to true
+ */
+ public static boolean shouldStoreTimestampWithoutZone(RuntimeConfig
sessionConf) {
+ String sessionConfValue =
sessionConf.get(READ_TIMESTAMP_AS_TIMESTAMP_WITHOUT_TIMEZONE, null);
Review comment:
for this case this is right property. This method checks should we use
`Types.TimestampType#withoutZone()` or `Types.TimestampType#withZone()` in new
tables.
Changed method name and added little bit more details in doc
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
