This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 37fcfb8 Fixing TIMESTAMP data type usage during segment creation
(#8407)
37fcfb8 is described below
commit 37fcfb8fd51120026169a0653031c3049b3a5583
Author: Xiang Fu <[email protected]>
AuthorDate: Thu Mar 24 19:52:21 2022 -0700
Fixing TIMESTAMP data type usage during segment creation (#8407)
---
.../transform/transformer/datetime/DateTimeTransformerFactory.java | 6 +++---
.../apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java | 3 ++-
.../spi/creator/name/NormalizedDateSegmentNameGenerator.java | 4 ++--
pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java | 4 ++--
4 files changed, 9 insertions(+), 8 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java
index 60552eb..88e28ff 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/transformer/datetime/DateTimeTransformerFactory.java
@@ -35,14 +35,14 @@ public class DateTimeTransformerFactory {
TimeFormat inputTimeFormat = inputFormat.getTimeFormat();
TimeFormat outputTimeFormat = outputFormat.getTimeFormat();
- if (inputTimeFormat == TimeFormat.EPOCH) {
- if (outputTimeFormat == TimeFormat.EPOCH) {
+ if (inputTimeFormat == TimeFormat.EPOCH || inputTimeFormat ==
TimeFormat.TIMESTAMP) {
+ if (outputTimeFormat == TimeFormat.EPOCH || outputTimeFormat ==
TimeFormat.TIMESTAMP) {
return new EpochToEpochTransformer(inputFormat, outputFormat,
outputGranularity);
} else {
return new EpochToSDFTransformer(inputFormat, outputFormat,
outputGranularity);
}
} else {
- if (outputTimeFormat == TimeFormat.EPOCH) {
+ if (outputTimeFormat == TimeFormat.EPOCH || outputTimeFormat ==
TimeFormat.TIMESTAMP) {
return new SDFToEpochTransformer(inputFormat, outputFormat,
outputGranularity);
} else {
return new SDFToSDFTransformer(inputFormat, outputFormat,
outputGranularity);
diff --git
a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
index 5ae8966..aad0e63 100644
---
a/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
+++
b/pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java
@@ -71,7 +71,8 @@ public class SegmentPreprocessingMapper
String timeType =
_jobConf.get(InternalConfigConstants.SEGMENT_TIME_TYPE);
String timeFormat =
_jobConf.get(InternalConfigConstants.SEGMENT_TIME_FORMAT);
DateTimeFormatSpec dateTimeFormatSpec;
- if (timeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString())) {
+ if (timeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()) ||
timeFormat.equals(
+ DateTimeFieldSpec.TimeFormat.TIMESTAMP.toString())) {
dateTimeFormatSpec = new DateTimeFormatSpec(1, timeType, timeFormat);
} else {
dateTimeFormatSpec = new DateTimeFormatSpec(1, timeType, timeFormat,
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
index a65ef57..decbc50 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/creator/name/NormalizedDateSegmentNameGenerator.java
@@ -72,10 +72,10 @@ public class NormalizedDateSegmentNameGenerator implements
SegmentNameGenerator
}
_outputSDF.setTimeZone(TimeZone.getTimeZone("UTC"));
- // Parse input time format: 'EPOCH' or 'SIMPLE_DATE_FORMAT' using pattern
+ // Parse input time format: 'EPOCH'/'TIMESTAMP' or 'SIMPLE_DATE_FORMAT'
using pattern
Preconditions.checkNotNull(dateTimeFormatSpec);
TimeFormat timeFormat = dateTimeFormatSpec.getTimeFormat();
- if (timeFormat == TimeFormat.EPOCH) {
+ if (timeFormat == TimeFormat.EPOCH || timeFormat ==
TimeFormat.TIMESTAMP) {
_inputTimeUnit = dateTimeFormatSpec.getColumnUnit();
_inputSDF = null;
} else {
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java
index 0193d31..e445b2e 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/Schema.java
@@ -752,8 +752,8 @@ public final class Schema implements Serializable {
TimeUnit incomingTimeUnit = incomingGranularitySpec.getTimeType();
String incomingTimeFormat = incomingGranularitySpec.getTimeFormat();
Preconditions.checkState(
-
incomingTimeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()) &&
outgoingTimeFormat
- .equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()),
+
(incomingTimeFormat.equals(DateTimeFieldSpec.TimeFormat.EPOCH.toString()) ||
incomingTimeFormat.equals(
+ DateTimeFieldSpec.TimeFormat.TIMESTAMP.toString())) &&
outgoingTimeFormat.equals(incomingTimeFormat),
"Conversion from incoming to outgoing is not supported for
SIMPLE_DATE_FORMAT");
String transformFunction =
constructTransformFunctionString(incomingName, incomingTimeSize,
incomingTimeUnit, outgoingTimeSize,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
