This is an automated email from the ASF dual-hosted git repository. nehapawar pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push: new 94aad60 Include transformFunction in timeFieldSpec toJsonObject (#5464) 94aad60 is described below commit 94aad60f520d627e257cac6c57c54abeafec37a2 Author: Neha Pawar <neha.pawa...@gmail.com> AuthorDate: Thu May 28 14:11:08 2020 -0700 Include transformFunction in timeFieldSpec toJsonObject (#5464) --- .../apache/pinot/common/data/FieldSpecTest.java | 10 ++++++--- pinot-common/src/test/resources/schemaTest.schema | 25 +++++++++++++++++++--- .../java/org/apache/pinot/spi/data/FieldSpec.java | 10 ++++++--- .../org/apache/pinot/spi/data/TimeFieldSpec.java | 1 + 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java b/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java index 9d29bd4..8063988 100644 --- a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java +++ b/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java @@ -262,7 +262,7 @@ public class FieldSpecTest { FieldSpec second; // Single-value boolean type dimension field with default null value. - String[] dimensionFields = {"\"name\":\"dimension\"", "\"dataType\":\"BOOLEAN\"", "\"defaultNullValue\":false"}; + String[] dimensionFields = {"\"name\":\"dimension\"", "\"dataType\":\"BOOLEAN\"", "\"defaultNullValue\":false", "\"transformFunction\":\"trim(foo)\""}; first = JsonUtils.stringToObject(getRandomOrderJsonString(dimensionFields), DimensionFieldSpec.class); second = JsonUtils.stringToObject(first.toJsonObject().toString(), DimensionFieldSpec.class); Assert.assertEquals(first, second, ERROR_MESSAGE); @@ -276,14 +276,18 @@ public class FieldSpecTest { // Time field with default value. String[] timeFields = - {"\"incomingGranularitySpec\":{\"timeUnitSize\":1, \"timeType\":\"MILLISECONDS\",\"dataType\":\"LONG\",\"name\":\"incomingTime\"}", "\"outgoingGranularitySpec\":{\"timeType\":\"SECONDS\",\"dataType\":\"INT\",\"name\":\"outgoingTime\"}", "\"defaultNullValue\":-1"}; + {"\"incomingGranularitySpec\":{\"timeUnitSize\":1, \"timeType\":\"MILLISECONDS\",\"dataType\":\"LONG\",\"name\":\"incomingTime\"}", + "\"outgoingGranularitySpec\":{\"timeType\":\"SECONDS\",\"dataType\":\"INT\",\"name\":\"outgoingTime\"}", + "\"defaultNullValue\":-1", + "\"transformFunction\":\"toEpochDays(millis)\""}; first = JsonUtils.stringToObject(getRandomOrderJsonString(timeFields), TimeFieldSpec.class); second = JsonUtils.stringToObject(first.toJsonObject().toString(), TimeFieldSpec.class); Assert.assertEquals(first, second, ERROR_MESSAGE); // DateTime field String[] dateTimeFields = - {"\"name\":\"Date\"", "\"dataType\":\"LONG\"", "\"format\":\"1:MILLISECONDS:EPOCH\"", "\"granularity\":\"5:MINUTES\"", "\"dateTimeType\":\"PRIMARY\""}; + {"\"name\":\"Date\"", "\"dataType\":\"LONG\"", "\"format\":\"1:MILLISECONDS:EPOCH\"", "\"granularity\":\"5:MINUTES\"", + "\"transformFunction\":\"fromEpochDays(daysSinceEpoch)\""}; first = JsonUtils.stringToObject(getRandomOrderJsonString(dateTimeFields), DateTimeFieldSpec.class); second = JsonUtils.stringToObject(first.toJsonObject().toString(), DateTimeFieldSpec.class); Assert.assertEquals(first, second, ERROR_MESSAGE); diff --git a/pinot-common/src/test/resources/schemaTest.schema b/pinot-common/src/test/resources/schemaTest.schema index 32a83c0..b7c7dcf 100644 --- a/pinot-common/src/test/resources/schemaTest.schema +++ b/pinot-common/src/test/resources/schemaTest.schema @@ -8,7 +8,8 @@ { "name": "longMetric", "dataType": "LONG", - "singleValueField": true + "singleValueField": true, + "transformFunction": "Groovy({foo/1000}, foo)" }, { "name": "floatMetric", @@ -37,7 +38,8 @@ "name": "longDimension", "dataType": "LONG", "singleValueField": false, - "delimiter": null + "delimiter": null, + "transformFunction": "trim(bar)" }, { "name": "floatDimension", @@ -69,7 +71,24 @@ "timeType": "DAYS", "name": "outgoing" }, - "defaultNullValue": 10000 + "defaultNullValue": 10000, + "transformFunction": "toEpochDays(millis)" }, + "dateTimeFieldSpecs": [ + { + "name": "dateTime1", + "dataType": "LONG", + "format": "1:MILLISECONDS:EPOCH", + "granularity": "15:MINUTES", + "transformFunction": "fromDateTime(dt, 'yyyyMMdd')" + }, + { + "name": "dateTime2", + "dataType": "STRING", + "format": "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", + "granularity": "1:DAYS", + "defaultNullValue": "20200101" + } + ], "schemaName": "schemaTest" } diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java index de3e3ee..aabbd7f 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java @@ -263,9 +263,7 @@ public abstract class FieldSpec implements Comparable<FieldSpec> { jsonObject.put("maxLength", _maxLength); } appendDefaultNullValue(jsonObject); - if (_transformFunction != null) { - jsonObject.put("transformFunction", _transformFunction); - } + appendTransformFunction(jsonObject); return jsonObject; } @@ -280,6 +278,12 @@ public abstract class FieldSpec implements Comparable<FieldSpec> { } } + protected void appendTransformFunction(ObjectNode jsonNode) { + if (_transformFunction != null) { + jsonNode.put("transformFunction", _transformFunction); + } + } + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @Override public boolean equals(Object o) { diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java index 5fefbf7..1699ab8 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java @@ -112,6 +112,7 @@ public final class TimeFieldSpec extends FieldSpec { jsonObject.set("outgoingGranularitySpec", _outgoingGranularitySpec.toJsonObject()); } appendDefaultNullValue(jsonObject); + appendTransformFunction(jsonObject); return jsonObject; } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org