This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c7ecd5e Include DATE_TIME in BaseDefaultColumnsHandler (#5379)
c7ecd5e is described below
commit c7ecd5e9a0d4a559e2e2ca628a6078ec1718f945
Author: Neha Pawar <[email protected]>
AuthorDate: Wed May 13 15:58:46 2020 -0700
Include DATE_TIME in BaseDefaultColumnsHandler (#5379)
---
.../defaultcolumn/BaseDefaultColumnHandler.java | 34 +++++++++++++++++-----
.../BaseDefaultColumnHandlerTest.java | 29 +++++++++++++++---
2 files changed, 52 insertions(+), 11 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
index df8b6cf..d7277b9 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandler.java
@@ -62,19 +62,23 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
// Present in schema but not in segment.
ADD_DIMENSION,
ADD_METRIC,
+ ADD_DATE_TIME,
// Present in segment but not in schema
REMOVE_DIMENSION,
REMOVE_METRIC,
+ REMOVE_DATE_TIME,
// Present in both segment and schema but one of the following updates is
needed
UPDATE_DIMENSION_DATA_TYPE,
UPDATE_DIMENSION_DEFAULT_VALUE,
UPDATE_DIMENSION_NUMBER_OF_VALUES,
UPDATE_METRIC_DATA_TYPE,
UPDATE_METRIC_DEFAULT_VALUE,
- UPDATE_METRIC_NUMBER_OF_VALUES;
+ UPDATE_METRIC_NUMBER_OF_VALUES,
+ UPDATE_DATE_TIME_DATA_TYPE,
+ UPDATE_DATE_TIME_DEFAULT_VALUE;
boolean isAddAction() {
- return this == ADD_DIMENSION || this == ADD_METRIC;
+ return this == ADD_DIMENSION || this == ADD_METRIC || this ==
ADD_DATE_TIME;
}
boolean isUpdateAction() {
@@ -82,7 +86,7 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
}
boolean isRemoveAction() {
- return this == REMOVE_DIMENSION || this == REMOVE_METRIC;
+ return this == REMOVE_DIMENSION || this == REMOVE_METRIC || this ==
REMOVE_DATE_TIME;
}
}
@@ -125,6 +129,8 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
LoaderUtils.getStringListFromSegmentProperties(V1Constants.MetadataKeys.Segment.DIMENSIONS,
_segmentProperties);
List<String> metricColumns =
LoaderUtils.getStringListFromSegmentProperties(V1Constants.MetadataKeys.Segment.METRICS,
_segmentProperties);
+ List<String> dateTimeColumns =
+
LoaderUtils.getStringListFromSegmentProperties(V1Constants.MetadataKeys.Segment.DATETIME_COLUMNS,
_segmentProperties);
for (Map.Entry<String, DefaultColumnAction> entry :
defaultColumnActionMap.entrySet()) {
String column = entry.getKey();
DefaultColumnAction action = entry.getValue();
@@ -135,18 +141,24 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
case ADD_METRIC:
metricColumns.add(column);
break;
+ case ADD_DATE_TIME:
+ dateTimeColumns.add(column);
+ break;
case REMOVE_DIMENSION:
dimensionColumns.remove(column);
break;
case REMOVE_METRIC:
metricColumns.remove(column);
break;
+ case REMOVE_DATE_TIME:
+ dateTimeColumns.remove(column);
default:
break;
}
}
_segmentProperties.setProperty(V1Constants.MetadataKeys.Segment.DIMENSIONS,
dimensionColumns);
_segmentProperties.setProperty(V1Constants.MetadataKeys.Segment.METRICS,
metricColumns);
+
_segmentProperties.setProperty(V1Constants.MetadataKeys.Segment.DATETIME_COLUMNS,
dateTimeColumns);
// Create a back up for origin metadata.
File metadataFile = _segmentProperties.getFile();
@@ -215,8 +227,7 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
} else if (isSingleValueInMetadata != isSingleValueInSchema) {
defaultColumnActionMap.put(column,
DefaultColumnAction.UPDATE_DIMENSION_NUMBER_OF_VALUES);
}
- } else {
- Preconditions.checkState(fieldTypeInMetadata ==
FieldSpec.FieldType.METRIC);
+ } else if (fieldTypeInMetadata == FieldSpec.FieldType.METRIC){
if (dataTypeInMetadata != dataTypeInSchema) {
defaultColumnActionMap.put(column,
DefaultColumnAction.UPDATE_METRIC_DATA_TYPE);
} else if (!defaultValueInSchema.equals(defaultValueInMetadata)) {
@@ -224,6 +235,12 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
} else if (isSingleValueInMetadata != isSingleValueInSchema) {
defaultColumnActionMap.put(column,
DefaultColumnAction.UPDATE_METRIC_NUMBER_OF_VALUES);
}
+ } else if (fieldTypeInMetadata == FieldSpec.FieldType.DATE_TIME){
+ if (dataTypeInMetadata != dataTypeInSchema) {
+ defaultColumnActionMap.put(column,
DefaultColumnAction.UPDATE_DATE_TIME_DATA_TYPE);
+ } else if (!defaultValueInSchema.equals(defaultValueInMetadata)) {
+ defaultColumnActionMap.put(column,
DefaultColumnAction.UPDATE_DATE_TIME_DEFAULT_VALUE);
+ }
}
} else {
// Column does not exist in the segment, add default value for it.
@@ -235,6 +252,8 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
case METRIC:
defaultColumnActionMap.put(column, DefaultColumnAction.ADD_METRIC);
break;
+ case DATE_TIME:
+ defaultColumnActionMap.put(column,
DefaultColumnAction.ADD_DATE_TIME);
default:
LOGGER.warn("Skip adding default column for column: {} with field
type: {}", column, fieldTypeInSchema);
break;
@@ -253,9 +272,10 @@ public abstract class BaseDefaultColumnHandler implements
DefaultColumnHandler {
FieldSpec.FieldType fieldTypeInMetadata =
columnMetadata.getFieldType();
if (fieldTypeInMetadata == FieldSpec.FieldType.DIMENSION) {
defaultColumnActionMap.put(column,
DefaultColumnAction.REMOVE_DIMENSION);
- } else {
- Preconditions.checkState(fieldTypeInMetadata ==
FieldSpec.FieldType.METRIC);
+ } else if (fieldTypeInMetadata == FieldSpec.FieldType.METRIC){
defaultColumnActionMap.put(column,
DefaultColumnAction.REMOVE_METRIC);
+ } else if (fieldTypeInMetadata == FieldSpec.FieldType.DATE_TIME) {
+ defaultColumnActionMap.put(column,
DefaultColumnAction.REMOVE_DATE_TIME);
}
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
index b1e8d83..d4132b0 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
@@ -168,8 +168,29 @@ public class BaseDefaultColumnHandlerTest {
Assert.assertEquals(defaultColumnHandler.computeDefaultColumnActionMap(),
ImmutableMap.of("column11", DefaultColumnAction.ADD_METRIC));
+ // Add metric in the schema
+ Schema schema4=
+ new
Schema.SchemaBuilder().setSchemaName("testTable").addSingleValueDimension("column1",
FieldSpec.DataType.INT)
+ .addSingleValueDimension("column2", FieldSpec.DataType.INT)
+ .addSingleValueDimension("column3", FieldSpec.DataType.STRING)
+ .addSingleValueDimension("column4", FieldSpec.DataType.STRING)
+ .addSingleValueDimension("column5", FieldSpec.DataType.STRING)
+ .addSingleValueDimension("column6", FieldSpec.DataType.INT)
+ .addSingleValueDimension("column7", FieldSpec.DataType.INT)
+ .addSingleValueDimension("column8", FieldSpec.DataType.INT)
+ .addSingleValueDimension("column9", FieldSpec.DataType.INT)
+ .addSingleValueDimension("column10", FieldSpec.DataType.INT)
+ .addSingleValueDimension("column13", FieldSpec.DataType.INT)
+ .addSingleValueDimension("count", FieldSpec.DataType.INT)
+ .addSingleValueDimension("daysSinceEpoch", FieldSpec.DataType.INT)
+ .addSingleValueDimension("weeksSinceEpochSunday",
FieldSpec.DataType.INT)
+ .addDateTime("column11", FieldSpec.DataType.INT, "1:HOURS:EPOCH",
"1:HOURS").build(); // add column11
+ defaultColumnHandler = new V3DefaultColumnHandler(segmentDirectory,
schema4, committedSegmentMetadata, writer);
+ Assert.assertEquals(defaultColumnHandler.computeDefaultColumnActionMap(),
+ ImmutableMap.of("column11", DefaultColumnAction.ADD_DATE_TIME));
+
// Do not remove non-autogenerated column in the segmentMetadata
- Schema schema4 =
+ Schema schema5 =
new
Schema.SchemaBuilder().setSchemaName("testTable").addSingleValueDimension("column1",
FieldSpec.DataType.INT)
.addSingleValueDimension("column3", FieldSpec.DataType.STRING) //
remove column2
.addSingleValueDimension("column4", FieldSpec.DataType.STRING)
@@ -183,11 +204,11 @@ public class BaseDefaultColumnHandlerTest {
.addSingleValueDimension("count", FieldSpec.DataType.INT)
.addSingleValueDimension("daysSinceEpoch", FieldSpec.DataType.INT)
.addSingleValueDimension("weeksSinceEpochSunday",
FieldSpec.DataType.INT).build();
- defaultColumnHandler = new V3DefaultColumnHandler(segmentDirectory,
schema4, committedSegmentMetadata, writer);
+ defaultColumnHandler = new V3DefaultColumnHandler(segmentDirectory,
schema5, committedSegmentMetadata, writer);
Assert.assertEquals(defaultColumnHandler.computeDefaultColumnActionMap(),
Collections.EMPTY_MAP);
// Do not update non-autogenerated column in the schema
- Schema schema5 =
+ Schema schema6 =
new
Schema.SchemaBuilder().setSchemaName("testTable").addSingleValueDimension("column1",
FieldSpec.DataType.INT)
.addSingleValueDimension("column2", FieldSpec.DataType.STRING) //
update datatype
.addSingleValueDimension("column3", FieldSpec.DataType.STRING)
@@ -202,7 +223,7 @@ public class BaseDefaultColumnHandlerTest {
.addSingleValueDimension("count", FieldSpec.DataType.INT)
.addSingleValueDimension("daysSinceEpoch", FieldSpec.DataType.INT)
.addSingleValueDimension("weeksSinceEpochSunday",
FieldSpec.DataType.INT).build();
- defaultColumnHandler = new V3DefaultColumnHandler(segmentDirectory,
schema5, committedSegmentMetadata, writer);
+ defaultColumnHandler = new V3DefaultColumnHandler(segmentDirectory,
schema6, committedSegmentMetadata, writer);
Assert.assertEquals(defaultColumnHandler.computeDefaultColumnActionMap(),
Collections.EMPTY_MAP);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
