This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 889889e include pinot-confluent-avro plugin to the distribution
(#5248)
add b20ace0 Evaluate schema transform expressions during ingestion (#5238)
No new revisions were added by this update.
Summary of changes:
.../recordtransformer/CompositeTransformer.java | 7 +-
.../recordtransformer/DataTypeTransformer.java | 2 +-
.../recordtransformer/ExpressionTransformer.java | 26 ++-
.../data/recordtransformer/TimeTransformer.java | 99 -----------
.../data/readers/RecordReaderSampleDataTest.java | 88 +++++-----
.../ExpressionTransformerTest.java | 162 ++++++++++++++++++
.../ExpressionTransformerTimeTest.java | 182 +++++++++++++++++++++
.../recordtransformer/TimeTransformerTest.java | 151 -----------------
.../groovy_expression_transformer.json | 62 +++++++
.../pinot/integration/tests/ClusterTest.java | 10 +-
.../tests/MapTypeClusterIntegrationTest.java | 10 +-
.../inputformat/avro/AvroRecordExtractor.java | 31 +++-
.../plugin/inputformat/avro/AvroRecordReader.java | 21 +--
.../pinot/plugin/inputformat/avro/AvroUtils.java | 70 ++++----
.../avro/AvroRecordExtractorMapTypeTest.java | 118 +++++++++++++
.../inputformat/avro/AvroRecordExtractorTest.java | 85 ++++++++++
.../avro/AvroRecordToPinotRowGeneratorTest.java | 13 +-
.../inputformat/avro/KafkaAvroMessageDecoder.java | 8 +-
.../inputformat/avro/SimpleAvroMessageDecoder.java | 8 +-
...aConfluentSchemaRegistryAvroMessageDecoder.java | 31 ++--
.../plugin/inputformat/csv/CSVRecordExtractor.java | 72 ++++++++
.../inputformat/csv/CSVRecordExtractorConfig.java | 19 ++-
.../plugin/inputformat/csv/CSVRecordReader.java | 26 +--
.../inputformat/csv/CSVRecordExtractorTest.java | 104 ++++++++++++
.../inputformat/csv/CSVRecordReaderTest.java | 30 ++++
.../inputformat/json/JSONRecordExtractor.java | 99 +++++++++++
.../plugin/inputformat/json/JSONRecordReader.java | 20 +--
...eaderTest.java => JSONRecordExtractorTest.java} | 34 ++--
.../inputformat/json/JSONRecordReaderTest.java | 26 +++
...RCRecordReader.java => ORCRecordExtractor.java} | 107 +++---------
.../inputformat/orc/ORCRecordExtractorConfig.java | 19 ++-
.../plugin/inputformat/orc/ORCRecordReader.java | 108 +-----------
.../inputformat/orc/ORCRecordExtractorTest.java | 102 ++++++++++++
.../parquet/ParquetRecordExtractor.java | 36 ++--
.../inputformat/parquet/ParquetRecordReader.java | 20 +--
.../parquet/ParquetRecordExtractorTest.java | 90 ++++++++++
.../inputformat/thrift/ThriftRecordExtractor.java | 58 +++++++
.../thrift/ThriftRecordExtractorConfig.java | 22 ++-
.../inputformat/thrift/ThriftRecordReader.java | 26 +--
.../pinot-kafka-base/pom.xml | 8 +
.../stream/kafka/KafkaJSONMessageDecoder.java | 38 +++--
pinot-spi/pom.xml | 4 +
.../java/org/apache/pinot/spi/data/Schema.java | 3 +-
.../evaluators/DefaultTimeSpecEvaluator.java | 73 +++++++++
.../function/evaluators/ExpressionEvaluator.java | 22 ++-
.../evaluators/ExpressionEvaluatorFactory.java | 114 +++++++++++++
.../evaluators/GroovyExpressionEvaluator.java | 102 ++++++++++++
.../data/readers/AbstractBaseRecordExtractor.java | 40 -----
.../apache/pinot/spi/data/readers/GenericRow.java | 9 +
.../pinot/spi/data/readers/RecordExtractor.java | 27 ++-
...eaderConfig.java => RecordExtractorConfig.java} | 6 +-
.../pinot/spi/data/readers/RecordReader.java | 2 +
.../pinot/spi/data/readers/RecordReaderUtils.java | 178 ++++----------------
.../pinot/spi/utils/SchemaFieldExtractorUtils.java | 133 +++++++++++++++
.../org/apache/pinot/spi/utils}/TimeConverter.java | 2 +-
.../evaluators/GroovyExpressionEvaluatorTest.java | 87 ++++++++++
.../data/readers/AbstractRecordExtractorTest.java | 155 ++++++++++++++++++
.../spi/data/readers/AbstractRecordReaderTest.java | 2 +-
.../spi/data/readers/RecordReaderUtilsTest.java | 54 +-----
.../spi/utils/SchemaFieldExtractorUtilsTest.java | 168 +++++++++++++++++++
.../apache/pinot/spi/utils}/TimeConverterTest.java | 2 +-
.../groovy_map_transform_functions_schema.json | 27 +++
.../groovy_transform_functions_schema.json | 44 +++++
pinot-tools/pom.xml | 5 +
pom.xml | 11 ++
65 files changed, 2538 insertions(+), 980 deletions(-)
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/TimeTransformer.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformerTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformerTimeTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/TimeTransformerTest.java
create mode 100644
pinot-core/src/test/resources/data/expression_transformer/groovy_expression_transformer.json
create mode 100644
pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractorMapTypeTest.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-avro-base/src/test/java/org/apache/pinot/plugin/inputformat/avro/AvroRecordExtractorTest.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractor.java
copy
pinot-core/src/main/java/org/apache/pinot/core/common/predicate/BaseInPredicate.java
=>
pinot-plugins/pinot-input-format/pinot-csv/src/main/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorConfig.java
(64%)
create mode 100644
pinot-plugins/pinot-input-format/pinot-csv/src/test/java/org/apache/pinot/plugin/inputformat/csv/CSVRecordExtractorTest.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONRecordExtractor.java
copy
pinot-plugins/pinot-input-format/pinot-json/src/test/java/org/apache/pinot/plugin/inputformat/json/{JSONRecordReaderTest.java
=> JSONRecordExtractorTest.java} (61%)
copy
pinot-plugins/pinot-input-format/pinot-orc/src/main/java/org/apache/pinot/plugin/inputformat/orc/{ORCRecordReader.java
=> ORCRecordExtractor.java} (59%)
copy
pinot-core/src/main/java/org/apache/pinot/core/minion/segment/DefaultRecordPartitioner.java
=>
pinot-plugins/pinot-input-format/pinot-orc/src/main/java/org/apache/pinot/plugin/inputformat/orc/ORCRecordExtractorConfig.java
(64%)
create mode 100644
pinot-plugins/pinot-input-format/pinot-orc/src/test/java/org/apache/pinot/plugin/inputformat/orc/ORCRecordExtractorTest.java
copy
pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/fakestream/FakeStreamLevelConsumer.java
=>
pinot-plugins/pinot-input-format/pinot-parquet/src/main/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordExtractor.java
(51%)
create mode 100644
pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordExtractorTest.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractor.java
copy
thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomalydetection/alertFilterAutotune/AlertFilterAutoTune.java
=>
pinot-plugins/pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractorConfig.java
(65%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/data/function/evaluators/DefaultTimeSpecEvaluator.java
copy
pinot-core/src/main/java/org/apache/pinot/core/minion/segment/RecordAggregator.java
=>
pinot-spi/src/main/java/org/apache/pinot/spi/data/function/evaluators/ExpressionEvaluator.java
(59%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/data/function/evaluators/ExpressionEvaluatorFactory.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/data/function/evaluators/GroovyExpressionEvaluator.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/AbstractBaseRecordExtractor.java
copy
pinot-spi/src/main/java/org/apache/pinot/spi/data/readers/{RecordReaderConfig.java
=> RecordExtractorConfig.java} (89%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/utils/SchemaFieldExtractorUtils.java
rename {pinot-common/src/main/java/org/apache/pinot/common/utils/time =>
pinot-spi/src/main/java/org/apache/pinot/spi/utils}/TimeConverter.java (98%)
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/data/function/evaluators/GroovyExpressionEvaluatorTest.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/data/readers/AbstractRecordExtractorTest.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/utils/SchemaFieldExtractorUtilsTest.java
rename {pinot-common/src/test/java/org/apache/pinot/common/utils/time =>
pinot-spi/src/test/java/org/apache/pinot/spi/utils}/TimeConverterTest.java (99%)
create mode 100644
pinot-spi/src/test/resources/groovy_map_transform_functions_schema.json
create mode 100644
pinot-spi/src/test/resources/groovy_transform_functions_schema.json
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
