This is an automated email from the ASF dual-hosted git repository. arina pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/drill.git.
from 8e6cdbe DRILL-7446: Fix Eclipse compilation issue in AbstractParquetGroupScan new ffd3c35 DRILL-7388: Kafka improvements new 120e10b DRILL-7448: Fix warnings when running Drill memory tests new 5b9380e DRILL-7441: Fix issues with fillEmpties, offset vectors new 0230627 DRILL-7445: Create batch copier based on result set framework The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: common/pom.xml | 8 - .../java/org/apache/drill/common/types/Types.java | 32 +- common/src/test/resources/logback-test.xml | 47 +- .../apache/drill/exec/hive/TestHiveStorage.java | 12 +- contrib/storage-kafka/README.md | 12 +- contrib/storage-kafka/pom.xml | 6 +- .../drill/exec/store/kafka/KafkaAsyncCloser.java | 105 ++++ .../drill/exec/store/kafka/KafkaGroupScan.java | 52 +- .../drill/exec/store/kafka/KafkaNodeProcessor.java | 1 - .../exec/store/kafka/KafkaPartitionScanSpec.java | 10 +- .../store/kafka/KafkaPartitionScanSpecBuilder.java | 111 ++-- .../store/kafka/KafkaPushDownFilterIntoScan.java | 25 +- .../drill/exec/store/kafka/KafkaRecordReader.java | 85 ++- .../exec/store/kafka/KafkaScanBatchCreator.java | 14 +- .../drill/exec/store/kafka/KafkaScanSpec.java | 3 +- .../drill/exec/store/kafka/KafkaStoragePlugin.java | 21 +- .../exec/store/kafka/KafkaStoragePluginConfig.java | 35 +- .../drill/exec/store/kafka/KafkaSubScan.java | 2 +- .../drill/exec/store/kafka/MessageIterator.java | 50 +- .../drill/exec/store/kafka/MetaDataField.java | 10 +- .../apache/drill/exec/store/kafka/ReadOptions.java | 95 +++ .../store/kafka/decoders/JsonMessageReader.java | 95 ++- .../exec/store/kafka/decoders/MessageReader.java | 10 +- .../store/kafka/decoders/MessageReaderFactory.java | 4 +- .../store/kafka/schema/KafkaMessageSchema.java | 14 +- .../store/kafka/schema/KafkaSchemaFactory.java | 5 +- .../main/resources/bootstrap-storage-plugins.json | 10 +- .../exec/store/kafka/KafkaFilterPushdownTest.java | 22 +- .../exec/store/kafka/KafkaMessageGenerator.java | 99 ++- .../drill/exec/store/kafka/KafkaQueriesTest.java | 173 +++++- .../drill/exec/store/kafka/KafkaTestBase.java | 12 +- .../exec/store/kafka/MessageIteratorTest.java | 5 +- .../drill/exec/store/kafka/TestKafkaSuit.java | 81 +-- .../store/kafka/cluster/EmbeddedKafkaCluster.java | 57 +- .../kafka/decoders/MessageReaderFactoryTest.java | 6 +- .../java/org/apache/drill/exec/ExecConstants.java | 10 + .../physical/impl/aggregate/BatchIterator.java | 2 - .../impl/protocol/IndirectContainerAccessor.java | 100 ++++ .../impl/protocol/VectorContainerAccessor.java | 41 +- .../physical/impl/validate/BatchValidator.java | 174 +++--- .../exec/physical/resultSet/ResultSetCopier.java | 189 ++++++ .../exec/physical/resultSet/ResultSetReader.java | 7 + .../resultSet/impl/ResultSetCopierImpl.java | 313 ++++++++++ .../resultSet/impl/ResultSetLoaderImpl.java | 1 + .../resultSet/impl/ResultSetReaderImpl.java | 3 + .../exec/physical/rowSet/RowSetFormatter.java | 13 +- .../exec/record/selection/SelectionVector2.java | 13 +- .../selection/SelectionVector2Builder.java} | 35 +- .../exec/server/options/SystemOptionManager.java | 3 + .../drill/exec/vector/complex/fn/JsonReader.java | 1 + .../drill/exec/work/batch/BaseRawBatchBuffer.java | 1 - .../java-exec/src/main/resources/drill-module.conf | 3 + .../test/java/org/apache/drill/TestBugFixes.java | 31 +- .../test/java/org/apache/drill/TestCTASJson.java | 20 +- .../java/org/apache/drill/TestFrameworkTest.java | 12 +- .../java/org/apache/drill/TestJoinNullable.java | 1 - .../exec/compile/TestLargeFileCompilation.java | 22 +- .../physical/impl/TestNestedDateTimeTimestamp.java | 10 +- .../physical/impl/join/TestMergeJoinAdvanced.java | 22 +- .../impl/protocol/TestOperatorRecordBatch.java | 9 +- .../physical/impl/validate/TestBatchValidator.java | 2 +- .../writer/TestCorruptParquetDateCorrection.java | 23 +- .../resultSet/impl/TestResultSetCopier.java | 663 +++++++++++++++++++++ .../resultSet/impl/TestResultSetReader.java | 13 +- .../exec/physical/rowSet/TestMapAccessors.java | 16 +- .../physical/rowSet/TestOffsetVectorWriter.java | 14 +- .../exec/physical/rowSet/TestScalarAccessors.java | 217 ++++++- .../drill/exec/planner/sql/TestDrillSQLWorker.java | 8 +- .../org/apache/drill/exec/server/TestOptions.java | 12 +- .../java/org/apache/drill/exec/sql/TestCTAS.java | 16 +- .../java/org/apache/drill/exec/sql/TestCTTAS.java | 30 +- .../complex/fn/TestJsonReaderWithSparseFiles.java | 44 +- .../exec/vector/complex/writer/TestJsonNanInf.java | 20 +- .../exec/vector/complex/writer/TestJsonReader.java | 11 +- .../TestLimit0VsRegularQueriesMetadata.java | 12 +- .../org/apache/drill/test/BaseDirTestWatcher.java | 12 +- .../java/org/apache/drill/test/BaseFixture.java | 27 +- .../java/org/apache/drill/test/BaseTestQuery.java | 108 ++-- .../java/org/apache/drill/test/ClusterFixture.java | 59 +- .../org/apache/drill/test/DrillTestWrapper.java | 8 +- .../org/apache/drill/test/SubOperatorTest.java | 1 + .../apache/drill/exec/memory/BaseAllocator.java | 14 +- .../main/codegen/templates/ColumnAccessors.java | 21 +- .../src/main/codegen/templates/ComplexWriters.java | 14 +- .../main/codegen/templates/FixedValueVectors.java | 22 +- .../src/main/codegen/templates/ListWriters.java | 1 - .../src/main/codegen/templates/MapWriters.java | 24 +- .../codegen/templates/NullableValueVectors.java | 119 ++-- .../codegen/templates/VariableLengthVectors.java | 190 +++--- .../drill/exec/record/MaterializedField.java | 1 + .../apache/drill/exec/vector/NullableVector.java | 8 +- .../drill/exec/vector/accessor/ColumnWriter.java | 9 + .../drill/exec/vector/accessor/ObjectReader.java | 7 + .../drill/exec/vector/accessor/ObjectWriter.java | 11 +- .../accessor/convert/AbstractWriteConverter.java | 6 + .../exec/vector/accessor/impl/VectorPrinter.java | 9 + .../accessor/reader/AbstractObjectReader.java | 1 + .../accessor/reader/AbstractTupleReader.java | 16 +- .../vector/accessor/reader/ArrayReaderImpl.java | 1 - .../vector/accessor/reader/BaseScalarReader.java | 12 + .../vector/accessor/reader/OffsetVectorReader.java | 1 + .../vector/accessor/reader/UnionReaderImpl.java | 12 +- .../accessor/writer/AbstractArrayWriter.java | 14 + .../accessor/writer/AbstractObjectWriter.java | 11 +- .../accessor/writer/AbstractTupleWriter.java | 22 +- .../vector/accessor/writer/BaseVarWidthWriter.java | 8 +- .../vector/accessor/writer/BitColumnWriter.java | 8 + .../exec/vector/accessor/writer/MapWriter.java | 9 +- .../accessor/writer/NullableScalarWriter.java | 12 +- .../accessor/writer/OffsetVectorWriterImpl.java | 28 +- .../vector/accessor/writer/UnionWriterImpl.java | 13 +- .../accessor/writer/dummy/DummyArrayWriter.java | 4 + .../accessor/writer/dummy/DummyScalarWriter.java | 4 + .../vector/complex/AbstractRepeatedMapVector.java | 32 +- .../vector/complex/BaseRepeatedValueVector.java | 16 +- .../exec/vector/complex/EmptyValuePopulator.java | 42 +- .../exec/vector/VariableLengthVectorTest.java | 52 +- .../org/apache/drill/exec/vector/VectorTest.java | 623 +++++++++++++++++++ 118 files changed, 3885 insertions(+), 1193 deletions(-) create mode 100644 contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaAsyncCloser.java create mode 100644 contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/ReadOptions.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/protocol/IndirectContainerAccessor.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/ResultSetCopier.java create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ResultSetCopierImpl.java copy exec/java-exec/src/main/java/org/apache/drill/exec/{physical/impl/aggregate/HashAggUpdater.java => record/selection/SelectionVector2Builder.java} (55%) create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetCopier.java create mode 100644 exec/vector/src/test/java/org/apache/drill/exec/vector/VectorTest.java