This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch xiangfu0/codex1/sorted-exchange-core
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit be31a33d114 Preserve partition workers for complete-input window sorts
omit 80558b56997 Align stage-equivalence field coverage with receive nodes
omit e07e184497c Remove the extra merge-cursor blank line
omit 810ca14cfde Keep complete window input when fallback stages use older
servers
omit f0f839c6b9f Complete ordered-window review fixes
omit 3194d04f5b1 Address ordered-window merge review feedback
omit 067bdcb5c75 [test] Verify homogeneous snapshot builds disable k-way
merge
omit 1cf85058a9a Align window input sort plan expectations with complete
input
omit d5f5d3d766c Retain complete input for global window receiver sorts
omit 9bc53a4ab67 [MSE] Express ordered streams with a k-way merge exchange
omit ead7520cede [build] Include Zstd API annotations for compiler
validation
omit 9205e896d33 [MSE] Use current planner test APIs
omit fd01b62a2f2 [MSE] Simplify sorted mailbox merging
omit 8d222b8cdc2 [MSE] Speed up four-sender sorted window merge
omit eaae4f9c621 [MSE] Address sender-sorted window review feedback
omit 15963008a34 [MSE] Minimize sender-sorted merge changes
omit cd200a52a73 [MSE] Avoid eager sort buffer allocation
omit 06223c4a090 [MSE] Limit sender-sorted merge to global windows
omit 7f360018b3f [MSE] Optimize sorted mailbox merge
omit baf1c493137 [MSE] Merge sender-sorted streams for ordered windows
add f526b645ce8 Bump software.amazon.awssdk:bom from 2.57.3 to 2.57.4
(#20103)
add 863356920f2 Bump org.roaringbitmap:RoaringBitmap from 1.6.25 to 1.6.27
(#20102)
add e12b8435c75 Add per-table SHADOW enforcement mode for storage and
query quotas (#19838)
add 1b552624f52 Add a broker hook to prepare each MSE stage before
dispatch (#20104)
add 3117dbb262a Avoid repeated force commits during rebalance (#20097)
add 96fe7141147 Merge final results of array, raw sketch and other
aggregation functions (#20098)
add eca9ea7afde Reuse the shared hybrid cluster for tenant rebalance tests
(#20084)
add 66eb74d69d1 Add requiresFetch flag to a segment to for prefetch
planning (#20068)
add d226fb5b358 Reduce instance selector memory by grouping segments with
the same candidates (#20077)
add df57b73726a Store sealed segment metadata in a mapped sidecar (#19998)
add bfdb305e148 Skip mailbox config scans for non-server membership
changes (#20107)
add e9be2ee203a Bump follow-redirects in
/pinot-controller/src/main/resources (#20110)
add 2b0da67bc43 Extract hybrid table retention check into overridable
shouldEnableHybridTableRetentionStrategy method (#20088)
add 9fe9d4d8f04 Spy a plain RetentionManager in hybrid retention override
tests so subclass suites can reuse them (#20119)
add 250cbe0b654 Bump com.github.jnr:jnr-ffi from 2.3.4 to 2.3.5 (#20132)
add ad1f044fe02 Bump joda-time:joda-time from 2.15.0 to 2.15.1 (#20131)
add 0ea960fb6a9 Bump org.apache.zookeeper:zookeeper from 3.9.6 to 3.9.7
(#20127)
add 300d124b1aa Bump org.apache.maven.plugins:maven-enforcer-plugin from
3.6.3 to 3.7.0 (#20126)
add 68b836d6f2a Bump wire.version from 7.1.0 to 7.1.2 (#20124)
add 4d76004ccad Bump com.google.cloud:libraries-bom from 26.90.0 to
26.91.0 (#20123)
add 63cd8e006d1 Let segment directories serve mapped segment metadata
(#20114)
add 677a903191f Bump org.roaringbitmap:RoaringBitmap from 1.6.27 to 1.6.29
(#20130)
add 5d81ce915d1 Bump com.github.luben:zstd-jni from 1.5.7-24 to 1.5.7-25
(#20125)
add 3ac4b6e6c7b Add option to require an existing server data directory
(#20115)
add 791cdbd86c7 Bump software.amazon.awssdk:bom from 2.57.4 to 2.57.7
(#20129)
add 3fc2fd8c126 Bump org.apache.maven.enforcer:enforcer-api from 3.6.3 to
3.7.0 (#20128)
add 25d05ce4a90 Bump org.mongodb:bson from 5.13.0 to 5.14.0 (#20122)
add 9bacd8af0f9 Handle literal ORDER BY keys in single-stage queries
(#20134)
add 772f7723380 [test] Stabilize table rebalance disk utilization
prechecks (#20135)
add 4f8a1b419a4 Bump com.github.jnr:jnr-ffi from 2.3.5 to 2.3.6 (#20139)
add 2c5ff02a3e4 Bump software.amazon.awssdk:bom from 2.57.7 to 2.57.8
(#20140)
add 361346ce7b8 Bump org.webjars:swagger-ui from 5.34.0 to 5.34.1 (#20142)
add 16321377e94 Bump com.github.jnr:jnr-constants from 0.11.2 to 0.11.3
(#20141)
add da086263777 Bump
org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_25 (#20143)
add 9f812e9a0e2 Bump com.github.jnr:jnr-posix from 3.2.5 to 3.2.6 (#20144)
add 660962fbdc1 Include column name in realtime indexing error logs
(#20138)
add badcdc0d131 Check for query termination before each transform
evaluation (#19907)
add 837acb93ef9 Add streaming selection ORDER BY combine for physically
sorted segments (#19120)
add 2974ced3548 Bump com.puppycrawl.tools:checkstyle from 14.3.0 to 14.4.0
(#20150)
add fd1d3f7b590 Name upsert validDocIds bitmaps read from or written to
disk *OnDiskBitmap (#20152)
add 3e7342c5319 Use a single Dependabot label (#20155)
add 2bacc8889b2 Bump dnsjava:dnsjava from 3.7.0 to 3.7.1 (#20156)
add 091eac79529 Fix HyperLogLogPlus max byte size and reject sp above 25
(#20154)
add bb054a49afe Add a README Docker quickstart and improve GitHub metadata
(#20164)
add 14b564428ee Make Utf8Utils.encodedLength agree with encode() on
unpaired surrogates (#20153)
add 684b7a4aa5f Add metric for local realtime segment builds with CRC
mismatch (#19450)
add 8190e43747f Re-run query optimizer after expression override for
non-hybrid tables (#20148)
add a7a9761605a Allow zero system activities CPU time in query killing
tests (#20159)
add 5765dab883d Let ControllerTest subclasses set the cluster configs a
controller starts with (#20158)
add 999b978f8b6 Add per-table hybrid retention strategy with cluster-level
fallback (#20116)
add 0a53497fddf Consolidate Kafka ingestion on the Kafka 4 plugin (#20112)
add 81296e98957 Bump com.google.guava:guava-bom from 33.7.2-jre to
33.8.0-jre (#20178)
add 2d2c436d481 Bump net.minidev:json-smart from 2.6.0 to 2.6.1 (#20181)
add ca5f5fc3e9e Bump com.google.errorprone:error_prone_annotations from
2.51.0 to 2.52.0 (#20180)
add 8d1bc907a08 Bump org.checkerframework:checker-qual from 4.3.1 to 4.4.0
(#20179)
add dd880ab964a Bump com.github.jnr:jnr-ffi from 2.3.6 to 2.3.7 (#20177)
add 519e604a826 Bump com.github.jnr:jnr-posix from 3.2.6 to 3.2.7 (#20173)
add 229902bfe1d Bump com.github.jnr:jnr-constants from 0.11.3 to 0.11.4
(#20175)
add 3f758a2fe24 Bump org.bitbucket.b_c:jose4j from 0.9.6 to 0.9.7 (#20174)
add 8d140dcfba3 Fix rlsFiltersApplied reporting true for multi-stage
queries when no RLS filter is applied (#20089)
add bd5cf5e6d47 Clarify review compatibility and style conventions (#20184)
add f60f17fd701 Reject cross-table functions under row-level security
(#20169)
add f54a4250126 [MSE] Merge sender-sorted streams for ordered windows
add fcab501e2ac [MSE] Optimize sorted mailbox merge
add bbebde5623c [MSE] Limit sender-sorted merge to global windows
add 5b51d3861b5 [MSE] Avoid eager sort buffer allocation
add 16f3427c0fa [MSE] Minimize sender-sorted merge changes
add 67b8673ee7a [MSE] Address sender-sorted window review feedback
add 849ab557b52 [MSE] Speed up four-sender sorted window merge
add d44817045e4 [MSE] Simplify sorted mailbox merging
add 6184dd16782 [MSE] Use current planner test APIs
add 9f6396330dd [build] Include Zstd API annotations for compiler
validation
add 3b82fb1048b [MSE] Express ordered streams with a k-way merge exchange
add 1a30e7659f9 Retain complete input for global window receiver sorts
add 1168d4a3dd9 Align window input sort plan expectations with complete
input
add 61b9dc7505b [test] Verify homogeneous snapshot builds disable k-way
merge
add c53a1a8e4fa Address ordered-window merge review feedback
add b79fb67d40d Complete ordered-window review fixes
add 2d0a32fc8f8 Keep complete window input when fallback stages use older
servers
add 4f5a4a73a43 Remove the extra merge-cursor blank line
add 9b284ae650a Align stage-equivalence field coverage with receive nodes
add cdcc8f2ad10 Preserve partition workers for complete-input window sorts
add 6ad5bda7978 Remove stale atomic import after upstream rebase
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (be31a33d114)
\
N -- N -- N refs/heads/xiangfu0/codex1/sorted-exchange-core
(6ad5bda7978)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 12 +-
.github/PULL_REQUEST_TEMPLATE.md | 1 -
.github/dependabot.yml | 3 +
README.md | 36 +
helm/pinot/pinot-github-events-setup.yml | 1 -
helm/pinot/pinot-realtime-quickstart.yml | 2 -
kb/code-review-principles.md | 23 +-
kb/plugin-modules.md | 1 -
kb/skills/review-naming-api.md | 4 +-
pinot-bom/pom.xml | 9 +-
.../HelixExternalViewBasedQueryQuotaManager.java | 25 +
.../pinot/broker/queryquota/QueryQuotaEntity.java | 10 +
.../pinot/broker/queryquota/QueryQuotaManager.java | 11 +
.../requesthandler/BaseBrokerRequestHandler.java | 33 +-
.../BaseSingleStageBrokerRequestHandler.java | 83 +-
.../MultiStageBrokerRequestHandler.java | 60 +-
.../AdaptiveServerSelector.java | 3 +-
.../PriorityPoolInstanceSelector.java | 182 +-
.../instanceselector/BalancedInstanceSelector.java | 91 +-
.../instanceselector/BaseInstanceSelector.java | 556 ++----
.../routing/instanceselector/CandidateGroup.java | 80 +
.../routing/instanceselector/GroupRouting.java | 64 +
.../routing/instanceselector/InstanceSelector.java | 49 +-
.../MultiStageReplicaGroupSelector.java | 305 +--
.../routing/instanceselector/NewSegmentState.java | 49 -
.../ReplicaGroupInstanceSelector.java | 330 ++--
.../routing/instanceselector/SegmentGroups.java | 75 +
.../instanceselector/SegmentGroupsBuilder.java | 546 +++++
.../routing/instanceselector/SegmentStates.java | 126 +-
.../instanceselector/SelectionCollector.java | 170 ++
.../StrictReplicaGroupInstanceSelector.java | 45 +-
.../instanceselector/TableReplicaHealth.java | 2 +-
.../routing/manager/BaseBrokerRoutingManager.java | 52 +-
...elixExternalViewBasedQueryQuotaManagerTest.java | 148 ++
.../BaseSingleStageBrokerRequestHandlerTest.java | 288 +++
.../BrokerQpsQuotaEnforcementModeTest.java | 219 +++
.../MultiStageBrokerRequestHandlerTest.java | 109 +-
.../PriorityPoolInstanceSelectorTest.java | 111 +-
.../CompactAssignmentInstanceSelectorTest.java | 5 +-
.../instanceselector/InstanceSelectorTest.java | 504 ++---
.../MultiStageReplicaGroupSelectorTest.java | 20 +-
.../MultiStageSelectionReferenceModelTest.java | 415 ++++
.../instanceselector/ReplicaGroupSelectorTest.java | 170 +-
.../SegmentStatesReferenceModelTest.java | 1180 +++++++++++
.../instanceselector/SegmentStatesTestUtils.java | 147 ++
.../SelectionReferenceModelTest.java | 598 ++++++
.../instanceselector/SelectionResultTestUtils.java | 69 +
.../routing/manager/BrokerRoutingManagerTest.java | 39 +-
.../pinot/common/metrics/ControllerMeter.java | 4 +-
.../apache/pinot/common/metrics/ServerMeter.java | 1 +
.../pinot/common/utils/TarCompressionUtils.java | 33 +-
.../common/utils/config/QueryOptionsUtils.java | 54 +
.../utils/helix/ImmutableSortedArrayMap.java | 10 +
.../pinot/common/utils/request/RequestUtils.java | 20 +
.../pinot/sql/parsers/dml/DeleteStatement.java | 2 +
.../parsers/rewriter/OrderByLiteralRemover.java | 45 +
.../sql/parsers/rewriter/QueryRewriterFactory.java | 3 +-
.../common/utils/TarCompressionUtilsTest.java | 21 +
.../common/utils/config/QueryOptionsUtilsTest.java | 53 +-
.../utils/config/TableConfigSerDeUtilsTest.java | 70 +-
.../utils/helix/ImmutableSortedArrayMapTest.java | 3 +
.../common/utils/request/RequestUtilsTest.java | 19 +
.../rewriter/OrderByLiteralRemoverTest.java | 75 +
.../parsers/rewriter/QueryRewriterFactoryTest.java | 55 +-
pinot-connectors/pinot-flink-connector/pom.xml | 29 +-
.../api/upload/SegmentValidationUtils.java | 7 +
.../helix/core/rebalance/TableRebalancer.java | 15 +-
.../helix/core/retention/RetentionManager.java | 21 +-
.../RealtimeSegmentValidationManager.java | 15 +-
.../controller/validation/StorageQuotaChecker.java | 28 +-
.../Homepage/Operations/AddIngestionComponent.tsx | 1 -
.../Operations/AddRealTimeIngestionComponent.tsx | 1 -
.../Homepage/Operations/AddRealtimeTableOp.tsx | 1 -
.../src/main/resources/package-lock.json | 6 +-
.../api/upload/SegmentValidationUtilsTest.java | 29 +
.../pinot/controller/helix/ControllerTest.java | 15 +
.../rebalance/TableRebalancerForceCommitTest.java | 352 ++++
.../helix/core/retention/RetentionManagerTest.java | 115 +-
.../RealtimeSegmentValidationManagerTest.java | 49 +
.../validation/StorageQuotaCheckerTest.java | 118 ++
.../core/data/manager/BaseTableDataManager.java | 18 +-
.../realtime/RealtimeSegmentDataManager.java | 41 +-
.../core/operator/InstanceResponseOperator.java | 20 +-
.../StreamingSelectionOrderByCombineOperator.java | 680 +++++++
.../operator/docvalsets/TransformBlockValSet.java | 29 +
.../SelectionPartiallyOrderedByDescOperation.java | 9 +-
.../SelectionPartiallyOrderedByLinearOperator.java | 14 +-
.../query/StreamingSelectionOrderByOperator.java | 621 ++++++
.../apache/pinot/core/plan/CombinePlanNode.java | 12 +
.../apache/pinot/core/plan/SelectionPlanNode.java | 102 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 213 +-
.../aggregation/function/AggregationFunction.java | 11 +-
.../function/AnyValueAggregationFunction.java | 5 +
...ValueIntegerTupleSketchAggregationFunction.java | 6 +
.../DistinctCountHLLPlusAggregationFunction.java | 2 +
...stinctCountRawCPCSketchAggregationFunction.java | 17 +
.../DistinctCountRawHLLAggregationFunction.java | 15 +-
...DistinctCountRawHLLPlusAggregationFunction.java | 15 +-
...inctCountRawThetaSketchAggregationFunction.java | 17 +
.../DistinctCountRawULLAggregationFunction.java | 15 +-
...stinctCountSmartHLLPlusAggregationFunction.java | 2 +
.../FrequentLongsSketchAggregationFunction.java | 10 +
.../FrequentStringsSketchAggregationFunction.java | 10 +
.../function/HistogramAggregationFunction.java | 9 +
.../function/IdSetAggregationFunction.java | 9 +
.../IntegerTupleSketchAggregationFunction.java | 11 +
.../PercentileRawEstAggregationFunction.java | 16 +-
.../PercentileRawKLLAggregationFunction.java | 14 +
.../PercentileRawKLLMVAggregationFunction.java | 14 +
.../PercentileRawTDigestAggregationFunction.java | 15 +-
.../function/StUnionAggregationFunction.java | 7 +
.../function/SumPrecisionAggregationFunction.java | 24 +-
.../function/array/BaseArrayAggFunction.java | 20 +
.../function/array/ListAggFunction.java | 15 +-
.../array/SumArrayDoubleAggregationFunction.java | 10 +
.../array/SumArrayLongAggregationFunction.java | 10 +
.../funnel/FunnelCountAggregationFunction.java | 5 +-
...FunnelStepDurationStatsAggregationFunction.java | 5 -
.../query/executor/ServerQueryExecutorV1Impl.java | 24 +-
.../pinot/core/query/prefetch/FetchPlanner.java | 10 +-
.../query/pruner/BloomFilterSegmentPruner.java | 5 +-
.../query/reduce/SelectionDataTableReducer.java | 2 +
.../core/query/request/context/QueryContext.java | 65 +
.../data/manager/BaseTableDataManagerTest.java | 73 +
.../realtime/RealtimeSegmentDataManagerTest.java | 181 +-
.../operator/combine/CombineSlowOperatorsTest.java | 86 +-
.../ConsumingSegmentBoundsStalenessTest.java | 213 ++
...reamingSelectionOrderByCombineOperatorTest.java | 2078 ++++++++++++++++++++
.../TransformBlockValSetTerminationTest.java | 115 ++
.../StreamingSelectionOrderByOperatorTest.java | 966 +++++++++
.../plan/maker/InstancePlanMakerImplV2Test.java | 67 +-
.../plan/maker/InstancePlanMakerPrefetchTest.java | 283 +++
.../function/AggregationFunctionFactoryTest.java | 24 +
.../function/AnyValueAggregationFunctionTest.java | 6 +
...tinctCountCPCSketchAggregationFunctionTest.java | 65 +-
.../DistinctCountULLAggregationFunctionTest.java | 15 +
.../aggregation/function/FinalResultMergeTest.java | 176 ++
.../IntegerTupleSketchAggregationFunctionTest.java | 65 +-
.../aggregation/function/ListAggFunctionTest.java | 34 +
.../function/StUnionAggregationFunctionTest.java | 14 +
.../SumPrecisionAggregationFunctionTest.java | 19 +
.../function/funnel/FunnelFinalResultTest.java | 16 +
.../core/query/executor/QueryExecutorTest.java | 44 +
.../query/pruner/BloomFilterSegmentPrunerTest.java | 50 +
.../apache/pinot/queries/ArrayAggQueriesTest.java | 27 +-
.../org/apache/pinot/queries/BaseQueriesTest.java | 2 +-
.../DistinctCountThetaSketchQueriesTest.java | 7 +
...nnerSegmentSelectionSingleValueQueriesTest.java | 176 ++
...erSegmentAggregationSingleValueQueriesTest.java | 16 +
.../TransformEvaluationDeadlineQueriesTest.java | 160 ++
pinot-distribution/pinot-assembly.xml | 4 +-
pinot-integration-test-base/pom.xml | 30 +-
.../tests/BaseClusterIntegrationTest.java | 2 +-
pinot-integration-tests/pom.xml | 30 +-
.../tests/BaseClusterIntegrationTestSet.java | 46 +-
.../tests/LLCRealtimeClusterIntegrationTest.java | 4 +-
.../tests/QueryKillingIntegrationTest.java | 8 +-
.../tests/QueryQuotaClusterIntegrationTest.java | 59 +
.../tests/RowLevelSecurityIntegrationTest.java | 32 +
.../SharedHybridClusterIntegrationTestSuite.java | 18 +-
.../tests/TableRebalanceIntegrationTest.java | 7 +-
.../tests/TenantRebalanceIntegrationTest.java | 72 +-
.../UpsertSnapshotRestartIntegrationTest.java | 7 +-
.../UpsertTableSegmentPreloadIntegrationTest.java | 2 +-
.../tests/suites/SharedHybridSuite.java | 4 +-
pinot-perf/pom.xml | 2 +-
.../pinot-input-format/pinot-json/README.md | 2 -
pinot-plugins/pinot-stream-ingestion/README.md | 45 +-
.../pinot-stream-ingestion/pinot-kafka-3.0/pom.xml | 119 --
.../stream/kafka30/KafkaConsumerFactory.java | 70 -
.../KafkaPartitionLevelConnectionHandler.java | 220 ---
.../kafka30/KafkaPartitionLevelConsumer.java | 225 ---
.../kafka30/KafkaStreamMetadataProvider.java | 490 -----
.../SynchronizedKafkaStreamMetadataProvider.java | 42 -
.../kafka30/KafkaAdminClientManagerTest.java | 130 --
.../KafkaPartitionLevelConnectionHandlerTest.java | 110 --
...tionLevelConsumerBackwardCompatibilityTest.java | 28 -
...titionLevelConsumerDataLossIntegrationTest.java | 242 ---
.../KafkaPartitionLevelConsumerSeekTest.java | 130 --
.../kafka30/KafkaPartitionLevelConsumerTest.java | 633 ------
.../KafkaPartitionLevelStreamConfigTest.java | 203 --
.../kafka30/KafkaStreamMetadataProviderTest.java | 454 -----
.../pinot-kafka-3.0/src/test/resources/log4j2.xml | 36 -
.../pinot-stream-ingestion/pinot-kafka-4.0/pom.xml | 46 +-
.../stream/kafka40}/server/KafkaDataProducer.java | 3 +-
.../kafka40}/server/KafkaServerStartable.java | 6 +-
...g.apache.pinot.spi.stream.StreamConsumerFactory | 2 +-
.../stream/kafka40/KafkaOAuthRuntimeTest.java | 65 +
...titionLevelConsumerDataLossIntegrationTest.java | 12 +-
.../kafka40/KafkaPartitionLevelConsumerTest.java | 45 +-
.../KafkaPartitionLevelStreamConfigTest.java | 33 +
.../kafka40}/server/EmbeddedKafkaCluster.java | 8 +-
.../kafka40}/server/KafkaServerStartableTest.java | 24 +-
.../stream/kafka40/utils/MiniKafkaCluster.java | 171 --
.../plugin/stream/kafka/KafkaConfigUtils.java | 99 +-
.../plugin/stream/kafka/KafkaConfigUtilsTest.java | 193 ++
pinot-plugins/pinot-stream-ingestion/pom.xml | 1 -
.../DistinctCountHLLPlusValueAggregator.java | 12 +-
.../dedup/BasePartitionDedupMetadataManager.java | 2 +-
.../indexsegment/immutable/EmptyIndexSegment.java | 8 +-
.../immutable/ImmutableSegmentImpl.java | 33 +-
.../immutable/ImmutableSegmentLoader.java | 34 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 18 +-
.../loader/DefaultSegmentDirectoryLoader.java | 4 +
.../loader/TierBasedSegmentDirectoryLoader.java | 4 +
.../converter/stats/CompactedColumnStatistics.java | 4 +-
.../stats/CompactedNoDictColumnStatistics.java | 6 +-
.../dictionary/StringOnHeapMutableDictionary.java | 5 +-
.../impl/forward/CLPMutableForwardIndex.java | 4 +-
.../impl/forward/SameValueMutableForwardIndex.java | 4 +-
.../realtime/impl/json/MutableJsonIndexImpl.java | 17 +-
.../SchemaConformingTransformer.java | 4 +-
.../creator/impl/SegmentDictionaryCreator.java | 2 +-
.../impl/openstruct/OpenStructColumnSplitter.java | 4 +-
.../stats/NoDictColumnStatisticsCollector.java | 4 +-
.../stats/StringColumnPreIndexStatsCollector.java | 6 +-
.../segment/index/loader/ForwardIndexHandler.java | 4 +-
.../segment/index/loader/IndexLoadingConfig.java | 22 +-
.../segment/index/loader/SegmentPreProcessor.java | 5 +-
.../defaultcolumn/DefaultColumnStatistics.java | 4 +-
.../readers/PinotSegmentColumnReaderImpl.java | 9 +-
.../segment/readers/PinotSegmentRecordReader.java | 11 +-
.../segment/store/SegmentLocalFSDirectory.java | 71 +-
.../PartitionIdVirtualColumnProvider.java | 17 +-
.../local/startree/StarTreeBuilderUtils.java | 4 +-
.../upsert/BasePartitionUpsertMetadataManager.java | 164 +-
.../segment/local/utils/HyperLogLogPlusUtils.java | 62 +-
.../segment/local/utils/SegmentPreloadUtils.java | 3 +-
.../segment/local/utils/TableConfigUtils.java | 18 +-
.../DistinctCountHLLPlusValueAggregatorTest.java | 55 +
.../local/aggregator/ValueAggregatorTest.java | 4 +
...apPartitionDedupMetadataManagerWithTTLTest.java | 2 +
.../local/dedup/DedupWithTimestampColumnTest.java | 1 +
.../immutable/EmptyIndexSegmentTest.java | 9 +-
.../immutable/ImmutableSegmentImplTest.java | 34 +-
...MutableSegmentImplIngestionAggregationTest.java | 47 +
.../stats/MutableNoDictColumnStatisticsTest.java | 6 +-
.../segment/index/SegmentMetadataImplTest.java | 610 ++++++
.../index/loader/IndexLoadingConfigTest.java | 26 +-
.../local/segment/index/loader/LoaderTest.java | 5 +-
.../defaultcolumn/DefaultColumnStatisticsTest.java | 6 +-
.../OpenStructPerKeyIndexReloadTest.java | 16 +
.../index/readers/ImmutableDictionaryTest.java | 4 +-
.../readers/PinotSegmentColumnReaderImplTest.java | 33 +
.../segment/store/SegmentLocalFSDirectoryTest.java | 25 +
.../v2/builder/MultipleTreesBuilderCloseTest.java | 4 +-
.../BasePartitionUpsertMetadataManagerTest.java | 232 ++-
...ertMetadataManagerForConsistentDeletesTest.java | 63 +-
...rrentMapPartitionUpsertMetadataManagerTest.java | 110 +-
.../local/upsert/UpsertSnapshotRestartTest.java | 6 +-
.../local/utils/HyperLogLogPlusUtilsTest.java | 89 +
.../local/utils/SegmentPreloadUtilsTest.java | 2 +-
.../segment/local/utils/TableConfigUtilsTest.java | 86 +
.../apache/pinot/segment/spi/ColumnMetadata.java | 28 +
.../org/apache/pinot/segment/spi/IndexSegment.java | 8 +
.../apache/pinot/segment/spi/SegmentMetadata.java | 1 +
.../org/apache/pinot/segment/spi/V1Constants.java | 13 +-
.../spi/index/metadata/ColumnMetadataImpl.java | 18 +
.../spi/index/metadata/MappedColumnMetadata.java | 405 ++++
.../index/metadata/MappedSegmentMetadataStore.java | 970 +++++++++
.../spi/index/metadata/SegmentMetadataImpl.java | 372 +++-
.../spi/loader/SegmentDirectoryLoaderContext.java | 17 +-
.../pinot/segment/spi/store/SegmentDirectory.java | 15 +
.../resources/SegmentCompressionStatsReader.java | 9 +-
.../pinot/server/api/resources/TablesResource.java | 45 +-
.../starter/helix/HelixInstanceDataManager.java | 30 +-
.../helix/HelixInstanceDataManagerConfig.java | 4 +
.../helix/ServerMailboxChannelRefreshHandler.java | 81 +-
.../pinot/server/api/TablesResourceTest.java | 52 +-
.../SegmentCompressionStatsReaderTest.java | 13 +
.../helix/HelixInstanceDataManagerConfigTest.java | 17 +
.../helix/HelixInstanceDataManagerTest.java | 68 +-
.../ServerMailboxChannelRefreshHandlerTest.java | 215 +-
.../apache/pinot/spi/config/table/QuotaConfig.java | 32 +-
.../spi/config/table/QuotaEnforcementMode.java | 42 +
.../SegmentsValidationAndRetentionConfig.java | 29 +
.../org/apache/pinot/spi/plugin/PluginManager.java | 72 +-
.../org/apache/pinot/spi/stream/StreamConfig.java | 2 +-
.../apache/pinot/spi/utils/CommonConstants.java | 69 +
.../java/org/apache/pinot/spi/utils/Utf8Utils.java | 78 +-
.../pinot/spi/config/table/QuotaConfigTest.java | 59 +
...nfigTest.java => QuotaEnforcementModeTest.java} | 24 +-
.../apache/pinot/spi/plugin/PluginManagerTest.java | 51 +-
.../org/apache/pinot/spi/utils/Utf8UtilsTest.java | 58 +-
pinot-tools/pom.xml | 2 +-
.../LaunchSparkDataIngestionJobCommand.java | 27 +-
.../src/main/resources/appAssemblerScriptTemplate | 3 +
.../src/main/resources/conf/pinot-server.conf | 4 +
.../LaunchSparkDataIngestionJobCommandTest.java | 16 +
pom.xml | 85 +-
290 files changed, 18970 insertions(+), 5805 deletions(-)
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/CandidateGroup.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/GroupRouting.java
delete mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/NewSegmentState.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/SegmentGroups.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/SegmentGroupsBuilder.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/SelectionCollector.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerQpsQuotaEnforcementModeTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/MultiStageSelectionReferenceModelTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SegmentStatesReferenceModelTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SegmentStatesTestUtils.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SelectionReferenceModelTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SelectionResultTestUtils.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/OrderByLiteralRemover.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/OrderByLiteralRemoverTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerForceCommitTest.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/combine/StreamingSelectionOrderByCombineOperator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/query/StreamingSelectionOrderByOperator.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/combine/ConsumingSegmentBoundsStalenessTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/combine/StreamingSelectionOrderByCombineOperatorTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/docvalsets/TransformBlockValSetTerminationTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/query/StreamingSelectionOrderByOperatorTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/plan/maker/InstancePlanMakerPrefetchTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/FinalResultMergeTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/TransformEvaluationDeadlineQueriesTest.java
delete mode 100644 pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/pom.xml
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaConsumerFactory.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConnectionHandler.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumer.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/KafkaStreamMetadataProvider.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30/SynchronizedKafkaStreamMetadataProvider.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaAdminClientManagerTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConnectionHandlerTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerBackwardCompatibilityTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerDataLossIntegrationTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerSeekTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelStreamConfigTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaStreamMetadataProviderTest.java
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/resources/log4j2.xml
rename
pinot-plugins/pinot-stream-ingestion/{pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30
=>
pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40}/server/KafkaDataProducer.java
(96%)
rename
pinot-plugins/pinot-stream-ingestion/{pinot-kafka-3.0/src/main/java/org/apache/pinot/plugin/stream/kafka30
=>
pinot-kafka-4.0/src/main/java/org/apache/pinot/plugin/stream/kafka40}/server/KafkaServerStartable.java
(99%)
rename pinot-plugins/pinot-stream-ingestion/{pinot-kafka-3.0 =>
pinot-kafka-4.0}/src/main/resources/META-INF/services/org.apache.pinot.spi.stream.StreamConsumerFactory
(92%)
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40/KafkaOAuthRuntimeTest.java
rename
pinot-plugins/pinot-stream-ingestion/{pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30
=>
pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40}/server/EmbeddedKafkaCluster.java
(97%)
rename
pinot-plugins/pinot-stream-ingestion/{pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30
=>
pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40}/server/KafkaServerStartableTest.java
(80%)
delete mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40/utils/MiniKafkaCluster.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/aggregator/DistinctCountHLLPlusValueAggregatorTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/HyperLogLogPlusUtilsTest.java
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/MappedColumnMetadata.java
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/MappedSegmentMetadataStore.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/QuotaEnforcementMode.java
copy
pinot-spi/src/test/java/org/apache/pinot/spi/config/table/{StarTreeIndexConfigTest.java
=> QuotaEnforcementModeTest.java} (55%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
