This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch
data-2616-disallow-nondeterministic-transforms
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard dfe8f01bacf Disallow non-deterministic ingestion transforms
add 011ca32aabe Bump org.jline:jline from 4.1.3 to 4.2.1 (#18788)
add 8702c2b8123 Bump software.amazon.awssdk:bom from 2.46.11 to 2.46.12
(#18787)
add b68ebccc36e Fix default HTTPS protocol to use HttpsSegmentFetcher
(#17675)
add 1c2a4fa13e6 Remove batch message mode and never allow enabling it
(#18795)
add d65b7337b10 [multistage] Fix count to return 0 when intermediate
result is null (#18471)
add 5bf9e68b40e Refresh server table config cache on update (#18796)
add 4e6445cc910 Add an overridable table-config-change callback distinct
from index-loading-config fetch (#18792)
add d469cb17507 Support forcing pre-partitioned exchanges for window
functions to avoid data shuffle using query hint (#17395)
add ff3c92fab5f OPEN_STRUCT — SPI utilities, index registration, stats
constructors (PR 2a/4) (#18710)
add a0d00297153 Allow dynamic toggling of adaptive routing metrics export
(#18286)
add a0df7fa8bc3 Fix typo: occurence -> occurrence in
RegexpReplaceVarFunctions and BenchmarkRegexpReplace (#18803)
add f44bdcd8a39 Fix typo: rename DEFUALT_PUSH_ATTEMPTS to
DEFAULT_PUSH_ATTEMPTS (#18801)
add ceb2ad72c4b Bump org.codehaus.jettison:jettison from 1.5.5 to 1.5.6
(#18799)
add f53f848f9cd Bump software.amazon.awssdk:bom from 2.46.12 to 2.46.13
(#18798)
add 4b87ef4cbe9 Add broker extension seams for query-thread estimation and
routing-metadata listeners (#18800)
add f76e05afc10 Fix typos in comments, Javadoc, and log messages (#18802)
add dd28703d459 Replace segmentDirectoryConfigs map with ReadMode in
SegmentDirectoryLoaderContext (#18806)
add 7265af3574a Fix controller UI crash on instance listing pages when
instance names lack type prefix (#18805)
add 431d541a03f Support firstWithTime/lastWithTime aggregation types for
MergeRollupTask (#18724)
add 5526d5b8601 Add OVERLAY scalar function for SQL-standard substring
replacement (#18790)
add 90ad923cd66 Bump software.amazon.awssdk:bom from 2.46.13 to 2.46.14
(#18811)
add ea0b761f4ca Bump flink.version from 2.2.1 to 2.3.0 (#18810)
add 648d285840d Bump actions/checkout from 6 to 7 (#18809)
add 7de143f4a9d Make dateTimeConvert scalar function robust to typed
time-value input (#18814)
add 0bceb35a8de Support GROUP BY GROUPING SETS / ROLLUP / CUBE in the
single-stage engine (#18662)
add 413edae9935 drop non-reproducible entries (#18812)
add ff7df4893c8 Add server ingestion OOM protection (#18784)
add dd4fcf56140 Rename PinotDataType.INTEGER to INT for naming consistency
(#18815)
add 3f58dba6317 Add optional source-field data type fix during ingestion
(#18816)
add 8f5af9ed27e Fix potential NPE in PinotHelixResourceManager tenant
deletability checks (#18777)
add 4660ed5700e Avoid materializing the full bitmap for count aggregation
with filters for SV columns (#18543)
add 14bc147d88d Add metrics to track server startup time spending on
different status callback (#18493)
add 54f29a9d0f1 Fix MSE handle search null handling issue by adding an or
clause with a null check when required (#18729)
add 1b9649950a1 Bump software.amazon.awssdk:bom from 2.46.14 to 2.46.15
(#18827)
add 179e9582671 Bump io.opentelemetry.contrib:opentelemetry-gcp-resources
(#18826)
add 5696fd1793e Bump com.gradle:develocity-maven-extension from 2.4.1 to
2.4.2 (#18825)
add 3397a4a041f Bump org.javassist:javassist from 3.31.0-GA to 3.32.0-GA
(#18824)
add 52b8fdcb5d8 Add isPrivateIp scalar function for IP address
classification (#18828)
add e44e35d657b Support AVG aggregation in MergeRollupTask and
RealtimeToOfflineSegmentsTask (#18822)
add b6e9f0388cc Finalize aggregate intermediate results in
AggregationResultsBlock.getRows() for server-return-final (#18835)
add 09309dd0769 [Bug fix] Skip flushing empty chunk in
OffHeapH3IndexCreator (#18834)
add 63bf8dcc8cf Centralize batch ingestion job spec constants into
SegmentGenerationJobUtils (#18413)
add 302783f42ca Fix ProtoBufRecordExtractor cache invalidation for schema
evolution (#18840)
add 2bdbd7fe094 Support a per-execution PlanMaker override in
QueryExecutor.execute (#18847)
add 225a550ff4d Bump org.eclipse.sisu:sisu-maven-plugin from 1.0.0 to
1.0.1 (#18846)
add 0ae60ca0752 Bump io.grpc:grpc-bom from 1.82.0 to 1.82.1 (#18845)
add b459a257faa Bump org.webjars:swagger-ui from 5.32.6 to 5.32.8 (#18844)
add b59a370ed22 Bump actions/cache from 5 to 6 (#18843)
add e72b0747cdf Bump software.amazon.awssdk:bom from 2.46.15 to 2.46.17
(#18838)
add ca608fa3906 Fix NPE in GapfillProcessor when TIMESERIESON column is
not in SELECT list (#18820)
add 23fabe268f7 Replace Collections singleton factory usages (#18837)
add e8bb63fa6ad Fix MutableJsonIndexImpl corrupting doc ID mapping on
unpaired UTF-16 surrogates (#18738)
add 3d148bafe29 Expose segment-level accessors on SegmentDataManager for
server observability endpoints (#18821)
add 25e378b5b07 Prune unused passthrough columns from UNNEST output
(opt-in) (#18782)
add dc4e9577177 add validateEnvironmentVariables (#18851)
add a9b5207a632 Add optional fieldId and aliases to FieldSpec (#18833)
add 235bd8e2d2c Update prom rule to emit table label for minion subtask
metrics (#18854)
add 3be62366ef5 Add TABLE_TENANT_INFO controller gauge for table-to-tenant
mapping (#18823)
add bf58476d375 Mark numMinionSubtasks* gauges as non-global to match PR
#18854 Prometheus rule (#18864)
add 9aadb762cba Bump org.apache.httpcomponents.core5:httpcore5 from 5.4.2
to 5.4.3 (#18849)
add bec3dcc36f1 Forward SegmentDirectoryLoaderContext in
DefaultSegmentDirectoryLoader (#18863)
add cd6dd61ce32 Rebuild upsert metadata from validDocIds snapshot on
reload for TTL tables (#18860)
add fd022b34e5f Bump com.github.jnr:jnr-constants from 0.10.4 to 0.11.0
(#18877)
add 9d04a166882 Bump com.github.jnr:jnr-posix from 3.1.22 to 3.2.1 (#18878)
add 0646ef3df7e Bump org.apache:apache from 38 to 39 (#18879)
add e6bd15e7d88 [Minor] Make Upsert/Dedup TTL_WATERMARK_NOT_SET public
accessible (#18865)
add 8cc41925e0f Bump com.github.jnr:jnr-ffi from 2.2.19 to 2.3.0 (#18880)
add 9c02213b99e Do not override user-configured validDocIdsType for upsert
compaction (#18850)
add 065d50c3f77 Extend FUNNEL_COUNT to support multiple CORRELATE_BY
columns (#18760)
add 2103aa27006 Preserve millisecond precision for TIMESTAMP comparisons
in the multi-stage engine (#18883)
add 85e10a02f42 Fix AIOOB in Lucene text-index reuse path on retried
segment conversion (#18885)
add b3124b6e9ee Expose the query executor init config via
QueryExecutor#getQueryExecutorConfig (#18886)
add 5ff23090e21 Bump com.gradle:develocity-maven-extension from 2.4.2 to
2.5.0 (#18887)
add d503fee570c Bump software.amazon.awssdk:bom from 2.46.17 to 2.46.18
(#18888)
add b97acc46c0f Bump com.gradle:common-custom-user-data-maven-extension
(#18889)
add 3718bc3b9e4 Bump org.apache.httpcomponents.client5:httpclient5 from
5.6.1 to 5.6.2 (#18890)
add f0329f33762 Enforce validDocIds consensus in upsert task generators
and add includeBitmaps to validDocIdsMetadata API (#18853)
add a796afce50f Bump org.apache.kerby:kerby-bom from 2.1.1 to 2.1.2
(#18862)
add dafe7ad3e48 Bump org.jline:jline from 4.2.1 to 4.3.1 (#18899)
add ec55ce8a087 Bump echarts from 6.0.0 to 6.1.0 in
/pinot-controller/src/main/resources (#18894)
add 52d04b440a5 Bump software.amazon.awssdk:bom from 2.46.18 to 2.46.19
(#18900)
add 91be61f0f4f Make the server query meter table level (#18882)
add 627b8d9e806 Fix column-to-column comparison in WHERE clause (#18587)
add ea74bb9037e Remove unused sortable flag from FieldSpec.DataType
(#18903)
add 7ee1b425328 Add config to set consumption rate limit at partition
level (#18904)
add 65a7bdd666d Fix Thrift checksum URL in base Docker images (#18893)
add 545210b8e0e Share a single pooled ByteBuf allocator across Netty query
transports instead of one per server channel (#18905)
add 6d319a5715a Avoid cloning the first bitmap when intersecting
bitmap-based doc id sets in AndDocIdSet (#18912)
add 2e7cbbb8c1a Bump org.checkerframework:checker-qual from 4.2.0 to 4.2.1
(#18907)
add 5041f74d80e Bump docker/login-action from 4.2.0 to 4.3.0 (#18908)
add e92bff56467 Bump docker/setup-qemu-action from 4.1.0 to 4.2.0 (#18909)
add d0540d29d22 Bump docker/setup-buildx-action from 4.1.0 to 4.2.0
(#18910)
add 5732375ef16 Bump software.amazon.awssdk:bom from 2.46.19 to 2.46.20
(#18911)
add d0ead2f626c Skip local build on CRC mismatch during CONSUMING->ONLINE
transition (#18895)
add bcf3f660a14 Add getUnavailableSegments broker hook for route-level
unavailable filtering (#18906)
add 19897c591be [Arrow] Typed zero-boxing column-major build for
single-value primitive columns (#18797)
add f9647dabd44 Consolidate IVF and HNSW vector indexes into segment
columns.psf (opt-in via storeInSegmentFile) (#18852)
add 92a0d98d8fd [UUID 1/8] Add logical UUID type foundation (pinot-spi)
(#18869)
add 5a451fa1b26 Bump software.amazon.awssdk:bom from 2.46.20 to 2.46.21
(#18917)
add 8bfeeaa2083 Bump log4j.version from 2.26.0 to 2.26.1 (#18916)
add a702e559103 Bump docker/login-action from 4.3.0 to 4.4.0 (#18923)
add 9d94f195e85 Bump at.yawk.lz4:lz4-java from 1.11.0 to 1.11.1 (#18922)
add 9ceca0f18e9 Bump org.apache.pulsar:pulsar-bom from 4.0.11 to 4.0.12
(#18921)
add bd488ae28d5 Fix GROUP BY DISTINCTCOUNT ClassCastException in
mergeDataTablesOnly (#18842)
add d424eecb91b Rework UUID type conversion to use java.util.UUID as the
external form (#18927)
add 2b8007eb634 Mitigate stale realtime segmentStoppedConsuming
notification (#18914)
add 1f1521b8389 Add peer download fallback to PredownloadScheduler (#18641)
add 91416079849 Close op chain when scheduling fails to release operator
resources (#18928)
add bb83fa618c2 Initialize ServerMetrics in PredownloadScheduler (#18608)
add be8d3b221ce Bump software.amazon.awssdk:bom from 2.46.21 to 2.47.0
(#18929)
add e5284a736be Simplify ColumnReader SPI: drop iterator API, replace type
checks with getValueType() (#18918)
add 26aa88a2dea Fix SchemaUtils to include validation failure reason in
error message (#18896)
add 50dcbffb4df Refactor splitPart(limit) to avoid full array allocation
(#18892)
add b6c3ca686af Fix vector index load failure on non-local segment
directories (storeInSegmentFile=true) (#18930)
add 2fc75fcf5a3 Bump software.amazon.awssdk:bom from 2.47.0 to 2.47.1
(#18938)
add 23aa0888f25 Bump com.google.cloud:libraries-bom from 26.84.0 to
26.85.0 (#18937)
add 93dcfc24e5f Bump com.fasterxml.jackson:jackson-bom from 2.22.0 to
2.22.1 (#18936)
add 637b6b3babd Add broker pruning support to partitioned leaf path and
logical tables in MSE (#18924)
add 46ff7489c59 Remove deprecated Kafka high-level-consumer (HLC) code
(#18931)
add 08c64f3d722 Fix multi-stage physical optimizer failing to query empty
tables (#18925)
add 82dd1d156eb Disallow non-deterministic ingestion transforms
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (dfe8f01bacf)
\
N -- N -- N
refs/heads/data-2616-disallow-nondeterministic-transforms (82dd1d156eb)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/copilot-instructions.md | 7 +
.github/workflows/build-all-pinot-docker-image.yml | 14 +-
.../workflows/build-pinot-base-docker-image.yml | 8 +-
.github/workflows/build-pinot-docker-image.yml | 8 +-
.github/workflows/build-superset-docker-image.yml | 8 +-
.../nightly-build-all-pinot-docker-image.yml | 14 +-
...htly-build-multi-arch-superset-docker-image.yml | 10 +-
.github/workflows/pinot_compatibility_checks.yml | 14 +-
.github/workflows/pinot_compatibility_tests.yml | 2 +-
.github/workflows/pinot_integration_tests.yml | 4 +-
...ulti_stage_query_engine_compatibility_tests.yml | 2 +-
.github/workflows/pinot_quickstart_tests.yml | 4 +-
.github/workflows/pinot_unit_tests.yml | 6 +-
.github/workflows/pinot_vuln_check.yml | 2 +-
.mvn/extensions.xml | 4 +-
AGENTS.md | 7 +
CLAUDE.md | 7 +
.../config/feature-test-2-realtime.json | 1 -
.../config/feature-test-2-realtime.json | 1 -
.../config/feature-test-3-realtime.json | 1 -
config/checkstyle.xml | 14 +
.../pinot-base-build/amazoncorretto.dockerfile | 9 +-
.../pinot-base-build/ms-openjdk.dockerfile | 9 +-
.../configs/controller.yml | 25 +-
.../airlineStats_realtime_table_config.json | 2 -
helm/pinot/pinot-github-events-setup.yml | 1 -
helm/pinot/pinot-realtime-quickstart.yml | 4 -
kb/code-review-principles.md | 18 +-
kb/skills/review-naming-api.md | 4 +
.../broker/api/resources/InstanceResource.java | 3 +-
.../broker/ZkBasicAuthAccessControlFactory.java | 3 +-
.../broker/broker/helix/BaseBrokerStarter.java | 4 +-
.../helix/MultiClusterHelixBrokerStarter.java | 7 +-
.../HelixExternalViewBasedQueryQuotaManager.java | 13 +-
.../BaseSingleStageBrokerRequestHandler.java | 16 +-
.../MultiStageBrokerRequestHandler.java | 15 +-
.../requesthandler/MultiStageQueryThrottler.java | 3 +-
.../requesthandler/TimeSeriesRequestHandler.java | 3 +-
.../PriorityPoolInstanceSelector.java | 3 +-
.../ServerSelectionContext.java | 3 +-
.../instanceselector/BaseInstanceSelector.java | 7 +-
.../ReplicaGroupInstanceSelector.java | 5 +-
.../routing/manager/BaseBrokerRoutingManager.java | 39 +-
.../SegmentPartitionMetadataManager.java | 5 +-
.../segmentpartition/SegmentPartitionUtils.java | 5 +-
.../routing/segmentpruner/TimeSegmentPruner.java | 5 +-
.../tablesampler/FirstNSegmentsTableSampler.java | 2 +-
.../broker/api/resources/PinotBrokerDebugTest.java | 6 +-
.../BasicAuthAccessControlFactoryGrpcTest.java | 7 +-
.../BrokerManagedAsyncExecutorProviderTest.java | 4 +-
.../broker/helix/ClusterChangeMediatorTest.java | 7 +-
.../MultiStageQueryThrottlerTest.java | 23 +-
.../AdaptiveServerSelectorTest.java | 3 +-
.../PriorityPoolInstanceSelectorTest.java | 9 +-
.../instanceselector/InstanceSelectorTest.java | 30 +-
.../MultiStageReplicaGroupSelectorTest.java | 3 +-
.../routing/intervalst/IntervalTreeTest.java | 4 +-
.../routing/manager/BrokerRoutingManagerTest.java | 14 +-
.../manager/MultiClusterRoutingManagerTest.java | 5 +-
.../SegmentZkMetadataFetcherTest.java | 10 +-
.../SegmentPartitionMetadataManagerTest.java | 78 +--
.../segmentpreselector/SegmentPreSelectorTest.java | 3 +-
.../timeboundary/TimeBoundaryManagerTest.java | 5 +-
.../pinot/client/DefaultSslContextProvider.java | 3 +-
.../java/org/apache/pinot/client/TlsProtocols.java | 3 +-
.../pinot/client/admin/BaseServiceAdminClient.java | 3 +-
.../pinot/client/admin/ClusterAdminClient.java | 3 +-
.../pinot/client/admin/PinotAdminTransport.java | 5 +-
.../pinot/client/admin/SegmentAdminClient.java | 13 +-
.../apache/pinot/client/grpc/GrpcConnection.java | 3 +-
.../pinot/client/utils/BrokerSelectorUtils.java | 3 +-
.../pinot/client/DynamicBrokerSelectorTest.java | 15 +-
.../apache/pinot/client/PreparedStatementTest.java | 4 +-
.../apache/pinot/client/ResultSetGroupTest.java | 6 +-
.../pinot/client/grpc/GrpcConnectionTest.java | 7 +-
.../org/apache/pinot/client/utils/DriverUtils.java | 3 +-
.../apache/pinot/client/PinotResultSetTest.java | 4 +-
pinot-common/src/main/codegen/templates/Parser.jj | 5 +-
.../pinot/common/audit/AuditRequestProcessor.java | 3 +-
.../pinot/common/auth/AuthProviderUtils.java | 7 +-
.../apache/pinot/common/auth/NullAuthProvider.java | 3 +-
.../pinot/common/auth/StaticTokenAuthProvider.java | 5 +-
.../apache/pinot/common/auth/UrlAuthProvider.java | 3 +-
.../common/config/provider/StaticTableCache.java | 2 +-
.../pinot/common/config/provider/TableCache.java | 3 +-
.../pinot/common/datablock/BaseDataBlock.java | 5 +-
.../pinot/common/datablock/MetadataBlock.java | 5 +-
.../common/evaluator/GroovyFunctionEvaluator.java | 3 +-
.../evaluator/TimeSpecFunctionEvaluator.java | 3 +-
.../failuredetector/NoOpFailureDetector.java | 3 +-
.../pinot/common/function/FunctionUtils.java | 4 +-
.../scalar/DataTypeConversionFunctions.java | 8 +-
.../common/function/scalar/DateTimeConvert.java | 26 +-
.../common/function/scalar/IpAddressFunctions.java | 52 ++
.../common/function/scalar/JsonFunctions.java | 2 +-
.../common/function/scalar/StringFunctions.java | 244 ++++++-
.../scalar/regexp/RegexpReplaceVarFunctions.java | 24 +-
.../pinot/common/metadata/ZKMetadataProvider.java | 30 +-
.../metadata/instance/InstanceZKMetadata.java | 158 -----
.../pinot/common/metrics/AbstractMetrics.java | 3 +-
.../apache/pinot/common/metrics/BrokerMetrics.java | 4 +-
.../pinot/common/metrics/ControllerGauge.java | 11 +
.../apache/pinot/common/metrics/MseMetrics.java | 4 +-
.../apache/pinot/common/metrics/ServerGauge.java | 9 +
.../apache/pinot/common/metrics/ServerMeter.java | 4 +
.../apache/pinot/common/metrics/ServerMetrics.java | 4 +-
.../apache/pinot/common/request/PinotQuery.java | 290 ++++++--
.../common/request/context/FilterContext.java | 3 +-
.../pinot/common/request/context/GroupingSets.java | 62 ++
.../common/request/context/LiteralContext.java | 4 +-
.../request/context/RequestContextUtils.java | 3 +-
.../response/encoder/ArrowResponseEncoder.java | 13 +-
.../resources/EndReplaceSegmentsRequest.java | 3 +-
.../resources/StartReplaceSegmentsRequest.java | 5 +-
.../resources/ValidDocIdsBitmapResponse.java | 22 +-
.../restlet/resources/ValidDocIdsMetadataInfo.java | 19 +-
.../common/utils/ClientSSLContextGenerator.java | 3 +-
.../org/apache/pinot/common/utils/DataSchema.java | 2 +-
.../common/utils/FileUploadDownloadClient.java | 9 +-
.../apache/pinot/common/utils/ServiceStatus.java | 10 +-
.../common/utils/config/QueryOptionsUtils.java | 3 +-
.../pinot/common/utils/config/TierConfigUtils.java | 3 +-
.../common/utils/fetcher/BaseSegmentFetcher.java | 39 ++
.../common/utils/fetcher/HttpSegmentFetcher.java | 16 +
.../common/utils/fetcher/HttpsSegmentFetcher.java | 6 +
.../pinot/common/utils/fetcher/SegmentFetcher.java | 6 +
.../utils/fetcher/SegmentFetcherFactory.java | 21 +-
.../common/utils/grpc/BaseGrpcQueryClient.java | 3 +-
.../common/utils/grpc/ServerGrpcQueryClient.java | 4 +-
.../pinot/common/utils/helix/HelixHelper.java | 5 +-
.../common/utils/helix/IdealStateGroupCommit.java | 7 +-
.../common/utils/helix/IdealStateSingleCommit.java | 7 +-
.../apache/pinot/common/utils/http/HttpClient.java | 3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 203 +++++-
.../parser/SqlPinotDropMaterializedView.java | 3 +-
.../sql/parsers/parser/SqlPinotRefreshClause.java | 3 +-
.../parser/SqlPinotShowCreateMaterializedView.java | 3 +-
.../parser/SqlPinotShowMaterializedViews.java | 3 +-
.../sql/parsers/parser/SqlPinotShowTables.java | 3 +-
...nAggregationGroupByToDistinctQueryRewriter.java | 6 +
.../rewriter/PredicateComparisonRewriter.java | 35 +-
pinot-common/src/main/proto/plan.proto | 5 +
.../datablock/ZeroCopyDataBlockSerdeTest.java | 6 +-
.../evaluator/InbuiltFunctionEvaluatorTest.java | 12 +-
.../ConnectionFailureDetectorTest.java | 21 +-
.../pinot/common/function/FunctionUtilsTest.java | 8 +-
.../scalar/DataTypeConversionFunctionsTest.java | 9 +
.../function/scalar/IpAddressFunctionsTest.java | 78 +++
.../function/scalar/StringFunctionsTest.java | 71 ++
.../common/metadata/InstanceZKMetadataTest.java | 71 --
.../metadata/MetadataEqualsHashCodeTest.java | 3 -
.../ControllerPrometheusMetricsTest.java | 17 +-
.../mapper/TimeSeriesResponseMapperTest.java | 3 +-
.../pinot/common/tier/TierConfigUtilsTest.java | 8 +-
.../utils/DefaultLogicalTableConfigSerDeTest.java | 5 +-
.../pinot/common/utils/EqualityUtilsTest.java | 7 +-
.../common/utils/FileUploadDownloadClientTest.java | 3 +-
.../common/utils/RoundRobinURIProviderTest.java | 41 +-
.../pinot/common/utils/ServiceStatusTest.java | 3 +-
.../apache/pinot/common/utils/URIUtilsTest.java | 7 +-
.../common/utils/config/InstanceUtilsTest.java | 15 +-
.../utils/config/QueryWorkloadConfigUtilsTest.java | 13 +-
.../utils/config/TableConfigSerDeUtilsTest.java | 22 +-
.../utils/fetcher/HttpSegmentFetcherTest.java | 52 ++
.../utils/fetcher/SegmentFetcherFactoryTest.java | 18 +-
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 26 +-
.../pinot/sql/parsers/GroupingSetsParserTest.java | 178 +++++
.../rewriter/PredicateComparisonRewriterTest.java | 116 ++--
pinot-common/src/thrift/query.thrift | 6 +
.../java/org/apache/pinot/compat/SegmentOp.java | 3 +-
.../java/org/apache/pinot/compat/StreamOp.java | 3 +-
.../pinot/connector/flink/sink/PinotSink.java | 9 +-
.../flink/sink/PinotSinkIntegrationTest.java | 3 +-
.../sink/PinotSinkUpsertTableIntegrationTest.java | 3 +-
.../pinot/controller/BaseControllerStarter.java | 10 +-
.../apache/pinot/controller/ControllerConf.java | 6 -
.../PinotApplicationQuotaRestletResource.java | 6 +-
.../api/resources/PinotClusterConfigs.java | 3 +-
.../api/resources/PinotControllerHealthCheck.java | 3 +-
.../resources/PinotDatabaseRestletResource.java | 3 +-
.../api/resources/PinotDdlRestletResource.java | 15 +-
.../PinotInstanceAssignmentRestletResource.java | 8 +-
.../PinotLeadControllerRestletResource.java | 3 +-
.../api/resources/PinotSchemaRestletResource.java | 2 +-
.../api/resources/PinotSegmentRestletResource.java | 7 +-
.../api/resources/PinotTableInstances.java | 3 +-
.../api/resources/PinotTableRestletResource.java | 3 +-
.../api/resources/PinotTaskRestletResource.java | 7 +-
.../api/resources/PinotTenantRestletResource.java | 3 +-
.../api/resources/TableConfigValidationUtils.java | 29 +
.../api/resources/TableConfigsRestletResource.java | 2 +-
.../controller/helix/SegmentStatusChecker.java | 66 ++
.../helix/core/PinotHelixResourceManager.java | 56 +-
.../helix/core/PinotTableIdealStateBuilder.java | 4 +-
.../helix/core/minion/ClusterInfoAccessor.java | 3 +-
.../helix/core/minion/CronJobScheduleJob.java | 6 +-
.../core/minion/PinotHelixTaskResourceManager.java | 15 +-
.../helix/core/minion/TaskMetricsEmitter.java | 3 +-
.../realtime/MissingConsumingSegmentFinder.java | 3 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 30 +-
.../helix/core/rebalance/RebalanceChecker.java | 5 +-
.../helix/core/rebalance/TableRebalancer.java | 2 +-
.../helix/core/relocation/SegmentRelocator.java | 13 +-
.../helix/core/util/HelixSetupUtils.java | 38 +-
.../rules/impl/AggregateMetricsRule.java | 3 +-
.../recommender/rules/utils/FixedLenBitset.java | 3 +-
.../utils/QueryInvertedSortedIndexRecommender.java | 23 +-
.../services/PinotTableReloadStatusReporter.java | 2 +-
.../util/ConsumingSegmentInfoReader.java | 3 +-
.../pinot/controller/util/FileIngestionHelper.java | 4 +-
.../util/ServerSegmentMetadataReader.java | 37 +-
.../pinot/controller/util/TableMetadataReader.java | 3 +-
.../pinot/controller/util/TableSizeReader.java | 3 +-
.../pinot/controller/util/TableTierReader.java | 3 +-
.../RealtimeSegmentValidationManager.java | 2 +-
.../workload/scheme/PropagationUtils.java | 12 +-
.../workload/scheme/TablePropagationScheme.java | 11 +-
.../Homepage/Operations/SchemaComponent.tsx | 4 +-
.../main/resources/app/utils/PinotMethodUtils.ts | 4 +-
.../src/main/resources/package-lock.json | 16 +-
pinot-controller/src/main/resources/package.json | 2 +-
.../ControllerStarterDynamicEnvTest.java | 8 +-
.../controller/ControllerStarterStatelessTest.java | 6 +-
.../ConsumingSegmentInfoReaderStatelessTest.java | 41 +-
.../api/PinotDdlRestletResourceTest.java | 3 +-
...anceAssignmentRestletResourceStatelessTest.java | 38 +-
.../api/PinotInstanceRestletResourceTest.java | 11 +-
.../api/PinotMinionRestletResourceTest.java | 5 +-
.../api/PinotSchemaRestletResourceTest.java | 16 +-
.../api/PinotSegmentRestletResourceTest.java | 9 +-
.../controller/api/PinotSegmentsMetadataTest.java | 3 +-
.../api/PinotTenantRestletResourceTest.java | 6 +-
.../pinot/controller/api/TableTierReaderTest.java | 41 +-
...DdlRestletResourceMaterializedViewUnitTest.java | 8 +-
.../resources/PinotDdlRestletResourceUnitTest.java | 26 +-
.../resources/PinotRealtimeTableResourceTest.java | 4 +-
.../resources/PinotTaskRestletResourceTest.java | 5 +-
.../resources/TableConfigValidationUtilsTest.java | 76 ++
.../pinot/controller/helix/ControllerTest.java | 20 +-
.../controller/helix/SegmentStatusCheckerTest.java | 178 +++++
.../PinotHelixResourceManagerAssignmentTest.java | 11 +-
...otHelixResourceManagerConfigValidationTest.java | 11 +-
...esourceManagerMaterializedViewBackfillTest.java | 21 +-
...ResourceManagerMaterializedViewListingTest.java | 4 +-
.../PinotHelixResourceManagerMinionDrainTest.java | 19 +-
.../PinotHelixResourceManagerMinionStatusTest.java | 46 +-
.../PinotHelixResourceManagerStatelessTest.java | 104 ++-
.../controller/helix/core/SegmentResetTest.java | 4 +-
.../instance/InstanceAssignmentTest.java | 170 ++---
...NonReplicaGroupTieredSegmentAssignmentTest.java | 3 +-
.../RealtimeReplicaGroupSegmentAssignmentTest.java | 11 +-
.../segment/SegmentAssignmentUtilsTest.java | 3 +-
.../StrictRealtimeSegmentAssignmentTest.java | 7 +-
.../BalancedNumSegmentAssignmentStrategyTest.java | 3 +-
.../ReplicaGroupSegmentAssignmentStrategyTest.java | 22 +-
...nReplicaGroupSegmentAssignmentStrategyTest.java | 14 +-
.../RoundRobinSegmentAssignmentStrategyTest.java | 7 +-
.../cleanup/StaleInstancesCleanupTaskTest.java | 17 +-
.../core/lineage/DefaultLineageManagerTest.java | 9 +-
.../core/lineage/LineageDeleteExclusionTest.java | 27 +-
.../LineageDeleteInterleavingIntegrationTest.java | 103 ++-
.../minion/PinotHelixTaskResourceManagerTest.java | 97 ++-
...ManagerConcurrentSchedulingIntegrationTest.java | 3 +-
.../PinotTaskManagerConcurrentSchedulingTest.java | 7 +-
.../PinotTaskManagerDistributedLockingTest.java | 3 +-
.../core/minion/PinotTaskManagerOnChangeTest.java | 7 +-
.../core/minion/PinotTaskManagerStatelessTest.java | 11 +-
.../minion/generator/TaskGeneratorUtilsTest.java | 9 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 91 ++-
.../realtime/SegmentCompletionFSMFactoryTest.java | 3 +-
.../core/rebalance/DataLossRiskAssessorTest.java | 8 +-
.../TableRebalancerClusterStatelessTest.java | 7 +-
.../helix/core/rebalance/TableRebalancerTest.java | 58 +-
.../rebalance/tenant/TenantRebalancerTest.java | 31 +-
.../helix/core/retention/RetentionManagerTest.java | 9 +-
.../core/retention/SegmentLineageCleanupTest.java | 3 +-
.../tuner/RealTimeAutoIndexTunerTest.java | 3 +-
.../pinot/controller/tuner/TunerRegistryTest.java | 3 +-
.../controller/util/BrokerServiceHelperTest.java | 6 +-
.../controller/utils/SegmentMetadataMockUtils.java | 5 +-
.../BrokerResourceValidationManagerTest.java | 3 +-
.../RealtimeOffsetAutoResetManagerTest.java | 17 +-
.../validation/StorageQuotaCheckerTest.java | 4 +-
.../pinot/core/auth/BasicAuthPrincipalUtils.java | 9 +-
.../apache/pinot/core/common/MinionConstants.java | 43 +-
.../core/data/manager/BaseTableDataManager.java | 72 +-
.../manager/offline/DimensionTableDataManager.java | 5 +-
.../provider/DefaultTableDataManagerProvider.java | 4 +-
.../manager/provider/TableDataManagerProvider.java | 7 +-
.../realtime/RealtimeConsumptionRateManager.java | 36 +-
.../realtime/RealtimeSegmentDataManager.java | 117 +++-
.../realtime/RealtimeSegmentMetadataUtils.java | 4 +-
.../manager/realtime/RealtimeTableDataManager.java | 21 +-
.../ServerIngestionOomProtectionManager.java | 373 ++++++++++
.../apache/pinot/core/data/table/IndexedTable.java | 4 +-
.../pinot/core/data/table/SortedRecordTable.java | 2 +-
.../pinot/core/data/table/SortedRecordsMerger.java | 2 +-
.../apache/pinot/core/data/table/TableResizer.java | 106 ++-
.../AcquireReleaseColumnsSegmentOperator.java | 3 +-
.../core/operator/BitmapDocIdSetOperator.java | 3 +-
.../pinot/core/operator/DocIdSetOperator.java | 3 +-
.../core/operator/InstanceResponseOperator.java | 3 +-
.../pinot/core/operator/ProjectionOperator.java | 3 +-
.../blocks/results/AggregationResultsBlock.java | 16 +-
.../blocks/results/GroupByResultsBlock.java | 13 +-
.../operator/blocks/results/ResultsBlockUtils.java | 15 +-
.../operator/combine/GroupByCombineOperator.java | 10 +-
.../ExpressionScanDocIdIterator.java | 3 +-
.../pinot/core/operator/docidsets/AndDocIdSet.java | 9 +-
.../operator/filter/BitmapBasedFilterOperator.java | 3 +-
.../core/operator/filter/EmptyFilterOperator.java | 3 +-
.../filter/ExactVectorScanFilterOperator.java | 3 +-
.../operator/filter/ExpressionFilterOperator.java | 3 +-
.../filter/H3InclusionIndexFilterOperator.java | 3 +-
.../operator/filter/H3IndexFilterOperator.java | 7 +-
.../filter/InvertedIndexFilterOperator.java | 55 +-
.../operator/filter/JsonMatchFilterOperator.java | 3 +-
.../core/operator/filter/MapFilterOperator.java | 3 +-
.../operator/filter/MatchAllFilterOperator.java | 3 +-
.../core/operator/filter/NotFilterOperator.java | 3 +-
.../filter/RangeIndexBasedFilterOperator.java | 3 +-
.../operator/filter/ScanBasedFilterOperator.java | 3 +-
.../filter/SortedIndexBasedFilterOperator.java | 7 +-
.../core/operator/filter/TestFilterOperator.java | 3 +-
.../operator/filter/TextMatchFilterOperator.java | 3 +-
.../filter/VectorRadiusFilterOperator.java | 3 +-
.../filter/VectorSimilarityFilterOperator.java | 5 +-
.../core/operator/query/AggregationOperator.java | 3 +-
.../query/DictionaryBasedDistinctOperator.java | 3 +-
.../core/operator/query/DistinctOperator.java | 3 +-
.../operator/query/EmptyAggregationOperator.java | 5 +-
.../operator/query/EmptySelectionOperator.java | 5 +-
.../operator/query/FastFilteredCountOperator.java | 3 +-
.../operator/query/FilteredGroupByOperator.java | 27 +-
.../pinot/core/operator/query/GroupByOperator.java | 29 +-
.../query/LinearSelectionOrderByOperator.java | 3 +-
.../query/NonScanBasedAggregationOperator.java | 3 +-
.../core/operator/query/SelectionOnlyOperator.java | 3 +-
.../operator/query/SelectionOrderByOperator.java | 3 +-
.../streaming/StreamingSelectionOnlyOperator.java | 3 +-
.../timeseries/TimeSeriesOperatorUtils.java | 3 +-
.../core/operator/transform/TransformOperator.java | 3 +-
.../function/ScalarTransformFunctionWrapper.java | 4 +-
.../org/apache/pinot/core/plan/ExplainInfo.java | 5 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 5 +-
.../function/AggregationFunctionUtils.java | 14 +-
.../BaseSingleInputAggregationFunction.java | 3 +-
.../function/CountAggregationFunction.java | 8 +-
.../function/CountMVAggregationFunction.java | 3 +-
...istinctCountThetaSketchAggregationFunction.java | 5 +-
.../function/funnel/AggregationStrategy.java | 181 ++++-
.../function/funnel/BitmapAggregationStrategy.java | 10 +
.../funnel/BitmapResultExtractionStrategy.java | 58 +-
.../function/funnel/DictIdsWrapper.java | 136 +++-
.../FunnelCountAggregationFunctionFactory.java | 5 +-
.../FunnelCountSortedAggregationFunction.java | 16 +-
.../funnel/SetResultExtractionStrategy.java | 30 +-
.../function/funnel/SortedAggregationResult.java | 108 ++-
.../function/funnel/SortedAggregationStrategy.java | 12 +
.../funnel/ThetaSketchAggregationStrategy.java | 19 +-
.../groupby/DefaultGroupByExecutor.java | 8 +-
.../groupby/GroupingSetsGroupKeyGenerator.java | 423 ++++++++++++
.../utils/exprminmax/ExprMinMaxObject.java | 7 +-
.../pinot/core/query/executor/QueryExecutor.java | 23 +
.../query/executor/ServerQueryExecutorV1Impl.java | 50 +-
.../core/query/executor/TableExecutionInfo.java | 3 +-
.../pinot/core/query/optimizer/QueryOptimizer.java | 3 +-
.../filter/IdenticalPredicateFilterOptimizer.java | 95 +--
.../statement/StringPredicateFilterOptimizer.java | 124 ----
.../core/query/prefetch/DefaultFetchPlanner.java | 3 +-
.../query/pruner/SelectionQuerySegmentPruner.java | 3 +-
.../query/reduce/AggregationDataTableReducer.java | 5 +-
.../core/query/reduce/BaseGapfillProcessor.java | 3 +-
.../query/reduce/ExplainPlanDataTableReducer.java | 8 +-
.../pinot/core/query/reduce/GapfillProcessor.java | 11 +-
.../core/query/reduce/GroupByDataTableReducer.java | 66 +-
.../core/query/reduce/PostAggregationHandler.java | 64 +-
.../core/query/reduce/ReducerDataSchemaUtils.java | 26 +-
.../query/reduce/SelectionDataTableReducer.java | 3 +-
.../reduce/SelectionOnlyStreamingReducer.java | 3 +-
.../core/query/request/context/QueryContext.java | 91 ++-
.../context/utils/QueryContextConverterUtils.java | 40 ++
.../query/selection/SelectionOperatorService.java | 3 +-
.../query/selection/SelectionOperatorUtils.java | 3 +-
.../core/routing/MultiClusterRoutingContext.java | 5 +-
.../processing/aggregator/AvgValueAggregator.java | 62 ++
.../aggregator/FirstWithTimeValueAggregator.java | 42 ++
.../aggregator/LastWithTimeValueAggregator.java | 42 ++
.../aggregator/ValueAggregatorFactory.java | 57 +-
.../framework/SegmentProcessorConfig.java | 24 +-
.../framework/SegmentProcessorFramework.java | 3 +-
.../genericrow/GenericRowFileManager.java | 25 +
.../genericrow/GenericRowFileReader.java | 13 +-
.../genericrow/GenericRowFileRecordReader.java | 9 +-
.../segment/processing/mapper/SegmentMapper.java | 7 +-
.../segment/processing/reducer/RollupReducer.java | 43 +-
.../processing/timehandler/EpochTimeHandler.java | 14 +
.../processing/timehandler/TimeHandler.java | 6 +
.../processing/timehandler/TimeHandlerFactory.java | 3 +-
.../processing/utils/SegmentProcessorUtils.java | 30 +-
.../apache/pinot/core/startree/StarTreeUtils.java | 3 +-
.../startree/operator/StarTreeFilterOperator.java | 7 +-
.../core/transport/InstanceRequestHandler.java | 1 +
.../PooledByteBufAllocatorWithLimits.java | 37 +-
.../apache/pinot/core/transport/QueryServer.java | 14 +-
.../pinot/core/transport/ServerChannels.java | 28 +-
.../routing/stats/ServerRoutingStatsManager.java | 100 ++-
.../org/apache/pinot/core/util/GapfillUtils.java | 75 ++
.../org/apache/pinot/core/util/GroupByUtils.java | 61 ++
.../pinot/core/util/SortedRangeIntersection.java | 5 +-
.../accounting/QueryResourceAggregatorTest.java | 20 +-
.../org/apache/pinot/core/auth/BasicAuthTest.java | 21 +-
.../apache/pinot/core/auth/ZkBasicAuthTest.java | 22 +-
.../apache/pinot/core/common/DataFetcherTest.java | 3 +-
.../common/datablock/DataBlockBuilderTest.java | 3 +-
.../core/common/datablock/DataBlockSerdeTest.java | 3 +-
.../core/data/function/ArrayFunctionsTest.java | 81 ++-
.../core/data/function/DateTimeFunctionsTest.java | 39 +-
.../data/manager/BaseTableDataManagerTest.java | 25 +
.../data/manager/DuoSegmentDataManagerTest.java | 4 +-
.../pinot/core/data/manager/TableIndexingTest.java | 10 +-
.../offline/DimensionTableDataManagerTest.java | 9 +-
.../RealtimeConsumptionRateManagerTest.java | 51 +-
.../realtime/RealtimeSegmentDataManagerTest.java | 367 +++++++++-
.../ServerIngestionOomProtectionManagerTest.java | 491 +++++++++++++
.../pinot/core/data/table/TableResizerTest.java | 54 ++
.../transform/StGeometryTypeFunctionTest.java | 10 +-
.../pinot/core/minion/SegmentPurgerTest.java | 9 +-
.../operator/InstanceResponseOperatorTest.java | 10 +-
.../operator/combine/CombineSlowOperatorsTest.java | 3 +-
.../dociditerators/SortedDocIdIteratorTest.java | 7 +-
.../core/operator/docidsets/AndDocIdSetTest.java | 128 ++++
.../core/operator/docidsets/OrDocIdSetTest.java | 7 +-
.../operator/filter/AndFilterOperatorTest.java | 5 +-
.../operator/filter/BaseFilterOperatorTest.java | 11 +-
.../operator/filter/FilterOperatorUtilsTest.java | 17 +-
.../filter/InvertedIndexFilterOperatorTest.java | 132 ++++
.../operator/filter/NotFilterOperatorTest.java | 3 +-
.../core/operator/filter/OrFilterOperatorTest.java | 5 +-
.../operator/filter/VectorSearchParamsTest.java | 3 +-
.../function/CaseTransformFunctionTest.java | 19 +-
.../DateTimeConversionTransformFunctionTest.java | 3 +-
.../periodictask/PeriodicTaskSchedulerTest.java | 3 +-
.../function/AbstractAggregationFunctionTest.java | 4 +-
.../function/AvgAggregationFunctionTest.java | 4 +-
.../function/CountAggregationFunctionTest.java | 24 +-
.../function/DistinctAggregationFunctionTest.java | 20 +-
.../DistinctAvgAggregationFunctionTest.java | 4 +-
.../DistinctCountAggregationFunctionTest.java | 36 +-
...stinctCountBitmapMVAggregationFunctionTest.java | 6 +-
.../DistinctSumAggregationFunctionTest.java | 4 +-
.../FirstWithTimeAggregationFunctionTest.java | 3 +-
.../LastWithTimeAggregationFunctionTest.java | 3 +-
.../function/MaxStringAggregationFunctionTest.java | 10 +-
.../MinMaxRangeAggregationFunctionTest.java | 6 +-
.../function/MinStringAggregationFunctionTest.java | 10 +-
.../function/ModeAggregationFunctionTest.java | 6 +-
.../function/StUnionAggregationFunctionTest.java | 3 +-
.../function/funnel/DictIdsWrapperTest.java | 128 ++++
.../funnel/SortedAggregationResultTest.java | 57 ++
.../query/aggregation/groupby/GroupByTrimTest.java | 3 +-
.../executor/LogicalTableExecutionInfoTest.java | 7 +-
.../core/query/executor/QueryExecutorTest.java | 12 +-
.../executor/SingleTableExecutionInfoTest.java | 7 +-
.../killing/CompositeQueryKillingStrategyTest.java | 5 +-
.../core/query/optimizer/QueryOptimizerTest.java | 7 +
.../StringPredicateFilterOptimizerTest.java | 68 --
.../query/pruner/BloomFilterSegmentPrunerTest.java | 5 +-
.../query/pruner/ColumnValueSegmentPrunerTest.java | 4 +-
.../query/pruner/SegmentPrunerServiceTest.java | 7 +-
.../core/query/reduce/MergeDataTablesOnlyTest.java | 61 ++
.../query/reduce/ReducerDataSchemaUtilsTest.java | 29 +
.../BrokerRequestToQueryContextConverterTest.java | 87 ++-
.../selection/SelectionOperatorServiceTest.java | 5 +-
.../core/query/selection/SelectionOrderByTest.java | 32 +-
.../query/utils/OrderByComparatorFactoryTest.java | 9 +-
.../fakestream/FakePartitionLevelConsumer.java | 3 +-
.../core/routing/MockRoutingManagerFactory.java | 9 +
.../aggregator/AvgValueAggregatorTest.java | 114 +++
.../BaseSegmentProcessorFrameworkTest.java | 154 +++++
.../segment/processing/framework/ReducerTest.java | 251 ++++++-
.../processing/framework/SegmentMapperTest.java | 35 +-
.../framework/SegmentProcessorFrameworkTest.java | 13 +-
.../processing/genericrow/GenericRowSerDeTest.java | 3 +-
.../utils/SegmentProcessorUtilsTest.java | 89 +++
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 5 +-
.../v2/DistinctCountCPCSketchStarTreeV2Test.java | 4 +-
.../v2/DistinctCountHLLStarTreeV2Test.java | 4 +-
.../v2/DistinctCountHLLWithMVStarTreeV2Test.java | 4 +-
...ctCountIntegerSumTupleSketchStarTreeV2Test.java | 4 +-
.../v2/DistinctCountRawHLLStarTreeV2Test.java | 4 +-
.../v2/DistinctCountThetaSketchStarTreeV2Test.java | 4 +-
.../v2/DistinctCountULLStarTreeV2Test.java | 4 +-
.../v2/PercentileTDigestStarTreeV2Test.java | 4 +-
.../v2/PercentileTDigestWithMVStarTreeV2Test.java | 4 +-
...reAggregatedDistinctCountHLLStarTreeV2Test.java | 4 +-
...eAggregatedPercentileTDigestStarTreeV2Test.java | 4 +-
.../startree/v2/SumPrecisionStarTreeV2Test.java | 4 +-
.../pinot/core/transport/QueryRoutingTest.java | 10 +-
.../pinot/core/transport/QueryServerTest.java | 11 +
.../pinot/core/transport/ServerChannelsTest.java | 31 +
.../stats/ServerRoutingStatsManagerTest.java | 135 +++-
.../apache/pinot/core/util/GroupByUtilsTest.java | 62 ++
.../apache/pinot/core/util/SchemaUtilsTest.java | 32 +-
.../queries/BaseFSTBasedRegexpLikeQueriesTest.java | 3 +-
.../apache/pinot/queries/DistinctQueriesTest.java | 6 +-
.../pinot/queries/FilteredAggregationsTest.java | 3 +-
.../pinot/queries/FirstWithTimeQueriesTest.java | 5 +-
.../apache/pinot/queries/GapfillQueriesTest.java | 18 +
.../apache/pinot/queries/H3IndexQueriesTest.java | 3 +-
...terSegmentAggregationMultiValueQueriesTest.java | 37 +-
...SegmentAggregationMultiValueRawQueriesTest.java | 33 +-
...erSegmentAggregationSingleValueQueriesTest.java | 197 +++---
.../pinot/queries/JsonExtractScalarTest.java | 23 +
.../pinot/queries/LastWithTimeQueriesTest.java | 5 +-
.../org/apache/pinot/queries/ModeQueriesTest.java | 17 +-
.../NoDictionaryCompressionQueriesTest.java | 11 +-
.../queries/NullHandlingEnabledQueriesTest.java | 11 +-
.../org/apache/pinot/queries/QueriesTestUtils.java | 5 +-
.../apache/pinot/queries/StUnionQueriesTest.java | 3 +-
.../pinot/queries/StatisticalQueriesTest.java | 3 +-
.../pinot/queries/TextSearchQueriesTest.java | 3 +-
.../tests/BaseClusterIntegrationTest.java | 7 +-
.../pinot/integration/tests/QueryGenerator.java | 9 +-
.../tests/BaseClusterIntegrationTestSet.java | 7 +-
.../tests/CommitTimeCompactionIntegrationTest.java | 9 +-
.../tests/CursorCronCleanupIntegrationTest.java | 5 +-
.../integration/tests/CursorIntegrationTest.java | 3 +-
.../tests/DimensionTableIntegrationTest.java | 3 +-
.../tests/ErrorCodesIntegrationTest.java | 5 +-
.../tests/HybridClusterIntegrationTest.java | 3 +-
.../IngestionConfigHybridIntegrationTest.java | 3 +-
.../tests/LLCRealtimeClusterIntegrationTest.java | 3 +-
.../MaterializedViewClusterIntegrationTest.java | 29 +-
.../MergeRollupMinionClusterIntegrationTest.java | 26 +-
.../MultiNodesOfflineClusterIntegrationTest.java | 4 +-
.../tests/MultiStageEngineIntegrationTest.java | 104 ++-
.../MultiStageWithoutStatsIntegrationTest.java | 5 +-
.../tests/NullHandlingIntegrationTest.java | 30 +
.../tests/OfflineClusterIntegrationTest.java | 6 +-
...PartialUpsertTableRebalanceIntegrationTest.java | 10 +-
...geMetadataPushMinionClusterIntegrationTest.java | 3 +-
.../tests/PurgeMinionClusterIntegrationTest.java | 42 +-
.../tests/QueryWorkloadIntegrationTest.java | 3 +-
...nsumptionRateLimiterClusterIntegrationTest.java | 3 +-
...fflineSegmentsMinionClusterIntegrationTest.java | 28 +-
...mentGenerationMinionClusterIntegrationTest.java | 3 +-
...gmentGenerationMinionRealtimeIngestionTest.java | 7 +-
...PartitionLLCRealtimeClusterIntegrationTest.java | 52 +-
.../SegmentWriterUploaderIntegrationTest.java | 3 +-
.../tests/SimpleMinionClusterIntegrationTest.java | 3 +-
.../tests/StaleSegmentCheckIntegrationTest.java | 11 +-
.../StarTreeFunctionParametersIntegrationTest.java | 9 +-
.../tests/TableRebalanceIntegrationTest.java | 27 +-
.../tests/TimeSeriesIntegrationTest.java | 3 +-
.../integration/tests/TlsIntegrationTest.java | 3 +-
.../UpsertCompactMergeTaskIntegrationTest.java | 16 +-
.../tests/UpsertTableIntegrationTest.java | 14 +-
.../tests/UrlAuthRealtimeIntegrationTest.java | 3 +-
.../tests/custom/AggregateMetricsTest.java | 9 +-
.../tests/custom/CLPEncodingRealtimeTest.java | 3 +-
.../integration/tests/custom/FunnelCountTest.java | 103 ++-
.../tests/custom/GroupingSetsQueriesTest.java | 720 +++++++++++++++++++
.../tests/custom/HnswConsolidatedVectorTest.java | 187 +++++
.../custom/IvfFlatConsolidatedVectorTest.java | 185 +++++
.../tests/custom/MultiColumnTextIndicesTest.java | 3 +-
.../tests/custom/RefreshSegmentMinionTest.java | 39 +-
.../integration/tests/custom/SSBQueryTest.java | 5 +-
.../integration/tests/custom/TimestampTest.java | 26 +
.../tests/custom/UnnestIntegrationTest.java | 38 +
.../BaseLogicalTableIntegrationTest.java | 29 +
.../KafkaPartitionSubsetChaosIntegrationTest.java | 3 +-
.../models/DummyTableUpsertMetadataManager.java | 3 +-
.../plugin/minion/tasks/TestTaskGenerator.java | 3 +-
.../ingestion/BaseKinesisIntegrationTest.java | 4 +-
.../realtime/ingestion/KinesisShardChangeTest.java | 3 +-
.../ingestion/RealtimeKinesisIntegrationTest.java | 3 +-
.../FailureInjectingRealtimeTableDataManager.java | 10 +
.../FailureInjectingTableDataManagerProvider.java | 6 +-
.../tpch/TPCHGeneratedQueryIntegrationTest.java | 6 +-
.../tests/tpch/TPCHQueryIntegrationTest.java | 6 +-
.../analysis/MaterializedViewAnalyzer.java | 3 +-
.../MaterializedViewConsistencyManager.java | 5 +-
.../MaterializedViewDefinitionMetadata.java | 3 +-
.../MaterializedViewDefinitionMetadataBuilder.java | 4 +-
.../rewrite/MaterializedViewMetadataCache.java | 4 +-
.../scheduler/MaterializedViewTaskScheduler.java | 3 +-
.../GrpcMaterializedViewQueryExecutorTest.java | 5 +-
...erializedViewDefinitionMetadataBuilderTest.java | 10 +-
.../metadata/MaterializedViewMetadataTest.java | 26 +-
.../AggregationSubsumptionStrategyTest.java | 5 +-
.../rewrite/ExactSubsumptionStrategyTest.java | 5 +-
...MaterializedViewMetadataCacheLifecycleTest.java | 7 +-
.../MaterializedViewQueryRewriteEngineTest.java | 3 +-
.../rewrite/ScanSubsumptionStrategyTest.java | 5 +-
.../MaterializedViewTaskSchedulerTest.java | 15 +-
.../org/apache/pinot/minion/BaseMinionStarter.java | 3 +-
.../perf/BenchmarkColumnValueSegmentPruner.java | 9 +-
.../org/apache/pinot/perf/BenchmarkDataBlock.java | 3 +-
.../pinot/perf/BenchmarkGrpcMailboxSend.java | 3 +-
.../BenchmarkInvertedIndexGetNumMatchingDocs.java | 239 +++++++
.../pinot/perf/BenchmarkJsonIndexDistinct.java | 12 +-
.../pinot/perf/BenchmarkOfflineIndexReader.java | 11 +-
.../apache/pinot/perf/BenchmarkQueriesMSQE.java | 5 +-
.../apache/pinot/perf/BenchmarkQueriesSSQE.java | 5 +-
.../apache/pinot/perf/BenchmarkRegexpReplace.java | 10 +-
.../perf/BenchmarkVectorFeatureWorkloads.java | 7 +-
.../pinot/perf/BenchmarkVectorFilterWorkloads.java | 17 +-
.../apache/pinot/perf/BenchmarkVectorIndex.java | 7 +-
.../perf/aggregation/BenchmarkAvgAggregation.java | 4 +-
.../BenchmarkDistinctCountAggregation.java | 4 +-
.../BenchmarkDistinctCountHLLThreshold.java | 3 +-
.../perf/aggregation/BenchmarkMinAggregation.java | 4 +-
.../perf/aggregation/BenchmarkModeAggregation.java | 4 +-
.../perf/aggregation/BenchmarkSumAggregation.java | 4 +-
.../aggregation/BenchmarkVarianceAggregation.java | 3 +-
.../batch/common/SegmentGenerationJobUtils.java | 7 +
.../hadoop/HadoopSegmentGenerationJobRunner.java | 15 +-
.../HadoopSegmentGenerationJobRunnerTest.java | 4 +-
.../spark3/SparkSegmentGenerationJobRunner.java | 10 +-
.../SparkSegmentGenerationJobRunnerTest.java | 8 +-
.../standalone/SegmentGenerationJobRunnerTest.java | 4 +-
.../filesystem/GcsPinotFSNullSafetyTest.java | 6 +-
.../filesystem/GcsPinotFSPaginatedListTest.java | 3 +-
.../filesystem/S3PinotFSPaginatedListTest.java | 5 +-
.../inputformat/arrow/ArrowColumnReader.java | 350 ++--------
.../arrow/ArrowColumnReaderFactory.java | 4 +-
.../arrow/ArrowFileColumnReaderFactory.java | 4 +-
.../arrow/ArrowToPinotTypeConverter.java | 40 ++
.../arrow/BatchedArrowColumnReader.java | 308 ++-------
.../arrow/ArrowColumnReaderFactoryTest.java | 57 +-
.../inputformat/arrow/ArrowColumnReaderTest.java | 87 +++
.../arrow/ArrowColumnarBuildIntegrationTest.java | 91 ++-
.../arrow/ArrowFileColumnReaderFactoryTest.java | 124 ++--
...aConfluentSchemaRegistryJsonMessageDecoder.java | 4 +-
.../json/confluent/JsonConfluentSchemaTest.java | 3 +-
...fluentSchemaRegistryProtoBufMessageDecoder.java | 4 +-
.../protobuf/ProtoBufConfluentSchemaTest.java | 3 +-
.../parquet/ParquetNativeRecordExtractor.java | 2 +-
.../protobuf/ProtoBufRecordExtractor.java | 14 +-
.../ProtoBufRecordExtractorCachingTest.java | 87 +++
.../protobuf/ProtoBufSchemaUtilsTest.java | 18 +-
.../compound/CompoundPinotMetricsFactory.java | 5 +-
.../DropwizardBrokerPrometheusMetricsTest.java | 2 +-
.../DropwizardControllerPrometheusMetricsTest.java | 2 +-
.../DropwizardMinionPrometheusMetricsTest.java | 2 +-
.../DropwizardServerPrometheusMetricsTest.java | 2 +-
.../pinot/plugin/minion/tasks/MergeTaskUtils.java | 51 +-
.../pinot/plugin/minion/tasks/MinionTaskUtils.java | 182 ++++-
.../minion/tasks/SegmentConversionUtils.java | 3 +-
.../MaterializedViewTaskExecutor.java | 9 +-
.../mergerollup/MergeRollupTaskGenerator.java | 21 +
.../DefaultMergeRollupTaskSegmentGroupManager.java | 3 +-
.../minion/tasks/purge/PurgeTaskExecutor.java | 3 +-
.../RealtimeToOfflineSegmentsTaskExecutor.java | 3 +-
.../RealtimeToOfflineSegmentsTaskGenerator.java | 17 +-
.../refreshsegment/RefreshSegmentTaskExecutor.java | 21 +-
.../SegmentGenerationAndPushTaskExecutor.java | 4 +-
.../SegmentGenerationAndPushTaskGenerator.java | 7 +-
.../UpsertCompactionTaskExecutor.java | 13 +-
.../UpsertCompactionTaskGenerator.java | 81 +--
.../UpsertCompactMergeTaskExecutor.java | 5 +-
.../UpsertCompactMergeTaskGenerator.java | 83 ++-
.../plugin/minion/tasks/MergeTaskUtilsTest.java | 71 +-
.../plugin/minion/tasks/MinionTaskUtilsTest.java | 116 +++-
.../mergerollup/DimensionValueTransformerTest.java | 8 +-
.../MergeRollupAvgTaskExecutorTest.java | 330 +++++++++
.../MergeRollupFirstLastTaskExecutorTest.java | 256 +++++++
.../mergerollup/MergeRollupTaskGeneratorTest.java | 133 +++-
.../RealtimeToOfflineSegmentsTaskExecutorTest.java | 99 ++-
...RealtimeToOfflineSegmentsTaskGeneratorTest.java | 98 ++-
.../UpsertCompactionTaskGeneratorTest.java | 114 ++-
.../UpsertCompactMergeTaskExecutorTest.java | 3 +-
.../UpsertCompactMergeTaskGeneratorTest.java | 91 ++-
.../segmentuploader/SegmentUploaderDefault.java | 5 +-
.../filebased/FileBasedSegmentWriter.java | 3 +-
.../filebased/FileBasedSegmentWriterTest.java | 17 +-
.../KafkaPartitionLevelConnectionHandler.java | 4 +-
.../kafka30/KafkaStreamMetadataProvider.java | 14 +-
.../kafka30/server/KafkaServerStartable.java | 9 +-
.../kafka30/server/EmbeddedKafkaCluster.java | 13 +-
.../KafkaPartitionLevelConnectionHandler.java | 4 +-
.../kafka40/KafkaStreamMetadataProvider.java | 14 +-
.../stream/kafka40/utils/MiniKafkaCluster.java | 8 +-
.../stream/kafka/KafkaStreamConfigProperties.java | 12 -
.../kafka/KafkaPartitionSubsetUtilsTest.java | 3 +-
.../pinot/plugin/stream/pulsar/PulsarConfig.java | 16 +-
.../PulsarPartitionLevelConnectionHandler.java | 2 +-
.../plugin/stream/pulsar/PulsarConfigTest.java | 12 +-
.../pinot/tsdb/m3ql/M3TimeSeriesPlanner.java | 3 +-
.../pinot/calcite/rel/hint/PinotHintOptions.java | 12 +
.../calcite/rel/hint/PinotHintStrategyTable.java | 1 +
.../rel/logical/PinotLogicalEnrichedJoin.java | 3 +-
.../rel/logical/PinotLogicalSortExchange.java | 29 +-
.../PinotAggregateExchangeNodeInsertRule.java | 5 +-
.../rel/rules/PinotJoinExchangeNodeInsertRule.java | 5 +-
.../rel/rules/PinotRelDistributionTraitRule.java | 11 +-
.../rel/rules/PinotSortExchangeNodeInsertRule.java | 4 +-
.../rules/PinotWindowExchangeNodeInsertRule.java | 15 +-
.../org/apache/pinot/query/QueryEnvironment.java | 31 +-
.../apache/pinot/query/context/PlannerContext.java | 4 +-
.../explain/AskingServerStageExplainer.java | 11 +-
.../planner/explain/ExplainNodeSimplifier.java | 3 +-
.../planner/logical/PinotLogicalQueryPlanner.java | 9 +-
.../planner/logical/PlanNodeToRelConverter.java | 33 +-
.../planner/logical/RelToPlanNodeConverter.java | 197 +++++-
.../query/planner/logical/RexExpressionUtils.java | 73 +-
.../planner/physical/DispatchableSubPlan.java | 24 +-
.../planner/physical/PinotDispatchPlanner.java | 47 +-
.../planner/physical/v2/PinotDataDistribution.java | 3 +-
.../v2/PlanFragmentAndMailboxAssignment.java | 9 +
.../physical/v2/nodes/PhysicalExchange.java | 3 +-
.../planner/physical/v2/nodes/PhysicalFilter.java | 3 +-
.../opt/rules/LeafStageWorkerAssignmentRule.java | 11 +-
.../v2/opt/rules/LiteModeWorkerAssignmentRule.java | 3 +-
.../v2/opt/rules/WorkerExchangeAssignmentRule.java | 11 +-
.../query/planner/plannode/ExplainedNode.java | 3 +-
.../pinot/query/planner/plannode/UnnestNode.java | 44 +-
.../query/planner/serde/PlanNodeDeserializer.java | 10 +-
.../query/planner/serde/PlanNodeSerializer.java | 4 +-
.../apache/pinot/query/routing/WorkerManager.java | 245 +++++--
.../org/apache/pinot/query/type/TypeFactory.java | 2 +
.../org/apache/pinot/query/type/TypeSystem.java | 15 +-
.../rel/rules/PinotEnrichedJoinRuleTest.java | 42 +-
.../PinotWindowExchangeNodeInsertRuleTest.java | 3 +-
.../rel/rules/PinotWindowSplitRuleTest.java | 9 +-
.../apache/pinot/query/QueryCompilationTest.java | 99 +++
.../pinot/query/QueryEnvironmentTestBase.java | 10 +-
.../planner/explain/ExplainNodeSimplifierTest.java | 10 +-
.../query/planner/explain/PlanNodeMergerTest.java | 10 +-
.../logical/RelToPlanNodeConverterTest.java | 18 +-
.../planner/logical/RexExpressionUtilsTest.java | 570 +++++++++++++++
.../query/planner/logical/StagesTestBase.java | 4 +-
.../planner/physical/DispatchableSubPlanTest.java | 18 +-
.../planner/physical/v2/EmptyTablePlanTest.java | 186 +++++
.../physical/v2/PinotDataDistributionTest.java | 8 +-
.../physical/v2/mapping/PinotDistMappingTest.java | 13 +-
.../rules/LeafStageWorkerAssignmentRuleTest.java | 12 +
.../query/planner/plannode/UnnestNodeTest.java | 32 +
.../query/planner/serde/PlanNodeSerDeTest.java | 40 ++
.../pinot/query/queries/UnnestSqlPlannerTest.java | 149 ++++
.../pinot/query/routing/WorkerManagerTest.java | 530 +++++++++++++-
.../apache/pinot/query/type/TypeFactoryTest.java | 22 +
.../query/validate/PinotTypeCoercionTest.java | 32 +-
.../resources/queries/ExplainPhysicalPlans.json | 34 +
.../resources/queries/LiteralEvaluationPlans.json | 6 +-
.../pinot/query/mailbox/GrpcSendingMailbox.java | 3 +-
.../query/mailbox/InMemorySendingMailbox.java | 7 +-
.../mailbox/channel/MailboxContentObserver.java | 2 +-
.../apache/pinot/query/runtime/QueryRunner.java | 17 +-
.../pinot/query/runtime/blocks/ErrorMseBlock.java | 5 +-
.../runtime/executor/OpChainSchedulerService.java | 13 +-
.../pinot/query/runtime/function/ArrayUdf.java | 3 +-
.../query/runtime/operator/AggregateOperator.java | 7 +-
.../runtime/operator/EnrichedHashJoinOperator.java | 3 +-
.../runtime/operator/MailboxSendOperator.java | 9 +-
.../query/runtime/operator/MultiStageOperator.java | 3 +-
.../operator/MultistageAggregationExecutor.java | 3 +-
.../operator/MultistageGroupByExecutor.java | 4 +-
.../query/runtime/operator/UnnestOperator.java | 28 +-
.../query/runtime/operator/join/LookupTable.java | 5 +-
.../utils/BlockingMultiStreamConsumer.java | 7 +-
.../plan/server/ServerPlanRequestVisitor.java | 5 +-
.../runtime/timeseries/LeafTimeSeriesOperator.java | 4 +-
.../PhysicalTimeSeriesServerPlanVisitor.java | 7 +-
.../TimeSeriesExchangeReceiveOperator.java | 3 +-
.../TimeSeriesExchangeReceivePlanNode.java | 3 +-
.../timeseries/TimeSeriesPhysicalTableScan.java | 3 +-
.../query/mailbox/GrpcSendingMailboxTest.java | 5 +-
.../query/mailbox/InMemorySendingMailboxTest.java | 4 +-
.../operator/EnrichedHashJoinOperatorTest.java | 9 +-
.../query/runtime/operator/LeafOperatorTest.java | 35 +-
.../pinot/query/runtime/operator/OpChainTest.java | 3 +-
.../query/runtime/operator/OperatorTestUtil.java | 9 +-
.../query/runtime/operator/UnnestOperatorTest.java | 113 +++
.../operator/WindowAggregateOperatorTest.java | 19 +-
.../operator/exchange/BlockExchangeTest.java | 11 +-
.../operator/exchange/HashExchangeTest.java | 2 +-
.../operator/exchange/RandomExchangeTest.java | 3 +-
.../operator/exchange/SingletonExchangeTest.java | 3 +-
.../plan/MultiStageStatsTreeEncoderTest.java | 3 +-
.../query/runtime/queries/QueryRunnerTest.java | 3 +-
.../PhysicalTimeSeriesServerPlanVisitorTest.java | 18 +-
.../TimeSeriesExchangeReceiveOperatorTest.java | 2 +-
.../timeseries/serde/TimeSeriesBlockSerdeTest.java | 5 +-
.../service/dispatch/QueryDispatcherTest.java | 3 +-
.../queries/DirectAggregateObjectIntermediate.json | 69 ++
.../src/test/resources/queries/QueryHints.json | 12 +
.../DataTypeColumnTransformer.java | 37 +-
.../local/data/manager/SegmentDataManager.java | 23 +-
.../local/data/manager/TableDataManager.java | 17 +-
.../dedup/BasePartitionDedupMetadataManager.java | 3 +-
.../immutable/ImmutableSegmentLoader.java | 47 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 35 +-
.../loader/DefaultSegmentDirectoryLoader.java | 17 +-
.../loader/TierBasedSegmentDirectoryLoader.java | 5 +-
.../invertedindex/RealtimeNgramFilteringIndex.java | 2 +-
.../realtime/impl/json/MutableJsonIndexImpl.java | 15 +-
.../writer/StatelessRealtimeSegmentWriter.java | 4 +-
.../recordtransformer/DataTypeTransformer.java | 44 +-
.../recordtransformer/RecordTransformerUtils.java | 35 +
.../segment/local/segment/creator/Fixtures.java | 2 -
.../segment/creator/impl/BaseSegmentCreator.java | 34 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 163 ++++-
.../impl/SegmentIndexCreationDriverImpl.java | 12 +
.../impl/inv/geospatial/OffHeapH3IndexCreator.java | 9 +-
.../stats/AbstractColumnStatisticsCollector.java | 29 +-
.../BigDecimalColumnPreIndexStatsCollector.java | 9 +
.../stats/BytesColumnPreIndexStatsCollector.java | 9 +
.../ColumnarSegmentPreIndexStatsContainer.java | 112 ++-
.../stats/DoubleColumnPreIndexStatsCollector.java | 9 +
.../stats/FloatColumnPreIndexStatsCollector.java | 9 +
.../stats/IntColumnPreIndexStatsCollector.java | 9 +
.../stats/LongColumnPreIndexStatsCollector.java | 9 +
.../stats/MapColumnPreIndexStatsCollector.java | 2 +-
.../creator/impl/stats/StatsCollectorUtil.java | 33 +-
.../stats/StringColumnPreIndexStatsCollector.java | 10 +
.../creator/impl/text/LuceneTextIndexCreator.java | 7 +
.../text/MultiColumnLuceneTextIndexCreator.java | 7 +
.../impl/vector/HnswVectorIndexCreator.java | 23 +
.../vector/lucene99/HnswVectorIndexCombined.java | 354 ++++++++++
.../local/segment/index/bloom/BloomIndexType.java | 3 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 68 +-
.../index/dictionary/DictionaryIndexType.java | 5 +-
.../segment/index/forward/ForwardIndexType.java | 8 +-
.../local/segment/index/h3/H3IndexType.java | 3 +-
.../segment/index/inverted/InvertedIndexType.java | 3 +-
.../local/segment/index/json/JsonIndexType.java | 3 +-
.../segment/index/loader/IndexLoadingConfig.java | 9 +-
.../segment/index/loader/SegmentPreProcessor.java | 3 +-
.../loader/invertedindex/VectorIndexHandler.java | 482 ++++++++++++-
.../index/nullvalue/NullValueIndexType.java | 3 +-
.../OpenStructIndexPlugin.java} | 10 +-
.../index/openstruct/OpenStructIndexType.java | 182 +++++
.../openstruct/OpenStructSupportedIndexes.java | 26 +-
.../local/segment/index/range/RangeIndexType.java | 5 +-
.../text/MultiColumnLuceneTextIndexReader.java | 3 +-
.../vector/HnswVectorIndexBufferReader.java | 119 ++++
.../readers/vector/HnswVectorIndexReader.java | 104 ++-
.../readers/vector/IvfFlatVectorIndexReader.java | 72 +-
.../readers/vector/IvfOnDiskVectorIndexReader.java | 218 +++---
.../readers/vector/IvfPqVectorIndexReader.java | 62 +-
.../segment/index/vector/IvfCombinedBuffers.java | 116 ++++
.../index/vector/IvfFlatVectorIndexCreator.java | 18 +-
.../segment/index/vector/IvfPqIndexFormat.java | 65 +-
.../index/vector/IvfPqVectorIndexCreator.java | 28 +-
.../segment/index/vector/VectorIndexType.java | 142 +++-
.../segment/readers/DefaultValueColumnReader.java | 224 +-----
.../readers/PinotSegmentColumnReaderImpl.java | 230 +------
.../local/segment/store/FilePerIndexDirectory.java | 15 +-
.../segment/store/SegmentLocalFSDirectory.java | 45 +-
.../segment/store/SingleFileIndexDirectory.java | 25 +-
.../local/segment/store/TextIndexUtils.java | 3 +-
.../local/segment/store/VectorIndexUtils.java | 175 ++++-
.../v2/builder/StarTreeV2BuilderConfig.java | 7 +-
.../upsert/BasePartitionUpsertMetadataManager.java | 34 +-
.../upsert/BaseTableUpsertMetadataManager.java | 3 +-
.../local/upsert/TableUpsertMetadataManager.java | 3 +-
.../pinot/segment/local/utils/IngestionUtils.java | 14 -
.../pinot/segment/local/utils/SchemaUtils.java | 6 +-
.../segment/local/utils/TableConfigUtils.java | 41 +-
.../DistinctCountCPCSketchValueAggregatorTest.java | 28 +-
...istinctCountThetaSketchValueAggregatorTest.java | 22 +-
.../DistinctCountULLValueAggregatorTest.java | 18 +-
.../IntegerTupleSketchValueAggregatorTest.java | 10 +-
.../DataTypeColumnTransformerTest.java | 174 ++---
.../indexsegment/mutable/IndexingFailureTest.java | 7 +-
.../MutableSegmentImplAggregateMetricsTest.java | 6 +-
.../MutableSegmentImplNullValueVectorTest.java | 6 +-
...ealtimeLuceneTextIndexResourceTrackingTest.java | 6 +-
.../recordtransformer/DataTypeTransformerTest.java | 33 +-
.../ExpressionTransformerTest.java | 21 +-
.../recordtransformer/RecordTransformerTest.java | 57 +-
.../segment/creator/DictionaryOptimiserTest.java | 3 +-
.../DictionaryOptimizerCardinalityTest.java | 3 +-
.../creator/impl/ColumnReaderInterfaceTest.java | 56 +-
.../AbstractColumnStatisticsCollectorTest.java | 4 +-
.../stats/MapColumnPreIndexStatsCollectorTest.java | 3 +-
.../stats/NoDictColumnStatisticsCollectorTest.java | 3 +-
.../stats/StatsCollectorUtilFieldSpecTest.java | 54 ++
.../MultiColumnLuceneTextIndexCreatorTest.java | 150 ++++
.../lucene99/HnswVectorIndexCombinedTest.java | 268 ++++++++
.../local/segment/index/ColumnMetadataTest.java | 3 +-
.../segment/local/segment/index/H3IndexTest.java | 59 +-
.../segment/local/segment/index/JsonIndexTest.java | 27 +-
.../segment/index/bloom/BloomIndexTypeTest.java | 8 +-
.../index/creator/LuceneTextIndexCreatorTest.java | 125 ++++
.../SegmentGenerationWithNoRecordsTest.java | 4 +-
.../index/loader/IndexLoadingConfigTest.java | 9 +-
.../invertedindex/VectorIndexHandlerTest.java | 764 +++++++++++++++++++++
.../index/openstruct/OpenStructIndexTypeTest.java | 86 +++
.../readers/vector/IvfFlatFilterAwareTest.java | 22 +-
.../readers/vector/IvfOnDiskFilterAwareTest.java | 21 +-
.../text/LuceneTextIndexConfigReloadTest.java | 15 +-
.../index/vector/IvfDiskFormatInspectionTest.java | 400 +++++++++++
.../index/vector/IvfFlatVectorIndexTest.java | 54 +-
.../segment/index/vector/IvfPqIndexFormatTest.java | 17 +-
.../segment/index/vector/IvfPqVectorIndexTest.java | 60 +-
.../index/vector/IvfReaderFailurePathTest.java | 410 +++++++++++
.../segment/index/vector/VectorIndexTypeTest.java | 277 ++++++++
.../index/vector/VectorSearchBenchmark.java | 3 +-
.../readers/DefaultValueColumnReaderTest.java | 415 +++--------
.../segment/readers/InMemoryColumnReader.java | 186 ++---
.../readers/PinotSegmentColumnReaderImplTest.java | 627 +++--------------
.../readers/PinotSegmentRecordReaderTest.java | 3 +-
.../segment/store/FilePerIndexDirectoryTest.java | 19 +-
.../segment/store/SegmentDirectoryPathsTest.java | 97 +++
.../store/SingleFileIndexDirectoryTest.java | 18 +-
.../segment/store/StarTreeIndexReaderTest.java | 6 +-
.../local/segment/store/VectorIndexUtilsTest.java | 199 ++++++
...ertMetadataManagerForConsistentDeletesTest.java | 13 +-
...rrentMapPartitionUpsertMetadataManagerTest.java | 57 +-
.../local/upsert/PartialUpsertHandlerTest.java | 5 +-
.../local/upsert/UpsertViewManagerTest.java | 9 +-
.../merger/PartialUpsertMergerFactoryTest.java | 4 +-
.../segment/local/utils/IngestionUtilsTest.java | 20 +-
.../local/utils/SegmentPreloadUtilsTest.java | 3 +-
.../segment/local/utils/TableConfigUtilsTest.java | 177 +++--
.../org/apache/pinot/segment/spi/V1Constants.java | 21 +
.../spi/datasource/OpenStructDataSource.java | 10 +
.../spi/index/ColumnConfigDeserializer.java | 3 +-
.../segment/spi/index/FieldIndexConfigsUtil.java | 46 ++
.../segment/spi/index/IndexConfigDeserializer.java | 9 +-
.../pinot/segment/spi/index/TextIndexConfig.java | 7 +-
.../spi/index/creator/VectorIndexConfig.java | 88 ++-
.../index/creator/VectorIndexConfigValidator.java | 3 +-
.../multicolumntext/MultiColumnTextMetadata.java | 13 +-
.../spi/index/reader/VectorIndexReader.java | 3 +-
.../spi/loader/SegmentDirectoryLoaderContext.java | 37 +-
.../pinot/segment/spi/memory/PinotDataBuffer.java | 3 +-
.../segment/spi/partition/PartitionFunction.java | 3 +-
.../segment/spi/store/SegmentDirectoryPaths.java | 45 +-
.../apache/pinot/segment/spi/utils/CsvParser.java | 3 +-
.../spi/index/FieldIndexConfigsUtilTest.java | 189 +++++
.../spi/index/startree/StarTreeV2MetadataTest.java | 3 +-
.../pinot/server/api/resources/DebugResource.java | 11 +-
.../server/api/resources/InstanceResource.java | 7 +-
.../server/api/resources/MmapDebugResource.java | 4 +-
.../server/api/resources/TableSizeResource.java | 16 +-
.../server/api/resources/TableTierResource.java | 17 +-
.../pinot/server/api/resources/TablesResource.java | 131 ++--
.../org/apache/pinot/server/conf/ServerConf.java | 5 +-
.../server/predownload/PredownloadMetrics.java | 14 +
.../server/predownload/PredownloadScheduler.java | 119 +++-
.../server/predownload/PredownloadSegmentInfo.java | 18 +-
.../server/predownload/PredownloadZKClient.java | 47 ++
.../server/starter/ServerMetricsInitUtils.java | 50 ++
.../server/starter/helix/BaseServerStarter.java | 83 ++-
.../starter/helix/HelixInstanceDataManager.java | 14 +-
.../helix/HelixInstanceDataManagerConfig.java | 2 +-
.../helix/SegmentMessageHandlerFactory.java | 5 +-
.../apache/pinot/server/api/AccessControlTest.java | 4 +-
.../apache/pinot/server/api/BaseResourceTest.java | 21 +-
.../pinot/server/api/TableTierResourceTest.java | 11 +-
.../server/predownload/PredownloadMetricsTest.java | 64 ++
.../predownload/PredownloadSchedulerTest.java | 504 +++++++++++++-
.../predownload/PredownloadZKClientTest.java | 72 ++
.../server/starter/ServerMetricsInitUtilsTest.java | 65 ++
.../ServerGrpcChannelBackoffResetHandlerTest.java | 40 +-
.../config/table/ingestion/IngestionConfig.java | 48 +-
.../config/table/ingestion/SourceFieldConfig.java | 76 ++
.../table/ingestion/StreamIngestionConfig.java | 14 +
.../java/org/apache/pinot/spi/data/FieldSpec.java | 172 +++--
.../apache/pinot/spi/data/OpenStructNaming.java | 37 +
.../pinot/spi/data/OpenStructTypeInference.java | 68 ++
.../java/org/apache/pinot/spi/data/Schema.java | 11 +-
.../org/apache/pinot/spi/data/TimeFieldSpec.java | 1 +
.../pinot/spi/data/readers/ColumnReader.java | 483 +++++--------
.../pinot/spi/data/readers/MultiValueResult.java | 64 +-
.../apache/pinot/spi/env/PinotConfiguration.java | 2 +-
.../spi/metrics/NoopPinotMetricsRegistry.java | 3 +-
.../apache/pinot/spi/metrics/PinotMetricUtils.java | 3 +-
.../org/apache/pinot/spi/plugin/PluginManager.java | 2 +-
.../pinot/spi/stream/PartitionLevelConsumer.java | 70 --
.../spi/stream/PartitionLevelStreamConfig.java | 30 -
.../org/apache/pinot/spi/stream/StreamConfig.java | 90 ++-
.../pinot/spi/stream/StreamConfigProperties.java | 16 +-
.../pinot/spi/stream/StreamConsumerFactory.java | 31 +-
.../pinot/spi/stream/StreamLevelConsumer.java | 62 --
.../pinot/spi/stream/StreamMetadataProvider.java | 3 +-
.../apache/pinot/spi/utils/CommonConstants.java | 35 +
.../pinot/spi/utils/GroovyTemplateUtils.java | 5 +-
.../java/org/apache/pinot/spi/utils/JsonUtils.java | 19 +-
.../org/apache/pinot/spi/utils/PinotDataType.java | 179 +++--
.../pinot/spi/utils/TimestampIndexUtils.java | 5 +-
.../java/org/apache/pinot/spi/utils/UuidKey.java | 123 ++++
.../java/org/apache/pinot/spi/utils/UuidUtils.java | 363 +++++++++-
.../spi/utils/builder/TableConfigBuilder.java | 3 +-
.../spi/config/table/TableTaskConfigTest.java | 9 +-
.../pinot/spi/config/table/UpsertConfigTest.java | 4 +-
.../table/ingestion/SourceFieldConfigTest.java | 71 ++
.../org/apache/pinot/spi/data/FieldSpecTest.java | 184 +++++
.../pinot/spi/data/OpenStructNamingTest.java | 40 +-
.../spi/data/OpenStructTypeInferenceTest.java | 66 ++
.../java/org/apache/pinot/spi/data/SchemaTest.java | 34 +-
.../pinot/spi/data/readers/ColumnReaderTest.java | 77 +++
.../pinot/spi/env/VersionedPropertyConfigTest.java | 2 +-
.../stream/PartitionGroupMetadataFetcherTest.java | 55 +-
.../pinot/spi/stream/StreamMetadataTest.java | 7 +-
.../pinot/spi/utils/IngestionConfigUtilsTest.java | 17 +-
.../org/apache/pinot/spi/utils/JsonUtilsTest.java | 58 +-
.../org/apache/pinot/spi/utils/ObfuscatorTest.java | 10 +-
.../apache/pinot/spi/utils/PinotDataTypeTest.java | 96 ++-
.../org/apache/pinot/spi/utils/UuidUtilsTest.java | 205 ++++++
.../apache/pinot/sql/ddl/compile/CompiledDdl.java | 2 +-
.../ddl/compile/CompiledDropMaterializedView.java | 4 +-
.../pinot/sql/ddl/compile/CompiledDropTable.java | 4 +-
.../CompiledShowCreateMaterializedView.java | 4 +-
.../sql/ddl/compile/CompiledShowCreateTable.java | 4 +-
.../ddl/compile/CompiledShowMaterializedViews.java | 4 +-
.../pinot/sql/ddl/compile/CompiledShowTables.java | 4 +-
.../pinot/sql/ddl/compile/PropertyMapping.java | 2 +-
.../inferer/MaterializedViewSchemaInfererTest.java | 9 +-
.../CanonicalDdlEmitterMaterializedViewTest.java | 2 +-
.../sql/ddl/reverse/CanonicalDdlEmitterTest.java | 9 +-
.../pinot/sql/ddl/roundtrip/RoundTripTest.java | 3 +-
.../tsdb/planner/TimeSeriesPlanFragmenter.java | 7 +-
.../tsdb/planner/TimeSeriesPlanFragmenterTest.java | 11 +-
.../java/org/apache/pinot/tsdb/spi/AggInfo.java | 2 +-
.../tsdb/spi/series/BaseTimeSeriesBuilder.java | 3 +-
.../tsdb/spi/plan/LeafTimeSeriesPlanNodeTest.java | 20 +-
.../pinot/tsdb/spi/series/TimeSeriesTest.java | 8 +-
.../org/apache/pinot/tools/AuthQuickstart.java | 3 +-
.../apache/pinot/tools/AuthZkBasicQuickstart.java | 3 +-
.../pinot/tools/ColocatedJoinQuickStart.java | 3 +-
.../apache/pinot/tools/GeoSpatialQuickStart.java | 3 +-
.../org/apache/pinot/tools/HybridQuickstart.java | 3 +-
.../java/org/apache/pinot/tools/JfrQuickstart.java | 3 +-
.../org/apache/pinot/tools/JoinQuickStart.java | 3 +-
.../pinot/tools/LookupJoinEngineQuickStart.java | 3 +-
.../tools/MultiStageResourceTrackerQuickStart.java | 5 +-
.../pinot/tools/MultistageEngineQuickStart.java | 3 +-
.../apache/pinot/tools/NullHandlingQuickstart.java | 5 +-
.../SingleStageResourceTrackingQuickStart.java | 5 +-
.../apache/pinot/tools/SortedColumnQuickstart.java | 7 +-
.../org/apache/pinot/tools/TPCHQuickStart.java | 3 +-
.../tools/TimeSeriesEngineAuthQuickStart.java | 3 +-
.../pinot/tools/TimeSeriesEngineQuickStart.java | 3 +-
.../pinot/tools/TimestampIndexQuickstart.java | 3 +-
.../org/apache/pinot/tools/UpsertQuickStart.java | 3 +-
.../admin/command/AbstractBaseAdminCommand.java | 3 +-
.../tools/admin/command/AddSchemaCommand.java | 4 +-
.../admin/command/AvroSchemaToPinotSchema.java | 3 +-
.../tools/admin/command/ImportDataCommand.java | 5 +-
.../tools/admin/command/JsonToPinotSchema.java | 3 +-
.../command/LaunchBackfillIngestionJobCommand.java | 3 +-
.../tools/admin/command/QuickstartRunner.java | 3 +-
.../admin/command/ShowClusterInfoCommand.java | 5 +-
.../org/apache/pinot/tools/perf/QueryRunner.java | 9 +-
.../githubevents/PullRequestMergedEvent.java | 9 +-
.../pinot/tools/utils/KafkaStarterUtils.java | 2 -
.../githubEvents_realtime_table_config.json | 1 -
.../airlineStats_realtime_table_config.json | 1 -
.../docker/airlineStats_realtime_table_config.json | 2 -
.../dailySales_realtime_table_config.json | 1 -
.../fineFoodReviews_realtime_table_config.json | 1 -
...neFoodReviews_part_0_realtime_table_config.json | 1 -
...neFoodReviews_part_1_realtime_table_config.json | 1 -
.../githubEvents_realtime_table_config.json | 1 -
.../meetupRsvp_realtime_table_config.json | 1 -
...eetupRsvpComplexType_realtime_table_config.json | 1 -
.../meetupRsvpJson_realtime_table_config.json | 1 -
...lRequestMergedEvents_realtime_table_config.json | 1 -
...lRequestMergedEvents_realtime_table_config.json | 1 -
...upsertJsonMeetupRsvp_realtime_table_config.json | 1 -
.../examples/stream/upsertMeetupRsvp/README.md | 66 ++
.../upsertMeetupRsvp_realtime_table_config.json | 4 +-
...ertPartialMeetupRsvp_realtime_table_config.json | 1 -
.../converter/PinotSegmentConverterTest.java | 4 +-
pom.xml | 46 +-
1070 files changed, 26398 insertions(+), 8503 deletions(-)
delete mode 100644
pinot-common/src/main/java/org/apache/pinot/common/metadata/instance/InstanceZKMetadata.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/request/context/GroupingSets.java
delete mode 100644
pinot-common/src/test/java/org/apache/pinot/common/metadata/InstanceZKMetadataTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/GroupingSetsParserTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/TableConfigValidationUtilsTest.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/data/manager/realtime/ServerIngestionOomProtectionManager.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/GroupingSetsGroupKeyGenerator.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizer.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/aggregator/AvgValueAggregator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/aggregator/FirstWithTimeValueAggregator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/aggregator/LastWithTimeValueAggregator.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/data/manager/realtime/ServerIngestionOomProtectionManagerTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/docidsets/AndDocIdSetTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/filter/InvertedIndexFilterOperatorTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/funnel/DictIdsWrapperTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/funnel/SortedAggregationResultTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/StringPredicateFilterOptimizerTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/aggregator/AvgValueAggregatorTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/utils/SegmentProcessorUtilsTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/GroupingSetsQueriesTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/HnswConsolidatedVectorTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/IvfFlatConsolidatedVectorTest.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkInvertedIndexGetNumMatchingDocs.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-arrow/src/test/java/org/apache/pinot/plugin/inputformat/arrow/ArrowColumnReaderTest.java
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupAvgTaskExecutorTest.java
create mode 100644
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/mergerollup/MergeRollupFirstLastTaskExecutorTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RexExpressionUtilsTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/planner/physical/v2/EmptyTablePlanTest.java
create mode 100644
pinot-query-runtime/src/test/resources/queries/DirectAggregateObjectIntermediate.json
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/lucene99/HnswVectorIndexCombined.java
copy
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/{fst/FstIndexPlugin.java
=> openstruct/OpenStructIndexPlugin.java} (73%)
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexType.java
copy pinot-spi/src/main/java/org/apache/pinot/spi/metrics/PinotHistogram.java
=>
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructSupportedIndexes.java
(52%)
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/vector/HnswVectorIndexBufferReader.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/vector/IvfCombinedBuffers.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/stats/StatsCollectorUtilFieldSpecTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/text/MultiColumnLuceneTextIndexCreatorTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/vector/lucene99/HnswVectorIndexCombinedTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexTypeTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/vector/IvfDiskFormatInspectionTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/vector/IvfReaderFailurePathTest.java
create mode 100644
pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/FieldIndexConfigsUtilTest.java
create mode 100644
pinot-server/src/main/java/org/apache/pinot/server/starter/ServerMetricsInitUtils.java
create mode 100644
pinot-server/src/test/java/org/apache/pinot/server/predownload/PredownloadMetricsTest.java
create mode 100644
pinot-server/src/test/java/org/apache/pinot/server/starter/ServerMetricsInitUtilsTest.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/SourceFieldConfig.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/data/OpenStructTypeInference.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/stream/PartitionLevelConsumer.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/stream/PartitionLevelStreamConfig.java
delete mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/stream/StreamLevelConsumer.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/utils/UuidKey.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/config/table/ingestion/SourceFieldConfigTest.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/data/OpenStructTypeInferenceTest.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/data/readers/ColumnReaderTest.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/utils/UuidUtilsTest.java
create mode 100644
pinot-tools/src/main/resources/examples/stream/upsertMeetupRsvp/README.md
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
