This is an automated email from the ASF dual-hosted git repository. abhishekrb pushed a commit to branch migrate_nested_push in repository https://gitbox.apache.org/repos/asf/druid.git
commit cd88b075bbed186886644d0b8293f4ca0d8ab555 Author: Abhishek Balaji Radhakrishnan <[email protected]> AuthorDate: Wed Dec 3 21:50:15 2025 -0800 Remove references to the old IT: ITNestedQueryPushDownTest --- .github/workflows/standard-its.yml | 60 +--- integration-tests/k8s_run_config_file.json | 16 -- .../tests/indexer/ITNestedQueryPushDownTest.java | 111 -------- .../queries/nestedquerypushdown_queries.json | 303 --------------------- 4 files changed, 1 insertion(+), 489 deletions(-) diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index 16e57e60aa6..1af7acd9c8e 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -112,62 +112,4 @@ jobs: testing_groups: -Dgroups=custom-coordinator-duties use_indexer: middleManager override_config_path: ./environment-configs/test-groups/custom-coordinator-duties - group: custom coordinator duties - - integration-k8s-leadership-tests: - needs: changes - name: (Compile=openjdk17, Run=openjdk17, Cluster Build On K8s) ITNestedQueryPushDownTest integration test - if: ${{ needs.changes.outputs.core == 'true' || needs.changes.outputs.common-extensions == 'true' }} - runs-on: ubuntu-22.04 - env: - MVN: mvn --no-snapshot-updates - MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true - CONFIG_FILE: k8s_run_config_file.json - IT_TEST: -Dit.test=ITNestedQueryPushDownTest - POD_NAME: int-test - POD_NAMESPACE: default - BUILD_DRUID_CLUSTER: true - steps: - - name: Checkout branch - uses: actions/checkout@v4 - - - name: setup java - uses: actions/setup-java@v4 - with: - java-version: '17' - distribution: 'zulu' - - # the build step produces SNAPSHOT artifacts into the local maven repository, - # we include github.sha in the cache key to make it specific to that build/jdk - - name: Restore Maven repository - id: maven-restore - uses: actions/cache/restore@v4 - with: - path: ~/.m2/repository - key: maven-${{ runner.os }}-17-${{ github.sha }} - restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }} - - - name: Maven build - if: steps.maven-restore.outputs.cache-hit != 'true' - run: | - ./it.sh ci - - - name: Run IT - id: test - timeout-minutes: 90 - run: | - set -x - mvn -B -ff install -pl '!web-console' -Pdist,bundle-contrib-exts -Pskip-static-checks,skip-tests -Dmaven.javadoc.skip=true -T1C - # Note: The above command relies on the correct version of the JARs being installed in the local m2 repository. - # For any changes, please rebuild it using the command from the previous step (./it.sh ci). - - MAVEN_OPTS='-Xmx2048m' ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP} -Dpod.name=${POD_NAME} -Dpod.namespace=${POD_NAMESPACE} -Dbuild.druid.cluster=${BUILD_DRUID_CLUSTER} - - - name: Debug on failure - if: ${{ failure() && steps.test.conclusion == 'failure' }} - run: | - for v in broker middlemanager router coordinator historical ; do - echo "------------------------druid-tiny-cluster-"$v"s-0-------------------------"; - /usr/local/bin/kubectl logs --tail 1000 druid-tiny-cluster-"$v"s-0 ||:; - /usr/local/bin/kubectl get events | grep druid-tiny-cluster-"$v"s-0 ||:; - done + group: custom coordinator duties \ No newline at end of file diff --git a/integration-tests/k8s_run_config_file.json b/integration-tests/k8s_run_config_file.json deleted file mode 100644 index 249e44b19ca..00000000000 --- a/integration-tests/k8s_run_config_file.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "broker_host" : "localhost", - "broker_port" : "30400", - "broker_tls_url" : "http://localhost:30100";, - "router_host" : "localhost", - "router_port" : "30400", - "router_tls_url" : "http://localhost:30400";, - "indexer_host" : "localhost", - "indexer_port" : "30400", - "historical_host" : "localhost", - "historical_port" : "30300", - "coordinator_host" : "localhost", - "coordinator_port" : "30400", - "middlemanager_host": "localhost", - "zookeeper_hosts": "localhost:30600" -} \ No newline at end of file diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITNestedQueryPushDownTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITNestedQueryPushDownTest.java deleted file mode 100644 index 8082a3681af..00000000000 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITNestedQueryPushDownTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.tests.indexer; - -import com.google.inject.Inject; -import org.apache.commons.io.IOUtils; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.testing.clients.ClientInfoResourceTestClient; -import org.apache.druid.testing.clients.CoordinatorResourceTestClient; -import org.apache.druid.testing.guice.DruidTestModuleFactory; -import org.apache.druid.testing.tools.ITRetryUtil; -import org.apache.druid.testing.tools.IntegrationTestingConfig; -import org.apache.druid.testing.utils.TestQueryHelper; -import org.apache.druid.tests.TestNGGroup; -import org.testng.annotations.BeforeSuite; -import org.testng.annotations.Guice; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -@Test(groups = TestNGGroup.QUERY) -@Guice(moduleFactory = DruidTestModuleFactory.class) -public class ITNestedQueryPushDownTest extends AbstractIndexerTest -{ - private static final String WIKITICKER_DATA_SOURCE = "wikiticker"; - private static final String WIKITICKER_INDEX_TASK = "/indexer/wikiticker_index_task.json"; - private static final String WIKITICKER_QUERIES_RESOURCE = "/queries/nestedquerypushdown_queries.json"; - - @Inject - private CoordinatorResourceTestClient coordinatorClient; - @Inject - private TestQueryHelper queryHelper; - - private static final Logger LOG = new Logger(ITNestedQueryPushDownTest.class); - - @Inject - private IntegrationTestingConfig config; - - @Inject - ClientInfoResourceTestClient clientInfoResourceTestClient; - - private String fullDatasourceName; - - @BeforeSuite - public void setFullDatasourceName() - { - fullDatasourceName = WIKITICKER_DATA_SOURCE + config.getExtraDatasourceNameSuffix(); - } - - @Test - public void testIndexData() - { - try { - loadData(); - - String queryResponseTemplate; - try { - InputStream is = AbstractITBatchIndexTest.class.getResourceAsStream(WIKITICKER_QUERIES_RESOURCE); - queryResponseTemplate = IOUtils.toString(is, StandardCharsets.UTF_8); - } - catch (IOException e) { - throw new ISE(e, "could not read query file: %s", WIKITICKER_QUERIES_RESOURCE); - } - - queryResponseTemplate = StringUtils.replace( - queryResponseTemplate, - "%%DATASOURCE%%", - fullDatasourceName - ); - - queryHelper.testQueriesFromString(queryResponseTemplate); - } - catch (Exception e) { - LOG.error(e, "Error while testing"); - throw new RuntimeException(e); - } - } - - private void loadData() throws Exception - { - String taskSpec = getResourceAsString(WIKITICKER_INDEX_TASK); - taskSpec = StringUtils.replace(taskSpec, "%%DATASOURCE%%", fullDatasourceName); - final String taskID = indexer.submitTask(taskSpec); - LOG.info("TaskID for loading index task %s", taskID); - indexer.waitUntilTaskCompletes(taskID); - ITRetryUtil.retryUntilTrue( - () -> coordinator.areSegmentsLoaded(fullDatasourceName), "Segment Load" - ); - } -} diff --git a/integration-tests/src/test/resources/queries/nestedquerypushdown_queries.json b/integration-tests/src/test/resources/queries/nestedquerypushdown_queries.json deleted file mode 100644 index c7a062c02bb..00000000000 --- a/integration-tests/src/test/resources/queries/nestedquerypushdown_queries.json +++ /dev/null @@ -1,303 +0,0 @@ -[ - { - "description": "Nested group by double agg query with force push down", - "query": { - "queryType": "groupBy", - "dataSource": { - "type": "query", - "query": { - "queryType": "groupBy", - "dataSource": "%%DATASOURCE%%", - "intervals": [ - "2015-09-12/2015-09-13" - ], - "granularity": "all", - "dimensions": [ - "channel", - "user" - ], - "metric": "added", - "aggregations": [ - { - "type": "longSum", - "name": "sumAdded", - "fieldName": "added" - } - ] - } - }, - "granularity": "all", - "dimension": "channel", - "aggregations": [ - { - "type": "longSum", - "name": "groupedSumAdded", - "fieldName": "sumAdded" - } - ], - "intervals": [ - "2015-09-12/2015-09-13" - ], - "context": { - "forcePushDownNestedQuery":"true" - } - }, - "expectedResults": [ - { - "version" : "v1", - "timestamp" : "2015-09-12T00:00:00.000Z", - "event" : { - "groupedSumAdded" : 9385573 - } - } - ] - }, - { - "description": "Nested group by query with force push down and renamed dimensions", - "query": { - "queryType": "groupBy", - "dataSource": { - "type": "query", - "query": { - "queryType": "groupBy", - "dataSource": "%%DATASOURCE%%", - "intervals": [ - "2015-09-12/2015-09-13" - ], - "granularity": "all", - "dimensions": [ - {"dimension" : "channel", "outputName" :"renamedChannel"}, - {"dimension" : "user", "outputName" :"renamedUser"} - ], - "metric": "added", - "aggregations": [ - { - "type": "longSum", - "name": "sumAdded", - "fieldName": "added" - } - ] - } - }, - "granularity": "all", - "dimension": "renamedChannel", - "aggregations": [ - { - "type": "longSum", - "name": "groupedSumAdded", - "fieldName": "sumAdded" - } - ], - "intervals": [ - "2015-09-12/2015-09-13" - ], - "context": { - "forcePushDownNestedQuery":"true" - } - }, - "expectedResults": [ - { - "version" : "v1", - "timestamp" : "2015-09-12T00:00:00.000Z", - "event" : { - "groupedSumAdded" : 9385573 - } - } - ] - }, - { - "description": "Nested group by query with force push down and filter on outer and inner query", - "query": { - "queryType": "groupBy", - "dataSource": { - "type": "query", - "query": { - "queryType": "groupBy", - "dataSource": "%%DATASOURCE%%", - "intervals": [ - "2015-09-12/2015-09-13" - ], - "granularity": "all", - "dimensions": [ - {"dimension" : "channel", "outputName" :"renamedChannel"}, - {"dimension" : "user", "outputName" :"renamedUser"} - ], - "metric": "added", - "aggregations": [ - { - "type": "longSum", - "name": "sumAdded", - "fieldName": "added" - } - ], - "filter": { - "type": "or", - "fields": [ - { - "type": "selector", - "dimension": "channel", - "value": "#zh.wikipedia" - }, - { - "type": "selector", - "dimension": "channel", - "value": "#es.wikipedia" - } - ] - } - } - }, - "granularity": "all", - "dimension": "renamedChannel", - "aggregations": [ - { - "type": "longSum", - "name": "groupedSumAdded", - "fieldName": "sumAdded" - } - ], - "intervals": [ - "2015-09-12/2015-09-13" - ], - "filter": { - "type": "and", - "fields": [ - { - "type": "selector", - "dimension": "renamedChannel", - "value": "#zh.wikipedia" - } - ] - }, - "context": { - "forcePushDownNestedQuery":"true" - } - }, - "expectedResults": [ - { - "version" : "v1", - "timestamp" : "2015-09-12T00:00:00.000Z", - "event" : { - "groupedSumAdded" : 191033 - } - } - ] - }, - { - "description": "Nested group by query with force push down and having clause", - "query": { - "queryType": "groupBy", - "dataSource": { - "type": "query", - "query": { - "queryType": "groupBy", - "dataSource": "%%DATASOURCE%%", - "intervals": [ - "2015-09-12/2015-09-13" - ], - "granularity": "all", - "dimensions": [ - {"dimension" : "channel"}, - {"dimension" : "user"} - ], - "metric": "added", - "aggregations": [ - { - "type": "longSum", - "name": "sumAdded", - "fieldName": "added" - } - ] - } - }, - "granularity": "all", - "aggregations": [ - { - "type": "longSum", - "name": "outerSum", - "fieldName": "sumAdded" - } - ], - "intervals": [ - "2015-09-12/2015-09-13" - ], - "having": { - "type": "or", - "havingSpecs": [ - { - "type": "greaterThan", - "aggregation": "outerSum", - "value": 9385570 - } - ] - }, - "context": { - "forcePushDownNestedQuery":"true" - } - }, - "expectedResults": [ - { - "version" : "v1", - "timestamp" : "2015-09-12T00:00:00.000Z", - "event" : { - "outerSum" : 9385573 - } - } - ] - }, - { - "description": "Nested group by query with force push down and having clause. This test asserts that the post processing was invoked.", - "query": { - "queryType": "groupBy", - "dataSource": { - "type": "query", - "query": { - "queryType": "groupBy", - "dataSource": "%%DATASOURCE%%", - "intervals": [ - "2015-09-12/2015-09-13" - ], - "granularity": "all", - "dimensions": [ - {"dimension" : "channel"}, - {"dimension" : "user"} - ], - "metric": "added", - "aggregations": [ - { - "type": "longSum", - "name": "sumAdded", - "fieldName": "added" - } - ] - } - }, - "granularity": "all", - "aggregations": [ - { - "type": "longSum", - "name": "outerSum", - "fieldName": "sumAdded" - } - ], - "intervals": [ - "2015-09-12/2015-09-13" - ], - "having": { - "type": "or", - "havingSpecs": [ - { - "type": "greaterThan", - "aggregation": "outerSum", - "value": 100000000 - } - ] - }, - "context": { - "forcePushDownNestedQuery":"true" - } - }, - "expectedResults": [ - ] - } -] \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
