This is an automated email from the ASF dual-hosted git repository. tarmstrong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit c4e7977f5ee1b740215eb486a074bc1326cff360 Author: Tim Armstrong <tarmstr...@cloudera.com> AuthorDate: Mon Nov 23 12:14:33 2020 -0800 IMPALA-10351,IMPALA-9812: enable mt_dop for DML by default This allows setting mt_dop for any query with any configuration. Before this patch it was not supported for DML. --unlock_mt_dop and --mt_dop_auto_fallback are now ignored. Testing: * Updated tests to reflect new behaviour. * Removed irrelevant tests for fallback/validation. * Ran exhaustive tests. Change-Id: I66331481260fe4b69d9e95b0200029b14d230ade Reviewed-on: http://gerrit.cloudera.org:8080/16775 Reviewed-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenk...@cloudera.com> --- be/src/common/global-flags.cc | 9 +-- be/src/util/backend-gflag-util.cc | 4 -- bin/run-all-tests.sh | 3 +- common/thrift/BackendGflags.thrift | 4 +- .../java/org/apache/impala/common/RuntimeEnv.java | 7 -- .../java/org/apache/impala/planner/Planner.java | 18 ----- .../org/apache/impala/service/BackendConfig.java | 8 --- .../org/apache/impala/planner/PlannerTest.java | 16 ++--- .../queries/PlannerTest/mt-dop-validation.test | 80 ++++++++++++++++------ .../queries/QueryTest/mt-dop-auto-fallback.test | 31 --------- tests/custom_cluster/test_mt_dop.py | 21 +----- 11 files changed, 69 insertions(+), 132 deletions(-) diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc index 339ff14..1b8c16f 100644 --- a/be/src/common/global-flags.cc +++ b/be/src/common/global-flags.cc @@ -290,13 +290,6 @@ DEFINE_double_hidden(invalidate_tables_fraction_on_memory_pressure, 0.1, "The fraction of tables to invalidate when CatalogdTableInvalidator considers the " "old GC generation to be almost full."); -DEFINE_bool_hidden(unlock_mt_dop, false, - "(Experimental) If true, allow specifying mt_dop for all queries."); - -DEFINE_bool_hidden(mt_dop_auto_fallback, false, - "(Experimental) If true, fall back to non-mt_dop if mt_dop query option is set and " - "a query does not support it. Has no effect if --unlock_mt_dop is true."); - DEFINE_bool_hidden(recursively_list_partitions, true, "If true, recursively list the content of partition directories."); @@ -399,6 +392,7 @@ REMOVED_FLAG(llama_registration_timeout_secs); REMOVED_FLAG(llama_registration_wait_secs); REMOVED_FLAG(local_nodemanager_url); REMOVED_FLAG(max_free_io_buffers); +REMOVED_FLAG(mt_dop_auto_fallback); REMOVED_FLAG(pull_incremental_statistics); REMOVED_FLAG(report_status_retry_interval_ms); REMOVED_FLAG(resource_broker_cnxn_attempts); @@ -417,6 +411,7 @@ REMOVED_FLAG(staging_cgroup); REMOVED_FLAG(status_report_interval); REMOVED_FLAG(status_report_max_retries); REMOVED_FLAG(suppress_unknown_disk_id_warnings); +REMOVED_FLAG(unlock_mt_dop); REMOVED_FLAG(use_krpc); REMOVED_FLAG(use_kudu_kinit); REMOVED_FLAG(use_statestore); diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc index 3a2c470..b43c63e 100644 --- a/be/src/util/backend-gflag-util.cc +++ b/be/src/util/backend-gflag-util.cc @@ -66,8 +66,6 @@ DECLARE_int32(kudu_error_buffer_size); DECLARE_int32(hms_event_polling_interval_s); DECLARE_bool(enable_insert_events); DECLARE_string(authorization_factory_class); -DECLARE_bool(unlock_mt_dop); -DECLARE_bool(mt_dop_auto_fallback); DECLARE_string(ranger_service_type); DECLARE_string(ranger_app_id); DECLARE_string(authorization_provider); @@ -153,8 +151,6 @@ Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) { cfg.__set_enable_insert_events(FLAGS_enable_insert_events); cfg.__set_impala_build_version(::GetDaemonBuildVersion()); cfg.__set_authorization_factory_class(FLAGS_authorization_factory_class); - cfg.__set_unlock_mt_dop(FLAGS_unlock_mt_dop); - cfg.__set_mt_dop_auto_fallback(FLAGS_mt_dop_auto_fallback); cfg.__set_ranger_service_type(FLAGS_ranger_service_type); cfg.__set_ranger_app_id(FLAGS_ranger_app_id); cfg.__set_authorization_provider(FLAGS_authorization_provider); diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh index 98fa122..329b866 100755 --- a/bin/run-all-tests.sh +++ b/bin/run-all-tests.sh @@ -169,11 +169,10 @@ TEST_RET_CODE=0 # Helper function to start Impala cluster. start_impala_cluster() { - # TODO: IMPALA-9812: remove --unlock_mt_dop when it is no longer needed. run-step "Starting Impala cluster" start-impala-cluster.log \ "${IMPALA_HOME}/bin/start-impala-cluster.py" \ --log_dir="${IMPALA_EE_TEST_LOGS_DIR}" \ - ${TEST_START_CLUSTER_ARGS} --impalad_args=--unlock_mt_dop=true + ${TEST_START_CLUSTER_ARGS} } run_ee_tests() { diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift index 017ad18..89ce733 100644 --- a/common/thrift/BackendGflags.thrift +++ b/common/thrift/BackendGflags.thrift @@ -118,7 +118,7 @@ struct TBackendGflags { 47: required string authorization_factory_class - 48: required bool unlock_mt_dop + // REMOVED: 48: required bool unlock_mt_dop 49: required string ranger_service_type @@ -146,7 +146,7 @@ struct TBackendGflags { 61: required string min_privilege_set_for_show_stmts - 62: required bool mt_dop_auto_fallback + // REMOVED: 62: required bool mt_dop_auto_fallback 63: required i32 num_expected_executors diff --git a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java index 4342cf0..2b216f8 100644 --- a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java +++ b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java @@ -30,10 +30,6 @@ public class RuntimeEnv { // Indicates whether this is an environment for testing. private boolean isTestEnv_; - // Whether we should do the same mt_dop validation in frontend tests as in the Impala - // service. - private boolean enableMtDopValidation_; - public RuntimeEnv() { reset(); } @@ -44,13 +40,10 @@ public class RuntimeEnv { public void reset() { numCores_ = Runtime.getRuntime().availableProcessors(); isTestEnv_ = false; - enableMtDopValidation_ = false; } public int getNumCores() { return numCores_; } public void setNumCores(int numCores) { this.numCores_ = numCores; } public void setTestEnv(boolean v) { isTestEnv_ = v; } public boolean isTestEnv() { return isTestEnv_; } - public boolean isMtDopValidationEnabled() { return enableMtDopValidation_; } - public void setEnableMtDopValidation(boolean v) { enableMtDopValidation_ = v; } } diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java index c02cc59..e62f6fe 100644 --- a/fe/src/main/java/org/apache/impala/planner/Planner.java +++ b/fe/src/main/java/org/apache/impala/planner/Planner.java @@ -125,24 +125,6 @@ public class Planner { invertJoins(singleNodePlan, ctx_.isSingleNodeExec()); singleNodePlan = useNljForSingularRowBuilds(singleNodePlan, ctx_.getRootAnalyzer()); - // Parallel plans are not supported by default for plans with table sinks because - // of issues like IMPALA-8125. We only allow such plans if --unlock_mt_dop=true is - // specified. - if (useParallelPlan() - && (!RuntimeEnv.INSTANCE.isTestEnv() - || RuntimeEnv.INSTANCE.isMtDopValidationEnabled()) - && !BackendConfig.INSTANCE.isMtDopUnlocked() - && ctx_.hasTableSink()) { - if (BackendConfig.INSTANCE.mtDopAutoFallback()) { - // Fall back to non-dop mode. This assumes that the mt_dop value is only used - // in the distributed planning process, which should be generally true as long - // as the value isn't cached in any plan nodes. - ctx_.getQueryOptions().setMt_dop(0); - } else { - throw new NotImplementedException("MT_DOP not supported for DML statements."); - } - } - singleNodePlanner.validatePlan(singleNodePlan); if (ctx_.isSingleNodeExec()) { diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java index c1a5264..50ccbce 100644 --- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java +++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java @@ -140,14 +140,6 @@ public class BackendConfig { return "".equals(val) ? null : val; } - public boolean isMtDopUnlocked() { - return backendCfg_.unlock_mt_dop; - } - - public boolean mtDopAutoFallback() { - return backendCfg_.mt_dop_auto_fallback; - } - public boolean recursivelyListPartitions() { return backendCfg_.recursively_list_partitions; } diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java index 78da229..96d3b19 100644 --- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java +++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java @@ -28,7 +28,6 @@ import org.apache.impala.catalog.FeHBaseTable; import org.apache.impala.catalog.HBaseColumn; import org.apache.impala.catalog.Type; import org.apache.impala.common.ImpalaException; -import org.apache.impala.common.RuntimeEnv; import org.apache.impala.datagenerator.HBaseTestDataRegionAssignment; import org.apache.impala.service.Frontend.PlanCtx; import org.apache.impala.testutil.TestUtils; @@ -41,7 +40,6 @@ import org.apache.impala.thrift.TQueryCtx; import org.apache.impala.thrift.TQueryOptions; import org.apache.impala.thrift.TRuntimeFilterMode; import org.junit.Assert; -import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; @@ -682,20 +680,14 @@ public class PlannerTest extends PlannerTestBase { @Test public void testMtDopValidation() { - // Tests that queries supported with mt_dop > 0 produce a parallel plan, or - // throw a NotImplementedException otherwise (e.g. plan has a distributed join). + // Tests that queries planned with mt_dop > 0 produce a parallel plan. + // Since IMPALA-9812 was fixed all plans are supported. Previously some plans + // were rejected. TQueryOptions options = defaultQueryOptions(); options.setMt_dop(3); options.setDisable_hdfs_num_rows_estimate(true); options.setExplain_level(TExplainLevel.EXTENDED); - try { - // Temporarily unset the test env such that unsupported queries with mt_dop > 0 - // throw an exception. Those are otherwise allowed for testing parallel plans. - RuntimeEnv.INSTANCE.setEnableMtDopValidation(true); - runPlannerTestFile("mt-dop-validation", options); - } finally { - RuntimeEnv.INSTANCE.setEnableMtDopValidation(false); - } + runPlannerTestFile("mt-dop-validation", options); } @Test diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test index 6f7695a..e02c5f1 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test @@ -229,7 +229,7 @@ Per-Instance Resources: mem-estimate=26.00MB mem-reservation=8.00KB thread-reser tuple-ids=0 row-size=4B cardinality=unavailable in pipelines: 00(GETNEXT) ==== -# Insert not allowed. +# Insert is allowed. insert into functional_parquet.alltypes partition(year,month) select * from functional_parquet.alltypessmall ---- PLAN @@ -257,9 +257,31 @@ WRITE TO HDFS [functional_parquet.alltypes, OVERWRITE=false, PARTITION-KEYS=(yea tuple-ids=0 row-size=80B cardinality=unavailable in pipelines: 00(GETNEXT) ---- PARALLELPLANS -NotImplementedException: MT_DOP not supported for DML statements. +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=4 +| Per-Instance Resources: mem-estimate=1.01GB mem-reservation=12.09MB thread-reservation=1 +WRITE TO HDFS [functional_parquet.alltypes, OVERWRITE=false, PARTITION-KEYS=(year,month)] +| partitions=4 +| output exprs: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col, year, month +| mem-estimate=1.00GB mem-reservation=0B thread-reservation=0 +| +01:SORT +| order by: year ASC NULLS LAST, month ASC NULLS LAST +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB thread-reservation=0 +| tuple-ids=2 row-size=80B cardinality=unavailable +| in pipelines: 01(GETNEXT), 00(OPEN) +| +00:SCAN HDFS [functional_parquet.alltypessmall, RANDOM] + HDFS partitions=4/4 files=4 size=14.76KB + stored statistics: + table: rows=unavailable size=unavailable + partitions: 0/4 rows=unavailable + columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col + extrapolated-rows=disabled max-scan-range-rows=unavailable + mem-estimate=16.00MB mem-reservation=88.00KB thread-reservation=0 + tuple-ids=0 row-size=80B cardinality=unavailable + in pipelines: 00(GETNEXT) ==== -# CTAS not allowed. +# CTAS is allowed. create table ctas_mt_dop_test as select * from functional_parquet.alltypes ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -270,7 +292,7 @@ WRITE TO HDFS [default.ctas_mt_dop_test, OVERWRITE=false] | mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 | 00:SCAN HDFS [functional_parquet.alltypes] - HDFS partitions=24/24 files=24 size=203.33KB + HDFS partitions=24/24 files=24 size=202.34KB stored statistics: table: rows=unavailable size=unavailable partitions: 0/24 rows=unavailable @@ -280,7 +302,23 @@ WRITE TO HDFS [default.ctas_mt_dop_test, OVERWRITE=false] tuple-ids=0 row-size=80B cardinality=unavailable in pipelines: 00(GETNEXT) ---- PARALLELPLANS -NotImplementedException: MT_DOP not supported for DML statements. +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9 +| Per-Instance Resources: mem-estimate=16.10MB mem-reservation=88.00KB thread-reservation=1 +WRITE TO HDFS [default.ctas_mt_dop_test, OVERWRITE=false] +| partitions=1 +| output exprs: functional_parquet.alltypes.id, functional_parquet.alltypes.bool_col, functional_parquet.alltypes.tinyint_col, functional_parquet.alltypes.smallint_col, functional_parquet.alltypes.int_col, functional_parquet.alltypes.bigint_col, functional_parquet.alltypes.float_col, functional_parquet.alltypes.double_col, functional_parquet.alltypes.date_string_col, functional_parquet.alltypes.string_col, functional_parquet.alltypes.timestamp_col, functional_parquet.alltypes.year, func [...] +| mem-estimate=100.00KB mem-reservation=0B thread-reservation=0 +| +00:SCAN HDFS [functional_parquet.alltypes, RANDOM] + HDFS partitions=24/24 files=24 size=202.34KB + stored statistics: + table: rows=unavailable size=unavailable + partitions: 0/24 rows=unavailable + columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col + extrapolated-rows=disabled max-scan-range-rows=unavailable + mem-estimate=16.00MB mem-reservation=88.00KB thread-reservation=0 + tuple-ids=0 row-size=80B cardinality=unavailable + in pipelines: 00(GETNEXT) ==== # Single-table scan/filter/agg/topn should work. select count(int_col) cnt from functional_parquet.alltypes @@ -309,7 +347,7 @@ PLAN-ROOT SINK | in pipelines: 01(GETNEXT), 00(OPEN) | 00:SCAN HDFS [functional_parquet.alltypes] - HDFS partitions=24/24 files=24 size=203.33KB + HDFS partitions=24/24 files=24 size=202.34KB predicates: id < CAST(10 AS INT) stored statistics: table: rows=unavailable size=unavailable @@ -365,7 +403,7 @@ Per-Instance Resources: mem-estimate=144.00MB mem-reservation=34.02MB thread-res | in pipelines: 00(GETNEXT) | 00:SCAN HDFS [functional_parquet.alltypes, RANDOM] - HDFS partitions=24/24 files=24 size=203.33KB + HDFS partitions=24/24 files=24 size=202.34KB predicates: id < CAST(10 AS INT) stored statistics: table: rows=unavailable size=unavailable @@ -405,7 +443,7 @@ PLAN-ROOT SINK | in pipelines: 01(GETNEXT), 00(OPEN) | 00:SCAN HDFS [functional_parquet.alltypes] - HDFS partitions=24/24 files=24 size=203.33KB + HDFS partitions=24/24 files=24 size=202.34KB predicates: id < CAST(10 AS INT) stored statistics: table: rows=unavailable size=unavailable @@ -454,7 +492,7 @@ Per-Instance Resources: mem-estimate=10.11MB mem-reservation=10.00MB thread-rese F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9 Per-Instance Resources: mem-estimate=16.00MB mem-reservation=16.00KB thread-reservation=1 00:SCAN HDFS [functional_parquet.alltypes, RANDOM] - HDFS partitions=24/24 files=24 size=203.33KB + HDFS partitions=24/24 files=24 size=202.34KB predicates: id < CAST(10 AS INT) stored statistics: table: rows=unavailable size=unavailable @@ -523,14 +561,14 @@ PLAN-ROOT SINK | in pipelines: 00(GETNEXT) | 00:SCAN HDFS [tpch_nested_parquet.customer c] - HDFS partitions=1/1 files=4 size=289.02MB + HDFS partitions=1/1 files=4 size=289.05MB predicates: c_custkey < CAST(10 AS BIGINT), !empty(c.c_orders) predicates on o: !empty(o.o_lineitems), o_orderkey < CAST(5 AS BIGINT) predicates on o_lineitems: l_linenumber < CAST(3 AS INT) stored statistics: - table: rows=150.00K size=289.02MB + table: rows=150.00K size=289.05MB columns missing stats: c_orders - extrapolated-rows=disabled max-scan-range-rows=50.42K + extrapolated-rows=disabled max-scan-range-rows=50.72K parquet statistics predicates: c_custkey < CAST(10 AS BIGINT) parquet statistics predicates on o: o_orderkey < CAST(5 AS BIGINT) parquet statistics predicates on o_lineitems: l_linenumber < CAST(3 AS INT) @@ -599,14 +637,14 @@ Per-Instance Resources: mem-estimate=104.00MB mem-reservation=104.00MB thread-re | in pipelines: 00(GETNEXT) | 00:SCAN HDFS [tpch_nested_parquet.customer c, RANDOM] - HDFS partitions=1/1 files=4 size=289.02MB + HDFS partitions=1/1 files=4 size=289.05MB predicates: c_custkey < CAST(10 AS BIGINT), !empty(c.c_orders) predicates on o: !empty(o.o_lineitems), o_orderkey < CAST(5 AS BIGINT) predicates on o_lineitems: l_linenumber < CAST(3 AS INT) stored statistics: - table: rows=150.00K size=289.02MB + table: rows=150.00K size=289.05MB columns missing stats: c_orders - extrapolated-rows=disabled max-scan-range-rows=50.42K + extrapolated-rows=disabled max-scan-range-rows=50.72K parquet statistics predicates: c_custkey < CAST(10 AS BIGINT) parquet statistics predicates on o: o_orderkey < CAST(5 AS BIGINT) parquet statistics predicates on o_lineitems: l_linenumber < CAST(3 AS INT) @@ -664,13 +702,13 @@ PLAN-ROOT SINK | in pipelines: 00(GETNEXT) | 00:SCAN HDFS [tpch_nested_parquet.customer c] - HDFS partitions=1/1 files=4 size=289.02MB + HDFS partitions=1/1 files=4 size=289.05MB predicates: !empty(c.c_orders), !empty(c.c_orders) predicates on o1: o1.o_orderkey < CAST(5 AS BIGINT) stored statistics: - table: rows=150.00K size=289.02MB + table: rows=150.00K size=289.05MB columns missing stats: c_orders, c_orders - extrapolated-rows=disabled max-scan-range-rows=50.42K + extrapolated-rows=disabled max-scan-range-rows=50.72K parquet statistics predicates on o1: o1.o_orderkey < CAST(5 AS BIGINT) parquet dictionary predicates on o1: o1.o_orderkey < CAST(5 AS BIGINT) mem-estimate=88.00MB mem-reservation=16.00MB thread-reservation=0 @@ -726,13 +764,13 @@ Per-Instance Resources: mem-estimate=89.94MB mem-reservation=17.94MB thread-rese | in pipelines: 00(GETNEXT) | 00:SCAN HDFS [tpch_nested_parquet.customer c, RANDOM] - HDFS partitions=1/1 files=4 size=289.02MB + HDFS partitions=1/1 files=4 size=289.05MB predicates: !empty(c.c_orders), !empty(c.c_orders) predicates on o1: o1.o_orderkey < CAST(5 AS BIGINT) stored statistics: - table: rows=150.00K size=289.02MB + table: rows=150.00K size=289.05MB columns missing stats: c_orders, c_orders - extrapolated-rows=disabled max-scan-range-rows=50.42K + extrapolated-rows=disabled max-scan-range-rows=50.72K parquet statistics predicates on o1: o1.o_orderkey < CAST(5 AS BIGINT) parquet dictionary predicates on o1: o1.o_orderkey < CAST(5 AS BIGINT) mem-estimate=88.00MB mem-reservation=16.00MB thread-reservation=0 diff --git a/testdata/workloads/functional-query/queries/QueryTest/mt-dop-auto-fallback.test b/testdata/workloads/functional-query/queries/QueryTest/mt-dop-auto-fallback.test deleted file mode 100644 index 6b601ba..0000000 --- a/testdata/workloads/functional-query/queries/QueryTest/mt-dop-auto-fallback.test +++ /dev/null @@ -1,31 +0,0 @@ -==== ----- QUERY -# IMPALA-9099: mt_dop is now allowed for joins, so we should run with mt_dop. -select min(l_orderkey), min(p_name) -from tpch.lineitem join tpch.part on l_partkey = p_partkey; ----- RESULTS -1,'almond antique blue royal burnished' ----- TYPES -BIGINT,STRING ----- RUNTIME_PROFILE -row_regex:.*Query Options \(set by configuration and planner\):.*MT_DOP=4.* -==== ----- QUERY -# IMPALA-9099: mt_dop is now allowed for joins, so we should run with mt_dop. -select min(l_orderkey), min(p_name) -from tpch_kudu.lineitem join tpch_kudu.part on l_partkey = p_partkey; ----- RESULTS -1,'almond antique blue royal burnished' ----- TYPES -BIGINT,STRING ----- RUNTIME_PROFILE -row_regex:.*Query Options \(set by configuration and planner\):.*MT_DOP=4.* -==== ----- QUERY -create table tmp as -select * from functional.alltypes ----- RUNTIME_PROFILE -row_regex:.*Query Options \(set by configuration and planner\):.*MT_DOP=0.* -row_regex:.*All 3 execution backends \(3 fragment instances\) started.* -row_regex:.*NumScannerThreadsStarted.* -==== diff --git a/tests/custom_cluster/test_mt_dop.py b/tests/custom_cluster/test_mt_dop.py index fd48e37..098df28 100644 --- a/tests/custom_cluster/test_mt_dop.py +++ b/tests/custom_cluster/test_mt_dop.py @@ -46,26 +46,7 @@ class TestMtDopFlags(CustomClusterTestSuite): def add_test_dimensions(cls): super(TestMtDopFlags, cls).add_test_dimensions() - @pytest.mark.execute_serially - @CustomClusterTestSuite.with_args(impalad_args="--mt_dop_auto_fallback=true") - @SkipIfNotHdfsMinicluster.tuned_for_minicluster - def test_mt_dop_fallback(self, vector, unique_database): - """Test inserts fall back to non-mt_dop correctly. - TODO: IMPALA-8966: remove this test when mt_dop is enabled across the board. - """ - vector = deepcopy(vector) - vector.get_value('exec_option')['mt_dop'] = 4 - # Targeted test case that verifies that the fallback actually switches to the - # non-mt-dop plans. - self.run_test_case('QueryTest/mt-dop-auto-fallback', vector, use_db=unique_database) - - # Check that the join and insert plans work as expected. - self.run_test_case('QueryTest/joins', vector, use_db="functional_parquet") - self.run_test_case('QueryTest/insert', vector, unique_database, - test_file_vars={'$ORIGINAL_DB': CustomClusterTestSuite - .get_db_name_from_format(vector.get_value('table_format'))}) - - @CustomClusterTestSuite.with_args(impalad_args="--unlock_mt_dop=true", cluster_size=1) + @CustomClusterTestSuite.with_args(cluster_size=1) def test_mt_dop_runtime_filters_one_node(self, vector): """Runtime filter tests, which assume 3 fragment instances, can also be run on a single node cluster to test multiple filter sources/destinations per backend."""