[
https://issues.apache.org/jira/browse/IMPALA-14918?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18075884#comment-18075884
]
Quanlong Huang commented on IMPALA-14918:
-----------------------------------------
Submitted a patch to print the stdout & stderr of the subprocess when it fails.
So we know what's wrong next time this happens.
[https://gerrit.cloudera.org/c/24231/]
> TestAdmissionController.test_pool_mem_limit_configs failed to start the
> cluster
> -------------------------------------------------------------------------------
>
> Key: IMPALA-14918
> URL: https://issues.apache.org/jira/browse/IMPALA-14918
> Project: IMPALA
> Issue Type: Bug
> Reporter: Quanlong Huang
> Assignee: Quanlong Huang
> Priority: Critical
>
> Saw this failed in a downstream build:
> {noformat}
> custom_cluster.test_admission_controller.TestAdmissionController.test_pool_mem_limit_configs["table_format:
> text/none" | "protocol: hs2" | "exec_option: {'abort_on_error': 1,
> 'batch_size': 0, 'disable_codegen': False, 'disable_codegen_rows_threshold':
> 5000, 'exec_single_node_rows_threshold': 0, 'num_nodes': 0, 'test_replan':
> 1}"] (from pytest){noformat}
> The error message is
> {noformat}
> failed on setup with "subprocess.CalledProcessError: Command ... returned
> non-zero exit status 1."{noformat}
> But I can't find the "*-out.log" files to see what's wrong there. The dir
> {{logs/custom_cluster_tests/TestAdmissionController/test_pool_mem_limit_configs}}
> is empty.
> h3. Stacktrace
> {noformat}
> custom_cluster/test_admission_controller.py:490: in setup_method
> super(TestAdmissionController, self).setup_method(method)
> __class__ = <class
> 'tests.custom_cluster.test_admission_controller.TestAdmissionController'>
> method = <bound method
> TestAdmissionController.test_pool_mem_limit_configs of
> <tests.custom_cluster.test_admission_controller.TestAdmissionController
> object at 0x7f377aac8d50>>
> self =
> <tests.custom_cluster.test_admission_controller.TestAdmissionController
> object at 0x7f377aac8d50>
> common/custom_cluster_test_suite.py:418: in setup_method
> self.cluster_setup(method.__dict__)
> __class__ = <class
> 'tests.common.custom_cluster_test_suite.CustomClusterTestSuite'>
> method = <bound method
> TestAdmissionController.test_pool_mem_limit_configs of
> <tests.custom_cluster.test_admission_controller.TestAdmissionController
> object at 0x7f377aac8d50>>
> self =
> <tests.custom_cluster.test_admission_controller.TestAdmissionController
> object at 0x7f377aac8d50>
> common/custom_cluster_test_suite.py:409: in cluster_setup
> raise e
> __class__ = <class
> 'tests.common.custom_cluster_test_suite.CustomClusterTestSuite'>
> arg = 'jvm_args'
> args = {'impalad_args': '-vmodule admission-controller=3
> -fair_scheduler_allocation_path /data/jenkins/workspace/impala-cdw-m...args':
> '-statestore_heartbeat_frequency_ms=100
> -statestore_priority_update_frequency_ms=100', 'with_args_method': True}
> cls = <class
> 'tests.custom_cluster.test_admission_controller.TestAdmissionController'>
> cluster_args = ['--impalad_args=-vmodule admission-controller=3
> -fair_scheduler_allocation_path
> /data/jenkins/workspace/impala-cdw-ma...e_heartbeat_frequency_ms=100
> -statestore_priority_update_frequency_ms=100 ',
> '--impalad_args=--default_query_options=']
> cluster_size = 3
> custom_class_path_val = ''
> disable_log_buffering = False
> impala_daemons = ['impalad_args', 'state_store_args',
> 'catalogd_args', 'admissiond_args']
> kwargs = {'cluster_size': 3, 'default_query_options': None,
> 'expected_num_impalads': 3, 'force_restart': False, ...}
> num_coordinators = 3
> use_exclusive_coordinators = False
> val = ''
> common/custom_cluster_test_suite.py:391: in cluster_setup
> cls._start_impala_cluster(cluster_args, **kwargs)
> __class__ = <class
> 'tests.common.custom_cluster_test_suite.CustomClusterTestSuite'>
> arg = 'jvm_args'
> args = {'impalad_args': '-vmodule admission-controller=3
> -fair_scheduler_allocation_path /data/jenkins/workspace/impala-cdw-m...args':
> '-statestore_heartbeat_frequency_ms=100
> -statestore_priority_update_frequency_ms=100', 'with_args_method': True}
> cls = <class
> 'tests.custom_cluster.test_admission_controller.TestAdmissionController'>
> cluster_args = ['--impalad_args=-vmodule admission-controller=3
> -fair_scheduler_allocation_path
> /data/jenkins/workspace/impala-cdw-ma...e_heartbeat_frequency_ms=100
> -statestore_priority_update_frequency_ms=100 ',
> '--impalad_args=--default_query_options=']
> cluster_size = 3
> custom_class_path_val = ''
> disable_log_buffering = False
> impala_daemons = ['impalad_args', 'state_store_args',
> 'catalogd_args', 'admissiond_args']
> kwargs = {'cluster_size': 3, 'default_query_options': None,
> 'expected_num_impalads': 3, 'force_restart': False, ...}
> num_coordinators = 3
> use_exclusive_coordinators = False
> val = ''
> common/custom_cluster_test_suite.py:684: in _start_impala_cluster
> check_call(cmd + options, close_fds=True)
> add_executors = False
> add_impalads = False
> cls = <class
> 'tests.custom_cluster.test_admission_controller.TestAdmissionController'>
> cluster_size = 3
> cmd =
> ['/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/bin/start-impala-cluster.py',
>
> '--state_stor...core-asan/repos/Impala/logs/custom_cluster_tests/TestAdmissionController/test_pool_mem_limit_configs',
> '--log_level=1']
> cmd_str =
> "/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/bin/start-impala-cluster.py
> '--state_store_a...tore_heartbeat_frequency_ms=100
> -statestore_priority_update_frequency_ms=100 '
> --impalad_args=--default_query_options="
> default_query_option_kvs = []
> default_query_options = None
> expected_num_impalads = 3
> expected_subscribers = 0
> force_restart = False
> ignore_pid_on_log_rotation = False
> impala_base_log_dir =
> '/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/logs/custom_cluster_tests'
> impala_log_dir = None
> impala_log_dir_per_test =
> '/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/logs/custom_cluster_tests/TestAdmissionController/test_pool_mem_limit_configs'
> impalad_timeout_s = 60
> log_level = 1
> log_symlinks = False
> num_coordinators = 3
> options = ['--impalad_args=-vmodule admission-controller=3
> -fair_scheduler_allocation_path
> /data/jenkins/workspace/impala-cdw-ma...e_heartbeat_frequency_ms=100
> -statestore_priority_update_frequency_ms=100 ',
> '--impalad_args=--default_query_options=']
> statestored_timeout_s = 60
> use_exclusive_coordinators = False
> wait_for_backends = True
> /data/jenkins/workspace/impala-cdw-master-staging-core-asan/Impala-Toolchain/toolchain-packages-gcc10.4.0/python-3.11.14/lib/python3.11/subprocess.py:413:
> in check_call
> raise CalledProcessError(retcode, cmd)
> E subprocess.CalledProcessError: Command
> '['/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/bin/start-impala-cluster.py',
> '--state_store_args=--statestore_update_frequency_ms=50
> --statestore_priority_update_frequency_ms=50
> --statestore_heartbeat_frequency_ms=50', '--cluster_size=3',
> '--num_coordinators=3',
> '--log_dir=/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/logs/custom_cluster_tests/TestAdmissionController/test_pool_mem_limit_configs',
> '--log_level=1', '--impalad_args=-vmodule admission-controller=3
> -fair_scheduler_allocation_path
> /data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/fe/src/test/resources/mem-limit-test-fair-scheduler.xml
> -llama_site_path
> /data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/fe/src/test/resources/mem-limit-test-llama-site.xml
> ', '--state_store_args=-statestore_heartbeat_frequency_ms=100
> -statestore_priority_update_frequency_ms=100 ',
> '--impalad_args=--default_query_options=']' returned non-zero exit status 1.
> cmd =
> ['/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/bin/start-impala-cluster.py',
>
> '--state_stor...asan/repos/Impala/logs/custom_cluster_tests/TestAdmissionController/test_pool_mem_limit_configs',
> '--log_level=1', ...]
> kwargs = {'close_fds': True}
> popenargs =
> (['/data/jenkins/workspace/impala-cdw-master-staging-core-asan/repos/Impala/bin/start-impala-cluster.py',
>
> '--state_sto...an/repos/Impala/logs/custom_cluster_tests/TestAdmissionController/test_pool_mem_limit_configs',
> '--log_level=1', ...],)
> retcode = 1{noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
