[
https://issues.apache.org/jira/browse/IMPALA-10559?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17299942#comment-17299942
]
Quanlong Huang commented on IMPALA-10559:
-----------------------------------------
I see a failure on the same query of the same test
(test_with_unlimited_scratch_limit). It's from a test of an unrelated branch.
Logs in impalad.INFO:
{code:java}
I0311 18:32:48.147064 18842 impala-beeswax-server.cc:541]
TClientRequest.queryOptions: TQueryOptions {
01: abort_on_error (bool) = true,
02: max_errors (i32) = 100,
03: disable_codegen (bool) = false,
04: batch_size (i32) = 0,
05: num_nodes (i32) = 0,
06: max_scan_range_length (i64) = 0,
07: num_scanner_threads (i32) = 0,
11: debug_action (string) = "",
12: mem_limit (i64) = 0,
15: hbase_caching (i32) = 0,
16: hbase_cache_blocks (bool) = false,
17: parquet_file_size (i64) = 0,
18: explain_level (i32) = 1,
19: sync_ddl (bool) = false,
24: disable_outermost_topn (bool) = false,
26: query_timeout_s (i32) = 0,
27: buffer_pool_limit (i64) = 33554432,
28: appx_count_distinct (bool) = false,
29: disable_unsafe_spills (bool) = false,
31: exec_single_node_rows_threshold (i32) = 0,
32: optimize_partition_key_scans (bool) = false,
33: replica_preference (i32) = 0,
34: schedule_random_replica (bool) = false,
36: disable_streaming_preaggregations (bool) = false,
37: runtime_filter_mode (i32) = 2,
38: runtime_bloom_filter_size (i32) = 1048576,
39: runtime_filter_wait_time_ms (i32) = 0,
40: disable_row_runtime_filtering (bool) = false,
41: max_num_runtime_filters (i32) = 10,
42: parquet_annotate_strings_utf8 (bool) = false,
43: parquet_fallback_schema_resolution (i32) = 0,
45: s3_skip_insert_staging (bool) = true,
46: runtime_filter_min_size (i32) = 1048576,
47: runtime_filter_max_size (i32) = 16777216,
48: prefetch_mode (i32) = 1,
49: strict_mode (bool) = false,
50: scratch_limit (i64) = -1,
51: enable_expr_rewrites (bool) = true,
52: decimal_v2 (bool) = true,
53: parquet_dictionary_filtering (bool) = true,
54: parquet_array_resolution (i32) = 0,
55: parquet_read_statistics (bool) = true,
56: default_join_distribution_mode (i32) = 0,
57: disable_codegen_rows_threshold (i32) = 5000,
58: default_spillable_buffer_size (i64) = 2097152,
59: min_spillable_buffer_size (i64) = 65536,
60: max_row_size (i64) = 524288,
61: idle_session_timeout (i32) = 0,
62: compute_stats_min_sample_size (i64) = 1073741824,
63: exec_time_limit_s (i32) = 0,
64: shuffle_distinct_exprs (bool) = true,
65: max_mem_estimate_for_admission (i64) = 0,
66: thread_reservation_limit (i32) = 3000,
67: thread_reservation_aggregate_limit (i32) = 0,
68: kudu_read_mode (i32) = 0,
69: allow_erasure_coded_files (bool) = false,
70: timezone (string) = "",
71: scan_bytes_limit (i64) = 0,
72: cpu_limit_s (i64) = 0,
73: topn_bytes_limit (i64) = 536870912,
74: client_identifier (string) =
"query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_unlimited_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_rows_th",
75: resource_trace_ratio (double) = 0,
76: num_remote_executor_candidates (i32) = 3,
77: num_rows_produced_limit (i64) = 0,
78: planner_testcase_mode (bool) = false,
79: default_file_format (i32) = 0,
80: parquet_timestamp_type (i32) = 0,
81: parquet_read_page_index (bool) = true,
82: parquet_write_page_index (bool) = true,
84: disable_hdfs_num_rows_estimate (bool) = false,
86: spool_query_results (bool) = true,
87: default_transactional_type (i32) = 0,
88: statement_expression_limit (i32) = 250000,
89: max_statement_length_bytes (i32) = 16777216,
90: disable_data_cache (bool) = false,
91: max_result_spooling_mem (i64) = 104857600,
92: max_spilled_result_spooling_mem (i64) = 1073741824,
93: disable_hbase_num_rows_estimate (bool) = false,
94: fetch_rows_timeout_ms (i64) = 10000,
95: now_string (string) = "",
96: parquet_object_store_split_size (i64) = 268435456,
97: mem_limit_executors (i64) = 0,
98: broadcast_bytes_limit (i64) = 34359738368,
99: preagg_bytes_limit (i64) = -1,
100: enable_cnf_rewrites (bool) = true,
101: max_cnf_exprs (i32) = 200,
102: kudu_snapshot_read_timestamp_micros (i64) = 0,
103: retry_failed_queries (bool) = false,
104: enabled_runtime_filter_types (i32) = 3,
105: async_codegen (bool) = false,
106: enable_distinct_semi_join_optimization (bool) = true,
107: sort_run_bytes_limit (i64) = -1,
108: max_fs_writers (i32) = 0,
109: refresh_updated_hms_partitions (bool) = false,
110: spool_all_results_for_retries (bool) = true,
112: use_local_tz_for_unix_timestamp_conversions (bool) = false,
113: convert_legacy_hive_parquet_utc_timestamps (bool) = false,
114: enable_outer_join_to_inner_transformation (bool) = false,
115: targeted_kudu_scan_range_length (i64) = -1,
116: report_skew_limit (double) = 1,
117: optimize_simple_limit (bool) = false,
118: use_dop_for_costing (bool) = true,
119: broadcast_to_partition_factor (double) = 1,
120: join_rows_produced_limit (i64) = 0,
121: utf8_mode (bool) = false,
122: analytic_rank_pushdown_threshold (i64) = 1000,
123: minmax_filter_threshold (double) = 0,
124: minmax_filtering_level (i32) = 1,
}
I0311 18:32:48.147261 18842 impala-server.cc:1191] Found local timezone "UTC".
I0311 18:32:48.151219 18842 impala-server.cc:1248]
c94da7291d86eab1:c681355400000000] Registered query
query_id=c94da7291d86eab1:c681355400000000
session_id=be4927bae7e0262b:877b42a046000a97
I0311 18:32:48.151445 18842 Frontend.java:1598]
c94da7291d86eab1:c681355400000000] Analyzing query: select o_orderdate,
o_custkey, o_comment
from tpch.orders
order by o_orderdate db: default
I0311 18:32:48.152427 18842 BaseAuthorizationChecker.java:110]
c94da7291d86eab1:c681355400000000] Authorization check took 0 ms
I0311 18:32:48.152485 18842 Frontend.java:1640]
c94da7291d86eab1:c681355400000000] Analysis and authorization finished.
I0311 18:32:48.153060 412 query-exec-mgr.cc:213] ReleaseQueryState(): deleted
query_id=7d4478947fa3bc66:182ad15200000000
I0311 18:32:48.158035 111 impala-server.cc:1495] Invalid or unknown query
handle: 7d4478947fa3bc66:182ad15200000000.
I0311 18:32:48.158064 111 control-service.cc:179] ReportExecStatus():
Received report for unknown query ID (probably closed or cancelled):
7d4478947fa3bc66:182ad15200000000 remote host=172.18.0.6:58716
I0311 18:32:48.163488 18855 admission-controller.cc:1696]
c94da7291d86eab1:c681355400000000] Trying to admit
id=c94da7291d86eab1:c681355400000000 in pool_name=root.default
executor_group_name=default per_host_mem_estimate=308.18 MB
dedicated_coord_mem_estimate=220.18 MB max_requests=-1 max_queued=200
max_mem=29.30 GB
I0311 18:32:48.163555 18855 admission-controller.cc:1704]
c94da7291d86eab1:c681355400000000] Stats: agg_num_running=1, agg_num_queued=0,
agg_mem_reserved=1.48 GB, local_host(local_mem_admitted=900.00 MB,
num_admitted_running=1, num_queued=0, backend_mem_reserved=608.18 MB,
topN_query_stats: queries=[7d4478947fa3bc66:182ad15200000000,
3646bbe46a6f1cb6:29f9f12700000000], total_mem_consumed=86.65 MB,
fraction_of_pool_total_mem=1; pool_level_stats: num_running=2, min=32.56 MB,
max=54.09 MB, pool_total_mem=86.65 MB, average_per_query=43.33 MB)
I0311 18:32:48.163615 18855 admission-controller.cc:1212]
c94da7291d86eab1:c681355400000000] Admitting query
id=c94da7291d86eab1:c681355400000000
I0311 18:32:48.163671 18855 impala-server.cc:2074]
c94da7291d86eab1:c681355400000000] Registering query locations
I0311 18:32:48.163700 18855 coordinator.cc:150]
c94da7291d86eab1:c681355400000000] Exec()
query_id=c94da7291d86eab1:c681355400000000 stmt=select o_orderdate, o_custkey,
o_comment
from tpch.orders
order by o_orderdate
I0311 18:32:48.164074 18855 coordinator.cc:474]
c94da7291d86eab1:c681355400000000] starting execution on 2 backends for
query_id=c94da7291d86eab1:c681355400000000
I0311 18:32:48.164805 111 control-service.cc:148]
c94da7291d86eab1:c681355400000000] ExecQueryFInstances():
query_id=c94da7291d86eab1:c681355400000000 coord=172.18.0.4:27000 #instances=2
I0311 18:32:48.165612 18855 coordinator.cc:533]
c94da7291d86eab1:c681355400000000] started execution on 2 backends for
query_id=c94da7291d86eab1:c681355400000000
I0311 18:32:48.165647 18858 query-state.cc:948]
c94da7291d86eab1:c681355400000001] Executing instance.
instance_id=c94da7291d86eab1:c681355400000001 fragment_idx=1
per_fragment_instance_idx=0 coord_state_idx=0 #in-flight=4
I0311 18:32:48.168483 18859 query-state.cc:948]
c94da7291d86eab1:c681355400000000] Executing instance.
instance_id=c94da7291d86eab1:c681355400000000 fragment_idx=0
per_fragment_instance_idx=0 coord_state_idx=0 #in-flight=5
I0311 18:32:53.759224 18842 impala-beeswax-server.cc:215]
get_results_metadata(): query_id=c94da7291d86eab1:c681355400000000
I0311 18:32:54.835614 18672 query-state.cc:957]
3646bbe46a6f1cb6:29f9f12700000007] Instance completed.
instance_id=3646bbe46a6f1cb6:29f9f12700000007 #in-flight=4 status=OK
I0311 18:32:54.836534 18674 query-state.cc:957]
3646bbe46a6f1cb6:29f9f12700000003] Instance completed.
instance_id=3646bbe46a6f1cb6:29f9f12700000003 #in-flight=3 status=OK
I0311 18:32:55.284286 130 krpc-data-stream-mgr.cc:427] Reduced stream ID
cache from 73 items, to 67, eviction took: 0
F0311 18:32:55.374138 18859 reservation-tracker.cc:436]
c94da7291d86eab1:c681355400000000] Check failed: used_reservation_.Load() +
child_reservations_.Load() <= reservation_.Load() (6291456 vs. 4194304) 4194304
+ 2097152 > 4194304 {code}
Stacktrace in impalad.ERROR
{code:java}
F0311 18:32:55.374138 18859 reservation-tracker.cc:436]
c94da7291d86eab1:c681355400000000] Check failed: used_reservation_.Load() +
child_reservations_.Load() <= reservation_.Load() (6291456 vs. 4194304) 4194304
+ 2097152 > 4194304
*** Check failure stack trace: ***
@ 0x548aefc google::LogMessage::Fail()
@ 0x548c7ec google::LogMessage::SendToLog()
@ 0x548a85a google::LogMessage::Flush()
@ 0x548e458 google::LogMessageFatal::~LogMessageFatal()
@ 0x2960bea impala::ReservationTracker::CheckConsistency()
@ 0x295fd06 impala::ReservationTracker::TransferReservationTo()
@ 0x2953337 impala::BufferPool::ClientHandle::SaveReservation()
@ 0x2f2133a impala::BufferedTupleStream::NextReadPage()
@ 0x2f22699 impala::BufferedTupleStream::UnpinStream()
@ 0x2f8f8dd impala::SpillableRowBatchQueue::AddBatch()
@ 0x2b6a1a8 impala::BufferedPlanRootSink::Send()
@ 0x24a6983 impala::FragmentInstanceState::ExecInternal()
@ 0x24a2989 impala::FragmentInstanceState::Exec()
@ 0x23e659f impala::QueryState::ExecFInstance()
@ 0x23e49a1 _ZZN6impala10QueryState15StartFInstancesEvENKUlvE_clEv
@ 0x23e956e
_ZN5boost6detail8function26void_function_obj_invoker0IZN6impala10QueryState15StartFInstancesEvEUlvE_vE6invokeERNS1_15function_bufferE
@ 0x228e471 boost::function0<>::operator()()
@ 0x2921a91 impala::Thread::SuperviseThread()
@ 0x292a3f2 boost::_bi::list5<>::operator()<>()
@ 0x292a316 boost::_bi::bind_t<>::operator()()
@ 0x292a2d7 boost::detail::thread_data<>::run()
@ 0x4191b81 thread_proxy
@ 0x7f0d4d9c36b9 start_thread
@ 0x7f0d4a4ef4dc clone {code}
> TestScratchLimit seems flaky
> ----------------------------
>
> Key: IMPALA-10559
> URL: https://issues.apache.org/jira/browse/IMPALA-10559
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Affects Versions: Impala 4.0
> Reporter: Yida Wu
> Priority: Blocker
> Labels: broken-build
>
>
> The TestScratchLimit testcase runs fine alone. But if runs TestScratchDir
> first, then runs TestScratchLimit, it will always fail as follow shows unless
> restarts the services (e.g. using "./buildall.sh -noclean -notests -format
> -start_minicluster -start_impala_cluster").
> {code:java}
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_low_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> query_test/test_scratch_limit.py::TestScratchLimit::test_without_specifying_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_high_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_zero_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_unlimited_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_zero_scratch_limit_no_memory_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw1] PASSED
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_zero_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw5] FAILED
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_low_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw4] FAILED
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_high_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw2] FAILED
> query_test/test_scratch_limit.py::TestScratchLimit::test_without_specifying_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw3] FAILED
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_unlimited_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw0] PASSED
> query_test/test_scratch_limit.py::TestScratchLimit::test_with_zero_scratch_limit_no_memory_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> ------------------------------------------- generated xml file:
> /home/yida/Impala/logs/ee_tests/results/TEST-impala-parallel.xml
> --------------------------------------------
> ==========================================================================
> short test summary info
> ==========================================================================
> FAIL
> query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_low_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> FAIL
> query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_high_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> FAIL
> query_test/test_scratch_limit.py::TestScratchLimit::()::test_without_specifying_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> FAIL
> query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_unlimited_scratch_limit[protocol:
> beeswax | exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> =================================================================================
> FAILURES
> ==================================================================================
> TestScratchLimit.test_with_low_scratch_limit[protocol: beeswax |
> exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw5] linux2 -- Python 2.7.16
> /home/yida/Impala/bin/../infra/python/env-gcc7.5.0/bin/python
> query_test/test_scratch_limit.py:90: in test_with_low_scratch_limit
> assert expected_error % scratch_limit_in_bytes in str(e)
> E assert ('Scratch space limit of %s bytes exceeded' % 25165824) in
> 'ImpalaBeeswaxException:\n Query aborted:Could not create files in any
> configured scratch directories (--scratch_dirs=...ous errors that may have
> prevented creating or writing scratch files. The following directories were
> at capacity: \n\n'
> E + where 'ImpalaBeeswaxException:\n Query aborted:Could not create files in
> any configured scratch directories (--scratch_dirs=...ous errors that may
> have prevented creating or writing scratch files. The following directories
> were at capacity: \n\n' = str(ImpalaBeeswaxException())
> ---------------------------------------------------------------------------
> Captured stderr setup
> ---------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_low_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_rows_threshol;
> -- connecting to: localhost:21000
> -- connecting to localhost:21050 with impyla
> -- 2021-03-03 07:12:08,115 INFO MainThread: Closing active operation
> -- connecting to localhost:28000 with impyla
> -- 2021-03-03 07:12:08,140 INFO MainThread: Closing active operation
> ---------------------------------------------------------------------------
> Captured stderr call
> ----------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_low_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_rows_threshol;
> SET scratch_limit=24m;
> SET batch_size=0;
> SET num_nodes=0;
> SET disable_codegen_rows_threshold=5000;
> SET disable_codegen=False;
> SET abort_on_error=1;
> SET exec_single_node_rows_threshold=0;
> SET buffer_pool_limit=32m;
> -- executing against localhost:21000
> select o_orderdate, o_custkey, o_comment
> from tpch.orders
> order by o_orderdate
> ;
> -- 2021-03-03 07:12:08,329 INFO MainThread: Started query
> f44b4a50bec6b10d:e45a053d00000000
> TestScratchLimit.test_with_high_scratch_limit[protocol: beeswax |
> exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw4] linux2 -- Python 2.7.16
> /home/yida/Impala/bin/../infra/python/env-gcc7.5.0/bin/python
> query_test/test_scratch_limit.py:74: in test_with_high_scratch_limit
> self.execute_query_expect_success(self.client, self.spilling_sort_query,
> exec_option)
> common/impala_test_suite.py:809: in wrapper
> return function(*args, **kwargs)
> common/impala_test_suite.py:817: in execute_query_expect_success
> result = cls.__execute_query(impalad_client, query, query_options, user)
> common/impala_test_suite.py:918: in __execute_query
> return impalad_client.execute(query, user=user)
> common/impala_connection.py:205: in execute
> return self.__beeswax_client.execute(sql_stmt, user=user)
> beeswax/impala_beeswax.py:187: in execute
> handle = self.__execute_query(query_string.strip(), user=user)
> beeswax/impala_beeswax.py:365: in __execute_query
> self.wait_for_finished(handle)
> beeswax/impala_beeswax.py:386: in wait_for_finished
> raise ImpalaBeeswaxException("Query aborted:" + error_log, None)
> E ImpalaBeeswaxException: ImpalaBeeswaxException:
> E Query aborted:Could not create files in any configured scratch directories
> (--scratch_dirs=) on backend 'yida-OptiPlex-7060:27001'. 0 of scratch is
> currently in use by this Impala Daemon (0 by this query). See logs for
> previous errors that may have prevented creating or writing scratch files.
> The following directories were at capacity:
> ---------------------------------------------------------------------------
> Captured stderr setup
> ---------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_high_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_rows_thresho;
> -- connecting to: localhost:21000
> -- connecting to localhost:21050 with impyla
> -- 2021-03-03 07:12:08,114 INFO MainThread: Closing active operation
> -- connecting to localhost:28000 with impyla
> -- 2021-03-03 07:12:08,136 INFO MainThread: Closing active operation
> ---------------------------------------------------------------------------
> Captured stderr call
> ----------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_high_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_rows_thresho;
> SET scratch_limit=500m;
> SET batch_size=0;
> SET num_nodes=0;
> SET disable_codegen_rows_threshold=5000;
> SET disable_codegen=False;
> SET abort_on_error=1;
> SET exec_single_node_rows_threshold=0;
> SET buffer_pool_limit=32m;
> -- executing against localhost:21000
> select o_orderdate, o_custkey, o_comment
> from tpch.orders
> order by o_orderdate
> ;
> -- 2021-03-03 07:12:08,329 INFO MainThread: Started query
> 814fccc9c1242331:83738b0800000000
> TestScratchLimit.test_without_specifying_scratch_limit[protocol: beeswax |
> exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw2] linux2 -- Python 2.7.16
> /home/yida/Impala/bin/../infra/python/env-gcc7.5.0/bin/python
> query_test/test_scratch_limit.py:118: in test_without_specifying_scratch_limit
> self.execute_query_expect_success(self.client, self.spilling_sort_query,
> exec_option)
> common/impala_test_suite.py:809: in wrapper
> return function(*args, **kwargs)
> common/impala_test_suite.py:817: in execute_query_expect_success
> result = cls.__execute_query(impalad_client, query, query_options, user)
> common/impala_test_suite.py:918: in __execute_query
> return impalad_client.execute(query, user=user)
> common/impala_connection.py:205: in execute
> return self.__beeswax_client.execute(sql_stmt, user=user)
> beeswax/impala_beeswax.py:187: in execute
> handle = self.__execute_query(query_string.strip(), user=user)
> beeswax/impala_beeswax.py:365: in __execute_query
> self.wait_for_finished(handle)
> beeswax/impala_beeswax.py:386: in wait_for_finished
> raise ImpalaBeeswaxException("Query aborted:" + error_log, None)
> E ImpalaBeeswaxException: ImpalaBeeswaxException:
> E Query aborted:Could not create files in any configured scratch directories
> (--scratch_dirs=) on backend 'yida-OptiPlex-7060:27002'. 0 of scratch is
> currently in use by this Impala Daemon (0 by this query). See logs for
> previous errors that may have prevented creating or writing scratch files.
> The following directories were at capacity:
> ---------------------------------------------------------------------------
> Captured stderr setup
> ---------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_without_specifying_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_row;
> -- connecting to: localhost:21000
> -- connecting to localhost:21050 with impyla
> -- 2021-03-03 07:12:08,114 INFO MainThread: Closing active operation
> -- connecting to localhost:28000 with impyla
> -- 2021-03-03 07:12:08,137 INFO MainThread: Closing active operation
> ---------------------------------------------------------------------------
> Captured stderr call
> ----------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_without_specifying_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_row;
> SET batch_size=0;
> SET num_nodes=0;
> SET disable_codegen_rows_threshold=5000;
> SET disable_codegen=False;
> SET abort_on_error=1;
> SET exec_single_node_rows_threshold=0;
> SET buffer_pool_limit=32m;
> -- executing against localhost:21000
> select o_orderdate, o_custkey, o_comment
> from tpch.orders
> order by o_orderdate
> ;
> -- 2021-03-03 07:12:08,329 INFO MainThread: Started query
> 944af93b81ee62cf:ab33f08800000000
> TestScratchLimit.test_with_unlimited_scratch_limit[protocol: beeswax |
> exec_option: {'batch_size': 0, 'num_nodes': 0,
> 'disable_codegen_rows_threshold': 5000, 'disable_codegen': False,
> 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format:
> text/none]
> [gw3] linux2 -- Python 2.7.16
> /home/yida/Impala/bin/../infra/python/env-gcc7.5.0/bin/python
> query_test/test_scratch_limit.py:110: in test_with_unlimited_scratch_limit
> self.execute_query_expect_success(self.client, self.spilling_sort_query,
> exec_option)
> common/impala_test_suite.py:809: in wrapper
> return function(*args, **kwargs)
> common/impala_test_suite.py:817: in execute_query_expect_success
> result = cls.__execute_query(impalad_client, query, query_options, user)
> common/impala_test_suite.py:918: in __execute_query
> return impalad_client.execute(query, user=user)
> common/impala_connection.py:205: in execute
> return self.__beeswax_client.execute(sql_stmt, user=user)
> beeswax/impala_beeswax.py:187: in execute
> handle = self.__execute_query(query_string.strip(), user=user)
> beeswax/impala_beeswax.py:365: in __execute_query
> self.wait_for_finished(handle)
> beeswax/impala_beeswax.py:386: in wait_for_finished
> raise ImpalaBeeswaxException("Query aborted:" + error_log, None)
> E ImpalaBeeswaxException: ImpalaBeeswaxException:
> E Query aborted:Could not create files in any configured scratch directories
> (--scratch_dirs=) on backend 'yida-OptiPlex-7060:27001'. 0 of scratch is
> currently in use by this Impala Daemon (0 by this query). See logs for
> previous errors that may have prevented creating or writing scratch files.
> The following directories were at capacity:
> ---------------------------------------------------------------------------
> Captured stderr setup
> ---------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_unlimited_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_rows_th;
> -- connecting to: localhost:21000
> -- connecting to localhost:21050 with impyla
> -- 2021-03-03 07:12:08,114 INFO MainThread: Closing active operation
> -- connecting to localhost:28000 with impyla
> -- 2021-03-03 07:12:08,146 INFO MainThread: Closing active operation
> ---------------------------------------------------------------------------
> Captured stderr call
> ----------------------------------------------------------------------------
> SET
> client_identifier=query_test/test_scratch_limit.py::TestScratchLimit::()::test_with_unlimited_scratch_limit[protocol:beeswax|exec_option:{'batch_size':0;'num_nodes':0;'disable_codegen_rows_threshold':5000;'disable_codegen':False;'abort_on_error':1;'exec_single_node_rows_th;
> SET scratch_limit=-1;
> SET batch_size=0;
> SET num_nodes=0;
> SET disable_codegen_rows_threshold=5000;
> SET disable_codegen=False;
> SET abort_on_error=1;
> SET exec_single_node_rows_threshold=0;
> SET buffer_pool_limit=32m;
> -- executing against localhost:21000
> select o_orderdate, o_custkey, o_comment
> from tpch.orders
> order by o_orderdate
> ;
> -- 2021-03-03 07:12:08,329 INFO MainThread: Started query
> 5c492cfd62287b03:f9c3370b00000000
> ==================================================================== 4
> failed, 2 passed in 14.11 seconds
> ====================================================================
> Running TestExecutor with args: ['--ignore', 'query_test', '--ignore',
> 'catalog_service', '--ignore', 'hs2', '--ignore', 'experiments', '--ignore',
> 'data_errors', '--ignore', 'performance', '--ignore', 'beeswax', '--ignore',
> 'authorization', '--ignore', 'metadata', '--ignore', 'shell', '--ignore',
> 'benchmark', '--ignore', 'infra', '--ignore', 'statestore', '--ignore',
> 'failure', '--ignore', 'webserver', '--ignore', 'comparison', '--ignore',
> 'stress', '--ignore', 'util', '--ignore', 'custom_cluster', '--ignore',
> 'common', '--ignore', 'observability', '--ignore', 'unittests',
> '--resultlog',
> '/home/yida/Impala/logs/ee_tests/results/TEST-impala-verify-metrics.log',
> '--junitxml',
> '/home/yida/Impala/logs/ee_tests/results/TEST-impala-verify-metrics.xml',
> 'verifiers/test_verify_metrics.py']
> rootLoggerLevel = INFO
> {code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-all-unsubscr...@impala.apache.org
For additional commands, e-mail: issues-all-h...@impala.apache.org
