adriangb commented on code in PR #24891: URL: https://github.com/apache/datafusion/pull/24891#discussion_r3918856939
########## datafusion/sqllogictest/test_files/repartition_memory_spill.slt: ########## @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Memory-limited (spilling) repartition. +# +# Regression test for <https://github.com/apache/datafusion/issues/24883>. +# +# Under a memory limit, RepartitionExec writes batches to a spill pool. Each +# output partition has one pool, and all input tasks share it. When two input +# tasks spilled at the same time, the pool had two open files. The reader +# waited on the older file, but the batch that it needed was in the newer +# file. The query then never completed. +# +# Two partitions give two writers per pool. The small batch size sends many +# small batches through the spill path, and the spill file stays below the +# rotation size. The Utf8View column makes each small batch count the full +# buffer of its source batch, so the repartition reservation fills fast. The +# memory limit is too small for the repartition and large enough for the +# aggregates. This is necessary to make RepartitionExec spill. +# +# NOTE: sqllogictest has no timeout. If this bug comes back, this file hangs. +# The Rust test `repartition_spill_pool_does_not_deadlock` runs the same query +# with a timeout. + +statement ok +SET datafusion.execution.target_partitions = 2 + +statement ok +SET datafusion.execution.batch_size = 64 + +statement ok +CREATE TABLE trace_events AS +SELECT v % 13 AS g, + CASE WHEN v % 29 = 0 THEN NULL + ELSE md5(CAST(v % 337 AS VARCHAR)) END AS trace_id +FROM generate_series(1, 40000) AS t(v) + +statement ok +CREATE VIEW tv AS +SELECT g, arrow_cast(trace_id, 'Utf8View') AS trace_id FROM trace_events + +statement ok +SET datafusion.runtime.memory_limit = '4M' + +# Each group has all 337 distinct trace ids. The query runs eight times, +# because one run did not always hang on the unfixed code. No ORDER BY: a +# sort needs its own memory reservation, which does not fit in the limit. +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +query II rowsort +SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +0 337 +1 337 +10 337 +11 337 +12 337 +2 337 +3 337 +4 337 +5 337 +6 337 +7 337 +8 337 +9 337 + +# Make sure that the query goes through the spill path of RepartitionExec. +query TT +EXPLAIN ANALYZE SELECT g, count(DISTINCT trace_id) AS n FROM tv GROUP BY g +---- +<slt:ignore> Review Comment: Hmm not sure what these are, we should check them ########## datafusion/sqllogictest/test_files/repartition_memory_spill.slt: ########## @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one Review Comment: I don't recall if there are any negatives to a new STL file. Should consider if it should be it's own file or folded into an existing suite. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
