This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new d2625c35d1 Minor: Improve memory helper trait documentaiton (#9025)
d2625c35d1 is described below
commit d2625c35d15dc962b1eb97e540baf5eefe8c99d9
Author: Andrew Lamb <[email protected]>
AuthorDate: Mon Jan 29 14:49:14 2024 -0500
Minor: Improve memory helper trait documentaiton (#9025)
---
datafusion/execution/src/memory_pool/mod.rs | 15 ++++++++++-----
datafusion/execution/src/memory_pool/proxy.rs | 23 ++++++++++++++++-------
2 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/datafusion/execution/src/memory_pool/mod.rs
b/datafusion/execution/src/memory_pool/mod.rs
index 58ed1ebff0..7816f15bc2 100644
--- a/datafusion/execution/src/memory_pool/mod.rs
+++ b/datafusion/execution/src/memory_pool/mod.rs
@@ -15,7 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-//! Manages all available memory during query execution
+//! [`MemoryPool`] for memory management during query execution, [`proxy]` for
+//! help with allocation accounting.
use datafusion_common::Result;
use std::{cmp::Ordering, sync::Arc};
@@ -56,7 +57,7 @@ pub use pool::*;
/// kills the process, DataFusion `ExecutionPlan`s (operators) that consume
/// large amounts of memory must first request their desired allocation from a
/// [`MemoryPool`] before allocating more. The request is typically managed
via
-/// a [`MemoryReservation`].
+/// a [`MemoryReservation`] and [`MemoryConsumer`].
///
/// If the allocation is successful, the operator should proceed and allocate
/// the desired memory. If the allocation fails, the operator must either first
@@ -107,9 +108,13 @@ pub trait MemoryPool: Send + Sync + std::fmt::Debug {
fn reserved(&self) -> usize;
}
-/// A memory consumer that can be tracked by [`MemoryReservation`] in
-/// a [`MemoryPool`]. All allocations are registered to a particular
-/// `MemoryConsumer`;
+/// A memory consumer is a named allocation traced by a particular
+/// [`MemoryReservation`] in a [`MemoryPool`]. All allocations are registered
to
+/// a particular `MemoryConsumer`;
+///
+/// For help with allocation accounting, see the [proxy] module.
+///
+/// [proxy]: crate::memory_pool::proxy
#[derive(Debug)]
pub struct MemoryConsumer {
name: String,
diff --git a/datafusion/execution/src/memory_pool/proxy.rs
b/datafusion/execution/src/memory_pool/proxy.rs
index ced977b3bf..d754bbaa34 100644
--- a/datafusion/execution/src/memory_pool/proxy.rs
+++ b/datafusion/execution/src/memory_pool/proxy.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-//! Utilities that help with tracking of memory allocations.
+//! [`VecAllocExt`] and [`RawTableAllocExt`] to help tracking of memory
allocations
use hashbrown::raw::{Bucket, RawTable};
@@ -24,12 +24,18 @@ pub trait VecAllocExt {
/// Item type.
type T;
- /// [Push](Vec::push) new element to vector and store additional allocated
bytes in `accounting` (additive).
+ /// [Push](Vec::push) new element to vector and increase
+ /// `accounting` by any newly allocated bytes.
+ ///
+ /// Note that allocation counts capacity, not size
fn push_accounted(&mut self, x: Self::T, accounting: &mut usize);
- /// Return the amount of memory allocated by this Vec (not
- /// recursively counting any heap allocations contained within the
- /// structure). Does not include the size of `self`
+ /// Return the amount of memory allocated by this Vec to store elements
+ /// (`size_of<T> * capacity`).
+ ///
+ /// Note this calculation is not recursive, and does not include any heap
+ /// allocations contained within the Vec's elements. Does not include the
+ /// size of `self`
fn allocated_size(&self) -> usize;
}
@@ -54,12 +60,15 @@ impl<T> VecAllocExt for Vec<T> {
}
}
-/// Extension trait for [`RawTable`] to account for allocations.
+/// Extension trait for hash browns [`RawTable`] to account for allocations.
pub trait RawTableAllocExt {
/// Item type.
type T;
- /// [Insert](RawTable::insert) new element into table and store additional
allocated bytes in `accounting` (additive).
+ /// [Insert](RawTable::insert) new element into table and increase
+ /// `accounting` by any newly allocated bytes.
+ ///
+ /// Returns the bucket where the element was inserted.
fn insert_accounted(
&mut self,
x: Self::T,
