Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_scavenger.c (292488 => 292489)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_scavenger.c 2022-04-06 19:06:37 UTC (rev 292488)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_scavenger.c 2022-04-06 19:09:18 UTC (rev 292489)
@@ -69,10 +69,6 @@
uint64_t pas_scavenger_max_epoch_delta = 300ll * 1000ll * 1000ll;
#endif
-static uint32_t pas_scavenger_tick_count = 0;
-/* Run thread-local-cache decommit once a N. It should be power of two. */
-#define PAS_THREAD_LOCAL_CACHE_DECOMMIT_PERIOD_COUNT 128 /* Roughly speaking, it runs once per 13 seconds. */
-
#if PAS_OS(DARWIN)
static _Atomic qos_class_t pas_scavenger_requested_qos_class = QOS_CLASS_USER_INITIATED;
@@ -196,7 +192,6 @@
double time_in_milliseconds;
double absolute_timeout_in_milliseconds_for_period_sleep;
pas_scavenger_activity_callback completion_callback;
- pas_thread_local_cache_decommit_action thread_local_cache_decommit_action;
bool should_go_again;
uint64_t epoch;
uint64_t delta;
@@ -213,7 +208,6 @@
pthread_set_qos_class_self_np(configured_qos_class, 0);
}
#endif
- ++pas_scavenger_tick_count;
should_go_again = false;
@@ -235,16 +229,9 @@
pas_utility_heap_for_all_allocators(pas_allocator_scavenge_request_stop_action,
pas_lock_is_not_held);
- thread_local_cache_decommit_action = pas_thread_local_cache_decommit_no_action;
- if ((pas_scavenger_tick_count % PAS_THREAD_LOCAL_CACHE_DECOMMIT_PERIOD_COUNT) == 0) {
- if (verbose)
- printf("Attempt to decommit unused TLC\n");
- thread_local_cache_decommit_action = pas_thread_local_cache_decommit_if_possible_action;
- }
should_go_again |=
pas_thread_local_cache_for_all(pas_allocator_scavenge_request_stop_action,
- pas_deallocator_scavenge_flush_log_if_clean_action,
- thread_local_cache_decommit_action);
+ pas_deallocator_scavenge_flush_log_if_clean_action);
should_go_again |= handle_expendable_memory(pas_expendable_memory_scavenge_periodic);
@@ -515,8 +502,7 @@
pas_scavenger_clear_all_caches_except_remote_tlcs();
pas_thread_local_cache_for_all(pas_allocator_scavenge_force_stop_action,
- pas_deallocator_scavenge_flush_log_action,
- pas_thread_local_cache_decommit_if_possible_action);
+ pas_deallocator_scavenge_flush_log_action);
}
void pas_scavenger_decommit_expendable_memory(void)
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c (292488 => 292489)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-04-06 19:06:37 UTC (rev 292488)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c 2022-04-06 19:09:18 UTC (rev 292489)
@@ -1022,8 +1022,7 @@
}
bool pas_thread_local_cache_for_all(pas_allocator_scavenge_action allocator_action,
- pas_deallocator_scavenge_action deallocator_action,
- pas_thread_local_cache_decommit_action thread_local_cache_decommit_action)
+ pas_deallocator_scavenge_action deallocator_action)
{
pas_thread_local_cache_node* node;
bool result;
@@ -1088,6 +1087,8 @@
pas_thread_local_cache_layout_segment* segment;
pas_thread_local_cache_layout_segment* begin_segment;
uintptr_t start_of_possible_decommit;
+ uintptr_t begin_page_index;
+ uintptr_t end_page_index;
uintptr_t page_index;
uintptr_t node_index;
uintptr_t begin_node_index;
@@ -1105,7 +1106,9 @@
node_index = 0;
for (PAS_THREAD_LOCAL_CACHE_LAYOUT_EACH_ALLOCATOR_WITH_SEGMENT_AND_INDEX(layout_node, segment, node_index)) {
pas_allocator_index allocator_index;
+ pas_allocator_index end_allocator_index;
pas_local_allocator_scavenger_data* scavenger_data;
+ pas_decommit_exclusion_range decommit_exclusion_range;
allocator_index =
pas_thread_local_cache_layout_node_get_allocator_index_generic(layout_node);
@@ -1113,44 +1116,45 @@
if (allocator_index >= cache->allocator_index_upper_bound)
break;
+ end_allocator_index =
+ allocator_index + pas_thread_local_cache_layout_node_num_allocator_indices(layout_node);
+
scavenger_data = (pas_local_allocator_scavenger_data*)
pas_thread_local_cache_get_local_allocator_direct(cache, allocator_index);
+
+ decommit_exclusion_range = pas_thread_local_cache_compute_decommit_exclusion_range(
+ cache, allocator_index, end_allocator_index);
+ PAS_ASSERT(start_of_possible_decommit
+ <= decommit_exclusion_range.start_of_possible_decommit);
+ PAS_ASSERT(start_of_possible_decommit
+ <= decommit_exclusion_range.end_of_possible_decommit);
+ if (pas_decommit_exclusion_range_is_contiguous(decommit_exclusion_range)) {
+ stop_allocator(
+ cache, allocator_action, allocator_index, scavenger_data, &result, &thread_suspend_data);
- if (thread_local_cache_decommit_action == pas_thread_local_cache_decommit_if_possible_action) {
- pas_allocator_index end_allocator_index = allocator_index + pas_thread_local_cache_layout_node_num_allocator_indices(layout_node);
- pas_decommit_exclusion_range decommit_exclusion_range = pas_thread_local_cache_compute_decommit_exclusion_range(cache, allocator_index, end_allocator_index);
- PAS_ASSERT(start_of_possible_decommit <= decommit_exclusion_range.start_of_possible_decommit);
- PAS_ASSERT(start_of_possible_decommit <= decommit_exclusion_range.end_of_possible_decommit);
-
- if (pas_decommit_exclusion_range_is_contiguous(decommit_exclusion_range)) {
- stop_allocator(
- cache, allocator_action, allocator_index, scavenger_data, &result, &thread_suspend_data);
-
- if (pas_local_allocator_scavenger_data_is_stopped(scavenger_data)) {
- if (!begin_segment) {
- begin_segment = segment;
- begin_node_index = node_index;
- } else {
- PAS_TESTING_ASSERT(
- pas_thread_local_cache_layout_node_get_allocator_index_generic(pas_thread_local_cache_layout_segment_get_node(begin_segment, begin_node_index))
- < allocator_index);
- }
+ if (pas_local_allocator_scavenger_data_is_stopped(scavenger_data)) {
+ if (!begin_segment) {
+ begin_segment = segment;
+ begin_node_index = node_index;
} else {
- decommit_exclusion_range =
- pas_decommit_exclusion_range_create_inverted(decommit_exclusion_range);
+ PAS_TESTING_ASSERT(
+ pas_thread_local_cache_layout_node_get_allocator_index_generic(pas_thread_local_cache_layout_segment_get_node(begin_segment, begin_node_index))
+ < allocator_index);
}
+ } else {
+ decommit_exclusion_range =
+ pas_decommit_exclusion_range_create_inverted(decommit_exclusion_range);
}
+ }
- if (pas_decommit_exclusion_range_is_inverted(decommit_exclusion_range)) {
- decommit_allocator_range(
- cache, begin_segment, begin_node_index, start_of_possible_decommit,
- decommit_exclusion_range.end_of_possible_decommit, segment, node_index);
- start_of_possible_decommit = decommit_exclusion_range.start_of_possible_decommit;
- begin_segment = NULL;
- begin_node_index = 0;
- }
- } else
- stop_allocator(cache, allocator_action, allocator_index, scavenger_data, &result, &thread_suspend_data);
+ if (pas_decommit_exclusion_range_is_inverted(decommit_exclusion_range)) {
+ decommit_allocator_range(
+ cache, begin_segment, begin_node_index, start_of_possible_decommit,
+ decommit_exclusion_range.end_of_possible_decommit, segment, node_index);
+ start_of_possible_decommit = decommit_exclusion_range.start_of_possible_decommit;
+ begin_segment = NULL;
+ begin_node_index = 0;
+ }
}
#if PAS_OS(DARWIN)
@@ -1158,31 +1162,29 @@
resume(cache, &thread_suspend_data);
#endif
- if (thread_local_cache_decommit_action == pas_thread_local_cache_decommit_if_possible_action) {
- uintptr_t begin_page_index =
- pas_round_up_to_power_of_2(start_of_possible_decommit, page_size)
- >> page_size_shift;
- uintptr_t end_page_index =
- pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity)
- >> page_size_shift;
+ begin_page_index =
+ pas_round_up_to_power_of_2(start_of_possible_decommit, page_size)
+ >> page_size_shift;
+ end_page_index =
+ pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity)
+ >> page_size_shift;
- for (page_index = begin_page_index; page_index < end_page_index; ++page_index) {
- if (!pas_bitvector_get(cache->pages_committed, page_index)) {
- decommit_allocator_range(
- cache, begin_segment, begin_node_index, start_of_possible_decommit,
- page_index << page_size_shift,
- segment, node_index);
- start_of_possible_decommit = (page_index + 1) << page_size_shift;
- begin_segment = NULL;
- begin_node_index = 0;
- }
+ for (page_index = begin_page_index; page_index < end_page_index; ++page_index) {
+ if (!pas_bitvector_get(cache->pages_committed, page_index)) {
+ decommit_allocator_range(
+ cache, begin_segment, begin_node_index, start_of_possible_decommit,
+ page_index << page_size_shift,
+ segment, node_index);
+ start_of_possible_decommit = (page_index + 1) << page_size_shift;
+ begin_segment = NULL;
+ begin_node_index = 0;
}
-
- decommit_allocator_range(
- cache, begin_segment, begin_node_index, start_of_possible_decommit,
- pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity),
- segment, node_index);
}
+
+ decommit_allocator_range(
+ cache, begin_segment, begin_node_index, start_of_possible_decommit,
+ pas_thread_local_cache_size_for_allocator_index_capacity(cache->allocator_index_capacity),
+ segment, node_index);
}
pas_lock_unlock(&node->scavenger_lock);
Modified: trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.h (292488 => 292489)
--- trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.h 2022-04-06 19:06:37 UTC (rev 292488)
+++ trunk/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.h 2022-04-06 19:09:18 UTC (rev 292489)
@@ -372,17 +372,10 @@
pas_thread_local_cache* thread_local_cache,
pas_lock_hold_mode heap_lock_hold_mode);
-enum pas_thread_local_cache_decommit_action {
- pas_thread_local_cache_decommit_no_action,
- pas_thread_local_cache_decommit_if_possible_action,
-};
-typedef enum pas_thread_local_cache_decommit_action pas_thread_local_cache_decommit_action;
-
/* Returns true if it's possible that we'll be able to return more memory if this was called
again. */
PAS_API bool pas_thread_local_cache_for_all(pas_allocator_scavenge_action allocator_action,
- pas_deallocator_scavenge_action deallocator_action,
- pas_thread_local_cache_decommit_action thread_local_cache_decommit_action);
+ pas_deallocator_scavenge_action deallocator_action);
PAS_API PAS_NEVER_INLINE void pas_thread_local_cache_append_deallocation_slow(
pas_thread_local_cache* thread_local_cache,
Modified: trunk/Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp (292488 => 292489)
--- trunk/Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp 2022-04-06 19:06:37 UTC (rev 292488)
+++ trunk/Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp 2022-04-06 19:09:18 UTC (rev 292489)
@@ -157,8 +157,7 @@
pas_heap_lock_unlock();
pas_thread_local_cache_for_all(pas_allocator_scavenge_request_stop_action,
- pas_deallocator_scavenge_no_action,
- pas_thread_local_cache_decommit_if_possible_action);
+ pas_deallocator_scavenge_no_action);
CHECK_EQUAL(numCommittedPagesInTLC(), numCommittedPagesAfterDecommit);
@@ -297,8 +296,7 @@
CHECK(pas_thread_local_cache_try_get()->deallocation_log_index);
pas_thread_local_cache_for_all(pas_allocator_scavenge_force_stop_action,
- pas_deallocator_scavenge_no_action,
- pas_thread_local_cache_decommit_if_possible_action);
+ pas_deallocator_scavenge_no_action);
CHECK_EQUAL(numCommittedPagesInTLC(), 1);
CHECK(pas_thread_local_cache_try_get()->deallocation_log_index);
@@ -342,8 +340,7 @@
CHECK(!pas_thread_local_cache_try_get()->deallocation_log_index);
pas_thread_local_cache_for_all(pas_allocator_scavenge_force_stop_action,
- pas_deallocator_scavenge_no_action,
- pas_thread_local_cache_decommit_if_possible_action);
+ pas_deallocator_scavenge_no_action);
for (void* object : objects)
bmalloc_deallocate(object);
