Branch: refs/heads/main
Home: https://github.com/WebKit/WebKit
Commit: c8cb35beacac025207d52d9cb86b058149e178b3
https://github.com/WebKit/WebKit/commit/c8cb35beacac025207d52d9cb86b058149e178b3
Author: Marcus Plutowski <[email protected]>
Date: 2026-09-29 (Tue, 29 Sep 2026)
Changed paths:
M Source/JavaScriptCore/heap/StructureAlignedMemoryAllocator.cpp
M Source/bmalloc/CMakeLists.txt
M Source/bmalloc/bmalloc.xcodeproj/project.pbxproj
M Source/bmalloc/bmalloc/CompactAllocationMode.h
M Source/bmalloc/bmalloc/SystemHeap.cpp
M Source/bmalloc/bmalloc/TZoneHeap.cpp
M Source/bmalloc/bmalloc/TZoneHeapManager.cpp
M Source/bmalloc/bmalloc/bmalloc.cpp
M Source/bmalloc/bmalloc/bmalloc.h
M Source/bmalloc/libpas/Documentation.md
M Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj
M Source/bmalloc/libpas/src/chaos/Chaos.cpp
M Source/bmalloc/libpas/src/libpas/bmalloc_heap.c
M Source/bmalloc/libpas/src/libpas/bmalloc_heap.h
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_config.c
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_config.h
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_flex.c
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_flex_internal.h
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_inlines.h
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_innards.h
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_iso.c
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_iso_internal.h
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_ref.h
M Source/bmalloc/libpas/src/libpas/bmalloc_heap_utils.c
M Source/bmalloc/libpas/src/libpas/bmalloc_prefault_supply.c
M Source/bmalloc/libpas/src/libpas/hotbit_heap.c
M Source/bmalloc/libpas/src/libpas/hotbit_heap.h
M Source/bmalloc/libpas/src/libpas/hotbit_heap_config.c
M Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h
M Source/bmalloc/libpas/src/libpas/iso_heap.c
M Source/bmalloc/libpas/src/libpas/iso_heap.h
M Source/bmalloc/libpas/src/libpas/iso_heap_config.c
M Source/bmalloc/libpas/src/libpas/iso_heap_inlines.h
M Source/bmalloc/libpas/src/libpas/iso_test_heap.c
M Source/bmalloc/libpas/src/libpas/iso_test_heap.h
M Source/bmalloc/libpas/src/libpas/iso_test_heap_config.c
M Source/bmalloc/libpas/src/libpas/jit_heap.c
M Source/bmalloc/libpas/src/libpas/minalign32_heap.c
M Source/bmalloc/libpas/src/libpas/minalign32_heap.h
M Source/bmalloc/libpas/src/libpas/minalign32_heap_config.c
M Source/bmalloc/libpas/src/libpas/pagesize64k_heap.c
M Source/bmalloc/libpas/src/libpas/pagesize64k_heap.h
M Source/bmalloc/libpas/src/libpas/pagesize64k_heap_config.c
R Source/bmalloc/libpas/src/libpas/pas_allocation_mode.h
M Source/bmalloc/libpas/src/libpas/pas_basic_heap_page_caches.h
M Source/bmalloc/libpas/src/libpas/pas_bitfit_allocator_inlines.h
M Source/bmalloc/libpas/src/libpas/pas_bitfit_page_config.h
M Source/bmalloc/libpas/src/libpas/pas_bitfit_page_config_inlines.h
M Source/bmalloc/libpas/src/libpas/pas_bitfit_page_inlines.h
M
Source/bmalloc/libpas/src/libpas/pas_create_basic_heap_page_caches_with_reserved_memory.c
M Source/bmalloc/libpas/src/libpas/pas_deallocate.c
M Source/bmalloc/libpas/src/libpas/pas_deallocate.h
M Source/bmalloc/libpas/src/libpas/pas_ensure_heap_with_page_caches.c
M Source/bmalloc/libpas/src/libpas/pas_enumerable_page_malloc.c
M Source/bmalloc/libpas/src/libpas/pas_enumerable_page_malloc.h
M Source/bmalloc/libpas/src/libpas/pas_heap.c
M Source/bmalloc/libpas/src/libpas/pas_heap.h
M Source/bmalloc/libpas/src/libpas/pas_heap_config.h
M Source/bmalloc/libpas/src/libpas/pas_heap_config_inlines.h
M Source/bmalloc/libpas/src/libpas/pas_heap_config_utils.c
M Source/bmalloc/libpas/src/libpas/pas_heap_config_utils.h
M Source/bmalloc/libpas/src/libpas/pas_heap_config_utils_inlines.h
M Source/bmalloc/libpas/src/libpas/pas_heap_kind.h
M Source/bmalloc/libpas/src/libpas/pas_heap_ref_prefix.h
M Source/bmalloc/libpas/src/libpas/pas_heap_runtime_config.h
M Source/bmalloc/libpas/src/libpas/pas_large_heap.c
M Source/bmalloc/libpas/src/libpas/pas_large_heap.h
M
Source/bmalloc/libpas/src/libpas/pas_large_heap_physical_page_sharing_cache.h
M Source/bmalloc/libpas/src/libpas/pas_local_allocator.h
M Source/bmalloc/libpas/src/libpas/pas_local_allocator_inlines.h
M Source/bmalloc/libpas/src/libpas/pas_mar_registry.h
M Source/bmalloc/libpas/src/libpas/pas_megapage_cache.h
M Source/bmalloc/libpas/src/libpas/pas_mte.h
A Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_free_heap.c
A Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_free_heap.h
A Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_heap_page_provider.c
A Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_heap_page_provider.h
M Source/bmalloc/libpas/src/libpas/pas_mte_config.c
M Source/bmalloc/libpas/src/libpas/pas_page_malloc.c
M Source/bmalloc/libpas/src/libpas/pas_page_malloc.h
M
Source/bmalloc/libpas/src/libpas/pas_probabilistic_guard_malloc_allocator.c
M
Source/bmalloc/libpas/src/libpas/pas_probabilistic_guard_malloc_allocator.h
M Source/bmalloc/libpas/src/libpas/pas_report_crash_pgm_report.h
M Source/bmalloc/libpas/src/libpas/pas_segregated_page_config.h
M Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_inlines.h
R Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_free_heap.c
R Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_free_heap.h
R
Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_heap_page_provider.c
R
Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_heap_page_provider.h
M Source/bmalloc/libpas/src/libpas/pas_stats.c
M Source/bmalloc/libpas/src/libpas/pas_stats.h
M Source/bmalloc/libpas/src/libpas/pas_system_heap.h
M Source/bmalloc/libpas/src/libpas/pas_try_allocate.h
M Source/bmalloc/libpas/src/libpas/pas_try_allocate_array.h
M Source/bmalloc/libpas/src/libpas/pas_try_allocate_common.h
M Source/bmalloc/libpas/src/libpas/pas_try_allocate_intrinsic.h
M Source/bmalloc/libpas/src/libpas/pas_try_allocate_primitive.h
M Source/bmalloc/libpas/src/libpas/pas_try_reallocate.h
M Source/bmalloc/libpas/src/libpas/pas_utility_heap.c
M Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap.c
M Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_config.c
M Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_config.h
M Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_inlines.h
M Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_utils.c
M Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_utils.h
M Source/bmalloc/libpas/src/libpas/thingy_heap.c
M Source/bmalloc/libpas/src/libpas/thingy_heap.h
M Source/bmalloc/libpas/src/libpas/thingy_heap_config.c
M Source/bmalloc/libpas/src/libpas/thingy_heap_prefix.h
M Source/bmalloc/libpas/src/mbmalloc/mbmalloc_bmalloc.c
M Source/bmalloc/libpas/src/mbmalloc/mbmalloc_hotbit.c
M Source/bmalloc/libpas/src/mbmalloc/mbmalloc_iso_common_primitive.c
M Source/bmalloc/libpas/src/test/AllocationZeroingTests.cpp
M Source/bmalloc/libpas/src/test/BitfitTests.cpp
M Source/bmalloc/libpas/src/test/BmallocTests.cpp
M Source/bmalloc/libpas/src/test/EnumerationTests.cpp
M Source/bmalloc/libpas/src/test/ExpendableMemoryTests.cpp
M Source/bmalloc/libpas/src/test/HeapRefAllocatorIndexTests.cpp
M Source/bmalloc/libpas/src/test/IsoDynamicPrimitiveHeapTests.cpp
M Source/bmalloc/libpas/src/test/IsoHeapChaosTests.cpp
M Source/bmalloc/libpas/src/test/IsoHeapPageSharingTests.cpp
M Source/bmalloc/libpas/src/test/IsoHeapPartialAndBaselineTests.cpp
M Source/bmalloc/libpas/src/test/IsoHeapReservedMemoryTests.cpp
M Source/bmalloc/libpas/src/test/LotsOfHeapsAndThreads.cpp
M Source/bmalloc/libpas/src/test/MemalignTests.cpp
M Source/bmalloc/libpas/src/test/PGMTests.cpp
M Source/bmalloc/libpas/src/test/RaceTests.cpp
M Source/bmalloc/libpas/src/test/ReallocFastPathTests.cpp
M Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp
M Source/bmalloc/libpas/src/test/TSDTests.cpp
M Source/bmalloc/libpas/src/test/ThingyAndUtilityHeapAllocationTests.cpp
M Source/bmalloc/libpas/src/test/ViewCacheTests.cpp
Log Message:
-----------
[libpas] Use tagged_bmalloc_heap for MTE allocations
https://bugs.webkit.org/show_bug.cgi?id=317893
rdar://180668556
Reviewed by Yusuke Suzuki and Dan Hecht.
Previously, the tagged_bmalloc_heap was compiled in but unused. This
patch enables it. In brief, this hoists the 'compactness' of objects
out of libpas: pas-heaps themselves do not need to differentiate
compact vs. non-compact.
This adds a new dispatch-pathway to the various functions in bmalloc.h,
as it's now possible to dispatch to entirely different pas_heaps
rather than just between different allocation paths within a single
pas_heap. Unlike the latter, deallocating from different pas_heaps
requires a dispatch on the fast-path, which was a ~0.3% regression on
Speedometer3.
Now that we can call the inline-only fast-paths separately from the
casual case, we're able to ameliorate the overhead of switching on the
free-path and leave the fastest fast-path (i.e. small-segregated
TLC-present non-MTE free) untouched by speculating that the allocation
is non-MTE, then checking the MTE case, and only then falling through to
the casual case. This means we don't need to check for MTE enablement
prior to freeing objects in the WebContent process.
One benefit of this change: the inline portion of fastMalloc goes from
141 instructions to 73. This is because the MTE tag-setting logic,
which was previously inlined for all heaps that could potentially tag
allocations, is now out-of-line behind the tagged heap, and is
eliminated at compile time for the default heap.
On the flipside, the inline portion of fastFree goes from 44 to 45.
The actual fast-path length in both cases is of course shorter.
There are two major motivations for this patch.
First, it eliminates the compact/non-compact distinction that had
spread through libpas' internals; compact/non-compact megapages,
compact/non-compact heap-refs, etc. were only necessary because
heaps had to service both tagged and untagged allocations.
Secondly, it eliminates the last reason heaps need to mmap multiple
'flavors' of memory; consequently, we will soon be able to extend
on the work of 313052@main to embed the necessary mmap flags inside
the pas_page_base_config itself, allowing libpas to be fully aware
of the memory-types it's allocating/handling, and thus handle them
with more granularity than the previous paradigm of "may mmap" or
"may not mmap".
It only needs to be enabled for FastMalloc, because TZoneMalloc is
disabled when MTE is enabled, and the tagged_bmalloc_heap doesn't expose
the iso-allocation pathways regardless. But since the dispatch layer is
underneath the bmalloc API anyways, neither FastMalloc nor TZoneMalloc
need to be aware of the distinction other than insofar as they pass a
compaction mode in.
That choice, to keep the tagged/untagged dispatch at the bmalloc level
rather than hoisting it to the FastMalloc/TZoneMalloc level, was
made because empirically it performs just as well, while maintaining
a better centralization of concerns (i.e. not forcing bmalloc consumers
to be aware of MTE).
Note that technically there’s no problem with using the tagged_bmalloc
heap for non-compact allocations everywhere (every process / all SoCs)
as MTE tagging will currently be disabled when unsupported regardless,
but doing so in WebContent would hurt performance due to the cost of
serving allocations from a non-designated-intrinsic heap.
One side effect of this patch is that sufficiently-large compact objects
are now eligible for libmalloc delegation; this is preferable in terms
of memory protections, but does change allocation patterns + memory
accounting.
* Source/bmalloc/bmalloc/bmalloc.cpp: add new MTE dispatch layer
(bmalloc::api::freeOutOfLine):
(bmalloc::api::isMTEEnabled): Deleted.
(bmalloc::api::mallocOutOfLine):
(bmalloc::api::tryReallocOutOfLine):
(bmalloc::api::reallocOutOfLine):
* Source/bmalloc/bmalloc/bmalloc.h: add new MTE dispatch layer
(bmalloc::api::isMTEEnabled):
(bmalloc::api::tryMalloc):
(bmalloc::api::malloc):
(bmalloc::api::tryZeroedMalloc):
(bmalloc::api::zeroedMalloc):
(bmalloc::api::tryMemalign):
(bmalloc::api::memalign):
(bmalloc::api::tryZeroedMemalign):
(bmalloc::api::zeroedMemalign):
(bmalloc::api::tryRealloc):
(bmalloc::api::realloc):
(bmalloc::api::free):
(bmalloc::api::isMTEEnabledUnsafe):
* Source/bmalloc/libpas/src/libpas/bmalloc_heap.c: strip pas_allocation_mode
(bmalloc_try_allocate_casual):
(bmalloc_allocate_casual):
(bmalloc_try_allocate_auxiliary_with_alignment_casual):
(bmalloc_allocate_auxiliary_with_alignment_casual):
(bmalloc_try_deallocate_casual):
(bmalloc_try_allocate_auxiliary):
(bmalloc_allocate_auxiliary):
(bmalloc_try_allocate):
(bmalloc_try_allocate_with_alignment):
(bmalloc_try_allocate_zeroed):
(bmalloc_try_allocate_zeroed_with_alignment):
(bmalloc_allocate):
(bmalloc_allocate_with_alignment):
(bmalloc_allocate_zeroed):
(bmalloc_allocate_zeroed_with_alignment):
(bmalloc_try_reallocate):
(bmalloc_reallocate):
(bmalloc_try_allocate_auxiliary_zeroed):
(bmalloc_allocate_auxiliary_zeroed):
(bmalloc_try_allocate_auxiliary_with_alignment):
(bmalloc_allocate_auxiliary_with_alignment):
(bmalloc_try_allocate_auxiliary_zeroed_with_alignment):
(bmalloc_allocate_auxiliary_zeroed_with_alignment):
(bmalloc_try_reallocate_auxiliary):
(bmalloc_reallocate_auxiliary):
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_inlines.h: strip
pas_allocation_mode
(bmalloc_try_allocate_auxiliary_inline):
(bmalloc_allocate_auxiliary_inline):
(bmalloc_try_allocate_auxiliary_zeroed_inline):
(bmalloc_allocate_auxiliary_zeroed_inline):
(bmalloc_try_allocate_auxiliary_with_alignment_inline):
(bmalloc_allocate_auxiliary_with_alignment_inline):
(bmalloc_try_allocate_auxiliary_zeroed_with_alignment_inline):
(bmalloc_allocate_auxiliary_zeroed_with_alignment_inline):
(bmalloc_try_reallocate_auxiliary_inline):
(bmalloc_reallocate_auxiliary_inline):
(bmalloc_try_allocate_inline):
(bmalloc_try_allocate_with_alignment_inline):
(bmalloc_try_allocate_zeroed_with_alignment_inline):
(bmalloc_try_allocate_zeroed_inline):
(bmalloc_allocate_inline):
(bmalloc_allocate_with_alignment_inline):
(bmalloc_allocate_zeroed_inline):
(bmalloc_allocate_zeroed_with_alignment_inline):
(bmalloc_try_reallocate_inline):
(bmalloc_reallocate_inline):
(bmalloc_deallocate_inline):
(bmalloc_try_deallocate_inline_only):
* Source/bmalloc/libpas/src/libpas/pas_mte_config.c: allow fast unsafe checking
(pas_mte_do_initialization):
(pas_mte_is_mte_on_for_process):
(pas_mte_do_and_check_initialization):
(pas_mte_is_enabled): Deleted.
(pas_report_config):
(pas_bmalloc_force_allocations_into_bitfit_heaps_where_available):
(pas_mte_force_nontaggable_user_allocations_into_large_heap):
* Source/JavaScriptCore/heap/StructureAlignedMemoryAllocator.cpp: use new
libpas API
(JSC::StructureMemoryManager::tryMallocStructureBlock):
* Source/bmalloc/bmalloc/SystemHeap.cpp: replace compaction mode with bool
(pas_system_heap_is_enabled):
(pas_system_heap_should_supplant_bmalloc):
(pas_system_heap_malloc):
(pas_system_heap_memalign):
(pas_system_heap_realloc):
(pas_system_heap_malloc_compact):
(pas_system_heap_memalign_compact):
(pas_system_heap_realloc_compact):
(pas_system_heap_malloc_untagged):
(pas_system_heap_memalign_untagged):
(pas_system_heap_realloc_untagged):
* Source/bmalloc/bmalloc/TZoneHeap.cpp: no longer differentiate
(bmalloc::api::tzoneAllocateNonCompactSlow):
(bmalloc::api::tzoneAllocateCompactSlow):
(bmalloc::api::tzoneAllocateCompact):
(bmalloc::api::tzoneAllocateNonCompact):
(bmalloc::api::tzoneFree):
compact/non-compact heap refs and thus buckets
* Source/bmalloc/bmalloc/TZoneHeapManager.cpp:
(bmalloc::api::determineTZoneMallocFallback):
(bmalloc::api::TZoneHeapManager::populateGroupBuckets):
* Source/bmalloc/libpas/Documentation.md:
* Source/bmalloc/libpas/src/libpas/bmalloc_heap.h: strip pas_allocation_mode
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_config.c:
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_config.h:
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_flex.c:
(bmalloc_try_allocate_flex_with_alignment_casual):
(bmalloc_allocate_flex_with_alignment_casual):
(bmalloc_try_allocate_flex):
(bmalloc_allocate_flex):
(bmalloc_try_allocate_zeroed_flex):
(bmalloc_allocate_zeroed_flex):
(bmalloc_try_allocate_flex_with_alignment):
(bmalloc_allocate_flex_with_alignment):
(bmalloc_try_reallocate_flex):
(bmalloc_reallocate_flex):
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_flex_internal.h:
(bmalloc_try_allocate_flex_inline):
(bmalloc_allocate_flex_inline):
(bmalloc_try_allocate_zeroed_flex_inline):
(bmalloc_allocate_zeroed_flex_inline):
(bmalloc_try_allocate_flex_with_alignment_inline):
(bmalloc_allocate_flex_with_alignment_inline):
(bmalloc_try_reallocate_flex_inline):
(bmalloc_reallocate_flex_inline):
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_innards.h:
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_iso.c:
(bmalloc_try_iso_allocate_casual):
(bmalloc_iso_allocate_casual):
(bmalloc_try_allocate_array_by_size_with_alignment_casual):
(bmalloc_allocate_array_by_size_with_alignment_casual):
(bmalloc_try_iso_allocate):
(bmalloc_iso_allocate):
(bmalloc_try_iso_allocate_array_by_size):
(bmalloc_iso_allocate_array_by_size):
(bmalloc_try_iso_allocate_zeroed_array_by_size):
(bmalloc_iso_allocate_zeroed_array_by_size):
(bmalloc_try_iso_allocate_array_by_size_with_alignment):
(bmalloc_iso_allocate_array_by_size_with_alignment):
(bmalloc_try_iso_reallocate_array_by_size):
(bmalloc_iso_reallocate_array_by_size):
(bmalloc_try_iso_allocate_array_by_count):
(bmalloc_iso_allocate_array_by_count):
(bmalloc_try_iso_allocate_array_by_count_with_alignment):
(bmalloc_iso_allocate_array_by_count_with_alignment):
(bmalloc_try_iso_reallocate_array_by_count):
(bmalloc_iso_reallocate_array_by_count):
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_iso_internal.h:
(bmalloc_try_iso_allocate_inline):
(bmalloc_iso_allocate_inline):
(bmalloc_try_iso_allocate_array_by_size_inline):
(bmalloc_try_iso_allocate_zeroed_array_by_size_inline):
(bmalloc_try_iso_allocate_array_by_size_with_alignment_inline):
(bmalloc_try_iso_allocate_array_by_count_inline):
(bmalloc_try_iso_allocate_array_by_count_with_alignment_inline):
(bmalloc_iso_allocate_array_by_size_inline):
(bmalloc_iso_allocate_zeroed_array_by_size_inline):
(bmalloc_iso_allocate_array_by_size_with_alignment_inline):
(bmalloc_iso_allocate_array_by_count_inline):
(bmalloc_iso_allocate_array_by_count_with_alignment_inline):
(bmalloc_try_iso_reallocate_array_by_size_inline):
(bmalloc_iso_reallocate_array_by_size_inline):
(bmalloc_try_iso_reallocate_array_by_count_inline):
(bmalloc_iso_reallocate_array_by_count_inline):
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_utils.c:
* Source/bmalloc/libpas/src/libpas/hotbit_heap.c: strip pas_allocation_mode
(hotbit_try_allocate):
(hotbit_try_allocate_with_alignment):
(hotbit_try_reallocate):
* Source/bmalloc/libpas/src/libpas/hotbit_heap.h:
* Source/bmalloc/libpas/src/libpas/hotbit_heap_config.c:
* Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h:
(hotbit_try_allocate_inline):
(hotbit_try_allocate_with_alignment_inline):
(hotbit_try_reallocate_inline):
* Source/bmalloc/libpas/src/libpas/iso_heap.c: strip pas_allocation_mode
(iso_try_allocate_common_primitive):
(iso_try_allocate_common_primitive_with_alignment):
(iso_try_allocate_common_primitive_zeroed):
(iso_allocate_common_primitive):
(iso_allocate_common_primitive_with_alignment):
(iso_allocate_common_primitive_zeroed):
(iso_try_reallocate_common_primitive):
(iso_reallocate_common_primitive):
(iso_try_allocate_dynamic_primitive):
(iso_try_allocate_dynamic_primitive_with_alignment):
(iso_try_allocate_dynamic_primitive_zeroed):
(iso_try_reallocate_dynamic_primitive):
(iso_try_allocate):
(iso_allocate):
(iso_try_allocate_array_by_count):
(iso_allocate_array_by_count):
(iso_try_allocate_array_by_count_zeroed):
(iso_allocate_array_by_count_zeroed):
(iso_try_reallocate_array_by_count):
(iso_reallocate_array_by_count):
(iso_try_allocate_primitive):
(iso_allocate_primitive):
(iso_try_allocate_primitive_zeroed):
(iso_allocate_primitive_zeroed):
(iso_try_allocate_primitive_with_alignment):
(iso_allocate_primitive_with_alignment):
(iso_try_reallocate_primitive):
(iso_reallocate_primitive):
(iso_try_allocate_for_flex):
* Source/bmalloc/libpas/src/libpas/iso_heap.h:
* Source/bmalloc/libpas/src/libpas/iso_heap_config.c:
* Source/bmalloc/libpas/src/libpas/iso_heap_inlines.h:
(iso_try_allocate_common_primitive_inline):
(iso_try_allocate_common_primitive_with_alignment_inline):
(iso_try_allocate_common_primitive_zeroed_inline):
(iso_allocate_common_primitive_inline):
(iso_allocate_common_primitive_with_alignment_inline):
(iso_allocate_common_primitive_zeroed_inline):
(iso_try_reallocate_common_primitive_inline):
(iso_reallocate_common_primitive_inline):
(iso_try_allocate_inline):
(iso_allocate_inline):
(iso_try_allocate_array_by_count_inline):
(iso_allocate_array_by_count_inline):
(iso_try_allocate_array_by_count_zeroed_inline):
(iso_allocate_array_by_count_zeroed_inline):
(iso_try_reallocate_array_by_count_inline):
(iso_reallocate_array_by_count_inline):
(iso_try_allocate_primitive_inline):
(iso_allocate_primitive_inline):
(iso_try_allocate_primitive_zeroed_inline):
(iso_allocate_primitive_zeroed_inline):
(iso_try_allocate_primitive_with_alignment_inline):
(iso_allocate_primitive_with_alignment_inline):
(iso_try_reallocate_primitive_inline):
(iso_reallocate_primitive_inline):
(iso_try_allocate_for_flex_inline):
* Source/bmalloc/libpas/src/libpas/iso_test_heap.c: strip pas_allocation_mode
(iso_test_allocate_common_primitive):
(iso_test_allocate):
(iso_test_allocate_array_by_count):
* Source/bmalloc/libpas/src/libpas/iso_test_heap.h:
* Source/bmalloc/libpas/src/libpas/iso_test_heap_config.c:
* Source/bmalloc/libpas/src/libpas/jit_heap.c: strip pas_allocation_mode
(jit_heap_try_allocate):
* Source/bmalloc/libpas/src/libpas/minalign32_heap.c: strip pas_allocation_mode
(minalign32_allocate_common_primitive):
(minalign32_allocate):
(minalign32_allocate_array_by_count):
* Source/bmalloc/libpas/src/libpas/minalign32_heap.h:
* Source/bmalloc/libpas/src/libpas/minalign32_heap_config.c:
* Source/bmalloc/libpas/src/libpas/pagesize64k_heap.c: strip pas_allocation_mode
(pagesize64k_allocate_common_primitive):
(pagesize64k_allocate):
(pagesize64k_allocate_array_by_count):
* Source/bmalloc/libpas/src/libpas/pagesize64k_heap.h:
* Source/bmalloc/libpas/src/libpas/pagesize64k_heap_config.c:
* Source/bmalloc/libpas/src/libpas/pas_basic_heap_page_caches.h:
* Source/bmalloc/libpas/src/libpas/pas_bitfit_allocator_inlines.h:
(pas_bitfit_allocator_try_allocate):
* Source/bmalloc/libpas/src/libpas/pas_bitfit_page_config.h:
* Source/bmalloc/libpas/src/libpas/pas_bitfit_page_config_inlines.h:
* Source/bmalloc/libpas/src/libpas/pas_bitfit_page_inlines.h:
(pas_bitfit_page_finish_allocation):
(pas_bitfit_page_allocate):
*
Source/bmalloc/libpas/src/libpas/pas_create_basic_heap_page_caches_with_reserved_memory.c:
(allocate_from_megapages):
(pas_create_basic_heap_page_caches_with_reserved_memory):
(allocate_from_compact_megapages): Deleted.
(allocate_from_large):
remove compact megapages
* Source/bmalloc/libpas/src/libpas/pas_deallocate.c:
(pas_try_deallocate_known_large):
(pas_try_deallocate_slow_no_cache):
* Source/bmalloc/libpas/src/libpas/pas_deallocate.h:
(pas_try_deallocate_casual_case):
(pas_try_deallocate_not_small_exclusive_segregated):
* Source/bmalloc/libpas/src/libpas/pas_heap.c:
(pas_heap_create):
* Source/bmalloc/libpas/src/libpas/pas_heap.h:
(pas_heap_for_large_heap):
* Source/bmalloc/libpas/src/libpas/pas_heap_config.h:
* Source/bmalloc/libpas/src/libpas/pas_heap_config_inlines.h:
* Source/bmalloc/libpas/src/libpas/pas_heap_config_utils.h:
* Source/bmalloc/libpas/src/libpas/pas_heap_config_utils_inlines.h:
* Source/bmalloc/libpas/src/libpas/pas_large_heap.c: in order to make
(allocate_impl):
(delegated_allocate_impl):
(should_delegate_user_allocation_to_system_malloc):
(pas_large_heap_try_allocate_and_forget):
(pas_large_heap_try_allocate_user_allocation):
(pas_large_heap_try_deallocate):
(pas_large_heap_construct):
(large_object_delegation_is_enabled):
the new try-deallocate stuff work properly, we need to make sure we
don't remove entries from the large heap unless we're sure they belong
to us, otherwise when we return out a failure the next person to come
along will fail to find it and die
* Source/bmalloc/libpas/src/libpas/pas_large_heap.h:
* Source/bmalloc/libpas/src/libpas/pas_local_allocator.h: remove
is_mte_tagged, since we decide this at the heap-config level now
* Source/bmalloc/libpas/src/libpas/pas_local_allocator_inlines.h:
(pas_local_allocator_try_allocate_with_free_bits):
(pas_local_allocator_try_allocate_inline_cases):
(pas_local_allocator_try_allocate_small_segregated_slow_impl):
(pas_local_allocator_try_allocate_small_segregated_slow):
(pas_local_allocator_try_allocate_out_of_line_cases):
(pas_local_allocator_try_allocate_slow_impl):
(pas_local_allocator_try_allocate_slow):
(pas_local_allocator_try_allocate_inline_only):
(pas_local_allocator_try_allocate):
* Source/bmalloc/libpas/src/libpas/pas_mar_registry.h:
* Source/bmalloc/libpas/src/libpas/pas_megapage_cache.h:
remove compact/non-compact megapage distinction
* Source/bmalloc/libpas/src/libpas/pas_mte.h: use heap-config as source
(pas_mte_generate_tag_and_tag_region):
(pas_mte_maybe_tag_allocated_region):
(pas_mte_retag_freed_region_if_tagged):
(pas_mte_compute_valid_tags_under_adjacent_tag_exclusion):
of truth for mte tagging decision
* Source/bmalloc/libpas/src/libpas/pas_probabilistic_guard_malloc_allocator.c:
strip pas_allocation_mode
(pas_probabilistic_guard_malloc_allocate):
* Source/bmalloc/libpas/src/libpas/pas_probabilistic_guard_malloc_allocator.h:
* Source/bmalloc/libpas/src/libpas/pas_segregated_page_config.h:
* Source/bmalloc/libpas/src/libpas/pas_segregated_page_config_inlines.h:
* Source/bmalloc/libpas/src/libpas/pas_system_heap.h: replace compaction mode
with bool
(pas_system_heap_allocate):
(pas_system_heap_malloc_untagged):
(pas_system_heap_memalign_untagged):
(pas_system_heap_realloc_untagged):
(pas_system_heap_malloc_compact): Deleted.
(pas_system_heap_memalign_compact): Deleted.
(pas_system_heap_realloc_compact): Deleted.
(pas_system_heap_malloc):
(pas_system_heap_memalign):
(pas_system_heap_realloc):
(pas_system_heap_free):
* Source/bmalloc/libpas/src/libpas/pas_try_allocate.h: split alloc casual path
(pas_try_allocate_impl_casual_case):
(pas_try_allocate_impl_inline_only):
* Source/bmalloc/libpas/src/libpas/pas_try_allocate_array.h:
(pas_try_allocate_array_impl_casual_case_with_heap):
(pas_try_allocate_array_impl_inline_only_with_heap):
(pas_try_allocate_array_impl_inline_only):
* Source/bmalloc/libpas/src/libpas/pas_try_allocate_common.h:
(pas_try_allocate_common_impl_fast_inline_only):
(pas_try_allocate_common_impl_fast):
(pas_try_allocate_pgm):
(pas_try_allocate_common_impl_slow):
(pas_try_allocate_common_impl):
* Source/bmalloc/libpas/src/libpas/pas_try_allocate_intrinsic.h:
(pas_try_allocate_intrinsic_impl_casual_case):
(pas_try_allocate_intrinsic_impl_inline_only):
* Source/bmalloc/libpas/src/libpas/pas_try_allocate_primitive.h:
(pas_try_allocate_primitive_impl_casual_case):
(pas_try_allocate_primitive_impl_inline_only):
* Source/bmalloc/libpas/src/libpas/pas_try_reallocate.h: strip
pas_allocation_mode
(pas_try_allocate_for_reallocate_and_copy):
(pas_try_reallocate_table_segregated_case):
(pas_try_reallocate_table_bitfit_case):
(pas_try_reallocate):
(pas_try_reallocate_intrinsic_allocate_callback):
(pas_try_reallocate_intrinsic):
(pas_try_reallocate_single_allocate_callback):
(pas_try_reallocate_single):
(pas_try_reallocate_array_allocate_callback):
(pas_try_reallocate_array_by_size):
(pas_try_reallocate_array_by_count):
(pas_try_reallocate_primitive_allocate_callback):
(pas_try_reallocate_primitive):
* Source/bmalloc/libpas/src/libpas/pas_utility_heap.c:
* Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap.c:
(tagged_bmalloc_try_allocate_casual):
(tagged_bmalloc_allocate_casual):
(tagged_bmalloc_try_deallocate_casual):
(tagged_bmalloc_deallocate_casual):
* Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_config.c:
(tagged_bmalloc_heap_config_activate):
* Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_inlines.h:
(tagged_bmalloc_try_allocate_inline):
(tagged_bmalloc_try_allocate_with_alignment_inline):
(tagged_bmalloc_try_allocate_zeroed_with_alignment_inline):
(tagged_bmalloc_try_allocate_zeroed_inline):
(tagged_bmalloc_allocate_inline):
(tagged_bmalloc_allocate_with_alignment_inline):
(tagged_bmalloc_allocate_zeroed_inline):
(tagged_bmalloc_allocate_zeroed_with_alignment_inline):
(tagged_bmalloc_try_reallocate_inline):
(tagged_bmalloc_reallocate_inline):
(tagged_bmalloc_try_deallocate_inline_only):
* Source/bmalloc/libpas/src/libpas/thingy_heap.c: strip pas_allocation_mode
(thingy_try_allocate_primitive):
(thingy_try_allocate_primitive_zeroed):
(thingy_try_allocate_primitive_with_alignment):
(thingy_try_reallocate_primitive):
(thingy_try_allocate):
(thingy_try_allocate_zeroed):
(thingy_try_allocate_array):
(thingy_try_allocate_zeroed_array):
(thingy_try_reallocate_array):
(thingy_utility_heap_allocate):
* Source/bmalloc/libpas/src/libpas/thingy_heap.h:
* Source/bmalloc/libpas/src/libpas/thingy_heap_config.c:
* Source/bmalloc/libpas/src/libpas/thingy_heap_prefix.h:
* Source/bmalloc/CMakeLists.txt:
* Source/bmalloc/bmalloc.xcodeproj/project.pbxproj:
* Source/bmalloc/bmalloc/CompactAllocationMode.h:
(bmalloc::asPasAllocationMode): Deleted.
* Source/bmalloc/libpas/src/chaos/Chaos.cpp:
(std::threadMain):
* Source/bmalloc/libpas/src/libpas/pas_allocation_mode.h: Removed.
* Source/bmalloc/libpas/src/libpas/pas_stats.c:
(pas_stats_page_alloc_counts_record):
(pas_stats_page_alloc_counts_dump_to_json):
* Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_config.h:
* Source/bmalloc/libpas/src/mbmalloc/mbmalloc_bmalloc.c:
(mbmalloc):
(mbmemalign):
(mbrealloc):
* Source/bmalloc/libpas/src/mbmalloc/mbmalloc_hotbit.c:
(mbmalloc):
(mbmemalign):
(mbrealloc):
* Source/bmalloc/libpas/src/mbmalloc/mbmalloc_iso_common_primitive.c:
(mbmalloc):
(mbmemalign):
(mbrealloc):
* Source/bmalloc/libpas/src/test/AllocationZeroingTests.cpp:
* Source/bmalloc/libpas/src/test/BitfitTests.cpp:
(std::testAllocateAlignedSmallerThanSizeClassAndSmallerThanLargestAvailable):
(std::testTakeLastEmptyLastEmptyPlusOneWatchRace):
* Source/bmalloc/libpas/src/test/BmallocTests.cpp:
(std::testBmallocAllocate):
(std::testBmallocAllocationZeroing):
(std::testBmallocAllocationAlignment):
(std::testBmallocDeallocate):
(std::testBmallocForceBitfitAfterAlloc):
(std::testBmallocDisableAllocationsAboveMTETaggingCeiling):
(std::testBmallocSmallIndexOverlap):
(std::testBmallocOwnsObject):
(addBmallocTests):
(std::testBmallocLargeObjectDelegation):
* Source/bmalloc/libpas/src/test/EnumerationTests.cpp:
(std::testBasicEnumeration):
(std::testPGMEnumerationBasic):
(std::testPGMEnumerationAddAndFree):
(std::testEnumerationInvalidCompactHeapBump):
* Source/bmalloc/libpas/src/test/ExpendableMemoryTests.cpp:
(std::testPayloadImpl):
(std::testRage):
(std::testRematerializeAfterSearchOfDecommitted):
(std::testBasicSizeClass):
(std::testSoManyHeaps):
* Source/bmalloc/libpas/src/test/HeapRefAllocatorIndexTests.cpp:
* Source/bmalloc/libpas/src/test/IsoDynamicPrimitiveHeapTests.cpp:
(std::allocate42):
(std::allocate42WithAlignment):
(std::allocate42Zeroed):
(std::reallocate42):
* Source/bmalloc/libpas/src/test/IsoHeapChaosTests.cpp:
(std::gigacageAllocate):
(std::testAllocationChaos):
(std::addAllTests):
(std::jit_heap_test_try_allocate): Deleted.
(std::createBmallocHeapRefForSize):
* Source/bmalloc/libpas/src/test/IsoHeapPageSharingTests.cpp:
(std::testTakePages):
(std::testTakePagesFromCorrectHeap):
(std::testLargeHeapTakesPagesFromCorrectSmallHeap):
(std::testLargeHeapTakesPagesFromCorrectSmallHeapAllocateAfterFree):
(std::testLargeHeapTakesPagesFromCorrectSmallHeapWithFancyOrder):
(std::testLargeHeapTakesPagesFromCorrectLargeHeap):
(std::testLargeHeapTakesPagesFromCorrectLargeHeapAllocateAfterFreeOnSmallHeap):
(std::testLargeHeapTakesPagesFromCorrectLargeHeapAllocateAfterFreeOnAnotherLargeHeap):
(std::testLargeHeapTakesPagesFromCorrectLargeHeapWithFancyOrder):
(std::testSmallHeapTakesPagesFromCorrectLargeHeap):
(std::testSmallHeapTakesPagesFromCorrectLargeHeapWithFancyOrder):
(std::testSmallHeapTakesPagesFromCorrectLargeHeapAllocateAfterFreeOnSmallHeap):
(std::testSmallHeapTakesPagesFromCorrectLargeHeapAllocateAfterFreeOnAnotherLargeHeap):
(std::allocateThingiesImpl):
(std::testScavengerEventuallyReturnsMemory):
(std::testScavengerEventuallyReturnsMemoryEvenWithoutManualShrink):
* Source/bmalloc/libpas/src/test/IsoHeapPartialAndBaselineTests.cpp:
(std::testTwoBaselinesEvictions):
* Source/bmalloc/libpas/src/test/IsoHeapReservedMemoryTests.cpp:
(std::testSizeProgression):
* Source/bmalloc/libpas/src/test/LotsOfHeapsAndThreads.cpp:
(std::testLotsOfHeapsAndThreads):
* Source/bmalloc/libpas/src/test/MemalignTests.cpp:
* Source/bmalloc/libpas/src/test/PGMTests.cpp:
(std::testPGMSingleAlloc):
(std::testPGMMultipleAlloc):
(std::testPGMAlignedAlloc):
(std::testPGMRealloc):
(std::testPGMMetaData):
(std::testPGMErrors):
(std::testPGMMetadataVectorManagement):
(std::testPGMMetadataVectorManagementFewDeallocations):
(std::testPGMMetadataDoubleFreeBehavior):
(std::testPGMMetadataVectorManagementRehash):
(std::testPGMBmallocAllocationBacktrace):
(std::testPGMAllocMetadataOnly):
(std::testPGMCrashReportTaskPassthrough):
* Source/bmalloc/libpas/src/test/RaceTests.cpp:
(std::testLocalAllocatorStopRace):
(std::testLocalAllocatorStopRaceAgainstScavenge):
(std::testMediumDirectoryTornInsertRace):
* Source/bmalloc/libpas/src/test/ReallocFastPathTests.cpp:
(std::testReallocFastPathGrowWithinSizeClass):
(std::testReallocFastPathShrinkWithinSizeClass):
(std::testReallocFastPathToExactUsableSize):
(std::testReallocGrowBeyondSizeClassCopiesData):
(std::testReallocFastPathSweepAcrossSizes):
(std::testReallocBitfitFastPath):
(std::testReallocExcessiveShrinkForcesCopy):
(std::testReallocLargeDataPreserved):
(std::testReallocFromNullAllocates):
* Source/bmalloc/libpas/src/test/TLCDecommitTests.cpp:
(std::testTLCDecommit):
(std::testChaosThenDecommit):
(std::testAllocateFromStoppedBaselineImpl):
* Source/bmalloc/libpas/src/test/TSDTests.cpp:
(std::destructor):
(std::testTSD):
* Source/bmalloc/libpas/src/test/ThingyAndUtilityHeapAllocationTests.cpp:
(std::testFreeListRefillSpans):
(std::testInternalScavenge):
(std::testInternalScavengeFromCorrectDirectory):
(std::testSpuriousEligibility):
(std::testBasicSizeClassNotSet):
(std::testSmallDoubleFree):
(std::testSmallFreeInner):
(std::testSmallFreeNextWithShrink):
(std::testSmallFreeNextWithoutShrink):
(std::testSmallFreeNextBeforeShrink):
(std::testAllocationChaos):
(std::testCombinedAllocationChaos):
(std::testLargeDoubleFree):
(std::testLargeOffsetFree):
(std::testReallocatePrimitive):
(std::testReallocateArray):
* Source/bmalloc/libpas/src/test/ViewCacheTests.cpp:
(std::testDisableViewCacheUsingBoundForNoViewCache):
(std::testEnableViewCacheAtSomeBoundForNoViewCache):
* Source/bmalloc/libpas/src/libpas/bmalloc_heap_ref.h:
* Source/bmalloc/libpas/src/libpas/pas_heap_ref_prefix.h:
* Source/bmalloc/libpas/libpas.xcodeproj/project.pbxproj:
* Source/bmalloc/libpas/src/libpas/pas_enumerable_page_malloc.c:
(pas_enumerable_page_malloc_try_allocate_without_deallocating_padding):
* Source/bmalloc/libpas/src/libpas/pas_enumerable_page_malloc.h:
* Source/bmalloc/libpas/src/libpas/pas_heap_kind.h:
(pas_heap_kind_get_string):
* Source/bmalloc/libpas/src/libpas/pas_large_heap_physical_page_sharing_cache.h:
* Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_free_heap.c: Renamed from
Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_free_heap.c.
* Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_free_heap.h: Renamed from
Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_free_heap.h.
* Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_heap_page_provider.c:
Renamed from
Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_heap_page_provider.c.
(pas_mte_bootstrap_heap_page_provider):
* Source/bmalloc/libpas/src/libpas/pas_mte_bootstrap_heap_page_provider.h:
Renamed from
Source/bmalloc/libpas/src/libpas/pas_small_medium_bootstrap_heap_page_provider.h.
* Source/bmalloc/libpas/src/libpas/pas_page_malloc.c:
(pas_page_malloc_try_map_pages):
(pas_page_malloc_try_allocate_without_deallocating_padding):
* Source/bmalloc/libpas/src/libpas/pas_page_malloc.h:
* Source/bmalloc/libpas/src/libpas/pas_stats.h:
* Source/bmalloc/libpas/src/libpas/pas_heap_config_utils.c:
(pas_heap_config_utils_allocate_aligned):
* Source/bmalloc/libpas/src/libpas/pas_report_crash_pgm_report.h:
* Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_utils.c:
(tagged_bmalloc_owns_object):
* Source/bmalloc/libpas/src/libpas/tagged_bmalloc_heap_utils.h:
* Source/bmalloc/libpas/src/libpas/bmalloc_prefault_supply.c:
(refill):
(bmalloc_prefault_supply_try_allocate):
* Source/bmalloc/libpas/src/libpas/pas_ensure_heap_with_page_caches.c:
(pas_ensure_heap_with_page_caches):
* Source/bmalloc/libpas/src/libpas/pas_heap_runtime_config.h:
Canonical link: https://commits.webkit.org/322175@main
To unsubscribe from these emails, change your notification settings at
https://github.com/WebKit/WebKit/settings/notifications
