Thanks for the patch! yet something to improve: On Mon, Nov 02, 2026 at 05:58:04PM +0200, Shmuel Fomberg wrote: > hv-memory-hint (hypercall HvExtCallMemoryHeatHint) allow a guest to > notify host which memory pages will be needed in the near future. > This is not a replacement for the balloon functionality, but can > help the kernel
given this is supposed to be an optimization, I expect to see some performance numbers showing it actually helps, and some documentation explaining when it does and when it doesn't. e.g. does it work with vhost-user (shmem)? hugetlb? pls document which configs were tested, with linux and windows guests. > make better decisions when memory is scare. scarse? > > Signed-off-by: Shmuel Fomberg <[email protected]> > --- > docs/system/i386/hyperv.rst | 14 +++ > hw/hyperv/hyperv.c | 158 +++++++++++++++++++++++++++++++ > include/hw/hyperv/hyperv-proto.h | 25 +++++ > include/hw/hyperv/hyperv.h | 13 +++ > include/qemu/madvise.h | 7 ++ > target/i386/cpu.c | 3 + > target/i386/cpu.h | 2 + > target/i386/kvm/hyperv-proto.h | 1 + > target/i386/kvm/hyperv.c | 24 +++++ > target/i386/kvm/kvm.c | 7 ++ > 10 files changed, 254 insertions(+) > > diff --git a/docs/system/i386/hyperv.rst b/docs/system/i386/hyperv.rst > index e1babffb0f..1e164f1863 100644 > --- a/docs/system/i386/hyperv.rst > +++ b/docs/system/i386/hyperv.rst > @@ -259,6 +259,15 @@ Existing enlightenments > > Recommended: ``hv-evmcs`` (Intel) > > +``hv-extcalls`` > + Advertises extended Hyper-V hypercalls (call codes 0x8000 and above) to > + the guest. > + > +``hv-memory-hint`` > + Enables guest memory heat hints (HvExtCallMemoryHeatHint). > + > + Requires: ``hv-extcalls`` > + > Supplementary features > ---------------------- > > @@ -276,6 +285,11 @@ Supplementary features > Windows guests from booting when used without proper configuration. If > needed, > ``hv-syndbg`` can be enabled additionally. > > + Note: ``hv-passthrough`` enables ``hv-extcalls`` when the host KVM > advertises > + it. It does not enable ``hv-memory-hint``, which is implemented in QEMU and > + has no KVM CPUID bit. Enable ``hv-memory-hint`` explicitly when guest heat > + hints should reach the host. > + I'm not sure what all this means. "KVM CPUID" is also obscure. Can you pls rewrite? Maybe along the lines of: if .... and .... then ... results in ... if .... and .... then ... is ignored > Note: ``hv-passthrough`` effectively prevents migration as the list of > enabled > enlightenments may differ between target and destination hosts. > > diff --git a/hw/hyperv/hyperv.c b/hw/hyperv/hyperv.c > index 900ff80213..b3fce7b563 100644 > --- a/hw/hyperv/hyperv.c > +++ b/hw/hyperv/hyperv.c > @@ -20,7 +20,9 @@ > #include "system/physmem.h" > #include "qemu/bitops.h" > #include "qemu/error-report.h" > +#include "qemu/int128.h" > #include "qemu/lockable.h" > +#include "qemu/madvise.h" > #include "qemu/queue.h" > #include "qemu/rcu.h" > #include "qemu/rcu_queue.h" > @@ -646,6 +648,162 @@ unmap: > return ret; > } > > +uint16_t hyperv_hcall_ext_capabilities(uint64_t outgpa, bool fast, > + bool memory_hint) > +{ > + uint16_t ret; > + hwaddr len; > + uint64_t *caps = NULL; > + > + if (fast) { > + return HV_STATUS_INVALID_HYPERCALL_CODE; > + } > + if (outgpa & (__alignof__(*caps) - 1)) { > + return HV_STATUS_INVALID_ALIGNMENT; > + } > + > + len = sizeof(*caps); > + caps = physical_memory_map(outgpa, &len, 1); > + if (!caps || len < sizeof(*caps)) { > + ret = HV_STATUS_INSUFFICIENT_MEMORY; > + goto unmap; > + } > + > + *caps = 0; > + if (memory_hint) { > + *caps = HV_EXT_CAP_MEMORY_HEAT_HINT; > + if (QEMU_MADV_COLD != QEMU_MADV_INVALID) { > + *caps |= HV_EXT_CAP_MEMORY_HEAT_HINT_COLD; > + } > + } > + ret = HV_STATUS_SUCCESS; > + > +unmap: > + if (caps) { > + physical_memory_unmap(caps, len, 1, ret == HV_STATUS_SUCCESS ? len : > 0); > + } > + return ret; > +} > + > +static bool hv_decode_gpa_page_range(uint64_t raw, hwaddr *gpa, hwaddr *len) > +{ > + uint64_t pages = (raw & HV_GPA_PAGE_RANGE_ADDITIONAL_MASK) + 1; shouldn't we check alignment and error out rather than force it? > + > + if (raw & HV_GPA_PAGE_RANGE_LARGEPAGE) { > + if (raw & HV_GPA_PAGE_RANGE_PAGE_SIZE_1GB) { > + *gpa = raw & HV_GPA_PAGE_RANGE_1G_GPA_MASK; > + *len = pages << 30; > + } else { > + *gpa = raw & HV_GPA_PAGE_RANGE_2M_GPA_MASK; > + *len = pages << 21; > + } > + } else { > + *gpa = raw & HV_GPA_PAGE_RANGE_4K_GPA_MASK; > + *len = pages << 12; > + } > + > + return *len != 0; how can len be 0? > +} > + > +static void hv_madvise_gpa_range(hwaddr gpa, hwaddr len, int advice) > +{ > + while (len) { > + MemoryRegionSection section; > + hwaddr chunk, start, skip; > + void *host; > + > + section = memory_region_find(get_system_memory(), gpa, len); > + if (!section.mr) { > + break; why break? we could have an unmapped hole followed by a mapped one. > + } > + > + start = section.offset_within_address_space; > + chunk = int128_get64(section.size); > + > + if (!chunk || start < gpa) { is this dead code? we got the mr from find can start be before? > + memory_region_unref(section.mr); > + break; > + } > + > + skip = start - gpa; > + if (skip >= len || chunk > len - skip) { can this trigger? or more dead code? > + memory_region_unref(section.mr); > + break; > + } > + > + gpa = start + chunk; > + len -= skip + chunk; > + > + if (memory_region_is_ram(section.mr) && > + !memory_region_is_rom(section.mr) && > + !memory_region_is_romd(section.mr)) { > + host = memory_region_get_ram_ptr(section.mr) + > + section.offset_within_region; > + qemu_madvise(host, chunk, advice); > + } > + memory_region_unref(section.mr); > + } > +} > + > +uint16_t hyperv_hcall_memory_hint(uint64_t in_gpa, uint16_t rep_count, > + bool fast) > +{ > + uint16_t ret = HV_STATUS_SUCCESS; > + hwaddr in_len, needed; > + struct hyperv_memory_heat_hint *in = NULL; > + unsigned int i; > + uint64_t heat_type; > + int advice; > + > + if (fast) { > + return HV_STATUS_INVALID_HYPERCALL_CODE; > + } > + if (rep_count < 1 || rep_count > HV_EXT_MEMORY_HEAT_HINT_MAX_RANGES) { > + return HV_STATUS_INVALID_HYPERCALL_INPUT; > + } > + if (in_gpa & (__alignof__(*in) - 1)) { > + return HV_STATUS_INVALID_ALIGNMENT; > + } > + > + needed = sizeof(in->heat_type) + sizeof(in->ranges[0]) * rep_count; > + in_len = TARGET_PAGE_SIZE; > + in = physical_memory_map(in_gpa, &in_len, 0); > + if (!in || in_len < needed) { > + ret = HV_STATUS_INSUFFICIENT_MEMORY; > + goto unmap; needed can be 4096 with rep_count 511, can it not? if the input is not page aligned or end of page is not ram, will this not fail? why not just map needed bytes? > + } > + > + heat_type = in->heat_type & 3; I would add #define HV_EXTMEM_HEAT_HINT_MASK 3 or (HV_EXTMEM_HEAT_HINT_HOT|HV_EXTMEM_HEAT_HINT_COLD|HV_EXTMEM_HEAT_HINT_COLD_DISCARD) also are you really supposed to just ignore rest of bits? add a code comment explaining why. > + switch (heat_type) { > + case HV_EXTMEM_HEAT_HINT_HOT: > + advice = QEMU_MADV_WILLNEED; > + break; > + case HV_EXTMEM_HEAT_HINT_COLD: > + case HV_EXTMEM_HEAT_HINT_COLD_DISCARD: > + advice = QEMU_MADV_COLD; > + break; > + default: > + ret = HV_STATUS_INVALID_PARAMETER; > + goto unmap; > + } Pls document what are these things supposed to do, with code comments. For example "discard" sounds like MADV_DONTNEED, MADV_REMOVE or MADV_FREE or a combination of these to me? "COLD" sounds kinda like MADV_COLD but there's also MADV_PAGEOUT which is same but more aggressive. > + > + for (i = 0; i < rep_count; i++) { > + hwaddr gpa, len; > + uint64_t raw = in->ranges[i]; > + > + if (!hv_decode_gpa_page_range(raw, &gpa, &len)) { > + continue; what's the point of this? does it ever trigger? > + } > + hv_madvise_gpa_range(gpa, len, advice); > + } > + > +unmap: > + if (in) { > + physical_memory_unmap(in, in_len, 0, 0); > + } > + return ret; > +} > + > static int set_event_flag_handler(uint32_t conn_id, EventNotifier *notifier) > { > int ret; > diff --git a/include/hw/hyperv/hyperv-proto.h > b/include/hw/hyperv/hyperv-proto.h > index fffc5ce342..67e6f72a90 100644 > --- a/include/hw/hyperv/hyperv-proto.h > +++ b/include/hw/hyperv/hyperv-proto.h > @@ -35,7 +35,32 @@ > #define HV_POST_DEBUG_DATA 0x0069 > #define HV_RETRIEVE_DEBUG_DATA 0x006a > #define HV_RESET_DEBUG_SESSION 0x006b > +#define HV_EXT_CALL_QUERY_CAPABILITIES 0x8001 > +#define HV_EXT_CALL_MEMORY_HEAT_HINT 0x8003 > #define HV_HYPERCALL_FAST (1u << 16) > +#define HV_HYPERCALL_REP_COMP_OFFSET 32 > +#define HV_HYPERCALL_REP_COMP_MASK 0xfff > + > +#define HV_EXT_CAP_MEMORY_HEAT_HINT (1ull << 1) > +#define HV_EXT_CAP_MEMORY_HEAT_HINT_COLD (1ull << 2) Here: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/include/hyperv/hvgdk_ext.h I see: #define HV_EXT_CAPABILITY_MEMORY_COLD_DISCARD_HINT BIT(8) > +#define HV_EXT_MEMORY_HEAT_HINT_MAX_RANGES 511 > + > +#define HV_EXTMEM_HEAT_HINT_COLD 0 > +#define HV_EXTMEM_HEAT_HINT_HOT 1 > +#define HV_EXTMEM_HEAT_HINT_COLD_DISCARD 2 > + > +struct hyperv_memory_heat_hint { > + uint64_t heat_type; > + uint64_t ranges[HV_EXT_MEMORY_HEAT_HINT_MAX_RANGES]; > +}; > + > +/* HV_GPA_PAGE_RANGE in each heat-hint range qword */ > +#define HV_GPA_PAGE_RANGE_ADDITIONAL_MASK 0x7ffull > +#define HV_GPA_PAGE_RANGE_LARGEPAGE (1ull << 11) > +#define HV_GPA_PAGE_RANGE_PAGE_SIZE_1GB (1ull << 12) > +#define HV_GPA_PAGE_RANGE_4K_GPA_MASK (~0xfffull) > +#define HV_GPA_PAGE_RANGE_2M_GPA_MASK (~0x1fffffull) > +#define HV_GPA_PAGE_RANGE_1G_GPA_MASK (~0x3fffffffull) > Can you try to move the linux bits out to a uapi file and import from there, instead of duplicating? > /* > * Message size > diff --git a/include/hw/hyperv/hyperv.h b/include/hw/hyperv/hyperv.h > index 23091301d0..8cccbb91ba 100644 > --- a/include/hw/hyperv/hyperv.h > +++ b/include/hw/hyperv/hyperv.h > @@ -70,6 +70,19 @@ uint16_t hyperv_hcall_post_message(uint64_t param, bool > fast); > * the connection as specified in @param. > */ > uint16_t hyperv_hcall_signal_event(uint64_t param, bool fast); > +/* > + * Process HV_EXT_CALL_QUERY_CAPABILITIES hypercall: list supported ext hcall > + * fill the capabilities bit array in @outgpa. > + */ > +uint16_t hyperv_hcall_ext_capabilities(uint64_t outgpa, bool fast, > + bool memory_hint); > +/* > + * Process HV_EXT_CALL_MEMORY_HEAT_HINT hypercall: receive a list of hot/cold > + * pages and call madvice on them. madvise > + */ > +uint16_t hyperv_hcall_memory_hint(uint64_t in_gpa, uint16_t rep_count, > + bool fast); > + > > static inline uint32_t hyperv_vp_index(CPUState *cs) > { > diff --git a/include/qemu/madvise.h b/include/qemu/madvise.h > index e155f59a0d..fd638c2254 100644 > --- a/include/qemu/madvise.h > +++ b/include/qemu/madvise.h > @@ -59,6 +59,11 @@ > #else > #define QEMU_MADV_POPULATE_WRITE QEMU_MADV_INVALID > #endif > +#ifdef MADV_COLD > +#define QEMU_MADV_COLD MADV_COLD > +#else > +#define QEMU_MADV_COLD QEMU_MADV_INVALID > +#endif > > #elif defined(CONFIG_POSIX_MADVISE) > > @@ -73,6 +78,7 @@ > #define QEMU_MADV_NOHUGEPAGE QEMU_MADV_INVALID > #define QEMU_MADV_REMOVE QEMU_MADV_DONTNEED > #define QEMU_MADV_POPULATE_WRITE QEMU_MADV_INVALID > +#define QEMU_MADV_COLD QEMU_MADV_INVALID > > #else /* no-op */ > > @@ -87,6 +93,7 @@ > #define QEMU_MADV_NOHUGEPAGE QEMU_MADV_INVALID > #define QEMU_MADV_REMOVE QEMU_MADV_INVALID > #define QEMU_MADV_POPULATE_WRITE QEMU_MADV_INVALID > +#define QEMU_MADV_COLD QEMU_MADV_INVALID > > #endif > > diff --git a/target/i386/cpu.c b/target/i386/cpu.c > index e65eca0459..263523d052 100644 > --- a/target/i386/cpu.c > +++ b/target/i386/cpu.c > @@ -10824,6 +10824,9 @@ static const Property x86_cpu_properties[] = { > HYPERV_FEAT_TLBFLUSH_EXT, 0), > DEFINE_PROP_BIT64("hv-tlbflush-direct", X86CPU, hyperv_features, > HYPERV_FEAT_TLBFLUSH_DIRECT, 0), > + DEFINE_PROP_BIT64("hv-extcalls", X86CPU, hyperv_features, > + HYPERV_FEAT_EXT_HYPERCALLS, 0), > + DEFINE_PROP_BOOL("hv-memory-hint", X86CPU, hyperv_memory_hint, false), > DEFINE_PROP_ON_OFF_AUTO("hv-no-nonarch-coresharing", X86CPU, > hyperv_no_nonarch_cs, ON_OFF_AUTO_OFF), > #ifdef CONFIG_SYNDBG > diff --git a/target/i386/cpu.h b/target/i386/cpu.h > index 3945e90371..3c9da61e17 100644 > --- a/target/i386/cpu.h > +++ b/target/i386/cpu.h > @@ -1489,6 +1489,7 @@ uint64_t x86_cpu_get_supported_feature_word(X86CPU > *cpu, FeatureWord w); > #define HYPERV_FEAT_XMM_INPUT 18 > #define HYPERV_FEAT_TLBFLUSH_EXT 19 > #define HYPERV_FEAT_TLBFLUSH_DIRECT 20 > +#define HYPERV_FEAT_EXT_HYPERCALLS 21 > > #ifndef HYPERV_SPINLOCK_NEVER_NOTIFY > #define HYPERV_SPINLOCK_NEVER_NOTIFY 0xFFFFFFFF > @@ -2374,6 +2375,7 @@ struct ArchCPU { > uint32_t hyperv_spinlock_attempts; > char *hyperv_vendor; > uint64_t hyperv_features; > + bool hyperv_memory_hint; > bool hyperv_passthrough; > OnOffAuto hyperv_no_nonarch_cs; > uint32_t hyperv_vendor_id[3]; > diff --git a/target/i386/kvm/hyperv-proto.h b/target/i386/kvm/hyperv-proto.h > index a9f056f2f3..e88913fd37 100644 > --- a/target/i386/kvm/hyperv-proto.h > +++ b/target/i386/kvm/hyperv-proto.h > @@ -66,6 +66,7 @@ > * HV_CPUID_FEATURES.EBX bits > */ > #define HV_PARTITION_DEBUGGING_ALLOWED (1u << 12) > +#define HV_ENABLE_EXTENDED_HYPERCALLS (1u << 20) > > /* > * HV_CPUID_ENLIGHTMENT_INFO.EAX bits > diff --git a/target/i386/kvm/hyperv.c b/target/i386/kvm/hyperv.c > index 420c76b5ff..9d97f5cb4e 100644 > --- a/target/i386/kvm/hyperv.c > +++ b/target/i386/kvm/hyperv.c > @@ -97,6 +97,13 @@ int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit > *exit) > bool fast = exit->u.hcall.input & HV_HYPERCALL_FAST; > uint64_t in_param = exit->u.hcall.params[0]; > uint64_t out_param = exit->u.hcall.params[1]; > + uint64_t rep_count = (exit->u.hcall.input >> 32) & 0xFFF; > + > + if ((code & 0x8000) && > + !hyperv_feat_enabled(cpu, HYPERV_FEAT_EXT_HYPERCALLS)) { > + exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE; > + return 0; > + } > > switch (code) { > case HV_POST_MESSAGE: > @@ -117,6 +124,23 @@ int kvm_hv_handle_exit(X86CPU *cpu, struct > kvm_hyperv_exit *exit) > exit->u.hcall.result = > hyperv_hcall_reset_dbg_session(out_param); > break; > + case HV_EXT_CALL_QUERY_CAPABILITIES: > + exit->u.hcall.result = hyperv_hcall_ext_capabilities( > + out_param, fast, cpu->hyperv_memory_hint); > + break; > + case HV_EXT_CALL_MEMORY_HEAT_HINT: > + if (!cpu->hyperv_memory_hint) { > + exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE; > + break; > + } > + exit->u.hcall.result = > + hyperv_hcall_memory_hint(in_param, rep_count, fast); > + if (exit->u.hcall.result == HV_STATUS_SUCCESS) { > + exit->u.hcall.result |= > + (rep_count & HV_HYPERCALL_REP_COMP_MASK) << > + HV_HYPERCALL_REP_COMP_OFFSET; > + } > + break; > default: > exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE; > } > diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c > index 072da8b9f1..46c983f963 100644 > --- a/target/i386/kvm/kvm.c > +++ b/target/i386/kvm/kvm.c > @@ -1137,6 +1137,13 @@ static struct { > }, > .dependencies = BIT(HYPERV_FEAT_VAPIC) > }, > + [HYPERV_FEAT_EXT_HYPERCALLS] = { > + .desc = "enable extended hypercalls (hv-extcalls)", > + .flags = { > + {.func = HV_CPUID_FEATURES, .reg = R_EBX, > + .bits = HV_ENABLE_EXTENDED_HYPERCALLS} > + } > + }, > }; > > static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max, > -- > 2.54.0 (Apple Git-157) > >
