Add read-only statistics registers to the migration BAR at 0x100-0x11C for monitoring dirty page tracking and DMA activity per VF.
The dma_writes counter is also reported in the dirty query buffer so the driver gets it alongside the bitmap without an extra MMIO read. Counters are reset on first DIRTY_CTRL=ENABLE or device reset, so the driver can read final values after DIRTY_CTRL=DISABLE. Signed-off-by: Cédric Le Goater <[email protected]> --- docs/system/devices/igb-migration.rst | 24 ++++++++++- hw/net/igb_core.h | 1 + hw/net/igb_migration.h | 22 +++++++++- hw/net/igb_migration.c | 59 +++++++++++++++++++++++++-- hw/net/trace-events | 2 +- 5 files changed, 102 insertions(+), 6 deletions(-) diff --git a/docs/system/devices/igb-migration.rst b/docs/system/devices/igb-migration.rst index 0e34e4b9fc68..3f861eed59cd 100644 --- a/docs/system/devices/igb-migration.rst +++ b/docs/system/devices/igb-migration.rst @@ -127,7 +127,9 @@ device-written completion fields:: 0x40 status device 0 = pending, 1 = complete 0x44 bitmap_size device Bytes written to bitmap 0x48 dirty_page_count device Number of set bits in bitmap - 0x4C reserved[12] - Pad to 64-byte cache line + 0x4C reserved - Padding + 0x50 dma_writes device Total DMA writes since enable + 0x58 reserved[10] - Pad to 64-byte cache line 0x80 bitmap[] device Dirty page bitmap The driver fills the request fields, issues ``DIRTY_CTRL=QUERY``, and @@ -135,6 +137,26 @@ polls ``status`` for completion. The device reads the request, writes the dirty bitmap and completion fields via DMA, then sets ``status = 1``. +Migration statistics +~~~~~~~~~~~~~~~~~~~~ + +The migration BAR includes read-only statistics counters in a dedicated +aperture at offset ``0x100``. These counters provide a live +per-migration-cycle view of activity without requiring a +``DIRTY_CTRL=QUERY``:: + + Offset Name Description + 0x100 MIG_STAT_DMA_WRITES Total DMA write operations tracked + 0x104 MIG_STAT_DMA_BYTES_LO Total DMA bytes written (low 32 bits) + 0x108 MIG_STAT_DMA_BYTES_HI Total DMA bytes written (high 32 bits) + 0x10C MIG_STAT_DIRTY_PAGES_SET Dirty pages marked since enable + 0x110 MIG_STAT_DIRTY_PAGES_CLR Dirty pages cleared by queries + 0x114 MIG_STAT_DIRTY_PAGE_COUNT Current dirty pages (set minus cleared) + 0x118 MIG_STAT_DIRTY_QUERY_CNT Number of QUERY operations + +All counters are reset on first ``DIRTY_CTRL=ENABLE`` or device reset, +so the driver can read final values after ``DIRTY_CTRL=DISABLE``. + Testing setup ~~~~~~~~~~~~~ diff --git a/hw/net/igb_core.h b/hw/net/igb_core.h index 07b4270708c4..91796aec8d7a 100644 --- a/hw/net/igb_core.h +++ b/hw/net/igb_core.h @@ -84,6 +84,7 @@ struct IGBCore { } tx[IGB_NUM_QUEUES]; IGBVfDirtyState vf_dirty[IGB_MAX_VF_FUNCTIONS]; + IgbVfMigStats vf_mig_stats[IGB_MAX_VF_FUNCTIONS]; struct NetRxPkt *rx_pkt; diff --git a/hw/net/igb_migration.h b/hw/net/igb_migration.h index 414412f6392e..6d2e3442781f 100644 --- a/hw/net/igb_migration.h +++ b/hw/net/igb_migration.h @@ -64,6 +64,15 @@ #define IGB_MIG_DIRTY_BUF_ADDR_HI 0x038 #define IGB_MIG_DIRTY_STATUS 0x03C +/* Migration statistics registers (read-only) */ +#define IGB_MIG_STAT_DMA_WRITES 0x100 +#define IGB_MIG_STAT_DMA_BYTES_LO 0x104 +#define IGB_MIG_STAT_DMA_BYTES_HI 0x108 +#define IGB_MIG_STAT_DIRTY_PAGES_SET 0x10C +#define IGB_MIG_STAT_DIRTY_PAGES_CLR 0x110 +#define IGB_MIG_STAT_DIRTY_PAGE_COUNT 0x114 +#define IGB_MIG_STAT_DIRTY_QUERY_CNT 0x118 + /* DEVICE_STATE values - mirrors VFIO migration states */ #define IGB_MIG_STATE_ERROR 0 #define IGB_MIG_STATE_STOP 1 @@ -126,7 +135,9 @@ struct igb_mig_dirty_query { uint32_t status; uint32_t bitmap_size; uint32_t dirty_page_count; - uint32_t reserved1[12]; + uint32_t reserved1; + uint64_t dma_writes; + uint32_t reserved2[10]; /* Cache line 2+: bitmap (written by device) */ uint8_t bitmap[]; @@ -154,6 +165,15 @@ typedef struct IgbVfMigState { uint32_t mig_dirty_status; } IgbVfMigState; +typedef struct IgbVfMigStats { + uint32_t dma_writes; + uint64_t dma_bytes; + uint32_t dirty_pages_set; + uint32_t dirty_pages_cleared; + uint32_t dirty_page_count; + uint32_t dirty_query_count; +} IgbVfMigStats; + typedef struct IgbVfState IgbVfState; void igb_pf_init_migration_bar(PCIDevice *dev); bool igbvf_add_migration_cap(PCIDevice *dev, Error **errp); diff --git a/hw/net/igb_migration.c b/hw/net/igb_migration.c index fce661f68e3d..7fce8d535909 100644 --- a/hw/net/igb_migration.c +++ b/hw/net/igb_migration.c @@ -458,6 +458,7 @@ void igb_core_dirty_track_dma(IGBCore *core, int vfn, dma_addr_t addr, dma_addr_t len) { IGBVfDirtyState *ds = &core->vf_dirty[vfn]; + IgbVfMigStats *stats = &core->vf_mig_stats[vfn]; bool matched = false; uint32_t i; @@ -465,6 +466,8 @@ void igb_core_dirty_track_dma(IGBCore *core, int vfn, return; } + stats->dma_writes++; + stats->dma_bytes += len; trace_igb_core_dirty_track_dma(vfn, addr, len); for (i = 0; i < ds->num_ranges; i++) { @@ -486,7 +489,10 @@ void igb_core_dirty_track_dma(IGBCore *core, int vfn, for (page = start_page; page <= end_page; page++) { if (page < r->nbits) { - set_bit(page, r->bitmap); + if (!test_and_set_bit(page, r->bitmap)) { + stats->dirty_pages_set++; + stats->dirty_page_count++; + } } } } @@ -510,6 +516,15 @@ static uint32_t igb_core_vf_dirty_enable(IgbVfState *s, uint64_t pgsize, IGBVfDirtyState *ds = igb_core_vf_dirty_state(s); IGBVfDirtyRange *r; + /* + * Reset stats on first enable so the driver can read them after + * disable + */ + if (ds->num_ranges == 0) { + memset(&igbvf_get_core(s)->vf_mig_stats[s->vfn], 0, + sizeof(IgbVfMigStats)); + } + if (ds->num_ranges >= IGB_MIG_CAPS_MAX_RANGES) { return IGB_MIG_DIRTY_STATUS_TOO_MANY_RANGES; } @@ -561,7 +576,9 @@ static bool igb_core_vf_dirty_query(IgbVfState *s, void *buf, size_t buf_size, size_t *out_size, uint64_t range_iova, uint64_t range_size) { - IGBVfDirtyState *ds = igb_core_vf_dirty_state(s); + IGBCore *core = igbvf_get_core(s); + IGBVfDirtyState *ds = &core->vf_dirty[s->vfn]; + IgbVfMigStats *stats = &core->vf_mig_stats[s->vfn]; uint32_t i; for (i = 0; i < ds->num_ranges; i++) { @@ -587,6 +604,13 @@ static bool igb_core_vf_dirty_query(IgbVfState *s, bitmap_clear(r->bitmap, start_page, n); } + { + uint32_t cleared = bitmap_count_one(buf, count); + stats->dirty_pages_cleared += cleared; + stats->dirty_page_count -= MIN(stats->dirty_page_count, cleared); + stats->dirty_query_count++; + } + *out_size = bitmap_empty(buf, count) ? 0 : DIV_ROUND_UP(count, 8); return true; } @@ -881,6 +905,7 @@ static void igbvf_mig_data_xfer(IgbVfState *s, uint32_t val) static uint64_t igbvf_mig_read(void *opaque, hwaddr addr, unsigned size) { IgbVfState *s = opaque; + IgbVfMigStats *stats = &igbvf_get_core(s)->vf_mig_stats[s->vfn]; IgbVfMigState *ms = &s->mig; uint64_t val = 0; @@ -915,6 +940,27 @@ static uint64_t igbvf_mig_read(void *opaque, hwaddr addr, unsigned size) case IGB_MIG_DIRTY_STATUS: val = ms->mig_dirty_status; break; + case IGB_MIG_STAT_DMA_WRITES: + val = stats->dma_writes; + break; + case IGB_MIG_STAT_DMA_BYTES_LO: + val = (uint32_t)stats->dma_bytes; + break; + case IGB_MIG_STAT_DMA_BYTES_HI: + val = (uint32_t)(stats->dma_bytes >> 32); + break; + case IGB_MIG_STAT_DIRTY_PAGES_SET: + val = stats->dirty_pages_set; + break; + case IGB_MIG_STAT_DIRTY_PAGES_CLR: + val = stats->dirty_pages_cleared; + break; + case IGB_MIG_STAT_DIRTY_PAGE_COUNT: + val = stats->dirty_page_count; + break; + case IGB_MIG_STAT_DIRTY_QUERY_CNT: + val = stats->dirty_query_count; + break; default: qemu_log_mask(LOG_GUEST_ERROR, "igbvf: VF%u bad migration BAR read at 0x%" @@ -942,6 +988,7 @@ static uint32_t igbvf_mig_dirty_count(const void *bitmap, size_t size) static void igbvf_mig_dirty_query(IgbVfState *s, uint64_t pgsize) { IgbVfMigState *ms = &s->mig; + IgbVfMigStats *stats = &igbvf_get_core(s)->vf_mig_stats[s->vfn]; PCIDevice *dev = pcie_sriov_get_pf(PCI_DEVICE(s)); uint64_t buf_addr = ms->mig_dirty_buf_addr; uint64_t range_iova = 0, range_size = 0; @@ -982,12 +1029,17 @@ static void igbvf_mig_dirty_query(IgbVfState *s, uint64_t pgsize) stl_le_pci_dma(dev, buf_addr + offsetof(struct igb_mig_dirty_query, dirty_page_count), dirty_pages, MEMTXATTRS_UNSPECIFIED); + stq_le_pci_dma(dev, + buf_addr + offsetof(struct igb_mig_dirty_query, dma_writes), + stats->dma_writes, + MEMTXATTRS_UNSPECIFIED); stl_le_pci_dma(dev, buf_addr + offsetof(struct igb_mig_dirty_query, status), valid ? IGB_MIG_DIRTY_STATUS_COMPLETE : 0, MEMTXATTRS_UNSPECIFIED); - trace_igbvf_mig_dirty_query(s->vfn, (uint64_t)out_size, dirty_pages); + trace_igbvf_mig_dirty_query(s->vfn, (uint64_t)out_size, dirty_pages, + stats->dma_writes); } static void igbvf_mig_dirty_ctrl(IgbVfState *s, uint32_t val) @@ -1135,6 +1187,7 @@ void igbvf_mig_state_reset(IgbVfState *s) ms->mig_dirty_range_size = 0; ms->mig_dirty_buf_addr = 0; ms->mig_dirty_status = IGB_MIG_DIRTY_STATUS_OK; + memset(&igbvf_get_core(s)->vf_mig_stats[s->vfn], 0, sizeof(IgbVfMigStats)); ms->mig_saved_vfre = true; ms->mig_saved_vfte = true; trace_igbvf_mig_reset(s->vfn); diff --git a/hw/net/trace-events b/hw/net/trace-events index 1e38d9ab3697..76584b43bf19 100644 --- a/hw/net/trace-events +++ b/hw/net/trace-events @@ -305,7 +305,7 @@ igbvf_mig_load_state(uint16_t vfn, uint32_t size, bool vfre, bool vfte) "VF%u: l igbvf_mig_reset(uint16_t vfn) "VF%u: migration state reset" igbvf_mig_dirty_enable(uint16_t vfn, uint64_t pgsize, uint64_t nbits) "VF%u: dirty tracking enabled pgsize=%"PRIu64" nbits=%"PRIu64 igbvf_mig_dirty_disable(uint16_t vfn) "VF%u: dirty tracking disabled" -igbvf_mig_dirty_query(uint16_t vfn, uint64_t size, uint32_t dirty_pages) "VF%u: dirty query returned %"PRIu64" bytes, %u dirty pages" +igbvf_mig_dirty_query(uint16_t vfn, uint64_t size, uint32_t dirty_pages, uint64_t dma_writes) "VF%u: dirty query returned %"PRIu64" bytes, %u dirty pages (dma_writes=%"PRIu64")" # igb_core.c - VF migration diagnostics igb_core_dirty_track_dma(int vfn, uint64_t addr, uint64_t len) "VF%d: dirty DMA addr=0x%"PRIx64" len=%"PRIu64 -- 2.55.0
