+
static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
struct xe_bo *src_bo,
struct xe_bo *dst_bo,
struct ttm_resource *src,
struct ttm_resource *dst,
- bool copy_only_ccs,
- bool is_vram_resolve)
+ struct sg_table *src_sg,
+ struct xe_migrate_copy_flags flags)
{
struct xe_gt *gt = m->tile->primary_gt;
struct xe_device *xe = gt_to_xe(gt);
@@ -881,20 +907,29 @@ static struct dma_fence *__xe_migrate_copy(struct
xe_migrate *m,
bool src_is_vram = mem_type_is_vram(src->mem_type);
bool dst_is_vram = mem_type_is_vram(dst->mem_type);
bool type_device = src_bo->ttm.type == ttm_bo_type_device;
- bool needs_ccs_emit = type_device && xe_migrate_needs_ccs_emit(xe);
+ bool needs_ccs_emit = type_device && xe_migrate_needs_ccs_emit(xe) &&
+ !flags.defrag_copy;
bool copy_ccs = xe_device_has_flat_ccs(xe) &&
- xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo);
- bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram);
+ xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo)
&&
+ !flags.defrag_copy;
+ bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram) &&
+ !flags.defrag_ccs && !flags.defrag_copy;
/*
* For decompression operation, always use the compression PAT index.
* Otherwise, only use the compression PAT index for device memory
* when copying from VRAM to system memory.
*/
- bool use_comp_pat = is_vram_resolve || (type_device &&
+ bool use_comp_pat = flags.is_vram_resolve || (type_device &&
xe_device_has_flat_ccs(xe) &&
GRAPHICS_VER(xe) >= 20 && src_is_vram &&
!dst_is_vram);
+ /*
+ * The defrag CCS pass migrates the aux state from the old pages
+ * (@src_sg) to the new pages with both sides accessed indirectly, so
+ * keep both src_is_pltt and dst_is_pltt set (no direct staging side).
+ */
+
/* Copying CCS between two different BOs is not supported yet. */
if (XE_WARN_ON(copy_ccs && src_bo != dst_bo))
return ERR_PTR(-EINVAL);
@@ -903,7 +938,7 @@ static struct dma_fence *__xe_migrate_copy(struct
xe_migrate *m,
return ERR_PTR(-EINVAL);
if (!src_is_vram)
- xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it);
+ xe_res_first_sg(src_sg ? src_sg : xe_bo_sg(src_bo), 0, size,
&src_it);
else
xe_res_first(src, 0, size, &src_it);
if (!dst_is_vram)
@@ -912,7 +947,8 @@ static struct dma_fence *__xe_migrate_copy(struct
xe_migrate *m,
xe_res_first(dst, 0, size, &dst_it);
if (copy_system_ccs)
- xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo),
+ xe_res_first_sg(src_sg ? src_sg : xe_bo_sg(src_bo),
+ xe_bo_ccs_pages_start(src_bo),
PAGE_ALIGN(xe_device_ccs_bytes(xe, size)),
&ccs_it);
@@ -942,7 +978,7 @@ static struct dma_fence *__xe_migrate_copy(struct
xe_migrate *m,
batch_size += pte_update_size(m, pte_flags, src, &src_it,
&src_L0,
&src_L0_ofs, &src_L0_pt, 0, 0,
avail_pts);
- if (copy_only_ccs) {
+ if (flags.copy_only_ccs) {
dst_L0_ofs = src_L0_ofs;
} else {
pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0;
@@ -963,7 +999,8 @@ static struct dma_fence *__xe_migrate_copy(struct
xe_migrate *m,
}
/* Add copy commands size here */
- batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) +
+ batch_size += ((flags.copy_only_ccs || flags.defrag_ccs) ? 0 :
+ emit_copy_cmd_len(xe)) +
((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0));
bb = xe_bb_new(gt, batch_size, usm);
@@ -975,13 +1012,15 @@ static struct dma_fence *__xe_migrate_copy(struct
xe_migrate *m,
if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it))
xe_res_next(&src_it, src_L0);
else
- emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs
|| use_comp_pat,
+ emit_pte(m, bb, src_L0_pt, src_is_vram,
+ copy_system_ccs || use_comp_pat ||
flags.defrag_ccs,
&src_it, src_L0, src);
if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it))
xe_res_next(&dst_it, src_L0);
- else if (!copy_only_ccs)
- emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs,
+ else if (!flags.copy_only_ccs)
+ emit_pte(m, bb, dst_L0_pt, dst_is_vram,
+ copy_system_ccs || flags.defrag_ccs,
&dst_it, src_L0, dst);
if (copy_system_ccs)
@@ -990,7 +1029,7 @@ static struct dma_fence *__xe_migrate_copy(struct
xe_migrate *m,
bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
update_idx = bb->len;
- if (!copy_only_ccs)
+ if (!flags.copy_only_ccs && !flags.defrag_ccs)
emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0,
XE_PAGE_SIZE);
if (needs_ccs_emit)
@@ -1078,7 +1117,69 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
struct ttm_resource *dst,
bool copy_only_ccs)
{
- return __xe_migrate_copy(m, src_bo, dst_bo, src, dst, copy_only_ccs,
false);
+ return __xe_migrate_copy(m, src_bo, dst_bo, src, dst, NULL,
+ (struct xe_migrate_copy_flags) {
+ .copy_only_ccs = copy_only_ccs,
+ });
+}
+
+/**
+ * xe_migrate_copy_defrag() - Migrate a BO's backing to a reallocated page set.
+ * @m: The migration context.
+ * @bo: The buffer object being defragmented. Both @src and @dst belong to @bo.
+ * @src: The source TTM resource (the old backing).
+ * @dst: The dst TTM resource (the freshly reallocated backing).
+ * @src_sg: Scatter-gather table of the source (old) backing pages.
+ * @need_ccs: Whether the BO has CCS aux state that must be migrated too.
+ *
+ * Used by the defrag path where a single BO has its backing reallocated at a
+ * more beneficial page order. The contents are relocated from the old pages
+ * (@src_sg) to the new pages (the BO's current backing) entirely on the GPU,
+ * in up to two passes:
+ *
+ * - Data pass (always): copy the data pages verbatim. The pages are mapped
+ * without the compression PAT, so the (possibly compressed) bytes are
copied
+ * as-is rather than being decompressed and recompressed, and no CCS aux
copy
+ * is appended.
+ *
+ * - CCS pass (@need_ccs): copy the CCS aux (compression) state. Both the old
+ * and new data pages are mapped with the compression PAT so the copy engine
+ * accesses the aux state indirectly, issuing one indirect -> indirect
+ * XY_CTRL_SURF_COPY_BLT; the data blit is skipped.
+ *
+ * The two passes are issued separately because they require opposite
+ * compression PAT settings on the data page mappings (cleared for the verbatim
+ * data copy, set for the indirect CCS access), which a single mapping cannot
+ * satisfy. Both passes run in order on the same migrate queue.
+ *
+ * Return: Pointer to a dma_fence representing the last copy batch, or an error
+ * pointer on failure.
+ */
+struct dma_fence *xe_migrate_copy_defrag(struct xe_migrate *m,
+ struct xe_bo *bo,
+ struct ttm_resource *src,
+ struct ttm_resource *dst,
+ struct sg_table *src_sg,
+ bool need_ccs)
+{
+ struct dma_fence *fence2, *fence =
+ __xe_migrate_copy(m, bo, bo, src, dst, src_sg,
+ (struct xe_migrate_copy_flags) {
+ .defrag_copy = true,
+ });
+
+ if (IS_ERR(fence) || !need_ccs)
+ return fence;
+
+ fence2 = __xe_migrate_copy(m, bo, bo, src, dst, src_sg,
+ (struct xe_migrate_copy_flags) {
+ .defrag_ccs = true,
+ });
+ if (IS_ERR(fence2))
+ dma_fence_wait(fence, false);
+ dma_fence_put(fence);
+
+ return fence2;
}
/**
@@ -1096,7 +1197,10 @@ struct dma_fence *xe_migrate_resolve(struct xe_migrate
*m,
struct xe_bo *bo,
struct ttm_resource *res)
{
- return __xe_migrate_copy(m, bo, bo, res, res, false, true);
+ return __xe_migrate_copy(m, bo, bo, res, res, NULL,
+ (struct xe_migrate_copy_flags) {
+ .is_vram_resolve = true,
+ });
}
/**
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 965c45889c72..e4f996942a6d 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -11,6 +11,7 @@
struct dma_fence;
struct drm_pagemap_addr;
struct iosys_map;
+struct sg_table;
struct ttm_resource;
struct xe_bo;
@@ -132,6 +133,13 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
struct ttm_resource *dst,
bool copy_only_ccs);
+struct dma_fence *xe_migrate_copy_defrag(struct xe_migrate *m,
+ struct xe_bo *bo,
+ struct ttm_resource *src,
+ struct ttm_resource *dst,
+ struct sg_table *src_sg,
+ bool need_ccs);
+
struct dma_fence *xe_migrate_resolve(struct xe_migrate *m,
struct xe_bo *bo,
struct ttm_resource *res);
--
2.34.1