Re: [Intel-gfx] [PATCH RFC] drm/i915/gt: Retry RING_HEAD reset until it sticks
On Fri, Jul 15, 2022 at 09:26:16AM +0100, Mauro Carvalho Chehab wrote: > From: Chris Wilson > > On Haswell, in particular, we see an issue where resets fails because Can we then make this platform specific? Only because some older hw doesn't behave like expected we shouldn't make this a default & global workaround. > the engine resumes from an incorrect RING_HEAD. Since the RING_HEAD > doesn't point to the remaining requests to re-run, but may instead point > into the uninitialised portion of the ring, the GPU may be then fed > invalid instructions from a privileged context, oft pushing the GPU into > an unrecoverable hang. > > If at first the write doesn't succeed, try, try again. > > References: https://gitlab.freedesktop.org/drm/intel/-/issues/5432 > Testcase: igt/i915_selftest/hangcheck > Signed-off-by: Chris Wilson > Cc: Andrzej Hajda > Cc: Mika Kuoppala > Signed-off-by: Mauro Carvalho Chehab > --- > .../gpu/drm/i915/gt/intel_ring_submission.c | 44 +-- > drivers/gpu/drm/i915/i915_utils.h | 10 + > 2 files changed, 40 insertions(+), 14 deletions(-) > > diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c > b/drivers/gpu/drm/i915/gt/intel_ring_submission.c > index d5d6f1fadcae..cc53feb1f8ed 100644 > --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c > +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c > @@ -190,6 +190,7 @@ static bool stop_ring(struct intel_engine_cs *engine) > static int xcs_resume(struct intel_engine_cs *engine) > { > struct intel_ring *ring = engine->legacy.ring; > + ktime_t kt; > > ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n", >ring->head, ring->tail); > @@ -228,9 +229,20 @@ static int xcs_resume(struct intel_engine_cs *engine) > set_pp_dir(engine); > > /* First wake the ring up to an empty/idle ring */ > - ENGINE_WRITE_FW(engine, RING_HEAD, ring->head); > + until_timeout_ns(kt, 2 * NSEC_PER_MSEC) { > + ENGINE_WRITE_FW(engine, RING_HEAD, ring->head); > + if (ENGINE_READ_FW(engine, RING_HEAD) == ring->head) > + break; > + } > + > ENGINE_WRITE_FW(engine, RING_TAIL, ring->head); > - ENGINE_POSTING_READ(engine, RING_TAIL); > + if (ENGINE_READ_FW(engine, RING_HEAD) != ENGINE_READ_FW(engine, > RING_TAIL)) { > + ENGINE_TRACE(engine, "failed to reset empty ring: [%x, %x]: > %x\n", > + ENGINE_READ_FW(engine, RING_HEAD), > + ENGINE_READ_FW(engine, RING_TAIL), > + ring->head); > + goto err; > + } commit message mentions until this point I'm afraid... everything below (except the new until_timeout_ns) looks like a different patch to me, or deserves some mention in the commit msg. > > ENGINE_WRITE_FW(engine, RING_CTL, > RING_CTL_SIZE(ring->size) | RING_VALID); > @@ -239,12 +251,16 @@ static int xcs_resume(struct intel_engine_cs *engine) > if (__intel_wait_for_register_fw(engine->uncore, >RING_CTL(engine->mmio_base), >RING_VALID, RING_VALID, > - 5000, 0, NULL)) > + 5000, 0, NULL)) { > + ENGINE_TRACE(engine, "failed to restart\n"); > goto err; > + } > > - if (GRAPHICS_VER(engine->i915) > 2) > + if (GRAPHICS_VER(engine->i915) > 2) { > ENGINE_WRITE_FW(engine, > RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); > + ENGINE_POSTING_READ(engine, RING_MI_MODE); > + } > > /* Now awake, let it get started */ > if (ring->tail != ring->head) { > @@ -257,16 +273,16 @@ static int xcs_resume(struct intel_engine_cs *engine) > return 0; > > err: > - drm_err(&engine->i915->drm, > - "%s initialization failed; " > - "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start > %08x [expected %08x]\n", > - engine->name, > - ENGINE_READ(engine, RING_CTL), > - ENGINE_READ(engine, RING_CTL) & RING_VALID, > - ENGINE_READ(engine, RING_HEAD), ring->head, > - ENGINE_READ(engine, RING_TAIL), ring->tail, > - ENGINE_READ(engine, RING_START), > - i915_ggtt_offset(ring->vma)); > + ENGINE_TRACE(engine, > + "initialization failed; " > + "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] > start %08x [expected %08x]\n", > + ENGINE_READ(engine, RING_CTL), > + ENGINE_READ(engine, RING_CTL) & RING_VALID, > + ENGINE_READ(engine, RING_HEAD), ring->head, > + ENGINE_READ(engine, RING_TAIL), ring->tail, > + ENGINE_READ(engine, RING_START), > + i915_ggtt_offset(ring->vma)); > + GEM
Re: [Intel-gfx] [PATCH RFC] drm/i915/gt: Retry RING_HEAD reset until it sticks
On 15.07.2022 10:26, Mauro Carvalho Chehab wrote: From: Chris Wilson On Haswell, in particular, we see an issue where resets fails because the engine resumes from an incorrect RING_HEAD. Since the RING_HEAD doesn't point to the remaining requests to re-run, but may instead point into the uninitialised portion of the ring, the GPU may be then fed invalid instructions from a privileged context, oft pushing the GPU into an unrecoverable hang. If at first the write doesn't succeed, try, try again. References: https://gitlab.freedesktop.org/drm/intel/-/issues/5432 Testcase: igt/i915_selftest/hangcheck Signed-off-by: Chris Wilson Cc: Andrzej Hajda Cc: Mika Kuoppala Signed-off-by: Mauro Carvalho Chehab That is pity hw does not provide reliable way of reset. Reviewed-by: Andrzej Hajda Regards Andrzej --- .../gpu/drm/i915/gt/intel_ring_submission.c | 44 +-- drivers/gpu/drm/i915/i915_utils.h | 10 + 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index d5d6f1fadcae..cc53feb1f8ed 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -190,6 +190,7 @@ static bool stop_ring(struct intel_engine_cs *engine) static int xcs_resume(struct intel_engine_cs *engine) { struct intel_ring *ring = engine->legacy.ring; + ktime_t kt; ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n", ring->head, ring->tail); @@ -228,9 +229,20 @@ static int xcs_resume(struct intel_engine_cs *engine) set_pp_dir(engine); /* First wake the ring up to an empty/idle ring */ - ENGINE_WRITE_FW(engine, RING_HEAD, ring->head); + until_timeout_ns(kt, 2 * NSEC_PER_MSEC) { + ENGINE_WRITE_FW(engine, RING_HEAD, ring->head); + if (ENGINE_READ_FW(engine, RING_HEAD) == ring->head) + break; + } + ENGINE_WRITE_FW(engine, RING_TAIL, ring->head); - ENGINE_POSTING_READ(engine, RING_TAIL); + if (ENGINE_READ_FW(engine, RING_HEAD) != ENGINE_READ_FW(engine, RING_TAIL)) { + ENGINE_TRACE(engine, "failed to reset empty ring: [%x, %x]: %x\n", +ENGINE_READ_FW(engine, RING_HEAD), +ENGINE_READ_FW(engine, RING_TAIL), +ring->head); + goto err; + } ENGINE_WRITE_FW(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID); @@ -239,12 +251,16 @@ static int xcs_resume(struct intel_engine_cs *engine) if (__intel_wait_for_register_fw(engine->uncore, RING_CTL(engine->mmio_base), RING_VALID, RING_VALID, -5000, 0, NULL)) +5000, 0, NULL)) { + ENGINE_TRACE(engine, "failed to restart\n"); goto err; + } - if (GRAPHICS_VER(engine->i915) > 2) + if (GRAPHICS_VER(engine->i915) > 2) { ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); + ENGINE_POSTING_READ(engine, RING_MI_MODE); + } /* Now awake, let it get started */ if (ring->tail != ring->head) { @@ -257,16 +273,16 @@ static int xcs_resume(struct intel_engine_cs *engine) return 0; err: - drm_err(&engine->i915->drm, - "%s initialization failed; " - "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", - engine->name, - ENGINE_READ(engine, RING_CTL), - ENGINE_READ(engine, RING_CTL) & RING_VALID, - ENGINE_READ(engine, RING_HEAD), ring->head, - ENGINE_READ(engine, RING_TAIL), ring->tail, - ENGINE_READ(engine, RING_START), - i915_ggtt_offset(ring->vma)); + ENGINE_TRACE(engine, +"initialization failed; " +"ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", +ENGINE_READ(engine, RING_CTL), +ENGINE_READ(engine, RING_CTL) & RING_VALID, +ENGINE_READ(engine, RING_HEAD), ring->head, +ENGINE_READ(engine, RING_TAIL), ring->tail, +ENGINE_READ(engine, RING_START), +i915_ggtt_offset(ring->vma)); + GEM_TRACE_DUMP(); return -EIO; } diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index c10d68cdc3ca..717fb6b9cc15 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -256,6 +256,16 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)