hl_device_fini() sets device_fini_pending and flush_delayed_work()s the
pending-reset work, but flush only waits for a running callback - it
does not prevent the callback from re-arming.
device_hard_reset_pending() re-arms itself via queue_delayed_work() when
hl_device_reset() fails with -EBUSY and the (unlocked)
device_fini_pending check still reads 0, so a callback that passed that
check before fini set it re-arms during the flush, and the fresh timer
survives: destroy_workqueue(hdev->reset_wq) cannot see a delayed work
pending on the timer wheel, so the work fires later, queues onto a
destroyed workqueue and runs with a freed hdev.  Use-after-free.

On a busy device (live compute contexts make hl_device_reset() keep
failing with -EBUSY) this work sits in a per-second retry loop and the
callback body is a real reset attempt, so hitting the window on device
removal/FLR is observable in practice, and retries are free.

Use disable_delayed_work_sync(), which waits for a running callback to
finish and leaves the work item disabled, so the callback's
queue_delayed_work() re-arm has no effect.

Found by static analysis.  The race was reproduced on a KASAN build
with a small module exercising this exact flush/re-arm/free pattern
(no Gaudi hardware available); PoC and splat are below.

Cc: [email protected]
Signed-off-by: Chengyu Liu <[email protected]>
---

Reproducer notes (PoC source below; a module parameter selects the
shape): mode=0 (flush_delayed_work + lockless guard, this bug's shape) produces

    [    2.22] BUG: KASAN: slab-use-after-free in 
__run_timer_base.part.0+0x807/0x900

after the teardown frees the container; mode=1
(cancel_delayed_work_sync, i.e. this fix) stays clean.

---------------------------- reproducer: pm_flush_poc.c 
----------------------------
// SPDX-License-Identifier: GPL-2.0
/*
 * pm_teardown_poc - mechanism PoC for the drv-teardown-sync C/D variants
 * (F-026 ipoib / F-028 epf-ntb / F-029 habanalabs).
 *
 * Modes (module parameter):
 *   0  F-029 vuln shape:  flush_delayed_work + lockless guard
 *   1  control:           cancel_delayed_work_sync (>=6.9 swallows re-arm)
 *   2  F-028 vuln shape:  cancel_delayed_work (non-sync) + unconditional re-arm
 *   3  F-026 vuln shape:  cancel_delayed_work (non-sync) + guard check/queue
 *                         race window (test passes, teardown slips in before
 *                         the queue_delayed_work)
 *
 * Every vuln mode must produce KASAN slab-use-after-free when the orphaned
 * work fires after kfree; the control mode must stay clean.
 */
#include <linux/module.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/delay.h>

static int mode;
module_param(mode, int, 0444);
MODULE_PARM_DESC(mode, "0=flush(F-029) 1=cancel_sync(ctrl) 
2=cancel_nosync(F-028) 3=cancel_nosync+guard-race(F-026)");

struct pm_td_ctx {
        bool                    fini_pending;
        struct delayed_work     work;
};

static struct pm_td_ctx *ctx;

static void pm_td_cb(struct work_struct *w)
{
        struct pm_td_ctx *c = container_of(to_delayed_work(w),
                                           struct pm_td_ctx, work);

        pr_info("PM-POC: cb running, fini_pending=%d\n", c->fini_pending);

        switch (mode) {
        case 0:         /* F-029: lockless guard then long body then re-arm */
                if (c->fini_pending)
                        return;
                msleep(300);
                schedule_delayed_work(&c->work, HZ / 10);
                pr_info("PM-POC: re-armed (flush shape)\n");
                break;
        case 2:         /* F-028: unconditional re-arm, cancel must catch 
running cb */
                msleep(300);    /* long body so cancel lands mid-run */
                schedule_delayed_work(&c->work, HZ / 10);
                pr_info("PM-POC: re-armed (no guard)\n");
                break;
        case 3:         /* F-026: guard check passes, THEN teardown slips in */
                if (c->fini_pending)
                        return;
                /* widen the check->queue race window like ipoib's 724->725 */
                msleep(300);
                schedule_delayed_work(&c->work, HZ / 10);
                pr_info("PM-POC: re-armed past guard (test-queue race)\n");
                break;
        default:
                break;
        }
}

static int __init pm_poc_init(void)
{
        ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
        if (!ctx)
                return -ENOMEM;
        ctx->fini_pending = false;
        INIT_DELAYED_WORK(&ctx->work, pm_td_cb);

        pr_info("PM-POC: arming, mode=%d\n", mode);
        schedule_delayed_work(&ctx->work, 0);

        msleep(50);     /* callback is mid-run now */

        /* teardown sequence for the selected variant */
        ctx->fini_pending = true;
        switch (mode) {
        case 1:
                cancel_delayed_work_sync(&ctx->work);
                break;
        case 0:
                flush_delayed_work(&ctx->work);
                break;
        default:
                cancel_delayed_work(&ctx->work);        /* non-sync: C variants 
*/
                break;
        }

        pr_info("PM-POC: teardown done, freeing ctx\n");
        kfree(ctx);

        msleep(600);
        pr_info("PM-POC: SURVIVED (no KASAN splat)\n");
        return 0;
}

static void __exit pm_poc_exit(void) { }

module_init(pm_poc_init);
module_exit(pm_poc_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("drv-teardown-sync C/D variant mechanism PoC 
(F-026/028/029)");

 drivers/accel/habanalabs/common/device.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/accel/habanalabs/common/device.c 
b/drivers/accel/habanalabs/common/device.c
index 09b27bac3a..4635dc8572 100644
--- a/drivers/accel/habanalabs/common/device.c
+++ b/drivers/accel/habanalabs/common/device.c
@@ -2487,7 +2487,7 @@ void hl_device_fini(struct hl_device *hdev)
        dev_info(hdev->dev, "Removing device %s\n", 
dev_name(&(hdev)->pdev->dev));
 
        hdev->device_fini_pending = 1;
-       flush_delayed_work(&hdev->device_reset_work.reset_work);
+       disable_delayed_work_sync(&hdev->device_reset_work.reset_work);
 
        if (hdev->pldm)
                reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT;
-- 
2.25.1

Reply via email to