[PATCH 6/7] HMM: add device page fault support.
From: Jérôme Glisse
This patch add helper for device page fault. Device page fault helper will
fill the mirror page table using the CPU page table all this synchronized
with any update to CPU page table.
Signed-off-by: Jérôme Glisse
Signed-off-by: Sherry Cheung
Signed-off-by: Subhash Gutti
Signed-off-by: Mark Hairgrove
Signed-off-by: John Hubbard
Signed-off-by: Jatin Kumar
---
include/linux/hmm.h | 1 +
mm/hmm.c| 384
2 files changed, 385 insertions(+)
diff --git a/include/linux/hmm.h b/include/linux/hmm.h
index dd34572..72e168b 100644
--- a/include/linux/hmm.h
+++ b/include/linux/hmm.h
@@ -259,6 +259,7 @@ struct hmm_mirror {
int hmm_mirror_register(struct hmm_mirror *mirror, struct hmm_device *device);
void hmm_mirror_unregister(struct hmm_mirror *mirror);
+int hmm_mirror_fault(struct hmm_mirror *mirror, struct hmm_event *event);
#endif /* CONFIG_HMM */
diff --git a/mm/hmm.c b/mm/hmm.c
index 90ebe75..5fb7e19 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -55,6 +55,9 @@ static struct srcu_struct srcu;
* @lock: Serialize the mirror list modifications.
* @kref: Reference counter
* @mmu_notifier: The mmu_notifier of this mm.
+ * @device_faults: List of all active device page faults.
+ * @ndevice_faults: Number of active device page faults.
+ * @wait_queue: Wait queue for event synchronization.
*
* For each process address space (mm_struct) there is one and only one hmm
* struct. hmm functions will redispatch to each devices the change made to
@@ -67,6 +70,9 @@ struct hmm {
spinlock_t lock;
struct kref kref;
struct mmu_notifier mmu_notifier;
+ struct list_headdevice_faults;
+ unsignedndevice_faults;
+ wait_queue_head_t wait_queue;
};
static struct mmu_notifier_ops hmm_notifier_ops;
@@ -88,6 +94,11 @@ static void hmm_mirror_update_pt(struct hmm_mirror *mirror,
* help dealing with all this.
*/
+static inline bool hmm_event_overlap(struct hmm_event *a, struct hmm_event *b)
+{
+ return !((a->end <= b->start) || (a->start >= b->end));
+}
+
static inline int hmm_event_init(struct hmm_event *event,
struct hmm *hmm,
unsigned long start,
@@ -149,6 +160,9 @@ static int hmm_init(struct hmm *hmm)
hmm->vm_end = TASK_SIZE;
kref_init(>kref);
INIT_HLIST_HEAD(>mirrors);
+ INIT_LIST_HEAD(>device_faults);
+ hmm->ndevice_faults = 0;
+ init_waitqueue_head(>wait_queue);
spin_lock_init(>lock);
/* register notifier */
@@ -205,6 +219,60 @@ static inline struct hmm *hmm_unref(struct hmm *hmm)
return NULL;
}
+static int hmm_device_fault_start(struct hmm *hmm, struct hmm_event *event)
+{
+ int ret = 0;
+
+ mmu_notifier_range_wait_valid(hmm->mm, event->start, event->end);
+
+ spin_lock(>lock);
+ if (mmu_notifier_range_is_valid(hmm->mm, event->start, event->end)) {
+ list_add_tail(>list, >device_faults);
+ hmm->ndevice_faults++;
+ event->backoff = false;
+ } else
+ ret = -EAGAIN;
+ spin_unlock(>lock);
+
+ wake_up(>wait_queue);
+
+ return ret;
+}
+
+static void hmm_device_fault_end(struct hmm *hmm, struct hmm_event *event)
+{
+ hmm_event_wait(event);
+
+ spin_lock(>lock);
+ list_del_init(>list);
+ hmm->ndevice_faults--;
+ spin_unlock(>lock);
+
+ wake_up(>wait_queue);
+}
+
+static void hmm_wait_device_fault(struct hmm *hmm, struct hmm_event *ievent)
+{
+ struct hmm_event *fevent;
+ unsigned long wait_for = 0;
+
+again:
+ spin_lock(>lock);
+ list_for_each_entry(fevent, >device_faults, list) {
+ if (!hmm_event_overlap(fevent, ievent))
+ continue;
+ fevent->backoff = true;
+ wait_for = hmm->ndevice_faults;
+ }
+ spin_unlock(>lock);
+
+ if (wait_for > 0) {
+ wait_event(hmm->wait_queue, wait_for != hmm->ndevice_faults);
+ wait_for = 0;
+ goto again;
+ }
+}
+
static void hmm_update(struct hmm *hmm, struct hmm_event *event)
{
struct hmm_mirror *mirror;
@@ -214,6 +282,8 @@ static void hmm_update(struct hmm *hmm, struct hmm_event
*event)
if (hmm->mm->hmm != hmm)
return;
+ hmm_wait_device_fault(hmm, event);
+
id = srcu_read_lock();
hlist_for_each_entry_rcu(mirror, >mirrors, mlist)
@@ -226,6 +296,35 @@ static void hmm_update(struct hmm *hmm, struct hmm_event
*event)
hmm_mirror_update_pt(mirror, event);
srcu_read_unlock(, id);
+
+ wake_up(>wait_queue);
+}
+
+static int hmm_mm_fault(struct hmm *hmm,
+ struct hmm_event *event,
+ struct vm_area_struct *vma,
+ unsigned
[PATCH 6/7] HMM: add device page fault support.
From: Jérôme Glisse jgli...@redhat.com
This patch add helper for device page fault. Device page fault helper will
fill the mirror page table using the CPU page table all this synchronized
with any update to CPU page table.
Signed-off-by: Jérôme Glisse jgli...@redhat.com
Signed-off-by: Sherry Cheung sche...@nvidia.com
Signed-off-by: Subhash Gutti sgu...@nvidia.com
Signed-off-by: Mark Hairgrove mhairgr...@nvidia.com
Signed-off-by: John Hubbard jhubb...@nvidia.com
Signed-off-by: Jatin Kumar jaku...@nvidia.com
---
include/linux/hmm.h | 1 +
mm/hmm.c| 384
2 files changed, 385 insertions(+)
diff --git a/include/linux/hmm.h b/include/linux/hmm.h
index dd34572..72e168b 100644
--- a/include/linux/hmm.h
+++ b/include/linux/hmm.h
@@ -259,6 +259,7 @@ struct hmm_mirror {
int hmm_mirror_register(struct hmm_mirror *mirror, struct hmm_device *device);
void hmm_mirror_unregister(struct hmm_mirror *mirror);
+int hmm_mirror_fault(struct hmm_mirror *mirror, struct hmm_event *event);
#endif /* CONFIG_HMM */
diff --git a/mm/hmm.c b/mm/hmm.c
index 90ebe75..5fb7e19 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -55,6 +55,9 @@ static struct srcu_struct srcu;
* @lock: Serialize the mirror list modifications.
* @kref: Reference counter
* @mmu_notifier: The mmu_notifier of this mm.
+ * @device_faults: List of all active device page faults.
+ * @ndevice_faults: Number of active device page faults.
+ * @wait_queue: Wait queue for event synchronization.
*
* For each process address space (mm_struct) there is one and only one hmm
* struct. hmm functions will redispatch to each devices the change made to
@@ -67,6 +70,9 @@ struct hmm {
spinlock_t lock;
struct kref kref;
struct mmu_notifier mmu_notifier;
+ struct list_headdevice_faults;
+ unsignedndevice_faults;
+ wait_queue_head_t wait_queue;
};
static struct mmu_notifier_ops hmm_notifier_ops;
@@ -88,6 +94,11 @@ static void hmm_mirror_update_pt(struct hmm_mirror *mirror,
* help dealing with all this.
*/
+static inline bool hmm_event_overlap(struct hmm_event *a, struct hmm_event *b)
+{
+ return !((a-end = b-start) || (a-start = b-end));
+}
+
static inline int hmm_event_init(struct hmm_event *event,
struct hmm *hmm,
unsigned long start,
@@ -149,6 +160,9 @@ static int hmm_init(struct hmm *hmm)
hmm-vm_end = TASK_SIZE;
kref_init(hmm-kref);
INIT_HLIST_HEAD(hmm-mirrors);
+ INIT_LIST_HEAD(hmm-device_faults);
+ hmm-ndevice_faults = 0;
+ init_waitqueue_head(hmm-wait_queue);
spin_lock_init(hmm-lock);
/* register notifier */
@@ -205,6 +219,60 @@ static inline struct hmm *hmm_unref(struct hmm *hmm)
return NULL;
}
+static int hmm_device_fault_start(struct hmm *hmm, struct hmm_event *event)
+{
+ int ret = 0;
+
+ mmu_notifier_range_wait_valid(hmm-mm, event-start, event-end);
+
+ spin_lock(hmm-lock);
+ if (mmu_notifier_range_is_valid(hmm-mm, event-start, event-end)) {
+ list_add_tail(event-list, hmm-device_faults);
+ hmm-ndevice_faults++;
+ event-backoff = false;
+ } else
+ ret = -EAGAIN;
+ spin_unlock(hmm-lock);
+
+ wake_up(hmm-wait_queue);
+
+ return ret;
+}
+
+static void hmm_device_fault_end(struct hmm *hmm, struct hmm_event *event)
+{
+ hmm_event_wait(event);
+
+ spin_lock(hmm-lock);
+ list_del_init(event-list);
+ hmm-ndevice_faults--;
+ spin_unlock(hmm-lock);
+
+ wake_up(hmm-wait_queue);
+}
+
+static void hmm_wait_device_fault(struct hmm *hmm, struct hmm_event *ievent)
+{
+ struct hmm_event *fevent;
+ unsigned long wait_for = 0;
+
+again:
+ spin_lock(hmm-lock);
+ list_for_each_entry(fevent, hmm-device_faults, list) {
+ if (!hmm_event_overlap(fevent, ievent))
+ continue;
+ fevent-backoff = true;
+ wait_for = hmm-ndevice_faults;
+ }
+ spin_unlock(hmm-lock);
+
+ if (wait_for 0) {
+ wait_event(hmm-wait_queue, wait_for != hmm-ndevice_faults);
+ wait_for = 0;
+ goto again;
+ }
+}
+
static void hmm_update(struct hmm *hmm, struct hmm_event *event)
{
struct hmm_mirror *mirror;
@@ -214,6 +282,8 @@ static void hmm_update(struct hmm *hmm, struct hmm_event
*event)
if (hmm-mm-hmm != hmm)
return;
+ hmm_wait_device_fault(hmm, event);
+
id = srcu_read_lock(srcu);
hlist_for_each_entry_rcu(mirror, hmm-mirrors, mlist)
@@ -226,6 +296,35 @@ static void hmm_update(struct hmm *hmm, struct hmm_event
*event)
hmm_mirror_update_pt(mirror, event);
srcu_read_unlock(srcu, id);
+
+ wake_up(hmm-wait_queue);
+}
+
