Re: [RFC v3 3/5] vhost-vdpa: Implement the GPA->IOVA tree
On Tue, Jan 21, 2025 at 4:25 PM Jonah Palmer wrote:
>
>
>
> On 1/16/25 2:00 PM, Eugenio Perez Martin wrote:
> > On Fri, Jan 10, 2025 at 6:09 PM Jonah Palmer
> > wrote:
> >>
> >> Implements the GPA->IOVA tree for handling mapping and unmapping for
> >> guest memory. This, alongside the SVQ IOVA->HVA tree & IOVA-only tree
> >> implemented in the previous patches, allows us to handle guest and
> >> host-only memory mapping operations separately via their own respective
> >> trees.
> >>
> >> The next patches will implement a method to determine if an incomming
> >
> > s/incomming/incoming/ (credits to google syntax highlight actually :) )
> >
>
> Whoops! Good catch. Maybe I should add a spellchecker plugin to my Vim
> configuration haha.
>
> >> address for translation is backed by guest or host-only memory.
> >>
> >> Signed-off-by: Jonah Palmer
> >> ---
> >> hw/virtio/vhost-iova-tree.c | 50 +
> >> hw/virtio/vhost-iova-tree.h | 4 +++
> >> hw/virtio/vhost-vdpa.c | 22 ++--
> >> include/qemu/iova-tree.h| 22
> >> util/iova-tree.c| 46 ++
> >> 5 files changed, 136 insertions(+), 8 deletions(-)
> >>
> >> diff --git a/hw/virtio/vhost-iova-tree.c b/hw/virtio/vhost-iova-tree.c
> >> index f6a5694857..540bc35660 100644
> >> --- a/hw/virtio/vhost-iova-tree.c
> >> +++ b/hw/virtio/vhost-iova-tree.c
> >> @@ -31,6 +31,9 @@ struct VhostIOVATree {
> >>
> >> /* Allocated IOVA addresses */
> >> IOVATree *iova_map;
> >> +
> >> +/* GPA to IOVA address memory maps */
> >> +IOVATree *gpa_iova_map;
> >> };
> >>
> >> /**
> >> @@ -48,6 +51,7 @@ VhostIOVATree *vhost_iova_tree_new(hwaddr iova_first,
> >> hwaddr iova_last)
> >>
> >> tree->iova_taddr_map = iova_tree_new();
> >> tree->iova_map = iova_tree_new();
> >> +tree->gpa_iova_map = gpa_tree_new();
> >> return tree;
> >> }
> >>
> >> @@ -58,6 +62,7 @@ void vhost_iova_tree_delete(VhostIOVATree *iova_tree)
> >> {
> >> iova_tree_destroy(iova_tree->iova_taddr_map);
> >> iova_tree_destroy(iova_tree->iova_map);
> >> +iova_tree_destroy(iova_tree->gpa_iova_map);
> >> g_free(iova_tree);
> >> }
> >>
> >> @@ -134,3 +139,48 @@ int vhost_iova_tree_insert(VhostIOVATree *iova_tree,
> >> DMAMap *map)
> >>
> >> return iova_tree_insert(iova_tree->iova_taddr_map, map);
> >> }
> >> +
> >> +/** Insert a new GPA->IOVA mapping to the GPA->IOVA tree
> >> + *
> >> + * @iova_tree: The VhostIOVATree
> >> + * @map: The GPA->IOVA mapping
> >> + *
> >> + * Returns:
> >> + * - IOVA_OK if the map fits in the container
> >> + * - IOVA_ERR_INVALID if the map does not make sense (e.g. size overflow)
> >> + * - IOVA_ERR_OVERLAP if the GPA range overlaps with an existing range
> >> + */
> >> +int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map)
> >> +{
> >> +if (map->iova + map->size < map->iova || map->perm == IOMMU_NONE) {
> >> +return IOVA_ERR_INVALID;
> >> +}
> >> +
> >> +return gpa_tree_insert(iova_tree->gpa_iova_map, map);
> >> +}
> >> +
> >> +/**
> >> + * Find the IOVA address stored from a guest memory address (GPA)
> >> + *
> >> + * @tree: The VhostIOVATree
> >> + * @map: The map with the guest memory address
> >> + *
> >> + * Returns the stored GPA->IOVA mapping, or NULL if not found.
> >> + */
> >> +const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *tree,
> >> + const DMAMap *map)
> >> +{
> >> +return iova_tree_find_iova(tree->gpa_iova_map, map);
> >> +}
> >> +
> >> +/**
> >> + * Remove existing mappings from the GPA->IOVA & IOVA trees
> >> + *
> >> + * @iova_tree: The VhostIOVATree
> >> + * @map: The guest memory address map to remove
> >> + */
> >> +void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map)
> >> +{
> >> +iova_tree_remove(iova_tree->gpa_iova_map, map);
> >> +iova_tree_remove(iova_tree->iova_map, map);
> >> +}
> >> diff --git a/hw/virtio/vhost-iova-tree.h b/hw/virtio/vhost-iova-tree.h
> >> index 8bf7b64786..3e3dcd04fe 100644
> >> --- a/hw/virtio/vhost-iova-tree.h
> >> +++ b/hw/virtio/vhost-iova-tree.h
> >> @@ -24,5 +24,9 @@ const DMAMap *vhost_iova_tree_find_iova(const
> >> VhostIOVATree *iova_tree,
> >> int vhost_iova_tree_map_alloc(VhostIOVATree *iova_tree, DMAMap *map);
> >> void vhost_iova_tree_remove(VhostIOVATree *iova_tree, DMAMap map);
> >> int vhost_iova_tree_insert(VhostIOVATree *iova_tree, DMAMap *map);
> >> +int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map);
> >> +const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *iova_tree,
> >> + const DMAMap *map);
> >> +void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map);
> >>
> >> #endif
> >> diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
> >> index f5803f35f4..8587f3f6c8 100644
> >> --- a/hw/virtio/vhost-vdpa.c
> >> +++
Re: [RFC v3 3/5] vhost-vdpa: Implement the GPA->IOVA tree
On 1/16/25 2:00 PM, Eugenio Perez Martin wrote:
On Fri, Jan 10, 2025 at 6:09 PM Jonah Palmer wrote:
Implements the GPA->IOVA tree for handling mapping and unmapping for
guest memory. This, alongside the SVQ IOVA->HVA tree & IOVA-only tree
implemented in the previous patches, allows us to handle guest and
host-only memory mapping operations separately via their own respective
trees.
The next patches will implement a method to determine if an incomming
s/incomming/incoming/ (credits to google syntax highlight actually :) )
Whoops! Good catch. Maybe I should add a spellchecker plugin to my Vim
configuration haha.
address for translation is backed by guest or host-only memory.
Signed-off-by: Jonah Palmer
---
hw/virtio/vhost-iova-tree.c | 50 +
hw/virtio/vhost-iova-tree.h | 4 +++
hw/virtio/vhost-vdpa.c | 22 ++--
include/qemu/iova-tree.h| 22
util/iova-tree.c| 46 ++
5 files changed, 136 insertions(+), 8 deletions(-)
diff --git a/hw/virtio/vhost-iova-tree.c b/hw/virtio/vhost-iova-tree.c
index f6a5694857..540bc35660 100644
--- a/hw/virtio/vhost-iova-tree.c
+++ b/hw/virtio/vhost-iova-tree.c
@@ -31,6 +31,9 @@ struct VhostIOVATree {
/* Allocated IOVA addresses */
IOVATree *iova_map;
+
+/* GPA to IOVA address memory maps */
+IOVATree *gpa_iova_map;
};
/**
@@ -48,6 +51,7 @@ VhostIOVATree *vhost_iova_tree_new(hwaddr iova_first, hwaddr
iova_last)
tree->iova_taddr_map = iova_tree_new();
tree->iova_map = iova_tree_new();
+tree->gpa_iova_map = gpa_tree_new();
return tree;
}
@@ -58,6 +62,7 @@ void vhost_iova_tree_delete(VhostIOVATree *iova_tree)
{
iova_tree_destroy(iova_tree->iova_taddr_map);
iova_tree_destroy(iova_tree->iova_map);
+iova_tree_destroy(iova_tree->gpa_iova_map);
g_free(iova_tree);
}
@@ -134,3 +139,48 @@ int vhost_iova_tree_insert(VhostIOVATree *iova_tree,
DMAMap *map)
return iova_tree_insert(iova_tree->iova_taddr_map, map);
}
+
+/** Insert a new GPA->IOVA mapping to the GPA->IOVA tree
+ *
+ * @iova_tree: The VhostIOVATree
+ * @map: The GPA->IOVA mapping
+ *
+ * Returns:
+ * - IOVA_OK if the map fits in the container
+ * - IOVA_ERR_INVALID if the map does not make sense (e.g. size overflow)
+ * - IOVA_ERR_OVERLAP if the GPA range overlaps with an existing range
+ */
+int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map)
+{
+if (map->iova + map->size < map->iova || map->perm == IOMMU_NONE) {
+return IOVA_ERR_INVALID;
+}
+
+return gpa_tree_insert(iova_tree->gpa_iova_map, map);
+}
+
+/**
+ * Find the IOVA address stored from a guest memory address (GPA)
+ *
+ * @tree: The VhostIOVATree
+ * @map: The map with the guest memory address
+ *
+ * Returns the stored GPA->IOVA mapping, or NULL if not found.
+ */
+const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *tree,
+ const DMAMap *map)
+{
+return iova_tree_find_iova(tree->gpa_iova_map, map);
+}
+
+/**
+ * Remove existing mappings from the GPA->IOVA & IOVA trees
+ *
+ * @iova_tree: The VhostIOVATree
+ * @map: The guest memory address map to remove
+ */
+void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map)
+{
+iova_tree_remove(iova_tree->gpa_iova_map, map);
+iova_tree_remove(iova_tree->iova_map, map);
+}
diff --git a/hw/virtio/vhost-iova-tree.h b/hw/virtio/vhost-iova-tree.h
index 8bf7b64786..3e3dcd04fe 100644
--- a/hw/virtio/vhost-iova-tree.h
+++ b/hw/virtio/vhost-iova-tree.h
@@ -24,5 +24,9 @@ const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree
*iova_tree,
int vhost_iova_tree_map_alloc(VhostIOVATree *iova_tree, DMAMap *map);
void vhost_iova_tree_remove(VhostIOVATree *iova_tree, DMAMap map);
int vhost_iova_tree_insert(VhostIOVATree *iova_tree, DMAMap *map);
+int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map);
+const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *iova_tree,
+ const DMAMap *map);
+void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map);
#endif
diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index f5803f35f4..8587f3f6c8 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -361,10 +361,10 @@ static void vhost_vdpa_listener_region_add(MemoryListener
*listener,
if (s->shadow_data) {
int r;
-mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr,
mem_region.size = int128_get64(llsize) - 1,
mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
+/* Allocate an IOVA range in the IOVA tree */
r = vhost_iova_tree_map_alloc(s->iova_tree, &mem_region);
if (unlikely(r != IOVA_OK)) {
error_report("Can't allocate a mapping (%d)", r);
@@ -372,6 +372,14 @@ static void vhost_vdpa_listener_region_add(MemoryListener
Re: [RFC v3 3/5] vhost-vdpa: Implement the GPA->IOVA tree
On Fri, Jan 10, 2025 at 6:09 PM Jonah Palmer wrote:
>
> Implements the GPA->IOVA tree for handling mapping and unmapping for
> guest memory. This, alongside the SVQ IOVA->HVA tree & IOVA-only tree
> implemented in the previous patches, allows us to handle guest and
> host-only memory mapping operations separately via their own respective
> trees.
>
> The next patches will implement a method to determine if an incomming
s/incomming/incoming/ (credits to google syntax highlight actually :) )
> address for translation is backed by guest or host-only memory.
>
> Signed-off-by: Jonah Palmer
> ---
> hw/virtio/vhost-iova-tree.c | 50 +
> hw/virtio/vhost-iova-tree.h | 4 +++
> hw/virtio/vhost-vdpa.c | 22 ++--
> include/qemu/iova-tree.h| 22
> util/iova-tree.c| 46 ++
> 5 files changed, 136 insertions(+), 8 deletions(-)
>
> diff --git a/hw/virtio/vhost-iova-tree.c b/hw/virtio/vhost-iova-tree.c
> index f6a5694857..540bc35660 100644
> --- a/hw/virtio/vhost-iova-tree.c
> +++ b/hw/virtio/vhost-iova-tree.c
> @@ -31,6 +31,9 @@ struct VhostIOVATree {
>
> /* Allocated IOVA addresses */
> IOVATree *iova_map;
> +
> +/* GPA to IOVA address memory maps */
> +IOVATree *gpa_iova_map;
> };
>
> /**
> @@ -48,6 +51,7 @@ VhostIOVATree *vhost_iova_tree_new(hwaddr iova_first,
> hwaddr iova_last)
>
> tree->iova_taddr_map = iova_tree_new();
> tree->iova_map = iova_tree_new();
> +tree->gpa_iova_map = gpa_tree_new();
> return tree;
> }
>
> @@ -58,6 +62,7 @@ void vhost_iova_tree_delete(VhostIOVATree *iova_tree)
> {
> iova_tree_destroy(iova_tree->iova_taddr_map);
> iova_tree_destroy(iova_tree->iova_map);
> +iova_tree_destroy(iova_tree->gpa_iova_map);
> g_free(iova_tree);
> }
>
> @@ -134,3 +139,48 @@ int vhost_iova_tree_insert(VhostIOVATree *iova_tree,
> DMAMap *map)
>
> return iova_tree_insert(iova_tree->iova_taddr_map, map);
> }
> +
> +/** Insert a new GPA->IOVA mapping to the GPA->IOVA tree
> + *
> + * @iova_tree: The VhostIOVATree
> + * @map: The GPA->IOVA mapping
> + *
> + * Returns:
> + * - IOVA_OK if the map fits in the container
> + * - IOVA_ERR_INVALID if the map does not make sense (e.g. size overflow)
> + * - IOVA_ERR_OVERLAP if the GPA range overlaps with an existing range
> + */
> +int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map)
> +{
> +if (map->iova + map->size < map->iova || map->perm == IOMMU_NONE) {
> +return IOVA_ERR_INVALID;
> +}
> +
> +return gpa_tree_insert(iova_tree->gpa_iova_map, map);
> +}
> +
> +/**
> + * Find the IOVA address stored from a guest memory address (GPA)
> + *
> + * @tree: The VhostIOVATree
> + * @map: The map with the guest memory address
> + *
> + * Returns the stored GPA->IOVA mapping, or NULL if not found.
> + */
> +const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *tree,
> + const DMAMap *map)
> +{
> +return iova_tree_find_iova(tree->gpa_iova_map, map);
> +}
> +
> +/**
> + * Remove existing mappings from the GPA->IOVA & IOVA trees
> + *
> + * @iova_tree: The VhostIOVATree
> + * @map: The guest memory address map to remove
> + */
> +void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map)
> +{
> +iova_tree_remove(iova_tree->gpa_iova_map, map);
> +iova_tree_remove(iova_tree->iova_map, map);
> +}
> diff --git a/hw/virtio/vhost-iova-tree.h b/hw/virtio/vhost-iova-tree.h
> index 8bf7b64786..3e3dcd04fe 100644
> --- a/hw/virtio/vhost-iova-tree.h
> +++ b/hw/virtio/vhost-iova-tree.h
> @@ -24,5 +24,9 @@ const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree
> *iova_tree,
> int vhost_iova_tree_map_alloc(VhostIOVATree *iova_tree, DMAMap *map);
> void vhost_iova_tree_remove(VhostIOVATree *iova_tree, DMAMap map);
> int vhost_iova_tree_insert(VhostIOVATree *iova_tree, DMAMap *map);
> +int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map);
> +const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *iova_tree,
> + const DMAMap *map);
> +void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map);
>
> #endif
> diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
> index f5803f35f4..8587f3f6c8 100644
> --- a/hw/virtio/vhost-vdpa.c
> +++ b/hw/virtio/vhost-vdpa.c
> @@ -361,10 +361,10 @@ static void
> vhost_vdpa_listener_region_add(MemoryListener *listener,
> if (s->shadow_data) {
> int r;
>
> -mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr,
> mem_region.size = int128_get64(llsize) - 1,
> mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
>
> +/* Allocate an IOVA range in the IOVA tree */
> r = vhost_iova_tree_map_alloc(s->iova_tree, &mem_region);
> if (unlikely(r != IOVA_OK)) {
> error_report("Can't allocate a mapping (%d)", r);
> @@ -372,6
[RFC v3 3/5] vhost-vdpa: Implement the GPA->IOVA tree
Implements the GPA->IOVA tree for handling mapping and unmapping for
guest memory. This, alongside the SVQ IOVA->HVA tree & IOVA-only tree
implemented in the previous patches, allows us to handle guest and
host-only memory mapping operations separately via their own respective
trees.
The next patches will implement a method to determine if an incomming
address for translation is backed by guest or host-only memory.
Signed-off-by: Jonah Palmer
---
hw/virtio/vhost-iova-tree.c | 50 +
hw/virtio/vhost-iova-tree.h | 4 +++
hw/virtio/vhost-vdpa.c | 22 ++--
include/qemu/iova-tree.h| 22
util/iova-tree.c| 46 ++
5 files changed, 136 insertions(+), 8 deletions(-)
diff --git a/hw/virtio/vhost-iova-tree.c b/hw/virtio/vhost-iova-tree.c
index f6a5694857..540bc35660 100644
--- a/hw/virtio/vhost-iova-tree.c
+++ b/hw/virtio/vhost-iova-tree.c
@@ -31,6 +31,9 @@ struct VhostIOVATree {
/* Allocated IOVA addresses */
IOVATree *iova_map;
+
+/* GPA to IOVA address memory maps */
+IOVATree *gpa_iova_map;
};
/**
@@ -48,6 +51,7 @@ VhostIOVATree *vhost_iova_tree_new(hwaddr iova_first, hwaddr
iova_last)
tree->iova_taddr_map = iova_tree_new();
tree->iova_map = iova_tree_new();
+tree->gpa_iova_map = gpa_tree_new();
return tree;
}
@@ -58,6 +62,7 @@ void vhost_iova_tree_delete(VhostIOVATree *iova_tree)
{
iova_tree_destroy(iova_tree->iova_taddr_map);
iova_tree_destroy(iova_tree->iova_map);
+iova_tree_destroy(iova_tree->gpa_iova_map);
g_free(iova_tree);
}
@@ -134,3 +139,48 @@ int vhost_iova_tree_insert(VhostIOVATree *iova_tree,
DMAMap *map)
return iova_tree_insert(iova_tree->iova_taddr_map, map);
}
+
+/** Insert a new GPA->IOVA mapping to the GPA->IOVA tree
+ *
+ * @iova_tree: The VhostIOVATree
+ * @map: The GPA->IOVA mapping
+ *
+ * Returns:
+ * - IOVA_OK if the map fits in the container
+ * - IOVA_ERR_INVALID if the map does not make sense (e.g. size overflow)
+ * - IOVA_ERR_OVERLAP if the GPA range overlaps with an existing range
+ */
+int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map)
+{
+if (map->iova + map->size < map->iova || map->perm == IOMMU_NONE) {
+return IOVA_ERR_INVALID;
+}
+
+return gpa_tree_insert(iova_tree->gpa_iova_map, map);
+}
+
+/**
+ * Find the IOVA address stored from a guest memory address (GPA)
+ *
+ * @tree: The VhostIOVATree
+ * @map: The map with the guest memory address
+ *
+ * Returns the stored GPA->IOVA mapping, or NULL if not found.
+ */
+const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *tree,
+ const DMAMap *map)
+{
+return iova_tree_find_iova(tree->gpa_iova_map, map);
+}
+
+/**
+ * Remove existing mappings from the GPA->IOVA & IOVA trees
+ *
+ * @iova_tree: The VhostIOVATree
+ * @map: The guest memory address map to remove
+ */
+void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map)
+{
+iova_tree_remove(iova_tree->gpa_iova_map, map);
+iova_tree_remove(iova_tree->iova_map, map);
+}
diff --git a/hw/virtio/vhost-iova-tree.h b/hw/virtio/vhost-iova-tree.h
index 8bf7b64786..3e3dcd04fe 100644
--- a/hw/virtio/vhost-iova-tree.h
+++ b/hw/virtio/vhost-iova-tree.h
@@ -24,5 +24,9 @@ const DMAMap *vhost_iova_tree_find_iova(const VhostIOVATree
*iova_tree,
int vhost_iova_tree_map_alloc(VhostIOVATree *iova_tree, DMAMap *map);
void vhost_iova_tree_remove(VhostIOVATree *iova_tree, DMAMap map);
int vhost_iova_tree_insert(VhostIOVATree *iova_tree, DMAMap *map);
+int vhost_iova_tree_insert_gpa(VhostIOVATree *iova_tree, DMAMap *map);
+const DMAMap *vhost_iova_tree_find_gpa(const VhostIOVATree *iova_tree,
+ const DMAMap *map);
+void vhost_iova_tree_remove_gpa(VhostIOVATree *iova_tree, DMAMap map);
#endif
diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
index f5803f35f4..8587f3f6c8 100644
--- a/hw/virtio/vhost-vdpa.c
+++ b/hw/virtio/vhost-vdpa.c
@@ -361,10 +361,10 @@ static void vhost_vdpa_listener_region_add(MemoryListener
*listener,
if (s->shadow_data) {
int r;
-mem_region.translated_addr = (hwaddr)(uintptr_t)vaddr,
mem_region.size = int128_get64(llsize) - 1,
mem_region.perm = IOMMU_ACCESS_FLAG(true, section->readonly),
+/* Allocate an IOVA range in the IOVA tree */
r = vhost_iova_tree_map_alloc(s->iova_tree, &mem_region);
if (unlikely(r != IOVA_OK)) {
error_report("Can't allocate a mapping (%d)", r);
@@ -372,6 +372,14 @@ static void vhost_vdpa_listener_region_add(MemoryListener
*listener,
}
iova = mem_region.iova;
+mem_region.translated_addr = section->offset_within_address_space;
+
+/* Add GPA->IOVA mapping to the GPA->IOVA tree */
+r = vhost_iova_tree_insert_gpa(s->iova_tree, &mem_region);
+if (unlikely(r != IOVA_OK)) {
