Re: zpool import skips wedges due to a race condition

[Taylor R Campbell]

> Date: Tue, 7 Sep 2021 22:30:45 +0100
> From: Alexander Nasonov <al...@yandex.ru>
> 
> When I run zfs import, it launches 32 threads and opens 32 disks in
> parallel, including cgd1 and dk24. But it can't open dk24 while
> cgd1 is still open (it fails with EBUSY).

I have the attached patch in my tree from a while ago to scan
non-wedge disks first, then wedges.  For some reason I didn't commit
it but I forget why.  (Well, it's a little kludgey to look at the
pathname to distinguish dkN devices, but...)

>From dbee261ca7b5837946e93ec601acb04ae0b0a8b1 Mon Sep 17 00:00:00 2001
From: Taylor R Campbell <riastr...@netbsd.org>
Date: Sat, 29 Feb 2020 18:19:45 +0000
Subject: [PATCH] Scan primary disks first; then dkwedges.

---
 .../dist/lib/libzfs/common/libzfs_import.c    | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c 
b/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c
index b935e49076a5..e8d29b86f4b2 100644
--- a/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c
+++ b/external/cddl/osnet/dist/lib/libzfs/common/libzfs_import.c
@@ -1175,6 +1175,9 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t 
*iarg)
        config_entry_t *ce, *cenext;
        name_entry_t *ne, *nenext;
        avl_tree_t slice_cache;
+#ifdef __NetBSD__
+       avl_tree_t dkwedge_cache;
+#endif
        rdsk_node_t *slice;
        void *cookie;
 
@@ -1266,6 +1269,8 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t 
*iarg)
                }
 #endif
 #ifdef __NetBSD__
+               avl_create(&dkwedge_cache, slice_cache_compare,
+                   sizeof (rdsk_node_t), offsetof(rdsk_node_t, rn_node));
                if (strcmp(rdsk, "/dev/") == 0) {
                        static const char mib_name[] = "hw.disknames";
                        size_t len;
@@ -1291,7 +1296,10 @@ zpool_find_import_impl(libzfs_handle_t *hdl, 
importargs_t *iarg)
                                slice->rn_dfd = dfd;
                                slice->rn_hdl = hdl;
                                slice->rn_nozpool = B_FALSE;
-                               avl_add(&slice_cache, slice);
+                               if (strncmp(name, "dk", 2) == 0)
+                                       avl_add(&dkwedge_cache, slice);
+                               else
+                                       avl_add(&slice_cache, slice);
                        }
                        free(disknames);
 
@@ -1333,6 +1341,16 @@ skipdir:
                    AVL_AFTER)))
                        (void) tpool_dispatch(t, zpool_open_func, slice);
                tpool_wait(t);
+#ifdef __NetBSD__
+               for (slice = avl_first(&dkwedge_cache); slice;
+                   (slice = avl_walk(&dkwedge_cache, slice,
+                   AVL_AFTER)))
+                       (void) tpool_dispatch(t, zpool_open_func, slice);
+               tpool_wait(t);
+               while ((slice = avl_destroy_nodes(&dkwedge_cache,
+                   &cookie)) != NULL)
+                       avl_add(&slice_cache, slice);
+#endif
                tpool_destroy(t);
 
                cookie = NULL;
@@ -1373,6 +1391,9 @@ skipdir:
                        free(slice->rn_name);
                        free(slice);
                }
+#ifdef __NetBSD__
+               avl_destroy(&dkwedge_cache);
+#endif
                avl_destroy(&slice_cache);
 
                (void) closedir(dirp);