The branch stable/15 has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=198cd4912f304a5074d018c34a919dd2d5068abc
commit 198cd4912f304a5074d018c34a919dd2d5068abc Author: John Baldwin <[email protected]> AuthorDate: 2025-10-09 19:24:27 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2025-12-18 18:41:50 +0000 ctld: Defer initialization of NVMeoF associations Wait until all of the configuration has been parsed before creating associations for NVMe portals. This ensures that any options specified in a transport group are honored when creating associations. To enable this, add a new virtual method portal::prepare invoked when applying a configuration prior to opening a socket (or reusing an existing socket) for a portal. Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D52844 (cherry picked from commit 1435a9b293e21f8fca1f654420c5075ea7434e8f) --- usr.sbin/ctld/ctld.cc | 5 ++++ usr.sbin/ctld/ctld.hh | 1 + usr.sbin/ctld/nvmf.cc | 77 +++++++++++++++++++++++---------------------------- usr.sbin/ctld/nvmf.hh | 16 ++++------- 4 files changed, 46 insertions(+), 53 deletions(-) diff --git a/usr.sbin/ctld/ctld.cc b/usr.sbin/ctld/ctld.cc index 10c12f25068e..331c029e282e 100644 --- a/usr.sbin/ctld/ctld.cc +++ b/usr.sbin/ctld/ctld.cc @@ -814,6 +814,11 @@ portal_group::open_sockets(struct conf &oldconf) } for (portal_up &portal : pg_portals) { + if (!portal->prepare()) { + cumulated_error++; + continue; + } + /* * Try to find already open portal and reuse the * listening socket. We don't care about what portal diff --git a/usr.sbin/ctld/ctld.hh b/usr.sbin/ctld/ctld.hh index cc88e6eb590e..3bf18f6a32c0 100644 --- a/usr.sbin/ctld/ctld.hh +++ b/usr.sbin/ctld/ctld.hh @@ -151,6 +151,7 @@ struct portal { p_protocol(protocol) {} virtual ~portal() = default; + virtual bool prepare() { return true; } bool reuse_socket(portal &oldp); bool init_socket(); virtual bool init_socket_options(int s __unused) { return true; } diff --git a/usr.sbin/ctld/nvmf.cc b/usr.sbin/ctld/nvmf.cc index d1240bfa4f6c..eb116903f5c1 100644 --- a/usr.sbin/ctld/nvmf.cc +++ b/usr.sbin/ctld/nvmf.cc @@ -34,11 +34,8 @@ struct nvmf_io_portal final : public nvmf_portal { nvmf_io_portal(struct portal_group *pg, const char *listen, - portal_protocol protocol, freebsd::addrinfo_up ai, - const struct nvmf_association_params &aparams, - nvmf_association_up na) : - nvmf_portal(pg, listen, protocol, std::move(ai), aparams, - std::move(na)) {} + portal_protocol protocol, freebsd::addrinfo_up ai) : + nvmf_portal(pg, listen, protocol, std::move(ai)) {} void handle_connection(freebsd::fd_up fd, const char *host, const struct sockaddr *client_sa) override; @@ -63,8 +60,6 @@ struct nvmf_transport_group final : public portal_group { override; private: - struct nvmf_association_params init_aparams(portal_protocol protocol); - static uint16_t last_port_id; }; @@ -143,48 +138,55 @@ parse_number(const nvlist_t *nvl, const char *key, uint64_t def, uint64_t minv, return def; } -struct nvmf_association_params -nvmf_transport_group::init_aparams(portal_protocol protocol) +bool +nvmf_portal::prepare() { - struct nvmf_association_params params; - memset(¶ms, 0, sizeof(params)); + memset(&p_aparams, 0, sizeof(p_aparams)); /* Options shared between discovery and I/O associations. */ - const nvlist_t *nvl = pg_options.get(); - params.tcp.header_digests = parse_bool(nvl, "HDGST", false); - params.tcp.data_digests = parse_bool(nvl, "DDGST", false); - uint64_t value = parse_number(nvl, "MAXH2CDATA", DEFAULT_MAXH2CDATA, - 4096, UINT32_MAX); + freebsd::nvlist_up nvl = portal_group()->options(); + p_aparams.tcp.header_digests = parse_bool(nvl.get(), "HDGST", false); + p_aparams.tcp.data_digests = parse_bool(nvl.get(), "DDGST", false); + uint64_t value = parse_number(nvl.get(), "MAXH2CDATA", + DEFAULT_MAXH2CDATA, 4096, UINT32_MAX); if (value % 4 != 0) { log_warnx("Invalid value \"%ju\" for option MAXH2CDATA", (uintmax_t)value); value = DEFAULT_MAXH2CDATA; } - params.tcp.maxh2cdata = value; + p_aparams.tcp.maxh2cdata = value; - switch (protocol) { + switch (protocol()) { case portal_protocol::NVME_TCP: - params.sq_flow_control = parse_bool(nvl, "SQFC", false); - params.dynamic_controller_model = true; - params.max_admin_qsize = parse_number(nvl, "max_admin_qsize", - NVME_MAX_ADMIN_ENTRIES, NVME_MIN_ADMIN_ENTRIES, - NVME_MAX_ADMIN_ENTRIES); - params.max_io_qsize = parse_number(nvl, "max_io_qsize", + p_aparams.sq_flow_control = parse_bool(nvl.get(), "SQFC", + false); + p_aparams.dynamic_controller_model = true; + p_aparams.max_admin_qsize = parse_number(nvl.get(), + "max_admin_qsize", NVME_MAX_ADMIN_ENTRIES, + NVME_MIN_ADMIN_ENTRIES, NVME_MAX_ADMIN_ENTRIES); + p_aparams.max_io_qsize = parse_number(nvl.get(), "max_io_qsize", NVME_MAX_IO_ENTRIES, NVME_MIN_IO_ENTRIES, NVME_MAX_IO_ENTRIES); - params.tcp.pda = 0; + p_aparams.tcp.pda = 0; break; case portal_protocol::NVME_DISCOVERY_TCP: - params.sq_flow_control = false; - params.dynamic_controller_model = true; - params.max_admin_qsize = NVME_MAX_ADMIN_ENTRIES; - params.tcp.pda = 0; + p_aparams.sq_flow_control = false; + p_aparams.dynamic_controller_model = true; + p_aparams.max_admin_qsize = NVME_MAX_ADMIN_ENTRIES; + p_aparams.tcp.pda = 0; break; default: __assert_unreachable(); } - return params; + p_association.reset(nvmf_allocate_association(NVMF_TRTYPE_TCP, true, + &p_aparams)); + if (!p_association) { + log_warn("Failed to create NVMe controller association"); + return false; + } + + return true; } portal_group_up @@ -209,15 +211,12 @@ bool nvmf_transport_group::add_portal(const char *value, portal_protocol protocol) { freebsd::addrinfo_up ai; - enum nvmf_trtype trtype; switch (protocol) { case portal_protocol::NVME_TCP: - trtype = NVMF_TRTYPE_TCP; ai = parse_addr_port(value, "4420"); break; case portal_protocol::NVME_DISCOVERY_TCP: - trtype = NVMF_TRTYPE_TCP; ai = parse_addr_port(value, "8009"); break; default: @@ -230,14 +229,6 @@ nvmf_transport_group::add_portal(const char *value, portal_protocol protocol) return false; } - struct nvmf_association_params aparams = init_aparams(protocol); - nvmf_association_up association(nvmf_allocate_association(trtype, true, - &aparams)); - if (!association) { - log_warn("Failed to create NVMe controller association"); - return false; - } - /* * XXX: getaddrinfo(3) may return multiple addresses; we should turn * those into multiple portals. @@ -246,10 +237,10 @@ nvmf_transport_group::add_portal(const char *value, portal_protocol protocol) portal_up portal; if (protocol == portal_protocol::NVME_DISCOVERY_TCP) { portal = std::make_unique<nvmf_discovery_portal>(this, value, - protocol, std::move(ai), aparams, std::move(association)); + protocol, std::move(ai)); } else { portal = std::make_unique<nvmf_io_portal>(this, value, - protocol, std::move(ai), aparams, std::move(association)); + protocol, std::move(ai)); need_tcp_transport = true; } diff --git a/usr.sbin/ctld/nvmf.hh b/usr.sbin/ctld/nvmf.hh index 0b4f8d45adfd..6f34a2858ef9 100644 --- a/usr.sbin/ctld/nvmf.hh +++ b/usr.sbin/ctld/nvmf.hh @@ -38,13 +38,12 @@ using nvmf_qpair_up = std::unique_ptr<nvmf_qpair, nvmf_qpair_deleter>; struct nvmf_portal : public portal { nvmf_portal(struct portal_group *pg, const char *listen, - portal_protocol protocol, freebsd::addrinfo_up ai, - const struct nvmf_association_params &aparams, - nvmf_association_up na) : - portal(pg, listen, protocol, std::move(ai)), - p_aparams(aparams), p_association(std::move(na)) {} + portal_protocol protocol, freebsd::addrinfo_up ai) : + portal(pg, listen, protocol, std::move(ai)) {} virtual ~nvmf_portal() override = default; + virtual bool prepare() override; + const struct nvmf_association_params *aparams() const { return &p_aparams; } @@ -58,11 +57,8 @@ private: struct nvmf_discovery_portal final : public nvmf_portal { nvmf_discovery_portal(struct portal_group *pg, const char *listen, - portal_protocol protocol, freebsd::addrinfo_up ai, - const struct nvmf_association_params &aparams, - nvmf_association_up na) : - nvmf_portal(pg, listen, protocol, std::move(ai), aparams, - std::move(na)) {} + portal_protocol protocol, freebsd::addrinfo_up ai) : + nvmf_portal(pg, listen, protocol, std::move(ai)) {} void handle_connection(freebsd::fd_up fd, const char *host, const struct sockaddr *client_sa) override;
