On Wed, 6 Apr 2022, Julien Grall wrote:
> But if we use the 'connection status' field, then you could just delay the
> initialization until you receive an event and the connection status is
> connected.
I prototyped this approach and I managed to validate it successfully.
See attached patches for xen and linux (on top of existing patches). I
allocated the page from init-dom0less (instead of Xen) to achieve best
compatibility with older kernels.
There are some rough edges in the two patches but let me know if you
have any comments on the overall approach.
diff --git a/tools/helpers/Makefile b/tools/helpers/Makefile
index 8e42997052..8d78ab1e90 100644
--- a/tools/helpers/Makefile
+++ b/tools/helpers/Makefile
@@ -46,7 +46,7 @@ init-xenstore-domain: $(INIT_XENSTORE_DOMAIN_OBJS)
$(CC) $(LDFLAGS) -o $@ $(INIT_XENSTORE_DOMAIN_OBJS) $(LDLIBS_libxentoollog) $(LDLIBS_libxenstore) $(LDLIBS_libxenctrl) $(LDLIBS_libxenguest) $(LDLIBS_libxenlight) $(APPEND_LDFLAGS)
init-dom0less: $(INIT_DOM0LESS_OBJS)
- $(CC) $(LDFLAGS) -o $@ $(INIT_DOM0LESS_OBJS) $(LDLIBS_libxenctrl) $(LDLIBS_libxenevtchn) $(LDLIBS_libxentoollog) $(LDLIBS_libxenstore) $(LDLIBS_libxenlight) $(LDLIBS_libxenguest) $(APPEND_LDFLAGS)
+ $(CC) $(LDFLAGS) -o $@ $(INIT_DOM0LESS_OBJS) $(LDLIBS_libxenctrl) $(LDLIBS_libxenevtchn) $(LDLIBS_libxentoollog) $(LDLIBS_libxenstore) $(LDLIBS_libxenlight) $(LDLIBS_libxenguest) $(LDLIBS_libxenforeignmemory) $(APPEND_LDFLAGS)
.PHONY: install
install: all
diff --git a/tools/helpers/init-dom0less.c b/tools/helpers/init-dom0less.c
index dc9ccee868..329aa44da6 100644
--- a/tools/helpers/init-dom0less.c
+++ b/tools/helpers/init-dom0less.c
@@ -9,9 +9,12 @@
#include <xenguest.h>
#include <libxl.h>
#include <xenevtchn.h>
+#include <xenforeignmemory.h>
#include "init-dom-json.h"
+#define XS_CONNECTION_STATE_OFFSET (2068/4)
+#define XS_CONNECTION_STATE_RECONNECTING 0x1
#define XENSTORE_PFN_OFFSET 1
#define STR_MAX_LENGTH 64
@@ -215,12 +218,18 @@ err:
static int init_domain(struct xs_handle *xsh, libxl_dominfo *info)
{
struct xc_interface_core *xch;
+ xenforeignmemory_handle *xfh;
libxl_uuid uuid;
uint64_t xenstore_evtchn, xenstore_pfn;
int rc;
+ uint32_t *page;
printf("Init dom0less domain: %u\n", info->domid);
xch = xc_interface_open(0, 0, 0);
+ xfh = xenforeignmemory_open(0, 0);
+
+ if (xch == NULL || xfh == NULL)
+ err(1, "Cannot open xc/xenforeignmemory interfaces\n");
rc = xc_hvm_param_get(xch, info->domid, HVM_PARAM_STORE_EVTCHN,
&xenstore_evtchn);
@@ -235,6 +244,14 @@ static int init_domain(struct xs_handle *xsh, libxl_dominfo *info)
return 1;
}
+ page = xenforeignmemory_map(xfh, info->domid, XS_READ | XS_WRITE, 1, &xenstore_pfn, NULL);
+ if (!page) {
+ printf("Error mapping xenstore page\n");
+ return 1;
+ }
+ page[XS_CONNECTION_STATE_OFFSET] = XS_CONNECTION_STATE_RECONNECTING;
+ xenforeignmemory_unmap(xfh, page, 1);
+
rc = xc_dom_gnttab_seed(xch, info->domid, true,
(xen_pfn_t)-1, xenstore_pfn, 0, 0);
if (rc)
diff --git a/tools/xenstore/xenstored_domain.c b/tools/xenstore/xenstored_domain.c
index 0543f49670..7bb8c64d33 100644
--- a/tools/xenstore/xenstored_domain.c
+++ b/tools/xenstore/xenstored_domain.c
@@ -494,6 +494,7 @@ static struct domain *introduce_domain(const void *ctx,
talloc_steal(domain->conn, domain);
/* Notify the domain that xenstore is available */
+ interface->connection = 0x0;
xenevtchn_notify(xce_handle, domain->port);
if (!is_master_domain && !restore)
diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c
index 51e52e175892..dc046d25789e 100644
--- a/drivers/xen/xenbus/xenbus_probe.c
+++ b/drivers/xen/xenbus/xenbus_probe.c
@@ -939,6 +939,7 @@ static int __init xenbus_init(void)
{
int err;
uint64_t v = 0;
+ bool wait = false;
xen_store_domain_type = XS_UNKNOWN;
if (!xen_domain())
@@ -992,17 +993,7 @@ static int __init xenbus_init(void)
goto out_error;
}
if (v == ~0ULL) {
- err = bind_evtchn_to_irqhandler(xen_store_evtchn,
- xenbus_late_init,
- 0, "xenstore_late_init",
- &xb_waitq);
- if (err < 0) {
- pr_err("xenstore_late_init couldn't bind irq err=%d\n",
- err);
- return err;
- }
-
- xs_init_irq = err;
+ wait = true;
} else {
/* Avoid truncation on 32-bit. */
#if BITS_PER_LONG == 32
@@ -1017,6 +1008,21 @@ static int __init xenbus_init(void)
xen_store_interface =
xen_remap(xen_store_gfn << XEN_PAGE_SHIFT,
XEN_PAGE_SIZE);
+ if (xen_store_interface->connection != 0)
+ wait = true;
+ }
+ if (wait) {
+ err = bind_evtchn_to_irqhandler(xen_store_evtchn,
+ xenbus_late_init,
+ 0, "xenstore_late_init",
+ &xb_waitq);
+ if (err < 0) {
+ pr_err("xenstore_late_init couldn't bind irq err=%d\n",
+ err);
+ return err;
+ }
+
+ xs_init_irq = err;
}
break;
default:
diff --git a/include/xen/interface/io/xs_wire.h b/include/xen/interface/io/xs_wire.h
index d40a44f09b16..cd7ae5ebb133 100644
--- a/include/xen/interface/io/xs_wire.h
+++ b/include/xen/interface/io/xs_wire.h
@@ -87,6 +87,9 @@ struct xenstore_domain_interface {
char rsp[XENSTORE_RING_SIZE]; /* Replies and async watch events. */
XENSTORE_RING_IDX req_cons, req_prod;
XENSTORE_RING_IDX rsp_cons, rsp_prod;
+ uint32_t server_features; /* Bitmap of features supported by the server */
+ uint32_t connection;
+ uint32_t error;
};
/* Violating this is very bad. See docs/misc/xenstore.txt. */
