The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/7844
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === - During cluster pre-join the local node's database row IDs can be different than the actual cluster row IDs. - This can cause inconsistencies when adding local OVS chassis IDs to OVN logical HA chassis groups as part of network start. - Makes the original OVN logical network name (derived from original database row ID) available during pre-join phase. - Modifies parent network's `ovn.ovs_bridge` value to also be derived from original parent network's row ID. - Re-arranges parent port setup to perform all DB modifications in single transaction.
From a3620edb4c9deeca588ab005321e9935098b2d61 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Thu, 3 Sep 2020 18:22:02 +0100 Subject: [PATCH 1/5] lxd/network/driver/ovn: Add and delete local chassis ID to HA chassis group on start/stop Allows each node to add its local OVS chassis ID to the logical OVN chassis group. Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/network/driver_ovn.go | 97 ++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go index 8414132368..2ec0c2576f 100644 --- a/lxd/network/driver_ovn.go +++ b/lxd/network/driver_ovn.go @@ -825,19 +825,6 @@ func (n *ovn) setup(update bool) error { revert.Add(func() { client.ChassisGroupDelete(n.getChassisGroupName()) }) - // Add local chassis to chassis group. - ovs := openvswitch.NewOVS() - chassisID, err := ovs.ChassisID() - if err != nil { - return errors.Wrapf(err, "Failed getting OVS Chassis ID") - } - - var priority uint = ovnChassisPriorityMax - err = client.ChassisGroupChassisAdd(n.getChassisGroupName(), chassisID, priority) - if err != nil { - return errors.Wrapf(err, "Failed adding OVS chassis %q with priority %d to chassis group %q", chassisID, priority, n.getChassisGroupName()) - } - // Create logical router. if update { client.LogicalRouterDelete(n.getRouterName()) @@ -1068,10 +1055,60 @@ func (n *ovn) setup(update bool) error { return nil } +// addChassisGroupEntry adds an entry for the local OVS chassis to the OVN logical network's chassis group. +func (n *ovn) addChassisGroupEntry() error { + client, err := n.getClient() + if err != nil { + return err + } + + // Add local chassis to chassis group. + ovs := openvswitch.NewOVS() + chassisID, err := ovs.ChassisID() + if err != nil { + return errors.Wrapf(err, "Failed getting OVS Chassis ID") + } + + var priority uint = ovnChassisPriorityMax + err = client.ChassisGroupChassisAdd(n.getChassisGroupName(), chassisID, priority) + if err != nil { + return errors.Wrapf(err, "Failed adding OVS chassis %q with priority %d to chassis group %q", chassisID, priority, n.getChassisGroupName()) + } + + return nil +} + +// deleteChassisGroupEntry deletes an entry for the local OVS chassis from the OVN logical network's chassis group. +func (n *ovn) deleteChassisGroupEntry() error { + client, err := n.getClient() + if err != nil { + return err + } + + // Add local chassis to chassis group. + ovs := openvswitch.NewOVS() + chassisID, err := ovs.ChassisID() + if err != nil { + return errors.Wrapf(err, "Failed getting OVS Chassis ID") + } + + err = client.ChassisGroupChassisDelete(n.getChassisGroupName(), chassisID) + if err != nil { + return errors.Wrapf(err, "Failed deleting OVS chassis %q from chassis group %q", chassisID, n.getChassisGroupName()) + } + + return nil +} + // Delete deletes a network. func (n *ovn) Delete(clientType cluster.ClientType) error { n.logger.Debug("Delete", log.Ctx{"clientType": clientType}) + err := n.Stop() + if err != nil { + return err + } + if clientType == cluster.ClientTypeNormal { client, err := n.getClient() if err != nil { @@ -1125,12 +1162,6 @@ func (n *ovn) Delete(clientType cluster.ClientType) error { } } - // Delete local parent uplink port. - err := n.deleteParentPort() - if err != nil { - return err - } - return n.common.delete(clientType) } @@ -1157,7 +1188,7 @@ func (n *ovn) Rename(newName string) error { return nil } -// Start starts configures the local OVS parent uplink port. +// Start starts adds the local OVS chassis ID to the OVN chass group and starts the local OVS parent uplink port. func (n *ovn) Start() error { n.logger.Debug("Start") @@ -1165,7 +1196,13 @@ func (n *ovn) Start() error { return fmt.Errorf("Cannot start pending network") } - err := n.startParentPort() + // Add local node's OVS chassis ID to logical chassis group. + err := n.addChassisGroupEntry() + if err != nil { + return err + } + + err = n.startParentPort() if err != nil { return err } @@ -1173,10 +1210,26 @@ func (n *ovn) Start() error { return nil } -// Stop stops is a no-op. +// Stop deletes the local OVS parent uplink port (if unused) and deletes the local OVS chassis ID from the +// OVN chass group func (n *ovn) Stop() error { n.logger.Debug("Stop") + // Delete local OVS chassis ID from logical OVN HA chassis group. + err := n.deleteChassisGroupEntry() + if err != nil { + return err + } + + // Delete local parent uplink port. + // This must occur after the local OVS chassis ID is removed from the OVN HA chassis group so that the + // OVN patch port connection is removed for this network and we can correctly detect whether there are + // any other OVN networks using this uplink bridge before removing it. + err = n.deleteParentPort() + if err != nil { + return err + } + return nil } From ec2c7eaf3a9f7c4f700f3da43953c1c5465861ed Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Thu, 3 Sep 2020 18:21:22 +0100 Subject: [PATCH 2/5] lxd/network/openvswitch/ovn: Adds ChassisGroupChassisDelete function Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/network/openvswitch/ovn.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lxd/network/openvswitch/ovn.go b/lxd/network/openvswitch/ovn.go index e5bb67b062..aebe5e210f 100644 --- a/lxd/network/openvswitch/ovn.go +++ b/lxd/network/openvswitch/ovn.go @@ -823,3 +823,33 @@ func (o *OVN) ChassisGroupChassisAdd(haChassisGroupName OVNChassisGroup, chassis return nil } + +// ChassisGroupChassisDelete deletes a chassis ID from an HA chassis group. +func (o *OVN) ChassisGroupChassisDelete(haChassisGroupName OVNChassisGroup, chassisID string) error { + // Check if chassis group exists. ovn-nbctl doesn't provide an "--if-exists" option for this. + existingChassisGroup, err := o.nbctl("--no-headings", "--data=bare", "--colum=name", "find", "ha_chassis_group", fmt.Sprintf("name=%s", string(haChassisGroupName))) + if err != nil { + return err + } + + // Nothing to do if chassis group doesn't exist. + if strings.TrimSpace(existingChassisGroup) == "" { + return nil + } + + // Check if chassis exists. ovn-nbctl doesn't provide an "--if-exists" option for this. + existingChassis, err := o.nbctl("--no-headings", "--data=bare", "--colum=chassis_name", "find", "ha_chassis", fmt.Sprintf("chassis_name=%s", chassisID)) + if err != nil { + return err + } + + // Remove chassis from group if exists. + if strings.TrimSpace(existingChassis) != "" { + _, err := o.nbctl("ha-chassis-group-remove-chassis", string(haChassisGroupName), chassisID) + if err != nil { + return err + } + } + + return nil +} From a3ce774924f0ce526e44b9173b5a934081d65650 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Fri, 4 Sep 2020 09:33:12 +0100 Subject: [PATCH 3/5] lxd/network/driver/ovn: Adds ovn.name setting to store OVN logical network name - Stored at network setup time so that if a cluster node is joined in the future it can access the correct OVN network name in the pre-join phase before the database IDs are synced. - Moves ovnParentOVSBridge config populated into setupParentPort as is not parent network type specific. - Wraps setupParentPort and the population of ovn.name in a single transaction so it is done atomically. Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/network/driver_ovn.go | 181 ++++++++++++++++++++++---------------- 1 file changed, 103 insertions(+), 78 deletions(-) diff --git a/lxd/network/driver_ovn.go b/lxd/network/driver_ovn.go index 2ec0c2576f..b22642d5c0 100644 --- a/lxd/network/driver_ovn.go +++ b/lxd/network/driver_ovn.go @@ -38,6 +38,10 @@ const ovnVolatileParentIPv6 = "volatile.network.ipv6.address" // associated veth interfaces when using the parent network as an OVN uplink. const ovnParentOVSBridge = "ovn.ovs_bridge" +// ovnName setting on the OVN network config indicating the name of the logical OVN network. Allows the consistent +// use of the OVN logical network name in cluster node pre-join phase before node joins clustered database. +const ovnName = "ovn.name" + // ovnParentVars OVN object variables derived from parent network. type ovnParentVars struct { // Router. @@ -95,7 +99,8 @@ func (n *ovn) Validate(config map[string]string) error { "dns.domain": validate.IsAny, "dns.search": validate.IsAny, - // Volatile keys populated automatically as needed. + // Populated automatically at network setup time. + ovnName: validate.IsAny, ovnVolatileParentIPv4: validate.Optional(validate.IsNetworkAddressV4), ovnVolatileParentIPv6: validate.Optional(validate.IsNetworkAddressV6), } @@ -141,9 +146,9 @@ func (n *ovn) getNetworkPrefix() string { return fmt.Sprintf("lxd-net%d", n.id) } -// getChassisGroup returns OVN chassis group name to use. +// getChassisGroup returns OVN chassis group name to use based on ovn.name setting in config. func (n *ovn) getChassisGroupName() openvswitch.OVNChassisGroup { - return openvswitch.OVNChassisGroup(n.getNetworkPrefix()) + return openvswitch.OVNChassisGroup(n.config[ovnName]) } // getRouterName returns OVN logical router name to use. @@ -264,23 +269,29 @@ func (n *ovn) getIntSwitchInstancePortPrefix() string { // setupParentPort initialises the parent uplink connection. Returns the derived ovnParentVars settings used // during the initial creation of the logical network. -func (n *ovn) setupParentPort(routerMAC net.HardwareAddr) (*ovnParentVars, error) { - parentNet, err := LoadByName(n.state, n.config["network"]) - if err != nil { - return nil, errors.Wrapf(err, "Failed loading parent network") +func (n *ovn) setupParentPort(tx *db.ClusterTx, parentNet Network, routerMAC net.HardwareAddr) (*ovnParentVars, error) { + // Store the OVN OVS bridge name in the parent network config if not set. + parentNetConf := parentNet.Config() + if parentNetConf[ovnParentOVSBridge] == "" { + parentNetConf[ovnParentOVSBridge] = fmt.Sprintf("lxdovn%d", parentNet.ID()) + err := tx.UpdateNetwork(parentNet.ID(), parentNet.Description(), parentNetConf) + if err != nil { + return nil, errors.Wrapf(err, "Failed saving parent network OVN OVS bridge name") + } } switch parentNet.Type() { case "bridge": - return n.setupParentPortBridge(parentNet, routerMAC) + return n.setupParentPortBridge(tx, parentNet, routerMAC) } return nil, fmt.Errorf("Network type %q unsupported as OVN parent", parentNet.Type()) } -// setupParentPortBridge allocates external IPs on the parent bridge. +// setupParentPortBridge allocates external uplink IPs from the parent bridge and stores them in the OVN network +// config. Also generates the OVS uplink bridge name and stores in the parent network's config. // Returns the derived ovnParentVars settings. -func (n *ovn) setupParentPortBridge(parentNet Network, routerMAC net.HardwareAddr) (*ovnParentVars, error) { +func (n *ovn) setupParentPortBridge(tx *db.ClusterTx, parentNet Network, routerMAC net.HardwareAddr) (*ovnParentVars, error) { v := &ovnParentVars{} bridgeNet, ok := parentNet.(*bridge) @@ -317,68 +328,57 @@ func (n *ovn) setupParentPortBridge(parentNet Network, routerMAC net.HardwareAdd // Decide whether we need to allocate new IP(s) and go to the expense of retrieving all allocated IPs. if (parentIPv4Net != nil && routerExtPortIPv4 == nil) || (parentIPv6Net != nil && routerExtPortIPv6 == nil) { - err := n.state.Cluster.Transaction(func(tx *db.ClusterTx) error { - allAllocatedIPv4, allAllocatedIPv6, err := n.parentAllAllocatedIPs(tx, parentNet.Name()) + allAllocatedIPv4, allAllocatedIPv6, err := n.parentAllAllocatedIPs(tx, parentNet.Name()) + if err != nil { + return nil, errors.Wrapf(err, "Failed to get all allocated IPs for parent") + } + + if parentIPv4Net != nil && routerExtPortIPv4 == nil { + if parentNetConf["ipv4.ovn.ranges"] == "" { + return nil, fmt.Errorf(`Missing required "ipv4.ovn.ranges" config key on parent network`) + } + + ipRanges, err := parseIPRanges(parentNetConf["ipv4.ovn.ranges"], parentNet.DHCPv4Subnet()) if err != nil { - return errors.Wrapf(err, "Failed to get all allocated IPs for parent") + return nil, errors.Wrapf(err, "Failed to parse parent IPv4 OVN ranges") } - if parentIPv4Net != nil && routerExtPortIPv4 == nil { - if parentNetConf["ipv4.ovn.ranges"] == "" { - return fmt.Errorf(`Missing required "ipv4.ovn.ranges" config key on parent network`) - } + routerExtPortIPv4, err = n.parentAllocateIP(ipRanges, allAllocatedIPv4) + if err != nil { + return nil, errors.Wrapf(err, "Failed to allocate parent IPv4 address") + } + + n.config[ovnVolatileParentIPv4] = routerExtPortIPv4.String() + } - ipRanges, err := parseIPRanges(parentNetConf["ipv4.ovn.ranges"], parentNet.DHCPv4Subnet()) + if parentIPv6Net != nil && routerExtPortIPv6 == nil { + // If IPv6 OVN ranges are specified by the parent, allocate from them. + if parentNetConf["ipv6.ovn.ranges"] != "" { + ipRanges, err := parseIPRanges(parentNetConf["ipv6.ovn.ranges"], parentNet.DHCPv6Subnet()) if err != nil { - return errors.Wrapf(err, "Failed to parse parent IPv4 OVN ranges") + return nil, errors.Wrapf(err, "Failed to parse parent IPv6 OVN ranges") } - routerExtPortIPv4, err = n.parentAllocateIP(ipRanges, allAllocatedIPv4) + routerExtPortIPv6, err = n.parentAllocateIP(ipRanges, allAllocatedIPv6) if err != nil { - return errors.Wrapf(err, "Failed to allocate parent IPv4 address") + return nil, errors.Wrapf(err, "Failed to allocate parent IPv6 address") } - n.config[ovnVolatileParentIPv4] = routerExtPortIPv4.String() - } - - if parentIPv6Net != nil && routerExtPortIPv6 == nil { - // If IPv6 OVN ranges are specified by the parent, allocate from them. - if parentNetConf["ipv6.ovn.ranges"] != "" { - ipRanges, err := parseIPRanges(parentNetConf["ipv6.ovn.ranges"], parentNet.DHCPv6Subnet()) - if err != nil { - return errors.Wrapf(err, "Failed to parse parent IPv6 OVN ranges") - } - - routerExtPortIPv6, err = n.parentAllocateIP(ipRanges, allAllocatedIPv6) - if err != nil { - return errors.Wrapf(err, "Failed to allocate parent IPv6 address") - } - - } else { - // Otherwise use EUI64 derived from MAC address. - routerExtPortIPv6, err = eui64.ParseMAC(parentIPv6Net.IP, routerMAC) - if err != nil { - return err - } + } else { + // Otherwise use EUI64 derived from MAC address. + routerExtPortIPv6, err = eui64.ParseMAC(parentIPv6Net.IP, routerMAC) + if err != nil { + return nil, err } - - n.config[ovnVolatileParentIPv6] = routerExtPortIPv6.String() - } - - networkID, err := tx.GetNetworkID(n.name) - if err != nil { - return errors.Wrapf(err, "Failed to get network ID for network %q", n.name) } - err = tx.UpdateNetwork(networkID, n.description, n.config) - if err != nil { - return errors.Wrapf(err, "Failed saving allocated parent network IPs") - } + n.config[ovnVolatileParentIPv6] = routerExtPortIPv6.String() + } - return nil - }) + // Store allocated IPs on parent network in network config. + err = tx.UpdateNetwork(n.id, n.description, n.config) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "Failed saving allocated parent network IPs") } } @@ -510,21 +510,7 @@ func (n *ovn) parentOperationLockName(parentNet Network) string { func (n *ovn) parentPortBridgeVars(parentNet Network) (*ovnParentPortBridgeVars, error) { parentConfig := parentNet.Config() if parentConfig[ovnParentOVSBridge] == "" { - // Generate random OVS bridge name for parent uplink. - parentConfig[ovnParentOVSBridge] = RandomDevName("ovn") - - // Store in parent config. - err := n.state.Cluster.Transaction(func(tx *db.ClusterTx) error { - err := tx.UpdateNetwork(parentNet.ID(), parentNet.Description(), parentConfig) - if err != nil { - return errors.Wrapf(err, "Failed saving parent network OVN OVS bridge name") - } - - return nil - }) - if err != nil { - return nil, err - } + return nil, fmt.Errorf("Missing %q setting in parent network config", ovnParentOVSBridge) } return &ovnParentPortBridgeVars{ @@ -782,8 +768,37 @@ func (n *ovn) setup(update bool) error { return err } - // Setup parent port (do this first to check parent is suitable). - parent, err := n.setupParentPort(routerMAC) + // Parent network must be in default project. + parentNet, err := LoadByName(n.state, n.config["network"]) + if err != nil { + return errors.Wrapf(err, "Failed loading parent network %q", n.config["network"]) + } + + var parent *ovnParentVars + + // Start a transaction as we may need to modify the config in both this network and in the parent network. + // So do it in an atomic fashion. + err = n.state.Cluster.Transaction(func(tx *db.ClusterTx) error { + // Setup parent port (do this first to check parent is suitable). + // This may need to modify both the network config and the parent network config. + parent, err = n.setupParentPort(tx, parentNet, routerMAC) + if err != nil { + return err + } + + // Set the OVN network name into config if missing (used for joining cluster nodes). + if n.config[ovnName] == "" { + n.config[ovnName] = n.getNetworkPrefix() + + // Store updated network config. + err = tx.UpdateNetwork(n.id, n.description, n.config) + if err != nil { + return errors.Wrapf(err, "Failed saving OVN network name to config") + } + } + + return nil + }) if err != nil { return err } @@ -1057,6 +1072,11 @@ func (n *ovn) setup(update bool) error { // addChassisGroupEntry adds an entry for the local OVS chassis to the OVN logical network's chassis group. func (n *ovn) addChassisGroupEntry() error { + chassisGroupName := n.getChassisGroupName() + if chassisGroupName == "" { + return fmt.Errorf("Cannot add chassis group entry, missing %q setting", ovnName) + } + client, err := n.getClient() if err != nil { return err @@ -1070,9 +1090,9 @@ func (n *ovn) addChassisGroupEntry() error { } var priority uint = ovnChassisPriorityMax - err = client.ChassisGroupChassisAdd(n.getChassisGroupName(), chassisID, priority) + err = client.ChassisGroupChassisAdd(chassisGroupName, chassisID, priority) if err != nil { - return errors.Wrapf(err, "Failed adding OVS chassis %q with priority %d to chassis group %q", chassisID, priority, n.getChassisGroupName()) + return errors.Wrapf(err, "Failed adding OVS chassis %q with priority %d to chassis group %q", chassisID, priority, chassisGroupName) } return nil @@ -1080,6 +1100,11 @@ func (n *ovn) addChassisGroupEntry() error { // deleteChassisGroupEntry deletes an entry for the local OVS chassis from the OVN logical network's chassis group. func (n *ovn) deleteChassisGroupEntry() error { + chassisGroupName := n.getChassisGroupName() + if chassisGroupName == "" { + return fmt.Errorf("Cannot delete chassis group entry, missing %q setting", ovnName) + } + client, err := n.getClient() if err != nil { return err @@ -1092,9 +1117,9 @@ func (n *ovn) deleteChassisGroupEntry() error { return errors.Wrapf(err, "Failed getting OVS Chassis ID") } - err = client.ChassisGroupChassisDelete(n.getChassisGroupName(), chassisID) + err = client.ChassisGroupChassisDelete(chassisGroupName, chassisID) if err != nil { - return errors.Wrapf(err, "Failed deleting OVS chassis %q from chassis group %q", chassisID, n.getChassisGroupName()) + return errors.Wrapf(err, "Failed deleting OVS chassis %q from chassis group %q", chassisID, chassisGroupName) } return nil From 1da05127ce526532196400a51fae93bfd751ea37 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Fri, 4 Sep 2020 10:52:32 +0100 Subject: [PATCH 4/5] doc/networks: Adds ovn.name to OVN network doc Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- doc/networks.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/networks.md b/doc/networks.md index 63af6893b9..d2509a5400 100644 --- a/doc/networks.md +++ b/doc/networks.md @@ -298,3 +298,4 @@ dns.search | string | - | - ipv4.address | string | standard mode | random unused subnet | IPv4 address for the bridge (CIDR notation). Use "none" to turn off IPv4 or "auto" to generate a new one ipv6.address | string | standard mode | random unused subnet | IPv6 address for the bridge (CIDR notation). Use "none" to turn off IPv6 or "auto" to generate a new one network | string | - | - | Parent network to use for outbound external network access +ovn.name | string | - | - | Name of OVN logical network (populated automatically at network setup time) From 71898bd2654e8e0e977f5ba9c319c5e85b5c69fd Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Fri, 4 Sep 2020 10:54:31 +0100 Subject: [PATCH 5/5] api: Adds network_ovn_name API extension Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- doc/api-extensions.md | 4 ++++ shared/version/api.go | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index fa749cb371..14d2316208 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -1158,3 +1158,7 @@ Adds the `ovn.ovs_bridge` setting to `bridge` networks to allow the `ovn` networ If missing, the first `ovn` network to specify a `bridge` network as its parent `network` will cause the setting to be populated with a random interface name prefixed with "ovn". + +## network\_ovn\_name +Adds the `ovn.name` setting to `ovn` networks to allow nodes joining cluster to access the logical OVN network +name during the pre-join stage before the node is connected to the cluster database. diff --git a/shared/version/api.go b/shared/version/api.go index 4771c18f21..ac07e18827 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -225,6 +225,7 @@ var APIExtensions = []string{ "container_syscall_intercept_bpf_devices", "network_type_ovn", "network_bridge_ovn_bridge", + "network_ovn_name", } // APIExtensionsCount returns the number of available API extensions.
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel