The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/7193
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === When using custom policy routing on the host, proxy ARP and proxy NDP will not function unless static routes for the instance's IPs are added to the custom policy routing table (in addition to the host's main routing table). This PR introduces a new `ipvlan` NIC config setting: `host_table` which can be set to the routing table ID that LXD will add static routes to for each of the instances IPs. Fixes #7123 Includes https://github.com/lxc/lxd/pull/7192
From f1f3cc1692fe4c98cdbee13b76f14d0fbe945bd9 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 10:11:50 +0100 Subject: [PATCH 1/9] lxd/device/nic: Adds host_table setting validation rule Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/device/nic.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lxd/device/nic.go b/lxd/device/nic.go index 2b0841c3f2..4af56b01a5 100644 --- a/lxd/device/nic.go +++ b/lxd/device/nic.go @@ -36,6 +36,7 @@ func nicValidationRules(requiredFields []string, optionalFields []string) map[st "vlan": shared.IsAny, "hwaddr": networkValidMAC, "host_name": shared.IsAny, + "host_table": shared.IsUint32, "limits.ingress": shared.IsAny, "limits.egress": shared.IsAny, "limits.max": shared.IsAny, From 84cbdec1a1f9f90050a4d80fb4f1057f8ff89d5c Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 10:13:12 +0100 Subject: [PATCH 2/9] lxd/device/nic/routed: Fix sysctl command suggestion when using vlans Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/device/nic_routed.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lxd/device/nic_routed.go b/lxd/device/nic_routed.go index 66372f1233..e0166e70d0 100644 --- a/lxd/device/nic_routed.go +++ b/lxd/device/nic_routed.go @@ -131,7 +131,8 @@ func (d *nicRouted) validateEnvironment() error { return fmt.Errorf("Error reading net sysctl %s: %v", ipv4FwdPath, err) } if sysctlVal != "1\n" { - return fmt.Errorf("Routed mode requires sysctl net.ipv4.conf.%s.forwarding=1", effectiveParentName) + // Replace . in parent name with / for sysctl formatting. + return fmt.Errorf("Routed mode requires sysctl net.ipv4.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } } @@ -143,7 +144,8 @@ func (d *nicRouted) validateEnvironment() error { return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err) } if sysctlVal != "1\n" { - return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", effectiveParentName) + // Replace . in parent name with / for sysctl formatting. + return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", effectiveParentName) @@ -152,7 +154,8 @@ func (d *nicRouted) validateEnvironment() error { return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err) } if sysctlVal != "1\n" { - return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", effectiveParentName) + // Replace . in parent name with / for sysctl formatting. + return fmt.Errorf("Routed mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } } From de45bd8ef3fc45fa2e5eeeaa519b05e8f0bdf698 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 10:13:37 +0100 Subject: [PATCH 3/9] lxd/device/nic/routed: Add host_table support Allows adding static routes for instance IPs to custom policy routing tables. Fixes #7152 Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/device/nic_routed.go | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/lxd/device/nic_routed.go b/lxd/device/nic_routed.go index e0166e70d0..5dbea050c2 100644 --- a/lxd/device/nic_routed.go +++ b/lxd/device/nic_routed.go @@ -39,6 +39,7 @@ func (d *nicRouted) validateConfig(instConf instance.ConfigReader) error { "mtu", "hwaddr", "host_name", + "host_table", "vlan", "ipv4.gateway", "ipv6.gateway", @@ -294,10 +295,7 @@ func (d *nicRouted) setupParentSysctls(parentName string) error { func (d *nicRouted) postStart() error { v := d.volatileGet() - // If host_name is defined (and it should be), then we add the dummy link-local gateway IPs - // to the host end of the veth pair. This ensures that liveness detection of the gateways - // inside the instance work and ensure that traffic doesn't periodically halt whilst ARP/NDP - // is re-detected. + // If volatile host_name is defined (and it should be), then configure the host-side interface. if v["host_name"] != "" { // Attempt to disable IPv6 router advertisement acceptance. err := util.SysctlSet(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", v["host_name"]), "0") @@ -318,17 +316,51 @@ func (d *nicRouted) postStart() error { } if d.config["ipv4.address"] != "" { + // Add dummy link-local gateway IPs to the host end of the veth pair. This ensures that + // liveness detection of the gateways inside the instance work and ensure that traffic + // doesn't periodically halt whilst ARP is re-detected. _, err := shared.RunCommand("ip", "-4", "addr", "add", fmt.Sprintf("%s/32", d.ipv4HostAddress()), "dev", v["host_name"]) if err != nil { return err } + + // Add static routes to instance IPs to custom routing tables if specified. + // This is in addition to the static route added by liblxc to the main routing table, which + // is still critical to ensure that reverse path filtering doesn't kick in blocking traffic + // from the instance. + if d.config["host_table"] != "" { + for _, addr := range strings.Split(d.config["ipv4.address"], ",") { + addr = strings.TrimSpace(addr) + _, err := shared.RunCommand("ip", "-4", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/32", addr), "dev", v["host_name"]) + if err != nil { + return err + } + } + } } if d.config["ipv6.address"] != "" { + // Add dummy link-local gateway IPs to the host end of the veth pair. This ensures that + // liveness detection of the gateways inside the instance work and ensure that traffic + // doesn't periodically halt whilst NDP is re-detected. _, err := shared.RunCommand("ip", "-6", "addr", "add", fmt.Sprintf("%s/128", d.ipv6HostAddress()), "dev", v["host_name"]) if err != nil { return err } + + // Add static routes to instance IPs to custom routing tables if specified. + // This is in addition to the static route added by liblxc to the main routing table, which + // is still critical to ensure that reverse path filtering doesn't kick in blocking traffic + // from the instance. + if d.config["host_table"] != "" { + for _, addr := range strings.Split(d.config["ipv6.address"], ",") { + addr = strings.TrimSpace(addr) + _, err := shared.RunCommand("ip", "-6", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/128", addr), "dev", v["host_name"]) + if err != nil { + return err + } + } + } } } From ac391675781727029b4f2e5adb457e561a83c4fe Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 10:16:50 +0100 Subject: [PATCH 4/9] api: Adds container_nic_routed_host_table extension Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- doc/api-extensions.md | 4 ++++ shared/version/api.go | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index 82fb120b81..7e1587551e 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -1013,3 +1013,7 @@ Exposes the die\_id information on each core. This introduces two new fields in `/1.0`, `os` and `os\_version`. Those are taken from the os-release data on the system. + +## container\_nic\_routed\_host\_table +This introduces the `host_table` NIC config key that can be used to add static routes for the instance's IPs to a +custom policy routing table by ID. diff --git a/shared/version/api.go b/shared/version/api.go index 87e58fd56b..6c914d1aeb 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -205,6 +205,7 @@ var APIExtensions = []string{ "resources_cpu_threads_numa", "resources_cpu_core_die", "api_os", + "container_nic_routed_host_table", } // APIExtensionsCount returns the number of available API extensions. From f98d3fd73cd9b96b82895e11dc7c63eddd83eead Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 10:18:53 +0100 Subject: [PATCH 5/9] doc: Adds documentation for routed NIC host_table setting Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- doc/instances.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/instances.md b/doc/instances.md index c212149f0c..e84c4ce2b8 100644 --- a/doc/instances.md +++ b/doc/instances.md @@ -469,6 +469,7 @@ Key | Type | Default | Required | Descriptio parent | string | - | no | The name of the host device to join the instance to name | string | kernel assigned | no | The name of the interface inside the instance host\_name | string | randomly assigned | no | The name of the interface inside the host +host\_table | integer | - | no | The custom policy routing table ID to add IP static routes to (in addition to main routing table). mtu | integer | parent MTU | no | The MTU of the new interface hwaddr | string | randomly assigned | no | The MAC address of the new interface ipv4.address | string | - | no | Comma delimited list of IPv4 static addresses to add to the instance From 8f7977dbe1ef323dd0930d8a396b9f6ebd27a450 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 10:45:35 +0100 Subject: [PATCH 6/9] lxd/device/nic/vlan: Improve validation of sysctl settings when vlan setting used Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/device/nic_ipvlan.go | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/lxd/device/nic_ipvlan.go b/lxd/device/nic_ipvlan.go index e51af7d96f..4247e4545a 100644 --- a/lxd/device/nic_ipvlan.go +++ b/lxd/device/nic_ipvlan.go @@ -66,45 +66,61 @@ func (d *nicIPVLAN) validateEnvironment() error { return fmt.Errorf("Requires name property to start") } + extensions := d.state.OS.LXCFeatures + if !extensions["network_ipvlan"] || !extensions["network_l2proxy"] || !extensions["network_gateway_device_route"] { + return fmt.Errorf("Requires liblxc has following API extensions: network_ipvlan, network_l2proxy, network_gateway_device_route") + } + if !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", d.config["parent"])) { return fmt.Errorf("Parent device '%s' doesn't exist", d.config["parent"]) } - extensions := d.state.OS.LXCFeatures - if !extensions["network_ipvlan"] || !extensions["network_l2proxy"] || !extensions["network_gateway_device_route"] { - return fmt.Errorf("Requires liblxc has following API extensions: network_ipvlan, network_l2proxy, network_gateway_device_route") + if d.config["parent"] == "" && d.config["vlan"] != "" { + return fmt.Errorf("The vlan setting can only be used when combined with a parent interface") + } + + // Generate effective parent name, including the VLAN part if option used. + effectiveParentName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) + + // If the effective parent doesn't exist and the vlan option is specified, it means we are going to create + // the VLAN parent at start, and we will configure the needed sysctls so don't need to check them yet. + if d.config["vlan"] != "" && !shared.PathExists(fmt.Sprintf("/sys/class/net/%s", effectiveParentName)) { + return nil } if d.config["ipv4.address"] != "" { // Check necessary sysctls are configured for use with l2proxy parent in IPVLAN l3s mode. - ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", d.config["parent"]) + ipv4FwdPath := fmt.Sprintf("net/ipv4/conf/%s/forwarding", effectiveParentName) sysctlVal, err := util.SysctlGet(ipv4FwdPath) if err != nil || sysctlVal != "1\n" { return fmt.Errorf("Error reading net sysctl %s: %v", ipv4FwdPath, err) } if sysctlVal != "1\n" { - return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv4.conf.%s.forwarding=1", d.config["parent"]) + // Replace . in parent name with / for sysctl formatting. + return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv4.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } } if d.config["ipv6.address"] != "" { // Check necessary sysctls are configured for use with l2proxy parent in IPVLAN l3s mode. - ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", d.config["parent"]) + ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", effectiveParentName) sysctlVal, err := util.SysctlGet(ipv6FwdPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err) } if sysctlVal != "1\n" { - return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.forwarding=1", d.config["parent"]) + // Replace . in parent name with / for sysctl formatting. + return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.forwarding=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } - ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", d.config["parent"]) + ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", effectiveParentName) sysctlVal, err = util.SysctlGet(ipv6ProxyNdpPath) if err != nil { return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err) } if sysctlVal != "1\n" { - return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", d.config["parent"]) + // Replace . in parent name with / for sysctl formatting. + return fmt.Errorf("IPVLAN in L3S mode requires sysctl net.ipv6.conf.%s.proxy_ndp=1", strings.ReplaceAll(effectiveParentName, ".", "/")) } } @@ -207,13 +223,13 @@ func (d *nicIPVLAN) setupParentSysctls(parentName string) error { ipv6FwdPath := fmt.Sprintf("net/ipv6/conf/%s/forwarding", parentName) err := util.SysctlSet(ipv6FwdPath, "1") if err != nil { - return fmt.Errorf("Error reading net sysctl %s: %v", ipv6FwdPath, err) + return fmt.Errorf("Error setting net sysctl %s: %v", ipv6FwdPath, err) } ipv6ProxyNdpPath := fmt.Sprintf("net/ipv6/conf/%s/proxy_ndp", parentName) err = util.SysctlSet(ipv6ProxyNdpPath, "1") if err != nil { - return fmt.Errorf("Error reading net sysctl %s: %v", ipv6ProxyNdpPath, err) + return fmt.Errorf("Error setting net sysctl %s: %v", ipv6ProxyNdpPath, err) } } From 73a92ebe0e9f16bc5b832020a4e72747ddd29507 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 11:48:22 +0100 Subject: [PATCH 7/9] lxd/device/nic/ipvlan: Adds host_table setting support Fixes #7123 Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- lxd/device/nic_ipvlan.go | 61 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/lxd/device/nic_ipvlan.go b/lxd/device/nic_ipvlan.go index 4247e4545a..c28386a764 100644 --- a/lxd/device/nic_ipvlan.go +++ b/lxd/device/nic_ipvlan.go @@ -31,6 +31,7 @@ func (d *nicIPVLAN) validateConfig(instConf instance.ConfigReader) error { "name", "mtu", "hwaddr", + "host_table", "vlan", "ipv4.gateway", "ipv6.gateway", @@ -202,6 +203,7 @@ func (d *nicIPVLAN) Start() (*deviceConfig.RunConfig, error) { } runConf.NetworkInterface = nic + runConf.PostHooks = append(runConf.PostHooks, d.postStart) return &runConf, nil } @@ -236,6 +238,39 @@ func (d *nicIPVLAN) setupParentSysctls(parentName string) error { return nil } +// postStart is run after the instance is started. +func (d *nicIPVLAN) postStart() error { + if d.config["ipv4.address"] != "" { + // Add static routes to instance IPs to custom routing tables if specified. + // This is in addition to the static route added by liblxc to the main routing table. + if d.config["host_table"] != "" { + for _, addr := range strings.Split(d.config["ipv4.address"], ",") { + addr = strings.TrimSpace(addr) + _, err := shared.RunCommand("ip", "-4", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/32", addr), "dev", "lo") + if err != nil { + return err + } + } + } + } + + if d.config["ipv6.address"] != "" { + // Add static routes to instance IPs to custom routing tables if specified. + // This is in addition to the static route added by liblxc to the main routing table. + if d.config["host_table"] != "" { + for _, addr := range strings.Split(d.config["ipv6.address"], ",") { + addr = strings.TrimSpace(addr) + _, err := shared.RunCommand("ip", "-6", "route", "add", "table", d.config["host_table"], fmt.Sprintf("%s/128", addr), "dev", "lo") + if err != nil { + return err + } + } + } + } + + return nil +} + // Stop is run when the device is removed from the instance. func (d *nicIPVLAN) Stop() (*deviceConfig.RunConfig, error) { runConf := deviceConfig.RunConfig{ @@ -253,6 +288,32 @@ func (d *nicIPVLAN) postStop() error { v := d.volatileGet() + if d.config["ipv4.address"] != "" { + // Remove static routes to instance IPs to custom routing tables if specified. + if d.config["host_table"] != "" { + for _, addr := range strings.Split(d.config["ipv4.address"], ",") { + addr = strings.TrimSpace(addr) + _, err := shared.RunCommand("ip", "-4", "route", "delete", "table", d.config["host_table"], fmt.Sprintf("%s/32", addr), "dev", "lo") + if err != nil { + return err + } + } + } + } + + if d.config["ipv6.address"] != "" { + // Remove static routes to instance IPs to custom routing tables if specified. + if d.config["host_table"] != "" { + for _, addr := range strings.Split(d.config["ipv6.address"], ",") { + addr = strings.TrimSpace(addr) + _, err := shared.RunCommand("ip", "-6", "route", "delete", "table", d.config["host_table"], fmt.Sprintf("%s/128", addr), "dev", "lo") + if err != nil { + return err + } + } + } + } + // This will delete the parent interface if we created it for VLAN parent. if shared.IsTrue(v["last_state.created"]) { parentName := network.GetHostDevice(d.config["parent"], d.config["vlan"]) From 498c15842cc37c598f41bc0b2ba0e4c854450ddd Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 11:50:44 +0100 Subject: [PATCH 8/9] api: Adds container_nic_ipvlan_host_table extension Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- doc/api-extensions.md | 4 ++++ shared/version/api.go | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index 7e1587551e..8b55a04693 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -1017,3 +1017,7 @@ Those are taken from the os-release data on the system. ## container\_nic\_routed\_host\_table This introduces the `host_table` NIC config key that can be used to add static routes for the instance's IPs to a custom policy routing table by ID. + +## container\_nic\_ipvlan\_host\_table +This introduces the `host_table` NIC config key that can be used to add static routes for the instance's IPs to a +custom policy routing table by ID. diff --git a/shared/version/api.go b/shared/version/api.go index 6c914d1aeb..9673f9cce5 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -206,6 +206,7 @@ var APIExtensions = []string{ "resources_cpu_core_die", "api_os", "container_nic_routed_host_table", + "container_nic_ipvlan_host_table", } // APIExtensionsCount returns the number of available API extensions. From ca7b4a4fd1953ce117d0c92fbfc6356c93b04c48 Mon Sep 17 00:00:00 2001 From: Thomas Parrott <thomas.parr...@canonical.com> Date: Wed, 15 Apr 2020 11:51:36 +0100 Subject: [PATCH 9/9] doc: Adds documentation for ipvlan NIC host_table setting Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com> --- doc/instances.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/instances.md b/doc/instances.md index e84c4ce2b8..0b6eb78504 100644 --- a/doc/instances.md +++ b/doc/instances.md @@ -329,6 +329,7 @@ parent | string | - | yes | The name o name | string | kernel assigned | no | The name of the interface inside the instance mtu | integer | parent MTU | no | The MTU of the new interface hwaddr | string | randomly assigned | no | The MAC address of the new interface +host\_table | integer | - | no | The custom policy routing table ID to add IP static routes to (in addition to main routing table). vlan | integer | - | no | The VLAN ID to attach to maas.subnet.ipv4 | string | - | no | MAAS IPv4 subnet to register the instance in maas.subnet.ipv6 | string | - | no | MAAS IPv6 subnet to register the instance in
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel