This patch adds local router support. A logical router can be declared local at northbound level via external_ids. This is communicated to the southbound database via northd which populates the external_id of the datapath_binding table. ovn-controller will also allow a chassis to be configured for 1 or more local routers. When ovn-controller configures openflow rules it looks at whether a logical router is local and it so it checks where its chassis instance local routers has this particular logical router configured as local; if it does, it allows the southbound flows of the local logical router to be translated into openflow rules and programmed on this chassis. If a logical router is labelled as local but a given chassis is not configured for that logical router to be local, then the corresponding openflow rules are not programmed on this chassis.
Signed-off-by: Darrell Ball <dlu...@gmail.com> --- ovn/controller/chassis.c | 32 +++++- ovn/controller/chassis.h | 4 +- ovn/controller/lflow.c | 16 ++- ovn/controller/lflow.h | 4 +- ovn/controller/ovn-controller.8.xml | 8 ++ ovn/controller/ovn-controller.c | 50 +++++++++- ovn/controller/ovn-controller.h | 4 + ovn/northd/ovn-northd.c | 17 +++- ovn/ovn-nb.xml | 6 +- ovn/ovn-sb.xml | 8 ++ tests/ovn.at | 189 ++++++++++++++++++++++++++++++++++++ 11 files changed, 324 insertions(+), 14 deletions(-) diff --git a/ovn/controller/chassis.c b/ovn/controller/chassis.c index d40181b..e58145b 100644 --- a/ovn/controller/chassis.c +++ b/ovn/controller/chassis.c @@ -24,6 +24,7 @@ #include "openvswitch/vlog.h" #include "ovn/lib/ovn-sb-idl.h" #include "ovn-controller.h" +#include "sset.h" VLOG_DEFINE_THIS_MODULE(chassis); @@ -63,8 +64,16 @@ get_bridge_mappings(const struct smap *ext_ids) return bridge_mappings ? bridge_mappings : ""; } +static const char * +get_local_routers(const struct smap *ext_ids) +{ + const char *local_routers = smap_get(ext_ids, "local-routers"); + return local_routers ? local_routers : ""; +} + void -chassis_run(struct controller_ctx *ctx, const char *chassis_id) +chassis_run(struct controller_ctx *ctx, const char *chassis_id, + struct sset *local_router_datapaths) { if (!ctx->ovnsb_idl_txn) { return; @@ -112,6 +121,8 @@ chassis_run(struct controller_ctx *ctx, const char *chassis_id) const char *bridge_mappings = get_bridge_mappings(&cfg->external_ids); + const char *local_routers = get_local_routers(&cfg->external_ids); + const struct sbrec_chassis *chassis_rec = get_chassis(ctx->ovnsb_idl, chassis_id); @@ -131,6 +142,17 @@ chassis_run(struct controller_ctx *ctx, const char *chassis_id) smap_destroy(&new_ids); } + const char *chassis_local_routers + = get_local_routers(&chassis_rec->external_ids); + if (strcmp(local_routers, chassis_local_routers)) { + struct smap new_ids; + smap_clone(&new_ids, &chassis_rec->external_ids); + smap_replace(&new_ids, "local-router", local_routers); + sbrec_chassis_verify_external_ids(chassis_rec); + sbrec_chassis_set_external_ids(chassis_rec, &new_ids); + smap_destroy(&new_ids); + } + /* Compare desired tunnels against those currently in the database. */ uint32_t cur_tunnels = 0; bool same = true; @@ -166,12 +188,16 @@ chassis_run(struct controller_ctx *ctx, const char *chassis_id) chassis_id); if (!chassis_rec) { - struct smap ext_ids = SMAP_CONST1(&ext_ids, "ovn-bridge-mappings", - bridge_mappings); + + struct smap ext_ids = SMAP_INITIALIZER(&ext_ids); + smap_add(&ext_ids, "ovn-bridge-mappings", + bridge_mappings); + smap_add(&ext_ids, "local-routers", local_routers); chassis_rec = sbrec_chassis_insert(ctx->ovnsb_idl_txn); sbrec_chassis_set_name(chassis_rec, chassis_id); sbrec_chassis_set_hostname(chassis_rec, hostname); sbrec_chassis_set_external_ids(chassis_rec, &ext_ids); + smap_destroy(&ext_ids); } int n_encaps = count_1bits(req_tunnels); diff --git a/ovn/controller/chassis.h b/ovn/controller/chassis.h index 26017d0..324e3cf 100644 --- a/ovn/controller/chassis.h +++ b/ovn/controller/chassis.h @@ -21,9 +21,11 @@ struct controller_ctx; struct ovsdb_idl; struct ovsrec_bridge; +struct sset; void chassis_register_ovs_idl(struct ovsdb_idl *); -void chassis_run(struct controller_ctx *, const char *chassis_id); +void chassis_run(struct controller_ctx *, const char *chassis_id, + struct sset *); bool chassis_cleanup(struct controller_ctx *, const char *chassis_id); #endif /* ovn/chassis.h */ diff --git a/ovn/controller/lflow.c b/ovn/controller/lflow.c index 96b7c66..d86646a 100644 --- a/ovn/controller/lflow.c +++ b/ovn/controller/lflow.c @@ -199,7 +199,8 @@ add_logical_flows(struct controller_ctx *ctx, const struct lport_index *lports, const struct mcgroup_index *mcgroups, const struct hmap *local_datapaths, const struct hmap *patched_datapaths, - const struct simap *ct_zones, struct hmap *flow_table) + const struct simap *ct_zones, struct hmap *flow_table, + const struct sset *local_routers) { uint32_t conj_id_ofs = 1; @@ -250,6 +251,13 @@ add_logical_flows(struct controller_ctx *ctx, const struct lport_index *lports, continue; } } + } else { + const char *local_router = + smap_get(&ldp->external_ids, "local-router"); + if (strcmp(local_router, "") && + !is_local_router(local_routers, local_router)) { + continue; + } } /* Determine translation of logical table IDs to physical table IDs. */ @@ -429,10 +437,12 @@ lflow_run(struct controller_ctx *ctx, const struct lport_index *lports, const struct mcgroup_index *mcgroups, const struct hmap *local_datapaths, const struct hmap *patched_datapaths, - const struct simap *ct_zones, struct hmap *flow_table) + const struct simap *ct_zones, struct hmap *flow_table, + const struct sset *local_router_datapaths) { add_logical_flows(ctx, lports, mcgroups, local_datapaths, - patched_datapaths, ct_zones, flow_table); + patched_datapaths, ct_zones, flow_table, + local_router_datapaths); add_neighbor_flows(ctx, lports, flow_table); } diff --git a/ovn/controller/lflow.h b/ovn/controller/lflow.h index a3fc50c..4144481 100644 --- a/ovn/controller/lflow.h +++ b/ovn/controller/lflow.h @@ -41,6 +41,7 @@ struct lport_index; struct mcgroup_index; struct simap; struct uuid; +struct sset; /* OpenFlow table numbers. * @@ -64,7 +65,8 @@ void lflow_run(struct controller_ctx *, const struct lport_index *, const struct hmap *local_datapaths, const struct hmap *patched_datapaths, const struct simap *ct_zones, - struct hmap *flow_table); + struct hmap *flow_table, + const struct sset *local_routers); void lflow_destroy(void); #endif /* ovn/lflow.h */ diff --git a/ovn/controller/ovn-controller.8.xml b/ovn/controller/ovn-controller.8.xml index 1ee3a6e..582a00b 100644 --- a/ovn/controller/ovn-controller.8.xml +++ b/ovn/controller/ovn-controller.8.xml @@ -154,6 +154,14 @@ value mapping two physical network names to two ovs bridges would be: <code>physnet1:br-eth0,physnet2:br-eth1</code>. </dd> + <dt><code>external_ids:local-routers</code></dt> + <dd> + A set of a local routers on this chassis instance. The logical flows of a + local logical router datapath are only translated into openflow + flows if the chassis is configured for that local logical router. + An example set with two local routers would be: + <code>local_router_1,local_router_2</code>. + </dd> </dl> <h1>Open vSwitch Database Usage</h1> diff --git a/ovn/controller/ovn-controller.c b/ovn/controller/ovn-controller.c index f68f842..e6c390a 100644 --- a/ovn/controller/ovn-controller.c +++ b/ovn/controller/ovn-controller.c @@ -51,6 +51,7 @@ #include "stream.h" #include "unixctl.h" #include "util.h" +#include "sset.h" VLOG_DEFINE_THIS_MODULE(main); @@ -252,6 +253,37 @@ get_ovnsb_remote_probe_interval(struct ovsdb_idl *ovs_idl, int *value) return false; } +bool is_local_router(const struct sset *local_routers, + const char *local_router) +{ + if (sset_find(local_routers, local_router)) { + return true; + } + return false; +} + +static void +build_local_routers(const struct smap *ext_ids, + struct sset *local_routers) +{ + const char *local_router_cfg = smap_get(ext_ids, "local-routers"); + if (!local_router_cfg) { + local_router_cfg = ""; + } + + char *cur, *next, *start; + next = start = xstrdup(local_router_cfg); + while ((cur = strsep(&next, ",")) && *cur) { + char *local_router = cur; + + if (sset_find(local_routers, local_router)) { + continue; + } + sset_add(local_routers, local_router); + } + free(start); +} + int main(int argc, char *argv[]) { @@ -353,11 +385,22 @@ main(int argc, char *argv[]) struct hmap patched_datapaths = HMAP_INITIALIZER(&patched_datapaths); + struct sset local_routers = + SSET_INITIALIZER(&local_routers); + const struct ovsrec_bridge *br_int = get_br_int(&ctx); const char *chassis_id = get_chassis_id(ctx.ovs_idl); if (chassis_id) { - chassis_run(&ctx, chassis_id); + chassis_run(&ctx, chassis_id, &local_routers); + + const struct sbrec_chassis *chassis_rec = + get_chassis(ctx.ovnsb_idl, chassis_id); + if (chassis_rec) { + build_local_routers(&chassis_rec->external_ids, + &local_routers); + } + encaps_run(&ctx, br_int, chassis_id); binding_run(&ctx, br_int, chassis_id, &ct_zones, ct_zone_bitmap, &local_datapaths); @@ -377,7 +420,8 @@ main(int argc, char *argv[]) struct hmap flow_table = HMAP_INITIALIZER(&flow_table); lflow_run(&ctx, &lports, &mcgroups, &local_datapaths, - &patched_datapaths, &ct_zones, &flow_table); + &patched_datapaths, &ct_zones, &flow_table, + &local_routers); if (chassis_id) { physical_run(&ctx, mff_ovn_geneve, br_int, chassis_id, &ct_zones, &flow_table, @@ -404,6 +448,8 @@ main(int argc, char *argv[]) } hmap_destroy(&patched_datapaths); + sset_destroy(&local_routers); + unixctl_server_run(unixctl); unixctl_server_wait(unixctl); diff --git a/ovn/controller/ovn-controller.h b/ovn/controller/ovn-controller.h index 9af7959..012560c 100644 --- a/ovn/controller/ovn-controller.h +++ b/ovn/controller/ovn-controller.h @@ -23,6 +23,7 @@ /* Linux supports a maximum of 64K zones, which seems like a fine default. */ #define MAX_CT_ZONES 65535 +struct sset; struct controller_ctx { struct ovsdb_idl *ovnsb_idl; struct ovsdb_idl_txn *ovnsb_idl_txn; @@ -53,6 +54,9 @@ struct patched_datapath { struct patched_datapath *get_patched_datapath(const struct hmap *, uint32_t tunnel_key); +bool is_local_router(const struct sset *local_routers, + const char *local_router); + const struct ovsrec_bridge *get_bridge(struct ovsdb_idl *, const char *br_name); diff --git a/ovn/northd/ovn-northd.c b/ovn/northd/ovn-northd.c index 9e03606..b271f7f 100644 --- a/ovn/northd/ovn-northd.c +++ b/ovn/northd/ovn-northd.c @@ -460,8 +460,21 @@ build_datapaths(struct northd_context *ctx, struct hmap *datapaths) char uuid_s[UUID_LEN + 1]; sprintf(uuid_s, UUID_FMT, UUID_ARGS(&od->key)); const char *key = od->nbs ? "logical-switch" : "logical-router"; - const struct smap id = SMAP_CONST1(&id, key, uuid_s); - sbrec_datapath_binding_set_external_ids(od->sb, &id); + struct smap ids = SMAP_INITIALIZER(&ids); + smap_add(&ids, key, uuid_s); + + if (od->nbr) { + const char *value = smap_get(&od->nbr->external_ids, + "local-router"); + + if (value) { + smap_add(&ids, "local-router", value); + } else { + smap_add(&ids, "local-router", ""); + } + } + sbrec_datapath_binding_set_external_ids(od->sb, &ids); + smap_destroy(&ids); sbrec_datapath_binding_set_tunnel_key(od->sb, tunnel_key); } diff --git a/ovn/ovn-nb.xml b/ovn/ovn-nb.xml index 34251af..5b84d6c 100644 --- a/ovn/ovn-nb.xml +++ b/ovn/ovn-nb.xml @@ -639,8 +639,10 @@ </column> <group title="Common Columns"> - <column name="external_ids"> - See <em>External IDs</em> at the beginning of this document. + <column name="external_ids" key="local-router"> + The CMS populates this optional key with the a local router name, which + <code>ovn-northd</code> writes into the southbound database + <code>datapath_binding</code> table. </column> </group> </table> diff --git a/ovn/ovn-sb.xml b/ovn/ovn-sb.xml index efd2f9a..aa0d8bd 100644 --- a/ovn/ovn-sb.xml +++ b/ovn/ovn-sb.xml @@ -1189,6 +1189,14 @@ tcp.flags = RST; corresponding <ref table="Logical_Router" db="OVN_Northbound"/> row in the <ref db="OVN_Northbound"/> database. </column> + + <column name="external_ids" key="local-router"> + <code>ovn-northd</code> populates this key with the a local router name, + which <code>ovn-northd</code> reads from the corresponding logical + router in the northbound database. The use of this external_id is + optional. + </column> + </group> <group title="Common Columns"> diff --git a/tests/ovn.at b/tests/ovn.at index 173dc27..b7b3ad5 100644 --- a/tests/ovn.at +++ b/tests/ovn.at @@ -2748,3 +2748,192 @@ OVS_APP_EXIT_AND_WAIT([ovs-vswitchd]) OVS_APP_EXIT_AND_WAIT([ovsdb-server]) AT_CLEANUP + +AT_SETUP([ovn -- 2 HVs, 3 LS, 1 lport/LS, 2 peer LRs, local router]) +AT_KEYWORDS([ovnlocalrouter]) +AT_SKIP_IF([test $HAVE_PYTHON = no]) +ovn_start + +# Logical network: +# Two LRs - R1 and R2 that are connected to each other as peers in 20.0.0.0/24 +# network. R1 has switch foo (192.168.1.0/24) connected to it. +# R2 has alice (172.16.1.0/24) and bob (172.16.2.0/24) connected to it. + +ovn-nbctl create Logical_Router name=R1 +ovn-nbctl create Logical_Router name=R2 external-ids:local-router=llr2 + +ovn-nbctl lswitch-add foo +ovn-nbctl lswitch-add alice +ovn-nbctl lswitch-add bob + +# Connect foo to R1 +ovn-nbctl -- --id=@lrp create Logical_Router_port name=foo \ +network=192.168.1.1/24 mac=\"00:00:00:01:02:03\" -- add Logical_Router R1 \ +ports @lrp -- lport-add foo rp-foo + +ovn-nbctl set Logical_port rp-foo type=router options:router-port=foo \ +addresses=\"00:00:00:01:02:03\" + +# Connect alice to R2 +ovn-nbctl -- --id=@lrp create Logical_Router_port name=alice \ +network=172.16.1.1/24 mac=\"00:00:00:01:02:04\" -- add Logical_Router R2 \ +ports @lrp -- lport-add alice rp-alice + +ovn-nbctl set Logical_port rp-alice type=router options:router-port=alice \ +addresses=\"00:00:00:01:02:04\" + +# Connect bob to R1 +ovn-nbctl -- --id=@lrp create Logical_Router_port name=bob-r1 \ +network=172.16.2.3/24 mac=\"00:00:00:01:02:06\" -- add Logical_Router R1 \ +ports @lrp -- lport-add bob rp-bob-r1 + +ovn-nbctl set Logical_port rp-bob-r1 type=router options:router-port=bob-r1 \ +addresses=\"00:00:00:01:02:06\" + +# Connect bob to R2 +ovn-nbctl -- --id=@lrp create Logical_Router_port name=bob \ +network=172.16.2.1/24 mac=\"00:00:00:01:02:05\" -- add Logical_Router R2 \ +ports @lrp -- lport-add bob rp-bob + +ovn-nbctl set Logical_port rp-bob type=router options:router-port=bob \ +addresses=\"00:00:00:01:02:05\" + +# Create logical port foo1 in foo +ovn-nbctl lport-add foo foo1 \ +-- lport-set-addresses foo1 "f0:00:00:01:02:03 192.168.1.2" + +# Create logical port alice1 in alice +ovn-nbctl lport-add alice alice1 \ +-- lport-set-addresses alice1 "f0:00:00:01:02:04 172.16.1.2" + +# Create logical port bob1 in bob +ovn-nbctl lport-add bob bob1 \ +-- lport-set-addresses bob1 "f0:00:00:01:02:05 172.16.2.2" + +# Create two hypervisor and create OVS ports corresponding to logical ports. +net_add n1 + +sim_add hv1 +as hv1 +ovs-vsctl add-br br-phys +ovn_attach n1 br-phys 192.168.0.1 +ovs-vsctl -- add-port br-int hv1-vif1 -- \ + set interface hv1-vif1 external-ids:iface-id=foo1 \ + options:tx_pcap=hv1/vif1-tx.pcap \ + options:rxq_pcap=hv1/vif1-rx.pcap \ + ofport-request=1 + + +sim_add hv2 +as hv2 +ovs-vsctl add-br br-phys +ovs-vsctl set open . external-ids:local-routers=llr2 +ovn_attach n1 br-phys 192.168.0.2 +ovs-vsctl -- add-port br-int hv2-vif1 -- \ + set interface hv2-vif1 external-ids:iface-id=bob1 \ + options:tx_pcap=hv2/vif1-tx.pcap \ + options:rxq_pcap=hv2/vif1-rx.pcap \ + ofport-request=1 + +ovs-vsctl -- add-port br-int hv2-vif2 -- \ + set interface hv2-vif2 external-ids:iface-id=alice1 \ + options:tx_pcap=hv2/vif2-tx.pcap \ + options:rxq_pcap=hv2/vif2-rx.pcap \ + ofport-request=2 + + +# Pre-populate the hypervisors' ARP tables so that we don't lose any +# packets for ARP resolution (native tunneling doesn't queue packets +# for ARP resolution). +ovn_populate_arp + +# Allow some time for ovn-northd and ovn-controller to catch up. +# XXX This should be more systematic. +sleep 1 + +ip_to_hex() { + printf "%02x%02x%02x%02x" "$@" +} +trim_zeros() { + sed 's/\(00\)\{1,\}$//' +} + +# Send ip packets between foo1 and bob1 +src_mac="f00000010203" +dst_mac="000000010203" +src_ip=`ip_to_hex 192 168 1 2` +dst_ip=`ip_to_hex 172 16 2 2` +packet=${dst_mac}${src_mac}08004500001c0000000040110000${src_ip}${dst_ip}0035111100080000 +as hv1 ovs-appctl netdev-dummy/receive hv1-vif1 $packet + +# Send ip packets between bob1 and alice1 +src_mac="f00000010205" +dst_mac="000000010205" +src_ip=`ip_to_hex 172 16 2 2` +dst_ip=`ip_to_hex 172 16 1 2` +packet=${dst_mac}${src_mac}08004500001c0000000040110000${src_ip}${dst_ip}0035111100080000 +as hv2 ovs-appctl netdev-dummy/receive hv2-vif1 $packet + +echo "---------NB dump-----" +ovn-nbctl show +echo "---------------------" +ovn-nbctl list logical_router +echo "---------------------" +ovn-nbctl list logical_router_port +echo "---------------------" + +echo "---------SB dump-----" +ovn-sbctl list datapath_binding +echo "---------------------" +ovn-sbctl list port_binding +echo "---------------------" +ovn-sbctl dump-flows + +#echo "------ hv1 dump ----------" +#as hv1 ovs-ofctl dump-flows br-int +#echo "------ hv2 dump ----------" +#as hv2 ovs-ofctl dump-flows br-int + +# Packet to Expect at bob1 +src_mac="000000010206" +dst_mac="f00000010205" +src_ip=`ip_to_hex 192 168 1 2` +dst_ip=`ip_to_hex 172 16 2 2` +expected=${dst_mac}${src_mac}08004500001c000000003f110100${src_ip}${dst_ip}0035111100080000 + +$PYTHON "$top_srcdir/utilities/ovs-pcap.in" hv2/vif1-tx.pcap | trim_zeros > received.packets +echo $expected | trim_zeros > expout +AT_CHECK([cat received.packets], [0], [expout]) + +# Packet to Expect at alice1 +src_mac="000000010204" +dst_mac="f00000010204" +src_ip=`ip_to_hex 172 16 2 2` +dst_ip=`ip_to_hex 172 16 1 2` +expected=${dst_mac}${src_mac}08004500001c000000003f110100${src_ip}${dst_ip}0035111100080000 + +$PYTHON "$top_srcdir/utilities/ovs-pcap.in" hv2/vif2-tx.pcap | trim_zeros > received1.packets +echo $expected | trim_zeros > expout +AT_CHECK([cat received1.packets], [0], [expout]) + +for sim in hv1 hv2; do + as $sim + OVS_APP_EXIT_AND_WAIT([ovn-controller]) + OVS_APP_EXIT_AND_WAIT([ovs-vswitchd]) + OVS_APP_EXIT_AND_WAIT([ovsdb-server]) +done + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as main +OVS_APP_EXIT_AND_WAIT([ovs-vswitchd]) +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +AT_CLEANUP -- 1.9.1 _______________________________________________ dev mailing list dev@openvswitch.org http://openvswitch.org/mailman/listinfo/dev