For each vrf we use we open a netlink watcher. This allows us to reconcile on changed route entries from outside routing agents.
Acked-by: Frode Nordahl <[email protected]> Acked-by: Lorenzo Bianconi <[email protected]> Acked-by: Dumitru Ceara <[email protected]> Signed-off-by: Felix Huettner <[email protected]> --- v6->v7: * Thanks to Dumitru for providing a patch for this to make the implementation significantly more efficient. v5->v6: * addressed review comments controller/automake.mk | 7 +- controller/ovn-controller.c | 51 +++++- controller/route-exchange.c | 6 +- controller/route-exchange.h | 3 + controller/route-table-notify-stub.c | 56 +++++++ controller/route-table-notify.c | 226 +++++++++++++++++++++++++++ controller/route-table-notify.h | 44 ++++++ tests/system-ovn.at | 8 - 8 files changed, 388 insertions(+), 13 deletions(-) create mode 100644 controller/route-table-notify-stub.c create mode 100644 controller/route-table-notify.c create mode 100644 controller/route-table-notify.h diff --git a/controller/automake.mk b/controller/automake.mk index fa3083d2d..c789c313c 100644 --- a/controller/automake.mk +++ b/controller/automake.mk @@ -55,6 +55,7 @@ controller_ovn_controller_SOURCES = \ controller/ecmp-next-hop-monitor.h \ controller/ecmp-next-hop-monitor.c \ controller/route-exchange.h \ + controller/route-table-notify.h \ controller/route.h \ controller/route.c @@ -62,10 +63,12 @@ if HAVE_NETLINK controller_ovn_controller_SOURCES += \ controller/route-exchange-netlink.h \ controller/route-exchange-netlink.c \ - controller/route-exchange.c + controller/route-exchange.c \ + controller/route-table-notify.c else controller_ovn_controller_SOURCES += \ - controller/route-exchange-stub.c + controller/route-exchange-stub.c \ + controller/route-table-notify-stub.c endif controller_ovn_controller_LDADD = lib/libovn.la $(OVS_LIBDIR)/libopenvswitch.la diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c index 0a7d39528..f9f7ea0ba 100644 --- a/controller/ovn-controller.c +++ b/controller/ovn-controller.c @@ -90,6 +90,7 @@ #include "ovn-dns.h" #include "route.h" #include "route-exchange.h" +#include "route-table-notify.h" VLOG_DEFINE_THIS_MODULE(main); @@ -5206,10 +5207,16 @@ en_route_exchange_run(struct engine_node *node, void *data) .sbrec_port_binding_by_name = sbrec_port_binding_by_name, .announce_routes = &route_data->announce_routes, }; - struct route_exchange_ctx_out r_ctx_out = { - }; + struct route_exchange_ctx_out r_ctx_out = {}; + + hmap_init(&r_ctx_out.route_table_watches); route_exchange_run(&r_ctx_in, &r_ctx_out); + route_table_notify_update_watches(&r_ctx_out.route_table_watches); + + route_table_watch_request_cleanup(&r_ctx_out.route_table_watches); + hmap_destroy(&r_ctx_out.route_table_watches); + engine_set_node_state(node, EN_UPDATED); } @@ -5227,6 +5234,38 @@ static void en_route_exchange_cleanup(void *data OVS_UNUSED) {} +struct ed_type_route_table_notify { + /* For incremental processing this could be tracked per datapath in + * the future. */ + bool changed; +}; + +static void +en_route_table_notify_run(struct engine_node *node, void *data) +{ + struct ed_type_route_table_notify *rtn = data; + if (rtn->changed) { + engine_set_node_state(node, EN_UPDATED); + } else { + engine_set_node_state(node, EN_UNCHANGED); + } + rtn->changed = false; +} + + +static void * +en_route_table_notify_init(struct engine_node *node OVS_UNUSED, + struct engine_arg *arg OVS_UNUSED) +{ + struct ed_type_route_table_notify *rtn = xzalloc(sizeof *rtn); + rtn->changed = true; + return rtn; +} + +static void +en_route_table_notify_cleanup(void *data OVS_UNUSED) +{} + /* Returns false if the northd internal version stored in SB_Global * and ovn-controller internal version don't match. */ @@ -5532,6 +5571,7 @@ main(int argc, char *argv[]) ENGINE_NODE(bfd_chassis, "bfd_chassis"); ENGINE_NODE(dns_cache, "dns_cache"); ENGINE_NODE(route, "route"); + ENGINE_NODE(route_table_notify, "route_table_notify"); ENGINE_NODE(route_exchange, "route_exchange"); #define SB_NODE(NAME, NAME_STR) ENGINE_NODE_SB(NAME, NAME_STR); @@ -5569,6 +5609,7 @@ main(int argc, char *argv[]) engine_noop_handler); engine_add_input(&en_route_exchange, &en_sb_port_binding, engine_noop_handler); + engine_add_input(&en_route_exchange, &en_route_table_notify, NULL); engine_add_input(&en_addr_sets, &en_sb_address_set, addr_sets_sb_address_set_handler); @@ -6087,6 +6128,10 @@ main(int argc, char *argv[]) &transport_zones, bridge_table); + struct ed_type_route_table_notify *rtn = + engine_get_internal_data(&en_route_table_notify); + rtn->changed = route_table_notify_run(); + stopwatch_start(CONTROLLER_LOOP_STOPWATCH_NAME, time_msec()); @@ -6369,6 +6414,7 @@ main(int argc, char *argv[]) } binding_wait(); + route_table_notify_wait(); } unixctl_server_run(unixctl); @@ -6523,6 +6569,7 @@ loop_done: ovsrcu_exit(); dns_resolve_destroy(); route_exchange_destroy(); + route_table_notify_destroy(); exit(retval); } diff --git a/controller/route-exchange.c b/controller/route-exchange.c index 7b91d3adb..8f740fa91 100644 --- a/controller/route-exchange.c +++ b/controller/route-exchange.c @@ -29,6 +29,7 @@ #include "ha-chassis.h" #include "local_data.h" #include "route.h" +#include "route-table-notify.h" #include "route-exchange.h" #include "route-exchange-netlink.h" @@ -166,7 +167,7 @@ sb_sync_learned_routes(const struct ovs_list *learned_routes, void route_exchange_run(struct route_exchange_ctx_in *r_ctx_in, - struct route_exchange_ctx_out *r_ctx_out OVS_UNUSED) + struct route_exchange_ctx_out *r_ctx_out) { struct sset old_maintained_vrfs = SSET_INITIALIZER(&old_maintained_vrfs); sset_swap(&_maintained_vrfs, &old_maintained_vrfs); @@ -208,6 +209,9 @@ route_exchange_run(struct route_exchange_ctx_in *r_ctx_in, r_ctx_in->sbrec_port_binding_by_name, r_ctx_in->sbrec_learned_route_by_datapath); + route_table_add_watch_request( + &r_ctx_out->route_table_watches, table_id); + re_nl_learned_routes_destroy(&received_routes); } diff --git a/controller/route-exchange.h b/controller/route-exchange.h index 66d8f3945..1e1a09da3 100644 --- a/controller/route-exchange.h +++ b/controller/route-exchange.h @@ -18,6 +18,8 @@ #ifndef ROUTE_EXCHANGE_H #define ROUTE_EXCHANGE_H 1 +#include "openvswitch/hmap.h" + struct route_exchange_ctx_in { struct ovsdb_idl_txn *ovnsb_idl_txn; struct ovsdb_idl_index *sbrec_port_binding_by_name; @@ -27,6 +29,7 @@ struct route_exchange_ctx_in { }; struct route_exchange_ctx_out { + struct hmap route_table_watches; }; void route_exchange_run(struct route_exchange_ctx_in *, diff --git a/controller/route-table-notify-stub.c b/controller/route-table-notify-stub.c new file mode 100644 index 000000000..f58897f03 --- /dev/null +++ b/controller/route-table-notify-stub.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2025, STACKIT GmbH & Co. KG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <config.h> + +#include <stdbool.h> + +#include "openvswitch/compiler.h" +#include "route-table-notify.h" + +bool +route_table_notify_run(void) +{ + return false; +} + +void +route_table_notify_wait(void) +{ +} + +void +route_table_add_watch_request(struct hmap *route_table_watches OVS_UNUSED, + uint32_t table_id OVS_UNUSED) +{ +} + +void +route_table_watch_request_cleanup(struct hmap *route_table_watches OVS_UNUSED) +{ +} + +void +route_table_notify_update_watches( + const struct hmap *route_table_watches OVS_UNUSED) +{ +} + +void +route_table_notify_destroy(void) +{ +} + diff --git a/controller/route-table-notify.c b/controller/route-table-notify.c new file mode 100644 index 000000000..88d4ce33e --- /dev/null +++ b/controller/route-table-notify.c @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2025, STACKIT GmbH & Co. KG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <config.h> + +#include <net/if.h> +#include <linux/rtnetlink.h> + +#include "netlink-notifier.h" +#include "openvswitch/vlog.h" + +#include "binding.h" +#include "hash.h" +#include "hmapx.h" +#include "route-table.h" +#include "route.h" +#include "route-table-notify.h" +#include "route-exchange-netlink.h" + +VLOG_DEFINE_THIS_MODULE(route_table_notify); + +struct route_table_watch_request { + struct hmap_node node; + uint32_t table_id; +}; + +struct route_table_watch_entry { + struct hmap_node node; + uint32_t table_id; +}; + +static struct hmap watches = HMAP_INITIALIZER(&watches); +static bool any_route_table_changed; +static struct route_table_msg nln_rtmsg_change; + +static struct nln *nl_route_handle; +static struct nln_notifier *nl_route_notifier_v4; +static struct nln_notifier *nl_route_notifier_v6; + +static void route_table_change(const void *change_, void *aux); + +static void +route_table_register_notifiers(void) +{ + VLOG_INFO("Adding route table watchers."); + ovs_assert(!nl_route_handle); + + nl_route_handle = nln_create(NETLINK_ROUTE, route_table_parse, + &nln_rtmsg_change); + ovs_assert(nl_route_handle); + + nl_route_notifier_v4 = + nln_notifier_create(nl_route_handle, RTNLGRP_IPV4_ROUTE, + route_table_change, NULL); + if (!nl_route_notifier_v4) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL(&rl, "Failed to create ipv4 route table watcher."); + } + + nl_route_notifier_v6 = + nln_notifier_create(nl_route_handle, RTNLGRP_IPV6_ROUTE, + route_table_change, NULL); + if (!nl_route_notifier_v6) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL(&rl, "Failed to create ipv6 route table watcher."); + } +} + +static void +route_table_deregister_notifiers(void) +{ + VLOG_INFO("Removing route table watchers."); + ovs_assert(nl_route_handle); + + nln_notifier_destroy(nl_route_notifier_v4); + nln_notifier_destroy(nl_route_notifier_v6); + nln_destroy(nl_route_handle); + nl_route_notifier_v4 = NULL; + nl_route_notifier_v6 = NULL; + nl_route_handle = NULL; +} + +static uint32_t +route_table_notify_hash_watch(uint32_t table_id) +{ + return hash_add(0, table_id); +} + +void +route_table_add_watch_request(struct hmap *route_table_watches, + uint32_t table_id) +{ + struct route_table_watch_request *wr = xzalloc(sizeof *wr); + wr->table_id = table_id; + hmap_insert(route_table_watches, &wr->node, + route_table_notify_hash_watch(wr->table_id)); +} + +void +route_table_watch_request_cleanup(struct hmap *route_table_watches) +{ + struct route_table_watch_request *wr; + HMAP_FOR_EACH_POP (wr, node, route_table_watches) { + free(wr); + } +} + +static struct route_table_watch_entry * +find_watch_entry(uint32_t table_id) +{ + struct route_table_watch_entry *we; + uint32_t hash = route_table_notify_hash_watch(table_id); + HMAP_FOR_EACH_WITH_HASH (we, node, hash, &watches) { + if (table_id == we->table_id) { + return we; + } + } + return NULL; +} + +static void +route_table_change(const void *change_, void *aux OVS_UNUSED) +{ + const struct route_table_msg *change = change_; + if (change && change->rd.rtm_protocol != RTPROT_OVN) { + if (find_watch_entry(change->rd.rta_table_id)) { + any_route_table_changed = true; + } + } +} + +static void +add_watch_entry(uint32_t table_id) +{ + VLOG_INFO("Registering new route table watcher for table %d.", + table_id); + + struct route_table_watch_entry *we; + uint32_t hash = route_table_notify_hash_watch(table_id); + we = xzalloc(sizeof *we); + we->table_id = table_id; + hmap_insert(&watches, &we->node, hash); + + if (!nl_route_handle) { + route_table_register_notifiers(); + } +} + +static void +remove_watch_entry(struct route_table_watch_entry *we) +{ + VLOG_INFO("Removing route table watcher for table %d.", we->table_id); + hmap_remove(&watches, &we->node); + free(we); + + if (hmap_is_empty(&watches)) { + route_table_deregister_notifiers(); + } +} + +bool +route_table_notify_run(void) +{ + any_route_table_changed = false; + + if (nl_route_handle) { + nln_run(nl_route_handle); + } + + return any_route_table_changed; +} + +void +route_table_notify_wait(void) +{ + if (nl_route_handle) { + nln_wait(nl_route_handle); + } +} + +void +route_table_notify_update_watches(const struct hmap *route_table_watches) +{ + struct hmapx sync_watches = HMAPX_INITIALIZER(&sync_watches); + struct route_table_watch_entry *we; + HMAP_FOR_EACH (we, node, &watches) { + hmapx_add(&sync_watches, we); + } + + struct route_table_watch_request *wr; + HMAP_FOR_EACH (wr, node, route_table_watches) { + we = find_watch_entry(wr->table_id); + if (we) { + hmapx_find_and_delete(&sync_watches, we); + } else { + add_watch_entry(wr->table_id); + } + } + + struct hmapx_node *node; + HMAPX_FOR_EACH (node, &sync_watches) { + remove_watch_entry(node->data); + } +} + +void +route_table_notify_destroy(void) +{ + struct route_table_watch_entry *we; + HMAP_FOR_EACH_SAFE (we, node, &watches) { + remove_watch_entry(we); + } +} diff --git a/controller/route-table-notify.h b/controller/route-table-notify.h new file mode 100644 index 000000000..a2bc05a49 --- /dev/null +++ b/controller/route-table-notify.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2025, STACKIT GmbH & Co. KG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ROUTE_TABLE_NOTIFY_H +#define ROUTE_TABLE_NOTIFY_H 1 + +#include <stdbool.h> +#include "openvswitch/hmap.h" + +/* Returns true if any route table has changed enough that we need + * to learn new routes. */ +bool route_table_notify_run(void); +void route_table_notify_wait(void); + +/* Add a watch request to the hmap. The hmap should later be passed to + * route_table_notify_update_watches*/ +void route_table_add_watch_request(struct hmap *route_table_watches, + uint32_t table_id); + +/* Cleanup all watch request in the provided hmap that where added using + * route_table_add_watch_request. */ +void route_table_watch_request_cleanup(struct hmap *route_table_watches); + +/* Updates the list of route table watches that are currently active. + * hmap should contain struct route_table_watch_request */ +void route_table_notify_update_watches(const struct hmap *route_table_watches); + +/* Cleans up all route table watches. */ +void route_table_notify_destroy(void); + +#endif /* ROUTE_TABLE_NOTIFY_H */ diff --git a/tests/system-ovn.at b/tests/system-ovn.at index 0584ccc17..8805eaf84 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -15677,8 +15677,6 @@ blackhole 198.51.100.0/24 proto 84 metric 1000]) # Now we test route learning. check_row_count Learned_Route 0 check ip route add 233.252.0.0/24 via 192.168.10.10 dev lo onlink vrf ovnvrf1337 -# For now we trigger a recompute as route watching is not yet implemented. -check ovn-appctl -t ovn-controller inc-engine/recompute check ovn-nbctl --wait=hv sync check_row_count Learned_Route 1 lp=$(fetch_column port_binding _uuid logical_port=internet-phys) @@ -15708,8 +15706,6 @@ check ovn-nbctl --wait=hv set Logical_Router_Port internet-phys \ options:dynamic-routing-port-name=mylearninglsp check ip route add 233.253.0.0/24 via 192.168.20.20 dev hv1-mll onlink vrf ovnvrf1337 -# For now we trigger a recompute as route watching is not yet implemented. -check ovn-appctl -t ovn-controller inc-engine/recompute check ovn-nbctl --wait=hv sync check_row_count Learned_Route 1 ip_prefix=233.253.0.0/24 nexthop=192.168.20.20 @@ -15943,8 +15939,6 @@ blackhole 198.51.100.0/24 proto 84 metric 1000]) # Now we test route learning. check_row_count Learned_Route 0 check ip route add 233.252.0.0/24 via 192.168.10.10 dev lo onlink vrf ovnvrf1337 -# For now we trigger a recompute as route watching is not yet implemented. -check ovn-appctl -t ovn-controller inc-engine/recompute check ovn-nbctl --wait=hv sync check_row_count Learned_Route 1 lp=$(fetch_column port_binding _uuid logical_port=internet-phys) @@ -15974,8 +15968,6 @@ check ovn-nbctl --wait=hv set Logical_Router_Port internet-phys \ options:dynamic-routing-port-name=mylearninglsp check ip route add 233.253.0.0/24 via 192.168.20.20 dev hv1-mll onlink vrf ovnvrf1337 -# For now we trigger a recompute as route watching is not yet implemented. -check ovn-appctl -t ovn-controller inc-engine/recompute check ovn-nbctl --wait=hv sync check_row_count Learned_Route 1 ip_prefix=233.253.0.0/24 nexthop=192.168.20.20 -- 2.47.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
