The controller uses a filtered two-pass approach to gate LB-derived routes by Service_Monitor health. First, route_run() collects LB-route candidates from the Advertised_Route table: rows whose external_ids:source=lb marker is set and whose tracked_port is local. If no candidates exist, the Load_Balancer and Service_Monitor tables are not scanned.
When candidates exist, the controller builds a per-VIP-IP gate by scanning the Load_Balancer.vips smap for entries whose VIP IP matches the route ip_prefix, extracting the backend (ip, port, protocol) tuples, and cross-referencing them against Service_Monitor rows for the route tracked_port (the backend LSP). When matching Service_Monitor rows exist but none report online, the controller skips installing the kernel route and records external_ids:status=withdrawn-monitor. When no matching Service_Monitor rows exist (e.g. the backend has no health check), the route is installed unconditionally with status=advertised. Each VIP IP is gated independently: a route for VIP A is withdrawn only when all of VIP A backends are offline, regardless of the health state of VIP B backends on the same LSP. Because the gate is keyed on external_ids:source=lb, non-LB routes (NAT, connected, static) that happen to share the same ip_prefix as an LB VIP are never gated. The Load_Balancer SB table is added as an engine input to the route node so the controller recomputes when LB VIPs change. Additionally: - Replicates Advertised_Route.external_ids (previously omitted from the SB IDL) so it can read the enable flag and publish status. - Requests a recompute when external_ids changes on a route row, so that out-of-band writers (e.g. an L7 health agent setting external_ids:enabled=false) take effect on the next recompute. - Honors external_ids:enabled as an administrative enable flag: when explicitly false, route_run() skips installing the route and records status=withdrawn-admin. The default (key absent) is enabled. - Publishes its advertisement decision to external_ids:status (advertised, withdrawn-admin, withdrawn-monitor). Only the owner chassis writes status. When ownership transitions the value may be stale until the new owner updates it. - Documents external_ids:enabled and external_ids:status under Advertised_Route in ovn-sb.xml. route_run() runs with a read-only SB txn, so it only records the status decision. The route-exchange node, which holds a writable txn, writes external_ids:status (write-on-change). Only the owner chassis records an entry, so there is a single writer per route. Signed-off-by: Dmitrii Shcherbakov <[email protected]> --- controller/ovn-controller.c | 61 ++- controller/route.c | 313 +++++++++++++- controller/route.h | 20 + ovn-sb.xml | 25 +- tests/ovn-inc-proc-graph-dump.at | 3 + tests/system-ovn.at | 691 +++++++++++++++++++++++++++++++ 6 files changed, 1106 insertions(+), 7 deletions(-) diff --git a/controller/ovn-controller.c b/controller/ovn-controller.c index cbe6576f6..df0ade41d 100644 --- a/controller/ovn-controller.c +++ b/controller/ovn-controller.c @@ -5276,6 +5276,11 @@ struct ed_type_route { /* Contains struct advertise_datapath_entry */ struct hmap announce_routes; + /* Contains struct advertised_route_status recorded by route_run() + * and published to Advertised_Route.external_ids:status by + * route_exchange_run(). */ + struct hmap advertised_route_status; + struct ovsdb_idl *ovnsb_idl; }; @@ -5306,6 +5311,10 @@ en_route_run(struct engine_node *node, void *data) const struct sbrec_advertised_route_table *advertised_route_table = EN_OVSDB_GET(engine_get_input("SB_advertised_route", node)); + const struct sbrec_service_monitor_table *service_monitor_table = + EN_OVSDB_GET(engine_get_input("SB_service_monitor", node)); + const struct sbrec_load_balancer_table *load_balancer_table = + EN_OVSDB_GET(engine_get_input("SB_load_balancer", node)); const struct ovsrec_open_vswitch *cfg = ovsrec_open_vswitch_table_first(ovs_table); @@ -5314,6 +5323,8 @@ en_route_run(struct engine_node *node, void *data) struct route_ctx_in r_ctx_in = { .advertised_route_table = advertised_route_table, + .service_monitor_table = service_monitor_table, + .load_balancer_table = load_balancer_table, .sbrec_port_binding_by_name = sbrec_port_binding_by_name, .chassis = chassis, .dynamic_routing_port_mapping = dynamic_routing_port_mapping, @@ -5327,9 +5338,11 @@ en_route_run(struct engine_node *node, void *data) .filtered_ports = &re_data->filtered_ports, .tracked_ports_remote = &re_data->tracked_ports_remote, .announce_routes = &re_data->announce_routes, + .advertised_route_status = &re_data->advertised_route_status, }; route_cleanup(&re_data->announce_routes); + advertised_route_status_clear(&re_data->advertised_route_status); tracked_datapaths_clear(r_ctx_out.tracked_re_datapaths); sset_clear(r_ctx_out.tracked_ports_local); sset_clear(r_ctx_out.tracked_ports_remote); @@ -5351,6 +5364,7 @@ en_route_init(struct engine_node *node OVS_UNUSED, sset_init(&data->tracked_ports_remote); sset_init(&data->filtered_ports); hmap_init(&data->announce_routes); + hmap_init(&data->advertised_route_status); data->ovnsb_idl = arg->sb_idl; return data; @@ -5367,6 +5381,8 @@ en_route_cleanup(void *data) sset_destroy(&re_data->filtered_ports); route_cleanup(&re_data->announce_routes); hmap_destroy(&re_data->announce_routes); + advertised_route_status_clear(&re_data->advertised_route_status); + hmap_destroy(&re_data->advertised_route_status); } static enum engine_input_handler_result @@ -5575,6 +5591,11 @@ route_sb_advertised_route_data_handler(struct engine_node *node, void *data) return EN_UNHANDLED; } + if (sbrec_advertised_route_is_updated( + sbrec_route, SBREC_ADVERTISED_ROUTE_COL_EXTERNAL_IDS)) { + return EN_UNHANDLED; + } + if (sbrec_route->tracked_port) { const char *name = sbrec_route->tracked_port->logical_port; if (!(sset_contains(&re_data->tracked_ports_local, name) || @@ -5626,6 +5647,23 @@ route_sb_datapath_binding_handler(struct engine_node *node, return EN_HANDLED_UNCHANGED; } +static enum engine_input_handler_result +route_sb_service_monitor_handler(struct engine_node *node, + void *data OVS_UNUSED) +{ + const struct sbrec_service_monitor_table *sm_table = + EN_OVSDB_GET(engine_get_input("SB_service_monitor", node)); + + const struct sbrec_service_monitor *sm; + SBREC_SERVICE_MONITOR_TABLE_FOR_EACH_TRACKED (sm, sm_table) { + if (sm->type && !strcmp(sm->type, "load-balancer")) { + return EN_UNHANDLED; + } + } + + return EN_HANDLED_UNCHANGED; +} + static int table_id_cmp(const void *a_, const void *b_) { @@ -5699,6 +5737,21 @@ en_route_exchange_run(struct engine_node *node, void *data) }; route_exchange_run(&r_ctx_in, &r_ctx_out); + + /* Publish per-route advertisement status to + * Advertised_Route.external_ids:status (write-on-change). + * Only the owner chassis records status entries. */ + if (r_ctx_in.ovnsb_idl_txn) { + struct advertised_route_status *s; + HMAP_FOR_EACH (s, node, &route_data->advertised_route_status) { + const char *cur = smap_get(&s->route->external_ids, "status"); + if (!cur || strcmp(cur, s->status)) { + sbrec_advertised_route_update_external_ids_setkey( + s->route, "status", s->status); + } + } + } + route_table_notify_update(&rt_notify->watches); re->sb_changes_pending = r_ctx_out.sb_changes_pending; @@ -6874,7 +6927,8 @@ evpn_arp_vtep_binding_handler(struct engine_node *node, void *data OVS_UNUSED) SB_NODE(acl_id) \ SB_NODE(advertised_route) \ SB_NODE(learned_route) \ - SB_NODE(advertised_mac_binding) + SB_NODE(advertised_mac_binding) \ + SB_NODE(service_monitor) enum sb_engine_node { #define SB_NODE(NAME) SB_##NAME, @@ -7007,6 +7061,9 @@ inc_proc_ovn_controller_init( route_sb_advertised_route_data_handler); engine_add_input(&en_route, &en_sb_datapath_binding, route_sb_datapath_binding_handler); + engine_add_input(&en_route, &en_sb_service_monitor, + route_sb_service_monitor_handler); + engine_add_input(&en_route, &en_sb_load_balancer, NULL); engine_add_input(&en_route_exchange, &en_route, NULL); engine_add_input(&en_route_exchange, &en_sb_learned_route, @@ -7590,8 +7647,6 @@ main(int argc, char *argv[]) ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_ha_chassis_col_external_ids); ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_ha_chassis_group_col_external_ids); - ovsdb_idl_omit(ovnsb_idl_loop.idl, - &sbrec_advertised_route_col_external_ids); ovsdb_idl_omit(ovnsb_idl_loop.idl, &sbrec_learned_route_col_external_ids); ovsdb_idl_omit(ovnsb_idl_loop.idl, diff --git a/controller/route.c b/controller/route.c index 13e6d3010..b828f32b0 100644 --- a/controller/route.c +++ b/controller/route.c @@ -18,6 +18,7 @@ #include <config.h> #include <net/if.h> +#include <arpa/inet.h> #include "vswitch-idl.h" #include "openvswitch/hmap.h" @@ -25,6 +26,7 @@ #include "openvswitch/ofp-parse.h" #include "lib/ovn-sb-idl.h" +#include "lib/ovn-util.h" #include "binding.h" #include "ha-chassis.h" @@ -38,6 +40,221 @@ VLOG_DEFINE_THIS_MODULE(exchange); #define PRIORITY_DEFAULT 1000 #define PRIORITY_LOCAL_BOUND 100 +struct sm_lb_key { + struct hmap_node node; + const char *logical_port; + const char *chassis_name; + const char *ip; + int64_t port; + const char *protocol; + bool online; +}; + +/* Per-candidate state for an LB-derived Advertised_Route that is + * eligible for Service_Monitor gating. Keyed by vip_ip so the LB + * table scan can update matching candidates without parsing backends + * for irrelevant VIPs. */ +struct lb_route_gate { + struct hmap_node node; + const struct sbrec_advertised_route *route; + char *vip_ip; + const char *tracked_lp; + bool seen_monitor; + bool any_online; +}; + +/* Extract the VIP IP string from a route ip_prefix (e.g. "172.16.1.20" + * from prefix/plen). Writes the result to buf and returns buf on + * success, or NULL on failure. */ +static const char * +route_prefix_to_ip_str(const struct in6_addr *prefix, char *buf, size_t buflen) +{ + if (IN6_IS_ADDR_V4MAPPED(prefix)) { + if (inet_ntop(AF_INET, &prefix->s6_addr[12], buf, buflen)) { + return buf; + } + } else { + if (inet_ntop(AF_INET6, &prefix->s6_addr, buf, buflen)) { + return buf; + } + } + return NULL; +} + +/* First pass: scan Advertised_Route rows and build a gate entry for + * each LB-derived route whose tracked_port is local. The returned + * hmap is keyed by vip_ip so the LB table scan can update matching + * candidates without parsing backends for irrelevant VIPs. + * + * If no gate candidates are found the hmap is empty and the caller + * can skip the SM index build and LB table scan entirely. */ +static void +build_lb_route_gates(struct hmap *gates, + const struct sbrec_advertised_route_table *ar_table, + struct ovsdb_idl_index *sbrec_port_binding_by_name, + const struct sbrec_chassis *chassis, + struct hmap *announce_routes) +{ + const struct sbrec_advertised_route *route; + SBREC_ADVERTISED_ROUTE_TABLE_FOR_EACH (route, ar_table) { + const char *source = smap_get(&route->external_ids, "source"); + if (!source || strcmp(source, "lb")) { + continue; + } + if (!route->tracked_port) { + continue; + } + if (!lport_is_local(sbrec_port_binding_by_name, chassis, + route->tracked_port->logical_port)) { + continue; + } + struct advertise_datapath_entry *ad = + advertise_datapath_find(announce_routes, route->datapath); + if (!ad) { + continue; + } + + struct in6_addr prefix; + unsigned int plen; + if (!ip46_parse_cidr(route->ip_prefix, &prefix, &plen)) { + continue; + } + char vip_ip_buf[INET6_ADDRSTRLEN]; + const char *vip_ip = route_prefix_to_ip_str(&prefix, vip_ip_buf, + sizeof vip_ip_buf); + if (!vip_ip) { + continue; + } + + struct lb_route_gate *g = xmalloc(sizeof *g); + *g = (struct lb_route_gate) { + .route = route, + .vip_ip = xstrdup(vip_ip), + .tracked_lp = route->tracked_port->logical_port, + .seen_monitor = false, + .any_online = false, + }; + hmap_insert(gates, &g->node, hash_string(g->vip_ip, 0)); + } +} + +static void +destroy_lb_route_gates(struct hmap *gates) +{ + struct lb_route_gate *g; + HMAP_FOR_EACH_POP (g, node, gates) { + free(g->vip_ip); + free(g); + } + hmap_destroy(gates); +} + +/* Scan LB VIPs but parse backend strings only for VIP IPs that + * appear in the gate set. For each matching backend, look up the + * SM index and update the gate's health state. */ +static void +evaluate_lb_route_gates(struct hmap *gates, + const struct hmap *sm_lb_index, + const struct sbrec_load_balancer_table *lb_table, + const char *chassis_name) +{ + const struct sbrec_load_balancer *lb; + SBREC_LOAD_BALANCER_TABLE_FOR_EACH (lb, lb_table) { + const char *protocol = lb->protocol ? lb->protocol : "tcp"; + struct smap_node *node; + SMAP_FOR_EACH (node, &lb->vips) { + char *vip_ip = NULL; + struct in6_addr vip_addr; + uint16_t vip_port; + int vip_af; + if (!ip_address_and_port_from_lb_key(node->key, &vip_ip, + &vip_addr, &vip_port, + &vip_af)) { + continue; + } + + uint32_t hash = hash_string(vip_ip, 0); + bool needed = false; + struct lb_route_gate *g; + HMAP_FOR_EACH_WITH_HASH (g, node, hash, gates) { + if (!strcmp(g->vip_ip, vip_ip)) { + needed = true; + break; + } + } + if (!needed) { + free(vip_ip); + continue; + } + + char *vips_copy = xstrdup(node->value); + char *saveptr = NULL; + for (char *tok = strtok_r(vips_copy, ",", &saveptr); + tok; tok = strtok_r(NULL, ",", &saveptr)) { + char *backend_ip = NULL; + struct in6_addr backend_addr; + uint16_t backend_port = 0; + int backend_af; + if (!ip_address_and_port_from_lb_key( + tok, &backend_ip, &backend_addr, + &backend_port, &backend_af)) { + free(backend_ip); + continue; + } + + HMAP_FOR_EACH_WITH_HASH (g, node, hash, gates) { + if (strcmp(g->vip_ip, vip_ip)) { + continue; + } + + uint32_t sm_hash = hash_string(g->tracked_lp, 0); + sm_hash = hash_string(chassis_name, sm_hash); + sm_hash = hash_string(backend_ip, sm_hash); + sm_hash = hash_int((uint32_t) backend_port, sm_hash); + sm_hash = hash_string(protocol, sm_hash); + + struct sm_lb_key *k; + HMAP_FOR_EACH_WITH_HASH (k, node, sm_hash, + sm_lb_index) { + if (k->port != backend_port || + strcmp(k->logical_port, g->tracked_lp) || + strcmp(k->chassis_name, chassis_name) || + strcmp(k->ip, backend_ip) || + strcmp(k->protocol, protocol)) { + continue; + } + g->seen_monitor = true; + g->any_online |= k->online; + } + } + free(backend_ip); + } + free(vips_copy); + free(vip_ip); + } + } +} + +/* Look up the gate decision for a specific route. Returns: + * -1 if no gate exists (route is not LB-derived or not local-bound) + * 0 if gate says withdraw (seen_monitor && !any_online) + * 1 if gate says install */ +static int +lb_route_gate_decision(const struct hmap *gates, + const struct sbrec_advertised_route *route) +{ + const struct lb_route_gate *g; + HMAP_FOR_EACH (g, node, gates) { + if (g->route == route) { + if (g->seen_monitor && !g->any_online) { + return 0; + } + return 1; + } + } + return -1; +} + static bool route_exchange_relevant_port(const struct sbrec_port_binding *pb) { @@ -216,6 +433,29 @@ advertised_datapath_alloc(const struct sbrec_datapath_binding *datapath) return ad; } +static void +route_record_status(struct route_ctx_out *r_ctx_out, + const struct sbrec_advertised_route *route, + const char *status) +{ + struct advertised_route_status *s = xmalloc(sizeof *s); + *s = (struct advertised_route_status) { + .route = route, + .status = status, + }; + hmap_insert(r_ctx_out->advertised_route_status, &s->node, + hash_pointer(route, 0)); +} + +void +advertised_route_status_clear(struct hmap *statuses) +{ + struct advertised_route_status *s; + HMAP_FOR_EACH_POP (s, node, statuses) { + free(s); + } +} + void route_run(struct route_ctx_in *r_ctx_in, struct route_ctx_out *r_ctx_out) @@ -315,6 +555,50 @@ route_run(struct route_ctx_in *r_ctx_in, } } + struct hmap lb_route_gates = HMAP_INITIALIZER(&lb_route_gates); + build_lb_route_gates(&lb_route_gates, + r_ctx_in->advertised_route_table, + r_ctx_in->sbrec_port_binding_by_name, + r_ctx_in->chassis, + r_ctx_out->announce_routes); + + struct hmap sm_lb_index = HMAP_INITIALIZER(&sm_lb_index); + if (!hmap_is_empty(&lb_route_gates) && r_ctx_in->service_monitor_table) { + const struct sbrec_service_monitor *sm; + SBREC_SERVICE_MONITOR_TABLE_FOR_EACH ( + sm, r_ctx_in->service_monitor_table) { + if (!sm->type || strcmp(sm->type, "load-balancer")) { + continue; + } + if (!sm->logical_port || !sm->chassis_name || + !sm->ip || !sm->protocol) { + continue; + } + struct sm_lb_key *k = xmalloc(sizeof *k); + uint32_t hash = hash_string(sm->logical_port, 0); + hash = hash_string(sm->chassis_name, hash); + hash = hash_string(sm->ip, hash); + hash = hash_int((uint32_t) sm->port, hash); + hash = hash_string(sm->protocol, hash); + *k = (struct sm_lb_key) { + .logical_port = sm->logical_port, + .chassis_name = sm->chassis_name, + .ip = sm->ip, + .port = sm->port, + .protocol = sm->protocol, + .online = sm->status && !strcmp(sm->status, "online"), + }; + hmap_insert(&sm_lb_index, &k->node, hash); + } + + if (!hmap_is_empty(&sm_lb_index) && + r_ctx_in->load_balancer_table) { + evaluate_lb_route_gates(&lb_route_gates, &sm_lb_index, + r_ctx_in->load_balancer_table, + r_ctx_in->chassis->name); + } + } + const struct sbrec_advertised_route *route; SBREC_ADVERTISED_ROUTE_TABLE_FOR_EACH (route, r_ctx_in->advertised_route_table) { @@ -345,6 +629,18 @@ route_run(struct route_ctx_in *r_ctx_in, sset_add(r_ctx_out->tracked_ports_local, route->logical_port->logical_port); + if (!smap_get_bool(&route->external_ids, "enabled", true)) { + if (route->tracked_port) { + if (lport_is_local(r_ctx_in->sbrec_port_binding_by_name, + r_ctx_in->chassis, + route->tracked_port->logical_port)) { + route_record_status(r_ctx_out, route, + "withdrawn-admin"); + } + } + continue; + } + unsigned int priority = PRIORITY_DEFAULT; if (route->tracked_port) { bool redistribute_local_bound_only = @@ -357,12 +653,18 @@ route_run(struct route_ctx_in *r_ctx_in, priority = PRIORITY_LOCAL_BOUND; sset_add(r_ctx_out->tracked_ports_local, route->tracked_port->logical_port); + + int gate = lb_route_gate_decision(&lb_route_gates, route); + if (gate == 0) { + route_record_status(r_ctx_out, route, + "withdrawn-monitor"); + continue; + } + route_record_status(r_ctx_out, route, "advertised"); } else { sset_add(r_ctx_out->tracked_ports_remote, route->tracked_port->logical_port); if (redistribute_local_bound_only) { - /* We're not advertising routes whose 'tracked_port' is - * not local, skip this route. */ continue; } } @@ -386,6 +688,13 @@ route_run(struct route_ctx_in *r_ctx_in, advertise_route_hash(&ar->addr, &ar->nexthop, plen)); } + struct sm_lb_key *k; + HMAP_FOR_EACH_POP (k, node, &sm_lb_index) { + free(k); + } + hmap_destroy(&sm_lb_index); + destroy_lb_route_gates(&lb_route_gates); + smap_destroy(&port_mapping); } diff --git a/controller/route.h b/controller/route.h index f1d03a9e5..75262065f 100644 --- a/controller/route.h +++ b/controller/route.h @@ -31,9 +31,12 @@ struct route_data; struct sbrec_chassis; struct sbrec_port_binding; struct sbrec_datapath_binding; +struct sbrec_advertised_route; struct route_ctx_in { const struct sbrec_advertised_route_table *advertised_route_table; + const struct sbrec_service_monitor_table *service_monitor_table; + const struct sbrec_load_balancer_table *load_balancer_table; struct ovsdb_idl_index *sbrec_port_binding_by_name; const struct sbrec_chassis *chassis; const char *dynamic_routing_port_mapping; @@ -58,6 +61,22 @@ struct route_ctx_out { /* Contains struct advertise_datapath_entry */ struct hmap *announce_routes; + + /* Contains struct advertised_route_status entries recorded by + * route_run() for publication to SB external_ids:status by + * route_exchange_run(). */ + struct hmap *advertised_route_status; +}; + +/* Per-route advertisement decision recorded by route_run() so that + * route_exchange_run() (which holds a writable SB txn) can publish it + * to Advertised_Route.external_ids:status. Only the chassis that + * owns the tracked_port records an entry. Status may be stale until + * a new owner updates it after an ownership transition. */ +struct advertised_route_status { + struct hmap_node node; + const struct sbrec_advertised_route *route; + const char *status; }; struct advertise_datapath_entry { @@ -100,6 +119,7 @@ struct advertise_route_entry advertise_route_from_route_data(const struct route_data *); void route_run(struct route_ctx_in *, struct route_ctx_out *); void route_cleanup(struct hmap *announce_routes); +void advertised_route_status_clear(struct hmap *statuses); uint32_t route_get_table_id(const struct sbrec_datapath_binding *); struct advertise_route_entry * advertise_route_find(unsigned int priority, const struct in6_addr *prefix, diff --git a/ovn-sb.xml b/ovn-sb.xml index e70b83a6e..75464f999 100644 --- a/ovn-sb.xml +++ b/ovn-sb.xml @@ -5465,8 +5465,9 @@ tcp.flags = RST; See <em>External IDs</em> at the beginning of this document. <p> - <code>ovn-northd</code> sets the following key on routes - derived from Load_Balancer VIPs: + The following keys are used by <code>ovn-northd</code> and + <code>ovn-controller</code> for Load_Balancer route gating + and observability: </p> <dl> @@ -5477,6 +5478,26 @@ tcp.flags = RST; <code>ovn-controller</code> uses this key to identify routes eligible for Service_Monitor-based gating. </dd> + + <dt><code>enabled</code></dt> + <dd> + Administrative enable flag. When explicitly set to + <code>false</code>, <code>ovn-controller</code> skips + installing the route regardless of Service_Monitor health. + The default (key absent) is enabled. Out-of-band writers + (e.g. an L7 health agent) may set this key to disable a route + ahead of the normal monitor-based withdrawal. + </dd> + + <dt><code>status</code></dt> + <dd> + Advertisement decision recorded by the chassis that owns the + route's <code>tracked_port</code>. Possible values: + <code>advertised</code>, <code>withdrawn-admin</code>, + <code>withdrawn-monitor</code>. The key is written only by + the owner chassis. It may be absent or stale when no chassis + currently owns the <code>tracked_port</code>. + </dd> </dl> </column> </table> diff --git a/tests/ovn-inc-proc-graph-dump.at b/tests/ovn-inc-proc-graph-dump.at index 264146301..0a1e0d57b 100644 --- a/tests/ovn-inc-proc-graph-dump.at +++ b/tests/ovn-inc-proc-graph-dump.at @@ -449,6 +449,7 @@ digraph "Incremental-Processing-Engine" { SB_chassis -> bfd_chassis [[label=""]]; SB_ha_chassis_group -> bfd_chassis [[label=""]]; SB_advertised_route [[style=filled, shape=box, fillcolor=white, label="SB_advertised_route"]]; + SB_service_monitor [[style=filled, shape=box, fillcolor=white, label="SB_service_monitor"]]; route [[style=filled, shape=box, fillcolor=white, label="route"]]; OVS_open_vswitch -> route [[label=""]]; SB_chassis -> route [[label=""]]; @@ -456,6 +457,8 @@ digraph "Incremental-Processing-Engine" { runtime_data -> route [[label="route_runtime_data_handler"]]; SB_advertised_route -> route [[label="route_sb_advertised_route_data_handler"]]; SB_datapath_binding -> route [[label="route_sb_datapath_binding_handler"]]; + SB_service_monitor -> route [[label="route_sb_service_monitor_handler"]]; + SB_load_balancer -> route [[label=""]]; SB_learned_route [[style=filled, shape=box, fillcolor=white, label="SB_learned_route"]]; route_table_notify [[style=filled, shape=box, fillcolor=white, label="route_table_notify"]]; route_exchange_status [[style=filled, shape=box, fillcolor=white, label="route_exchange_status"]]; diff --git a/tests/system-ovn.at b/tests/system-ovn.at index ed5d63fd3..876bbb8a5 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -21992,3 +21992,694 @@ OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d AT_CLEANUP ]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - LB redistribute gated by Service_Monitor.status]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1339]) + +# Verifies the controller-side gate that skips installing the +# advertise-only kernel route for an LB-derived Advertised_Route +# whose matching local Service_Monitor row, joined on +# (logical_port, chassis_name, type='load-balancer', ip, port, +# protocol), is not reporting online. The backend service selector +# (IP, port, protocol) is derived at runtime from the +# Load_Balancer.vips smap rather than stored on the Advertised_Route +# row. When no Service_Monitor row exists the route is installed +# unconditionally. +# +# Topology: two chassis-bound LRs (lr-origin, lr-target) share a +# common LS (ls-share). lr-origin has redistribute=lb on its LRP into +# ls-share, so northd emits one Advertised_Route row per backend LSP +# on lr-origin's datapath with tracked_port = the backend LSP. +# lr-target owns the LB. The backend LSP (be0) is on ls-share, bound +# to hv1. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +# Shared LS holding both LR LRPs and the backend LSP. +check ovn-nbctl ls-add ls-share + +# lr-origin: GW, vrf 1339, has the redistribute LRP into ls-share. +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1339 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +# lr-target: GW, owns the LB. Also attached to ls-share so lr-origin +# can walk LR-LS-LR to discover the LB. +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +# Backend LSP on the same LS. Veth-backed so it's claimed locally. +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") + +# LB with ip_port_mappings -> backend LSP "be0". options:distributed=true +# is required for northd to populate ovn_northd_lb_backend.logical_port +# (see ovn_lb_vip_backends_ip_port_mappings_init). The +# Load_Balancer_Health_Check is what makes ovn-northd populate the +# Service_Monitor SB table for the backend: a manually-created row +# would be garbage-collected as orphaned. ovn-controller then probes +# the backend and updates Service_Monitor.status, which is exactly +# what the controller-side gate observes. +check ovn-nbctl \ + -- lb-add lb0 172.16.1.10:80 192.168.0.10:80 \ + -- set Load_Balancer lb0 options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb0 +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.10\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb0 health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# Baseline: no listener on be0:80 -> ovn-controller's probe fails -> +# Service_Monitor.status=offline -> the gate withdraws the route from +# the VRF on this chassis. The blackhole route for 172.16.1.10 must +# NOT appear in ovnvrf1339. +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1339 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 +wait_row_count Service_Monitor 1 logical_port=be0 status=offline + +# Confirm the gate is firing: no route for the LB VIP. +AT_CHECK([ + ip route list vrf ovnvrf1339 | grep -c "blackhole 172.16.1.10" || true +], [0], [0 +]) + +# Start a TCP listener on be0:80. ovn-controller's probe now succeeds +# and Service_Monitor.status flips to online, so the gate lets the route +# through and the route shows up in the VRF. OVS_START_L7 uses netstat +# for readiness which may not be available, so start test-l7.py directly +# and use Service_Monitor.status as the readiness signal instead. +be0_pid_file=$(mktemp be0_http.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be0_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +# Kill the listener. The probe fails again, status=offline, and the +# route is withdrawn from the VRF so that any routing speaker reading +# it retracts the advertisement. +kill `cat $be0_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +# Remove the health_check from the LB -> northd deletes the SM row -> +# gate is inactive again -> route reinstalled (the route is unconditional +# for unmonitored backends). +check ovn-nbctl clear Load_Balancer lb0 health_check +wait_row_count Service_Monitor 0 logical_port=be0 +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1339 | grep -q "blackhole 172.16.1.10"]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) + +AT_CLEANUP +]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - shared backend LSP gates per-VIP]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1340]) + +# Two distinct LBs (lb-a:80 and lb-b:443) share one backend LSP (be0). +# Each LB has its own Load_Balancer_Health_Check probing the backend +# port specific to that LB. This is the K8s-style multi-Service-per-pod +# topology: one backend LSP serves several VIPs whose health states are +# independent. +# +# The controller derives each route's backend selector from the +# Load_Balancer.vips smap at runtime, matching the route's ip_prefix +# against VIP keys. Each VIP's kernel-route presence depends only on +# its own backend port's SM row. Bringing one listener up and leaving +# the other down must install exactly the corresponding VIP's route, +# never both, never neither. This is a regression test for the +# previous coarser (tracked_port, chassis) matching that would +# advertise both VIPs whenever any backend port was healthy. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +check ovn-nbctl ls-add ls-share + +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1340 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +# One backend LSP shared by both LBs, with two listeners on different +# ports (:80 and :443) inside the same netns. +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") + +# lb-a: VIP 172.16.1.10:80 -> 192.168.0.10:80 +check ovn-nbctl \ + -- lb-add lb-a 172.16.1.10:80 192.168.0.10:80 \ + -- set Load_Balancer lb-a options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-a +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.10\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-a health_check @hc + +# lb-b: VIP 172.16.1.11:443 -> 192.168.0.10:443 (same backend LSP). +check ovn-nbctl \ + -- lb-add lb-b 172.16.1.11:443 192.168.0.10:443 \ + -- set Load_Balancer lb-b options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-b +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.11\:443" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-b health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# Baseline: nothing listening -> both SM rows offline -> neither VIP +# advertised. VRF exists (maintain-vrf=true) but carries no LB route. +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1340 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +wait_row_count Service_Monitor 1 logical_port=be0 port=443 status=offline +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1." || true +], [0], [0 +]) + +# Bring up the :80 listener only. SM for :80 flips to online, SM for +# :443 stays offline. Tuple match must install ONLY the :80 route. +be80_pid_file=$(mktemp be0_http80.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be80_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1340 | grep -q "blackhole 172.16.1.10"]) +# The :443 VIP must NOT be advertised: this is the regression the +# per-VIP-IP gate ensures. Under the coarser (tracked_port, chassis) match +# the :80 SM going online would suffice to advertise 172.16.1.11 too. +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1.11" || true +], [0], [0 +]) + +# Bring up the :443 listener too. test-l7.py only binds :80, so spin +# up a separate Python TCP listener for :443. +be443_pid_file=$(mktemp be0_tcp443.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [python3 -c "import socket; s=socket.socket(); s.bind(('0.0.0.0',443)); s.listen(1) +while True: + c,_=s.accept(); c.close()"], + [$be443_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=443 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1340 | grep -q "blackhole 172.16.1.11"]) +# :80 route should still be there. +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1.10" +], [0], [1 +]) + +# Kill ONLY the :80 listener. :443 still healthy. Only the 172.16.1.10 +# VIP must withdraw. 172.16.1.11 stays. +kill `cat $be80_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1340 | grep -q "blackhole 172.16.1.10"]) +AT_CHECK([ + ip route list vrf ovnvrf1340 | grep -c "blackhole 172.16.1.11" +], [0], [1 +]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) + +AT_CLEANUP +]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - shared VIP IP and cross-VIP isolation]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1341]) + +# This test exercises the per-VIP-IP route emission and the +# per-VIP-IP controller gate over a single backend LSP that hosts +# services for two distinct VIP IPs: +# +# - VIP A 172.16.1.20: two LB listeners (lb-c:80, lb-d:443) share +# the same VIP IP and the same backend LSP (be0). Northd emits +# one Advertised_Route for 172.16.1.20 (one per VIP IP, backend +# LSP pair). The controller derives all backends for this VIP IP +# from the Load_Balancer table and gates the route as a unit: +# installed while ANY listener for the VIP IP is healthy locally. +# +# - VIP B 172.16.1.30: a third LB (lb-e:8080) advertises a +# different VIP IP off the same backend LSP. Its health must +# not influence VIP A and vice versa. +# +# The cross-VIP isolation case (VIP A all offline, VIP B online) is +# the regression catcher: the controller looks up backends by VIP IP +# in the Load_Balancer table, so VIP B's healthy backend on a +# different VIP IP cannot affect VIP A's gate. An earlier coarser +# match keyed on (logical_port, chassis) alone would let any healthy +# backend keep all VIPs advertised. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +check ovn-nbctl ls-add ls-share + +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1341 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") + +# VIP A, listener 1: lb-c carries 172.16.1.20:80 -> 192.168.0.10:80. +check ovn-nbctl \ + -- lb-add lb-c 172.16.1.20:80 192.168.0.10:80 \ + -- set Load_Balancer lb-c options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-c +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.20\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-c health_check @hc + +# VIP A, listener 2: lb-d carries 172.16.1.20:443 -> 192.168.0.10:443. +check ovn-nbctl \ + -- lb-add lb-d 172.16.1.20:443 192.168.0.10:443 \ + -- set Load_Balancer lb-d options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-d +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.20\:443" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-d health_check @hc + +# VIP B: lb-e carries 172.16.1.30:8080 -> 192.168.0.10:8080. +check ovn-nbctl \ + -- lb-add lb-e 172.16.1.30:8080 192.168.0.10:8080 \ + -- set Load_Balancer lb-e options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb-e +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.30\:8080" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb-e health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# Northd emits one Advertised_Route per (VIP IP, backend LSP): one +# for VIP A (172.16.1.20) and one for VIP B (172.16.1.30), two total +# on lr-origin's datapath. +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1341 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +wait_row_count Service_Monitor 1 logical_port=be0 port=443 status=offline +wait_row_count Service_Monitor 1 logical_port=be0 port=8080 status=offline +wait_row_count sb:Advertised_Route 1 ip_prefix='"172.16.1.20"' +wait_row_count sb:Advertised_Route 1 ip_prefix='"172.16.1.30"' + +# Baseline: nothing listening -> neither VIP A nor VIP B advertised. +AT_CHECK([ + ip route list vrf ovnvrf1341 | grep -c "blackhole 172.16.1." || true +], [0], [0 +]) + +# Bring up VIP B's :8080 listener only. VIP B's route derives its +# backend from the LB table (172.16.1.30:8080 -> 192.168.0.10:8080) +# and the matching SM (be0, 192.168.0.10, 8080, tcp) goes online, so +# the route for 172.16.1.30 appears. Critically, VIP A's route +# derives its backends from LB VIPs 172.16.1.20:80 and 172.16.1.20:443, +# whose SMs (be0, 192.168.0.10, 80/443, tcp) both stay offline, so +# 172.16.1.20 must NOT be advertised. +be8080_pid_file=$(mktemp be0_tcp8080.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [python3 -c "import socket; s=socket.socket(); s.bind(('0.0.0.0',8080)); s.listen(1) +while True: + c,_=s.accept(); c.close()"], + [$be8080_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=8080 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1341 | grep -q "blackhole 172.16.1.30"]) +AT_CHECK([ + ip route list vrf ovnvrf1341 | grep -c "blackhole 172.16.1.20" || true +], [0], [0 +]) + +# Bring up VIP A's :80 listener. VIP A's route derives its backends +# from LB VIPs 172.16.1.20:80 and 172.16.1.20:443. The :80 backend's +# SM is now online, so the route passes its gate and 172.16.1.20 +# appears, even though the :443 backend is still offline. This is +# the "advertise while any listener for this VIP IP is healthy" +# semantic. +be80_pid_file=$(mktemp be0_http80.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be80_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=online +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1341 | grep -q "blackhole 172.16.1.20"]) + +# Kill VIP A's :80 listener. VIP A is back to no healthy local +# listener -> route for 172.16.1.20 withdraws. VIP B's listener +# is still up, so 172.16.1.30 stays advertised. This is the +# cross-VIP isolation case: VIP B's backends are derived from a +# different VIP IP (172.16.1.30) in the LB table, so its online +# SM cannot affect VIP A's gate and keep VIP A advertised. +kill `cat $be80_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 port=80 status=offline +OVS_WAIT_UNTIL([ + ! ip route list vrf ovnvrf1341 | grep -q "blackhole 172.16.1.20"]) +AT_CHECK([ + ip route list vrf ovnvrf1341 | grep -c "blackhole 172.16.1.30" +], [0], [1 +]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) + +AT_CLEANUP +]) + +OVN_FOR_EACH_NORTHD([ +AT_SETUP([dynamic-routing - non-LB route overlap and LB route status]) +AT_KEYWORDS([dynamic-routing]) + +VRF_RESERVE([1342]) + +# Regression test for false gating of non-LB routes: when a NAT +# external IP overlaps with an LB VIP IP, northd emits two +# Advertised_Route rows with the same ip_prefix. Only the LB-derived +# row carries external_ids:source=lb. The controller gate must skip +# non-LB rows: the NAT route stays installed regardless of +# Service_Monitor status, while the LB route is gated normally. +# +# The test also verifies that the controller publishes +# external_ids:status on the LB-derived route through each transition: +# withdrawn-monitor, advertised, and withdrawn-admin. + +ovn_start +OVS_TRAFFIC_VSWITCHD_START() + +ADD_BR([br-int]) +check ovs-vsctl \ + -- set Open_vSwitch . external-ids:system-id=hv1 \ + -- set Open_vSwitch . external-ids:ovn-remote=unix:$ovs_base/ovn-sb/ovn-sb.sock \ + -- set Open_vSwitch . external-ids:ovn-encap-type=geneve \ + -- set Open_vSwitch . external-ids:ovn-encap-ip=169.0.0.1 \ + -- set bridge br-int fail-mode=secure other-config:disable-in-band=true + +start_daemon ovn-controller + +check ovn-nbctl ls-add ls-share + +# lr-origin: GW with redistribute="lb,nat". The NAT rule creates a +# non-LB Advertised_Route for 172.16.1.10 (no source=lb marker). +check ovn-nbctl lr-add lr-origin \ + -- set Logical_Router lr-origin options:chassis=hv1 \ + options:dynamic-routing=true \ + options:dynamic-routing-vrf-id=1342 +check ovn-nbctl lrp-add lr-origin lr-origin-share 00:de:ad:00:00:01 \ + 192.168.0.1/24 \ + -- set Logical_Router_Port lr-origin-share \ + options:dynamic-routing-redistribute="lb,nat" \ + options:dynamic-routing-maintain-vrf=true +check ovn-nbctl lsp-add ls-share share-lr-origin \ + -- set Logical_Switch_Port share-lr-origin type=router \ + options:router-port=lr-origin-share \ + -- lsp-set-addresses share-lr-origin router + +# NAT rule whose external IP matches the LB VIP. Northd emits a NAT +# Advertised_Route for 172.16.1.10 with no source=lb marker. +check ovn-nbctl lr-nat-add lr-origin dnat_and_snat 172.16.1.10 192.168.0.10 + +# lr-target: owns the LB. +check ovn-nbctl lr-add lr-target \ + -- set Logical_Router lr-target options:chassis=hv1 +check ovn-nbctl lrp-add lr-target lr-target-share 00:de:ad:00:00:02 \ + 192.168.0.2/24 +check ovn-nbctl lsp-add ls-share share-lr-target \ + -- set Logical_Switch_Port share-lr-target type=router \ + options:router-port=lr-target-share \ + -- lsp-set-addresses share-lr-target router + +# Backend LSP. +check ovn-nbctl lsp-add ls-share be0 +check ovn-nbctl lsp-set-addresses be0 "00:de:ad:00:00:10 192.168.0.10" +ADD_NAMESPACES(be0_ns) +ADD_VETH(be0, be0_ns, br-int, "192.168.0.10/24", "00:de:ad:00:00:10") + +# LB with VIP 172.16.1.10:80 (same IP as NAT external IP). +check ovn-nbctl \ + -- lb-add lb0 172.16.1.10:80 192.168.0.10:80 \ + -- set Load_Balancer lb0 options:distributed=true \ + ip_port_mappings:192.168.0.10="be0:192.168.0.2" \ + -- lr-lb-add lr-target lb0 +check_uuid ovn-nbctl --id=@hc create Load_Balancer_Health_Check \ + vip="172.16.1.10\:80" \ + options:interval=2 options:timeout=1 \ + options:success_count=1 options:failure_count=1 \ + -- add Load_Balancer lb0 health_check @hc + +check ovn-nbctl --wait=hv sync +wait_for_ports_up +OVS_CTL_TIMEOUT=30 + +# Two AR rows for 172.16.1.10: NAT-derived (no source) and LB-derived +# (source=lb). +OVS_WAIT_UNTIL([test -n "`ip vrf show ovnvrf1342 2>/dev/null`"]) +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +wait_row_count sb:Advertised_Route 2 ip_prefix='"172.16.1.10"' + +# SM offline: LB route is gated (withdrawn-monitor), but the NAT route +# is NOT gated and keeps the blackhole route in the VRF. This is the +# regression assertion: without the source=lb marker, both routes +# would be gated and the route would disappear. +OVS_WAIT_UNTIL([ + ip route list vrf ovnvrf1342 | grep -q "blackhole 172.16.1.10"]) + +# Verify status=withdrawn-monitor on the LB-derived route (has +# external_ids:source=lb). +lb_ar=$(ovn-sbctl --columns=_uuid,external_ids find Advertised_Route \ + ip_prefix=172.16.1.10 | awk ' + /_uuid/ { uuid = $3 } + /source.*lb/ { print uuid; exit } + ') +test -n "$lb_ar" +OVS_WAIT_UNTIL([ + ovn-sbctl --columns=external_ids find Advertised_Route \ + ip_prefix=172.16.1.10 | + grep 'source=lb' | grep -q 'withdrawn-monitor']) + +# Bring up the listener. SM goes online, LB route passes the gate. +be_pid_file=$(mktemp be0_http.XXX.pid) +NETNS_DAEMONIZE([be0_ns], + [[$PYTHON $srcdir/test-l7.py http]], [$be_pid_file]) +wait_row_count Service_Monitor 1 logical_port=be0 status=online +OVS_WAIT_UNTIL([ + ovn-sbctl --columns=external_ids find Advertised_Route \ + ip_prefix=172.16.1.10 | + grep 'source=lb' | grep -q 'status.*advertised']) +# Both NAT and LB routes installed (different priorities). +AT_CHECK([ + ip route list vrf ovnvrf1342 | grep -c "blackhole 172.16.1.10" +], [0], [2 +]) + +# Admin-disable the LB route. Controller records withdrawn-admin. +# The NAT route is unaffected and keeps the VRF route. +check ovn-sbctl set Advertised_Route $lb_ar external_ids:enabled=false +OVS_WAIT_UNTIL([ + ovn-sbctl --columns=external_ids find Advertised_Route \ + ip_prefix=172.16.1.10 | + grep 'source=lb' | grep -q 'withdrawn-admin']) +AT_CHECK([ + ip route list vrf ovnvrf1342 | grep -c "blackhole 172.16.1.10" +], [0], [1 +]) + +# Re-enable the LB route. Status flips back to advertised. +check ovn-sbctl remove Advertised_Route $lb_ar external_ids enabled +OVS_WAIT_UNTIL([ + ovn-sbctl --columns=external_ids find Advertised_Route \ + ip_prefix=172.16.1.10 | + grep 'source=lb' | grep -q 'status.*advertised']) + +# Kill the listener. SM goes offline, LB route back to withdrawn-monitor. +kill `cat $be_pid_file` +wait_row_count Service_Monitor 1 logical_port=be0 status=offline +OVS_WAIT_UNTIL([ + ovn-sbctl --columns=external_ids find Advertised_Route \ + ip_prefix=172.16.1.10 | + grep 'source=lb' | grep -q 'withdrawn-monitor']) +# NAT route still keeps the VRF route. +AT_CHECK([ + ip route list vrf ovnvrf1342 | grep -c "blackhole 172.16.1.10" +], [0], [1 +]) + +OVS_APP_EXIT_AND_WAIT([ovn-controller]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +as +OVS_TRAFFIC_VSWITCHD_STOP(["/failed to query port patch-.*/d +/connection dropped.*/d"]) + +AT_CLEANUP +]) -- 2.53.0 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
