move the pf_state_tree_id type from pfvar.h to pfvar_priv.h. this is like the pf_state_tree change from yesterday. the pf_state_tree_id type is private to the kernel, just like the pf_state_tree type.
moving it from RB to RBT saves another 12k in pf.o on amd64. i also changed some hand rolled for loops over to RBT_FOREACH_SAFE. they look right? ok? Index: pf.c =================================================================== RCS file: /cvs/src/sys/net/pf.c,v retrieving revision 1.1166 diff -u -p -r1.1166 pf.c --- pf.c 4 Jan 2023 02:00:49 -0000 1.1166 +++ pf.c 4 Jan 2023 03:08:04 -0000 @@ -305,8 +305,8 @@ struct pf_pool_limit pf_pool_limits[PF_L static __inline int pf_src_compare(struct pf_src_node *, struct pf_src_node *); static inline int pf_state_compare_key(const struct pf_state_key *, const struct pf_state_key *); -static __inline int pf_state_compare_id(struct pf_state *, - struct pf_state *); +static inline int pf_state_compare_id(const struct pf_state *, + const struct pf_state *); #ifdef INET6 static __inline void pf_cksum_uncover(u_int16_t *, u_int16_t, u_int8_t); static __inline void pf_cksum_cover(u_int16_t *, u_int16_t, u_int8_t); @@ -320,8 +320,7 @@ struct pf_state_list pf_state_list = PF_ RB_GENERATE(pf_src_tree, pf_src_node, entry, pf_src_compare); RBT_GENERATE(pf_state_tree, pf_state_key, sk_entry, pf_state_compare_key); -RB_GENERATE(pf_state_tree_id, pf_state, - entry_id, pf_state_compare_id); +RBT_GENERATE(pf_state_tree_id, pf_state, entry_id, pf_state_compare_id); int pf_addr_compare(const struct pf_addr *a, const struct pf_addr *b, @@ -541,7 +540,7 @@ pf_src_connlimit(struct pf_state **state struct pf_state *st; pf_status.lcounters[LCNT_OVERLOAD_FLUSH]++; - RB_FOREACH(st, pf_state_tree_id, &tree_id) { + RBT_FOREACH(st, pf_state_tree_id, &tree_id) { sk = st->key[PF_SK_WIRE]; /* * Kill states from this source. (Only those @@ -715,8 +714,8 @@ pf_state_compare_key(const struct pf_sta return (0); } -static __inline int -pf_state_compare_id(struct pf_state *a, struct pf_state *b) +static inline int +pf_state_compare_id(const struct pf_state *a, const struct pf_state *b) { if (a->id > b->id) return (1); @@ -1054,7 +1053,7 @@ pf_state_insert(struct pfi_kif *kif, str s->id = htobe64(pf_status.stateid++); s->creatorid = pf_status.hostid; } - if (RB_INSERT(pf_state_tree_id, &tree_id, s) != NULL) { + if (RBT_INSERT(pf_state_tree_id, &tree_id, s) != NULL) { if (pf_status.debug >= LOG_NOTICE) { log(LOG_NOTICE, "pf: state insert failed: " "id: %016llx creatorid: %08x", @@ -1085,7 +1084,7 @@ pf_find_state_byid(struct pf_state_cmp * { pf_status.fcounters[FCNT_STATE_SEARCH]++; - return (RB_FIND(pf_state_tree_id, &tree_id, (struct pf_state *)key)); + return (RBT_FIND(pf_state_tree_id, &tree_id, (struct pf_state *)key)); } int @@ -1733,7 +1732,7 @@ pf_remove_state(struct pf_state *cur) if (cur->key[PF_SK_STACK]->proto == IPPROTO_TCP) pf_set_protostate(cur, PF_PEER_BOTH, TCPS_CLOSED); - RB_REMOVE(pf_state_tree_id, &tree_id, cur); + RBT_REMOVE(pf_state_tree_id, &tree_id, cur); #if NPFLOW > 0 if (cur->state_flags & PFSTATE_PFLOW) export_pflow(cur); Index: pf_ioctl.c =================================================================== RCS file: /cvs/src/sys/net/pf_ioctl.c,v retrieving revision 1.394 diff -u -p -r1.394 pf_ioctl.c --- pf_ioctl.c 4 Jan 2023 02:00:49 -0000 1.394 +++ pf_ioctl.c 4 Jan 2023 03:08:05 -0000 @@ -1796,10 +1796,7 @@ pfioctl(dev_t dev, u_long cmd, caddr_t a NET_LOCK(); PF_LOCK(); PF_STATE_ENTER_WRITE(); - for (s = RB_MIN(pf_state_tree_id, &tree_id); s; - s = nexts) { - nexts = RB_NEXT(pf_state_tree_id, &tree_id, s); - + RBT_FOREACH_SAFE(s, pf_state_tree_id, &tree_id, nexts) { if (s->direction == PF_OUT) { sk = s->key[PF_SK_STACK]; srcaddr = &sk->addr[1]; @@ -2828,7 +2825,7 @@ pfioctl(dev_t dev, u_long cmd, caddr_t a NET_LOCK(); PF_LOCK(); PF_STATE_ENTER_WRITE(); - RB_FOREACH(state, pf_state_tree_id, &tree_id) + RBT_FOREACH(state, pf_state_tree_id, &tree_id) pf_src_tree_remove_state(state); PF_STATE_EXIT_WRITE(); RB_FOREACH(n, pf_src_tree, &tree_src_tracking) @@ -2861,7 +2858,7 @@ pfioctl(dev_t dev, u_long cmd, caddr_t a if (sn->states != 0) { PF_ASSERT_LOCKED(); PF_STATE_ENTER_WRITE(); - RB_FOREACH(s, pf_state_tree_id, + RBT_FOREACH(s, pf_state_tree_id, &tree_id) pf_state_rm_src_node(s, sn); PF_STATE_EXIT_WRITE(); Index: pf_lb.c =================================================================== RCS file: /cvs/src/sys/net/pf_lb.c,v retrieving revision 1.72 diff -u -p -r1.72 pf_lb.c --- pf_lb.c 31 Aug 2022 11:29:12 -0000 1.72 +++ pf_lb.c 4 Jan 2023 03:08:05 -0000 @@ -311,10 +311,8 @@ pf_map_addr_sticky(sa_family_t af, struc } if (sns[type]->states != 0) { /* XXX expensive */ - RB_FOREACH(s, pf_state_tree_id, - &tree_id) - pf_state_rm_src_node(s, - sns[type]); + RBT_FOREACH(s, pf_state_tree_id, &tree_id) + pf_state_rm_src_node(s, sns[type]); } sns[type]->expire = 1; pf_remove_src_node(sns[type]); Index: pfvar.h =================================================================== RCS file: /cvs/src/sys/net/pfvar.h,v retrieving revision 1.526 diff -u -p -r1.526 pfvar.h --- pfvar.h 4 Jan 2023 02:00:49 -0000 1.526 +++ pfvar.h 4 Jan 2023 03:08:05 -0000 @@ -1581,10 +1581,6 @@ RB_HEAD(pf_src_tree, pf_src_node); RB_PROTOTYPE(pf_src_tree, pf_src_node, entry, pf_src_compare); extern struct pf_src_tree tree_src_tracking; -RB_HEAD(pf_state_tree_id, pf_state); -RB_PROTOTYPE(pf_state_tree_id, pf_state, - entry_id, pf_state_compare_id); -extern struct pf_state_tree_id tree_id; extern struct pf_state_list pf_state_list; TAILQ_HEAD(pf_queuehead, pf_queuespec); Index: pfvar_priv.h =================================================================== RCS file: /cvs/src/sys/net/pfvar_priv.h,v retrieving revision 1.28 diff -u -p -r1.28 pfvar_priv.h --- pfvar_priv.h 4 Jan 2023 02:00:49 -0000 1.28 +++ pfvar_priv.h 4 Jan 2023 03:08:05 -0000 @@ -129,6 +129,10 @@ struct pf_state { u_int8_t snapped; /* [S] */ }; +RBT_HEAD(pf_state_tree_id, pf_state); +RBT_PROTOTYPE(pf_state_tree_id, pf_state, entry_id, pf_state_compare_id); +extern struct pf_state_tree_id tree_id; + /* * * states are linked into a global list to support the following Index: if_pfsync.c =================================================================== RCS file: /cvs/src/sys/net/if_pfsync.c,v retrieving revision 1.311 diff -u -p -r1.311 if_pfsync.c --- if_pfsync.c 11 Nov 2022 11:47:13 -0000 1.311 +++ if_pfsync.c 4 Jan 2023 03:08:06 -0000 @@ -621,8 +621,7 @@ pfsync_in_clr(caddr_t buf, int len, int continue; PF_STATE_ENTER_WRITE(); - for (st = RB_MIN(pf_state_tree_id, &tree_id); st; st = nexts) { - nexts = RB_NEXT(pf_state_tree_id, &tree_id, st); + RBT_FOREACH_SAFE(st, pf_state_tree_id, &tree_id, nexts) { if (st->creatorid == creatorid && ((kif && st->kif == kif) || !kif)) { SET(st->state_flags, PFSTATE_NOSYNC);