please skip over the #if 0,
thats debugging to find the right values.

As i said work in progress, feedback welcome.

Sebastian Benoit([email protected]) on 2018.02.09 04:49:43 +0100:
> Hi,
> 
> when bgpd changes a lot of routes (and runs with fib-updates yes), the
> routing socket of ospfd (and ospf6d, ripd, eigrpd) will get a lot of routing
> messages. When the kernel cannot write all the messages intothe recipients
> route socket, it will set the desync flag, and the daemon has to read the
> full routing table and reevaluate the routes. That comes with a cost.
> 
> claudio and i are investigating varios aproaches to minimize this.
> 
> One way is to increase the buffer size on the route socket. That alone does
> not help, the receiver simply does not get a chance to read the data fast
> enough. So here i tryto give it a chance:
> 
> I add the socket option SO_RDAEMON. A receiver sets it on a route socket.
> When the sockets reaches 1/4 of the current maximum size,the kernel will
> yield() at the end of route_input().
> 
> With this i can shrink the buffer size to 1/2 of the current setting (8
> times what it was until yesterday) and not get a DESYNC on coupling a full
> table from bgpd.
> 
> To get this working i had to disable the sb_mbmax calculation in sbreserve()
> because that triggers a lot earlier. There is another mail from claudio
> asking on advise how to change sbreserve().
> 
> So right now this is a proof of concept.
> Opinions?
> 
> diff --git sys/kern/uipc_socket.c sys/kern/uipc_socket.c
> index 2f7ecfd2371..e3d24843a1c 100644
> --- sys/kern/uipc_socket.c
> +++ sys/kern/uipc_socket.c
> @@ -1600,6 +1600,7 @@ sosetopt(struct socket *so, int level, int optname, 
> struct mbuf *m)
>               case SO_OOBINLINE:
>               case SO_TIMESTAMP:
>               case SO_ZEROIZE:
> +             case SO_RDAEMON:
>                       if (m == NULL || m->m_len < sizeof (int))
>                               return (EINVAL);
>                       if (*mtod(m, int *))
> @@ -1770,6 +1771,7 @@ sogetopt(struct socket *so, int level, int optname, 
> struct mbuf *m)
>               case SO_OOBINLINE:
>               case SO_TIMESTAMP:
>               case SO_ZEROIZE:
> +             case SO_RDAEMON:
>                       *mtod(m, int *) = so->so_options & optname;
>                       break;
>  
> diff --git sys/kern/uipc_socket2.c sys/kern/uipc_socket2.c
> index a7c6ba1f576..e60fc8d8256 100644
> --- sys/kern/uipc_socket2.c
> +++ sys/kern/uipc_socket2.c
> @@ -464,8 +464,11 @@ sbreserve(struct socket *so, struct sockbuf *sb, u_long 
> cc)
>       if (cc == 0 || cc > sb_max)
>               return (1);
>       sb->sb_hiwat = cc;
> +#if 0
>       sb->sb_mbmax = max(3 * MAXMCLBYTES,
>           min(cc * 2, sb_max + (sb_max / MCLBYTES) * MSIZE));
> +#endif
> +     sb->sb_mbmax = ULONG_MAX/4;
>       if (sb->sb_lowat > sb->sb_hiwat)
>               sb->sb_lowat = sb->sb_hiwat;
>       return (0);
> diff --git sys/net/rtsock.c sys/net/rtsock.c
> index 6a516bf00cd..dc918e1aa93 100644
> --- sys/net/rtsock.c
> +++ sys/net/rtsock.c
> @@ -388,6 +388,10 @@ route_input(struct mbuf *m0, struct socket *so, 
> sa_family_t sa_family)
>       int sockets = 0;
>       struct socket *last = NULL;
>       struct sockaddr *sosrc, *sodst;
> +     int yield_cnt = 0;
> +#if 0
> +     static int yield_logged = 0;
> +#endif
>  
>       KERNEL_ASSERT_LOCKED();
>  
> @@ -457,7 +461,21 @@ route_input(struct mbuf *m0, struct socket *so, 
> sa_family_t sa_family)
>  
>               if (last) {
>                       struct mbuf *n;
> +                     if ((last->so_options & SO_RDAEMON) &&
> +                         (last->so_rcv.sb_cc > (sb_max/4)))
> +                             yield_cnt++;
>                       if ((n = m_copym(m, 0, M_COPYALL, M_NOWAIT)) != NULL) {
> +#if 0
> +                             if (sbspace(last, &last->so_rcv) < (2*MSIZE)) {
> +                                     printf("sbspace 1 hiwat %lu cc "
> +                                         "%lu mbmax %lu mbcnt %lu f %hd\n",
> +                                         last->so_rcv.sb_hiwat,
> +                                         last->so_rcv.sb_cc,
> +                                         last->so_rcv.sb_mbmax,
> +                                         last->so_rcv.sb_mbcnt,
> +                                         last->so_options);
> +                             }
> +#endif
>                               if (sbspace(last, &last->so_rcv) < (2*MSIZE) ||
>                                   sbappendaddr(last, &last->so_rcv, sosrc,
>                                   n, (struct mbuf *)NULL) == 0) {
> @@ -479,6 +497,20 @@ route_input(struct mbuf *m0, struct socket *so, 
> sa_family_t sa_family)
>               last = rp->rcb_socket;
>       }
>       if (last) {
> +             if ((last->so_options & SO_RDAEMON) &&
> +                 (last->so_rcv.sb_cc > (sb_max/4)))
> +                     yield_cnt++;
> +#if 0
> +             if (sbspace(last, &last->so_rcv) < (2*MSIZE)) {
> +                     printf("sbspace 2 hiwat %lu cc "
> +                         "%lu mbmax %lu mbcnt %lu f %hd\n",
> +                         last->so_rcv.sb_hiwat,
> +                         last->so_rcv.sb_cc,
> +                         last->so_rcv.sb_mbmax,
> +                         last->so_rcv.sb_mbcnt,
> +                         last->so_options);
> +             }
> +#endif
>               if (sbspace(last, &last->so_rcv) < (2 * MSIZE) ||
>                   sbappendaddr(last, &last->so_rcv, sosrc,
>                   m, (struct mbuf *)NULL) == 0) {
> @@ -493,6 +525,24 @@ route_input(struct mbuf *m0, struct socket *so, 
> sa_family_t sa_family)
>               }
>       } else
>               m_freem(m);
> +
> +     if (yield_cnt && so != NULL) {
> +#if 0
> +             if ((yield_logged % 2000) == 0) {
> +                     printf("%s: yield hiwat %lu cc "
> +                         "%lu mbmax %lu mbcnt %lu f %hd\n",
> +                         __func__,
> +                         last->so_rcv.sb_hiwat,
> +                         last->so_rcv.sb_cc,
> +                         last->so_rcv.sb_mbmax,
> +                         last->so_rcv.sb_mbcnt,
> +                         last->so_options);
> +             }
> +             yield_logged++;
> +#endif
> +             NET_ASSERT_UNLOCKED();
> +             yield();
> +     }
>  }
>  
>  struct rt_msghdr *
> diff --git sys/sys/socket.h sys/sys/socket.h
> index a1ff94653f8..fc78c14296c 100644
> --- sys/sys/socket.h
> +++ sys/sys/socket.h
> @@ -97,6 +97,7 @@ typedef     __sa_family_t   sa_family_t;    /* sockaddr 
> address family type */
>  #define SO_TIMESTAMP 0x0800          /* timestamp received dgram traffic */
>  #define SO_BINDANY   0x1000          /* allow bind to any address */
>  #define SO_ZEROIZE   0x2000          /* zero out all mbufs sent over socket 
> */
> +#define      SO_RDAEMON      0x4000          /* optimize bulk routing msg */
>  
>  /*
>   * Additional options, not kept in so_options.
> diff --git usr.sbin/ospfd/kroute.c usr.sbin/ospfd/kroute.c
> index 17febefbdcb..caa1c9fe591 100644
> --- usr.sbin/ospfd/kroute.c
> +++ usr.sbin/ospfd/kroute.c
> @@ -129,7 +129,7 @@ kif_init(void)
>  int
>  kr_init(int fs, u_int rdomain)
>  {
> -     int             opt = 0, rcvbuf, default_rcvbuf;
> +     int             opt = 0, rcvbuf, default_rcvbuf, rdaemon_flag = 1;
>       socklen_t       optlen;
>  
>       kr_state.fib_sync = fs;
> @@ -159,6 +159,11 @@ kr_init(int fs, u_int rdomain)
>                   rcvbuf /= 2)
>                       ;       /* nothing */
>  
> +     if (setsockopt(kr_state.fd, SOL_SOCKET, SO_RDAEMON,
> +         &rdaemon_flag, sizeof(rdaemon_flag)) == -1)
> +             log_info("%s: setsockopt SO_RDAEMON failed",
> +                 __func__);
> +
>       kr_state.pid = getpid();
>       kr_state.rtseq = 1;
>  
> diff --git usr.sbin/ospfd/ospfd.h usr.sbin/ospfd/ospfd.h
> index af082b1079c..f9f52cf3ea4 100644
> --- usr.sbin/ospfd/ospfd.h
> +++ usr.sbin/ospfd/ospfd.h
> @@ -46,7 +46,7 @@
>  #define      READ_BUF_SIZE           65535
>  #define      PKG_DEF_SIZE            512     /* compromise */
>  #define      RT_BUF_SIZE             16384
> -#define      MAX_RTSOCK_BUF          (2 * 1024 * 1024)
> +#define      MAX_RTSOCK_BUF          (1 * 1024 * 1024)
>  
>  #define      OSPFD_FLAG_NO_FIB_UPDATE        0x0001
>  #define      OSPFD_FLAG_STUB_ROUTER          0x0002
> 

Reply via email to