From 0d0a919f69d9190ea020032bdfe7f0a482f5116d Mon Sep 17 00:00:00 2001 From: bluhm Date: Wed, 6 Dec 2023 22:57:14 +0000 Subject: [PATCH] Add tests that create and delete cloned routes during connect(2). --- regress/sys/netinet/bindconnect/Makefile | 53 ++++- regress/sys/netinet/bindconnect/README | 25 ++- regress/sys/netinet/bindconnect/bindconnect.c | 208 +++++++++++++++--- 3 files changed, 246 insertions(+), 40 deletions(-) diff --git a/regress/sys/netinet/bindconnect/Makefile b/regress/sys/netinet/bindconnect/Makefile index 2884042f645..258a6a3f8c7 100644 --- a/regress/sys/netinet/bindconnect/Makefile +++ b/regress/sys/netinet/bindconnect/Makefile @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile,v 1.1 2023/12/06 14:41:52 bluhm Exp $ +# $OpenBSD: Makefile,v 1.2 2023/12/06 22:57:14 bluhm Exp $ PROG= bindconnect LDADD= -lpthread @@ -7,22 +7,63 @@ WARNINGS= yes CLEANFILES= ktrace.out -${REGRESS_TARGETS}: ${PROG} +LOCAL_NET ?= + +REGRESS_ROOT_TARGETS += setup-maxfiles run-100000 run-localnet-connect-delete + +REGRESS_SETUP_ONCE += setup-maxfiles +setup-maxfiles: + [[ $$(sysctl -n kern.maxfiles) -ge 110000 ]] || \ + ${SUDO} sysctl kern.maxfiles=110000 + +REGRESS_SETUP += ${PROG} + +.if ! empty(LOCAL_NET) +REGRESS_CLEANUP += cleanup-delete +.endif +cleanup-delete: + -${SUDO} time ./${PROG} -s 0 -o 0 -b 0 -c 0 -d 1 -N ${LOCAL_NET} -t 1 REGRESS_TARGETS += run-default run-default: - ${SUDO} time ${KTRACE} ./${PROG} + time ${KTRACE} ./${PROG} REGRESS_TARGETS += run-bind run-bind: - ${SUDO} time ${KTRACE} ./${PROG} -n 10 -s 2 -o 1 -b 5 -c 0 + time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 5 -c 0 REGRESS_TARGETS += run-connect run-connect: - ${SUDO} time ${KTRACE} ./${PROG} -n 10 -s 2 -o 1 -b 0 -c 5 + time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 0 -c 5 REGRESS_TARGETS += run-bind-connect run-bind-connect: - ${SUDO} time ${KTRACE} ./${PROG} -n 10 -s 2 -o 1 -b 3 -c 3 + time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 3 -c 3 + +REGRESS_TARGETS += run-100000 +run-100000: + ${SUDO} time ${KTRACE} ./${PROG} -n 100000 -s 2 -o 1 -b 3 -c 3 + +REGRESS_TARGETS += run-reuseport +run-reuseport: + time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 3 -c 3 -r + +.if empty(LOCAL_NET) +REGRESS_SKIP_TARGETS += run-localnet-connect run-localnet-bind-connect \ + run-localnet-connect-delete +.endif + +REGRESS_TARGETS += run-localnet-connect +run-localnet-connect: + time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -c 5 -N ${LOCAL_NET} + +REGRESS_TARGETS += run-localnet-bind-connect +run-localnet-bind-connect: + time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 3 -c 3 -N ${LOCAL_NET} + +REGRESS_TARGETS += run-localnet-connect-delete +run-localnet-connect-delete: + ${SUDO} time ${KTRACE} \ + ./${PROG} -n 16 -s 2 -o 1 -b 0 -c 5 -d 3 -N ${LOCAL_NET} .include diff --git a/regress/sys/netinet/bindconnect/README b/regress/sys/netinet/bindconnect/README index b8a00ad2f7d..0c168e63e10 100644 --- a/regress/sys/netinet/bindconnect/README +++ b/regress/sys/netinet/bindconnect/README @@ -1,12 +1,16 @@ Stress test bind(2) and connect(2) system calls in OpenBSD regress. -bindconnect [-b bind] [-c connect] [-n num] [-o close] [-s socket] [-t time] - -b bind threads binding sockets, default 1 - -c connect threads connecting sockets, default 1 - -n num number of file descriptors, default 100 - -o close threads closing sockets, default 1 - -s socket threads creating sockets, default 1 - -t time run time in seconds, default 10 +bindconnect [-r] [-b bind] [-c connect] [-d delroute] +[-N addr/net] [-n num] [-o close] [-s socket] [-t time] + -b bind threads binding sockets, default 1 + -c connect threads connecting sockets, default 1 + -d delroute threads deleting cloned routes, default 0 + -N addr/net connect to any address within network + -n num number of file descriptors, default 128 + -o close threads closing sockets, default 1 + -r set reuse port socket option + -s socket threads creating sockets, default 1 + -t time run time in seconds, default 10 Separate threads are started to run socket(2), close(2), bind(2), and connect(2) system calls concurrently. The number of sockets @@ -15,4 +19,9 @@ system calls operate on random file descriptors. By setting the number of threads for each system call and the number of available file descriptors, the focus for the stress test can be changed. -Currently only IPv4 UDP sockets with 127.0.0.1 are supported. +Currently only IPv4 UDP sockets are supported. Per default the +address to bind and connect is 127.0.0.1. LOCAL_NET environment +variable allows to bind on a local address and connect to all +directly attached hosts. This triggers creation of cloned routes +during source address selection. To stress test routing table, +these routes can be deleted in another thread. diff --git a/regress/sys/netinet/bindconnect/bindconnect.c b/regress/sys/netinet/bindconnect/bindconnect.c index e94f80987ff..19966d9bbb0 100644 --- a/regress/sys/netinet/bindconnect/bindconnect.c +++ b/regress/sys/netinet/bindconnect/bindconnect.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bindconnect.c,v 1.1 2023/12/06 14:41:52 bluhm Exp $ */ +/* $OpenBSD: bindconnect.c,v 1.2 2023/12/06 22:57:14 bluhm Exp $ */ /* * Copyright (c) 2023 Alexander Bluhm @@ -19,7 +19,9 @@ #include #include +#include #include +#include #include #include @@ -32,23 +34,31 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b)) int fd_base; -unsigned int fd_num = 100; +unsigned int fd_num = 128; unsigned int run_time = 10; -unsigned int socket_num = 1, close_num = 1, bind_num = 1, connect_num = 1; +unsigned int socket_num = 1, close_num = 1, bind_num = 1, connect_num = 1, + delroute_num = 0; +int reuse_port = 0; +struct in_addr addr, mask; +int prefix = -1, route_sock = -1; static void __dead usage(void) { fprintf(stderr, - "bindconnect [-b bind] [-c connect] [-n num] [-o close]\n" - "[-s socket] [-t time]\n" - " -b bind threads binding sockets, default %u\n" - " -c connect threads connecting sockets, default %u\n" - " -n num number of file descriptors, default %u\n" - " -o close threads closing sockets, default %u\n" - " -s socket threads creating sockets, default %u\n" - " -t time run time in seconds, default %u\n", - bind_num, connect_num, fd_num, close_num, socket_num, run_time); + "bindconnect [-r] [-b bind] [-c connect] [-d delroute]\n" + "[-N addr/net] [-n num] [-o close] [-s socket] [-t time]\n" + " -b bind threads binding sockets, default %u\n" + " -c connect threads connecting sockets, default %u\n" + " -d delroute threads deleting cloned routes, default %u\n" + " -N addr/net connect to any address within network\n" + " -n num number of file descriptors, default %u\n" + " -o close threads closing sockets, default %u\n" + " -r set reuse port socket option\n" + " -s socket threads creating sockets, default %u\n" + " -t time run time in seconds, default %u\n", + bind_num, connect_num, delroute_num, fd_num, close_num, socket_num, + run_time); exit(2); } @@ -58,14 +68,30 @@ sintosa(struct sockaddr_in *sin) return ((struct sockaddr *)(sin)); } +static void +in_prefixlen2mask(struct in_addr *maskp, int plen) +{ + if (plen == 0) + maskp->s_addr = 0; + else + maskp->s_addr = htonl(0xffffffff << (32 - plen)); +} + static void * thread_socket(void *arg) { volatile int *run = arg; unsigned long count; + int fd; for (count = 0; *run; count++) { - socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + int opt; + + fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (fd < 0 || !reuse_port) + continue; + opt = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt)); } return (void *)count; @@ -99,6 +125,9 @@ thread_bind(void *arg) sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (prefix >= 0) + sin.sin_addr = addr; + for (count = 0; *run; count++) { fd = fd_base + arc4random_uniform(fd_num); bind(fd, sintosa(&sin), sizeof(sin)); @@ -119,31 +148,92 @@ thread_connect(void *arg) sin.sin_len = sizeof(sin); sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - sin.sin_port = arc4random(); + + if (prefix >= 0) + sin.sin_addr = addr; for (count = 0; *run; count++) { fd = fd_base + arc4random_uniform(fd_num); + if (prefix >=0 && prefix != 32) { + sin.sin_addr.s_addr &= mask.s_addr; + sin.sin_addr.s_addr |= ~mask.s_addr & arc4random(); + } + sin.sin_port = arc4random(); connect(fd, sintosa(&sin), sizeof(sin)); } return (void *)count; } +static void * +thread_delroute(void *arg) +{ + volatile int *run = arg; + unsigned long count; + int seq = 0; + struct { + struct rt_msghdr m_rtm; + char m_space[512]; + } m_rtmsg; + struct sockaddr_in sin; + +#define rtm \ + m_rtmsg.m_rtm +#define ROUNDUP(a) \ + ((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long)) +#define ADVANCE(x, n) \ + (x += ROUNDUP((n)->sa_len)) +#define NEXTADDR(w, sa) \ + if (rtm.rtm_addrs & (w)) { \ + int l = ROUNDUP(sa->sa_len); \ + memcpy(cp, sa, l); \ + cp += l; \ + } + + memset(&m_rtmsg, 0, sizeof(m_rtmsg)); + rtm.rtm_type = RTM_DELETE; + rtm.rtm_flags = RTF_HOST; + rtm.rtm_version = RTM_VERSION; + rtm.rtm_addrs = RTA_DST; + rtm.rtm_priority = 0; /* XXX */ + rtm.rtm_hdrlen = sizeof(rtm); + + memset(&sin, 0, sizeof(sin)); + sin.sin_len = sizeof(sin); + sin.sin_family = AF_INET; + sin.sin_addr = addr; + + for (count = 0; *run; count++) { + char *cp = m_rtmsg.m_space; + + rtm.rtm_seq = ++seq; + sin.sin_addr.s_addr &= mask.s_addr; + sin.sin_addr.s_addr |= ~mask.s_addr & arc4random(); + NEXTADDR(RTA_DST, sintosa(&sin)); + rtm.rtm_msglen = cp - (char *)&m_rtmsg; + write(route_sock, &m_rtmsg, rtm.rtm_msglen); + } + +#undef rtm +#undef ROUNDUP +#undef ADVANCE +#undef NEXTADDR + + return (void *)count; +} + int main(int argc, char *argv[]) { struct rlimit rlim; - pthread_t *tsocket, *tclose, *tbind, *tconnect; - const char *errstr; + pthread_t *tsocket, *tclose, *tbind, *tconnect, *tdelroute; + const char *errstr, *addr_net = NULL; int ch, run; unsigned int n; - unsigned long socket_count, close_count, bind_count, connect_count; + unsigned long socket_count, close_count, bind_count, connect_count, + delroute_count; - fd_base = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - if (fd_base < 0) - err(1, "socket fd_base"); - - while ((ch = getopt(argc, argv, "b:c:n:o:s:t:")) != -1) { + while ((ch = getopt(argc, argv, "b:c:d:N:n:o:rs:t:")) != -1) { switch (ch) { case 'b': bind_num = strtonum(optarg, 0, UINT_MAX, &errstr); @@ -155,9 +245,16 @@ main(int argc, char *argv[]) if (errstr != NULL) errx(1, "connect is %s: %s", errstr, optarg); break; + case 'd': + delroute_num = strtonum(optarg, 0, UINT_MAX, &errstr); + if (errstr != NULL) + errx(1, "delroute is %s: %s", errstr, optarg); + break; + case 'N': + addr_net = optarg; + break; case 'n': - fd_num = strtonum(optarg, 1, INT_MAX - fd_base, - &errstr); + fd_num = strtonum(optarg, 1, INT_MAX, &errstr); if (errstr != NULL) errx(1, "num is %s: %s", errstr, optarg); break; @@ -166,6 +263,9 @@ main(int argc, char *argv[]) if (errstr != NULL) errx(1, "close is %s: %s", errstr, optarg); break; + case 'r': + reuse_port = 1; + break; case 's': socket_num = strtonum(optarg, 0, UINT_MAX, &errstr); if (errstr != NULL) @@ -185,6 +285,27 @@ main(int argc, char *argv[]) if (argc > 0) usage(); + if (addr_net != NULL) { + prefix = inet_net_pton(AF_INET, addr_net, &addr, sizeof(addr)); + if (prefix < 0) + err(1, "inet_net_pton %s", addr_net); + in_prefixlen2mask(&mask, prefix); + } + if (delroute_num > 0) { + if (prefix < 0 || prefix == 32) + errx(1, "delroute %u needs addr/net", delroute_num); + route_sock = socket(AF_ROUTE, SOCK_RAW, AF_INET); + if (route_sock < 0) + err(1, "socket route"); + if (shutdown(route_sock, SHUT_RD) < 0) + err(1, "shutdown read route"); + } + + fd_base = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (fd_base < 0) + err(1, "socket fd_base"); + if (fd_base > INT_MAX - (int)fd_num) + err(1, "fd base %d and num %u overflow", fd_base, fd_num); if (closefrom(fd_base) < 0) err(1, "closefrom %d", fd_base); @@ -196,6 +317,7 @@ main(int argc, char *argv[]) err(1, "setrlimit %llu", rlim.rlim_cur); run = 1; + tsocket = calloc(socket_num, sizeof(pthread_t)); if (tsocket == NULL) err(1, "tsocket"); @@ -204,6 +326,7 @@ main(int argc, char *argv[]) if (errno) err(1, "pthread_create socket %u", n); } + tclose = calloc(close_num, sizeof(pthread_t)); if (tclose == NULL) err(1, "tclose"); @@ -212,6 +335,7 @@ main(int argc, char *argv[]) if (errno) err(1, "pthread_create close %u", n); } + tbind = calloc(bind_num, sizeof(pthread_t)); if (tbind == NULL) err(1, "tbind"); @@ -220,6 +344,7 @@ main(int argc, char *argv[]) if (errno) err(1, "pthread_create bind %u", n); } + tconnect = calloc(connect_num, sizeof(pthread_t)); if (tconnect == NULL) err(1, "tconnect"); @@ -230,6 +355,16 @@ main(int argc, char *argv[]) err(1, "pthread_create connect %u", n); } + tdelroute = calloc(delroute_num, sizeof(pthread_t)); + if (tdelroute == NULL) + err(1, "tdelroute"); + for (n = 0; n < delroute_num; n++) { + errno = pthread_create(&tdelroute[n], NULL, thread_delroute, + &run); + if (errno) + err(1, "pthread_create delroute %u", n); + } + if (run_time > 0) { if (sleep(run_time) < 0) err(1, "sleep %u", run_time); @@ -245,6 +380,8 @@ main(int argc, char *argv[]) err(1, "pthread_join socket %u", n); socket_count += count; } + free(tsocket); + close_count = 0; for (n = 0; n < close_num; n++) { unsigned long count; @@ -254,6 +391,8 @@ main(int argc, char *argv[]) err(1, "pthread_join close %u", n); close_count += count; } + free(tclose); + bind_count = 0; for (n = 0; n < bind_num; n++) { unsigned long count; @@ -263,6 +402,8 @@ main(int argc, char *argv[]) err(1, "pthread_join bind %u", n); bind_count += count; } + free(tbind); + connect_count = 0; for (n = 0; n < connect_num; n++) { unsigned long count; @@ -272,8 +413,23 @@ main(int argc, char *argv[]) err(1, "pthread_join connect %u", n); connect_count += count; } - printf("count: socket %lu, close %lu, bind %lu, connect %lu\n", - socket_count, close_count, bind_count, connect_count); + free(tconnect); + + delroute_count = 0; + for (n = 0; n < delroute_num; n++) { + unsigned long count; + + errno = pthread_join(tdelroute[n], (void **)&count); + if (errno) + err(1, "pthread_join delroute %u", n); + delroute_count += count; + } + free(tdelroute); + + printf("count: socket %lu, close %lu, bind %lu, connect %lu, " + "delroute %lu\n", + socket_count, close_count, bind_count, connect_count, + delroute_count); return 0; } -- 2.20.1