interop: work around extreme REGRESS_SKIP_SLOW slowness

A few years back beck introduced REGRESS_SKIP_SLOW dances with the idea
that this should speed up the interop tests for us devs because this also
checked interop between opensslX and opensslY, which we don't particularly
care about. This never really worked. On a mac m1 mini the result is this:

REGRESS_SKIP_SLOW unset
    9m56.69s real     3m42.24s user     3m00.70s system
REGRESS_SKIP_SLOW=yes
   11m04.61s real     7m29.61s user     1m40.29s system

The problem is that REGRESS_SKIP_SLOW simply wasn't designed to handle
the huge number of tests we have here. There are many nested .for loops
resulting in several thousand tests. Each test has a name of length ~80.
REGRESS_SKIP_SLOW concatenates them into a several hundred kilobytes
long string in REGRESS_SKIP_TARGETS, iterates over all regress targets and
tests with ".if ${REGRESS_SKIP_TARGETS:M${RT}}" if it should skip them.
This means that during a regress run, make spends a lot of time linearly
scanning a huge string.

I ran into this when I added OpenSSL 3.0 tests to the already existing
1.0.2 and 1.1 tests with the result that with REGRESS_SLOW_TARGTS set
it took the better part of an hour while without it it took about 15 min.

The hack here is simply to avoid using REGRESS_SLOW_TARGTES here and
handle the situation differently.

patch, REGRESS_SKIP_SLOW=yes
    5m42.32s real     2m09.98s user     1m45.21s system

The real solution would be to fix this in bsd.regress.mk, which someone
who understands make well is very welcome to do. For now, I'm happy with
this.

Debugged with jsing a few months ago

diff --git a/regress/lib/libssl/interop/cert/Makefile b/regress/lib/libssl/interop/cert/Makefile
index 32b7a4f..47f4422 100644 (file)
--- a/regress/lib/libssl/interop/cert/Makefile
+++ b/regress/lib/libssl/interop/cert/Makefile
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile,v 1.9 2023/02/01 14:39:09 tb Exp $
+# $OpenBSD: Makefile,v 1.10 2023/04/19 15:34:23 tb Exp $
 
 # Connect a client to a server.  Both can be current libressl, or
 # openssl 1.1 or 3.0.  Create client and server certificates
@@ -39,7 +39,8 @@ FAIL_${cca}_${sca}_${ccert}_${scert}_${cv}_${sv} = !
 .if ("${clib}" == "libressl" || "${slib}" == "libressl")
 REGRESS_TARGETS +=     run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv}
 .else
-REGRESS_SLOW_TARGETS +=        run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv}
+# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow.
+SLOW_TARGETS +=        run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv}
 .endif
 
 run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv}: \
@@ -76,6 +77,12 @@ run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${s
 .endfor
 .endfor
 
+.include <bsd.own.mk>
+REGRESS_SKIP_SLOW ?= no
+.if ${REGRESS_SKIP_SLOW:L} != "yes"
+REGRESS_TARGETS += ${SLOW_TARGETS}
+.endif
+
 REGRESS_TARGETS +=     run-bob
 run-bob:
        @echo Bob, be happy!  Tests finished.

diff --git a/regress/lib/libssl/interop/cipher/Makefile b/regress/lib/libssl/interop/cipher/Makefile
index 3cb4330..85d927a 100644 (file)
--- a/regress/lib/libssl/interop/cipher/Makefile
+++ b/regress/lib/libssl/interop/cipher/Makefile
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile,v 1.11 2023/02/01 14:39:09 tb Exp $
+# $OpenBSD: Makefile,v 1.12 2023/04/19 15:34:23 tb Exp $
 
 # Connect a client to a server.  Both can be current libressl, or
 # openssl 1.1 or 3.0.  Create lists of supported ciphers
@@ -105,7 +105,8 @@ DHPARAM_${cipher}_${slib} =
 .if ("${clib}" == "libressl" || "${slib}" == "libressl")
 REGRESS_TARGETS +=     run-cipher-${cipher}-client-${clib}-server-${slib}
 .else
-REGRESS_SLOW_TARGETS +=        run-cipher-${cipher}-client-${clib}-server-${slib}
+# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow.
+SLOW_TARGETS +=        run-cipher-${cipher}-client-${clib}-server-${slib}
 .endif
 run-cipher-${cipher}-client-${clib}-server-${slib} \
 client-cipher-${cipher}-client-${clib}-server-${slib}.out \
@@ -127,7 +128,8 @@ server-cipher-${cipher}-client-${clib}-server-${slib}.out: dh.param \
 .if ("${clib}" == "libressl" || "${slib}" == "libressl")
 REGRESS_TARGETS +=     check-cipher-${cipher}-client-${clib}-server-${slib}
 .else
-REGRESS_SLOW_TARGETS +=        check-cipher-${cipher}-client-${clib}-server-${slib}
+# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow.
+SLOW_TARGETS +=        check-cipher-${cipher}-client-${clib}-server-${slib}
 .endif
 check-cipher-${cipher}-client-${clib}-server-${slib}: \
     client-cipher-${cipher}-client-${clib}-server-${slib}.out \
@@ -161,4 +163,10 @@ check-cipher-${cipher}-client-${clib}-server-${slib}: \
 .endfor
 .endfor
 
+.include <bsd.own.mk>
+REGRESS_SKIP_SLOW ?= no
+.if ${REGRESS_SKIP_SLOW:L} != "yes"
+REGRESS_TARGETS += ${SLOW_TARGETS}
+.endif
+
 .include <bsd.regress.mk>

diff --git a/regress/lib/libssl/interop/version/Makefile b/regress/lib/libssl/interop/version/Makefile
index 1e0af57..9d0ae41 100644 (file)
--- a/regress/lib/libssl/interop/version/Makefile
+++ b/regress/lib/libssl/interop/version/Makefile
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile,v 1.5 2023/02/01 14:39:09 tb Exp $
+# $OpenBSD: Makefile,v 1.6 2023/04/19 15:34:23 tb Exp $
 
 # Connect a client to a server.  Both can be current libressl, or
 # openssl 1.1 or openssl 3.0.  Pin client or server to a fixed TLS
@@ -37,7 +37,8 @@ FAIL_${cver}_${sver} = !
 .if ("${clib}" == "libressl" || "${slib}" == "libressl")
 REGRESS_TARGETS +=     run-version-client-${clib}-${cver}-server-${slib}-${sver}
 .else
-REGRESS_SLOW_TARGETS +=        run-version-client-${clib}-${cver}-server-${slib}-${sver}
+# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow.
+SLOW_TARGETS +=        run-version-client-${clib}-${cver}-server-${slib}-${sver}
 .endif
 
 run-version-client-${clib}-${cver}-server-${slib}-${sver} \
@@ -65,7 +66,8 @@ server-version-client-${clib}-${cver}-server-${slib}-${sver}.out: \
 .if ("${clib}" == "libressl" || "${slib}" == "libressl")
 REGRESS_TARGETS +=     check-version-client-${clib}-${cver}-server-${slib}-${sver}
 .else
-REGRESS_SLOW_TARGETS +=        check-version-client-${clib}-${cver}-server-${slib}-${sver}
+# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow.
+SLOW_TARGETS +=        check-version-client-${clib}-${cver}-server-${slib}-${sver}
 .endif
 
 check-version-client-${clib}-${cver}-server-${slib}-${sver}: \
@@ -98,4 +100,10 @@ check-version-client-${clib}-${cver}-server-${slib}-${sver}: \
 .endfor
 .endfor
 
+.include <bsd.own.mk>
+REGRESS_SKIP_SLOW ?= no
+.if ${REGRESS_SKIP_SLOW:L} != "yes"
+REGRESS_TARGETS += ${SLOW_TARGETS}
+.endif
+
 .include <bsd.regress.mk>