From: tb Date: Wed, 19 Apr 2023 15:34:23 +0000 (+0000) Subject: interop: work around extreme REGRESS_SKIP_SLOW slowness X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=2c34d847be01bf339394fb11d1e9410ca78ac2e3;p=openbsd interop: work around extreme REGRESS_SKIP_SLOW slowness A few years back beck introduced REGRESS_SKIP_SLOW dances with the idea that this should speed up the interop tests for us devs because this also checked interop between opensslX and opensslY, which we don't particularly care about. This never really worked. On a mac m1 mini the result is this: REGRESS_SKIP_SLOW unset 9m56.69s real 3m42.24s user 3m00.70s system REGRESS_SKIP_SLOW=yes 11m04.61s real 7m29.61s user 1m40.29s system The problem is that REGRESS_SKIP_SLOW simply wasn't designed to handle the huge number of tests we have here. There are many nested .for loops resulting in several thousand tests. Each test has a name of length ~80. REGRESS_SKIP_SLOW concatenates them into a several hundred kilobytes long string in REGRESS_SKIP_TARGETS, iterates over all regress targets and tests with ".if ${REGRESS_SKIP_TARGETS:M${RT}}" if it should skip them. This means that during a regress run, make spends a lot of time linearly scanning a huge string. I ran into this when I added OpenSSL 3.0 tests to the already existing 1.0.2 and 1.1 tests with the result that with REGRESS_SLOW_TARGTS set it took the better part of an hour while without it it took about 15 min. The hack here is simply to avoid using REGRESS_SLOW_TARGTES here and handle the situation differently. patch, REGRESS_SKIP_SLOW=yes 5m42.32s real 2m09.98s user 1m45.21s system The real solution would be to fix this in bsd.regress.mk, which someone who understands make well is very welcome to do. For now, I'm happy with this. Debugged with jsing a few months ago --- diff --git a/regress/lib/libssl/interop/cert/Makefile b/regress/lib/libssl/interop/cert/Makefile index 32b7a4f4f11..47f4422d6e0 100644 --- a/regress/lib/libssl/interop/cert/Makefile +++ b/regress/lib/libssl/interop/cert/Makefile @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile,v 1.9 2023/02/01 14:39:09 tb Exp $ +# $OpenBSD: Makefile,v 1.10 2023/04/19 15:34:23 tb Exp $ # Connect a client to a server. Both can be current libressl, or # openssl 1.1 or 3.0. Create client and server certificates @@ -39,7 +39,8 @@ FAIL_${cca}_${sca}_${ccert}_${scert}_${cv}_${sv} = ! .if ("${clib}" == "libressl" || "${slib}" == "libressl") REGRESS_TARGETS += run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv} .else -REGRESS_SLOW_TARGETS += run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv} +# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow. +SLOW_TARGETS += run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv} .endif run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${sv}: \ @@ -76,6 +77,12 @@ run-cert-client-${clib}-${cca}-${ccert}-${cv}-server-${slib}-${sca}-${scert}-${s .endfor .endfor +.include +REGRESS_SKIP_SLOW ?= no +.if ${REGRESS_SKIP_SLOW:L} != "yes" +REGRESS_TARGETS += ${SLOW_TARGETS} +.endif + REGRESS_TARGETS += run-bob run-bob: @echo Bob, be happy! Tests finished. diff --git a/regress/lib/libssl/interop/cipher/Makefile b/regress/lib/libssl/interop/cipher/Makefile index 3cb4330d311..85d927a92dc 100644 --- a/regress/lib/libssl/interop/cipher/Makefile +++ b/regress/lib/libssl/interop/cipher/Makefile @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile,v 1.11 2023/02/01 14:39:09 tb Exp $ +# $OpenBSD: Makefile,v 1.12 2023/04/19 15:34:23 tb Exp $ # Connect a client to a server. Both can be current libressl, or # openssl 1.1 or 3.0. Create lists of supported ciphers @@ -105,7 +105,8 @@ DHPARAM_${cipher}_${slib} = .if ("${clib}" == "libressl" || "${slib}" == "libressl") REGRESS_TARGETS += run-cipher-${cipher}-client-${clib}-server-${slib} .else -REGRESS_SLOW_TARGETS += run-cipher-${cipher}-client-${clib}-server-${slib} +# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow. +SLOW_TARGETS += run-cipher-${cipher}-client-${clib}-server-${slib} .endif run-cipher-${cipher}-client-${clib}-server-${slib} \ client-cipher-${cipher}-client-${clib}-server-${slib}.out \ @@ -127,7 +128,8 @@ server-cipher-${cipher}-client-${clib}-server-${slib}.out: dh.param \ .if ("${clib}" == "libressl" || "${slib}" == "libressl") REGRESS_TARGETS += check-cipher-${cipher}-client-${clib}-server-${slib} .else -REGRESS_SLOW_TARGETS += check-cipher-${cipher}-client-${clib}-server-${slib} +# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow. +SLOW_TARGETS += check-cipher-${cipher}-client-${clib}-server-${slib} .endif check-cipher-${cipher}-client-${clib}-server-${slib}: \ client-cipher-${cipher}-client-${clib}-server-${slib}.out \ @@ -161,4 +163,10 @@ check-cipher-${cipher}-client-${clib}-server-${slib}: \ .endfor .endfor +.include +REGRESS_SKIP_SLOW ?= no +.if ${REGRESS_SKIP_SLOW:L} != "yes" +REGRESS_TARGETS += ${SLOW_TARGETS} +.endif + .include diff --git a/regress/lib/libssl/interop/version/Makefile b/regress/lib/libssl/interop/version/Makefile index 1e0af57a66b..9d0ae418bad 100644 --- a/regress/lib/libssl/interop/version/Makefile +++ b/regress/lib/libssl/interop/version/Makefile @@ -1,4 +1,4 @@ -# $OpenBSD: Makefile,v 1.5 2023/02/01 14:39:09 tb Exp $ +# $OpenBSD: Makefile,v 1.6 2023/04/19 15:34:23 tb Exp $ # Connect a client to a server. Both can be current libressl, or # openssl 1.1 or openssl 3.0. Pin client or server to a fixed TLS @@ -37,7 +37,8 @@ FAIL_${cver}_${sver} = ! .if ("${clib}" == "libressl" || "${slib}" == "libressl") REGRESS_TARGETS += run-version-client-${clib}-${cver}-server-${slib}-${sver} .else -REGRESS_SLOW_TARGETS += run-version-client-${clib}-${cver}-server-${slib}-${sver} +# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow. +SLOW_TARGETS += run-version-client-${clib}-${cver}-server-${slib}-${sver} .endif run-version-client-${clib}-${cver}-server-${slib}-${sver} \ @@ -65,7 +66,8 @@ server-version-client-${clib}-${cver}-server-${slib}-${sver}.out: \ .if ("${clib}" == "libressl" || "${slib}" == "libressl") REGRESS_TARGETS += check-version-client-${clib}-${cver}-server-${slib}-${sver} .else -REGRESS_SLOW_TARGETS += check-version-client-${clib}-${cver}-server-${slib}-${sver} +# Don't use REGRESS_SLOW_TARGETS since its handling in bsd.regress.mk is slow. +SLOW_TARGETS += check-version-client-${clib}-${cver}-server-${slib}-${sver} .endif check-version-client-${clib}-${cver}-server-${slib}-${sver}: \ @@ -98,4 +100,10 @@ check-version-client-${clib}-${cver}-server-${slib}-${sver}: \ .endfor .endfor +.include +REGRESS_SKIP_SLOW ?= no +.if ${REGRESS_SKIP_SLOW:L} != "yes" +REGRESS_TARGETS += ${SLOW_TARGETS} +.endif + .include