Retire asynchronous crypto API as it is no longer required by any driver and

adds unnecessary complexity.  Dedicated crypto offloading devices are not common
anymore.  Modern CPU crypto acceleration works synchronously, eliminating the need
for callbacks.

Replace all occurrences of crypto_dispatch() with crypto_invoke(), which is
blocking and only returns after the operation has completed or an error occured.
Invoke callback functions directly from the consumer (e.g. IPsec, softraid)
instead of relying on the crypto driver to call crypto_done().

ok bluhm@ mvs@ patrick@

diff --git a/sys/arch/amd64/amd64/aesni.c b/sys/arch/amd64/amd64/aesni.c
index 12ed40b..d99c1c7 100644 (file)
--- a/sys/arch/amd64/amd64/aesni.c
+++ b/sys/arch/amd64/amd64/aesni.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: aesni.c,v 1.51 2021/10/13 13:08:58 bluhm Exp $        */
+/*     $OpenBSD: aesni.c,v 1.52 2021/10/23 15:42:34 tobhe Exp $        */
 /*-
  * Copyright (c) 2003 Jason Wright
  * Copyright (c) 2003, 2004 Theo de Raadt
@@ -699,7 +699,6 @@ aesni_process(struct cryptop *crp)
 out:
        smr_read_leave();
        crp->crp_etype = err;
-       crypto_done(crp);
        return (err);
 }
 

diff --git a/sys/arch/amd64/amd64/via.c b/sys/arch/amd64/amd64/via.c
index d6f04c3..041b4a1 100644 (file)
--- a/sys/arch/amd64/amd64/via.c
+++ b/sys/arch/amd64/amd64/via.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: via.c,v 1.35 2021/10/13 13:08:58 bluhm Exp $  */
+/*     $OpenBSD: via.c,v 1.36 2021/10/23 15:42:34 tobhe Exp $  */
 /*     $NetBSD: machdep.c,v 1.214 1996/11/10 03:16:17 thorpej Exp $    */
 
 /*-
@@ -460,7 +460,6 @@ viac3_crypto_process(struct cryptop *crp)
        }
 out:
        crp->crp_etype = err;
-       crypto_done(crp);
        return (err);
 }
 

diff --git a/sys/arch/arm64/arm64/cryptox.c b/sys/arch/arm64/arm64/cryptox.c
index 0b4ff18..c97da3a 100644 (file)
--- a/sys/arch/arm64/arm64/cryptox.c
+++ b/sys/arch/arm64/arm64/cryptox.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cryptox.c,v 1.3 2021/10/13 13:08:58 bluhm Exp $       */
+/*     $OpenBSD: cryptox.c,v 1.4 2021/10/23 15:42:35 tobhe Exp $       */
 /*
  * Copyright (c) 2003 Jason Wright
  * Copyright (c) 2003, 2004 Theo de Raadt
@@ -486,6 +486,5 @@ cryptox_process(struct cryptop *crp)
 out:
        smr_read_leave();
        crp->crp_etype = err;
-       crypto_done(crp);
        return (err);
 }

diff --git a/sys/arch/i386/i386/via.c b/sys/arch/i386/i386/via.c
index 9822bec..3a93e6c 100644 (file)
--- a/sys/arch/i386/i386/via.c
+++ b/sys/arch/i386/i386/via.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: via.c,v 1.48 2021/10/13 13:08:58 bluhm Exp $  */
+/*     $OpenBSD: via.c,v 1.49 2021/10/23 15:42:35 tobhe Exp $  */
 /*     $NetBSD: machdep.c,v 1.214 1996/11/10 03:16:17 thorpej Exp $    */
 
 /*-
@@ -468,7 +468,6 @@ viac3_crypto_process(struct cryptop *crp)
        }
 out:
        crp->crp_etype = err;
-       crypto_done(crp);
        return (err);
 }
 

diff --git a/sys/arch/i386/pci/glxsb.c b/sys/arch/i386/pci/glxsb.c
index 2dca0de..e09cea3 100644 (file)
--- a/sys/arch/i386/pci/glxsb.c
+++ b/sys/arch/i386/pci/glxsb.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: glxsb.c,v 1.38 2021/10/13 13:08:58 bluhm Exp $        */
+/*     $OpenBSD: glxsb.c,v 1.39 2021/10/23 15:42:35 tobhe Exp $        */
 
 /*
  * Copyright (c) 2006 Tom Cosgrove <tom@openbsd.org>
@@ -825,7 +825,6 @@ glxsb_crypto_process(struct cryptop *crp)
 
 out:
        crp->crp_etype = err;
-       crypto_done(crp);
        splx(s);
        return (err);
 }

diff --git a/sys/arch/octeon/dev/octcrypto.c b/sys/arch/octeon/dev/octcrypto.c
index 2644967..29017de 100644 (file)
--- a/sys/arch/octeon/dev/octcrypto.c
+++ b/sys/arch/octeon/dev/octcrypto.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: octcrypto.c,v 1.6 2021/10/13 13:08:58 bluhm Exp $     */
+/*     $OpenBSD: octcrypto.c,v 1.7 2021/10/23 15:42:35 tobhe Exp $     */
 
 /*
  * Copyright (c) 2018 Visa Hankala
@@ -658,7 +658,6 @@ out:
        smr_read_leave();
 
        crp->crp_etype = error;
-       crypto_done(crp);
        return error;
 }
 

diff --git a/sys/crypto/crypto.c b/sys/crypto/crypto.c
index c4bc606..6efb0d7 100644 (file)
--- a/sys/crypto/crypto.c
+++ b/sys/crypto/crypto.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: crypto.c,v 1.89 2021/10/21 23:03:48 tobhe Exp $       */
+/*     $OpenBSD: crypto.c,v 1.90 2021/10/23 15:42:35 tobhe Exp $       */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -376,33 +376,6 @@ crypto_unregister(u_int32_t driverid, int alg)
        return 0;
 }
 
-/*
- * Add crypto request to a queue, to be processed by a kernel thread.
- */
-void
-crypto_dispatch(struct cryptop *crp)
-{
-       int lock = 1, s;
-       u_int32_t hid;
-
-       s = splvm();
-       hid = (crp->crp_sid >> 32) & 0xffffffff;
-       if (hid < crypto_drivers_num) {
-               if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_MPSAFE)
-                       lock = 0;
-       }
-       splx(s);
-
-       /* XXXSMP crypto_invoke() is not MP safe */
-       lock = 1;
-
-       if (lock)
-               KERNEL_LOCK();
-       crypto_invoke(crp);
-       if (lock)
-               KERNEL_UNLOCK();
-}
-
 /*
  * Dispatch a crypto request to the appropriate crypto devices.
  */
@@ -416,7 +389,6 @@ crypto_invoke(struct cryptop *crp)
 
        /* Sanity checks. */
        KASSERT(crp != NULL);
-       KASSERT(crp->crp_callback != NULL);
 
        KERNEL_ASSERT_LOCKED();
 
@@ -466,7 +438,6 @@ crypto_invoke(struct cryptop *crp)
 
        crp->crp_etype = EAGAIN;
  done:
-       crypto_done(crp);
        splx(s);
 }
 
@@ -518,14 +489,3 @@ crypto_init(void)
        pool_init(&cryptop_pool, sizeof(struct cryptop), 0, IPL_VM, 0,
            "cryptop", NULL);
 }
-
-/*
- * Invoke the callback on behalf of the driver.
- */
-void
-crypto_done(struct cryptop *crp)
-{
-       crp->crp_flags |= CRYPTO_F_DONE;
-
-       crp->crp_callback(crp);
-}

diff --git a/sys/crypto/cryptodev.h b/sys/crypto/cryptodev.h
index 5caa512..4af72c7 100644 (file)
--- a/sys/crypto/cryptodev.h
+++ b/sys/crypto/cryptodev.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cryptodev.h,v 1.79 2021/10/22 12:30:53 bluhm Exp $    */
+/*     $OpenBSD: cryptodev.h,v 1.80 2021/10/23 15:42:35 tobhe Exp $    */
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -169,18 +169,14 @@ struct cryptop {
 #define CRYPTO_F_IMBUF 0x0001  /* Input/output are mbuf chains, otherwise contig */
 #define CRYPTO_F_IOV   0x0002  /* Input/output are uio */
 #define CRYPTO_F_MPSAFE        0x0004  /* Do not use kernel lock for callback */
-#define CRYPTO_F_DONE  0x0010  /* request completed */
 
        void            *crp_buf;       /* Data to be processed */
-       void            *crp_opaque;    /* Opaque pointer, passed along */
 
        struct cryptodesc *crp_desc;    /* List of processing descriptors */
        struct cryptodesc crp_sdesc[2]; /* Static array for small ops */
        int              crp_ndesc;     /* Amount of descriptors to use */
        int              crp_ndescalloc;/* Amount of descriptors allocated */
 
-       void (*crp_callback)(struct cryptop *); /* Callback function */
-
        caddr_t         crp_mac;
 };
 
@@ -214,14 +210,12 @@ void      crypto_init(void);
 
 int    crypto_newsession(u_int64_t *, struct cryptoini *, int);
 int    crypto_freesession(u_int64_t);
-void   crypto_dispatch(struct cryptop *);
 int    crypto_register(u_int32_t, int *,
            int (*)(u_int32_t *, struct cryptoini *), int (*)(u_int64_t),
            int (*)(struct cryptop *));
 int    crypto_unregister(u_int32_t, int);
 int32_t        crypto_get_driverid(u_int8_t);
 void   crypto_invoke(struct cryptop *);
-void   crypto_done(struct cryptop *);
 
 void   cuio_copydata(struct uio *, int, int, caddr_t);
 void   cuio_copyback(struct uio *, int, int, const void *);

diff --git a/sys/crypto/cryptosoft.c b/sys/crypto/cryptosoft.c
index 3fc7e3c..dcb815a 100644 (file)
--- a/sys/crypto/cryptosoft.c
+++ b/sys/crypto/cryptosoft.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: cryptosoft.c,v 1.89 2021/10/13 13:08:58 bluhm Exp $   */
+/*     $OpenBSD: cryptosoft.c,v 1.90 2021/10/23 15:42:35 tobhe Exp $   */
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -1129,7 +1129,6 @@ swcr_process(struct cryptop *crp)
        }
 
 done:
-       crypto_done(crp);
        return 0;
 }
 

diff --git a/sys/dev/softraid_crypto.c b/sys/dev/softraid_crypto.c
index 867e9f6..376deb1 100644 (file)
--- a/sys/dev/softraid_crypto.c
+++ b/sys/dev/softraid_crypto.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_crypto.c,v 1.143 2021/10/22 05:06:37 anton Exp $ */
+/* $OpenBSD: softraid_crypto.c,v 1.144 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * Copyright (c) 2007 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2008 Hans-Joerg Hoexer <hshoexer@openbsd.org>
@@ -87,13 +87,13 @@ int         sr_crypto_meta_opt_handler_internal(struct sr_discipline *,
                    struct sr_crypto *, struct sr_meta_opt_hdr *);
 int            sr_crypto_meta_opt_handler(struct sr_discipline *,
                    struct sr_meta_opt_hdr *);
-void           sr_crypto_write(struct cryptop *);
+void           sr_crypto_write(struct sr_crypto_wu *);
 int            sr_crypto_rw(struct sr_workunit *);
 int            sr_crypto_dev_rw(struct sr_workunit *, struct sr_crypto_wu *);
 void           sr_crypto_done_internal(struct sr_workunit *,
                    struct sr_crypto *);
 void           sr_crypto_done(struct sr_workunit *);
-void           sr_crypto_read(struct cryptop *);
+void           sr_crypto_read(struct sr_crypto_wu *);
 void           sr_crypto_calculate_check_hmac_sha1(u_int8_t *, int,
                   u_int8_t *, int, u_char *);
 void           sr_crypto_hotplug(struct sr_discipline *, struct disk *, int);
@@ -322,7 +322,6 @@ sr_crypto_prepare(struct sr_workunit *wu, struct sr_crypto *mdd_crypto,
        keyndx = blkno >> SR_CRYPTO_KEY_BLKSHIFT;
        crwu->cr_crp->crp_sid = mdd_crypto->scr_sid[keyndx];
 
-       crwu->cr_crp->crp_opaque = crwu;
        crwu->cr_crp->crp_ilen = xs->datalen;
        crwu->cr_crp->crp_alloctype = M_DEVBUF;
        crwu->cr_crp->crp_flags = CRYPTO_F_IOV;
@@ -1168,8 +1167,8 @@ sr_crypto_rw(struct sr_workunit *wu)
        if (wu->swu_xs->flags & SCSI_DATA_OUT) {
                mdd_crypto = &wu->swu_dis->mds.mdd_crypto;
                crwu = sr_crypto_prepare(wu, mdd_crypto, 1);
-               crwu->cr_crp->crp_callback = sr_crypto_write;
-               crypto_dispatch(crwu->cr_crp);
+               crypto_invoke(crwu->cr_crp);
+               sr_crypto_write(crwu);
                rv = crwu->cr_crp->crp_etype;
        } else
                rv = sr_crypto_dev_rw(wu, NULL);
@@ -1178,16 +1177,15 @@ sr_crypto_rw(struct sr_workunit *wu)
 }
 
 void
-sr_crypto_write(struct cryptop *crp)
+sr_crypto_write(struct sr_crypto_wu *crwu)
 {
-       struct sr_crypto_wu     *crwu = crp->crp_opaque;
        struct sr_workunit      *wu = &crwu->cr_wu;
        int                     s;
 
        DNPRINTF(SR_D_INTR, "%s: sr_crypto_write: wu %p xs: %p\n",
            DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
 
-       if (crp->crp_etype) {
+       if (crwu->cr_crp->crp_etype) {
                /* fail io */
                wu->swu_xs->error = XS_DRIVER_STUFFUP;
                s = splbio();
@@ -1246,10 +1244,10 @@ sr_crypto_done_internal(struct sr_workunit *wu, struct sr_crypto *mdd_crypto)
        /* If this was a successful read, initiate decryption of the data. */
        if (ISSET(xs->flags, SCSI_DATA_IN) && xs->error == XS_NOERROR) {
                crwu = sr_crypto_prepare(wu, mdd_crypto, 0);
-               crwu->cr_crp->crp_callback = sr_crypto_read;
-               DNPRINTF(SR_D_INTR, "%s: sr_crypto_done: crypto_dispatch %p\n",
+               DNPRINTF(SR_D_INTR, "%s: sr_crypto_done: crypto_invoke %p\n",
                    DEVNAME(wu->swu_dis->sd_sc), crwu->cr_crp);
-               crypto_dispatch(crwu->cr_crp);
+               crypto_invoke(crwu->cr_crp);
+               sr_crypto_read(crwu);
                return;
        }
 
@@ -1266,16 +1264,15 @@ sr_crypto_done(struct sr_workunit *wu)
 }
 
 void
-sr_crypto_read(struct cryptop *crp)
+sr_crypto_read(struct sr_crypto_wu *crwu)
 {
-       struct sr_crypto_wu     *crwu = crp->crp_opaque;
        struct sr_workunit      *wu = &crwu->cr_wu;
        int                     s;
 
        DNPRINTF(SR_D_INTR, "%s: sr_crypto_read: wu %p xs: %p\n",
            DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
 
-       if (crp->crp_etype)
+       if (crwu->cr_crp->crp_etype)
                wu->swu_xs->error = XS_DRIVER_STUFFUP;
 
        s = splbio();

diff --git a/sys/dev/softraid_raid1c.c b/sys/dev/softraid_raid1c.c
index d23323b..d428709 100644 (file)
--- a/sys/dev/softraid_raid1c.c
+++ b/sys/dev/softraid_raid1c.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_raid1c.c,v 1.4 2021/10/13 22:43:44 bluhm Exp $ */
+/* $OpenBSD: softraid_raid1c.c,v 1.5 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * Copyright (c) 2007 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2008 Hans-Joerg Hoexer <hshoexer@openbsd.org>
@@ -58,7 +58,7 @@ void  sr_raid1c_free_resources(struct sr_discipline *sd);
 int    sr_raid1c_ioctl(struct sr_discipline *sd, struct bioc_discipline *bd);
 int    sr_raid1c_meta_opt_handler(struct sr_discipline *,
            struct sr_meta_opt_hdr *);
-void   sr_raid1c_write(struct cryptop *);
+void   sr_raid1c_write(struct sr_crypto_wu *);
 int    sr_raid1c_rw(struct sr_workunit *);
 int    sr_raid1c_dev_rw(struct sr_workunit *, struct sr_crypto_wu *);
 void   sr_raid1c_done(struct sr_workunit *wu);
@@ -312,16 +312,15 @@ bad:
 }
 
 void
-sr_raid1c_write(struct cryptop *crp)
+sr_raid1c_write(struct sr_crypto_wu *crwu)
 {
-       struct sr_crypto_wu     *crwu = crp->crp_opaque;
        struct sr_workunit      *wu = &crwu->cr_wu;
        int                     s;
 
        DNPRINTF(SR_D_INTR, "%s: sr_raid1c_write: wu %p xs: %p\n",
            DEVNAME(wu->swu_dis->sd_sc), wu, wu->swu_xs);
 
-       if (crp->crp_etype) {
+       if (crwu->cr_crp->crp_etype) {
                /* fail io */
                wu->swu_xs->error = XS_DRIVER_STUFFUP;
                s = splbio();
@@ -358,8 +357,8 @@ sr_raid1c_rw(struct sr_workunit *wu)
            !ISSET(wu->swu_flags, SR_WUF_REBUILD)) {
                mdd_raid1c = &wu->swu_dis->mds.mdd_raid1c;
                crwu = sr_crypto_prepare(wu, &mdd_raid1c->sr1c_crypto, 1);
-               crwu->cr_crp->crp_callback = sr_raid1c_write;
-               crypto_dispatch(crwu->cr_crp);
+               crypto_invoke(crwu->cr_crp);
+               sr_raid1c_write(crwu);
                rv = crwu->cr_crp->crp_etype;
        } else
                rv = sr_raid1c_dev_rw(wu, NULL);

diff --git a/sys/netinet/ip_ah.c b/sys/netinet/ip_ah.c
index a10c2a7..e64026d 100644 (file)
--- a/sys/netinet/ip_ah.c
+++ b/sys/netinet/ip_ah.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ip_ah.c,v 1.158 2021/10/22 15:44:20 bluhm Exp $ */
+/*     $OpenBSD: ip_ah.c,v 1.159 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -531,7 +531,7 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        struct tdb_crypto *tc = NULL;
        u_int32_t btsx, esn;
        u_int8_t hl;
-       int error, rplen;
+       int error, rplen, clen;
        u_int64_t ibytes;
 #ifdef ENCDEBUG
        char buf[INET6_ADDRSTRLEN];
@@ -686,9 +686,7 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
        crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
        crp->crp_buf = (caddr_t)m;
-       crp->crp_callback = ipsec_input_cb;
        crp->crp_sid = tdb->tdb_cryptoid;
-       crp->crp_opaque = (caddr_t)tc;
 
        /* These are passed as-is to the callback. */
        tc->tc_skip = skip;
@@ -699,7 +697,34 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
        tc->tc_rpl = tdb->tdb_rpl;
 
-       crypto_dispatch(crp);
+       KERNEL_LOCK();
+       crypto_invoke(crp);
+       while (crp->crp_etype == EAGAIN) {
+               /* Reset the session ID */
+               if (tdb->tdb_cryptoid != 0)
+                       tdb->tdb_cryptoid = crp->crp_sid;
+               crypto_invoke(crp);
+       }
+       KERNEL_UNLOCK();
+       if (crp->crp_etype) {
+               DPRINTF("crypto error %d", crp->crp_etype);
+               ipsecstat_inc(ipsec_noxform);
+               error = crp->crp_etype;
+               goto drop;
+       }
+
+       /* Length of data after processing */
+       clen = crp->crp_olen;
+
+       /* Release the crypto descriptors */
+       crypto_freereq(crp);
+
+       error = ah_input_cb(tdb, tc, m, clen);
+       if (error) {
+               ipsecstat_inc(ipsec_idrops);
+               tdb->tdb_idrops++;
+       }
+
        return 0;
 
  drop:
@@ -893,7 +918,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        struct cryptop *crp = NULL;
        u_int64_t replay64;
        u_int16_t iplen;
-       int error, rplen, roff;
+       int error, rplen, roff, ilen, olen;
        u_int8_t prot;
        struct ah *ah;
 #if NBPFILTER > 0
@@ -1133,9 +1158,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
        crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
        crp->crp_buf = (caddr_t)m;
-       crp->crp_callback = ipsec_output_cb;
        crp->crp_sid = tdb->tdb_cryptoid;
-       crp->crp_opaque = (caddr_t)tc;
 
        /* These are passed as-is to the callback. */
        tc->tc_skip = skip;
@@ -1145,7 +1168,34 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        tc->tc_rdomain = tdb->tdb_rdomain;
        memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
 
-       crypto_dispatch(crp);
+       KERNEL_LOCK();
+       crypto_invoke(crp);
+       while (crp->crp_etype == EAGAIN) {
+               /* Reset the session ID */
+               if (tdb->tdb_cryptoid != 0)
+                       tdb->tdb_cryptoid = crp->crp_sid;
+               crypto_invoke(crp);
+       }
+       KERNEL_UNLOCK();
+       if (crp->crp_etype) {
+               DPRINTF("crypto error %d", crp->crp_etype);
+               ipsecstat_inc(ipsec_noxform);
+               error = crp->crp_etype;
+               goto drop;
+       }
+
+       ilen = crp->crp_ilen;
+       olen = crp->crp_olen;
+
+       /* Release the crypto descriptors */
+       crypto_freereq(crp);
+
+       error = ah_output_cb(tdb, tc, m, ilen, olen);
+       if (error) {
+               ipsecstat_inc(ipsec_odrops);
+               tdb->tdb_odrops++;
+       }
+
        return 0;
 
  drop:

diff --git a/sys/netinet/ip_esp.c b/sys/netinet/ip_esp.c
index 1b96a2b..391cfe0 100644 (file)
--- a/sys/netinet/ip_esp.c
+++ b/sys/netinet/ip_esp.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ip_esp.c,v 1.177 2021/10/22 15:44:20 bluhm Exp $ */
+/*     $OpenBSD: ip_esp.c,v 1.178 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -347,7 +347,7 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        struct cryptodesc *crde = NULL, *crda = NULL;
        struct cryptop *crp = NULL;
        struct tdb_crypto *tc = NULL;
-       int plen, alen, hlen, error;
+       int plen, alen, hlen, error, clen;
        u_int32_t btsx, esn;
 #ifdef ENCDEBUG
        char buf[INET6_ADDRSTRLEN];
@@ -498,9 +498,7 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
        crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
        crp->crp_buf = (caddr_t)m;
-       crp->crp_callback = ipsec_input_cb;
        crp->crp_sid = tdb->tdb_cryptoid;
-       crp->crp_opaque = (caddr_t)tc;
 
        /* These are passed as-is to the callback */
        tc->tc_skip = skip;
@@ -526,7 +524,33 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
                        crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
        }
 
-       crypto_dispatch(crp);
+       KERNEL_LOCK();
+       crypto_invoke(crp);
+       while (crp->crp_etype == EAGAIN) {
+               /* Reset the session ID */
+               if (tdb->tdb_cryptoid != 0)
+                       tdb->tdb_cryptoid = crp->crp_sid;
+               crypto_invoke(crp);
+       }
+       KERNEL_UNLOCK();
+       if (crp->crp_etype) {
+               DPRINTF("crypto error %d", crp->crp_etype);
+               ipsecstat_inc(ipsec_noxform);
+               error = crp->crp_etype;
+               goto drop;
+       }
+
+       clen = crp->crp_olen;
+
+       /* Release the crypto descriptors */
+       crypto_freereq(crp);
+
+       error = esp_input_cb(tdb, tc, m, clen);
+       if (error) {
+               ipsecstat_inc(ipsec_idrops);
+               tdb->tdb_idrops++;
+       }
+
        return 0;
 
  drop:
@@ -742,7 +766,7 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 {
        const struct enc_xform *espx = tdb->tdb_encalgxform;
        const struct auth_hash *esph = tdb->tdb_authalgxform;
-       int ilen, hlen, rlen, padding, blks, alen, roff, error;
+       int ilen, olen, hlen, rlen, padding, blks, alen, roff, error;
        u_int64_t replay64;
        u_int32_t replay;
        struct mbuf *mi, *mo = (struct mbuf *) NULL;
@@ -980,8 +1004,6 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
        crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
        crp->crp_buf = (caddr_t)m;
-       crp->crp_callback = ipsec_output_cb;
-       crp->crp_opaque = (caddr_t)tc;
        crp->crp_sid = tdb->tdb_cryptoid;
 
        if (esph) {
@@ -1010,7 +1032,34 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
                        crda->crd_len = m->m_pkthdr.len - (skip + alen);
        }
 
-       crypto_dispatch(crp);
+       KERNEL_LOCK();
+       crypto_invoke(crp);
+       while (crp->crp_etype == EAGAIN) {
+               /* Reset the session ID */
+               if (tdb->tdb_cryptoid != 0)
+                       tdb->tdb_cryptoid = crp->crp_sid;
+               crypto_invoke(crp);
+       }
+       KERNEL_UNLOCK();
+       if (crp->crp_etype) {
+               DPRINTF("crypto error %d", crp->crp_etype);
+               ipsecstat_inc(ipsec_noxform);
+               error = crp->crp_etype;
+               goto drop;
+       }
+
+       ilen = crp->crp_ilen;
+       olen = crp->crp_olen;
+
+       /* Release the crypto descriptors */
+       crypto_freereq(crp);
+
+       error = esp_output_cb(tdb, tc, m, ilen, olen);
+       if (error) {
+               ipsecstat_inc(ipsec_odrops);
+               tdb->tdb_odrops++;
+       }
+
        return 0;
 
  drop:

diff --git a/sys/netinet/ip_ipcomp.c b/sys/netinet/ip_ipcomp.c
index 902955c..1854df8 100644 (file)
--- a/sys/netinet/ip_ipcomp.c
+++ b/sys/netinet/ip_ipcomp.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ip_ipcomp.c,v 1.79 2021/10/22 15:44:20 bluhm Exp $ */
+/* $OpenBSD: ip_ipcomp.c,v 1.80 2021/10/23 15:42:35 tobhe Exp $ */
 
 /*
  * Copyright (c) 2001 Jean-Jacques Bernard-Gundol (jj@wabbitt.org)
@@ -135,7 +135,7 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 {
        const struct comp_algo *ipcompx = tdb->tdb_compalgxform;
        struct tdb_crypto *tc;
-       int hlen;
+       int hlen, error, clen;
 
        struct cryptodesc *crdc = NULL;
        struct cryptop *crp;
@@ -172,9 +172,7 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
        crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
        crp->crp_buf = (caddr_t)m;
-       crp->crp_callback = ipsec_input_cb;
        crp->crp_sid = tdb->tdb_cryptoid;
-       crp->crp_opaque = (caddr_t)tc;
 
        /* These are passed as-is to the callback */
        tc->tc_skip = skip;
@@ -184,7 +182,35 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        tc->tc_rdomain = tdb->tdb_rdomain;
        tc->tc_dst = tdb->tdb_dst;
 
-       crypto_dispatch(crp);
+       KERNEL_LOCK();
+       crypto_invoke(crp);
+       while (crp->crp_etype == EAGAIN) {
+               /* Reset the session ID */
+               if (tdb->tdb_cryptoid != 0)
+                       tdb->tdb_cryptoid = crp->crp_sid;
+               crypto_invoke(crp);
+       }
+       KERNEL_UNLOCK();
+       if (crp->crp_etype) {
+               DPRINTF("crypto error %d", crp->crp_etype);
+               ipsecstat_inc(ipsec_noxform);
+               free(tc, M_XDATA, 0);
+               m_freem(m);
+               crypto_freereq(crp);
+               return crp->crp_etype;
+       }
+
+       clen = crp->crp_olen;
+
+       /* Release the crypto descriptors */
+       crypto_freereq(crp);
+
+       error = ipcomp_input_cb(tdb, tc, m, clen);
+       if (error) {
+               ipsecstat_inc(ipsec_idrops);
+               tdb->tdb_idrops++;
+       }
+
        return 0;
 }
 
@@ -319,7 +345,7 @@ int
 ipcomp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 {
        const struct comp_algo *ipcompx = tdb->tdb_compalgxform;
-       int error, hlen;
+       int error, hlen, ilen, olen;
        struct cryptodesc *crdc = NULL;
        struct cryptop *crp = NULL;
        struct tdb_crypto *tc;
@@ -474,11 +500,38 @@ ipcomp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
        crp->crp_ilen = m->m_pkthdr.len;        /* Total input length */
        crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
        crp->crp_buf = (caddr_t)m;
-       crp->crp_callback = ipsec_output_cb;
-       crp->crp_opaque = (caddr_t)tc;
        crp->crp_sid = tdb->tdb_cryptoid;
 
-       crypto_dispatch(crp);
+       KERNEL_LOCK();
+       crypto_invoke(crp);
+       while (crp->crp_etype == EAGAIN) {
+               /* Reset the session ID */
+               if (tdb->tdb_cryptoid != 0)
+                       tdb->tdb_cryptoid = crp->crp_sid;
+               crypto_invoke(crp);
+       }
+       KERNEL_UNLOCK();
+       if (crp->crp_etype) {
+               DPRINTF("crypto error %d", crp->crp_etype);
+               ipsecstat_inc(ipsec_noxform);
+               free(tc, M_XDATA, 0);
+               m_freem(m);
+               crypto_freereq(crp);
+               return crp->crp_etype;
+       }
+
+       ilen = crp->crp_ilen;
+       olen = crp->crp_olen;
+
+       /* Release the crypto descriptors */
+       crypto_freereq(crp);
+
+       error = ipcomp_output_cb(tdb, tc, m, crp->crp_ilen, crp->crp_olen);
+       if (error) {
+               ipsecstat_inc(ipsec_odrops);
+               tdb->tdb_odrops++;
+       }
+
        return 0;
 
  drop:

diff --git a/sys/netinet/ip_ipsp.h b/sys/netinet/ip_ipsp.h
index 9bb3946..55452ca 100644 (file)
--- a/sys/netinet/ip_ipsp.h
+++ b/sys/netinet/ip_ipsp.h
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ip_ipsp.h,v 1.210 2021/10/13 14:36:31 bluhm Exp $     */
+/*     $OpenBSD: ip_ipsp.h,v 1.211 2021/10/23 15:42:35 tobhe Exp $     */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
@@ -648,8 +648,6 @@ void        ipsp_init(void);
 void   ipsec_init(void);
 int    ipsec_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int    ipsec_common_input(struct mbuf *, int, int, int, int, int);
-void   ipsec_input_cb(struct cryptop *);
-void   ipsec_output_cb(struct cryptop *);
 int    ipsec_common_input_cb(struct mbuf *, struct tdb *, int, int);
 int    ipsec_delete_policy(struct ipsec_policy *);
 ssize_t        ipsec_hdrsz(struct tdb *);

diff --git a/sys/netinet/ipsec_input.c b/sys/netinet/ipsec_input.c
index 2a9a378..12b8450 100644 (file)
--- a/sys/netinet/ipsec_input.c
+++ b/sys/netinet/ipsec_input.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ipsec_input.c,v 1.185 2021/10/22 15:44:20 bluhm Exp $ */
+/*     $OpenBSD: ipsec_input.c,v 1.186 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -366,78 +366,6 @@ ipsec_common_input(struct mbuf *m, int skip, int protoff, int af, int sproto,
        return error;
 }
 
-void
-ipsec_input_cb(struct cryptop *crp)
-{
-       struct tdb_crypto *tc = (struct tdb_crypto *) crp->crp_opaque;
-       struct mbuf *m = (struct mbuf *) crp->crp_buf;
-       struct tdb *tdb = NULL;
-       int clen, error;
-
-       NET_ASSERT_LOCKED();
-
-       if (m == NULL) {
-               DPRINTF("bogus returned buffer from crypto");
-               ipsecstat_inc(ipsec_crypto);
-               goto drop;
-       }
-
-       tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
-       if (tdb == NULL) {
-               DPRINTF("TDB is expired while in crypto");
-               ipsecstat_inc(ipsec_notdb);
-               goto drop;
-       }
-
-       /* Check for crypto errors */
-       if (crp->crp_etype) {
-               if (crp->crp_etype == EAGAIN) {
-                       /* Reset the session ID */
-                       if (tdb->tdb_cryptoid != 0)
-                               tdb->tdb_cryptoid = crp->crp_sid;
-                       crypto_dispatch(crp);
-                       return;
-               }
-               DPRINTF("crypto error %d", crp->crp_etype);
-               ipsecstat_inc(ipsec_noxform);
-               goto drop;
-       }
-
-       /* Length of data after processing */
-       clen = crp->crp_olen;
-
-       /* Release the crypto descriptors */
-       crypto_freereq(crp);
-
-       switch (tdb->tdb_sproto) {
-       case IPPROTO_ESP:
-               error = esp_input_cb(tdb, tc, m, clen);
-               break;
-       case IPPROTO_AH:
-               error = ah_input_cb(tdb, tc, m, clen);
-               break;
-       case IPPROTO_IPCOMP:
-               error = ipcomp_input_cb(tdb, tc, m, clen);
-               break;
-       default:
-               panic("%s: unknown/unsupported security protocol %d",
-                   __func__, tdb->tdb_sproto);
-       }
-       if (error) {
-               ipsecstat_inc(ipsec_idrops);
-               tdb->tdb_idrops++;
-       }
-       return;
-
- drop:
-       m_freem(m);
-       free(tc, M_XDATA, 0);
-       crypto_freereq(crp);
-       ipsecstat_inc(ipsec_idrops);
-       if (tdb != NULL)
-               tdb->tdb_idrops++;
-}
-
 /*
  * IPsec input callback, called by the transform callback. Takes care of
  * filtering and other sanity checks on the processed packet.

diff --git a/sys/netinet/ipsec_output.c b/sys/netinet/ipsec_output.c
index ed11171..43b94b8 100644 (file)
--- a/sys/netinet/ipsec_output.c
+++ b/sys/netinet/ipsec_output.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: ipsec_output.c,v 1.90 2021/10/22 15:44:20 bluhm Exp $ */
+/*     $OpenBSD: ipsec_output.c,v 1.91 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -377,81 +377,6 @@ ipsp_process_packet(struct mbuf *m, struct tdb *tdb, int af, int tunalready)
        return error;
 }
 
-/*
- * IPsec output callback, called directly by the crypto driver.
- */
-void
-ipsec_output_cb(struct cryptop *crp)
-{
-       struct tdb_crypto *tc = (struct tdb_crypto *) crp->crp_opaque;
-       struct mbuf *m = (struct mbuf *) crp->crp_buf;
-       struct tdb *tdb = NULL;
-       int error, ilen, olen;
-
-       NET_ASSERT_LOCKED();
-
-       if (m == NULL) {
-               DPRINTF("bogus returned buffer from crypto");
-               ipsecstat_inc(ipsec_crypto);
-               goto drop;
-       }
-
-       tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
-       if (tdb == NULL) {
-               DPRINTF("TDB is expired while in crypto");
-               ipsecstat_inc(ipsec_notdb);
-               goto drop;
-       }
-
-       /* Check for crypto errors. */
-       if (crp->crp_etype) {
-               if (crp->crp_etype == EAGAIN) {
-                       /* Reset the session ID */
-                       if (tdb->tdb_cryptoid != 0)
-                               tdb->tdb_cryptoid = crp->crp_sid;
-                       crypto_dispatch(crp);
-                       return;
-               }
-               DPRINTF("crypto error %d", crp->crp_etype);
-               ipsecstat_inc(ipsec_noxform);
-               goto drop;
-       }
-
-       olen = crp->crp_olen;
-       ilen = crp->crp_ilen;
-
-       /* Release crypto descriptors. */
-       crypto_freereq(crp);
-
-       switch (tdb->tdb_sproto) {
-       case IPPROTO_ESP:
-               error = esp_output_cb(tdb, tc, m, ilen, olen);
-               break;
-       case IPPROTO_AH:
-               error = ah_output_cb(tdb, tc, m, ilen, olen);
-               break;
-       case IPPROTO_IPCOMP:
-               error = ipcomp_output_cb(tdb, tc, m, ilen, olen);
-               break;
-       default:
-               panic("%s: unhandled security protocol %d",
-                   __func__, tdb->tdb_sproto);
-       }
-       if (error) {
-               ipsecstat_inc(ipsec_odrops);
-               tdb->tdb_odrops++;
-       }
-       return;
-
- drop:
-       m_freem(m);
-       free(tc, M_XDATA, 0);
-       crypto_freereq(crp);
-       ipsecstat_inc(ipsec_odrops);
-       if (tdb != NULL)
-               tdb->tdb_odrops++;
-}
-
 /*
  * Called by the IPsec output transform callbacks, to transmit the packet
  * or do further processing, as necessary.