use an intrmap when establishing interrupts for queues.

mcx is still hardcoded/limited to 1 queue for now, but this lets
different mcx devices use different cpus for handling packets.

looks good jmatthew@

diff --git a/sys/dev/pci/if_mcx.c b/sys/dev/pci/if_mcx.c
index 4624a2f..55d55d7 100644 (file)
--- a/sys/dev/pci/if_mcx.c
+++ b/sys/dev/pci/if_mcx.c
@@ -1,4 +1,4 @@
-/*     $OpenBSD: if_mcx.c,v 1.94 2021/01/20 10:04:26 jmatthew Exp $ */
+/*     $OpenBSD: if_mcx.c,v 1.95 2021/01/25 01:45:55 dlg Exp $ */
 
 /*
  * Copyright (c) 2017 David Gwynne <dlg@openbsd.org>
@@ -34,6 +34,7 @@
 #include <sys/task.h>
 #include <sys/atomic.h>
 #include <sys/timetc.h>
+#include <sys/intrmap.h>
 
 #include <machine/bus.h>
 #include <machine/intr.h>
@@ -2477,8 +2478,8 @@ struct mcx_softc {
        uint32_t                 sc_mhz;
        uint32_t                 sc_khz;
 
+       struct intrmap          *sc_intrmap;
        struct mcx_queues        sc_queues[MCX_MAX_QUEUES];
-       unsigned int             sc_nqueues;
 
        int                      sc_mcam_reg;
 
@@ -2878,7 +2879,12 @@ mcx_attach(struct device *parent, struct device *self, void *aux)
            ether_sprintf(sc->sc_ac.ac_enaddr));
 
        msix = pci_intr_msix_count(pa->pa_pc, pa->pa_tag);
-       sc->sc_nqueues = 1;
+       sc->sc_intrmap = intrmap_create(&sc->sc_dev, msix, MCX_MAX_QUEUES,
+           INTRMAP_POWEROF2);
+       if (sc->sc_intrmap == NULL) {
+               printf("%s: unable to create interrupt map\n", DEVNAME(sc));
+               goto teardown;
+       }
 
        strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
        ifp->if_softc = sc;
@@ -2905,9 +2911,9 @@ mcx_attach(struct device *parent, struct device *self, void *aux)
        if_attach(ifp);
        ether_ifattach(ifp);
 
-       if_attach_iqueues(ifp, sc->sc_nqueues);
-       if_attach_queues(ifp, sc->sc_nqueues);
-       for (i = 0; i < sc->sc_nqueues; i++) {
+       if_attach_iqueues(ifp, intrmap_count(sc->sc_intrmap));
+       if_attach_queues(ifp, intrmap_count(sc->sc_intrmap));
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
                struct ifiqueue *ifiq = ifp->if_iqs[i];
                struct ifqueue *ifq = ifp->if_ifqs[i];
                struct mcx_queues *q = &sc->sc_queues[i];
@@ -2923,13 +2929,13 @@ mcx_attach(struct device *parent, struct device *self, void *aux)
                if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
                        printf("%s: unable to alloc uar %d\n",
                            DEVNAME(sc), i);
-                       goto teardown;
+                       goto intrdisestablish;
                }
 
                if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
                        printf("%s: unable to create event queue %d\n",
                            DEVNAME(sc), i);
-                       goto teardown;
+                       goto intrdisestablish;
                }
 
                rx->rx_softc = sc;
@@ -2944,12 +2950,18 @@ mcx_attach(struct device *parent, struct device *self, void *aux)
                if (pci_intr_map_msix(pa, vec, &ih) != 0) {
                        printf("%s: unable to map queue interrupt %d\n",
                            DEVNAME(sc), i);
-                       goto teardown;
+                       goto intrdisestablish;
                }
                snprintf(q->q_name, sizeof(q->q_name), "%s:%d",
                    DEVNAME(sc), i);
-               q->q_ihc = pci_intr_establish(sc->sc_pc, ih,
-                   IPL_NET | IPL_MPSAFE, mcx_cq_intr, q, q->q_name);
+               q->q_ihc = pci_intr_establish_cpu(sc->sc_pc, ih,
+                   IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i),
+                   mcx_cq_intr, q, q->q_name);
+               if (q->q_ihc == NULL) {
+                       printf("%s: unable to establish interrupt %d\n",
+                           DEVNAME(sc), i);
+                       goto intrdisestablish;
+               }
        }
 
        timeout_set(&sc->sc_calibrate, mcx_calibrate, sc);
@@ -2976,6 +2988,16 @@ mcx_attach(struct device *parent, struct device *self, void *aux)
        mcx_timecounter_attach(sc);
        return;
 
+intrdisestablish:
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
+               struct mcx_queues *q = &sc->sc_queues[i];
+               if (q->q_ihc == NULL)
+                       continue;
+               pci_intr_disestablish(sc->sc_pc, q->q_ihc);
+               q->q_ihc = NULL;
+       }
+       intrmap_destroy(sc->sc_intrmap);
+       sc->sc_intrmap = NULL;
 teardown:
        mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
        /* error printed by mcx_teardown_hca, and we're already unwinding */
@@ -7156,7 +7178,7 @@ mcx_up(struct mcx_softc *sc)
        if (mcx_create_tis(sc, &sc->sc_tis) != 0)
                goto down;
 
-       for (i = 0; i < sc->sc_nqueues; i++) {
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
                if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
                        goto down;
                }
@@ -7264,10 +7286,11 @@ mcx_up(struct mcx_softc *sc)
         * since we also restrict the number of queues to a power of two,
         * we can just put each rx queue in once.
         */
-       for (i = 0; i < sc->sc_nqueues; i++)
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++)
                rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;
 
-       if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0)
+       if (mcx_create_rqt(sc, intrmap_count(sc->sc_intrmap), rqns,
+           &sc->sc_rqt) != 0)
                goto down;
 
        start = 0;
@@ -7297,7 +7320,7 @@ mcx_up(struct mcx_softc *sc)
                flow_index++;
        }
 
-       for (i = 0; i < sc->sc_nqueues; i++) {
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
                struct mcx_queues *q = &sc->sc_queues[i];
                rx = &q->q_rx;
                tx = &q->q_tx;
@@ -7371,7 +7394,7 @@ mcx_down(struct mcx_softc *sc)
                flow_index++;
        }
        intr_barrier(sc->sc_ihc);
-       for (i = 0; i < sc->sc_nqueues; i++) {
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
                struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq;
                ifq_barrier(ifq);
 
@@ -7400,7 +7423,7 @@ mcx_down(struct mcx_softc *sc)
                sc->sc_rqt = -1;
        }
 
-       for (i = 0; i < sc->sc_nqueues; i++) {
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
                struct mcx_queues *q = &sc->sc_queues[i];
                struct mcx_rx *rx = &q->q_rx;
                struct mcx_tx *tx = &q->q_tx;
@@ -7607,12 +7630,12 @@ mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri)
        unsigned int i;
        int error;
 
-       ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), M_TEMP,
-           M_WAITOK|M_ZERO|M_CANFAIL);
+       ifrs = mallocarray(intrmap_count(sc->sc_intrmap), sizeof(*ifrs),
+           M_TEMP, M_WAITOK|M_ZERO|M_CANFAIL);
        if (ifrs == NULL)
                return (ENOMEM);
 
-       for (i = 0; i < sc->sc_nqueues; i++) {
+       for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) {
                struct mcx_rx *rx = &sc->sc_queues[i].q_rx;
                struct if_rxring_info *ifr = &ifrs[i];
 
@@ -8564,7 +8587,7 @@ mcx_kstat_attach_queues(struct mcx_softc *sc)
        struct kstat_kv *kvs;
        int q, i;
 
-       for (q = 0; q < sc->sc_nqueues; q++) {
+       for (q = 0; q < intrmap_count(sc->sc_intrmap); q++) {
                ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
                    KSTAT_T_KV, 0);
                if (ks == NULL) {