From 5df6f9650b0f9fc05757fef099a582bd245d48c0 Mon Sep 17 00:00:00 2001 From: dlg Date: Mon, 25 Jan 2021 01:45:55 +0000 Subject: [PATCH] use an intrmap when establishing interrupts for queues. mcx is still hardcoded/limited to 1 queue for now, but this lets different mcx devices use different cpus for handling packets. looks good jmatthew@ --- sys/dev/pci/if_mcx.c | 65 ++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/sys/dev/pci/if_mcx.c b/sys/dev/pci/if_mcx.c index 4624a2f6520..55d55d78c07 100644 --- a/sys/dev/pci/if_mcx.c +++ b/sys/dev/pci/if_mcx.c @@ -1,4 +1,4 @@ -/* $OpenBSD: if_mcx.c,v 1.94 2021/01/20 10:04:26 jmatthew Exp $ */ +/* $OpenBSD: if_mcx.c,v 1.95 2021/01/25 01:45:55 dlg Exp $ */ /* * Copyright (c) 2017 David Gwynne @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -2477,8 +2478,8 @@ struct mcx_softc { uint32_t sc_mhz; uint32_t sc_khz; + struct intrmap *sc_intrmap; struct mcx_queues sc_queues[MCX_MAX_QUEUES]; - unsigned int sc_nqueues; int sc_mcam_reg; @@ -2878,7 +2879,12 @@ mcx_attach(struct device *parent, struct device *self, void *aux) ether_sprintf(sc->sc_ac.ac_enaddr)); msix = pci_intr_msix_count(pa->pa_pc, pa->pa_tag); - sc->sc_nqueues = 1; + sc->sc_intrmap = intrmap_create(&sc->sc_dev, msix, MCX_MAX_QUEUES, + INTRMAP_POWEROF2); + if (sc->sc_intrmap == NULL) { + printf("%s: unable to create interrupt map\n", DEVNAME(sc)); + goto teardown; + } strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); ifp->if_softc = sc; @@ -2905,9 +2911,9 @@ mcx_attach(struct device *parent, struct device *self, void *aux) if_attach(ifp); ether_ifattach(ifp); - if_attach_iqueues(ifp, sc->sc_nqueues); - if_attach_queues(ifp, sc->sc_nqueues); - for (i = 0; i < sc->sc_nqueues; i++) { + if_attach_iqueues(ifp, intrmap_count(sc->sc_intrmap)); + if_attach_queues(ifp, intrmap_count(sc->sc_intrmap)); + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { struct ifiqueue *ifiq = ifp->if_iqs[i]; struct ifqueue *ifq = ifp->if_ifqs[i]; struct mcx_queues *q = &sc->sc_queues[i]; @@ -2923,13 +2929,13 @@ mcx_attach(struct device *parent, struct device *self, void *aux) if (mcx_alloc_uar(sc, &q->q_uar) != 0) { printf("%s: unable to alloc uar %d\n", DEVNAME(sc), i); - goto teardown; + goto intrdisestablish; } if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) { printf("%s: unable to create event queue %d\n", DEVNAME(sc), i); - goto teardown; + goto intrdisestablish; } rx->rx_softc = sc; @@ -2944,12 +2950,18 @@ mcx_attach(struct device *parent, struct device *self, void *aux) if (pci_intr_map_msix(pa, vec, &ih) != 0) { printf("%s: unable to map queue interrupt %d\n", DEVNAME(sc), i); - goto teardown; + goto intrdisestablish; } snprintf(q->q_name, sizeof(q->q_name), "%s:%d", DEVNAME(sc), i); - q->q_ihc = pci_intr_establish(sc->sc_pc, ih, - IPL_NET | IPL_MPSAFE, mcx_cq_intr, q, q->q_name); + q->q_ihc = pci_intr_establish_cpu(sc->sc_pc, ih, + IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), + mcx_cq_intr, q, q->q_name); + if (q->q_ihc == NULL) { + printf("%s: unable to establish interrupt %d\n", + DEVNAME(sc), i); + goto intrdisestablish; + } } timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); @@ -2976,6 +2988,16 @@ mcx_attach(struct device *parent, struct device *self, void *aux) mcx_timecounter_attach(sc); return; +intrdisestablish: + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { + struct mcx_queues *q = &sc->sc_queues[i]; + if (q->q_ihc == NULL) + continue; + pci_intr_disestablish(sc->sc_pc, q->q_ihc); + q->q_ihc = NULL; + } + intrmap_destroy(sc->sc_intrmap); + sc->sc_intrmap = NULL; teardown: mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); /* error printed by mcx_teardown_hca, and we're already unwinding */ @@ -7156,7 +7178,7 @@ mcx_up(struct mcx_softc *sc) if (mcx_create_tis(sc, &sc->sc_tis) != 0) goto down; - for (i = 0; i < sc->sc_nqueues; i++) { + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) { goto down; } @@ -7264,10 +7286,11 @@ mcx_up(struct mcx_softc *sc) * since we also restrict the number of queues to a power of two, * we can just put each rx queue in once. */ - for (i = 0; i < sc->sc_nqueues; i++) + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) rqns[i] = sc->sc_queues[i].q_rx.rx_rqn; - if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0) + if (mcx_create_rqt(sc, intrmap_count(sc->sc_intrmap), rqns, + &sc->sc_rqt) != 0) goto down; start = 0; @@ -7297,7 +7320,7 @@ mcx_up(struct mcx_softc *sc) flow_index++; } - for (i = 0; i < sc->sc_nqueues; i++) { + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { struct mcx_queues *q = &sc->sc_queues[i]; rx = &q->q_rx; tx = &q->q_tx; @@ -7371,7 +7394,7 @@ mcx_down(struct mcx_softc *sc) flow_index++; } intr_barrier(sc->sc_ihc); - for (i = 0; i < sc->sc_nqueues; i++) { + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq; ifq_barrier(ifq); @@ -7400,7 +7423,7 @@ mcx_down(struct mcx_softc *sc) sc->sc_rqt = -1; } - for (i = 0; i < sc->sc_nqueues; i++) { + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { struct mcx_queues *q = &sc->sc_queues[i]; struct mcx_rx *rx = &q->q_rx; struct mcx_tx *tx = &q->q_tx; @@ -7607,12 +7630,12 @@ mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri) unsigned int i; int error; - ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), M_TEMP, - M_WAITOK|M_ZERO|M_CANFAIL); + ifrs = mallocarray(intrmap_count(sc->sc_intrmap), sizeof(*ifrs), + M_TEMP, M_WAITOK|M_ZERO|M_CANFAIL); if (ifrs == NULL) return (ENOMEM); - for (i = 0; i < sc->sc_nqueues; i++) { + for (i = 0; i < intrmap_count(sc->sc_intrmap); i++) { struct mcx_rx *rx = &sc->sc_queues[i].q_rx; struct if_rxring_info *ifr = &ifrs[i]; @@ -8564,7 +8587,7 @@ mcx_kstat_attach_queues(struct mcx_softc *sc) struct kstat_kv *kvs; int q, i; - for (q = 0; q < sc->sc_nqueues; q++) { + for (q = 0; q < intrmap_count(sc->sc_intrmap); q++) { ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q, KSTAT_T_KV, 0); if (ks == NULL) { -- 2.20.1