diff --git a/src/libs/compat/freebsd_iflib/compat/net/iflib.h b/src/libs/compat/freebsd_iflib/compat/net/iflib.h index c73c458fce..4536ab6359 100644 --- a/src/libs/compat/freebsd_iflib/compat/net/iflib.h +++ b/src/libs/compat/freebsd_iflib/compat/net/iflib.h @@ -48,7 +48,7 @@ typedef uint16_t qidx_t; struct iflib_ctx; typedef struct iflib_ctx *if_ctx_t; struct if_shared_ctx; -typedef struct if_shared_ctx *if_shared_ctx_t; +typedef const struct if_shared_ctx *if_shared_ctx_t; struct if_int_delay_info; typedef struct if_int_delay_info *if_int_delay_info_t; struct if_pseudo; @@ -94,7 +94,6 @@ typedef struct if_rxd_info { typedef struct if_rxd_update { uint64_t *iru_paddrs; - caddr_t *iru_vaddrs; qidx_t *iru_idxs; qidx_t iru_pidx; uint16_t iru_qsidx; @@ -167,13 +166,13 @@ typedef struct pci_vendor_info { uint32_t pvi_class_mask; const char *pvi_name; } pci_vendor_info_t; - #define PVID(vendor, devid, name) {vendor, devid, 0, 0, 0, 0, name} #define PVID_OEM(vendor, devid, svid, sdevid, revid, name) {vendor, devid, svid, sdevid, revid, 0, name} #define PVID_END {0, 0, 0, 0, 0, 0, NULL} -#define IFLIB_PNP_DESCR "U32:vendor;U32:device;U32:subvendor;U32:subdevice;" \ - "U32:revision;U32:class;D:#" +/* No drivers in tree currently match on anything except vendor:device. */ +#define IFLIB_PNP_DESCR "U32:vendor;U32:device;U32:#;U32:#;" \ + "U32:#;U32:#;D:#" #define IFLIB_PNP_INFO(b, u, t) \ MODULE_PNP_INFO(IFLIB_PNP_DESCR, b, u, t, nitems(t) - 1) @@ -219,6 +218,9 @@ typedef struct if_softc_ctx { uint32_t __spare2__; iflib_intr_mode_t isc_intr; + uint16_t isc_rxd_buf_size[8]; /* set at init time by driver, 0 + means use iflib-calculated size + based on isc_max_frame_size */ uint16_t isc_max_frame_size; /* set at init time by driver */ uint16_t isc_min_frame_size; /* set at init time by driver, only used if IFLIB_NEED_ETHER_PAD is set. */ @@ -285,15 +287,32 @@ typedef struct iflib_dma_info { #define IFLIB_MAGIC 0xCAFEF00D typedef enum { + /* Interrupt or softirq handles only receive */ IFLIB_INTR_RX, + + /* Interrupt or softirq handles only transmit */ IFLIB_INTR_TX, + + /* + * Interrupt will check for both pending receive + * and available tx credits and dispatch a task + * for one or both depending on the disposition + * of the respective queues. + */ IFLIB_INTR_RXTX, + + /* + * Other interrupt - typically link status and + * or error conditions. + */ IFLIB_INTR_ADMIN, + + /* Softirq (task) for iov handling */ IFLIB_INTR_IOV, } iflib_intr_type_t; /* - * Interface has a separate command queue for RX + * Interface has a separate completion queue for RX */ #define IFLIB_HAS_RXCQ 0x01 /* @@ -305,7 +324,7 @@ typedef enum { */ #define IFLIB_IS_VF 0x04 /* - * Interface has a separate command queue for TX + * Interface has a separate completion queue for TX */ #define IFLIB_HAS_TXCQ 0x08 /* @@ -365,6 +384,25 @@ typedef enum { * Driver will pass the media */ #define IFLIB_DRIVER_MEDIA 0x20000 +/* + * When using a single hardware interrupt for the interface, only process RX + * interrupts instead of doing combined RX/TX processing. + */ +#define IFLIB_SINGLE_IRQ_RX_ONLY 0x40000 +/* + * Don't need/want most of the niceties of + * emulating ethernet + */ +#define IFLIB_PSEUDO_ETHER 0x80000 + +/* + * These enum values are used in iflib_needs_restart to indicate to iflib + * functions whether or not the interface needs restarting when certain events + * happen. + */ +enum iflib_restart_event { + IFLIB_RESTART_VLAN_CONFIG, +}; /* * field accessors diff --git a/src/libs/compat/freebsd_iflib/iflib.c b/src/libs/compat/freebsd_iflib/iflib.c index ef52deb9ad..8fd077e563 100644 --- a/src/libs/compat/freebsd_iflib/iflib.c +++ b/src/libs/compat/freebsd_iflib/iflib.c @@ -50,7 +50,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include #include @@ -69,6 +68,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #ifndef __HAIKU__ @@ -81,7 +81,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #ifndef __HAIKU__ #include #endif @@ -112,6 +111,7 @@ __FBSDID("$FreeBSD$"); #endif #include + /* * enable accounting of every mbuf as it comes in to and goes out of * iflib's software descriptor references @@ -133,7 +133,6 @@ __FBSDID("$FreeBSD$"); * access. */ - /* * File organization: * - private structures @@ -146,6 +145,9 @@ __FBSDID("$FreeBSD$"); */ MALLOC_DEFINE(M_IFLIB, "iflib", "ifnet library"); +#define IFLIB_RXEOF_MORE (1U << 0) +#define IFLIB_RXEOF_EMPTY (2U << 0) + struct iflib_txq; typedef struct iflib_txq *iflib_txq_t; struct iflib_rxq; @@ -157,6 +159,7 @@ struct iflib_ctx; static void iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid); static void iflib_timer(void *arg); +static void iflib_tqg_detach(if_ctx_t ctx); typedef struct iflib_filter_info { driver_filter_t *ifi_filter; @@ -211,6 +214,8 @@ struct iflib_ctx { uint16_t ifc_sysctl_core_offset; #define CORE_OFFSET_UNSPECIFIED 0xffff uint8_t ifc_sysctl_separate_txrx; + uint8_t ifc_sysctl_use_logical_cores; + bool ifc_cpus_are_physical_cores; qidx_t ifc_sysctl_ntxds[8]; qidx_t ifc_sysctl_nrxds[8]; @@ -222,8 +227,6 @@ struct iflib_ctx { #define isc_rxd_pkt_get ifc_txrx.ift_rxd_pkt_get #define isc_rxd_refill ifc_txrx.ift_rxd_refill #define isc_rxd_flush ifc_txrx.ift_rxd_flush -#define isc_rxd_refill ifc_txrx.ift_rxd_refill -#define isc_rxd_refill ifc_txrx.ift_rxd_refill #define isc_legacy_intr ifc_txrx.ift_legacy_intr eventhandler_tag ifc_vlan_attach_event; eventhandler_tag ifc_vlan_detach_event; @@ -365,6 +368,9 @@ struct iflib_txq { qidx_t ift_size; uint16_t ift_id; struct callout ift_timer; +#ifdef DEV_NETMAP + struct callout ift_netmap_timer; +#endif /* DEV_NETMAP */ if_txsd_vec_t ift_sds; uint8_t ift_qstatus; @@ -374,7 +380,7 @@ struct iflib_txq { bus_dma_tag_t ift_buf_tag; bus_dma_tag_t ift_tso_buf_tag; iflib_dma_info_t ift_ifdi; -#define MTX_NAME_LEN 16 +#define MTX_NAME_LEN 32 char ift_mtx_name[MTX_NAME_LEN]; bus_dma_segment_t ift_segs[IFLIB_MAX_TX_SEGS] __aligned(CACHE_LINE_SIZE); #ifdef IFLIB_DIAGNOSTICS @@ -410,8 +416,7 @@ struct iflib_fl { bus_dma_tag_t ifl_buf_tag; iflib_dma_info_t ifl_ifdi; uint64_t ifl_bus_addrs[IFLIB_MAX_RX_REFRESH] __aligned(CACHE_LINE_SIZE); - caddr_t ifl_vm_addrs[IFLIB_MAX_RX_REFRESH]; - qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH]; + qidx_t ifl_rxd_idxs[IFLIB_MAX_RX_REFRESH]; } __aligned(CACHE_LINE_SIZE); static inline qidx_t @@ -449,7 +454,7 @@ struct iflib_rxq { #endif /* * If there is a separate completion queue (IFLIB_HAS_RXCQ), this is - * the command queue consumer index. Otherwise it's unused. + * the completion queue consumer index. Otherwise it's unused. */ qidx_t ifr_cq_cidx; uint16_t ifr_id; @@ -461,6 +466,7 @@ struct iflib_rxq { struct lro_ctrl ifr_lc; #endif struct grouptask ifr_task; + struct callout ifr_watchdog; struct iflib_filter_info ifr_filter_info; iflib_dma_info_t ifr_ifdi; @@ -474,7 +480,6 @@ struct iflib_rxq { typedef struct if_rxsd { caddr_t *ifsd_cl; iflib_fl_t ifsd_fl; - qidx_t ifsd_cidx; } *if_rxsd_t; /* multiple of word size */ @@ -500,7 +505,6 @@ typedef struct if_rxd_info_pad { CTASSERT(sizeof(struct if_pkt_info_pad) == sizeof(struct if_pkt_info)); CTASSERT(sizeof(struct if_rxd_info_pad) == sizeof(struct if_rxd_info)); - static inline void pkt_info_zero(if_pkt_info_t pi) { @@ -512,7 +516,7 @@ pkt_info_zero(if_pkt_info_t pi) #ifndef __LP64__ pi_pad->pkt_val[6] = 0; pi_pad->pkt_val[7] = 0; pi_pad->pkt_val[8] = 0; pi_pad->pkt_val[9] = 0; pi_pad->pkt_val[10] = 0; -#endif +#endif } #ifndef __HAIKU__ @@ -602,11 +606,11 @@ TASKQGROUP_DEFINE(if_config_tqg, 1, 1); #endif /* !INVARIANTS */ #endif -static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD, 0, - "iflib driver parameters"); +static SYSCTL_NODE(_net, OID_AUTO, iflib, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "iflib driver parameters"); /* - * XXX need to ensure that this can't accidentally cause the head to be moved backwards + * XXX need to ensure that this can't accidentally cause the head to be moved backwards */ static int iflib_min_tx_latency = 0; SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW, @@ -614,6 +618,9 @@ SYSCTL_INT(_net_iflib, OID_AUTO, min_tx_latency, CTLFLAG_RW, static int iflib_no_tx_batch = 0; SYSCTL_INT(_net_iflib, OID_AUTO, no_tx_batch, CTLFLAG_RW, &iflib_no_tx_batch, 0, "minimize transmit latency at the possible expense of throughput"); +static int iflib_timer_default = 1000; +SYSCTL_INT(_net_iflib, OID_AUTO, timer_default, CTLFLAG_RW, + &iflib_timer_default, 0, "number of ticks between iflib_timer calls"); #if IFLIB_DEBUG_COUNTERS @@ -641,7 +648,6 @@ SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills, CTLFLAG_RD, SYSCTL_INT(_net_iflib, OID_AUTO, fl_refills_large, CTLFLAG_RD, &iflib_fl_refills_large, 0, "# large refills"); - static int iflib_txq_drain_flushing; static int iflib_txq_drain_oactive; static int iflib_txq_drain_notready; @@ -653,7 +659,6 @@ SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_oactive, CTLFLAG_RD, SYSCTL_INT(_net_iflib, OID_AUTO, txq_drain_notready, CTLFLAG_RD, &iflib_txq_drain_notready, 0, "# drain notready"); - static int iflib_encap_load_mbuf_fail; static int iflib_encap_pad_mbuf_fail; static int iflib_encap_txq_avail_fail; @@ -734,6 +739,8 @@ static int iflib_altq_if_transmit(if_t ifp, struct mbuf *m); #endif static int iflib_register(if_ctx_t); static void iflib_deregister(if_ctx_t); +static void iflib_unregister_vlan_handlers(if_ctx_t ctx); +static uint16_t iflib_get_mbuf_size_for(unsigned int size); static void iflib_init_locked(if_ctx_t ctx); static void iflib_add_device_sysctl_pre(if_ctx_t ctx); static void iflib_add_device_sysctl_post(if_ctx_t ctx); @@ -752,14 +759,34 @@ struct cpu_offset { SLIST_ENTRY(cpu_offset) entries; cpuset_t set; unsigned int refcount; - uint16_t offset; + uint16_t next_cpuid; }; static struct mtx cpu_offset_mtx; MTX_SYSINIT(iflib_cpu_offset, &cpu_offset_mtx, "iflib_cpu_offset lock", MTX_DEF); #endif -NETDUMP_DEFINE(iflib); +DEBUGNET_DEFINE(iflib); + +static int +iflib_num_rx_descs(if_ctx_t ctx) +{ + if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; + if_shared_ctx_t sctx = ctx->ifc_sctx; + uint16_t first_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; + + return scctx->isc_nrxd[first_rxq]; +} + +static int +iflib_num_tx_descs(if_ctx_t ctx) +{ + if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; + if_shared_ctx_t sctx = ctx->ifc_sctx; + uint16_t first_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; + + return scctx->isc_ntxd[first_txq]; +} #ifdef DEV_NETMAP #include @@ -768,7 +795,8 @@ NETDUMP_DEFINE(iflib); MODULE_DEPEND(iflib, netmap, 1, 1, 1); -static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init); +static int netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init); +static void iflib_netmap_timer(void *arg); /* * device-specific sysctl variables: @@ -808,21 +836,25 @@ iflib_netmap_register(struct netmap_adapter *na, int onoff) int status; CTX_LOCK(ctx); - IFDI_INTR_DISABLE(ctx); - - /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); - if (!CTX_IS_VF(ctx)) IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); - /* enable or disable flags and callbacks in na and ifp */ + iflib_stop(ctx); + + /* + * Enable (or disable) netmap flags, and intercept (or restore) + * ifp->if_transmit. This is done once the device has been stopped + * to prevent race conditions. Also, this must be done after + * calling netmap_disable_all_rings() and before calling + * netmap_enable_all_rings(), so that these two functions see the + * updated state of the NAF_NETMAP_ON bit. + */ if (onoff) { nm_set_native_flags(na); } else { nm_clear_native_flags(na); } - iflib_stop(ctx); + iflib_init_locked(ctx); IFDI_CRCSTRIP_SET(ctx, onoff, iflib_crcstrip); // XXX why twice ? status = ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1; @@ -833,50 +865,99 @@ iflib_netmap_register(struct netmap_adapter *na, int onoff) } static int -netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, bool init) +iflib_netmap_config(struct netmap_adapter *na, struct nm_config_info *info) +{ + if_t ifp = na->ifp; + if_ctx_t ctx = ifp->if_softc; + iflib_rxq_t rxq = &ctx->ifc_rxqs[0]; + iflib_fl_t fl = &rxq->ifr_fl[0]; + + info->num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets; + info->num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets; + info->num_tx_descs = iflib_num_tx_descs(ctx); + info->num_rx_descs = iflib_num_rx_descs(ctx); + info->rx_buf_maxsize = fl->ifl_buf_size; + nm_prinf("txr %u rxr %u txd %u rxd %u rbufsz %u", + info->num_tx_rings, info->num_rx_rings, info->num_tx_descs, + info->num_rx_descs, info->rx_buf_maxsize); + + return 0; +} + +static int +netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, bool init) { struct netmap_adapter *na = kring->na; u_int const lim = kring->nkr_num_slots - 1; - u_int head = kring->rhead; struct netmap_ring *ring = kring->ring; bus_dmamap_t *map; struct if_rxd_update iru; if_ctx_t ctx = rxq->ifr_ctx; iflib_fl_t fl = &rxq->ifr_fl[0]; - uint32_t refill_pidx, nic_i; + u_int nic_i_first, nic_i; + u_int nm_i; + int i, n; #if IFLIB_DEBUG_COUNTERS int rf_count = 0; #endif - if (nm_i == head && __predict_true(!init)) - return 0; + /* + * This function is used both at initialization and in rxsync. + * At initialization we need to prepare (with isc_rxd_refill()) + * all the netmap buffers currently owned by the kernel, in + * such a way to keep fl->ifl_pidx and kring->nr_hwcur in sync + * (except for kring->nkr_hwofs). These may be less than + * kring->nkr_num_slots if netmap_reset() was called while + * an application using the kring that still owned some + * buffers. + * At rxsync time, both indexes point to the next buffer to be + * refilled. + * In any case we publish (with isc_rxd_flush()) up to + * (fl->ifl_pidx - 1) % N (included), to avoid the NIC tail/prod + * pointer to overrun the head/cons pointer, although this is + * not necessary for some NICs (e.g. vmx). + */ + if (__predict_false(init)) { + n = kring->nkr_num_slots - nm_kr_rxspace(kring); + } else { + n = kring->rhead - kring->nr_hwcur; + if (n == 0) + return (0); /* Nothing to do. */ + if (n < 0) + n += kring->nkr_num_slots; + } + iru_init(&iru, rxq, 0 /* flid */); map = fl->ifl_sds.ifsd_map; - refill_pidx = netmap_idx_k2n(kring, nm_i); - /* - * IMPORTANT: we must leave one free slot in the ring, - * so move head back by one unit - */ - head = nm_prev(head, lim); - nic_i = UINT_MAX; + nic_i = fl->ifl_pidx; + nm_i = netmap_idx_n2k(kring, nic_i); + if (__predict_false(init)) { + /* + * On init/reset, nic_i must be 0, and we must + * start to refill from hwtail (see netmap_reset()). + */ + MPASS(nic_i == 0); + MPASS(nm_i == kring->nr_hwtail); + } else + MPASS(nm_i == kring->nr_hwcur); DBG_COUNTER_INC(fl_refills); - while (nm_i != head) { + while (n > 0) { #if IFLIB_DEBUG_COUNTERS if (++rf_count == 9) DBG_COUNTER_INC(fl_refills_large); #endif - for (int tmp_pidx = 0; tmp_pidx < IFLIB_MAX_RX_REFRESH && nm_i != head; tmp_pidx++) { + nic_i_first = nic_i; + for (i = 0; n > 0 && i < IFLIB_MAX_RX_REFRESH; n--, i++) { struct netmap_slot *slot = &ring->slot[nm_i]; - void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[tmp_pidx]); - uint32_t nic_i_dma = refill_pidx; - nic_i = netmap_idx_k2n(kring, nm_i); + void *addr = PNMB(na, slot, &fl->ifl_bus_addrs[i]); - MPASS(tmp_pidx < IFLIB_MAX_RX_REFRESH); + MPASS(i < IFLIB_MAX_RX_REFRESH); if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ return netmap_ring_reinit(kring); - fl->ifl_vm_addrs[tmp_pidx] = addr; + fl->ifl_rxd_idxs[i] = nic_i; + if (__predict_false(init)) { netmap_load_map(na, fl->ifl_buf_tag, map[nic_i], addr); @@ -885,36 +966,37 @@ netmap_fl_refill(iflib_rxq_t rxq, struct netmap_kring *kring, uint32_t nm_i, boo netmap_reload_map(na, fl->ifl_buf_tag, map[nic_i], addr); } + bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i], + BUS_DMASYNC_PREREAD); slot->flags &= ~NS_BUF_CHANGED; nm_i = nm_next(nm_i, lim); - fl->ifl_rxd_idxs[tmp_pidx] = nic_i = nm_next(nic_i, lim); - if (nm_i != head && tmp_pidx < IFLIB_MAX_RX_REFRESH-1) - continue; - - iru.iru_pidx = refill_pidx; - iru.iru_count = tmp_pidx+1; - ctx->isc_rxd_refill(ctx->ifc_softc, &iru); - refill_pidx = nic_i; - for (int n = 0; n < iru.iru_count; n++) { - bus_dmamap_sync(fl->ifl_buf_tag, map[nic_i_dma], - BUS_DMASYNC_PREREAD); - /* XXX - change this to not use the netmap func*/ - nic_i_dma = nm_next(nic_i_dma, lim); - } + nic_i = nm_next(nic_i, lim); } + + iru.iru_pidx = nic_i_first; + iru.iru_count = i; + ctx->isc_rxd_refill(ctx->ifc_softc, &iru); } - kring->nr_hwcur = head; + fl->ifl_pidx = nic_i; + /* + * At the end of the loop we must have refilled everything + * we could possibly refill. + */ + MPASS(nm_i == kring->rhead); + kring->nr_hwcur = nm_i; bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - if (__predict_true(nic_i != UINT_MAX)) { - ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, nic_i); - DBG_COUNTER_INC(rxd_flush); - } + ctx->isc_rxd_flush(ctx->ifc_softc, rxq->ifr_id, fl->ifl_id, + nm_prev(nic_i, lim)); + DBG_COUNTER_INC(rxd_flush); + return (0); } +#define NETMAP_TX_TIMER_US 90 + /* * Reconcile kernel and user view of the transmit ring. * @@ -941,6 +1023,7 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags) u_int const lim = kring->nkr_num_slots - 1; u_int const head = kring->rhead; struct if_pkt_info pi; + int tx_pkts = 0, tx_bytes = 0; /* * interrupts on every tx packet are expensive so request @@ -978,6 +1061,8 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags) nm_i = kring->nr_hwcur; if (nm_i != head) { /* we have new packets to send */ + uint32_t pkt_len = 0, seg_idx = 0; + int nic_i_start = -1, flags = 0; pkt_info_zero(&pi); pi.ipi_segs = txq->ift_segs; pi.ipi_qsidx = kring->ring_id; @@ -992,22 +1077,44 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags) u_int len = slot->len; uint64_t paddr; void *addr = PNMB(na, slot, &paddr); - int flags = (slot->flags & NS_REPORT || + + flags |= (slot->flags & NS_REPORT || nic_i == 0 || nic_i == report_frequency) ? IPI_TX_INTR : 0; - /* device-specific */ - pi.ipi_len = len; - pi.ipi_segs[0].ds_addr = paddr; - pi.ipi_segs[0].ds_len = len; - pi.ipi_nsegs = 1; - pi.ipi_ndescs = 0; - pi.ipi_pidx = nic_i; - pi.ipi_flags = flags; + /* + * If this is the first packet fragment, save the + * index of the first NIC slot for later. + */ + if (nic_i_start < 0) + nic_i_start = nic_i; - /* Fill the slot in the NIC ring. */ - ctx->isc_txd_encap(ctx->ifc_softc, &pi); - DBG_COUNTER_INC(tx_encap); + pi.ipi_segs[seg_idx].ds_addr = paddr; + pi.ipi_segs[seg_idx].ds_len = len; + if (len) { + pkt_len += len; + seg_idx++; + } + + if (!(slot->flags & NS_MOREFRAG)) { + pi.ipi_len = pkt_len; + pi.ipi_nsegs = seg_idx; + pi.ipi_pidx = nic_i_start; + pi.ipi_ndescs = 0; + pi.ipi_flags = flags; + + /* Prepare the NIC TX ring. */ + ctx->isc_txd_encap(ctx->ifc_softc, &pi); + DBG_COUNTER_INC(tx_encap); + + /* Update transmit counters */ + tx_bytes += pi.ipi_len; + tx_pkts++; + + /* Reinit per-packet info for the next one. */ + flags = seg_idx = pkt_len = 0; + nic_i_start = -1; + } /* prefetch for next round */ __builtin_prefetch(&ring->slot[nm_i + 1]); @@ -1026,7 +1133,7 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags) txq->ift_sds.ifsd_map[nic_i], BUS_DMASYNC_PREWRITE); - slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); + slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED | NS_MOREFRAG); nm_i = nm_next(nm_i, lim); nic_i = nm_next(nic_i, lim); } @@ -1044,9 +1151,8 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags) * Second part: reclaim buffers for completed transmissions. * * If there are unclaimed buffers, attempt to reclaim them. - * If none are reclaimed, and TX IRQs are not in use, do an initial - * minimal delay, then trigger the tx handler which will spin in the - * group task queue. + * If we don't manage to reclaim them all, and TX IRQs are not in use, + * trigger a per-tx-queue timer to try again later. */ if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { if (iflib_tx_credits_update(ctx, txq)) { @@ -1055,11 +1161,18 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags) kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); } } + if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) if (kring->nr_hwtail != nm_prev(kring->nr_hwcur, lim)) { - callout_reset_on(&txq->ift_timer, hz < 2000 ? 1 : hz / 1000, - iflib_timer, txq, txq->ift_timer.c_cpu); - } + callout_reset_sbt_on(&txq->ift_netmap_timer, + NETMAP_TX_TIMER_US * SBT_1US, SBT_1US, + iflib_netmap_timer, txq, + txq->ift_netmap_timer.c_cpu, 0); + } + + if_inc_counter(ifp, IFCOUNTER_OBYTES, tx_bytes); + if_inc_counter(ifp, IFCOUNTER_OPACKETS, tx_pkts); + return (0); } @@ -1082,80 +1195,112 @@ iflib_netmap_rxsync(struct netmap_kring *kring, int flags) struct netmap_adapter *na = kring->na; struct netmap_ring *ring = kring->ring; if_t ifp = na->ifp; - iflib_fl_t fl; uint32_t nm_i; /* index into the netmap ring */ uint32_t nic_i; /* index into the NIC ring */ - u_int i, n; + u_int n; u_int const lim = kring->nkr_num_slots - 1; - u_int const head = kring->rhead; int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; - struct if_rxd_info ri; + int i = 0, rx_bytes = 0, rx_pkts = 0; if_ctx_t ctx = ifp->if_softc; + if_shared_ctx_t sctx = ctx->ifc_sctx; + if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; iflib_rxq_t rxq = &ctx->ifc_rxqs[kring->ring_id]; - if (head > lim) - return netmap_ring_reinit(kring); + iflib_fl_t fl = &rxq->ifr_fl[0]; + struct if_rxd_info ri; + qidx_t *cidxp; /* - * XXX netmap_fl_refill() only ever (re)fills free list 0 so far. + * netmap only uses free list 0, to avoid out of order consumption + * of receive buffers */ - for (i = 0, fl = rxq->ifr_fl; i < rxq->ifr_nfl; i++, fl++) { - bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - } + bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); /* * First part: import newly received packets. * * nm_i is the index of the next free slot in the netmap ring, - * nic_i is the index of the next received packet in the NIC ring, - * and they may differ in case if_init() has been called while + * nic_i is the index of the next received packet in the NIC ring + * (or in the free list 0 if IFLIB_HAS_RXCQ is set), and they may + * differ in case if_init() has been called while * in netmap mode. For the receive ring we have * - * nic_i = rxr->next_check; + * nic_i = fl->ifl_cidx; * nm_i = kring->nr_hwtail (previous) * and * nm_i == (nic_i + kring->nkr_hwofs) % ring_size * - * rxr->next_check is set to 0 on a ring reinit + * fl->ifl_cidx is set to 0 on a ring reinit */ if (netmap_no_pendintr || force_update) { + uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim); + bool have_rxcq = sctx->isc_flags & IFLIB_HAS_RXCQ; int crclen = iflib_crcstrip ? 0 : 4; int error, avail; - for (i = 0; i < rxq->ifr_nfl; i++) { - fl = &rxq->ifr_fl[i]; - nic_i = fl->ifl_cidx; - nm_i = netmap_idx_n2k(kring, nic_i); - avail = ctx->isc_rxd_available(ctx->ifc_softc, - rxq->ifr_id, nic_i, USHRT_MAX); - for (n = 0; avail > 0; n++, avail--) { - rxd_info_zero(&ri); - ri.iri_frags = rxq->ifr_frags; - ri.iri_qsidx = kring->ring_id; - ri.iri_ifp = ctx->ifc_ifp; - ri.iri_cidx = nic_i; + /* + * For the free list consumer index, we use the same + * logic as in iflib_rxeof(). + */ + if (have_rxcq) + cidxp = &rxq->ifr_cq_cidx; + else + cidxp = &fl->ifl_cidx; + avail = ctx->isc_rxd_available(ctx->ifc_softc, + rxq->ifr_id, *cidxp, USHRT_MAX); + + nic_i = fl->ifl_cidx; + nm_i = netmap_idx_n2k(kring, nic_i); + MPASS(nm_i == kring->nr_hwtail); + for (n = 0; avail > 0 && nm_i != hwtail_lim; n++, avail--) { + rxd_info_zero(&ri); + ri.iri_frags = rxq->ifr_frags; + ri.iri_qsidx = kring->ring_id; + ri.iri_ifp = ctx->ifc_ifp; + ri.iri_cidx = *cidxp; + + error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); + for (i = 0; i < ri.iri_nfrags; i++) { + if (error) { + ring->slot[nm_i].len = 0; + ring->slot[nm_i].flags = 0; + } else { + ring->slot[nm_i].len = ri.iri_frags[i].irf_len; + if (i == (ri.iri_nfrags - 1)) { + ring->slot[nm_i].len -= crclen; + ring->slot[nm_i].flags = 0; + + /* Update receive counters */ + rx_bytes += ri.iri_len; + rx_pkts++; + } else + ring->slot[nm_i].flags = NS_MOREFRAG; + } - error = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); - ring->slot[nm_i].len = error ? 0 : ri.iri_len - crclen; - ring->slot[nm_i].flags = 0; bus_dmamap_sync(fl->ifl_buf_tag, fl->ifl_sds.ifsd_map[nic_i], BUS_DMASYNC_POSTREAD); nm_i = nm_next(nm_i, lim); - nic_i = nm_next(nic_i, lim); + fl->ifl_cidx = nic_i = nm_next(nic_i, lim); } - if (n) { /* update the state variables */ - if (netmap_no_pendintr && !force_update) { - /* diagnostics */ - iflib_rx_miss ++; - iflib_rx_miss_bufs += n; - } - fl->ifl_cidx = nic_i; - kring->nr_hwtail = nm_i; + + if (have_rxcq) { + *cidxp = ri.iri_cidx; + while (*cidxp >= scctx->isc_nrxd[0]) + *cidxp -= scctx->isc_nrxd[0]; } - kring->nr_kflags &= ~NKR_PENDINTR; + } + if (n) { /* update the state variables */ + if (netmap_no_pendintr && !force_update) { + /* diagnostics */ + iflib_rx_miss ++; + iflib_rx_miss_bufs += n; + } + kring->nr_hwtail = nm_i; + } + kring->nr_kflags &= ~NKR_PENDINTR; } /* * Second part: skip past packets that userspace has released. @@ -1165,10 +1310,12 @@ iflib_netmap_rxsync(struct netmap_kring *kring, int flags) * nic_i is the index in the NIC ring, and * nm_i == (nic_i + kring->nkr_hwofs) % ring_size */ - /* XXX not sure how this will work with multiple free lists */ - nm_i = kring->nr_hwcur; + netmap_fl_refill(rxq, kring, false); - return (netmap_fl_refill(rxq, kring, nm_i, false)); + if_inc_counter(ifp, IFCOUNTER_IBYTES, rx_bytes); + if_inc_counter(ifp, IFCOUNTER_IPACKETS, rx_pkts); + + return (0); } static void @@ -1185,32 +1332,31 @@ iflib_netmap_intr(struct netmap_adapter *na, int onoff) CTX_UNLOCK(ctx); } - static int iflib_netmap_attach(if_ctx_t ctx) { struct netmap_adapter na; - if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; bzero(&na, sizeof(na)); na.ifp = ctx->ifc_ifp; - na.na_flags = NAF_BDG_MAYSLEEP; + na.na_flags = NAF_BDG_MAYSLEEP | NAF_MOREFRAG; MPASS(ctx->ifc_softc_ctx.isc_ntxqsets); MPASS(ctx->ifc_softc_ctx.isc_nrxqsets); - na.num_tx_desc = scctx->isc_ntxd[0]; - na.num_rx_desc = scctx->isc_nrxd[0]; + na.num_tx_desc = iflib_num_tx_descs(ctx); + na.num_rx_desc = iflib_num_rx_descs(ctx); na.nm_txsync = iflib_netmap_txsync; na.nm_rxsync = iflib_netmap_rxsync; na.nm_register = iflib_netmap_register; na.nm_intr = iflib_netmap_intr; + na.nm_config = iflib_netmap_config; na.num_tx_rings = ctx->ifc_softc_ctx.isc_ntxqsets; na.num_rx_rings = ctx->ifc_softc_ctx.isc_nrxqsets; return (netmap_attach(&na)); } -static void +static int iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq) { struct netmap_adapter *na = NA(ctx->ifc_ifp); @@ -1218,9 +1364,8 @@ iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq) slot = netmap_reset(na, NR_TX, txq->ift_id, 0); if (slot == NULL) - return; + return (0); for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxd[0]; i++) { - /* * In netmap mode, set the map for the packet buffer. * NOTE: Some drivers (not this one) also need to set @@ -1232,57 +1377,48 @@ iflib_netmap_txq_init(if_ctx_t ctx, iflib_txq_t txq) netmap_load_map(na, txq->ift_buf_tag, txq->ift_sds.ifsd_map[i], NMB(na, slot + si)); } + return (1); } -static void +static int iflib_netmap_rxq_init(if_ctx_t ctx, iflib_rxq_t rxq) { struct netmap_adapter *na = NA(ctx->ifc_ifp); - struct netmap_kring *kring = na->rx_rings[rxq->ifr_id]; + struct netmap_kring *kring; struct netmap_slot *slot; - uint32_t nm_i; slot = netmap_reset(na, NR_RX, rxq->ifr_id, 0); if (slot == NULL) - return; - nm_i = netmap_idx_n2k(kring, 0); - netmap_fl_refill(rxq, kring, nm_i, true); + return (0); + kring = na->rx_rings[rxq->ifr_id]; + netmap_fl_refill(rxq, kring, true); + return (1); } static void -iflib_netmap_timer_adjust(if_ctx_t ctx, iflib_txq_t txq, uint32_t *reset_on) +iflib_netmap_timer(void *arg) { - struct netmap_kring *kring; - uint16_t txqid; + iflib_txq_t txq = arg; + if_ctx_t ctx = txq->ift_ctx; - txqid = txq->ift_id; - kring = NA(ctx->ifc_ifp)->tx_rings[txqid]; - - if (kring->nr_hwcur != nm_next(kring->nr_hwtail, kring->nkr_num_slots - 1)) { - bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, - BUS_DMASYNC_POSTREAD); - if (ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) - netmap_tx_irq(ctx->ifc_ifp, txqid); - if (!(ctx->ifc_flags & IFC_NETMAP_TX_IRQ)) { - if (hz < 2000) - *reset_on = 1; - else - *reset_on = hz / 1000; - } - } + /* + * Wake up the netmap application, to give it a chance to + * call txsync and reclaim more completed TX buffers. + */ + netmap_tx_irq(ctx->ifc_ifp, txq->ift_id); } #define iflib_netmap_detach(ifp) netmap_detach(ifp) #else -#define iflib_netmap_txq_init(ctx, txq) -#define iflib_netmap_rxq_init(ctx, rxq) +#define iflib_netmap_txq_init(ctx, txq) (0) +#define iflib_netmap_rxq_init(ctx, rxq) (0) #define iflib_netmap_detach(ifp) +#define netmap_enable_all_rings(ifp) +#define netmap_disable_all_rings(ifp) #define iflib_netmap_attach(ctx) (0) #define netmap_rx_irq(ifp, qid, budget) (0) -#define netmap_tx_irq(ifp, qid) do {} while (0) -#define iflib_netmap_timer_adjust(ctx, txq, reset_on) #endif #if defined(__i386__) || defined(__amd64__) @@ -1311,7 +1447,6 @@ iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid) fl = &rxq->ifr_fl[flid]; iru->iru_paddrs = fl->ifl_bus_addrs; - iru->iru_vaddrs = &fl->ifl_vm_addrs[0]; iru->iru_idxs = fl->ifl_rxd_idxs; iru->iru_qsidx = rxq->ifr_id; iru->iru_buf_size = fl->ifl_buf_size; @@ -1438,29 +1573,6 @@ iflib_dma_free_multi(iflib_dma_info_t *dmalist, int count) iflib_dma_free(*dmaiter); } -#ifdef EARLY_AP_STARTUP -static const int iflib_started = 1; -#else -/* - * We used to abuse the smp_started flag to decide if the queues have been - * fully initialized (by late taskqgroup_adjust() calls in a SYSINIT()). - * That gave bad races, since the SYSINIT() runs strictly after smp_started - * is set. Run a SYSINIT() strictly after that to just set a usable - * completion flag. - */ - -static int iflib_started; - -static void -iflib_record_started(void *arg) -{ - iflib_started = 1; -} - -SYSINIT(iflib_record_started, SI_SUB_SMP + 1, SI_ORDER_FIRST, - iflib_record_started, NULL); -#endif - static int iflib_fast_intr(void *arg) { @@ -1468,9 +1580,6 @@ iflib_fast_intr(void *arg) struct grouptask *gtask = info->ifi_task; int result; - if (!iflib_started) - return (FILTER_STRAY); - DBG_COUNTER_INC(fast_intrs); if (info->ifi_filter != NULL) { result = info->ifi_filter(info->ifi_filter_arg); @@ -1479,7 +1588,7 @@ iflib_fast_intr(void *arg) } GROUPTASK_ENQUEUE(gtask); - return (FILTER_SCHEDULE_THREAD); + return (FILTER_HANDLED); } static int @@ -1495,9 +1604,6 @@ iflib_fast_intr_rxtx(void *arg) qidx_t txqid; bool intr_enable, intr_legacy; - if (!iflib_started) - return (FILTER_STRAY); - DBG_COUNTER_INC(fast_intrs); if (info->ifi_filter != NULL) { result = info->ifi_filter(info->ifi_filter_arg); @@ -1539,10 +1645,9 @@ iflib_fast_intr_rxtx(void *arg) } if (intr_enable) IFDI_INTR_ENABLE(ctx); - return (FILTER_SCHEDULE_THREAD); + return (FILTER_HANDLED); } - static int iflib_fast_intr_ctx(void *arg) { @@ -1550,9 +1655,6 @@ iflib_fast_intr_ctx(void *arg) struct grouptask *gtask = info->ifi_task; int result; - if (!iflib_started) - return (FILTER_STRAY); - DBG_COUNTER_INC(fast_intrs); if (info->ifi_filter != NULL) { result = info->ifi_filter(info->ifi_filter_arg); @@ -1561,7 +1663,7 @@ iflib_fast_intr_ctx(void *arg) } GROUPTASK_ENQUEUE(gtask); - return (FILTER_SCHEDULE_THREAD); + return (FILTER_HANDLED); } static int @@ -1601,7 +1703,6 @@ _iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid, return (0); } - /********************************************************************* * * Allocate DMA resources for TX buffers as well as memory for the TX @@ -1621,7 +1722,6 @@ iflib_txsd_alloc(iflib_txq_t txq) bus_size_t tsomaxsize; int err, nsegments, ntsosegments; bool tso; - int i; nsegments = scctx->isc_tx_nsegments; ntsosegments = scctx->isc_tx_tso_segments_max; @@ -1702,7 +1802,7 @@ iflib_txsd_alloc(iflib_txq_t txq) err = ENOMEM; goto fail; } - for (i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { + for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { err = bus_dmamap_create(txq->ift_buf_tag, 0, &txq->ift_sds.ifsd_map[i]); if (err != 0) { @@ -1730,20 +1830,16 @@ iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int i) { bus_dmamap_t map; - map = NULL; - if (txq->ift_sds.ifsd_map != NULL) + if (txq->ift_sds.ifsd_map != NULL) { map = txq->ift_sds.ifsd_map[i]; - if (map != NULL) { bus_dmamap_sync(txq->ift_buf_tag, map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txq->ift_buf_tag, map); bus_dmamap_destroy(txq->ift_buf_tag, map); txq->ift_sds.ifsd_map[i] = NULL; } - map = NULL; - if (txq->ift_sds.ifsd_tso_map != NULL) + if (txq->ift_sds.ifsd_tso_map != NULL) { map = txq->ift_sds.ifsd_tso_map[i]; - if (map != NULL) { bus_dmamap_sync(txq->ift_tso_buf_tag, map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txq->ift_tso_buf_tag, map); @@ -1756,10 +1852,17 @@ static void iflib_txq_destroy(iflib_txq_t txq) { if_ctx_t ctx = txq->ift_ctx; - int i; - for (i = 0; i < txq->ift_size; i++) + for (int i = 0; i < txq->ift_size; i++) iflib_txsd_destroy(ctx, txq, i); + + if (txq->ift_br != NULL) { + ifmp_ring_free(txq->ift_br); + txq->ift_br = NULL; + } + + mtx_destroy(&txq->ift_mtx); + if (txq->ift_sds.ifsd_map != NULL) { free(txq->ift_sds.ifsd_map, M_IFLIB); txq->ift_sds.ifsd_map = NULL; @@ -1780,6 +1883,9 @@ iflib_txq_destroy(iflib_txq_t txq) bus_dma_tag_destroy(txq->ift_tso_buf_tag); txq->ift_tso_buf_tag = NULL; } + if (txq->ift_ifdi != NULL) { + free(txq->ift_ifdi, M_IFLIB); + } } static void @@ -1802,7 +1908,7 @@ iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i) bus_dmamap_unload(txq->ift_tso_buf_tag, txq->ift_sds.ifsd_tso_map[i]); } - m_free(*mp); + m_freem(*mp); DBG_COUNTER_INC(tx_frees); *mp = NULL; } @@ -1856,13 +1962,12 @@ iflib_rxsd_alloc(iflib_rxq_t rxq) device_t dev = ctx->ifc_dev; iflib_fl_t fl; int err; - int i; MPASS(scctx->isc_nrxd[0] > 0); MPASS(scctx->isc_nrxd[rxq->ifr_fl_offset] > 0); fl = rxq->ifr_fl; - for (i = 0; i < rxq->ifr_nfl; i++, fl++) { + for (int i = 0; i < rxq->ifr_nfl; i++, fl++) { fl->ifl_size = scctx->isc_nrxd[rxq->ifr_fl_offset]; /* this isn't necessarily the same */ /* Set up DMA tag for RX buffers. */ err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ @@ -1923,9 +2028,7 @@ iflib_rxsd_alloc(iflib_rxq_t rxq) err = ENOMEM; goto fail; } - { - int i; - for (i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { + for (int i = 0; i < scctx->isc_nrxd[rxq->ifr_fl_offset]; i++) { err = bus_dmamap_create(fl->ifl_buf_tag, 0, &fl->ifl_sds.ifsd_map[i]); if (err != 0) { @@ -1933,7 +2036,6 @@ iflib_rxsd_alloc(iflib_rxq_t rxq) goto fail; } } - } } return (0); @@ -1942,7 +2044,6 @@ fail: return (err); } - /* * Internal service routines */ @@ -1964,16 +2065,17 @@ _rxq_refill_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) } /** - * _iflib_fl_refill - refill an rxq free-buffer list + * iflib_fl_refill - refill an rxq free-buffer list * @ctx: the iflib context * @fl: the free list to refill * @count: the number of new buffers to allocate * * (Re)populate an rxq free-buffer list with up to @count new packet buffers. - * The caller must assure that @count does not exceed the queue's capacity. + * The caller must assure that @count does not exceed the queue's capacity + * minus one (since we always leave a descriptor unavailable). */ -static void -_iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) +static uint8_t +iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) { struct if_rxd_update iru; struct rxq_refill_cb_arg cb_arg; @@ -1985,6 +2087,8 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) int err, frag_idx, i, idx, n, pidx; qidx_t credits; + MPASS(count <= fl->ifl_size - fl->ifl_credits - 1); + sd_m = fl->ifl_sds.ifsd_m; sd_map = fl->ifl_sds.ifsd_map; sd_cl = fl->ifl_sds.ifsd_cl; @@ -2010,12 +2114,13 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) if (n > 8) DBG_COUNTER_INC(fl_refills_large); iru_init(&iru, fl->ifl_rxq, fl->ifl_id); - while (n--) { + while (n-- > 0) { /* * We allocate an uninitialized mbuf + cluster, mbuf is * initialized after rx. * - * If the cluster is still set then we know a minimum sized packet was received + * If the cluster is still set then we know a minimum sized + * packet was received */ bit_ffc_at(fl->ifl_rx_bitmap, frag_idx, fl->ifl_size, &frag_idx); @@ -2023,7 +2128,12 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) bit_ffc(fl->ifl_rx_bitmap, fl->ifl_size, &frag_idx); MPASS(frag_idx >= 0); if ((cl = sd_cl[frag_idx]) == NULL) { +#ifndef __HAIKU__ + cl = uma_zalloc(fl->ifl_zone, M_NOWAIT); + if (__predict_false(cl == NULL)) +#else if ((cl = m_cljget(NULL, M_NOWAIT, fl->ifl_buf_size)) == NULL) +#endif break; cb_arg.error = 0; @@ -2031,18 +2141,16 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) err = bus_dmamap_load(fl->ifl_buf_tag, sd_map[frag_idx], cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, BUS_DMA_NOWAIT); - if (err != 0 || cb_arg.error) { - /* - * !zone_pack ? - */ + if (__predict_false(err != 0 || cb_arg.error)) { #ifndef __HAIKU__ - if (fl->ifl_zone == zone_pack) - uma_zfree(fl->ifl_zone, cl); + uma_zfree(fl->ifl_zone, cl); +#else + m_free(cl); #endif break; } - sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; + sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; sd_cl[frag_idx] = cl; #if MEMORY_LOGGING fl->ifl_cl_enqueued++; @@ -2054,9 +2162,9 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) BUS_DMASYNC_PREREAD); if (sd_m[frag_idx] == NULL) { - if ((m = m_gethdr(M_NOWAIT, MT_NOINIT)) == NULL) { + m = m_gethdr(M_NOWAIT, MT_NOINIT); + if (__predict_false(m == NULL)) break; - } sd_m[frag_idx] = m; } bit_set(fl->ifl_rx_bitmap, frag_idx); @@ -2067,48 +2175,65 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int count) DBG_COUNTER_INC(rx_allocs); fl->ifl_rxd_idxs[i] = frag_idx; fl->ifl_bus_addrs[i] = bus_addr; - fl->ifl_vm_addrs[i] = cl; credits++; i++; MPASS(credits <= fl->ifl_size); if (++idx == fl->ifl_size) { +#ifdef INVARIANTS fl->ifl_gen = 1; +#endif idx = 0; } if (n == 0 || i == IFLIB_MAX_RX_REFRESH) { iru.iru_pidx = pidx; iru.iru_count = i; ctx->isc_rxd_refill(ctx->ifc_softc, &iru); - i = 0; - pidx = idx; fl->ifl_pidx = idx; fl->ifl_credits = credits; + pidx = idx; + i = 0; } } - if (i) { - iru.iru_pidx = pidx; - iru.iru_count = i; - ctx->isc_rxd_refill(ctx->ifc_softc, &iru); - fl->ifl_pidx = idx; - fl->ifl_credits = credits; + if (n < count - 1) { + if (i != 0) { + iru.iru_pidx = pidx; + iru.iru_count = i; + ctx->isc_rxd_refill(ctx->ifc_softc, &iru); + fl->ifl_pidx = idx; + fl->ifl_credits = credits; + } + DBG_COUNTER_INC(rxd_flush); + bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, + fl->ifl_id, fl->ifl_pidx); + if (__predict_true(bit_test(fl->ifl_rx_bitmap, frag_idx))) { + fl->ifl_fragidx = frag_idx + 1; + if (fl->ifl_fragidx == fl->ifl_size) + fl->ifl_fragidx = 0; + } else { + fl->ifl_fragidx = frag_idx; + } } - DBG_COUNTER_INC(rxd_flush); - if (fl->ifl_pidx == 0) - pidx = fl->ifl_size - 1; - else - pidx = fl->ifl_pidx - 1; - bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - ctx->isc_rxd_flush(ctx->ifc_softc, fl->ifl_rxq->ifr_id, fl->ifl_id, pidx); - fl->ifl_fragidx = frag_idx; + return (n == -1 ? 0 : IFLIB_RXEOF_EMPTY); } -static __inline void -__iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) +static inline uint8_t +iflib_fl_refill_all(if_ctx_t ctx, iflib_fl_t fl) { - /* we avoid allowing pidx to catch up with cidx as it confuses ixl */ + /* + * We leave an unused descriptor to avoid pidx to catch up with cidx. + * This is important as it confuses most NICs. For instance, + * Intel NICs have (per receive ring) RDH and RDT registers, where + * RDH points to the next receive descriptor to be used by the NIC, + * and RDT for the next receive descriptor to be published by the + * driver to the NIC (RDT - 1 is thus the last valid one). + * The condition RDH == RDT means no descriptors are available to + * the NIC, and thus it would be ambiguous if it also meant that + * all the descriptors are available to the NIC. + */ int32_t reclaimable = fl->ifl_size - fl->ifl_credits - 1; #ifdef INVARIANTS int32_t delta = fl->ifl_size - get_inuse(fl->ifl_size, fl->ifl_cidx, fl->ifl_pidx, fl->ifl_gen) - 1; @@ -2118,13 +2243,15 @@ __iflib_fl_refill_lt(if_ctx_t ctx, iflib_fl_t fl, int max) MPASS(reclaimable == delta); if (reclaimable > 0) - _iflib_fl_refill(ctx, fl, min(max, reclaimable)); + return (iflib_fl_refill(ctx, fl, reclaimable)); + return (0); } uint8_t iflib_in_detach(if_ctx_t ctx) { bool in_detach; + STATE_LOCK(ctx); in_detach = !!(ctx->ifc_flags & IFC_IN_DETACH); STATE_UNLOCK(ctx); @@ -2147,36 +2274,30 @@ iflib_fl_bufs_free(iflib_fl_t fl) bus_dmamap_sync(fl->ifl_buf_tag, sd_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(fl->ifl_buf_tag, sd_map); - if (*sd_cl != NULL) { #ifndef __HAIKU__ - uma_zfree(fl->ifl_zone, *sd_cl); + uma_zfree(fl->ifl_zone, *sd_cl); #else - struct mbuf* mb = m_get(0, MT_DATA); - m_cljset(mb, *sd_cl, fl->ifl_cltype); - m_free(mb); + struct mbuf* mb = m_get(0, MT_DATA); + m_cljset(mb, *sd_cl, fl->ifl_cltype); + m_free(mb); #endif - } - // XXX: Should this get moved out? - if (iflib_in_detach(fl->ifl_rxq->ifr_ctx)) - bus_dmamap_destroy(fl->ifl_buf_tag, sd_map); + *sd_cl = NULL; if (*sd_m != NULL) { m_init(*sd_m, M_NOWAIT, MT_DATA, 0); #ifndef __HAIKU__ - uma_zfree(zone_mbuf, *sd_m); + m_free_raw(*sd_m); #else m_free(*sd_m); #endif + *sd_m = NULL; } } else { - MPASS(*sd_cl == NULL); MPASS(*sd_m == NULL); } #if MEMORY_LOGGING fl->ifl_m_dequeued++; fl->ifl_cl_dequeued++; #endif - *sd_cl = NULL; - *sd_m = NULL; } #ifdef INVARIANTS for (i = 0; i < fl->ifl_size; i++) { @@ -2193,7 +2314,7 @@ iflib_fl_bufs_free(iflib_fl_t fl) /********************************************************************* * - * Initialize a receive ring and its buffers. + * Initialize a free list and its buffers. * **********************************************************************/ static int @@ -2201,6 +2322,8 @@ iflib_fl_setup(iflib_fl_t fl) { iflib_rxq_t rxq = fl->ifl_rxq; if_ctx_t ctx = rxq->ifr_ctx; + if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; + int qidx; bit_nclear(fl->ifl_rx_bitmap, 0, fl->ifl_size - 1); /* @@ -2209,7 +2332,16 @@ iflib_fl_setup(iflib_fl_t fl) iflib_fl_bufs_free(fl); /* Now replenish the mbufs */ MPASS(fl->ifl_credits == 0); - fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; + qidx = rxq->ifr_fl_offset + fl->ifl_id; + if (scctx->isc_rxd_buf_size[qidx] != 0) + fl->ifl_buf_size = scctx->isc_rxd_buf_size[qidx]; + else + fl->ifl_buf_size = ctx->ifc_rx_mbuf_sz; + /* + * ifl_buf_size may be a driver-supplied value, so pull it up + * to the selected mbuf size. + */ + fl->ifl_buf_size = iflib_get_mbuf_size_for(fl->ifl_buf_size); if (fl->ifl_buf_size > ctx->ifc_max_fl_buf_size) ctx->ifc_max_fl_buf_size = fl->ifl_buf_size; fl->ifl_cltype = m_gettype(fl->ifl_buf_size); @@ -2217,13 +2349,15 @@ iflib_fl_setup(iflib_fl_t fl) fl->ifl_zone = m_getzone(fl->ifl_buf_size); #endif - - /* avoid pre-allocating zillions of clusters to an idle card - * potentially speeding up attach + /* + * Avoid pre-allocating zillions of clusters to an idle card + * potentially speeding up attach. In any case make sure + * to leave a descriptor unavailable. See the comment in + * iflib_fl_refill_all(). */ - _iflib_fl_refill(ctx, fl, min(128, fl->ifl_size)); - MPASS(min(128, fl->ifl_size) == fl->ifl_credits); - if (min(128, fl->ifl_size) != fl->ifl_credits) + MPASS(fl->ifl_size > 0); + (void)iflib_fl_refill(ctx, fl, min(128, fl->ifl_size - 1)); + if (min(128, fl->ifl_size - 1) != fl->ifl_credits) return (ENOBUFS); /* * handle failure @@ -2252,9 +2386,6 @@ iflib_rx_sds_free(iflib_rxq_t rxq) if (fl->ifl_buf_tag != NULL) { if (fl->ifl_sds.ifsd_map != NULL) { for (j = 0; j < fl->ifl_size; j++) { - if (fl->ifl_sds.ifsd_map[j] == - NULL) - continue; bus_dmamap_sync( fl->ifl_buf_tag, fl->ifl_sds.ifsd_map[j], @@ -2262,6 +2393,9 @@ iflib_rx_sds_free(iflib_rxq_t rxq) bus_dmamap_unload( fl->ifl_buf_tag, fl->ifl_sds.ifsd_map[j]); + bus_dmamap_destroy( + fl->ifl_buf_tag, + fl->ifl_sds.ifsd_map[j]); } } bus_dma_tag_destroy(fl->ifl_buf_tag); @@ -2271,13 +2405,17 @@ iflib_rx_sds_free(iflib_rxq_t rxq) free(fl->ifl_sds.ifsd_cl, M_IFLIB); free(fl->ifl_sds.ifsd_ba, M_IFLIB); free(fl->ifl_sds.ifsd_map, M_IFLIB); + free(fl->ifl_rx_bitmap, M_IFLIB); fl->ifl_sds.ifsd_m = NULL; fl->ifl_sds.ifsd_cl = NULL; fl->ifl_sds.ifsd_ba = NULL; fl->ifl_sds.ifsd_map = NULL; + fl->ifl_rx_bitmap = NULL; } free(rxq->ifr_fl, M_IFLIB); rxq->ifr_fl = NULL; + free(rxq->ifr_ifdi, M_IFLIB); + rxq->ifr_ifdi = NULL; rxq->ifr_cq_cidx = 0; } } @@ -2292,7 +2430,6 @@ iflib_timer(void *arg) if_ctx_t ctx = txq->ift_ctx; if_softc_ctx_t sctx = &ctx->ifc_softc_ctx; uint64_t this_tick = ticks; - uint32_t reset_on = hz / 2; if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) return; @@ -2302,7 +2439,7 @@ iflib_timer(void *arg) ** can be done without the lock because its RO ** and the HUNG state will be static if set. */ - if (this_tick - txq->ift_last_timer_tick >= hz / 2) { + if (this_tick - txq->ift_last_timer_tick >= iflib_timer_default) { txq->ift_last_timer_tick = this_tick; IFDI_TIMER(ctx, txq->ift_id); if ((txq->ift_qstatus == IFLIB_QUEUE_HUNG) && @@ -2310,21 +2447,22 @@ iflib_timer(void *arg) (sctx->isc_pause_frames == 0))) goto hung; - if (ifmp_ring_is_stalled(txq->ift_br)) + if (txq->ift_qstatus != IFLIB_QUEUE_IDLE && + ifmp_ring_is_stalled(txq->ift_br)) { + KASSERT(ctx->ifc_link_state == LINK_STATE_UP, + ("queue can't be marked as hung if interface is down")); txq->ift_qstatus = IFLIB_QUEUE_HUNG; + } txq->ift_cleaned_prev = txq->ift_cleaned; } -#ifdef DEV_NETMAP - if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) - iflib_netmap_timer_adjust(ctx, txq, &reset_on); -#endif /* handle any laggards */ if (txq->ift_db_pending) GROUPTASK_ENQUEUE(&txq->ift_task); sctx->isc_pause_frames = 0; - if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) - callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); + if (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING) + callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, + txq, txq->ift_timer.c_cpu); return; hung: @@ -2338,6 +2476,16 @@ iflib_timer(void *arg) STATE_UNLOCK(ctx); } +static uint16_t +iflib_get_mbuf_size_for(unsigned int size) +{ + + if (size <= MCLBYTES) + return (MCLBYTES); + else + return (MJUMPAGESIZE); +} + static void iflib_calc_rx_mbuf_sz(if_ctx_t ctx) { @@ -2347,10 +2495,8 @@ iflib_calc_rx_mbuf_sz(if_ctx_t ctx) * XXX don't set the max_frame_size to larger * than the hardware can handle */ - if (sctx->isc_max_frame_size <= MCLBYTES) - ctx->ifc_rx_mbuf_sz = MCLBYTES; - else - ctx->ifc_rx_mbuf_sz = MJUMPAGESIZE; + ctx->ifc_rx_mbuf_sz = + iflib_get_mbuf_size_for(sctx->isc_max_frame_size); } uint32_t @@ -2374,6 +2520,12 @@ iflib_init_locked(if_ctx_t ctx) if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); IFDI_INTR_DISABLE(ctx); + /* + * See iflib_stop(). Useful in case iflib_init_locked() is + * called without first calling iflib_stop(). + */ + netmap_disable_all_rings(ifp); + tx_ip_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP); tx_ip6_csum_flags = scctx->isc_tx_csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | CSUM_IP6_SCTP); /* Set hardware offload abilities */ @@ -2390,6 +2542,9 @@ iflib_init_locked(if_ctx_t ctx) for (i = 0, txq = ctx->ifc_txqs; i < sctx->isc_ntxqsets; i++, txq++) { CALLOUT_LOCK(txq); callout_stop(&txq->ift_timer); +#ifdef DEV_NETMAP + callout_stop(&txq->ift_netmap_timer); +#endif /* DEV_NETMAP */ CALLOUT_UNLOCK(txq); iflib_netmap_txq_init(ctx, txq); } @@ -2407,10 +2562,8 @@ iflib_init_locked(if_ctx_t ctx) IFDI_INIT(ctx); MPASS(if_getdrvflags(ifp) == i); for (i = 0, rxq = ctx->ifc_rxqs; i < sctx->isc_nrxqsets; i++, rxq++) { - /* XXX this should really be done on a per-queue basis */ - if (if_getcapenable(ifp) & IFCAP_NETMAP) { - MPASS(rxq->ifr_id == i); - iflib_netmap_rxq_init(ctx, rxq); + if (iflib_netmap_rxq_init(ctx, rxq) > 0) { + /* This rxq is in netmap mode. Skip normal init. */ continue; } for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { @@ -2427,8 +2580,11 @@ done: IFDI_INTR_ENABLE(ctx); txq = ctx->ifc_txqs; for (i = 0; i < sctx->isc_ntxqsets; i++, txq++) - callout_reset_on(&txq->ift_timer, hz/2, iflib_timer, txq, + callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq, txq->ift_timer.c_cpu); + + /* Re-enable txsync/rxsync. */ + netmap_enable_all_rings(ifp); } static int @@ -2439,7 +2595,7 @@ iflib_media_change(if_t ifp) CTX_LOCK(ctx); if ((err = IFDI_MEDIA_CHANGE(ctx)) == 0) - iflib_init_locked(ctx); + iflib_if_init_locked(ctx); CTX_UNLOCK(ctx); return (err); } @@ -2474,6 +2630,13 @@ iflib_stop(if_ctx_t ctx) IFDI_STOP(ctx); DELAY(1000); + /* + * Stop any pending txsync/rxsync and prevent new ones + * form starting. Processes blocked in poll() will get + * POLLERR. + */ + netmap_disable_all_rings(ctx->ifc_ifp); + iflib_debug_reset(); /* Wait for current tx queue users to exit to disarm watchdog timer. */ for (i = 0; i < scctx->isc_ntxqsets; i++, txq++) { @@ -2481,6 +2644,9 @@ iflib_stop(if_ctx_t ctx) CALLOUT_LOCK(txq); callout_stop(&txq->ift_timer); +#ifdef DEV_NETMAP + callout_stop(&txq->ift_netmap_timer); +#endif /* DEV_NETMAP */ CALLOUT_UNLOCK(txq); /* clean any enqueued buffers */ @@ -2536,7 +2702,6 @@ prefetch_pkts(iflib_fl_t fl, int cidx) int nrxd = fl->ifl_size; caddr_t next_rxd; - nextptr = (cidx + CACHE_PTR_INCREMENT) & (nrxd-1); prefetch(&fl->ifl_sds.ifsd_m[nextptr]); prefetch(&fl->ifl_sds.ifsd_cl[nextptr]); @@ -2567,7 +2732,6 @@ rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, bool unload, if_rxsd_t sd, cidx = irf->irf_idx; fl = &rxq->ifr_fl[flid]; sd->ifsd_fl = fl; - sd->ifsd_cidx = cidx; m = fl->ifl_sds.ifsd_m[cidx]; sd->ifsd_cl = &fl->ifl_sds.ifsd_cl[cidx]; fl->ifl_credits--; @@ -2579,14 +2743,12 @@ rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, bool unload, if_rxsd_t sd, next = (cidx + CACHE_PTR_INCREMENT) & (fl->ifl_size-1); prefetch(&fl->ifl_sds.ifsd_map[next]); map = fl->ifl_sds.ifsd_map[cidx]; - next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); - /* not valid assert if bxe really does SGE from non-contiguous elements */ - MPASS(fl->ifl_cidx == cidx); bus_dmamap_sync(fl->ifl_buf_tag, map, BUS_DMASYNC_POSTREAD); #ifndef __HAIKU__ - if (rxq->pfil != NULL && PFIL_HOOKED_IN(rxq->pfil) && pf_rv != NULL) { + if (rxq->pfil != NULL && PFIL_HOOKED_IN(rxq->pfil) && pf_rv != NULL && + irf->irf_len != 0) { payload = *sd->ifsd_cl; payload += ri->iri_pad; len = ri->iri_len - ri->iri_pad; @@ -2622,10 +2784,11 @@ rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, bool unload, if_rxsd_t sd, #endif { fl->ifl_sds.ifsd_m[cidx] = NULL; - *pf_rv = PFIL_PASS; + if (pf_rv != NULL) + *pf_rv = PFIL_PASS; } - if (unload) + if (unload && irf->irf_len != 0) bus_dmamap_unload(fl->ifl_buf_tag, map); fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); if (__predict_false(fl->ifl_cidx == 0)) @@ -2715,6 +2878,7 @@ iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri) /* should I merge this back in now that the two paths are basically duplicated? */ if (ri->iri_nfrags == 1 && + ri->iri_frags[0].irf_len != 0 && ri->iri_frags[0].irf_len <= MIN(IFLIB_RX_COPY_THRESH, MHLEN)) { m = rxd_frag_to_sd(rxq, &ri->iri_frags[0], false, &sd, &pf_rv, ri); @@ -2735,6 +2899,8 @@ iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri) } } else { m = assemble_segments(rxq, ri, &sd, &pf_rv); + if (m == NULL) + return (NULL); if (pf_rv != PFIL_PASS #ifndef __HAIKU__ && pf_rv != PFIL_REALLOCED @@ -2759,10 +2925,10 @@ iflib_get_ip_forwarding(struct lro_ctrl *lc, bool *v4, bool *v6) { CURVNET_SET(lc->ifp->if_vnet); #if defined(INET6) - *v6 = VNET(ip6_forwarding); + *v6 = V_ip6_forwarding; #endif #if defined(INET) - *v4 = VNET(ipforwarding); + *v4 = V_ipforwarding; #endif CURVNET_RESTORE(); } @@ -2800,7 +2966,15 @@ iflib_get_ip_forwarding(struct lro_ctrl *lc __unused, bool *v4 __unused, bool *v } #endif -static bool +static void +_task_fn_rx_watchdog(void *context) +{ + iflib_rxq_t rxq = context; + + GROUPTASK_ENQUEUE(&rxq->ifr_task); +} + +static uint8_t iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) { if_t ifp; @@ -2814,6 +2988,7 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) iflib_fl_t fl; int lro_enabled; bool v4_forwarding, v6_forwarding, lro_possible; + uint8_t retval = 0; /* * XXX early demux data packets so that if_input processing only handles @@ -2821,6 +2996,10 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) */ struct mbuf *m, *mh, *mt, *mf; +#ifndef __HAIKU__ + NET_EPOCH_ASSERT(); +#endif + lro_possible = v4_forwarding = v6_forwarding = false; ifp = ctx->ifc_ifp; mh = mt = NULL; @@ -2832,9 +3011,9 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) cidxp = &rxq->ifr_fl[0].ifl_cidx; if ((avail = iflib_rxd_avail(ctx, rxq, *cidxp, budget)) == 0) { for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) - __iflib_fl_refill_lt(ctx, fl, budget + 8); + retval |= iflib_fl_refill_all(ctx, fl); DBG_COUNTER_INC(rx_unavail); - return (false); + return (retval); } /* pfil needs the vnet to be set */ @@ -2892,7 +3071,7 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) CURVNET_RESTORE(); /* make sure that we can refill faster than drain */ for (i = 0, fl = &rxq->ifr_fl[0]; i < sctx->isc_nfl; i++, fl++) - __iflib_fl_refill_lt(ctx, fl, budget + 8); + retval |= iflib_fl_refill_all(ctx, fl); lro_enabled = (if_getcapenable(ifp) & IFCAP_LRO); #ifndef __HAIKU__ @@ -2908,8 +3087,6 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) if (!IP_ALIGNED(m) && (m = iflib_fixup_rx(m)) == NULL) continue; #endif - rx_bytes += m->m_pkthdr.len; - rx_pkts++; #ifndef __HAIKU__ #if defined(INET6) || defined(INET) if (lro_enabled) { @@ -2966,15 +3143,15 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) tcp_lro_flush_all(&rxq->ifr_lc); #endif #endif - if (avail) - return true; - return (iflib_rxd_avail(ctx, rxq, *cidxp, 1)); + if (avail != 0 || iflib_rxd_avail(ctx, rxq, *cidxp, 1) != 0) + retval |= IFLIB_RXEOF_MORE; + return (retval); err: STATE_LOCK(ctx); ctx->ifc_flags |= IFC_DO_RESET; iflib_admin_intr_deferred(ctx); STATE_UNLOCK(ctx); - return (false); + return (0); } #define TXD_NOTIFY_COUNT(txq) (((txq)->ift_size / (txq)->ift_update_freq)-1) @@ -3022,26 +3199,37 @@ txq_max_rs_deferred(iflib_txq_t txq) /* XXX we should be setting this to something other than zero */ #define RECLAIM_THRESH(ctx) ((ctx)->ifc_sctx->isc_tx_reclaim_thresh) -#define MAX_TX_DESC(ctx) max((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max, \ +#define MAX_TX_DESC(ctx) MAX((ctx)->ifc_softc_ctx.isc_tx_tso_segments_max, \ (ctx)->ifc_softc_ctx.isc_tx_nsegments) static inline bool -iflib_txd_db_check(if_ctx_t ctx, iflib_txq_t txq, int ring, qidx_t in_use) +iflib_txd_db_check(iflib_txq_t txq, int ring) { + if_ctx_t ctx = txq->ift_ctx; qidx_t dbval, max; - bool rang; - rang = false; - max = TXQ_MAX_DB_DEFERRED(txq, in_use); - if (ring || txq->ift_db_pending >= max) { + max = TXQ_MAX_DB_DEFERRED(txq, txq->ift_in_use); + + /* force || threshold exceeded || at the edge of the ring */ + if (ring || (txq->ift_db_pending >= max) || (TXQ_AVAIL(txq) <= MAX_TX_DESC(ctx) + 2)) { + + /* + * 'npending' is used if the card's doorbell is in terms of the number of descriptors + * pending flush (BRCM). 'pidx' is used in cases where the card's doorbeel uses the + * producer index explicitly (INTC). + */ dbval = txq->ift_npending ? txq->ift_npending : txq->ift_pidx; bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); ctx->isc_txd_flush(ctx->ifc_softc, txq->ift_id, dbval); + + /* + * Absent bugs there are zero packets pending so reset pending counts to zero. + */ txq->ift_db_pending = txq->ift_npending = 0; - rang = true; + return (true); } - return (rang); + return (false); } #ifdef PKT_DEBUG @@ -3495,13 +3683,14 @@ defrag: txq->ift_gen = 1; } /* - * drivers can need as many as + * drivers can need as many as * two sentinels */ MPASS(ndesc <= pi.ipi_nsegs + 2); MPASS(pi.ipi_new_pidx != pidx); MPASS(ndesc > 0); txq->ift_in_use += ndesc; + txq->ift_db_pending += ndesc; /* * We update the last software descriptor again here because there may @@ -3609,7 +3798,6 @@ iflib_completed_tx_reclaim(iflib_txq_t txq, int thresh) printf("%s processed=%ju cleaned=%ju tx_nsegments=%d reclaim=%d thresh=%d\n", __FUNCTION__, txq->ift_processed, txq->ift_cleaned, txq->ift_ctx->ifc_softc_ctx.isc_tx_nsegments, reclaim, thresh); - } #endif return (0); @@ -3669,8 +3857,8 @@ iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) if_ctx_t ctx = txq->ift_ctx; if_t ifp = ctx->ifc_ifp; struct mbuf *m, **mp; - int avail, bytes_sent, consumed, count, err, i, in_use_prev; - int mcast_sent, pkt_sent, reclaimed, txq_avail; + int avail, bytes_sent, skipped, count, err, i; + int mcast_sent, pkt_sent, reclaimed; bool do_prefetch, rang, ring; if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING) || @@ -3679,13 +3867,17 @@ iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) return (0); } reclaimed = iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx)); - rang = iflib_txd_db_check(ctx, txq, reclaimed, txq->ift_in_use); + rang = iflib_txd_db_check(txq, reclaimed && txq->ift_db_pending); avail = IDXDIFF(pidx, cidx, r->size); + if (__predict_false(ctx->ifc_flags & IFC_QFLUSH)) { + /* + * The driver is unloading so we need to free all pending packets. + */ DBG_COUNTER_INC(txq_drain_flushing); for (i = 0; i < avail; i++) { if (__predict_true(r->items[(cidx + i) & (r->size-1)] != (void *)txq)) - m_free(r->items[(cidx + i) & (r->size-1)]); + m_freem(r->items[(cidx + i) & (r->size-1)]); r->items[(cidx + i) & (r->size-1)] = NULL; } return (avail); @@ -3699,9 +3891,13 @@ iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) DBG_COUNTER_INC(txq_drain_oactive); return (0); } + + /* + * If we've reclaimed any packets this queue cannot be hung. + */ if (reclaimed) txq->ift_qstatus = IFLIB_QUEUE_IDLE; - consumed = mcast_sent = bytes_sent = pkt_sent = 0; + skipped = mcast_sent = bytes_sent = pkt_sent = 0; count = MIN(avail, TX_BATCH_SIZE); #ifdef INVARIANTS if (iflib_verbose_debug) @@ -3709,54 +3905,57 @@ iflib_txq_drain(struct ifmp_ring *r, uint32_t cidx, uint32_t pidx) avail, ctx->ifc_flags, TXQ_AVAIL(txq)); #endif do_prefetch = (ctx->ifc_flags & IFC_PREFETCH); - txq_avail = TXQ_AVAIL(txq); err = 0; - for (i = 0; i < count && txq_avail > MAX_TX_DESC(ctx) + 2; i++) { + for (i = 0; i < count && TXQ_AVAIL(txq) >= MAX_TX_DESC(ctx) + 2; i++) { int rem = do_prefetch ? count - i : 0; mp = _ring_peek_one(r, cidx, i, rem); MPASS(mp != NULL && *mp != NULL); + + /* + * Completion interrupts will use the address of the txq + * as a sentinel to enqueue _something_ in order to acquire + * the lock on the mp_ring (there's no direct lock call). + * We obviously whave to check for these sentinel cases + * and skip them. + */ if (__predict_false(*mp == (struct mbuf *)txq)) { - consumed++; + skipped++; continue; } - in_use_prev = txq->ift_in_use; err = iflib_encap(txq, mp); if (__predict_false(err)) { /* no room - bail out */ if (err == ENOBUFS) break; - consumed++; + skipped++; /* we can't send this packet - skip it */ continue; } - consumed++; pkt_sent++; m = *mp; DBG_COUNTER_INC(tx_sent); bytes_sent += m->m_pkthdr.len; mcast_sent += !!(m->m_flags & M_MCAST); - txq_avail = TXQ_AVAIL(txq); - txq->ift_db_pending += (txq->ift_in_use - in_use_prev); - ETHER_BPF_MTAP(ifp, m); if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) break; - rang = iflib_txd_db_check(ctx, txq, false, in_use_prev); + ETHER_BPF_MTAP(ifp, m); + rang = iflib_txd_db_check(txq, false); } /* deliberate use of bitwise or to avoid gratuitous short-circuit */ - ring = rang ? false : (iflib_min_tx_latency | err) || (TXQ_AVAIL(txq) < MAX_TX_DESC(ctx)); - iflib_txd_db_check(ctx, txq, ring, txq->ift_in_use); + ring = rang ? false : (iflib_min_tx_latency | err); + iflib_txd_db_check(txq, ring); if_inc_counter(ifp, IFCOUNTER_OBYTES, bytes_sent); if_inc_counter(ifp, IFCOUNTER_OPACKETS, pkt_sent); if (mcast_sent) if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast_sent); #ifdef INVARIANTS if (iflib_verbose_debug) - printf("consumed=%d\n", consumed); + printf("consumed=%d\n", skipped + pkt_sent); #endif - return (consumed); + return (skipped + pkt_sent); } static uint32_t @@ -3811,28 +4010,18 @@ _task_fn_tx(void *context) { iflib_txq_t txq = context; if_ctx_t ctx = txq->ift_ctx; -#if defined(ALTQ) || defined(DEV_NETMAP) if_t ifp = ctx->ifc_ifp; -#endif int abdicate = ctx->ifc_sysctl_tx_abdicate; #ifdef IFLIB_DIAGNOSTICS txq->ift_cpu_exec_count[curcpu]++; #endif - if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) return; #ifdef DEV_NETMAP - if (if_getcapenable(ifp) & IFCAP_NETMAP) { - bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, - BUS_DMASYNC_POSTREAD); - if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) - netmap_tx_irq(ifp, txq->ift_id); - if (ctx->ifc_flags & IFC_LEGACY) - IFDI_INTR_ENABLE(ctx); - else - IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); - return; - } + if ((if_getcapenable(ifp) & IFCAP_NETMAP) && + netmap_tx_irq(ifp, txq->ift_id)) + goto skip_ifmp; #endif #ifdef ALTQ if (ALTQ_IS_ENABLED(&ifp->if_snd)) @@ -3847,6 +4036,9 @@ _task_fn_tx(void *context) */ if (abdicate) ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); +#ifdef DEV_NETMAP +skip_ifmp: +#endif if (ctx->ifc_flags & IFC_LEGACY) IFDI_INTR_ENABLE(ctx); else @@ -3858,8 +4050,12 @@ _task_fn_rx(void *context) { iflib_rxq_t rxq = context; if_ctx_t ctx = rxq->ifr_ctx; - bool more; + uint8_t more; uint16_t budget; +#ifdef DEV_NETMAP + u_int work = 0; + int nmirq; +#endif #ifdef IFLIB_DIAGNOSTICS rxq->ifr_cpu_exec_count[curcpu]++; @@ -3867,19 +4063,21 @@ _task_fn_rx(void *context) DBG_COUNTER_INC(task_fn_rxs); if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) return; - more = true; #ifdef DEV_NETMAP - if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) { - u_int work = 0; - if (netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work)) { - more = false; - } + nmirq = netmap_rx_irq(ctx->ifc_ifp, rxq->ifr_id, &work); + if (nmirq != NM_IRQ_PASS) { + more = (nmirq == NM_IRQ_RESCHED) ? IFLIB_RXEOF_MORE : 0; + goto skip_rxeof; } #endif budget = ctx->ifc_sysctl_rx_budget; if (budget == 0) budget = 16; /* XXX */ - if (more == false || (more = iflib_rxeof(rxq, budget)) == false) { + more = iflib_rxeof(rxq, budget); +#ifdef DEV_NETMAP +skip_rxeof: +#endif + if ((more & IFLIB_RXEOF_MORE) == 0) { if (ctx->ifc_flags & IFC_LEGACY) IFDI_INTR_ENABLE(ctx); else @@ -3888,8 +4086,15 @@ _task_fn_rx(void *context) } if (__predict_false(!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING))) return; - if (more) + + if (more & IFLIB_RXEOF_MORE) GROUPTASK_ENQUEUE(&rxq->ifr_task); + else if (more & IFLIB_RXEOF_EMPTY) +#ifndef __HAIKU__ + callout_reset_curcpu(&rxq->ifr_watchdog, 1, &_task_fn_rx_watchdog, rxq); +#else + callout_reset(&rxq->ifr_watchdog, 1, &_task_fn_rx_watchdog, rxq); +#endif } static void @@ -3900,7 +4105,6 @@ _task_fn_admin(void *context) iflib_txq_t txq; int i; bool oactive, running, do_reset, do_watchdog, in_detach; - uint32_t reset_on = hz / 2; STATE_LOCK(ctx); running = (if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING); @@ -3928,12 +4132,8 @@ _task_fn_admin(void *context) } IFDI_UPDATE_ADMIN_STATUS(ctx); for (txq = ctx->ifc_txqs, i = 0; i < sctx->isc_ntxqsets; i++, txq++) { -#ifdef DEV_NETMAP - reset_on = hz / 2; - if (if_getcapenable(ctx->ifc_ifp) & IFCAP_NETMAP) - iflib_netmap_timer_adjust(ctx, txq, &reset_on); -#endif - callout_reset_on(&txq->ift_timer, reset_on, iflib_timer, txq, txq->ift_timer.c_cpu); + callout_reset_on(&txq->ift_timer, iflib_timer_default, iflib_timer, txq, + txq->ift_timer.c_cpu); } IFDI_LINK_INTR_ENABLE(ctx); if (do_reset) @@ -3946,7 +4146,6 @@ _task_fn_admin(void *context) iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET); } - static void _task_fn_iov(void *context) { @@ -3991,7 +4190,6 @@ iflib_if_init_locked(if_ctx_t ctx) iflib_init_locked(ctx); } - static void iflib_if_init(void *arg) { @@ -4159,11 +4357,10 @@ iflib_if_qflush(if_t ifp) if_qflush(ifp); } - #define IFCAP_FLAGS (IFCAP_HWCSUM_IPV6 | IFCAP_HWCSUM | IFCAP_LRO | \ IFCAP_TSO | IFCAP_VLAN_HWTAGGING | IFCAP_HWSTATS | \ IFCAP_VLAN_MTU | IFCAP_VLAN_HWFILTER | \ - IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM) + IFCAP_VLAN_HWTSO | IFCAP_VLAN_HWCSUM | IFCAP_MEXTPG) static int iflib_if_ioctl(if_t ifp, u_long command, caddr_t data) @@ -4232,7 +4429,9 @@ iflib_if_ioctl(if_t ifp, u_long command, caddr_t data) if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { if ((if_getflags(ifp) ^ ctx->ifc_if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) { + CTX_UNLOCK(ctx); err = IFDI_PROMISC_SET(ctx, if_getflags(ifp)); + CTX_LOCK(ctx); } } else reinit = 1; @@ -4292,7 +4491,7 @@ iflib_if_ioctl(if_t ifp, u_long command, caddr_t data) oldmask = if_getcapenable(ifp); mask = ifr->ifr_reqcap ^ oldmask; - mask &= ctx->ifc_softc_ctx.isc_capabilities; + mask &= ctx->ifc_softc_ctx.isc_capabilities | IFCAP_MEXTPG; setmask = 0; #ifdef TCP_OFFLOAD setmask |= mask & (IFCAP_TOE4|IFCAP_TOE6); @@ -4371,11 +4570,17 @@ iflib_vlan_register(void *arg, if_t ifp, uint16_t vtag) if ((vtag == 0) || (vtag > 4095)) return; + if (iflib_in_detach(ctx)) + return; + CTX_LOCK(ctx); + /* Driver may need all untagged packets to be flushed */ + if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG)) + iflib_stop(ctx); IFDI_VLAN_REGISTER(ctx, vtag); - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - iflib_if_init_locked(ctx); + /* Re-init to load the changes, if required */ + if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG)) + iflib_init_locked(ctx); CTX_UNLOCK(ctx); } @@ -4391,10 +4596,13 @@ iflib_vlan_unregister(void *arg, if_t ifp, uint16_t vtag) return; CTX_LOCK(ctx); + /* Driver may need all tagged packets to be flushed */ + if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG)) + iflib_stop(ctx); IFDI_VLAN_UNREGISTER(ctx, vtag); - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - iflib_if_init_locked(ctx); + /* Re-init to load the changes, if required */ + if (IFDI_NEEDS_RESTART(ctx, IFLIB_RESTART_VLAN_CONFIG)) + iflib_init_locked(ctx); CTX_UNLOCK(ctx); } @@ -4446,7 +4654,6 @@ iflib_device_probe(device_t dev) (ent->pvi_subdevice_id == 0)) && ((pci_rev_id == ent->pvi_rev_id) || (ent->pvi_rev_id == 0))) { - device_set_desc_copy(dev, ent->pvi_name); /* this needs to be changed to zero if the bus probing code * ever stops re-probing on best match because the sctx @@ -4576,42 +4783,298 @@ iflib_rem_pfil(if_ctx_t ctx) #endif } + +#ifndef __HAIKU__ +/* + * Advance forward by n members of the cpuset ctx->ifc_cpus starting from + * cpuid and wrapping as necessary. + */ +static unsigned int +cpuid_advance(if_ctx_t ctx, unsigned int cpuid, unsigned int n) +{ + unsigned int first_valid; + unsigned int last_valid; + + /* cpuid should always be in the valid set */ + MPASS(CPU_ISSET(cpuid, &ctx->ifc_cpus)); + + /* valid set should never be empty */ + MPASS(!CPU_EMPTY(&ctx->ifc_cpus)); + + first_valid = CPU_FFS(&ctx->ifc_cpus) - 1; + last_valid = CPU_FLS(&ctx->ifc_cpus) - 1; + n = n % CPU_COUNT(&ctx->ifc_cpus); + while (n > 0) { + do { + cpuid++; + if (cpuid > last_valid) + cpuid = first_valid; + } while (!CPU_ISSET(cpuid, &ctx->ifc_cpus)); + n--; + } + + return (cpuid); +} +#endif + +#if defined(SMP) && defined(SCHED_ULE) +extern struct cpu_group *cpu_top; /* CPU topology */ + +static int +find_child_with_core(int cpu, struct cpu_group *grp) +{ + int i; + + if (grp->cg_children == 0) + return -1; + + MPASS(grp->cg_child); + for (i = 0; i < grp->cg_children; i++) { + if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) + return i; + } + + return -1; +} + + +/* + * Find an L2 neighbor of the given CPU or return -1 if none found. This + * does not distinguish among multiple L2 neighbors if the given CPU has + * more than one (it will always return the same result in that case). + */ +static int +find_l2_neighbor(int cpu) +{ + struct cpu_group *grp; + int i; + + grp = cpu_top; + if (grp == NULL) + return -1; + + /* + * Find the smallest CPU group that contains the given core. + */ + i = 0; + while ((i = find_child_with_core(cpu, grp)) != -1) { + /* + * If the smallest group containing the given CPU has less + * than two members, we conclude the given CPU has no + * L2 neighbor. + */ + if (grp->cg_child[i].cg_count <= 1) + return (-1); + grp = &grp->cg_child[i]; + } + + /* Must share L2. */ + if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) + return -1; + + /* + * Select the first member of the set that isn't the reference + * CPU, which at this point is guaranteed to exist. + */ + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, &grp->cg_mask) && i != cpu) + return (i); + } + + /* Should never be reached */ + return (-1); +} + +#else +static int +find_l2_neighbor(int cpu) +{ + + return (-1); +} +#endif + +#ifndef __HAIKU__ +/* + * CPU mapping behaviors + * --------------------- + * 'separate txrx' refers to the separate_txrx sysctl + * 'use logical' refers to the use_logical_cores sysctl + * 'INTR CPUS' indicates whether bus_get_cpus(INTR_CPUS) succeeded + * + * separate use INTR + * txrx logical CPUS result + * ---------- --------- ------ ------------------------------------------------ + * - - X RX and TX queues mapped to consecutive physical + * cores with RX/TX pairs on same core and excess + * of either following + * - X X RX and TX queues mapped to consecutive cores + * of any type with RX/TX pairs on same core and + * excess of either following + * X - X RX and TX queues mapped to consecutive physical + * cores; all RX then all TX + * X X X RX queues mapped to consecutive physical cores + * first, then TX queues mapped to L2 neighbor of + * the corresponding RX queue if one exists, + * otherwise to consecutive physical cores + * - n/a - RX and TX queues mapped to consecutive cores of + * any type with RX/TX pairs on same core and excess + * of either following + * X n/a - RX and TX queues mapped to consecutive cores of + * any type; all RX then all TX + */ +static unsigned int +get_cpuid_for_queue(if_ctx_t ctx, unsigned int base_cpuid, unsigned int qid, + bool is_tx) +{ + if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; + unsigned int core_index; + + if (ctx->ifc_sysctl_separate_txrx) { + /* + * When using separate CPUs for TX and RX, the assignment + * will always be of a consecutive CPU out of the set of + * context CPUs, except for the specific case where the + * context CPUs are phsyical cores, the use of logical cores + * has been enabled, the assignment is for TX, the TX qid + * corresponds to an RX qid, and the CPU assigned to the + * corresponding RX queue has an L2 neighbor. + */ + if (ctx->ifc_sysctl_use_logical_cores && + ctx->ifc_cpus_are_physical_cores && + is_tx && qid < scctx->isc_nrxqsets) { + int l2_neighbor; + unsigned int rx_cpuid; + + rx_cpuid = cpuid_advance(ctx, base_cpuid, qid); + l2_neighbor = find_l2_neighbor(rx_cpuid); + if (l2_neighbor != -1) { + return (l2_neighbor); + } + /* + * ... else fall through to the normal + * consecutive-after-RX assignment scheme. + * + * Note that we are assuming that all RX queue CPUs + * have an L2 neighbor, or all do not. If a mixed + * scenario is possible, we will have to keep track + * separately of how many queues prior to this one + * were not able to be assigned to an L2 neighbor. + */ + } + if (is_tx) + core_index = scctx->isc_nrxqsets + qid; + else + core_index = qid; + } else { + core_index = qid; + } + + return (cpuid_advance(ctx, base_cpuid, core_index)); +} +#else +#define get_cpuid_for_queue(...) CPU_FIRST() +#endif + static uint16_t get_ctx_core_offset(if_ctx_t ctx) { #ifndef __HAIKU__ if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; struct cpu_offset *op; - uint16_t qc; - uint16_t ret = ctx->ifc_sysctl_core_offset; + cpuset_t assigned_cpus; + unsigned int cores_consumed; + unsigned int base_cpuid = ctx->ifc_sysctl_core_offset; + unsigned int first_valid; + unsigned int last_valid; + unsigned int i; - if (ret != CORE_OFFSET_UNSPECIFIED) - return (ret); + first_valid = CPU_FFS(&ctx->ifc_cpus) - 1; + last_valid = CPU_FLS(&ctx->ifc_cpus) - 1; - if (ctx->ifc_sysctl_separate_txrx) - qc = scctx->isc_ntxqsets + scctx->isc_nrxqsets; - else - qc = max(scctx->isc_ntxqsets, scctx->isc_nrxqsets); + if (base_cpuid != CORE_OFFSET_UNSPECIFIED) { + /* + * Align the user-chosen base CPU ID to the next valid CPU + * for this device. If the chosen base CPU ID is smaller + * than the first valid CPU or larger than the last valid + * CPU, we assume the user does not know what the valid + * range is for this device and is thinking in terms of a + * zero-based reference frame, and so we shift the given + * value into the valid range (and wrap accordingly) so the + * intent is translated to the proper frame of reference. + * If the base CPU ID is within the valid first/last, but + * does not correspond to a valid CPU, it is advanced to the + * next valid CPU (wrapping if necessary). + */ + if (base_cpuid < first_valid || base_cpuid > last_valid) { + /* shift from zero-based to first_valid-based */ + base_cpuid += first_valid; + /* wrap to range [first_valid, last_valid] */ + base_cpuid = (base_cpuid - first_valid) % + (last_valid - first_valid + 1); + } + if (!CPU_ISSET(base_cpuid, &ctx->ifc_cpus)) { + /* + * base_cpuid is in [first_valid, last_valid], but + * not a member of the valid set. In this case, + * there will always be a member of the valid set + * with a CPU ID that is greater than base_cpuid, + * and we simply advance to it. + */ + while (!CPU_ISSET(base_cpuid, &ctx->ifc_cpus)) + base_cpuid++; + } + return (base_cpuid); + } + + /* + * Determine how many cores will be consumed by performing the CPU + * assignments and counting how many of the assigned CPUs correspond + * to CPUs in the set of context CPUs. This is done using the CPU + * ID first_valid as the base CPU ID, as the base CPU must be within + * the set of context CPUs. + * + * Note not all assigned CPUs will be in the set of context CPUs + * when separate CPUs are being allocated to TX and RX queues, + * assignment to logical cores has been enabled, the set of context + * CPUs contains only physical CPUs, and TX queues are mapped to L2 + * neighbors of CPUs that RX queues have been mapped to - in this + * case we do only want to count how many CPUs in the set of context + * CPUs have been consumed, as that determines the next CPU in that + * set to start allocating at for the next device for which + * core_offset is not set. + */ + CPU_ZERO(&assigned_cpus); + for (i = 0; i < scctx->isc_ntxqsets; i++) + CPU_SET(get_cpuid_for_queue(ctx, first_valid, i, true), + &assigned_cpus); + for (i = 0; i < scctx->isc_nrxqsets; i++) + CPU_SET(get_cpuid_for_queue(ctx, first_valid, i, false), + &assigned_cpus); + CPU_AND(&assigned_cpus, &ctx->ifc_cpus); + cores_consumed = CPU_COUNT(&assigned_cpus); mtx_lock(&cpu_offset_mtx); SLIST_FOREACH(op, &cpu_offsets, entries) { if (CPU_CMP(&ctx->ifc_cpus, &op->set) == 0) { - ret = op->offset; - op->offset += qc; + base_cpuid = op->next_cpuid; + op->next_cpuid = cpuid_advance(ctx, op->next_cpuid, + cores_consumed); MPASS(op->refcount < UINT_MAX); op->refcount++; break; } } - if (ret == CORE_OFFSET_UNSPECIFIED) { - ret = 0; + if (base_cpuid == CORE_OFFSET_UNSPECIFIED) { + base_cpuid = first_valid; op = malloc(sizeof(struct cpu_offset), M_IFLIB, M_NOWAIT | M_ZERO); if (op == NULL) { device_printf(ctx->ifc_dev, "allocation for cpu offset failed.\n"); } else { - op->offset = qc; + op->next_cpuid = cpuid_advance(ctx, base_cpuid, + cores_consumed); op->refcount = 1; CPU_COPY(&ctx->ifc_cpus, &op->set); SLIST_INSERT_HEAD(&cpu_offsets, op, entries); @@ -4619,7 +5082,7 @@ get_ctx_core_offset(if_ctx_t ctx) } mtx_unlock(&cpu_offset_mtx); - return (ret); + return (base_cpuid); #else return 0; #endif @@ -4656,7 +5119,7 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct kobjop_desc_t kobj_desc; kobj_method_t *kobj_method; int err, msix, rid; - uint16_t main_rxq, main_txq; + int num_txd, num_rxd; ctx = malloc(sizeof(* ctx), M_IFLIB, M_WAITOK|M_ZERO); @@ -4696,29 +5159,30 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct MPASS(scctx->isc_tx_csum_flags); #endif - if_setcapabilities(ifp, scctx->isc_capabilities | IFCAP_HWSTATS); - if_setcapenable(ifp, scctx->isc_capenable | IFCAP_HWSTATS); + if_setcapabilities(ifp, + scctx->isc_capabilities | IFCAP_HWSTATS | IFCAP_MEXTPG); + if_setcapenable(ifp, + scctx->isc_capenable | IFCAP_HWSTATS | IFCAP_MEXTPG); if (scctx->isc_ntxqsets == 0 || (scctx->isc_ntxqsets_max && scctx->isc_ntxqsets_max < scctx->isc_ntxqsets)) scctx->isc_ntxqsets = scctx->isc_ntxqsets_max; if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; - main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; - main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; + num_txd = iflib_num_tx_descs(ctx); + num_rxd = iflib_num_rx_descs(ctx); /* XXX change for per-queue sizes */ device_printf(dev, "Using %d TX descriptors and %d RX descriptors\n", - scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); + num_txd, num_rxd); - if (scctx->isc_tx_nsegments > scctx->isc_ntxd[main_txq] / - MAX_SINGLE_PACKET_FRACTION) - scctx->isc_tx_nsegments = max(1, scctx->isc_ntxd[main_txq] / + if (scctx->isc_tx_nsegments > num_txd / MAX_SINGLE_PACKET_FRACTION) + scctx->isc_tx_nsegments = max(1, num_txd / MAX_SINGLE_PACKET_FRACTION); - if (scctx->isc_tx_tso_segments_max > scctx->isc_ntxd[main_txq] / + if (scctx->isc_tx_tso_segments_max > num_txd / MAX_SINGLE_PACKET_FRACTION) scctx->isc_tx_tso_segments_max = max(1, - scctx->isc_ntxd[main_txq] / MAX_SINGLE_PACKET_FRACTION); + num_txd / MAX_SINGLE_PACKET_FRACTION); /* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */ if (if_getcapabilities(ifp) & IFCAP_TSO) { @@ -4755,7 +5219,9 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct if (bus_get_cpus(dev, INTR_CPUS, sizeof(ctx->ifc_cpus), &ctx->ifc_cpus) != 0) { device_printf(dev, "Unable to fetch CPU list\n"); CPU_COPY(&all_cpus, &ctx->ifc_cpus); - } + ctx->ifc_cpus_are_physical_cores = false; + } else + ctx->ifc_cpus_are_physical_cores = true; MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0); #endif @@ -4792,18 +5258,6 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct */ ctx->ifc_sysctl_core_offset = get_ctx_core_offset(ctx); - /* - * Group taskqueues aren't properly set up until SMP is started, - * so we disable interrupts until we can handle them post - * SI_SUB_SMP. - * - * XXX: disabling interrupts doesn't actually work, at least for - * the non-MSI case. When they occur before SI_SUB_SMP completes, - * we do null handling and depend on this not causing too large an - * interrupt storm. - */ - IFDI_INTR_DISABLE(ctx); - if (msix > 1) { /* * When using MSI-X, ensure that ifdi_{r,t}x_queue_intr_enable @@ -4813,7 +5267,7 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct #ifdef __HAIKU__ kobj_method = kobj_lookup_method(ctx->ops.cls, NULL, #else - kobj_method = kobj_lookup_method(((kobj_t)ctx), NULL, + kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, #endif kobj_desc); if (kobj_method == &kobj_desc->deflt) { @@ -4826,7 +5280,7 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct #ifdef __HAIKU__ kobj_method = kobj_lookup_method(ctx->ops.cls, NULL, #else - kobj_method = kobj_lookup_method(((kobj_t)ctx), NULL, + kobj_method = kobj_lookup_method(((kobj_t)ctx)->ops->cls, NULL, #endif kobj_desc); if (kobj_method == &kobj_desc->deflt) { @@ -4861,7 +5315,7 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct device_printf(dev, "Cannot use iflib with only 1 MSI-X interrupt!\n"); err = ENODEV; - goto fail_intr_free; + goto fail_queues; } ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet); @@ -4885,7 +5339,7 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct } *ctxp = ctx; - NETDUMP_SET(ctx->ifc_ifp, iflib); + DEBUGNET_SET(ctx->ifc_ifp, iflib); if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); iflib_add_device_sysctl_post(ctx); @@ -4897,13 +5351,14 @@ iflib_device_register(device_t dev, void *sc, if_shared_ctx_t sctx, if_ctx_t *ct fail_detach: ether_ifdetach(ctx->ifc_ifp); -fail_intr_free: - iflib_free_intr_mem(ctx); fail_queues: + iflib_tqg_detach(ctx); iflib_tx_structures_free(ctx); iflib_rx_structures_free(ctx); - taskqgroup_detach(qgroup_if_config_tqg, &ctx->ifc_admin_task); IFDI_DETACH(ctx); + IFDI_QUEUES_FREE(ctx); +fail_intr_free: + iflib_free_intr_mem(ctx); fail_unlock: CTX_UNLOCK(ctx); iflib_deregister(ctx); @@ -4919,14 +5374,13 @@ int iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, struct iflib_cloneattach_ctx *clctx) { + int num_txd, num_rxd; int err; if_ctx_t ctx; if_t ifp; if_softc_ctx_t scctx; int i; void *sc; - uint16_t main_txq; - uint16_t main_rxq; ctx = malloc(sizeof(*ctx), M_IFLIB, M_WAITOK|M_ZERO); sc = malloc(sctx->isc_driver->size, M_IFLIB, M_WAITOK|M_ZERO); @@ -4960,12 +5414,8 @@ iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, if ((err = IFDI_CLONEATTACH(ctx, clctx->cc_ifc, clctx->cc_name, clctx->cc_params)) != 0) { device_printf(dev, "IFDI_CLONEATTACH failed %d\n", err); - goto fail_ctx_free; + goto fail_unlock; } - ifmedia_add(ctx->ifc_mediap, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(ctx->ifc_mediap, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(ctx->ifc_mediap, IFM_ETHER | IFM_AUTO); - #ifdef INVARIANTS if (scctx->isc_capabilities & IFCAP_TXCSUM) MPASS(scctx->isc_tx_csum_flags); @@ -4976,7 +5426,14 @@ iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, ifp->if_flags |= IFF_NOGROUP; if (sctx->isc_flags & IFLIB_PSEUDO) { - ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet); + ifmedia_add(ctx->ifc_mediap, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(ctx->ifc_mediap, IFM_ETHER | IFM_AUTO); + if (sctx->isc_flags & IFLIB_PSEUDO_ETHER) { + ether_ifattach(ctx->ifc_ifp, ctx->ifc_mac.octet); + } else { + if_attach(ctx->ifc_ifp); + bpfattach(ctx->ifc_ifp, DLT_NULL, sizeof(u_int32_t)); + } if ((err = IFDI_ATTACH_POST(ctx)) != 0) { device_printf(dev, "IFDI_ATTACH_POST failed %d\n", err); @@ -4996,8 +5453,13 @@ iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, if_setgetcounterfn(ctx->ifc_ifp, iflib_if_get_counter); iflib_add_device_sysctl_post(ctx); ctx->ifc_flags |= IFC_INIT_DONE; + CTX_UNLOCK(ctx); return (0); } + ifmedia_add(ctx->ifc_mediap, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); + ifmedia_add(ctx->ifc_mediap, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(ctx->ifc_mediap, IFM_ETHER | IFM_AUTO); + _iflib_pre_assert(scctx); ctx->ifc_txrx = *scctx->isc_txrx; @@ -5006,21 +5468,20 @@ iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, if (scctx->isc_nrxqsets == 0 || (scctx->isc_nrxqsets_max && scctx->isc_nrxqsets_max < scctx->isc_nrxqsets)) scctx->isc_nrxqsets = scctx->isc_nrxqsets_max; - main_txq = (sctx->isc_flags & IFLIB_HAS_TXCQ) ? 1 : 0; - main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; + num_txd = iflib_num_tx_descs(ctx); + num_rxd = iflib_num_rx_descs(ctx); /* XXX change for per-queue sizes */ device_printf(dev, "Using %d TX descriptors and %d RX descriptors\n", - scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); + num_txd, num_rxd); - if (scctx->isc_tx_nsegments > scctx->isc_ntxd[main_txq] / - MAX_SINGLE_PACKET_FRACTION) - scctx->isc_tx_nsegments = max(1, scctx->isc_ntxd[main_txq] / + if (scctx->isc_tx_nsegments > num_txd / MAX_SINGLE_PACKET_FRACTION) + scctx->isc_tx_nsegments = max(1, num_txd / MAX_SINGLE_PACKET_FRACTION); - if (scctx->isc_tx_tso_segments_max > scctx->isc_ntxd[main_txq] / + if (scctx->isc_tx_tso_segments_max > num_txd / MAX_SINGLE_PACKET_FRACTION) scctx->isc_tx_tso_segments_max = max(1, - scctx->isc_ntxd[main_txq] / MAX_SINGLE_PACKET_FRACTION); + num_txd / MAX_SINGLE_PACKET_FRACTION); /* TSO parameters - dig these out of the data sheet - simply correspond to tag setup */ if (if_getcapabilities(ifp) & IFCAP_TSO) { @@ -5098,10 +5559,12 @@ iflib_pseudo_register(device_t dev, if_shared_ctx_t sctx, if_ctx_t *ctxp, fail_detach: ether_ifdetach(ctx->ifc_ifp); fail_queues: + iflib_tqg_detach(ctx); iflib_tx_structures_free(ctx); iflib_rx_structures_free(ctx); fail_iflib_detach: IFDI_DETACH(ctx); + IFDI_QUEUES_FREE(ctx); fail_unlock: CTX_UNLOCK(ctx); iflib_deregister(ctx); @@ -5115,35 +5578,24 @@ int iflib_pseudo_deregister(if_ctx_t ctx) { if_t ifp = ctx->ifc_ifp; - iflib_txq_t txq; - iflib_rxq_t rxq; - int i, j; - struct taskqgroup *tqg; - iflib_fl_t fl; + if_shared_ctx_t sctx = ctx->ifc_sctx; - ether_ifdetach(ifp); - /* XXX drain any dependent tasks */ - tqg = qgroup_if_io_tqg; - for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { - callout_drain(&txq->ift_timer); - if (txq->ift_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &txq->ift_task); + /* Unregister VLAN event handlers early */ + iflib_unregister_vlan_handlers(ctx); + + if ((sctx->isc_flags & IFLIB_PSEUDO) && + (sctx->isc_flags & IFLIB_PSEUDO_ETHER) == 0) { + bpfdetach(ifp); + if_detach(ifp); + } else { + ether_ifdetach(ifp); } - for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { - if (rxq->ifr_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &rxq->ifr_task); - - for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) - free(fl->ifl_rx_bitmap, M_IFLIB); - } - tqg = qgroup_if_config_tqg; - if (ctx->ifc_admin_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &ctx->ifc_admin_task); - if (ctx->ifc_vflr_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &ctx->ifc_vflr_task); + iflib_tqg_detach(ctx); iflib_tx_structures_free(ctx); iflib_rx_structures_free(ctx); + IFDI_DETACH(ctx); + IFDI_QUEUES_FREE(ctx); iflib_deregister(ctx); @@ -5171,12 +5623,7 @@ int iflib_device_deregister(if_ctx_t ctx) { if_t ifp = ctx->ifc_ifp; - iflib_txq_t txq; - iflib_rxq_t rxq; device_t dev = ctx->ifc_dev; - int i, j; - struct taskqgroup *tqg; - iflib_fl_t fl; /* Make sure VLANS are not using driver */ if (if_vlantrunkinuse(ifp)) { @@ -5194,42 +5641,27 @@ iflib_device_deregister(if_ctx_t ctx) ctx->ifc_flags |= IFC_IN_DETACH; STATE_UNLOCK(ctx); + /* Unregister VLAN handlers before calling iflib_stop() */ + iflib_unregister_vlan_handlers(ctx); + + iflib_netmap_detach(ifp); + ether_ifdetach(ifp); + CTX_LOCK(ctx); iflib_stop(ctx); CTX_UNLOCK(ctx); - /* Unregister VLAN events */ - if (ctx->ifc_vlan_attach_event != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); - if (ctx->ifc_vlan_detach_event != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); - - iflib_netmap_detach(ifp); - ether_ifdetach(ifp); iflib_rem_pfil(ctx); if (ctx->ifc_led_dev != NULL) led_destroy(ctx->ifc_led_dev); - /* XXX drain any dependent tasks */ - tqg = qgroup_if_io_tqg; - for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { - callout_drain(&txq->ift_timer); - if (txq->ift_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &txq->ift_task); - } - for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { - if (rxq->ifr_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &rxq->ifr_task); - for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) - free(fl->ifl_rx_bitmap, M_IFLIB); - } - tqg = qgroup_if_config_tqg; - if (ctx->ifc_admin_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &ctx->ifc_admin_task); - if (ctx->ifc_vflr_task.gt_uniq != NULL) - taskqgroup_detach(tqg, &ctx->ifc_vflr_task); + iflib_tqg_detach(ctx); + iflib_tx_structures_free(ctx); + iflib_rx_structures_free(ctx); + CTX_LOCK(ctx); IFDI_DETACH(ctx); + IFDI_QUEUES_FREE(ctx); CTX_UNLOCK(ctx); /* ether_ifdetach calls if_qflush - lock must be destroy afterwards*/ @@ -5237,9 +5669,6 @@ iflib_device_deregister(if_ctx_t ctx) bus_generic_detach(dev); - iflib_tx_structures_free(ctx); - iflib_rx_structures_free(ctx); - iflib_deregister(ctx); device_set_softc(ctx->ifc_dev, NULL); @@ -5250,6 +5679,35 @@ iflib_device_deregister(if_ctx_t ctx) return (0); } +static void +iflib_tqg_detach(if_ctx_t ctx) +{ + iflib_txq_t txq; + iflib_rxq_t rxq; + int i; + struct taskqgroup *tqg; + + /* XXX drain any dependent tasks */ + tqg = qgroup_if_io_tqg; + for (txq = ctx->ifc_txqs, i = 0; i < NTXQSETS(ctx); i++, txq++) { + callout_drain(&txq->ift_timer); +#ifdef DEV_NETMAP + callout_drain(&txq->ift_netmap_timer); +#endif /* DEV_NETMAP */ + if (txq->ift_task.gt_uniq != NULL) + taskqgroup_detach(tqg, &txq->ift_task); + } + for (i = 0, rxq = ctx->ifc_rxqs; i < NRXQSETS(ctx); i++, rxq++) { + if (rxq->ifr_task.gt_uniq != NULL) + taskqgroup_detach(tqg, &rxq->ifr_task); + } + tqg = qgroup_if_config_tqg; + if (ctx->ifc_admin_task.gt_uniq != NULL) + taskqgroup_detach(tqg, &ctx->ifc_admin_task); + if (ctx->ifc_vflr_task.gt_uniq != NULL) + taskqgroup_detach(tqg, &ctx->ifc_vflr_task); +} + static void iflib_free_intr_mem(if_ctx_t ctx) { @@ -5298,19 +5756,17 @@ iflib_device_shutdown(device_t dev) return bus_generic_suspend(dev); } - int iflib_device_resume(device_t dev) { if_ctx_t ctx = device_get_softc(dev); iflib_txq_t txq = ctx->ifc_txqs; - int i; CTX_LOCK(ctx); IFDI_RESUME(ctx); iflib_if_init_locked(ctx); CTX_UNLOCK(ctx); - for (i = 0; i < NTXQSETS(ctx); i++, txq++) + for (int i = 0; i < NTXQSETS(ctx); i++, txq++) iflib_txq_check_drain(txq, IFLIB_RESTART_BUDGET); return (bus_generic_resume(dev)); @@ -5365,6 +5821,7 @@ iflib_device_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) static int iflib_module_init(void) { + iflib_timer_default = hz / 2; return (0); } @@ -5394,7 +5851,6 @@ iflib_module_event_handler(module_t mod, int what, void *arg) * **********************************************************************/ - static void _iflib_assert(if_shared_ctx_t sctx) { @@ -5448,12 +5904,22 @@ iflib_register(if_ctx_t ctx) driver_t *driver = sctx->isc_driver; device_t dev = ctx->ifc_dev; if_t ifp; + u_char type; + int iflags; - _iflib_assert(sctx); + if ((sctx->isc_flags & IFLIB_PSEUDO) == 0) + _iflib_assert(sctx); CTX_LOCK_INIT(ctx); STATE_LOCK_INIT(ctx, device_get_nameunit(ctx->ifc_dev)); - ifp = ctx->ifc_ifp = if_alloc(IFT_ETHER); + if (sctx->isc_flags & IFLIB_PSEUDO) { + if (sctx->isc_flags & IFLIB_PSEUDO_ETHER) + type = IFT_ETHER; + else + type = IFT_PPP; + } else + type = IFT_ETHER; + ifp = ctx->ifc_ifp = if_alloc(type); if (ifp == NULL) { device_printf(dev, "can not allocate ifnet structure\n"); return (ENOMEM); @@ -5464,9 +5930,6 @@ iflib_register(if_ctx_t ctx) */ kobj_init((kobj_t) ctx, (kobj_class_t) driver); kobj_class_compile((kobj_class_t) driver); -#ifndef __HAIKU__ - driver->refs++; -#endif if_initname(ifp, device_get_name(dev), device_get_unit(dev)); if_setsoftc(ifp, ctx); @@ -5481,8 +5944,18 @@ iflib_register(if_ctx_t ctx) if_settransmitfn(ifp, iflib_if_transmit); #endif if_setqflushfn(ifp, iflib_if_qflush); - if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); +#ifndef __HAIKU__ + iflags = IFF_MULTICAST | IFF_KNOWSEPOCH; +#else + iflags = IFF_MULTICAST; +#endif + if ((sctx->isc_flags & IFLIB_PSEUDO) && + (sctx->isc_flags & IFLIB_PSEUDO_ETHER) == 0) + iflags |= IFF_POINTOPOINT; + else + iflags |= IFF_BROADCAST | IFF_SIMPLEX; + if_setflags(ifp, iflags); ctx->ifc_vlan_attach_event = EVENTHANDLER_REGISTER(vlan_config, iflib_vlan_register, ctx, EVENTHANDLER_PRI_FIRST); @@ -5491,7 +5964,6 @@ iflib_register(if_ctx_t ctx) EVENTHANDLER_PRI_FIRST); if ((sctx->isc_flags & IFLIB_DRIVER_MEDIA) == 0) { - dprintf("DRVMEDia"); ctx->ifc_mediap = &ctx->ifc_media; ifmedia_init(ctx->ifc_mediap, IFM_IMASK, iflib_media_change, iflib_media_status); @@ -5499,6 +5971,21 @@ iflib_register(if_ctx_t ctx) return (0); } +static void +iflib_unregister_vlan_handlers(if_ctx_t ctx) +{ + /* Unregister VLAN events */ + if (ctx->ifc_vlan_attach_event != NULL) { + EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); + ctx->ifc_vlan_attach_event = NULL; + } + if (ctx->ifc_vlan_detach_event != NULL) { + EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); + ctx->ifc_vlan_detach_event = NULL; + } + +} + static void iflib_deregister(if_ctx_t ctx) { @@ -5507,15 +5994,8 @@ iflib_deregister(if_ctx_t ctx) /* Remove all media */ ifmedia_removeall(&ctx->ifc_media); - /* Unregister VLAN events */ - if (ctx->ifc_vlan_attach_event != NULL) { - EVENTHANDLER_DEREGISTER(vlan_config, ctx->ifc_vlan_attach_event); - ctx->ifc_vlan_attach_event = NULL; - } - if (ctx->ifc_vlan_detach_event != NULL) { - EVENTHANDLER_DEREGISTER(vlan_unconfig, ctx->ifc_vlan_detach_event); - ctx->ifc_vlan_detach_event = NULL; - } + /* Ensure that VLAN event handlers are unregistered */ + iflib_unregister_vlan_handlers(ctx); #ifndef __HAIKU__ /* Release kobject reference */ @@ -5549,11 +6029,14 @@ iflib_queues_alloc(if_ctx_t ctx) uint8_t nrxqs = sctx->isc_nrxqs; uint8_t ntxqs = sctx->isc_ntxqs; int nfree_lists = sctx->isc_nfl ? sctx->isc_nfl : 1; + int fl_offset = (sctx->isc_flags & IFLIB_HAS_RXCQ ? 1 : 0); caddr_t *vaddrs; uint64_t *paddrs; KASSERT(ntxqs > 0, ("number of queues per qset must be at least 1")); KASSERT(nrxqs > 0, ("number of queues per qset must be at least 1")); + KASSERT(nrxqs >= fl_offset + nfree_lists, + ("there must be at least a rxq for each free list")); /* Allocate the TX ring struct memory */ if (!(ctx->ifc_txqs = @@ -5607,10 +6090,6 @@ iflib_queues_alloc(if_ctx_t ctx) } else { txq->ift_br_offset = 0; } -#ifndef __HAIKU__ - /* XXX fix this */ - txq->ift_timer.c_cpu = cpu; -#endif if (iflib_txsd_alloc(txq)) { device_printf(dev, "Critical Failure setting up TX buffers\n"); @@ -5623,6 +6102,13 @@ iflib_queues_alloc(if_ctx_t ctx) device_get_nameunit(dev), txq->ift_id); mtx_init(&txq->ift_mtx, txq->ift_mtx_name, NULL, MTX_DEF); callout_init_mtx(&txq->ift_timer, &txq->ift_mtx, 0); +#ifndef __HAIKU__ + txq->ift_timer.c_cpu = cpu; +#endif +#ifdef DEV_NETMAP + callout_init_mtx(&txq->ift_netmap_timer, &txq->ift_mtx, 0); + txq->ift_netmap_timer.c_cpu = cpu; +#endif /* DEV_NETMAP */ err = ifmp_ring_alloc(&txq->ift_br, 2048, txq, iflib_txq_drain, iflib_txq_can_drain, M_IFLIB, M_WAITOK); @@ -5635,6 +6121,7 @@ iflib_queues_alloc(if_ctx_t ctx) for (rxconf = i = 0; i < nrxqsets; i++, rxconf++, rxq++) { /* Set up some basics */ + callout_init(&rxq->ifr_watchdog, 1); if ((ifdip = malloc(sizeof(struct iflib_dma_info) * nrxqs, M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { @@ -5659,11 +6146,7 @@ iflib_queues_alloc(if_ctx_t ctx) } rxq->ifr_ctx = ctx; rxq->ifr_id = i; - if (sctx->isc_flags & IFLIB_HAS_RXCQ) { - rxq->ifr_fl_offset = 1; - } else { - rxq->ifr_fl_offset = 0; - } + rxq->ifr_fl_offset = fl_offset; rxq->ifr_nfl = nfree_lists; if (!(fl = (iflib_fl_t) malloc(sizeof(struct iflib_fl) * nfree_lists, M_IFLIB, M_NOWAIT | M_ZERO))) { @@ -5686,7 +6169,7 @@ iflib_queues_alloc(if_ctx_t ctx) goto err_rx_desc; } - for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) + for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) fl->ifl_rx_bitmap = bit_alloc(fl->ifl_size, M_IFLIB, M_WAITOK); } @@ -5771,13 +6254,12 @@ iflib_tx_structures_free(if_ctx_t ctx) int i, j; for (i = 0; i < NTXQSETS(ctx); i++, txq++) { - iflib_txq_destroy(txq); for (j = 0; j < sctx->isc_ntxqs; j++) iflib_dma_free(&txq->ift_ifdi[j]); + iflib_txq_destroy(txq); } free(ctx->ifc_txqs, M_IFLIB); ctx->ifc_txqs = NULL; - IFDI_QUEUES_FREE(ctx); } /********************************************************************* @@ -5791,7 +6273,7 @@ iflib_rx_structures_setup(if_ctx_t ctx) iflib_rxq_t rxq = ctx->ifc_rxqs; int q; #if defined(INET6) || defined(INET) - int i, err; + int err, i; #endif for (q = 0; q < ctx->ifc_softc_ctx.isc_nrxqsets; q++, rxq++) { @@ -5835,9 +6317,12 @@ static void iflib_rx_structures_free(if_ctx_t ctx) { iflib_rxq_t rxq = ctx->ifc_rxqs; - int i; + if_shared_ctx_t sctx = ctx->ifc_sctx; + int i, j; for (i = 0; i < ctx->ifc_softc_ctx.isc_nrxqsets; i++, rxq++) { + for (j = 0; j < sctx->isc_nrxqs; j++) + iflib_dma_free(&rxq->ifr_ifdi[j]); iflib_rx_sds_free(rxq); #if defined(INET6) || defined(INET) if (if_getcapabilities(ctx->ifc_ifp) & IFCAP_LRO) @@ -5876,128 +6361,6 @@ iflib_irq_alloc(if_ctx_t ctx, if_irq_t irq, int rid, return (_iflib_irq_alloc(ctx, irq, rid, filter, handler, arg, name)); } -#ifdef SMP -static int -find_nth(if_ctx_t ctx, int qid) -{ - cpuset_t cpus; - int i, cpuid, eqid, count; - - CPU_COPY(&ctx->ifc_cpus, &cpus); - count = CPU_COUNT(&cpus); - eqid = qid % count; - /* clear up to the qid'th bit */ - for (i = 0; i < eqid; i++) { - cpuid = CPU_FFS(&cpus); - MPASS(cpuid != 0); - CPU_CLR(cpuid-1, &cpus); - } - cpuid = CPU_FFS(&cpus); - MPASS(cpuid != 0); - return (cpuid-1); -} - -#ifdef SCHED_ULE -extern struct cpu_group *cpu_top; /* CPU topology */ - -static int -find_child_with_core(int cpu, struct cpu_group *grp) -{ - int i; - - if (grp->cg_children == 0) - return -1; - - MPASS(grp->cg_child); - for (i = 0; i < grp->cg_children; i++) { - if (CPU_ISSET(cpu, &grp->cg_child[i].cg_mask)) - return i; - } - - return -1; -} - -/* - * Find the nth "close" core to the specified core - * "close" is defined as the deepest level that shares - * at least an L2 cache. With threads, this will be - * threads on the same core. If the shared cache is L3 - * or higher, simply returns the same core. - */ -static int -find_close_core(int cpu, int core_offset) -{ - struct cpu_group *grp; - int i; - int fcpu; - cpuset_t cs; - - grp = cpu_top; - if (grp == NULL) - return cpu; - i = 0; - while ((i = find_child_with_core(cpu, grp)) != -1) { - /* If the child only has one cpu, don't descend */ - if (grp->cg_child[i].cg_count <= 1) - break; - grp = &grp->cg_child[i]; - } - - /* If they don't share at least an L2 cache, use the same CPU */ - if (grp->cg_level > CG_SHARE_L2 || grp->cg_level == CG_SHARE_NONE) - return cpu; - - /* Now pick one */ - CPU_COPY(&grp->cg_mask, &cs); - - /* Add the selected CPU offset to core offset. */ - for (i = 0; (fcpu = CPU_FFS(&cs)) != 0; i++) { - if (fcpu - 1 == cpu) - break; - CPU_CLR(fcpu - 1, &cs); - } - MPASS(fcpu); - - core_offset += i; - - CPU_COPY(&grp->cg_mask, &cs); - for (i = core_offset % grp->cg_count; i > 0; i--) { - MPASS(CPU_FFS(&cs)); - CPU_CLR(CPU_FFS(&cs) - 1, &cs); - } - MPASS(CPU_FFS(&cs)); - return CPU_FFS(&cs) - 1; -} -#else -static int -find_close_core(int cpu, int core_offset __unused) -{ - return cpu; -} -#endif - -static int -get_core_offset(if_ctx_t ctx, iflib_intr_type_t type, int qid) -{ - switch (type) { - case IFLIB_INTR_TX: - /* TX queues get cores which share at least an L2 cache with the corresponding RX queue */ - /* XXX handle multiple RX threads per core and more than two core per L2 group */ - return qid / CPU_COUNT(&ctx->ifc_cpus) + 1; - case IFLIB_INTR_RX: - case IFLIB_INTR_RXTX: - /* RX queues get the specified core */ - return qid / CPU_COUNT(&ctx->ifc_cpus); - default: - return -1; - } -} -#else -#define get_core_offset(ctx, type, qid) CPU_FIRST() -#define find_close_core(cpuid, tid) CPU_FIRST() -#define find_nth(ctx, gid) CPU_FIRST() -#endif - /* Just to avoid copy/paste */ static inline int iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, @@ -6005,21 +6368,14 @@ iflib_irq_set_affinity(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, const char *name) { device_t dev; - int co, cpuid, err, tid; + unsigned int base_cpuid, cpuid; + int err; dev = ctx->ifc_dev; - co = ctx->ifc_sysctl_core_offset; - if (ctx->ifc_sysctl_separate_txrx && type == IFLIB_INTR_TX) - co += ctx->ifc_softc_ctx.isc_nrxqsets; - cpuid = find_nth(ctx, qid + co); - tid = get_core_offset(ctx, type, qid); - if (tid < 0) { - device_printf(dev, "get_core_offset failed\n"); - return (EOPNOTSUPP); - } - cpuid = find_close_core(cpuid, tid); - err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, irq->ii_res, - name); + base_cpuid = ctx->ifc_sysctl_core_offset; + cpuid = get_cpuid_for_queue(ctx, base_cpuid, qid, type == IFLIB_INTR_TX); + err = taskqgroup_attach_cpu(tqg, gtask, uniq, cpuid, dev, + irq ? irq->ii_res : NULL, name); if (err) { device_printf(dev, "taskqgroup_attach_cpu failed %d\n", err); return (err); @@ -6067,7 +6423,7 @@ iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, tqg = qgroup_if_io_tqg; fn = _task_fn_rx; intr_fast = iflib_fast_intr; - GROUPTASK_INIT(gtask, 0, fn, q); + NET_GROUPTASK_INIT(gtask, 0, fn, q); break; case IFLIB_INTR_RXTX: q = &ctx->ifc_rxqs[qid]; @@ -6076,7 +6432,7 @@ iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, tqg = qgroup_if_io_tqg; fn = _task_fn_rx; intr_fast = iflib_fast_intr_rxtx; - GROUPTASK_INIT(gtask, 0, fn, q); + NET_GROUPTASK_INIT(gtask, 0, fn, q); break; case IFLIB_INTR_ADMIN: q = ctx; @@ -6108,8 +6464,8 @@ iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, return (0); if (tqrid != -1) { - err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg, - q, name); + err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg, q, + name); if (err) return (err); } else { @@ -6122,6 +6478,7 @@ iflib_irq_alloc_generic(if_ctx_t ctx, if_irq_t irq, int rid, void iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, void *arg, int qid, const char *name) { + device_t dev; struct grouptask *gtask; struct taskqgroup *tqg; gtask_fn_t *fn; @@ -6134,31 +6491,30 @@ iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq, iflib_intr_type_t type, gtask = &ctx->ifc_txqs[qid].ift_task; tqg = qgroup_if_io_tqg; fn = _task_fn_tx; + GROUPTASK_INIT(gtask, 0, fn, q); break; case IFLIB_INTR_RX: q = &ctx->ifc_rxqs[qid]; gtask = &ctx->ifc_rxqs[qid].ifr_task; tqg = qgroup_if_io_tqg; fn = _task_fn_rx; + NET_GROUPTASK_INIT(gtask, 0, fn, q); break; case IFLIB_INTR_IOV: q = ctx; gtask = &ctx->ifc_vflr_task; tqg = qgroup_if_config_tqg; fn = _task_fn_iov; + GROUPTASK_INIT(gtask, 0, fn, q); break; default: panic("unknown net intr type"); } - GROUPTASK_INIT(gtask, 0, fn, q); - if (irq != NULL) { - err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg, - q, name); - if (err) - taskqgroup_attach(tqg, gtask, q, ctx->ifc_dev, - irq->ii_res, name); - } else { - taskqgroup_attach(tqg, gtask, q, NULL, NULL, name); + err = iflib_irq_set_affinity(ctx, irq, type, qid, gtask, tqg, q, name); + if (err) { + dev = ctx->ifc_dev; + taskqgroup_attach(tqg, gtask, q, dev, irq ? irq->ii_res : NULL, + name); } } @@ -6189,29 +6545,30 @@ iflib_legacy_setup(if_ctx_t ctx, driver_filter_t filter, void *filter_arg, int * struct grouptask *gtask; struct resource *res; struct taskqgroup *tqg; - gtask_fn_t *fn; void *q; int err, tqrid; + bool rx_only; q = &ctx->ifc_rxqs[0]; info = &rxq[0].ifr_filter_info; gtask = &rxq[0].ifr_task; tqg = qgroup_if_io_tqg; tqrid = *rid; - fn = _task_fn_rx; + rx_only = (ctx->ifc_sctx->isc_flags & IFLIB_SINGLE_IRQ_RX_ONLY) != 0; ctx->ifc_flags |= IFC_LEGACY; info->ifi_filter = filter; info->ifi_filter_arg = filter_arg; info->ifi_task = gtask; - info->ifi_ctx = q; + info->ifi_ctx = rx_only ? ctx : q; dev = ctx->ifc_dev; /* We allocate a single interrupt resource */ - if ((err = _iflib_irq_alloc(ctx, irq, tqrid, iflib_fast_intr_rxtx, - NULL, info, name)) != 0) + err = _iflib_irq_alloc(ctx, irq, tqrid, rx_only ? iflib_fast_intr_ctx : + iflib_fast_intr_rxtx, NULL, info, name); + if (err != 0) return (err); - GROUPTASK_INIT(gtask, 0, fn, q); + NET_GROUPTASK_INIT(gtask, 0, _task_fn_rx, q); res = irq->ii_res; taskqgroup_attach(tqg, gtask, q, dev, res, name); @@ -6246,13 +6603,8 @@ iflib_rx_intr_deferred(if_ctx_t ctx, int rxqid) void iflib_admin_intr_deferred(if_ctx_t ctx) { -#ifdef INVARIANTS - struct grouptask *gtask; - - gtask = &ctx->ifc_admin_task; - MPASS(gtask != NULL && gtask->gt_taskqueue != NULL); -#endif + MPASS(ctx->ifc_admin_task.gt_taskqueue != NULL); GROUPTASK_ENQUEUE(&ctx->ifc_admin_task); } @@ -6285,7 +6637,7 @@ void iflib_config_gtask_deinit(struct grouptask *gtask) { - taskqgroup_detach(qgroup_if_config_tqg, gtask); + taskqgroup_detach(qgroup_if_config_tqg, gtask); } void @@ -6302,8 +6654,7 @@ iflib_link_state_change(if_ctx_t ctx, int link_state, uint64_t baudrate) } /* If link down, disable watchdog */ if ((ctx->ifc_link_state == LINK_STATE_UP) && (link_state == LINK_STATE_DOWN)) { - int i; - for (i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++) + for (int i = 0; i < ctx->ifc_softc_ctx.isc_ntxqsets; i++, txq++) txq->ift_qstatus = IFLIB_QUEUE_IDLE; } ctx->ifc_link_state = link_state; @@ -6355,7 +6706,7 @@ iflib_add_int_delay_sysctl(if_ctx_t ctx, const char *name, info->iidi_value = value; SYSCTL_ADD_PROC(device_get_sysctl_ctx(ctx->ifc_dev), SYSCTL_CHILDREN(device_get_sysctl_tree(ctx->ifc_dev)), - OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, + OID_AUTO, name, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, info, 0, iflib_sysctl_int_delay, "I", description); } @@ -6622,7 +6973,7 @@ iflib_add_device_sysctl_pre(if_ctx_t ctx) ctx_list = device_get_sysctl_ctx(dev); child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); ctx->ifc_sysctl_node = node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, "iflib", - CTLFLAG_RD, NULL, "IFLIB fields"); + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "IFLIB fields"); oid_list = SYSCTL_CHILDREN(node); SYSCTL_ADD_CONST_STRING(ctx_list, oid_list, OID_AUTO, "driver_version", @@ -6654,16 +7005,19 @@ iflib_add_device_sysctl_pre(if_ctx_t ctx) SYSCTL_ADD_U8(ctx_list, oid_list, OID_AUTO, "separate_txrx", CTLFLAG_RDTUN, &ctx->ifc_sysctl_separate_txrx, 0, "use separate cores for TX and RX"); + SYSCTL_ADD_U8(ctx_list, oid_list, OID_AUTO, "use_logical_cores", + CTLFLAG_RDTUN, &ctx->ifc_sysctl_use_logical_cores, 0, + "try to make use of logical cores for TX and RX"); /* XXX change for per-queue sizes */ SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_ntxds", - CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NTXD_HANDLER, - mp_ndesc_handler, "A", - "list of # of TX descriptors to use, 0 = use default #"); + CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, ctx, + IFLIB_NTXD_HANDLER, mp_ndesc_handler, "A", + "list of # of TX descriptors to use, 0 = use default #"); SYSCTL_ADD_PROC(ctx_list, oid_list, OID_AUTO, "override_nrxds", - CTLTYPE_STRING|CTLFLAG_RWTUN, ctx, IFLIB_NRXD_HANDLER, - mp_ndesc_handler, "A", - "list of # of RX descriptors to use, 0 = use default #"); + CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, ctx, + IFLIB_NRXD_HANDLER, mp_ndesc_handler, "A", + "list of # of RX descriptors to use, 0 = use default #"); #endif } @@ -6698,8 +7052,11 @@ iflib_add_device_sysctl_post(if_ctx_t ctx) for (i = 0, txq = ctx->ifc_txqs; i < scctx->isc_ntxqsets; i++, txq++) { snprintf(namebuf, NAME_BUFLEN, qfmt, i); queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "Queue Name"); + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); + SYSCTL_ADD_INT(ctx_list, queue_list, OID_AUTO, "cpu", + CTLFLAG_RD, + &txq->ift_task.gt_cpu, 0, "cpu this queue is bound to"); #if MEMORY_LOGGING SYSCTL_ADD_QUAD(ctx_list, queue_list, OID_AUTO, "txq_dequeued", CTLFLAG_RD, @@ -6748,8 +7105,9 @@ iflib_add_device_sysctl_post(if_ctx_t ctx) CTLFLAG_RD, &txq->ift_cleaned, "total cleaned"); SYSCTL_ADD_PROC(ctx_list, queue_list, OID_AUTO, "ring_state", - CTLTYPE_STRING | CTLFLAG_RD, __DEVOLATILE(uint64_t *, &txq->ift_br->state), - 0, mp_ring_state_handler, "A", "soft ring state"); + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, + __DEVOLATILE(uint64_t *, &txq->ift_br->state), 0, + mp_ring_state_handler, "A", "soft ring state"); SYSCTL_ADD_COUNTER_U64(ctx_list, queue_list, OID_AUTO, "r_enqueues", CTLFLAG_RD, &txq->ift_br->enqueues, "# of enqueues to the mp_ring for this queue"); @@ -6779,8 +7137,11 @@ iflib_add_device_sysctl_post(if_ctx_t ctx) for (i = 0, rxq = ctx->ifc_rxqs; i < scctx->isc_nrxqsets; i++, rxq++) { snprintf(namebuf, NAME_BUFLEN, qfmt, i); queue_node = SYSCTL_ADD_NODE(ctx_list, child, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "Queue Name"); + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); + SYSCTL_ADD_INT(ctx_list, queue_list, OID_AUTO, "cpu", + CTLFLAG_RD, + &rxq->ifr_task.gt_cpu, 0, "cpu this queue is bound to"); if (sctx->isc_flags & IFLIB_HAS_RXCQ) { SYSCTL_ADD_U16(ctx_list, queue_list, OID_AUTO, "rxq_cq_cidx", CTLFLAG_RD, @@ -6790,7 +7151,7 @@ iflib_add_device_sysctl_post(if_ctx_t ctx) for (j = 0, fl = rxq->ifr_fl; j < rxq->ifr_nfl; j++, fl++) { snprintf(namebuf, NAME_BUFLEN, "rxq_fl%d", j); fl_node = SYSCTL_ADD_NODE(ctx_list, queue_list, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "freelist Name"); + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist Name"); fl_list = SYSCTL_CHILDREN(fl_node); SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "pidx", CTLFLAG_RD, @@ -6801,6 +7162,9 @@ iflib_add_device_sysctl_post(if_ctx_t ctx) SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "credits", CTLFLAG_RD, &fl->ifl_credits, 1, "credits available"); + SYSCTL_ADD_U16(ctx_list, fl_list, OID_AUTO, "buf_size", + CTLFLAG_RD, + &fl->ifl_buf_size, 1, "buffer size"); #if MEMORY_LOGGING SYSCTL_ADD_QUAD(ctx_list, fl_list, OID_AUTO, "fl_m_enqueued", CTLFLAG_RD, @@ -6815,7 +7179,6 @@ iflib_add_device_sysctl_post(if_ctx_t ctx) CTLFLAG_RD, &fl->ifl_cl_dequeued, "clusters freed"); #endif - } } #endif @@ -6857,9 +7220,9 @@ iflib_fixup_rx(struct mbuf *m) } #endif -#ifdef NETDUMP +#ifdef DEBUGNET static void -iflib_netdump_init(if_t ifp, int *nrxr, int *ncl, int *clsize) +iflib_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize) { if_ctx_t ctx; @@ -6872,7 +7235,7 @@ iflib_netdump_init(if_t ifp, int *nrxr, int *ncl, int *clsize) } static void -iflib_netdump_event(if_t ifp, enum netdump_ev event) +iflib_debugnet_event(if_t ifp, enum debugnet_ev event) { if_ctx_t ctx; if_softc_ctx_t scctx; @@ -6884,8 +7247,8 @@ iflib_netdump_event(if_t ifp, enum netdump_ev event) scctx = &ctx->ifc_softc_ctx; switch (event) { + case DEBUGNET_START: #ifndef __HAIKU__ - case NETDUMP_START: for (i = 0; i < scctx->isc_nrxqsets; i++) { rxq = &ctx->ifc_rxqs[i]; for (j = 0; j < rxq->ifr_nfl; j++) { @@ -6902,7 +7265,7 @@ iflib_netdump_event(if_t ifp, enum netdump_ev event) } static int -iflib_netdump_transmit(if_t ifp, struct mbuf *m) +iflib_debugnet_transmit(if_t ifp, struct mbuf *m) { if_ctx_t ctx; iflib_txq_t txq; @@ -6916,13 +7279,14 @@ iflib_netdump_transmit(if_t ifp, struct mbuf *m) txq = &ctx->ifc_txqs[0]; error = iflib_encap(txq, &m); if (error == 0) - (void)iflib_txd_db_check(ctx, txq, true, txq->ift_in_use); + (void)iflib_txd_db_check(txq, true); return (error); } static int -iflib_netdump_poll(if_t ifp, int count) +iflib_debugnet_poll(if_t ifp, int count) { + struct epoch_tracker et; if_ctx_t ctx; if_softc_ctx_t scctx; iflib_txq_t txq; @@ -6938,8 +7302,10 @@ iflib_netdump_poll(if_t ifp, int count) txq = &ctx->ifc_txqs[0]; (void)iflib_completed_tx_reclaim(txq, RECLAIM_THRESH(ctx)); + NET_EPOCH_ENTER(et); for (i = 0; i < scctx->isc_nrxqsets; i++) (void)iflib_rxeof(&ctx->ifc_rxqs[i], 16 /* XXX */); + NET_EPOCH_EXIT(et); return (0); } -#endif /* NETDUMP */ +#endif /* DEBUGNET */ diff --git a/src/libs/compat/freebsd_iflib/mp_ring.c b/src/libs/compat/freebsd_iflib/mp_ring.c index 26e7922495..da1ce7676b 100644 --- a/src/libs/compat/freebsd_iflib/mp_ring.c +++ b/src/libs/compat/freebsd_iflib/mp_ring.c @@ -109,7 +109,6 @@ drain_ring_locked(struct ifmp_ring *r, union ring_state os, uint16_t prev, int b total = 0; while (cidx != pidx) { - /* Items from cidx to pidx are available for consumption. */ n = r->drain(r, cidx, pidx); if (n == 0) { @@ -184,7 +183,6 @@ drain_ring_lockless(struct ifmp_ring *r, union ring_state os, uint16_t prev, int total = 0; while (cidx != pidx) { - /* Items from cidx to pidx are available for consumption. */ n = r->drain(r, cidx, pidx); if (n == 0) { @@ -491,7 +489,6 @@ ifmp_ring_check_drainage(struct ifmp_ring *r, int budget) ns.state = os.state; ns.flags = BUSY; - #ifdef MP_RING_NO_64BIT_ATOMICS mtx_lock(&r->lock); if (r->state != os.state) { @@ -509,7 +506,6 @@ ifmp_ring_check_drainage(struct ifmp_ring *r, int budget) if (!atomic_cmpset_acq_64(&r->state, os.state, ns.state)) return; - drain_ring_lockless(r, ns, os.flags, budget); #endif } diff --git a/src/libs/compat/freebsd_iflib/nv_impl.h b/src/libs/compat/freebsd_iflib/nv_impl.h index 56e01a2bfb..1875c739be 100644 --- a/src/libs/compat/freebsd_iflib/nv_impl.h +++ b/src/libs/compat/freebsd_iflib/nv_impl.h @@ -52,13 +52,13 @@ typedef struct nvpair nvpair_t; #define NV_FLAG_IN_ARRAY 0x100 #ifdef _KERNEL -#define nv_malloc(size) malloc((size), M_NVLIST, M_WAITOK) +#define nv_malloc(size) malloc((size), M_NVLIST, M_NOWAIT) #define nv_calloc(n, size) mallocarray((n), (size), M_NVLIST, \ - M_WAITOK | M_ZERO) + M_NOWAIT | M_ZERO) #define nv_realloc(buf, size) realloc((buf), (size), M_NVLIST, \ - M_WAITOK) + M_NOWAIT) #define nv_free(buf) free((buf), M_NVLIST) -#define nv_strdup(buf) strdup((buf), M_NVLIST) +#define nv_strdup(buf) strdup_flags((buf), M_NVLIST, M_NOWAIT) #define nv_vasprintf(ptr, ...) vasprintf(ptr, M_NVLIST, __VA_ARGS__) #define ERRNO_SET(var) do { } while (0) @@ -103,6 +103,7 @@ bool nvlist_move_nvpair(nvlist_t *nvl, nvpair_t *nvp); void nvlist_set_parent(nvlist_t *nvl, nvpair_t *parent); void nvlist_set_array_next(nvlist_t *nvl, nvpair_t *ele); +nvpair_t *nvlist_get_array_next_nvpair(nvlist_t *nvl); const nvpair_t *nvlist_get_nvpair(const nvlist_t *nvl, const char *name); diff --git a/src/libs/compat/freebsd_iflib/nvlist.c b/src/libs/compat/freebsd_iflib/nvlist.c index d228b0616d..31ab62abeb 100644 --- a/src/libs/compat/freebsd_iflib/nvlist.c +++ b/src/libs/compat/freebsd_iflib/nvlist.c @@ -99,6 +99,7 @@ struct nvlist { int nvl_magic; int nvl_error; int nvl_flags; + size_t nvl_datasize; nvpair_t *nvl_parent; nvpair_t *nvl_array_next; struct nvl_head nvl_head; @@ -139,6 +140,7 @@ nvlist_create(int flags) nvl->nvl_flags = flags; nvl->nvl_parent = NULL; nvl->nvl_array_next = NULL; + nvl->nvl_datasize = sizeof(struct nvlist_header); TAILQ_INIT(&nvl->nvl_head); nvl->nvl_magic = NVLIST_MAGIC; @@ -247,6 +249,60 @@ nvlist_set_array_next(nvlist_t *nvl, nvpair_t *ele) nvl->nvl_array_next = ele; } +static void +nvlist_update_size(nvlist_t *nvl, nvpair_t *new, ssize_t mul) +{ + ssize_t size; + size_t nitems; + const nvlist_t *nvlistnew; + const nvlist_t * const *nvlarray; + nvlist_t *parent; + unsigned int ii; + + NVLIST_ASSERT(nvl); + NVPAIR_ASSERT(new); + PJDLOG_ASSERT(mul == 1 || mul == -1); + + size = nvpair_header_size(); + size += strlen(nvpair_name(new)) + 1; + + if (nvpair_type(new) == NV_TYPE_NVLIST) { + nvlistnew = nvpair_get_nvlist(new); + size += nvlistnew->nvl_datasize; + size += nvpair_header_size() + 1; + } else if (nvpair_type(new) == NV_TYPE_NVLIST_ARRAY) { + nvlarray = nvpair_get_nvlist_array(new, &nitems); + PJDLOG_ASSERT(nitems > 0); + + size += (nvpair_header_size() + 1) * nitems; + for (ii = 0; ii < nitems; ii++) { + PJDLOG_ASSERT(nvlarray[ii]->nvl_error == 0); + size += nvlarray[ii]->nvl_datasize; + } + } else { + size += nvpair_size(new); + } + + size *= mul; + + nvl->nvl_datasize += size; + + parent = nvl; + while ((parent = __DECONST(nvlist_t *, + nvlist_get_parent(parent, NULL))) != NULL) { + parent->nvl_datasize += size; + } +} + +nvpair_t * +nvlist_get_array_next_nvpair(nvlist_t *nvl) +{ + + NVLIST_ASSERT(nvl); + + return (nvl->nvl_array_next); +} + bool nvlist_in_array(const nvlist_t *nvl) { @@ -631,78 +687,8 @@ nvlist_fdump(const nvlist_t *nvl, FILE *fp) size_t nvlist_size(const nvlist_t *nvl) { - const nvlist_t *tmpnvl; - const nvlist_t * const *nvlarray; - const nvpair_t *nvp, *tmpnvp; - void *cookie; - size_t size, nitems; - unsigned int ii; - NVLIST_ASSERT(nvl); - PJDLOG_ASSERT(nvl->nvl_error == 0); - - size = sizeof(struct nvlist_header); - nvp = nvlist_first_nvpair(nvl); - while (nvp != NULL) { - size += nvpair_header_size(); - size += strlen(nvpair_name(nvp)) + 1; - if (nvpair_type(nvp) == NV_TYPE_NVLIST) { - size += sizeof(struct nvlist_header); - size += nvpair_header_size() + 1; - tmpnvl = nvpair_get_nvlist(nvp); - PJDLOG_ASSERT(tmpnvl->nvl_error == 0); - tmpnvp = nvlist_first_nvpair(tmpnvl); - if (tmpnvp != NULL) { - nvl = tmpnvl; - nvp = tmpnvp; - continue; - } - } else if (nvpair_type(nvp) == NV_TYPE_NVLIST_ARRAY) { - nvlarray = nvpair_get_nvlist_array(nvp, &nitems); - PJDLOG_ASSERT(nitems > 0); - - size += (nvpair_header_size() + 1) * nitems; - size += sizeof(struct nvlist_header) * nitems; - - tmpnvl = NULL; - tmpnvp = NULL; - for (ii = 0; ii < nitems; ii++) { - PJDLOG_ASSERT(nvlarray[ii]->nvl_error == 0); - tmpnvp = nvlist_first_nvpair(nvlarray[ii]); - if (tmpnvp != NULL) { - tmpnvl = nvlarray[ii]; - break; - } - } - if (tmpnvp != NULL) { - nvp = tmpnvp; - nvl = tmpnvl; - continue; - } - - } else { - size += nvpair_size(nvp); - } - - while ((nvp = nvlist_next_nvpair(nvl, nvp)) == NULL) { - do { - cookie = NULL; - nvl = nvlist_get_pararr(nvl, &cookie); - if (nvl == NULL) - goto out; - if (nvlist_in_array(nvl) && cookie == NULL) { - nvp = nvlist_first_nvpair(nvl); - } else { - nvp = cookie; - } - } while (nvp == NULL); - if (nvlist_in_array(nvl) && cookie == NULL) - break; - } - } - -out: - return (size); + return (nvl->nvl_datasize); } #ifndef _KERNEL @@ -1365,7 +1351,7 @@ nvlist_first_nvpair(const nvlist_t *nvl) } nvpair_t * -nvlist_next_nvpair(const nvlist_t *nvl, const nvpair_t *nvp) +nvlist_next_nvpair(const nvlist_t *nvl __unused, const nvpair_t *nvp) { nvpair_t *retnvp; @@ -1381,7 +1367,7 @@ nvlist_next_nvpair(const nvlist_t *nvl, const nvpair_t *nvp) } nvpair_t * -nvlist_prev_nvpair(const nvlist_t *nvl, const nvpair_t *nvp) +nvlist_prev_nvpair(const nvlist_t *nvl __unused, const nvpair_t *nvp) { nvpair_t *retnvp; @@ -1474,12 +1460,9 @@ nvlist_add_nvpair(nvlist_t *nvl, const nvpair_t *nvp) } nvpair_insert(&nvl->nvl_head, newnvp, nvl); + nvlist_update_size(nvl, newnvp, 1); } -void -nvlist_add_stringv(nvlist_t *nvl, const char *name, const char *valuefmt, - va_list valueap); - void nvlist_add_stringf(nvlist_t *nvl, const char *name, const char *valuefmt, ...) { @@ -1626,10 +1609,12 @@ nvlist_append_##type##_array(nvlist_t *nvl, const char *name, vtype value)\ nvlist_add_##type##_array(nvl, name, &value, 1); \ return; \ } \ + nvlist_update_size(nvl, nvp, -1); \ if (nvpair_append_##type##_array(nvp, value) == -1) { \ nvl->nvl_error = ERRNO_OR_DEFAULT(ENOMEM); \ ERRNO_SET(nvl->nvl_error); \ } \ + nvlist_update_size(nvl, nvp, 1); \ } NVLIST_APPEND_ARRAY(const bool, bool, BOOL) @@ -1664,6 +1649,7 @@ nvlist_move_nvpair(nvlist_t *nvl, nvpair_t *nvp) } nvpair_insert(&nvl->nvl_head, nvp, nvl); + nvlist_update_size(nvl, nvp, 1); return (true); } @@ -2015,6 +2001,7 @@ nvlist_remove_nvpair(nvlist_t *nvl, nvpair_t *nvp) PJDLOG_ASSERT(nvpair_nvlist(nvp) == nvl); nvpair_remove(&nvl->nvl_head, nvp, nvl); + nvlist_update_size(nvl, nvp, -1); } void diff --git a/src/libs/compat/freebsd_network/compat/net/debugnet.h b/src/libs/compat/freebsd_network/compat/net/debugnet.h new file mode 100644 index 0000000000..613f4cf257 --- /dev/null +++ b/src/libs/compat/freebsd_network/compat/net/debugnet.h @@ -0,0 +1,14 @@ +/* + * Copyright 2021, Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef _FBSD_COMPAT_NET_DEBUGNET_H +#define _FBSD_COMPAT_NET_DEBUGNET_H + + +#define DEBUGNET_DEFINE(driver) +#define DEBUGNET_REINIT(ifp) +#define DEBUGNET_SET(ifp, driver) + + +#endif /* _FBSD_COMPAT_NET_DEBUGNET_H */ diff --git a/src/libs/compat/freebsd_network/compat/netinet/netdump/netdump.h b/src/libs/compat/freebsd_network/compat/netinet/netdump/netdump.h deleted file mode 100644 index ef07076098..0000000000 --- a/src/libs/compat/freebsd_network/compat/netinet/netdump/netdump.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2018, Haiku Inc. All rights reserved. - * Distributed under the terms of the MIT License. - */ -#ifndef _FBSD_COMPAT_NETINET_NETDUMP_H_ -#define _FBSD_COMPAT_NETINET_NETDUMP_H_ - - -#define NETDUMP_DEFINE(driver) -#define NETDUMP_REINIT(ifp) -#define NETDUMP_SET(ifp, driver) - - -#endif /* _FBSD_COMPAT_NETINET_NETDUMP_H_ */