Home | History | Annotate | Line # | Download | only in ixgbe
ixgbe_netmap.c revision 1.1.4.3
      1  1.1.4.2     snj /******************************************************************************
      2  1.1.4.2     snj 
      3  1.1.4.2     snj   Copyright (c) 2001-2017, Intel Corporation
      4  1.1.4.2     snj   All rights reserved.
      5  1.1.4.2     snj 
      6  1.1.4.2     snj   Redistribution and use in source and binary forms, with or without
      7  1.1.4.2     snj   modification, are permitted provided that the following conditions are met:
      8  1.1.4.2     snj 
      9  1.1.4.2     snj    1. Redistributions of source code must retain the above copyright notice,
     10  1.1.4.2     snj       this list of conditions and the following disclaimer.
     11  1.1.4.2     snj 
     12  1.1.4.2     snj    2. Redistributions in binary form must reproduce the above copyright
     13  1.1.4.2     snj       notice, this list of conditions and the following disclaimer in the
     14  1.1.4.2     snj       documentation and/or other materials provided with the distribution.
     15  1.1.4.2     snj 
     16  1.1.4.2     snj    3. Neither the name of the Intel Corporation nor the names of its
     17  1.1.4.2     snj       contributors may be used to endorse or promote products derived from
     18  1.1.4.2     snj       this software without specific prior written permission.
     19  1.1.4.2     snj 
     20  1.1.4.2     snj   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
     21  1.1.4.2     snj   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     22  1.1.4.2     snj   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     23  1.1.4.2     snj   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
     24  1.1.4.2     snj   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  1.1.4.2     snj   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  1.1.4.2     snj   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  1.1.4.2     snj   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  1.1.4.2     snj   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  1.1.4.2     snj   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  1.1.4.2     snj   POSSIBILITY OF SUCH DAMAGE.
     31  1.1.4.2     snj 
     32  1.1.4.2     snj ******************************************************************************/
     33  1.1.4.2     snj /*$FreeBSD: head/sys/dev/ixgbe/ixgbe_netmap.c 320688 2017-07-05 17:27:03Z erj $*/
     34  1.1.4.2     snj 
     35  1.1.4.2     snj /*
     36  1.1.4.2     snj  * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved.
     37  1.1.4.2     snj  *
     38  1.1.4.2     snj  * Redistribution and use in source and binary forms, with or without
     39  1.1.4.2     snj  * modification, are permitted provided that the following conditions
     40  1.1.4.2     snj  * are met:
     41  1.1.4.2     snj  * 1. Redistributions of source code must retain the above copyright
     42  1.1.4.2     snj  *    notice, this list of conditions and the following disclaimer.
     43  1.1.4.2     snj  * 2. Redistributions in binary form must reproduce the above copyright
     44  1.1.4.2     snj  *    notice, this list of conditions and the following disclaimer in the
     45  1.1.4.2     snj  *    documentation and/or other materials provided with the distribution.
     46  1.1.4.2     snj  *
     47  1.1.4.2     snj  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     48  1.1.4.2     snj  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     49  1.1.4.2     snj  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     50  1.1.4.2     snj  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     51  1.1.4.2     snj  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     52  1.1.4.2     snj  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     53  1.1.4.2     snj  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     54  1.1.4.2     snj  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     55  1.1.4.2     snj  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     56  1.1.4.2     snj  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     57  1.1.4.2     snj  * SUCH DAMAGE.
     58  1.1.4.2     snj  */
     59  1.1.4.2     snj 
     60  1.1.4.2     snj /*
     61  1.1.4.2     snj  * $FreeBSD: head/sys/dev/ixgbe/ixgbe_netmap.c 320688 2017-07-05 17:27:03Z erj $
     62  1.1.4.2     snj  *
     63  1.1.4.2     snj  * netmap support for: ixgbe
     64  1.1.4.2     snj  *
     65  1.1.4.2     snj  * This file is meant to be a reference on how to implement
     66  1.1.4.2     snj  * netmap support for a network driver.
     67  1.1.4.2     snj  * This file contains code but only static or inline functions used
     68  1.1.4.2     snj  * by a single driver. To avoid replication of code we just #include
     69  1.1.4.2     snj  * it near the beginning of the standard driver.
     70  1.1.4.2     snj  */
     71  1.1.4.2     snj 
     72  1.1.4.2     snj #ifdef DEV_NETMAP
     73  1.1.4.2     snj /*
     74  1.1.4.2     snj  * Some drivers may need the following headers. Others
     75  1.1.4.2     snj  * already include them by default
     76  1.1.4.2     snj 
     77  1.1.4.2     snj #include <vm/vm.h>
     78  1.1.4.2     snj #include <vm/pmap.h>
     79  1.1.4.2     snj 
     80  1.1.4.2     snj  */
     81  1.1.4.2     snj #include "ixgbe.h"
     82  1.1.4.2     snj 
     83  1.1.4.2     snj /*
     84  1.1.4.2     snj  * device-specific sysctl variables:
     85  1.1.4.2     snj  *
     86  1.1.4.2     snj  * ix_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
     87  1.1.4.2     snj  *	During regular operations the CRC is stripped, but on some
     88  1.1.4.2     snj  *	hardware reception of frames not multiple of 64 is slower,
     89  1.1.4.2     snj  *	so using crcstrip=0 helps in benchmarks.
     90  1.1.4.2     snj  *
     91  1.1.4.2     snj  * ix_rx_miss, ix_rx_miss_bufs:
     92  1.1.4.2     snj  *	count packets that might be missed due to lost interrupts.
     93  1.1.4.2     snj  */
     94  1.1.4.2     snj SYSCTL_DECL(_dev_netmap);
     95  1.1.4.2     snj static int ix_rx_miss, ix_rx_miss_bufs;
     96  1.1.4.2     snj int ix_crcstrip;
     97  1.1.4.2     snj SYSCTL_INT(_dev_netmap, OID_AUTO, ix_crcstrip,
     98  1.1.4.2     snj     CTLFLAG_RW, &ix_crcstrip, 0, "strip CRC on rx frames");
     99  1.1.4.2     snj SYSCTL_INT(_dev_netmap, OID_AUTO, ix_rx_miss,
    100  1.1.4.2     snj     CTLFLAG_RW, &ix_rx_miss, 0, "potentially missed rx intr");
    101  1.1.4.2     snj SYSCTL_INT(_dev_netmap, OID_AUTO, ix_rx_miss_bufs,
    102  1.1.4.2     snj     CTLFLAG_RW, &ix_rx_miss_bufs, 0, "potentially missed rx intr bufs");
    103  1.1.4.2     snj 
    104  1.1.4.2     snj 
    105  1.1.4.2     snj static void
    106  1.1.4.2     snj set_crcstrip(struct ixgbe_hw *hw, int onoff)
    107  1.1.4.2     snj {
    108  1.1.4.2     snj 	/* crc stripping is set in two places:
    109  1.1.4.2     snj 	 * IXGBE_HLREG0 (modified on init_locked and hw reset)
    110  1.1.4.2     snj 	 * IXGBE_RDRXCTL (set by the original driver in
    111  1.1.4.2     snj 	 *	ixgbe_setup_hw_rsc() called in init_locked.
    112  1.1.4.2     snj 	 *	We disable the setting when netmap is compiled in).
    113  1.1.4.2     snj 	 * We update the values here, but also in ixgbe.c because
    114  1.1.4.2     snj 	 * init_locked sometimes is called outside our control.
    115  1.1.4.2     snj 	 */
    116  1.1.4.2     snj 	uint32_t hl, rxc;
    117  1.1.4.2     snj 
    118  1.1.4.2     snj 	hl = IXGBE_READ_REG(hw, IXGBE_HLREG0);
    119  1.1.4.2     snj 	rxc = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
    120  1.1.4.2     snj 	if (netmap_verbose)
    121  1.1.4.2     snj 		D("%s read  HLREG 0x%x rxc 0x%x",
    122  1.1.4.2     snj 			onoff ? "enter" : "exit", hl, rxc);
    123  1.1.4.2     snj 	/* hw requirements ... */
    124  1.1.4.2     snj 	rxc &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
    125  1.1.4.2     snj 	rxc |= IXGBE_RDRXCTL_RSCACKC;
    126  1.1.4.2     snj 	if (onoff && !ix_crcstrip) {
    127  1.1.4.2     snj 		/* keep the crc. Fast rx */
    128  1.1.4.2     snj 		hl &= ~IXGBE_HLREG0_RXCRCSTRP;
    129  1.1.4.2     snj 		rxc &= ~IXGBE_RDRXCTL_CRCSTRIP;
    130  1.1.4.2     snj 	} else {
    131  1.1.4.2     snj 		/* reset default mode */
    132  1.1.4.2     snj 		hl |= IXGBE_HLREG0_RXCRCSTRP;
    133  1.1.4.2     snj 		rxc |= IXGBE_RDRXCTL_CRCSTRIP;
    134  1.1.4.2     snj 	}
    135  1.1.4.2     snj 	if (netmap_verbose)
    136  1.1.4.2     snj 		D("%s write HLREG 0x%x rxc 0x%x",
    137  1.1.4.2     snj 			onoff ? "enter" : "exit", hl, rxc);
    138  1.1.4.2     snj 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hl);
    139  1.1.4.2     snj 	IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rxc);
    140  1.1.4.2     snj }
    141  1.1.4.2     snj 
    142  1.1.4.2     snj 
    143  1.1.4.2     snj /*
    144  1.1.4.2     snj  * Register/unregister. We are already under netmap lock.
    145  1.1.4.2     snj  * Only called on the first register or the last unregister.
    146  1.1.4.2     snj  */
    147  1.1.4.2     snj static int
    148  1.1.4.2     snj ixgbe_netmap_reg(struct netmap_adapter *na, int onoff)
    149  1.1.4.2     snj {
    150  1.1.4.2     snj 	struct ifnet *ifp = na->ifp;
    151  1.1.4.2     snj 	struct adapter *adapter = ifp->if_softc;
    152  1.1.4.2     snj 
    153  1.1.4.2     snj 	IXGBE_CORE_LOCK(adapter);
    154  1.1.4.2     snj 	adapter->stop_locked(adapter);
    155  1.1.4.2     snj 
    156  1.1.4.2     snj 	set_crcstrip(&adapter->hw, onoff);
    157  1.1.4.2     snj 	/* enable or disable flags and callbacks in na and ifp */
    158  1.1.4.2     snj 	if (onoff) {
    159  1.1.4.2     snj 		nm_set_native_flags(na);
    160  1.1.4.2     snj 	} else {
    161  1.1.4.2     snj 		nm_clear_native_flags(na);
    162  1.1.4.2     snj 	}
    163  1.1.4.2     snj 	adapter->init_locked(adapter);	/* also enables intr */
    164  1.1.4.2     snj 	set_crcstrip(&adapter->hw, onoff); // XXX why twice ?
    165  1.1.4.2     snj 	IXGBE_CORE_UNLOCK(adapter);
    166  1.1.4.2     snj 	return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
    167  1.1.4.2     snj }
    168  1.1.4.2     snj 
    169  1.1.4.2     snj 
    170  1.1.4.2     snj /*
    171  1.1.4.2     snj  * Reconcile kernel and user view of the transmit ring.
    172  1.1.4.2     snj  *
    173  1.1.4.2     snj  * All information is in the kring.
    174  1.1.4.2     snj  * Userspace wants to send packets up to the one before kring->rhead,
    175  1.1.4.2     snj  * kernel knows kring->nr_hwcur is the first unsent packet.
    176  1.1.4.2     snj  *
    177  1.1.4.2     snj  * Here we push packets out (as many as possible), and possibly
    178  1.1.4.2     snj  * reclaim buffers from previously completed transmission.
    179  1.1.4.2     snj  *
    180  1.1.4.2     snj  * The caller (netmap) guarantees that there is only one instance
    181  1.1.4.2     snj  * running at any time. Any interference with other driver
    182  1.1.4.2     snj  * methods should be handled by the individual drivers.
    183  1.1.4.2     snj  */
    184  1.1.4.2     snj static int
    185  1.1.4.2     snj ixgbe_netmap_txsync(struct netmap_kring *kring, int flags)
    186  1.1.4.2     snj {
    187  1.1.4.2     snj 	struct netmap_adapter *na = kring->na;
    188  1.1.4.2     snj 	struct ifnet *ifp = na->ifp;
    189  1.1.4.2     snj 	struct netmap_ring *ring = kring->ring;
    190  1.1.4.2     snj 	u_int nm_i;	/* index into the netmap ring */
    191  1.1.4.2     snj 	u_int nic_i;	/* index into the NIC ring */
    192  1.1.4.2     snj 	u_int n;
    193  1.1.4.2     snj 	u_int const lim = kring->nkr_num_slots - 1;
    194  1.1.4.2     snj 	u_int const head = kring->rhead;
    195  1.1.4.2     snj 	/*
    196  1.1.4.2     snj 	 * interrupts on every tx packet are expensive so request
    197  1.1.4.2     snj 	 * them every half ring, or where NS_REPORT is set
    198  1.1.4.2     snj 	 */
    199  1.1.4.2     snj 	u_int report_frequency = kring->nkr_num_slots >> 1;
    200  1.1.4.2     snj 
    201  1.1.4.2     snj 	/* device-specific */
    202  1.1.4.2     snj 	struct adapter *adapter = ifp->if_softc;
    203  1.1.4.2     snj 	struct tx_ring *txr = &adapter->tx_rings[kring->ring_id];
    204  1.1.4.2     snj 	int reclaim_tx;
    205  1.1.4.2     snj 
    206  1.1.4.2     snj 	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
    207  1.1.4.2     snj 			BUS_DMASYNC_POSTREAD);
    208  1.1.4.2     snj 
    209  1.1.4.2     snj 	/*
    210  1.1.4.2     snj 	 * First part: process new packets to send.
    211  1.1.4.2     snj 	 * nm_i is the current index in the netmap ring,
    212  1.1.4.2     snj 	 * nic_i is the corresponding index in the NIC ring.
    213  1.1.4.2     snj 	 * The two numbers differ because upon a *_init() we reset
    214  1.1.4.2     snj 	 * the NIC ring but leave the netmap ring unchanged.
    215  1.1.4.2     snj 	 * For the transmit ring, we have
    216  1.1.4.2     snj 	 *
    217  1.1.4.2     snj 	 *		nm_i = kring->nr_hwcur
    218  1.1.4.2     snj 	 *		nic_i = IXGBE_TDT (not tracked in the driver)
    219  1.1.4.2     snj 	 * and
    220  1.1.4.2     snj 	 * 		nm_i == (nic_i + kring->nkr_hwofs) % ring_size
    221  1.1.4.2     snj 	 *
    222  1.1.4.2     snj 	 * In this driver kring->nkr_hwofs >= 0, but for other
    223  1.1.4.2     snj 	 * drivers it might be negative as well.
    224  1.1.4.2     snj 	 */
    225  1.1.4.2     snj 
    226  1.1.4.2     snj 	/*
    227  1.1.4.2     snj 	 * If we have packets to send (kring->nr_hwcur != kring->rhead)
    228  1.1.4.2     snj 	 * iterate over the netmap ring, fetch length and update
    229  1.1.4.2     snj 	 * the corresponding slot in the NIC ring. Some drivers also
    230  1.1.4.2     snj 	 * need to update the buffer's physical address in the NIC slot
    231  1.1.4.2     snj 	 * even NS_BUF_CHANGED is not set (PNMB computes the addresses).
    232  1.1.4.2     snj 	 *
    233  1.1.4.2     snj 	 * The netmap_reload_map() calls is especially expensive,
    234  1.1.4.2     snj 	 * even when (as in this case) the tag is 0, so do only
    235  1.1.4.2     snj 	 * when the buffer has actually changed.
    236  1.1.4.2     snj 	 *
    237  1.1.4.2     snj 	 * If possible do not set the report/intr bit on all slots,
    238  1.1.4.2     snj 	 * but only a few times per ring or when NS_REPORT is set.
    239  1.1.4.2     snj 	 *
    240  1.1.4.2     snj 	 * Finally, on 10G and faster drivers, it might be useful
    241  1.1.4.2     snj 	 * to prefetch the next slot and txr entry.
    242  1.1.4.2     snj 	 */
    243  1.1.4.2     snj 
    244  1.1.4.2     snj 	nm_i = kring->nr_hwcur;
    245  1.1.4.2     snj 	if (nm_i != head) {	/* we have new packets to send */
    246  1.1.4.2     snj 		nic_i = netmap_idx_k2n(kring, nm_i);
    247  1.1.4.2     snj 
    248  1.1.4.2     snj 		__builtin_prefetch(&ring->slot[nm_i]);
    249  1.1.4.2     snj 		__builtin_prefetch(&txr->tx_buffers[nic_i]);
    250  1.1.4.2     snj 
    251  1.1.4.2     snj 		for (n = 0; nm_i != head; n++) {
    252  1.1.4.2     snj 			struct netmap_slot *slot = &ring->slot[nm_i];
    253  1.1.4.2     snj 			u_int len = slot->len;
    254  1.1.4.2     snj 			uint64_t paddr;
    255  1.1.4.2     snj 			void *addr = PNMB(na, slot, &paddr);
    256  1.1.4.2     snj 
    257  1.1.4.2     snj 			/* device-specific */
    258  1.1.4.2     snj 			union ixgbe_adv_tx_desc *curr = &txr->tx_base[nic_i];
    259  1.1.4.2     snj 			struct ixgbe_tx_buf *txbuf = &txr->tx_buffers[nic_i];
    260  1.1.4.2     snj 			int flags = (slot->flags & NS_REPORT ||
    261  1.1.4.2     snj 				nic_i == 0 || nic_i == report_frequency) ?
    262  1.1.4.2     snj 				IXGBE_TXD_CMD_RS : 0;
    263  1.1.4.2     snj 
    264  1.1.4.2     snj 			/* prefetch for next round */
    265  1.1.4.2     snj 			__builtin_prefetch(&ring->slot[nm_i + 1]);
    266  1.1.4.2     snj 			__builtin_prefetch(&txr->tx_buffers[nic_i + 1]);
    267  1.1.4.2     snj 
    268  1.1.4.2     snj 			NM_CHECK_ADDR_LEN(na, addr, len);
    269  1.1.4.2     snj 
    270  1.1.4.2     snj 			if (slot->flags & NS_BUF_CHANGED) {
    271  1.1.4.2     snj 				/* buffer has changed, reload map */
    272  1.1.4.2     snj 				netmap_reload_map(na, txr->txtag, txbuf->map, addr);
    273  1.1.4.2     snj 			}
    274  1.1.4.2     snj 			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
    275  1.1.4.2     snj 
    276  1.1.4.2     snj 			/* Fill the slot in the NIC ring. */
    277  1.1.4.2     snj 			/* Use legacy descriptor, they are faster? */
    278  1.1.4.2     snj 			curr->read.buffer_addr = htole64(paddr);
    279  1.1.4.2     snj 			curr->read.olinfo_status = 0;
    280  1.1.4.2     snj 			curr->read.cmd_type_len = htole32(len | flags |
    281  1.1.4.2     snj 				IXGBE_ADVTXD_DCMD_IFCS | IXGBE_TXD_CMD_EOP);
    282  1.1.4.2     snj 
    283  1.1.4.2     snj 			/* make sure changes to the buffer are synced */
    284  1.1.4.2     snj 			bus_dmamap_sync(txr->txtag, txbuf->map,
    285  1.1.4.2     snj 				BUS_DMASYNC_PREWRITE);
    286  1.1.4.2     snj 
    287  1.1.4.2     snj 			nm_i = nm_next(nm_i, lim);
    288  1.1.4.2     snj 			nic_i = nm_next(nic_i, lim);
    289  1.1.4.2     snj 		}
    290  1.1.4.2     snj 		kring->nr_hwcur = head;
    291  1.1.4.2     snj 
    292  1.1.4.2     snj 		/* synchronize the NIC ring */
    293  1.1.4.2     snj 		bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
    294  1.1.4.2     snj 			BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
    295  1.1.4.2     snj 
    296  1.1.4.2     snj 		/* (re)start the tx unit up to slot nic_i (excluded) */
    297  1.1.4.2     snj 		IXGBE_WRITE_REG(&adapter->hw, txr->tail, nic_i);
    298  1.1.4.2     snj 	}
    299  1.1.4.2     snj 
    300  1.1.4.2     snj 	/*
    301  1.1.4.2     snj 	 * Second part: reclaim buffers for completed transmissions.
    302  1.1.4.2     snj 	 * Because this is expensive (we read a NIC register etc.)
    303  1.1.4.2     snj 	 * we only do it in specific cases (see below).
    304  1.1.4.2     snj 	 */
    305  1.1.4.2     snj 	if (flags & NAF_FORCE_RECLAIM) {
    306  1.1.4.2     snj 		reclaim_tx = 1; /* forced reclaim */
    307  1.1.4.2     snj 	} else if (!nm_kr_txempty(kring)) {
    308  1.1.4.2     snj 		reclaim_tx = 0; /* have buffers, no reclaim */
    309  1.1.4.2     snj 	} else {
    310  1.1.4.2     snj 		/*
    311  1.1.4.2     snj 		 * No buffers available. Locate previous slot with
    312  1.1.4.2     snj 		 * REPORT_STATUS set.
    313  1.1.4.2     snj 		 * If the slot has DD set, we can reclaim space,
    314  1.1.4.2     snj 		 * otherwise wait for the next interrupt.
    315  1.1.4.2     snj 		 * This enables interrupt moderation on the tx
    316  1.1.4.2     snj 		 * side though it might reduce throughput.
    317  1.1.4.2     snj 		 */
    318  1.1.4.2     snj 		struct ixgbe_legacy_tx_desc *txd =
    319  1.1.4.2     snj 		    (struct ixgbe_legacy_tx_desc *)txr->tx_base;
    320  1.1.4.2     snj 
    321  1.1.4.2     snj 		nic_i = txr->next_to_clean + report_frequency;
    322  1.1.4.2     snj 		if (nic_i > lim)
    323  1.1.4.2     snj 			nic_i -= lim + 1;
    324  1.1.4.2     snj 		// round to the closest with dd set
    325  1.1.4.2     snj 		nic_i = (nic_i < kring->nkr_num_slots / 4 ||
    326  1.1.4.2     snj 			 nic_i >= kring->nkr_num_slots*3/4) ?
    327  1.1.4.2     snj 			0 : report_frequency;
    328  1.1.4.2     snj 		reclaim_tx = txd[nic_i].upper.fields.status & IXGBE_TXD_STAT_DD;	// XXX cpu_to_le32 ?
    329  1.1.4.2     snj 	}
    330  1.1.4.2     snj 	if (reclaim_tx) {
    331  1.1.4.2     snj 		/*
    332  1.1.4.2     snj 		 * Record completed transmissions.
    333  1.1.4.2     snj 		 * We (re)use the driver's txr->next_to_clean to keep
    334  1.1.4.2     snj 		 * track of the most recently completed transmission.
    335  1.1.4.2     snj 		 *
    336  1.1.4.2     snj 		 * The datasheet discourages the use of TDH to find
    337  1.1.4.2     snj 		 * out the number of sent packets, but we only set
    338  1.1.4.2     snj 		 * REPORT_STATUS in a few slots so TDH is the only
    339  1.1.4.2     snj 		 * good way.
    340  1.1.4.2     snj 		 */
    341  1.1.4.2     snj 		nic_i = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(kring->ring_id));
    342  1.1.4.2     snj 		if (nic_i >= kring->nkr_num_slots) { /* XXX can it happen ? */
    343  1.1.4.2     snj 			D("TDH wrap %d", nic_i);
    344  1.1.4.2     snj 			nic_i -= kring->nkr_num_slots;
    345  1.1.4.2     snj 		}
    346  1.1.4.2     snj 		if (nic_i != txr->next_to_clean) {
    347  1.1.4.2     snj 			/* some tx completed, increment avail */
    348  1.1.4.2     snj 			txr->next_to_clean = nic_i;
    349  1.1.4.2     snj 			kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
    350  1.1.4.2     snj 		}
    351  1.1.4.2     snj 	}
    352  1.1.4.2     snj 
    353  1.1.4.2     snj 	return 0;
    354  1.1.4.2     snj }
    355  1.1.4.2     snj 
    356  1.1.4.2     snj 
    357  1.1.4.2     snj /*
    358  1.1.4.2     snj  * Reconcile kernel and user view of the receive ring.
    359  1.1.4.2     snj  * Same as for the txsync, this routine must be efficient.
    360  1.1.4.2     snj  * The caller guarantees a single invocations, but races against
    361  1.1.4.2     snj  * the rest of the driver should be handled here.
    362  1.1.4.2     snj  *
    363  1.1.4.2     snj  * On call, kring->rhead is the first packet that userspace wants
    364  1.1.4.2     snj  * to keep, and kring->rcur is the wakeup point.
    365  1.1.4.2     snj  * The kernel has previously reported packets up to kring->rtail.
    366  1.1.4.2     snj  *
    367  1.1.4.2     snj  * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
    368  1.1.4.2     snj  * of whether or not we received an interrupt.
    369  1.1.4.2     snj  */
    370  1.1.4.2     snj static int
    371  1.1.4.2     snj ixgbe_netmap_rxsync(struct netmap_kring *kring, int flags)
    372  1.1.4.2     snj {
    373  1.1.4.2     snj 	struct netmap_adapter *na = kring->na;
    374  1.1.4.2     snj 	struct ifnet *ifp = na->ifp;
    375  1.1.4.2     snj 	struct netmap_ring *ring = kring->ring;
    376  1.1.4.2     snj 	u_int nm_i;	/* index into the netmap ring */
    377  1.1.4.2     snj 	u_int nic_i;	/* index into the NIC ring */
    378  1.1.4.2     snj 	u_int n;
    379  1.1.4.2     snj 	u_int const lim = kring->nkr_num_slots - 1;
    380  1.1.4.2     snj 	u_int const head = kring->rhead;
    381  1.1.4.2     snj 	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
    382  1.1.4.2     snj 
    383  1.1.4.2     snj 	/* device-specific */
    384  1.1.4.2     snj 	struct adapter *adapter = ifp->if_softc;
    385  1.1.4.2     snj 	struct rx_ring *rxr = &adapter->rx_rings[kring->ring_id];
    386  1.1.4.2     snj 
    387  1.1.4.2     snj 	if (head > lim)
    388  1.1.4.2     snj 		return netmap_ring_reinit(kring);
    389  1.1.4.2     snj 
    390  1.1.4.2     snj 	/* XXX check sync modes */
    391  1.1.4.2     snj 	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
    392  1.1.4.2     snj 			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
    393  1.1.4.2     snj 
    394  1.1.4.2     snj 	/*
    395  1.1.4.2     snj 	 * First part: import newly received packets.
    396  1.1.4.2     snj 	 *
    397  1.1.4.2     snj 	 * nm_i is the index of the next free slot in the netmap ring,
    398  1.1.4.2     snj 	 * nic_i is the index of the next received packet in the NIC ring,
    399  1.1.4.2     snj 	 * and they may differ in case if_init() has been called while
    400  1.1.4.2     snj 	 * in netmap mode. For the receive ring we have
    401  1.1.4.2     snj 	 *
    402  1.1.4.2     snj 	 *	nic_i = rxr->next_to_check;
    403  1.1.4.2     snj 	 *	nm_i = kring->nr_hwtail (previous)
    404  1.1.4.2     snj 	 * and
    405  1.1.4.2     snj 	 *	nm_i == (nic_i + kring->nkr_hwofs) % ring_size
    406  1.1.4.2     snj 	 *
    407  1.1.4.2     snj 	 * rxr->next_to_check is set to 0 on a ring reinit
    408  1.1.4.2     snj 	 */
    409  1.1.4.2     snj 	if (netmap_no_pendintr || force_update) {
    410  1.1.4.2     snj 		int crclen = (ix_crcstrip) ? 0 : 4;
    411  1.1.4.2     snj 
    412  1.1.4.2     snj 		nic_i = rxr->next_to_check; // or also k2n(kring->nr_hwtail)
    413  1.1.4.2     snj 		nm_i = netmap_idx_n2k(kring, nic_i);
    414  1.1.4.2     snj 
    415  1.1.4.2     snj 		for (n = 0; ; n++) {
    416  1.1.4.2     snj 			union ixgbe_adv_rx_desc *curr = &rxr->rx_base[nic_i];
    417  1.1.4.2     snj 			uint32_t staterr = le32toh(curr->wb.upper.status_error);
    418  1.1.4.2     snj 
    419  1.1.4.2     snj 			if ((staterr & IXGBE_RXD_STAT_DD) == 0)
    420  1.1.4.2     snj 				break;
    421  1.1.4.2     snj 			ring->slot[nm_i].len = le16toh(curr->wb.upper.length) - crclen;
    422  1.1.4.3  martin 			ring->slot[nm_i].flags = 0;
    423  1.1.4.2     snj 			bus_dmamap_sync(rxr->ptag,
    424  1.1.4.2     snj 			    rxr->rx_buffers[nic_i].pmap, BUS_DMASYNC_POSTREAD);
    425  1.1.4.2     snj 			nm_i = nm_next(nm_i, lim);
    426  1.1.4.2     snj 			nic_i = nm_next(nic_i, lim);
    427  1.1.4.2     snj 		}
    428  1.1.4.2     snj 		if (n) { /* update the state variables */
    429  1.1.4.2     snj 			if (netmap_no_pendintr && !force_update) {
    430  1.1.4.2     snj 				/* diagnostics */
    431  1.1.4.2     snj 				ix_rx_miss ++;
    432  1.1.4.2     snj 				ix_rx_miss_bufs += n;
    433  1.1.4.2     snj 			}
    434  1.1.4.2     snj 			rxr->next_to_check = nic_i;
    435  1.1.4.2     snj 			kring->nr_hwtail = nm_i;
    436  1.1.4.2     snj 		}
    437  1.1.4.2     snj 		kring->nr_kflags &= ~NKR_PENDINTR;
    438  1.1.4.2     snj 	}
    439  1.1.4.2     snj 
    440  1.1.4.2     snj 	/*
    441  1.1.4.2     snj 	 * Second part: skip past packets that userspace has released.
    442  1.1.4.2     snj 	 * (kring->nr_hwcur to kring->rhead excluded),
    443  1.1.4.2     snj 	 * and make the buffers available for reception.
    444  1.1.4.2     snj 	 * As usual nm_i is the index in the netmap ring,
    445  1.1.4.2     snj 	 * nic_i is the index in the NIC ring, and
    446  1.1.4.2     snj 	 * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
    447  1.1.4.2     snj 	 */
    448  1.1.4.2     snj 	nm_i = kring->nr_hwcur;
    449  1.1.4.2     snj 	if (nm_i != head) {
    450  1.1.4.2     snj 		nic_i = netmap_idx_k2n(kring, nm_i);
    451  1.1.4.2     snj 		for (n = 0; nm_i != head; n++) {
    452  1.1.4.2     snj 			struct netmap_slot *slot = &ring->slot[nm_i];
    453  1.1.4.2     snj 			uint64_t paddr;
    454  1.1.4.2     snj 			void *addr = PNMB(na, slot, &paddr);
    455  1.1.4.2     snj 
    456  1.1.4.2     snj 			union ixgbe_adv_rx_desc *curr = &rxr->rx_base[nic_i];
    457  1.1.4.2     snj 			struct ixgbe_rx_buf *rxbuf = &rxr->rx_buffers[nic_i];
    458  1.1.4.2     snj 
    459  1.1.4.2     snj 			if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
    460  1.1.4.2     snj 				goto ring_reset;
    461  1.1.4.2     snj 
    462  1.1.4.2     snj 			if (slot->flags & NS_BUF_CHANGED) {
    463  1.1.4.2     snj 				/* buffer has changed, reload map */
    464  1.1.4.2     snj 				netmap_reload_map(na, rxr->ptag, rxbuf->pmap, addr);
    465  1.1.4.2     snj 				slot->flags &= ~NS_BUF_CHANGED;
    466  1.1.4.2     snj 			}
    467  1.1.4.2     snj 			curr->wb.upper.status_error = 0;
    468  1.1.4.2     snj 			curr->read.pkt_addr = htole64(paddr);
    469  1.1.4.2     snj 			bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
    470  1.1.4.2     snj 			    BUS_DMASYNC_PREREAD);
    471  1.1.4.2     snj 			nm_i = nm_next(nm_i, lim);
    472  1.1.4.2     snj 			nic_i = nm_next(nic_i, lim);
    473  1.1.4.2     snj 		}
    474  1.1.4.2     snj 		kring->nr_hwcur = head;
    475  1.1.4.2     snj 
    476  1.1.4.2     snj 		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
    477  1.1.4.2     snj 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
    478  1.1.4.2     snj 		/*
    479  1.1.4.2     snj 		 * IMPORTANT: we must leave one free slot in the ring,
    480  1.1.4.2     snj 		 * so move nic_i back by one unit
    481  1.1.4.2     snj 		 */
    482  1.1.4.2     snj 		nic_i = nm_prev(nic_i, lim);
    483  1.1.4.2     snj 		IXGBE_WRITE_REG(&adapter->hw, rxr->tail, nic_i);
    484  1.1.4.2     snj 	}
    485  1.1.4.2     snj 
    486  1.1.4.2     snj 	return 0;
    487  1.1.4.2     snj 
    488  1.1.4.2     snj ring_reset:
    489  1.1.4.2     snj 	return netmap_ring_reinit(kring);
    490  1.1.4.2     snj }
    491  1.1.4.2     snj 
    492  1.1.4.2     snj 
    493  1.1.4.2     snj /*
    494  1.1.4.2     snj  * The attach routine, called near the end of ixgbe_attach(),
    495  1.1.4.2     snj  * fills the parameters for netmap_attach() and calls it.
    496  1.1.4.2     snj  * It cannot fail, in the worst case (such as no memory)
    497  1.1.4.2     snj  * netmap mode will be disabled and the driver will only
    498  1.1.4.2     snj  * operate in standard mode.
    499  1.1.4.2     snj  */
    500  1.1.4.2     snj void
    501  1.1.4.2     snj ixgbe_netmap_attach(struct adapter *adapter)
    502  1.1.4.2     snj {
    503  1.1.4.2     snj 	struct netmap_adapter na;
    504  1.1.4.2     snj 
    505  1.1.4.2     snj 	bzero(&na, sizeof(na));
    506  1.1.4.2     snj 
    507  1.1.4.2     snj 	na.ifp = adapter->ifp;
    508  1.1.4.2     snj 	na.na_flags = NAF_BDG_MAYSLEEP;
    509  1.1.4.2     snj 	na.num_tx_desc = adapter->num_tx_desc;
    510  1.1.4.2     snj 	na.num_rx_desc = adapter->num_rx_desc;
    511  1.1.4.2     snj 	na.nm_txsync = ixgbe_netmap_txsync;
    512  1.1.4.2     snj 	na.nm_rxsync = ixgbe_netmap_rxsync;
    513  1.1.4.2     snj 	na.nm_register = ixgbe_netmap_reg;
    514  1.1.4.2     snj 	na.num_tx_rings = na.num_rx_rings = adapter->num_queues;
    515  1.1.4.2     snj 	netmap_attach(&na);
    516  1.1.4.2     snj }
    517  1.1.4.2     snj 
    518  1.1.4.2     snj #endif /* DEV_NETMAP */
    519  1.1.4.2     snj 
    520  1.1.4.2     snj /* end of file */
    521