ixgbe_netmap.c revision 1.1.4.3 1 1.1.4.2 snj /******************************************************************************
2 1.1.4.2 snj
3 1.1.4.2 snj Copyright (c) 2001-2017, Intel Corporation
4 1.1.4.2 snj All rights reserved.
5 1.1.4.2 snj
6 1.1.4.2 snj Redistribution and use in source and binary forms, with or without
7 1.1.4.2 snj modification, are permitted provided that the following conditions are met:
8 1.1.4.2 snj
9 1.1.4.2 snj 1. Redistributions of source code must retain the above copyright notice,
10 1.1.4.2 snj this list of conditions and the following disclaimer.
11 1.1.4.2 snj
12 1.1.4.2 snj 2. Redistributions in binary form must reproduce the above copyright
13 1.1.4.2 snj notice, this list of conditions and the following disclaimer in the
14 1.1.4.2 snj documentation and/or other materials provided with the distribution.
15 1.1.4.2 snj
16 1.1.4.2 snj 3. Neither the name of the Intel Corporation nor the names of its
17 1.1.4.2 snj contributors may be used to endorse or promote products derived from
18 1.1.4.2 snj this software without specific prior written permission.
19 1.1.4.2 snj
20 1.1.4.2 snj THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 1.1.4.2 snj AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 1.1.4.2 snj IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 1.1.4.2 snj ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 1.1.4.2 snj LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 1.1.4.2 snj CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 1.1.4.2 snj SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 1.1.4.2 snj INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 1.1.4.2 snj CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 1.1.4.2 snj ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 1.1.4.2 snj POSSIBILITY OF SUCH DAMAGE.
31 1.1.4.2 snj
32 1.1.4.2 snj ******************************************************************************/
33 1.1.4.2 snj /*$FreeBSD: head/sys/dev/ixgbe/ixgbe_netmap.c 320688 2017-07-05 17:27:03Z erj $*/
34 1.1.4.2 snj
35 1.1.4.2 snj /*
36 1.1.4.2 snj * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved.
37 1.1.4.2 snj *
38 1.1.4.2 snj * Redistribution and use in source and binary forms, with or without
39 1.1.4.2 snj * modification, are permitted provided that the following conditions
40 1.1.4.2 snj * are met:
41 1.1.4.2 snj * 1. Redistributions of source code must retain the above copyright
42 1.1.4.2 snj * notice, this list of conditions and the following disclaimer.
43 1.1.4.2 snj * 2. Redistributions in binary form must reproduce the above copyright
44 1.1.4.2 snj * notice, this list of conditions and the following disclaimer in the
45 1.1.4.2 snj * documentation and/or other materials provided with the distribution.
46 1.1.4.2 snj *
47 1.1.4.2 snj * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
48 1.1.4.2 snj * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
49 1.1.4.2 snj * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
50 1.1.4.2 snj * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
51 1.1.4.2 snj * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
52 1.1.4.2 snj * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
53 1.1.4.2 snj * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
54 1.1.4.2 snj * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
55 1.1.4.2 snj * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
56 1.1.4.2 snj * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
57 1.1.4.2 snj * SUCH DAMAGE.
58 1.1.4.2 snj */
59 1.1.4.2 snj
60 1.1.4.2 snj /*
61 1.1.4.2 snj * $FreeBSD: head/sys/dev/ixgbe/ixgbe_netmap.c 320688 2017-07-05 17:27:03Z erj $
62 1.1.4.2 snj *
63 1.1.4.2 snj * netmap support for: ixgbe
64 1.1.4.2 snj *
65 1.1.4.2 snj * This file is meant to be a reference on how to implement
66 1.1.4.2 snj * netmap support for a network driver.
67 1.1.4.2 snj * This file contains code but only static or inline functions used
68 1.1.4.2 snj * by a single driver. To avoid replication of code we just #include
69 1.1.4.2 snj * it near the beginning of the standard driver.
70 1.1.4.2 snj */
71 1.1.4.2 snj
72 1.1.4.2 snj #ifdef DEV_NETMAP
73 1.1.4.2 snj /*
74 1.1.4.2 snj * Some drivers may need the following headers. Others
75 1.1.4.2 snj * already include them by default
76 1.1.4.2 snj
77 1.1.4.2 snj #include <vm/vm.h>
78 1.1.4.2 snj #include <vm/pmap.h>
79 1.1.4.2 snj
80 1.1.4.2 snj */
81 1.1.4.2 snj #include "ixgbe.h"
82 1.1.4.2 snj
83 1.1.4.2 snj /*
84 1.1.4.2 snj * device-specific sysctl variables:
85 1.1.4.2 snj *
86 1.1.4.2 snj * ix_crcstrip: 0: keep CRC in rx frames (default), 1: strip it.
87 1.1.4.2 snj * During regular operations the CRC is stripped, but on some
88 1.1.4.2 snj * hardware reception of frames not multiple of 64 is slower,
89 1.1.4.2 snj * so using crcstrip=0 helps in benchmarks.
90 1.1.4.2 snj *
91 1.1.4.2 snj * ix_rx_miss, ix_rx_miss_bufs:
92 1.1.4.2 snj * count packets that might be missed due to lost interrupts.
93 1.1.4.2 snj */
94 1.1.4.2 snj SYSCTL_DECL(_dev_netmap);
95 1.1.4.2 snj static int ix_rx_miss, ix_rx_miss_bufs;
96 1.1.4.2 snj int ix_crcstrip;
97 1.1.4.2 snj SYSCTL_INT(_dev_netmap, OID_AUTO, ix_crcstrip,
98 1.1.4.2 snj CTLFLAG_RW, &ix_crcstrip, 0, "strip CRC on rx frames");
99 1.1.4.2 snj SYSCTL_INT(_dev_netmap, OID_AUTO, ix_rx_miss,
100 1.1.4.2 snj CTLFLAG_RW, &ix_rx_miss, 0, "potentially missed rx intr");
101 1.1.4.2 snj SYSCTL_INT(_dev_netmap, OID_AUTO, ix_rx_miss_bufs,
102 1.1.4.2 snj CTLFLAG_RW, &ix_rx_miss_bufs, 0, "potentially missed rx intr bufs");
103 1.1.4.2 snj
104 1.1.4.2 snj
105 1.1.4.2 snj static void
106 1.1.4.2 snj set_crcstrip(struct ixgbe_hw *hw, int onoff)
107 1.1.4.2 snj {
108 1.1.4.2 snj /* crc stripping is set in two places:
109 1.1.4.2 snj * IXGBE_HLREG0 (modified on init_locked and hw reset)
110 1.1.4.2 snj * IXGBE_RDRXCTL (set by the original driver in
111 1.1.4.2 snj * ixgbe_setup_hw_rsc() called in init_locked.
112 1.1.4.2 snj * We disable the setting when netmap is compiled in).
113 1.1.4.2 snj * We update the values here, but also in ixgbe.c because
114 1.1.4.2 snj * init_locked sometimes is called outside our control.
115 1.1.4.2 snj */
116 1.1.4.2 snj uint32_t hl, rxc;
117 1.1.4.2 snj
118 1.1.4.2 snj hl = IXGBE_READ_REG(hw, IXGBE_HLREG0);
119 1.1.4.2 snj rxc = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
120 1.1.4.2 snj if (netmap_verbose)
121 1.1.4.2 snj D("%s read HLREG 0x%x rxc 0x%x",
122 1.1.4.2 snj onoff ? "enter" : "exit", hl, rxc);
123 1.1.4.2 snj /* hw requirements ... */
124 1.1.4.2 snj rxc &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
125 1.1.4.2 snj rxc |= IXGBE_RDRXCTL_RSCACKC;
126 1.1.4.2 snj if (onoff && !ix_crcstrip) {
127 1.1.4.2 snj /* keep the crc. Fast rx */
128 1.1.4.2 snj hl &= ~IXGBE_HLREG0_RXCRCSTRP;
129 1.1.4.2 snj rxc &= ~IXGBE_RDRXCTL_CRCSTRIP;
130 1.1.4.2 snj } else {
131 1.1.4.2 snj /* reset default mode */
132 1.1.4.2 snj hl |= IXGBE_HLREG0_RXCRCSTRP;
133 1.1.4.2 snj rxc |= IXGBE_RDRXCTL_CRCSTRIP;
134 1.1.4.2 snj }
135 1.1.4.2 snj if (netmap_verbose)
136 1.1.4.2 snj D("%s write HLREG 0x%x rxc 0x%x",
137 1.1.4.2 snj onoff ? "enter" : "exit", hl, rxc);
138 1.1.4.2 snj IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hl);
139 1.1.4.2 snj IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rxc);
140 1.1.4.2 snj }
141 1.1.4.2 snj
142 1.1.4.2 snj
143 1.1.4.2 snj /*
144 1.1.4.2 snj * Register/unregister. We are already under netmap lock.
145 1.1.4.2 snj * Only called on the first register or the last unregister.
146 1.1.4.2 snj */
147 1.1.4.2 snj static int
148 1.1.4.2 snj ixgbe_netmap_reg(struct netmap_adapter *na, int onoff)
149 1.1.4.2 snj {
150 1.1.4.2 snj struct ifnet *ifp = na->ifp;
151 1.1.4.2 snj struct adapter *adapter = ifp->if_softc;
152 1.1.4.2 snj
153 1.1.4.2 snj IXGBE_CORE_LOCK(adapter);
154 1.1.4.2 snj adapter->stop_locked(adapter);
155 1.1.4.2 snj
156 1.1.4.2 snj set_crcstrip(&adapter->hw, onoff);
157 1.1.4.2 snj /* enable or disable flags and callbacks in na and ifp */
158 1.1.4.2 snj if (onoff) {
159 1.1.4.2 snj nm_set_native_flags(na);
160 1.1.4.2 snj } else {
161 1.1.4.2 snj nm_clear_native_flags(na);
162 1.1.4.2 snj }
163 1.1.4.2 snj adapter->init_locked(adapter); /* also enables intr */
164 1.1.4.2 snj set_crcstrip(&adapter->hw, onoff); // XXX why twice ?
165 1.1.4.2 snj IXGBE_CORE_UNLOCK(adapter);
166 1.1.4.2 snj return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1);
167 1.1.4.2 snj }
168 1.1.4.2 snj
169 1.1.4.2 snj
170 1.1.4.2 snj /*
171 1.1.4.2 snj * Reconcile kernel and user view of the transmit ring.
172 1.1.4.2 snj *
173 1.1.4.2 snj * All information is in the kring.
174 1.1.4.2 snj * Userspace wants to send packets up to the one before kring->rhead,
175 1.1.4.2 snj * kernel knows kring->nr_hwcur is the first unsent packet.
176 1.1.4.2 snj *
177 1.1.4.2 snj * Here we push packets out (as many as possible), and possibly
178 1.1.4.2 snj * reclaim buffers from previously completed transmission.
179 1.1.4.2 snj *
180 1.1.4.2 snj * The caller (netmap) guarantees that there is only one instance
181 1.1.4.2 snj * running at any time. Any interference with other driver
182 1.1.4.2 snj * methods should be handled by the individual drivers.
183 1.1.4.2 snj */
184 1.1.4.2 snj static int
185 1.1.4.2 snj ixgbe_netmap_txsync(struct netmap_kring *kring, int flags)
186 1.1.4.2 snj {
187 1.1.4.2 snj struct netmap_adapter *na = kring->na;
188 1.1.4.2 snj struct ifnet *ifp = na->ifp;
189 1.1.4.2 snj struct netmap_ring *ring = kring->ring;
190 1.1.4.2 snj u_int nm_i; /* index into the netmap ring */
191 1.1.4.2 snj u_int nic_i; /* index into the NIC ring */
192 1.1.4.2 snj u_int n;
193 1.1.4.2 snj u_int const lim = kring->nkr_num_slots - 1;
194 1.1.4.2 snj u_int const head = kring->rhead;
195 1.1.4.2 snj /*
196 1.1.4.2 snj * interrupts on every tx packet are expensive so request
197 1.1.4.2 snj * them every half ring, or where NS_REPORT is set
198 1.1.4.2 snj */
199 1.1.4.2 snj u_int report_frequency = kring->nkr_num_slots >> 1;
200 1.1.4.2 snj
201 1.1.4.2 snj /* device-specific */
202 1.1.4.2 snj struct adapter *adapter = ifp->if_softc;
203 1.1.4.2 snj struct tx_ring *txr = &adapter->tx_rings[kring->ring_id];
204 1.1.4.2 snj int reclaim_tx;
205 1.1.4.2 snj
206 1.1.4.2 snj bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
207 1.1.4.2 snj BUS_DMASYNC_POSTREAD);
208 1.1.4.2 snj
209 1.1.4.2 snj /*
210 1.1.4.2 snj * First part: process new packets to send.
211 1.1.4.2 snj * nm_i is the current index in the netmap ring,
212 1.1.4.2 snj * nic_i is the corresponding index in the NIC ring.
213 1.1.4.2 snj * The two numbers differ because upon a *_init() we reset
214 1.1.4.2 snj * the NIC ring but leave the netmap ring unchanged.
215 1.1.4.2 snj * For the transmit ring, we have
216 1.1.4.2 snj *
217 1.1.4.2 snj * nm_i = kring->nr_hwcur
218 1.1.4.2 snj * nic_i = IXGBE_TDT (not tracked in the driver)
219 1.1.4.2 snj * and
220 1.1.4.2 snj * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
221 1.1.4.2 snj *
222 1.1.4.2 snj * In this driver kring->nkr_hwofs >= 0, but for other
223 1.1.4.2 snj * drivers it might be negative as well.
224 1.1.4.2 snj */
225 1.1.4.2 snj
226 1.1.4.2 snj /*
227 1.1.4.2 snj * If we have packets to send (kring->nr_hwcur != kring->rhead)
228 1.1.4.2 snj * iterate over the netmap ring, fetch length and update
229 1.1.4.2 snj * the corresponding slot in the NIC ring. Some drivers also
230 1.1.4.2 snj * need to update the buffer's physical address in the NIC slot
231 1.1.4.2 snj * even NS_BUF_CHANGED is not set (PNMB computes the addresses).
232 1.1.4.2 snj *
233 1.1.4.2 snj * The netmap_reload_map() calls is especially expensive,
234 1.1.4.2 snj * even when (as in this case) the tag is 0, so do only
235 1.1.4.2 snj * when the buffer has actually changed.
236 1.1.4.2 snj *
237 1.1.4.2 snj * If possible do not set the report/intr bit on all slots,
238 1.1.4.2 snj * but only a few times per ring or when NS_REPORT is set.
239 1.1.4.2 snj *
240 1.1.4.2 snj * Finally, on 10G and faster drivers, it might be useful
241 1.1.4.2 snj * to prefetch the next slot and txr entry.
242 1.1.4.2 snj */
243 1.1.4.2 snj
244 1.1.4.2 snj nm_i = kring->nr_hwcur;
245 1.1.4.2 snj if (nm_i != head) { /* we have new packets to send */
246 1.1.4.2 snj nic_i = netmap_idx_k2n(kring, nm_i);
247 1.1.4.2 snj
248 1.1.4.2 snj __builtin_prefetch(&ring->slot[nm_i]);
249 1.1.4.2 snj __builtin_prefetch(&txr->tx_buffers[nic_i]);
250 1.1.4.2 snj
251 1.1.4.2 snj for (n = 0; nm_i != head; n++) {
252 1.1.4.2 snj struct netmap_slot *slot = &ring->slot[nm_i];
253 1.1.4.2 snj u_int len = slot->len;
254 1.1.4.2 snj uint64_t paddr;
255 1.1.4.2 snj void *addr = PNMB(na, slot, &paddr);
256 1.1.4.2 snj
257 1.1.4.2 snj /* device-specific */
258 1.1.4.2 snj union ixgbe_adv_tx_desc *curr = &txr->tx_base[nic_i];
259 1.1.4.2 snj struct ixgbe_tx_buf *txbuf = &txr->tx_buffers[nic_i];
260 1.1.4.2 snj int flags = (slot->flags & NS_REPORT ||
261 1.1.4.2 snj nic_i == 0 || nic_i == report_frequency) ?
262 1.1.4.2 snj IXGBE_TXD_CMD_RS : 0;
263 1.1.4.2 snj
264 1.1.4.2 snj /* prefetch for next round */
265 1.1.4.2 snj __builtin_prefetch(&ring->slot[nm_i + 1]);
266 1.1.4.2 snj __builtin_prefetch(&txr->tx_buffers[nic_i + 1]);
267 1.1.4.2 snj
268 1.1.4.2 snj NM_CHECK_ADDR_LEN(na, addr, len);
269 1.1.4.2 snj
270 1.1.4.2 snj if (slot->flags & NS_BUF_CHANGED) {
271 1.1.4.2 snj /* buffer has changed, reload map */
272 1.1.4.2 snj netmap_reload_map(na, txr->txtag, txbuf->map, addr);
273 1.1.4.2 snj }
274 1.1.4.2 snj slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
275 1.1.4.2 snj
276 1.1.4.2 snj /* Fill the slot in the NIC ring. */
277 1.1.4.2 snj /* Use legacy descriptor, they are faster? */
278 1.1.4.2 snj curr->read.buffer_addr = htole64(paddr);
279 1.1.4.2 snj curr->read.olinfo_status = 0;
280 1.1.4.2 snj curr->read.cmd_type_len = htole32(len | flags |
281 1.1.4.2 snj IXGBE_ADVTXD_DCMD_IFCS | IXGBE_TXD_CMD_EOP);
282 1.1.4.2 snj
283 1.1.4.2 snj /* make sure changes to the buffer are synced */
284 1.1.4.2 snj bus_dmamap_sync(txr->txtag, txbuf->map,
285 1.1.4.2 snj BUS_DMASYNC_PREWRITE);
286 1.1.4.2 snj
287 1.1.4.2 snj nm_i = nm_next(nm_i, lim);
288 1.1.4.2 snj nic_i = nm_next(nic_i, lim);
289 1.1.4.2 snj }
290 1.1.4.2 snj kring->nr_hwcur = head;
291 1.1.4.2 snj
292 1.1.4.2 snj /* synchronize the NIC ring */
293 1.1.4.2 snj bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
294 1.1.4.2 snj BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
295 1.1.4.2 snj
296 1.1.4.2 snj /* (re)start the tx unit up to slot nic_i (excluded) */
297 1.1.4.2 snj IXGBE_WRITE_REG(&adapter->hw, txr->tail, nic_i);
298 1.1.4.2 snj }
299 1.1.4.2 snj
300 1.1.4.2 snj /*
301 1.1.4.2 snj * Second part: reclaim buffers for completed transmissions.
302 1.1.4.2 snj * Because this is expensive (we read a NIC register etc.)
303 1.1.4.2 snj * we only do it in specific cases (see below).
304 1.1.4.2 snj */
305 1.1.4.2 snj if (flags & NAF_FORCE_RECLAIM) {
306 1.1.4.2 snj reclaim_tx = 1; /* forced reclaim */
307 1.1.4.2 snj } else if (!nm_kr_txempty(kring)) {
308 1.1.4.2 snj reclaim_tx = 0; /* have buffers, no reclaim */
309 1.1.4.2 snj } else {
310 1.1.4.2 snj /*
311 1.1.4.2 snj * No buffers available. Locate previous slot with
312 1.1.4.2 snj * REPORT_STATUS set.
313 1.1.4.2 snj * If the slot has DD set, we can reclaim space,
314 1.1.4.2 snj * otherwise wait for the next interrupt.
315 1.1.4.2 snj * This enables interrupt moderation on the tx
316 1.1.4.2 snj * side though it might reduce throughput.
317 1.1.4.2 snj */
318 1.1.4.2 snj struct ixgbe_legacy_tx_desc *txd =
319 1.1.4.2 snj (struct ixgbe_legacy_tx_desc *)txr->tx_base;
320 1.1.4.2 snj
321 1.1.4.2 snj nic_i = txr->next_to_clean + report_frequency;
322 1.1.4.2 snj if (nic_i > lim)
323 1.1.4.2 snj nic_i -= lim + 1;
324 1.1.4.2 snj // round to the closest with dd set
325 1.1.4.2 snj nic_i = (nic_i < kring->nkr_num_slots / 4 ||
326 1.1.4.2 snj nic_i >= kring->nkr_num_slots*3/4) ?
327 1.1.4.2 snj 0 : report_frequency;
328 1.1.4.2 snj reclaim_tx = txd[nic_i].upper.fields.status & IXGBE_TXD_STAT_DD; // XXX cpu_to_le32 ?
329 1.1.4.2 snj }
330 1.1.4.2 snj if (reclaim_tx) {
331 1.1.4.2 snj /*
332 1.1.4.2 snj * Record completed transmissions.
333 1.1.4.2 snj * We (re)use the driver's txr->next_to_clean to keep
334 1.1.4.2 snj * track of the most recently completed transmission.
335 1.1.4.2 snj *
336 1.1.4.2 snj * The datasheet discourages the use of TDH to find
337 1.1.4.2 snj * out the number of sent packets, but we only set
338 1.1.4.2 snj * REPORT_STATUS in a few slots so TDH is the only
339 1.1.4.2 snj * good way.
340 1.1.4.2 snj */
341 1.1.4.2 snj nic_i = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(kring->ring_id));
342 1.1.4.2 snj if (nic_i >= kring->nkr_num_slots) { /* XXX can it happen ? */
343 1.1.4.2 snj D("TDH wrap %d", nic_i);
344 1.1.4.2 snj nic_i -= kring->nkr_num_slots;
345 1.1.4.2 snj }
346 1.1.4.2 snj if (nic_i != txr->next_to_clean) {
347 1.1.4.2 snj /* some tx completed, increment avail */
348 1.1.4.2 snj txr->next_to_clean = nic_i;
349 1.1.4.2 snj kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim);
350 1.1.4.2 snj }
351 1.1.4.2 snj }
352 1.1.4.2 snj
353 1.1.4.2 snj return 0;
354 1.1.4.2 snj }
355 1.1.4.2 snj
356 1.1.4.2 snj
357 1.1.4.2 snj /*
358 1.1.4.2 snj * Reconcile kernel and user view of the receive ring.
359 1.1.4.2 snj * Same as for the txsync, this routine must be efficient.
360 1.1.4.2 snj * The caller guarantees a single invocations, but races against
361 1.1.4.2 snj * the rest of the driver should be handled here.
362 1.1.4.2 snj *
363 1.1.4.2 snj * On call, kring->rhead is the first packet that userspace wants
364 1.1.4.2 snj * to keep, and kring->rcur is the wakeup point.
365 1.1.4.2 snj * The kernel has previously reported packets up to kring->rtail.
366 1.1.4.2 snj *
367 1.1.4.2 snj * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective
368 1.1.4.2 snj * of whether or not we received an interrupt.
369 1.1.4.2 snj */
370 1.1.4.2 snj static int
371 1.1.4.2 snj ixgbe_netmap_rxsync(struct netmap_kring *kring, int flags)
372 1.1.4.2 snj {
373 1.1.4.2 snj struct netmap_adapter *na = kring->na;
374 1.1.4.2 snj struct ifnet *ifp = na->ifp;
375 1.1.4.2 snj struct netmap_ring *ring = kring->ring;
376 1.1.4.2 snj u_int nm_i; /* index into the netmap ring */
377 1.1.4.2 snj u_int nic_i; /* index into the NIC ring */
378 1.1.4.2 snj u_int n;
379 1.1.4.2 snj u_int const lim = kring->nkr_num_slots - 1;
380 1.1.4.2 snj u_int const head = kring->rhead;
381 1.1.4.2 snj int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
382 1.1.4.2 snj
383 1.1.4.2 snj /* device-specific */
384 1.1.4.2 snj struct adapter *adapter = ifp->if_softc;
385 1.1.4.2 snj struct rx_ring *rxr = &adapter->rx_rings[kring->ring_id];
386 1.1.4.2 snj
387 1.1.4.2 snj if (head > lim)
388 1.1.4.2 snj return netmap_ring_reinit(kring);
389 1.1.4.2 snj
390 1.1.4.2 snj /* XXX check sync modes */
391 1.1.4.2 snj bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
392 1.1.4.2 snj BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
393 1.1.4.2 snj
394 1.1.4.2 snj /*
395 1.1.4.2 snj * First part: import newly received packets.
396 1.1.4.2 snj *
397 1.1.4.2 snj * nm_i is the index of the next free slot in the netmap ring,
398 1.1.4.2 snj * nic_i is the index of the next received packet in the NIC ring,
399 1.1.4.2 snj * and they may differ in case if_init() has been called while
400 1.1.4.2 snj * in netmap mode. For the receive ring we have
401 1.1.4.2 snj *
402 1.1.4.2 snj * nic_i = rxr->next_to_check;
403 1.1.4.2 snj * nm_i = kring->nr_hwtail (previous)
404 1.1.4.2 snj * and
405 1.1.4.2 snj * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
406 1.1.4.2 snj *
407 1.1.4.2 snj * rxr->next_to_check is set to 0 on a ring reinit
408 1.1.4.2 snj */
409 1.1.4.2 snj if (netmap_no_pendintr || force_update) {
410 1.1.4.2 snj int crclen = (ix_crcstrip) ? 0 : 4;
411 1.1.4.2 snj
412 1.1.4.2 snj nic_i = rxr->next_to_check; // or also k2n(kring->nr_hwtail)
413 1.1.4.2 snj nm_i = netmap_idx_n2k(kring, nic_i);
414 1.1.4.2 snj
415 1.1.4.2 snj for (n = 0; ; n++) {
416 1.1.4.2 snj union ixgbe_adv_rx_desc *curr = &rxr->rx_base[nic_i];
417 1.1.4.2 snj uint32_t staterr = le32toh(curr->wb.upper.status_error);
418 1.1.4.2 snj
419 1.1.4.2 snj if ((staterr & IXGBE_RXD_STAT_DD) == 0)
420 1.1.4.2 snj break;
421 1.1.4.2 snj ring->slot[nm_i].len = le16toh(curr->wb.upper.length) - crclen;
422 1.1.4.3 martin ring->slot[nm_i].flags = 0;
423 1.1.4.2 snj bus_dmamap_sync(rxr->ptag,
424 1.1.4.2 snj rxr->rx_buffers[nic_i].pmap, BUS_DMASYNC_POSTREAD);
425 1.1.4.2 snj nm_i = nm_next(nm_i, lim);
426 1.1.4.2 snj nic_i = nm_next(nic_i, lim);
427 1.1.4.2 snj }
428 1.1.4.2 snj if (n) { /* update the state variables */
429 1.1.4.2 snj if (netmap_no_pendintr && !force_update) {
430 1.1.4.2 snj /* diagnostics */
431 1.1.4.2 snj ix_rx_miss ++;
432 1.1.4.2 snj ix_rx_miss_bufs += n;
433 1.1.4.2 snj }
434 1.1.4.2 snj rxr->next_to_check = nic_i;
435 1.1.4.2 snj kring->nr_hwtail = nm_i;
436 1.1.4.2 snj }
437 1.1.4.2 snj kring->nr_kflags &= ~NKR_PENDINTR;
438 1.1.4.2 snj }
439 1.1.4.2 snj
440 1.1.4.2 snj /*
441 1.1.4.2 snj * Second part: skip past packets that userspace has released.
442 1.1.4.2 snj * (kring->nr_hwcur to kring->rhead excluded),
443 1.1.4.2 snj * and make the buffers available for reception.
444 1.1.4.2 snj * As usual nm_i is the index in the netmap ring,
445 1.1.4.2 snj * nic_i is the index in the NIC ring, and
446 1.1.4.2 snj * nm_i == (nic_i + kring->nkr_hwofs) % ring_size
447 1.1.4.2 snj */
448 1.1.4.2 snj nm_i = kring->nr_hwcur;
449 1.1.4.2 snj if (nm_i != head) {
450 1.1.4.2 snj nic_i = netmap_idx_k2n(kring, nm_i);
451 1.1.4.2 snj for (n = 0; nm_i != head; n++) {
452 1.1.4.2 snj struct netmap_slot *slot = &ring->slot[nm_i];
453 1.1.4.2 snj uint64_t paddr;
454 1.1.4.2 snj void *addr = PNMB(na, slot, &paddr);
455 1.1.4.2 snj
456 1.1.4.2 snj union ixgbe_adv_rx_desc *curr = &rxr->rx_base[nic_i];
457 1.1.4.2 snj struct ixgbe_rx_buf *rxbuf = &rxr->rx_buffers[nic_i];
458 1.1.4.2 snj
459 1.1.4.2 snj if (addr == NETMAP_BUF_BASE(na)) /* bad buf */
460 1.1.4.2 snj goto ring_reset;
461 1.1.4.2 snj
462 1.1.4.2 snj if (slot->flags & NS_BUF_CHANGED) {
463 1.1.4.2 snj /* buffer has changed, reload map */
464 1.1.4.2 snj netmap_reload_map(na, rxr->ptag, rxbuf->pmap, addr);
465 1.1.4.2 snj slot->flags &= ~NS_BUF_CHANGED;
466 1.1.4.2 snj }
467 1.1.4.2 snj curr->wb.upper.status_error = 0;
468 1.1.4.2 snj curr->read.pkt_addr = htole64(paddr);
469 1.1.4.2 snj bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
470 1.1.4.2 snj BUS_DMASYNC_PREREAD);
471 1.1.4.2 snj nm_i = nm_next(nm_i, lim);
472 1.1.4.2 snj nic_i = nm_next(nic_i, lim);
473 1.1.4.2 snj }
474 1.1.4.2 snj kring->nr_hwcur = head;
475 1.1.4.2 snj
476 1.1.4.2 snj bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
477 1.1.4.2 snj BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
478 1.1.4.2 snj /*
479 1.1.4.2 snj * IMPORTANT: we must leave one free slot in the ring,
480 1.1.4.2 snj * so move nic_i back by one unit
481 1.1.4.2 snj */
482 1.1.4.2 snj nic_i = nm_prev(nic_i, lim);
483 1.1.4.2 snj IXGBE_WRITE_REG(&adapter->hw, rxr->tail, nic_i);
484 1.1.4.2 snj }
485 1.1.4.2 snj
486 1.1.4.2 snj return 0;
487 1.1.4.2 snj
488 1.1.4.2 snj ring_reset:
489 1.1.4.2 snj return netmap_ring_reinit(kring);
490 1.1.4.2 snj }
491 1.1.4.2 snj
492 1.1.4.2 snj
493 1.1.4.2 snj /*
494 1.1.4.2 snj * The attach routine, called near the end of ixgbe_attach(),
495 1.1.4.2 snj * fills the parameters for netmap_attach() and calls it.
496 1.1.4.2 snj * It cannot fail, in the worst case (such as no memory)
497 1.1.4.2 snj * netmap mode will be disabled and the driver will only
498 1.1.4.2 snj * operate in standard mode.
499 1.1.4.2 snj */
500 1.1.4.2 snj void
501 1.1.4.2 snj ixgbe_netmap_attach(struct adapter *adapter)
502 1.1.4.2 snj {
503 1.1.4.2 snj struct netmap_adapter na;
504 1.1.4.2 snj
505 1.1.4.2 snj bzero(&na, sizeof(na));
506 1.1.4.2 snj
507 1.1.4.2 snj na.ifp = adapter->ifp;
508 1.1.4.2 snj na.na_flags = NAF_BDG_MAYSLEEP;
509 1.1.4.2 snj na.num_tx_desc = adapter->num_tx_desc;
510 1.1.4.2 snj na.num_rx_desc = adapter->num_rx_desc;
511 1.1.4.2 snj na.nm_txsync = ixgbe_netmap_txsync;
512 1.1.4.2 snj na.nm_rxsync = ixgbe_netmap_rxsync;
513 1.1.4.2 snj na.nm_register = ixgbe_netmap_reg;
514 1.1.4.2 snj na.num_tx_rings = na.num_rx_rings = adapter->num_queues;
515 1.1.4.2 snj netmap_attach(&na);
516 1.1.4.2 snj }
517 1.1.4.2 snj
518 1.1.4.2 snj #endif /* DEV_NETMAP */
519 1.1.4.2 snj
520 1.1.4.2 snj /* end of file */
521