if_gfe.c revision 1.57.4.1 1 /* $NetBSD: if_gfe.c,v 1.57.4.1 2021/03/22 02:01:01 thorpej Exp $ */
2
3 /*
4 * Copyright (c) 2002 Allegro Networks, Inc., Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. All advertising materials mentioning features or use of this software
16 * must display the following acknowledgement:
17 * This product includes software developed for the NetBSD Project by
18 * Allegro Networks, Inc., and Wasabi Systems, Inc.
19 * 4. The name of Allegro Networks, Inc. may not be used to endorse
20 * or promote products derived from this software without specific prior
21 * written permission.
22 * 5. The name of Wasabi Systems, Inc. may not be used to endorse
23 * or promote products derived from this software without specific prior
24 * written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY ALLEGRO NETWORKS, INC. AND
27 * WASABI SYSTEMS, INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
28 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
29 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
30 * IN NO EVENT SHALL EITHER ALLEGRO NETWORKS, INC. OR WASABI SYSTEMS, INC.
31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 * POSSIBILITY OF SUCH DAMAGE.
38 */
39
40 /*
41 * if_gfe.c -- GT ethernet MAC driver
42 */
43
44 #include <sys/cdefs.h>
45 __KERNEL_RCSID(0, "$NetBSD: if_gfe.c,v 1.57.4.1 2021/03/22 02:01:01 thorpej Exp $");
46
47 #include "opt_inet.h"
48
49 #include <sys/param.h>
50 #include <sys/bus.h>
51 #include <sys/callout.h>
52 #include <sys/device.h>
53 #include <sys/errno.h>
54 #include <sys/ioctl.h>
55 #include <sys/mbuf.h>
56 #include <sys/mutex.h>
57 #include <sys/socket.h>
58
59 #include <net/if.h>
60 #include <net/if_dl.h>
61 #include <net/if_ether.h>
62 #include <net/if_media.h>
63
64 #ifdef INET
65 #include <netinet/in.h>
66 #include <netinet/if_inarp.h>
67 #endif
68 #include <net/bpf.h>
69 #include <sys/rndsource.h>
70
71 #include <dev/mii/mii.h>
72 #include <dev/mii/miivar.h>
73
74 #include <dev/marvell/gtreg.h>
75 #include <dev/marvell/gtvar.h>
76 #include <dev/marvell/gtethreg.h>
77 #include <dev/marvell/if_gfevar.h>
78 #include <dev/marvell/marvellreg.h>
79 #include <dev/marvell/marvellvar.h>
80
81 #include <prop/proplib.h>
82
83 #include "locators.h"
84
85
86 #define GE_READ(sc, reg) \
87 bus_space_read_4((sc)->sc_memt, (sc)->sc_memh, (reg))
88 #define GE_WRITE(sc, reg, v) \
89 bus_space_write_4((sc)->sc_memt, (sc)->sc_memh, (reg), (v))
90
91 #define GE_DEBUG
92 #if 0
93 #define GE_NOHASH
94 #define GE_NORX
95 #endif
96
97 #ifdef GE_DEBUG
98 #define GE_DPRINTF(sc, a) \
99 do { \
100 if ((sc)->sc_ec.ec_if.if_flags & IFF_DEBUG) \
101 printf a; \
102 } while (0 /* CONSTCOND */)
103 #define GE_FUNC_ENTER(sc, func) GE_DPRINTF(sc, ("[" func))
104 #define GE_FUNC_EXIT(sc, str) GE_DPRINTF(sc, (str "]"))
105 #else
106 #define GE_DPRINTF(sc, a) do { } while (0)
107 #define GE_FUNC_ENTER(sc, func) do { } while (0)
108 #define GE_FUNC_EXIT(sc, str) do { } while (0)
109 #endif
110 enum gfe_whack_op {
111 GE_WHACK_START, GE_WHACK_RESTART,
112 GE_WHACK_CHANGE, GE_WHACK_STOP
113 };
114
115 enum gfe_hash_op {
116 GE_HASH_ADD, GE_HASH_REMOVE,
117 };
118
119 #if 1
120 #define htogt32(a) htobe32(a)
121 #define gt32toh(a) be32toh(a)
122 #else
123 #define htogt32(a) htole32(a)
124 #define gt32toh(a) le32toh(a)
125 #endif
126
127 #define GE_RXDSYNC(sc, rxq, n, ops) \
128 bus_dmamap_sync((sc)->sc_dmat, (rxq)->rxq_desc_mem.gdm_map, \
129 (n) * sizeof((rxq)->rxq_descs[0]), sizeof((rxq)->rxq_descs[0]), \
130 (ops))
131 #define GE_RXDPRESYNC(sc, rxq, n) \
132 GE_RXDSYNC(sc, rxq, n, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)
133 #define GE_RXDPOSTSYNC(sc, rxq, n) \
134 GE_RXDSYNC(sc, rxq, n, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)
135
136 #define GE_TXDSYNC(sc, txq, n, ops) \
137 bus_dmamap_sync((sc)->sc_dmat, (txq)->txq_desc_mem.gdm_map, \
138 (n) * sizeof((txq)->txq_descs[0]), sizeof((txq)->txq_descs[0]), \
139 (ops))
140 #define GE_TXDPRESYNC(sc, txq, n) \
141 GE_TXDSYNC(sc, txq, n, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)
142 #define GE_TXDPOSTSYNC(sc, txq, n) \
143 GE_TXDSYNC(sc, txq, n, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)
144
145 #define STATIC
146
147
148 STATIC int gfec_match(device_t, cfdata_t, void *);
149 STATIC void gfec_attach(device_t, device_t, void *);
150
151 STATIC int gfec_print(void *, const char *);
152 STATIC int gfec_search(device_t, cfdata_t, const int *, void *);
153
154 STATIC int gfec_enet_phy(device_t, int);
155 STATIC int gfec_mii_read(device_t, int, int, uint16_t *);
156 STATIC int gfec_mii_write(device_t, int, int, uint16_t);
157 STATIC void gfec_mii_statchg(struct ifnet *);
158
159 STATIC int gfe_match(device_t, cfdata_t, void *);
160 STATIC void gfe_attach(device_t, device_t, void *);
161
162 STATIC int gfe_dmamem_alloc(struct gfe_softc *, struct gfe_dmamem *, int,
163 size_t, int);
164 STATIC void gfe_dmamem_free(struct gfe_softc *, struct gfe_dmamem *);
165
166 STATIC int gfe_ifioctl(struct ifnet *, u_long, void *);
167 STATIC void gfe_ifstart(struct ifnet *);
168 STATIC void gfe_ifwatchdog(struct ifnet *);
169
170 STATIC void gfe_tick(void *arg);
171
172 STATIC void gfe_tx_restart(void *);
173 STATIC int gfe_tx_enqueue(struct gfe_softc *, enum gfe_txprio);
174 STATIC uint32_t gfe_tx_done(struct gfe_softc *, enum gfe_txprio, uint32_t);
175 STATIC void gfe_tx_cleanup(struct gfe_softc *, enum gfe_txprio, int);
176 STATIC int gfe_tx_txqalloc(struct gfe_softc *, enum gfe_txprio);
177 STATIC int gfe_tx_start(struct gfe_softc *, enum gfe_txprio);
178 STATIC void gfe_tx_stop(struct gfe_softc *, enum gfe_whack_op);
179
180 STATIC void gfe_rx_cleanup(struct gfe_softc *, enum gfe_rxprio);
181 STATIC void gfe_rx_get(struct gfe_softc *, enum gfe_rxprio);
182 STATIC int gfe_rx_prime(struct gfe_softc *);
183 STATIC uint32_t gfe_rx_process(struct gfe_softc *, uint32_t, uint32_t);
184 STATIC int gfe_rx_rxqalloc(struct gfe_softc *, enum gfe_rxprio);
185 STATIC int gfe_rx_rxqinit(struct gfe_softc *, enum gfe_rxprio);
186 STATIC void gfe_rx_stop(struct gfe_softc *, enum gfe_whack_op);
187
188 STATIC int gfe_intr(void *);
189
190 STATIC int gfe_whack(struct gfe_softc *, enum gfe_whack_op);
191
192 STATIC int gfe_hash_compute(struct gfe_softc *, const uint8_t [ETHER_ADDR_LEN]);
193 STATIC int gfe_hash_entry_op(struct gfe_softc *, enum gfe_hash_op,
194 enum gfe_rxprio, const uint8_t [ETHER_ADDR_LEN]);
195 STATIC int gfe_hash_multichg(struct ethercom *, const struct ether_multi *,
196 u_long);
197 STATIC int gfe_hash_fill(struct gfe_softc *);
198 STATIC int gfe_hash_alloc(struct gfe_softc *);
199
200
201 CFATTACH_DECL_NEW(gfec, sizeof(struct gfec_softc),
202 gfec_match, gfec_attach, NULL, NULL);
203 CFATTACH_DECL_NEW(gfe, sizeof(struct gfe_softc),
204 gfe_match, gfe_attach, NULL, NULL);
205
206
207 /* ARGSUSED */
208 int
209 gfec_match(device_t parent, cfdata_t cf, void *aux)
210 {
211 struct marvell_attach_args *mva = aux;
212
213 if (strcmp(mva->mva_name, cf->cf_name) != 0)
214 return 0;
215 if (mva->mva_offset == MVA_OFFSET_DEFAULT)
216 return 0;
217
218 mva->mva_size = ETHC_SIZE;
219 return 1;
220 }
221
222 /* ARGSUSED */
223 void
224 gfec_attach(device_t parent, device_t self, void *aux)
225 {
226 struct gfec_softc *sc = device_private(self);
227 struct marvell_attach_args *mva = aux, gfea;
228 static int gfe_irqs[] = { 32, 33, 34 };
229 int i;
230
231 aprint_naive("\n");
232 aprint_normal(": Ethernet Controller\n");
233
234 sc->sc_dev = self;
235 sc->sc_iot = mva->mva_iot;
236 if (bus_space_subregion(mva->mva_iot, mva->mva_ioh, mva->mva_offset,
237 mva->mva_size, &sc->sc_ioh)) {
238 aprint_error_dev(self, "Cannot map registers\n");
239 return;
240 }
241
242 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NET);
243
244 for (i = 0; i < ETH_NUM; i++) {
245 gfea.mva_name = "gfe";
246 gfea.mva_model = mva->mva_model;
247 gfea.mva_iot = sc->sc_iot;
248 gfea.mva_ioh = sc->sc_ioh;
249 gfea.mva_unit = i;
250 gfea.mva_dmat = mva->mva_dmat;
251 gfea.mva_irq = gfe_irqs[i];
252 config_found(sc->sc_dev, &gfea, gfec_print,
253 CFARG_SUBMATCH, gfec_search,
254 CFARG_IATTR, "gfec",
255 CFARG_EOL);
256 }
257 }
258
259 int
260 gfec_print(void *aux, const char *pnp)
261 {
262 struct marvell_attach_args *gfea = aux;
263
264 if (pnp)
265 aprint_normal("%s at %s port %d",
266 gfea->mva_name, pnp, gfea->mva_unit);
267 else {
268 if (gfea->mva_unit != GFECCF_PORT_DEFAULT)
269 aprint_normal(" port %d", gfea->mva_unit);
270 if (gfea->mva_irq != GFECCF_IRQ_DEFAULT)
271 aprint_normal(" irq %d", gfea->mva_irq);
272 }
273 return UNCONF;
274 }
275
276 /* ARGSUSED */
277 int
278 gfec_search(device_t parent, cfdata_t cf, const int *ldesc, void *aux)
279 {
280 struct marvell_attach_args *gfea = aux;
281
282 if (cf->cf_loc[GFECCF_PORT] == gfea->mva_unit &&
283 cf->cf_loc[GFECCF_IRQ] != GFECCF_IRQ_DEFAULT)
284 gfea->mva_irq = cf->cf_loc[GFECCF_IRQ];
285
286 return config_match(parent, cf, aux);
287 }
288
289 int
290 gfec_enet_phy(device_t dev, int unit)
291 {
292 struct gfec_softc *sc = device_private(dev);
293 uint32_t epar;
294
295 epar = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ETH_EPAR);
296 return ETH_EPAR_PhyAD_GET(epar, unit);
297 }
298
299 int
300 gfec_mii_read(device_t dev, int phy, int reg, uint16_t *val)
301 {
302 struct gfec_softc *csc = device_private(device_parent(dev));
303 uint32_t data;
304 int count = 10000;
305
306 mutex_enter(&csc->sc_mtx);
307
308 do {
309 DELAY(10);
310 data = bus_space_read_4(csc->sc_iot, csc->sc_ioh, ETH_ESMIR);
311 } while ((data & ETH_ESMIR_Busy) && count-- > 0);
312
313 if (count == 0) {
314 aprint_error_dev(dev,
315 "mii read for phy %d reg %d busied out\n", phy, reg);
316 mutex_exit(&csc->sc_mtx);
317 return ETIMEDOUT;
318 }
319
320 bus_space_write_4(csc->sc_iot, csc->sc_ioh, ETH_ESMIR,
321 ETH_ESMIR_READ(phy, reg));
322
323 count = 10000;
324 do {
325 DELAY(10);
326 data = bus_space_read_4(csc->sc_iot, csc->sc_ioh, ETH_ESMIR);
327 } while ((data & ETH_ESMIR_ReadValid) == 0 && count-- > 0);
328
329 mutex_exit(&csc->sc_mtx);
330
331 if (count == 0) {
332 aprint_error_dev(dev,
333 "mii read for phy %d reg %d timed out\n", phy, reg);
334 return ETIMEDOUT;
335 }
336 #if defined(GTMIIDEBUG)
337 aprint_normal_dev(dev, "mii_read(%d, %d): %#x data %#x\n",
338 phy, reg, data, ETH_ESMIR_Value_GET(data));
339 #endif
340 *val = ETH_ESMIR_Value_GET(data);
341 return 0;
342 }
343
344 int
345 gfec_mii_write(device_t dev, int phy, int reg, uint16_t value)
346 {
347 struct gfec_softc *csc = device_private(device_parent(dev));
348 uint32_t data;
349 int count = 10000;
350
351 mutex_enter(&csc->sc_mtx);
352
353 do {
354 DELAY(10);
355 data = bus_space_read_4(csc->sc_iot, csc->sc_ioh, ETH_ESMIR);
356 } while ((data & ETH_ESMIR_Busy) && count-- > 0);
357
358 if (count == 0) {
359 aprint_error_dev(dev,
360 "mii write for phy %d reg %d busied out (busy)\n",
361 phy, reg);
362 mutex_exit(&csc->sc_mtx);
363 return ETIMEDOUT;
364 }
365
366 bus_space_write_4(csc->sc_iot, csc->sc_ioh, ETH_ESMIR,
367 ETH_ESMIR_WRITE(phy, reg, value));
368
369 count = 10000;
370 do {
371 DELAY(10);
372 data = bus_space_read_4(csc->sc_iot, csc->sc_ioh, ETH_ESMIR);
373 } while ((data & ETH_ESMIR_Busy) && count-- > 0);
374
375 mutex_exit(&csc->sc_mtx);
376
377 if (count == 0) {
378 aprint_error_dev(dev,
379 "mii write for phy %d reg %d timed out\n", phy, reg);
380 return ETIMEDOUT;
381 }
382 #if defined(GTMIIDEBUG)
383 aprint_normal_dev(dev, "mii_write(%d, %d, %#hx)\n", phy, reg, value);
384 #endif
385 return 0;
386 }
387
388 void
389 gfec_mii_statchg(struct ifnet *ifp)
390 {
391 /* struct gfe_softc *sc = ifp->if_softc; */
392 /* do nothing? */
393 }
394
395 /* ARGSUSED */
396 int
397 gfe_match(device_t parent, cfdata_t cf, void *aux)
398 {
399
400 return 1;
401 }
402
403 /* ARGSUSED */
404 void
405 gfe_attach(device_t parent, device_t self, void *aux)
406 {
407 struct marvell_attach_args *mva = aux;
408 struct gfe_softc * const sc = device_private(self);
409 struct ifnet * const ifp = &sc->sc_ec.ec_if;
410 struct mii_data * const mii = &sc->sc_mii;
411 uint32_t sdcr;
412 int phyaddr, error;
413 prop_data_t ea;
414 uint8_t enaddr[6];
415
416 aprint_naive("\n");
417 aprint_normal(": Ethernet Controller\n");
418
419 if (bus_space_subregion(mva->mva_iot, mva->mva_ioh,
420 mva->mva_offset, mva->mva_size, &sc->sc_memh)) {
421 aprint_error_dev(self, "failed to map registers\n");
422 return;
423 }
424 sc->sc_dev = self;
425 sc->sc_memt = mva->mva_iot;
426 sc->sc_dmat = mva->mva_dmat;
427 sc->sc_macno = (mva->mva_offset == ETH_BASE(0)) ? 0 :
428 ((mva->mva_offset == ETH_BASE(1)) ? 1 : 2);
429
430 callout_init(&sc->sc_co, 0);
431
432 phyaddr = gfec_enet_phy(parent, sc->sc_macno);
433
434 ea = prop_dictionary_get(device_properties(sc->sc_dev), "mac-addr");
435 if (ea != NULL) {
436 KASSERT(prop_object_type(ea) == PROP_TYPE_DATA);
437 KASSERT(prop_data_size(ea) == ETHER_ADDR_LEN);
438 memcpy(enaddr, prop_data_data_nocopy(ea), ETHER_ADDR_LEN);
439 }
440
441 sc->sc_pcr = GE_READ(sc, ETH_EPCR);
442 sc->sc_pcxr = GE_READ(sc, ETH_EPCXR);
443 sc->sc_intrmask = GE_READ(sc, ETH_EIMR) | ETH_IR_MIIPhySTC;
444
445 aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(enaddr));
446
447 #if defined(DEBUG)
448 printf("pcr %#x, pcxr %#x\n", sc->sc_pcr, sc->sc_pcxr);
449 #endif
450
451 sc->sc_pcxr &= ~ETH_EPCXR_PRIOrx_Override;
452 if (device_cfdata(self)->cf_flags & 1) {
453 aprint_normal_dev(self, "phy %d (rmii)\n", phyaddr);
454 sc->sc_pcxr |= ETH_EPCXR_RMIIEn;
455 } else {
456 aprint_normal_dev(self, "phy %d (mii)\n", phyaddr);
457 sc->sc_pcxr &= ~ETH_EPCXR_RMIIEn;
458 }
459 if (device_cfdata(self)->cf_flags & 2)
460 sc->sc_flags |= GE_NOFREE;
461 /* Set Max Frame Length is 1536 */
462 sc->sc_pcxr &= ~ETH_EPCXR_MFL_SET(ETH_EPCXR_MFL_MASK);
463 sc->sc_pcxr |= ETH_EPCXR_MFL_SET(ETH_EPCXR_MFL_1536);
464 sc->sc_max_frame_length = 1536;
465
466 if (sc->sc_pcr & ETH_EPCR_EN) {
467 int tries = 1000;
468 /*
469 * Abort transmitter and receiver and wait for them to quiese
470 */
471 GE_WRITE(sc, ETH_ESDCMR, ETH_ESDCMR_AR | ETH_ESDCMR_AT);
472 do {
473 delay(100);
474 if (tries-- <= 0) {
475 aprint_error_dev(self, "Abort TX/RX failed\n");
476 break;
477 }
478 } while (GE_READ(sc, ETH_ESDCMR) &
479 (ETH_ESDCMR_AR | ETH_ESDCMR_AT));
480 }
481
482 sc->sc_pcr &=
483 ~(ETH_EPCR_EN | ETH_EPCR_RBM | ETH_EPCR_PM | ETH_EPCR_PBF);
484
485 #if defined(DEBUG)
486 printf("pcr %#x, pcxr %#x\n", sc->sc_pcr, sc->sc_pcxr);
487 #endif
488
489 /*
490 * Now turn off the GT. If it didn't quiese, too ***ing bad.
491 */
492 GE_WRITE(sc, ETH_EPCR, sc->sc_pcr);
493 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
494 sdcr = GE_READ(sc, ETH_ESDCR);
495 ETH_ESDCR_BSZ_SET(sdcr, ETH_ESDCR_BSZ_4);
496 sdcr |= ETH_ESDCR_RIFB;
497 GE_WRITE(sc, ETH_ESDCR, sdcr);
498
499 mii->mii_ifp = ifp;
500 mii->mii_readreg = gfec_mii_read;
501 mii->mii_writereg = gfec_mii_write;
502 mii->mii_statchg = gfec_mii_statchg;
503
504 sc->sc_ec.ec_mii = mii;
505 ifmedia_init(&mii->mii_media, 0, ether_mediachange, ether_mediastatus);
506
507 mii_attach(sc->sc_dev, mii, 0xffffffff, phyaddr,
508 MII_OFFSET_ANY, MIIF_NOISOLATE);
509 if (LIST_FIRST(&mii->mii_phys) == NULL) {
510 ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE, 0, NULL);
511 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE);
512 } else
513 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
514
515 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ);
516 ifp->if_softc = sc;
517 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
518 #if 0
519 ifp->if_flags |= IFF_DEBUG;
520 #endif
521 ifp->if_ioctl = gfe_ifioctl;
522 ifp->if_start = gfe_ifstart;
523 ifp->if_watchdog = gfe_ifwatchdog;
524
525 if (sc->sc_flags & GE_NOFREE) {
526 error = gfe_rx_rxqalloc(sc, GE_RXPRIO_HI);
527 if (!error)
528 error = gfe_rx_rxqalloc(sc, GE_RXPRIO_MEDHI);
529 if (!error)
530 error = gfe_rx_rxqalloc(sc, GE_RXPRIO_MEDLO);
531 if (!error)
532 error = gfe_rx_rxqalloc(sc, GE_RXPRIO_LO);
533 if (!error)
534 error = gfe_tx_txqalloc(sc, GE_TXPRIO_HI);
535 if (!error)
536 error = gfe_hash_alloc(sc);
537 if (error)
538 aprint_error_dev(self,
539 "failed to allocate resources: %d\n", error);
540 }
541
542 if_attach(ifp);
543 ether_ifattach(ifp, enaddr);
544 bpf_attach(ifp, DLT_EN10MB, sizeof(struct ether_header));
545 rnd_attach_source(&sc->sc_rnd_source, device_xname(self), RND_TYPE_NET,
546 RND_FLAG_DEFAULT);
547 marvell_intr_establish(mva->mva_irq, IPL_NET, gfe_intr, sc);
548 }
549
550 int
551 gfe_dmamem_alloc(struct gfe_softc *sc, struct gfe_dmamem *gdm, int maxsegs,
552 size_t size, int flags)
553 {
554 int error = 0;
555 GE_FUNC_ENTER(sc, "gfe_dmamem_alloc");
556
557 KASSERT(gdm->gdm_kva == NULL);
558 gdm->gdm_size = size;
559 gdm->gdm_maxsegs = maxsegs;
560
561 error = bus_dmamem_alloc(sc->sc_dmat, gdm->gdm_size, PAGE_SIZE,
562 gdm->gdm_size, gdm->gdm_segs, gdm->gdm_maxsegs, &gdm->gdm_nsegs,
563 BUS_DMA_NOWAIT);
564 if (error)
565 goto fail;
566
567 error = bus_dmamem_map(sc->sc_dmat, gdm->gdm_segs, gdm->gdm_nsegs,
568 gdm->gdm_size, &gdm->gdm_kva, flags | BUS_DMA_NOWAIT);
569 if (error)
570 goto fail;
571
572 error = bus_dmamap_create(sc->sc_dmat, gdm->gdm_size, gdm->gdm_nsegs,
573 gdm->gdm_size, 0, BUS_DMA_ALLOCNOW |BUS_DMA_NOWAIT, &gdm->gdm_map);
574 if (error)
575 goto fail;
576
577 error = bus_dmamap_load(sc->sc_dmat, gdm->gdm_map, gdm->gdm_kva,
578 gdm->gdm_size, NULL, BUS_DMA_NOWAIT);
579 if (error)
580 goto fail;
581
582 /* invalidate from cache */
583 bus_dmamap_sync(sc->sc_dmat, gdm->gdm_map, 0, gdm->gdm_size,
584 BUS_DMASYNC_PREREAD);
585 fail:
586 if (error) {
587 gfe_dmamem_free(sc, gdm);
588 GE_DPRINTF(sc, (":err=%d", error));
589 }
590 GE_DPRINTF(sc, (":kva=%p/%#x,map=%p,nsegs=%d,pa=%x/%x",
591 gdm->gdm_kva, gdm->gdm_size, gdm->gdm_map, gdm->gdm_map->dm_nsegs,
592 gdm->gdm_map->dm_segs->ds_addr, gdm->gdm_map->dm_segs->ds_len));
593 GE_FUNC_EXIT(sc, "");
594 return error;
595 }
596
597 void
598 gfe_dmamem_free(struct gfe_softc *sc, struct gfe_dmamem *gdm)
599 {
600 GE_FUNC_ENTER(sc, "gfe_dmamem_free");
601 if (gdm->gdm_map)
602 bus_dmamap_destroy(sc->sc_dmat, gdm->gdm_map);
603 if (gdm->gdm_kva)
604 bus_dmamem_unmap(sc->sc_dmat, gdm->gdm_kva, gdm->gdm_size);
605 if (gdm->gdm_nsegs > 0)
606 bus_dmamem_free(sc->sc_dmat, gdm->gdm_segs, gdm->gdm_nsegs);
607 gdm->gdm_map = NULL;
608 gdm->gdm_kva = NULL;
609 gdm->gdm_nsegs = 0;
610 GE_FUNC_EXIT(sc, "");
611 }
612
613 int
614 gfe_ifioctl(struct ifnet *ifp, u_long cmd, void *data)
615 {
616 struct gfe_softc * const sc = ifp->if_softc;
617 struct ifreq *ifr = (struct ifreq *) data;
618 struct ifaddr *ifa = (struct ifaddr *) data;
619 int s, error = 0;
620
621 GE_FUNC_ENTER(sc, "gfe_ifioctl");
622 s = splnet();
623
624 switch (cmd) {
625 case SIOCINITIFADDR:
626 ifp->if_flags |= IFF_UP;
627 error = gfe_whack(sc, GE_WHACK_START);
628 switch (ifa->ifa_addr->sa_family) {
629 #ifdef INET
630 case AF_INET:
631 if (error == 0)
632 arp_ifinit(ifp, ifa);
633 break;
634 #endif
635 default:
636 break;
637 }
638 break;
639
640 case SIOCSIFFLAGS:
641 if ((error = ifioctl_common(ifp, cmd, data)) != 0)
642 break;
643 /* XXX re-use ether_ioctl() */
644 switch (ifp->if_flags & (IFF_UP | IFF_RUNNING)) {
645 case IFF_UP | IFF_RUNNING:/* active->active, update */
646 error = gfe_whack(sc, GE_WHACK_CHANGE);
647 break;
648 case IFF_RUNNING: /* not up, so we stop */
649 error = gfe_whack(sc, GE_WHACK_STOP);
650 break;
651 case IFF_UP: /* not running, so we start */
652 error = gfe_whack(sc, GE_WHACK_START);
653 break;
654 case 0: /* idle->idle: do nothing */
655 break;
656 }
657 break;
658
659 case SIOCSIFMTU:
660 if (ifr->ifr_mtu > ETHERMTU || ifr->ifr_mtu < ETHERMIN) {
661 error = EINVAL;
662 break;
663 }
664 if ((error = ifioctl_common(ifp, cmd, data)) == ENETRESET)
665 error = 0;
666 break;
667
668 default:
669 if ((error = ether_ioctl(ifp, cmd, data)) == ENETRESET) {
670 if (ifp->if_flags & IFF_RUNNING)
671 error = gfe_whack(sc, GE_WHACK_CHANGE);
672 else
673 error = 0;
674 }
675 break;
676 }
677 splx(s);
678 GE_FUNC_EXIT(sc, "");
679 return error;
680 }
681
682 void
683 gfe_ifstart(struct ifnet *ifp)
684 {
685 struct gfe_softc * const sc = ifp->if_softc;
686 struct mbuf *m;
687
688 GE_FUNC_ENTER(sc, "gfe_ifstart");
689
690 if ((ifp->if_flags & IFF_RUNNING) == 0) {
691 GE_FUNC_EXIT(sc, "$");
692 return;
693 }
694
695 for (;;) {
696 IF_DEQUEUE(&ifp->if_snd, m);
697 if (m == NULL) {
698 ifp->if_flags &= ~IFF_OACTIVE;
699 GE_FUNC_EXIT(sc, "");
700 return;
701 }
702
703 /*
704 * No space in the pending queue? try later.
705 */
706 if (IF_QFULL(&sc->sc_txq[GE_TXPRIO_HI].txq_pendq))
707 break;
708
709 /*
710 * Try to enqueue a mbuf to the device. If that fails, we
711 * can always try to map the next mbuf.
712 */
713 IF_ENQUEUE(&sc->sc_txq[GE_TXPRIO_HI].txq_pendq, m);
714 GE_DPRINTF(sc, (">"));
715 #ifndef GE_NOTX
716 (void) gfe_tx_enqueue(sc, GE_TXPRIO_HI);
717 #endif
718 }
719
720 /*
721 * Attempt to queue the mbuf for send failed.
722 */
723 IF_PREPEND(&ifp->if_snd, m);
724 ifp->if_flags |= IFF_OACTIVE;
725 GE_FUNC_EXIT(sc, "%%");
726 }
727
728 void
729 gfe_ifwatchdog(struct ifnet *ifp)
730 {
731 struct gfe_softc * const sc = ifp->if_softc;
732 struct gfe_txqueue * const txq = &sc->sc_txq[GE_TXPRIO_HI];
733
734 GE_FUNC_ENTER(sc, "gfe_ifwatchdog");
735 aprint_error_dev(sc->sc_dev, "device timeout");
736 if (ifp->if_flags & IFF_RUNNING) {
737 uint32_t curtxdnum;
738
739 curtxdnum = (GE_READ(sc, txq->txq_ectdp) -
740 txq->txq_desc_busaddr) / sizeof(txq->txq_descs[0]);
741 GE_TXDPOSTSYNC(sc, txq, txq->txq_fi);
742 GE_TXDPOSTSYNC(sc, txq, curtxdnum);
743 aprint_error(" (fi=%d(%#x),lo=%d,cur=%d(%#x),icm=%#x) ",
744 txq->txq_fi, txq->txq_descs[txq->txq_fi].ed_cmdsts,
745 txq->txq_lo, curtxdnum, txq->txq_descs[curtxdnum].ed_cmdsts,
746 GE_READ(sc, ETH_EICR));
747 GE_TXDPRESYNC(sc, txq, txq->txq_fi);
748 GE_TXDPRESYNC(sc, txq, curtxdnum);
749 }
750 aprint_error("\n");
751 if_statinc(ifp, if_oerrors);
752 (void) gfe_whack(sc, GE_WHACK_RESTART);
753 GE_FUNC_EXIT(sc, "");
754 }
755
756 int
757 gfe_rx_rxqalloc(struct gfe_softc *sc, enum gfe_rxprio rxprio)
758 {
759 struct gfe_rxqueue * const rxq = &sc->sc_rxq[rxprio];
760 int error;
761
762 GE_FUNC_ENTER(sc, "gfe_rx_rxqalloc");
763 GE_DPRINTF(sc, ("(%d)", rxprio));
764
765 error = gfe_dmamem_alloc(sc, &rxq->rxq_desc_mem, 1,
766 GE_RXDESC_MEMSIZE, BUS_DMA_NOCACHE);
767 if (error) {
768 GE_FUNC_EXIT(sc, "!!");
769 return error;
770 }
771
772 error = gfe_dmamem_alloc(sc, &rxq->rxq_buf_mem, GE_RXBUF_NSEGS,
773 GE_RXBUF_MEMSIZE, 0);
774 if (error) {
775 GE_FUNC_EXIT(sc, "!!!");
776 return error;
777 }
778 GE_FUNC_EXIT(sc, "");
779 return error;
780 }
781
782 int
783 gfe_rx_rxqinit(struct gfe_softc *sc, enum gfe_rxprio rxprio)
784 {
785 struct gfe_rxqueue * const rxq = &sc->sc_rxq[rxprio];
786 volatile struct gt_eth_desc *rxd;
787 const bus_dma_segment_t *ds;
788 int idx;
789 bus_addr_t nxtaddr;
790 bus_size_t boff;
791
792 GE_FUNC_ENTER(sc, "gfe_rx_rxqinit");
793 GE_DPRINTF(sc, ("(%d)", rxprio));
794
795 if ((sc->sc_flags & GE_NOFREE) == 0) {
796 int error = gfe_rx_rxqalloc(sc, rxprio);
797 if (error) {
798 GE_FUNC_EXIT(sc, "!");
799 return error;
800 }
801 } else {
802 KASSERT(rxq->rxq_desc_mem.gdm_kva != NULL);
803 KASSERT(rxq->rxq_buf_mem.gdm_kva != NULL);
804 }
805
806 memset(rxq->rxq_desc_mem.gdm_kva, 0, GE_RXDESC_MEMSIZE);
807
808 rxq->rxq_descs =
809 (volatile struct gt_eth_desc *) rxq->rxq_desc_mem.gdm_kva;
810 rxq->rxq_desc_busaddr = rxq->rxq_desc_mem.gdm_map->dm_segs[0].ds_addr;
811 rxq->rxq_bufs = (struct gfe_rxbuf *) rxq->rxq_buf_mem.gdm_kva;
812 rxq->rxq_fi = 0;
813 rxq->rxq_active = GE_RXDESC_MAX;
814 boff = 0;
815 ds = rxq->rxq_buf_mem.gdm_map->dm_segs;
816 nxtaddr = rxq->rxq_desc_busaddr + sizeof(*rxd);
817 for (idx = 0, rxd = rxq->rxq_descs; idx < GE_RXDESC_MAX;
818 idx++, rxd++, nxtaddr += sizeof(*rxd)) {
819 rxd->ed_lencnt = htogt32(GE_RXBUF_SIZE << 16);
820 rxd->ed_cmdsts = htogt32(RX_CMD_F|RX_CMD_L|RX_CMD_O|RX_CMD_EI);
821 rxd->ed_bufptr = htogt32(ds->ds_addr + boff);
822 /*
823 * update the nxtptr to point to the next txd.
824 */
825 if (idx == GE_RXDESC_MAX - 1)
826 nxtaddr = rxq->rxq_desc_busaddr;
827 rxd->ed_nxtptr = htogt32(nxtaddr);
828 boff += GE_RXBUF_SIZE;
829 if (boff == ds->ds_len) {
830 ds++;
831 boff = 0;
832 }
833 }
834 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_desc_mem.gdm_map, 0,
835 rxq->rxq_desc_mem.gdm_map->dm_mapsize,
836 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
837 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_buf_mem.gdm_map, 0,
838 rxq->rxq_buf_mem.gdm_map->dm_mapsize,
839 BUS_DMASYNC_PREREAD);
840
841 rxq->rxq_intrbits = ETH_IR_RxBuffer | ETH_IR_RxError;
842 switch (rxprio) {
843 case GE_RXPRIO_HI:
844 rxq->rxq_intrbits |= ETH_IR_RxBuffer_3 | ETH_IR_RxError_3;
845 rxq->rxq_efrdp = ETH_EFRDP3;
846 rxq->rxq_ecrdp = ETH_ECRDP3;
847 break;
848 case GE_RXPRIO_MEDHI:
849 rxq->rxq_intrbits |= ETH_IR_RxBuffer_2 | ETH_IR_RxError_2;
850 rxq->rxq_efrdp = ETH_EFRDP2;
851 rxq->rxq_ecrdp = ETH_ECRDP2;
852 break;
853 case GE_RXPRIO_MEDLO:
854 rxq->rxq_intrbits |= ETH_IR_RxBuffer_1 | ETH_IR_RxError_1;
855 rxq->rxq_efrdp = ETH_EFRDP1;
856 rxq->rxq_ecrdp = ETH_ECRDP1;
857 break;
858 case GE_RXPRIO_LO:
859 rxq->rxq_intrbits |= ETH_IR_RxBuffer_0 | ETH_IR_RxError_0;
860 rxq->rxq_efrdp = ETH_EFRDP0;
861 rxq->rxq_ecrdp = ETH_ECRDP0;
862 break;
863 }
864 GE_FUNC_EXIT(sc, "");
865 return 0;
866 }
867
868 void
869 gfe_rx_get(struct gfe_softc *sc, enum gfe_rxprio rxprio)
870 {
871 struct ifnet * const ifp = &sc->sc_ec.ec_if;
872 struct gfe_rxqueue * const rxq = &sc->sc_rxq[rxprio];
873 struct mbuf *m = rxq->rxq_curpkt;
874
875 GE_FUNC_ENTER(sc, "gfe_rx_get");
876 GE_DPRINTF(sc, ("(%d)", rxprio));
877
878 while (rxq->rxq_active > 0) {
879 volatile struct gt_eth_desc *rxd = &rxq->rxq_descs[rxq->rxq_fi];
880 struct gfe_rxbuf *rxb = &rxq->rxq_bufs[rxq->rxq_fi];
881 const struct ether_header *eh;
882 unsigned int cmdsts;
883 size_t buflen;
884
885 GE_RXDPOSTSYNC(sc, rxq, rxq->rxq_fi);
886 cmdsts = gt32toh(rxd->ed_cmdsts);
887 GE_DPRINTF(sc, (":%d=%#x", rxq->rxq_fi, cmdsts));
888 rxq->rxq_cmdsts = cmdsts;
889 /*
890 * Sometimes the GE "forgets" to reset the ownership bit.
891 * But if the length has been rewritten, the packet is ours
892 * so pretend the O bit is set.
893 */
894 buflen = gt32toh(rxd->ed_lencnt) & 0xffff;
895 if ((cmdsts & RX_CMD_O) && buflen == 0) {
896 GE_RXDPRESYNC(sc, rxq, rxq->rxq_fi);
897 break;
898 }
899
900 /*
901 * If this is not a single buffer packet with no errors
902 * or for some reason it's bigger than our frame size,
903 * ignore it and go to the next packet.
904 */
905 if ((cmdsts & (RX_CMD_F | RX_CMD_L | RX_STS_ES)) !=
906 (RX_CMD_F | RX_CMD_L) ||
907 (buflen > sc->sc_max_frame_length)) {
908 GE_DPRINTF(sc, ("!"));
909 --rxq->rxq_active;
910 if_statinc(ifp, if_ipackets);
911 if_statinc(ifp, if_ierrors);
912 goto give_it_back;
913 }
914
915 /* CRC is included with the packet; trim it off. */
916 buflen -= ETHER_CRC_LEN;
917
918 if (m == NULL) {
919 MGETHDR(m, M_DONTWAIT, MT_DATA);
920 if (m == NULL) {
921 GE_DPRINTF(sc, ("?"));
922 break;
923 }
924 }
925 if ((m->m_flags & M_EXT) == 0 && buflen > MHLEN - 2) {
926 MCLGET(m, M_DONTWAIT);
927 if ((m->m_flags & M_EXT) == 0) {
928 GE_DPRINTF(sc, ("?"));
929 break;
930 }
931 }
932 m->m_data += 2;
933 m->m_len = 0;
934 m->m_pkthdr.len = 0;
935 m_set_rcvif(m, ifp);
936 rxq->rxq_cmdsts = cmdsts;
937 --rxq->rxq_active;
938
939 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_buf_mem.gdm_map,
940 rxq->rxq_fi * sizeof(*rxb), buflen, BUS_DMASYNC_POSTREAD);
941
942 KASSERT(m->m_len == 0 && m->m_pkthdr.len == 0);
943 memcpy(m->m_data + m->m_len, rxb->rxb_data, buflen);
944 m->m_len = buflen;
945 m->m_pkthdr.len = buflen;
946
947 eh = (const struct ether_header *) m->m_data;
948 if ((ifp->if_flags & IFF_PROMISC) ||
949 (rxq->rxq_cmdsts & RX_STS_M) == 0 ||
950 (rxq->rxq_cmdsts & RX_STS_HE) ||
951 (eh->ether_dhost[0] & 1) != 0 ||
952 memcmp(eh->ether_dhost, CLLADDR(ifp->if_sadl),
953 ETHER_ADDR_LEN) == 0) {
954 if_percpuq_enqueue(ifp->if_percpuq, m);
955 m = NULL;
956 GE_DPRINTF(sc, (">"));
957 } else {
958 m->m_len = 0;
959 m->m_pkthdr.len = 0;
960 GE_DPRINTF(sc, ("+"));
961 }
962 rxq->rxq_cmdsts = 0;
963
964 give_it_back:
965 rxd->ed_lencnt &= ~0xffff; /* zero out length */
966 rxd->ed_cmdsts = htogt32(RX_CMD_F|RX_CMD_L|RX_CMD_O|RX_CMD_EI);
967 #if 0
968 GE_DPRINTF(sc, ("([%d]->%08lx.%08lx.%08lx.%08lx)",
969 rxq->rxq_fi,
970 ((unsigned long *)rxd)[0], ((unsigned long *)rxd)[1],
971 ((unsigned long *)rxd)[2], ((unsigned long *)rxd)[3]));
972 #endif
973 GE_RXDPRESYNC(sc, rxq, rxq->rxq_fi);
974 if (++rxq->rxq_fi == GE_RXDESC_MAX)
975 rxq->rxq_fi = 0;
976 rxq->rxq_active++;
977 }
978 rxq->rxq_curpkt = m;
979 GE_FUNC_EXIT(sc, "");
980 }
981
982 uint32_t
983 gfe_rx_process(struct gfe_softc *sc, uint32_t cause, uint32_t intrmask)
984 {
985 struct ifnet * const ifp = &sc->sc_ec.ec_if;
986 struct gfe_rxqueue *rxq;
987 uint32_t rxbits;
988 #define RXPRIO_DECODER 0xffffaa50
989 GE_FUNC_ENTER(sc, "gfe_rx_process");
990
991 rxbits = ETH_IR_RxBuffer_GET(cause);
992 while (rxbits) {
993 enum gfe_rxprio rxprio = (RXPRIO_DECODER >> (rxbits * 2)) & 3;
994 GE_DPRINTF(sc, ("%1x", rxbits));
995 rxbits &= ~(1 << rxprio);
996 gfe_rx_get(sc, rxprio);
997 }
998
999 rxbits = ETH_IR_RxError_GET(cause);
1000 while (rxbits) {
1001 enum gfe_rxprio rxprio = (RXPRIO_DECODER >> (rxbits * 2)) & 3;
1002 uint32_t masks[(GE_RXDESC_MAX + 31) / 32];
1003 int idx;
1004 rxbits &= ~(1 << rxprio);
1005 rxq = &sc->sc_rxq[rxprio];
1006 sc->sc_idlemask |= (rxq->rxq_intrbits & ETH_IR_RxBits);
1007 intrmask &= ~(rxq->rxq_intrbits & ETH_IR_RxBits);
1008 if ((sc->sc_tickflags & GE_TICK_RX_RESTART) == 0) {
1009 sc->sc_tickflags |= GE_TICK_RX_RESTART;
1010 callout_reset(&sc->sc_co, 1, gfe_tick, sc);
1011 }
1012 if_statinc(ifp, if_ierrors);
1013 GE_DPRINTF(sc, ("%s: rx queue %d filled at %u\n",
1014 device_xname(sc->sc_dev), rxprio, rxq->rxq_fi));
1015 memset(masks, 0, sizeof(masks));
1016 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_desc_mem.gdm_map,
1017 0, rxq->rxq_desc_mem.gdm_size,
1018 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1019 for (idx = 0; idx < GE_RXDESC_MAX; idx++) {
1020 volatile struct gt_eth_desc *rxd = &rxq->rxq_descs[idx];
1021
1022 if (RX_CMD_O & gt32toh(rxd->ed_cmdsts))
1023 masks[idx/32] |= 1 << (idx & 31);
1024 }
1025 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_desc_mem.gdm_map,
1026 0, rxq->rxq_desc_mem.gdm_size,
1027 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1028 #if defined(DEBUG)
1029 printf("%s: rx queue %d filled at %u=%#x(%#x/%#x)\n",
1030 device_xname(sc->sc_dev), rxprio, rxq->rxq_fi,
1031 rxq->rxq_cmdsts, masks[0], masks[1]);
1032 #endif
1033 }
1034 if ((intrmask & ETH_IR_RxBits) == 0)
1035 intrmask &= ~(ETH_IR_RxBuffer | ETH_IR_RxError);
1036
1037 GE_FUNC_EXIT(sc, "");
1038 return intrmask;
1039 }
1040
1041 int
1042 gfe_rx_prime(struct gfe_softc *sc)
1043 {
1044 struct gfe_rxqueue *rxq;
1045 int error;
1046
1047 GE_FUNC_ENTER(sc, "gfe_rx_prime");
1048
1049 error = gfe_rx_rxqinit(sc, GE_RXPRIO_HI);
1050 if (error)
1051 goto bail;
1052 rxq = &sc->sc_rxq[GE_RXPRIO_HI];
1053 if ((sc->sc_flags & GE_RXACTIVE) == 0) {
1054 GE_WRITE(sc, ETH_EFRDP3, rxq->rxq_desc_busaddr);
1055 GE_WRITE(sc, ETH_ECRDP3, rxq->rxq_desc_busaddr);
1056 }
1057 sc->sc_intrmask |= rxq->rxq_intrbits;
1058
1059 error = gfe_rx_rxqinit(sc, GE_RXPRIO_MEDHI);
1060 if (error)
1061 goto bail;
1062 if ((sc->sc_flags & GE_RXACTIVE) == 0) {
1063 rxq = &sc->sc_rxq[GE_RXPRIO_MEDHI];
1064 GE_WRITE(sc, ETH_EFRDP2, rxq->rxq_desc_busaddr);
1065 GE_WRITE(sc, ETH_ECRDP2, rxq->rxq_desc_busaddr);
1066 sc->sc_intrmask |= rxq->rxq_intrbits;
1067 }
1068
1069 error = gfe_rx_rxqinit(sc, GE_RXPRIO_MEDLO);
1070 if (error)
1071 goto bail;
1072 if ((sc->sc_flags & GE_RXACTIVE) == 0) {
1073 rxq = &sc->sc_rxq[GE_RXPRIO_MEDLO];
1074 GE_WRITE(sc, ETH_EFRDP1, rxq->rxq_desc_busaddr);
1075 GE_WRITE(sc, ETH_ECRDP1, rxq->rxq_desc_busaddr);
1076 sc->sc_intrmask |= rxq->rxq_intrbits;
1077 }
1078
1079 error = gfe_rx_rxqinit(sc, GE_RXPRIO_LO);
1080 if (error)
1081 goto bail;
1082 if ((sc->sc_flags & GE_RXACTIVE) == 0) {
1083 rxq = &sc->sc_rxq[GE_RXPRIO_LO];
1084 GE_WRITE(sc, ETH_EFRDP0, rxq->rxq_desc_busaddr);
1085 GE_WRITE(sc, ETH_ECRDP0, rxq->rxq_desc_busaddr);
1086 sc->sc_intrmask |= rxq->rxq_intrbits;
1087 }
1088
1089 bail:
1090 GE_FUNC_EXIT(sc, "");
1091 return error;
1092 }
1093
1094 void
1095 gfe_rx_cleanup(struct gfe_softc *sc, enum gfe_rxprio rxprio)
1096 {
1097 struct gfe_rxqueue *rxq = &sc->sc_rxq[rxprio];
1098 GE_FUNC_ENTER(sc, "gfe_rx_cleanup");
1099 if (rxq == NULL) {
1100 GE_FUNC_EXIT(sc, "");
1101 return;
1102 }
1103
1104 if (rxq->rxq_curpkt)
1105 m_freem(rxq->rxq_curpkt);
1106 if ((sc->sc_flags & GE_NOFREE) == 0) {
1107 gfe_dmamem_free(sc, &rxq->rxq_desc_mem);
1108 gfe_dmamem_free(sc, &rxq->rxq_buf_mem);
1109 }
1110 GE_FUNC_EXIT(sc, "");
1111 }
1112
1113 void
1114 gfe_rx_stop(struct gfe_softc *sc, enum gfe_whack_op op)
1115 {
1116 GE_FUNC_ENTER(sc, "gfe_rx_stop");
1117 sc->sc_flags &= ~GE_RXACTIVE;
1118 sc->sc_idlemask &= ~(ETH_IR_RxBits | ETH_IR_RxBuffer | ETH_IR_RxError);
1119 sc->sc_intrmask &= ~(ETH_IR_RxBits | ETH_IR_RxBuffer | ETH_IR_RxError);
1120 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
1121 GE_WRITE(sc, ETH_ESDCMR, ETH_ESDCMR_AR);
1122 do {
1123 delay(10);
1124 } while (GE_READ(sc, ETH_ESDCMR) & ETH_ESDCMR_AR);
1125 gfe_rx_cleanup(sc, GE_RXPRIO_HI);
1126 gfe_rx_cleanup(sc, GE_RXPRIO_MEDHI);
1127 gfe_rx_cleanup(sc, GE_RXPRIO_MEDLO);
1128 gfe_rx_cleanup(sc, GE_RXPRIO_LO);
1129 GE_FUNC_EXIT(sc, "");
1130 }
1131
1132 void
1133 gfe_tick(void *arg)
1134 {
1135 struct gfe_softc * const sc = arg;
1136 uint32_t intrmask;
1137 unsigned int tickflags;
1138 int s;
1139
1140 GE_FUNC_ENTER(sc, "gfe_tick");
1141
1142 s = splnet();
1143
1144 tickflags = sc->sc_tickflags;
1145 sc->sc_tickflags = 0;
1146 intrmask = sc->sc_intrmask;
1147 if (tickflags & GE_TICK_TX_IFSTART)
1148 gfe_ifstart(&sc->sc_ec.ec_if);
1149 if (tickflags & GE_TICK_RX_RESTART) {
1150 intrmask |= sc->sc_idlemask;
1151 if (sc->sc_idlemask & (ETH_IR_RxBuffer_3 | ETH_IR_RxError_3)) {
1152 struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_HI];
1153 rxq->rxq_fi = 0;
1154 GE_WRITE(sc, ETH_EFRDP3, rxq->rxq_desc_busaddr);
1155 GE_WRITE(sc, ETH_ECRDP3, rxq->rxq_desc_busaddr);
1156 }
1157 if (sc->sc_idlemask & (ETH_IR_RxBuffer_2 | ETH_IR_RxError_2)) {
1158 struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_MEDHI];
1159 rxq->rxq_fi = 0;
1160 GE_WRITE(sc, ETH_EFRDP2, rxq->rxq_desc_busaddr);
1161 GE_WRITE(sc, ETH_ECRDP2, rxq->rxq_desc_busaddr);
1162 }
1163 if (sc->sc_idlemask & (ETH_IR_RxBuffer_1 | ETH_IR_RxError_1)) {
1164 struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_MEDLO];
1165 rxq->rxq_fi = 0;
1166 GE_WRITE(sc, ETH_EFRDP1, rxq->rxq_desc_busaddr);
1167 GE_WRITE(sc, ETH_ECRDP1, rxq->rxq_desc_busaddr);
1168 }
1169 if (sc->sc_idlemask & (ETH_IR_RxBuffer_0 | ETH_IR_RxError_0)) {
1170 struct gfe_rxqueue *rxq = &sc->sc_rxq[GE_RXPRIO_LO];
1171 rxq->rxq_fi = 0;
1172 GE_WRITE(sc, ETH_EFRDP0, rxq->rxq_desc_busaddr);
1173 GE_WRITE(sc, ETH_ECRDP0, rxq->rxq_desc_busaddr);
1174 }
1175 sc->sc_idlemask = 0;
1176 }
1177 if (intrmask != sc->sc_intrmask) {
1178 sc->sc_intrmask = intrmask;
1179 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
1180 }
1181 gfe_intr(sc);
1182 splx(s);
1183
1184 GE_FUNC_EXIT(sc, "");
1185 }
1186
1187 int
1188 gfe_tx_enqueue(struct gfe_softc *sc, enum gfe_txprio txprio)
1189 {
1190 const int dcache_line_size = curcpu()->ci_ci.dcache_line_size;
1191 struct ifnet * const ifp = &sc->sc_ec.ec_if;
1192 struct gfe_txqueue * const txq = &sc->sc_txq[txprio];
1193 volatile struct gt_eth_desc * const txd = &txq->txq_descs[txq->txq_lo];
1194 uint32_t intrmask = sc->sc_intrmask;
1195 size_t buflen;
1196 struct mbuf *m;
1197
1198 GE_FUNC_ENTER(sc, "gfe_tx_enqueue");
1199
1200 /*
1201 * Anything in the pending queue to enqueue? if not, punt. Likewise
1202 * if the txq is not yet created.
1203 * otherwise grab its dmamap.
1204 */
1205 if (txq == NULL || (m = txq->txq_pendq.ifq_head) == NULL) {
1206 GE_FUNC_EXIT(sc, "-");
1207 return 0;
1208 }
1209
1210 /*
1211 * Have we [over]consumed our limit of descriptors?
1212 * Do we have enough free descriptors?
1213 */
1214 if (GE_TXDESC_MAX == txq->txq_nactive + 2) {
1215 volatile struct gt_eth_desc * const txd2 = &txq->txq_descs[txq->txq_fi];
1216 uint32_t cmdsts;
1217 size_t pktlen;
1218 GE_TXDPOSTSYNC(sc, txq, txq->txq_fi);
1219 cmdsts = gt32toh(txd2->ed_cmdsts);
1220 if (cmdsts & TX_CMD_O) {
1221 int nextin;
1222 /*
1223 * Sometime the Discovery forgets to update the
1224 * last descriptor. See if we own the descriptor
1225 * after it (since we know we've turned that to
1226 * the discovery and if we owned it, the Discovery
1227 * gave it back). If we do, we know the Discovery
1228 * gave back this one but forgot to mark it as ours.
1229 */
1230 nextin = txq->txq_fi + 1;
1231 if (nextin == GE_TXDESC_MAX)
1232 nextin = 0;
1233 GE_TXDPOSTSYNC(sc, txq, nextin);
1234 if (gt32toh(txq->txq_descs[nextin].ed_cmdsts) & TX_CMD_O) {
1235 GE_TXDPRESYNC(sc, txq, txq->txq_fi);
1236 GE_TXDPRESYNC(sc, txq, nextin);
1237 GE_FUNC_EXIT(sc, "@");
1238 return 0;
1239 }
1240 #ifdef DEBUG
1241 printf("%s: txenqueue: transmitter resynced at %d\n",
1242 device_xname(sc->sc_dev), txq->txq_fi);
1243 #endif
1244 }
1245 if (++txq->txq_fi == GE_TXDESC_MAX)
1246 txq->txq_fi = 0;
1247 txq->txq_inptr = gt32toh(txd2->ed_bufptr) - txq->txq_buf_busaddr;
1248 pktlen = (gt32toh(txd2->ed_lencnt) >> 16) & 0xffff;
1249 txq->txq_inptr += roundup(pktlen, dcache_line_size);
1250 txq->txq_nactive--;
1251
1252 /* statistics */
1253 if_statinc(ifp, if_opackets);
1254 if (cmdsts & TX_STS_ES)
1255 if_statinc(ifp, if_oerrors);
1256 GE_DPRINTF(sc, ("%%"));
1257 }
1258
1259 buflen = roundup(m->m_pkthdr.len, dcache_line_size);
1260
1261 /*
1262 * If this packet would wrap around the end of the buffer, reset back
1263 * to the beginning.
1264 */
1265 if (txq->txq_outptr + buflen > GE_TXBUF_SIZE) {
1266 txq->txq_ei_gapcount += GE_TXBUF_SIZE - txq->txq_outptr;
1267 txq->txq_outptr = 0;
1268 }
1269
1270 /*
1271 * Make sure the output packet doesn't run over the beginning of
1272 * what we've already given the GT.
1273 */
1274 if (txq->txq_nactive > 0 && txq->txq_outptr <= txq->txq_inptr &&
1275 txq->txq_outptr + buflen > txq->txq_inptr) {
1276 intrmask |= txq->txq_intrbits &
1277 (ETH_IR_TxBufferHigh | ETH_IR_TxBufferLow);
1278 if (sc->sc_intrmask != intrmask) {
1279 sc->sc_intrmask = intrmask;
1280 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
1281 }
1282 GE_FUNC_EXIT(sc, "#");
1283 return 0;
1284 }
1285
1286 /*
1287 * The end-of-list descriptor we put on last time is the starting point
1288 * for this packet. The GT is supposed to terminate list processing on
1289 * a NULL nxtptr but that currently is broken so a CPU-owned descriptor
1290 * must terminate the list.
1291 */
1292 intrmask = sc->sc_intrmask;
1293
1294 m_copydata(m, 0, m->m_pkthdr.len,
1295 (char *)txq->txq_buf_mem.gdm_kva + (int)txq->txq_outptr);
1296 bus_dmamap_sync(sc->sc_dmat, txq->txq_buf_mem.gdm_map,
1297 txq->txq_outptr, buflen, BUS_DMASYNC_PREWRITE);
1298 txd->ed_bufptr = htogt32(txq->txq_buf_busaddr + txq->txq_outptr);
1299 txd->ed_lencnt = htogt32(m->m_pkthdr.len << 16);
1300 GE_TXDPRESYNC(sc, txq, txq->txq_lo);
1301
1302 /*
1303 * Request a buffer interrupt every 2/3 of the way thru the transmit
1304 * buffer.
1305 */
1306 txq->txq_ei_gapcount += buflen;
1307 if (txq->txq_ei_gapcount > 2 * GE_TXBUF_SIZE / 3) {
1308 txd->ed_cmdsts = htogt32(TX_CMD_FIRST |TX_CMD_LAST |TX_CMD_EI);
1309 txq->txq_ei_gapcount = 0;
1310 } else {
1311 txd->ed_cmdsts = htogt32(TX_CMD_FIRST | TX_CMD_LAST);
1312 }
1313 #if 0
1314 GE_DPRINTF(sc, ("([%d]->%08lx.%08lx.%08lx.%08lx)", txq->txq_lo,
1315 ((unsigned long *)txd)[0], ((unsigned long *)txd)[1],
1316 ((unsigned long *)txd)[2], ((unsigned long *)txd)[3]));
1317 #endif
1318 GE_TXDPRESYNC(sc, txq, txq->txq_lo);
1319
1320 txq->txq_outptr += buflen;
1321 /*
1322 * Tell the SDMA engine to "Fetch!"
1323 */
1324 GE_WRITE(sc, ETH_ESDCMR,
1325 txq->txq_esdcmrbits & (ETH_ESDCMR_TXDH | ETH_ESDCMR_TXDL));
1326
1327 GE_DPRINTF(sc, ("(%d)", txq->txq_lo));
1328
1329 /*
1330 * Update the last out appropriately.
1331 */
1332 txq->txq_nactive++;
1333 if (++txq->txq_lo == GE_TXDESC_MAX)
1334 txq->txq_lo = 0;
1335
1336 /*
1337 * Move mbuf from the pending queue to the snd queue.
1338 */
1339 IF_DEQUEUE(&txq->txq_pendq, m);
1340 bpf_mtap(ifp, m, BPF_D_OUT);
1341 m_freem(m);
1342 ifp->if_flags &= ~IFF_OACTIVE;
1343
1344 /*
1345 * Since we have put an item into the packet queue, we now want
1346 * an interrupt when the transmit queue finishes processing the
1347 * list. But only update the mask if needs changing.
1348 */
1349 intrmask |= txq->txq_intrbits & (ETH_IR_TxEndHigh | ETH_IR_TxEndLow);
1350 if (sc->sc_intrmask != intrmask) {
1351 sc->sc_intrmask = intrmask;
1352 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
1353 }
1354 if (ifp->if_timer == 0)
1355 ifp->if_timer = 5;
1356 GE_FUNC_EXIT(sc, "*");
1357 return 1;
1358 }
1359
1360 uint32_t
1361 gfe_tx_done(struct gfe_softc *sc, enum gfe_txprio txprio, uint32_t intrmask)
1362 {
1363 struct gfe_txqueue * const txq = &sc->sc_txq[txprio];
1364 struct ifnet * const ifp = &sc->sc_ec.ec_if;
1365
1366 GE_FUNC_ENTER(sc, "gfe_tx_done");
1367
1368 if (txq == NULL) {
1369 GE_FUNC_EXIT(sc, "");
1370 return intrmask;
1371 }
1372
1373 while (txq->txq_nactive > 0) {
1374 const int dcache_line_size = curcpu()->ci_ci.dcache_line_size;
1375 volatile struct gt_eth_desc *txd = &txq->txq_descs[txq->txq_fi];
1376 uint32_t cmdsts;
1377 size_t pktlen;
1378
1379 GE_TXDPOSTSYNC(sc, txq, txq->txq_fi);
1380 if ((cmdsts = gt32toh(txd->ed_cmdsts)) & TX_CMD_O) {
1381 int nextin;
1382
1383 if (txq->txq_nactive == 1) {
1384 GE_TXDPRESYNC(sc, txq, txq->txq_fi);
1385 GE_FUNC_EXIT(sc, "");
1386 return intrmask;
1387 }
1388 /*
1389 * Sometimes the Discovery forgets to update the
1390 * ownership bit in the descriptor. See if we own the
1391 * descriptor after it (since we know we've turned
1392 * that to the Discovery and if we own it now then the
1393 * Discovery gave it back). If we do, we know the
1394 * Discovery gave back this one but forgot to mark it
1395 * as ours.
1396 */
1397 nextin = txq->txq_fi + 1;
1398 if (nextin == GE_TXDESC_MAX)
1399 nextin = 0;
1400 GE_TXDPOSTSYNC(sc, txq, nextin);
1401 if (gt32toh(txq->txq_descs[nextin].ed_cmdsts) & TX_CMD_O) {
1402 GE_TXDPRESYNC(sc, txq, txq->txq_fi);
1403 GE_TXDPRESYNC(sc, txq, nextin);
1404 GE_FUNC_EXIT(sc, "");
1405 return intrmask;
1406 }
1407 #ifdef DEBUG
1408 printf("%s: txdone: transmitter resynced at %d\n",
1409 device_xname(sc->sc_dev), txq->txq_fi);
1410 #endif
1411 }
1412 #if 0
1413 GE_DPRINTF(sc, ("([%d]<-%08lx.%08lx.%08lx.%08lx)",
1414 txq->txq_lo,
1415 ((unsigned long *)txd)[0], ((unsigned long *)txd)[1],
1416 ((unsigned long *)txd)[2], ((unsigned long *)txd)[3]));
1417 #endif
1418 GE_DPRINTF(sc, ("(%d)", txq->txq_fi));
1419 if (++txq->txq_fi == GE_TXDESC_MAX)
1420 txq->txq_fi = 0;
1421 txq->txq_inptr = gt32toh(txd->ed_bufptr) - txq->txq_buf_busaddr;
1422 pktlen = (gt32toh(txd->ed_lencnt) >> 16) & 0xffff;
1423 bus_dmamap_sync(sc->sc_dmat, txq->txq_buf_mem.gdm_map,
1424 txq->txq_inptr, pktlen, BUS_DMASYNC_POSTWRITE);
1425 txq->txq_inptr += roundup(pktlen, dcache_line_size);
1426
1427 /* statistics */
1428 if_statinc(ifp, if_opackets);
1429 if (cmdsts & TX_STS_ES)
1430 if_statinc(ifp, if_oerrors);
1431
1432 /* txd->ed_bufptr = 0; */
1433
1434 ifp->if_timer = 5;
1435 --txq->txq_nactive;
1436 }
1437 if (txq->txq_nactive != 0)
1438 panic("%s: transmit fifo%d empty but active count (%d) > 0!",
1439 device_xname(sc->sc_dev), txprio, txq->txq_nactive);
1440 ifp->if_timer = 0;
1441 intrmask &=
1442 ~(txq->txq_intrbits & (ETH_IR_TxEndHigh | ETH_IR_TxEndLow));
1443 intrmask &=
1444 ~(txq->txq_intrbits & (ETH_IR_TxBufferHigh | ETH_IR_TxBufferLow));
1445 GE_FUNC_EXIT(sc, "");
1446 return intrmask;
1447 }
1448
1449 int
1450 gfe_tx_txqalloc(struct gfe_softc *sc, enum gfe_txprio txprio)
1451 {
1452 struct gfe_txqueue * const txq = &sc->sc_txq[txprio];
1453 int error;
1454
1455 GE_FUNC_ENTER(sc, "gfe_tx_txqalloc");
1456
1457 error = gfe_dmamem_alloc(sc, &txq->txq_desc_mem, 1,
1458 GE_TXDESC_MEMSIZE, BUS_DMA_NOCACHE);
1459 if (error) {
1460 GE_FUNC_EXIT(sc, "");
1461 return error;
1462 }
1463 error = gfe_dmamem_alloc(sc, &txq->txq_buf_mem, 1, GE_TXBUF_SIZE, 0);
1464 if (error) {
1465 gfe_dmamem_free(sc, &txq->txq_desc_mem);
1466 GE_FUNC_EXIT(sc, "");
1467 return error;
1468 }
1469 GE_FUNC_EXIT(sc, "");
1470 return 0;
1471 }
1472
1473 int
1474 gfe_tx_start(struct gfe_softc *sc, enum gfe_txprio txprio)
1475 {
1476 struct gfe_txqueue * const txq = &sc->sc_txq[txprio];
1477 volatile struct gt_eth_desc *txd;
1478 unsigned int i;
1479 bus_addr_t addr;
1480
1481 GE_FUNC_ENTER(sc, "gfe_tx_start");
1482
1483 sc->sc_intrmask &=
1484 ~(ETH_IR_TxEndHigh |
1485 ETH_IR_TxBufferHigh |
1486 ETH_IR_TxEndLow |
1487 ETH_IR_TxBufferLow);
1488
1489 if (sc->sc_flags & GE_NOFREE) {
1490 KASSERT(txq->txq_desc_mem.gdm_kva != NULL);
1491 KASSERT(txq->txq_buf_mem.gdm_kva != NULL);
1492 } else {
1493 int error = gfe_tx_txqalloc(sc, txprio);
1494 if (error) {
1495 GE_FUNC_EXIT(sc, "!");
1496 return error;
1497 }
1498 }
1499
1500 txq->txq_descs =
1501 (volatile struct gt_eth_desc *) txq->txq_desc_mem.gdm_kva;
1502 txq->txq_desc_busaddr = txq->txq_desc_mem.gdm_map->dm_segs[0].ds_addr;
1503 txq->txq_buf_busaddr = txq->txq_buf_mem.gdm_map->dm_segs[0].ds_addr;
1504
1505 txq->txq_pendq.ifq_maxlen = 10;
1506 txq->txq_ei_gapcount = 0;
1507 txq->txq_nactive = 0;
1508 txq->txq_fi = 0;
1509 txq->txq_lo = 0;
1510 txq->txq_inptr = GE_TXBUF_SIZE;
1511 txq->txq_outptr = 0;
1512 for (i = 0, txd = txq->txq_descs,
1513 addr = txq->txq_desc_busaddr + sizeof(*txd);
1514 i < GE_TXDESC_MAX - 1; i++, txd++, addr += sizeof(*txd)) {
1515 /*
1516 * update the nxtptr to point to the next txd.
1517 */
1518 txd->ed_cmdsts = 0;
1519 txd->ed_nxtptr = htogt32(addr);
1520 }
1521 txq->txq_descs[GE_TXDESC_MAX-1].ed_nxtptr =
1522 htogt32(txq->txq_desc_busaddr);
1523 bus_dmamap_sync(sc->sc_dmat, txq->txq_desc_mem.gdm_map, 0,
1524 GE_TXDESC_MEMSIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1525
1526 switch (txprio) {
1527 case GE_TXPRIO_HI:
1528 txq->txq_intrbits = ETH_IR_TxEndHigh | ETH_IR_TxBufferHigh;
1529 txq->txq_esdcmrbits = ETH_ESDCMR_TXDH;
1530 txq->txq_epsrbits = ETH_EPSR_TxHigh;
1531 txq->txq_ectdp = ETH_ECTDP1;
1532 GE_WRITE(sc, ETH_ECTDP1, txq->txq_desc_busaddr);
1533 break;
1534
1535 case GE_TXPRIO_LO:
1536 txq->txq_intrbits = ETH_IR_TxEndLow | ETH_IR_TxBufferLow;
1537 txq->txq_esdcmrbits = ETH_ESDCMR_TXDL;
1538 txq->txq_epsrbits = ETH_EPSR_TxLow;
1539 txq->txq_ectdp = ETH_ECTDP0;
1540 GE_WRITE(sc, ETH_ECTDP0, txq->txq_desc_busaddr);
1541 break;
1542
1543 case GE_TXPRIO_NONE:
1544 break;
1545 }
1546 #if 0
1547 GE_DPRINTF(sc, ("(ectdp=%#x", txq->txq_ectdp));
1548 GE_WRITE(sc->sc_dev, txq->txq_ectdp, txq->txq_desc_busaddr);
1549 GE_DPRINTF(sc, (")"));
1550 #endif
1551
1552 /*
1553 * If we are restarting, there may be packets in the pending queue
1554 * waiting to be enqueued. Try enqueuing packets from both priority
1555 * queues until the pending queue is empty or there no room for them
1556 * on the device.
1557 */
1558 while (gfe_tx_enqueue(sc, txprio))
1559 continue;
1560
1561 GE_FUNC_EXIT(sc, "");
1562 return 0;
1563 }
1564
1565 void
1566 gfe_tx_cleanup(struct gfe_softc *sc, enum gfe_txprio txprio, int flush)
1567 {
1568 struct gfe_txqueue * const txq = &sc->sc_txq[txprio];
1569
1570 GE_FUNC_ENTER(sc, "gfe_tx_cleanup");
1571 if (txq == NULL) {
1572 GE_FUNC_EXIT(sc, "");
1573 return;
1574 }
1575
1576 if (!flush) {
1577 GE_FUNC_EXIT(sc, "");
1578 return;
1579 }
1580
1581 if ((sc->sc_flags & GE_NOFREE) == 0) {
1582 gfe_dmamem_free(sc, &txq->txq_desc_mem);
1583 gfe_dmamem_free(sc, &txq->txq_buf_mem);
1584 }
1585 GE_FUNC_EXIT(sc, "-F");
1586 }
1587
1588 void
1589 gfe_tx_stop(struct gfe_softc *sc, enum gfe_whack_op op)
1590 {
1591 GE_FUNC_ENTER(sc, "gfe_tx_stop");
1592
1593 GE_WRITE(sc, ETH_ESDCMR, ETH_ESDCMR_STDH | ETH_ESDCMR_STDL);
1594
1595 sc->sc_intrmask = gfe_tx_done(sc, GE_TXPRIO_HI, sc->sc_intrmask);
1596 sc->sc_intrmask = gfe_tx_done(sc, GE_TXPRIO_LO, sc->sc_intrmask);
1597 sc->sc_intrmask &=
1598 ~(ETH_IR_TxEndHigh |
1599 ETH_IR_TxBufferHigh |
1600 ETH_IR_TxEndLow |
1601 ETH_IR_TxBufferLow);
1602
1603 gfe_tx_cleanup(sc, GE_TXPRIO_HI, op == GE_WHACK_STOP);
1604 gfe_tx_cleanup(sc, GE_TXPRIO_LO, op == GE_WHACK_STOP);
1605
1606 sc->sc_ec.ec_if.if_timer = 0;
1607 GE_FUNC_EXIT(sc, "");
1608 }
1609
1610 int
1611 gfe_intr(void *arg)
1612 {
1613 struct gfe_softc * const sc = arg;
1614 uint32_t cause;
1615 uint32_t intrmask = sc->sc_intrmask;
1616 int claim = 0;
1617 int cnt;
1618
1619 GE_FUNC_ENTER(sc, "gfe_intr");
1620
1621 for (cnt = 0; cnt < 4; cnt++) {
1622 if (sc->sc_intrmask != intrmask) {
1623 sc->sc_intrmask = intrmask;
1624 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
1625 }
1626 cause = GE_READ(sc, ETH_EICR);
1627 cause &= sc->sc_intrmask;
1628 GE_DPRINTF(sc, (".%#x", cause));
1629 if (cause == 0)
1630 break;
1631
1632 claim = 1;
1633
1634 GE_WRITE(sc, ETH_EICR, ~cause);
1635 #ifndef GE_NORX
1636 if (cause & (ETH_IR_RxBuffer | ETH_IR_RxError))
1637 intrmask = gfe_rx_process(sc, cause, intrmask);
1638 #endif
1639
1640 #ifndef GE_NOTX
1641 if (cause & (ETH_IR_TxBufferHigh | ETH_IR_TxEndHigh))
1642 intrmask = gfe_tx_done(sc, GE_TXPRIO_HI, intrmask);
1643 if (cause & (ETH_IR_TxBufferLow | ETH_IR_TxEndLow))
1644 intrmask = gfe_tx_done(sc, GE_TXPRIO_LO, intrmask);
1645 #endif
1646 if (cause & ETH_IR_MIIPhySTC) {
1647 sc->sc_flags |= GE_PHYSTSCHG;
1648 /* intrmask &= ~ETH_IR_MIIPhySTC; */
1649 }
1650 }
1651
1652 while (gfe_tx_enqueue(sc, GE_TXPRIO_HI))
1653 continue;
1654 while (gfe_tx_enqueue(sc, GE_TXPRIO_LO))
1655 continue;
1656
1657 GE_FUNC_EXIT(sc, "");
1658 return claim;
1659 }
1660
1661 int
1662 gfe_whack(struct gfe_softc *sc, enum gfe_whack_op op)
1663 {
1664 int error = 0;
1665 GE_FUNC_ENTER(sc, "gfe_whack");
1666
1667 switch (op) {
1668 case GE_WHACK_RESTART:
1669 #ifndef GE_NOTX
1670 gfe_tx_stop(sc, op);
1671 #endif
1672 /* sc->sc_ec.ec_if.if_flags &= ~IFF_RUNNING; */
1673 /* FALLTHROUGH */
1674 case GE_WHACK_START:
1675 #ifndef GE_NOHASH
1676 if (error == 0 && sc->sc_hashtable == NULL) {
1677 error = gfe_hash_alloc(sc);
1678 if (error)
1679 break;
1680 }
1681 if (op != GE_WHACK_RESTART)
1682 gfe_hash_fill(sc);
1683 #endif
1684 #ifndef GE_NORX
1685 if (op != GE_WHACK_RESTART) {
1686 error = gfe_rx_prime(sc);
1687 if (error)
1688 break;
1689 }
1690 #endif
1691 #ifndef GE_NOTX
1692 error = gfe_tx_start(sc, GE_TXPRIO_HI);
1693 if (error)
1694 break;
1695 #endif
1696 sc->sc_ec.ec_if.if_flags |= IFF_RUNNING;
1697 GE_WRITE(sc, ETH_EPCR, sc->sc_pcr | ETH_EPCR_EN);
1698 GE_WRITE(sc, ETH_EPCXR, sc->sc_pcxr);
1699 GE_WRITE(sc, ETH_EICR, 0);
1700 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
1701 #ifndef GE_NOHASH
1702 GE_WRITE(sc, ETH_EHTPR,
1703 sc->sc_hash_mem.gdm_map->dm_segs->ds_addr);
1704 #endif
1705 #ifndef GE_NORX
1706 GE_WRITE(sc, ETH_ESDCMR, ETH_ESDCMR_ERD);
1707 sc->sc_flags |= GE_RXACTIVE;
1708 #endif
1709 /* FALLTHROUGH */
1710 case GE_WHACK_CHANGE:
1711 GE_DPRINTF(sc, ("(pcr=%#x,imr=%#x)",
1712 GE_READ(sc, ETH_EPCR), GE_READ(sc, ETH_EIMR)));
1713 GE_WRITE(sc, ETH_EPCR, sc->sc_pcr | ETH_EPCR_EN);
1714 GE_WRITE(sc, ETH_EIMR, sc->sc_intrmask);
1715 gfe_ifstart(&sc->sc_ec.ec_if);
1716 GE_DPRINTF(sc, ("(ectdp0=%#x, ectdp1=%#x)",
1717 GE_READ(sc, ETH_ECTDP0), GE_READ(sc, ETH_ECTDP1)));
1718 GE_FUNC_EXIT(sc, "");
1719 return error;
1720 case GE_WHACK_STOP:
1721 break;
1722 }
1723
1724 #ifdef GE_DEBUG
1725 if (error)
1726 GE_DPRINTF(sc, (" failed: %d\n", error));
1727 #endif
1728 GE_WRITE(sc, ETH_EPCR, sc->sc_pcr);
1729 GE_WRITE(sc, ETH_EIMR, 0);
1730 sc->sc_ec.ec_if.if_flags &= ~IFF_RUNNING;
1731 #ifndef GE_NOTX
1732 gfe_tx_stop(sc, GE_WHACK_STOP);
1733 #endif
1734 #ifndef GE_NORX
1735 gfe_rx_stop(sc, GE_WHACK_STOP);
1736 #endif
1737 #ifndef GE_NOHASH
1738 if ((sc->sc_flags & GE_NOFREE) == 0) {
1739 gfe_dmamem_free(sc, &sc->sc_hash_mem);
1740 sc->sc_hashtable = NULL;
1741 }
1742 #endif
1743
1744 GE_FUNC_EXIT(sc, "");
1745 return error;
1746 }
1747
1748 int
1749 gfe_hash_compute(struct gfe_softc *sc, const uint8_t eaddr[ETHER_ADDR_LEN])
1750 {
1751 uint32_t w0, add0, add1;
1752 uint32_t result;
1753
1754 GE_FUNC_ENTER(sc, "gfe_hash_compute");
1755 add0 = ((uint32_t) eaddr[5] << 0) |
1756 ((uint32_t) eaddr[4] << 8) |
1757 ((uint32_t) eaddr[3] << 16);
1758
1759 add0 = ((add0 & 0x00f0f0f0) >> 4) | ((add0 & 0x000f0f0f) << 4);
1760 add0 = ((add0 & 0x00cccccc) >> 2) | ((add0 & 0x00333333) << 2);
1761 add0 = ((add0 & 0x00aaaaaa) >> 1) | ((add0 & 0x00555555) << 1);
1762
1763 add1 = ((uint32_t) eaddr[2] << 0) |
1764 ((uint32_t) eaddr[1] << 8) |
1765 ((uint32_t) eaddr[0] << 16);
1766
1767 add1 = ((add1 & 0x00f0f0f0) >> 4) | ((add1 & 0x000f0f0f) << 4);
1768 add1 = ((add1 & 0x00cccccc) >> 2) | ((add1 & 0x00333333) << 2);
1769 add1 = ((add1 & 0x00aaaaaa) >> 1) | ((add1 & 0x00555555) << 1);
1770
1771 GE_DPRINTF(sc, ("%s=", ether_sprintf(eaddr)));
1772 /*
1773 * hashResult is the 15 bits Hash entry address.
1774 * ethernetADD is a 48 bit number, which is derived from the Ethernet
1775 * MAC address, by nibble swapping in every byte (i.e MAC address
1776 * of 0x123456789abc translates to ethernetADD of 0x21436587a9cb).
1777 */
1778
1779 if ((sc->sc_pcr & ETH_EPCR_HM) == 0) {
1780 /*
1781 * hashResult[14:0] = hashFunc0(ethernetADD[47:0])
1782 *
1783 * hashFunc0 calculates the hashResult in the following manner:
1784 * hashResult[ 8:0] = ethernetADD[14:8,1,0]
1785 * XOR ethernetADD[23:15] XOR ethernetADD[32:24]
1786 */
1787 result = (add0 & 3) | ((add0 >> 6) & ~3);
1788 result ^= (add0 >> 15) ^ (add1 >> 0);
1789 result &= 0x1ff;
1790 /*
1791 * hashResult[14:9] = ethernetADD[7:2]
1792 */
1793 result |= (add0 & ~3) << 7; /* excess bits will be masked */
1794 GE_DPRINTF(sc, ("0(%#x)", result & 0x7fff));
1795 } else {
1796 #define TRIBITFLIP 073516240 /* yes its in octal */
1797 /*
1798 * hashResult[14:0] = hashFunc1(ethernetADD[47:0])
1799 *
1800 * hashFunc1 calculates the hashResult in the following manner:
1801 * hashResult[08:00] = ethernetADD[06:14]
1802 * XOR ethernetADD[15:23] XOR ethernetADD[24:32]
1803 */
1804 w0 = ((add0 >> 6) ^ (add0 >> 15) ^ (add1)) & 0x1ff;
1805 /*
1806 * Now bitswap those 9 bits
1807 */
1808 result = 0;
1809 result |= ((TRIBITFLIP >> (((w0 >> 0) & 7) * 3)) & 7) << 6;
1810 result |= ((TRIBITFLIP >> (((w0 >> 3) & 7) * 3)) & 7) << 3;
1811 result |= ((TRIBITFLIP >> (((w0 >> 6) & 7) * 3)) & 7) << 0;
1812
1813 /*
1814 * hashResult[14:09] = ethernetADD[00:05]
1815 */
1816 result |= ((TRIBITFLIP >> (((add0 >> 0) & 7) * 3)) & 7) << 12;
1817 result |= ((TRIBITFLIP >> (((add0 >> 3) & 7) * 3)) & 7) << 9;
1818 GE_DPRINTF(sc, ("1(%#x)", result));
1819 }
1820 GE_FUNC_EXIT(sc, "");
1821 return result & ((sc->sc_pcr & ETH_EPCR_HS_512) ? 0x7ff : 0x7fff);
1822 }
1823
1824 int
1825 gfe_hash_entry_op(struct gfe_softc *sc, enum gfe_hash_op op,
1826 enum gfe_rxprio prio, const uint8_t eaddr[ETHER_ADDR_LEN])
1827 {
1828 uint64_t he;
1829 uint64_t *maybe_he_p = NULL;
1830 int limit;
1831 int hash;
1832 int maybe_hash = 0;
1833
1834 GE_FUNC_ENTER(sc, "gfe_hash_entry_op");
1835
1836 hash = gfe_hash_compute(sc, eaddr);
1837
1838 if (sc->sc_hashtable == NULL) {
1839 panic("%s:%d: hashtable == NULL!", device_xname(sc->sc_dev),
1840 __LINE__);
1841 }
1842
1843 /*
1844 * Assume we are going to insert so create the hash entry we
1845 * are going to insert. We also use it to match entries we
1846 * will be removing.
1847 */
1848 he = ((uint64_t) eaddr[5] << 43) |
1849 ((uint64_t) eaddr[4] << 35) |
1850 ((uint64_t) eaddr[3] << 27) |
1851 ((uint64_t) eaddr[2] << 19) |
1852 ((uint64_t) eaddr[1] << 11) |
1853 ((uint64_t) eaddr[0] << 3) |
1854 HSH_PRIO_INS(prio) | HSH_V | HSH_R;
1855
1856 /*
1857 * The GT will search upto 12 entries for a hit, so we must mimic that.
1858 */
1859 hash &= sc->sc_hashmask / sizeof(he);
1860 for (limit = HSH_LIMIT; limit > 0 ; --limit) {
1861 /*
1862 * Does the GT wrap at the end, stop at the, or overrun the
1863 * end? Assume it wraps for now. Stash a copy of the
1864 * current hash entry.
1865 */
1866 uint64_t *he_p = &sc->sc_hashtable[hash];
1867 uint64_t thishe = *he_p;
1868
1869 /*
1870 * If the hash entry isn't valid, that break the chain. And
1871 * this entry a good candidate for reuse.
1872 */
1873 if ((thishe & HSH_V) == 0) {
1874 maybe_he_p = he_p;
1875 break;
1876 }
1877
1878 /*
1879 * If the hash entry has the same address we are looking for
1880 * then ... if we are removing and the skip bit is set, its
1881 * already been removed. if are adding and the skip bit is
1882 * clear, then its already added. In either return EBUSY
1883 * indicating the op has already been done. Otherwise flip
1884 * the skip bit and return 0.
1885 */
1886 if (((he ^ thishe) & HSH_ADDR_MASK) == 0) {
1887 if (((op == GE_HASH_REMOVE) && (thishe & HSH_S)) ||
1888 ((op == GE_HASH_ADD) && (thishe & HSH_S) == 0))
1889 return EBUSY;
1890 *he_p = thishe ^ HSH_S;
1891 bus_dmamap_sync(sc->sc_dmat, sc->sc_hash_mem.gdm_map,
1892 hash * sizeof(he), sizeof(he),
1893 BUS_DMASYNC_PREWRITE);
1894 GE_FUNC_EXIT(sc, "^");
1895 return 0;
1896 }
1897
1898 /*
1899 * If we haven't found a slot for the entry and this entry
1900 * is currently being skipped, return this entry.
1901 */
1902 if (maybe_he_p == NULL && (thishe & HSH_S)) {
1903 maybe_he_p = he_p;
1904 maybe_hash = hash;
1905 }
1906
1907 hash = (hash + 1) & (sc->sc_hashmask / sizeof(he));
1908 }
1909
1910 /*
1911 * If we got here, then there was no entry to remove.
1912 */
1913 if (op == GE_HASH_REMOVE) {
1914 GE_FUNC_EXIT(sc, "?");
1915 return ENOENT;
1916 }
1917
1918 /*
1919 * If we couldn't find a slot, return an error.
1920 */
1921 if (maybe_he_p == NULL) {
1922 GE_FUNC_EXIT(sc, "!");
1923 return ENOSPC;
1924 }
1925
1926 /* Update the entry.
1927 */
1928 *maybe_he_p = he;
1929 bus_dmamap_sync(sc->sc_dmat, sc->sc_hash_mem.gdm_map,
1930 maybe_hash * sizeof(he), sizeof(he), BUS_DMASYNC_PREWRITE);
1931 GE_FUNC_EXIT(sc, "+");
1932 return 0;
1933 }
1934
1935 int
1936 gfe_hash_multichg(struct ethercom *ec, const struct ether_multi *enm,
1937 u_long cmd)
1938 {
1939 struct gfe_softc *sc = ec->ec_if.if_softc;
1940 int error;
1941 enum gfe_hash_op op;
1942 enum gfe_rxprio prio;
1943
1944 GE_FUNC_ENTER(sc, "hash_multichg");
1945 /*
1946 * Is this a wildcard entry? If so and its being removed, recompute.
1947 */
1948 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN) != 0) {
1949 if (cmd == SIOCDELMULTI) {
1950 GE_FUNC_EXIT(sc, "");
1951 return ENETRESET;
1952 }
1953
1954 /*
1955 * Switch in
1956 */
1957 sc->sc_flags |= GE_ALLMULTI;
1958 if ((sc->sc_pcr & ETH_EPCR_PM) == 0) {
1959 sc->sc_pcr |= ETH_EPCR_PM;
1960 GE_WRITE(sc, ETH_EPCR, sc->sc_pcr);
1961 GE_FUNC_EXIT(sc, "");
1962 return 0;
1963 }
1964 GE_FUNC_EXIT(sc, "");
1965 return ENETRESET;
1966 }
1967
1968 prio = GE_RXPRIO_MEDLO;
1969 op = (cmd == SIOCDELMULTI ? GE_HASH_REMOVE : GE_HASH_ADD);
1970
1971 if (sc->sc_hashtable == NULL) {
1972 GE_FUNC_EXIT(sc, "");
1973 return 0;
1974 }
1975
1976 error = gfe_hash_entry_op(sc, op, prio, enm->enm_addrlo);
1977 if (error == EBUSY) {
1978 aprint_error_dev(sc->sc_dev, "multichg: tried to %s %s again\n",
1979 cmd == SIOCDELMULTI ? "remove" : "add",
1980 ether_sprintf(enm->enm_addrlo));
1981 GE_FUNC_EXIT(sc, "");
1982 return 0;
1983 }
1984
1985 if (error == ENOENT) {
1986 aprint_error_dev(sc->sc_dev,
1987 "multichg: failed to remove %s: not in table\n",
1988 ether_sprintf(enm->enm_addrlo));
1989 GE_FUNC_EXIT(sc, "");
1990 return 0;
1991 }
1992
1993 if (error == ENOSPC) {
1994 aprint_error_dev(sc->sc_dev, "multichg:"
1995 " failed to add %s: no space; regenerating table\n",
1996 ether_sprintf(enm->enm_addrlo));
1997 GE_FUNC_EXIT(sc, "");
1998 return ENETRESET;
1999 }
2000 GE_DPRINTF(sc, ("%s: multichg: %s: %s succeeded\n",
2001 device_xname(sc->sc_dev),
2002 cmd == SIOCDELMULTI ? "remove" : "add",
2003 ether_sprintf(enm->enm_addrlo)));
2004 GE_FUNC_EXIT(sc, "");
2005 return 0;
2006 }
2007
2008 int
2009 gfe_hash_fill(struct gfe_softc *sc)
2010 {
2011 struct ethercom *ec = &sc->sc_ec;
2012 struct ether_multistep step;
2013 struct ether_multi *enm;
2014 int error;
2015
2016 GE_FUNC_ENTER(sc, "gfe_hash_fill");
2017
2018 error = gfe_hash_entry_op(sc, GE_HASH_ADD, GE_RXPRIO_HI,
2019 CLLADDR(ec->ec_if.if_sadl));
2020 if (error) {
2021 GE_FUNC_EXIT(sc, "!");
2022 return error;
2023 }
2024
2025 sc->sc_flags &= ~GE_ALLMULTI;
2026 if ((ec->ec_if.if_flags & IFF_PROMISC) == 0)
2027 sc->sc_pcr &= ~ETH_EPCR_PM;
2028 ETHER_LOCK(ec);
2029 ETHER_FIRST_MULTI(step, ec, enm);
2030 while (enm != NULL) {
2031 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
2032 sc->sc_flags |= GE_ALLMULTI;
2033 sc->sc_pcr |= ETH_EPCR_PM;
2034 } else {
2035 error = gfe_hash_entry_op(sc, GE_HASH_ADD,
2036 GE_RXPRIO_MEDLO, enm->enm_addrlo);
2037 if (error == ENOSPC)
2038 break;
2039 }
2040 ETHER_NEXT_MULTI(step, enm);
2041 }
2042 ETHER_UNLOCK(ec);
2043
2044 GE_FUNC_EXIT(sc, "");
2045 return error;
2046 }
2047
2048 int
2049 gfe_hash_alloc(struct gfe_softc *sc)
2050 {
2051 int error;
2052 GE_FUNC_ENTER(sc, "gfe_hash_alloc");
2053 sc->sc_hashmask = (sc->sc_pcr & ETH_EPCR_HS_512 ? 16 : 256)*1024 - 1;
2054 error = gfe_dmamem_alloc(sc, &sc->sc_hash_mem, 1, sc->sc_hashmask + 1,
2055 BUS_DMA_NOCACHE);
2056 if (error) {
2057 aprint_error_dev(sc->sc_dev,
2058 "failed to allocate %d bytes for hash table: %d\n",
2059 sc->sc_hashmask + 1, error);
2060 GE_FUNC_EXIT(sc, "");
2061 return error;
2062 }
2063 sc->sc_hashtable = (uint64_t *) sc->sc_hash_mem.gdm_kva;
2064 memset(sc->sc_hashtable, 0, sc->sc_hashmask + 1);
2065 bus_dmamap_sync(sc->sc_dmat, sc->sc_hash_mem.gdm_map,
2066 0, sc->sc_hashmask + 1, BUS_DMASYNC_PREWRITE);
2067 GE_FUNC_EXIT(sc, "");
2068 return 0;
2069 }
2070