if_aq.c revision 1.17.2.4 1 /* $NetBSD: if_aq.c,v 1.17.2.4 2023/06/03 14:46:07 martin Exp $ */
2
3 /**
4 * aQuantia Corporation Network Driver
5 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * (1) Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
13 * disclaimer.
14 *
15 * (2) Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials provided
18 * with the distribution.
19 *
20 * (3) The name of the author may not be used to endorse or promote
21 * products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
25 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
26 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
28 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
30 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
33 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
34 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38 /*-
39 * Copyright (c) 2020 Ryo Shimizu <ryo (at) nerv.org>
40 * All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 *
51 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
52 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
53 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
54 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
55 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
56 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
57 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
59 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
60 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
61 * POSSIBILITY OF SUCH DAMAGE.
62 */
63
64 #include <sys/cdefs.h>
65 __KERNEL_RCSID(0, "$NetBSD: if_aq.c,v 1.17.2.4 2023/06/03 14:46:07 martin Exp $");
66
67 #ifdef _KERNEL_OPT
68 #include "opt_if_aq.h"
69 #include "sysmon_envsys.h"
70 #endif
71
72 #include <sys/param.h>
73 #include <sys/types.h>
74 #include <sys/bitops.h>
75 #include <sys/cprng.h>
76 #include <sys/cpu.h>
77 #include <sys/interrupt.h>
78 #include <sys/module.h>
79 #include <sys/pcq.h>
80
81 #include <net/bpf.h>
82 #include <net/if.h>
83 #include <net/if_dl.h>
84 #include <net/if_media.h>
85 #include <net/if_ether.h>
86 #include <net/rss_config.h>
87
88 #include <dev/pci/pcivar.h>
89 #include <dev/pci/pcireg.h>
90 #include <dev/pci/pcidevs.h>
91 #include <dev/sysmon/sysmonvar.h>
92
93 /* driver configuration */
94 #define CONFIG_INTR_MODERATION_ENABLE true /* delayed interrupt */
95 #undef CONFIG_LRO_SUPPORT /* no LRO not suppoted */
96 #undef CONFIG_NO_TXRX_INDEPENDENT /* share TX/RX interrupts */
97
98 #define AQ_NINTR_MAX (AQ_RSSQUEUE_MAX + AQ_RSSQUEUE_MAX + 1)
99 /* TX + RX + LINK. must be <= 32 */
100 #define AQ_LINKSTAT_IRQ 31 /* for legacy mode */
101
102 #define AQ_TXD_NUM 2048 /* per ring. 8*n && 32~8184 */
103 #define AQ_RXD_NUM 2048 /* per ring. 8*n && 32~8184 */
104 /* minimum required to send a packet (vlan needs additional TX descriptor) */
105 #define AQ_TXD_MIN (1 + 1)
106
107
108 /* hardware specification */
109 #define AQ_RINGS_NUM 32
110 #define AQ_RSSQUEUE_MAX 8
111 #define AQ_RX_DESCRIPTOR_MIN 32
112 #define AQ_TX_DESCRIPTOR_MIN 32
113 #define AQ_RX_DESCRIPTOR_MAX 8184
114 #define AQ_TX_DESCRIPTOR_MAX 8184
115 #define AQ_TRAFFICCLASS_NUM 8
116 #define AQ_RSS_HASHKEY_SIZE 40
117 #define AQ_RSS_INDIRECTION_TABLE_MAX 64
118
119 /*
120 * TERMINOLOGY
121 * MPI = MAC PHY INTERFACE?
122 * RPO = RX Protocol Offloading
123 * TPO = TX Protocol Offloading
124 * RPF = RX Packet Filter
125 * TPB = TX Packet buffer
126 * RPB = RX Packet buffer
127 */
128
129 /* registers */
130 #define AQ_FW_SOFTRESET_REG 0x0000
131 #define AQ_FW_SOFTRESET_RESET __BIT(15) /* soft reset bit */
132 #define AQ_FW_SOFTRESET_DIS __BIT(14) /* reset disable */
133
134 #define AQ_FW_VERSION_REG 0x0018
135 #define AQ_HW_REVISION_REG 0x001c
136 #define AQ_GLB_NVR_INTERFACE1_REG 0x0100
137
138 #define AQ_FW_MBOX_CMD_REG 0x0200
139 #define AQ_FW_MBOX_CMD_EXECUTE 0x00008000
140 #define AQ_FW_MBOX_CMD_BUSY 0x00000100
141 #define AQ_FW_MBOX_ADDR_REG 0x0208
142 #define AQ_FW_MBOX_VAL_REG 0x020c
143
144 #define FW2X_LED_MIN_VERSION 0x03010026 /* >= 3.1.38 */
145 #define FW2X_LED_REG 0x031c
146 #define FW2X_LED_DEFAULT 0x00000000
147 #define FW2X_LED_NONE 0x0000003f
148 #define FW2X_LINKLED __BITS(0,1)
149 #define FW2X_LINKLED_ACTIVE 0
150 #define FW2X_LINKLED_ON 1
151 #define FW2X_LINKLED_BLINK 2
152 #define FW2X_LINKLED_OFF 3
153 #define FW2X_STATUSLED __BITS(2,5)
154 #define FW2X_STATUSLED_ORANGE 0
155 #define FW2X_STATUSLED_ORANGE_BLINK 2
156 #define FW2X_STATUSLED_OFF 3
157 #define FW2X_STATUSLED_GREEN 4
158 #define FW2X_STATUSLED_ORANGE_GREEN_BLINK 8
159 #define FW2X_STATUSLED_GREEN_BLINK 10
160
161 #define FW_MPI_MBOX_ADDR_REG 0x0360
162 #define FW1X_MPI_INIT1_REG 0x0364
163 #define FW1X_MPI_CONTROL_REG 0x0368
164 #define FW1X_MPI_STATE_REG 0x036c
165 #define FW1X_MPI_STATE_MODE __BITS(7,0)
166 #define FW1X_MPI_STATE_SPEED __BITS(32,16)
167 #define FW1X_MPI_STATE_DISABLE_DIRTYWAKE __BITS(25)
168 #define FW1X_MPI_STATE_DOWNSHIFT __BITS(31,28)
169 #define FW1X_MPI_INIT2_REG 0x0370
170 #define FW1X_MPI_EFUSEADDR_REG 0x0374
171
172 #define FW2X_MPI_EFUSEADDR_REG 0x0364
173 #define FW2X_MPI_CONTROL_REG 0x0368 /* 64bit */
174 #define FW2X_MPI_STATE_REG 0x0370 /* 64bit */
175 #define FW_BOOT_EXIT_CODE_REG 0x0388
176 #define RBL_STATUS_DEAD 0x0000dead
177 #define RBL_STATUS_SUCCESS 0x0000abba
178 #define RBL_STATUS_FAILURE 0x00000bad
179 #define RBL_STATUS_HOST_BOOT 0x0000f1a7
180
181 #define AQ_FW_GLB_CPU_SEM_REG(i) (0x03a0 + (i) * 4)
182 #define AQ_FW_SEM_RAM_REG AQ_FW_GLB_CPU_SEM_REG(2)
183
184 #define AQ_FW_GLB_CTL2_REG 0x0404
185 #define AQ_FW_GLB_CTL2_MCP_UP_FORCE_INTERRUPT __BIT(1)
186
187 #define AQ_GLB_GENERAL_PROVISIONING9_REG 0x0520
188 #define AQ_GLB_NVR_PROVISIONING2_REG 0x0534
189
190 #define FW_MPI_DAISY_CHAIN_STATUS_REG 0x0704
191
192 #define AQ_PCI_REG_CONTROL_6_REG 0x1014
193
194 // msix bitmap */
195 #define AQ_INTR_STATUS_REG 0x2000 /* intr status */
196 #define AQ_INTR_STATUS_CLR_REG 0x2050 /* intr status clear */
197 #define AQ_INTR_MASK_REG 0x2060 /* intr mask set */
198 #define AQ_INTR_MASK_CLR_REG 0x2070 /* intr mask clear */
199 #define AQ_INTR_AUTOMASK_REG 0x2090
200
201 /* AQ_INTR_IRQ_MAP_TXRX_REG[AQ_RINGS_NUM] 0x2100-0x2140 */
202 #define AQ_INTR_IRQ_MAP_TXRX_REG(i) (0x2100 + ((i) / 2) * 4)
203 #define AQ_INTR_IRQ_MAP_TX_REG(i) AQ_INTR_IRQ_MAP_TXRX_REG(i)
204 #define AQ_INTR_IRQ_MAP_TX_IRQMAP(i) (__BITS(28,24) >> (((i) & 1)*8))
205 #define AQ_INTR_IRQ_MAP_TX_EN(i) (__BIT(31) >> (((i) & 1)*8))
206 #define AQ_INTR_IRQ_MAP_RX_REG(i) AQ_INTR_IRQ_MAP_TXRX_REG(i)
207 #define AQ_INTR_IRQ_MAP_RX_IRQMAP(i) (__BITS(12,8) >> (((i) & 1)*8))
208 #define AQ_INTR_IRQ_MAP_RX_EN(i) (__BIT(15) >> (((i) & 1)*8))
209
210 /* AQ_GEN_INTR_MAP_REG[AQ_RINGS_NUM] 0x2180-0x2200 */
211 #define AQ_GEN_INTR_MAP_REG(i) (0x2180 + (i) * 4)
212 #define AQ_B0_ERR_INT 8U
213
214 #define AQ_INTR_CTRL_REG 0x2300
215 #define AQ_INTR_CTRL_IRQMODE __BITS(1,0)
216 #define AQ_INTR_CTRL_IRQMODE_LEGACY 0
217 #define AQ_INTR_CTRL_IRQMODE_MSI 1
218 #define AQ_INTR_CTRL_IRQMODE_MSIX 2
219 #define AQ_INTR_CTRL_MULTIVEC __BIT(2)
220 #define AQ_INTR_CTRL_AUTO_MASK __BIT(5)
221 #define AQ_INTR_CTRL_CLR_ON_READ __BIT(7)
222 #define AQ_INTR_CTRL_RESET_DIS __BIT(29)
223 #define AQ_INTR_CTRL_RESET_IRQ __BIT(31)
224
225 #define AQ_MBOXIF_POWER_GATING_CONTROL_REG 0x32a8
226
227 #define FW_MPI_RESETCTRL_REG 0x4000
228 #define FW_MPI_RESETCTRL_RESET_DIS __BIT(29)
229
230 #define RX_SYSCONTROL_REG 0x5000
231 #define RX_SYSCONTROL_RPB_DMA_LOOPBACK __BIT(6)
232 #define RX_SYSCONTROL_RPF_TPO_LOOPBACK __BIT(8)
233 #define RX_SYSCONTROL_RESET_DIS __BIT(29)
234
235 #define RX_TCP_RSS_HASH_REG 0x5040
236 #define RX_TCP_RSS_HASH_RPF2 __BITS(19,16)
237 #define RX_TCP_RSS_HASH_TYPE __BITS(15,0)
238
239 /* for RPF_*_REG.ACTION */
240 #define RPF_ACTION_DISCARD 0
241 #define RPF_ACTION_HOST 1
242 #define RPF_ACTION_MANAGEMENT 2
243 #define RPF_ACTION_HOST_MANAGEMENT 3
244 #define RPF_ACTION_WOL 4
245
246 #define RPF_L2BC_REG 0x5100
247 #define RPF_L2BC_EN __BIT(0)
248 #define RPF_L2BC_PROMISC __BIT(3)
249 #define RPF_L2BC_ACTION __BITS(12,14)
250 #define RPF_L2BC_THRESHOLD __BITS(31,16)
251
252 /* RPF_L2UC_*_REG[34] (actual [38]?) */
253 #define RPF_L2UC_LSW_REG(i) (0x5110 + (i) * 8)
254 #define RPF_L2UC_MSW_REG(i) (0x5114 + (i) * 8)
255 #define RPF_L2UC_MSW_MACADDR_HI __BITS(15,0)
256 #define RPF_L2UC_MSW_ACTION __BITS(18,16)
257 #define RPF_L2UC_MSW_EN __BIT(31)
258 #define AQ_HW_MAC_OWN 0 /* index of own address */
259 #define AQ_HW_MAC_NUM 34
260
261 /* RPF_MCAST_FILTER_REG[8] 0x5250-0x5270 */
262 #define RPF_MCAST_FILTER_REG(i) (0x5250 + (i) * 4)
263 #define RPF_MCAST_FILTER_EN __BIT(31)
264 #define RPF_MCAST_FILTER_MASK_REG 0x5270
265 #define RPF_MCAST_FILTER_MASK_ALLMULTI __BIT(14)
266
267 #define RPF_VLAN_MODE_REG 0x5280
268 #define RPF_VLAN_MODE_PROMISC __BIT(1)
269 #define RPF_VLAN_MODE_ACCEPT_UNTAGGED __BIT(2)
270 #define RPF_VLAN_MODE_UNTAGGED_ACTION __BITS(5,3)
271
272 #define RPF_VLAN_TPID_REG 0x5284
273 #define RPF_VLAN_TPID_OUTER __BITS(31,16)
274 #define RPF_VLAN_TPID_INNER __BITS(15,0)
275
276 /* RPF_VLAN_FILTER_REG[RPF_VLAN_MAX_FILTERS] 0x5290-0x52d0 */
277 #define RPF_VLAN_MAX_FILTERS 16
278 #define RPF_VLAN_FILTER_REG(i) (0x5290 + (i) * 4)
279 #define RPF_VLAN_FILTER_EN __BIT(31)
280 #define RPF_VLAN_FILTER_RXQ_EN __BIT(28)
281 #define RPF_VLAN_FILTER_RXQ __BITS(24,20)
282 #define RPF_VLAN_FILTER_ACTION __BITS(18,16)
283 #define RPF_VLAN_FILTER_ID __BITS(11,0)
284
285 /* RPF_ETHERTYPE_FILTER_REG[AQ_RINGS_NUM] 0x5300-0x5380 */
286 #define RPF_ETHERTYPE_FILTER_REG(i) (0x5300 + (i) * 4)
287 #define RPF_ETHERTYPE_FILTER_EN __BIT(31)
288 #define RPF_ETHERTYPE_FILTER_PRIO_EN __BIT(30)
289 #define RPF_ETHERTYPE_FILTER_RXQF_EN __BIT(29)
290 #define RPF_ETHERTYPE_FILTER_PRIO __BITS(28,26)
291 #define RPF_ETHERTYPE_FILTER_RXQF __BITS(24,20)
292 #define RPF_ETHERTYPE_FILTER_MNG_RXQF __BIT(19)
293 #define RPF_ETHERTYPE_FILTER_ACTION __BITS(18,16)
294 #define RPF_ETHERTYPE_FILTER_VAL __BITS(15,0)
295
296 /* RPF_L3_FILTER_REG[8] 0x5380-0x53a0 */
297 #define RPF_L3_FILTER_REG(i) (0x5380 + (i) * 4)
298 #define RPF_L3_FILTER_L4_EN __BIT(31)
299 #define RPF_L3_FILTER_IPV6_EN __BIT(30)
300 #define RPF_L3_FILTER_SRCADDR_EN __BIT(29)
301 #define RPF_L3_FILTER_DSTADDR_EN __BIT(28)
302 #define RPF_L3_FILTER_L4_SRCPORT_EN __BIT(27)
303 #define RPF_L3_FILTER_L4_DSTPORT_EN __BIT(26)
304 #define RPF_L3_FILTER_L4_PROTO_EN __BIT(25)
305 #define RPF_L3_FILTER_ARP_EN __BIT(24)
306 #define RPF_L3_FILTER_L4_RXQUEUE_EN __BIT(23)
307 #define RPF_L3_FILTER_L4_RXQUEUE_MANAGEMENT_EN __BIT(22)
308 #define RPF_L3_FILTER_L4_ACTION __BITS(16,18)
309 #define RPF_L3_FILTER_L4_RXQUEUE __BITS(12,8)
310 #define RPF_L3_FILTER_L4_PROTO __BITS(2,0)
311 #define RPF_L3_FILTER_L4_PROTO_TCP 0
312 #define RPF_L3_FILTER_L4_PROTO_UDP 1
313 #define RPF_L3_FILTER_L4_PROTO_SCTP 2
314 #define RPF_L3_FILTER_L4_PROTO_ICMP 3
315 /* parameters of RPF_L3_FILTER_REG[8] */
316 #define RPF_L3_FILTER_SRCADDR_REG(i) (0x53b0 + (i) * 4)
317 #define RPF_L3_FILTER_DSTADDR_REG(i) (0x53d0 + (i) * 4)
318 #define RPF_L3_FILTER_L4_SRCPORT_REG(i) (0x5400 + (i) * 4)
319 #define RPF_L3_FILTER_L4_DSTPORT_REG(i) (0x5420 + (i) * 4)
320
321 #define RX_FLR_RSS_CONTROL1_REG 0x54c0
322 #define RX_FLR_RSS_CONTROL1_EN __BIT(31)
323
324 #define RPF_RPB_RX_TC_UPT_REG 0x54c4
325 #define RPF_RPB_RX_TC_UPT_MASK(i) (0x00000007 << ((i) * 4))
326
327 #define RPF_RSS_KEY_ADDR_REG 0x54d0
328 #define RPF_RSS_KEY_ADDR __BITS(4,0)
329 #define RPF_RSS_KEY_WR_EN __BIT(5)
330 #define RPF_RSS_KEY_WR_DATA_REG 0x54d4
331 #define RPF_RSS_KEY_RD_DATA_REG 0x54d8
332
333 #define RPF_RSS_REDIR_ADDR_REG 0x54e0
334 #define RPF_RSS_REDIR_ADDR __BITS(3,0)
335 #define RPF_RSS_REDIR_WR_EN __BIT(4)
336
337 #define RPF_RSS_REDIR_WR_DATA_REG 0x54e4
338 #define RPF_RSS_REDIR_WR_DATA __BITS(15,0)
339
340 #define RPO_HWCSUM_REG 0x5580
341 #define RPO_HWCSUM_IP4CSUM_EN __BIT(1)
342 #define RPO_HWCSUM_L4CSUM_EN __BIT(0) /* TCP/UDP/SCTP */
343
344 #define RPO_LRO_ENABLE_REG 0x5590
345
346 #define RPO_LRO_CONF_REG 0x5594
347 #define RPO_LRO_CONF_QSESSION_LIMIT __BITS(13,12)
348 #define RPO_LRO_CONF_TOTAL_DESC_LIMIT __BITS(6,5)
349 #define RPO_LRO_CONF_PATCHOPTIMIZATION_EN __BIT(15)
350 #define RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT __BITS(4,0)
351 #define RPO_LRO_RSC_MAX_REG 0x5598
352
353 /* RPO_LRO_LDES_MAX_REG[32/8] 0x55a0-0x55b0 */
354 #define RPO_LRO_LDES_MAX_REG(i) (0x55a0 + (i / 8) * 4)
355 #define RPO_LRO_LDES_MAX_MASK(i) (0x00000003 << ((i & 7) * 4))
356 #define RPO_LRO_TB_DIV_REG 0x5620
357 #define RPO_LRO_TB_DIV __BITS(20,31)
358 #define RPO_LRO_INACTIVE_IVAL_REG 0x5620
359 #define RPO_LRO_INACTIVE_IVAL __BITS(10,19)
360 #define RPO_LRO_MAX_COALESCING_IVAL_REG 0x5620
361 #define RPO_LRO_MAX_COALESCING_IVAL __BITS(9,0)
362
363 #define RPB_RPF_RX_REG 0x5700
364 #define RPB_RPF_RX_TC_MODE __BIT(8)
365 #define RPB_RPF_RX_FC_MODE __BITS(5,4)
366 #define RPB_RPF_RX_BUF_EN __BIT(0)
367
368 /* RPB_RXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x5710-0x5790 */
369 #define RPB_RXB_BUFSIZE_REG(i) (0x5710 + (i) * 0x10)
370 #define RPB_RXB_BUFSIZE __BITS(8,0)
371 #define RPB_RXB_XOFF_REG(i) (0x5714 + (i) * 0x10)
372 #define RPB_RXB_XOFF_EN __BIT(31)
373 #define RPB_RXB_XOFF_THRESH_HI __BITS(29,16)
374 #define RPB_RXB_XOFF_THRESH_LO __BITS(13,0)
375
376 #define RX_DMA_DESC_CACHE_INIT_REG 0x5a00
377 #define RX_DMA_DESC_CACHE_INIT __BIT(0)
378
379 #define RX_DMA_INT_DESC_WRWB_EN_REG 0x05a30
380 #define RX_DMA_INT_DESC_WRWB_EN __BIT(2)
381 #define RX_DMA_INT_DESC_MODERATE_EN __BIT(3)
382
383 /* RX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x5a40-0x5ac0 */
384 #define RX_INTR_MODERATION_CTL_REG(i) (0x5a40 + (i) * 4)
385 #define RX_INTR_MODERATION_CTL_EN __BIT(1)
386 #define RX_INTR_MODERATION_CTL_MIN __BITS(15,8)
387 #define RX_INTR_MODERATION_CTL_MAX __BITS(24,16)
388
389 /* RX_DMA_DESC_*[AQ_RINGS_NUM] 0x5b00-0x5f00 */
390 #define RX_DMA_DESC_BASE_ADDRLSW_REG(i) (0x5b00 + (i) * 0x20)
391 #define RX_DMA_DESC_BASE_ADDRMSW_REG(i) (0x5b04 + (i) * 0x20)
392 #define RX_DMA_DESC_REG(i) (0x5b08 + (i) * 0x20)
393 #define RX_DMA_DESC_LEN __BITS(12,3) /* RXD_NUM/8 */
394 #define RX_DMA_DESC_RESET __BIT(25)
395 #define RX_DMA_DESC_HEADER_SPLIT __BIT(28)
396 #define RX_DMA_DESC_VLAN_STRIP __BIT(29)
397 #define RX_DMA_DESC_EN __BIT(31)
398 #define RX_DMA_DESC_HEAD_PTR_REG(i) (0x5b0c + (i) * 0x20)
399 #define RX_DMA_DESC_HEAD_PTR __BITS(12,0)
400 #define RX_DMA_DESC_TAIL_PTR_REG(i) (0x5b10 + (i) * 0x20)
401 #define RX_DMA_DESC_BUFSIZE_REG(i) (0x5b18 + (i) * 0x20)
402 #define RX_DMA_DESC_BUFSIZE_DATA __BITS(4,0)
403 #define RX_DMA_DESC_BUFSIZE_HDR __BITS(12,8)
404
405 /* RX_DMA_DCAD_REG[AQ_RINGS_NUM] 0x6100-0x6180 */
406 #define RX_DMA_DCAD_REG(i) (0x6100 + (i) * 4)
407 #define RX_DMA_DCAD_CPUID __BITS(7,0)
408 #define RX_DMA_DCAD_PAYLOAD_EN __BIT(29)
409 #define RX_DMA_DCAD_HEADER_EN __BIT(30)
410 #define RX_DMA_DCAD_DESC_EN __BIT(31)
411
412 #define RX_DMA_DCA_REG 0x6180
413 #define RX_DMA_DCA_EN __BIT(31)
414 #define RX_DMA_DCA_MODE __BITS(3,0)
415
416 /* counters */
417 #define RX_DMA_GOOD_PKT_COUNTERLSW 0x6800
418 #define RX_DMA_GOOD_OCTET_COUNTERLSW 0x6808
419 #define RX_DMA_DROP_PKT_CNT_REG 0x6818
420 #define RX_DMA_COALESCED_PKT_CNT_REG 0x6820
421
422 #define TX_SYSCONTROL_REG 0x7000
423 #define TX_SYSCONTROL_TPB_DMA_LOOPBACK __BIT(6)
424 #define TX_SYSCONTROL_TPO_PKT_LOOPBACK __BIT(7)
425 #define TX_SYSCONTROL_RESET_DIS __BIT(29)
426
427 #define TX_TPO2_REG 0x7040
428 #define TX_TPO2_EN __BIT(16)
429
430 #define TPS_DESC_VM_ARB_MODE_REG 0x7300
431 #define TPS_DESC_VM_ARB_MODE __BIT(0)
432 #define TPS_DESC_RATE_REG 0x7310
433 #define TPS_DESC_RATE_TA_RST __BIT(31)
434 #define TPS_DESC_RATE_LIM __BITS(10,0)
435 #define TPS_DESC_TC_ARB_MODE_REG 0x7200
436 #define TPS_DESC_TC_ARB_MODE __BITS(1,0)
437 #define TPS_DATA_TC_ARB_MODE_REG 0x7100
438 #define TPS_DATA_TC_ARB_MODE __BIT(0)
439
440 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7110-0x7130 */
441 #define TPS_DATA_TCT_REG(i) (0x7110 + (i) * 4)
442 #define TPS_DATA_TCT_CREDIT_MAX __BITS(16,27)
443 #define TPS_DATA_TCT_WEIGHT __BITS(8,0)
444 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7210-0x7230 */
445 #define TPS_DESC_TCT_REG(i) (0x7210 + (i) * 4)
446 #define TPS_DESC_TCT_CREDIT_MAX __BITS(16,27)
447 #define TPS_DESC_TCT_WEIGHT __BITS(8,0)
448
449 #define AQ_HW_TXBUF_MAX 160
450 #define AQ_HW_RXBUF_MAX 320
451
452 #define TPO_HWCSUM_REG 0x7800
453 #define TPO_HWCSUM_IP4CSUM_EN __BIT(1)
454 #define TPO_HWCSUM_L4CSUM_EN __BIT(0) /* TCP/UDP/SCTP */
455
456 #define TDM_LSO_EN_REG 0x7810
457
458 #define THM_LSO_TCP_FLAG1_REG 0x7820
459 #define THM_LSO_TCP_FLAG1_FIRST __BITS(11,0)
460 #define THM_LSO_TCP_FLAG1_MID __BITS(27,16)
461 #define THM_LSO_TCP_FLAG2_REG 0x7824
462 #define THM_LSO_TCP_FLAG2_LAST __BITS(11,0)
463
464 #define TPB_TX_BUF_REG 0x7900
465 #define TPB_TX_BUF_EN __BIT(0)
466 #define TPB_TX_BUF_SCP_INS_EN __BIT(2)
467 #define TPB_TX_BUF_TC_MODE_EN __BIT(8)
468
469 /* TPB_TXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x7910-7990 */
470 #define TPB_TXB_BUFSIZE_REG(i) (0x7910 + (i) * 0x10)
471 #define TPB_TXB_BUFSIZE __BITS(7,0)
472 #define TPB_TXB_THRESH_REG(i) (0x7914 + (i) * 0x10)
473 #define TPB_TXB_THRESH_HI __BITS(16,28)
474 #define TPB_TXB_THRESH_LO __BITS(12,0)
475
476 #define AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG 0x7b20
477 #define TX_DMA_INT_DESC_WRWB_EN_REG 0x7b40
478 #define TX_DMA_INT_DESC_WRWB_EN __BIT(1)
479 #define TX_DMA_INT_DESC_MODERATE_EN __BIT(4)
480
481 /* TX_DMA_DESC_*[AQ_RINGS_NUM] 0x7c00-0x8400 */
482 #define TX_DMA_DESC_BASE_ADDRLSW_REG(i) (0x7c00 + (i) * 0x40)
483 #define TX_DMA_DESC_BASE_ADDRMSW_REG(i) (0x7c04 + (i) * 0x40)
484 #define TX_DMA_DESC_REG(i) (0x7c08 + (i) * 0x40)
485 #define TX_DMA_DESC_LEN __BITS(12, 3) /* TXD_NUM/8 */
486 #define TX_DMA_DESC_EN __BIT(31)
487 #define TX_DMA_DESC_HEAD_PTR_REG(i) (0x7c0c + (i) * 0x40)
488 #define TX_DMA_DESC_HEAD_PTR __BITS(12,0)
489 #define TX_DMA_DESC_TAIL_PTR_REG(i) (0x7c10 + (i) * 0x40)
490 #define TX_DMA_DESC_WRWB_THRESH_REG(i) (0x7c18 + (i) * 0x40)
491 #define TX_DMA_DESC_WRWB_THRESH __BITS(14,8)
492
493 /* TDM_DCAD_REG[AQ_RINGS_NUM] 0x8400-0x8480 */
494 #define TDM_DCAD_REG(i) (0x8400 + (i) * 4)
495 #define TDM_DCAD_CPUID __BITS(7,0)
496 #define TDM_DCAD_CPUID_EN __BIT(31)
497
498 #define TDM_DCA_REG 0x8480
499 #define TDM_DCA_EN __BIT(31)
500 #define TDM_DCA_MODE __BITS(3,0)
501
502 /* TX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x8980-0x8a00 */
503 #define TX_INTR_MODERATION_CTL_REG(i) (0x8980 + (i) * 4)
504 #define TX_INTR_MODERATION_CTL_EN __BIT(1)
505 #define TX_INTR_MODERATION_CTL_MIN __BITS(15,8)
506 #define TX_INTR_MODERATION_CTL_MAX __BITS(24,16)
507
508 #define FW1X_CTRL_10G __BIT(0)
509 #define FW1X_CTRL_5G __BIT(1)
510 #define FW1X_CTRL_5GSR __BIT(2)
511 #define FW1X_CTRL_2G5 __BIT(3)
512 #define FW1X_CTRL_1G __BIT(4)
513 #define FW1X_CTRL_100M __BIT(5)
514
515 #define FW2X_CTRL_10BASET_HD __BIT(0)
516 #define FW2X_CTRL_10BASET_FD __BIT(1)
517 #define FW2X_CTRL_100BASETX_HD __BIT(2)
518 #define FW2X_CTRL_100BASET4_HD __BIT(3)
519 #define FW2X_CTRL_100BASET2_HD __BIT(4)
520 #define FW2X_CTRL_100BASETX_FD __BIT(5)
521 #define FW2X_CTRL_100BASET2_FD __BIT(6)
522 #define FW2X_CTRL_1000BASET_HD __BIT(7)
523 #define FW2X_CTRL_1000BASET_FD __BIT(8)
524 #define FW2X_CTRL_2P5GBASET_FD __BIT(9)
525 #define FW2X_CTRL_5GBASET_FD __BIT(10)
526 #define FW2X_CTRL_10GBASET_FD __BIT(11)
527 #define FW2X_CTRL_RESERVED1 __BIT(32)
528 #define FW2X_CTRL_10BASET_EEE __BIT(33)
529 #define FW2X_CTRL_RESERVED2 __BIT(34)
530 #define FW2X_CTRL_PAUSE __BIT(35)
531 #define FW2X_CTRL_ASYMMETRIC_PAUSE __BIT(36)
532 #define FW2X_CTRL_100BASETX_EEE __BIT(37)
533 #define FW2X_CTRL_RESERVED3 __BIT(38)
534 #define FW2X_CTRL_RESERVED4 __BIT(39)
535 #define FW2X_CTRL_1000BASET_FD_EEE __BIT(40)
536 #define FW2X_CTRL_2P5GBASET_FD_EEE __BIT(41)
537 #define FW2X_CTRL_5GBASET_FD_EEE __BIT(42)
538 #define FW2X_CTRL_10GBASET_FD_EEE __BIT(43)
539 #define FW2X_CTRL_RESERVED5 __BIT(44)
540 #define FW2X_CTRL_RESERVED6 __BIT(45)
541 #define FW2X_CTRL_RESERVED7 __BIT(46)
542 #define FW2X_CTRL_RESERVED8 __BIT(47)
543 #define FW2X_CTRL_RESERVED9 __BIT(48)
544 #define FW2X_CTRL_CABLE_DIAG __BIT(49)
545 #define FW2X_CTRL_TEMPERATURE __BIT(50)
546 #define FW2X_CTRL_DOWNSHIFT __BIT(51)
547 #define FW2X_CTRL_PTP_AVB_EN __BIT(52)
548 #define FW2X_CTRL_MEDIA_DETECT __BIT(53)
549 #define FW2X_CTRL_LINK_DROP __BIT(54)
550 #define FW2X_CTRL_SLEEP_PROXY __BIT(55)
551 #define FW2X_CTRL_WOL __BIT(56)
552 #define FW2X_CTRL_MAC_STOP __BIT(57)
553 #define FW2X_CTRL_EXT_LOOPBACK __BIT(58)
554 #define FW2X_CTRL_INT_LOOPBACK __BIT(59)
555 #define FW2X_CTRL_EFUSE_AGENT __BIT(60)
556 #define FW2X_CTRL_WOL_TIMER __BIT(61)
557 #define FW2X_CTRL_STATISTICS __BIT(62)
558 #define FW2X_CTRL_TRANSACTION_ID __BIT(63)
559
560 #define FW2X_SNPRINTB \
561 "\177\020" \
562 "b\x23" "PAUSE\0" \
563 "b\x24" "ASYMMETRIC-PAUSE\0" \
564 "b\x31" "CABLE-DIAG\0" \
565 "b\x32" "TEMPERATURE\0" \
566 "b\x33" "DOWNSHIFT\0" \
567 "b\x34" "PTP-AVB\0" \
568 "b\x35" "MEDIA-DETECT\0" \
569 "b\x36" "LINK-DROP\0" \
570 "b\x37" "SLEEP-PROXY\0" \
571 "b\x38" "WOL\0" \
572 "b\x39" "MAC-STOP\0" \
573 "b\x3a" "EXT-LOOPBACK\0" \
574 "b\x3b" "INT-LOOPBACK\0" \
575 "b\x3c" "EFUSE-AGENT\0" \
576 "b\x3d" "WOL-TIMER\0" \
577 "b\x3e" "STATISTICS\0" \
578 "b\x3f" "TRANSACTION-ID\0" \
579 "\0"
580
581 #define FW2X_CTRL_RATE_100M FW2X_CTRL_100BASETX_FD
582 #define FW2X_CTRL_RATE_1G FW2X_CTRL_1000BASET_FD
583 #define FW2X_CTRL_RATE_2G5 FW2X_CTRL_2P5GBASET_FD
584 #define FW2X_CTRL_RATE_5G FW2X_CTRL_5GBASET_FD
585 #define FW2X_CTRL_RATE_10G FW2X_CTRL_10GBASET_FD
586 #define FW2X_CTRL_RATE_MASK \
587 (FW2X_CTRL_RATE_100M | \
588 FW2X_CTRL_RATE_1G | \
589 FW2X_CTRL_RATE_2G5 | \
590 FW2X_CTRL_RATE_5G | \
591 FW2X_CTRL_RATE_10G)
592 #define FW2X_CTRL_EEE_MASK \
593 (FW2X_CTRL_10BASET_EEE | \
594 FW2X_CTRL_100BASETX_EEE | \
595 FW2X_CTRL_1000BASET_FD_EEE | \
596 FW2X_CTRL_2P5GBASET_FD_EEE | \
597 FW2X_CTRL_5GBASET_FD_EEE | \
598 FW2X_CTRL_10GBASET_FD_EEE)
599
600 typedef enum aq_fw_bootloader_mode {
601 FW_BOOT_MODE_UNKNOWN = 0,
602 FW_BOOT_MODE_FLB,
603 FW_BOOT_MODE_RBL_FLASH,
604 FW_BOOT_MODE_RBL_HOST_BOOTLOAD
605 } aq_fw_bootloader_mode_t;
606
607 #define AQ_WRITE_REG(sc, reg, val) \
608 bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val))
609
610 #define AQ_READ_REG(sc, reg) \
611 bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))
612
613 #define AQ_READ64_REG(sc, reg) \
614 ((uint64_t)AQ_READ_REG(sc, reg) | \
615 (((uint64_t)AQ_READ_REG(sc, (reg) + 4)) << 32))
616
617 #define AQ_WRITE64_REG(sc, reg, val) \
618 do { \
619 AQ_WRITE_REG(sc, reg, (uint32_t)val); \
620 AQ_WRITE_REG(sc, reg + 4, (uint32_t)(val >> 32)); \
621 } while (/* CONSTCOND */0)
622
623 #define AQ_READ_REG_BIT(sc, reg, mask) \
624 __SHIFTOUT(AQ_READ_REG(sc, reg), mask)
625
626 #define AQ_WRITE_REG_BIT(sc, reg, mask, val) \
627 do { \
628 uint32_t _v; \
629 _v = AQ_READ_REG((sc), (reg)); \
630 _v &= ~(mask); \
631 if ((val) != 0) \
632 _v |= __SHIFTIN((val), (mask)); \
633 AQ_WRITE_REG((sc), (reg), _v); \
634 } while (/* CONSTCOND */ 0)
635
636 #define WAIT_FOR(expr, us, n, errp) \
637 do { \
638 unsigned int _n; \
639 for (_n = n; (!(expr)) && _n != 0; --_n) { \
640 delay((us)); \
641 } \
642 if ((errp != NULL)) { \
643 if (_n == 0) \
644 *(errp) = ETIMEDOUT; \
645 else \
646 *(errp) = 0; \
647 } \
648 } while (/* CONSTCOND */ 0)
649
650 #define msec_delay(x) DELAY(1000 * (x))
651
652 typedef struct aq_mailbox_header {
653 uint32_t version;
654 uint32_t transaction_id;
655 int32_t error;
656 } __packed aq_mailbox_header_t;
657
658 typedef struct aq_hw_stats_s {
659 uint32_t uprc;
660 uint32_t mprc;
661 uint32_t bprc;
662 uint32_t erpt;
663 uint32_t uptc;
664 uint32_t mptc;
665 uint32_t bptc;
666 uint32_t erpr;
667 uint32_t mbtc;
668 uint32_t bbtc;
669 uint32_t mbrc;
670 uint32_t bbrc;
671 uint32_t ubrc;
672 uint32_t ubtc;
673 uint32_t ptc;
674 uint32_t prc;
675 uint32_t dpc; /* not exists in fw2x_msm_statistics */
676 uint32_t cprc; /* not exists in fw2x_msm_statistics */
677 } __packed aq_hw_stats_s_t;
678
679 typedef struct fw1x_mailbox {
680 aq_mailbox_header_t header;
681 aq_hw_stats_s_t msm;
682 } __packed fw1x_mailbox_t;
683
684 typedef struct fw2x_msm_statistics {
685 uint32_t uprc;
686 uint32_t mprc;
687 uint32_t bprc;
688 uint32_t erpt;
689 uint32_t uptc;
690 uint32_t mptc;
691 uint32_t bptc;
692 uint32_t erpr;
693 uint32_t mbtc;
694 uint32_t bbtc;
695 uint32_t mbrc;
696 uint32_t bbrc;
697 uint32_t ubrc;
698 uint32_t ubtc;
699 uint32_t ptc;
700 uint32_t prc;
701 } __packed fw2x_msm_statistics_t;
702
703 typedef struct fw2x_phy_cable_diag_data {
704 uint32_t lane_data[4];
705 } __packed fw2x_phy_cable_diag_data_t;
706
707 typedef struct fw2x_capabilities {
708 uint32_t caps_lo;
709 uint32_t caps_hi;
710 } __packed fw2x_capabilities_t;
711
712 typedef struct fw2x_mailbox { /* struct fwHostInterface */
713 aq_mailbox_header_t header;
714 fw2x_msm_statistics_t msm; /* msmStatistics_t msm; */
715
716 uint32_t phy_info1;
717 #define PHYINFO1_FAULT_CODE __BITS(31,16)
718 #define PHYINFO1_PHY_H_BIT __BITS(0,15)
719 uint32_t phy_info2;
720 #define PHYINFO2_TEMPERATURE __BITS(15,0)
721 #define PHYINFO2_CABLE_LEN __BITS(23,16)
722
723 fw2x_phy_cable_diag_data_t diag_data;
724 uint32_t reserved[8];
725
726 fw2x_capabilities_t caps;
727
728 /* ... */
729 } __packed fw2x_mailbox_t;
730
731 typedef enum aq_link_speed {
732 AQ_LINK_NONE = 0,
733 AQ_LINK_100M = (1 << 0),
734 AQ_LINK_1G = (1 << 1),
735 AQ_LINK_2G5 = (1 << 2),
736 AQ_LINK_5G = (1 << 3),
737 AQ_LINK_10G = (1 << 4)
738 } aq_link_speed_t;
739 #define AQ_LINK_ALL (AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | \
740 AQ_LINK_5G | AQ_LINK_10G )
741 #define AQ_LINK_AUTO AQ_LINK_ALL
742
743 typedef enum aq_link_fc {
744 AQ_FC_NONE = 0,
745 AQ_FC_RX = __BIT(0),
746 AQ_FC_TX = __BIT(1),
747 AQ_FC_ALL = (AQ_FC_RX | AQ_FC_TX)
748 } aq_link_fc_t;
749
750 typedef enum aq_link_eee {
751 AQ_EEE_DISABLE = 0,
752 AQ_EEE_ENABLE = 1
753 } aq_link_eee_t;
754
755 typedef enum aq_hw_fw_mpi_state {
756 MPI_DEINIT = 0,
757 MPI_RESET = 1,
758 MPI_INIT = 2,
759 MPI_POWER = 4
760 } aq_hw_fw_mpi_state_t;
761
762 enum aq_media_type {
763 AQ_MEDIA_TYPE_UNKNOWN = 0,
764 AQ_MEDIA_TYPE_FIBRE,
765 AQ_MEDIA_TYPE_TP
766 };
767
768 struct aq_rx_desc_read {
769 uint64_t buf_addr;
770 uint64_t hdr_addr;
771 } __packed;
772
773 struct aq_rx_desc_wb {
774 uint32_t type;
775 #define RXDESC_TYPE_RSSTYPE __BITS(3,0)
776 #define RXDESC_TYPE_RSSTYPE_NONE 0
777 #define RXDESC_TYPE_RSSTYPE_IPV4 2
778 #define RXDESC_TYPE_RSSTYPE_IPV6 3
779 #define RXDESC_TYPE_RSSTYPE_IPV4_TCP 4
780 #define RXDESC_TYPE_RSSTYPE_IPV6_TCP 5
781 #define RXDESC_TYPE_RSSTYPE_IPV4_UDP 6
782 #define RXDESC_TYPE_RSSTYPE_IPV6_UDP 7
783 #define RXDESC_TYPE_PKTTYPE_ETHER __BITS(5,4)
784 #define RXDESC_TYPE_PKTTYPE_ETHER_IPV4 0
785 #define RXDESC_TYPE_PKTTYPE_ETHER_IPV6 1
786 #define RXDESC_TYPE_PKTTYPE_ETHER_OTHERS 2
787 #define RXDESC_TYPE_PKTTYPE_ETHER_ARP 3
788 #define RXDESC_TYPE_PKTTYPE_PROTO __BITS(8,6)
789 #define RXDESC_TYPE_PKTTYPE_PROTO_TCP 0
790 #define RXDESC_TYPE_PKTTYPE_PROTO_UDP 1
791 #define RXDESC_TYPE_PKTTYPE_PROTO_SCTP 2
792 #define RXDESC_TYPE_PKTTYPE_PROTO_ICMP 3
793 #define RXDESC_TYPE_PKTTYPE_PROTO_OTHERS 4
794 #define RXDESC_TYPE_PKTTYPE_VLAN __BIT(9)
795 #define RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE __BIT(10)
796 #define RXDESC_TYPE_MAC_DMA_ERR __BIT(12)
797 #define RXDESC_TYPE_RESERVED __BITS(18,13)
798 #define RXDESC_TYPE_IPV4_CSUM_CHECKED __BIT(19) /* PKTTYPE_ETHER_IPV4 */
799 #define RXDESC_TYPE_TCPUDP_CSUM_CHECKED __BIT(20)
800 #define RXDESC_TYPE_SPH __BIT(21)
801 #define RXDESC_TYPE_HDR_LEN __BITS(31,22)
802 uint32_t rss_hash;
803 uint16_t status;
804 #define RXDESC_STATUS_DD __BIT(0)
805 #define RXDESC_STATUS_EOP __BIT(1)
806 #define RXDESC_STATUS_MACERR __BIT(2)
807 #define RXDESC_STATUS_IPV4_CSUM_NG __BIT(3)
808 #define RXDESC_STATUS_TCPUDP_CSUM_ERROR __BIT(4)
809 #define RXDESC_STATUS_TCPUDP_CSUM_OK __BIT(5)
810
811 #define RXDESC_STATUS_STAT __BITS(2,5)
812 #define RXDESC_STATUS_ESTAT __BITS(6,11)
813 #define RXDESC_STATUS_RSC_CNT __BITS(12,15)
814 uint16_t pkt_len;
815 uint16_t next_desc_ptr;
816 uint16_t vlan;
817 } __packed;
818
819 typedef union aq_rx_desc {
820 struct aq_rx_desc_read read;
821 struct aq_rx_desc_wb wb;
822 } __packed aq_rx_desc_t;
823
824 typedef struct aq_tx_desc {
825 uint64_t buf_addr;
826 uint32_t ctl1;
827 #define AQ_TXDESC_CTL1_TYPE_MASK 0x00000003
828 #define AQ_TXDESC_CTL1_TYPE_TXD 0x00000001
829 #define AQ_TXDESC_CTL1_TYPE_TXC 0x00000002
830 #define AQ_TXDESC_CTL1_BLEN __BITS(19,4) /* TXD */
831 #define AQ_TXDESC_CTL1_DD __BIT(20) /* TXD */
832 #define AQ_TXDESC_CTL1_EOP __BIT(21) /* TXD */
833 #define AQ_TXDESC_CTL1_CMD_VLAN __BIT(22) /* TXD */
834 #define AQ_TXDESC_CTL1_CMD_FCS __BIT(23) /* TXD */
835 #define AQ_TXDESC_CTL1_CMD_IP4CSUM __BIT(24) /* TXD */
836 #define AQ_TXDESC_CTL1_CMD_L4CSUM __BIT(25) /* TXD */
837 #define AQ_TXDESC_CTL1_CMD_LSO __BIT(26) /* TXD */
838 #define AQ_TXDESC_CTL1_CMD_WB __BIT(27) /* TXD */
839 #define AQ_TXDESC_CTL1_CMD_VXLAN __BIT(28) /* TXD */
840 #define AQ_TXDESC_CTL1_VID __BITS(15,4) /* TXC */
841 #define AQ_TXDESC_CTL1_LSO_IPV6 __BIT(21) /* TXC */
842 #define AQ_TXDESC_CTL1_LSO_TCP __BIT(22) /* TXC */
843 uint32_t ctl2;
844 #define AQ_TXDESC_CTL2_LEN __BITS(31,14)
845 #define AQ_TXDESC_CTL2_CTX_EN __BIT(13)
846 #define AQ_TXDESC_CTL2_CTX_IDX __BIT(12)
847 } __packed aq_tx_desc_t;
848
849 struct aq_txring {
850 struct aq_softc *txr_sc;
851 int txr_index;
852 kmutex_t txr_mutex;
853 bool txr_active;
854
855 pcq_t *txr_pcq;
856 void *txr_softint;
857
858 aq_tx_desc_t *txr_txdesc; /* aq_tx_desc_t[AQ_TXD_NUM] */
859 bus_dmamap_t txr_txdesc_dmamap;
860 bus_dma_segment_t txr_txdesc_seg[1];
861 bus_size_t txr_txdesc_size;
862
863 struct {
864 struct mbuf *m;
865 bus_dmamap_t dmamap;
866 } txr_mbufs[AQ_TXD_NUM];
867 unsigned int txr_prodidx;
868 unsigned int txr_considx;
869 int txr_nfree;
870
871 /* counters */
872 uint64_t txr_opackets;
873 uint64_t txr_obytes;
874 uint64_t txr_omcasts;
875 uint64_t txr_oerrors;
876 };
877
878 struct aq_rxring {
879 struct aq_softc *rxr_sc;
880 int rxr_index;
881 kmutex_t rxr_mutex;
882 bool rxr_active;
883
884 aq_rx_desc_t *rxr_rxdesc; /* aq_rx_desc_t[AQ_RXD_NUM] */
885 bus_dmamap_t rxr_rxdesc_dmamap;
886 bus_dma_segment_t rxr_rxdesc_seg[1];
887 bus_size_t rxr_rxdesc_size;
888 struct {
889 struct mbuf *m;
890 bus_dmamap_t dmamap;
891 } rxr_mbufs[AQ_RXD_NUM];
892 unsigned int rxr_readidx;
893
894 /* counters */
895 uint64_t rxr_ipackets;
896 uint64_t rxr_ibytes;
897 uint64_t rxr_ierrors;
898 uint64_t rxr_iqdrops;
899 };
900
901 struct aq_queue {
902 struct aq_softc *sc;
903 struct aq_txring txring;
904 struct aq_rxring rxring;
905 };
906
907 struct aq_softc;
908 struct aq_firmware_ops {
909 int (*reset)(struct aq_softc *);
910 int (*set_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t,
911 aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
912 int (*get_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t *,
913 aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
914 int (*get_stats)(struct aq_softc *, aq_hw_stats_s_t *);
915 #if NSYSMON_ENVSYS > 0
916 int (*get_temperature)(struct aq_softc *, uint32_t *);
917 #endif
918 };
919
920 #ifdef AQ_EVENT_COUNTERS
921 #define AQ_EVCNT_DECL(name) \
922 char sc_evcount_##name##_name[32]; \
923 struct evcnt sc_evcount_##name##_ev;
924 #define AQ_EVCNT_ATTACH(sc, name, desc, evtype) \
925 do { \
926 snprintf((sc)->sc_evcount_##name##_name, \
927 sizeof((sc)->sc_evcount_##name##_name), \
928 "%s", desc); \
929 evcnt_attach_dynamic(&(sc)->sc_evcount_##name##_ev, \
930 (evtype), NULL, device_xname((sc)->sc_dev), \
931 (sc)->sc_evcount_##name##_name); \
932 } while (/*CONSTCOND*/0)
933 #define AQ_EVCNT_ATTACH_MISC(sc, name, desc) \
934 AQ_EVCNT_ATTACH(sc, name, desc, EVCNT_TYPE_MISC)
935 #define AQ_EVCNT_DETACH(sc, name) \
936 evcnt_detach(&(sc)->sc_evcount_##name##_ev)
937 #define AQ_EVCNT_ADD(sc, name, val) \
938 ((sc)->sc_evcount_##name##_ev.ev_count += (val))
939 #endif /* AQ_EVENT_COUNTERS */
940
941 #define AQ_LOCK(sc) mutex_enter(&(sc)->sc_mutex);
942 #define AQ_UNLOCK(sc) mutex_exit(&(sc)->sc_mutex);
943
944 /* lock for FW2X_MPI_{CONTROL,STATE]_REG read-modify-write */
945 #define AQ_MPI_LOCK(sc) mutex_enter(&(sc)->sc_mpi_mutex);
946 #define AQ_MPI_UNLOCK(sc) mutex_exit(&(sc)->sc_mpi_mutex);
947
948
949 struct aq_softc {
950 device_t sc_dev;
951
952 bus_space_tag_t sc_iot;
953 bus_space_handle_t sc_ioh;
954 bus_size_t sc_iosize;
955 bus_dma_tag_t sc_dmat;
956
957 void *sc_ihs[AQ_NINTR_MAX];
958 pci_intr_handle_t *sc_intrs;
959
960 int sc_tx_irq[AQ_RSSQUEUE_MAX];
961 int sc_rx_irq[AQ_RSSQUEUE_MAX];
962 int sc_linkstat_irq;
963 bool sc_use_txrx_independent_intr;
964 bool sc_poll_linkstat;
965 bool sc_detect_linkstat;
966
967 #if NSYSMON_ENVSYS > 0
968 struct sysmon_envsys *sc_sme;
969 envsys_data_t sc_sensor_temp;
970 #endif
971
972 callout_t sc_tick_ch;
973
974 int sc_nintrs;
975 bool sc_msix;
976
977 struct aq_queue sc_queue[AQ_RSSQUEUE_MAX];
978 int sc_nqueues;
979
980 pci_chipset_tag_t sc_pc;
981 pcitag_t sc_pcitag;
982 uint16_t sc_product;
983 uint16_t sc_revision;
984
985 kmutex_t sc_mutex;
986 kmutex_t sc_mpi_mutex;
987
988 const struct aq_firmware_ops *sc_fw_ops;
989 uint64_t sc_fw_caps;
990 enum aq_media_type sc_media_type;
991 aq_link_speed_t sc_available_rates;
992
993 aq_link_speed_t sc_link_rate;
994 aq_link_fc_t sc_link_fc;
995 aq_link_eee_t sc_link_eee;
996
997 uint32_t sc_fw_version;
998 #define FW_VERSION_MAJOR(sc) (((sc)->sc_fw_version >> 24) & 0xff)
999 #define FW_VERSION_MINOR(sc) (((sc)->sc_fw_version >> 16) & 0xff)
1000 #define FW_VERSION_BUILD(sc) ((sc)->sc_fw_version & 0xffff)
1001 uint32_t sc_features;
1002 #define FEATURES_MIPS 0x00000001
1003 #define FEATURES_TPO2 0x00000002
1004 #define FEATURES_RPF2 0x00000004
1005 #define FEATURES_MPI_AQ 0x00000008
1006 #define FEATURES_REV_A0 0x10000000
1007 #define FEATURES_REV_A (FEATURES_REV_A0)
1008 #define FEATURES_REV_B0 0x20000000
1009 #define FEATURES_REV_B1 0x40000000
1010 #define FEATURES_REV_B (FEATURES_REV_B0|FEATURES_REV_B1)
1011 uint32_t sc_mbox_addr;
1012
1013 bool sc_rbl_enabled;
1014 bool sc_fast_start_enabled;
1015 bool sc_flash_present;
1016
1017 bool sc_intr_moderation_enable;
1018 bool sc_rss_enable;
1019
1020 struct ethercom sc_ethercom;
1021 struct ether_addr sc_enaddr;
1022 struct ifmedia sc_media;
1023 int sc_ec_capenable; /* last ec_capenable */
1024 unsigned short sc_if_flags; /* last if_flags */
1025
1026 #ifdef AQ_EVENT_COUNTERS
1027 aq_hw_stats_s_t sc_statistics[2];
1028 int sc_statistics_idx;
1029 bool sc_poll_statistics;
1030
1031 AQ_EVCNT_DECL(uprc);
1032 AQ_EVCNT_DECL(mprc);
1033 AQ_EVCNT_DECL(bprc);
1034 AQ_EVCNT_DECL(erpt);
1035 AQ_EVCNT_DECL(uptc);
1036 AQ_EVCNT_DECL(mptc);
1037 AQ_EVCNT_DECL(bptc);
1038 AQ_EVCNT_DECL(erpr);
1039 AQ_EVCNT_DECL(mbtc);
1040 AQ_EVCNT_DECL(bbtc);
1041 AQ_EVCNT_DECL(mbrc);
1042 AQ_EVCNT_DECL(bbrc);
1043 AQ_EVCNT_DECL(ubrc);
1044 AQ_EVCNT_DECL(ubtc);
1045 AQ_EVCNT_DECL(ptc);
1046 AQ_EVCNT_DECL(prc);
1047 AQ_EVCNT_DECL(dpc);
1048 AQ_EVCNT_DECL(cprc);
1049 #endif
1050 };
1051
1052 static int aq_match(device_t, cfdata_t, void *);
1053 static void aq_attach(device_t, device_t, void *);
1054 static int aq_detach(device_t, int);
1055
1056 static int aq_setup_msix(struct aq_softc *, struct pci_attach_args *, int,
1057 bool, bool);
1058 static int aq_setup_legacy(struct aq_softc *, struct pci_attach_args *,
1059 pci_intr_type_t);
1060 static int aq_establish_msix_intr(struct aq_softc *, bool, bool);
1061
1062 static int aq_ifmedia_change(struct ifnet * const);
1063 static void aq_ifmedia_status(struct ifnet * const, struct ifmediareq *);
1064 static int aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set);
1065 static int aq_ifflags_cb(struct ethercom *);
1066 static int aq_init(struct ifnet *);
1067 static void aq_send_common_locked(struct ifnet *, struct aq_softc *,
1068 struct aq_txring *, bool);
1069 static int aq_transmit(struct ifnet *, struct mbuf *);
1070 static void aq_deferred_transmit(void *);
1071 static void aq_start(struct ifnet *);
1072 static void aq_stop(struct ifnet *, int);
1073 static void aq_watchdog(struct ifnet *);
1074 static int aq_ioctl(struct ifnet *, unsigned long, void *);
1075
1076 static int aq_txrx_rings_alloc(struct aq_softc *);
1077 static void aq_txrx_rings_free(struct aq_softc *);
1078 static int aq_tx_pcq_alloc(struct aq_softc *, struct aq_txring *);
1079 static void aq_tx_pcq_free(struct aq_softc *, struct aq_txring *);
1080
1081 static void aq_initmedia(struct aq_softc *);
1082 static void aq_enable_intr(struct aq_softc *, bool, bool);
1083
1084 #if NSYSMON_ENVSYS > 0
1085 static void aq_temp_refresh(struct sysmon_envsys *, envsys_data_t *);
1086 #endif
1087 static void aq_tick(void *);
1088 static int aq_legacy_intr(void *);
1089 static int aq_link_intr(void *);
1090 static int aq_txrx_intr(void *);
1091 static int aq_tx_intr(void *);
1092 static int aq_rx_intr(void *);
1093
1094 static int aq_set_linkmode(struct aq_softc *, aq_link_speed_t, aq_link_fc_t,
1095 aq_link_eee_t);
1096 static int aq_get_linkmode(struct aq_softc *, aq_link_speed_t *, aq_link_fc_t *,
1097 aq_link_eee_t *);
1098
1099 static int aq_fw_reset(struct aq_softc *);
1100 static int aq_fw_version_init(struct aq_softc *);
1101 static int aq_hw_init(struct aq_softc *);
1102 static int aq_hw_init_ucp(struct aq_softc *);
1103 static int aq_hw_reset(struct aq_softc *);
1104 static int aq_fw_downld_dwords(struct aq_softc *, uint32_t, uint32_t *,
1105 uint32_t);
1106 static int aq_get_mac_addr(struct aq_softc *);
1107 static int aq_init_rss(struct aq_softc *);
1108 static int aq_set_capability(struct aq_softc *);
1109
1110 static int fw1x_reset(struct aq_softc *);
1111 static int fw1x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
1112 aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
1113 static int fw1x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
1114 aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
1115 static int fw1x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
1116
1117 static int fw2x_reset(struct aq_softc *);
1118 static int fw2x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
1119 aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
1120 static int fw2x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
1121 aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
1122 static int fw2x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
1123 #if NSYSMON_ENVSYS > 0
1124 static int fw2x_get_temperature(struct aq_softc *, uint32_t *);
1125 #endif
1126
1127 static const struct aq_firmware_ops aq_fw1x_ops = {
1128 .reset = fw1x_reset,
1129 .set_mode = fw1x_set_mode,
1130 .get_mode = fw1x_get_mode,
1131 .get_stats = fw1x_get_stats,
1132 #if NSYSMON_ENVSYS > 0
1133 .get_temperature = NULL
1134 #endif
1135 };
1136
1137 static const struct aq_firmware_ops aq_fw2x_ops = {
1138 .reset = fw2x_reset,
1139 .set_mode = fw2x_set_mode,
1140 .get_mode = fw2x_get_mode,
1141 .get_stats = fw2x_get_stats,
1142 #if NSYSMON_ENVSYS > 0
1143 .get_temperature = fw2x_get_temperature
1144 #endif
1145 };
1146
1147 CFATTACH_DECL3_NEW(aq, sizeof(struct aq_softc),
1148 aq_match, aq_attach, aq_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN);
1149
1150 static const struct aq_product {
1151 pci_vendor_id_t aq_vendor;
1152 pci_product_id_t aq_product;
1153 const char *aq_name;
1154 enum aq_media_type aq_media_type;
1155 aq_link_speed_t aq_available_rates;
1156 } aq_products[] = {
1157 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100,
1158 "Aquantia AQC100 10 Gigabit Network Adapter",
1159 AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
1160 },
1161 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107,
1162 "Aquantia AQC107 10 Gigabit Network Adapter",
1163 AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
1164 },
1165 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108,
1166 "Aquantia AQC108 5 Gigabit Network Adapter",
1167 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1168 },
1169 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109,
1170 "Aquantia AQC109 2.5 Gigabit Network Adapter",
1171 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1172 },
1173 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111,
1174 "Aquantia AQC111 5 Gigabit Network Adapter",
1175 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1176 },
1177 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112,
1178 "Aquantia AQC112 2.5 Gigabit Network Adapter",
1179 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1180 },
1181 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100S,
1182 "Aquantia AQC100S 10 Gigabit Network Adapter",
1183 AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
1184 },
1185 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107S,
1186 "Aquantia AQC107S 10 Gigabit Network Adapter",
1187 AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
1188 },
1189 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108S,
1190 "Aquantia AQC108S 5 Gigabit Network Adapter",
1191 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1192 },
1193 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109S,
1194 "Aquantia AQC109S 2.5 Gigabit Network Adapter",
1195 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1196 },
1197 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111S,
1198 "Aquantia AQC111S 5 Gigabit Network Adapter",
1199 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1200 },
1201 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112S,
1202 "Aquantia AQC112S 2.5 Gigabit Network Adapter",
1203 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1204 },
1205 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D100,
1206 "Aquantia D100 10 Gigabit Network Adapter",
1207 AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
1208 },
1209 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D107,
1210 "Aquantia D107 10 Gigabit Network Adapter",
1211 AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
1212 },
1213 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D108,
1214 "Aquantia D108 5 Gigabit Network Adapter",
1215 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
1216 },
1217 { PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D109,
1218 "Aquantia D109 2.5 Gigabit Network Adapter",
1219 AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
1220 }
1221 };
1222
1223 static const struct aq_product *
1224 aq_lookup(const struct pci_attach_args *pa)
1225 {
1226 unsigned int i;
1227
1228 for (i = 0; i < __arraycount(aq_products); i++) {
1229 if (PCI_VENDOR(pa->pa_id) == aq_products[i].aq_vendor &&
1230 PCI_PRODUCT(pa->pa_id) == aq_products[i].aq_product)
1231 return &aq_products[i];
1232 }
1233 return NULL;
1234 }
1235
1236 static int
1237 aq_match(device_t parent, cfdata_t cf, void *aux)
1238 {
1239 struct pci_attach_args *pa = aux;
1240
1241 if (aq_lookup(pa) != NULL)
1242 return 1;
1243
1244 return 0;
1245 }
1246
1247 static void
1248 aq_attach(device_t parent, device_t self, void *aux)
1249 {
1250 struct aq_softc *sc = device_private(self);
1251 struct pci_attach_args *pa = aux;
1252 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1253 pci_chipset_tag_t pc;
1254 pcitag_t tag;
1255 pcireg_t command, memtype, bar;
1256 const struct aq_product *aqp;
1257 int error;
1258
1259 sc->sc_dev = self;
1260 mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_NET);
1261 mutex_init(&sc->sc_mpi_mutex, MUTEX_DEFAULT, IPL_NET);
1262
1263 sc->sc_pc = pc = pa->pa_pc;
1264 sc->sc_pcitag = tag = pa->pa_tag;
1265 sc->sc_dmat = pci_dma64_available(pa) ? pa->pa_dmat64 : pa->pa_dmat;
1266
1267 command = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
1268 command |= PCI_COMMAND_MASTER_ENABLE;
1269 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, command);
1270
1271 sc->sc_product = PCI_PRODUCT(pa->pa_id);
1272 sc->sc_revision = PCI_REVISION(pa->pa_class);
1273
1274 aqp = aq_lookup(pa);
1275 KASSERT(aqp != NULL);
1276
1277 pci_aprint_devinfo_fancy(pa, "Ethernet controller", aqp->aq_name, 1);
1278
1279 bar = pci_conf_read(pc, tag, PCI_BAR(0));
1280 if ((PCI_MAPREG_MEM_ADDR(bar) == 0) ||
1281 (PCI_MAPREG_TYPE(bar) != PCI_MAPREG_TYPE_MEM)) {
1282 aprint_error_dev(sc->sc_dev, "wrong BAR type\n");
1283 return;
1284 }
1285 memtype = pci_mapreg_type(pc, tag, PCI_BAR(0));
1286 if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->sc_iot, &sc->sc_ioh,
1287 NULL, &sc->sc_iosize) != 0) {
1288 aprint_error_dev(sc->sc_dev, "unable to map register\n");
1289 return;
1290 }
1291
1292 sc->sc_nqueues = MIN(ncpu, AQ_RSSQUEUE_MAX);
1293
1294 /* max queue num is 8, and must be 2^n */
1295 if (ncpu >= 8)
1296 sc->sc_nqueues = 8;
1297 else if (ncpu >= 4)
1298 sc->sc_nqueues = 4;
1299 else if (ncpu >= 2)
1300 sc->sc_nqueues = 2;
1301 else
1302 sc->sc_nqueues = 1;
1303
1304 int msixcount = pci_msix_count(pa->pa_pc, pa->pa_tag);
1305 #ifndef CONFIG_NO_TXRX_INDEPENDENT
1306 if (msixcount >= (sc->sc_nqueues * 2 + 1)) {
1307 /* TX intrs + RX intrs + LINKSTAT intrs */
1308 sc->sc_use_txrx_independent_intr = true;
1309 sc->sc_poll_linkstat = false;
1310 sc->sc_msix = true;
1311 } else if (msixcount >= (sc->sc_nqueues * 2)) {
1312 /* TX intrs + RX intrs */
1313 sc->sc_use_txrx_independent_intr = true;
1314 sc->sc_poll_linkstat = true;
1315 sc->sc_msix = true;
1316 } else
1317 #endif
1318 if (msixcount >= (sc->sc_nqueues + 1)) {
1319 /* TX/RX intrs LINKSTAT intrs */
1320 sc->sc_use_txrx_independent_intr = false;
1321 sc->sc_poll_linkstat = false;
1322 sc->sc_msix = true;
1323 } else if (msixcount >= sc->sc_nqueues) {
1324 /* TX/RX intrs */
1325 sc->sc_use_txrx_independent_intr = false;
1326 sc->sc_poll_linkstat = true;
1327 sc->sc_msix = true;
1328 } else {
1329 /* giving up using MSI-X */
1330 sc->sc_msix = false;
1331 }
1332
1333 /* XXX: on FIBRE, linkstat interrupt does not occur on boot? */
1334 if (aqp->aq_media_type == AQ_MEDIA_TYPE_FIBRE)
1335 sc->sc_poll_linkstat = true;
1336
1337 #ifdef AQ_FORCE_POLL_LINKSTAT
1338 sc->sc_poll_linkstat = true;
1339 #endif
1340
1341 aprint_debug_dev(sc->sc_dev,
1342 "ncpu=%d, pci_msix_count=%d."
1343 " allocate %d interrupts for %d%s queues%s\n",
1344 ncpu, msixcount,
1345 (sc->sc_use_txrx_independent_intr ?
1346 (sc->sc_nqueues * 2) : sc->sc_nqueues) +
1347 (sc->sc_poll_linkstat ? 0 : 1),
1348 sc->sc_nqueues,
1349 sc->sc_use_txrx_independent_intr ? "*2" : "",
1350 sc->sc_poll_linkstat ? "" : ", and link status");
1351
1352 if (sc->sc_msix)
1353 error = aq_setup_msix(sc, pa, sc->sc_nqueues,
1354 sc->sc_use_txrx_independent_intr, !sc->sc_poll_linkstat);
1355 else
1356 error = ENODEV;
1357
1358 if (error != 0) {
1359 /* if MSI-X failed, fallback to MSI with single queue */
1360 sc->sc_use_txrx_independent_intr = false;
1361 sc->sc_poll_linkstat = false;
1362 sc->sc_msix = false;
1363 sc->sc_nqueues = 1;
1364 error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_MSI);
1365 }
1366 if (error != 0) {
1367 /* if MSI failed, fallback to INTx */
1368 error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_INTX);
1369 }
1370 if (error != 0)
1371 return;
1372
1373 callout_init(&sc->sc_tick_ch, 0);
1374 callout_setfunc(&sc->sc_tick_ch, aq_tick, sc);
1375
1376 sc->sc_intr_moderation_enable = CONFIG_INTR_MODERATION_ENABLE;
1377
1378 if (sc->sc_msix && (sc->sc_nqueues > 1))
1379 sc->sc_rss_enable = true;
1380 else
1381 sc->sc_rss_enable = false;
1382
1383 error = aq_txrx_rings_alloc(sc);
1384 if (error != 0)
1385 goto attach_failure;
1386
1387 error = aq_fw_reset(sc);
1388 if (error != 0)
1389 goto attach_failure;
1390
1391 error = aq_fw_version_init(sc);
1392 if (error != 0)
1393 goto attach_failure;
1394
1395 error = aq_hw_init_ucp(sc);
1396 if (error < 0)
1397 goto attach_failure;
1398
1399 KASSERT(sc->sc_mbox_addr != 0);
1400 error = aq_hw_reset(sc);
1401 if (error != 0)
1402 goto attach_failure;
1403
1404 aq_get_mac_addr(sc);
1405 aq_init_rss(sc);
1406
1407 error = aq_hw_init(sc); /* initialize and interrupts */
1408 if (error != 0)
1409 goto attach_failure;
1410
1411 sc->sc_media_type = aqp->aq_media_type;
1412 sc->sc_available_rates = aqp->aq_available_rates;
1413
1414 sc->sc_ethercom.ec_ifmedia = &sc->sc_media;
1415 ifmedia_init(&sc->sc_media, IFM_IMASK,
1416 aq_ifmedia_change, aq_ifmedia_status);
1417 aq_initmedia(sc);
1418
1419 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
1420 ifp->if_softc = sc;
1421 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1422 ifp->if_baudrate = IF_Gbps(10);
1423 ifp->if_init = aq_init;
1424 ifp->if_ioctl = aq_ioctl;
1425 if (sc->sc_msix && (sc->sc_nqueues > 1))
1426 ifp->if_transmit = aq_transmit;
1427 ifp->if_start = aq_start;
1428 ifp->if_stop = aq_stop;
1429 ifp->if_watchdog = aq_watchdog;
1430 IFQ_SET_READY(&ifp->if_snd);
1431
1432 /* initialize capabilities */
1433 sc->sc_ethercom.ec_capabilities = 0;
1434 sc->sc_ethercom.ec_capenable = 0;
1435 #if notyet
1436 /* TODO */
1437 sc->sc_ethercom.ec_capabilities |= ETHERCAP_EEE;
1438 #endif
1439 sc->sc_ethercom.ec_capabilities |=
1440 ETHERCAP_JUMBO_MTU |
1441 ETHERCAP_VLAN_MTU |
1442 ETHERCAP_VLAN_HWTAGGING |
1443 ETHERCAP_VLAN_HWFILTER;
1444 sc->sc_ethercom.ec_capenable |=
1445 ETHERCAP_VLAN_HWTAGGING |
1446 ETHERCAP_VLAN_HWFILTER;
1447
1448 ifp->if_capabilities = 0;
1449 ifp->if_capenable = 0;
1450 #ifdef CONFIG_LRO_SUPPORT
1451 ifp->if_capabilities |= IFCAP_LRO;
1452 ifp->if_capenable |= IFCAP_LRO;
1453 #endif
1454 #if notyet
1455 /* TSO */
1456 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
1457 #endif
1458
1459 #if notyet
1460 /*
1461 * XXX:
1462 * Rx L4 CSUM doesn't work well for fragment packet.
1463 * aq marks 'CHEDKED' and 'BAD' for them.
1464 * we need to ignore (clear) hw-csum flags if the packet is fragmented
1465 *
1466 * TODO: test with LRO enabled
1467 */
1468 ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv6_Rx;
1469 ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv6_Rx;
1470 #endif
1471 /* TX hardware checksum offloadding */
1472 ifp->if_capabilities |= IFCAP_CSUM_IPv4_Tx;
1473 ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv6_Tx;
1474 ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv6_Tx;
1475 /* RX hardware checksum offloadding */
1476 ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx;
1477
1478 if_attach(ifp);
1479 if_deferred_start_init(ifp, NULL);
1480 ether_ifattach(ifp, sc->sc_enaddr.ether_addr_octet);
1481 ether_set_vlan_cb(&sc->sc_ethercom, aq_vlan_cb);
1482 ether_set_ifflags_cb(&sc->sc_ethercom, aq_ifflags_cb);
1483
1484 aq_enable_intr(sc, true, false); /* only intr about link */
1485
1486 /* update media */
1487 aq_ifmedia_change(ifp);
1488
1489 #if NSYSMON_ENVSYS > 0
1490 /* temperature monitoring */
1491 if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_temperature != NULL &&
1492 (sc->sc_fw_caps & FW2X_CTRL_TEMPERATURE) != 0) {
1493
1494 sc->sc_sme = sysmon_envsys_create();
1495 sc->sc_sme->sme_name = device_xname(self);
1496 sc->sc_sme->sme_cookie = sc;
1497 sc->sc_sme->sme_flags = 0;
1498 sc->sc_sme->sme_refresh = aq_temp_refresh;
1499 sc->sc_sensor_temp.units = ENVSYS_STEMP;
1500 sc->sc_sensor_temp.state = ENVSYS_SINVALID;
1501 snprintf(sc->sc_sensor_temp.desc, ENVSYS_DESCLEN, "PHY");
1502
1503 sysmon_envsys_sensor_attach(sc->sc_sme, &sc->sc_sensor_temp);
1504 sysmon_envsys_register(sc->sc_sme);
1505
1506 /*
1507 * for unknown reasons, the first call of fw2x_get_temperature()
1508 * will always fail (firmware matter?), so run once now.
1509 */
1510 aq_temp_refresh(sc->sc_sme, &sc->sc_sensor_temp);
1511 }
1512 #endif
1513
1514 #ifdef AQ_EVENT_COUNTERS
1515 /* get starting statistics values */
1516 if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_stats != NULL &&
1517 sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[0]) == 0) {
1518 sc->sc_poll_statistics = true;
1519 }
1520
1521 AQ_EVCNT_ATTACH_MISC(sc, uprc, "RX unicast packet");
1522 AQ_EVCNT_ATTACH_MISC(sc, bprc, "RX broadcast packet");
1523 AQ_EVCNT_ATTACH_MISC(sc, mprc, "RX multicast packet");
1524 AQ_EVCNT_ATTACH_MISC(sc, erpr, "RX error packet");
1525 AQ_EVCNT_ATTACH_MISC(sc, ubrc, "RX unicast bytes");
1526 AQ_EVCNT_ATTACH_MISC(sc, bbrc, "RX broadcast bytes");
1527 AQ_EVCNT_ATTACH_MISC(sc, mbrc, "RX multicast bytes");
1528 AQ_EVCNT_ATTACH_MISC(sc, prc, "RX good packet");
1529 AQ_EVCNT_ATTACH_MISC(sc, uptc, "TX unicast packet");
1530 AQ_EVCNT_ATTACH_MISC(sc, bptc, "TX broadcast packet");
1531 AQ_EVCNT_ATTACH_MISC(sc, mptc, "TX multicast packet");
1532 AQ_EVCNT_ATTACH_MISC(sc, erpt, "TX error packet");
1533 AQ_EVCNT_ATTACH_MISC(sc, ubtc, "TX unicast bytes");
1534 AQ_EVCNT_ATTACH_MISC(sc, bbtc, "TX broadcast bytes");
1535 AQ_EVCNT_ATTACH_MISC(sc, mbtc, "TX multicast bytes");
1536 AQ_EVCNT_ATTACH_MISC(sc, ptc, "TX good packet");
1537 AQ_EVCNT_ATTACH_MISC(sc, dpc, "DMA drop packet");
1538 AQ_EVCNT_ATTACH_MISC(sc, cprc, "RX coalesced packet");
1539 #endif
1540
1541 return;
1542
1543 attach_failure:
1544 aq_detach(self, 0);
1545 }
1546
1547 static int
1548 aq_detach(device_t self, int flags __unused)
1549 {
1550 struct aq_softc *sc = device_private(self);
1551 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1552 int i, s;
1553
1554 if (sc->sc_iosize != 0) {
1555 if (ifp->if_softc != NULL) {
1556 s = splnet();
1557 aq_stop(ifp, 0);
1558 splx(s);
1559 }
1560
1561 for (i = 0; i < AQ_NINTR_MAX; i++) {
1562 if (sc->sc_ihs[i] != NULL) {
1563 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
1564 sc->sc_ihs[i] = NULL;
1565 }
1566 }
1567 if (sc->sc_nintrs > 0) {
1568 pci_intr_release(sc->sc_pc, sc->sc_intrs,
1569 sc->sc_nintrs);
1570 sc->sc_intrs = NULL;
1571 sc->sc_nintrs = 0;
1572 }
1573
1574 aq_txrx_rings_free(sc);
1575
1576 if (ifp->if_softc != NULL) {
1577 ether_ifdetach(ifp);
1578 if_detach(ifp);
1579 }
1580
1581 aprint_debug_dev(sc->sc_dev, "%s: bus_space_unmap\n", __func__);
1582 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
1583 sc->sc_iosize = 0;
1584 }
1585
1586 callout_stop(&sc->sc_tick_ch);
1587
1588 #if NSYSMON_ENVSYS > 0
1589 if (sc->sc_sme != NULL) {
1590 /* all sensors associated with this will also be detached */
1591 sysmon_envsys_unregister(sc->sc_sme);
1592 sc->sc_sme = NULL;
1593 }
1594 #endif
1595
1596 #ifdef AQ_EVENT_COUNTERS
1597 AQ_EVCNT_DETACH(sc, uprc);
1598 AQ_EVCNT_DETACH(sc, mprc);
1599 AQ_EVCNT_DETACH(sc, bprc);
1600 AQ_EVCNT_DETACH(sc, erpt);
1601 AQ_EVCNT_DETACH(sc, uptc);
1602 AQ_EVCNT_DETACH(sc, mptc);
1603 AQ_EVCNT_DETACH(sc, bptc);
1604 AQ_EVCNT_DETACH(sc, erpr);
1605 AQ_EVCNT_DETACH(sc, mbtc);
1606 AQ_EVCNT_DETACH(sc, bbtc);
1607 AQ_EVCNT_DETACH(sc, mbrc);
1608 AQ_EVCNT_DETACH(sc, bbrc);
1609 AQ_EVCNT_DETACH(sc, ubrc);
1610 AQ_EVCNT_DETACH(sc, ubtc);
1611 AQ_EVCNT_DETACH(sc, ptc);
1612 AQ_EVCNT_DETACH(sc, prc);
1613 AQ_EVCNT_DETACH(sc, dpc);
1614 AQ_EVCNT_DETACH(sc, cprc);
1615 #endif
1616
1617 mutex_destroy(&sc->sc_mpi_mutex);
1618 mutex_destroy(&sc->sc_mutex);
1619
1620 return 0;
1621 }
1622
1623 static int
1624 aq_establish_intr(struct aq_softc *sc, int intno, kcpuset_t *affinity,
1625 int (*func)(void *), void *arg, const char *xname)
1626 {
1627 char intrbuf[PCI_INTRSTR_LEN];
1628 pci_chipset_tag_t pc = sc->sc_pc;
1629 void *vih;
1630 const char *intrstr = NULL;
1631
1632 intrstr = pci_intr_string(pc, sc->sc_intrs[intno], intrbuf,
1633 sizeof(intrbuf));
1634
1635 pci_intr_setattr(pc, &sc->sc_intrs[intno], PCI_INTR_MPSAFE, true);
1636
1637 vih = pci_intr_establish_xname(pc, sc->sc_intrs[intno],
1638 IPL_NET, func, arg, xname);
1639 if (vih == NULL) {
1640 aprint_error_dev(sc->sc_dev,
1641 "unable to establish MSI-X%s%s for %s\n",
1642 intrstr ? " at " : "",
1643 intrstr ? intrstr : "", xname);
1644 return EIO;
1645 }
1646 sc->sc_ihs[intno] = vih;
1647
1648 if (affinity != NULL) {
1649 /* Round-robin affinity */
1650 kcpuset_zero(affinity);
1651 kcpuset_set(affinity, intno % ncpu);
1652 interrupt_distribute(vih, affinity, NULL);
1653 }
1654
1655 return 0;
1656 }
1657
1658 static int
1659 aq_establish_msix_intr(struct aq_softc *sc, bool txrx_independent,
1660 bool linkintr)
1661 {
1662 kcpuset_t *affinity;
1663 int error, intno, i;
1664 char intr_xname[INTRDEVNAMEBUF];
1665
1666 kcpuset_create(&affinity, false);
1667
1668 intno = 0;
1669
1670 if (txrx_independent) {
1671 for (i = 0; i < sc->sc_nqueues; i++) {
1672 snprintf(intr_xname, sizeof(intr_xname), "%s RX%d",
1673 device_xname(sc->sc_dev), i);
1674 sc->sc_rx_irq[i] = intno;
1675 error = aq_establish_intr(sc, intno++, affinity,
1676 aq_rx_intr, &sc->sc_queue[i].rxring, intr_xname);
1677 if (error != 0)
1678 goto fail;
1679 }
1680 for (i = 0; i < sc->sc_nqueues; i++) {
1681 snprintf(intr_xname, sizeof(intr_xname), "%s TX%d",
1682 device_xname(sc->sc_dev), i);
1683 sc->sc_tx_irq[i] = intno;
1684 error = aq_establish_intr(sc, intno++, affinity,
1685 aq_tx_intr, &sc->sc_queue[i].txring, intr_xname);
1686 if (error != 0)
1687 goto fail;
1688 }
1689 } else {
1690 for (i = 0; i < sc->sc_nqueues; i++) {
1691 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
1692 device_xname(sc->sc_dev), i);
1693 sc->sc_rx_irq[i] = intno;
1694 sc->sc_tx_irq[i] = intno;
1695 error = aq_establish_intr(sc, intno++, affinity,
1696 aq_txrx_intr, &sc->sc_queue[i], intr_xname);
1697 if (error != 0)
1698 goto fail;
1699 }
1700 }
1701
1702 if (linkintr) {
1703 snprintf(intr_xname, sizeof(intr_xname), "%s LINK",
1704 device_xname(sc->sc_dev));
1705 sc->sc_linkstat_irq = intno;
1706 error = aq_establish_intr(sc, intno++, affinity,
1707 aq_link_intr, sc, intr_xname);
1708 if (error != 0)
1709 goto fail;
1710 }
1711
1712 kcpuset_destroy(affinity);
1713 return 0;
1714
1715 fail:
1716 for (i = 0; i < AQ_NINTR_MAX; i++) {
1717 if (sc->sc_ihs[i] != NULL) {
1718 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
1719 sc->sc_ihs[i] = NULL;
1720 }
1721 }
1722
1723 kcpuset_destroy(affinity);
1724 return ENOMEM;
1725 }
1726
1727 static int
1728 aq_setup_msix(struct aq_softc *sc, struct pci_attach_args *pa, int nqueue,
1729 bool txrx_independent, bool linkintr)
1730 {
1731 int error, nintr;
1732
1733 if (txrx_independent)
1734 nintr = nqueue * 2;
1735 else
1736 nintr = nqueue;
1737
1738 if (linkintr)
1739 nintr++;
1740
1741 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, nintr);
1742 if (error != 0) {
1743 aprint_error_dev(sc->sc_dev,
1744 "failed to allocate MSI-X interrupts\n");
1745 goto fail;
1746 }
1747
1748 error = aq_establish_msix_intr(sc, txrx_independent, linkintr);
1749 if (error == 0) {
1750 sc->sc_nintrs = nintr;
1751 } else {
1752 pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
1753 sc->sc_nintrs = 0;
1754 }
1755 fail:
1756 return error;
1757
1758 }
1759
1760 static int
1761 aq_setup_legacy(struct aq_softc *sc, struct pci_attach_args *pa,
1762 pci_intr_type_t inttype)
1763 {
1764 int counts[PCI_INTR_TYPE_SIZE];
1765 int error, nintr;
1766
1767 nintr = 1;
1768
1769 memset(counts, 0, sizeof(counts));
1770 counts[inttype] = nintr;
1771
1772 error = pci_intr_alloc(pa, &sc->sc_intrs, counts, inttype);
1773 if (error != 0) {
1774 aprint_error_dev(sc->sc_dev,
1775 "failed to allocate%s interrupts\n",
1776 (inttype == PCI_INTR_TYPE_MSI) ? " MSI" : "");
1777 return error;
1778 }
1779 error = aq_establish_intr(sc, 0, NULL, aq_legacy_intr, sc,
1780 device_xname(sc->sc_dev));
1781 if (error == 0) {
1782 sc->sc_nintrs = nintr;
1783 } else {
1784 pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
1785 sc->sc_nintrs = 0;
1786 }
1787 return error;
1788 }
1789
1790 static void
1791 global_software_reset(struct aq_softc *sc)
1792 {
1793 uint32_t v;
1794
1795 AQ_WRITE_REG_BIT(sc, RX_SYSCONTROL_REG, RX_SYSCONTROL_RESET_DIS, 0);
1796 AQ_WRITE_REG_BIT(sc, TX_SYSCONTROL_REG, TX_SYSCONTROL_RESET_DIS, 0);
1797 AQ_WRITE_REG_BIT(sc, FW_MPI_RESETCTRL_REG,
1798 FW_MPI_RESETCTRL_RESET_DIS, 0);
1799
1800 v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
1801 v &= ~AQ_FW_SOFTRESET_DIS;
1802 v |= AQ_FW_SOFTRESET_RESET;
1803 AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
1804 }
1805
1806 static int
1807 mac_soft_reset_rbl(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
1808 {
1809 int timo;
1810
1811 aprint_debug_dev(sc->sc_dev, "RBL> MAC reset STARTED!\n");
1812
1813 AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
1814 AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
1815 AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
1816
1817 /* MAC FW will reload PHY FW if 1E.1000.3 was cleaned - #undone */
1818 AQ_WRITE_REG(sc, FW_BOOT_EXIT_CODE_REG, RBL_STATUS_DEAD);
1819
1820 global_software_reset(sc);
1821
1822 AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e0);
1823
1824 /* Wait for RBL to finish boot process. */
1825 #define RBL_TIMEOUT_MS 10000
1826 uint16_t rbl_status;
1827 for (timo = RBL_TIMEOUT_MS; timo > 0; timo--) {
1828 rbl_status = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG) & 0xffff;
1829 if (rbl_status != 0 && rbl_status != RBL_STATUS_DEAD)
1830 break;
1831 msec_delay(1);
1832 }
1833 if (timo <= 0) {
1834 aprint_error_dev(sc->sc_dev,
1835 "RBL> RBL restart failed: timeout\n");
1836 return EBUSY;
1837 }
1838 switch (rbl_status) {
1839 case RBL_STATUS_SUCCESS:
1840 if (mode != NULL)
1841 *mode = FW_BOOT_MODE_RBL_FLASH;
1842 aprint_debug_dev(sc->sc_dev, "RBL> reset complete! [Flash]\n");
1843 break;
1844 case RBL_STATUS_HOST_BOOT:
1845 if (mode != NULL)
1846 *mode = FW_BOOT_MODE_RBL_HOST_BOOTLOAD;
1847 aprint_debug_dev(sc->sc_dev,
1848 "RBL> reset complete! [Host Bootload]\n");
1849 break;
1850 case RBL_STATUS_FAILURE:
1851 default:
1852 aprint_error_dev(sc->sc_dev,
1853 "unknown RBL status 0x%x\n", rbl_status);
1854 return EBUSY;
1855 }
1856
1857 return 0;
1858 }
1859
1860 static int
1861 mac_soft_reset_flb(struct aq_softc *sc)
1862 {
1863 uint32_t v;
1864 int timo;
1865
1866 AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
1867 /*
1868 * Let Felicity hardware to complete SMBUS transaction before
1869 * Global software reset.
1870 */
1871 msec_delay(50);
1872
1873 /*
1874 * If SPI burst transaction was interrupted(before running the script),
1875 * global software reset may not clear SPI interface.
1876 * Clean it up manually before global reset.
1877 */
1878 AQ_WRITE_REG(sc, AQ_GLB_NVR_PROVISIONING2_REG, 0x00a0);
1879 AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x009f);
1880 AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x809f);
1881 msec_delay(50);
1882
1883 v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
1884 v &= ~AQ_FW_SOFTRESET_DIS;
1885 v |= AQ_FW_SOFTRESET_RESET;
1886 AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
1887
1888 /* Kickstart. */
1889 AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
1890 AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
1891 if (!sc->sc_fast_start_enabled)
1892 AQ_WRITE_REG(sc, AQ_GLB_GENERAL_PROVISIONING9_REG, 1);
1893
1894 /*
1895 * For the case SPI burst transaction was interrupted (by MCP reset
1896 * above), wait until it is completed by hardware.
1897 */
1898 msec_delay(50);
1899
1900 /* MAC Kickstart */
1901 if (!sc->sc_fast_start_enabled) {
1902 AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x180e0);
1903
1904 uint32_t flb_status;
1905 for (timo = 0; timo < 1000; timo++) {
1906 flb_status = AQ_READ_REG(sc,
1907 FW_MPI_DAISY_CHAIN_STATUS_REG) & 0x10;
1908 if (flb_status != 0)
1909 break;
1910 msec_delay(1);
1911 }
1912 if (flb_status == 0) {
1913 aprint_error_dev(sc->sc_dev,
1914 "FLB> MAC kickstart failed: timed out\n");
1915 return ETIMEDOUT;
1916 }
1917 aprint_debug_dev(sc->sc_dev,
1918 "FLB> MAC kickstart done, %d ms\n", timo);
1919 /* FW reset */
1920 AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
1921 /*
1922 * Let Felicity hardware complete SMBUS transaction before
1923 * Global software reset.
1924 */
1925 msec_delay(50);
1926 sc->sc_fast_start_enabled = true;
1927 }
1928 AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
1929
1930 /* PHY Kickstart: #undone */
1931 global_software_reset(sc);
1932
1933 for (timo = 0; timo < 1000; timo++) {
1934 if (AQ_READ_REG(sc, AQ_FW_VERSION_REG) != 0)
1935 break;
1936 msec_delay(10);
1937 }
1938 if (timo >= 1000) {
1939 aprint_error_dev(sc->sc_dev, "FLB> Global Soft Reset failed\n");
1940 return ETIMEDOUT;
1941 }
1942 aprint_debug_dev(sc->sc_dev, "FLB> F/W restart: %d ms\n", timo * 10);
1943 return 0;
1944
1945 }
1946
1947 static int
1948 mac_soft_reset(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
1949 {
1950 if (sc->sc_rbl_enabled)
1951 return mac_soft_reset_rbl(sc, mode);
1952
1953 if (mode != NULL)
1954 *mode = FW_BOOT_MODE_FLB;
1955 return mac_soft_reset_flb(sc);
1956 }
1957
1958 static int
1959 aq_fw_read_version(struct aq_softc *sc)
1960 {
1961 int i, error = EBUSY;
1962 #define MAC_FW_START_TIMEOUT_MS 10000
1963 for (i = 0; i < MAC_FW_START_TIMEOUT_MS; i++) {
1964 sc->sc_fw_version = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
1965 if (sc->sc_fw_version != 0) {
1966 error = 0;
1967 break;
1968 }
1969 delay(1000);
1970 }
1971 return error;
1972 }
1973
1974 static int
1975 aq_fw_reset(struct aq_softc *sc)
1976 {
1977 uint32_t ver, v, bootExitCode;
1978 int i, error;
1979
1980 ver = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
1981
1982 for (i = 1000; i > 0; i--) {
1983 v = AQ_READ_REG(sc, FW_MPI_DAISY_CHAIN_STATUS_REG);
1984 bootExitCode = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG);
1985 if (v != 0x06000000 || bootExitCode != 0)
1986 break;
1987 }
1988 if (i <= 0) {
1989 aprint_error_dev(sc->sc_dev,
1990 "F/W reset failed. Neither RBL nor FLB started\n");
1991 return ETIMEDOUT;
1992 }
1993 sc->sc_rbl_enabled = (bootExitCode != 0);
1994
1995 /*
1996 * Having FW version 0 is an indicator that cold start
1997 * is in progress. This means two things:
1998 * 1) Driver have to wait for FW/HW to finish boot (500ms giveup)
1999 * 2) Driver may skip reset sequence and save time.
2000 */
2001 if (sc->sc_fast_start_enabled && (ver != 0)) {
2002 error = aq_fw_read_version(sc);
2003 /* Skip reset as it just completed */
2004 if (error == 0)
2005 return 0;
2006 }
2007
2008 aq_fw_bootloader_mode_t mode = FW_BOOT_MODE_UNKNOWN;
2009 error = mac_soft_reset(sc, &mode);
2010 if (error != 0) {
2011 aprint_error_dev(sc->sc_dev, "MAC reset failed: %d\n", error);
2012 return error;
2013 }
2014
2015 switch (mode) {
2016 case FW_BOOT_MODE_FLB:
2017 aprint_debug_dev(sc->sc_dev,
2018 "FLB> F/W successfully loaded from flash.\n");
2019 sc->sc_flash_present = true;
2020 return aq_fw_read_version(sc);
2021 case FW_BOOT_MODE_RBL_FLASH:
2022 aprint_debug_dev(sc->sc_dev,
2023 "RBL> F/W loaded from flash. Host Bootload disabled.\n");
2024 sc->sc_flash_present = true;
2025 return aq_fw_read_version(sc);
2026 case FW_BOOT_MODE_UNKNOWN:
2027 aprint_error_dev(sc->sc_dev,
2028 "F/W bootload error: unknown bootloader type\n");
2029 return ENOTSUP;
2030 case FW_BOOT_MODE_RBL_HOST_BOOTLOAD:
2031 aprint_debug_dev(sc->sc_dev, "RBL> Host Bootload mode\n");
2032 break;
2033 }
2034
2035 /*
2036 * XXX: TODO: add support Host Boot
2037 */
2038 aprint_error_dev(sc->sc_dev,
2039 "RBL> F/W Host Bootload not implemented\n");
2040 return ENOTSUP;
2041 }
2042
2043 static int
2044 aq_hw_reset(struct aq_softc *sc)
2045 {
2046 int error;
2047
2048 /* disable irq */
2049 AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS, 0);
2050
2051 /* apply */
2052 AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_IRQ, 1);
2053
2054 /* wait ack 10 times by 1ms */
2055 WAIT_FOR(
2056 (AQ_READ_REG(sc, AQ_INTR_CTRL_REG) & AQ_INTR_CTRL_RESET_IRQ) == 0,
2057 1000, 10, &error);
2058 if (error != 0) {
2059 aprint_error_dev(sc->sc_dev,
2060 "atlantic: IRQ reset failed: %d\n", error);
2061 return error;
2062 }
2063
2064 return sc->sc_fw_ops->reset(sc);
2065 }
2066
2067 static int
2068 aq_hw_init_ucp(struct aq_softc *sc)
2069 {
2070 int timo;
2071
2072 if (FW_VERSION_MAJOR(sc) == 1) {
2073 if (AQ_READ_REG(sc, FW1X_MPI_INIT2_REG) == 0) {
2074 uint32_t data;
2075 cprng_fast(&data, sizeof(data));
2076 data &= 0xfefefefe;
2077 data |= 0x02020202;
2078 AQ_WRITE_REG(sc, FW1X_MPI_INIT2_REG, data);
2079 }
2080 AQ_WRITE_REG(sc, FW1X_MPI_INIT1_REG, 0);
2081 }
2082
2083 for (timo = 100; timo > 0; timo--) {
2084 sc->sc_mbox_addr = AQ_READ_REG(sc, FW_MPI_MBOX_ADDR_REG);
2085 if (sc->sc_mbox_addr != 0)
2086 break;
2087 delay(1000);
2088 }
2089
2090 #define AQ_FW_MIN_VERSION 0x01050006
2091 #define AQ_FW_MIN_VERSION_STR "1.5.6"
2092 if (sc->sc_fw_version < AQ_FW_MIN_VERSION) {
2093 aprint_error_dev(sc->sc_dev,
2094 "atlantic: wrong FW version: " AQ_FW_MIN_VERSION_STR
2095 " or later required, this is %d.%d.%d\n",
2096 FW_VERSION_MAJOR(sc),
2097 FW_VERSION_MINOR(sc),
2098 FW_VERSION_BUILD(sc));
2099 return ENOTSUP;
2100 }
2101
2102 return 0;
2103 }
2104
2105 static int
2106 aq_fw_version_init(struct aq_softc *sc)
2107 {
2108 int error = 0;
2109 char fw_vers[sizeof("F/W version xxxxx.xxxxx.xxxxx")];
2110
2111 if (FW_VERSION_MAJOR(sc) == 1) {
2112 sc->sc_fw_ops = &aq_fw1x_ops;
2113 } else if ((FW_VERSION_MAJOR(sc) == 2) || (FW_VERSION_MAJOR(sc) == 3)) {
2114 sc->sc_fw_ops = &aq_fw2x_ops;
2115 } else {
2116 aprint_error_dev(sc->sc_dev,
2117 "Unsupported F/W version %d.%d.%d\n",
2118 FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc),
2119 FW_VERSION_BUILD(sc));
2120 return ENOTSUP;
2121 }
2122 snprintf(fw_vers, sizeof(fw_vers), "F/W version %d.%d.%d",
2123 FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc), FW_VERSION_BUILD(sc));
2124
2125 /* detect revision */
2126 uint32_t hwrev = AQ_READ_REG(sc, AQ_HW_REVISION_REG);
2127 switch (hwrev & 0x0000000f) {
2128 case 0x01:
2129 aprint_normal_dev(sc->sc_dev, "Atlantic revision A0, %s\n",
2130 fw_vers);
2131 sc->sc_features |= FEATURES_REV_A0 |
2132 FEATURES_MPI_AQ | FEATURES_MIPS;
2133 break;
2134 case 0x02:
2135 aprint_normal_dev(sc->sc_dev, "Atlantic revision B0, %s\n",
2136 fw_vers);
2137 sc->sc_features |= FEATURES_REV_B0 |
2138 FEATURES_MPI_AQ | FEATURES_MIPS |
2139 FEATURES_TPO2 | FEATURES_RPF2;
2140 break;
2141 case 0x0A:
2142 aprint_normal_dev(sc->sc_dev, "Atlantic revision B1, %s\n",
2143 fw_vers);
2144 sc->sc_features |= FEATURES_REV_B1 |
2145 FEATURES_MPI_AQ | FEATURES_MIPS |
2146 FEATURES_TPO2 | FEATURES_RPF2;
2147 break;
2148 default:
2149 aprint_error_dev(sc->sc_dev,
2150 "Unknown revision (0x%08x)\n", hwrev);
2151 error = ENOTSUP;
2152 break;
2153 }
2154 return error;
2155 }
2156
2157 static int
2158 fw1x_reset(struct aq_softc *sc)
2159 {
2160 struct aq_mailbox_header mbox;
2161 const int retryCount = 1000;
2162 uint32_t tid0;
2163 int i;
2164
2165 tid0 = ~0; /*< Initial value of MBOX transactionId. */
2166
2167 for (i = 0; i < retryCount; ++i) {
2168 /*
2169 * Read the beginning of Statistics structure to capture
2170 * the Transaction ID.
2171 */
2172 aq_fw_downld_dwords(sc, sc->sc_mbox_addr,
2173 (uint32_t *)&mbox, sizeof(mbox) / sizeof(uint32_t));
2174
2175 /* Successfully read the stats. */
2176 if (tid0 == ~0U) {
2177 /* We have read the initial value. */
2178 tid0 = mbox.transaction_id;
2179 continue;
2180 } else if (mbox.transaction_id != tid0) {
2181 /*
2182 * Compare transaction ID to initial value.
2183 * If it's different means f/w is alive.
2184 * We're done.
2185 */
2186 return 0;
2187 }
2188
2189 /*
2190 * Transaction ID value haven't changed since last time.
2191 * Try reading the stats again.
2192 */
2193 delay(10);
2194 }
2195 aprint_error_dev(sc->sc_dev, "F/W 1.x reset finalize timeout\n");
2196 return EBUSY;
2197 }
2198
2199 static int
2200 fw1x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
2201 aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
2202 {
2203 uint32_t mpictrl = 0;
2204 uint32_t mpispeed = 0;
2205
2206 if (speed & AQ_LINK_10G)
2207 mpispeed |= FW1X_CTRL_10G;
2208 if (speed & AQ_LINK_5G)
2209 mpispeed |= (FW1X_CTRL_5G | FW1X_CTRL_5GSR);
2210 if (speed & AQ_LINK_2G5)
2211 mpispeed |= FW1X_CTRL_2G5;
2212 if (speed & AQ_LINK_1G)
2213 mpispeed |= FW1X_CTRL_1G;
2214 if (speed & AQ_LINK_100M)
2215 mpispeed |= FW1X_CTRL_100M;
2216
2217 mpictrl |= __SHIFTIN(mode, FW1X_MPI_STATE_MODE);
2218 mpictrl |= __SHIFTIN(mpispeed, FW1X_MPI_STATE_SPEED);
2219 AQ_WRITE_REG(sc, FW1X_MPI_CONTROL_REG, mpictrl);
2220 return 0;
2221 }
2222
2223 static int
2224 fw1x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
2225 aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
2226 {
2227 uint32_t mpistate, mpi_speed;
2228 aq_link_speed_t speed = AQ_LINK_NONE;
2229
2230 mpistate = AQ_READ_REG(sc, FW1X_MPI_STATE_REG);
2231
2232 if (modep != NULL)
2233 *modep = __SHIFTOUT(mpistate, FW1X_MPI_STATE_MODE);
2234
2235 mpi_speed = __SHIFTOUT(mpistate, FW1X_MPI_STATE_SPEED);
2236 if (mpi_speed & FW1X_CTRL_10G)
2237 speed = AQ_LINK_10G;
2238 else if (mpi_speed & (FW1X_CTRL_5G|FW1X_CTRL_5GSR))
2239 speed = AQ_LINK_5G;
2240 else if (mpi_speed & FW1X_CTRL_2G5)
2241 speed = AQ_LINK_2G5;
2242 else if (mpi_speed & FW1X_CTRL_1G)
2243 speed = AQ_LINK_1G;
2244 else if (mpi_speed & FW1X_CTRL_100M)
2245 speed = AQ_LINK_100M;
2246
2247 if (speedp != NULL)
2248 *speedp = speed;
2249
2250 if (fcp != NULL)
2251 *fcp = AQ_FC_NONE;
2252
2253 if (eeep != NULL)
2254 *eeep = AQ_EEE_DISABLE;
2255
2256 return 0;
2257 }
2258
2259 static int
2260 fw1x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
2261 {
2262 int error;
2263
2264 error = aq_fw_downld_dwords(sc,
2265 sc->sc_mbox_addr + offsetof(fw1x_mailbox_t, msm), (uint32_t *)stats,
2266 sizeof(aq_hw_stats_s_t) / sizeof(uint32_t));
2267 if (error < 0) {
2268 device_printf(sc->sc_dev,
2269 "fw1x> download statistics data FAILED, error %d", error);
2270 return error;
2271 }
2272
2273 stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
2274 stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
2275 return 0;
2276 }
2277
2278 static int
2279 fw2x_reset(struct aq_softc *sc)
2280 {
2281 fw2x_capabilities_t caps = { 0 };
2282 int error;
2283
2284 error = aq_fw_downld_dwords(sc,
2285 sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, caps),
2286 (uint32_t *)&caps, sizeof caps / sizeof(uint32_t));
2287 if (error != 0) {
2288 aprint_error_dev(sc->sc_dev,
2289 "fw2x> can't get F/W capabilities mask, error %d\n",
2290 error);
2291 return error;
2292 }
2293 sc->sc_fw_caps = caps.caps_lo | ((uint64_t)caps.caps_hi << 32);
2294
2295 char buf[256];
2296 snprintb(buf, sizeof(buf), FW2X_SNPRINTB, sc->sc_fw_caps);
2297 aprint_verbose_dev(sc->sc_dev, "fw2x> F/W capabilities=%s\n", buf);
2298
2299 return 0;
2300 }
2301
2302 static int
2303 fw2x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
2304 aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
2305 {
2306 uint64_t mpi_ctrl;
2307 int error = 0;
2308
2309 AQ_MPI_LOCK(sc);
2310
2311 mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
2312
2313 switch (mode) {
2314 case MPI_INIT:
2315 mpi_ctrl &= ~FW2X_CTRL_RATE_MASK;
2316 if (speed & AQ_LINK_10G)
2317 mpi_ctrl |= FW2X_CTRL_RATE_10G;
2318 if (speed & AQ_LINK_5G)
2319 mpi_ctrl |= FW2X_CTRL_RATE_5G;
2320 if (speed & AQ_LINK_2G5)
2321 mpi_ctrl |= FW2X_CTRL_RATE_2G5;
2322 if (speed & AQ_LINK_1G)
2323 mpi_ctrl |= FW2X_CTRL_RATE_1G;
2324 if (speed & AQ_LINK_100M)
2325 mpi_ctrl |= FW2X_CTRL_RATE_100M;
2326
2327 mpi_ctrl &= ~FW2X_CTRL_LINK_DROP;
2328
2329 mpi_ctrl &= ~FW2X_CTRL_EEE_MASK;
2330 if (eee == AQ_EEE_ENABLE)
2331 mpi_ctrl |= FW2X_CTRL_EEE_MASK;
2332
2333 mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
2334 if (fc & AQ_FC_RX)
2335 mpi_ctrl |= FW2X_CTRL_PAUSE;
2336 if (fc & AQ_FC_TX)
2337 mpi_ctrl |= FW2X_CTRL_ASYMMETRIC_PAUSE;
2338 break;
2339 case MPI_DEINIT:
2340 mpi_ctrl &= ~(FW2X_CTRL_RATE_MASK | FW2X_CTRL_EEE_MASK);
2341 mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
2342 break;
2343 default:
2344 device_printf(sc->sc_dev, "fw2x> unknown MPI state %d\n", mode);
2345 error = EINVAL;
2346 goto failure;
2347 }
2348 AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
2349
2350 failure:
2351 AQ_MPI_UNLOCK(sc);
2352 return error;
2353 }
2354
2355 static int
2356 fw2x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
2357 aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
2358 {
2359 uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
2360
2361 if (modep != NULL) {
2362 uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
2363 if (mpi_ctrl & FW2X_CTRL_RATE_MASK)
2364 *modep = MPI_INIT;
2365 else
2366 *modep = MPI_DEINIT;
2367 }
2368
2369 aq_link_speed_t speed = AQ_LINK_NONE;
2370 if (mpi_state & FW2X_CTRL_RATE_10G)
2371 speed = AQ_LINK_10G;
2372 else if (mpi_state & FW2X_CTRL_RATE_5G)
2373 speed = AQ_LINK_5G;
2374 else if (mpi_state & FW2X_CTRL_RATE_2G5)
2375 speed = AQ_LINK_2G5;
2376 else if (mpi_state & FW2X_CTRL_RATE_1G)
2377 speed = AQ_LINK_1G;
2378 else if (mpi_state & FW2X_CTRL_RATE_100M)
2379 speed = AQ_LINK_100M;
2380
2381 if (speedp != NULL)
2382 *speedp = speed;
2383
2384 aq_link_fc_t fc = AQ_FC_NONE;
2385 if (mpi_state & FW2X_CTRL_PAUSE)
2386 fc |= AQ_FC_RX;
2387 if (mpi_state & FW2X_CTRL_ASYMMETRIC_PAUSE)
2388 fc |= AQ_FC_TX;
2389 if (fcp != NULL)
2390 *fcp = fc;
2391
2392 /* XXX: TODO: EEE */
2393 if (eeep != NULL)
2394 *eeep = AQ_EEE_DISABLE;
2395
2396 return 0;
2397 }
2398
2399 static int
2400 toggle_mpi_ctrl_and_wait(struct aq_softc *sc, uint64_t mask,
2401 uint32_t timeout_ms, uint32_t try_count)
2402 {
2403 uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
2404 uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
2405 int error;
2406
2407 /* First, check that control and state values are consistent */
2408 if ((mpi_ctrl & mask) != (mpi_state & mask)) {
2409 device_printf(sc->sc_dev,
2410 "fw2x> MPI control (%#llx) and state (%#llx)"
2411 " are not consistent for mask %#llx!\n",
2412 (unsigned long long)mpi_ctrl, (unsigned long long)mpi_state,
2413 (unsigned long long)mask);
2414 return EINVAL;
2415 }
2416
2417 /* Invert bits (toggle) in control register */
2418 mpi_ctrl ^= mask;
2419 AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
2420
2421 /* Clear all bits except masked */
2422 mpi_ctrl &= mask;
2423
2424 /* Wait for FW reflecting change in state register */
2425 WAIT_FOR((AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG) & mask) == mpi_ctrl,
2426 1000 * timeout_ms, try_count, &error);
2427 if (error != 0) {
2428 device_printf(sc->sc_dev,
2429 "f/w2x> timeout while waiting for response"
2430 " in state register for bit %#llx!",
2431 (unsigned long long)mask);
2432 return error;
2433 }
2434 return 0;
2435 }
2436
2437 static int
2438 fw2x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
2439 {
2440 int error;
2441
2442 AQ_MPI_LOCK(sc);
2443 /* Say to F/W to update the statistics */
2444 error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_STATISTICS, 1, 25);
2445 if (error != 0) {
2446 device_printf(sc->sc_dev,
2447 "fw2x> statistics update error %d\n", error);
2448 goto failure;
2449 }
2450
2451 CTASSERT(sizeof(fw2x_msm_statistics_t) <= sizeof(struct aq_hw_stats_s));
2452 error = aq_fw_downld_dwords(sc,
2453 sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, msm), (uint32_t *)stats,
2454 sizeof(fw2x_msm_statistics_t) / sizeof(uint32_t));
2455 if (error != 0) {
2456 device_printf(sc->sc_dev,
2457 "fw2x> download statistics data FAILED, error %d", error);
2458 goto failure;
2459 }
2460 stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
2461 stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
2462
2463 failure:
2464 AQ_MPI_UNLOCK(sc);
2465 return error;
2466 }
2467
2468 #if NSYSMON_ENVSYS > 0
2469 static int
2470 fw2x_get_temperature(struct aq_softc *sc, uint32_t *temp)
2471 {
2472 int error;
2473 uint32_t value, celsius;
2474
2475 AQ_MPI_LOCK(sc);
2476
2477 /* Say to F/W to update the temperature */
2478 error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_TEMPERATURE, 1, 25);
2479 if (error != 0)
2480 goto failure;
2481
2482 error = aq_fw_downld_dwords(sc,
2483 sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, phy_info2),
2484 &value, sizeof(value) / sizeof(uint32_t));
2485 if (error != 0)
2486 goto failure;
2487
2488 /* 1/256 decrees C to microkelvin */
2489 celsius = __SHIFTOUT(value, PHYINFO2_TEMPERATURE);
2490 if (celsius == 0) {
2491 error = EIO;
2492 goto failure;
2493 }
2494 *temp = celsius * (1000000 / 256) + 273150000;
2495
2496 failure:
2497 AQ_MPI_UNLOCK(sc);
2498 return 0;
2499 }
2500 #endif
2501
2502 static int
2503 aq_fw_downld_dwords(struct aq_softc *sc, uint32_t addr, uint32_t *p,
2504 uint32_t cnt)
2505 {
2506 uint32_t v;
2507 int error = 0;
2508
2509 WAIT_FOR(AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG) == 1, 1, 10000, &error);
2510 if (error != 0) {
2511 AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);
2512 v = AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG);
2513 if (v == 0) {
2514 device_printf(sc->sc_dev,
2515 "%s:%d: timeout\n", __func__, __LINE__);
2516 return ETIMEDOUT;
2517 }
2518 }
2519
2520 AQ_WRITE_REG(sc, AQ_FW_MBOX_ADDR_REG, addr);
2521
2522 error = 0;
2523 for (; cnt > 0 && error == 0; cnt--) {
2524 /* execute mailbox interface */
2525 AQ_WRITE_REG_BIT(sc, AQ_FW_MBOX_CMD_REG,
2526 AQ_FW_MBOX_CMD_EXECUTE, 1);
2527 if (sc->sc_features & FEATURES_REV_B1) {
2528 WAIT_FOR(AQ_READ_REG(sc, AQ_FW_MBOX_ADDR_REG) != addr,
2529 1, 1000, &error);
2530 } else {
2531 WAIT_FOR((AQ_READ_REG(sc, AQ_FW_MBOX_CMD_REG) &
2532 AQ_FW_MBOX_CMD_BUSY) == 0,
2533 1, 1000, &error);
2534 }
2535 *p++ = AQ_READ_REG(sc, AQ_FW_MBOX_VAL_REG);
2536 addr += sizeof(uint32_t);
2537 }
2538 AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);
2539
2540 if (error != 0)
2541 device_printf(sc->sc_dev,
2542 "%s:%d: timeout\n", __func__, __LINE__);
2543
2544 return error;
2545 }
2546
2547 /* read my mac address */
2548 static int
2549 aq_get_mac_addr(struct aq_softc *sc)
2550 {
2551 uint32_t mac_addr[2];
2552 uint32_t efuse_shadow_addr;
2553 int err;
2554
2555 efuse_shadow_addr = 0;
2556 if (FW_VERSION_MAJOR(sc) >= 2)
2557 efuse_shadow_addr = AQ_READ_REG(sc, FW2X_MPI_EFUSEADDR_REG);
2558 else
2559 efuse_shadow_addr = AQ_READ_REG(sc, FW1X_MPI_EFUSEADDR_REG);
2560
2561 if (efuse_shadow_addr == 0) {
2562 aprint_error_dev(sc->sc_dev, "cannot get efuse addr\n");
2563 return ENXIO;
2564 }
2565
2566 memset(mac_addr, 0, sizeof(mac_addr));
2567 err = aq_fw_downld_dwords(sc, efuse_shadow_addr + (40 * 4),
2568 mac_addr, __arraycount(mac_addr));
2569 if (err < 0)
2570 return err;
2571
2572 if (mac_addr[0] == 0 && mac_addr[1] == 0) {
2573 aprint_error_dev(sc->sc_dev, "mac address not found\n");
2574 return ENXIO;
2575 }
2576
2577 mac_addr[0] = bswap32(mac_addr[0]);
2578 mac_addr[1] = bswap32(mac_addr[1]);
2579
2580 memcpy(sc->sc_enaddr.ether_addr_octet,
2581 (uint8_t *)mac_addr, ETHER_ADDR_LEN);
2582 aprint_normal_dev(sc->sc_dev, "Etheraddr: %s\n",
2583 ether_sprintf(sc->sc_enaddr.ether_addr_octet));
2584
2585 return 0;
2586 }
2587
2588 /* set multicast filter. index 0 for own address */
2589 static int
2590 aq_set_mac_addr(struct aq_softc *sc, int index, uint8_t *enaddr)
2591 {
2592 uint32_t h, l;
2593
2594 if (index >= AQ_HW_MAC_NUM)
2595 return EINVAL;
2596
2597 if (enaddr == NULL) {
2598 /* disable */
2599 AQ_WRITE_REG_BIT(sc,
2600 RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
2601 return 0;
2602 }
2603
2604 h = (enaddr[0] << 8) | (enaddr[1]);
2605 l = ((uint32_t)enaddr[2] << 24) | (enaddr[3] << 16) |
2606 (enaddr[4] << 8) | (enaddr[5]);
2607
2608 /* disable, set, and enable */
2609 AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
2610 AQ_WRITE_REG(sc, RPF_L2UC_LSW_REG(index), l);
2611 AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index),
2612 RPF_L2UC_MSW_MACADDR_HI, h);
2613 AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_ACTION, 1);
2614 AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 1);
2615
2616 return 0;
2617 }
2618
2619 static int
2620 aq_set_capability(struct aq_softc *sc)
2621 {
2622 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2623 int ip4csum_tx =
2624 ((ifp->if_capenable & IFCAP_CSUM_IPv4_Tx) == 0) ? 0 : 1;
2625 int ip4csum_rx =
2626 ((ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) == 0) ? 0 : 1;
2627 int l4csum_tx = ((ifp->if_capenable &
2628 (IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx |
2629 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_UDPv6_Tx)) == 0) ? 0 : 1;
2630 int l4csum_rx =
2631 ((ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
2632 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) == 0) ? 0 : 1;
2633 uint32_t lso =
2634 ((ifp->if_capenable & (IFCAP_TSOv4 | IFCAP_TSOv6)) == 0) ?
2635 0 : 0xffffffff;
2636 uint32_t lro = ((ifp->if_capenable & IFCAP_LRO) == 0) ?
2637 0 : 0xffffffff;
2638 uint32_t i, v;
2639
2640 /* TX checksums offloads*/
2641 AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_IP4CSUM_EN, ip4csum_tx);
2642 AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_L4CSUM_EN, l4csum_tx);
2643
2644 /* RX checksums offloads*/
2645 AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_IP4CSUM_EN, ip4csum_rx);
2646 AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_L4CSUM_EN, l4csum_rx);
2647
2648 /* LSO offloads*/
2649 AQ_WRITE_REG(sc, TDM_LSO_EN_REG, lso);
2650
2651 #define AQ_B0_LRO_RXD_MAX 16
2652 v = (8 < AQ_B0_LRO_RXD_MAX) ? 3 :
2653 (4 < AQ_B0_LRO_RXD_MAX) ? 2 :
2654 (2 < AQ_B0_LRO_RXD_MAX) ? 1 : 0;
2655 for (i = 0; i < AQ_RINGS_NUM; i++) {
2656 AQ_WRITE_REG_BIT(sc, RPO_LRO_LDES_MAX_REG(i),
2657 RPO_LRO_LDES_MAX_MASK(i), v);
2658 }
2659
2660 AQ_WRITE_REG_BIT(sc, RPO_LRO_TB_DIV_REG, RPO_LRO_TB_DIV, 0x61a);
2661 AQ_WRITE_REG_BIT(sc, RPO_LRO_INACTIVE_IVAL_REG,
2662 RPO_LRO_INACTIVE_IVAL, 0);
2663 /*
2664 * the LRO timebase divider is 5 uS (0x61a),
2665 * to get a maximum coalescing interval of 250 uS,
2666 * we need to multiply by 50(0x32) to get
2667 * the default value 250 uS
2668 */
2669 AQ_WRITE_REG_BIT(sc, RPO_LRO_MAX_COALESCING_IVAL_REG,
2670 RPO_LRO_MAX_COALESCING_IVAL, 50);
2671 AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
2672 RPO_LRO_CONF_QSESSION_LIMIT, 1);
2673 AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
2674 RPO_LRO_CONF_TOTAL_DESC_LIMIT, 2);
2675 AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
2676 RPO_LRO_CONF_PATCHOPTIMIZATION_EN, 0);
2677 AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
2678 RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT, 10);
2679 AQ_WRITE_REG(sc, RPO_LRO_RSC_MAX_REG, 1);
2680 AQ_WRITE_REG(sc, RPO_LRO_ENABLE_REG, lro);
2681
2682 return 0;
2683 }
2684
2685 static int
2686 aq_set_filter(struct aq_softc *sc)
2687 {
2688 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2689 struct ethercom *ec = &sc->sc_ethercom;
2690 struct ether_multi *enm;
2691 struct ether_multistep step;
2692 int idx, error = 0;
2693
2694 if (ifp->if_flags & IFF_PROMISC) {
2695 AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_PROMISC,
2696 (ifp->if_flags & IFF_PROMISC) ? 1 : 0);
2697 ec->ec_flags |= ETHER_F_ALLMULTI;
2698 goto done;
2699 }
2700
2701 /* clear all table */
2702 for (idx = 0; idx < AQ_HW_MAC_NUM; idx++) {
2703 if (idx == AQ_HW_MAC_OWN) /* already used for own */
2704 continue;
2705 aq_set_mac_addr(sc, idx, NULL);
2706 }
2707
2708 /* don't accept all multicast */
2709 AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
2710 RPF_MCAST_FILTER_MASK_ALLMULTI, 0);
2711 AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
2712 RPF_MCAST_FILTER_EN, 0);
2713
2714 idx = 0;
2715 ETHER_LOCK(ec);
2716 ETHER_FIRST_MULTI(step, ec, enm);
2717 while (enm != NULL) {
2718 if (idx == AQ_HW_MAC_OWN)
2719 idx++;
2720
2721 if ((idx >= AQ_HW_MAC_NUM) ||
2722 memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
2723 /*
2724 * too many filters.
2725 * fallback to accept all multicast addresses.
2726 */
2727 AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
2728 RPF_MCAST_FILTER_MASK_ALLMULTI, 1);
2729 AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
2730 RPF_MCAST_FILTER_EN, 1);
2731 ec->ec_flags |= ETHER_F_ALLMULTI;
2732 ETHER_UNLOCK(ec);
2733 goto done;
2734 }
2735
2736 /* add a filter */
2737 aq_set_mac_addr(sc, idx++, enm->enm_addrlo);
2738
2739 ETHER_NEXT_MULTI(step, enm);
2740 }
2741 ec->ec_flags &= ~ETHER_F_ALLMULTI;
2742 ETHER_UNLOCK(ec);
2743
2744 done:
2745 return error;
2746 }
2747
2748 static int
2749 aq_ifmedia_change(struct ifnet * const ifp)
2750 {
2751 struct aq_softc *sc = ifp->if_softc;
2752 aq_link_speed_t rate = AQ_LINK_NONE;
2753 aq_link_fc_t fc = AQ_FC_NONE;
2754 aq_link_eee_t eee = AQ_EEE_DISABLE;
2755
2756 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
2757 return EINVAL;
2758
2759 switch (IFM_SUBTYPE(sc->sc_media.ifm_media)) {
2760 case IFM_AUTO:
2761 rate = AQ_LINK_AUTO;
2762 break;
2763 case IFM_NONE:
2764 rate = AQ_LINK_NONE;
2765 break;
2766 case IFM_100_TX:
2767 rate = AQ_LINK_100M;
2768 break;
2769 case IFM_1000_T:
2770 rate = AQ_LINK_1G;
2771 break;
2772 case IFM_2500_T:
2773 rate = AQ_LINK_2G5;
2774 break;
2775 case IFM_5000_T:
2776 rate = AQ_LINK_5G;
2777 break;
2778 case IFM_10G_T:
2779 rate = AQ_LINK_10G;
2780 break;
2781 default:
2782 device_printf(sc->sc_dev, "unknown media: 0x%X\n",
2783 IFM_SUBTYPE(sc->sc_media.ifm_media));
2784 return ENODEV;
2785 }
2786
2787 if (sc->sc_media.ifm_media & IFM_FLOW)
2788 fc = AQ_FC_ALL;
2789
2790 /* XXX: todo EEE */
2791
2792 /* re-initialize hardware with new parameters */
2793 aq_set_linkmode(sc, rate, fc, eee);
2794
2795 return 0;
2796 }
2797
2798 static void
2799 aq_ifmedia_status(struct ifnet * const ifp, struct ifmediareq *ifmr)
2800 {
2801 struct aq_softc *sc = ifp->if_softc;
2802
2803 /* update ifm_active */
2804 ifmr->ifm_active = IFM_ETHER;
2805 if (sc->sc_link_fc & AQ_FC_RX)
2806 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
2807 if (sc->sc_link_fc & AQ_FC_TX)
2808 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
2809
2810 switch (sc->sc_link_rate) {
2811 case AQ_LINK_100M:
2812 /* XXX: need to detect fulldup or halfdup */
2813 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
2814 break;
2815 case AQ_LINK_1G:
2816 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
2817 break;
2818 case AQ_LINK_2G5:
2819 ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
2820 break;
2821 case AQ_LINK_5G:
2822 ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
2823 break;
2824 case AQ_LINK_10G:
2825 ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
2826 break;
2827 default:
2828 ifmr->ifm_active |= IFM_NONE;
2829 break;
2830 }
2831
2832 /* update ifm_status */
2833 ifmr->ifm_status = IFM_AVALID;
2834 if (sc->sc_link_rate != AQ_LINK_NONE)
2835 ifmr->ifm_status |= IFM_ACTIVE;
2836 }
2837
2838 static void
2839 aq_initmedia(struct aq_softc *sc)
2840 {
2841 #define IFMEDIA_ETHER_ADD(sc, media) \
2842 ifmedia_add(&(sc)->sc_media, IFM_ETHER | media, 0, NULL);
2843
2844 IFMEDIA_ETHER_ADD(sc, IFM_NONE);
2845 if (sc->sc_available_rates & AQ_LINK_100M) {
2846 IFMEDIA_ETHER_ADD(sc, IFM_100_TX);
2847 IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FLOW);
2848 IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FDX | IFM_FLOW);
2849 }
2850 if (sc->sc_available_rates & AQ_LINK_1G) {
2851 IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX);
2852 IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX | IFM_FLOW);
2853 }
2854 if (sc->sc_available_rates & AQ_LINK_2G5) {
2855 IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX);
2856 IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX | IFM_FLOW);
2857 }
2858 if (sc->sc_available_rates & AQ_LINK_5G) {
2859 IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX);
2860 IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX | IFM_FLOW);
2861 }
2862 if (sc->sc_available_rates & AQ_LINK_10G) {
2863 IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX);
2864 IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX | IFM_FLOW);
2865 }
2866 IFMEDIA_ETHER_ADD(sc, IFM_AUTO);
2867 IFMEDIA_ETHER_ADD(sc, IFM_AUTO | IFM_FLOW);
2868
2869 /* default: auto without flowcontrol */
2870 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
2871 aq_set_linkmode(sc, AQ_LINK_AUTO, AQ_FC_NONE, AQ_EEE_DISABLE);
2872 }
2873
2874 static int
2875 aq_set_linkmode(struct aq_softc *sc, aq_link_speed_t speed, aq_link_fc_t fc,
2876 aq_link_eee_t eee)
2877 {
2878 return sc->sc_fw_ops->set_mode(sc, MPI_INIT, speed, fc, eee);
2879 }
2880
2881 static int
2882 aq_get_linkmode(struct aq_softc *sc, aq_link_speed_t *speed, aq_link_fc_t *fc,
2883 aq_link_eee_t *eee)
2884 {
2885 aq_hw_fw_mpi_state_t mode;
2886 int error;
2887
2888 error = sc->sc_fw_ops->get_mode(sc, &mode, speed, fc, eee);
2889 if (error != 0)
2890 return error;
2891 if (mode != MPI_INIT)
2892 return ENXIO;
2893
2894 return 0;
2895 }
2896
2897 static void
2898 aq_hw_init_tx_path(struct aq_softc *sc)
2899 {
2900 /* Tx TC/RSS number config */
2901 AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_TC_MODE_EN, 1);
2902
2903 AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
2904 THM_LSO_TCP_FLAG1_FIRST, 0x0ff6);
2905 AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
2906 THM_LSO_TCP_FLAG1_MID, 0x0ff6);
2907 AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG2_REG,
2908 THM_LSO_TCP_FLAG2_LAST, 0x0f7f);
2909
2910 /* misc */
2911 AQ_WRITE_REG(sc, TX_TPO2_REG,
2912 (sc->sc_features & FEATURES_TPO2) ? TX_TPO2_EN : 0);
2913 AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_EN, 0);
2914 AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_MODE, 0);
2915
2916 AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_SCP_INS_EN, 1);
2917 }
2918
2919 static void
2920 aq_hw_init_rx_path(struct aq_softc *sc)
2921 {
2922 int i;
2923
2924 /* clear setting */
2925 AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 0);
2926 AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 0);
2927 AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG, 0);
2928 for (i = 0; i < 32; i++) {
2929 AQ_WRITE_REG_BIT(sc, RPF_ETHERTYPE_FILTER_REG(i),
2930 RPF_ETHERTYPE_FILTER_EN, 0);
2931 }
2932
2933 if (sc->sc_rss_enable) {
2934 /* Rx TC/RSS number config */
2935 AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 1);
2936
2937 /* Rx flow control */
2938 AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 1);
2939
2940 /* RSS Ring selection */
2941 switch (sc->sc_nqueues) {
2942 case 2:
2943 AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
2944 RX_FLR_RSS_CONTROL1_EN | 0x11111111);
2945 break;
2946 case 4:
2947 AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
2948 RX_FLR_RSS_CONTROL1_EN | 0x22222222);
2949 break;
2950 case 8:
2951 AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
2952 RX_FLR_RSS_CONTROL1_EN | 0x33333333);
2953 break;
2954 }
2955 }
2956
2957 /* L2 and Multicast filters */
2958 for (i = 0; i < AQ_HW_MAC_NUM; i++) {
2959 AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_EN, 0);
2960 AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_ACTION,
2961 RPF_ACTION_HOST);
2962 }
2963 AQ_WRITE_REG(sc, RPF_MCAST_FILTER_MASK_REG, 0);
2964 AQ_WRITE_REG(sc, RPF_MCAST_FILTER_REG(0), 0x00010fff);
2965
2966 /* Vlan filters */
2967 AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_OUTER,
2968 ETHERTYPE_QINQ);
2969 AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_INNER,
2970 ETHERTYPE_VLAN);
2971 AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);
2972
2973 if (sc->sc_features & FEATURES_REV_B) {
2974 AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
2975 RPF_VLAN_MODE_ACCEPT_UNTAGGED, 1);
2976 AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
2977 RPF_VLAN_MODE_UNTAGGED_ACTION, RPF_ACTION_HOST);
2978 }
2979
2980 /* misc */
2981 if (sc->sc_features & FEATURES_RPF2)
2982 AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, RX_TCP_RSS_HASH_RPF2);
2983 else
2984 AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, 0);
2985
2986 /*
2987 * XXX: RX_TCP_RSS_HASH_REG:
2988 * linux set 0x000f0000
2989 * freebsd set 0x000f001e
2990 */
2991 /* RSS hash type set for IP/TCP */
2992 AQ_WRITE_REG_BIT(sc, RX_TCP_RSS_HASH_REG,
2993 RX_TCP_RSS_HASH_TYPE, 0x001e);
2994
2995 AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_EN, 1);
2996 AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_ACTION, RPF_ACTION_HOST);
2997 AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_THRESHOLD, 0xffff);
2998
2999 AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_EN, 0);
3000 AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_MODE, 0);
3001 }
3002
3003 static void
3004 aq_hw_interrupt_moderation_set(struct aq_softc *sc)
3005 {
3006 int i;
3007
3008 if (sc->sc_intr_moderation_enable) {
3009 unsigned int tx_min, rx_min; /* 0-255 */
3010 unsigned int tx_max, rx_max; /* 0-511? */
3011
3012 switch (sc->sc_link_rate) {
3013 case AQ_LINK_100M:
3014 tx_min = 0x4f;
3015 tx_max = 0xff;
3016 rx_min = 0x04;
3017 rx_max = 0x50;
3018 break;
3019 case AQ_LINK_1G:
3020 default:
3021 tx_min = 0x4f;
3022 tx_max = 0xff;
3023 rx_min = 0x30;
3024 rx_max = 0x80;
3025 break;
3026 case AQ_LINK_2G5:
3027 tx_min = 0x4f;
3028 tx_max = 0xff;
3029 rx_min = 0x18;
3030 rx_max = 0xe0;
3031 break;
3032 case AQ_LINK_5G:
3033 tx_min = 0x4f;
3034 tx_max = 0xff;
3035 rx_min = 0x0c;
3036 rx_max = 0x70;
3037 break;
3038 case AQ_LINK_10G:
3039 tx_min = 0x4f;
3040 tx_max = 0x1ff;
3041 rx_min = 0x06; /* freebsd use 80 */
3042 rx_max = 0x38; /* freebsd use 120 */
3043 break;
3044 }
3045
3046 AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
3047 TX_DMA_INT_DESC_WRWB_EN, 0);
3048 AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
3049 TX_DMA_INT_DESC_MODERATE_EN, 1);
3050 AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
3051 RX_DMA_INT_DESC_WRWB_EN, 0);
3052 AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
3053 RX_DMA_INT_DESC_MODERATE_EN, 1);
3054
3055 for (i = 0; i < AQ_RINGS_NUM; i++) {
3056 AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i),
3057 __SHIFTIN(tx_min, TX_INTR_MODERATION_CTL_MIN) |
3058 __SHIFTIN(tx_max, TX_INTR_MODERATION_CTL_MAX) |
3059 TX_INTR_MODERATION_CTL_EN);
3060 }
3061 for (i = 0; i < AQ_RINGS_NUM; i++) {
3062 AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i),
3063 __SHIFTIN(rx_min, RX_INTR_MODERATION_CTL_MIN) |
3064 __SHIFTIN(rx_max, RX_INTR_MODERATION_CTL_MAX) |
3065 RX_INTR_MODERATION_CTL_EN);
3066 }
3067
3068 } else {
3069 AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
3070 TX_DMA_INT_DESC_WRWB_EN, 1);
3071 AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
3072 TX_DMA_INT_DESC_MODERATE_EN, 0);
3073 AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
3074 RX_DMA_INT_DESC_WRWB_EN, 1);
3075 AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
3076 RX_DMA_INT_DESC_MODERATE_EN, 0);
3077
3078 for (i = 0; i < AQ_RINGS_NUM; i++) {
3079 AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i), 0);
3080 }
3081 for (i = 0; i < AQ_RINGS_NUM; i++) {
3082 AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i), 0);
3083 }
3084 }
3085 }
3086
3087 static void
3088 aq_hw_qos_set(struct aq_softc *sc)
3089 {
3090 uint32_t tc = 0;
3091 uint32_t buff_size;
3092
3093 /* TPS Descriptor rate init */
3094 AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_TA_RST, 0);
3095 AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_LIM, 0xa);
3096
3097 /* TPS VM init */
3098 AQ_WRITE_REG_BIT(sc, TPS_DESC_VM_ARB_MODE_REG, TPS_DESC_VM_ARB_MODE, 0);
3099
3100 /* TPS TC credits init */
3101 AQ_WRITE_REG_BIT(sc, TPS_DESC_TC_ARB_MODE_REG, TPS_DESC_TC_ARB_MODE, 0);
3102 AQ_WRITE_REG_BIT(sc, TPS_DATA_TC_ARB_MODE_REG, TPS_DATA_TC_ARB_MODE, 0);
3103
3104 AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
3105 TPS_DATA_TCT_CREDIT_MAX, 0xfff);
3106 AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
3107 TPS_DATA_TCT_WEIGHT, 0x64);
3108 AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
3109 TPS_DESC_TCT_CREDIT_MAX, 0x50);
3110 AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
3111 TPS_DESC_TCT_WEIGHT, 0x1e);
3112
3113 /* Tx buf size */
3114 tc = 0;
3115 buff_size = AQ_HW_TXBUF_MAX;
3116 AQ_WRITE_REG_BIT(sc, TPB_TXB_BUFSIZE_REG(tc), TPB_TXB_BUFSIZE,
3117 buff_size);
3118 AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_HI,
3119 (buff_size * (1024 / 32) * 66) / 100);
3120 AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_LO,
3121 (buff_size * (1024 / 32) * 50) / 100);
3122
3123 /* QoS Rx buf size per TC */
3124 tc = 0;
3125 buff_size = AQ_HW_RXBUF_MAX;
3126 AQ_WRITE_REG_BIT(sc, RPB_RXB_BUFSIZE_REG(tc), RPB_RXB_BUFSIZE,
3127 buff_size);
3128 AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_EN, 0);
3129 AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_HI,
3130 (buff_size * (1024 / 32) * 66) / 100);
3131 AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_LO,
3132 (buff_size * (1024 / 32) * 50) / 100);
3133
3134 /* QoS 802.1p priority -> TC mapping */
3135 int i_priority;
3136 for (i_priority = 0; i_priority < 8; i_priority++) {
3137 AQ_WRITE_REG_BIT(sc, RPF_RPB_RX_TC_UPT_REG,
3138 RPF_RPB_RX_TC_UPT_MASK(i_priority), 0);
3139 }
3140 }
3141
3142 /* called once from aq_attach */
3143 static int
3144 aq_init_rss(struct aq_softc *sc)
3145 {
3146 CTASSERT(AQ_RSS_HASHKEY_SIZE == RSS_KEYSIZE);
3147 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
3148 uint8_t rss_table[AQ_RSS_INDIRECTION_TABLE_MAX];
3149 unsigned int i;
3150 int error;
3151
3152 /* initialize rss key */
3153 rss_getkey((uint8_t *)rss_key);
3154
3155 /* hash to ring table */
3156 for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
3157 rss_table[i] = i % sc->sc_nqueues;
3158 }
3159
3160 /*
3161 * set rss key
3162 */
3163 for (i = 0; i < __arraycount(rss_key); i++) {
3164 uint32_t key_data = sc->sc_rss_enable ? ntohl(rss_key[i]) : 0;
3165 AQ_WRITE_REG(sc, RPF_RSS_KEY_WR_DATA_REG, key_data);
3166 AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
3167 RPF_RSS_KEY_ADDR, __arraycount(rss_key) - 1 - i);
3168 AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
3169 RPF_RSS_KEY_WR_EN, 1);
3170 WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
3171 RPF_RSS_KEY_WR_EN) == 0, 1000, 10, &error);
3172 if (error != 0) {
3173 device_printf(sc->sc_dev, "%s: rss key write timeout\n",
3174 __func__);
3175 goto rss_set_timeout;
3176 }
3177 }
3178
3179 /*
3180 * set rss indirection table
3181 *
3182 * AQ's rss redirect table is consist of 3bit*64 (192bit) packed array.
3183 * we'll make it by __BITMAP(3) macros.
3184 */
3185 __BITMAP_TYPE(, uint16_t, 3 * AQ_RSS_INDIRECTION_TABLE_MAX) bit3x64;
3186 __BITMAP_ZERO(&bit3x64);
3187
3188 #define AQ_3BIT_PACKED_ARRAY_SET(bitmap, idx, val) \
3189 do { \
3190 if (val & 1) { \
3191 __BITMAP_SET((idx) * 3, (bitmap)); \
3192 } else { \
3193 __BITMAP_CLR((idx) * 3, (bitmap)); \
3194 } \
3195 if (val & 2) { \
3196 __BITMAP_SET((idx) * 3 + 1, (bitmap)); \
3197 } else { \
3198 __BITMAP_CLR((idx) * 3 + 1, (bitmap)); \
3199 } \
3200 if (val & 4) { \
3201 __BITMAP_SET((idx) * 3 + 2, (bitmap)); \
3202 } else { \
3203 __BITMAP_CLR((idx) * 3 + 2, (bitmap)); \
3204 } \
3205 } while (0 /* CONSTCOND */)
3206
3207 for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
3208 AQ_3BIT_PACKED_ARRAY_SET(&bit3x64, i, rss_table[i]);
3209 }
3210
3211 /* write 192bit data in steps of 16bit */
3212 for (i = 0; i < (int)__arraycount(bit3x64._b); i++) {
3213 AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_WR_DATA_REG,
3214 RPF_RSS_REDIR_WR_DATA, bit3x64._b[i]);
3215 AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
3216 RPF_RSS_REDIR_ADDR, i);
3217 AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
3218 RPF_RSS_REDIR_WR_EN, 1);
3219
3220 WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
3221 RPF_RSS_REDIR_WR_EN) == 0, 1000, 10, &error);
3222 if (error != 0)
3223 break;
3224 }
3225
3226 rss_set_timeout:
3227 return error;
3228 }
3229
3230 static void
3231 aq_hw_l3_filter_set(struct aq_softc *sc)
3232 {
3233 int i;
3234
3235 /* clear all filter */
3236 for (i = 0; i < 8; i++) {
3237 AQ_WRITE_REG_BIT(sc, RPF_L3_FILTER_REG(i),
3238 RPF_L3_FILTER_L4_EN, 0);
3239 }
3240 }
3241
3242 static void
3243 aq_set_vlan_filters(struct aq_softc *sc)
3244 {
3245 struct ethercom *ec = &sc->sc_ethercom;
3246 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
3247 struct vlanid_list *vlanidp;
3248 int i;
3249
3250 ETHER_LOCK(ec);
3251
3252 /* disable all vlan filters */
3253 for (i = 0; i < RPF_VLAN_MAX_FILTERS; i++)
3254 AQ_WRITE_REG(sc, RPF_VLAN_FILTER_REG(i), 0);
3255
3256 /* count VID */
3257 i = 0;
3258 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list)
3259 i++;
3260
3261 if (((sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWFILTER) == 0) ||
3262 (ifp->if_flags & IFF_PROMISC) ||
3263 (i > RPF_VLAN_MAX_FILTERS)) {
3264 /*
3265 * no vlan hwfilter, in promiscuous mode, or too many VID?
3266 * must receive all VID
3267 */
3268 AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
3269 RPF_VLAN_MODE_PROMISC, 1);
3270 goto done;
3271 }
3272
3273 /* receive only selected VID */
3274 AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);
3275 i = 0;
3276 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
3277 AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
3278 RPF_VLAN_FILTER_EN, 1);
3279 AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
3280 RPF_VLAN_FILTER_RXQ_EN, 0);
3281 AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
3282 RPF_VLAN_FILTER_RXQ, 0);
3283 AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
3284 RPF_VLAN_FILTER_ACTION, RPF_ACTION_HOST);
3285 AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
3286 RPF_VLAN_FILTER_ID, vlanidp->vid);
3287 i++;
3288 }
3289
3290 done:
3291 ETHER_UNLOCK(ec);
3292 }
3293
3294 static int
3295 aq_hw_init(struct aq_softc *sc)
3296 {
3297 uint32_t v;
3298
3299 /* Force limit MRRS on RDM/TDM to 2K */
3300 v = AQ_READ_REG(sc, AQ_PCI_REG_CONTROL_6_REG);
3301 AQ_WRITE_REG(sc, AQ_PCI_REG_CONTROL_6_REG, (v & ~0x0707) | 0x0404);
3302
3303 /*
3304 * TX DMA total request limit. B0 hardware is not capable to
3305 * handle more than (8K-MRRS) incoming DMA data.
3306 * Value 24 in 256byte units
3307 */
3308 AQ_WRITE_REG(sc, AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG, 24);
3309
3310 aq_hw_init_tx_path(sc);
3311 aq_hw_init_rx_path(sc);
3312
3313 aq_hw_interrupt_moderation_set(sc);
3314
3315 aq_set_mac_addr(sc, AQ_HW_MAC_OWN, sc->sc_enaddr.ether_addr_octet);
3316 aq_set_linkmode(sc, AQ_LINK_NONE, AQ_FC_NONE, AQ_EEE_DISABLE);
3317
3318 aq_hw_qos_set(sc);
3319
3320 /* Enable interrupt */
3321 int irqmode;
3322 if (sc->sc_msix)
3323 irqmode = AQ_INTR_CTRL_IRQMODE_MSIX;
3324 else
3325 irqmode = AQ_INTR_CTRL_IRQMODE_MSI;
3326
3327 AQ_WRITE_REG(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS);
3328 AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_MULTIVEC,
3329 sc->sc_msix ? 1 : 0);
3330 AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_IRQMODE, irqmode);
3331
3332 AQ_WRITE_REG(sc, AQ_INTR_AUTOMASK_REG, 0xffffffff);
3333
3334 AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(0),
3335 ((AQ_B0_ERR_INT << 24) | (1U << 31)) |
3336 ((AQ_B0_ERR_INT << 16) | (1 << 23))
3337 );
3338
3339 /* link interrupt */
3340 if (!sc->sc_msix)
3341 sc->sc_linkstat_irq = AQ_LINKSTAT_IRQ;
3342 AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(3),
3343 __BIT(7) | sc->sc_linkstat_irq);
3344
3345 return 0;
3346 }
3347
3348 static int
3349 aq_update_link_status(struct aq_softc *sc)
3350 {
3351 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
3352 aq_link_speed_t rate = AQ_LINK_NONE;
3353 aq_link_fc_t fc = AQ_FC_NONE;
3354 aq_link_eee_t eee = AQ_EEE_DISABLE;
3355 unsigned int speed;
3356 int changed = 0;
3357
3358 aq_get_linkmode(sc, &rate, &fc, &eee);
3359
3360 if (sc->sc_link_rate != rate)
3361 changed = 1;
3362 if (sc->sc_link_fc != fc)
3363 changed = 1;
3364 if (sc->sc_link_eee != eee)
3365 changed = 1;
3366
3367 if (changed) {
3368 switch (rate) {
3369 case AQ_LINK_100M:
3370 speed = 100;
3371 break;
3372 case AQ_LINK_1G:
3373 speed = 1000;
3374 break;
3375 case AQ_LINK_2G5:
3376 speed = 2500;
3377 break;
3378 case AQ_LINK_5G:
3379 speed = 5000;
3380 break;
3381 case AQ_LINK_10G:
3382 speed = 10000;
3383 break;
3384 case AQ_LINK_NONE:
3385 default:
3386 speed = 0;
3387 break;
3388 }
3389
3390 if (sc->sc_link_rate == AQ_LINK_NONE) {
3391 /* link DOWN -> UP */
3392 device_printf(sc->sc_dev, "link is UP: speed=%u\n",
3393 speed);
3394 if_link_state_change(ifp, LINK_STATE_UP);
3395 } else if (rate == AQ_LINK_NONE) {
3396 /* link UP -> DOWN */
3397 device_printf(sc->sc_dev, "link is DOWN\n");
3398 if_link_state_change(ifp, LINK_STATE_DOWN);
3399 } else {
3400 device_printf(sc->sc_dev,
3401 "link mode changed: speed=%u, fc=0x%x, eee=%x\n",
3402 speed, fc, eee);
3403 }
3404
3405 sc->sc_link_rate = rate;
3406 sc->sc_link_fc = fc;
3407 sc->sc_link_eee = eee;
3408
3409 /* update interrupt timing according to new link speed */
3410 aq_hw_interrupt_moderation_set(sc);
3411 }
3412
3413 return changed;
3414 }
3415
3416 #ifdef AQ_EVENT_COUNTERS
3417 static void
3418 aq_update_statistics(struct aq_softc *sc)
3419 {
3420 int prev = sc->sc_statistics_idx;
3421 int cur = prev ^ 1;
3422
3423 sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[cur]);
3424
3425 /*
3426 * aq's internal statistics counter is 32bit.
3427 * cauculate delta, and add to evcount
3428 */
3429 #define ADD_DELTA(cur, prev, name) \
3430 do { \
3431 uint32_t n; \
3432 n = (uint32_t)(sc->sc_statistics[cur].name - \
3433 sc->sc_statistics[prev].name); \
3434 if (n != 0) { \
3435 AQ_EVCNT_ADD(sc, name, n); \
3436 } \
3437 } while (/*CONSTCOND*/0);
3438
3439 ADD_DELTA(cur, prev, uprc);
3440 ADD_DELTA(cur, prev, mprc);
3441 ADD_DELTA(cur, prev, bprc);
3442 ADD_DELTA(cur, prev, prc);
3443 ADD_DELTA(cur, prev, erpr);
3444 ADD_DELTA(cur, prev, uptc);
3445 ADD_DELTA(cur, prev, mptc);
3446 ADD_DELTA(cur, prev, bptc);
3447 ADD_DELTA(cur, prev, ptc);
3448 ADD_DELTA(cur, prev, erpt);
3449 ADD_DELTA(cur, prev, mbtc);
3450 ADD_DELTA(cur, prev, bbtc);
3451 ADD_DELTA(cur, prev, mbrc);
3452 ADD_DELTA(cur, prev, bbrc);
3453 ADD_DELTA(cur, prev, ubrc);
3454 ADD_DELTA(cur, prev, ubtc);
3455 ADD_DELTA(cur, prev, dpc);
3456 ADD_DELTA(cur, prev, cprc);
3457
3458 sc->sc_statistics_idx = cur;
3459 }
3460 #endif /* AQ_EVENT_COUNTERS */
3461
3462 /* allocate and map one DMA block */
3463 static int
3464 _alloc_dma(struct aq_softc *sc, bus_size_t size, bus_size_t *sizep,
3465 void **addrp, bus_dmamap_t *mapp, bus_dma_segment_t *seg)
3466 {
3467 int nsegs, error;
3468
3469 if ((error = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, seg,
3470 1, &nsegs, 0)) != 0) {
3471 aprint_error_dev(sc->sc_dev,
3472 "unable to allocate DMA buffer, error=%d\n", error);
3473 goto fail_alloc;
3474 }
3475
3476 if ((error = bus_dmamem_map(sc->sc_dmat, seg, 1, size, addrp,
3477 BUS_DMA_COHERENT)) != 0) {
3478 aprint_error_dev(sc->sc_dev,
3479 "unable to map DMA buffer, error=%d\n", error);
3480 goto fail_map;
3481 }
3482
3483 if ((error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
3484 0, mapp)) != 0) {
3485 aprint_error_dev(sc->sc_dev,
3486 "unable to create DMA map, error=%d\n", error);
3487 goto fail_create;
3488 }
3489
3490 if ((error = bus_dmamap_load(sc->sc_dmat, *mapp, *addrp, size, NULL,
3491 0)) != 0) {
3492 aprint_error_dev(sc->sc_dev,
3493 "unable to load DMA map, error=%d\n", error);
3494 goto fail_load;
3495 }
3496
3497 *sizep = size;
3498 return 0;
3499
3500 fail_load:
3501 bus_dmamap_destroy(sc->sc_dmat, *mapp);
3502 *mapp = NULL;
3503 fail_create:
3504 bus_dmamem_unmap(sc->sc_dmat, *addrp, size);
3505 *addrp = NULL;
3506 fail_map:
3507 bus_dmamem_free(sc->sc_dmat, seg, 1);
3508 memset(seg, 0, sizeof(*seg));
3509 fail_alloc:
3510 *sizep = 0;
3511 return error;
3512 }
3513
3514 static void
3515 _free_dma(struct aq_softc *sc, bus_size_t *sizep, void **addrp,
3516 bus_dmamap_t *mapp, bus_dma_segment_t *seg)
3517 {
3518 if (*mapp != NULL) {
3519 bus_dmamap_destroy(sc->sc_dmat, *mapp);
3520 *mapp = NULL;
3521 }
3522 if (*addrp != NULL) {
3523 bus_dmamem_unmap(sc->sc_dmat, *addrp, *sizep);
3524 *addrp = NULL;
3525 }
3526 if (*sizep != 0) {
3527 bus_dmamem_free(sc->sc_dmat, seg, 1);
3528 memset(seg, 0, sizeof(*seg));
3529 *sizep = 0;
3530 }
3531 }
3532
3533 static int
3534 aq_txring_alloc(struct aq_softc *sc, struct aq_txring *txring)
3535 {
3536 int i, error;
3537
3538 /* allocate tx descriptors */
3539 error = _alloc_dma(sc, sizeof(aq_tx_desc_t) * AQ_TXD_NUM,
3540 &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
3541 &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
3542 if (error != 0)
3543 return error;
3544
3545 memset(txring->txr_txdesc, 0, sizeof(aq_tx_desc_t) * AQ_TXD_NUM);
3546
3547 /* fill tx ring with dmamap */
3548 for (i = 0; i < AQ_TXD_NUM; i++) {
3549 #define AQ_MAXDMASIZE (16 * 1024)
3550 #define AQ_NTXSEGS 32
3551 /* XXX: TODO: error check */
3552 bus_dmamap_create(sc->sc_dmat, AQ_MAXDMASIZE, AQ_NTXSEGS,
3553 AQ_MAXDMASIZE, 0, 0, &txring->txr_mbufs[i].dmamap);
3554 }
3555 return 0;
3556 }
3557
3558 static void
3559 aq_txring_free(struct aq_softc *sc, struct aq_txring *txring)
3560 {
3561 int i;
3562
3563 _free_dma(sc, &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
3564 &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
3565
3566 for (i = 0; i < AQ_TXD_NUM; i++) {
3567 if (txring->txr_mbufs[i].dmamap != NULL) {
3568 if (txring->txr_mbufs[i].m != NULL) {
3569 bus_dmamap_unload(sc->sc_dmat,
3570 txring->txr_mbufs[i].dmamap);
3571 m_freem(txring->txr_mbufs[i].m);
3572 txring->txr_mbufs[i].m = NULL;
3573 }
3574 bus_dmamap_destroy(sc->sc_dmat,
3575 txring->txr_mbufs[i].dmamap);
3576 txring->txr_mbufs[i].dmamap = NULL;
3577 }
3578 }
3579 }
3580
3581 static int
3582 aq_rxring_alloc(struct aq_softc *sc, struct aq_rxring *rxring)
3583 {
3584 int i, error;
3585
3586 /* allocate rx descriptors */
3587 error = _alloc_dma(sc, sizeof(aq_rx_desc_t) * AQ_RXD_NUM,
3588 &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
3589 &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
3590 if (error != 0)
3591 return error;
3592
3593 memset(rxring->rxr_rxdesc, 0, sizeof(aq_rx_desc_t) * AQ_RXD_NUM);
3594
3595 /* fill rxring with dmamaps */
3596 for (i = 0; i < AQ_RXD_NUM; i++) {
3597 rxring->rxr_mbufs[i].m = NULL;
3598 /* XXX: TODO: error check */
3599 bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0, 0,
3600 &rxring->rxr_mbufs[i].dmamap);
3601 }
3602 return 0;
3603 }
3604
3605 static void
3606 aq_rxdrain(struct aq_softc *sc, struct aq_rxring *rxring)
3607 {
3608 int i;
3609
3610 /* free all mbufs allocated for RX */
3611 for (i = 0; i < AQ_RXD_NUM; i++) {
3612 if (rxring->rxr_mbufs[i].m != NULL) {
3613 bus_dmamap_unload(sc->sc_dmat,
3614 rxring->rxr_mbufs[i].dmamap);
3615 m_freem(rxring->rxr_mbufs[i].m);
3616 rxring->rxr_mbufs[i].m = NULL;
3617 }
3618 }
3619 }
3620
3621 static void
3622 aq_rxring_free(struct aq_softc *sc, struct aq_rxring *rxring)
3623 {
3624 int i;
3625
3626 /* free all mbufs and dmamaps */
3627 aq_rxdrain(sc, rxring);
3628 for (i = 0; i < AQ_RXD_NUM; i++) {
3629 if (rxring->rxr_mbufs[i].dmamap != NULL) {
3630 bus_dmamap_destroy(sc->sc_dmat,
3631 rxring->rxr_mbufs[i].dmamap);
3632 rxring->rxr_mbufs[i].dmamap = NULL;
3633 }
3634 }
3635
3636 /* free RX descriptor */
3637 _free_dma(sc, &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
3638 &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
3639 }
3640
3641 static void
3642 aq_rxring_setmbuf(struct aq_softc *sc, struct aq_rxring *rxring, int idx,
3643 struct mbuf *m)
3644 {
3645 int error;
3646
3647 /* if mbuf already exists, unload and free */
3648 if (rxring->rxr_mbufs[idx].m != NULL) {
3649 bus_dmamap_unload(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap);
3650 m_freem(rxring->rxr_mbufs[idx].m);
3651 rxring->rxr_mbufs[idx].m = NULL;
3652 }
3653
3654 rxring->rxr_mbufs[idx].m = m;
3655
3656 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3657 error = bus_dmamap_load_mbuf(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap,
3658 m, BUS_DMA_READ | BUS_DMA_NOWAIT);
3659 if (error) {
3660 device_printf(sc->sc_dev,
3661 "unable to load rx DMA map %d, error = %d\n", idx, error);
3662 panic("%s: unable to load rx DMA map. error=%d",
3663 __func__, error);
3664 }
3665 bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
3666 rxring->rxr_mbufs[idx].dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
3667 }
3668
3669 static inline void
3670 aq_rxring_reset_desc(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
3671 {
3672 /* refill rxdesc, and sync */
3673 rxring->rxr_rxdesc[idx].read.buf_addr =
3674 htole64(rxring->rxr_mbufs[idx].dmamap->dm_segs[0].ds_addr);
3675 rxring->rxr_rxdesc[idx].read.hdr_addr = 0;
3676 bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
3677 sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
3678 BUS_DMASYNC_PREWRITE);
3679 }
3680
3681 static struct mbuf *
3682 aq_alloc_mbuf(void)
3683 {
3684 struct mbuf *m;
3685
3686 MGETHDR(m, M_DONTWAIT, MT_DATA);
3687 if (m == NULL)
3688 return NULL;
3689
3690 MCLGET(m, M_DONTWAIT);
3691 if ((m->m_flags & M_EXT) == 0) {
3692 m_freem(m);
3693 return NULL;
3694 }
3695
3696 return m;
3697 }
3698
3699 /* allocate mbuf and unload dmamap */
3700 static int
3701 aq_rxring_add(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
3702 {
3703 struct mbuf *m;
3704
3705 m = aq_alloc_mbuf();
3706 if (m == NULL)
3707 return ENOBUFS;
3708
3709 aq_rxring_setmbuf(sc, rxring, idx, m);
3710 return 0;
3711 }
3712
3713 static int
3714 aq_txrx_rings_alloc(struct aq_softc *sc)
3715 {
3716 int n, error;
3717
3718 for (n = 0; n < sc->sc_nqueues; n++) {
3719 sc->sc_queue[n].sc = sc;
3720 sc->sc_queue[n].txring.txr_sc = sc;
3721 sc->sc_queue[n].txring.txr_index = n;
3722 mutex_init(&sc->sc_queue[n].txring.txr_mutex, MUTEX_DEFAULT,
3723 IPL_NET);
3724 error = aq_txring_alloc(sc, &sc->sc_queue[n].txring);
3725 if (error != 0)
3726 goto failure;
3727
3728 error = aq_tx_pcq_alloc(sc, &sc->sc_queue[n].txring);
3729 if (error != 0)
3730 goto failure;
3731
3732 sc->sc_queue[n].rxring.rxr_sc = sc;
3733 sc->sc_queue[n].rxring.rxr_index = n;
3734 mutex_init(&sc->sc_queue[n].rxring.rxr_mutex, MUTEX_DEFAULT,
3735 IPL_NET);
3736 error = aq_rxring_alloc(sc, &sc->sc_queue[n].rxring);
3737 if (error != 0)
3738 break;
3739 }
3740
3741 failure:
3742 return error;
3743 }
3744
3745 static void
3746 aq_txrx_rings_free(struct aq_softc *sc)
3747 {
3748 int n;
3749
3750 for (n = 0; n < sc->sc_nqueues; n++) {
3751 aq_txring_free(sc, &sc->sc_queue[n].txring);
3752 mutex_destroy(&sc->sc_queue[n].txring.txr_mutex);
3753
3754 aq_tx_pcq_free(sc, &sc->sc_queue[n].txring);
3755
3756 aq_rxring_free(sc, &sc->sc_queue[n].rxring);
3757 mutex_destroy(&sc->sc_queue[n].rxring.rxr_mutex);
3758 }
3759 }
3760
3761 static int
3762 aq_tx_pcq_alloc(struct aq_softc *sc, struct aq_txring *txring)
3763 {
3764 int error = 0;
3765 txring->txr_softint = NULL;
3766
3767 txring->txr_pcq = pcq_create(AQ_TXD_NUM, KM_NOSLEEP);
3768 if (txring->txr_pcq == NULL) {
3769 aprint_error_dev(sc->sc_dev,
3770 "unable to allocate pcq for TXring[%d]\n",
3771 txring->txr_index);
3772 error = ENOMEM;
3773 goto done;
3774 }
3775
3776 txring->txr_softint = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
3777 aq_deferred_transmit, txring);
3778 if (txring->txr_softint == NULL) {
3779 aprint_error_dev(sc->sc_dev,
3780 "unable to establish softint for TXring[%d]\n",
3781 txring->txr_index);
3782 error = ENOENT;
3783 }
3784
3785 done:
3786 return error;
3787 }
3788
3789 static void
3790 aq_tx_pcq_free(struct aq_softc *sc, struct aq_txring *txring)
3791 {
3792 struct mbuf *m;
3793
3794 if (txring->txr_softint != NULL) {
3795 softint_disestablish(txring->txr_softint);
3796 txring->txr_softint = NULL;
3797 }
3798
3799 if (txring->txr_pcq != NULL) {
3800 while ((m = pcq_get(txring->txr_pcq)) != NULL)
3801 m_freem(m);
3802 pcq_destroy(txring->txr_pcq);
3803 txring->txr_pcq = NULL;
3804 }
3805 }
3806
3807 #if NSYSMON_ENVSYS > 0
3808 static void
3809 aq_temp_refresh(struct sysmon_envsys *sme, envsys_data_t *edata)
3810 {
3811 struct aq_softc *sc;
3812 uint32_t temp;
3813 int error;
3814
3815 sc = sme->sme_cookie;
3816
3817 error = sc->sc_fw_ops->get_temperature(sc, &temp);
3818 if (error == 0) {
3819 edata->value_cur = temp;
3820 edata->state = ENVSYS_SVALID;
3821 } else {
3822 edata->state = ENVSYS_SINVALID;
3823 }
3824 }
3825 #endif
3826
3827 static void
3828 aq_tick(void *arg)
3829 {
3830 struct aq_softc *sc = arg;
3831
3832 if (sc->sc_poll_linkstat || sc->sc_detect_linkstat) {
3833 sc->sc_detect_linkstat = false;
3834 aq_update_link_status(sc);
3835 }
3836
3837 #ifdef AQ_EVENT_COUNTERS
3838 if (sc->sc_poll_statistics)
3839 aq_update_statistics(sc);
3840 #endif
3841
3842 if (sc->sc_poll_linkstat
3843 #ifdef AQ_EVENT_COUNTERS
3844 || sc->sc_poll_statistics
3845 #endif
3846 ) {
3847 callout_schedule(&sc->sc_tick_ch, hz);
3848 }
3849 }
3850
3851 /* interrupt enable/disable */
3852 static void
3853 aq_enable_intr(struct aq_softc *sc, bool link, bool txrx)
3854 {
3855 uint32_t imask = 0;
3856 int i;
3857
3858 if (txrx) {
3859 for (i = 0; i < sc->sc_nqueues; i++) {
3860 imask |= __BIT(sc->sc_tx_irq[i]);
3861 imask |= __BIT(sc->sc_rx_irq[i]);
3862 }
3863 }
3864
3865 if (link)
3866 imask |= __BIT(sc->sc_linkstat_irq);
3867
3868 AQ_WRITE_REG(sc, AQ_INTR_MASK_REG, imask);
3869 AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
3870 }
3871
3872 static int
3873 aq_legacy_intr(void *arg)
3874 {
3875 struct aq_softc *sc = arg;
3876 uint32_t status;
3877 int nintr = 0;
3878
3879 status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
3880 AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
3881
3882 if (status & __BIT(sc->sc_linkstat_irq)) {
3883 sc->sc_detect_linkstat = true;
3884 callout_schedule(&sc->sc_tick_ch, 0);
3885 nintr++;
3886 }
3887
3888 if (status & __BIT(sc->sc_rx_irq[0])) {
3889 nintr += aq_rx_intr(&sc->sc_queue[0].rxring);
3890 }
3891
3892 if (status & __BIT(sc->sc_tx_irq[0])) {
3893 nintr += aq_tx_intr(&sc->sc_queue[0].txring);
3894 }
3895
3896 return nintr;
3897 }
3898
3899 static int
3900 aq_txrx_intr(void *arg)
3901 {
3902 struct aq_queue *queue = arg;
3903 struct aq_softc *sc = queue->sc;
3904 struct aq_txring *txring = &queue->txring;
3905 struct aq_rxring *rxring = &queue->rxring;
3906 uint32_t status;
3907 int nintr = 0;
3908 int txringidx, rxringidx, txirq, rxirq;
3909
3910 txringidx = txring->txr_index;
3911 rxringidx = rxring->rxr_index;
3912 txirq = sc->sc_tx_irq[txringidx];
3913 rxirq = sc->sc_rx_irq[rxringidx];
3914
3915 status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
3916 if ((status & (__BIT(txirq) | __BIT(rxirq))) == 0) {
3917 /* stray interrupt? */
3918 return 0;
3919 }
3920
3921 nintr += aq_rx_intr(rxring);
3922 nintr += aq_tx_intr(txring);
3923
3924 return nintr;
3925 }
3926
3927 static int
3928 aq_link_intr(void *arg)
3929 {
3930 struct aq_softc *sc = arg;
3931 uint32_t status;
3932 int nintr = 0;
3933
3934 status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
3935 if (status & __BIT(sc->sc_linkstat_irq)) {
3936 sc->sc_detect_linkstat = true;
3937 callout_schedule(&sc->sc_tick_ch, 0);
3938 AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG,
3939 __BIT(sc->sc_linkstat_irq));
3940 nintr++;
3941 }
3942
3943 return nintr;
3944 }
3945
3946 static void
3947 aq_txring_reset(struct aq_softc *sc, struct aq_txring *txring, bool start)
3948 {
3949 const int ringidx = txring->txr_index;
3950 int i;
3951
3952 mutex_enter(&txring->txr_mutex);
3953
3954 txring->txr_prodidx = 0;
3955 txring->txr_considx = 0;
3956 txring->txr_nfree = AQ_TXD_NUM;
3957 txring->txr_active = false;
3958
3959 /* free mbufs untransmitted */
3960 for (i = 0; i < AQ_TXD_NUM; i++) {
3961 if (txring->txr_mbufs[i].m != NULL) {
3962 m_freem(txring->txr_mbufs[i].m);
3963 txring->txr_mbufs[i].m = NULL;
3964 }
3965 }
3966
3967 /* disable DMA */
3968 AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_EN, 0);
3969
3970 if (start) {
3971 /* TX descriptor physical address */
3972 paddr_t paddr = txring->txr_txdesc_dmamap->dm_segs[0].ds_addr;
3973 AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
3974 AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
3975 (uint32_t)((uint64_t)paddr >> 32));
3976
3977 /* TX descriptor size */
3978 AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_LEN,
3979 AQ_TXD_NUM / 8);
3980
3981 /* reload TAIL pointer */
3982 txring->txr_prodidx = txring->txr_considx =
3983 AQ_READ_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(ringidx));
3984 AQ_WRITE_REG(sc, TX_DMA_DESC_WRWB_THRESH_REG(ringidx), 0);
3985
3986 /* Mapping interrupt vector */
3987 AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
3988 AQ_INTR_IRQ_MAP_TX_IRQMAP(ringidx), sc->sc_tx_irq[ringidx]);
3989 AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
3990 AQ_INTR_IRQ_MAP_TX_EN(ringidx), true);
3991
3992 /* enable DMA */
3993 AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx),
3994 TX_DMA_DESC_EN, 1);
3995
3996 const int cpuid = 0; /* XXX? */
3997 AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
3998 TDM_DCAD_CPUID, cpuid);
3999 AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
4000 TDM_DCAD_CPUID_EN, 0);
4001
4002 txring->txr_active = true;
4003 }
4004
4005 mutex_exit(&txring->txr_mutex);
4006 }
4007
4008 static int
4009 aq_rxring_reset(struct aq_softc *sc, struct aq_rxring *rxring, bool start)
4010 {
4011 const int ringidx = rxring->rxr_index;
4012 int i;
4013 int error = 0;
4014
4015 mutex_enter(&rxring->rxr_mutex);
4016 rxring->rxr_active = false;
4017
4018 /* disable DMA */
4019 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_EN, 0);
4020
4021 /* free all RX mbufs */
4022 aq_rxdrain(sc, rxring);
4023
4024 if (start) {
4025 for (i = 0; i < AQ_RXD_NUM; i++) {
4026 error = aq_rxring_add(sc, rxring, i);
4027 if (error != 0) {
4028 aq_rxdrain(sc, rxring);
4029 return error;
4030 }
4031 aq_rxring_reset_desc(sc, rxring, i);
4032 }
4033
4034 /* RX descriptor physical address */
4035 paddr_t paddr = rxring->rxr_rxdesc_dmamap->dm_segs[0].ds_addr;
4036 AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
4037 AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
4038 (uint32_t)((uint64_t)paddr >> 32));
4039
4040 /* RX descriptor size */
4041 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_LEN,
4042 AQ_RXD_NUM / 8);
4043
4044 /* maximum receive frame size */
4045 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
4046 RX_DMA_DESC_BUFSIZE_DATA, MCLBYTES / 1024);
4047 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
4048 RX_DMA_DESC_BUFSIZE_HDR, 0 / 1024);
4049
4050 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
4051 RX_DMA_DESC_HEADER_SPLIT, 0);
4052 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
4053 RX_DMA_DESC_VLAN_STRIP,
4054 (sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
4055 1 : 0);
4056
4057 /*
4058 * reload TAIL pointer, and update readidx
4059 * (HEAD pointer cannot write)
4060 */
4061 rxring->rxr_readidx = AQ_READ_REG_BIT(sc,
4062 RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR);
4063 AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx),
4064 (rxring->rxr_readidx + AQ_RXD_NUM - 1) % AQ_RXD_NUM);
4065
4066 /* Rx ring set mode */
4067
4068 /* Mapping interrupt vector */
4069 AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
4070 AQ_INTR_IRQ_MAP_RX_IRQMAP(ringidx), sc->sc_rx_irq[ringidx]);
4071 AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
4072 AQ_INTR_IRQ_MAP_RX_EN(ringidx), 1);
4073
4074 const int cpuid = 0; /* XXX? */
4075 AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
4076 RX_DMA_DCAD_CPUID, cpuid);
4077 AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
4078 RX_DMA_DCAD_DESC_EN, 0);
4079 AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
4080 RX_DMA_DCAD_HEADER_EN, 0);
4081 AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
4082 RX_DMA_DCAD_PAYLOAD_EN, 0);
4083
4084 /* enable DMA. start receiving */
4085 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
4086 RX_DMA_DESC_EN, 1);
4087
4088 rxring->rxr_active = true;
4089 }
4090
4091 mutex_exit(&rxring->rxr_mutex);
4092 return error;
4093 }
4094
4095 #define TXRING_NEXTIDX(idx) \
4096 (((idx) >= (AQ_TXD_NUM - 1)) ? 0 : ((idx) + 1))
4097 #define RXRING_NEXTIDX(idx) \
4098 (((idx) >= (AQ_RXD_NUM - 1)) ? 0 : ((idx) + 1))
4099
4100 static int
4101 aq_encap_txring(struct aq_softc *sc, struct aq_txring *txring, struct mbuf **mp)
4102 {
4103 bus_dmamap_t map;
4104 struct mbuf *m = *mp;
4105 uint32_t ctl1, ctl1_ctx, ctl2;
4106 int idx, i, error;
4107
4108 idx = txring->txr_prodidx;
4109 map = txring->txr_mbufs[idx].dmamap;
4110
4111 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
4112 BUS_DMA_WRITE | BUS_DMA_NOWAIT);
4113 if (error == EFBIG) {
4114 struct mbuf *n;
4115 n = m_defrag(m, M_DONTWAIT);
4116 if (n == NULL)
4117 return EFBIG;
4118 /* m_defrag() preserve m */
4119 KASSERT(n == m);
4120 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
4121 BUS_DMA_WRITE | BUS_DMA_NOWAIT);
4122 }
4123 if (error != 0)
4124 return error;
4125
4126 /*
4127 * check spaces of free descriptors.
4128 * +1 is additional descriptor for context (vlan, etc,.)
4129 */
4130 if ((map->dm_nsegs + 1) > txring->txr_nfree) {
4131 device_printf(sc->sc_dev,
4132 "TX: not enough descriptors left %d for %d segs\n",
4133 txring->txr_nfree, map->dm_nsegs + 1);
4134 bus_dmamap_unload(sc->sc_dmat, map);
4135 return ENOBUFS;
4136 }
4137
4138 /* sync dma for mbuf */
4139 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
4140 BUS_DMASYNC_PREWRITE);
4141
4142 ctl1_ctx = 0;
4143 ctl2 = __SHIFTIN(m->m_pkthdr.len, AQ_TXDESC_CTL2_LEN);
4144
4145 if (vlan_has_tag(m)) {
4146 ctl1 = AQ_TXDESC_CTL1_TYPE_TXC;
4147 ctl1 |= __SHIFTIN(vlan_get_tag(m), AQ_TXDESC_CTL1_VID);
4148
4149 ctl1_ctx |= AQ_TXDESC_CTL1_CMD_VLAN;
4150 ctl2 |= AQ_TXDESC_CTL2_CTX_EN;
4151
4152 /* fill context descriptor and forward index */
4153 txring->txr_txdesc[idx].buf_addr = 0;
4154 txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
4155 txring->txr_txdesc[idx].ctl2 = 0;
4156
4157 idx = TXRING_NEXTIDX(idx);
4158 txring->txr_nfree--;
4159 }
4160
4161 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
4162 ctl1_ctx |= AQ_TXDESC_CTL1_CMD_IP4CSUM;
4163 if (m->m_pkthdr.csum_flags &
4164 (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) {
4165 ctl1_ctx |= AQ_TXDESC_CTL1_CMD_L4CSUM;
4166 }
4167
4168 /* fill descriptor(s) */
4169 for (i = 0; i < map->dm_nsegs; i++) {
4170 ctl1 = ctl1_ctx | AQ_TXDESC_CTL1_TYPE_TXD |
4171 __SHIFTIN(map->dm_segs[i].ds_len, AQ_TXDESC_CTL1_BLEN);
4172 ctl1 |= AQ_TXDESC_CTL1_CMD_FCS;
4173
4174 if (i == 0) {
4175 /* remember mbuf of these descriptors */
4176 txring->txr_mbufs[idx].m = m;
4177 } else {
4178 txring->txr_mbufs[idx].m = NULL;
4179 }
4180
4181 if (i == map->dm_nsegs - 1) {
4182 /* last segment, mark an EndOfPacket, and cause intr */
4183 ctl1 |= AQ_TXDESC_CTL1_EOP | AQ_TXDESC_CTL1_CMD_WB;
4184 }
4185
4186 txring->txr_txdesc[idx].buf_addr =
4187 htole64(map->dm_segs[i].ds_addr);
4188 txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
4189 txring->txr_txdesc[idx].ctl2 = htole32(ctl2);
4190
4191 bus_dmamap_sync(sc->sc_dmat, txring->txr_txdesc_dmamap,
4192 sizeof(aq_tx_desc_t) * idx, sizeof(aq_tx_desc_t),
4193 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4194
4195 idx = TXRING_NEXTIDX(idx);
4196 txring->txr_nfree--;
4197 }
4198
4199 txring->txr_prodidx = idx;
4200
4201 return 0;
4202 }
4203
4204 static int
4205 aq_tx_intr(void *arg)
4206 {
4207 struct aq_txring *txring = arg;
4208 struct aq_softc *sc = txring->txr_sc;
4209 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
4210 struct mbuf *m;
4211 const int ringidx = txring->txr_index;
4212 unsigned int idx, hw_head, n = 0;
4213
4214 mutex_enter(&txring->txr_mutex);
4215
4216 if (!txring->txr_active)
4217 goto tx_intr_done;
4218
4219 hw_head = AQ_READ_REG_BIT(sc, TX_DMA_DESC_HEAD_PTR_REG(ringidx),
4220 TX_DMA_DESC_HEAD_PTR);
4221 if (hw_head == txring->txr_considx) {
4222 goto tx_intr_done;
4223 }
4224
4225 for (idx = txring->txr_considx; idx != hw_head;
4226 idx = TXRING_NEXTIDX(idx), n++) {
4227
4228 if ((m = txring->txr_mbufs[idx].m) != NULL) {
4229 bus_dmamap_unload(sc->sc_dmat,
4230 txring->txr_mbufs[idx].dmamap);
4231
4232 txring->txr_opackets++;
4233 txring->txr_obytes += m->m_pkthdr.len;
4234 if (m->m_flags & M_MCAST)
4235 txring->txr_omcasts++;
4236
4237 m_freem(m);
4238 txring->txr_mbufs[idx].m = NULL;
4239 }
4240
4241 txring->txr_nfree++;
4242 }
4243 txring->txr_considx = idx;
4244
4245 if (ringidx == 0 && txring->txr_nfree >= AQ_TXD_MIN)
4246 ifp->if_flags &= ~IFF_OACTIVE;
4247
4248 /* no more pending TX packet, cancel watchdog */
4249 if (txring->txr_nfree >= AQ_TXD_NUM)
4250 ifp->if_timer = 0;
4251
4252 tx_intr_done:
4253 mutex_exit(&txring->txr_mutex);
4254
4255 AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_tx_irq[ringidx]));
4256 return n;
4257 }
4258
4259 static int
4260 aq_rx_intr(void *arg)
4261 {
4262 struct aq_rxring *rxring = arg;
4263 struct aq_softc *sc = rxring->rxr_sc;
4264 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
4265 const int ringidx = rxring->rxr_index;
4266 aq_rx_desc_t *rxd;
4267 struct mbuf *m, *m0, *mprev, *new_m;
4268 uint32_t rxd_type, rxd_hash __unused;
4269 uint16_t rxd_status, rxd_pktlen;
4270 uint16_t rxd_nextdescptr __unused, rxd_vlan __unused;
4271 unsigned int idx, n = 0;
4272
4273 mutex_enter(&rxring->rxr_mutex);
4274
4275 if (!rxring->rxr_active)
4276 goto rx_intr_done;
4277
4278 if (rxring->rxr_readidx == AQ_READ_REG_BIT(sc,
4279 RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR)) {
4280 goto rx_intr_done;
4281 }
4282
4283 m0 = mprev = NULL;
4284 for (idx = rxring->rxr_readidx;
4285 idx != AQ_READ_REG_BIT(sc, RX_DMA_DESC_HEAD_PTR_REG(ringidx),
4286 RX_DMA_DESC_HEAD_PTR); idx = RXRING_NEXTIDX(idx), n++) {
4287
4288 bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
4289 sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
4290 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4291
4292 rxd = &rxring->rxr_rxdesc[idx];
4293 rxd_status = le16toh(rxd->wb.status);
4294
4295 if ((rxd_status & RXDESC_STATUS_DD) == 0)
4296 break; /* not yet done */
4297
4298 rxd_type = le32toh(rxd->wb.type);
4299 rxd_pktlen = le16toh(rxd->wb.pkt_len);
4300 rxd_nextdescptr = le16toh(rxd->wb.next_desc_ptr);
4301 rxd_hash = le32toh(rxd->wb.rss_hash);
4302 rxd_vlan = le16toh(rxd->wb.vlan);
4303
4304 if ((rxd_status & RXDESC_STATUS_MACERR) ||
4305 (rxd_type & RXDESC_TYPE_MAC_DMA_ERR)) {
4306 rxring->rxr_ierrors++;
4307 goto rx_next;
4308 }
4309
4310 bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
4311 rxring->rxr_mbufs[idx].dmamap->dm_mapsize,
4312 BUS_DMASYNC_POSTREAD);
4313 m = rxring->rxr_mbufs[idx].m;
4314
4315 new_m = aq_alloc_mbuf();
4316 if (new_m == NULL) {
4317 /*
4318 * cannot allocate new mbuf.
4319 * discard this packet, and reuse mbuf for next.
4320 */
4321 rxring->rxr_iqdrops++;
4322 goto rx_next;
4323 }
4324 rxring->rxr_mbufs[idx].m = NULL;
4325 aq_rxring_setmbuf(sc, rxring, idx, new_m);
4326
4327 if (m0 == NULL) {
4328 m0 = m;
4329 } else {
4330 if (m->m_flags & M_PKTHDR)
4331 m_remove_pkthdr(m);
4332 mprev->m_next = m;
4333 }
4334 mprev = m;
4335
4336 if ((rxd_status & RXDESC_STATUS_EOP) == 0) {
4337 m->m_len = MCLBYTES;
4338 } else {
4339 /* last buffer */
4340 m->m_len = rxd_pktlen % MCLBYTES;
4341 m0->m_pkthdr.len = rxd_pktlen;
4342 /* VLAN offloading */
4343 if ((sc->sc_ethercom.ec_capenable &
4344 ETHERCAP_VLAN_HWTAGGING) &&
4345 (__SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_VLAN) ||
4346 __SHIFTOUT(rxd_type,
4347 RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE))) {
4348 vlan_set_tag(m0, rxd_vlan);
4349 }
4350
4351 /* Checksum offloading */
4352 unsigned int pkttype_eth =
4353 __SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_ETHER);
4354 if ((ifp->if_capabilities & IFCAP_CSUM_IPv4_Rx) &&
4355 (pkttype_eth == RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
4356 __SHIFTOUT(rxd_type,
4357 RXDESC_TYPE_IPV4_CSUM_CHECKED)) {
4358 m0->m_pkthdr.csum_flags |= M_CSUM_IPv4;
4359 if (__SHIFTOUT(rxd_status,
4360 RXDESC_STATUS_IPV4_CSUM_NG))
4361 m0->m_pkthdr.csum_flags |=
4362 M_CSUM_IPv4_BAD;
4363 }
4364 #if notyet
4365 /*
4366 * XXX: aq always marks BAD for fragmented packet.
4367 * we should peek L3 header, and ignore cksum flags
4368 * if the packet is fragmented.
4369 */
4370 if (__SHIFTOUT(rxd_type,
4371 RXDESC_TYPE_TCPUDP_CSUM_CHECKED)) {
4372 bool checked = false;
4373 unsigned int pkttype_proto =
4374 __SHIFTOUT(rxd_type,
4375 RXDESC_TYPE_PKTTYPE_PROTO);
4376
4377 if (pkttype_proto ==
4378 RXDESC_TYPE_PKTTYPE_PROTO_TCP) {
4379 if ((pkttype_eth ==
4380 RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
4381 (ifp->if_capabilities &
4382 IFCAP_CSUM_TCPv4_Rx)) {
4383 m0->m_pkthdr.csum_flags |=
4384 M_CSUM_TCPv4;
4385 checked = true;
4386 } else if ((pkttype_eth ==
4387 RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
4388 (ifp->if_capabilities &
4389 IFCAP_CSUM_TCPv6_Rx)) {
4390 m0->m_pkthdr.csum_flags |=
4391 M_CSUM_TCPv6;
4392 checked = true;
4393 }
4394 } else if (pkttype_proto ==
4395 RXDESC_TYPE_PKTTYPE_PROTO_UDP) {
4396 if ((pkttype_eth ==
4397 RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
4398 (ifp->if_capabilities &
4399 IFCAP_CSUM_UDPv4_Rx)) {
4400 m0->m_pkthdr.csum_flags |=
4401 M_CSUM_UDPv4;
4402 checked = true;
4403 } else if ((pkttype_eth ==
4404 RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
4405 (ifp->if_capabilities &
4406 IFCAP_CSUM_UDPv6_Rx)) {
4407 m0->m_pkthdr.csum_flags |=
4408 M_CSUM_UDPv6;
4409 checked = true;
4410 }
4411 }
4412 if (checked &&
4413 (__SHIFTOUT(rxd_status,
4414 RXDESC_STATUS_TCPUDP_CSUM_ERROR) ||
4415 !__SHIFTOUT(rxd_status,
4416 RXDESC_STATUS_TCPUDP_CSUM_OK))) {
4417 m0->m_pkthdr.csum_flags |=
4418 M_CSUM_TCP_UDP_BAD;
4419 }
4420 }
4421 #endif
4422 m_set_rcvif(m0, ifp);
4423 rxring->rxr_ipackets++;
4424 rxring->rxr_ibytes += m0->m_pkthdr.len;
4425 if_percpuq_enqueue(ifp->if_percpuq, m0);
4426 m0 = mprev = NULL;
4427 }
4428
4429 rx_next:
4430 aq_rxring_reset_desc(sc, rxring, idx);
4431 AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx), idx);
4432 }
4433 rxring->rxr_readidx = idx;
4434
4435 rx_intr_done:
4436 mutex_exit(&rxring->rxr_mutex);
4437
4438 AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_rx_irq[ringidx]));
4439 return n;
4440 }
4441
4442 static int
4443 aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set)
4444 {
4445 struct ifnet *ifp = &ec->ec_if;
4446 struct aq_softc *sc = ifp->if_softc;
4447
4448 aq_set_vlan_filters(sc);
4449 return 0;
4450 }
4451
4452 static int
4453 aq_ifflags_cb(struct ethercom *ec)
4454 {
4455 struct ifnet *ifp = &ec->ec_if;
4456 struct aq_softc *sc = ifp->if_softc;
4457 int i, ecchange, error = 0;
4458 unsigned short iffchange;
4459
4460 AQ_LOCK(sc);
4461
4462 iffchange = ifp->if_flags ^ sc->sc_if_flags;
4463 if ((iffchange & IFF_PROMISC) != 0)
4464 error = aq_set_filter(sc);
4465
4466 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable;
4467 if (ecchange & ETHERCAP_VLAN_HWTAGGING) {
4468 for (i = 0; i < AQ_RINGS_NUM; i++) {
4469 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(i),
4470 RX_DMA_DESC_VLAN_STRIP,
4471 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
4472 1 : 0);
4473 }
4474 }
4475
4476 /* vlan configuration depends on also interface promiscuous mode */
4477 if ((ecchange & ETHERCAP_VLAN_HWFILTER) || (iffchange & IFF_PROMISC))
4478 aq_set_vlan_filters(sc);
4479
4480 sc->sc_ec_capenable = ec->ec_capenable;
4481 sc->sc_if_flags = ifp->if_flags;
4482
4483 AQ_UNLOCK(sc);
4484
4485 return error;
4486 }
4487
4488 static int
4489 aq_init(struct ifnet *ifp)
4490 {
4491 struct aq_softc *sc = ifp->if_softc;
4492 int i, error = 0;
4493
4494 AQ_LOCK(sc);
4495
4496 aq_set_vlan_filters(sc);
4497 aq_set_capability(sc);
4498
4499 for (i = 0; i < sc->sc_nqueues; i++) {
4500 aq_txring_reset(sc, &sc->sc_queue[i].txring, true);
4501 }
4502
4503 /* invalidate RX descriptor cache */
4504 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
4505 AQ_READ_REG_BIT(sc,
4506 RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
4507
4508 /* start RX */
4509 for (i = 0; i < sc->sc_nqueues; i++) {
4510 error = aq_rxring_reset(sc, &sc->sc_queue[i].rxring, true);
4511 if (error != 0) {
4512 device_printf(sc->sc_dev, "%s: cannot allocate rxbuf\n",
4513 __func__);
4514 goto aq_init_failure;
4515 }
4516 }
4517 aq_init_rss(sc);
4518 aq_hw_l3_filter_set(sc);
4519
4520 /* need to start callout? */
4521 if (sc->sc_poll_linkstat
4522 #ifdef AQ_EVENT_COUNTERS
4523 || sc->sc_poll_statistics
4524 #endif
4525 ) {
4526 callout_schedule(&sc->sc_tick_ch, hz);
4527 }
4528
4529 /* ready */
4530 ifp->if_flags |= IFF_RUNNING;
4531 ifp->if_flags &= ~IFF_OACTIVE;
4532
4533 /* start TX and RX */
4534 aq_enable_intr(sc, true, true);
4535 AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 1);
4536 AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 1);
4537
4538 aq_init_failure:
4539 sc->sc_if_flags = ifp->if_flags;
4540
4541 AQ_UNLOCK(sc);
4542
4543 return error;
4544 }
4545
4546 static void
4547 aq_send_common_locked(struct ifnet *ifp, struct aq_softc *sc,
4548 struct aq_txring *txring, bool is_transmit)
4549 {
4550 struct mbuf *m;
4551 int npkt, error;
4552
4553 if ((ifp->if_flags & IFF_RUNNING) == 0)
4554 return;
4555
4556 for (npkt = 0; ; npkt++) {
4557 if (is_transmit)
4558 m = pcq_peek(txring->txr_pcq);
4559 else
4560 IFQ_POLL(&ifp->if_snd, m);
4561
4562 if (m == NULL)
4563 break;
4564
4565 if (txring->txr_nfree < AQ_TXD_MIN)
4566 break;
4567
4568 if (is_transmit)
4569 pcq_get(txring->txr_pcq);
4570 else
4571 IFQ_DEQUEUE(&ifp->if_snd, m);
4572
4573 error = aq_encap_txring(sc, txring, &m);
4574 if (error != 0) {
4575 /* too many mbuf chains? or not enough descriptors? */
4576 m_freem(m);
4577 txring->txr_oerrors++;
4578 if (txring->txr_index == 0 && error == ENOBUFS)
4579 ifp->if_flags |= IFF_OACTIVE;
4580 break;
4581 }
4582
4583 /* update tail ptr */
4584 AQ_WRITE_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index),
4585 txring->txr_prodidx);
4586
4587 /* Pass the packet to any BPF listeners */
4588 bpf_mtap(ifp, m, BPF_D_OUT);
4589 }
4590
4591 if (txring->txr_index == 0 && txring->txr_nfree < AQ_TXD_MIN)
4592 ifp->if_flags |= IFF_OACTIVE;
4593
4594 if (npkt)
4595 ifp->if_timer = 5;
4596 }
4597
4598 static void
4599 aq_start(struct ifnet *ifp)
4600 {
4601 struct aq_softc *sc;
4602 struct aq_txring *txring;
4603
4604 sc = ifp->if_softc;
4605 txring = &sc->sc_queue[0].txring; /* aq_start() always use TX ring[0] */
4606
4607 mutex_enter(&txring->txr_mutex);
4608 if (txring->txr_active && !ISSET(ifp->if_flags, IFF_OACTIVE))
4609 aq_send_common_locked(ifp, sc, txring, false);
4610 mutex_exit(&txring->txr_mutex);
4611 }
4612
4613 static inline unsigned int
4614 aq_select_txqueue(struct aq_softc *sc, struct mbuf *m)
4615 {
4616 return (cpu_index(curcpu()) % sc->sc_nqueues);
4617 }
4618
4619 static int
4620 aq_transmit(struct ifnet *ifp, struct mbuf *m)
4621 {
4622 struct aq_softc *sc = ifp->if_softc;
4623 struct aq_txring *txring;
4624 int ringidx;
4625
4626 ringidx = aq_select_txqueue(sc, m);
4627 txring = &sc->sc_queue[ringidx].txring;
4628
4629 if (__predict_false(!pcq_put(txring->txr_pcq, m))) {
4630 m_freem(m);
4631 return ENOBUFS;
4632 }
4633
4634 if (mutex_tryenter(&txring->txr_mutex)) {
4635 aq_send_common_locked(ifp, sc, txring, true);
4636 mutex_exit(&txring->txr_mutex);
4637 } else {
4638 kpreempt_disable();
4639 softint_schedule(txring->txr_softint);
4640 kpreempt_enable();
4641 }
4642 return 0;
4643 }
4644
4645 static void
4646 aq_deferred_transmit(void *arg)
4647 {
4648 struct aq_txring *txring = arg;
4649 struct aq_softc *sc = txring->txr_sc;
4650 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
4651
4652 mutex_enter(&txring->txr_mutex);
4653 if (pcq_peek(txring->txr_pcq) != NULL)
4654 aq_send_common_locked(ifp, sc, txring, true);
4655 mutex_exit(&txring->txr_mutex);
4656 }
4657
4658 static void
4659 aq_stop(struct ifnet *ifp, int disable)
4660 {
4661 struct aq_softc *sc = ifp->if_softc;
4662 int i;
4663
4664 AQ_LOCK(sc);
4665
4666 ifp->if_timer = 0;
4667
4668 /* disable tx/rx interrupts */
4669 aq_enable_intr(sc, true, false);
4670
4671 AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 0);
4672 for (i = 0; i < sc->sc_nqueues; i++) {
4673 aq_txring_reset(sc, &sc->sc_queue[i].txring, false);
4674 }
4675
4676 AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 0);
4677 for (i = 0; i < sc->sc_nqueues; i++) {
4678 aq_rxring_reset(sc, &sc->sc_queue[i].rxring, false);
4679 }
4680
4681 /* invalidate RX descriptor cache */
4682 AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
4683 AQ_READ_REG_BIT(sc,
4684 RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
4685
4686 ifp->if_timer = 0;
4687
4688 if (!disable) {
4689 /* when pmf stop, disable link status intr and callout */
4690 aq_enable_intr(sc, false, false);
4691 callout_stop(&sc->sc_tick_ch);
4692 }
4693
4694 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
4695
4696 AQ_UNLOCK(sc);
4697 }
4698
4699 static void
4700 aq_watchdog(struct ifnet *ifp)
4701 {
4702 struct aq_softc *sc = ifp->if_softc;
4703 struct aq_txring *txring;
4704 int n, head, tail;
4705
4706 AQ_LOCK(sc);
4707
4708 device_printf(sc->sc_dev, "%s: INTR_MASK/STATUS = %08x/%08x\n",
4709 __func__, AQ_READ_REG(sc, AQ_INTR_MASK_REG),
4710 AQ_READ_REG(sc, AQ_INTR_STATUS_REG));
4711
4712 for (n = 0; n < sc->sc_nqueues; n++) {
4713 txring = &sc->sc_queue[n].txring;
4714 head = AQ_READ_REG_BIT(sc,
4715 TX_DMA_DESC_HEAD_PTR_REG(txring->txr_index),
4716 TX_DMA_DESC_HEAD_PTR),
4717 tail = AQ_READ_REG(sc,
4718 TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index));
4719
4720 device_printf(sc->sc_dev, "%s: TXring[%d] HEAD/TAIL=%d/%d\n",
4721 __func__, txring->txr_index, head, tail);
4722
4723 aq_tx_intr(txring);
4724 }
4725
4726 AQ_UNLOCK(sc);
4727
4728 aq_init(ifp);
4729 }
4730
4731 static int
4732 aq_ioctl(struct ifnet *ifp, unsigned long cmd, void *data)
4733 {
4734 struct aq_softc *sc __unused;
4735 struct ifreq *ifr __unused;
4736 uint64_t opackets, oerrors, obytes, omcasts;
4737 uint64_t ipackets, ierrors, ibytes, iqdrops;
4738 int error, i, s;
4739
4740 sc = (struct aq_softc *)ifp->if_softc;
4741 ifr = (struct ifreq *)data;
4742 error = 0;
4743
4744 switch (cmd) {
4745 case SIOCGIFDATA:
4746 case SIOCZIFDATA:
4747 opackets = oerrors = obytes = omcasts = 0;
4748 ipackets = ierrors = ibytes = iqdrops = 0;
4749 for (i = 0; i < sc->sc_nqueues; i++) {
4750 struct aq_txring *txring = &sc->sc_queue[i].txring;
4751 mutex_enter(&txring->txr_mutex);
4752 if (cmd == SIOCZIFDATA) {
4753 txring->txr_opackets = 0;
4754 txring->txr_obytes = 0;
4755 txring->txr_omcasts = 0;
4756 txring->txr_oerrors = 0;
4757 } else {
4758 opackets += txring->txr_opackets;
4759 oerrors += txring->txr_oerrors;
4760 obytes += txring->txr_obytes;
4761 omcasts += txring->txr_omcasts;
4762 }
4763 mutex_exit(&txring->txr_mutex);
4764
4765 struct aq_rxring *rxring = &sc->sc_queue[i].rxring;
4766 mutex_enter(&rxring->rxr_mutex);
4767 if (cmd == SIOCZIFDATA) {
4768 rxring->rxr_ipackets = 0;
4769 rxring->rxr_ibytes = 0;
4770 rxring->rxr_ierrors = 0;
4771 rxring->rxr_iqdrops = 0;
4772 } else {
4773 ipackets += rxring->rxr_ipackets;
4774 ierrors += rxring->rxr_ierrors;
4775 ibytes += rxring->rxr_ibytes;
4776 iqdrops += rxring->rxr_iqdrops;
4777 }
4778 mutex_exit(&rxring->rxr_mutex);
4779 }
4780 ifp->if_opackets = opackets;
4781 ifp->if_oerrors = oerrors;
4782 ifp->if_obytes = obytes;
4783 ifp->if_omcasts = omcasts;
4784 ifp->if_ipackets = ipackets;
4785 ifp->if_ierrors = ierrors;
4786 ifp->if_ibytes = ibytes;
4787 ifp->if_iqdrops = iqdrops;
4788 break;
4789 }
4790
4791 s = splnet();
4792 error = ether_ioctl(ifp, cmd, data);
4793 splx(s);
4794
4795 if (error != ENETRESET)
4796 return error;
4797
4798 switch (cmd) {
4799 case SIOCSIFCAP:
4800 error = aq_set_capability(sc);
4801 break;
4802 case SIOCADDMULTI:
4803 case SIOCDELMULTI:
4804 if ((ifp->if_flags & IFF_RUNNING) == 0)
4805 break;
4806
4807 /*
4808 * Multicast list has changed; set the hardware filter
4809 * accordingly.
4810 */
4811 error = aq_set_filter(sc);
4812 break;
4813 }
4814
4815 return error;
4816 }
4817
4818
4819 MODULE(MODULE_CLASS_DRIVER, if_aq, "pci");
4820
4821 #ifdef _MODULE
4822 #include "ioconf.c"
4823 #endif
4824
4825 static int
4826 if_aq_modcmd(modcmd_t cmd, void *opaque)
4827 {
4828 int error = 0;
4829
4830 switch (cmd) {
4831 case MODULE_CMD_INIT:
4832 #ifdef _MODULE
4833 error = config_init_component(cfdriver_ioconf_if_aq,
4834 cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
4835 #endif
4836 return error;
4837 case MODULE_CMD_FINI:
4838 #ifdef _MODULE
4839 error = config_fini_component(cfdriver_ioconf_if_aq,
4840 cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
4841 #endif
4842 return error;
4843 default:
4844 return ENOTTY;
4845 }
4846 }
4847