Home | History | Annotate | Line # | Download | only in pci
if_aq.c revision 1.17.2.4
      1 /*	$NetBSD: if_aq.c,v 1.17.2.4 2023/06/03 14:46:07 martin Exp $	*/
      2 
      3 /**
      4  * aQuantia Corporation Network Driver
      5  * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  *
     11  *   (1) Redistributions of source code must retain the above
     12  *   copyright notice, this list of conditions and the following
     13  *   disclaimer.
     14  *
     15  *   (2) Redistributions in binary form must reproduce the above
     16  *   copyright notice, this list of conditions and the following
     17  *   disclaimer in the documentation and/or other materials provided
     18  *   with the distribution.
     19  *
     20  *   (3) The name of the author may not be used to endorse or promote
     21  *   products derived from this software without specific prior
     22  *   written permission.
     23  *
     24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     25  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     27  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
     28  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
     30  * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
     33  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
     34  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     35  *
     36  */
     37 
     38 /*-
     39  * Copyright (c) 2020 Ryo Shimizu <ryo (at) nerv.org>
     40  * All rights reserved.
     41  *
     42  * Redistribution and use in source and binary forms, with or without
     43  * modification, are permitted provided that the following conditions
     44  * are met:
     45  * 1. Redistributions of source code must retain the above copyright
     46  *    notice, this list of conditions and the following disclaimer.
     47  * 2. Redistributions in binary form must reproduce the above copyright
     48  *    notice, this list of conditions and the following disclaimer in the
     49  *    documentation and/or other materials provided with the distribution.
     50  *
     51  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     52  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     53  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     54  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
     55  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     56  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     57  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     58  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
     59  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
     60  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     61  * POSSIBILITY OF SUCH DAMAGE.
     62  */
     63 
     64 #include <sys/cdefs.h>
     65 __KERNEL_RCSID(0, "$NetBSD: if_aq.c,v 1.17.2.4 2023/06/03 14:46:07 martin Exp $");
     66 
     67 #ifdef _KERNEL_OPT
     68 #include "opt_if_aq.h"
     69 #include "sysmon_envsys.h"
     70 #endif
     71 
     72 #include <sys/param.h>
     73 #include <sys/types.h>
     74 #include <sys/bitops.h>
     75 #include <sys/cprng.h>
     76 #include <sys/cpu.h>
     77 #include <sys/interrupt.h>
     78 #include <sys/module.h>
     79 #include <sys/pcq.h>
     80 
     81 #include <net/bpf.h>
     82 #include <net/if.h>
     83 #include <net/if_dl.h>
     84 #include <net/if_media.h>
     85 #include <net/if_ether.h>
     86 #include <net/rss_config.h>
     87 
     88 #include <dev/pci/pcivar.h>
     89 #include <dev/pci/pcireg.h>
     90 #include <dev/pci/pcidevs.h>
     91 #include <dev/sysmon/sysmonvar.h>
     92 
     93 /* driver configuration */
     94 #define CONFIG_INTR_MODERATION_ENABLE	true	/* delayed interrupt */
     95 #undef CONFIG_LRO_SUPPORT			/* no LRO not suppoted */
     96 #undef CONFIG_NO_TXRX_INDEPENDENT		/* share TX/RX interrupts */
     97 
     98 #define AQ_NINTR_MAX			(AQ_RSSQUEUE_MAX + AQ_RSSQUEUE_MAX + 1)
     99 					/* TX + RX + LINK. must be <= 32 */
    100 #define AQ_LINKSTAT_IRQ			31	/* for legacy mode */
    101 
    102 #define AQ_TXD_NUM			2048	/* per ring. 8*n && 32~8184 */
    103 #define AQ_RXD_NUM			2048	/* per ring. 8*n && 32~8184 */
    104 /* minimum required to send a packet (vlan needs additional TX descriptor) */
    105 #define AQ_TXD_MIN			(1 + 1)
    106 
    107 
    108 /* hardware specification */
    109 #define AQ_RINGS_NUM			32
    110 #define AQ_RSSQUEUE_MAX			8
    111 #define AQ_RX_DESCRIPTOR_MIN		32
    112 #define AQ_TX_DESCRIPTOR_MIN		32
    113 #define AQ_RX_DESCRIPTOR_MAX		8184
    114 #define AQ_TX_DESCRIPTOR_MAX		8184
    115 #define AQ_TRAFFICCLASS_NUM		8
    116 #define AQ_RSS_HASHKEY_SIZE		40
    117 #define AQ_RSS_INDIRECTION_TABLE_MAX	64
    118 
    119 /*
    120  * TERMINOLOGY
    121  *	MPI = MAC PHY INTERFACE?
    122  *	RPO = RX Protocol Offloading
    123  *	TPO = TX Protocol Offloading
    124  *	RPF = RX Packet Filter
    125  *	TPB = TX Packet buffer
    126  *	RPB = RX Packet buffer
    127  */
    128 
    129 /* registers */
    130 #define AQ_FW_SOFTRESET_REG			0x0000
    131 #define  AQ_FW_SOFTRESET_RESET			__BIT(15) /* soft reset bit */
    132 #define  AQ_FW_SOFTRESET_DIS			__BIT(14) /* reset disable */
    133 
    134 #define AQ_FW_VERSION_REG			0x0018
    135 #define AQ_HW_REVISION_REG			0x001c
    136 #define AQ_GLB_NVR_INTERFACE1_REG		0x0100
    137 
    138 #define AQ_FW_MBOX_CMD_REG			0x0200
    139 #define  AQ_FW_MBOX_CMD_EXECUTE			0x00008000
    140 #define  AQ_FW_MBOX_CMD_BUSY			0x00000100
    141 #define AQ_FW_MBOX_ADDR_REG			0x0208
    142 #define AQ_FW_MBOX_VAL_REG			0x020c
    143 
    144 #define FW2X_LED_MIN_VERSION			0x03010026	/* >= 3.1.38 */
    145 #define FW2X_LED_REG				0x031c
    146 #define  FW2X_LED_DEFAULT			0x00000000
    147 #define  FW2X_LED_NONE				0x0000003f
    148 #define  FW2X_LINKLED				__BITS(0,1)
    149 #define   FW2X_LINKLED_ACTIVE			0
    150 #define   FW2X_LINKLED_ON			1
    151 #define   FW2X_LINKLED_BLINK			2
    152 #define   FW2X_LINKLED_OFF			3
    153 #define  FW2X_STATUSLED				__BITS(2,5)
    154 #define   FW2X_STATUSLED_ORANGE			0
    155 #define   FW2X_STATUSLED_ORANGE_BLINK		2
    156 #define   FW2X_STATUSLED_OFF			3
    157 #define   FW2X_STATUSLED_GREEN			4
    158 #define   FW2X_STATUSLED_ORANGE_GREEN_BLINK	8
    159 #define   FW2X_STATUSLED_GREEN_BLINK		10
    160 
    161 #define FW_MPI_MBOX_ADDR_REG			0x0360
    162 #define FW1X_MPI_INIT1_REG			0x0364
    163 #define FW1X_MPI_CONTROL_REG			0x0368
    164 #define FW1X_MPI_STATE_REG			0x036c
    165 #define  FW1X_MPI_STATE_MODE			__BITS(7,0)
    166 #define  FW1X_MPI_STATE_SPEED			__BITS(32,16)
    167 #define  FW1X_MPI_STATE_DISABLE_DIRTYWAKE	__BITS(25)
    168 #define  FW1X_MPI_STATE_DOWNSHIFT		__BITS(31,28)
    169 #define FW1X_MPI_INIT2_REG			0x0370
    170 #define FW1X_MPI_EFUSEADDR_REG			0x0374
    171 
    172 #define FW2X_MPI_EFUSEADDR_REG			0x0364
    173 #define FW2X_MPI_CONTROL_REG			0x0368	/* 64bit */
    174 #define FW2X_MPI_STATE_REG			0x0370	/* 64bit */
    175 #define FW_BOOT_EXIT_CODE_REG			0x0388
    176 #define  RBL_STATUS_DEAD			0x0000dead
    177 #define  RBL_STATUS_SUCCESS			0x0000abba
    178 #define  RBL_STATUS_FAILURE			0x00000bad
    179 #define  RBL_STATUS_HOST_BOOT			0x0000f1a7
    180 
    181 #define AQ_FW_GLB_CPU_SEM_REG(i)		(0x03a0 + (i) * 4)
    182 #define AQ_FW_SEM_RAM_REG			AQ_FW_GLB_CPU_SEM_REG(2)
    183 
    184 #define AQ_FW_GLB_CTL2_REG			0x0404
    185 #define  AQ_FW_GLB_CTL2_MCP_UP_FORCE_INTERRUPT	__BIT(1)
    186 
    187 #define AQ_GLB_GENERAL_PROVISIONING9_REG	0x0520
    188 #define AQ_GLB_NVR_PROVISIONING2_REG		0x0534
    189 
    190 #define FW_MPI_DAISY_CHAIN_STATUS_REG		0x0704
    191 
    192 #define AQ_PCI_REG_CONTROL_6_REG		0x1014
    193 
    194 // msix bitmap */
    195 #define AQ_INTR_STATUS_REG			0x2000	/* intr status */
    196 #define AQ_INTR_STATUS_CLR_REG			0x2050	/* intr status clear */
    197 #define AQ_INTR_MASK_REG			0x2060	/* intr mask set */
    198 #define AQ_INTR_MASK_CLR_REG			0x2070	/* intr mask clear */
    199 #define AQ_INTR_AUTOMASK_REG			0x2090
    200 
    201 /* AQ_INTR_IRQ_MAP_TXRX_REG[AQ_RINGS_NUM] 0x2100-0x2140 */
    202 #define AQ_INTR_IRQ_MAP_TXRX_REG(i)		(0x2100 + ((i) / 2) * 4)
    203 #define AQ_INTR_IRQ_MAP_TX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
    204 #define  AQ_INTR_IRQ_MAP_TX_IRQMAP(i)		(__BITS(28,24) >> (((i) & 1)*8))
    205 #define  AQ_INTR_IRQ_MAP_TX_EN(i)		(__BIT(31)     >> (((i) & 1)*8))
    206 #define AQ_INTR_IRQ_MAP_RX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
    207 #define  AQ_INTR_IRQ_MAP_RX_IRQMAP(i)		(__BITS(12,8)  >> (((i) & 1)*8))
    208 #define  AQ_INTR_IRQ_MAP_RX_EN(i)		(__BIT(15)     >> (((i) & 1)*8))
    209 
    210 /* AQ_GEN_INTR_MAP_REG[AQ_RINGS_NUM] 0x2180-0x2200 */
    211 #define AQ_GEN_INTR_MAP_REG(i)			(0x2180 + (i) * 4)
    212 #define  AQ_B0_ERR_INT				8U
    213 
    214 #define AQ_INTR_CTRL_REG			0x2300
    215 #define  AQ_INTR_CTRL_IRQMODE			__BITS(1,0)
    216 #define  AQ_INTR_CTRL_IRQMODE_LEGACY		0
    217 #define  AQ_INTR_CTRL_IRQMODE_MSI		1
    218 #define  AQ_INTR_CTRL_IRQMODE_MSIX		2
    219 #define  AQ_INTR_CTRL_MULTIVEC			__BIT(2)
    220 #define  AQ_INTR_CTRL_AUTO_MASK			__BIT(5)
    221 #define  AQ_INTR_CTRL_CLR_ON_READ		__BIT(7)
    222 #define  AQ_INTR_CTRL_RESET_DIS			__BIT(29)
    223 #define  AQ_INTR_CTRL_RESET_IRQ			__BIT(31)
    224 
    225 #define AQ_MBOXIF_POWER_GATING_CONTROL_REG	0x32a8
    226 
    227 #define FW_MPI_RESETCTRL_REG			0x4000
    228 #define  FW_MPI_RESETCTRL_RESET_DIS		__BIT(29)
    229 
    230 #define RX_SYSCONTROL_REG			0x5000
    231 #define  RX_SYSCONTROL_RPB_DMA_LOOPBACK		__BIT(6)
    232 #define  RX_SYSCONTROL_RPF_TPO_LOOPBACK		__BIT(8)
    233 #define  RX_SYSCONTROL_RESET_DIS		__BIT(29)
    234 
    235 #define RX_TCP_RSS_HASH_REG			0x5040
    236 #define  RX_TCP_RSS_HASH_RPF2			__BITS(19,16)
    237 #define  RX_TCP_RSS_HASH_TYPE			__BITS(15,0)
    238 
    239 /* for RPF_*_REG.ACTION */
    240 #define RPF_ACTION_DISCARD			0
    241 #define RPF_ACTION_HOST				1
    242 #define RPF_ACTION_MANAGEMENT			2
    243 #define RPF_ACTION_HOST_MANAGEMENT		3
    244 #define RPF_ACTION_WOL				4
    245 
    246 #define RPF_L2BC_REG				0x5100
    247 #define  RPF_L2BC_EN				__BIT(0)
    248 #define  RPF_L2BC_PROMISC			__BIT(3)
    249 #define  RPF_L2BC_ACTION			__BITS(12,14)
    250 #define  RPF_L2BC_THRESHOLD			__BITS(31,16)
    251 
    252 /* RPF_L2UC_*_REG[34] (actual [38]?) */
    253 #define RPF_L2UC_LSW_REG(i)			(0x5110 + (i) * 8)
    254 #define RPF_L2UC_MSW_REG(i)			(0x5114 + (i) * 8)
    255 #define  RPF_L2UC_MSW_MACADDR_HI		__BITS(15,0)
    256 #define  RPF_L2UC_MSW_ACTION			__BITS(18,16)
    257 #define  RPF_L2UC_MSW_EN			__BIT(31)
    258 #define AQ_HW_MAC_OWN			0	/* index of own address */
    259 #define AQ_HW_MAC_NUM			34
    260 
    261 /* RPF_MCAST_FILTER_REG[8] 0x5250-0x5270 */
    262 #define RPF_MCAST_FILTER_REG(i)			(0x5250 + (i) * 4)
    263 #define  RPF_MCAST_FILTER_EN			__BIT(31)
    264 #define RPF_MCAST_FILTER_MASK_REG		0x5270
    265 #define  RPF_MCAST_FILTER_MASK_ALLMULTI		__BIT(14)
    266 
    267 #define RPF_VLAN_MODE_REG			0x5280
    268 #define  RPF_VLAN_MODE_PROMISC			__BIT(1)
    269 #define  RPF_VLAN_MODE_ACCEPT_UNTAGGED		__BIT(2)
    270 #define  RPF_VLAN_MODE_UNTAGGED_ACTION		__BITS(5,3)
    271 
    272 #define RPF_VLAN_TPID_REG			0x5284
    273 #define  RPF_VLAN_TPID_OUTER			__BITS(31,16)
    274 #define  RPF_VLAN_TPID_INNER			__BITS(15,0)
    275 
    276 /* RPF_VLAN_FILTER_REG[RPF_VLAN_MAX_FILTERS] 0x5290-0x52d0 */
    277 #define RPF_VLAN_MAX_FILTERS			16
    278 #define RPF_VLAN_FILTER_REG(i)			(0x5290 + (i) * 4)
    279 #define  RPF_VLAN_FILTER_EN			__BIT(31)
    280 #define  RPF_VLAN_FILTER_RXQ_EN			__BIT(28)
    281 #define  RPF_VLAN_FILTER_RXQ			__BITS(24,20)
    282 #define  RPF_VLAN_FILTER_ACTION			__BITS(18,16)
    283 #define  RPF_VLAN_FILTER_ID			__BITS(11,0)
    284 
    285 /* RPF_ETHERTYPE_FILTER_REG[AQ_RINGS_NUM] 0x5300-0x5380 */
    286 #define RPF_ETHERTYPE_FILTER_REG(i)		(0x5300 + (i) * 4)
    287 #define  RPF_ETHERTYPE_FILTER_EN		__BIT(31)
    288 #define  RPF_ETHERTYPE_FILTER_PRIO_EN		__BIT(30)
    289 #define  RPF_ETHERTYPE_FILTER_RXQF_EN		__BIT(29)
    290 #define  RPF_ETHERTYPE_FILTER_PRIO		__BITS(28,26)
    291 #define  RPF_ETHERTYPE_FILTER_RXQF		__BITS(24,20)
    292 #define  RPF_ETHERTYPE_FILTER_MNG_RXQF		__BIT(19)
    293 #define  RPF_ETHERTYPE_FILTER_ACTION		__BITS(18,16)
    294 #define  RPF_ETHERTYPE_FILTER_VAL		__BITS(15,0)
    295 
    296 /* RPF_L3_FILTER_REG[8] 0x5380-0x53a0 */
    297 #define RPF_L3_FILTER_REG(i)			(0x5380 + (i) * 4)
    298 #define  RPF_L3_FILTER_L4_EN			__BIT(31)
    299 #define  RPF_L3_FILTER_IPV6_EN			__BIT(30)
    300 #define  RPF_L3_FILTER_SRCADDR_EN		__BIT(29)
    301 #define  RPF_L3_FILTER_DSTADDR_EN		__BIT(28)
    302 #define  RPF_L3_FILTER_L4_SRCPORT_EN		__BIT(27)
    303 #define  RPF_L3_FILTER_L4_DSTPORT_EN		__BIT(26)
    304 #define  RPF_L3_FILTER_L4_PROTO_EN		__BIT(25)
    305 #define  RPF_L3_FILTER_ARP_EN			__BIT(24)
    306 #define  RPF_L3_FILTER_L4_RXQUEUE_EN		__BIT(23)
    307 #define  RPF_L3_FILTER_L4_RXQUEUE_MANAGEMENT_EN	__BIT(22)
    308 #define  RPF_L3_FILTER_L4_ACTION		__BITS(16,18)
    309 #define  RPF_L3_FILTER_L4_RXQUEUE		__BITS(12,8)
    310 #define  RPF_L3_FILTER_L4_PROTO			__BITS(2,0)
    311 #define   RPF_L3_FILTER_L4_PROTO_TCP		0
    312 #define   RPF_L3_FILTER_L4_PROTO_UDP		1
    313 #define   RPF_L3_FILTER_L4_PROTO_SCTP		2
    314 #define   RPF_L3_FILTER_L4_PROTO_ICMP		3
    315 /* parameters of RPF_L3_FILTER_REG[8] */
    316 #define RPF_L3_FILTER_SRCADDR_REG(i)		(0x53b0 + (i) * 4)
    317 #define RPF_L3_FILTER_DSTADDR_REG(i)		(0x53d0 + (i) * 4)
    318 #define RPF_L3_FILTER_L4_SRCPORT_REG(i)		(0x5400 + (i) * 4)
    319 #define RPF_L3_FILTER_L4_DSTPORT_REG(i)		(0x5420 + (i) * 4)
    320 
    321 #define RX_FLR_RSS_CONTROL1_REG			0x54c0
    322 #define  RX_FLR_RSS_CONTROL1_EN			__BIT(31)
    323 
    324 #define RPF_RPB_RX_TC_UPT_REG			0x54c4
    325 #define  RPF_RPB_RX_TC_UPT_MASK(i)		(0x00000007 << ((i) * 4))
    326 
    327 #define RPF_RSS_KEY_ADDR_REG			0x54d0
    328 #define  RPF_RSS_KEY_ADDR			__BITS(4,0)
    329 #define  RPF_RSS_KEY_WR_EN			__BIT(5)
    330 #define RPF_RSS_KEY_WR_DATA_REG			0x54d4
    331 #define RPF_RSS_KEY_RD_DATA_REG			0x54d8
    332 
    333 #define RPF_RSS_REDIR_ADDR_REG			0x54e0
    334 #define  RPF_RSS_REDIR_ADDR			__BITS(3,0)
    335 #define  RPF_RSS_REDIR_WR_EN			__BIT(4)
    336 
    337 #define RPF_RSS_REDIR_WR_DATA_REG		0x54e4
    338 #define  RPF_RSS_REDIR_WR_DATA			__BITS(15,0)
    339 
    340 #define RPO_HWCSUM_REG				0x5580
    341 #define  RPO_HWCSUM_IP4CSUM_EN			__BIT(1)
    342 #define  RPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */
    343 
    344 #define RPO_LRO_ENABLE_REG			0x5590
    345 
    346 #define RPO_LRO_CONF_REG			0x5594
    347 #define  RPO_LRO_CONF_QSESSION_LIMIT		__BITS(13,12)
    348 #define  RPO_LRO_CONF_TOTAL_DESC_LIMIT		__BITS(6,5)
    349 #define  RPO_LRO_CONF_PATCHOPTIMIZATION_EN	__BIT(15)
    350 #define  RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT	__BITS(4,0)
    351 #define RPO_LRO_RSC_MAX_REG			0x5598
    352 
    353 /* RPO_LRO_LDES_MAX_REG[32/8] 0x55a0-0x55b0 */
    354 #define RPO_LRO_LDES_MAX_REG(i)			(0x55a0 + (i / 8) * 4)
    355 #define  RPO_LRO_LDES_MAX_MASK(i)		(0x00000003 << ((i & 7) * 4))
    356 #define RPO_LRO_TB_DIV_REG			0x5620
    357 #define  RPO_LRO_TB_DIV				__BITS(20,31)
    358 #define RPO_LRO_INACTIVE_IVAL_REG		0x5620
    359 #define  RPO_LRO_INACTIVE_IVAL			__BITS(10,19)
    360 #define RPO_LRO_MAX_COALESCING_IVAL_REG		0x5620
    361 #define  RPO_LRO_MAX_COALESCING_IVAL		__BITS(9,0)
    362 
    363 #define RPB_RPF_RX_REG				0x5700
    364 #define  RPB_RPF_RX_TC_MODE			__BIT(8)
    365 #define  RPB_RPF_RX_FC_MODE			__BITS(5,4)
    366 #define  RPB_RPF_RX_BUF_EN			__BIT(0)
    367 
    368 /* RPB_RXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x5710-0x5790 */
    369 #define RPB_RXB_BUFSIZE_REG(i)			(0x5710 + (i) * 0x10)
    370 #define  RPB_RXB_BUFSIZE			__BITS(8,0)
    371 #define RPB_RXB_XOFF_REG(i)			(0x5714 + (i) * 0x10)
    372 #define  RPB_RXB_XOFF_EN			__BIT(31)
    373 #define  RPB_RXB_XOFF_THRESH_HI			__BITS(29,16)
    374 #define  RPB_RXB_XOFF_THRESH_LO			__BITS(13,0)
    375 
    376 #define RX_DMA_DESC_CACHE_INIT_REG		0x5a00
    377 #define  RX_DMA_DESC_CACHE_INIT			__BIT(0)
    378 
    379 #define RX_DMA_INT_DESC_WRWB_EN_REG		0x05a30
    380 #define  RX_DMA_INT_DESC_WRWB_EN		__BIT(2)
    381 #define  RX_DMA_INT_DESC_MODERATE_EN		__BIT(3)
    382 
    383 /* RX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x5a40-0x5ac0 */
    384 #define RX_INTR_MODERATION_CTL_REG(i)		(0x5a40 + (i) * 4)
    385 #define  RX_INTR_MODERATION_CTL_EN		__BIT(1)
    386 #define  RX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
    387 #define  RX_INTR_MODERATION_CTL_MAX		__BITS(24,16)
    388 
    389 /* RX_DMA_DESC_*[AQ_RINGS_NUM] 0x5b00-0x5f00 */
    390 #define RX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x5b00 + (i) * 0x20)
    391 #define RX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x5b04 + (i) * 0x20)
    392 #define RX_DMA_DESC_REG(i)			(0x5b08 + (i) * 0x20)
    393 #define  RX_DMA_DESC_LEN			__BITS(12,3)	/* RXD_NUM/8 */
    394 #define  RX_DMA_DESC_RESET			__BIT(25)
    395 #define  RX_DMA_DESC_HEADER_SPLIT		__BIT(28)
    396 #define  RX_DMA_DESC_VLAN_STRIP			__BIT(29)
    397 #define  RX_DMA_DESC_EN				__BIT(31)
    398 #define RX_DMA_DESC_HEAD_PTR_REG(i)		(0x5b0c + (i) * 0x20)
    399 #define  RX_DMA_DESC_HEAD_PTR			__BITS(12,0)
    400 #define RX_DMA_DESC_TAIL_PTR_REG(i)		(0x5b10 + (i) * 0x20)
    401 #define RX_DMA_DESC_BUFSIZE_REG(i)		(0x5b18 + (i) * 0x20)
    402 #define  RX_DMA_DESC_BUFSIZE_DATA		__BITS(4,0)
    403 #define  RX_DMA_DESC_BUFSIZE_HDR		__BITS(12,8)
    404 
    405 /* RX_DMA_DCAD_REG[AQ_RINGS_NUM] 0x6100-0x6180 */
    406 #define RX_DMA_DCAD_REG(i)			(0x6100 + (i) * 4)
    407 #define  RX_DMA_DCAD_CPUID			__BITS(7,0)
    408 #define  RX_DMA_DCAD_PAYLOAD_EN			__BIT(29)
    409 #define  RX_DMA_DCAD_HEADER_EN			__BIT(30)
    410 #define  RX_DMA_DCAD_DESC_EN			__BIT(31)
    411 
    412 #define RX_DMA_DCA_REG				0x6180
    413 #define  RX_DMA_DCA_EN				__BIT(31)
    414 #define  RX_DMA_DCA_MODE			__BITS(3,0)
    415 
    416 /* counters */
    417 #define RX_DMA_GOOD_PKT_COUNTERLSW		0x6800
    418 #define RX_DMA_GOOD_OCTET_COUNTERLSW		0x6808
    419 #define RX_DMA_DROP_PKT_CNT_REG			0x6818
    420 #define RX_DMA_COALESCED_PKT_CNT_REG		0x6820
    421 
    422 #define TX_SYSCONTROL_REG			0x7000
    423 #define  TX_SYSCONTROL_TPB_DMA_LOOPBACK		__BIT(6)
    424 #define  TX_SYSCONTROL_TPO_PKT_LOOPBACK		__BIT(7)
    425 #define  TX_SYSCONTROL_RESET_DIS		__BIT(29)
    426 
    427 #define TX_TPO2_REG				0x7040
    428 #define  TX_TPO2_EN				__BIT(16)
    429 
    430 #define TPS_DESC_VM_ARB_MODE_REG		0x7300
    431 #define  TPS_DESC_VM_ARB_MODE			__BIT(0)
    432 #define TPS_DESC_RATE_REG			0x7310
    433 #define  TPS_DESC_RATE_TA_RST			__BIT(31)
    434 #define  TPS_DESC_RATE_LIM			__BITS(10,0)
    435 #define TPS_DESC_TC_ARB_MODE_REG		0x7200
    436 #define  TPS_DESC_TC_ARB_MODE			__BITS(1,0)
    437 #define TPS_DATA_TC_ARB_MODE_REG		0x7100
    438 #define  TPS_DATA_TC_ARB_MODE			__BIT(0)
    439 
    440 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7110-0x7130 */
    441 #define TPS_DATA_TCT_REG(i)			(0x7110 + (i) * 4)
    442 #define  TPS_DATA_TCT_CREDIT_MAX		__BITS(16,27)
    443 #define  TPS_DATA_TCT_WEIGHT			__BITS(8,0)
    444 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7210-0x7230 */
    445 #define TPS_DESC_TCT_REG(i)			(0x7210 + (i) * 4)
    446 #define  TPS_DESC_TCT_CREDIT_MAX		__BITS(16,27)
    447 #define  TPS_DESC_TCT_WEIGHT			__BITS(8,0)
    448 
    449 #define AQ_HW_TXBUF_MAX		160
    450 #define AQ_HW_RXBUF_MAX		320
    451 
    452 #define TPO_HWCSUM_REG				0x7800
    453 #define  TPO_HWCSUM_IP4CSUM_EN			__BIT(1)
    454 #define  TPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */
    455 
    456 #define TDM_LSO_EN_REG				0x7810
    457 
    458 #define THM_LSO_TCP_FLAG1_REG			0x7820
    459 #define  THM_LSO_TCP_FLAG1_FIRST		__BITS(11,0)
    460 #define  THM_LSO_TCP_FLAG1_MID			__BITS(27,16)
    461 #define THM_LSO_TCP_FLAG2_REG			0x7824
    462 #define  THM_LSO_TCP_FLAG2_LAST			__BITS(11,0)
    463 
    464 #define TPB_TX_BUF_REG				0x7900
    465 #define  TPB_TX_BUF_EN				__BIT(0)
    466 #define  TPB_TX_BUF_SCP_INS_EN			__BIT(2)
    467 #define  TPB_TX_BUF_TC_MODE_EN			__BIT(8)
    468 
    469 /* TPB_TXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x7910-7990 */
    470 #define TPB_TXB_BUFSIZE_REG(i)			(0x7910 + (i) * 0x10)
    471 #define  TPB_TXB_BUFSIZE			__BITS(7,0)
    472 #define TPB_TXB_THRESH_REG(i)			(0x7914 + (i) * 0x10)
    473 #define  TPB_TXB_THRESH_HI			__BITS(16,28)
    474 #define  TPB_TXB_THRESH_LO			__BITS(12,0)
    475 
    476 #define AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG	0x7b20
    477 #define TX_DMA_INT_DESC_WRWB_EN_REG		0x7b40
    478 #define  TX_DMA_INT_DESC_WRWB_EN		__BIT(1)
    479 #define  TX_DMA_INT_DESC_MODERATE_EN		__BIT(4)
    480 
    481 /* TX_DMA_DESC_*[AQ_RINGS_NUM] 0x7c00-0x8400 */
    482 #define TX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x7c00 + (i) * 0x40)
    483 #define TX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x7c04 + (i) * 0x40)
    484 #define TX_DMA_DESC_REG(i)			(0x7c08 + (i) * 0x40)
    485 #define  TX_DMA_DESC_LEN			__BITS(12, 3)	/* TXD_NUM/8 */
    486 #define  TX_DMA_DESC_EN				__BIT(31)
    487 #define TX_DMA_DESC_HEAD_PTR_REG(i)		(0x7c0c + (i) * 0x40)
    488 #define  TX_DMA_DESC_HEAD_PTR			__BITS(12,0)
    489 #define TX_DMA_DESC_TAIL_PTR_REG(i)		(0x7c10 + (i) * 0x40)
    490 #define TX_DMA_DESC_WRWB_THRESH_REG(i)		(0x7c18 + (i) * 0x40)
    491 #define  TX_DMA_DESC_WRWB_THRESH		__BITS(14,8)
    492 
    493 /* TDM_DCAD_REG[AQ_RINGS_NUM] 0x8400-0x8480 */
    494 #define TDM_DCAD_REG(i)				(0x8400 + (i) * 4)
    495 #define  TDM_DCAD_CPUID				__BITS(7,0)
    496 #define  TDM_DCAD_CPUID_EN			__BIT(31)
    497 
    498 #define TDM_DCA_REG				0x8480
    499 #define  TDM_DCA_EN				__BIT(31)
    500 #define  TDM_DCA_MODE				__BITS(3,0)
    501 
    502 /* TX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x8980-0x8a00 */
    503 #define TX_INTR_MODERATION_CTL_REG(i)		(0x8980 + (i) * 4)
    504 #define  TX_INTR_MODERATION_CTL_EN		__BIT(1)
    505 #define  TX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
    506 #define  TX_INTR_MODERATION_CTL_MAX		__BITS(24,16)
    507 
    508 #define FW1X_CTRL_10G				__BIT(0)
    509 #define FW1X_CTRL_5G				__BIT(1)
    510 #define FW1X_CTRL_5GSR				__BIT(2)
    511 #define FW1X_CTRL_2G5				__BIT(3)
    512 #define FW1X_CTRL_1G				__BIT(4)
    513 #define FW1X_CTRL_100M				__BIT(5)
    514 
    515 #define FW2X_CTRL_10BASET_HD			__BIT(0)
    516 #define FW2X_CTRL_10BASET_FD			__BIT(1)
    517 #define FW2X_CTRL_100BASETX_HD			__BIT(2)
    518 #define FW2X_CTRL_100BASET4_HD			__BIT(3)
    519 #define FW2X_CTRL_100BASET2_HD			__BIT(4)
    520 #define FW2X_CTRL_100BASETX_FD			__BIT(5)
    521 #define FW2X_CTRL_100BASET2_FD			__BIT(6)
    522 #define FW2X_CTRL_1000BASET_HD			__BIT(7)
    523 #define FW2X_CTRL_1000BASET_FD			__BIT(8)
    524 #define FW2X_CTRL_2P5GBASET_FD			__BIT(9)
    525 #define FW2X_CTRL_5GBASET_FD			__BIT(10)
    526 #define FW2X_CTRL_10GBASET_FD			__BIT(11)
    527 #define FW2X_CTRL_RESERVED1			__BIT(32)
    528 #define FW2X_CTRL_10BASET_EEE			__BIT(33)
    529 #define FW2X_CTRL_RESERVED2			__BIT(34)
    530 #define FW2X_CTRL_PAUSE				__BIT(35)
    531 #define FW2X_CTRL_ASYMMETRIC_PAUSE		__BIT(36)
    532 #define FW2X_CTRL_100BASETX_EEE			__BIT(37)
    533 #define FW2X_CTRL_RESERVED3			__BIT(38)
    534 #define FW2X_CTRL_RESERVED4			__BIT(39)
    535 #define FW2X_CTRL_1000BASET_FD_EEE		__BIT(40)
    536 #define FW2X_CTRL_2P5GBASET_FD_EEE		__BIT(41)
    537 #define FW2X_CTRL_5GBASET_FD_EEE		__BIT(42)
    538 #define FW2X_CTRL_10GBASET_FD_EEE		__BIT(43)
    539 #define FW2X_CTRL_RESERVED5			__BIT(44)
    540 #define FW2X_CTRL_RESERVED6			__BIT(45)
    541 #define FW2X_CTRL_RESERVED7			__BIT(46)
    542 #define FW2X_CTRL_RESERVED8			__BIT(47)
    543 #define FW2X_CTRL_RESERVED9			__BIT(48)
    544 #define FW2X_CTRL_CABLE_DIAG			__BIT(49)
    545 #define FW2X_CTRL_TEMPERATURE			__BIT(50)
    546 #define FW2X_CTRL_DOWNSHIFT			__BIT(51)
    547 #define FW2X_CTRL_PTP_AVB_EN			__BIT(52)
    548 #define FW2X_CTRL_MEDIA_DETECT			__BIT(53)
    549 #define FW2X_CTRL_LINK_DROP			__BIT(54)
    550 #define FW2X_CTRL_SLEEP_PROXY			__BIT(55)
    551 #define FW2X_CTRL_WOL				__BIT(56)
    552 #define FW2X_CTRL_MAC_STOP			__BIT(57)
    553 #define FW2X_CTRL_EXT_LOOPBACK			__BIT(58)
    554 #define FW2X_CTRL_INT_LOOPBACK			__BIT(59)
    555 #define FW2X_CTRL_EFUSE_AGENT			__BIT(60)
    556 #define FW2X_CTRL_WOL_TIMER			__BIT(61)
    557 #define FW2X_CTRL_STATISTICS			__BIT(62)
    558 #define FW2X_CTRL_TRANSACTION_ID		__BIT(63)
    559 
    560 #define FW2X_SNPRINTB			\
    561 	"\177\020"			\
    562 	"b\x23" "PAUSE\0"		\
    563 	"b\x24" "ASYMMETRIC-PAUSE\0"	\
    564 	"b\x31" "CABLE-DIAG\0"		\
    565 	"b\x32" "TEMPERATURE\0"		\
    566 	"b\x33" "DOWNSHIFT\0"		\
    567 	"b\x34" "PTP-AVB\0"		\
    568 	"b\x35" "MEDIA-DETECT\0"	\
    569 	"b\x36" "LINK-DROP\0"		\
    570 	"b\x37" "SLEEP-PROXY\0"		\
    571 	"b\x38" "WOL\0"			\
    572 	"b\x39" "MAC-STOP\0"		\
    573 	"b\x3a" "EXT-LOOPBACK\0"	\
    574 	"b\x3b" "INT-LOOPBACK\0"	\
    575 	"b\x3c" "EFUSE-AGENT\0"		\
    576 	"b\x3d" "WOL-TIMER\0"		\
    577 	"b\x3e" "STATISTICS\0"		\
    578 	"b\x3f" "TRANSACTION-ID\0"	\
    579 	"\0"
    580 
    581 #define FW2X_CTRL_RATE_100M			FW2X_CTRL_100BASETX_FD
    582 #define FW2X_CTRL_RATE_1G			FW2X_CTRL_1000BASET_FD
    583 #define FW2X_CTRL_RATE_2G5			FW2X_CTRL_2P5GBASET_FD
    584 #define FW2X_CTRL_RATE_5G			FW2X_CTRL_5GBASET_FD
    585 #define FW2X_CTRL_RATE_10G			FW2X_CTRL_10GBASET_FD
    586 #define FW2X_CTRL_RATE_MASK		\
    587 	(FW2X_CTRL_RATE_100M |		\
    588 	 FW2X_CTRL_RATE_1G |		\
    589 	 FW2X_CTRL_RATE_2G5 |		\
    590 	 FW2X_CTRL_RATE_5G |		\
    591 	 FW2X_CTRL_RATE_10G)
    592 #define FW2X_CTRL_EEE_MASK		\
    593 	(FW2X_CTRL_10BASET_EEE |	\
    594 	 FW2X_CTRL_100BASETX_EEE |	\
    595 	 FW2X_CTRL_1000BASET_FD_EEE |	\
    596 	 FW2X_CTRL_2P5GBASET_FD_EEE |	\
    597 	 FW2X_CTRL_5GBASET_FD_EEE |	\
    598 	 FW2X_CTRL_10GBASET_FD_EEE)
    599 
    600 typedef enum aq_fw_bootloader_mode {
    601 	FW_BOOT_MODE_UNKNOWN = 0,
    602 	FW_BOOT_MODE_FLB,
    603 	FW_BOOT_MODE_RBL_FLASH,
    604 	FW_BOOT_MODE_RBL_HOST_BOOTLOAD
    605 } aq_fw_bootloader_mode_t;
    606 
    607 #define AQ_WRITE_REG(sc, reg, val)				\
    608 	bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val))
    609 
    610 #define AQ_READ_REG(sc, reg)					\
    611 	bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))
    612 
    613 #define AQ_READ64_REG(sc, reg)					\
    614 	((uint64_t)AQ_READ_REG(sc, reg) |			\
    615 	(((uint64_t)AQ_READ_REG(sc, (reg) + 4)) << 32))
    616 
    617 #define AQ_WRITE64_REG(sc, reg, val)				\
    618 	do {							\
    619 		AQ_WRITE_REG(sc, reg, (uint32_t)val);		\
    620 		AQ_WRITE_REG(sc, reg + 4, (uint32_t)(val >> 32)); \
    621 	} while (/* CONSTCOND */0)
    622 
    623 #define AQ_READ_REG_BIT(sc, reg, mask)				\
    624 	__SHIFTOUT(AQ_READ_REG(sc, reg), mask)
    625 
    626 #define AQ_WRITE_REG_BIT(sc, reg, mask, val)			\
    627 	do {							\
    628 		uint32_t _v;					\
    629 		_v = AQ_READ_REG((sc), (reg));			\
    630 		_v &= ~(mask);					\
    631 		if ((val) != 0)					\
    632 			_v |= __SHIFTIN((val), (mask));		\
    633 		AQ_WRITE_REG((sc), (reg), _v);			\
    634 	} while (/* CONSTCOND */ 0)
    635 
    636 #define WAIT_FOR(expr, us, n, errp)				\
    637 	do {							\
    638 		unsigned int _n;				\
    639 		for (_n = n; (!(expr)) && _n != 0; --_n) {	\
    640 			delay((us));				\
    641 		}						\
    642 		if ((errp != NULL)) {				\
    643 			if (_n == 0)				\
    644 				*(errp) = ETIMEDOUT;		\
    645 			else					\
    646 				*(errp) = 0;			\
    647 		}						\
    648 	} while (/* CONSTCOND */ 0)
    649 
    650 #define msec_delay(x)	DELAY(1000 * (x))
    651 
    652 typedef struct aq_mailbox_header {
    653 	uint32_t version;
    654 	uint32_t transaction_id;
    655 	int32_t error;
    656 } __packed aq_mailbox_header_t;
    657 
    658 typedef struct aq_hw_stats_s {
    659 	uint32_t uprc;
    660 	uint32_t mprc;
    661 	uint32_t bprc;
    662 	uint32_t erpt;
    663 	uint32_t uptc;
    664 	uint32_t mptc;
    665 	uint32_t bptc;
    666 	uint32_t erpr;
    667 	uint32_t mbtc;
    668 	uint32_t bbtc;
    669 	uint32_t mbrc;
    670 	uint32_t bbrc;
    671 	uint32_t ubrc;
    672 	uint32_t ubtc;
    673 	uint32_t ptc;
    674 	uint32_t prc;
    675 	uint32_t dpc;	/* not exists in fw2x_msm_statistics */
    676 	uint32_t cprc;	/* not exists in fw2x_msm_statistics */
    677 } __packed aq_hw_stats_s_t;
    678 
    679 typedef struct fw1x_mailbox {
    680 	aq_mailbox_header_t header;
    681 	aq_hw_stats_s_t msm;
    682 } __packed fw1x_mailbox_t;
    683 
    684 typedef struct fw2x_msm_statistics {
    685 	uint32_t uprc;
    686 	uint32_t mprc;
    687 	uint32_t bprc;
    688 	uint32_t erpt;
    689 	uint32_t uptc;
    690 	uint32_t mptc;
    691 	uint32_t bptc;
    692 	uint32_t erpr;
    693 	uint32_t mbtc;
    694 	uint32_t bbtc;
    695 	uint32_t mbrc;
    696 	uint32_t bbrc;
    697 	uint32_t ubrc;
    698 	uint32_t ubtc;
    699 	uint32_t ptc;
    700 	uint32_t prc;
    701 } __packed fw2x_msm_statistics_t;
    702 
    703 typedef struct fw2x_phy_cable_diag_data {
    704 	uint32_t lane_data[4];
    705 } __packed fw2x_phy_cable_diag_data_t;
    706 
    707 typedef struct fw2x_capabilities {
    708 	uint32_t caps_lo;
    709 	uint32_t caps_hi;
    710 } __packed fw2x_capabilities_t;
    711 
    712 typedef struct fw2x_mailbox {		/* struct fwHostInterface */
    713 	aq_mailbox_header_t header;
    714 	fw2x_msm_statistics_t msm;	/* msmStatistics_t msm; */
    715 
    716 	uint32_t phy_info1;
    717 #define PHYINFO1_FAULT_CODE	__BITS(31,16)
    718 #define PHYINFO1_PHY_H_BIT	__BITS(0,15)
    719 	uint32_t phy_info2;
    720 #define PHYINFO2_TEMPERATURE	__BITS(15,0)
    721 #define PHYINFO2_CABLE_LEN	__BITS(23,16)
    722 
    723 	fw2x_phy_cable_diag_data_t diag_data;
    724 	uint32_t reserved[8];
    725 
    726 	fw2x_capabilities_t caps;
    727 
    728 	/* ... */
    729 } __packed fw2x_mailbox_t;
    730 
    731 typedef enum aq_link_speed {
    732 	AQ_LINK_NONE	= 0,
    733 	AQ_LINK_100M	= (1 << 0),
    734 	AQ_LINK_1G	= (1 << 1),
    735 	AQ_LINK_2G5	= (1 << 2),
    736 	AQ_LINK_5G	= (1 << 3),
    737 	AQ_LINK_10G	= (1 << 4)
    738 } aq_link_speed_t;
    739 #define AQ_LINK_ALL	(AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | \
    740 			 AQ_LINK_5G | AQ_LINK_10G )
    741 #define AQ_LINK_AUTO	AQ_LINK_ALL
    742 
    743 typedef enum aq_link_fc {
    744 	AQ_FC_NONE = 0,
    745 	AQ_FC_RX = __BIT(0),
    746 	AQ_FC_TX = __BIT(1),
    747 	AQ_FC_ALL = (AQ_FC_RX | AQ_FC_TX)
    748 } aq_link_fc_t;
    749 
    750 typedef enum aq_link_eee {
    751 	AQ_EEE_DISABLE = 0,
    752 	AQ_EEE_ENABLE = 1
    753 } aq_link_eee_t;
    754 
    755 typedef enum aq_hw_fw_mpi_state {
    756 	MPI_DEINIT	= 0,
    757 	MPI_RESET	= 1,
    758 	MPI_INIT	= 2,
    759 	MPI_POWER	= 4
    760 } aq_hw_fw_mpi_state_t;
    761 
    762 enum aq_media_type {
    763 	AQ_MEDIA_TYPE_UNKNOWN = 0,
    764 	AQ_MEDIA_TYPE_FIBRE,
    765 	AQ_MEDIA_TYPE_TP
    766 };
    767 
    768 struct aq_rx_desc_read {
    769 	uint64_t buf_addr;
    770 	uint64_t hdr_addr;
    771 } __packed;
    772 
    773 struct aq_rx_desc_wb {
    774 	uint32_t type;
    775 #define RXDESC_TYPE_RSSTYPE		__BITS(3,0)
    776 #define  RXDESC_TYPE_RSSTYPE_NONE		0
    777 #define  RXDESC_TYPE_RSSTYPE_IPV4		2
    778 #define  RXDESC_TYPE_RSSTYPE_IPV6		3
    779 #define  RXDESC_TYPE_RSSTYPE_IPV4_TCP		4
    780 #define  RXDESC_TYPE_RSSTYPE_IPV6_TCP		5
    781 #define  RXDESC_TYPE_RSSTYPE_IPV4_UDP		6
    782 #define  RXDESC_TYPE_RSSTYPE_IPV6_UDP		7
    783 #define RXDESC_TYPE_PKTTYPE_ETHER	__BITS(5,4)
    784 #define  RXDESC_TYPE_PKTTYPE_ETHER_IPV4		0
    785 #define  RXDESC_TYPE_PKTTYPE_ETHER_IPV6		1
    786 #define  RXDESC_TYPE_PKTTYPE_ETHER_OTHERS	2
    787 #define  RXDESC_TYPE_PKTTYPE_ETHER_ARP		3
    788 #define RXDESC_TYPE_PKTTYPE_PROTO	__BITS(8,6)
    789 #define  RXDESC_TYPE_PKTTYPE_PROTO_TCP		0
    790 #define  RXDESC_TYPE_PKTTYPE_PROTO_UDP		1
    791 #define  RXDESC_TYPE_PKTTYPE_PROTO_SCTP		2
    792 #define  RXDESC_TYPE_PKTTYPE_PROTO_ICMP		3
    793 #define  RXDESC_TYPE_PKTTYPE_PROTO_OTHERS	4
    794 #define RXDESC_TYPE_PKTTYPE_VLAN	__BIT(9)
    795 #define RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE	__BIT(10)
    796 #define RXDESC_TYPE_MAC_DMA_ERR		__BIT(12)
    797 #define RXDESC_TYPE_RESERVED		__BITS(18,13)
    798 #define RXDESC_TYPE_IPV4_CSUM_CHECKED	__BIT(19)	/* PKTTYPE_ETHER_IPV4 */
    799 #define RXDESC_TYPE_TCPUDP_CSUM_CHECKED	__BIT(20)
    800 #define RXDESC_TYPE_SPH			__BIT(21)
    801 #define RXDESC_TYPE_HDR_LEN		__BITS(31,22)
    802 	uint32_t rss_hash;
    803 	uint16_t status;
    804 #define RXDESC_STATUS_DD		__BIT(0)
    805 #define RXDESC_STATUS_EOP		__BIT(1)
    806 #define RXDESC_STATUS_MACERR		__BIT(2)
    807 #define RXDESC_STATUS_IPV4_CSUM_NG	__BIT(3)
    808 #define RXDESC_STATUS_TCPUDP_CSUM_ERROR	__BIT(4)
    809 #define RXDESC_STATUS_TCPUDP_CSUM_OK	__BIT(5)
    810 
    811 #define RXDESC_STATUS_STAT		__BITS(2,5)
    812 #define RXDESC_STATUS_ESTAT		__BITS(6,11)
    813 #define RXDESC_STATUS_RSC_CNT		__BITS(12,15)
    814 	uint16_t pkt_len;
    815 	uint16_t next_desc_ptr;
    816 	uint16_t vlan;
    817 } __packed;
    818 
    819 typedef union aq_rx_desc {
    820 	struct aq_rx_desc_read read;
    821 	struct aq_rx_desc_wb wb;
    822 } __packed aq_rx_desc_t;
    823 
    824 typedef struct aq_tx_desc {
    825 	uint64_t buf_addr;
    826 	uint32_t ctl1;
    827 #define AQ_TXDESC_CTL1_TYPE_MASK	0x00000003
    828 #define AQ_TXDESC_CTL1_TYPE_TXD		0x00000001
    829 #define AQ_TXDESC_CTL1_TYPE_TXC		0x00000002
    830 #define AQ_TXDESC_CTL1_BLEN		__BITS(19,4)	/* TXD */
    831 #define AQ_TXDESC_CTL1_DD		__BIT(20)	/* TXD */
    832 #define AQ_TXDESC_CTL1_EOP		__BIT(21)	/* TXD */
    833 #define AQ_TXDESC_CTL1_CMD_VLAN		__BIT(22)	/* TXD */
    834 #define AQ_TXDESC_CTL1_CMD_FCS		__BIT(23)	/* TXD */
    835 #define AQ_TXDESC_CTL1_CMD_IP4CSUM	__BIT(24)	/* TXD */
    836 #define AQ_TXDESC_CTL1_CMD_L4CSUM	__BIT(25)	/* TXD */
    837 #define AQ_TXDESC_CTL1_CMD_LSO		__BIT(26)	/* TXD */
    838 #define AQ_TXDESC_CTL1_CMD_WB		__BIT(27)	/* TXD */
    839 #define AQ_TXDESC_CTL1_CMD_VXLAN	__BIT(28)	/* TXD */
    840 #define AQ_TXDESC_CTL1_VID		__BITS(15,4)	/* TXC */
    841 #define AQ_TXDESC_CTL1_LSO_IPV6		__BIT(21)	/* TXC */
    842 #define AQ_TXDESC_CTL1_LSO_TCP		__BIT(22)	/* TXC */
    843 	uint32_t ctl2;
    844 #define AQ_TXDESC_CTL2_LEN		__BITS(31,14)
    845 #define AQ_TXDESC_CTL2_CTX_EN		__BIT(13)
    846 #define AQ_TXDESC_CTL2_CTX_IDX		__BIT(12)
    847 } __packed aq_tx_desc_t;
    848 
    849 struct aq_txring {
    850 	struct aq_softc *txr_sc;
    851 	int txr_index;
    852 	kmutex_t txr_mutex;
    853 	bool txr_active;
    854 
    855 	pcq_t *txr_pcq;
    856 	void *txr_softint;
    857 
    858 	aq_tx_desc_t *txr_txdesc;	/* aq_tx_desc_t[AQ_TXD_NUM] */
    859 	bus_dmamap_t txr_txdesc_dmamap;
    860 	bus_dma_segment_t txr_txdesc_seg[1];
    861 	bus_size_t txr_txdesc_size;
    862 
    863 	struct {
    864 		struct mbuf *m;
    865 		bus_dmamap_t dmamap;
    866 	} txr_mbufs[AQ_TXD_NUM];
    867 	unsigned int txr_prodidx;
    868 	unsigned int txr_considx;
    869 	int txr_nfree;
    870 
    871 	/* counters */
    872 	uint64_t txr_opackets;
    873 	uint64_t txr_obytes;
    874 	uint64_t txr_omcasts;
    875 	uint64_t txr_oerrors;
    876 };
    877 
    878 struct aq_rxring {
    879 	struct aq_softc *rxr_sc;
    880 	int rxr_index;
    881 	kmutex_t rxr_mutex;
    882 	bool rxr_active;
    883 
    884 	aq_rx_desc_t *rxr_rxdesc;	/* aq_rx_desc_t[AQ_RXD_NUM] */
    885 	bus_dmamap_t rxr_rxdesc_dmamap;
    886 	bus_dma_segment_t rxr_rxdesc_seg[1];
    887 	bus_size_t rxr_rxdesc_size;
    888 	struct {
    889 		struct mbuf *m;
    890 		bus_dmamap_t dmamap;
    891 	} rxr_mbufs[AQ_RXD_NUM];
    892 	unsigned int rxr_readidx;
    893 
    894 	/* counters */
    895 	uint64_t rxr_ipackets;
    896 	uint64_t rxr_ibytes;
    897 	uint64_t rxr_ierrors;
    898 	uint64_t rxr_iqdrops;
    899 };
    900 
    901 struct aq_queue {
    902 	struct aq_softc *sc;
    903 	struct aq_txring txring;
    904 	struct aq_rxring rxring;
    905 };
    906 
    907 struct aq_softc;
    908 struct aq_firmware_ops {
    909 	int (*reset)(struct aq_softc *);
    910 	int (*set_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t,
    911 	    aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
    912 	int (*get_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t *,
    913 	    aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
    914 	int (*get_stats)(struct aq_softc *, aq_hw_stats_s_t *);
    915 #if NSYSMON_ENVSYS > 0
    916 	int (*get_temperature)(struct aq_softc *, uint32_t *);
    917 #endif
    918 };
    919 
    920 #ifdef AQ_EVENT_COUNTERS
    921 #define AQ_EVCNT_DECL(name)						\
    922 	char sc_evcount_##name##_name[32];				\
    923 	struct evcnt sc_evcount_##name##_ev;
    924 #define AQ_EVCNT_ATTACH(sc, name, desc, evtype)				\
    925 	do {								\
    926 		snprintf((sc)->sc_evcount_##name##_name,		\
    927 		    sizeof((sc)->sc_evcount_##name##_name),		\
    928 		    "%s", desc);					\
    929 		evcnt_attach_dynamic(&(sc)->sc_evcount_##name##_ev,	\
    930 		    (evtype), NULL, device_xname((sc)->sc_dev),		\
    931 		    (sc)->sc_evcount_##name##_name);			\
    932 	} while (/*CONSTCOND*/0)
    933 #define AQ_EVCNT_ATTACH_MISC(sc, name, desc)				\
    934 	AQ_EVCNT_ATTACH(sc, name, desc, EVCNT_TYPE_MISC)
    935 #define AQ_EVCNT_DETACH(sc, name)					\
    936 	evcnt_detach(&(sc)->sc_evcount_##name##_ev)
    937 #define AQ_EVCNT_ADD(sc, name, val)					\
    938 	((sc)->sc_evcount_##name##_ev.ev_count += (val))
    939 #endif /* AQ_EVENT_COUNTERS */
    940 
    941 #define AQ_LOCK(sc)		mutex_enter(&(sc)->sc_mutex);
    942 #define AQ_UNLOCK(sc)		mutex_exit(&(sc)->sc_mutex);
    943 
    944 /* lock for FW2X_MPI_{CONTROL,STATE]_REG read-modify-write */
    945 #define AQ_MPI_LOCK(sc)		mutex_enter(&(sc)->sc_mpi_mutex);
    946 #define AQ_MPI_UNLOCK(sc)	mutex_exit(&(sc)->sc_mpi_mutex);
    947 
    948 
    949 struct aq_softc {
    950 	device_t sc_dev;
    951 
    952 	bus_space_tag_t sc_iot;
    953 	bus_space_handle_t sc_ioh;
    954 	bus_size_t sc_iosize;
    955 	bus_dma_tag_t sc_dmat;
    956 
    957 	void *sc_ihs[AQ_NINTR_MAX];
    958 	pci_intr_handle_t *sc_intrs;
    959 
    960 	int sc_tx_irq[AQ_RSSQUEUE_MAX];
    961 	int sc_rx_irq[AQ_RSSQUEUE_MAX];
    962 	int sc_linkstat_irq;
    963 	bool sc_use_txrx_independent_intr;
    964 	bool sc_poll_linkstat;
    965 	bool sc_detect_linkstat;
    966 
    967 #if NSYSMON_ENVSYS > 0
    968 	struct sysmon_envsys *sc_sme;
    969 	envsys_data_t sc_sensor_temp;
    970 #endif
    971 
    972 	callout_t sc_tick_ch;
    973 
    974 	int sc_nintrs;
    975 	bool sc_msix;
    976 
    977 	struct aq_queue sc_queue[AQ_RSSQUEUE_MAX];
    978 	int sc_nqueues;
    979 
    980 	pci_chipset_tag_t sc_pc;
    981 	pcitag_t sc_pcitag;
    982 	uint16_t sc_product;
    983 	uint16_t sc_revision;
    984 
    985 	kmutex_t sc_mutex;
    986 	kmutex_t sc_mpi_mutex;
    987 
    988 	const struct aq_firmware_ops *sc_fw_ops;
    989 	uint64_t sc_fw_caps;
    990 	enum aq_media_type sc_media_type;
    991 	aq_link_speed_t sc_available_rates;
    992 
    993 	aq_link_speed_t sc_link_rate;
    994 	aq_link_fc_t sc_link_fc;
    995 	aq_link_eee_t sc_link_eee;
    996 
    997 	uint32_t sc_fw_version;
    998 #define FW_VERSION_MAJOR(sc)	(((sc)->sc_fw_version >> 24) & 0xff)
    999 #define FW_VERSION_MINOR(sc)	(((sc)->sc_fw_version >> 16) & 0xff)
   1000 #define FW_VERSION_BUILD(sc)	((sc)->sc_fw_version & 0xffff)
   1001 	uint32_t sc_features;
   1002 #define FEATURES_MIPS		0x00000001
   1003 #define FEATURES_TPO2		0x00000002
   1004 #define FEATURES_RPF2		0x00000004
   1005 #define FEATURES_MPI_AQ		0x00000008
   1006 #define FEATURES_REV_A0		0x10000000
   1007 #define FEATURES_REV_A		(FEATURES_REV_A0)
   1008 #define FEATURES_REV_B0		0x20000000
   1009 #define FEATURES_REV_B1		0x40000000
   1010 #define FEATURES_REV_B		(FEATURES_REV_B0|FEATURES_REV_B1)
   1011 	uint32_t sc_mbox_addr;
   1012 
   1013 	bool sc_rbl_enabled;
   1014 	bool sc_fast_start_enabled;
   1015 	bool sc_flash_present;
   1016 
   1017 	bool sc_intr_moderation_enable;
   1018 	bool sc_rss_enable;
   1019 
   1020 	struct ethercom sc_ethercom;
   1021 	struct ether_addr sc_enaddr;
   1022 	struct ifmedia sc_media;
   1023 	int sc_ec_capenable;		/* last ec_capenable */
   1024 	unsigned short sc_if_flags;	/* last if_flags */
   1025 
   1026 #ifdef AQ_EVENT_COUNTERS
   1027 	aq_hw_stats_s_t sc_statistics[2];
   1028 	int sc_statistics_idx;
   1029 	bool sc_poll_statistics;
   1030 
   1031 	AQ_EVCNT_DECL(uprc);
   1032 	AQ_EVCNT_DECL(mprc);
   1033 	AQ_EVCNT_DECL(bprc);
   1034 	AQ_EVCNT_DECL(erpt);
   1035 	AQ_EVCNT_DECL(uptc);
   1036 	AQ_EVCNT_DECL(mptc);
   1037 	AQ_EVCNT_DECL(bptc);
   1038 	AQ_EVCNT_DECL(erpr);
   1039 	AQ_EVCNT_DECL(mbtc);
   1040 	AQ_EVCNT_DECL(bbtc);
   1041 	AQ_EVCNT_DECL(mbrc);
   1042 	AQ_EVCNT_DECL(bbrc);
   1043 	AQ_EVCNT_DECL(ubrc);
   1044 	AQ_EVCNT_DECL(ubtc);
   1045 	AQ_EVCNT_DECL(ptc);
   1046 	AQ_EVCNT_DECL(prc);
   1047 	AQ_EVCNT_DECL(dpc);
   1048 	AQ_EVCNT_DECL(cprc);
   1049 #endif
   1050 };
   1051 
   1052 static int aq_match(device_t, cfdata_t, void *);
   1053 static void aq_attach(device_t, device_t, void *);
   1054 static int aq_detach(device_t, int);
   1055 
   1056 static int aq_setup_msix(struct aq_softc *, struct pci_attach_args *, int,
   1057     bool, bool);
   1058 static int aq_setup_legacy(struct aq_softc *, struct pci_attach_args *,
   1059     pci_intr_type_t);
   1060 static int aq_establish_msix_intr(struct aq_softc *, bool, bool);
   1061 
   1062 static int aq_ifmedia_change(struct ifnet * const);
   1063 static void aq_ifmedia_status(struct ifnet * const, struct ifmediareq *);
   1064 static int aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set);
   1065 static int aq_ifflags_cb(struct ethercom *);
   1066 static int aq_init(struct ifnet *);
   1067 static void aq_send_common_locked(struct ifnet *, struct aq_softc *,
   1068     struct aq_txring *, bool);
   1069 static int aq_transmit(struct ifnet *, struct mbuf *);
   1070 static void aq_deferred_transmit(void *);
   1071 static void aq_start(struct ifnet *);
   1072 static void aq_stop(struct ifnet *, int);
   1073 static void aq_watchdog(struct ifnet *);
   1074 static int aq_ioctl(struct ifnet *, unsigned long, void *);
   1075 
   1076 static int aq_txrx_rings_alloc(struct aq_softc *);
   1077 static void aq_txrx_rings_free(struct aq_softc *);
   1078 static int aq_tx_pcq_alloc(struct aq_softc *, struct aq_txring *);
   1079 static void aq_tx_pcq_free(struct aq_softc *, struct aq_txring *);
   1080 
   1081 static void aq_initmedia(struct aq_softc *);
   1082 static void aq_enable_intr(struct aq_softc *, bool, bool);
   1083 
   1084 #if NSYSMON_ENVSYS > 0
   1085 static void aq_temp_refresh(struct sysmon_envsys *, envsys_data_t *);
   1086 #endif
   1087 static void aq_tick(void *);
   1088 static int aq_legacy_intr(void *);
   1089 static int aq_link_intr(void *);
   1090 static int aq_txrx_intr(void *);
   1091 static int aq_tx_intr(void *);
   1092 static int aq_rx_intr(void *);
   1093 
   1094 static int aq_set_linkmode(struct aq_softc *, aq_link_speed_t, aq_link_fc_t,
   1095     aq_link_eee_t);
   1096 static int aq_get_linkmode(struct aq_softc *, aq_link_speed_t *, aq_link_fc_t *,
   1097     aq_link_eee_t *);
   1098 
   1099 static int aq_fw_reset(struct aq_softc *);
   1100 static int aq_fw_version_init(struct aq_softc *);
   1101 static int aq_hw_init(struct aq_softc *);
   1102 static int aq_hw_init_ucp(struct aq_softc *);
   1103 static int aq_hw_reset(struct aq_softc *);
   1104 static int aq_fw_downld_dwords(struct aq_softc *, uint32_t, uint32_t *,
   1105     uint32_t);
   1106 static int aq_get_mac_addr(struct aq_softc *);
   1107 static int aq_init_rss(struct aq_softc *);
   1108 static int aq_set_capability(struct aq_softc *);
   1109 
   1110 static int fw1x_reset(struct aq_softc *);
   1111 static int fw1x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
   1112     aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
   1113 static int fw1x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
   1114     aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
   1115 static int fw1x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
   1116 
   1117 static int fw2x_reset(struct aq_softc *);
   1118 static int fw2x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
   1119     aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
   1120 static int fw2x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
   1121     aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
   1122 static int fw2x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
   1123 #if NSYSMON_ENVSYS > 0
   1124 static int fw2x_get_temperature(struct aq_softc *, uint32_t *);
   1125 #endif
   1126 
   1127 static const struct aq_firmware_ops aq_fw1x_ops = {
   1128 	.reset = fw1x_reset,
   1129 	.set_mode = fw1x_set_mode,
   1130 	.get_mode = fw1x_get_mode,
   1131 	.get_stats = fw1x_get_stats,
   1132 #if NSYSMON_ENVSYS > 0
   1133 	.get_temperature = NULL
   1134 #endif
   1135 };
   1136 
   1137 static const struct aq_firmware_ops aq_fw2x_ops = {
   1138 	.reset = fw2x_reset,
   1139 	.set_mode = fw2x_set_mode,
   1140 	.get_mode = fw2x_get_mode,
   1141 	.get_stats = fw2x_get_stats,
   1142 #if NSYSMON_ENVSYS > 0
   1143 	.get_temperature = fw2x_get_temperature
   1144 #endif
   1145 };
   1146 
   1147 CFATTACH_DECL3_NEW(aq, sizeof(struct aq_softc),
   1148     aq_match, aq_attach, aq_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN);
   1149 
   1150 static const struct aq_product {
   1151 	pci_vendor_id_t aq_vendor;
   1152 	pci_product_id_t aq_product;
   1153 	const char *aq_name;
   1154 	enum aq_media_type aq_media_type;
   1155 	aq_link_speed_t aq_available_rates;
   1156 } aq_products[] = {
   1157 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100,
   1158 	  "Aquantia AQC100 10 Gigabit Network Adapter",
   1159 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
   1160 	},
   1161 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107,
   1162 	  "Aquantia AQC107 10 Gigabit Network Adapter",
   1163 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
   1164 	},
   1165 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108,
   1166 	  "Aquantia AQC108 5 Gigabit Network Adapter",
   1167 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
   1168 	},
   1169 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109,
   1170 	  "Aquantia AQC109 2.5 Gigabit Network Adapter",
   1171 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
   1172 	},
   1173 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111,
   1174 	  "Aquantia AQC111 5 Gigabit Network Adapter",
   1175 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
   1176 	},
   1177 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112,
   1178 	  "Aquantia AQC112 2.5 Gigabit Network Adapter",
   1179 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
   1180 	},
   1181 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100S,
   1182 	  "Aquantia AQC100S 10 Gigabit Network Adapter",
   1183 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
   1184 	},
   1185 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107S,
   1186 	  "Aquantia AQC107S 10 Gigabit Network Adapter",
   1187 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
   1188 	},
   1189 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108S,
   1190 	  "Aquantia AQC108S 5 Gigabit Network Adapter",
   1191 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
   1192 	},
   1193 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109S,
   1194 	  "Aquantia AQC109S 2.5 Gigabit Network Adapter",
   1195 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
   1196 	},
   1197 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111S,
   1198 	  "Aquantia AQC111S 5 Gigabit Network Adapter",
   1199 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
   1200 	},
   1201 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112S,
   1202 	  "Aquantia AQC112S 2.5 Gigabit Network Adapter",
   1203 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
   1204 	},
   1205 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D100,
   1206 	  "Aquantia D100 10 Gigabit Network Adapter",
   1207 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
   1208 	},
   1209 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D107,
   1210 	  "Aquantia D107 10 Gigabit Network Adapter",
   1211 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
   1212 	},
   1213 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D108,
   1214 	  "Aquantia D108 5 Gigabit Network Adapter",
   1215 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
   1216 	},
   1217 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D109,
   1218 	  "Aquantia D109 2.5 Gigabit Network Adapter",
   1219 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
   1220 	}
   1221 };
   1222 
   1223 static const struct aq_product *
   1224 aq_lookup(const struct pci_attach_args *pa)
   1225 {
   1226 	unsigned int i;
   1227 
   1228 	for (i = 0; i < __arraycount(aq_products); i++) {
   1229 		if (PCI_VENDOR(pa->pa_id)  == aq_products[i].aq_vendor &&
   1230 		    PCI_PRODUCT(pa->pa_id) == aq_products[i].aq_product)
   1231 			return &aq_products[i];
   1232 	}
   1233 	return NULL;
   1234 }
   1235 
   1236 static int
   1237 aq_match(device_t parent, cfdata_t cf, void *aux)
   1238 {
   1239 	struct pci_attach_args *pa = aux;
   1240 
   1241 	if (aq_lookup(pa) != NULL)
   1242 		return 1;
   1243 
   1244 	return 0;
   1245 }
   1246 
   1247 static void
   1248 aq_attach(device_t parent, device_t self, void *aux)
   1249 {
   1250 	struct aq_softc *sc = device_private(self);
   1251 	struct pci_attach_args *pa = aux;
   1252 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1253 	pci_chipset_tag_t pc;
   1254 	pcitag_t tag;
   1255 	pcireg_t command, memtype, bar;
   1256 	const struct aq_product *aqp;
   1257 	int error;
   1258 
   1259 	sc->sc_dev = self;
   1260 	mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_NET);
   1261 	mutex_init(&sc->sc_mpi_mutex, MUTEX_DEFAULT, IPL_NET);
   1262 
   1263 	sc->sc_pc = pc = pa->pa_pc;
   1264 	sc->sc_pcitag = tag = pa->pa_tag;
   1265 	sc->sc_dmat = pci_dma64_available(pa) ? pa->pa_dmat64 : pa->pa_dmat;
   1266 
   1267 	command = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
   1268 	command |= PCI_COMMAND_MASTER_ENABLE;
   1269 	pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, command);
   1270 
   1271 	sc->sc_product = PCI_PRODUCT(pa->pa_id);
   1272 	sc->sc_revision = PCI_REVISION(pa->pa_class);
   1273 
   1274 	aqp = aq_lookup(pa);
   1275 	KASSERT(aqp != NULL);
   1276 
   1277 	pci_aprint_devinfo_fancy(pa, "Ethernet controller", aqp->aq_name, 1);
   1278 
   1279 	bar = pci_conf_read(pc, tag, PCI_BAR(0));
   1280 	if ((PCI_MAPREG_MEM_ADDR(bar) == 0) ||
   1281 	    (PCI_MAPREG_TYPE(bar) != PCI_MAPREG_TYPE_MEM)) {
   1282 		aprint_error_dev(sc->sc_dev, "wrong BAR type\n");
   1283 		return;
   1284 	}
   1285 	memtype = pci_mapreg_type(pc, tag, PCI_BAR(0));
   1286 	if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->sc_iot, &sc->sc_ioh,
   1287 	    NULL, &sc->sc_iosize) != 0) {
   1288 		aprint_error_dev(sc->sc_dev, "unable to map register\n");
   1289 		return;
   1290 	}
   1291 
   1292 	sc->sc_nqueues = MIN(ncpu, AQ_RSSQUEUE_MAX);
   1293 
   1294 	/* max queue num is 8, and must be 2^n */
   1295 	if (ncpu >= 8)
   1296 		sc->sc_nqueues = 8;
   1297 	else if (ncpu >= 4)
   1298 		sc->sc_nqueues = 4;
   1299 	else if (ncpu >= 2)
   1300 		sc->sc_nqueues = 2;
   1301 	else
   1302 		sc->sc_nqueues = 1;
   1303 
   1304 	int msixcount = pci_msix_count(pa->pa_pc, pa->pa_tag);
   1305 #ifndef CONFIG_NO_TXRX_INDEPENDENT
   1306 	if (msixcount >= (sc->sc_nqueues * 2 + 1)) {
   1307 		/* TX intrs + RX intrs + LINKSTAT intrs */
   1308 		sc->sc_use_txrx_independent_intr = true;
   1309 		sc->sc_poll_linkstat = false;
   1310 		sc->sc_msix = true;
   1311 	} else if (msixcount >= (sc->sc_nqueues * 2)) {
   1312 		/* TX intrs + RX intrs */
   1313 		sc->sc_use_txrx_independent_intr = true;
   1314 		sc->sc_poll_linkstat = true;
   1315 		sc->sc_msix = true;
   1316 	} else
   1317 #endif
   1318 	if (msixcount >= (sc->sc_nqueues + 1)) {
   1319 		/* TX/RX intrs LINKSTAT intrs */
   1320 		sc->sc_use_txrx_independent_intr = false;
   1321 		sc->sc_poll_linkstat = false;
   1322 		sc->sc_msix = true;
   1323 	} else if (msixcount >= sc->sc_nqueues) {
   1324 		/* TX/RX intrs */
   1325 		sc->sc_use_txrx_independent_intr = false;
   1326 		sc->sc_poll_linkstat = true;
   1327 		sc->sc_msix = true;
   1328 	} else {
   1329 		/* giving up using MSI-X */
   1330 		sc->sc_msix = false;
   1331 	}
   1332 
   1333 	/* XXX: on FIBRE, linkstat interrupt does not occur on boot? */
   1334 	if (aqp->aq_media_type == AQ_MEDIA_TYPE_FIBRE)
   1335 		sc->sc_poll_linkstat = true;
   1336 
   1337 #ifdef AQ_FORCE_POLL_LINKSTAT
   1338 	sc->sc_poll_linkstat = true;
   1339 #endif
   1340 
   1341 	aprint_debug_dev(sc->sc_dev,
   1342 	    "ncpu=%d, pci_msix_count=%d."
   1343 	    " allocate %d interrupts for %d%s queues%s\n",
   1344 	    ncpu, msixcount,
   1345 	    (sc->sc_use_txrx_independent_intr ?
   1346 	    (sc->sc_nqueues * 2) : sc->sc_nqueues) +
   1347 	    (sc->sc_poll_linkstat ? 0 : 1),
   1348 	    sc->sc_nqueues,
   1349 	    sc->sc_use_txrx_independent_intr ? "*2" : "",
   1350 	    sc->sc_poll_linkstat ? "" : ", and link status");
   1351 
   1352 	if (sc->sc_msix)
   1353 		error = aq_setup_msix(sc, pa, sc->sc_nqueues,
   1354 		    sc->sc_use_txrx_independent_intr, !sc->sc_poll_linkstat);
   1355 	else
   1356 		error = ENODEV;
   1357 
   1358 	if (error != 0) {
   1359 		/* if MSI-X failed, fallback to MSI with single queue */
   1360 		sc->sc_use_txrx_independent_intr = false;
   1361 		sc->sc_poll_linkstat = false;
   1362 		sc->sc_msix = false;
   1363 		sc->sc_nqueues = 1;
   1364 		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_MSI);
   1365 	}
   1366 	if (error != 0) {
   1367 		/* if MSI failed, fallback to INTx */
   1368 		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_INTX);
   1369 	}
   1370 	if (error != 0)
   1371 		return;
   1372 
   1373 	callout_init(&sc->sc_tick_ch, 0);
   1374 	callout_setfunc(&sc->sc_tick_ch, aq_tick, sc);
   1375 
   1376 	sc->sc_intr_moderation_enable = CONFIG_INTR_MODERATION_ENABLE;
   1377 
   1378 	if (sc->sc_msix && (sc->sc_nqueues > 1))
   1379 		sc->sc_rss_enable = true;
   1380 	else
   1381 		sc->sc_rss_enable = false;
   1382 
   1383 	error = aq_txrx_rings_alloc(sc);
   1384 	if (error != 0)
   1385 		goto attach_failure;
   1386 
   1387 	error = aq_fw_reset(sc);
   1388 	if (error != 0)
   1389 		goto attach_failure;
   1390 
   1391 	error = aq_fw_version_init(sc);
   1392 	if (error != 0)
   1393 		goto attach_failure;
   1394 
   1395 	error = aq_hw_init_ucp(sc);
   1396 	if (error < 0)
   1397 		goto attach_failure;
   1398 
   1399 	KASSERT(sc->sc_mbox_addr != 0);
   1400 	error = aq_hw_reset(sc);
   1401 	if (error != 0)
   1402 		goto attach_failure;
   1403 
   1404 	aq_get_mac_addr(sc);
   1405 	aq_init_rss(sc);
   1406 
   1407 	error = aq_hw_init(sc);	/* initialize and interrupts */
   1408 	if (error != 0)
   1409 		goto attach_failure;
   1410 
   1411 	sc->sc_media_type = aqp->aq_media_type;
   1412 	sc->sc_available_rates = aqp->aq_available_rates;
   1413 
   1414 	sc->sc_ethercom.ec_ifmedia = &sc->sc_media;
   1415 	ifmedia_init(&sc->sc_media, IFM_IMASK,
   1416 	    aq_ifmedia_change, aq_ifmedia_status);
   1417 	aq_initmedia(sc);
   1418 
   1419 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
   1420 	ifp->if_softc = sc;
   1421 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
   1422 	ifp->if_baudrate = IF_Gbps(10);
   1423 	ifp->if_init = aq_init;
   1424 	ifp->if_ioctl = aq_ioctl;
   1425 	if (sc->sc_msix && (sc->sc_nqueues > 1))
   1426 		ifp->if_transmit = aq_transmit;
   1427 	ifp->if_start = aq_start;
   1428 	ifp->if_stop = aq_stop;
   1429 	ifp->if_watchdog = aq_watchdog;
   1430 	IFQ_SET_READY(&ifp->if_snd);
   1431 
   1432 	/* initialize capabilities */
   1433 	sc->sc_ethercom.ec_capabilities = 0;
   1434 	sc->sc_ethercom.ec_capenable = 0;
   1435 #if notyet
   1436 	/* TODO */
   1437 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_EEE;
   1438 #endif
   1439 	sc->sc_ethercom.ec_capabilities |=
   1440 	    ETHERCAP_JUMBO_MTU |
   1441 	    ETHERCAP_VLAN_MTU |
   1442 	    ETHERCAP_VLAN_HWTAGGING |
   1443 	    ETHERCAP_VLAN_HWFILTER;
   1444 	sc->sc_ethercom.ec_capenable |=
   1445 	    ETHERCAP_VLAN_HWTAGGING |
   1446 	    ETHERCAP_VLAN_HWFILTER;
   1447 
   1448 	ifp->if_capabilities = 0;
   1449 	ifp->if_capenable = 0;
   1450 #ifdef CONFIG_LRO_SUPPORT
   1451 	ifp->if_capabilities |= IFCAP_LRO;
   1452 	ifp->if_capenable |= IFCAP_LRO;
   1453 #endif
   1454 #if notyet
   1455 	/* TSO */
   1456 	ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
   1457 #endif
   1458 
   1459 #if notyet
   1460 	/*
   1461 	 * XXX:
   1462 	 *   Rx L4 CSUM doesn't work well for fragment packet.
   1463 	 *   aq marks 'CHEDKED' and 'BAD' for them.
   1464 	 *   we need to ignore (clear) hw-csum flags if the packet is fragmented
   1465 	 *
   1466 	 *   TODO: test with LRO enabled
   1467 	 */
   1468 	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv6_Rx;
   1469 	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv6_Rx;
   1470 #endif
   1471 	/* TX hardware checksum offloadding */
   1472 	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Tx;
   1473 	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv6_Tx;
   1474 	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv6_Tx;
   1475 	/* RX hardware checksum offloadding */
   1476 	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx;
   1477 
   1478 	if_attach(ifp);
   1479 	if_deferred_start_init(ifp, NULL);
   1480 	ether_ifattach(ifp, sc->sc_enaddr.ether_addr_octet);
   1481 	ether_set_vlan_cb(&sc->sc_ethercom, aq_vlan_cb);
   1482 	ether_set_ifflags_cb(&sc->sc_ethercom, aq_ifflags_cb);
   1483 
   1484 	aq_enable_intr(sc, true, false);	/* only intr about link */
   1485 
   1486 	/* update media */
   1487 	aq_ifmedia_change(ifp);
   1488 
   1489 #if NSYSMON_ENVSYS > 0
   1490 	/* temperature monitoring */
   1491 	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_temperature != NULL &&
   1492 	    (sc->sc_fw_caps & FW2X_CTRL_TEMPERATURE) != 0) {
   1493 
   1494 		sc->sc_sme = sysmon_envsys_create();
   1495 		sc->sc_sme->sme_name = device_xname(self);
   1496 		sc->sc_sme->sme_cookie = sc;
   1497 		sc->sc_sme->sme_flags = 0;
   1498 		sc->sc_sme->sme_refresh = aq_temp_refresh;
   1499 		sc->sc_sensor_temp.units = ENVSYS_STEMP;
   1500 		sc->sc_sensor_temp.state = ENVSYS_SINVALID;
   1501 		snprintf(sc->sc_sensor_temp.desc, ENVSYS_DESCLEN, "PHY");
   1502 
   1503 		sysmon_envsys_sensor_attach(sc->sc_sme, &sc->sc_sensor_temp);
   1504 		sysmon_envsys_register(sc->sc_sme);
   1505 
   1506 		/*
   1507 		 * for unknown reasons, the first call of fw2x_get_temperature()
   1508 		 * will always fail (firmware matter?), so run once now.
   1509 		 */
   1510 		aq_temp_refresh(sc->sc_sme, &sc->sc_sensor_temp);
   1511 	}
   1512 #endif
   1513 
   1514 #ifdef AQ_EVENT_COUNTERS
   1515 	/* get starting statistics values */
   1516 	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_stats != NULL &&
   1517 	    sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[0]) == 0) {
   1518 		sc->sc_poll_statistics = true;
   1519 	}
   1520 
   1521 	AQ_EVCNT_ATTACH_MISC(sc, uprc, "RX unicast packet");
   1522 	AQ_EVCNT_ATTACH_MISC(sc, bprc, "RX broadcast packet");
   1523 	AQ_EVCNT_ATTACH_MISC(sc, mprc, "RX multicast packet");
   1524 	AQ_EVCNT_ATTACH_MISC(sc, erpr, "RX error packet");
   1525 	AQ_EVCNT_ATTACH_MISC(sc, ubrc, "RX unicast bytes");
   1526 	AQ_EVCNT_ATTACH_MISC(sc, bbrc, "RX broadcast bytes");
   1527 	AQ_EVCNT_ATTACH_MISC(sc, mbrc, "RX multicast bytes");
   1528 	AQ_EVCNT_ATTACH_MISC(sc, prc, "RX good packet");
   1529 	AQ_EVCNT_ATTACH_MISC(sc, uptc, "TX unicast packet");
   1530 	AQ_EVCNT_ATTACH_MISC(sc, bptc, "TX broadcast packet");
   1531 	AQ_EVCNT_ATTACH_MISC(sc, mptc, "TX multicast packet");
   1532 	AQ_EVCNT_ATTACH_MISC(sc, erpt, "TX error packet");
   1533 	AQ_EVCNT_ATTACH_MISC(sc, ubtc, "TX unicast bytes");
   1534 	AQ_EVCNT_ATTACH_MISC(sc, bbtc, "TX broadcast bytes");
   1535 	AQ_EVCNT_ATTACH_MISC(sc, mbtc, "TX multicast bytes");
   1536 	AQ_EVCNT_ATTACH_MISC(sc, ptc, "TX good packet");
   1537 	AQ_EVCNT_ATTACH_MISC(sc, dpc, "DMA drop packet");
   1538 	AQ_EVCNT_ATTACH_MISC(sc, cprc, "RX coalesced packet");
   1539 #endif
   1540 
   1541 	return;
   1542 
   1543  attach_failure:
   1544 	aq_detach(self, 0);
   1545 }
   1546 
   1547 static int
   1548 aq_detach(device_t self, int flags __unused)
   1549 {
   1550 	struct aq_softc *sc = device_private(self);
   1551 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1552 	int i, s;
   1553 
   1554 	if (sc->sc_iosize != 0) {
   1555 		if (ifp->if_softc != NULL) {
   1556 			s = splnet();
   1557 			aq_stop(ifp, 0);
   1558 			splx(s);
   1559 		}
   1560 
   1561 		for (i = 0; i < AQ_NINTR_MAX; i++) {
   1562 			if (sc->sc_ihs[i] != NULL) {
   1563 				pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
   1564 				sc->sc_ihs[i] = NULL;
   1565 			}
   1566 		}
   1567 		if (sc->sc_nintrs > 0) {
   1568 			pci_intr_release(sc->sc_pc, sc->sc_intrs,
   1569 			    sc->sc_nintrs);
   1570 			sc->sc_intrs = NULL;
   1571 			sc->sc_nintrs = 0;
   1572 		}
   1573 
   1574 		aq_txrx_rings_free(sc);
   1575 
   1576 		if (ifp->if_softc != NULL) {
   1577 			ether_ifdetach(ifp);
   1578 			if_detach(ifp);
   1579 		}
   1580 
   1581 		aprint_debug_dev(sc->sc_dev, "%s: bus_space_unmap\n", __func__);
   1582 		bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
   1583 		sc->sc_iosize = 0;
   1584 	}
   1585 
   1586 	callout_stop(&sc->sc_tick_ch);
   1587 
   1588 #if NSYSMON_ENVSYS > 0
   1589 	if (sc->sc_sme != NULL) {
   1590 		/* all sensors associated with this will also be detached */
   1591 		sysmon_envsys_unregister(sc->sc_sme);
   1592 		sc->sc_sme = NULL;
   1593 	}
   1594 #endif
   1595 
   1596 #ifdef AQ_EVENT_COUNTERS
   1597 	AQ_EVCNT_DETACH(sc, uprc);
   1598 	AQ_EVCNT_DETACH(sc, mprc);
   1599 	AQ_EVCNT_DETACH(sc, bprc);
   1600 	AQ_EVCNT_DETACH(sc, erpt);
   1601 	AQ_EVCNT_DETACH(sc, uptc);
   1602 	AQ_EVCNT_DETACH(sc, mptc);
   1603 	AQ_EVCNT_DETACH(sc, bptc);
   1604 	AQ_EVCNT_DETACH(sc, erpr);
   1605 	AQ_EVCNT_DETACH(sc, mbtc);
   1606 	AQ_EVCNT_DETACH(sc, bbtc);
   1607 	AQ_EVCNT_DETACH(sc, mbrc);
   1608 	AQ_EVCNT_DETACH(sc, bbrc);
   1609 	AQ_EVCNT_DETACH(sc, ubrc);
   1610 	AQ_EVCNT_DETACH(sc, ubtc);
   1611 	AQ_EVCNT_DETACH(sc, ptc);
   1612 	AQ_EVCNT_DETACH(sc, prc);
   1613 	AQ_EVCNT_DETACH(sc, dpc);
   1614 	AQ_EVCNT_DETACH(sc, cprc);
   1615 #endif
   1616 
   1617 	mutex_destroy(&sc->sc_mpi_mutex);
   1618 	mutex_destroy(&sc->sc_mutex);
   1619 
   1620 	return 0;
   1621 }
   1622 
   1623 static int
   1624 aq_establish_intr(struct aq_softc *sc, int intno, kcpuset_t *affinity,
   1625     int (*func)(void *), void *arg, const char *xname)
   1626 {
   1627 	char intrbuf[PCI_INTRSTR_LEN];
   1628 	pci_chipset_tag_t pc = sc->sc_pc;
   1629 	void *vih;
   1630 	const char *intrstr = NULL;
   1631 
   1632 	intrstr = pci_intr_string(pc, sc->sc_intrs[intno], intrbuf,
   1633 	    sizeof(intrbuf));
   1634 
   1635 	pci_intr_setattr(pc, &sc->sc_intrs[intno], PCI_INTR_MPSAFE, true);
   1636 
   1637 	vih = pci_intr_establish_xname(pc, sc->sc_intrs[intno],
   1638 	    IPL_NET, func, arg, xname);
   1639 	if (vih == NULL) {
   1640 		aprint_error_dev(sc->sc_dev,
   1641 		    "unable to establish MSI-X%s%s for %s\n",
   1642 		    intrstr ? " at " : "",
   1643 		    intrstr ? intrstr : "", xname);
   1644 		return EIO;
   1645 	}
   1646 	sc->sc_ihs[intno] = vih;
   1647 
   1648 	if (affinity != NULL) {
   1649 		/* Round-robin affinity */
   1650 		kcpuset_zero(affinity);
   1651 		kcpuset_set(affinity, intno % ncpu);
   1652 		interrupt_distribute(vih, affinity, NULL);
   1653 	}
   1654 
   1655 	return 0;
   1656 }
   1657 
   1658 static int
   1659 aq_establish_msix_intr(struct aq_softc *sc, bool txrx_independent,
   1660     bool linkintr)
   1661 {
   1662 	kcpuset_t *affinity;
   1663 	int error, intno, i;
   1664 	char intr_xname[INTRDEVNAMEBUF];
   1665 
   1666 	kcpuset_create(&affinity, false);
   1667 
   1668 	intno = 0;
   1669 
   1670 	if (txrx_independent) {
   1671 		for (i = 0; i < sc->sc_nqueues; i++) {
   1672 			snprintf(intr_xname, sizeof(intr_xname), "%s RX%d",
   1673 			    device_xname(sc->sc_dev), i);
   1674 			sc->sc_rx_irq[i] = intno;
   1675 			error = aq_establish_intr(sc, intno++, affinity,
   1676 			   aq_rx_intr, &sc->sc_queue[i].rxring, intr_xname);
   1677 			if (error != 0)
   1678 				goto fail;
   1679 		}
   1680 		for (i = 0; i < sc->sc_nqueues; i++) {
   1681 			snprintf(intr_xname, sizeof(intr_xname), "%s TX%d",
   1682 			    device_xname(sc->sc_dev), i);
   1683 			sc->sc_tx_irq[i] = intno;
   1684 			error = aq_establish_intr(sc, intno++, affinity,
   1685 			    aq_tx_intr, &sc->sc_queue[i].txring, intr_xname);
   1686 			if (error != 0)
   1687 				goto fail;
   1688 		}
   1689 	} else {
   1690 		for (i = 0; i < sc->sc_nqueues; i++) {
   1691 			snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
   1692 			    device_xname(sc->sc_dev), i);
   1693 			sc->sc_rx_irq[i] = intno;
   1694 			sc->sc_tx_irq[i] = intno;
   1695 			error = aq_establish_intr(sc, intno++, affinity,
   1696 			    aq_txrx_intr, &sc->sc_queue[i], intr_xname);
   1697 			if (error != 0)
   1698 				goto fail;
   1699 		}
   1700 	}
   1701 
   1702 	if (linkintr) {
   1703 		snprintf(intr_xname, sizeof(intr_xname), "%s LINK",
   1704 		    device_xname(sc->sc_dev));
   1705 		sc->sc_linkstat_irq = intno;
   1706 		error = aq_establish_intr(sc, intno++, affinity,
   1707 		    aq_link_intr, sc, intr_xname);
   1708 		if (error != 0)
   1709 			goto fail;
   1710 	}
   1711 
   1712 	kcpuset_destroy(affinity);
   1713 	return 0;
   1714 
   1715  fail:
   1716 	for (i = 0; i < AQ_NINTR_MAX; i++) {
   1717 		if (sc->sc_ihs[i] != NULL) {
   1718 			pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
   1719 			sc->sc_ihs[i] = NULL;
   1720 		}
   1721 	}
   1722 
   1723 	kcpuset_destroy(affinity);
   1724 	return ENOMEM;
   1725 }
   1726 
   1727 static int
   1728 aq_setup_msix(struct aq_softc *sc, struct pci_attach_args *pa, int nqueue,
   1729     bool txrx_independent, bool linkintr)
   1730 {
   1731 	int error, nintr;
   1732 
   1733 	if (txrx_independent)
   1734 		nintr = nqueue * 2;
   1735 	else
   1736 		nintr = nqueue;
   1737 
   1738 	if (linkintr)
   1739 		nintr++;
   1740 
   1741 	error = pci_msix_alloc_exact(pa, &sc->sc_intrs, nintr);
   1742 	if (error != 0) {
   1743 		aprint_error_dev(sc->sc_dev,
   1744 		    "failed to allocate MSI-X interrupts\n");
   1745 		goto fail;
   1746 	}
   1747 
   1748 	error = aq_establish_msix_intr(sc, txrx_independent, linkintr);
   1749 	if (error == 0) {
   1750 		sc->sc_nintrs = nintr;
   1751 	} else {
   1752 		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
   1753 		sc->sc_nintrs = 0;
   1754 	}
   1755  fail:
   1756 	return error;
   1757 
   1758 }
   1759 
   1760 static int
   1761 aq_setup_legacy(struct aq_softc *sc, struct pci_attach_args *pa,
   1762     pci_intr_type_t inttype)
   1763 {
   1764 	int counts[PCI_INTR_TYPE_SIZE];
   1765 	int error, nintr;
   1766 
   1767 	nintr = 1;
   1768 
   1769 	memset(counts, 0, sizeof(counts));
   1770 	counts[inttype] = nintr;
   1771 
   1772 	error = pci_intr_alloc(pa, &sc->sc_intrs, counts, inttype);
   1773 	if (error != 0) {
   1774 		aprint_error_dev(sc->sc_dev,
   1775 		    "failed to allocate%s interrupts\n",
   1776 		    (inttype == PCI_INTR_TYPE_MSI) ? " MSI" : "");
   1777 		return error;
   1778 	}
   1779 	error = aq_establish_intr(sc, 0, NULL, aq_legacy_intr, sc,
   1780 	    device_xname(sc->sc_dev));
   1781 	if (error == 0) {
   1782 		sc->sc_nintrs = nintr;
   1783 	} else {
   1784 		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
   1785 		sc->sc_nintrs = 0;
   1786 	}
   1787 	return error;
   1788 }
   1789 
   1790 static void
   1791 global_software_reset(struct aq_softc *sc)
   1792 {
   1793 	uint32_t v;
   1794 
   1795 	AQ_WRITE_REG_BIT(sc, RX_SYSCONTROL_REG, RX_SYSCONTROL_RESET_DIS, 0);
   1796 	AQ_WRITE_REG_BIT(sc, TX_SYSCONTROL_REG, TX_SYSCONTROL_RESET_DIS, 0);
   1797 	AQ_WRITE_REG_BIT(sc, FW_MPI_RESETCTRL_REG,
   1798 	    FW_MPI_RESETCTRL_RESET_DIS, 0);
   1799 
   1800 	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
   1801 	v &= ~AQ_FW_SOFTRESET_DIS;
   1802 	v |= AQ_FW_SOFTRESET_RESET;
   1803 	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
   1804 }
   1805 
   1806 static int
   1807 mac_soft_reset_rbl(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
   1808 {
   1809 	int timo;
   1810 
   1811 	aprint_debug_dev(sc->sc_dev, "RBL> MAC reset STARTED!\n");
   1812 
   1813 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
   1814 	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
   1815 	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
   1816 
   1817 	/* MAC FW will reload PHY FW if 1E.1000.3 was cleaned - #undone */
   1818 	AQ_WRITE_REG(sc, FW_BOOT_EXIT_CODE_REG, RBL_STATUS_DEAD);
   1819 
   1820 	global_software_reset(sc);
   1821 
   1822 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e0);
   1823 
   1824 	/* Wait for RBL to finish boot process. */
   1825 #define RBL_TIMEOUT_MS	10000
   1826 	uint16_t rbl_status;
   1827 	for (timo = RBL_TIMEOUT_MS; timo > 0; timo--) {
   1828 		rbl_status = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG) & 0xffff;
   1829 		if (rbl_status != 0 && rbl_status != RBL_STATUS_DEAD)
   1830 			break;
   1831 		msec_delay(1);
   1832 	}
   1833 	if (timo <= 0) {
   1834 		aprint_error_dev(sc->sc_dev,
   1835 		    "RBL> RBL restart failed: timeout\n");
   1836 		return EBUSY;
   1837 	}
   1838 	switch (rbl_status) {
   1839 	case RBL_STATUS_SUCCESS:
   1840 		if (mode != NULL)
   1841 			*mode = FW_BOOT_MODE_RBL_FLASH;
   1842 		aprint_debug_dev(sc->sc_dev, "RBL> reset complete! [Flash]\n");
   1843 		break;
   1844 	case RBL_STATUS_HOST_BOOT:
   1845 		if (mode != NULL)
   1846 			*mode = FW_BOOT_MODE_RBL_HOST_BOOTLOAD;
   1847 		aprint_debug_dev(sc->sc_dev,
   1848 		    "RBL> reset complete! [Host Bootload]\n");
   1849 		break;
   1850 	case RBL_STATUS_FAILURE:
   1851 	default:
   1852 		aprint_error_dev(sc->sc_dev,
   1853 		    "unknown RBL status 0x%x\n", rbl_status);
   1854 		return EBUSY;
   1855 	}
   1856 
   1857 	return 0;
   1858 }
   1859 
   1860 static int
   1861 mac_soft_reset_flb(struct aq_softc *sc)
   1862 {
   1863 	uint32_t v;
   1864 	int timo;
   1865 
   1866 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
   1867 	/*
   1868 	 * Let Felicity hardware to complete SMBUS transaction before
   1869 	 * Global software reset.
   1870 	 */
   1871 	msec_delay(50);
   1872 
   1873 	/*
   1874 	 * If SPI burst transaction was interrupted(before running the script),
   1875 	 * global software reset may not clear SPI interface.
   1876 	 * Clean it up manually before global reset.
   1877 	 */
   1878 	AQ_WRITE_REG(sc, AQ_GLB_NVR_PROVISIONING2_REG, 0x00a0);
   1879 	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x009f);
   1880 	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x809f);
   1881 	msec_delay(50);
   1882 
   1883 	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
   1884 	v &= ~AQ_FW_SOFTRESET_DIS;
   1885 	v |= AQ_FW_SOFTRESET_RESET;
   1886 	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
   1887 
   1888 	/* Kickstart. */
   1889 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
   1890 	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
   1891 	if (!sc->sc_fast_start_enabled)
   1892 		AQ_WRITE_REG(sc, AQ_GLB_GENERAL_PROVISIONING9_REG, 1);
   1893 
   1894 	/*
   1895 	 * For the case SPI burst transaction was interrupted (by MCP reset
   1896 	 * above), wait until it is completed by hardware.
   1897 	 */
   1898 	msec_delay(50);
   1899 
   1900 	/* MAC Kickstart */
   1901 	if (!sc->sc_fast_start_enabled) {
   1902 		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x180e0);
   1903 
   1904 		uint32_t flb_status;
   1905 		for (timo = 0; timo < 1000; timo++) {
   1906 			flb_status = AQ_READ_REG(sc,
   1907 			    FW_MPI_DAISY_CHAIN_STATUS_REG) & 0x10;
   1908 			if (flb_status != 0)
   1909 				break;
   1910 			msec_delay(1);
   1911 		}
   1912 		if (flb_status == 0) {
   1913 			aprint_error_dev(sc->sc_dev,
   1914 			    "FLB> MAC kickstart failed: timed out\n");
   1915 			return ETIMEDOUT;
   1916 		}
   1917 		aprint_debug_dev(sc->sc_dev,
   1918 		    "FLB> MAC kickstart done, %d ms\n", timo);
   1919 		/* FW reset */
   1920 		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
   1921 		/*
   1922 		 * Let Felicity hardware complete SMBUS transaction before
   1923 		 * Global software reset.
   1924 		 */
   1925 		msec_delay(50);
   1926 		sc->sc_fast_start_enabled = true;
   1927 	}
   1928 	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
   1929 
   1930 	/* PHY Kickstart: #undone */
   1931 	global_software_reset(sc);
   1932 
   1933 	for (timo = 0; timo < 1000; timo++) {
   1934 		if (AQ_READ_REG(sc, AQ_FW_VERSION_REG) != 0)
   1935 			break;
   1936 		msec_delay(10);
   1937 	}
   1938 	if (timo >= 1000) {
   1939 		aprint_error_dev(sc->sc_dev, "FLB> Global Soft Reset failed\n");
   1940 		return ETIMEDOUT;
   1941 	}
   1942 	aprint_debug_dev(sc->sc_dev, "FLB> F/W restart: %d ms\n", timo * 10);
   1943 	return 0;
   1944 
   1945 }
   1946 
   1947 static int
   1948 mac_soft_reset(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
   1949 {
   1950 	if (sc->sc_rbl_enabled)
   1951 		return mac_soft_reset_rbl(sc, mode);
   1952 
   1953 	if (mode != NULL)
   1954 		*mode = FW_BOOT_MODE_FLB;
   1955 	return mac_soft_reset_flb(sc);
   1956 }
   1957 
   1958 static int
   1959 aq_fw_read_version(struct aq_softc *sc)
   1960 {
   1961 	int i, error = EBUSY;
   1962 #define MAC_FW_START_TIMEOUT_MS	10000
   1963 	for (i = 0; i < MAC_FW_START_TIMEOUT_MS; i++) {
   1964 		sc->sc_fw_version = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
   1965 		if (sc->sc_fw_version != 0) {
   1966 			error = 0;
   1967 			break;
   1968 		}
   1969 		delay(1000);
   1970 	}
   1971 	return error;
   1972 }
   1973 
   1974 static int
   1975 aq_fw_reset(struct aq_softc *sc)
   1976 {
   1977 	uint32_t ver, v, bootExitCode;
   1978 	int i, error;
   1979 
   1980 	ver = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
   1981 
   1982 	for (i = 1000; i > 0; i--) {
   1983 		v = AQ_READ_REG(sc, FW_MPI_DAISY_CHAIN_STATUS_REG);
   1984 		bootExitCode = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG);
   1985 		if (v != 0x06000000 || bootExitCode != 0)
   1986 			break;
   1987 	}
   1988 	if (i <= 0) {
   1989 		aprint_error_dev(sc->sc_dev,
   1990 		    "F/W reset failed. Neither RBL nor FLB started\n");
   1991 		return ETIMEDOUT;
   1992 	}
   1993 	sc->sc_rbl_enabled = (bootExitCode != 0);
   1994 
   1995 	/*
   1996 	 * Having FW version 0 is an indicator that cold start
   1997 	 * is in progress. This means two things:
   1998 	 * 1) Driver have to wait for FW/HW to finish boot (500ms giveup)
   1999 	 * 2) Driver may skip reset sequence and save time.
   2000 	 */
   2001 	if (sc->sc_fast_start_enabled && (ver != 0)) {
   2002 		error = aq_fw_read_version(sc);
   2003 		/* Skip reset as it just completed */
   2004 		if (error == 0)
   2005 			return 0;
   2006 	}
   2007 
   2008 	aq_fw_bootloader_mode_t mode = FW_BOOT_MODE_UNKNOWN;
   2009 	error = mac_soft_reset(sc, &mode);
   2010 	if (error != 0) {
   2011 		aprint_error_dev(sc->sc_dev, "MAC reset failed: %d\n", error);
   2012 		return error;
   2013 	}
   2014 
   2015 	switch (mode) {
   2016 	case FW_BOOT_MODE_FLB:
   2017 		aprint_debug_dev(sc->sc_dev,
   2018 		    "FLB> F/W successfully loaded from flash.\n");
   2019 		sc->sc_flash_present = true;
   2020 		return aq_fw_read_version(sc);
   2021 	case FW_BOOT_MODE_RBL_FLASH:
   2022 		aprint_debug_dev(sc->sc_dev,
   2023 		    "RBL> F/W loaded from flash. Host Bootload disabled.\n");
   2024 		sc->sc_flash_present = true;
   2025 		return aq_fw_read_version(sc);
   2026 	case FW_BOOT_MODE_UNKNOWN:
   2027 		aprint_error_dev(sc->sc_dev,
   2028 		    "F/W bootload error: unknown bootloader type\n");
   2029 		return ENOTSUP;
   2030 	case FW_BOOT_MODE_RBL_HOST_BOOTLOAD:
   2031 		aprint_debug_dev(sc->sc_dev, "RBL> Host Bootload mode\n");
   2032 		break;
   2033 	}
   2034 
   2035 	/*
   2036 	 * XXX: TODO: add support Host Boot
   2037 	 */
   2038 	aprint_error_dev(sc->sc_dev,
   2039 	    "RBL> F/W Host Bootload not implemented\n");
   2040 	return ENOTSUP;
   2041 }
   2042 
   2043 static int
   2044 aq_hw_reset(struct aq_softc *sc)
   2045 {
   2046 	int error;
   2047 
   2048 	/* disable irq */
   2049 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS, 0);
   2050 
   2051 	/* apply */
   2052 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_IRQ, 1);
   2053 
   2054 	/* wait ack 10 times by 1ms */
   2055 	WAIT_FOR(
   2056 	    (AQ_READ_REG(sc, AQ_INTR_CTRL_REG) & AQ_INTR_CTRL_RESET_IRQ) == 0,
   2057 	    1000, 10, &error);
   2058 	if (error != 0) {
   2059 		aprint_error_dev(sc->sc_dev,
   2060 		    "atlantic: IRQ reset failed: %d\n", error);
   2061 		return error;
   2062 	}
   2063 
   2064 	return sc->sc_fw_ops->reset(sc);
   2065 }
   2066 
   2067 static int
   2068 aq_hw_init_ucp(struct aq_softc *sc)
   2069 {
   2070 	int timo;
   2071 
   2072 	if (FW_VERSION_MAJOR(sc) == 1) {
   2073 		if (AQ_READ_REG(sc, FW1X_MPI_INIT2_REG) == 0) {
   2074 			uint32_t data;
   2075 			cprng_fast(&data, sizeof(data));
   2076 			data &= 0xfefefefe;
   2077 			data |= 0x02020202;
   2078 			AQ_WRITE_REG(sc, FW1X_MPI_INIT2_REG, data);
   2079 		}
   2080 		AQ_WRITE_REG(sc, FW1X_MPI_INIT1_REG, 0);
   2081 	}
   2082 
   2083 	for (timo = 100; timo > 0; timo--) {
   2084 		sc->sc_mbox_addr = AQ_READ_REG(sc, FW_MPI_MBOX_ADDR_REG);
   2085 		if (sc->sc_mbox_addr != 0)
   2086 			break;
   2087 		delay(1000);
   2088 	}
   2089 
   2090 #define AQ_FW_MIN_VERSION	0x01050006
   2091 #define AQ_FW_MIN_VERSION_STR	"1.5.6"
   2092 	if (sc->sc_fw_version < AQ_FW_MIN_VERSION) {
   2093 		aprint_error_dev(sc->sc_dev,
   2094 		    "atlantic: wrong FW version: " AQ_FW_MIN_VERSION_STR
   2095 		    " or later required, this is %d.%d.%d\n",
   2096 		    FW_VERSION_MAJOR(sc),
   2097 		    FW_VERSION_MINOR(sc),
   2098 		    FW_VERSION_BUILD(sc));
   2099 		return ENOTSUP;
   2100 	}
   2101 
   2102 	return 0;
   2103 }
   2104 
   2105 static int
   2106 aq_fw_version_init(struct aq_softc *sc)
   2107 {
   2108 	int error = 0;
   2109 	char fw_vers[sizeof("F/W version xxxxx.xxxxx.xxxxx")];
   2110 
   2111 	if (FW_VERSION_MAJOR(sc) == 1) {
   2112 		sc->sc_fw_ops = &aq_fw1x_ops;
   2113 	} else if ((FW_VERSION_MAJOR(sc) == 2) || (FW_VERSION_MAJOR(sc) == 3)) {
   2114 		sc->sc_fw_ops = &aq_fw2x_ops;
   2115 	} else {
   2116 		aprint_error_dev(sc->sc_dev,
   2117 		    "Unsupported F/W version %d.%d.%d\n",
   2118 		    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc),
   2119 		    FW_VERSION_BUILD(sc));
   2120 		return ENOTSUP;
   2121 	}
   2122 	snprintf(fw_vers, sizeof(fw_vers), "F/W version %d.%d.%d",
   2123 	    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc), FW_VERSION_BUILD(sc));
   2124 
   2125 	/* detect revision */
   2126 	uint32_t hwrev = AQ_READ_REG(sc, AQ_HW_REVISION_REG);
   2127 	switch (hwrev & 0x0000000f) {
   2128 	case 0x01:
   2129 		aprint_normal_dev(sc->sc_dev, "Atlantic revision A0, %s\n",
   2130 		    fw_vers);
   2131 		sc->sc_features |= FEATURES_REV_A0 |
   2132 		    FEATURES_MPI_AQ | FEATURES_MIPS;
   2133 		break;
   2134 	case 0x02:
   2135 		aprint_normal_dev(sc->sc_dev, "Atlantic revision B0, %s\n",
   2136 		    fw_vers);
   2137 		sc->sc_features |= FEATURES_REV_B0 |
   2138 		    FEATURES_MPI_AQ | FEATURES_MIPS |
   2139 		    FEATURES_TPO2 | FEATURES_RPF2;
   2140 		break;
   2141 	case 0x0A:
   2142 		aprint_normal_dev(sc->sc_dev, "Atlantic revision B1, %s\n",
   2143 		    fw_vers);
   2144 		sc->sc_features |= FEATURES_REV_B1 |
   2145 		    FEATURES_MPI_AQ | FEATURES_MIPS |
   2146 		    FEATURES_TPO2 | FEATURES_RPF2;
   2147 		break;
   2148 	default:
   2149 		aprint_error_dev(sc->sc_dev,
   2150 		    "Unknown revision (0x%08x)\n", hwrev);
   2151 		error = ENOTSUP;
   2152 		break;
   2153 	}
   2154 	return error;
   2155 }
   2156 
   2157 static int
   2158 fw1x_reset(struct aq_softc *sc)
   2159 {
   2160 	struct aq_mailbox_header mbox;
   2161 	const int retryCount = 1000;
   2162 	uint32_t tid0;
   2163 	int i;
   2164 
   2165 	tid0 = ~0;	/*< Initial value of MBOX transactionId. */
   2166 
   2167 	for (i = 0; i < retryCount; ++i) {
   2168 		/*
   2169 		 * Read the beginning of Statistics structure to capture
   2170 		 * the Transaction ID.
   2171 		 */
   2172 		aq_fw_downld_dwords(sc, sc->sc_mbox_addr,
   2173 		    (uint32_t *)&mbox, sizeof(mbox) / sizeof(uint32_t));
   2174 
   2175 		/* Successfully read the stats. */
   2176 		if (tid0 == ~0U) {
   2177 			/* We have read the initial value. */
   2178 			tid0 = mbox.transaction_id;
   2179 			continue;
   2180 		} else if (mbox.transaction_id != tid0) {
   2181 			/*
   2182 			 * Compare transaction ID to initial value.
   2183 			 * If it's different means f/w is alive.
   2184 			 * We're done.
   2185 			 */
   2186 			return 0;
   2187 		}
   2188 
   2189 		/*
   2190 		 * Transaction ID value haven't changed since last time.
   2191 		 * Try reading the stats again.
   2192 		 */
   2193 		delay(10);
   2194 	}
   2195 	aprint_error_dev(sc->sc_dev, "F/W 1.x reset finalize timeout\n");
   2196 	return EBUSY;
   2197 }
   2198 
   2199 static int
   2200 fw1x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
   2201     aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
   2202 {
   2203 	uint32_t mpictrl = 0;
   2204 	uint32_t mpispeed = 0;
   2205 
   2206 	if (speed & AQ_LINK_10G)
   2207 		mpispeed |= FW1X_CTRL_10G;
   2208 	if (speed & AQ_LINK_5G)
   2209 		mpispeed |= (FW1X_CTRL_5G | FW1X_CTRL_5GSR);
   2210 	if (speed & AQ_LINK_2G5)
   2211 		mpispeed |= FW1X_CTRL_2G5;
   2212 	if (speed & AQ_LINK_1G)
   2213 		mpispeed |= FW1X_CTRL_1G;
   2214 	if (speed & AQ_LINK_100M)
   2215 		mpispeed |= FW1X_CTRL_100M;
   2216 
   2217 	mpictrl |= __SHIFTIN(mode, FW1X_MPI_STATE_MODE);
   2218 	mpictrl |= __SHIFTIN(mpispeed, FW1X_MPI_STATE_SPEED);
   2219 	AQ_WRITE_REG(sc, FW1X_MPI_CONTROL_REG, mpictrl);
   2220 	return 0;
   2221 }
   2222 
   2223 static int
   2224 fw1x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
   2225     aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
   2226 {
   2227 	uint32_t mpistate, mpi_speed;
   2228 	aq_link_speed_t speed = AQ_LINK_NONE;
   2229 
   2230 	mpistate = AQ_READ_REG(sc, FW1X_MPI_STATE_REG);
   2231 
   2232 	if (modep != NULL)
   2233 		*modep = __SHIFTOUT(mpistate, FW1X_MPI_STATE_MODE);
   2234 
   2235 	mpi_speed = __SHIFTOUT(mpistate, FW1X_MPI_STATE_SPEED);
   2236 	if (mpi_speed & FW1X_CTRL_10G)
   2237 		speed = AQ_LINK_10G;
   2238 	else if (mpi_speed & (FW1X_CTRL_5G|FW1X_CTRL_5GSR))
   2239 		speed = AQ_LINK_5G;
   2240 	else if (mpi_speed & FW1X_CTRL_2G5)
   2241 		speed = AQ_LINK_2G5;
   2242 	else if (mpi_speed & FW1X_CTRL_1G)
   2243 		speed = AQ_LINK_1G;
   2244 	else if (mpi_speed & FW1X_CTRL_100M)
   2245 		speed = AQ_LINK_100M;
   2246 
   2247 	if (speedp != NULL)
   2248 		*speedp = speed;
   2249 
   2250 	if (fcp != NULL)
   2251 		*fcp = AQ_FC_NONE;
   2252 
   2253 	if (eeep != NULL)
   2254 		*eeep = AQ_EEE_DISABLE;
   2255 
   2256 	return 0;
   2257 }
   2258 
   2259 static int
   2260 fw1x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
   2261 {
   2262 	int error;
   2263 
   2264 	error = aq_fw_downld_dwords(sc,
   2265 	    sc->sc_mbox_addr + offsetof(fw1x_mailbox_t, msm), (uint32_t *)stats,
   2266 	    sizeof(aq_hw_stats_s_t) / sizeof(uint32_t));
   2267 	if (error < 0) {
   2268 		device_printf(sc->sc_dev,
   2269 		    "fw1x> download statistics data FAILED, error %d", error);
   2270 		return error;
   2271 	}
   2272 
   2273 	stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
   2274 	stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
   2275 	return 0;
   2276 }
   2277 
   2278 static int
   2279 fw2x_reset(struct aq_softc *sc)
   2280 {
   2281 	fw2x_capabilities_t caps = { 0 };
   2282 	int error;
   2283 
   2284 	error = aq_fw_downld_dwords(sc,
   2285 	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, caps),
   2286 	    (uint32_t *)&caps, sizeof caps / sizeof(uint32_t));
   2287 	if (error != 0) {
   2288 		aprint_error_dev(sc->sc_dev,
   2289 		    "fw2x> can't get F/W capabilities mask, error %d\n",
   2290 		    error);
   2291 		return error;
   2292 	}
   2293 	sc->sc_fw_caps = caps.caps_lo | ((uint64_t)caps.caps_hi << 32);
   2294 
   2295 	char buf[256];
   2296 	snprintb(buf, sizeof(buf), FW2X_SNPRINTB, sc->sc_fw_caps);
   2297 	aprint_verbose_dev(sc->sc_dev, "fw2x> F/W capabilities=%s\n", buf);
   2298 
   2299 	return 0;
   2300 }
   2301 
   2302 static int
   2303 fw2x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
   2304     aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
   2305 {
   2306 	uint64_t mpi_ctrl;
   2307 	int error = 0;
   2308 
   2309 	AQ_MPI_LOCK(sc);
   2310 
   2311 	mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
   2312 
   2313 	switch (mode) {
   2314 	case MPI_INIT:
   2315 		mpi_ctrl &= ~FW2X_CTRL_RATE_MASK;
   2316 		if (speed & AQ_LINK_10G)
   2317 			mpi_ctrl |= FW2X_CTRL_RATE_10G;
   2318 		if (speed & AQ_LINK_5G)
   2319 			mpi_ctrl |= FW2X_CTRL_RATE_5G;
   2320 		if (speed & AQ_LINK_2G5)
   2321 			mpi_ctrl |= FW2X_CTRL_RATE_2G5;
   2322 		if (speed & AQ_LINK_1G)
   2323 			mpi_ctrl |= FW2X_CTRL_RATE_1G;
   2324 		if (speed & AQ_LINK_100M)
   2325 			mpi_ctrl |= FW2X_CTRL_RATE_100M;
   2326 
   2327 		mpi_ctrl &= ~FW2X_CTRL_LINK_DROP;
   2328 
   2329 		mpi_ctrl &= ~FW2X_CTRL_EEE_MASK;
   2330 		if (eee == AQ_EEE_ENABLE)
   2331 			mpi_ctrl |= FW2X_CTRL_EEE_MASK;
   2332 
   2333 		mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
   2334 		if (fc & AQ_FC_RX)
   2335 			mpi_ctrl |= FW2X_CTRL_PAUSE;
   2336 		if (fc & AQ_FC_TX)
   2337 			mpi_ctrl |= FW2X_CTRL_ASYMMETRIC_PAUSE;
   2338 		break;
   2339 	case MPI_DEINIT:
   2340 		mpi_ctrl &= ~(FW2X_CTRL_RATE_MASK | FW2X_CTRL_EEE_MASK);
   2341 		mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
   2342 		break;
   2343 	default:
   2344 		device_printf(sc->sc_dev, "fw2x> unknown MPI state %d\n", mode);
   2345 		error =  EINVAL;
   2346 		goto failure;
   2347 	}
   2348 	AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
   2349 
   2350  failure:
   2351 	AQ_MPI_UNLOCK(sc);
   2352 	return error;
   2353 }
   2354 
   2355 static int
   2356 fw2x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
   2357     aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
   2358 {
   2359 	uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
   2360 
   2361 	if (modep != NULL) {
   2362 		uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
   2363 		if (mpi_ctrl & FW2X_CTRL_RATE_MASK)
   2364 			*modep = MPI_INIT;
   2365 		else
   2366 			*modep = MPI_DEINIT;
   2367 	}
   2368 
   2369 	aq_link_speed_t speed = AQ_LINK_NONE;
   2370 	if (mpi_state & FW2X_CTRL_RATE_10G)
   2371 		speed = AQ_LINK_10G;
   2372 	else if (mpi_state & FW2X_CTRL_RATE_5G)
   2373 		speed = AQ_LINK_5G;
   2374 	else if (mpi_state & FW2X_CTRL_RATE_2G5)
   2375 		speed = AQ_LINK_2G5;
   2376 	else if (mpi_state & FW2X_CTRL_RATE_1G)
   2377 		speed = AQ_LINK_1G;
   2378 	else if (mpi_state & FW2X_CTRL_RATE_100M)
   2379 		speed = AQ_LINK_100M;
   2380 
   2381 	if (speedp != NULL)
   2382 		*speedp = speed;
   2383 
   2384 	aq_link_fc_t fc = AQ_FC_NONE;
   2385 	if (mpi_state & FW2X_CTRL_PAUSE)
   2386 		fc |= AQ_FC_RX;
   2387 	if (mpi_state & FW2X_CTRL_ASYMMETRIC_PAUSE)
   2388 		fc |= AQ_FC_TX;
   2389 	if (fcp != NULL)
   2390 		*fcp = fc;
   2391 
   2392 	/* XXX: TODO: EEE */
   2393 	if (eeep != NULL)
   2394 		*eeep = AQ_EEE_DISABLE;
   2395 
   2396 	return 0;
   2397 }
   2398 
   2399 static int
   2400 toggle_mpi_ctrl_and_wait(struct aq_softc *sc, uint64_t mask,
   2401     uint32_t timeout_ms, uint32_t try_count)
   2402 {
   2403 	uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
   2404 	uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
   2405 	int error;
   2406 
   2407 	/* First, check that control and state values are consistent */
   2408 	if ((mpi_ctrl & mask) != (mpi_state & mask)) {
   2409 		device_printf(sc->sc_dev,
   2410 		    "fw2x> MPI control (%#llx) and state (%#llx)"
   2411 		    " are not consistent for mask %#llx!\n",
   2412 		    (unsigned long long)mpi_ctrl, (unsigned long long)mpi_state,
   2413 		    (unsigned long long)mask);
   2414 		return EINVAL;
   2415 	}
   2416 
   2417 	/* Invert bits (toggle) in control register */
   2418 	mpi_ctrl ^= mask;
   2419 	AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);
   2420 
   2421 	/* Clear all bits except masked */
   2422 	mpi_ctrl &= mask;
   2423 
   2424 	/* Wait for FW reflecting change in state register */
   2425 	WAIT_FOR((AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG) & mask) == mpi_ctrl,
   2426 	    1000 * timeout_ms, try_count, &error);
   2427 	if (error != 0) {
   2428 		device_printf(sc->sc_dev,
   2429 		    "f/w2x> timeout while waiting for response"
   2430 		    " in state register for bit %#llx!",
   2431 		    (unsigned long long)mask);
   2432 		return error;
   2433 	}
   2434 	return 0;
   2435 }
   2436 
   2437 static int
   2438 fw2x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
   2439 {
   2440 	int error;
   2441 
   2442 	AQ_MPI_LOCK(sc);
   2443 	/* Say to F/W to update the statistics */
   2444 	error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_STATISTICS, 1, 25);
   2445 	if (error != 0) {
   2446 		device_printf(sc->sc_dev,
   2447 		    "fw2x> statistics update error %d\n", error);
   2448 		goto failure;
   2449 	}
   2450 
   2451 	CTASSERT(sizeof(fw2x_msm_statistics_t) <= sizeof(struct aq_hw_stats_s));
   2452 	error = aq_fw_downld_dwords(sc,
   2453 	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, msm), (uint32_t *)stats,
   2454 	    sizeof(fw2x_msm_statistics_t) / sizeof(uint32_t));
   2455 	if (error != 0) {
   2456 		device_printf(sc->sc_dev,
   2457 		    "fw2x> download statistics data FAILED, error %d", error);
   2458 		goto failure;
   2459 	}
   2460 	stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
   2461 	stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
   2462 
   2463  failure:
   2464 	AQ_MPI_UNLOCK(sc);
   2465 	return error;
   2466 }
   2467 
   2468 #if NSYSMON_ENVSYS > 0
   2469 static int
   2470 fw2x_get_temperature(struct aq_softc *sc, uint32_t *temp)
   2471 {
   2472 	int error;
   2473 	uint32_t value, celsius;
   2474 
   2475 	AQ_MPI_LOCK(sc);
   2476 
   2477 	/* Say to F/W to update the temperature */
   2478 	error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_TEMPERATURE, 1, 25);
   2479 	if (error != 0)
   2480 		goto failure;
   2481 
   2482 	error = aq_fw_downld_dwords(sc,
   2483 	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, phy_info2),
   2484 	    &value, sizeof(value) / sizeof(uint32_t));
   2485 	if (error != 0)
   2486 		goto failure;
   2487 
   2488 	/* 1/256 decrees C to microkelvin */
   2489 	celsius = __SHIFTOUT(value, PHYINFO2_TEMPERATURE);
   2490 	if (celsius == 0) {
   2491 		error = EIO;
   2492 		goto failure;
   2493 	}
   2494 	*temp = celsius * (1000000 / 256) + 273150000;
   2495 
   2496  failure:
   2497 	AQ_MPI_UNLOCK(sc);
   2498 	return 0;
   2499 }
   2500 #endif
   2501 
   2502 static int
   2503 aq_fw_downld_dwords(struct aq_softc *sc, uint32_t addr, uint32_t *p,
   2504     uint32_t cnt)
   2505 {
   2506 	uint32_t v;
   2507 	int error = 0;
   2508 
   2509 	WAIT_FOR(AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG) == 1, 1, 10000, &error);
   2510 	if (error != 0) {
   2511 		AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);
   2512 		v = AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG);
   2513 		if (v == 0) {
   2514 			device_printf(sc->sc_dev,
   2515 			    "%s:%d: timeout\n", __func__, __LINE__);
   2516 			return ETIMEDOUT;
   2517 		}
   2518 	}
   2519 
   2520 	AQ_WRITE_REG(sc, AQ_FW_MBOX_ADDR_REG, addr);
   2521 
   2522 	error = 0;
   2523 	for (; cnt > 0 && error == 0; cnt--) {
   2524 		/* execute mailbox interface */
   2525 		AQ_WRITE_REG_BIT(sc, AQ_FW_MBOX_CMD_REG,
   2526 		    AQ_FW_MBOX_CMD_EXECUTE, 1);
   2527 		if (sc->sc_features & FEATURES_REV_B1) {
   2528 			WAIT_FOR(AQ_READ_REG(sc, AQ_FW_MBOX_ADDR_REG) != addr,
   2529 			    1, 1000, &error);
   2530 		} else {
   2531 			WAIT_FOR((AQ_READ_REG(sc, AQ_FW_MBOX_CMD_REG) &
   2532 			    AQ_FW_MBOX_CMD_BUSY) == 0,
   2533 			    1, 1000, &error);
   2534 		}
   2535 		*p++ = AQ_READ_REG(sc, AQ_FW_MBOX_VAL_REG);
   2536 		addr += sizeof(uint32_t);
   2537 	}
   2538 	AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);
   2539 
   2540 	if (error != 0)
   2541 		device_printf(sc->sc_dev,
   2542 		    "%s:%d: timeout\n", __func__, __LINE__);
   2543 
   2544 	return error;
   2545 }
   2546 
   2547 /* read my mac address */
   2548 static int
   2549 aq_get_mac_addr(struct aq_softc *sc)
   2550 {
   2551 	uint32_t mac_addr[2];
   2552 	uint32_t efuse_shadow_addr;
   2553 	int err;
   2554 
   2555 	efuse_shadow_addr = 0;
   2556 	if (FW_VERSION_MAJOR(sc) >= 2)
   2557 		efuse_shadow_addr = AQ_READ_REG(sc, FW2X_MPI_EFUSEADDR_REG);
   2558 	else
   2559 		efuse_shadow_addr = AQ_READ_REG(sc, FW1X_MPI_EFUSEADDR_REG);
   2560 
   2561 	if (efuse_shadow_addr == 0) {
   2562 		aprint_error_dev(sc->sc_dev, "cannot get efuse addr\n");
   2563 		return ENXIO;
   2564 	}
   2565 
   2566 	memset(mac_addr, 0, sizeof(mac_addr));
   2567 	err = aq_fw_downld_dwords(sc, efuse_shadow_addr + (40 * 4),
   2568 	    mac_addr, __arraycount(mac_addr));
   2569 	if (err < 0)
   2570 		return err;
   2571 
   2572 	if (mac_addr[0] == 0 && mac_addr[1] == 0) {
   2573 		aprint_error_dev(sc->sc_dev, "mac address not found\n");
   2574 		return ENXIO;
   2575 	}
   2576 
   2577 	mac_addr[0] = bswap32(mac_addr[0]);
   2578 	mac_addr[1] = bswap32(mac_addr[1]);
   2579 
   2580 	memcpy(sc->sc_enaddr.ether_addr_octet,
   2581 	    (uint8_t *)mac_addr, ETHER_ADDR_LEN);
   2582 	aprint_normal_dev(sc->sc_dev, "Etheraddr: %s\n",
   2583 	    ether_sprintf(sc->sc_enaddr.ether_addr_octet));
   2584 
   2585 	return 0;
   2586 }
   2587 
   2588 /* set multicast filter. index 0 for own address */
   2589 static int
   2590 aq_set_mac_addr(struct aq_softc *sc, int index, uint8_t *enaddr)
   2591 {
   2592 	uint32_t h, l;
   2593 
   2594 	if (index >= AQ_HW_MAC_NUM)
   2595 		return EINVAL;
   2596 
   2597 	if (enaddr == NULL) {
   2598 		/* disable */
   2599 		AQ_WRITE_REG_BIT(sc,
   2600 		    RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
   2601 		return 0;
   2602 	}
   2603 
   2604 	h = (enaddr[0] <<  8) | (enaddr[1]);
   2605 	l = ((uint32_t)enaddr[2] << 24) | (enaddr[3] << 16) |
   2606 	    (enaddr[4] <<  8) | (enaddr[5]);
   2607 
   2608 	/* disable, set, and enable */
   2609 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
   2610 	AQ_WRITE_REG(sc, RPF_L2UC_LSW_REG(index), l);
   2611 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index),
   2612 	    RPF_L2UC_MSW_MACADDR_HI, h);
   2613 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_ACTION, 1);
   2614 	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 1);
   2615 
   2616 	return 0;
   2617 }
   2618 
   2619 static int
   2620 aq_set_capability(struct aq_softc *sc)
   2621 {
   2622 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2623 	int ip4csum_tx =
   2624 	    ((ifp->if_capenable & IFCAP_CSUM_IPv4_Tx) == 0) ? 0 : 1;
   2625 	int ip4csum_rx =
   2626 	    ((ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) == 0) ? 0 : 1;
   2627 	int l4csum_tx = ((ifp->if_capenable &
   2628 	   (IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx |
   2629 	   IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_UDPv6_Tx)) == 0) ? 0 : 1;
   2630 	int l4csum_rx =
   2631 	   ((ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
   2632 	   IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) == 0) ? 0 : 1;
   2633 	uint32_t lso =
   2634 	   ((ifp->if_capenable & (IFCAP_TSOv4 | IFCAP_TSOv6)) == 0) ?
   2635 	   0 : 0xffffffff;
   2636 	uint32_t lro = ((ifp->if_capenable & IFCAP_LRO) == 0) ?
   2637 	    0 : 0xffffffff;
   2638 	uint32_t i, v;
   2639 
   2640 	/* TX checksums offloads*/
   2641 	AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_IP4CSUM_EN, ip4csum_tx);
   2642 	AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_L4CSUM_EN, l4csum_tx);
   2643 
   2644 	/* RX checksums offloads*/
   2645 	AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_IP4CSUM_EN, ip4csum_rx);
   2646 	AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_L4CSUM_EN, l4csum_rx);
   2647 
   2648 	/* LSO offloads*/
   2649 	AQ_WRITE_REG(sc, TDM_LSO_EN_REG, lso);
   2650 
   2651 #define AQ_B0_LRO_RXD_MAX	16
   2652 	v = (8 < AQ_B0_LRO_RXD_MAX) ? 3 :
   2653 	    (4 < AQ_B0_LRO_RXD_MAX) ? 2 :
   2654 	    (2 < AQ_B0_LRO_RXD_MAX) ? 1 : 0;
   2655 	for (i = 0; i < AQ_RINGS_NUM; i++) {
   2656 		AQ_WRITE_REG_BIT(sc, RPO_LRO_LDES_MAX_REG(i),
   2657 		    RPO_LRO_LDES_MAX_MASK(i), v);
   2658 	}
   2659 
   2660 	AQ_WRITE_REG_BIT(sc, RPO_LRO_TB_DIV_REG, RPO_LRO_TB_DIV, 0x61a);
   2661 	AQ_WRITE_REG_BIT(sc, RPO_LRO_INACTIVE_IVAL_REG,
   2662 	    RPO_LRO_INACTIVE_IVAL, 0);
   2663 	/*
   2664 	 * the LRO timebase divider is 5 uS (0x61a),
   2665 	 * to get a maximum coalescing interval of 250 uS,
   2666 	 * we need to multiply by 50(0x32) to get
   2667 	 * the default value 250 uS
   2668 	 */
   2669 	AQ_WRITE_REG_BIT(sc, RPO_LRO_MAX_COALESCING_IVAL_REG,
   2670 	    RPO_LRO_MAX_COALESCING_IVAL, 50);
   2671 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
   2672 	    RPO_LRO_CONF_QSESSION_LIMIT, 1);
   2673 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
   2674 	    RPO_LRO_CONF_TOTAL_DESC_LIMIT, 2);
   2675 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
   2676 	    RPO_LRO_CONF_PATCHOPTIMIZATION_EN, 0);
   2677 	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
   2678 	    RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT, 10);
   2679 	AQ_WRITE_REG(sc, RPO_LRO_RSC_MAX_REG, 1);
   2680 	AQ_WRITE_REG(sc, RPO_LRO_ENABLE_REG, lro);
   2681 
   2682 	return 0;
   2683 }
   2684 
   2685 static int
   2686 aq_set_filter(struct aq_softc *sc)
   2687 {
   2688 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2689 	struct ethercom *ec = &sc->sc_ethercom;
   2690 	struct ether_multi *enm;
   2691 	struct ether_multistep step;
   2692 	int idx, error = 0;
   2693 
   2694 	if (ifp->if_flags & IFF_PROMISC) {
   2695 		AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_PROMISC,
   2696 		    (ifp->if_flags & IFF_PROMISC) ? 1 : 0);
   2697 		ec->ec_flags |= ETHER_F_ALLMULTI;
   2698 		goto done;
   2699 	}
   2700 
   2701 	/* clear all table */
   2702 	for (idx = 0; idx < AQ_HW_MAC_NUM; idx++) {
   2703 		if (idx == AQ_HW_MAC_OWN)	/* already used for own */
   2704 			continue;
   2705 		aq_set_mac_addr(sc, idx, NULL);
   2706 	}
   2707 
   2708 	/* don't accept all multicast */
   2709 	AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
   2710 	    RPF_MCAST_FILTER_MASK_ALLMULTI, 0);
   2711 	AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
   2712 	    RPF_MCAST_FILTER_EN, 0);
   2713 
   2714 	idx = 0;
   2715 	ETHER_LOCK(ec);
   2716 	ETHER_FIRST_MULTI(step, ec, enm);
   2717 	while (enm != NULL) {
   2718 		if (idx == AQ_HW_MAC_OWN)
   2719 			idx++;
   2720 
   2721 		if ((idx >= AQ_HW_MAC_NUM) ||
   2722 		    memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
   2723 			/*
   2724 			 * too many filters.
   2725 			 * fallback to accept all multicast addresses.
   2726 			 */
   2727 			AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
   2728 			    RPF_MCAST_FILTER_MASK_ALLMULTI, 1);
   2729 			AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
   2730 			    RPF_MCAST_FILTER_EN, 1);
   2731 			ec->ec_flags |= ETHER_F_ALLMULTI;
   2732 			ETHER_UNLOCK(ec);
   2733 			goto done;
   2734 		}
   2735 
   2736 		/* add a filter */
   2737 		aq_set_mac_addr(sc, idx++, enm->enm_addrlo);
   2738 
   2739 		ETHER_NEXT_MULTI(step, enm);
   2740 	}
   2741 	ec->ec_flags &= ~ETHER_F_ALLMULTI;
   2742 	ETHER_UNLOCK(ec);
   2743 
   2744  done:
   2745 	return error;
   2746 }
   2747 
   2748 static int
   2749 aq_ifmedia_change(struct ifnet * const ifp)
   2750 {
   2751 	struct aq_softc *sc = ifp->if_softc;
   2752 	aq_link_speed_t rate = AQ_LINK_NONE;
   2753 	aq_link_fc_t fc = AQ_FC_NONE;
   2754 	aq_link_eee_t eee = AQ_EEE_DISABLE;
   2755 
   2756 	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
   2757 		return EINVAL;
   2758 
   2759 	switch (IFM_SUBTYPE(sc->sc_media.ifm_media)) {
   2760 	case IFM_AUTO:
   2761 		rate = AQ_LINK_AUTO;
   2762 		break;
   2763 	case IFM_NONE:
   2764 		rate = AQ_LINK_NONE;
   2765 		break;
   2766 	case IFM_100_TX:
   2767 		rate = AQ_LINK_100M;
   2768 		break;
   2769 	case IFM_1000_T:
   2770 		rate = AQ_LINK_1G;
   2771 		break;
   2772 	case IFM_2500_T:
   2773 		rate = AQ_LINK_2G5;
   2774 		break;
   2775 	case IFM_5000_T:
   2776 		rate = AQ_LINK_5G;
   2777 		break;
   2778 	case IFM_10G_T:
   2779 		rate = AQ_LINK_10G;
   2780 		break;
   2781 	default:
   2782 		device_printf(sc->sc_dev, "unknown media: 0x%X\n",
   2783 		    IFM_SUBTYPE(sc->sc_media.ifm_media));
   2784 		return ENODEV;
   2785 	}
   2786 
   2787 	if (sc->sc_media.ifm_media & IFM_FLOW)
   2788 		fc = AQ_FC_ALL;
   2789 
   2790 	/* XXX: todo EEE */
   2791 
   2792 	/* re-initialize hardware with new parameters */
   2793 	aq_set_linkmode(sc, rate, fc, eee);
   2794 
   2795 	return 0;
   2796 }
   2797 
   2798 static void
   2799 aq_ifmedia_status(struct ifnet * const ifp, struct ifmediareq *ifmr)
   2800 {
   2801 	struct aq_softc *sc = ifp->if_softc;
   2802 
   2803 	/* update ifm_active */
   2804 	ifmr->ifm_active = IFM_ETHER;
   2805 	if (sc->sc_link_fc & AQ_FC_RX)
   2806 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
   2807 	if (sc->sc_link_fc & AQ_FC_TX)
   2808 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
   2809 
   2810 	switch (sc->sc_link_rate) {
   2811 	case AQ_LINK_100M:
   2812 		/* XXX: need to detect fulldup or halfdup */
   2813 		ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
   2814 		break;
   2815 	case AQ_LINK_1G:
   2816 		ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
   2817 		break;
   2818 	case AQ_LINK_2G5:
   2819 		ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
   2820 		break;
   2821 	case AQ_LINK_5G:
   2822 		ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
   2823 		break;
   2824 	case AQ_LINK_10G:
   2825 		ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
   2826 		break;
   2827 	default:
   2828 		ifmr->ifm_active |= IFM_NONE;
   2829 		break;
   2830 	}
   2831 
   2832 	/* update ifm_status */
   2833 	ifmr->ifm_status = IFM_AVALID;
   2834 	if (sc->sc_link_rate != AQ_LINK_NONE)
   2835 		ifmr->ifm_status |= IFM_ACTIVE;
   2836 }
   2837 
   2838 static void
   2839 aq_initmedia(struct aq_softc *sc)
   2840 {
   2841 #define IFMEDIA_ETHER_ADD(sc, media)	\
   2842 	ifmedia_add(&(sc)->sc_media, IFM_ETHER | media, 0, NULL);
   2843 
   2844 	IFMEDIA_ETHER_ADD(sc, IFM_NONE);
   2845 	if (sc->sc_available_rates & AQ_LINK_100M) {
   2846 		IFMEDIA_ETHER_ADD(sc, IFM_100_TX);
   2847 		IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FLOW);
   2848 		IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FDX | IFM_FLOW);
   2849 	}
   2850 	if (sc->sc_available_rates & AQ_LINK_1G) {
   2851 		IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX);
   2852 		IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX | IFM_FLOW);
   2853 	}
   2854 	if (sc->sc_available_rates & AQ_LINK_2G5) {
   2855 		IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX);
   2856 		IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX | IFM_FLOW);
   2857 	}
   2858 	if (sc->sc_available_rates & AQ_LINK_5G) {
   2859 		IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX);
   2860 		IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX | IFM_FLOW);
   2861 	}
   2862 	if (sc->sc_available_rates & AQ_LINK_10G) {
   2863 		IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX);
   2864 		IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX | IFM_FLOW);
   2865 	}
   2866 	IFMEDIA_ETHER_ADD(sc, IFM_AUTO);
   2867 	IFMEDIA_ETHER_ADD(sc, IFM_AUTO | IFM_FLOW);
   2868 
   2869 	/* default: auto without flowcontrol */
   2870 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
   2871 	aq_set_linkmode(sc, AQ_LINK_AUTO, AQ_FC_NONE, AQ_EEE_DISABLE);
   2872 }
   2873 
   2874 static int
   2875 aq_set_linkmode(struct aq_softc *sc, aq_link_speed_t speed, aq_link_fc_t fc,
   2876     aq_link_eee_t eee)
   2877 {
   2878 	return sc->sc_fw_ops->set_mode(sc, MPI_INIT, speed, fc, eee);
   2879 }
   2880 
   2881 static int
   2882 aq_get_linkmode(struct aq_softc *sc, aq_link_speed_t *speed, aq_link_fc_t *fc,
   2883    aq_link_eee_t *eee)
   2884 {
   2885 	aq_hw_fw_mpi_state_t mode;
   2886 	int error;
   2887 
   2888 	error = sc->sc_fw_ops->get_mode(sc, &mode, speed, fc, eee);
   2889 	if (error != 0)
   2890 		return error;
   2891 	if (mode != MPI_INIT)
   2892 		return ENXIO;
   2893 
   2894 	return 0;
   2895 }
   2896 
   2897 static void
   2898 aq_hw_init_tx_path(struct aq_softc *sc)
   2899 {
   2900 	/* Tx TC/RSS number config */
   2901 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_TC_MODE_EN, 1);
   2902 
   2903 	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
   2904 	    THM_LSO_TCP_FLAG1_FIRST, 0x0ff6);
   2905 	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
   2906 	    THM_LSO_TCP_FLAG1_MID,   0x0ff6);
   2907 	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG2_REG,
   2908 	   THM_LSO_TCP_FLAG2_LAST,  0x0f7f);
   2909 
   2910 	/* misc */
   2911 	AQ_WRITE_REG(sc, TX_TPO2_REG,
   2912 	   (sc->sc_features & FEATURES_TPO2) ? TX_TPO2_EN : 0);
   2913 	AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_EN, 0);
   2914 	AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_MODE, 0);
   2915 
   2916 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_SCP_INS_EN, 1);
   2917 }
   2918 
   2919 static void
   2920 aq_hw_init_rx_path(struct aq_softc *sc)
   2921 {
   2922 	int i;
   2923 
   2924 	/* clear setting */
   2925 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 0);
   2926 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 0);
   2927 	AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG, 0);
   2928 	for (i = 0; i < 32; i++) {
   2929 		AQ_WRITE_REG_BIT(sc, RPF_ETHERTYPE_FILTER_REG(i),
   2930 		   RPF_ETHERTYPE_FILTER_EN, 0);
   2931 	}
   2932 
   2933 	if (sc->sc_rss_enable) {
   2934 		/* Rx TC/RSS number config */
   2935 		AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 1);
   2936 
   2937 		/* Rx flow control */
   2938 		AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 1);
   2939 
   2940 		/* RSS Ring selection */
   2941 		switch (sc->sc_nqueues) {
   2942 		case 2:
   2943 			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
   2944 			    RX_FLR_RSS_CONTROL1_EN | 0x11111111);
   2945 			break;
   2946 		case 4:
   2947 			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
   2948 			    RX_FLR_RSS_CONTROL1_EN | 0x22222222);
   2949 			break;
   2950 		case 8:
   2951 			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
   2952 			    RX_FLR_RSS_CONTROL1_EN | 0x33333333);
   2953 			break;
   2954 		}
   2955 	}
   2956 
   2957 	/* L2 and Multicast filters */
   2958 	for (i = 0; i < AQ_HW_MAC_NUM; i++) {
   2959 		AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_EN, 0);
   2960 		AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_ACTION,
   2961 		    RPF_ACTION_HOST);
   2962 	}
   2963 	AQ_WRITE_REG(sc, RPF_MCAST_FILTER_MASK_REG, 0);
   2964 	AQ_WRITE_REG(sc, RPF_MCAST_FILTER_REG(0), 0x00010fff);
   2965 
   2966 	/* Vlan filters */
   2967 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_OUTER,
   2968 	    ETHERTYPE_QINQ);
   2969 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_INNER,
   2970 	    ETHERTYPE_VLAN);
   2971 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);
   2972 
   2973 	if (sc->sc_features & FEATURES_REV_B) {
   2974 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
   2975 		    RPF_VLAN_MODE_ACCEPT_UNTAGGED, 1);
   2976 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
   2977 		    RPF_VLAN_MODE_UNTAGGED_ACTION, RPF_ACTION_HOST);
   2978 	}
   2979 
   2980 	/* misc */
   2981 	if (sc->sc_features & FEATURES_RPF2)
   2982 		AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, RX_TCP_RSS_HASH_RPF2);
   2983 	else
   2984 		AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, 0);
   2985 
   2986 	/*
   2987 	 * XXX: RX_TCP_RSS_HASH_REG:
   2988 	 *  linux   set 0x000f0000
   2989 	 *  freebsd set 0x000f001e
   2990 	 */
   2991 	/* RSS hash type set for IP/TCP */
   2992 	AQ_WRITE_REG_BIT(sc, RX_TCP_RSS_HASH_REG,
   2993 	    RX_TCP_RSS_HASH_TYPE, 0x001e);
   2994 
   2995 	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_EN, 1);
   2996 	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_ACTION, RPF_ACTION_HOST);
   2997 	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_THRESHOLD, 0xffff);
   2998 
   2999 	AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_EN, 0);
   3000 	AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_MODE, 0);
   3001 }
   3002 
   3003 static void
   3004 aq_hw_interrupt_moderation_set(struct aq_softc *sc)
   3005 {
   3006 	int i;
   3007 
   3008 	if (sc->sc_intr_moderation_enable) {
   3009 		unsigned int tx_min, rx_min;	/* 0-255 */
   3010 		unsigned int tx_max, rx_max;	/* 0-511? */
   3011 
   3012 		switch (sc->sc_link_rate) {
   3013 		case AQ_LINK_100M:
   3014 			tx_min = 0x4f;
   3015 			tx_max = 0xff;
   3016 			rx_min = 0x04;
   3017 			rx_max = 0x50;
   3018 			break;
   3019 		case AQ_LINK_1G:
   3020 		default:
   3021 			tx_min = 0x4f;
   3022 			tx_max = 0xff;
   3023 			rx_min = 0x30;
   3024 			rx_max = 0x80;
   3025 			break;
   3026 		case AQ_LINK_2G5:
   3027 			tx_min = 0x4f;
   3028 			tx_max = 0xff;
   3029 			rx_min = 0x18;
   3030 			rx_max = 0xe0;
   3031 			break;
   3032 		case AQ_LINK_5G:
   3033 			tx_min = 0x4f;
   3034 			tx_max = 0xff;
   3035 			rx_min = 0x0c;
   3036 			rx_max = 0x70;
   3037 			break;
   3038 		case AQ_LINK_10G:
   3039 			tx_min = 0x4f;
   3040 			tx_max = 0x1ff;
   3041 			rx_min = 0x06;	/* freebsd use 80 */
   3042 			rx_max = 0x38;	/* freebsd use 120 */
   3043 			break;
   3044 		}
   3045 
   3046 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
   3047 		    TX_DMA_INT_DESC_WRWB_EN, 0);
   3048 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
   3049 		    TX_DMA_INT_DESC_MODERATE_EN, 1);
   3050 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
   3051 		    RX_DMA_INT_DESC_WRWB_EN, 0);
   3052 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
   3053 		    RX_DMA_INT_DESC_MODERATE_EN, 1);
   3054 
   3055 		for (i = 0; i < AQ_RINGS_NUM; i++) {
   3056 			AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i),
   3057 			    __SHIFTIN(tx_min, TX_INTR_MODERATION_CTL_MIN) |
   3058 			    __SHIFTIN(tx_max, TX_INTR_MODERATION_CTL_MAX) |
   3059 			    TX_INTR_MODERATION_CTL_EN);
   3060 		}
   3061 		for (i = 0; i < AQ_RINGS_NUM; i++) {
   3062 			AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i),
   3063 			    __SHIFTIN(rx_min, RX_INTR_MODERATION_CTL_MIN) |
   3064 			    __SHIFTIN(rx_max, RX_INTR_MODERATION_CTL_MAX) |
   3065 			    RX_INTR_MODERATION_CTL_EN);
   3066 		}
   3067 
   3068 	} else {
   3069 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
   3070 		    TX_DMA_INT_DESC_WRWB_EN, 1);
   3071 		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
   3072 		    TX_DMA_INT_DESC_MODERATE_EN, 0);
   3073 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
   3074 		    RX_DMA_INT_DESC_WRWB_EN, 1);
   3075 		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
   3076 		    RX_DMA_INT_DESC_MODERATE_EN, 0);
   3077 
   3078 		for (i = 0; i < AQ_RINGS_NUM; i++) {
   3079 			AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i), 0);
   3080 		}
   3081 		for (i = 0; i < AQ_RINGS_NUM; i++) {
   3082 			AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i), 0);
   3083 		}
   3084 	}
   3085 }
   3086 
   3087 static void
   3088 aq_hw_qos_set(struct aq_softc *sc)
   3089 {
   3090 	uint32_t tc = 0;
   3091 	uint32_t buff_size;
   3092 
   3093 	/* TPS Descriptor rate init */
   3094 	AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_TA_RST, 0);
   3095 	AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_LIM, 0xa);
   3096 
   3097 	/* TPS VM init */
   3098 	AQ_WRITE_REG_BIT(sc, TPS_DESC_VM_ARB_MODE_REG, TPS_DESC_VM_ARB_MODE, 0);
   3099 
   3100 	/* TPS TC credits init */
   3101 	AQ_WRITE_REG_BIT(sc, TPS_DESC_TC_ARB_MODE_REG, TPS_DESC_TC_ARB_MODE, 0);
   3102 	AQ_WRITE_REG_BIT(sc, TPS_DATA_TC_ARB_MODE_REG, TPS_DATA_TC_ARB_MODE, 0);
   3103 
   3104 	AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
   3105 	    TPS_DATA_TCT_CREDIT_MAX, 0xfff);
   3106 	AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
   3107 	    TPS_DATA_TCT_WEIGHT, 0x64);
   3108 	AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
   3109 	    TPS_DESC_TCT_CREDIT_MAX, 0x50);
   3110 	AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
   3111 	    TPS_DESC_TCT_WEIGHT, 0x1e);
   3112 
   3113 	/* Tx buf size */
   3114 	tc = 0;
   3115 	buff_size = AQ_HW_TXBUF_MAX;
   3116 	AQ_WRITE_REG_BIT(sc, TPB_TXB_BUFSIZE_REG(tc), TPB_TXB_BUFSIZE,
   3117 	    buff_size);
   3118 	AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_HI,
   3119 	    (buff_size * (1024 / 32) * 66) / 100);
   3120 	AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_LO,
   3121 	    (buff_size * (1024 / 32) * 50) / 100);
   3122 
   3123 	/* QoS Rx buf size per TC */
   3124 	tc = 0;
   3125 	buff_size = AQ_HW_RXBUF_MAX;
   3126 	AQ_WRITE_REG_BIT(sc, RPB_RXB_BUFSIZE_REG(tc), RPB_RXB_BUFSIZE,
   3127 	    buff_size);
   3128 	AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_EN, 0);
   3129 	AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_HI,
   3130 	    (buff_size * (1024 / 32) * 66) / 100);
   3131 	AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_LO,
   3132 	    (buff_size * (1024 / 32) * 50) / 100);
   3133 
   3134 	/* QoS 802.1p priority -> TC mapping */
   3135 	int i_priority;
   3136 	for (i_priority = 0; i_priority < 8; i_priority++) {
   3137 		AQ_WRITE_REG_BIT(sc, RPF_RPB_RX_TC_UPT_REG,
   3138 		    RPF_RPB_RX_TC_UPT_MASK(i_priority), 0);
   3139 	}
   3140 }
   3141 
   3142 /* called once from aq_attach */
   3143 static int
   3144 aq_init_rss(struct aq_softc *sc)
   3145 {
   3146 	CTASSERT(AQ_RSS_HASHKEY_SIZE == RSS_KEYSIZE);
   3147 	uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
   3148 	uint8_t rss_table[AQ_RSS_INDIRECTION_TABLE_MAX];
   3149 	unsigned int i;
   3150 	int error;
   3151 
   3152 	/* initialize rss key */
   3153 	rss_getkey((uint8_t *)rss_key);
   3154 
   3155 	/* hash to ring table */
   3156 	for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
   3157 		rss_table[i] = i % sc->sc_nqueues;
   3158 	}
   3159 
   3160 	/*
   3161 	 * set rss key
   3162 	 */
   3163 	for (i = 0; i < __arraycount(rss_key); i++) {
   3164 		uint32_t key_data = sc->sc_rss_enable ? ntohl(rss_key[i]) : 0;
   3165 		AQ_WRITE_REG(sc, RPF_RSS_KEY_WR_DATA_REG, key_data);
   3166 		AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
   3167 		    RPF_RSS_KEY_ADDR, __arraycount(rss_key) - 1 - i);
   3168 		AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
   3169 		    RPF_RSS_KEY_WR_EN, 1);
   3170 		WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
   3171 		    RPF_RSS_KEY_WR_EN) == 0, 1000, 10, &error);
   3172 		if (error != 0) {
   3173 			device_printf(sc->sc_dev, "%s: rss key write timeout\n",
   3174 			    __func__);
   3175 			goto rss_set_timeout;
   3176 		}
   3177 	}
   3178 
   3179 	/*
   3180 	 * set rss indirection table
   3181 	 *
   3182 	 * AQ's rss redirect table is consist of 3bit*64 (192bit) packed array.
   3183 	 * we'll make it by __BITMAP(3) macros.
   3184 	 */
   3185 	__BITMAP_TYPE(, uint16_t, 3 * AQ_RSS_INDIRECTION_TABLE_MAX) bit3x64;
   3186 	__BITMAP_ZERO(&bit3x64);
   3187 
   3188 #define AQ_3BIT_PACKED_ARRAY_SET(bitmap, idx, val)		\
   3189 	do {							\
   3190 		if (val & 1) {					\
   3191 			__BITMAP_SET((idx) * 3, (bitmap));	\
   3192 		} else {					\
   3193 			__BITMAP_CLR((idx) * 3, (bitmap));	\
   3194 		}						\
   3195 		if (val & 2) {					\
   3196 			__BITMAP_SET((idx) * 3 + 1, (bitmap));	\
   3197 		} else {					\
   3198 			__BITMAP_CLR((idx) * 3 + 1, (bitmap));	\
   3199 		}						\
   3200 		if (val & 4) {					\
   3201 			__BITMAP_SET((idx) * 3 + 2, (bitmap));	\
   3202 		} else {					\
   3203 			__BITMAP_CLR((idx) * 3 + 2, (bitmap));	\
   3204 		}						\
   3205 	} while (0 /* CONSTCOND */)
   3206 
   3207 	for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
   3208 		AQ_3BIT_PACKED_ARRAY_SET(&bit3x64, i, rss_table[i]);
   3209 	}
   3210 
   3211 	/* write 192bit data in steps of 16bit */
   3212 	for (i = 0; i < (int)__arraycount(bit3x64._b); i++) {
   3213 		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_WR_DATA_REG,
   3214 		    RPF_RSS_REDIR_WR_DATA, bit3x64._b[i]);
   3215 		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
   3216 		    RPF_RSS_REDIR_ADDR, i);
   3217 		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
   3218 		    RPF_RSS_REDIR_WR_EN, 1);
   3219 
   3220 		WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
   3221 		    RPF_RSS_REDIR_WR_EN) == 0, 1000, 10, &error);
   3222 		if (error != 0)
   3223 			break;
   3224 	}
   3225 
   3226  rss_set_timeout:
   3227 	return error;
   3228 }
   3229 
   3230 static void
   3231 aq_hw_l3_filter_set(struct aq_softc *sc)
   3232 {
   3233 	int i;
   3234 
   3235 	/* clear all filter */
   3236 	for (i = 0; i < 8; i++) {
   3237 		AQ_WRITE_REG_BIT(sc, RPF_L3_FILTER_REG(i),
   3238 		    RPF_L3_FILTER_L4_EN, 0);
   3239 	}
   3240 }
   3241 
   3242 static void
   3243 aq_set_vlan_filters(struct aq_softc *sc)
   3244 {
   3245 	struct ethercom *ec = &sc->sc_ethercom;
   3246 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   3247 	struct vlanid_list *vlanidp;
   3248 	int i;
   3249 
   3250 	ETHER_LOCK(ec);
   3251 
   3252 	/* disable all vlan filters */
   3253 	for (i = 0; i < RPF_VLAN_MAX_FILTERS; i++)
   3254 		AQ_WRITE_REG(sc, RPF_VLAN_FILTER_REG(i), 0);
   3255 
   3256 	/* count VID */
   3257 	i = 0;
   3258 	SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list)
   3259 		i++;
   3260 
   3261 	if (((sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWFILTER) == 0) ||
   3262 	    (ifp->if_flags & IFF_PROMISC) ||
   3263 	    (i > RPF_VLAN_MAX_FILTERS)) {
   3264 		/*
   3265 		 * no vlan hwfilter, in promiscuous mode, or too many VID?
   3266 		 * must receive all VID
   3267 		 */
   3268 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
   3269 		    RPF_VLAN_MODE_PROMISC, 1);
   3270 		goto done;
   3271 	}
   3272 
   3273 	/* receive only selected VID */
   3274 	AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);
   3275 	i = 0;
   3276 	SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
   3277 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
   3278 		    RPF_VLAN_FILTER_EN, 1);
   3279 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
   3280 		    RPF_VLAN_FILTER_RXQ_EN, 0);
   3281 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
   3282 		    RPF_VLAN_FILTER_RXQ, 0);
   3283 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
   3284 		    RPF_VLAN_FILTER_ACTION, RPF_ACTION_HOST);
   3285 		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
   3286 		    RPF_VLAN_FILTER_ID, vlanidp->vid);
   3287 		i++;
   3288 	}
   3289 
   3290  done:
   3291 	ETHER_UNLOCK(ec);
   3292 }
   3293 
   3294 static int
   3295 aq_hw_init(struct aq_softc *sc)
   3296 {
   3297 	uint32_t v;
   3298 
   3299 	/* Force limit MRRS on RDM/TDM to 2K */
   3300 	v = AQ_READ_REG(sc, AQ_PCI_REG_CONTROL_6_REG);
   3301 	AQ_WRITE_REG(sc, AQ_PCI_REG_CONTROL_6_REG, (v & ~0x0707) | 0x0404);
   3302 
   3303 	/*
   3304 	 * TX DMA total request limit. B0 hardware is not capable to
   3305 	 * handle more than (8K-MRRS) incoming DMA data.
   3306 	 * Value 24 in 256byte units
   3307 	 */
   3308 	AQ_WRITE_REG(sc, AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG, 24);
   3309 
   3310 	aq_hw_init_tx_path(sc);
   3311 	aq_hw_init_rx_path(sc);
   3312 
   3313 	aq_hw_interrupt_moderation_set(sc);
   3314 
   3315 	aq_set_mac_addr(sc, AQ_HW_MAC_OWN, sc->sc_enaddr.ether_addr_octet);
   3316 	aq_set_linkmode(sc, AQ_LINK_NONE, AQ_FC_NONE, AQ_EEE_DISABLE);
   3317 
   3318 	aq_hw_qos_set(sc);
   3319 
   3320 	/* Enable interrupt */
   3321 	int irqmode;
   3322 	if (sc->sc_msix)
   3323 		irqmode =  AQ_INTR_CTRL_IRQMODE_MSIX;
   3324 	else
   3325 		irqmode =  AQ_INTR_CTRL_IRQMODE_MSI;
   3326 
   3327 	AQ_WRITE_REG(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS);
   3328 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_MULTIVEC,
   3329 	    sc->sc_msix ? 1 : 0);
   3330 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_IRQMODE, irqmode);
   3331 
   3332 	AQ_WRITE_REG(sc, AQ_INTR_AUTOMASK_REG, 0xffffffff);
   3333 
   3334 	AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(0),
   3335 	    ((AQ_B0_ERR_INT << 24) | (1U << 31)) |
   3336 	    ((AQ_B0_ERR_INT << 16) | (1 << 23))
   3337 	);
   3338 
   3339 	/* link interrupt */
   3340 	if (!sc->sc_msix)
   3341 		sc->sc_linkstat_irq = AQ_LINKSTAT_IRQ;
   3342 	AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(3),
   3343 	    __BIT(7) | sc->sc_linkstat_irq);
   3344 
   3345 	return 0;
   3346 }
   3347 
   3348 static int
   3349 aq_update_link_status(struct aq_softc *sc)
   3350 {
   3351 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   3352 	aq_link_speed_t rate = AQ_LINK_NONE;
   3353 	aq_link_fc_t fc = AQ_FC_NONE;
   3354 	aq_link_eee_t eee = AQ_EEE_DISABLE;
   3355 	unsigned int speed;
   3356 	int changed = 0;
   3357 
   3358 	aq_get_linkmode(sc, &rate, &fc, &eee);
   3359 
   3360 	if (sc->sc_link_rate != rate)
   3361 		changed = 1;
   3362 	if (sc->sc_link_fc != fc)
   3363 		changed = 1;
   3364 	if (sc->sc_link_eee != eee)
   3365 		changed = 1;
   3366 
   3367 	if (changed) {
   3368 		switch (rate) {
   3369 		case AQ_LINK_100M:
   3370 			speed = 100;
   3371 			break;
   3372 		case AQ_LINK_1G:
   3373 			speed = 1000;
   3374 			break;
   3375 		case AQ_LINK_2G5:
   3376 			speed = 2500;
   3377 			break;
   3378 		case AQ_LINK_5G:
   3379 			speed = 5000;
   3380 			break;
   3381 		case AQ_LINK_10G:
   3382 			speed = 10000;
   3383 			break;
   3384 		case AQ_LINK_NONE:
   3385 		default:
   3386 			speed = 0;
   3387 			break;
   3388 		}
   3389 
   3390 		if (sc->sc_link_rate == AQ_LINK_NONE) {
   3391 			/* link DOWN -> UP */
   3392 			device_printf(sc->sc_dev, "link is UP: speed=%u\n",
   3393 			    speed);
   3394 			if_link_state_change(ifp, LINK_STATE_UP);
   3395 		} else if (rate == AQ_LINK_NONE) {
   3396 			/* link UP -> DOWN */
   3397 			device_printf(sc->sc_dev, "link is DOWN\n");
   3398 			if_link_state_change(ifp, LINK_STATE_DOWN);
   3399 		} else {
   3400 			device_printf(sc->sc_dev,
   3401 			    "link mode changed: speed=%u, fc=0x%x, eee=%x\n",
   3402 			    speed, fc, eee);
   3403 		}
   3404 
   3405 		sc->sc_link_rate = rate;
   3406 		sc->sc_link_fc = fc;
   3407 		sc->sc_link_eee = eee;
   3408 
   3409 		/* update interrupt timing according to new link speed */
   3410 		aq_hw_interrupt_moderation_set(sc);
   3411 	}
   3412 
   3413 	return changed;
   3414 }
   3415 
   3416 #ifdef AQ_EVENT_COUNTERS
   3417 static void
   3418 aq_update_statistics(struct aq_softc *sc)
   3419 {
   3420 	int prev = sc->sc_statistics_idx;
   3421 	int cur = prev ^ 1;
   3422 
   3423 	sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[cur]);
   3424 
   3425 	/*
   3426 	 * aq's internal statistics counter is 32bit.
   3427 	 * cauculate delta, and add to evcount
   3428 	 */
   3429 #define ADD_DELTA(cur, prev, name)				\
   3430 	do {							\
   3431 		uint32_t n;					\
   3432 		n = (uint32_t)(sc->sc_statistics[cur].name -	\
   3433 		    sc->sc_statistics[prev].name);		\
   3434 		if (n != 0) {					\
   3435 			AQ_EVCNT_ADD(sc, name, n);		\
   3436 		}						\
   3437 	} while (/*CONSTCOND*/0);
   3438 
   3439 	ADD_DELTA(cur, prev, uprc);
   3440 	ADD_DELTA(cur, prev, mprc);
   3441 	ADD_DELTA(cur, prev, bprc);
   3442 	ADD_DELTA(cur, prev, prc);
   3443 	ADD_DELTA(cur, prev, erpr);
   3444 	ADD_DELTA(cur, prev, uptc);
   3445 	ADD_DELTA(cur, prev, mptc);
   3446 	ADD_DELTA(cur, prev, bptc);
   3447 	ADD_DELTA(cur, prev, ptc);
   3448 	ADD_DELTA(cur, prev, erpt);
   3449 	ADD_DELTA(cur, prev, mbtc);
   3450 	ADD_DELTA(cur, prev, bbtc);
   3451 	ADD_DELTA(cur, prev, mbrc);
   3452 	ADD_DELTA(cur, prev, bbrc);
   3453 	ADD_DELTA(cur, prev, ubrc);
   3454 	ADD_DELTA(cur, prev, ubtc);
   3455 	ADD_DELTA(cur, prev, dpc);
   3456 	ADD_DELTA(cur, prev, cprc);
   3457 
   3458 	sc->sc_statistics_idx = cur;
   3459 }
   3460 #endif /* AQ_EVENT_COUNTERS */
   3461 
   3462 /* allocate and map one DMA block */
   3463 static int
   3464 _alloc_dma(struct aq_softc *sc, bus_size_t size, bus_size_t *sizep,
   3465     void **addrp, bus_dmamap_t *mapp, bus_dma_segment_t *seg)
   3466 {
   3467 	int nsegs, error;
   3468 
   3469 	if ((error = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, seg,
   3470 	    1, &nsegs, 0)) != 0) {
   3471 		aprint_error_dev(sc->sc_dev,
   3472 		    "unable to allocate DMA buffer, error=%d\n", error);
   3473 		goto fail_alloc;
   3474 	}
   3475 
   3476 	if ((error = bus_dmamem_map(sc->sc_dmat, seg, 1, size, addrp,
   3477 	    BUS_DMA_COHERENT)) != 0) {
   3478 		aprint_error_dev(sc->sc_dev,
   3479 		    "unable to map DMA buffer, error=%d\n", error);
   3480 		goto fail_map;
   3481 	}
   3482 
   3483 	if ((error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
   3484 	    0, mapp)) != 0) {
   3485 		aprint_error_dev(sc->sc_dev,
   3486 		    "unable to create DMA map, error=%d\n", error);
   3487 		goto fail_create;
   3488 	}
   3489 
   3490 	if ((error = bus_dmamap_load(sc->sc_dmat, *mapp, *addrp, size, NULL,
   3491 	    0)) != 0) {
   3492 		aprint_error_dev(sc->sc_dev,
   3493 		    "unable to load DMA map, error=%d\n", error);
   3494 		goto fail_load;
   3495 	}
   3496 
   3497 	*sizep = size;
   3498 	return 0;
   3499 
   3500  fail_load:
   3501 	bus_dmamap_destroy(sc->sc_dmat, *mapp);
   3502 	*mapp = NULL;
   3503  fail_create:
   3504 	bus_dmamem_unmap(sc->sc_dmat, *addrp, size);
   3505 	*addrp = NULL;
   3506  fail_map:
   3507 	bus_dmamem_free(sc->sc_dmat, seg, 1);
   3508 	memset(seg, 0, sizeof(*seg));
   3509  fail_alloc:
   3510 	*sizep = 0;
   3511 	return error;
   3512 }
   3513 
   3514 static void
   3515 _free_dma(struct aq_softc *sc, bus_size_t *sizep, void **addrp,
   3516     bus_dmamap_t *mapp, bus_dma_segment_t *seg)
   3517 {
   3518 	if (*mapp != NULL) {
   3519 		bus_dmamap_destroy(sc->sc_dmat, *mapp);
   3520 		*mapp = NULL;
   3521 	}
   3522 	if (*addrp != NULL) {
   3523 		bus_dmamem_unmap(sc->sc_dmat, *addrp, *sizep);
   3524 		*addrp = NULL;
   3525 	}
   3526 	if (*sizep != 0) {
   3527 		bus_dmamem_free(sc->sc_dmat, seg, 1);
   3528 		memset(seg, 0, sizeof(*seg));
   3529 		*sizep = 0;
   3530 	}
   3531 }
   3532 
   3533 static int
   3534 aq_txring_alloc(struct aq_softc *sc, struct aq_txring *txring)
   3535 {
   3536 	int i, error;
   3537 
   3538 	/* allocate tx descriptors */
   3539 	error = _alloc_dma(sc, sizeof(aq_tx_desc_t) * AQ_TXD_NUM,
   3540 	    &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
   3541 	    &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
   3542 	if (error != 0)
   3543 		return error;
   3544 
   3545 	memset(txring->txr_txdesc, 0, sizeof(aq_tx_desc_t) * AQ_TXD_NUM);
   3546 
   3547 	/* fill tx ring with dmamap */
   3548 	for (i = 0; i < AQ_TXD_NUM; i++) {
   3549 #define AQ_MAXDMASIZE	(16 * 1024)
   3550 #define AQ_NTXSEGS	32
   3551 		/* XXX: TODO: error check */
   3552 		bus_dmamap_create(sc->sc_dmat, AQ_MAXDMASIZE, AQ_NTXSEGS,
   3553 		    AQ_MAXDMASIZE, 0, 0, &txring->txr_mbufs[i].dmamap);
   3554 	}
   3555 	return 0;
   3556 }
   3557 
   3558 static void
   3559 aq_txring_free(struct aq_softc *sc, struct aq_txring *txring)
   3560 {
   3561 	int i;
   3562 
   3563 	_free_dma(sc, &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
   3564 	    &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
   3565 
   3566 	for (i = 0; i < AQ_TXD_NUM; i++) {
   3567 		if (txring->txr_mbufs[i].dmamap != NULL) {
   3568 			if (txring->txr_mbufs[i].m != NULL) {
   3569 				bus_dmamap_unload(sc->sc_dmat,
   3570 				    txring->txr_mbufs[i].dmamap);
   3571 				m_freem(txring->txr_mbufs[i].m);
   3572 				txring->txr_mbufs[i].m = NULL;
   3573 			}
   3574 			bus_dmamap_destroy(sc->sc_dmat,
   3575 			    txring->txr_mbufs[i].dmamap);
   3576 			txring->txr_mbufs[i].dmamap = NULL;
   3577 		}
   3578 	}
   3579 }
   3580 
   3581 static int
   3582 aq_rxring_alloc(struct aq_softc *sc, struct aq_rxring *rxring)
   3583 {
   3584 	int i, error;
   3585 
   3586 	/* allocate rx descriptors */
   3587 	error = _alloc_dma(sc, sizeof(aq_rx_desc_t) * AQ_RXD_NUM,
   3588 	    &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
   3589 	    &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
   3590 	if (error != 0)
   3591 		return error;
   3592 
   3593 	memset(rxring->rxr_rxdesc, 0, sizeof(aq_rx_desc_t) * AQ_RXD_NUM);
   3594 
   3595 	/* fill rxring with dmamaps */
   3596 	for (i = 0; i < AQ_RXD_NUM; i++) {
   3597 		rxring->rxr_mbufs[i].m = NULL;
   3598 		/* XXX: TODO: error check */
   3599 		bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0, 0,
   3600 		    &rxring->rxr_mbufs[i].dmamap);
   3601 	}
   3602 	return 0;
   3603 }
   3604 
   3605 static void
   3606 aq_rxdrain(struct aq_softc *sc, struct aq_rxring *rxring)
   3607 {
   3608 	int i;
   3609 
   3610 	/* free all mbufs allocated for RX */
   3611 	for (i = 0; i < AQ_RXD_NUM; i++) {
   3612 		if (rxring->rxr_mbufs[i].m != NULL) {
   3613 			bus_dmamap_unload(sc->sc_dmat,
   3614 			    rxring->rxr_mbufs[i].dmamap);
   3615 			m_freem(rxring->rxr_mbufs[i].m);
   3616 			rxring->rxr_mbufs[i].m = NULL;
   3617 		}
   3618 	}
   3619 }
   3620 
   3621 static void
   3622 aq_rxring_free(struct aq_softc *sc, struct aq_rxring *rxring)
   3623 {
   3624 	int i;
   3625 
   3626 	/* free all mbufs and dmamaps */
   3627 	aq_rxdrain(sc, rxring);
   3628 	for (i = 0; i < AQ_RXD_NUM; i++) {
   3629 		if (rxring->rxr_mbufs[i].dmamap != NULL) {
   3630 			bus_dmamap_destroy(sc->sc_dmat,
   3631 			    rxring->rxr_mbufs[i].dmamap);
   3632 			rxring->rxr_mbufs[i].dmamap = NULL;
   3633 		}
   3634 	}
   3635 
   3636 	/* free RX descriptor */
   3637 	_free_dma(sc, &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
   3638 	    &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
   3639 }
   3640 
   3641 static void
   3642 aq_rxring_setmbuf(struct aq_softc *sc, struct aq_rxring *rxring, int idx,
   3643     struct mbuf *m)
   3644 {
   3645 	int error;
   3646 
   3647 	/* if mbuf already exists, unload and free */
   3648 	if (rxring->rxr_mbufs[idx].m != NULL) {
   3649 		bus_dmamap_unload(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap);
   3650 		m_freem(rxring->rxr_mbufs[idx].m);
   3651 		rxring->rxr_mbufs[idx].m = NULL;
   3652 	}
   3653 
   3654 	rxring->rxr_mbufs[idx].m = m;
   3655 
   3656 	m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
   3657 	error = bus_dmamap_load_mbuf(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap,
   3658 	    m, BUS_DMA_READ | BUS_DMA_NOWAIT);
   3659 	if (error) {
   3660 		device_printf(sc->sc_dev,
   3661 		    "unable to load rx DMA map %d, error = %d\n", idx, error);
   3662 		panic("%s: unable to load rx DMA map. error=%d",
   3663 		    __func__, error);
   3664 	}
   3665 	bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
   3666 	    rxring->rxr_mbufs[idx].dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
   3667 }
   3668 
   3669 static inline void
   3670 aq_rxring_reset_desc(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
   3671 {
   3672 	/* refill rxdesc, and sync */
   3673 	rxring->rxr_rxdesc[idx].read.buf_addr =
   3674 	   htole64(rxring->rxr_mbufs[idx].dmamap->dm_segs[0].ds_addr);
   3675 	rxring->rxr_rxdesc[idx].read.hdr_addr = 0;
   3676 	bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
   3677 	    sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
   3678 	    BUS_DMASYNC_PREWRITE);
   3679 }
   3680 
   3681 static struct mbuf *
   3682 aq_alloc_mbuf(void)
   3683 {
   3684 	struct mbuf *m;
   3685 
   3686 	MGETHDR(m, M_DONTWAIT, MT_DATA);
   3687 	if (m == NULL)
   3688 		return NULL;
   3689 
   3690 	MCLGET(m, M_DONTWAIT);
   3691 	if ((m->m_flags & M_EXT) == 0) {
   3692 		m_freem(m);
   3693 		return NULL;
   3694 	}
   3695 
   3696 	return m;
   3697 }
   3698 
   3699 /* allocate mbuf and unload dmamap */
   3700 static int
   3701 aq_rxring_add(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
   3702 {
   3703 	struct mbuf *m;
   3704 
   3705 	m = aq_alloc_mbuf();
   3706 	if (m == NULL)
   3707 		return ENOBUFS;
   3708 
   3709 	aq_rxring_setmbuf(sc, rxring, idx, m);
   3710 	return 0;
   3711 }
   3712 
   3713 static int
   3714 aq_txrx_rings_alloc(struct aq_softc *sc)
   3715 {
   3716 	int n, error;
   3717 
   3718 	for (n = 0; n < sc->sc_nqueues; n++) {
   3719 		sc->sc_queue[n].sc = sc;
   3720 		sc->sc_queue[n].txring.txr_sc = sc;
   3721 		sc->sc_queue[n].txring.txr_index = n;
   3722 		mutex_init(&sc->sc_queue[n].txring.txr_mutex, MUTEX_DEFAULT,
   3723 		    IPL_NET);
   3724 		error = aq_txring_alloc(sc, &sc->sc_queue[n].txring);
   3725 		if (error != 0)
   3726 			goto failure;
   3727 
   3728 		error = aq_tx_pcq_alloc(sc, &sc->sc_queue[n].txring);
   3729 		if (error != 0)
   3730 			goto failure;
   3731 
   3732 		sc->sc_queue[n].rxring.rxr_sc = sc;
   3733 		sc->sc_queue[n].rxring.rxr_index = n;
   3734 		mutex_init(&sc->sc_queue[n].rxring.rxr_mutex, MUTEX_DEFAULT,
   3735 		   IPL_NET);
   3736 		error = aq_rxring_alloc(sc, &sc->sc_queue[n].rxring);
   3737 		if (error != 0)
   3738 			break;
   3739 	}
   3740 
   3741  failure:
   3742 	return error;
   3743 }
   3744 
   3745 static void
   3746 aq_txrx_rings_free(struct aq_softc *sc)
   3747 {
   3748 	int n;
   3749 
   3750 	for (n = 0; n < sc->sc_nqueues; n++) {
   3751 		aq_txring_free(sc, &sc->sc_queue[n].txring);
   3752 		mutex_destroy(&sc->sc_queue[n].txring.txr_mutex);
   3753 
   3754 		aq_tx_pcq_free(sc, &sc->sc_queue[n].txring);
   3755 
   3756 		aq_rxring_free(sc, &sc->sc_queue[n].rxring);
   3757 		mutex_destroy(&sc->sc_queue[n].rxring.rxr_mutex);
   3758 	}
   3759 }
   3760 
   3761 static int
   3762 aq_tx_pcq_alloc(struct aq_softc *sc, struct aq_txring *txring)
   3763 {
   3764 	int error = 0;
   3765 	txring->txr_softint = NULL;
   3766 
   3767 	txring->txr_pcq = pcq_create(AQ_TXD_NUM, KM_NOSLEEP);
   3768 	if (txring->txr_pcq == NULL) {
   3769 		aprint_error_dev(sc->sc_dev,
   3770 		    "unable to allocate pcq for TXring[%d]\n",
   3771 		    txring->txr_index);
   3772 		error = ENOMEM;
   3773 		goto done;
   3774 	}
   3775 
   3776 	txring->txr_softint = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
   3777 	    aq_deferred_transmit, txring);
   3778 	if (txring->txr_softint == NULL) {
   3779 		aprint_error_dev(sc->sc_dev,
   3780 		    "unable to establish softint for TXring[%d]\n",
   3781 		    txring->txr_index);
   3782 		error = ENOENT;
   3783 	}
   3784 
   3785  done:
   3786 	return error;
   3787 }
   3788 
   3789 static void
   3790 aq_tx_pcq_free(struct aq_softc *sc, struct aq_txring *txring)
   3791 {
   3792 	struct mbuf *m;
   3793 
   3794 	if (txring->txr_softint != NULL) {
   3795 		softint_disestablish(txring->txr_softint);
   3796 		txring->txr_softint = NULL;
   3797 	}
   3798 
   3799 	if (txring->txr_pcq != NULL) {
   3800 		while ((m = pcq_get(txring->txr_pcq)) != NULL)
   3801 			m_freem(m);
   3802 		pcq_destroy(txring->txr_pcq);
   3803 		txring->txr_pcq = NULL;
   3804 	}
   3805 }
   3806 
   3807 #if NSYSMON_ENVSYS > 0
   3808 static void
   3809 aq_temp_refresh(struct sysmon_envsys *sme, envsys_data_t *edata)
   3810 {
   3811 	struct aq_softc *sc;
   3812 	uint32_t temp;
   3813 	int error;
   3814 
   3815 	sc = sme->sme_cookie;
   3816 
   3817 	error = sc->sc_fw_ops->get_temperature(sc, &temp);
   3818 	if (error == 0) {
   3819 		edata->value_cur = temp;
   3820 		edata->state = ENVSYS_SVALID;
   3821 	} else {
   3822 		edata->state = ENVSYS_SINVALID;
   3823 	}
   3824 }
   3825 #endif
   3826 
   3827 static void
   3828 aq_tick(void *arg)
   3829 {
   3830 	struct aq_softc *sc = arg;
   3831 
   3832 	if (sc->sc_poll_linkstat || sc->sc_detect_linkstat) {
   3833 		sc->sc_detect_linkstat = false;
   3834 		aq_update_link_status(sc);
   3835 	}
   3836 
   3837 #ifdef AQ_EVENT_COUNTERS
   3838 	if (sc->sc_poll_statistics)
   3839 		aq_update_statistics(sc);
   3840 #endif
   3841 
   3842 	if (sc->sc_poll_linkstat
   3843 #ifdef AQ_EVENT_COUNTERS
   3844 	    || sc->sc_poll_statistics
   3845 #endif
   3846 	    ) {
   3847 		callout_schedule(&sc->sc_tick_ch, hz);
   3848 	}
   3849 }
   3850 
   3851 /* interrupt enable/disable */
   3852 static void
   3853 aq_enable_intr(struct aq_softc *sc, bool link, bool txrx)
   3854 {
   3855 	uint32_t imask = 0;
   3856 	int i;
   3857 
   3858 	if (txrx) {
   3859 		for (i = 0; i < sc->sc_nqueues; i++) {
   3860 			imask |= __BIT(sc->sc_tx_irq[i]);
   3861 			imask |= __BIT(sc->sc_rx_irq[i]);
   3862 		}
   3863 	}
   3864 
   3865 	if (link)
   3866 		imask |= __BIT(sc->sc_linkstat_irq);
   3867 
   3868 	AQ_WRITE_REG(sc, AQ_INTR_MASK_REG, imask);
   3869 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
   3870 }
   3871 
   3872 static int
   3873 aq_legacy_intr(void *arg)
   3874 {
   3875 	struct aq_softc *sc = arg;
   3876 	uint32_t status;
   3877 	int nintr = 0;
   3878 
   3879 	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
   3880 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
   3881 
   3882 	if (status & __BIT(sc->sc_linkstat_irq)) {
   3883 		sc->sc_detect_linkstat = true;
   3884 		callout_schedule(&sc->sc_tick_ch, 0);
   3885 		nintr++;
   3886 	}
   3887 
   3888 	if (status & __BIT(sc->sc_rx_irq[0])) {
   3889 		nintr += aq_rx_intr(&sc->sc_queue[0].rxring);
   3890 	}
   3891 
   3892 	if (status & __BIT(sc->sc_tx_irq[0])) {
   3893 		nintr += aq_tx_intr(&sc->sc_queue[0].txring);
   3894 	}
   3895 
   3896 	return nintr;
   3897 }
   3898 
   3899 static int
   3900 aq_txrx_intr(void *arg)
   3901 {
   3902 	struct aq_queue *queue = arg;
   3903 	struct aq_softc *sc = queue->sc;
   3904 	struct aq_txring *txring = &queue->txring;
   3905 	struct aq_rxring *rxring = &queue->rxring;
   3906 	uint32_t status;
   3907 	int nintr = 0;
   3908 	int txringidx, rxringidx, txirq, rxirq;
   3909 
   3910 	txringidx = txring->txr_index;
   3911 	rxringidx = rxring->rxr_index;
   3912 	txirq = sc->sc_tx_irq[txringidx];
   3913 	rxirq = sc->sc_rx_irq[rxringidx];
   3914 
   3915 	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
   3916 	if ((status & (__BIT(txirq) | __BIT(rxirq))) == 0) {
   3917 		/* stray interrupt? */
   3918 		return 0;
   3919 	}
   3920 
   3921 	nintr += aq_rx_intr(rxring);
   3922 	nintr += aq_tx_intr(txring);
   3923 
   3924 	return nintr;
   3925 }
   3926 
   3927 static int
   3928 aq_link_intr(void *arg)
   3929 {
   3930 	struct aq_softc *sc = arg;
   3931 	uint32_t status;
   3932 	int nintr = 0;
   3933 
   3934 	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
   3935 	if (status & __BIT(sc->sc_linkstat_irq)) {
   3936 		sc->sc_detect_linkstat = true;
   3937 		callout_schedule(&sc->sc_tick_ch, 0);
   3938 		AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG,
   3939 		    __BIT(sc->sc_linkstat_irq));
   3940 		nintr++;
   3941 	}
   3942 
   3943 	return nintr;
   3944 }
   3945 
   3946 static void
   3947 aq_txring_reset(struct aq_softc *sc, struct aq_txring *txring, bool start)
   3948 {
   3949 	const int ringidx = txring->txr_index;
   3950 	int i;
   3951 
   3952 	mutex_enter(&txring->txr_mutex);
   3953 
   3954 	txring->txr_prodidx = 0;
   3955 	txring->txr_considx = 0;
   3956 	txring->txr_nfree = AQ_TXD_NUM;
   3957 	txring->txr_active = false;
   3958 
   3959 	/* free mbufs untransmitted */
   3960 	for (i = 0; i < AQ_TXD_NUM; i++) {
   3961 		if (txring->txr_mbufs[i].m != NULL) {
   3962 			m_freem(txring->txr_mbufs[i].m);
   3963 			txring->txr_mbufs[i].m = NULL;
   3964 		}
   3965 	}
   3966 
   3967 	/* disable DMA */
   3968 	AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_EN, 0);
   3969 
   3970 	if (start) {
   3971 		/* TX descriptor physical address */
   3972 		paddr_t paddr = txring->txr_txdesc_dmamap->dm_segs[0].ds_addr;
   3973 		AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
   3974 		AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
   3975 		    (uint32_t)((uint64_t)paddr >> 32));
   3976 
   3977 		/* TX descriptor size */
   3978 		AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_LEN,
   3979 		    AQ_TXD_NUM / 8);
   3980 
   3981 		/* reload TAIL pointer */
   3982 		txring->txr_prodidx = txring->txr_considx =
   3983 		    AQ_READ_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(ringidx));
   3984 		AQ_WRITE_REG(sc, TX_DMA_DESC_WRWB_THRESH_REG(ringidx), 0);
   3985 
   3986 		/* Mapping interrupt vector */
   3987 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
   3988 		    AQ_INTR_IRQ_MAP_TX_IRQMAP(ringidx), sc->sc_tx_irq[ringidx]);
   3989 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
   3990 		    AQ_INTR_IRQ_MAP_TX_EN(ringidx), true);
   3991 
   3992 		/* enable DMA */
   3993 		AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx),
   3994 		    TX_DMA_DESC_EN, 1);
   3995 
   3996 		const int cpuid = 0;	/* XXX? */
   3997 		AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
   3998 		    TDM_DCAD_CPUID, cpuid);
   3999 		AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
   4000 		    TDM_DCAD_CPUID_EN, 0);
   4001 
   4002 		txring->txr_active = true;
   4003 	}
   4004 
   4005 	mutex_exit(&txring->txr_mutex);
   4006 }
   4007 
   4008 static int
   4009 aq_rxring_reset(struct aq_softc *sc, struct aq_rxring *rxring, bool start)
   4010 {
   4011 	const int ringidx = rxring->rxr_index;
   4012 	int i;
   4013 	int error = 0;
   4014 
   4015 	mutex_enter(&rxring->rxr_mutex);
   4016 	rxring->rxr_active = false;
   4017 
   4018 	/* disable DMA */
   4019 	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_EN, 0);
   4020 
   4021 	/* free all RX mbufs */
   4022 	aq_rxdrain(sc, rxring);
   4023 
   4024 	if (start) {
   4025 		for (i = 0; i < AQ_RXD_NUM; i++) {
   4026 			error = aq_rxring_add(sc, rxring, i);
   4027 			if (error != 0) {
   4028 				aq_rxdrain(sc, rxring);
   4029 				return error;
   4030 			}
   4031 			aq_rxring_reset_desc(sc, rxring, i);
   4032 		}
   4033 
   4034 		/* RX descriptor physical address */
   4035 		paddr_t paddr = rxring->rxr_rxdesc_dmamap->dm_segs[0].ds_addr;
   4036 		AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
   4037 		AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
   4038 		    (uint32_t)((uint64_t)paddr >> 32));
   4039 
   4040 		/* RX descriptor size */
   4041 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_LEN,
   4042 		    AQ_RXD_NUM / 8);
   4043 
   4044 		/* maximum receive frame size */
   4045 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
   4046 		    RX_DMA_DESC_BUFSIZE_DATA, MCLBYTES / 1024);
   4047 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
   4048 		    RX_DMA_DESC_BUFSIZE_HDR, 0 / 1024);
   4049 
   4050 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
   4051 		    RX_DMA_DESC_HEADER_SPLIT, 0);
   4052 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
   4053 		    RX_DMA_DESC_VLAN_STRIP,
   4054 		    (sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
   4055 		    1 : 0);
   4056 
   4057 		/*
   4058 		 * reload TAIL pointer, and update readidx
   4059 		 * (HEAD pointer cannot write)
   4060 		 */
   4061 		rxring->rxr_readidx = AQ_READ_REG_BIT(sc,
   4062 		    RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR);
   4063 		AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx),
   4064 		    (rxring->rxr_readidx + AQ_RXD_NUM - 1) % AQ_RXD_NUM);
   4065 
   4066 		/* Rx ring set mode */
   4067 
   4068 		/* Mapping interrupt vector */
   4069 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
   4070 		    AQ_INTR_IRQ_MAP_RX_IRQMAP(ringidx), sc->sc_rx_irq[ringidx]);
   4071 		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
   4072 		    AQ_INTR_IRQ_MAP_RX_EN(ringidx), 1);
   4073 
   4074 		const int cpuid = 0;	/* XXX? */
   4075 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
   4076 		    RX_DMA_DCAD_CPUID, cpuid);
   4077 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
   4078 		    RX_DMA_DCAD_DESC_EN, 0);
   4079 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
   4080 		    RX_DMA_DCAD_HEADER_EN, 0);
   4081 		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
   4082 		    RX_DMA_DCAD_PAYLOAD_EN, 0);
   4083 
   4084 		/* enable DMA. start receiving */
   4085 		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
   4086 		    RX_DMA_DESC_EN, 1);
   4087 
   4088 		rxring->rxr_active = true;
   4089 	}
   4090 
   4091 	mutex_exit(&rxring->rxr_mutex);
   4092 	return error;
   4093 }
   4094 
   4095 #define TXRING_NEXTIDX(idx)	\
   4096 	(((idx) >= (AQ_TXD_NUM - 1)) ? 0 : ((idx) + 1))
   4097 #define RXRING_NEXTIDX(idx)	\
   4098 	(((idx) >= (AQ_RXD_NUM - 1)) ? 0 : ((idx) + 1))
   4099 
   4100 static int
   4101 aq_encap_txring(struct aq_softc *sc, struct aq_txring *txring, struct mbuf **mp)
   4102 {
   4103 	bus_dmamap_t map;
   4104 	struct mbuf *m = *mp;
   4105 	uint32_t ctl1, ctl1_ctx, ctl2;
   4106 	int idx, i, error;
   4107 
   4108 	idx = txring->txr_prodidx;
   4109 	map = txring->txr_mbufs[idx].dmamap;
   4110 
   4111 	error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
   4112 	    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   4113 	if (error == EFBIG) {
   4114 		struct mbuf *n;
   4115 		n = m_defrag(m, M_DONTWAIT);
   4116 		if (n == NULL)
   4117 			return EFBIG;
   4118 		/* m_defrag() preserve m */
   4119 		KASSERT(n == m);
   4120 		error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
   4121 		    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   4122 	}
   4123 	if (error != 0)
   4124 		return error;
   4125 
   4126 	/*
   4127 	 * check spaces of free descriptors.
   4128 	 * +1 is additional descriptor for context (vlan, etc,.)
   4129 	 */
   4130 	if ((map->dm_nsegs + 1) > txring->txr_nfree) {
   4131 		device_printf(sc->sc_dev,
   4132 		    "TX: not enough descriptors left %d for %d segs\n",
   4133 		    txring->txr_nfree, map->dm_nsegs + 1);
   4134 		bus_dmamap_unload(sc->sc_dmat, map);
   4135 		return ENOBUFS;
   4136 	}
   4137 
   4138 	/* sync dma for mbuf */
   4139 	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   4140 	    BUS_DMASYNC_PREWRITE);
   4141 
   4142 	ctl1_ctx = 0;
   4143 	ctl2 = __SHIFTIN(m->m_pkthdr.len, AQ_TXDESC_CTL2_LEN);
   4144 
   4145 	if (vlan_has_tag(m)) {
   4146 		ctl1 = AQ_TXDESC_CTL1_TYPE_TXC;
   4147 		ctl1 |= __SHIFTIN(vlan_get_tag(m), AQ_TXDESC_CTL1_VID);
   4148 
   4149 		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_VLAN;
   4150 		ctl2 |= AQ_TXDESC_CTL2_CTX_EN;
   4151 
   4152 		/* fill context descriptor and forward index */
   4153 		txring->txr_txdesc[idx].buf_addr = 0;
   4154 		txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
   4155 		txring->txr_txdesc[idx].ctl2 = 0;
   4156 
   4157 		idx = TXRING_NEXTIDX(idx);
   4158 		txring->txr_nfree--;
   4159 	}
   4160 
   4161 	if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
   4162 		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_IP4CSUM;
   4163 	if (m->m_pkthdr.csum_flags &
   4164 	    (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) {
   4165 		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_L4CSUM;
   4166 	}
   4167 
   4168 	/* fill descriptor(s) */
   4169 	for (i = 0; i < map->dm_nsegs; i++) {
   4170 		ctl1 = ctl1_ctx | AQ_TXDESC_CTL1_TYPE_TXD |
   4171 		    __SHIFTIN(map->dm_segs[i].ds_len, AQ_TXDESC_CTL1_BLEN);
   4172 		ctl1 |= AQ_TXDESC_CTL1_CMD_FCS;
   4173 
   4174 		if (i == 0) {
   4175 			/* remember mbuf of these descriptors */
   4176 			txring->txr_mbufs[idx].m = m;
   4177 		} else {
   4178 			txring->txr_mbufs[idx].m = NULL;
   4179 		}
   4180 
   4181 		if (i == map->dm_nsegs - 1) {
   4182 			/* last segment, mark an EndOfPacket, and cause intr */
   4183 			ctl1 |= AQ_TXDESC_CTL1_EOP | AQ_TXDESC_CTL1_CMD_WB;
   4184 		}
   4185 
   4186 		txring->txr_txdesc[idx].buf_addr =
   4187 		    htole64(map->dm_segs[i].ds_addr);
   4188 		txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
   4189 		txring->txr_txdesc[idx].ctl2 = htole32(ctl2);
   4190 
   4191 		bus_dmamap_sync(sc->sc_dmat, txring->txr_txdesc_dmamap,
   4192 		    sizeof(aq_tx_desc_t) * idx, sizeof(aq_tx_desc_t),
   4193 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
   4194 
   4195 		idx = TXRING_NEXTIDX(idx);
   4196 		txring->txr_nfree--;
   4197 	}
   4198 
   4199 	txring->txr_prodidx = idx;
   4200 
   4201 	return 0;
   4202 }
   4203 
   4204 static int
   4205 aq_tx_intr(void *arg)
   4206 {
   4207 	struct aq_txring *txring = arg;
   4208 	struct aq_softc *sc = txring->txr_sc;
   4209 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   4210 	struct mbuf *m;
   4211 	const int ringidx = txring->txr_index;
   4212 	unsigned int idx, hw_head, n = 0;
   4213 
   4214 	mutex_enter(&txring->txr_mutex);
   4215 
   4216 	if (!txring->txr_active)
   4217 		goto tx_intr_done;
   4218 
   4219 	hw_head = AQ_READ_REG_BIT(sc, TX_DMA_DESC_HEAD_PTR_REG(ringidx),
   4220 	    TX_DMA_DESC_HEAD_PTR);
   4221 	if (hw_head == txring->txr_considx) {
   4222 		goto tx_intr_done;
   4223 	}
   4224 
   4225 	for (idx = txring->txr_considx; idx != hw_head;
   4226 	    idx = TXRING_NEXTIDX(idx), n++) {
   4227 
   4228 		if ((m = txring->txr_mbufs[idx].m) != NULL) {
   4229 			bus_dmamap_unload(sc->sc_dmat,
   4230 			    txring->txr_mbufs[idx].dmamap);
   4231 
   4232 			txring->txr_opackets++;
   4233 			txring->txr_obytes += m->m_pkthdr.len;
   4234 			if (m->m_flags & M_MCAST)
   4235 				txring->txr_omcasts++;
   4236 
   4237 			m_freem(m);
   4238 			txring->txr_mbufs[idx].m = NULL;
   4239 		}
   4240 
   4241 		txring->txr_nfree++;
   4242 	}
   4243 	txring->txr_considx = idx;
   4244 
   4245 	if (ringidx == 0 && txring->txr_nfree >= AQ_TXD_MIN)
   4246 		ifp->if_flags &= ~IFF_OACTIVE;
   4247 
   4248 	/* no more pending TX packet, cancel watchdog */
   4249 	if (txring->txr_nfree >= AQ_TXD_NUM)
   4250 		ifp->if_timer = 0;
   4251 
   4252  tx_intr_done:
   4253 	mutex_exit(&txring->txr_mutex);
   4254 
   4255 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_tx_irq[ringidx]));
   4256 	return n;
   4257 }
   4258 
   4259 static int
   4260 aq_rx_intr(void *arg)
   4261 {
   4262 	struct aq_rxring *rxring = arg;
   4263 	struct aq_softc *sc = rxring->rxr_sc;
   4264 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   4265 	const int ringidx = rxring->rxr_index;
   4266 	aq_rx_desc_t *rxd;
   4267 	struct mbuf *m, *m0, *mprev, *new_m;
   4268 	uint32_t rxd_type, rxd_hash __unused;
   4269 	uint16_t rxd_status, rxd_pktlen;
   4270 	uint16_t rxd_nextdescptr __unused, rxd_vlan __unused;
   4271 	unsigned int idx, n = 0;
   4272 
   4273 	mutex_enter(&rxring->rxr_mutex);
   4274 
   4275 	if (!rxring->rxr_active)
   4276 		goto rx_intr_done;
   4277 
   4278 	if (rxring->rxr_readidx == AQ_READ_REG_BIT(sc,
   4279 	    RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR)) {
   4280 		goto rx_intr_done;
   4281 	}
   4282 
   4283 	m0 = mprev = NULL;
   4284 	for (idx = rxring->rxr_readidx;
   4285 	    idx != AQ_READ_REG_BIT(sc, RX_DMA_DESC_HEAD_PTR_REG(ringidx),
   4286 	    RX_DMA_DESC_HEAD_PTR); idx = RXRING_NEXTIDX(idx), n++) {
   4287 
   4288 		bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
   4289 		    sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
   4290 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
   4291 
   4292 		rxd = &rxring->rxr_rxdesc[idx];
   4293 		rxd_status = le16toh(rxd->wb.status);
   4294 
   4295 		if ((rxd_status & RXDESC_STATUS_DD) == 0)
   4296 			break;	/* not yet done */
   4297 
   4298 		rxd_type = le32toh(rxd->wb.type);
   4299 		rxd_pktlen = le16toh(rxd->wb.pkt_len);
   4300 		rxd_nextdescptr = le16toh(rxd->wb.next_desc_ptr);
   4301 		rxd_hash = le32toh(rxd->wb.rss_hash);
   4302 		rxd_vlan = le16toh(rxd->wb.vlan);
   4303 
   4304 		if ((rxd_status & RXDESC_STATUS_MACERR) ||
   4305 		    (rxd_type & RXDESC_TYPE_MAC_DMA_ERR)) {
   4306 			rxring->rxr_ierrors++;
   4307 			goto rx_next;
   4308 		}
   4309 
   4310 		bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
   4311 		    rxring->rxr_mbufs[idx].dmamap->dm_mapsize,
   4312 		    BUS_DMASYNC_POSTREAD);
   4313 		m = rxring->rxr_mbufs[idx].m;
   4314 
   4315 		new_m = aq_alloc_mbuf();
   4316 		if (new_m == NULL) {
   4317 			/*
   4318 			 * cannot allocate new mbuf.
   4319 			 * discard this packet, and reuse mbuf for next.
   4320 			 */
   4321 			rxring->rxr_iqdrops++;
   4322 			goto rx_next;
   4323 		}
   4324 		rxring->rxr_mbufs[idx].m = NULL;
   4325 		aq_rxring_setmbuf(sc, rxring, idx, new_m);
   4326 
   4327 		if (m0 == NULL) {
   4328 			m0 = m;
   4329 		} else {
   4330 			if (m->m_flags & M_PKTHDR)
   4331 				m_remove_pkthdr(m);
   4332 			mprev->m_next = m;
   4333 		}
   4334 		mprev = m;
   4335 
   4336 		if ((rxd_status & RXDESC_STATUS_EOP) == 0) {
   4337 			m->m_len = MCLBYTES;
   4338 		} else {
   4339 			/* last buffer */
   4340 			m->m_len = rxd_pktlen % MCLBYTES;
   4341 			m0->m_pkthdr.len = rxd_pktlen;
   4342 			/* VLAN offloading */
   4343 			if ((sc->sc_ethercom.ec_capenable &
   4344 			    ETHERCAP_VLAN_HWTAGGING) &&
   4345 			    (__SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_VLAN) ||
   4346 			    __SHIFTOUT(rxd_type,
   4347 			    RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE))) {
   4348 				vlan_set_tag(m0, rxd_vlan);
   4349 			}
   4350 
   4351 			/* Checksum offloading */
   4352 			unsigned int pkttype_eth =
   4353 			    __SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_ETHER);
   4354 			if ((ifp->if_capabilities & IFCAP_CSUM_IPv4_Rx) &&
   4355 			    (pkttype_eth == RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
   4356 			    __SHIFTOUT(rxd_type,
   4357 			    RXDESC_TYPE_IPV4_CSUM_CHECKED)) {
   4358 				m0->m_pkthdr.csum_flags |= M_CSUM_IPv4;
   4359 				if (__SHIFTOUT(rxd_status,
   4360 				    RXDESC_STATUS_IPV4_CSUM_NG))
   4361 					m0->m_pkthdr.csum_flags |=
   4362 					    M_CSUM_IPv4_BAD;
   4363 			}
   4364 #if notyet
   4365 			/*
   4366 			 * XXX: aq always marks BAD for fragmented packet.
   4367 			 * we should peek L3 header, and ignore cksum flags
   4368 			 * if the packet is fragmented.
   4369 			 */
   4370 			if (__SHIFTOUT(rxd_type,
   4371 			    RXDESC_TYPE_TCPUDP_CSUM_CHECKED)) {
   4372 				bool checked = false;
   4373 				unsigned int pkttype_proto =
   4374 				    __SHIFTOUT(rxd_type,
   4375 				    RXDESC_TYPE_PKTTYPE_PROTO);
   4376 
   4377 				if (pkttype_proto ==
   4378 				    RXDESC_TYPE_PKTTYPE_PROTO_TCP) {
   4379 					if ((pkttype_eth ==
   4380 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
   4381 					    (ifp->if_capabilities &
   4382 					    IFCAP_CSUM_TCPv4_Rx)) {
   4383 						m0->m_pkthdr.csum_flags |=
   4384 						    M_CSUM_TCPv4;
   4385 						checked = true;
   4386 					} else if ((pkttype_eth ==
   4387 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
   4388 					    (ifp->if_capabilities &
   4389 					    IFCAP_CSUM_TCPv6_Rx)) {
   4390 						m0->m_pkthdr.csum_flags |=
   4391 						    M_CSUM_TCPv6;
   4392 						checked = true;
   4393 					}
   4394 				} else if (pkttype_proto ==
   4395 				    RXDESC_TYPE_PKTTYPE_PROTO_UDP) {
   4396 					if ((pkttype_eth ==
   4397 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
   4398 					    (ifp->if_capabilities &
   4399 					    IFCAP_CSUM_UDPv4_Rx)) {
   4400 						m0->m_pkthdr.csum_flags |=
   4401 						    M_CSUM_UDPv4;
   4402 						checked = true;
   4403 					} else if ((pkttype_eth ==
   4404 					    RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
   4405 					    (ifp->if_capabilities &
   4406 					    IFCAP_CSUM_UDPv6_Rx)) {
   4407 						m0->m_pkthdr.csum_flags |=
   4408 						    M_CSUM_UDPv6;
   4409 						checked = true;
   4410 					}
   4411 				}
   4412 				if (checked &&
   4413 				    (__SHIFTOUT(rxd_status,
   4414 				    RXDESC_STATUS_TCPUDP_CSUM_ERROR) ||
   4415 				    !__SHIFTOUT(rxd_status,
   4416 				    RXDESC_STATUS_TCPUDP_CSUM_OK))) {
   4417 					m0->m_pkthdr.csum_flags |=
   4418 					    M_CSUM_TCP_UDP_BAD;
   4419 				}
   4420 			}
   4421 #endif
   4422 			m_set_rcvif(m0, ifp);
   4423 			rxring->rxr_ipackets++;
   4424 			rxring->rxr_ibytes += m0->m_pkthdr.len;
   4425 			if_percpuq_enqueue(ifp->if_percpuq, m0);
   4426 			m0 = mprev = NULL;
   4427 		}
   4428 
   4429  rx_next:
   4430 		aq_rxring_reset_desc(sc, rxring, idx);
   4431 		AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx), idx);
   4432 	}
   4433 	rxring->rxr_readidx = idx;
   4434 
   4435  rx_intr_done:
   4436 	mutex_exit(&rxring->rxr_mutex);
   4437 
   4438 	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_rx_irq[ringidx]));
   4439 	return n;
   4440 }
   4441 
   4442 static int
   4443 aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set)
   4444 {
   4445 	struct ifnet *ifp = &ec->ec_if;
   4446 	struct aq_softc *sc = ifp->if_softc;
   4447 
   4448 	aq_set_vlan_filters(sc);
   4449 	return 0;
   4450 }
   4451 
   4452 static int
   4453 aq_ifflags_cb(struct ethercom *ec)
   4454 {
   4455 	struct ifnet *ifp = &ec->ec_if;
   4456 	struct aq_softc *sc = ifp->if_softc;
   4457 	int i, ecchange, error = 0;
   4458 	unsigned short iffchange;
   4459 
   4460 	AQ_LOCK(sc);
   4461 
   4462 	iffchange = ifp->if_flags ^ sc->sc_if_flags;
   4463 	if ((iffchange & IFF_PROMISC) != 0)
   4464 		error = aq_set_filter(sc);
   4465 
   4466 	ecchange = ec->ec_capenable ^ sc->sc_ec_capenable;
   4467 	if (ecchange & ETHERCAP_VLAN_HWTAGGING) {
   4468 		for (i = 0; i < AQ_RINGS_NUM; i++) {
   4469 			AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(i),
   4470 			    RX_DMA_DESC_VLAN_STRIP,
   4471 			    (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
   4472 			    1 : 0);
   4473 		}
   4474 	}
   4475 
   4476 	/* vlan configuration depends on also interface promiscuous mode */
   4477 	if ((ecchange & ETHERCAP_VLAN_HWFILTER) || (iffchange & IFF_PROMISC))
   4478 		aq_set_vlan_filters(sc);
   4479 
   4480 	sc->sc_ec_capenable = ec->ec_capenable;
   4481 	sc->sc_if_flags = ifp->if_flags;
   4482 
   4483 	AQ_UNLOCK(sc);
   4484 
   4485 	return error;
   4486 }
   4487 
   4488 static int
   4489 aq_init(struct ifnet *ifp)
   4490 {
   4491 	struct aq_softc *sc = ifp->if_softc;
   4492 	int i, error = 0;
   4493 
   4494 	AQ_LOCK(sc);
   4495 
   4496 	aq_set_vlan_filters(sc);
   4497 	aq_set_capability(sc);
   4498 
   4499 	for (i = 0; i < sc->sc_nqueues; i++) {
   4500 		aq_txring_reset(sc, &sc->sc_queue[i].txring, true);
   4501 	}
   4502 
   4503 	/* invalidate RX descriptor cache */
   4504 	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
   4505 	    AQ_READ_REG_BIT(sc,
   4506 	    RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
   4507 
   4508 	/* start RX */
   4509 	for (i = 0; i < sc->sc_nqueues; i++) {
   4510 		error = aq_rxring_reset(sc, &sc->sc_queue[i].rxring, true);
   4511 		if (error != 0) {
   4512 			device_printf(sc->sc_dev, "%s: cannot allocate rxbuf\n",
   4513 			    __func__);
   4514 			goto aq_init_failure;
   4515 		}
   4516 	}
   4517 	aq_init_rss(sc);
   4518 	aq_hw_l3_filter_set(sc);
   4519 
   4520 	/* need to start callout? */
   4521 	if (sc->sc_poll_linkstat
   4522 #ifdef AQ_EVENT_COUNTERS
   4523 	    || sc->sc_poll_statistics
   4524 #endif
   4525 	    ) {
   4526 		callout_schedule(&sc->sc_tick_ch, hz);
   4527 	}
   4528 
   4529 	/* ready */
   4530 	ifp->if_flags |= IFF_RUNNING;
   4531 	ifp->if_flags &= ~IFF_OACTIVE;
   4532 
   4533 	/* start TX and RX */
   4534 	aq_enable_intr(sc, true, true);
   4535 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 1);
   4536 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 1);
   4537 
   4538  aq_init_failure:
   4539 	sc->sc_if_flags = ifp->if_flags;
   4540 
   4541 	AQ_UNLOCK(sc);
   4542 
   4543 	return error;
   4544 }
   4545 
   4546 static void
   4547 aq_send_common_locked(struct ifnet *ifp, struct aq_softc *sc,
   4548     struct aq_txring *txring, bool is_transmit)
   4549 {
   4550 	struct mbuf *m;
   4551 	int npkt, error;
   4552 
   4553 	if ((ifp->if_flags & IFF_RUNNING) == 0)
   4554 		return;
   4555 
   4556 	for (npkt = 0; ; npkt++) {
   4557 		if (is_transmit)
   4558 			m = pcq_peek(txring->txr_pcq);
   4559 		else
   4560 			IFQ_POLL(&ifp->if_snd, m);
   4561 
   4562 		if (m == NULL)
   4563 			break;
   4564 
   4565 		if (txring->txr_nfree < AQ_TXD_MIN)
   4566 			break;
   4567 
   4568 		if (is_transmit)
   4569 			pcq_get(txring->txr_pcq);
   4570 		else
   4571 			IFQ_DEQUEUE(&ifp->if_snd, m);
   4572 
   4573 		error = aq_encap_txring(sc, txring, &m);
   4574 		if (error != 0) {
   4575 			/* too many mbuf chains? or not enough descriptors? */
   4576 			m_freem(m);
   4577 			txring->txr_oerrors++;
   4578 			if (txring->txr_index == 0 && error == ENOBUFS)
   4579 				ifp->if_flags |= IFF_OACTIVE;
   4580 			break;
   4581 		}
   4582 
   4583 		/* update tail ptr */
   4584 		AQ_WRITE_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index),
   4585 		    txring->txr_prodidx);
   4586 
   4587 		/* Pass the packet to any BPF listeners */
   4588 		bpf_mtap(ifp, m, BPF_D_OUT);
   4589 	}
   4590 
   4591 	if (txring->txr_index == 0 && txring->txr_nfree < AQ_TXD_MIN)
   4592 		ifp->if_flags |= IFF_OACTIVE;
   4593 
   4594 	if (npkt)
   4595 		ifp->if_timer = 5;
   4596 }
   4597 
   4598 static void
   4599 aq_start(struct ifnet *ifp)
   4600 {
   4601 	struct aq_softc *sc;
   4602 	struct aq_txring *txring;
   4603 
   4604 	sc = ifp->if_softc;
   4605 	txring = &sc->sc_queue[0].txring; /* aq_start() always use TX ring[0] */
   4606 
   4607 	mutex_enter(&txring->txr_mutex);
   4608 	if (txring->txr_active && !ISSET(ifp->if_flags, IFF_OACTIVE))
   4609 		aq_send_common_locked(ifp, sc, txring, false);
   4610 	mutex_exit(&txring->txr_mutex);
   4611 }
   4612 
   4613 static inline unsigned int
   4614 aq_select_txqueue(struct aq_softc *sc, struct mbuf *m)
   4615 {
   4616 	return (cpu_index(curcpu()) % sc->sc_nqueues);
   4617 }
   4618 
   4619 static int
   4620 aq_transmit(struct ifnet *ifp, struct mbuf *m)
   4621 {
   4622 	struct aq_softc *sc = ifp->if_softc;
   4623 	struct aq_txring *txring;
   4624 	int ringidx;
   4625 
   4626 	ringidx = aq_select_txqueue(sc, m);
   4627 	txring = &sc->sc_queue[ringidx].txring;
   4628 
   4629 	if (__predict_false(!pcq_put(txring->txr_pcq, m))) {
   4630 		m_freem(m);
   4631 		return ENOBUFS;
   4632 	}
   4633 
   4634 	if (mutex_tryenter(&txring->txr_mutex)) {
   4635 		aq_send_common_locked(ifp, sc, txring, true);
   4636 		mutex_exit(&txring->txr_mutex);
   4637 	} else {
   4638 		kpreempt_disable();
   4639 		softint_schedule(txring->txr_softint);
   4640 		kpreempt_enable();
   4641 	}
   4642 	return 0;
   4643 }
   4644 
   4645 static void
   4646 aq_deferred_transmit(void *arg)
   4647 {
   4648 	struct aq_txring *txring = arg;
   4649 	struct aq_softc *sc = txring->txr_sc;
   4650 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   4651 
   4652 	mutex_enter(&txring->txr_mutex);
   4653 	if (pcq_peek(txring->txr_pcq) != NULL)
   4654 		aq_send_common_locked(ifp, sc, txring, true);
   4655 	mutex_exit(&txring->txr_mutex);
   4656 }
   4657 
   4658 static void
   4659 aq_stop(struct ifnet *ifp, int disable)
   4660 {
   4661 	struct aq_softc *sc = ifp->if_softc;
   4662 	int i;
   4663 
   4664 	AQ_LOCK(sc);
   4665 
   4666 	ifp->if_timer = 0;
   4667 
   4668 	/* disable tx/rx interrupts */
   4669 	aq_enable_intr(sc, true, false);
   4670 
   4671 	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 0);
   4672 	for (i = 0; i < sc->sc_nqueues; i++) {
   4673 		aq_txring_reset(sc, &sc->sc_queue[i].txring, false);
   4674 	}
   4675 
   4676 	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 0);
   4677 	for (i = 0; i < sc->sc_nqueues; i++) {
   4678 		aq_rxring_reset(sc, &sc->sc_queue[i].rxring, false);
   4679 	}
   4680 
   4681 	/* invalidate RX descriptor cache */
   4682 	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
   4683 	    AQ_READ_REG_BIT(sc,
   4684 	    RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);
   4685 
   4686 	ifp->if_timer = 0;
   4687 
   4688 	if (!disable) {
   4689 		/* when pmf stop, disable link status intr and callout */
   4690 		aq_enable_intr(sc, false, false);
   4691 		callout_stop(&sc->sc_tick_ch);
   4692 	}
   4693 
   4694 	ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
   4695 
   4696 	AQ_UNLOCK(sc);
   4697 }
   4698 
   4699 static void
   4700 aq_watchdog(struct ifnet *ifp)
   4701 {
   4702 	struct aq_softc *sc = ifp->if_softc;
   4703 	struct aq_txring *txring;
   4704 	int n, head, tail;
   4705 
   4706 	AQ_LOCK(sc);
   4707 
   4708 	device_printf(sc->sc_dev, "%s: INTR_MASK/STATUS = %08x/%08x\n",
   4709 	    __func__, AQ_READ_REG(sc, AQ_INTR_MASK_REG),
   4710 	    AQ_READ_REG(sc, AQ_INTR_STATUS_REG));
   4711 
   4712 	for (n = 0; n < sc->sc_nqueues; n++) {
   4713 		txring = &sc->sc_queue[n].txring;
   4714 		head = AQ_READ_REG_BIT(sc,
   4715 		    TX_DMA_DESC_HEAD_PTR_REG(txring->txr_index),
   4716 		    TX_DMA_DESC_HEAD_PTR),
   4717 		tail = AQ_READ_REG(sc,
   4718 		    TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index));
   4719 
   4720 		device_printf(sc->sc_dev, "%s: TXring[%d] HEAD/TAIL=%d/%d\n",
   4721 		    __func__, txring->txr_index, head, tail);
   4722 
   4723 		aq_tx_intr(txring);
   4724 	}
   4725 
   4726 	AQ_UNLOCK(sc);
   4727 
   4728 	aq_init(ifp);
   4729 }
   4730 
   4731 static int
   4732 aq_ioctl(struct ifnet *ifp, unsigned long cmd, void *data)
   4733 {
   4734 	struct aq_softc *sc __unused;
   4735 	struct ifreq *ifr __unused;
   4736 	uint64_t opackets, oerrors, obytes, omcasts;
   4737 	uint64_t ipackets, ierrors, ibytes, iqdrops;
   4738 	int error, i, s;
   4739 
   4740 	sc = (struct aq_softc *)ifp->if_softc;
   4741 	ifr = (struct ifreq *)data;
   4742 	error = 0;
   4743 
   4744 	switch (cmd) {
   4745 	case SIOCGIFDATA:
   4746 	case SIOCZIFDATA:
   4747 		opackets = oerrors = obytes = omcasts = 0;
   4748 		ipackets = ierrors = ibytes = iqdrops = 0;
   4749 		for (i = 0; i < sc->sc_nqueues; i++) {
   4750 			struct aq_txring *txring = &sc->sc_queue[i].txring;
   4751 			mutex_enter(&txring->txr_mutex);
   4752 			if (cmd == SIOCZIFDATA) {
   4753 				txring->txr_opackets = 0;
   4754 				txring->txr_obytes = 0;
   4755 				txring->txr_omcasts = 0;
   4756 				txring->txr_oerrors = 0;
   4757 			} else {
   4758 				opackets += txring->txr_opackets;
   4759 				oerrors += txring->txr_oerrors;
   4760 				obytes += txring->txr_obytes;
   4761 				omcasts += txring->txr_omcasts;
   4762 			}
   4763 			mutex_exit(&txring->txr_mutex);
   4764 
   4765 			struct aq_rxring *rxring = &sc->sc_queue[i].rxring;
   4766 			mutex_enter(&rxring->rxr_mutex);
   4767 			if (cmd == SIOCZIFDATA) {
   4768 				rxring->rxr_ipackets = 0;
   4769 				rxring->rxr_ibytes = 0;
   4770 				rxring->rxr_ierrors = 0;
   4771 				rxring->rxr_iqdrops = 0;
   4772 			} else {
   4773 				ipackets += rxring->rxr_ipackets;
   4774 				ierrors += rxring->rxr_ierrors;
   4775 				ibytes += rxring->rxr_ibytes;
   4776 				iqdrops += rxring->rxr_iqdrops;
   4777 			}
   4778 			mutex_exit(&rxring->rxr_mutex);
   4779 		}
   4780 		ifp->if_opackets = opackets;
   4781 		ifp->if_oerrors = oerrors;
   4782 		ifp->if_obytes = obytes;
   4783 		ifp->if_omcasts = omcasts;
   4784 		ifp->if_ipackets = ipackets;
   4785 		ifp->if_ierrors = ierrors;
   4786 		ifp->if_ibytes = ibytes;
   4787 		ifp->if_iqdrops = iqdrops;
   4788 		break;
   4789 	}
   4790 
   4791 	s = splnet();
   4792 	error = ether_ioctl(ifp, cmd, data);
   4793 	splx(s);
   4794 
   4795 	if (error != ENETRESET)
   4796 		return error;
   4797 
   4798 	switch (cmd) {
   4799 	case SIOCSIFCAP:
   4800 		error = aq_set_capability(sc);
   4801 		break;
   4802 	case SIOCADDMULTI:
   4803 	case SIOCDELMULTI:
   4804 		if ((ifp->if_flags & IFF_RUNNING) == 0)
   4805 			break;
   4806 
   4807 		/*
   4808 		 * Multicast list has changed; set the hardware filter
   4809 		 * accordingly.
   4810 		 */
   4811 		error = aq_set_filter(sc);
   4812 		break;
   4813 	}
   4814 
   4815 	return error;
   4816 }
   4817 
   4818 
   4819 MODULE(MODULE_CLASS_DRIVER, if_aq, "pci");
   4820 
   4821 #ifdef _MODULE
   4822 #include "ioconf.c"
   4823 #endif
   4824 
   4825 static int
   4826 if_aq_modcmd(modcmd_t cmd, void *opaque)
   4827 {
   4828 	int error = 0;
   4829 
   4830 	switch (cmd) {
   4831 	case MODULE_CMD_INIT:
   4832 #ifdef _MODULE
   4833 		error = config_init_component(cfdriver_ioconf_if_aq,
   4834 		    cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
   4835 #endif
   4836 		return error;
   4837 	case MODULE_CMD_FINI:
   4838 #ifdef _MODULE
   4839 		error = config_fini_component(cfdriver_ioconf_if_aq,
   4840 		    cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
   4841 #endif
   4842 		return error;
   4843 	default:
   4844 		return ENOTTY;
   4845 	}
   4846 }
   4847