Home | History | Annotate | Line # | Download | only in pci
if_mcx.c revision 1.26
      1  1.26   msaitoh /*	$NetBSD: if_mcx.c,v 1.26 2023/10/26 03:44:12 msaitoh Exp $ */
      2  1.20  jmcneill /*	$OpenBSD: if_mcx.c,v 1.101 2021/06/02 19:16:11 patrick Exp $ */
      3   1.1  jmcneill 
      4   1.1  jmcneill /*
      5   1.1  jmcneill  * Copyright (c) 2017 David Gwynne <dlg (at) openbsd.org>
      6   1.1  jmcneill  * Copyright (c) 2019 Jonathan Matthew <jmatthew (at) openbsd.org>
      7   1.1  jmcneill  *
      8   1.1  jmcneill  * Permission to use, copy, modify, and distribute this software for any
      9   1.1  jmcneill  * purpose with or without fee is hereby granted, provided that the above
     10   1.1  jmcneill  * copyright notice and this permission notice appear in all copies.
     11   1.1  jmcneill  *
     12   1.1  jmcneill  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     13   1.1  jmcneill  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     14   1.1  jmcneill  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     15   1.1  jmcneill  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     16   1.1  jmcneill  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     17   1.1  jmcneill  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     18   1.1  jmcneill  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     19   1.1  jmcneill  */
     20   1.1  jmcneill 
     21   1.1  jmcneill #ifdef _KERNEL_OPT
     22   1.1  jmcneill #include "opt_net_mpsafe.h"
     23   1.1  jmcneill #endif
     24   1.1  jmcneill 
     25  1.14  jmcneill #include <sys/cdefs.h>
     26  1.26   msaitoh __KERNEL_RCSID(0, "$NetBSD: if_mcx.c,v 1.26 2023/10/26 03:44:12 msaitoh Exp $");
     27  1.14  jmcneill 
     28   1.1  jmcneill #include <sys/param.h>
     29   1.1  jmcneill #include <sys/systm.h>
     30   1.1  jmcneill #include <sys/sockio.h>
     31   1.1  jmcneill #include <sys/mbuf.h>
     32   1.1  jmcneill #include <sys/kernel.h>
     33   1.1  jmcneill #include <sys/socket.h>
     34   1.1  jmcneill #include <sys/device.h>
     35   1.1  jmcneill #include <sys/pool.h>
     36   1.1  jmcneill #include <sys/queue.h>
     37   1.1  jmcneill #include <sys/callout.h>
     38   1.1  jmcneill #include <sys/workqueue.h>
     39   1.1  jmcneill #include <sys/atomic.h>
     40  1.15  jmcneill #include <sys/timetc.h>
     41   1.1  jmcneill #include <sys/kmem.h>
     42   1.1  jmcneill #include <sys/bus.h>
     43  1.15  jmcneill #include <sys/interrupt.h>
     44  1.16  jmcneill #include <sys/pcq.h>
     45  1.16  jmcneill #include <sys/cpu.h>
     46  1.23    andvar #include <sys/bitops.h>
     47   1.1  jmcneill 
     48   1.1  jmcneill #include <machine/intr.h>
     49   1.1  jmcneill 
     50   1.1  jmcneill #include <net/if.h>
     51   1.1  jmcneill #include <net/if_dl.h>
     52   1.1  jmcneill #include <net/if_ether.h>
     53   1.1  jmcneill #include <net/if_media.h>
     54  1.15  jmcneill #include <net/if_vlanvar.h>
     55  1.15  jmcneill #include <net/toeplitz.h>
     56   1.1  jmcneill 
     57   1.1  jmcneill #include <net/bpf.h>
     58   1.1  jmcneill 
     59   1.1  jmcneill #include <netinet/in.h>
     60   1.1  jmcneill 
     61   1.1  jmcneill #include <dev/pci/pcireg.h>
     62   1.1  jmcneill #include <dev/pci/pcivar.h>
     63   1.1  jmcneill #include <dev/pci/pcidevs.h>
     64   1.1  jmcneill 
     65  1.15  jmcneill /* TODO: Port kstat key/value stuff to evcnt/sysmon */
     66  1.15  jmcneill #define	NKSTAT		0
     67  1.15  jmcneill 
     68  1.12   thorpej /* XXX This driver is not yet MP-safe; don't claim to be! */
     69  1.12   thorpej /* #ifdef NET_MPSAFE */
     70  1.12   thorpej /* #define	MCX_MPSAFE	1 */
     71  1.12   thorpej /* #define	CALLOUT_FLAGS	CALLOUT_MPSAFE */
     72  1.12   thorpej /* #else */
     73   1.1  jmcneill #define	CALLOUT_FLAGS	0
     74  1.12   thorpej /* #endif */
     75   1.1  jmcneill 
     76  1.16  jmcneill #define	MCX_TXQ_NUM		2048
     77  1.16  jmcneill 
     78   1.1  jmcneill #define BUS_DMASYNC_PRERW	(BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
     79   1.1  jmcneill #define BUS_DMASYNC_POSTRW	(BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
     80   1.1  jmcneill 
     81   1.1  jmcneill #define MCX_HCA_BAR	PCI_MAPREG_START /* BAR 0 */
     82   1.1  jmcneill 
     83  1.15  jmcneill #define MCX_FW_VER			0x0000
     84  1.15  jmcneill #define  MCX_FW_VER_MAJOR(_v)			((_v) & 0xffff)
     85  1.15  jmcneill #define  MCX_FW_VER_MINOR(_v)			((_v) >> 16)
     86  1.15  jmcneill #define MCX_CMDIF_FW_SUBVER		0x0004
     87  1.15  jmcneill #define  MCX_FW_VER_SUBMINOR(_v)		((_v) & 0xffff)
     88  1.15  jmcneill #define  MCX_CMDIF(_v)				((_v) >> 16)
     89   1.1  jmcneill 
     90  1.15  jmcneill #define MCX_ISSI			1 /* as per the PRM */
     91  1.15  jmcneill #define MCX_CMD_IF_SUPPORTED		5
     92   1.1  jmcneill 
     93  1.15  jmcneill #define MCX_HARDMTU			9500
     94   1.1  jmcneill 
     95  1.15  jmcneill #define MCX_PAGE_SHIFT			12
     96  1.15  jmcneill #define MCX_PAGE_SIZE			(1 << MCX_PAGE_SHIFT)
     97   1.1  jmcneill 
     98   1.1  jmcneill /* queue sizes */
     99  1.15  jmcneill #define MCX_LOG_EQ_SIZE			7
    100  1.15  jmcneill #define MCX_LOG_CQ_SIZE			12
    101  1.15  jmcneill #define MCX_LOG_RQ_SIZE			10
    102  1.15  jmcneill #define MCX_LOG_SQ_SIZE			11
    103  1.15  jmcneill 
    104  1.15  jmcneill #define MCX_MAX_QUEUES			16
    105   1.1  jmcneill 
    106   1.1  jmcneill /* completion event moderation - about 10khz, or 90% of the cq */
    107  1.15  jmcneill #define MCX_CQ_MOD_PERIOD		50
    108  1.15  jmcneill #define MCX_CQ_MOD_COUNTER		\
    109  1.15  jmcneill 	(((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)
    110  1.15  jmcneill 
    111  1.15  jmcneill #define MCX_LOG_SQ_ENTRY_SIZE		6
    112  1.15  jmcneill #define MCX_SQ_ENTRY_MAX_SLOTS		4
    113  1.15  jmcneill #define MCX_SQ_SEGS_PER_SLOT		\
    114   1.1  jmcneill 	(sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
    115  1.15  jmcneill #define MCX_SQ_MAX_SEGMENTS		\
    116   1.1  jmcneill 	1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)
    117   1.1  jmcneill 
    118  1.15  jmcneill #define MCX_LOG_FLOW_TABLE_SIZE		5
    119  1.15  jmcneill #define MCX_NUM_STATIC_FLOWS		4 /* promisc, allmulti, ucast, bcast */
    120  1.15  jmcneill #define MCX_NUM_MCAST_FLOWS 		\
    121   1.1  jmcneill 	((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)
    122   1.1  jmcneill 
    123  1.15  jmcneill #define MCX_SQ_INLINE_SIZE		18
    124  1.15  jmcneill CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE);
    125   1.1  jmcneill 
    126   1.1  jmcneill /* doorbell offsets */
    127  1.15  jmcneill #define MCX_DOORBELL_AREA_SIZE		MCX_PAGE_SIZE
    128   1.1  jmcneill 
    129  1.15  jmcneill #define MCX_CQ_DOORBELL_BASE		0
    130  1.15  jmcneill #define MCX_CQ_DOORBELL_STRIDE		64
    131  1.15  jmcneill 
    132  1.15  jmcneill #define MCX_WQ_DOORBELL_BASE		MCX_PAGE_SIZE/2
    133  1.15  jmcneill #define MCX_WQ_DOORBELL_STRIDE		64
    134  1.15  jmcneill /* make sure the doorbells fit */
    135  1.15  jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE);
    136  1.15  jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE <
    137  1.15  jmcneill     MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE);
    138  1.15  jmcneill 
    139  1.15  jmcneill #define MCX_WQ_DOORBELL_MASK		0xffff
    140   1.1  jmcneill 
    141   1.1  jmcneill /* uar registers */
    142  1.15  jmcneill #define MCX_UAR_CQ_DOORBELL		0x20
    143  1.15  jmcneill #define MCX_UAR_EQ_DOORBELL_ARM		0x40
    144  1.15  jmcneill #define MCX_UAR_EQ_DOORBELL		0x48
    145  1.15  jmcneill #define MCX_UAR_BF			0x800
    146   1.1  jmcneill 
    147  1.15  jmcneill #define MCX_CMDQ_ADDR_HI		0x0010
    148  1.15  jmcneill #define MCX_CMDQ_ADDR_LO		0x0014
    149   1.1  jmcneill #define MCX_CMDQ_ADDR_NMASK		0xfff
    150   1.1  jmcneill #define MCX_CMDQ_LOG_SIZE(_v)		((_v) >> 4 & 0xf)
    151   1.1  jmcneill #define MCX_CMDQ_LOG_STRIDE(_v)		((_v) >> 0 & 0xf)
    152   1.1  jmcneill #define MCX_CMDQ_INTERFACE_MASK		(0x3 << 8)
    153   1.1  jmcneill #define MCX_CMDQ_INTERFACE_FULL_DRIVER	(0x0 << 8)
    154   1.1  jmcneill #define MCX_CMDQ_INTERFACE_DISABLED	(0x1 << 8)
    155   1.1  jmcneill 
    156   1.1  jmcneill #define MCX_CMDQ_DOORBELL		0x0018
    157   1.1  jmcneill 
    158  1.15  jmcneill #define MCX_STATE			0x01fc
    159  1.15  jmcneill #define MCX_STATE_MASK				(1U << 31)
    160  1.15  jmcneill #define MCX_STATE_INITIALIZING			(1 << 31)
    161  1.15  jmcneill #define MCX_STATE_READY				(0 << 31)
    162  1.15  jmcneill #define MCX_STATE_INTERFACE_MASK		(0x3 << 24)
    163  1.15  jmcneill #define MCX_STATE_INTERFACE_FULL_DRIVER		(0x0 << 24)
    164  1.15  jmcneill #define MCX_STATE_INTERFACE_DISABLED		(0x1 << 24)
    165  1.15  jmcneill 
    166  1.15  jmcneill #define MCX_INTERNAL_TIMER		0x1000
    167  1.15  jmcneill #define MCX_INTERNAL_TIMER_H		0x1000
    168  1.15  jmcneill #define MCX_INTERNAL_TIMER_L		0x1004
    169  1.15  jmcneill 
    170  1.15  jmcneill #define MCX_CLEAR_INT			0x100c
    171  1.15  jmcneill 
    172  1.15  jmcneill #define MCX_REG_OP_WRITE		0
    173  1.15  jmcneill #define MCX_REG_OP_READ			1
    174  1.15  jmcneill 
    175  1.15  jmcneill #define MCX_REG_PMLP			0x5002
    176  1.15  jmcneill #define MCX_REG_PMTU			0x5003
    177  1.15  jmcneill #define MCX_REG_PTYS			0x5004
    178  1.15  jmcneill #define MCX_REG_PAOS			0x5006
    179  1.15  jmcneill #define MCX_REG_PFCC			0x5007
    180  1.15  jmcneill #define MCX_REG_PPCNT			0x5008
    181  1.15  jmcneill #define MCX_REG_MTCAP			0x9009 /* mgmt temp capabilities */
    182  1.15  jmcneill #define MCX_REG_MTMP			0x900a /* mgmt temp */
    183  1.15  jmcneill #define MCX_REG_MCIA			0x9014
    184  1.15  jmcneill #define MCX_REG_MCAM			0x907f
    185  1.15  jmcneill 
    186  1.15  jmcneill #define MCX_ETHER_CAP_SGMII		0
    187  1.15  jmcneill #define MCX_ETHER_CAP_1000_KX		1
    188  1.15  jmcneill #define MCX_ETHER_CAP_10G_CX4		2
    189  1.15  jmcneill #define MCX_ETHER_CAP_10G_KX4		3
    190  1.15  jmcneill #define MCX_ETHER_CAP_10G_KR		4
    191  1.15  jmcneill #define MCX_ETHER_CAP_20G_KR2		5
    192  1.15  jmcneill #define MCX_ETHER_CAP_40G_CR4		6
    193  1.15  jmcneill #define MCX_ETHER_CAP_40G_KR4		7
    194  1.15  jmcneill #define MCX_ETHER_CAP_56G_R4		8
    195  1.15  jmcneill #define MCX_ETHER_CAP_10G_CR		12
    196  1.15  jmcneill #define MCX_ETHER_CAP_10G_SR		13
    197  1.15  jmcneill #define MCX_ETHER_CAP_10G_LR		14
    198  1.15  jmcneill #define MCX_ETHER_CAP_40G_SR4		15
    199  1.15  jmcneill #define MCX_ETHER_CAP_40G_LR4		16
    200  1.15  jmcneill #define MCX_ETHER_CAP_50G_SR2		18
    201  1.15  jmcneill #define MCX_ETHER_CAP_100G_CR4		20
    202  1.15  jmcneill #define MCX_ETHER_CAP_100G_SR4		21
    203  1.15  jmcneill #define MCX_ETHER_CAP_100G_KR4		22
    204  1.15  jmcneill #define MCX_ETHER_CAP_100G_LR4		23
    205  1.15  jmcneill #define MCX_ETHER_CAP_100_TX		24
    206  1.15  jmcneill #define MCX_ETHER_CAP_1000_T		25
    207  1.15  jmcneill #define MCX_ETHER_CAP_10G_T		26
    208  1.15  jmcneill #define MCX_ETHER_CAP_25G_CR		27
    209  1.15  jmcneill #define MCX_ETHER_CAP_25G_KR		28
    210  1.15  jmcneill #define MCX_ETHER_CAP_25G_SR		29
    211  1.15  jmcneill #define MCX_ETHER_CAP_50G_CR2		30
    212  1.15  jmcneill #define MCX_ETHER_CAP_50G_KR2		31
    213  1.15  jmcneill 
    214  1.15  jmcneill #define MCX_MAX_CQE			32
    215  1.15  jmcneill 
    216  1.15  jmcneill #define MCX_CMD_QUERY_HCA_CAP		0x100
    217  1.15  jmcneill #define MCX_CMD_QUERY_ADAPTER		0x101
    218  1.15  jmcneill #define MCX_CMD_INIT_HCA		0x102
    219  1.15  jmcneill #define MCX_CMD_TEARDOWN_HCA		0x103
    220  1.15  jmcneill #define MCX_CMD_ENABLE_HCA		0x104
    221  1.15  jmcneill #define MCX_CMD_DISABLE_HCA		0x105
    222  1.15  jmcneill #define MCX_CMD_QUERY_PAGES		0x107
    223  1.15  jmcneill #define MCX_CMD_MANAGE_PAGES		0x108
    224  1.15  jmcneill #define MCX_CMD_SET_HCA_CAP		0x109
    225  1.15  jmcneill #define MCX_CMD_QUERY_ISSI		0x10a
    226  1.15  jmcneill #define MCX_CMD_SET_ISSI		0x10b
    227  1.15  jmcneill #define MCX_CMD_SET_DRIVER_VERSION	0x10d
    228  1.15  jmcneill #define MCX_CMD_QUERY_SPECIAL_CONTEXTS	0x203
    229  1.15  jmcneill #define MCX_CMD_CREATE_EQ		0x301
    230  1.15  jmcneill #define MCX_CMD_DESTROY_EQ		0x302
    231  1.15  jmcneill #define MCX_CMD_QUERY_EQ		0x303
    232  1.15  jmcneill #define MCX_CMD_CREATE_CQ		0x400
    233  1.15  jmcneill #define MCX_CMD_DESTROY_CQ		0x401
    234  1.15  jmcneill #define MCX_CMD_QUERY_CQ		0x402
    235  1.15  jmcneill #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754
    236   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
    237  1.15  jmcneill 					0x755
    238  1.15  jmcneill #define MCX_CMD_QUERY_VPORT_COUNTERS 	0x770
    239  1.15  jmcneill #define MCX_CMD_ALLOC_PD		0x800
    240  1.15  jmcneill #define MCX_CMD_ALLOC_UAR		0x802
    241  1.15  jmcneill #define MCX_CMD_ACCESS_REG		0x805
    242  1.15  jmcneill #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN	0x816
    243  1.15  jmcneill #define MCX_CMD_CREATE_TIR		0x900
    244  1.15  jmcneill #define MCX_CMD_DESTROY_TIR		0x902
    245  1.15  jmcneill #define MCX_CMD_CREATE_SQ		0x904
    246  1.15  jmcneill #define MCX_CMD_MODIFY_SQ		0x905
    247  1.15  jmcneill #define MCX_CMD_DESTROY_SQ		0x906
    248  1.15  jmcneill #define MCX_CMD_QUERY_SQ		0x907
    249  1.15  jmcneill #define MCX_CMD_CREATE_RQ		0x908
    250  1.15  jmcneill #define MCX_CMD_MODIFY_RQ		0x909
    251  1.15  jmcneill #define MCX_CMD_DESTROY_RQ		0x90a
    252  1.15  jmcneill #define MCX_CMD_QUERY_RQ		0x90b
    253  1.15  jmcneill #define MCX_CMD_CREATE_TIS		0x912
    254  1.15  jmcneill #define MCX_CMD_DESTROY_TIS		0x914
    255  1.15  jmcneill #define MCX_CMD_CREATE_RQT		0x916
    256  1.15  jmcneill #define MCX_CMD_DESTROY_RQT		0x918
    257  1.15  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ROOT	0x92f
    258  1.15  jmcneill #define MCX_CMD_CREATE_FLOW_TABLE	0x930
    259  1.15  jmcneill #define MCX_CMD_DESTROY_FLOW_TABLE	0x931
    260  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE	0x932
    261  1.15  jmcneill #define MCX_CMD_CREATE_FLOW_GROUP	0x933
    262  1.15  jmcneill #define MCX_CMD_DESTROY_FLOW_GROUP	0x934
    263  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_GROUP	0x935
    264  1.15  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ENTRY	0x936
    265  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY	0x937
    266  1.15  jmcneill #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY	0x938
    267  1.15  jmcneill #define MCX_CMD_ALLOC_FLOW_COUNTER	0x939
    268  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_COUNTER	0x93b
    269  1.15  jmcneill 
    270  1.15  jmcneill #define MCX_QUEUE_STATE_RST		0
    271  1.15  jmcneill #define MCX_QUEUE_STATE_RDY		1
    272  1.15  jmcneill #define MCX_QUEUE_STATE_ERR		3
    273   1.1  jmcneill 
    274  1.15  jmcneill #define MCX_FLOW_TABLE_TYPE_RX		0
    275  1.15  jmcneill #define MCX_FLOW_TABLE_TYPE_TX		1
    276   1.1  jmcneill 
    277  1.15  jmcneill #define MCX_CMDQ_INLINE_DATASIZE	16
    278   1.1  jmcneill 
    279   1.1  jmcneill struct mcx_cmdq_entry {
    280   1.1  jmcneill 	uint8_t			cq_type;
    281   1.1  jmcneill #define MCX_CMDQ_TYPE_PCIE		0x7
    282   1.1  jmcneill 	uint8_t			cq_reserved0[3];
    283   1.1  jmcneill 
    284   1.1  jmcneill 	uint32_t		cq_input_length;
    285   1.1  jmcneill 	uint64_t		cq_input_ptr;
    286   1.1  jmcneill 	uint8_t			cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
    287   1.1  jmcneill 
    288   1.1  jmcneill 	uint8_t			cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
    289   1.1  jmcneill 	uint64_t		cq_output_ptr;
    290   1.1  jmcneill 	uint32_t		cq_output_length;
    291   1.1  jmcneill 
    292   1.1  jmcneill 	uint8_t			cq_token;
    293   1.1  jmcneill 	uint8_t			cq_signature;
    294   1.1  jmcneill 	uint8_t			cq_reserved1[1];
    295   1.1  jmcneill 	uint8_t			cq_status;
    296   1.1  jmcneill #define MCX_CQ_STATUS_SHIFT		1
    297   1.1  jmcneill #define MCX_CQ_STATUS_MASK		(0x7f << MCX_CQ_STATUS_SHIFT)
    298   1.1  jmcneill #define MCX_CQ_STATUS_OK		(0x00 << MCX_CQ_STATUS_SHIFT)
    299   1.1  jmcneill #define MCX_CQ_STATUS_INT_ERR		(0x01 << MCX_CQ_STATUS_SHIFT)
    300   1.1  jmcneill #define MCX_CQ_STATUS_BAD_OPCODE	(0x02 << MCX_CQ_STATUS_SHIFT)
    301   1.1  jmcneill #define MCX_CQ_STATUS_BAD_PARAM		(0x03 << MCX_CQ_STATUS_SHIFT)
    302   1.1  jmcneill #define MCX_CQ_STATUS_BAD_SYS_STATE	(0x04 << MCX_CQ_STATUS_SHIFT)
    303   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE	(0x05 << MCX_CQ_STATUS_SHIFT)
    304   1.1  jmcneill #define MCX_CQ_STATUS_RESOURCE_BUSY	(0x06 << MCX_CQ_STATUS_SHIFT)
    305   1.1  jmcneill #define MCX_CQ_STATUS_EXCEED_LIM	(0x08 << MCX_CQ_STATUS_SHIFT)
    306   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RES_STATE	(0x09 << MCX_CQ_STATUS_SHIFT)
    307   1.1  jmcneill #define MCX_CQ_STATUS_BAD_INDEX		(0x0a << MCX_CQ_STATUS_SHIFT)
    308   1.1  jmcneill #define MCX_CQ_STATUS_NO_RESOURCES	(0x0f << MCX_CQ_STATUS_SHIFT)
    309   1.1  jmcneill #define MCX_CQ_STATUS_BAD_INPUT_LEN	(0x50 << MCX_CQ_STATUS_SHIFT)
    310   1.1  jmcneill #define MCX_CQ_STATUS_BAD_OUTPUT_LEN	(0x51 << MCX_CQ_STATUS_SHIFT)
    311   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
    312   1.1  jmcneill 					(0x10 << MCX_CQ_STATUS_SHIFT)
    313   1.1  jmcneill #define MCX_CQ_STATUS_BAD_SIZE		(0x40 << MCX_CQ_STATUS_SHIFT)
    314   1.1  jmcneill #define MCX_CQ_STATUS_OWN_MASK		0x1
    315   1.1  jmcneill #define MCX_CQ_STATUS_OWN_SW		0x0
    316   1.1  jmcneill #define MCX_CQ_STATUS_OWN_HW		0x1
    317   1.1  jmcneill } __packed __aligned(8);
    318   1.1  jmcneill 
    319   1.1  jmcneill #define MCX_CMDQ_MAILBOX_DATASIZE	512
    320   1.1  jmcneill 
    321   1.1  jmcneill struct mcx_cmdq_mailbox {
    322   1.1  jmcneill 	uint8_t			mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
    323   1.1  jmcneill 	uint8_t			mb_reserved0[48];
    324   1.1  jmcneill 	uint64_t		mb_next_ptr;
    325   1.1  jmcneill 	uint32_t		mb_block_number;
    326   1.1  jmcneill 	uint8_t			mb_reserved1[1];
    327   1.1  jmcneill 	uint8_t			mb_token;
    328   1.1  jmcneill 	uint8_t			mb_ctrl_signature;
    329   1.1  jmcneill 	uint8_t			mb_signature;
    330   1.1  jmcneill } __packed __aligned(8);
    331   1.1  jmcneill 
    332   1.1  jmcneill #define MCX_CMDQ_MAILBOX_ALIGN	(1 << 10)
    333   1.1  jmcneill #define MCX_CMDQ_MAILBOX_SIZE	roundup(sizeof(struct mcx_cmdq_mailbox), \
    334   1.1  jmcneill 				    MCX_CMDQ_MAILBOX_ALIGN)
    335   1.1  jmcneill /*
    336  1.18    andvar  * command mailbox structures
    337   1.1  jmcneill  */
    338   1.1  jmcneill 
    339   1.1  jmcneill struct mcx_cmd_enable_hca_in {
    340   1.1  jmcneill 	uint16_t		cmd_opcode;
    341   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    342   1.1  jmcneill 	uint16_t		cmd_op_mod;
    343   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    344   1.1  jmcneill 	uint16_t		cmd_function_id;
    345   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    346   1.1  jmcneill } __packed __aligned(4);
    347   1.1  jmcneill 
    348   1.1  jmcneill struct mcx_cmd_enable_hca_out {
    349   1.1  jmcneill 	uint8_t			cmd_status;
    350   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    351   1.1  jmcneill 	uint32_t		cmd_syndrome;
    352   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    353   1.1  jmcneill } __packed __aligned(4);
    354   1.1  jmcneill 
    355   1.1  jmcneill struct mcx_cmd_init_hca_in {
    356   1.1  jmcneill 	uint16_t		cmd_opcode;
    357   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    358   1.1  jmcneill 	uint16_t		cmd_op_mod;
    359   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    360   1.1  jmcneill } __packed __aligned(4);
    361   1.1  jmcneill 
    362   1.1  jmcneill struct mcx_cmd_init_hca_out {
    363   1.1  jmcneill 	uint8_t			cmd_status;
    364   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    365   1.1  jmcneill 	uint32_t		cmd_syndrome;
    366   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    367   1.1  jmcneill } __packed __aligned(4);
    368   1.1  jmcneill 
    369   1.1  jmcneill struct mcx_cmd_teardown_hca_in {
    370   1.1  jmcneill 	uint16_t		cmd_opcode;
    371   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    372   1.1  jmcneill 	uint16_t		cmd_op_mod;
    373   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    374   1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_GRACEFUL	0x0
    375   1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_PANIC	0x1
    376   1.1  jmcneill 	uint16_t		cmd_profile;
    377   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    378   1.1  jmcneill } __packed __aligned(4);
    379   1.1  jmcneill 
    380   1.1  jmcneill struct mcx_cmd_teardown_hca_out {
    381   1.1  jmcneill 	uint8_t			cmd_status;
    382   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    383   1.1  jmcneill 	uint32_t		cmd_syndrome;
    384   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    385   1.1  jmcneill } __packed __aligned(4);
    386   1.1  jmcneill 
    387   1.1  jmcneill struct mcx_cmd_access_reg_in {
    388   1.1  jmcneill 	uint16_t		cmd_opcode;
    389   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    390   1.1  jmcneill 	uint16_t		cmd_op_mod;
    391   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    392   1.1  jmcneill 	uint16_t		cmd_register_id;
    393   1.1  jmcneill 	uint32_t		cmd_argument;
    394   1.1  jmcneill } __packed __aligned(4);
    395   1.1  jmcneill 
    396   1.1  jmcneill struct mcx_cmd_access_reg_out {
    397   1.1  jmcneill 	uint8_t			cmd_status;
    398   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    399   1.1  jmcneill 	uint32_t		cmd_syndrome;
    400   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    401   1.1  jmcneill } __packed __aligned(4);
    402   1.1  jmcneill 
    403   1.1  jmcneill struct mcx_reg_pmtu {
    404   1.1  jmcneill 	uint8_t			rp_reserved1;
    405   1.1  jmcneill 	uint8_t			rp_local_port;
    406   1.1  jmcneill 	uint8_t			rp_reserved2[2];
    407   1.1  jmcneill 	uint16_t		rp_max_mtu;
    408   1.1  jmcneill 	uint8_t			rp_reserved3[2];
    409   1.1  jmcneill 	uint16_t		rp_admin_mtu;
    410   1.1  jmcneill 	uint8_t			rp_reserved4[2];
    411   1.1  jmcneill 	uint16_t		rp_oper_mtu;
    412   1.1  jmcneill 	uint8_t			rp_reserved5[2];
    413   1.1  jmcneill } __packed __aligned(4);
    414   1.1  jmcneill 
    415   1.1  jmcneill struct mcx_reg_ptys {
    416   1.1  jmcneill 	uint8_t			rp_reserved1;
    417   1.1  jmcneill 	uint8_t			rp_local_port;
    418   1.1  jmcneill 	uint8_t			rp_reserved2;
    419   1.1  jmcneill 	uint8_t			rp_proto_mask;
    420   1.1  jmcneill #define MCX_REG_PTYS_PROTO_MASK_ETH		(1 << 2)
    421   1.1  jmcneill 	uint8_t			rp_reserved3[8];
    422   1.1  jmcneill 	uint32_t		rp_eth_proto_cap;
    423   1.1  jmcneill 	uint8_t			rp_reserved4[8];
    424   1.1  jmcneill 	uint32_t		rp_eth_proto_admin;
    425   1.1  jmcneill 	uint8_t			rp_reserved5[8];
    426   1.1  jmcneill 	uint32_t		rp_eth_proto_oper;
    427   1.1  jmcneill 	uint8_t			rp_reserved6[24];
    428   1.1  jmcneill } __packed __aligned(4);
    429   1.1  jmcneill 
    430   1.1  jmcneill struct mcx_reg_paos {
    431   1.1  jmcneill 	uint8_t			rp_reserved1;
    432   1.1  jmcneill 	uint8_t			rp_local_port;
    433   1.1  jmcneill 	uint8_t			rp_admin_status;
    434   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP		1
    435   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DOWN		2
    436   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE	3
    437   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED	4
    438   1.1  jmcneill 	uint8_t			rp_oper_status;
    439   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_UP		1
    440   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_DOWN		2
    441   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_FAILED		4
    442   1.1  jmcneill 	uint8_t			rp_admin_state_update;
    443   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN	(1 << 7)
    444   1.1  jmcneill 	uint8_t			rp_reserved2[11];
    445   1.1  jmcneill } __packed __aligned(4);
    446   1.1  jmcneill 
    447   1.1  jmcneill struct mcx_reg_pfcc {
    448   1.1  jmcneill 	uint8_t			rp_reserved1;
    449   1.1  jmcneill 	uint8_t			rp_local_port;
    450   1.1  jmcneill 	uint8_t			rp_reserved2[3];
    451   1.1  jmcneill 	uint8_t			rp_prio_mask_tx;
    452   1.1  jmcneill 	uint8_t			rp_reserved3;
    453   1.1  jmcneill 	uint8_t			rp_prio_mask_rx;
    454   1.1  jmcneill 	uint8_t			rp_pptx_aptx;
    455   1.1  jmcneill 	uint8_t			rp_pfctx;
    456   1.1  jmcneill 	uint8_t			rp_fctx_dis;
    457   1.1  jmcneill 	uint8_t			rp_reserved4;
    458   1.1  jmcneill 	uint8_t			rp_pprx_aprx;
    459   1.1  jmcneill 	uint8_t			rp_pfcrx;
    460   1.1  jmcneill 	uint8_t			rp_reserved5[2];
    461   1.1  jmcneill 	uint16_t		rp_dev_stall_min;
    462   1.1  jmcneill 	uint16_t		rp_dev_stall_crit;
    463   1.1  jmcneill 	uint8_t			rp_reserved6[12];
    464   1.1  jmcneill } __packed __aligned(4);
    465   1.1  jmcneill 
    466   1.1  jmcneill #define MCX_PMLP_MODULE_NUM_MASK	0xff
    467   1.1  jmcneill struct mcx_reg_pmlp {
    468   1.1  jmcneill 	uint8_t			rp_rxtx;
    469   1.1  jmcneill 	uint8_t			rp_local_port;
    470   1.1  jmcneill 	uint8_t			rp_reserved0;
    471   1.1  jmcneill 	uint8_t			rp_width;
    472   1.1  jmcneill 	uint32_t		rp_lane0_mapping;
    473   1.1  jmcneill 	uint32_t		rp_lane1_mapping;
    474   1.1  jmcneill 	uint32_t		rp_lane2_mapping;
    475   1.1  jmcneill 	uint32_t		rp_lane3_mapping;
    476   1.1  jmcneill 	uint8_t			rp_reserved1[44];
    477   1.1  jmcneill } __packed __aligned(4);
    478   1.1  jmcneill 
    479  1.15  jmcneill struct mcx_reg_ppcnt {
    480  1.15  jmcneill 	uint8_t			ppcnt_swid;
    481  1.15  jmcneill 	uint8_t			ppcnt_local_port;
    482  1.15  jmcneill 	uint8_t			ppcnt_pnat;
    483  1.15  jmcneill 	uint8_t			ppcnt_grp;
    484  1.15  jmcneill #define MCX_REG_PPCNT_GRP_IEEE8023		0x00
    485  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC2863		0x01
    486  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC2819		0x02
    487  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC3635		0x03
    488  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_PRIO		0x10
    489  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_TC		0x11
    490  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER		0x11
    491  1.15  jmcneill 
    492  1.15  jmcneill 	uint8_t			ppcnt_clr;
    493  1.15  jmcneill 	uint8_t			ppcnt_reserved1[2];
    494  1.15  jmcneill 	uint8_t			ppcnt_prio_tc;
    495  1.15  jmcneill #define MCX_REG_PPCNT_CLR			(1 << 7)
    496  1.15  jmcneill 
    497  1.15  jmcneill 	uint8_t			ppcnt_counter_set[248];
    498  1.15  jmcneill } __packed __aligned(8);
    499  1.15  jmcneill CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256);
    500  1.15  jmcneill CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) %
    501  1.15  jmcneill     sizeof(uint64_t)) == 0);
    502  1.15  jmcneill 
    503  1.15  jmcneill enum mcx_ppcnt_ieee8023 {
    504  1.15  jmcneill 	frames_transmitted_ok,
    505  1.15  jmcneill 	frames_received_ok,
    506  1.15  jmcneill 	frame_check_sequence_errors,
    507  1.15  jmcneill 	alignment_errors,
    508  1.15  jmcneill 	octets_transmitted_ok,
    509  1.15  jmcneill 	octets_received_ok,
    510  1.15  jmcneill 	multicast_frames_xmitted_ok,
    511  1.15  jmcneill 	broadcast_frames_xmitted_ok,
    512  1.15  jmcneill 	multicast_frames_received_ok,
    513  1.15  jmcneill 	broadcast_frames_received_ok,
    514  1.15  jmcneill 	in_range_length_errors,
    515  1.15  jmcneill 	out_of_range_length_field,
    516  1.15  jmcneill 	frame_too_long_errors,
    517  1.15  jmcneill 	symbol_error_during_carrier,
    518  1.15  jmcneill 	mac_control_frames_transmitted,
    519  1.15  jmcneill 	mac_control_frames_received,
    520  1.15  jmcneill 	unsupported_opcodes_received,
    521  1.15  jmcneill 	pause_mac_ctrl_frames_received,
    522  1.15  jmcneill 	pause_mac_ctrl_frames_transmitted,
    523  1.15  jmcneill 
    524  1.15  jmcneill 	mcx_ppcnt_ieee8023_count
    525  1.15  jmcneill };
    526  1.15  jmcneill CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98);
    527  1.15  jmcneill 
    528  1.15  jmcneill enum mcx_ppcnt_rfc2863 {
    529  1.15  jmcneill 	in_octets,
    530  1.15  jmcneill 	in_ucast_pkts,
    531  1.15  jmcneill 	in_discards,
    532  1.15  jmcneill 	in_errors,
    533  1.15  jmcneill 	in_unknown_protos,
    534  1.15  jmcneill 	out_octets,
    535  1.15  jmcneill 	out_ucast_pkts,
    536  1.15  jmcneill 	out_discards,
    537  1.15  jmcneill 	out_errors,
    538  1.15  jmcneill 	in_multicast_pkts,
    539  1.15  jmcneill 	in_broadcast_pkts,
    540  1.15  jmcneill 	out_multicast_pkts,
    541  1.15  jmcneill 	out_broadcast_pkts,
    542  1.15  jmcneill 
    543  1.15  jmcneill 	mcx_ppcnt_rfc2863_count
    544  1.15  jmcneill };
    545  1.15  jmcneill CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68);
    546  1.15  jmcneill 
    547  1.15  jmcneill enum mcx_ppcnt_rfc2819 {
    548  1.15  jmcneill 	drop_events,
    549  1.15  jmcneill 	octets,
    550  1.15  jmcneill 	pkts,
    551  1.15  jmcneill 	broadcast_pkts,
    552  1.15  jmcneill 	multicast_pkts,
    553  1.15  jmcneill 	crc_align_errors,
    554  1.15  jmcneill 	undersize_pkts,
    555  1.15  jmcneill 	oversize_pkts,
    556  1.15  jmcneill 	fragments,
    557  1.15  jmcneill 	jabbers,
    558  1.15  jmcneill 	collisions,
    559  1.15  jmcneill 	pkts64octets,
    560  1.15  jmcneill 	pkts65to127octets,
    561  1.15  jmcneill 	pkts128to255octets,
    562  1.15  jmcneill 	pkts256to511octets,
    563  1.15  jmcneill 	pkts512to1023octets,
    564  1.15  jmcneill 	pkts1024to1518octets,
    565  1.15  jmcneill 	pkts1519to2047octets,
    566  1.15  jmcneill 	pkts2048to4095octets,
    567  1.15  jmcneill 	pkts4096to8191octets,
    568  1.15  jmcneill 	pkts8192to10239octets,
    569  1.15  jmcneill 
    570  1.15  jmcneill 	mcx_ppcnt_rfc2819_count
    571  1.15  jmcneill };
    572  1.15  jmcneill CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8);
    573  1.15  jmcneill 
    574  1.15  jmcneill enum mcx_ppcnt_rfc3635 {
    575  1.15  jmcneill 	dot3stats_alignment_errors,
    576  1.15  jmcneill 	dot3stats_fcs_errors,
    577  1.15  jmcneill 	dot3stats_single_collision_frames,
    578  1.15  jmcneill 	dot3stats_multiple_collision_frames,
    579  1.15  jmcneill 	dot3stats_sqe_test_errors,
    580  1.15  jmcneill 	dot3stats_deferred_transmissions,
    581  1.15  jmcneill 	dot3stats_late_collisions,
    582  1.15  jmcneill 	dot3stats_excessive_collisions,
    583  1.15  jmcneill 	dot3stats_internal_mac_transmit_errors,
    584  1.15  jmcneill 	dot3stats_carrier_sense_errors,
    585  1.15  jmcneill 	dot3stats_frame_too_longs,
    586  1.15  jmcneill 	dot3stats_internal_mac_receive_errors,
    587  1.15  jmcneill 	dot3stats_symbol_errors,
    588  1.15  jmcneill 	dot3control_in_unknown_opcodes,
    589  1.15  jmcneill 	dot3in_pause_frames,
    590  1.15  jmcneill 	dot3out_pause_frames,
    591  1.15  jmcneill 
    592  1.15  jmcneill 	mcx_ppcnt_rfc3635_count
    593  1.15  jmcneill };
    594  1.15  jmcneill CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80);
    595  1.15  jmcneill 
    596  1.15  jmcneill struct mcx_reg_mcam {
    597  1.15  jmcneill 	uint8_t			_reserved1[1];
    598  1.15  jmcneill 	uint8_t			mcam_feature_group;
    599  1.15  jmcneill 	uint8_t			_reserved2[1];
    600  1.15  jmcneill 	uint8_t			mcam_access_reg_group;
    601  1.15  jmcneill 	uint8_t			_reserved3[4];
    602  1.15  jmcneill 	uint8_t			mcam_access_reg_cap_mask[16];
    603  1.15  jmcneill 	uint8_t			_reserved4[16];
    604  1.15  jmcneill 	uint8_t			mcam_feature_cap_mask[16];
    605  1.15  jmcneill 	uint8_t			_reserved5[16];
    606  1.15  jmcneill } __packed __aligned(4);
    607  1.15  jmcneill 
    608  1.15  jmcneill #define MCX_BITFIELD_BIT(bf, b)	(bf[(sizeof bf - 1) - (b / 8)] & (b % 8))
    609  1.15  jmcneill 
    610  1.15  jmcneill #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP	6
    611  1.15  jmcneill 
    612  1.15  jmcneill struct mcx_reg_mtcap {
    613  1.15  jmcneill 	uint8_t			_reserved1[3];
    614  1.15  jmcneill 	uint8_t			mtcap_sensor_count;
    615  1.15  jmcneill 	uint8_t			_reserved2[4];
    616  1.15  jmcneill 
    617  1.15  jmcneill 	uint64_t		mtcap_sensor_map;
    618  1.15  jmcneill };
    619  1.15  jmcneill 
    620  1.15  jmcneill struct mcx_reg_mtmp {
    621  1.15  jmcneill 	uint8_t			_reserved1[2];
    622  1.15  jmcneill 	uint16_t		mtmp_sensor_index;
    623  1.15  jmcneill 
    624  1.15  jmcneill 	uint8_t			_reserved2[2];
    625  1.15  jmcneill 	uint16_t		mtmp_temperature;
    626  1.15  jmcneill 
    627  1.15  jmcneill 	uint16_t		mtmp_mte_mtr;
    628  1.15  jmcneill #define MCX_REG_MTMP_MTE		(1 << 15)
    629  1.15  jmcneill #define MCX_REG_MTMP_MTR		(1 << 14)
    630  1.15  jmcneill 	uint16_t		mtmp_max_temperature;
    631  1.15  jmcneill 
    632  1.15  jmcneill 	uint16_t		mtmp_tee;
    633  1.15  jmcneill #define MCX_REG_MTMP_TEE_NOPE		(0 << 14)
    634  1.15  jmcneill #define MCX_REG_MTMP_TEE_GENERATE	(1 << 14)
    635  1.15  jmcneill #define MCX_REG_MTMP_TEE_GENERATE_ONE	(2 << 14)
    636  1.15  jmcneill 	uint16_t		mtmp_temperature_threshold_hi;
    637  1.15  jmcneill 
    638  1.15  jmcneill 	uint8_t			_reserved3[2];
    639  1.15  jmcneill 	uint16_t		mtmp_temperature_threshold_lo;
    640  1.15  jmcneill 
    641  1.15  jmcneill 	uint8_t			_reserved4[4];
    642  1.15  jmcneill 
    643  1.15  jmcneill 	uint8_t			mtmp_sensor_name[8];
    644  1.15  jmcneill };
    645  1.15  jmcneill CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20);
    646  1.15  jmcneill CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18);
    647  1.15  jmcneill 
    648   1.1  jmcneill #define MCX_MCIA_EEPROM_BYTES	32
    649   1.1  jmcneill struct mcx_reg_mcia {
    650   1.1  jmcneill 	uint8_t			rm_l;
    651   1.1  jmcneill 	uint8_t			rm_module;
    652   1.1  jmcneill 	uint8_t			rm_reserved0;
    653   1.1  jmcneill 	uint8_t			rm_status;
    654   1.1  jmcneill 	uint8_t			rm_i2c_addr;
    655   1.1  jmcneill 	uint8_t			rm_page_num;
    656   1.1  jmcneill 	uint16_t		rm_dev_addr;
    657   1.1  jmcneill 	uint16_t		rm_reserved1;
    658   1.1  jmcneill 	uint16_t		rm_size;
    659   1.1  jmcneill 	uint32_t		rm_reserved2;
    660   1.1  jmcneill 	uint8_t			rm_data[48];
    661   1.1  jmcneill } __packed __aligned(4);
    662   1.1  jmcneill 
    663   1.1  jmcneill struct mcx_cmd_query_issi_in {
    664   1.1  jmcneill 	uint16_t		cmd_opcode;
    665   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    666   1.1  jmcneill 	uint16_t		cmd_op_mod;
    667   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    668   1.1  jmcneill } __packed __aligned(4);
    669   1.1  jmcneill 
    670   1.1  jmcneill struct mcx_cmd_query_issi_il_out {
    671   1.1  jmcneill 	uint8_t			cmd_status;
    672   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    673   1.1  jmcneill 	uint32_t		cmd_syndrome;
    674   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    675   1.1  jmcneill 	uint16_t		cmd_current_issi;
    676   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    677   1.1  jmcneill } __packed __aligned(4);
    678   1.1  jmcneill 
    679   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
    680   1.1  jmcneill 
    681   1.1  jmcneill struct mcx_cmd_query_issi_mb_out {
    682   1.1  jmcneill 	uint8_t			cmd_reserved2[16];
    683   1.1  jmcneill 	uint8_t			cmd_supported_issi[80]; /* very big endian */
    684   1.1  jmcneill } __packed __aligned(4);
    685   1.1  jmcneill 
    686   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
    687   1.1  jmcneill 
    688   1.1  jmcneill struct mcx_cmd_set_issi_in {
    689   1.1  jmcneill 	uint16_t		cmd_opcode;
    690   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    691   1.1  jmcneill 	uint16_t		cmd_op_mod;
    692   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    693   1.1  jmcneill 	uint16_t		cmd_current_issi;
    694   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    695   1.1  jmcneill } __packed __aligned(4);
    696   1.1  jmcneill 
    697   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
    698   1.1  jmcneill 
    699   1.1  jmcneill struct mcx_cmd_set_issi_out {
    700   1.1  jmcneill 	uint8_t			cmd_status;
    701   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    702   1.1  jmcneill 	uint32_t		cmd_syndrome;
    703   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    704   1.1  jmcneill } __packed __aligned(4);
    705   1.1  jmcneill 
    706   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
    707   1.1  jmcneill 
    708   1.1  jmcneill struct mcx_cmd_query_pages_in {
    709   1.1  jmcneill 	uint16_t		cmd_opcode;
    710   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    711   1.1  jmcneill 	uint16_t		cmd_op_mod;
    712   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_BOOT	0x01
    713   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_INIT	0x02
    714   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_REGULAR	0x03
    715   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    716   1.1  jmcneill } __packed __aligned(4);
    717   1.1  jmcneill 
    718   1.1  jmcneill struct mcx_cmd_query_pages_out {
    719   1.1  jmcneill 	uint8_t			cmd_status;
    720   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    721   1.1  jmcneill 	uint32_t		cmd_syndrome;
    722   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    723   1.1  jmcneill 	uint16_t		cmd_func_id;
    724  1.15  jmcneill 	int32_t			cmd_num_pages;
    725   1.1  jmcneill } __packed __aligned(4);
    726   1.1  jmcneill 
    727   1.1  jmcneill struct mcx_cmd_manage_pages_in {
    728   1.1  jmcneill 	uint16_t		cmd_opcode;
    729   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    730   1.1  jmcneill 	uint16_t		cmd_op_mod;
    731   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
    732   1.1  jmcneill 					0x00
    733   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
    734   1.1  jmcneill 					0x01
    735   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
    736   1.1  jmcneill 					0x02
    737   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    738   1.1  jmcneill 	uint16_t		cmd_func_id;
    739   1.1  jmcneill 	uint32_t		cmd_input_num_entries;
    740   1.1  jmcneill } __packed __aligned(4);
    741   1.1  jmcneill 
    742   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
    743   1.1  jmcneill 
    744   1.1  jmcneill struct mcx_cmd_manage_pages_out {
    745   1.1  jmcneill 	uint8_t			cmd_status;
    746   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    747   1.1  jmcneill 	uint32_t		cmd_syndrome;
    748   1.1  jmcneill 	uint32_t		cmd_output_num_entries;
    749   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    750   1.1  jmcneill } __packed __aligned(4);
    751   1.1  jmcneill 
    752   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
    753   1.1  jmcneill 
    754   1.1  jmcneill struct mcx_cmd_query_hca_cap_in {
    755   1.1  jmcneill 	uint16_t		cmd_opcode;
    756   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    757   1.1  jmcneill 	uint16_t		cmd_op_mod;
    758   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_MAX	(0x0 << 0)
    759   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_CURRENT	(0x1 << 0)
    760   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_DEVICE	(0x0 << 1)
    761   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD	(0x1 << 1)
    762   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_FLOW	(0x7 << 1)
    763   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    764   1.1  jmcneill } __packed __aligned(4);
    765   1.1  jmcneill 
    766   1.1  jmcneill struct mcx_cmd_query_hca_cap_out {
    767   1.1  jmcneill 	uint8_t			cmd_status;
    768   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    769   1.1  jmcneill 	uint32_t		cmd_syndrome;
    770   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    771   1.1  jmcneill } __packed __aligned(4);
    772   1.1  jmcneill 
    773   1.1  jmcneill #define MCX_HCA_CAP_LEN			0x1000
    774   1.1  jmcneill #define MCX_HCA_CAP_NMAILBOXES		\
    775   1.1  jmcneill 	(MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
    776   1.1  jmcneill 
    777   1.1  jmcneill #if __GNUC_PREREQ__(4, 3)
    778   1.1  jmcneill #define __counter__		__COUNTER__
    779   1.1  jmcneill #else
    780   1.1  jmcneill #define __counter__		__LINE__
    781   1.1  jmcneill #endif
    782   1.1  jmcneill 
    783   1.1  jmcneill #define __token(_tok, _num)	_tok##_num
    784   1.1  jmcneill #define _token(_tok, _num)	__token(_tok, _num)
    785   1.1  jmcneill #define __reserved__		_token(__reserved, __counter__)
    786   1.1  jmcneill 
    787   1.1  jmcneill struct mcx_cap_device {
    788   1.1  jmcneill 	uint8_t			reserved0[16];
    789   1.1  jmcneill 
    790   1.1  jmcneill 	uint8_t			log_max_srq_sz;
    791   1.1  jmcneill 	uint8_t			log_max_qp_sz;
    792   1.1  jmcneill 	uint8_t			__reserved__[1];
    793   1.1  jmcneill 	uint8_t			log_max_qp; /* 5 bits */
    794   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_QP	0x1f
    795   1.1  jmcneill 
    796   1.1  jmcneill 	uint8_t			__reserved__[1];
    797   1.1  jmcneill 	uint8_t			log_max_srq; /* 5 bits */
    798   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SRQ	0x1f
    799   1.1  jmcneill 	uint8_t			__reserved__[2];
    800   1.1  jmcneill 
    801   1.1  jmcneill 	uint8_t			__reserved__[1];
    802   1.1  jmcneill 	uint8_t			log_max_cq_sz;
    803   1.1  jmcneill 	uint8_t			__reserved__[1];
    804   1.1  jmcneill 	uint8_t			log_max_cq; /* 5 bits */
    805   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_CQ	0x1f
    806   1.1  jmcneill 
    807   1.1  jmcneill 	uint8_t			log_max_eq_sz;
    808   1.1  jmcneill 	uint8_t			log_max_mkey; /* 6 bits */
    809   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MKEY	0x3f
    810   1.1  jmcneill 	uint8_t			__reserved__[1];
    811   1.1  jmcneill 	uint8_t			log_max_eq; /* 4 bits */
    812   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_EQ	0x0f
    813   1.1  jmcneill 
    814   1.1  jmcneill 	uint8_t			max_indirection;
    815   1.1  jmcneill 	uint8_t			log_max_mrw_sz; /* 7 bits */
    816   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ	0x7f
    817   1.1  jmcneill 	uint8_t			teardown_log_max_msf_list_size;
    818   1.1  jmcneill #define MCX_CAP_DEVICE_FORCE_TEARDOWN	0x80
    819   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
    820   1.1  jmcneill 					0x3f
    821   1.1  jmcneill 	uint8_t			log_max_klm_list_size; /* 6 bits */
    822   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
    823   1.1  jmcneill 					0x3f
    824   1.1  jmcneill 
    825   1.1  jmcneill 	uint8_t			__reserved__[1];
    826   1.1  jmcneill 	uint8_t			log_max_ra_req_dc; /* 6 bits */
    827   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC	0x3f
    828   1.1  jmcneill 	uint8_t			__reserved__[1];
    829   1.1  jmcneill 	uint8_t			log_max_ra_res_dc; /* 6 bits */
    830   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
    831   1.1  jmcneill 					0x3f
    832   1.1  jmcneill 
    833   1.1  jmcneill 	uint8_t			__reserved__[1];
    834   1.1  jmcneill 	uint8_t			log_max_ra_req_qp; /* 6 bits */
    835   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
    836   1.1  jmcneill 					0x3f
    837   1.1  jmcneill 	uint8_t			__reserved__[1];
    838   1.1  jmcneill 	uint8_t			log_max_ra_res_qp; /* 6 bits */
    839   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
    840   1.1  jmcneill 					0x3f
    841   1.1  jmcneill 
    842   1.1  jmcneill 	uint8_t			flags1;
    843   1.1  jmcneill #define MCX_CAP_DEVICE_END_PAD		0x80
    844   1.1  jmcneill #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED	0x40
    845   1.1  jmcneill #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
    846   1.1  jmcneill 					0x20
    847   1.1  jmcneill #define MCX_CAP_DEVICE_START_PAD	0x10
    848   1.1  jmcneill #define MCX_CAP_DEVICE_128BYTE_CACHELINE \
    849   1.1  jmcneill 					0x08
    850   1.1  jmcneill 	uint8_t			__reserved__[1];
    851   1.1  jmcneill 	uint16_t		gid_table_size;
    852   1.1  jmcneill 
    853   1.1  jmcneill 	uint16_t		flags2;
    854   1.1  jmcneill #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT	0x8000
    855   1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_COUNTERS	0x4000
    856   1.1  jmcneill #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
    857   1.1  jmcneill 					0x2000
    858   1.1  jmcneill #define MCX_CAP_DEVICE_DEBUG		0x1000
    859   1.1  jmcneill #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
    860   1.1  jmcneill 					0x8000
    861   1.1  jmcneill #define MCX_CAP_DEVICE_RQ_DELAY_DROP	0x4000
    862   1.1  jmcneill #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK	0x03ff
    863   1.1  jmcneill 	uint16_t		pkey_table_size;
    864   1.1  jmcneill 
    865   1.1  jmcneill 	uint8_t			flags3;
    866   1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
    867   1.1  jmcneill 					0x80
    868   1.1  jmcneill #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
    869   1.1  jmcneill 					0x40
    870   1.1  jmcneill #define MCX_CAP_DEVICE_IB_VIRTUAL	0x20
    871   1.1  jmcneill #define MCX_CAP_DEVICE_ETH_VIRTUAL	0x10
    872   1.1  jmcneill #define MCX_CAP_DEVICE_ETS		0x04
    873   1.1  jmcneill #define MCX_CAP_DEVICE_NIC_FLOW_TABLE	0x02
    874   1.1  jmcneill #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
    875   1.1  jmcneill 					0x01
    876   1.1  jmcneill 	uint8_t			local_ca_ack_delay; /* 5 bits */
    877   1.1  jmcneill #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
    878   1.1  jmcneill 					0x1f
    879  1.15  jmcneill #define MCX_CAP_DEVICE_MCAM_REG		0x40
    880   1.1  jmcneill 	uint8_t			port_type;
    881   1.1  jmcneill #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
    882   1.1  jmcneill 					0x80
    883   1.1  jmcneill #define MCX_CAP_DEVICE_PORT_TYPE	0x03
    884  1.15  jmcneill #define MCX_CAP_DEVICE_PORT_TYPE_ETH	0x01
    885   1.1  jmcneill 	uint8_t			num_ports;
    886   1.1  jmcneill 
    887   1.1  jmcneill 	uint8_t			snapshot_log_max_msg;
    888   1.1  jmcneill #define MCX_CAP_DEVICE_SNAPSHOT		0x80
    889   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSG	0x1f
    890   1.1  jmcneill 	uint8_t			max_tc; /* 4 bits */
    891   1.1  jmcneill #define MCX_CAP_DEVICE_MAX_TC		0x0f
    892   1.1  jmcneill 	uint8_t			flags4;
    893   1.1  jmcneill #define MCX_CAP_DEVICE_TEMP_WARN_EVENT	0x80
    894   1.1  jmcneill #define MCX_CAP_DEVICE_DCBX		0x40
    895   1.1  jmcneill #define MCX_CAP_DEVICE_ROL_S		0x02
    896   1.1  jmcneill #define MCX_CAP_DEVICE_ROL_G		0x01
    897   1.1  jmcneill 	uint8_t			wol;
    898   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_S		0x40
    899   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_G		0x20
    900   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_A		0x10
    901   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_B		0x08
    902   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_M		0x04
    903   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_U		0x02
    904   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_P		0x01
    905   1.1  jmcneill 
    906   1.1  jmcneill 	uint16_t		stat_rate_support;
    907   1.1  jmcneill 	uint8_t			__reserved__[1];
    908   1.1  jmcneill 	uint8_t			cqe_version; /* 4 bits */
    909   1.1  jmcneill #define MCX_CAP_DEVICE_CQE_VERSION	0x0f
    910   1.1  jmcneill 
    911   1.1  jmcneill 	uint32_t		flags5;
    912   1.1  jmcneill #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
    913   1.1  jmcneill 					0x80000000
    914   1.1  jmcneill #define MCX_CAP_DEVICE_STRIDING_RQ	0x40000000
    915   1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
    916   1.1  jmcneill 					0x10000000
    917   1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
    918   1.1  jmcneill 					0x08000000
    919   1.1  jmcneill #define MCX_CAP_DEVICE_DC_CONNECT_CP	0x00040000
    920   1.1  jmcneill #define MCX_CAP_DEVICE_DC_CNAK_DRACE	0x00020000
    921   1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    922   1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    923   1.1  jmcneill #define MCX_CAP_DEVICE_CMDIF_CHECKSUM	0x0000c000
    924   1.1  jmcneill #define MCX_CAP_DEVICE_SIGERR_QCE	0x00002000
    925   1.1  jmcneill #define MCX_CAP_DEVICE_WQ_SIGNATURE	0x00000800
    926   1.1  jmcneill #define MCX_CAP_DEVICE_SCTR_DATA_CQE	0x00000400
    927   1.1  jmcneill #define MCX_CAP_DEVICE_SHO		0x00000100
    928   1.1  jmcneill #define MCX_CAP_DEVICE_TPH		0x00000080
    929   1.1  jmcneill #define MCX_CAP_DEVICE_RF		0x00000040
    930   1.1  jmcneill #define MCX_CAP_DEVICE_DCT		0x00000020
    931   1.1  jmcneill #define MCX_CAP_DEVICE_QOS		0x00000010
    932   1.1  jmcneill #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS	0x00000008
    933   1.1  jmcneill #define MCX_CAP_DEVICE_ROCE		0x00000004
    934   1.1  jmcneill #define MCX_CAP_DEVICE_ATOMIC		0x00000002
    935   1.1  jmcneill 
    936   1.1  jmcneill 	uint32_t		flags6;
    937   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_OI		0x80000000
    938   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_RESIZE	0x40000000
    939   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_MODERATION	0x20000000
    940   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
    941   1.1  jmcneill 					0x10000000
    942   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_INVALIDATE	0x08000000
    943   1.1  jmcneill #define MCX_CAP_DEVICE_RESERVED_AT_255	0x04000000
    944   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_EQ_REMAP	0x02000000
    945   1.1  jmcneill #define MCX_CAP_DEVICE_PG		0x01000000
    946   1.1  jmcneill #define MCX_CAP_DEVICE_BLOCK_LB_MC	0x00800000
    947   1.1  jmcneill #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
    948   1.1  jmcneill 					0x00400000
    949   1.1  jmcneill #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
    950   1.1  jmcneill 					0x00200000
    951   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
    952   1.1  jmcneill 					0x00100000
    953   1.1  jmcneill #define MCX_CAP_DEVICE_CD		0x00080000
    954   1.1  jmcneill #define MCX_CAP_DEVICE_ATM		0x00040000
    955   1.1  jmcneill #define MCX_CAP_DEVICE_APM		0x00020000
    956   1.1  jmcneill #define MCX_CAP_DEVICE_IMAICL		0x00010000
    957   1.1  jmcneill #define MCX_CAP_DEVICE_QKV		0x00000200
    958   1.1  jmcneill #define MCX_CAP_DEVICE_PKV		0x00000100
    959   1.1  jmcneill #define MCX_CAP_DEVICE_SET_DETH_SQPN	0x00000080
    960   1.1  jmcneill #define MCX_CAP_DEVICE_XRC		0x00000008
    961   1.1  jmcneill #define MCX_CAP_DEVICE_UD		0x00000004
    962   1.1  jmcneill #define MCX_CAP_DEVICE_UC		0x00000002
    963   1.1  jmcneill #define MCX_CAP_DEVICE_RC		0x00000001
    964   1.1  jmcneill 
    965   1.1  jmcneill 	uint8_t			uar_flags;
    966   1.1  jmcneill #define MCX_CAP_DEVICE_UAR_4K		0x80
    967   1.1  jmcneill 	uint8_t			uar_sz;	/* 6 bits */
    968   1.1  jmcneill #define MCX_CAP_DEVICE_UAR_SZ		0x3f
    969   1.1  jmcneill 	uint8_t			__reserved__[1];
    970   1.1  jmcneill 	uint8_t			log_pg_sz;
    971   1.1  jmcneill 
    972   1.1  jmcneill 	uint8_t			flags7;
    973   1.1  jmcneill #define MCX_CAP_DEVICE_BF		0x80
    974   1.1  jmcneill #define MCX_CAP_DEVICE_DRIVER_VERSION	0x40
    975   1.1  jmcneill #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
    976   1.1  jmcneill 					0x20
    977   1.1  jmcneill 	uint8_t			log_bf_reg_size; /* 5 bits */
    978   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE	0x1f
    979   1.1  jmcneill 	uint8_t			__reserved__[2];
    980   1.1  jmcneill 
    981   1.1  jmcneill 	uint16_t		num_of_diagnostic_counters;
    982   1.1  jmcneill 	uint16_t		max_wqe_sz_sq;
    983   1.1  jmcneill 
    984   1.1  jmcneill 	uint8_t			__reserved__[2];
    985   1.1  jmcneill 	uint16_t		max_wqe_sz_rq;
    986   1.1  jmcneill 
    987   1.1  jmcneill 	uint8_t			__reserved__[2];
    988   1.1  jmcneill 	uint16_t		max_wqe_sz_sq_dc;
    989   1.1  jmcneill 
    990   1.1  jmcneill 	uint32_t		max_qp_mcg; /* 25 bits */
    991   1.1  jmcneill #define MCX_CAP_DEVICE_MAX_QP_MCG	0x1ffffff
    992   1.1  jmcneill 
    993   1.1  jmcneill 	uint8_t			__reserved__[3];
    994   1.1  jmcneill 	uint8_t			log_max_mcq;
    995   1.1  jmcneill 
    996   1.1  jmcneill 	uint8_t			log_max_transport_domain; /* 5 bits */
    997   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
    998   1.1  jmcneill 					0x1f
    999   1.1  jmcneill 	uint8_t			log_max_pd; /* 5 bits */
   1000   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_PD	0x1f
   1001   1.1  jmcneill 	uint8_t			__reserved__[1];
   1002   1.1  jmcneill 	uint8_t			log_max_xrcd; /* 5 bits */
   1003   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_XRCD	0x1f
   1004   1.1  jmcneill 
   1005   1.1  jmcneill 	uint8_t			__reserved__[2];
   1006   1.1  jmcneill 	uint16_t		max_flow_counter;
   1007   1.1  jmcneill 
   1008   1.1  jmcneill 	uint8_t			log_max_rq; /* 5 bits */
   1009   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQ	0x1f
   1010   1.1  jmcneill 	uint8_t			log_max_sq; /* 5 bits */
   1011   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SQ	0x1f
   1012   1.1  jmcneill 	uint8_t			log_max_tir; /* 5 bits */
   1013   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIR	0x1f
   1014   1.1  jmcneill 	uint8_t			log_max_tis; /* 5 bits */
   1015   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS	0x1f
   1016   1.1  jmcneill 
   1017   1.1  jmcneill 	uint8_t 		flags8;
   1018   1.1  jmcneill #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
   1019   1.1  jmcneill 					0x80
   1020   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RMP	0x1f
   1021   1.1  jmcneill 	uint8_t			log_max_rqt; /* 5 bits */
   1022   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT	0x1f
   1023   1.1  jmcneill 	uint8_t			log_max_rqt_size; /* 5 bits */
   1024   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE	0x1f
   1025   1.1  jmcneill 	uint8_t			log_max_tis_per_sq; /* 5 bits */
   1026   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
   1027   1.1  jmcneill 					0x1f
   1028  1.15  jmcneill 
   1029  1.15  jmcneill 	uint8_t			flags9;
   1030  1.15  jmcneill #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \
   1031  1.15  jmcneill 					0x80
   1032  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \
   1033  1.15  jmcneill 					0x1f
   1034  1.15  jmcneill 	uint8_t			log_min_stride_sz_rq; /* 5 bits */
   1035  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \
   1036  1.15  jmcneill 					0x1f
   1037  1.15  jmcneill 	uint8_t			log_max_stride_sz_sq; /* 5 bits */
   1038  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \
   1039  1.15  jmcneill 					0x1f
   1040  1.15  jmcneill 	uint8_t			log_min_stride_sz_sq; /* 5 bits */
   1041  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \
   1042  1.15  jmcneill 					0x1f
   1043  1.15  jmcneill 
   1044  1.15  jmcneill 	uint8_t			log_max_hairpin_queues;
   1045  1.15  jmcneill #define MXC_CAP_DEVICE_HAIRPIN		0x80
   1046  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \
   1047  1.15  jmcneill 					0x1f
   1048  1.15  jmcneill 	uint8_t			log_min_hairpin_queues;
   1049  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \
   1050  1.15  jmcneill 					0x1f
   1051  1.15  jmcneill 	uint8_t			log_max_hairpin_num_packets;
   1052  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \
   1053  1.15  jmcneill 					0x1f
   1054  1.15  jmcneill 	uint8_t			log_max_mq_sz;
   1055  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \
   1056  1.15  jmcneill 					0x1f
   1057  1.15  jmcneill 
   1058  1.15  jmcneill 	uint8_t			log_min_hairpin_wq_data_sz;
   1059  1.15  jmcneill #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \
   1060  1.15  jmcneill 					0x80
   1061  1.15  jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \
   1062  1.15  jmcneill 					0x40
   1063  1.15  jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \
   1064  1.15  jmcneill 					0x20
   1065  1.15  jmcneill #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \
   1066  1.15  jmcneill 					0x1f
   1067  1.15  jmcneill 	uint8_t			log_max_vlan_list;
   1068  1.15  jmcneill #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \
   1069  1.15  jmcneill 					0x80
   1070  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \
   1071  1.15  jmcneill 					0x1f
   1072  1.15  jmcneill 	uint8_t			log_max_current_mc_list;
   1073  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \
   1074  1.15  jmcneill 					0x1f
   1075  1.15  jmcneill 	uint8_t			log_max_current_uc_list;
   1076  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \
   1077  1.15  jmcneill 					0x1f
   1078  1.15  jmcneill 
   1079  1.15  jmcneill 	uint8_t			__reserved__[4];
   1080  1.15  jmcneill 
   1081  1.15  jmcneill 	uint32_t		create_qp_start_hint; /* 24 bits */
   1082  1.15  jmcneill 
   1083  1.15  jmcneill 	uint8_t			log_max_uctx; /* 5 bits */
   1084  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UCTX	0x1f
   1085  1.15  jmcneill 	uint8_t			log_max_umem; /* 5 bits */
   1086  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UMEM	0x1f
   1087  1.15  jmcneill 	uint16_t		max_num_eqs;
   1088  1.15  jmcneill 
   1089  1.15  jmcneill 	uint8_t			log_max_l2_table; /* 5 bits */
   1090  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE	0x1f
   1091  1.15  jmcneill 	uint8_t			__reserved__[1];
   1092  1.15  jmcneill 	uint16_t		log_uar_page_sz;
   1093  1.15  jmcneill 
   1094  1.15  jmcneill 	uint8_t			__reserved__[8];
   1095  1.15  jmcneill 
   1096  1.15  jmcneill 	uint32_t		device_frequency_mhz;
   1097  1.15  jmcneill 	uint32_t		device_frequency_khz;
   1098   1.1  jmcneill } __packed __aligned(8);
   1099   1.1  jmcneill 
   1100   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
   1101   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
   1102   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
   1103   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
   1104   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
   1105   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
   1106  1.15  jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98);
   1107  1.15  jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c);
   1108   1.1  jmcneill CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
   1109   1.1  jmcneill 
   1110   1.1  jmcneill struct mcx_cmd_set_driver_version_in {
   1111   1.1  jmcneill 	uint16_t		cmd_opcode;
   1112   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1113   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1114   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1115   1.1  jmcneill } __packed __aligned(4);
   1116   1.1  jmcneill 
   1117   1.1  jmcneill struct mcx_cmd_set_driver_version_out {
   1118   1.1  jmcneill 	uint8_t			cmd_status;
   1119   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1120   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1121   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1122   1.1  jmcneill } __packed __aligned(4);
   1123   1.1  jmcneill 
   1124   1.1  jmcneill struct mcx_cmd_set_driver_version {
   1125   1.1  jmcneill 	uint8_t			cmd_driver_version[64];
   1126   1.1  jmcneill } __packed __aligned(8);
   1127   1.1  jmcneill 
   1128   1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_in {
   1129   1.1  jmcneill 	uint16_t		cmd_opcode;
   1130   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1131   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1132   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1133   1.1  jmcneill 	uint32_t		cmd_field_select;
   1134   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR	0x04
   1135   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC	0x10
   1136   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU	0x40
   1137   1.1  jmcneill } __packed __aligned(4);
   1138   1.1  jmcneill 
   1139   1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_out {
   1140   1.1  jmcneill 	uint8_t			cmd_status;
   1141   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1142   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1143   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1144   1.1  jmcneill } __packed __aligned(4);
   1145   1.1  jmcneill 
   1146   1.1  jmcneill struct mcx_cmd_query_nic_vport_context_in {
   1147   1.1  jmcneill 	uint16_t		cmd_opcode;
   1148   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1149   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1150   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1151   1.1  jmcneill 	uint8_t			cmd_allowed_list_type;
   1152   1.1  jmcneill 	uint8_t			cmd_reserved2[3];
   1153   1.1  jmcneill } __packed __aligned(4);
   1154   1.1  jmcneill 
   1155   1.1  jmcneill struct mcx_cmd_query_nic_vport_context_out {
   1156   1.1  jmcneill 	uint8_t			cmd_status;
   1157   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1158   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1159   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1160   1.1  jmcneill } __packed __aligned(4);
   1161   1.1  jmcneill 
   1162   1.1  jmcneill struct mcx_nic_vport_ctx {
   1163   1.1  jmcneill 	uint32_t		vp_min_wqe_inline_mode;
   1164   1.1  jmcneill 	uint8_t			vp_reserved0[32];
   1165   1.1  jmcneill 	uint32_t		vp_mtu;
   1166   1.1  jmcneill 	uint8_t			vp_reserved1[200];
   1167   1.1  jmcneill 	uint16_t		vp_flags;
   1168   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_UC_MAC			(0)
   1169   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_MC_MAC			(1 << 24)
   1170   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_VLAN			(2 << 24)
   1171   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_ALL			(1 << 13)
   1172   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_MCAST			(1 << 14)
   1173   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_UCAST			(1 << 15)
   1174   1.1  jmcneill 	uint16_t		vp_allowed_list_size;
   1175   1.1  jmcneill 	uint64_t		vp_perm_addr;
   1176   1.1  jmcneill 	uint8_t			vp_reserved2[4];
   1177   1.1  jmcneill 	/* allowed list follows */
   1178   1.1  jmcneill } __packed __aligned(4);
   1179   1.1  jmcneill 
   1180   1.1  jmcneill struct mcx_counter {
   1181   1.1  jmcneill 	uint64_t		packets;
   1182   1.1  jmcneill 	uint64_t		octets;
   1183   1.1  jmcneill } __packed __aligned(4);
   1184   1.1  jmcneill 
   1185   1.1  jmcneill struct mcx_nic_vport_counters {
   1186   1.1  jmcneill 	struct mcx_counter	rx_err;
   1187   1.1  jmcneill 	struct mcx_counter	tx_err;
   1188   1.1  jmcneill 	uint8_t			reserved0[64]; /* 0x30 */
   1189   1.1  jmcneill 	struct mcx_counter	rx_bcast;
   1190   1.1  jmcneill 	struct mcx_counter	tx_bcast;
   1191   1.1  jmcneill 	struct mcx_counter	rx_ucast;
   1192   1.1  jmcneill 	struct mcx_counter	tx_ucast;
   1193   1.1  jmcneill 	struct mcx_counter	rx_mcast;
   1194   1.1  jmcneill 	struct mcx_counter	tx_mcast;
   1195   1.1  jmcneill 	uint8_t			reserved1[0x210 - 0xd0];
   1196   1.1  jmcneill } __packed __aligned(4);
   1197   1.1  jmcneill 
   1198   1.1  jmcneill struct mcx_cmd_query_vport_counters_in {
   1199   1.1  jmcneill 	uint16_t		cmd_opcode;
   1200   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1201   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1202   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1203   1.1  jmcneill } __packed __aligned(4);
   1204   1.1  jmcneill 
   1205   1.1  jmcneill struct mcx_cmd_query_vport_counters_mb_in {
   1206   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1207   1.1  jmcneill 	uint8_t			cmd_clear;
   1208   1.1  jmcneill 	uint8_t			cmd_reserved1[7];
   1209   1.1  jmcneill } __packed __aligned(4);
   1210   1.1  jmcneill 
   1211   1.1  jmcneill struct mcx_cmd_query_vport_counters_out {
   1212   1.1  jmcneill 	uint8_t			cmd_status;
   1213   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1214   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1215   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1216   1.1  jmcneill } __packed __aligned(4);
   1217   1.1  jmcneill 
   1218   1.1  jmcneill struct mcx_cmd_query_flow_counter_in {
   1219   1.1  jmcneill 	uint16_t		cmd_opcode;
   1220   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1221   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1222   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1223   1.1  jmcneill } __packed __aligned(4);
   1224   1.1  jmcneill 
   1225   1.1  jmcneill struct mcx_cmd_query_flow_counter_mb_in {
   1226   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1227   1.1  jmcneill 	uint8_t			cmd_clear;
   1228   1.1  jmcneill 	uint8_t			cmd_reserved1[5];
   1229   1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
   1230   1.1  jmcneill } __packed __aligned(4);
   1231   1.1  jmcneill 
   1232   1.1  jmcneill struct mcx_cmd_query_flow_counter_out {
   1233   1.1  jmcneill 	uint8_t			cmd_status;
   1234   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1235   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1236   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1237   1.1  jmcneill } __packed __aligned(4);
   1238   1.1  jmcneill 
   1239   1.1  jmcneill struct mcx_cmd_alloc_uar_in {
   1240   1.1  jmcneill 	uint16_t		cmd_opcode;
   1241   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1242   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1243   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1244   1.1  jmcneill } __packed __aligned(4);
   1245   1.1  jmcneill 
   1246   1.1  jmcneill struct mcx_cmd_alloc_uar_out {
   1247   1.1  jmcneill 	uint8_t			cmd_status;
   1248   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1249   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1250   1.1  jmcneill 	uint32_t		cmd_uar;
   1251   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1252   1.1  jmcneill } __packed __aligned(4);
   1253   1.1  jmcneill 
   1254   1.1  jmcneill struct mcx_cmd_query_special_ctx_in {
   1255   1.1  jmcneill 	uint16_t		cmd_opcode;
   1256   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1257   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1258   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1259   1.1  jmcneill } __packed __aligned(4);
   1260   1.1  jmcneill 
   1261   1.1  jmcneill struct mcx_cmd_query_special_ctx_out {
   1262   1.1  jmcneill 	uint8_t			cmd_status;
   1263   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1264   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1265   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1266   1.1  jmcneill 	uint32_t		cmd_resd_lkey;
   1267   1.1  jmcneill } __packed __aligned(4);
   1268   1.1  jmcneill 
   1269   1.1  jmcneill struct mcx_eq_ctx {
   1270   1.1  jmcneill 	uint32_t		eq_status;
   1271  1.15  jmcneill #define MCX_EQ_CTX_STATE_SHIFT		8
   1272  1.15  jmcneill #define MCX_EQ_CTX_STATE_MASK		(0xf << MCX_EQ_CTX_STATE_SHIFT)
   1273  1.15  jmcneill #define MCX_EQ_CTX_STATE_ARMED		0x9
   1274  1.15  jmcneill #define MCX_EQ_CTX_STATE_FIRED		0xa
   1275   1.1  jmcneill #define MCX_EQ_CTX_OI_SHIFT		17
   1276   1.1  jmcneill #define MCX_EQ_CTX_OI			(1 << MCX_EQ_CTX_OI_SHIFT)
   1277   1.1  jmcneill #define MCX_EQ_CTX_EC_SHIFT		18
   1278   1.1  jmcneill #define MCX_EQ_CTX_EC			(1 << MCX_EQ_CTX_EC_SHIFT)
   1279   1.1  jmcneill #define MCX_EQ_CTX_STATUS_SHIFT		28
   1280   1.1  jmcneill #define MCX_EQ_CTX_STATUS_MASK		(0xf << MCX_EQ_CTX_STATUS_SHIFT)
   1281  1.15  jmcneill #define MCX_EQ_CTX_STATUS_OK		0x0
   1282  1.15  jmcneill #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa
   1283   1.1  jmcneill 	uint32_t		eq_reserved1;
   1284   1.1  jmcneill 	uint32_t		eq_page_offset;
   1285   1.1  jmcneill #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT	5
   1286   1.1  jmcneill 	uint32_t		eq_uar_size;
   1287   1.1  jmcneill #define MCX_EQ_CTX_UAR_PAGE_MASK	0xffffff
   1288   1.1  jmcneill #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT	24
   1289   1.1  jmcneill 	uint32_t		eq_reserved2;
   1290   1.1  jmcneill 	uint8_t			eq_reserved3[3];
   1291   1.1  jmcneill 	uint8_t			eq_intr;
   1292   1.1  jmcneill 	uint32_t		eq_log_page_size;
   1293   1.1  jmcneill #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1294   1.1  jmcneill 	uint32_t		eq_reserved4[3];
   1295   1.1  jmcneill 	uint32_t		eq_consumer_counter;
   1296   1.1  jmcneill 	uint32_t		eq_producer_counter;
   1297   1.1  jmcneill #define MCX_EQ_CTX_COUNTER_MASK		0xffffff
   1298   1.1  jmcneill 	uint32_t		eq_reserved5[4];
   1299   1.1  jmcneill } __packed __aligned(4);
   1300   1.1  jmcneill 
   1301   1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
   1302   1.1  jmcneill 
   1303   1.1  jmcneill struct mcx_cmd_create_eq_in {
   1304   1.1  jmcneill 	uint16_t		cmd_opcode;
   1305   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1306   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1307   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1308   1.1  jmcneill } __packed __aligned(4);
   1309   1.1  jmcneill 
   1310   1.1  jmcneill struct mcx_cmd_create_eq_mb_in {
   1311   1.1  jmcneill 	struct mcx_eq_ctx	cmd_eq_ctx;
   1312   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1313   1.1  jmcneill 	uint64_t		cmd_event_bitmask;
   1314   1.1  jmcneill #define MCX_EVENT_TYPE_COMPLETION	0x00
   1315   1.1  jmcneill #define MCX_EVENT_TYPE_CQ_ERROR		0x04
   1316   1.1  jmcneill #define MCX_EVENT_TYPE_INTERNAL_ERROR	0x08
   1317   1.1  jmcneill #define MCX_EVENT_TYPE_PORT_CHANGE	0x09
   1318   1.1  jmcneill #define MCX_EVENT_TYPE_CMD_COMPLETION	0x0a
   1319   1.1  jmcneill #define MCX_EVENT_TYPE_PAGE_REQUEST	0x0b
   1320   1.1  jmcneill #define MCX_EVENT_TYPE_LAST_WQE		0x13
   1321   1.1  jmcneill 	uint8_t			cmd_reserved1[176];
   1322   1.1  jmcneill } __packed __aligned(4);
   1323   1.1  jmcneill 
   1324   1.1  jmcneill struct mcx_cmd_create_eq_out {
   1325   1.1  jmcneill 	uint8_t			cmd_status;
   1326   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1327   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1328   1.1  jmcneill 	uint32_t		cmd_eqn;
   1329   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1330   1.1  jmcneill } __packed __aligned(4);
   1331   1.1  jmcneill 
   1332  1.15  jmcneill struct mcx_cmd_query_eq_in {
   1333  1.15  jmcneill 	uint16_t		cmd_opcode;
   1334  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1335  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1336  1.15  jmcneill 	uint32_t		cmd_eqn;
   1337  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1338  1.15  jmcneill } __packed __aligned(4);
   1339  1.15  jmcneill 
   1340  1.15  jmcneill struct mcx_cmd_query_eq_out {
   1341  1.15  jmcneill 	uint8_t			cmd_status;
   1342  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1343  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1344  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1345  1.15  jmcneill } __packed __aligned(4);
   1346  1.15  jmcneill 
   1347   1.1  jmcneill struct mcx_eq_entry {
   1348   1.1  jmcneill 	uint8_t			eq_reserved1;
   1349   1.1  jmcneill 	uint8_t			eq_event_type;
   1350   1.1  jmcneill 	uint8_t			eq_reserved2;
   1351   1.1  jmcneill 	uint8_t			eq_event_sub_type;
   1352   1.1  jmcneill 
   1353   1.1  jmcneill 	uint8_t			eq_reserved3[28];
   1354   1.1  jmcneill 	uint32_t		eq_event_data[7];
   1355   1.1  jmcneill 	uint8_t			eq_reserved4[2];
   1356   1.1  jmcneill 	uint8_t			eq_signature;
   1357   1.1  jmcneill 	uint8_t			eq_owner;
   1358   1.1  jmcneill #define MCX_EQ_ENTRY_OWNER_INIT			1
   1359   1.1  jmcneill } __packed __aligned(4);
   1360   1.1  jmcneill 
   1361   1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_entry) == 64);
   1362   1.1  jmcneill 
   1363   1.1  jmcneill struct mcx_cmd_alloc_pd_in {
   1364   1.1  jmcneill 	uint16_t		cmd_opcode;
   1365   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1366   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1367   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1368   1.1  jmcneill } __packed __aligned(4);
   1369   1.1  jmcneill 
   1370   1.1  jmcneill struct mcx_cmd_alloc_pd_out {
   1371   1.1  jmcneill 	uint8_t			cmd_status;
   1372   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1373   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1374   1.1  jmcneill 	uint32_t		cmd_pd;
   1375   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1376   1.1  jmcneill } __packed __aligned(4);
   1377   1.1  jmcneill 
   1378   1.1  jmcneill struct mcx_cmd_alloc_td_in {
   1379   1.1  jmcneill 	uint16_t		cmd_opcode;
   1380   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1381   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1382   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1383   1.1  jmcneill } __packed __aligned(4);
   1384   1.1  jmcneill 
   1385   1.1  jmcneill struct mcx_cmd_alloc_td_out {
   1386   1.1  jmcneill 	uint8_t			cmd_status;
   1387   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1388   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1389   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1390   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1391   1.1  jmcneill } __packed __aligned(4);
   1392   1.1  jmcneill 
   1393   1.1  jmcneill struct mcx_cmd_create_tir_in {
   1394   1.1  jmcneill 	uint16_t		cmd_opcode;
   1395   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1396   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1397   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1398   1.1  jmcneill } __packed __aligned(4);
   1399   1.1  jmcneill 
   1400   1.1  jmcneill struct mcx_cmd_create_tir_mb_in {
   1401   1.1  jmcneill 	uint8_t			cmd_reserved0[20];
   1402   1.1  jmcneill 	uint32_t		cmd_disp_type;
   1403  1.15  jmcneill #define MCX_TIR_CTX_DISP_TYPE_DIRECT	0
   1404  1.15  jmcneill #define MCX_TIR_CTX_DISP_TYPE_INDIRECT	1
   1405   1.1  jmcneill #define MCX_TIR_CTX_DISP_TYPE_SHIFT	28
   1406   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1407   1.1  jmcneill 	uint32_t		cmd_lro;
   1408   1.1  jmcneill 	uint8_t			cmd_reserved2[8];
   1409   1.1  jmcneill 	uint32_t		cmd_inline_rqn;
   1410   1.1  jmcneill 	uint32_t		cmd_indir_table;
   1411   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1412  1.15  jmcneill #define MCX_TIR_CTX_HASH_TOEPLITZ	2
   1413  1.15  jmcneill #define MCX_TIR_CTX_HASH_SHIFT		28
   1414   1.1  jmcneill 	uint8_t			cmd_rx_hash_key[40];
   1415   1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_outer;
   1416  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_SRC_IP	(1 << 0)
   1417  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_DST_IP	(1 << 1)
   1418  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_SPORT	(1 << 2)
   1419  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_DPORT	(1 << 3)
   1420  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV4	(0 << 31)
   1421  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV6	(1 << 31)
   1422  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_TCP	(0 << 30)
   1423  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_UDP	(1 << 30)
   1424   1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_inner;
   1425   1.1  jmcneill 	uint8_t			cmd_reserved3[152];
   1426   1.1  jmcneill } __packed __aligned(4);
   1427   1.1  jmcneill 
   1428   1.1  jmcneill struct mcx_cmd_create_tir_out {
   1429   1.1  jmcneill 	uint8_t			cmd_status;
   1430   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1431   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1432   1.1  jmcneill 	uint32_t		cmd_tirn;
   1433   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1434   1.1  jmcneill } __packed __aligned(4);
   1435   1.1  jmcneill 
   1436   1.1  jmcneill struct mcx_cmd_destroy_tir_in {
   1437   1.1  jmcneill 	uint16_t		cmd_opcode;
   1438   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1439   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1440   1.1  jmcneill 	uint32_t		cmd_tirn;
   1441   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1442   1.1  jmcneill } __packed __aligned(4);
   1443   1.1  jmcneill 
   1444   1.1  jmcneill struct mcx_cmd_destroy_tir_out {
   1445   1.1  jmcneill 	uint8_t			cmd_status;
   1446   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1447   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1448   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1449   1.1  jmcneill } __packed __aligned(4);
   1450   1.1  jmcneill 
   1451   1.1  jmcneill struct mcx_cmd_create_tis_in {
   1452   1.1  jmcneill 	uint16_t		cmd_opcode;
   1453   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1454   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1455   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1456   1.1  jmcneill } __packed __aligned(4);
   1457   1.1  jmcneill 
   1458   1.1  jmcneill struct mcx_cmd_create_tis_mb_in {
   1459   1.1  jmcneill 	uint8_t			cmd_reserved[16];
   1460   1.1  jmcneill 	uint32_t		cmd_prio;
   1461   1.1  jmcneill 	uint8_t			cmd_reserved1[32];
   1462   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1463   1.1  jmcneill 	uint8_t			cmd_reserved2[120];
   1464   1.1  jmcneill } __packed __aligned(4);
   1465   1.1  jmcneill 
   1466   1.1  jmcneill struct mcx_cmd_create_tis_out {
   1467   1.1  jmcneill 	uint8_t			cmd_status;
   1468   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1469   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1470   1.1  jmcneill 	uint32_t		cmd_tisn;
   1471   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1472   1.1  jmcneill } __packed __aligned(4);
   1473   1.1  jmcneill 
   1474   1.1  jmcneill struct mcx_cmd_destroy_tis_in {
   1475   1.1  jmcneill 	uint16_t		cmd_opcode;
   1476   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1477   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1478   1.1  jmcneill 	uint32_t		cmd_tisn;
   1479   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1480   1.1  jmcneill } __packed __aligned(4);
   1481   1.1  jmcneill 
   1482   1.1  jmcneill struct mcx_cmd_destroy_tis_out {
   1483   1.1  jmcneill 	uint8_t			cmd_status;
   1484   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1485   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1486   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1487   1.1  jmcneill } __packed __aligned(4);
   1488   1.1  jmcneill 
   1489  1.15  jmcneill struct mcx_cmd_create_rqt_in {
   1490  1.15  jmcneill 	uint16_t		cmd_opcode;
   1491  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1492  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1493  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1494  1.15  jmcneill } __packed __aligned(4);
   1495  1.15  jmcneill 
   1496  1.15  jmcneill struct mcx_rqt_ctx {
   1497  1.15  jmcneill 	uint8_t			cmd_reserved0[20];
   1498  1.15  jmcneill 	uint16_t		cmd_reserved1;
   1499  1.15  jmcneill 	uint16_t		cmd_rqt_max_size;
   1500  1.15  jmcneill 	uint16_t		cmd_reserved2;
   1501  1.15  jmcneill 	uint16_t		cmd_rqt_actual_size;
   1502  1.15  jmcneill 	uint8_t			cmd_reserved3[212];
   1503  1.15  jmcneill } __packed __aligned(4);
   1504  1.15  jmcneill 
   1505  1.15  jmcneill struct mcx_cmd_create_rqt_mb_in {
   1506  1.15  jmcneill 	uint8_t			cmd_reserved0[16];
   1507  1.15  jmcneill 	struct mcx_rqt_ctx	cmd_rqt;
   1508  1.15  jmcneill } __packed __aligned(4);
   1509  1.15  jmcneill 
   1510  1.15  jmcneill struct mcx_cmd_create_rqt_out {
   1511  1.15  jmcneill 	uint8_t			cmd_status;
   1512  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1513  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1514  1.15  jmcneill 	uint32_t		cmd_rqtn;
   1515  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1516  1.15  jmcneill } __packed __aligned(4);
   1517  1.15  jmcneill 
   1518  1.15  jmcneill struct mcx_cmd_destroy_rqt_in {
   1519  1.15  jmcneill 	uint16_t		cmd_opcode;
   1520  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1521  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1522  1.15  jmcneill 	uint32_t		cmd_rqtn;
   1523  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1524  1.15  jmcneill } __packed __aligned(4);
   1525  1.15  jmcneill 
   1526  1.15  jmcneill struct mcx_cmd_destroy_rqt_out {
   1527  1.15  jmcneill 	uint8_t			cmd_status;
   1528  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1529  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1530  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1531  1.15  jmcneill } __packed __aligned(4);
   1532  1.15  jmcneill 
   1533   1.1  jmcneill struct mcx_cq_ctx {
   1534   1.1  jmcneill 	uint32_t		cq_status;
   1535  1.15  jmcneill #define MCX_CQ_CTX_STATUS_SHIFT		28
   1536  1.15  jmcneill #define MCX_CQ_CTX_STATUS_MASK		(0xf << MCX_CQ_CTX_STATUS_SHIFT)
   1537  1.15  jmcneill #define MCX_CQ_CTX_STATUS_OK		0x0
   1538  1.15  jmcneill #define MCX_CQ_CTX_STATUS_OVERFLOW	0x9
   1539  1.15  jmcneill #define MCX_CQ_CTX_STATUS_WRITE_FAIL	0xa
   1540  1.15  jmcneill #define MCX_CQ_CTX_STATE_SHIFT		8
   1541  1.15  jmcneill #define MCX_CQ_CTX_STATE_MASK		(0xf << MCX_CQ_CTX_STATE_SHIFT)
   1542  1.15  jmcneill #define MCX_CQ_CTX_STATE_SOLICITED	0x6
   1543  1.15  jmcneill #define MCX_CQ_CTX_STATE_ARMED		0x9
   1544  1.15  jmcneill #define MCX_CQ_CTX_STATE_FIRED		0xa
   1545   1.1  jmcneill 	uint32_t		cq_reserved1;
   1546   1.1  jmcneill 	uint32_t		cq_page_offset;
   1547   1.1  jmcneill 	uint32_t		cq_uar_size;
   1548   1.1  jmcneill #define MCX_CQ_CTX_UAR_PAGE_MASK	0xffffff
   1549   1.1  jmcneill #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT	24
   1550   1.1  jmcneill 	uint32_t		cq_period_max_count;
   1551   1.1  jmcneill #define MCX_CQ_CTX_PERIOD_SHIFT		16
   1552   1.1  jmcneill 	uint32_t		cq_eqn;
   1553   1.1  jmcneill 	uint32_t		cq_log_page_size;
   1554   1.1  jmcneill #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1555   1.1  jmcneill 	uint32_t		cq_reserved2;
   1556   1.1  jmcneill 	uint32_t		cq_last_notified;
   1557   1.1  jmcneill 	uint32_t		cq_last_solicit;
   1558   1.1  jmcneill 	uint32_t		cq_consumer_counter;
   1559   1.1  jmcneill 	uint32_t		cq_producer_counter;
   1560   1.1  jmcneill 	uint8_t			cq_reserved3[8];
   1561   1.1  jmcneill 	uint64_t		cq_doorbell;
   1562   1.1  jmcneill } __packed __aligned(4);
   1563   1.1  jmcneill 
   1564   1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
   1565   1.1  jmcneill 
   1566   1.1  jmcneill struct mcx_cmd_create_cq_in {
   1567   1.1  jmcneill 	uint16_t		cmd_opcode;
   1568   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1569   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1570   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1571   1.1  jmcneill } __packed __aligned(4);
   1572   1.1  jmcneill 
   1573   1.1  jmcneill struct mcx_cmd_create_cq_mb_in {
   1574   1.1  jmcneill 	struct mcx_cq_ctx	cmd_cq_ctx;
   1575   1.1  jmcneill 	uint8_t			cmd_reserved1[192];
   1576   1.1  jmcneill } __packed __aligned(4);
   1577   1.1  jmcneill 
   1578   1.1  jmcneill struct mcx_cmd_create_cq_out {
   1579   1.1  jmcneill 	uint8_t			cmd_status;
   1580   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1581   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1582   1.1  jmcneill 	uint32_t		cmd_cqn;
   1583   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1584   1.1  jmcneill } __packed __aligned(4);
   1585   1.1  jmcneill 
   1586   1.1  jmcneill struct mcx_cmd_destroy_cq_in {
   1587   1.1  jmcneill 	uint16_t		cmd_opcode;
   1588   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1589   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1590   1.1  jmcneill 	uint32_t		cmd_cqn;
   1591   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1592   1.1  jmcneill } __packed __aligned(4);
   1593   1.1  jmcneill 
   1594   1.1  jmcneill struct mcx_cmd_destroy_cq_out {
   1595   1.1  jmcneill 	uint8_t			cmd_status;
   1596   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1597   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1598   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1599   1.1  jmcneill } __packed __aligned(4);
   1600   1.1  jmcneill 
   1601  1.15  jmcneill struct mcx_cmd_query_cq_in {
   1602  1.15  jmcneill 	uint16_t		cmd_opcode;
   1603  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1604  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1605  1.15  jmcneill 	uint32_t		cmd_cqn;
   1606  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1607  1.15  jmcneill } __packed __aligned(4);
   1608  1.15  jmcneill 
   1609  1.15  jmcneill struct mcx_cmd_query_cq_out {
   1610  1.15  jmcneill 	uint8_t			cmd_status;
   1611  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1612  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1613  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1614  1.15  jmcneill } __packed __aligned(4);
   1615  1.15  jmcneill 
   1616   1.1  jmcneill struct mcx_cq_entry {
   1617   1.1  jmcneill 	uint32_t		__reserved__;
   1618   1.1  jmcneill 	uint32_t		cq_lro;
   1619   1.1  jmcneill 	uint32_t		cq_lro_ack_seq_num;
   1620   1.1  jmcneill 	uint32_t		cq_rx_hash;
   1621   1.1  jmcneill 	uint8_t			cq_rx_hash_type;
   1622   1.1  jmcneill 	uint8_t			cq_ml_path;
   1623   1.1  jmcneill 	uint16_t		__reserved__;
   1624   1.1  jmcneill 	uint32_t		cq_checksum;
   1625   1.1  jmcneill 	uint32_t		__reserved__;
   1626   1.1  jmcneill 	uint32_t		cq_flags;
   1627  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L4_OK		(1 << 26)
   1628  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L3_OK		(1 << 25)
   1629  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L2_OK		(1 << 24)
   1630  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_CV			(1 << 16)
   1631  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK		(0xffff)
   1632  1.15  jmcneill 
   1633   1.1  jmcneill 	uint32_t		cq_lro_srqn;
   1634   1.1  jmcneill 	uint32_t		__reserved__[2];
   1635   1.1  jmcneill 	uint32_t		cq_byte_cnt;
   1636   1.1  jmcneill 	uint64_t		cq_timestamp;
   1637   1.1  jmcneill 	uint8_t			cq_rx_drops;
   1638   1.1  jmcneill 	uint8_t			cq_flow_tag[3];
   1639   1.1  jmcneill 	uint16_t		cq_wqe_count;
   1640   1.1  jmcneill 	uint8_t			cq_signature;
   1641   1.1  jmcneill 	uint8_t			cq_opcode_owner;
   1642   1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_OWNER			(1 << 0)
   1643   1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_SE			(1 << 1)
   1644   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_SHIFT		2
   1645   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SHIFT		4
   1646   1.1  jmcneill 
   1647   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_NO_INLINE		0
   1648   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_32		1
   1649   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_64		2
   1650   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_COMPRESSED		3
   1651   1.1  jmcneill 
   1652   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ			0
   1653   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND		2
   1654   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ_ERR		13
   1655   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND_ERR		14
   1656   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_INVALID		15
   1657   1.1  jmcneill 
   1658   1.1  jmcneill } __packed __aligned(4);
   1659   1.1  jmcneill 
   1660   1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_entry) == 64);
   1661   1.1  jmcneill 
   1662   1.1  jmcneill struct mcx_cq_doorbell {
   1663   1.1  jmcneill 	uint32_t		 db_update_ci;
   1664   1.1  jmcneill 	uint32_t		 db_arm_ci;
   1665   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT	28
   1666   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD			(1 << 24)
   1667   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CI_MASK		(0xffffff)
   1668   1.1  jmcneill } __packed __aligned(8);
   1669   1.1  jmcneill 
   1670   1.1  jmcneill struct mcx_wq_ctx {
   1671   1.1  jmcneill 	uint8_t			 wq_type;
   1672   1.1  jmcneill #define MCX_WQ_CTX_TYPE_CYCLIC			(1 << 4)
   1673   1.1  jmcneill #define MCX_WQ_CTX_TYPE_SIGNATURE		(1 << 3)
   1674   1.1  jmcneill 	uint8_t			 wq_reserved0[5];
   1675   1.1  jmcneill 	uint16_t		 wq_lwm;
   1676   1.1  jmcneill 	uint32_t		 wq_pd;
   1677   1.1  jmcneill 	uint32_t		 wq_uar_page;
   1678   1.1  jmcneill 	uint64_t		 wq_doorbell;
   1679   1.1  jmcneill 	uint32_t		 wq_hw_counter;
   1680   1.1  jmcneill 	uint32_t		 wq_sw_counter;
   1681   1.1  jmcneill 	uint16_t		 wq_log_stride;
   1682   1.1  jmcneill 	uint8_t			 wq_log_page_sz;
   1683   1.1  jmcneill 	uint8_t			 wq_log_size;
   1684   1.1  jmcneill 	uint8_t			 wq_reserved1[156];
   1685   1.1  jmcneill } __packed __aligned(4);
   1686   1.1  jmcneill 
   1687   1.1  jmcneill CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
   1688   1.1  jmcneill 
   1689   1.1  jmcneill struct mcx_sq_ctx {
   1690   1.1  jmcneill 	uint32_t		sq_flags;
   1691   1.5   msaitoh #define MCX_SQ_CTX_RLKEY			(1U << 31)
   1692   1.1  jmcneill #define MCX_SQ_CTX_FRE_SHIFT			(1 << 29)
   1693   1.1  jmcneill #define MCX_SQ_CTX_FLUSH_IN_ERROR		(1 << 28)
   1694   1.1  jmcneill #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT		24
   1695   1.1  jmcneill #define MCX_SQ_CTX_STATE_SHIFT			20
   1696  1.15  jmcneill #define MCX_SQ_CTX_STATE_MASK			(0xf << 20)
   1697  1.15  jmcneill #define MCX_SQ_CTX_STATE_RST			0
   1698  1.15  jmcneill #define MCX_SQ_CTX_STATE_RDY			1
   1699  1.15  jmcneill #define MCX_SQ_CTX_STATE_ERR			3
   1700   1.1  jmcneill 	uint32_t		sq_user_index;
   1701   1.1  jmcneill 	uint32_t		sq_cqn;
   1702   1.1  jmcneill 	uint32_t		sq_reserved1[5];
   1703   1.1  jmcneill 	uint32_t		sq_tis_lst_sz;
   1704   1.1  jmcneill #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT		16
   1705   1.1  jmcneill 	uint32_t		sq_reserved2[2];
   1706   1.1  jmcneill 	uint32_t		sq_tis_num;
   1707   1.1  jmcneill 	struct mcx_wq_ctx	sq_wq;
   1708   1.1  jmcneill } __packed __aligned(4);
   1709   1.1  jmcneill 
   1710   1.1  jmcneill struct mcx_sq_entry_seg {
   1711   1.1  jmcneill 	uint32_t		sqs_byte_count;
   1712   1.1  jmcneill 	uint32_t		sqs_lkey;
   1713   1.1  jmcneill 	uint64_t		sqs_addr;
   1714   1.1  jmcneill } __packed __aligned(4);
   1715   1.1  jmcneill 
   1716   1.1  jmcneill struct mcx_sq_entry {
   1717   1.1  jmcneill 	/* control segment */
   1718   1.1  jmcneill 	uint32_t		sqe_opcode_index;
   1719   1.1  jmcneill #define MCX_SQE_WQE_INDEX_SHIFT			8
   1720   1.1  jmcneill #define MCX_SQE_WQE_OPCODE_NOP			0x00
   1721   1.1  jmcneill #define MCX_SQE_WQE_OPCODE_SEND			0x0a
   1722   1.1  jmcneill 	uint32_t		sqe_ds_sq_num;
   1723   1.1  jmcneill #define MCX_SQE_SQ_NUM_SHIFT			8
   1724   1.1  jmcneill 	uint32_t		sqe_signature;
   1725   1.1  jmcneill #define MCX_SQE_SIGNATURE_SHIFT			24
   1726   1.1  jmcneill #define MCX_SQE_SOLICITED_EVENT			0x02
   1727   1.1  jmcneill #define MCX_SQE_CE_CQE_ON_ERR			0x00
   1728   1.1  jmcneill #define MCX_SQE_CE_CQE_FIRST_ERR		0x04
   1729   1.1  jmcneill #define MCX_SQE_CE_CQE_ALWAYS			0x08
   1730   1.1  jmcneill #define MCX_SQE_CE_CQE_SOLICIT			0x0C
   1731   1.1  jmcneill #define MCX_SQE_FM_NO_FENCE			0x00
   1732   1.1  jmcneill #define MCX_SQE_FM_SMALL_FENCE			0x40
   1733   1.1  jmcneill 	uint32_t		sqe_mkey;
   1734   1.1  jmcneill 
   1735   1.1  jmcneill 	/* ethernet segment */
   1736   1.1  jmcneill 	uint32_t		sqe_reserved1;
   1737   1.1  jmcneill 	uint32_t		sqe_mss_csum;
   1738   1.1  jmcneill #define MCX_SQE_L4_CSUM				(1 << 31)
   1739   1.1  jmcneill #define MCX_SQE_L3_CSUM				(1 << 30)
   1740   1.1  jmcneill 	uint32_t		sqe_reserved2;
   1741   1.1  jmcneill 	uint16_t		sqe_inline_header_size;
   1742   1.1  jmcneill 	uint16_t		sqe_inline_headers[9];
   1743   1.1  jmcneill 
   1744   1.1  jmcneill 	/* data segment */
   1745   1.1  jmcneill 	struct mcx_sq_entry_seg sqe_segs[1];
   1746   1.1  jmcneill } __packed __aligned(64);
   1747   1.1  jmcneill 
   1748   1.1  jmcneill CTASSERT(sizeof(struct mcx_sq_entry) == 64);
   1749   1.1  jmcneill 
   1750   1.1  jmcneill struct mcx_cmd_create_sq_in {
   1751   1.1  jmcneill 	uint16_t		cmd_opcode;
   1752   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1753   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1754   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1755   1.1  jmcneill } __packed __aligned(4);
   1756   1.1  jmcneill 
   1757   1.1  jmcneill struct mcx_cmd_create_sq_out {
   1758   1.1  jmcneill 	uint8_t			cmd_status;
   1759   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1760   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1761   1.1  jmcneill 	uint32_t		cmd_sqn;
   1762   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1763   1.1  jmcneill } __packed __aligned(4);
   1764   1.1  jmcneill 
   1765   1.1  jmcneill struct mcx_cmd_modify_sq_in {
   1766   1.1  jmcneill 	uint16_t		cmd_opcode;
   1767   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1768   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1769   1.1  jmcneill 	uint32_t		cmd_sq_state;
   1770   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1771   1.1  jmcneill } __packed __aligned(4);
   1772   1.1  jmcneill 
   1773   1.1  jmcneill struct mcx_cmd_modify_sq_mb_in {
   1774   1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1775   1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1776   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1777   1.1  jmcneill 	struct mcx_sq_ctx	cmd_sq_ctx;
   1778   1.1  jmcneill } __packed __aligned(4);
   1779   1.1  jmcneill 
   1780   1.1  jmcneill struct mcx_cmd_modify_sq_out {
   1781   1.1  jmcneill 	uint8_t			cmd_status;
   1782   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1783   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1784   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1785   1.1  jmcneill } __packed __aligned(4);
   1786   1.1  jmcneill 
   1787   1.1  jmcneill struct mcx_cmd_destroy_sq_in {
   1788   1.1  jmcneill 	uint16_t		cmd_opcode;
   1789   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1790   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1791   1.1  jmcneill 	uint32_t		cmd_sqn;
   1792   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1793   1.1  jmcneill } __packed __aligned(4);
   1794   1.1  jmcneill 
   1795   1.1  jmcneill struct mcx_cmd_destroy_sq_out {
   1796   1.1  jmcneill 	uint8_t			cmd_status;
   1797   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1798   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1799   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1800   1.1  jmcneill } __packed __aligned(4);
   1801   1.1  jmcneill 
   1802   1.1  jmcneill 
   1803   1.1  jmcneill struct mcx_rq_ctx {
   1804   1.1  jmcneill 	uint32_t		rq_flags;
   1805   1.5   msaitoh #define MCX_RQ_CTX_RLKEY			(1U << 31)
   1806   1.1  jmcneill #define MCX_RQ_CTX_VLAN_STRIP_DIS		(1 << 28)
   1807   1.1  jmcneill #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT		24
   1808   1.1  jmcneill #define MCX_RQ_CTX_STATE_SHIFT			20
   1809  1.15  jmcneill #define MCX_RQ_CTX_STATE_MASK			(0xf << 20)
   1810  1.15  jmcneill #define MCX_RQ_CTX_STATE_RST			0
   1811  1.15  jmcneill #define MCX_RQ_CTX_STATE_RDY			1
   1812  1.15  jmcneill #define MCX_RQ_CTX_STATE_ERR			3
   1813   1.1  jmcneill #define MCX_RQ_CTX_FLUSH_IN_ERROR		(1 << 18)
   1814   1.1  jmcneill 	uint32_t		rq_user_index;
   1815   1.1  jmcneill 	uint32_t		rq_cqn;
   1816   1.1  jmcneill 	uint32_t		rq_reserved1;
   1817   1.1  jmcneill 	uint32_t		rq_rmpn;
   1818   1.1  jmcneill 	uint32_t		rq_reserved2[7];
   1819   1.1  jmcneill 	struct mcx_wq_ctx	rq_wq;
   1820   1.1  jmcneill } __packed __aligned(4);
   1821   1.1  jmcneill 
   1822   1.1  jmcneill struct mcx_rq_entry {
   1823   1.1  jmcneill 	uint32_t		rqe_byte_count;
   1824   1.1  jmcneill 	uint32_t		rqe_lkey;
   1825   1.1  jmcneill 	uint64_t		rqe_addr;
   1826   1.1  jmcneill } __packed __aligned(16);
   1827   1.1  jmcneill 
   1828   1.1  jmcneill struct mcx_cmd_create_rq_in {
   1829   1.1  jmcneill 	uint16_t		cmd_opcode;
   1830   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1831   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1832   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1833   1.1  jmcneill } __packed __aligned(4);
   1834   1.1  jmcneill 
   1835   1.1  jmcneill struct mcx_cmd_create_rq_out {
   1836   1.1  jmcneill 	uint8_t			cmd_status;
   1837   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1838   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1839   1.1  jmcneill 	uint32_t		cmd_rqn;
   1840   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1841   1.1  jmcneill } __packed __aligned(4);
   1842   1.1  jmcneill 
   1843   1.1  jmcneill struct mcx_cmd_modify_rq_in {
   1844   1.1  jmcneill 	uint16_t		cmd_opcode;
   1845   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1846   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1847   1.1  jmcneill 	uint32_t		cmd_rq_state;
   1848   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1849   1.1  jmcneill } __packed __aligned(4);
   1850   1.1  jmcneill 
   1851   1.1  jmcneill struct mcx_cmd_modify_rq_mb_in {
   1852   1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1853   1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1854   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1855   1.1  jmcneill 	struct mcx_rq_ctx	cmd_rq_ctx;
   1856   1.1  jmcneill } __packed __aligned(4);
   1857   1.1  jmcneill 
   1858   1.1  jmcneill struct mcx_cmd_modify_rq_out {
   1859   1.1  jmcneill 	uint8_t			cmd_status;
   1860   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1861   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1862   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1863   1.1  jmcneill } __packed __aligned(4);
   1864   1.1  jmcneill 
   1865   1.1  jmcneill struct mcx_cmd_destroy_rq_in {
   1866   1.1  jmcneill 	uint16_t		cmd_opcode;
   1867   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1868   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1869   1.1  jmcneill 	uint32_t		cmd_rqn;
   1870   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1871   1.1  jmcneill } __packed __aligned(4);
   1872   1.1  jmcneill 
   1873   1.1  jmcneill struct mcx_cmd_destroy_rq_out {
   1874   1.1  jmcneill 	uint8_t			cmd_status;
   1875   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1876   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1877   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1878   1.1  jmcneill } __packed __aligned(4);
   1879   1.1  jmcneill 
   1880   1.1  jmcneill struct mcx_cmd_create_flow_table_in {
   1881   1.1  jmcneill 	uint16_t		cmd_opcode;
   1882   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1883   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1884   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1885   1.1  jmcneill } __packed __aligned(4);
   1886   1.1  jmcneill 
   1887   1.1  jmcneill struct mcx_flow_table_ctx {
   1888   1.1  jmcneill 	uint8_t			ft_miss_action;
   1889   1.1  jmcneill 	uint8_t			ft_level;
   1890   1.1  jmcneill 	uint8_t			ft_reserved0;
   1891   1.1  jmcneill 	uint8_t			ft_log_size;
   1892   1.1  jmcneill 	uint32_t		ft_table_miss_id;
   1893   1.1  jmcneill 	uint8_t			ft_reserved1[28];
   1894   1.1  jmcneill } __packed __aligned(4);
   1895   1.1  jmcneill 
   1896   1.1  jmcneill struct mcx_cmd_create_flow_table_mb_in {
   1897   1.1  jmcneill 	uint8_t			cmd_table_type;
   1898   1.1  jmcneill 	uint8_t			cmd_reserved0[7];
   1899   1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   1900   1.1  jmcneill } __packed __aligned(4);
   1901   1.1  jmcneill 
   1902   1.1  jmcneill struct mcx_cmd_create_flow_table_out {
   1903   1.1  jmcneill 	uint8_t			cmd_status;
   1904   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1905   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1906   1.1  jmcneill 	uint32_t		cmd_table_id;
   1907   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1908   1.1  jmcneill } __packed __aligned(4);
   1909   1.1  jmcneill 
   1910   1.1  jmcneill struct mcx_cmd_destroy_flow_table_in {
   1911   1.1  jmcneill 	uint16_t		cmd_opcode;
   1912   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1913   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1914   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1915   1.1  jmcneill } __packed __aligned(4);
   1916   1.1  jmcneill 
   1917   1.1  jmcneill struct mcx_cmd_destroy_flow_table_mb_in {
   1918   1.1  jmcneill 	uint8_t			cmd_table_type;
   1919   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1920   1.1  jmcneill 	uint32_t		cmd_table_id;
   1921   1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   1922   1.1  jmcneill } __packed __aligned(4);
   1923   1.1  jmcneill 
   1924   1.1  jmcneill struct mcx_cmd_destroy_flow_table_out {
   1925   1.1  jmcneill 	uint8_t			cmd_status;
   1926   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1927   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1928   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1929   1.1  jmcneill } __packed __aligned(4);
   1930   1.1  jmcneill 
   1931   1.1  jmcneill struct mcx_cmd_set_flow_table_root_in {
   1932   1.1  jmcneill 	uint16_t		cmd_opcode;
   1933   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1934   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1935   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1936   1.1  jmcneill } __packed __aligned(4);
   1937   1.1  jmcneill 
   1938   1.1  jmcneill struct mcx_cmd_set_flow_table_root_mb_in {
   1939   1.1  jmcneill 	uint8_t			cmd_table_type;
   1940   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1941   1.1  jmcneill 	uint32_t		cmd_table_id;
   1942   1.1  jmcneill 	uint8_t			cmd_reserved1[56];
   1943   1.1  jmcneill } __packed __aligned(4);
   1944   1.1  jmcneill 
   1945   1.1  jmcneill struct mcx_cmd_set_flow_table_root_out {
   1946   1.1  jmcneill 	uint8_t			cmd_status;
   1947   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1948   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1949   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1950   1.1  jmcneill } __packed __aligned(4);
   1951   1.1  jmcneill 
   1952   1.1  jmcneill struct mcx_flow_match {
   1953   1.1  jmcneill 	/* outer headers */
   1954   1.1  jmcneill 	uint8_t			mc_src_mac[6];
   1955   1.1  jmcneill 	uint16_t		mc_ethertype;
   1956   1.1  jmcneill 	uint8_t			mc_dest_mac[6];
   1957   1.1  jmcneill 	uint16_t		mc_first_vlan;
   1958   1.1  jmcneill 	uint8_t			mc_ip_proto;
   1959   1.1  jmcneill 	uint8_t			mc_ip_dscp_ecn;
   1960   1.1  jmcneill 	uint8_t			mc_vlan_flags;
   1961  1.15  jmcneill #define MCX_FLOW_MATCH_IP_FRAG	(1 << 5)
   1962   1.1  jmcneill 	uint8_t			mc_tcp_flags;
   1963   1.1  jmcneill 	uint16_t		mc_tcp_sport;
   1964   1.1  jmcneill 	uint16_t		mc_tcp_dport;
   1965   1.1  jmcneill 	uint32_t		mc_reserved0;
   1966   1.1  jmcneill 	uint16_t		mc_udp_sport;
   1967   1.1  jmcneill 	uint16_t		mc_udp_dport;
   1968   1.1  jmcneill 	uint8_t			mc_src_ip[16];
   1969   1.1  jmcneill 	uint8_t			mc_dest_ip[16];
   1970   1.1  jmcneill 
   1971   1.1  jmcneill 	/* misc parameters */
   1972   1.1  jmcneill 	uint8_t			mc_reserved1[8];
   1973   1.1  jmcneill 	uint16_t		mc_second_vlan;
   1974   1.1  jmcneill 	uint8_t			mc_reserved2[2];
   1975   1.1  jmcneill 	uint8_t			mc_second_vlan_flags;
   1976   1.1  jmcneill 	uint8_t			mc_reserved3[15];
   1977   1.1  jmcneill 	uint32_t		mc_outer_ipv6_flow_label;
   1978   1.1  jmcneill 	uint8_t			mc_reserved4[32];
   1979   1.1  jmcneill 
   1980   1.1  jmcneill 	uint8_t			mc_reserved[384];
   1981   1.1  jmcneill } __packed __aligned(4);
   1982   1.1  jmcneill 
   1983   1.1  jmcneill CTASSERT(sizeof(struct mcx_flow_match) == 512);
   1984   1.1  jmcneill 
   1985   1.1  jmcneill struct mcx_cmd_create_flow_group_in {
   1986   1.1  jmcneill 	uint16_t		cmd_opcode;
   1987   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1988   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1989   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1990   1.1  jmcneill } __packed __aligned(4);
   1991   1.1  jmcneill 
   1992   1.1  jmcneill struct mcx_cmd_create_flow_group_mb_in {
   1993   1.1  jmcneill 	uint8_t			cmd_table_type;
   1994   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1995   1.1  jmcneill 	uint32_t		cmd_table_id;
   1996   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1997   1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   1998   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   1999   1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   2000   1.1  jmcneill 	uint8_t			cmd_reserved3[23];
   2001   1.1  jmcneill 	uint8_t			cmd_match_criteria_enable;
   2002   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER	(1 << 0)
   2003   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_MISC		(1 << 1)
   2004   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_INNER	(1 << 2)
   2005   1.1  jmcneill 	struct mcx_flow_match	cmd_match_criteria;
   2006   1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   2007   1.1  jmcneill } __packed __aligned(4);
   2008   1.1  jmcneill 
   2009   1.1  jmcneill struct mcx_cmd_create_flow_group_out {
   2010   1.1  jmcneill 	uint8_t			cmd_status;
   2011   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2012   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2013   1.1  jmcneill 	uint32_t		cmd_group_id;
   2014   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2015   1.1  jmcneill } __packed __aligned(4);
   2016   1.1  jmcneill 
   2017   1.1  jmcneill struct mcx_flow_ctx {
   2018   1.1  jmcneill 	uint8_t			fc_reserved0[4];
   2019   1.1  jmcneill 	uint32_t		fc_group_id;
   2020   1.1  jmcneill 	uint32_t		fc_flow_tag;
   2021   1.1  jmcneill 	uint32_t		fc_action;
   2022   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_ALLOW		(1 << 0)
   2023   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_DROP		(1 << 1)
   2024   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_FORWARD		(1 << 2)
   2025   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_COUNT		(1 << 3)
   2026   1.1  jmcneill 	uint32_t		fc_dest_list_size;
   2027   1.1  jmcneill 	uint32_t		fc_counter_list_size;
   2028   1.1  jmcneill 	uint8_t			fc_reserved1[40];
   2029   1.1  jmcneill 	struct mcx_flow_match	fc_match_value;
   2030   1.1  jmcneill 	uint8_t			fc_reserved2[192];
   2031   1.1  jmcneill } __packed __aligned(4);
   2032   1.1  jmcneill 
   2033   1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE	(1 << 24)
   2034   1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR		(2 << 24)
   2035   1.1  jmcneill 
   2036   1.1  jmcneill struct mcx_cmd_destroy_flow_group_in {
   2037   1.1  jmcneill 	uint16_t		cmd_opcode;
   2038   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2039   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2040   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2041   1.1  jmcneill } __packed __aligned(4);
   2042   1.1  jmcneill 
   2043   1.1  jmcneill struct mcx_cmd_destroy_flow_group_mb_in {
   2044   1.1  jmcneill 	uint8_t			cmd_table_type;
   2045   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2046   1.1  jmcneill 	uint32_t		cmd_table_id;
   2047   1.1  jmcneill 	uint32_t		cmd_group_id;
   2048   1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   2049   1.1  jmcneill } __packed __aligned(4);
   2050   1.1  jmcneill 
   2051   1.1  jmcneill struct mcx_cmd_destroy_flow_group_out {
   2052   1.1  jmcneill 	uint8_t			cmd_status;
   2053   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2054   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2055   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2056   1.1  jmcneill } __packed __aligned(4);
   2057   1.1  jmcneill 
   2058   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_in {
   2059   1.1  jmcneill 	uint16_t		cmd_opcode;
   2060   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2061   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2062   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2063   1.1  jmcneill } __packed __aligned(4);
   2064   1.1  jmcneill 
   2065   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_mb_in {
   2066   1.1  jmcneill 	uint8_t			cmd_table_type;
   2067   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2068   1.1  jmcneill 	uint32_t		cmd_table_id;
   2069   1.1  jmcneill 	uint32_t		cmd_modify_enable_mask;
   2070   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2071   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2072   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2073   1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   2074   1.1  jmcneill } __packed __aligned(4);
   2075   1.1  jmcneill 
   2076   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_out {
   2077   1.1  jmcneill 	uint8_t			cmd_status;
   2078   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2079   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2080   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2081   1.1  jmcneill } __packed __aligned(4);
   2082   1.1  jmcneill 
   2083   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_in {
   2084   1.1  jmcneill 	uint16_t		cmd_opcode;
   2085   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2086   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2087   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2088   1.1  jmcneill } __packed __aligned(4);
   2089   1.1  jmcneill 
   2090   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_in {
   2091   1.1  jmcneill 	uint8_t			cmd_table_type;
   2092   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2093   1.1  jmcneill 	uint32_t		cmd_table_id;
   2094   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2095   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2096   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2097   1.1  jmcneill } __packed __aligned(4);
   2098   1.1  jmcneill 
   2099   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_out {
   2100   1.1  jmcneill 	uint8_t			cmd_status;
   2101   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2102   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2103   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2104   1.1  jmcneill } __packed __aligned(4);
   2105   1.1  jmcneill 
   2106   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_out {
   2107   1.1  jmcneill 	uint8_t			cmd_reserved0[48];
   2108   1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   2109   1.1  jmcneill } __packed __aligned(4);
   2110   1.1  jmcneill 
   2111   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_in {
   2112   1.1  jmcneill 	uint16_t		cmd_opcode;
   2113   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2114   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2115   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2116   1.1  jmcneill } __packed __aligned(4);
   2117   1.1  jmcneill 
   2118   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in {
   2119   1.1  jmcneill 	uint8_t			cmd_table_type;
   2120   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2121   1.1  jmcneill 	uint32_t		cmd_table_id;
   2122   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2123   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2124   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2125   1.1  jmcneill } __packed __aligned(4);
   2126   1.1  jmcneill 
   2127   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_out {
   2128   1.1  jmcneill 	uint8_t			cmd_status;
   2129   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2130   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2131   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2132   1.1  jmcneill } __packed __aligned(4);
   2133   1.1  jmcneill 
   2134   1.1  jmcneill struct mcx_cmd_query_flow_group_in {
   2135   1.1  jmcneill 	uint16_t		cmd_opcode;
   2136   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2137   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2138   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2139   1.1  jmcneill } __packed __aligned(4);
   2140   1.1  jmcneill 
   2141   1.1  jmcneill struct mcx_cmd_query_flow_group_mb_in {
   2142   1.1  jmcneill 	uint8_t			cmd_table_type;
   2143   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2144   1.1  jmcneill 	uint32_t		cmd_table_id;
   2145   1.1  jmcneill 	uint32_t		cmd_group_id;
   2146   1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   2147   1.1  jmcneill } __packed __aligned(4);
   2148   1.1  jmcneill 
   2149   1.1  jmcneill struct mcx_cmd_query_flow_group_out {
   2150   1.1  jmcneill 	uint8_t			cmd_status;
   2151   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2152   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2153   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2154   1.1  jmcneill } __packed __aligned(4);
   2155   1.1  jmcneill 
   2156   1.1  jmcneill struct mcx_cmd_query_flow_group_mb_out {
   2157   1.1  jmcneill 	uint8_t			cmd_reserved0[12];
   2158   1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   2159   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2160   1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   2161   1.1  jmcneill 	uint8_t			cmd_reserved2[20];
   2162   1.1  jmcneill 	uint32_t		cmd_match_criteria_enable;
   2163   1.1  jmcneill 	uint8_t			cmd_match_criteria[512];
   2164   1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   2165   1.1  jmcneill } __packed __aligned(4);
   2166   1.1  jmcneill 
   2167   1.1  jmcneill struct mcx_cmd_query_flow_table_in {
   2168   1.1  jmcneill 	uint16_t		cmd_opcode;
   2169   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2170   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2171   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2172   1.1  jmcneill } __packed __aligned(4);
   2173   1.1  jmcneill 
   2174   1.1  jmcneill struct mcx_cmd_query_flow_table_mb_in {
   2175   1.1  jmcneill 	uint8_t			cmd_table_type;
   2176   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2177   1.1  jmcneill 	uint32_t		cmd_table_id;
   2178   1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   2179   1.1  jmcneill } __packed __aligned(4);
   2180   1.1  jmcneill 
   2181   1.1  jmcneill struct mcx_cmd_query_flow_table_out {
   2182   1.1  jmcneill 	uint8_t			cmd_status;
   2183   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2184   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2185   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2186   1.1  jmcneill } __packed __aligned(4);
   2187   1.1  jmcneill 
   2188   1.1  jmcneill struct mcx_cmd_query_flow_table_mb_out {
   2189   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2190   1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   2191   1.1  jmcneill } __packed __aligned(4);
   2192   1.1  jmcneill 
   2193   1.1  jmcneill struct mcx_cmd_alloc_flow_counter_in {
   2194   1.1  jmcneill 	uint16_t		cmd_opcode;
   2195   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2196   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2197   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2198   1.1  jmcneill } __packed __aligned(4);
   2199   1.1  jmcneill 
   2200   1.1  jmcneill struct mcx_cmd_query_rq_in {
   2201   1.1  jmcneill 	uint16_t		cmd_opcode;
   2202   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2203   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2204   1.1  jmcneill 	uint32_t		cmd_rqn;
   2205   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2206   1.1  jmcneill } __packed __aligned(4);
   2207   1.1  jmcneill 
   2208   1.1  jmcneill struct mcx_cmd_query_rq_out {
   2209   1.1  jmcneill 	uint8_t			cmd_status;
   2210   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2211   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2212   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2213   1.1  jmcneill } __packed __aligned(4);
   2214   1.1  jmcneill 
   2215   1.1  jmcneill struct mcx_cmd_query_rq_mb_out {
   2216   1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   2217   1.1  jmcneill 	struct mcx_rq_ctx	cmd_ctx;
   2218   1.1  jmcneill };
   2219   1.1  jmcneill 
   2220   1.1  jmcneill struct mcx_cmd_query_sq_in {
   2221   1.1  jmcneill 	uint16_t		cmd_opcode;
   2222   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2223   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2224   1.1  jmcneill 	uint32_t		cmd_sqn;
   2225   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2226   1.1  jmcneill } __packed __aligned(4);
   2227   1.1  jmcneill 
   2228   1.1  jmcneill struct mcx_cmd_query_sq_out {
   2229   1.1  jmcneill 	uint8_t			cmd_status;
   2230   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2231   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2232   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2233   1.1  jmcneill } __packed __aligned(4);
   2234   1.1  jmcneill 
   2235   1.1  jmcneill struct mcx_cmd_query_sq_mb_out {
   2236   1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   2237   1.1  jmcneill 	struct mcx_sq_ctx	cmd_ctx;
   2238   1.1  jmcneill };
   2239   1.1  jmcneill 
   2240   1.1  jmcneill struct mcx_cmd_alloc_flow_counter_out {
   2241   1.1  jmcneill 	uint8_t			cmd_status;
   2242   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2243   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2244   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
   2245   1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
   2246   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   2247   1.1  jmcneill } __packed __aligned(4);
   2248   1.1  jmcneill 
   2249   1.1  jmcneill struct mcx_wq_doorbell {
   2250   1.1  jmcneill 	uint32_t		 db_recv_counter;
   2251   1.1  jmcneill 	uint32_t		 db_send_counter;
   2252   1.1  jmcneill } __packed __aligned(8);
   2253   1.1  jmcneill 
   2254   1.1  jmcneill struct mcx_dmamem {
   2255   1.1  jmcneill 	bus_dmamap_t		 mxm_map;
   2256   1.1  jmcneill 	bus_dma_segment_t	 mxm_seg;
   2257   1.1  jmcneill 	int			 mxm_nsegs;
   2258   1.1  jmcneill 	size_t			 mxm_size;
   2259   1.1  jmcneill 	void			*mxm_kva;
   2260   1.1  jmcneill };
   2261   1.1  jmcneill #define MCX_DMA_MAP(_mxm)	((_mxm)->mxm_map)
   2262   1.1  jmcneill #define MCX_DMA_DVA(_mxm)	((_mxm)->mxm_map->dm_segs[0].ds_addr)
   2263   1.1  jmcneill #define MCX_DMA_KVA(_mxm)	((void *)(_mxm)->mxm_kva)
   2264  1.15  jmcneill #define MCX_DMA_OFF(_mxm, _off)	((void *)((char *)(_mxm)->mxm_kva + (_off)))
   2265   1.1  jmcneill #define MCX_DMA_LEN(_mxm)	((_mxm)->mxm_size)
   2266   1.1  jmcneill 
   2267   1.1  jmcneill struct mcx_hwmem {
   2268   1.1  jmcneill 	bus_dmamap_t		 mhm_map;
   2269   1.1  jmcneill 	bus_dma_segment_t	*mhm_segs;
   2270   1.1  jmcneill 	unsigned int		 mhm_seg_count;
   2271   1.1  jmcneill 	unsigned int		 mhm_npages;
   2272   1.1  jmcneill };
   2273   1.1  jmcneill 
   2274   1.1  jmcneill struct mcx_slot {
   2275   1.1  jmcneill 	bus_dmamap_t		 ms_map;
   2276   1.1  jmcneill 	struct mbuf		*ms_m;
   2277   1.1  jmcneill };
   2278   1.1  jmcneill 
   2279  1.15  jmcneill struct mcx_eq {
   2280  1.15  jmcneill 	int			 eq_n;
   2281  1.15  jmcneill 	uint32_t		 eq_cons;
   2282  1.15  jmcneill 	struct mcx_dmamem	 eq_mem;
   2283  1.15  jmcneill };
   2284  1.15  jmcneill 
   2285   1.1  jmcneill struct mcx_cq {
   2286   1.1  jmcneill 	int			 cq_n;
   2287   1.1  jmcneill 	struct mcx_dmamem	 cq_mem;
   2288  1.15  jmcneill 	bus_addr_t		 cq_doorbell;
   2289   1.1  jmcneill 	uint32_t		 cq_cons;
   2290   1.1  jmcneill 	uint32_t		 cq_count;
   2291   1.1  jmcneill };
   2292   1.1  jmcneill 
   2293   1.1  jmcneill struct mcx_calibration {
   2294   1.1  jmcneill 	uint64_t		 c_timestamp;	/* previous mcx chip time */
   2295   1.1  jmcneill 	uint64_t		 c_uptime;	/* previous kernel nanouptime */
   2296   1.1  jmcneill 	uint64_t		 c_tbase;	/* mcx chip time */
   2297   1.1  jmcneill 	uint64_t		 c_ubase;	/* kernel nanouptime */
   2298  1.15  jmcneill 	uint64_t		 c_ratio;
   2299   1.1  jmcneill };
   2300   1.1  jmcneill 
   2301   1.1  jmcneill #define MCX_CALIBRATE_FIRST    2
   2302  1.15  jmcneill #define MCX_CALIBRATE_NORMAL   32
   2303   1.1  jmcneill 
   2304   1.1  jmcneill struct mcx_rxring {
   2305   1.1  jmcneill 	u_int			 rxr_total;
   2306   1.1  jmcneill 	u_int			 rxr_inuse;
   2307   1.1  jmcneill };
   2308   1.1  jmcneill 
   2309   1.1  jmcneill MBUFQ_HEAD(mcx_mbufq);
   2310   1.1  jmcneill 
   2311  1.15  jmcneill struct mcx_rx {
   2312  1.15  jmcneill 	struct mcx_softc	*rx_softc;
   2313  1.15  jmcneill 
   2314  1.15  jmcneill 	int			 rx_rqn;
   2315  1.15  jmcneill 	struct mcx_dmamem	 rx_rq_mem;
   2316  1.15  jmcneill 	struct mcx_slot		*rx_slots;
   2317  1.15  jmcneill 	bus_addr_t		 rx_doorbell;
   2318  1.15  jmcneill 
   2319  1.15  jmcneill 	uint32_t		 rx_prod;
   2320  1.15  jmcneill 	callout_t		 rx_refill;
   2321  1.15  jmcneill 	struct mcx_rxring	 rx_rxr;
   2322  1.15  jmcneill } __aligned(64);
   2323  1.15  jmcneill 
   2324  1.15  jmcneill struct mcx_tx {
   2325  1.15  jmcneill 	struct mcx_softc	*tx_softc;
   2326  1.16  jmcneill 	kmutex_t		 tx_lock;
   2327  1.16  jmcneill 	pcq_t			*tx_pcq;
   2328  1.16  jmcneill 	void			*tx_softint;
   2329  1.15  jmcneill 
   2330  1.15  jmcneill 	int			 tx_uar;
   2331  1.15  jmcneill 	int			 tx_sqn;
   2332  1.15  jmcneill 	struct mcx_dmamem	 tx_sq_mem;
   2333  1.15  jmcneill 	struct mcx_slot		*tx_slots;
   2334  1.15  jmcneill 	bus_addr_t		 tx_doorbell;
   2335  1.15  jmcneill 	int			 tx_bf_offset;
   2336  1.15  jmcneill 
   2337  1.15  jmcneill 	uint32_t		 tx_cons;
   2338  1.15  jmcneill 	uint32_t		 tx_prod;
   2339  1.15  jmcneill } __aligned(64);
   2340  1.15  jmcneill 
   2341  1.15  jmcneill struct mcx_queues {
   2342  1.15  jmcneill 	void			*q_ihc;
   2343  1.15  jmcneill 	struct mcx_softc	*q_sc;
   2344  1.15  jmcneill 	int			 q_uar;
   2345  1.15  jmcneill 	int			 q_index;
   2346  1.15  jmcneill 	struct mcx_rx		 q_rx;
   2347  1.15  jmcneill 	struct mcx_tx		 q_tx;
   2348  1.15  jmcneill 	struct mcx_cq		 q_cq;
   2349  1.15  jmcneill 	struct mcx_eq		 q_eq;
   2350  1.15  jmcneill #if NKSTAT > 0
   2351  1.15  jmcneill 	struct kstat		*q_kstat;
   2352  1.15  jmcneill #endif
   2353  1.15  jmcneill };
   2354  1.15  jmcneill 
   2355  1.15  jmcneill struct mcx_flow_group {
   2356  1.15  jmcneill 	int			 g_id;
   2357  1.15  jmcneill 	int			 g_table;
   2358  1.15  jmcneill 	int			 g_start;
   2359  1.15  jmcneill 	int			 g_size;
   2360  1.15  jmcneill };
   2361  1.15  jmcneill 
   2362  1.15  jmcneill #define MCX_FLOW_GROUP_PROMISC		0
   2363  1.15  jmcneill #define MCX_FLOW_GROUP_ALLMULTI		1
   2364  1.15  jmcneill #define MCX_FLOW_GROUP_MAC		2
   2365  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_L4		3
   2366  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_L3		4
   2367  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_NONE		5
   2368  1.15  jmcneill #define	MCX_NUM_FLOW_GROUPS		6
   2369  1.15  jmcneill 
   2370  1.15  jmcneill #define MCX_HASH_SEL_L3		MCX_TIR_CTX_HASH_SEL_SRC_IP | \
   2371  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_DST_IP
   2372  1.15  jmcneill #define MCX_HASH_SEL_L4		MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \
   2373  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_DPORT
   2374  1.15  jmcneill 
   2375  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP  |\
   2376  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV4
   2377  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6_TCP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \
   2378  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV6
   2379  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
   2380  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV4
   2381  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
   2382  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV6
   2383  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4
   2384  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6
   2385  1.15  jmcneill 
   2386  1.15  jmcneill /*
   2387  1.15  jmcneill  * There are a few different pieces involved in configuring RSS.
   2388  1.15  jmcneill  * A Receive Queue Table (RQT) is the indirection table that maps packets to
   2389  1.15  jmcneill  * different rx queues based on a hash value.  We only create one, because
   2390  1.15  jmcneill  * we want to scatter any traffic we can apply RSS to across all our rx
   2391  1.15  jmcneill  * queues.  Anything else will only be delivered to the first rx queue,
   2392  1.15  jmcneill  * which doesn't require an RQT.
   2393  1.15  jmcneill  *
   2394  1.15  jmcneill  * A Transport Interface Receive (TIR) delivers packets to either a single rx
   2395  1.15  jmcneill  * queue or an RQT, and in the latter case, specifies the set of fields
   2396  1.15  jmcneill  * hashed, the hash function, and the hash key.  We need one of these for each
   2397  1.15  jmcneill  * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6,
   2398  1.15  jmcneill  * and one for non-RSS traffic.
   2399  1.15  jmcneill  *
   2400  1.15  jmcneill  * Flow tables hold flow table entries in sequence.  The first entry that
   2401  1.15  jmcneill  * matches a packet is applied, sending the packet to either another flow
   2402  1.15  jmcneill  * table or a TIR.  We use one flow table to select packets based on
   2403  1.15  jmcneill  * destination MAC address, and a second to apply RSS.  The entries in the
   2404  1.15  jmcneill  * first table send matching packets to the second, and the entries in the
   2405  1.15  jmcneill  * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR.
   2406  1.15  jmcneill  *
   2407  1.15  jmcneill  * The flow table entry that delivers packets to an RSS TIR must include match
   2408  1.15  jmcneill  * criteria that ensure packets delivered to the TIR include all the fields
   2409  1.15  jmcneill  * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must
   2410  1.15  jmcneill  * only accept v4 TCP packets.  Accordingly, we need flow table entries for
   2411  1.15  jmcneill  * each TIR.
   2412  1.15  jmcneill  *
   2413  1.15  jmcneill  * All of this is a lot more flexible than we need, and we can describe most
   2414  1.15  jmcneill  * of the stuff we need with a simple array.
   2415  1.15  jmcneill  *
   2416  1.15  jmcneill  * An RSS config creates a TIR with hashing enabled on a set of fields,
   2417  1.15  jmcneill  * pointing to either the first rx queue or the RQT containing all the rx
   2418  1.15  jmcneill  * queues, and a flow table entry that matches on an ether type and
   2419  1.15  jmcneill  * optionally an ip proto, that delivers packets to the TIR.
   2420  1.15  jmcneill  */
   2421  1.15  jmcneill static struct mcx_rss_rule {
   2422  1.15  jmcneill 	int			hash_sel;
   2423  1.15  jmcneill 	int			flow_group;
   2424  1.15  jmcneill 	int			ethertype;
   2425  1.15  jmcneill 	int			ip_proto;
   2426  1.15  jmcneill } mcx_rss_config[] = {
   2427  1.15  jmcneill 	/* udp and tcp for v4/v6 */
   2428  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4,
   2429  1.15  jmcneill 	  ETHERTYPE_IP, IPPROTO_TCP },
   2430  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4,
   2431  1.15  jmcneill 	  ETHERTYPE_IPV6, IPPROTO_TCP },
   2432  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4,
   2433  1.15  jmcneill 	  ETHERTYPE_IP, IPPROTO_UDP },
   2434  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4,
   2435  1.15  jmcneill 	  ETHERTYPE_IPV6, IPPROTO_UDP },
   2436  1.15  jmcneill 
   2437  1.15  jmcneill 	/* other v4/v6 */
   2438  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3,
   2439  1.15  jmcneill 	  ETHERTYPE_IP, 0 },
   2440  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3,
   2441  1.15  jmcneill 	  ETHERTYPE_IPV6, 0 },
   2442  1.15  jmcneill 
   2443  1.15  jmcneill 	/* non v4/v6 */
   2444  1.15  jmcneill 	{ 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 }
   2445  1.15  jmcneill };
   2446  1.15  jmcneill 
   2447   1.1  jmcneill struct mcx_softc {
   2448   1.1  jmcneill 	device_t		 sc_dev;
   2449   1.1  jmcneill 	struct ethercom		 sc_ec;
   2450   1.1  jmcneill 	struct ifmedia		 sc_media;
   2451   1.1  jmcneill 	uint64_t		 sc_media_status;
   2452   1.1  jmcneill 	uint64_t		 sc_media_active;
   2453  1.12   thorpej 	kmutex_t		 sc_media_mutex;
   2454   1.1  jmcneill 
   2455   1.1  jmcneill 	pci_chipset_tag_t	 sc_pc;
   2456   1.1  jmcneill 	pci_intr_handle_t	*sc_intrs;
   2457  1.15  jmcneill 	void			*sc_ihc;
   2458   1.1  jmcneill 	pcitag_t		 sc_tag;
   2459   1.1  jmcneill 
   2460   1.1  jmcneill 	bus_dma_tag_t		 sc_dmat;
   2461   1.1  jmcneill 	bus_space_tag_t		 sc_memt;
   2462   1.1  jmcneill 	bus_space_handle_t	 sc_memh;
   2463   1.1  jmcneill 	bus_size_t		 sc_mems;
   2464   1.1  jmcneill 
   2465   1.1  jmcneill 	struct mcx_dmamem	 sc_cmdq_mem;
   2466   1.1  jmcneill 	unsigned int		 sc_cmdq_mask;
   2467   1.1  jmcneill 	unsigned int		 sc_cmdq_size;
   2468   1.1  jmcneill 
   2469   1.1  jmcneill 	unsigned int		 sc_cmdq_token;
   2470   1.1  jmcneill 
   2471   1.1  jmcneill 	struct mcx_hwmem	 sc_boot_pages;
   2472   1.1  jmcneill 	struct mcx_hwmem	 sc_init_pages;
   2473   1.1  jmcneill 	struct mcx_hwmem	 sc_regular_pages;
   2474   1.1  jmcneill 
   2475   1.1  jmcneill 	int			 sc_uar;
   2476   1.1  jmcneill 	int			 sc_pd;
   2477   1.1  jmcneill 	int			 sc_tdomain;
   2478   1.1  jmcneill 	uint32_t		 sc_lkey;
   2479  1.15  jmcneill 	int			 sc_tis;
   2480  1.15  jmcneill 	int			 sc_tir[__arraycount(mcx_rss_config)];
   2481  1.15  jmcneill 	int			 sc_rqt;
   2482   1.1  jmcneill 
   2483   1.1  jmcneill 	struct mcx_dmamem	 sc_doorbell_mem;
   2484   1.1  jmcneill 
   2485  1.15  jmcneill 	struct mcx_eq		 sc_admin_eq;
   2486  1.15  jmcneill 	struct mcx_eq		 sc_queue_eq;
   2487  1.15  jmcneill 
   2488   1.1  jmcneill 	int			 sc_hardmtu;
   2489  1.15  jmcneill 	int			 sc_rxbufsz;
   2490  1.15  jmcneill 
   2491  1.15  jmcneill 	int			 sc_bf_size;
   2492  1.15  jmcneill 	int			 sc_max_rqt_size;
   2493   1.1  jmcneill 
   2494   1.1  jmcneill 	struct workqueue	*sc_workq;
   2495   1.1  jmcneill 	struct work		 sc_port_change;
   2496   1.1  jmcneill 
   2497  1.15  jmcneill 	int			 sc_mac_flow_table_id;
   2498  1.15  jmcneill 	int			 sc_rss_flow_table_id;
   2499  1.15  jmcneill 	struct mcx_flow_group	 sc_flow_group[MCX_NUM_FLOW_GROUPS];
   2500   1.1  jmcneill 	int			 sc_promisc_flow_enabled;
   2501   1.1  jmcneill 	int			 sc_allmulti_flow_enabled;
   2502   1.1  jmcneill 	int			 sc_mcast_flow_base;
   2503   1.1  jmcneill 	int			 sc_extra_mcast;
   2504   1.1  jmcneill 	uint8_t			 sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];
   2505   1.1  jmcneill 
   2506   1.1  jmcneill 	struct mcx_calibration	 sc_calibration[2];
   2507   1.1  jmcneill 	unsigned int		 sc_calibration_gen;
   2508   1.1  jmcneill 	callout_t		 sc_calibrate;
   2509  1.15  jmcneill 	uint32_t		 sc_mhz;
   2510  1.15  jmcneill 	uint32_t		 sc_khz;
   2511   1.1  jmcneill 
   2512  1.15  jmcneill 	struct mcx_queues	*sc_queues;
   2513  1.15  jmcneill 	unsigned int		 sc_nqueues;
   2514   1.1  jmcneill 
   2515  1.15  jmcneill 	int			 sc_mcam_reg;
   2516   1.1  jmcneill 
   2517  1.15  jmcneill #if NKSTAT > 0
   2518  1.15  jmcneill 	struct kstat		*sc_kstat_ieee8023;
   2519  1.15  jmcneill 	struct kstat		*sc_kstat_rfc2863;
   2520  1.15  jmcneill 	struct kstat		*sc_kstat_rfc2819;
   2521  1.15  jmcneill 	struct kstat		*sc_kstat_rfc3635;
   2522  1.15  jmcneill 	unsigned int		 sc_kstat_mtmp_count;
   2523  1.15  jmcneill 	struct kstat		**sc_kstat_mtmp;
   2524  1.15  jmcneill #endif
   2525   1.1  jmcneill 
   2526  1.15  jmcneill 	struct timecounter	 sc_timecounter;
   2527   1.1  jmcneill };
   2528   1.1  jmcneill #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
   2529   1.1  jmcneill 
   2530   1.1  jmcneill static int	mcx_match(device_t, cfdata_t, void *);
   2531   1.1  jmcneill static void	mcx_attach(device_t, device_t, void *);
   2532   1.1  jmcneill 
   2533  1.15  jmcneill static void *	mcx_establish_intr(struct mcx_softc *, int, kcpuset_t *,
   2534  1.15  jmcneill 		    int (*)(void *), void *, const char *);
   2535  1.15  jmcneill 
   2536   1.1  jmcneill static void	mcx_rxr_init(struct mcx_rxring *, u_int, u_int);
   2537   1.1  jmcneill static u_int	mcx_rxr_get(struct mcx_rxring *, u_int);
   2538   1.1  jmcneill static void	mcx_rxr_put(struct mcx_rxring *, u_int);
   2539   1.1  jmcneill static u_int	mcx_rxr_inuse(struct mcx_rxring *);
   2540   1.1  jmcneill 
   2541  1.15  jmcneill #if NKSTAT > 0
   2542  1.15  jmcneill static void	mcx_kstat_attach(struct mcx_softc *);
   2543  1.15  jmcneill #endif
   2544  1.15  jmcneill 
   2545  1.15  jmcneill static void	mcx_timecounter_attach(struct mcx_softc *);
   2546  1.15  jmcneill 
   2547   1.1  jmcneill static int	mcx_version(struct mcx_softc *);
   2548   1.1  jmcneill static int	mcx_init_wait(struct mcx_softc *);
   2549   1.1  jmcneill static int	mcx_enable_hca(struct mcx_softc *);
   2550   1.1  jmcneill static int	mcx_teardown_hca(struct mcx_softc *, uint16_t);
   2551   1.1  jmcneill static int	mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
   2552   1.1  jmcneill 		    int);
   2553   1.1  jmcneill static int	mcx_issi(struct mcx_softc *);
   2554   1.1  jmcneill static int	mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
   2555   1.1  jmcneill static int	mcx_hca_max_caps(struct mcx_softc *);
   2556   1.1  jmcneill static int	mcx_hca_set_caps(struct mcx_softc *);
   2557   1.1  jmcneill static int	mcx_init_hca(struct mcx_softc *);
   2558   1.1  jmcneill static int	mcx_set_driver_version(struct mcx_softc *);
   2559   1.1  jmcneill static int	mcx_iff(struct mcx_softc *);
   2560  1.15  jmcneill static int	mcx_alloc_uar(struct mcx_softc *, int *);
   2561   1.1  jmcneill static int	mcx_alloc_pd(struct mcx_softc *);
   2562   1.1  jmcneill static int	mcx_alloc_tdomain(struct mcx_softc *);
   2563  1.15  jmcneill static int	mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int,
   2564  1.15  jmcneill 		    uint64_t, int);
   2565   1.1  jmcneill static int	mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *);
   2566   1.1  jmcneill static int	mcx_query_special_contexts(struct mcx_softc *);
   2567   1.1  jmcneill static int	mcx_set_port_mtu(struct mcx_softc *, int);
   2568  1.15  jmcneill static int	mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int,
   2569  1.15  jmcneill 		    int);
   2570  1.15  jmcneill static int	mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *);
   2571  1.15  jmcneill static int	mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int,
   2572  1.15  jmcneill 		    int);
   2573  1.15  jmcneill static int	mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *);
   2574  1.15  jmcneill static int	mcx_ready_sq(struct mcx_softc *, struct mcx_tx *);
   2575  1.15  jmcneill static int	mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int);
   2576  1.15  jmcneill static int	mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *);
   2577  1.15  jmcneill static int	mcx_ready_rq(struct mcx_softc *, struct mcx_rx *);
   2578  1.15  jmcneill static int	mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *,
   2579  1.15  jmcneill 		    int *);
   2580  1.15  jmcneill static int	mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t,
   2581  1.15  jmcneill 		    int *);
   2582  1.15  jmcneill static int	mcx_destroy_tir(struct mcx_softc *, int);
   2583  1.15  jmcneill static int	mcx_create_tis(struct mcx_softc *, int *);
   2584  1.15  jmcneill static int	mcx_destroy_tis(struct mcx_softc *, int);
   2585  1.15  jmcneill static int	mcx_create_rqt(struct mcx_softc *, int, int *, int *);
   2586  1.15  jmcneill static int	mcx_destroy_rqt(struct mcx_softc *, int);
   2587  1.15  jmcneill static int	mcx_create_flow_table(struct mcx_softc *, int, int, int *);
   2588  1.15  jmcneill static int	mcx_set_flow_table_root(struct mcx_softc *, int);
   2589  1.15  jmcneill static int	mcx_destroy_flow_table(struct mcx_softc *, int);
   2590  1.15  jmcneill static int	mcx_create_flow_group(struct mcx_softc *, int, int, int,
   2591   1.1  jmcneill 		    int, int, struct mcx_flow_match *);
   2592   1.1  jmcneill static int	mcx_destroy_flow_group(struct mcx_softc *, int);
   2593  1.15  jmcneill static int	mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int,
   2594  1.15  jmcneill 		    const uint8_t *, uint32_t);
   2595  1.15  jmcneill static int	mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int,
   2596  1.15  jmcneill 		    int, int, uint32_t);
   2597   1.1  jmcneill static int	mcx_delete_flow_table_entry(struct mcx_softc *, int, int);
   2598   1.1  jmcneill 
   2599  1.15  jmcneill #if NKSTAT > 0
   2600  1.15  jmcneill static int	mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *);
   2601  1.15  jmcneill static int	mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *);
   2602  1.15  jmcneill static int	mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *);
   2603  1.15  jmcneill static int	mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *);
   2604  1.15  jmcneill #endif
   2605  1.15  jmcneill 
   2606   1.1  jmcneill #if 0
   2607  1.15  jmcneill static int	mcx_dump_flow_table(struct mcx_softc *, int);
   2608  1.15  jmcneill static int	mcx_dump_flow_table_entry(struct mcx_softc *, int, int);
   2609  1.15  jmcneill static int	mcx_dump_flow_group(struct mcx_softc *, int);
   2610   1.1  jmcneill #endif
   2611   1.1  jmcneill 
   2612   1.1  jmcneill 
   2613   1.1  jmcneill /*
   2614   1.1  jmcneill static void	mcx_cmdq_dump(const struct mcx_cmdq_entry *);
   2615   1.1  jmcneill static void	mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
   2616   1.1  jmcneill */
   2617   1.1  jmcneill static void	mcx_refill(void *);
   2618  1.15  jmcneill static int	mcx_process_rx(struct mcx_softc *, struct mcx_rx *,
   2619  1.15  jmcneill 		    struct mcx_cq_entry *, struct mcx_mbufq *,
   2620  1.15  jmcneill 		    const struct mcx_calibration *);
   2621  1.15  jmcneill static int	mcx_process_txeof(struct mcx_softc *, struct mcx_tx *,
   2622  1.15  jmcneill 		    struct mcx_cq_entry *);
   2623  1.15  jmcneill static void	mcx_process_cq(struct mcx_softc *, struct mcx_queues *,
   2624  1.15  jmcneill 		    struct mcx_cq *);
   2625  1.15  jmcneill 
   2626  1.15  jmcneill static void	mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int);
   2627  1.15  jmcneill static void	mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int);
   2628  1.15  jmcneill static int	mcx_admin_intr(void *);
   2629  1.15  jmcneill static int	mcx_cq_intr(void *);
   2630   1.1  jmcneill 
   2631   1.1  jmcneill static int	mcx_init(struct ifnet *);
   2632   1.1  jmcneill static void	mcx_stop(struct ifnet *, int);
   2633   1.1  jmcneill static int	mcx_ioctl(struct ifnet *, u_long, void *);
   2634   1.1  jmcneill static void	mcx_start(struct ifnet *);
   2635  1.16  jmcneill static int	mcx_transmit(struct ifnet *, struct mbuf *);
   2636  1.16  jmcneill static void	mcx_deferred_transmit(void *);
   2637   1.1  jmcneill static void	mcx_media_add_types(struct mcx_softc *);
   2638   1.1  jmcneill static void	mcx_media_status(struct ifnet *, struct ifmediareq *);
   2639   1.1  jmcneill static int	mcx_media_change(struct ifnet *);
   2640   1.1  jmcneill #if 0
   2641   1.1  jmcneill static int	mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
   2642   1.1  jmcneill #endif
   2643   1.1  jmcneill static void	mcx_port_change(struct work *, void *);
   2644   1.1  jmcneill 
   2645   1.1  jmcneill static void	mcx_calibrate_first(struct mcx_softc *);
   2646   1.1  jmcneill static void	mcx_calibrate(void *);
   2647   1.1  jmcneill 
   2648   1.1  jmcneill static inline uint32_t
   2649   1.1  jmcneill 		mcx_rd(struct mcx_softc *, bus_size_t);
   2650   1.1  jmcneill static inline void
   2651   1.1  jmcneill 		mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
   2652   1.1  jmcneill static inline void
   2653   1.1  jmcneill 		mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
   2654   1.1  jmcneill 
   2655   1.1  jmcneill static uint64_t	mcx_timer(struct mcx_softc *);
   2656   1.1  jmcneill 
   2657   1.1  jmcneill static int	mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
   2658   1.1  jmcneill 		    bus_size_t, u_int align);
   2659   1.1  jmcneill static void	mcx_dmamem_zero(struct mcx_dmamem *);
   2660   1.1  jmcneill static void	mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
   2661   1.1  jmcneill 
   2662   1.1  jmcneill static int	mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
   2663   1.1  jmcneill 		    unsigned int);
   2664   1.1  jmcneill static void	mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
   2665   1.1  jmcneill 
   2666   1.1  jmcneill CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL);
   2667   1.1  jmcneill 
   2668   1.1  jmcneill static const struct {
   2669   1.1  jmcneill 	pci_vendor_id_t		vendor;
   2670   1.1  jmcneill 	pci_product_id_t	product;
   2671   1.1  jmcneill } mcx_devices[] = {
   2672   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700 },
   2673  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700VF },
   2674   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710 },
   2675  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710VF },
   2676   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800 },
   2677  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800VF },
   2678   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800 },
   2679  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800VF },
   2680  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28908 },
   2681  1.26   msaitoh 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT2892 },
   2682  1.26   msaitoh 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT2894 },
   2683   1.1  jmcneill };
   2684   1.1  jmcneill 
   2685  1.13  jmcneill struct mcx_eth_proto_capability {
   2686  1.13  jmcneill 	uint64_t	cap_media;
   2687  1.13  jmcneill 	uint64_t	cap_baudrate;
   2688  1.13  jmcneill };
   2689  1.13  jmcneill 
   2690  1.13  jmcneill static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = {
   2691  1.13  jmcneill 	[MCX_ETHER_CAP_SGMII]		= { IFM_1000_SGMII,	IF_Gbps(1) },
   2692  1.13  jmcneill 	[MCX_ETHER_CAP_1000_KX]		= { IFM_1000_KX,	IF_Gbps(1) },
   2693  1.13  jmcneill 	[MCX_ETHER_CAP_10G_CX4]		= { IFM_10G_CX4,	IF_Gbps(10) },
   2694  1.13  jmcneill 	[MCX_ETHER_CAP_10G_KX4]		= { IFM_10G_KX4,	IF_Gbps(10) },
   2695  1.13  jmcneill 	[MCX_ETHER_CAP_10G_KR]		= { IFM_10G_KR,		IF_Gbps(10) },
   2696  1.13  jmcneill 	[MCX_ETHER_CAP_20G_KR2]		= { IFM_20G_KR2,	IF_Gbps(20) },
   2697  1.13  jmcneill 	[MCX_ETHER_CAP_40G_CR4]		= { IFM_40G_CR4,	IF_Gbps(40) },
   2698  1.13  jmcneill 	[MCX_ETHER_CAP_40G_KR4]		= { IFM_40G_KR4,	IF_Gbps(40) },
   2699  1.13  jmcneill 	[MCX_ETHER_CAP_56G_R4]		= { IFM_56G_R4,		IF_Gbps(56) },
   2700  1.13  jmcneill 	[MCX_ETHER_CAP_10G_CR]		= { IFM_10G_CR1,	IF_Gbps(10) },
   2701  1.13  jmcneill 	[MCX_ETHER_CAP_10G_SR]		= { IFM_10G_SR,		IF_Gbps(10) },
   2702  1.13  jmcneill 	[MCX_ETHER_CAP_10G_LR]		= { IFM_10G_LR,		IF_Gbps(10) },
   2703  1.13  jmcneill 	[MCX_ETHER_CAP_40G_SR4]		= { IFM_40G_SR4,	IF_Gbps(40) },
   2704  1.13  jmcneill 	[MCX_ETHER_CAP_40G_LR4]		= { IFM_40G_LR4,	IF_Gbps(40) },
   2705  1.13  jmcneill 	[MCX_ETHER_CAP_50G_SR2]		= { IFM_50G_SR2,	IF_Gbps(50) },
   2706  1.13  jmcneill 	[MCX_ETHER_CAP_100G_CR4]	= { IFM_100G_CR4,	IF_Gbps(100) },
   2707  1.13  jmcneill 	[MCX_ETHER_CAP_100G_SR4]	= { IFM_100G_SR4,	IF_Gbps(100) },
   2708  1.13  jmcneill 	[MCX_ETHER_CAP_100G_KR4]	= { IFM_100G_KR4,	IF_Gbps(100) },
   2709  1.13  jmcneill 	[MCX_ETHER_CAP_100G_LR4]	= { IFM_100G_LR4,	IF_Gbps(100) },
   2710  1.13  jmcneill 	[MCX_ETHER_CAP_100_TX]		= { IFM_100_TX,		IF_Mbps(100) },
   2711  1.13  jmcneill 	[MCX_ETHER_CAP_1000_T]		= { IFM_1000_T,		IF_Gbps(1) },
   2712  1.13  jmcneill 	[MCX_ETHER_CAP_10G_T]		= { IFM_10G_T,		IF_Gbps(10) },
   2713  1.13  jmcneill 	[MCX_ETHER_CAP_25G_CR]		= { IFM_25G_CR,		IF_Gbps(25) },
   2714  1.13  jmcneill 	[MCX_ETHER_CAP_25G_KR]		= { IFM_25G_KR,		IF_Gbps(25) },
   2715  1.13  jmcneill 	[MCX_ETHER_CAP_25G_SR]		= { IFM_25G_SR,		IF_Gbps(25) },
   2716  1.13  jmcneill 	[MCX_ETHER_CAP_50G_CR2]		= { IFM_50G_CR2,	IF_Gbps(50) },
   2717  1.13  jmcneill 	[MCX_ETHER_CAP_50G_KR2]		= { IFM_50G_KR2,	IF_Gbps(50) },
   2718   1.1  jmcneill };
   2719   1.1  jmcneill 
   2720   1.1  jmcneill static int
   2721  1.13  jmcneill mcx_get_id(uint32_t val)
   2722  1.13  jmcneill {
   2723  1.13  jmcneill 	return be32toh(val) & 0x00ffffff;
   2724  1.13  jmcneill }
   2725  1.13  jmcneill 
   2726  1.13  jmcneill static int
   2727   1.1  jmcneill mcx_match(device_t parent, cfdata_t cf, void *aux)
   2728   1.1  jmcneill {
   2729   1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2730   1.1  jmcneill 	int n;
   2731   1.1  jmcneill 
   2732   1.1  jmcneill 	for (n = 0; n < __arraycount(mcx_devices); n++) {
   2733   1.1  jmcneill 		if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor &&
   2734   1.1  jmcneill 		    PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product)
   2735   1.1  jmcneill 			return 1;
   2736   1.1  jmcneill 	}
   2737   1.1  jmcneill 
   2738   1.1  jmcneill 	return 0;
   2739   1.1  jmcneill }
   2740   1.1  jmcneill 
   2741   1.1  jmcneill void
   2742   1.1  jmcneill mcx_attach(device_t parent, device_t self, void *aux)
   2743   1.1  jmcneill {
   2744   1.1  jmcneill 	struct mcx_softc *sc = device_private(self);
   2745   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   2746   1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2747  1.22  jmcneill 	struct ifcapreq ifcr;
   2748   1.1  jmcneill 	uint8_t enaddr[ETHER_ADDR_LEN];
   2749   1.1  jmcneill 	int counts[PCI_INTR_TYPE_SIZE];
   2750  1.15  jmcneill 	char intrxname[32];
   2751   1.1  jmcneill 	pcireg_t memtype;
   2752   1.1  jmcneill 	uint32_t r;
   2753   1.1  jmcneill 	unsigned int cq_stride;
   2754   1.1  jmcneill 	unsigned int cq_size;
   2755  1.15  jmcneill 	int i, msix;
   2756  1.15  jmcneill 	kcpuset_t *affinity;
   2757   1.1  jmcneill 
   2758   1.1  jmcneill 	sc->sc_dev = self;
   2759   1.1  jmcneill 	sc->sc_pc = pa->pa_pc;
   2760   1.1  jmcneill 	sc->sc_tag = pa->pa_tag;
   2761  1.11   thorpej 	if (pci_dma64_available(pa))
   2762  1.11   thorpej 		sc->sc_dmat = pa->pa_dmat64;
   2763  1.11   thorpej 	else
   2764  1.11   thorpej 		sc->sc_dmat = pa->pa_dmat;
   2765   1.1  jmcneill 
   2766   1.1  jmcneill 	/* Map the PCI memory space */
   2767   1.1  jmcneill 	memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
   2768   1.1  jmcneill 	if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
   2769  1.21  jmcneill #ifdef __NetBSD__
   2770  1.21  jmcneill 	    0,
   2771  1.21  jmcneill #else
   2772  1.21  jmcneill 	    BUS_SPACE_MAP_PREFETCHABLE,
   2773  1.21  jmcneill #endif
   2774  1.21  jmcneill 	    &sc->sc_memt, &sc->sc_memh,
   2775   1.1  jmcneill 	    NULL, &sc->sc_mems)) {
   2776   1.1  jmcneill 		aprint_error(": unable to map register memory\n");
   2777   1.1  jmcneill 		return;
   2778   1.1  jmcneill 	}
   2779   1.1  jmcneill 
   2780   1.1  jmcneill 	pci_aprint_devinfo(pa, "Ethernet controller");
   2781   1.1  jmcneill 
   2782  1.12   thorpej 	mutex_init(&sc->sc_media_mutex, MUTEX_DEFAULT, IPL_SOFTNET);
   2783  1.12   thorpej 
   2784   1.1  jmcneill 	if (mcx_version(sc) != 0) {
   2785   1.1  jmcneill 		/* error printed by mcx_version */
   2786   1.1  jmcneill 		goto unmap;
   2787   1.1  jmcneill 	}
   2788   1.1  jmcneill 
   2789   1.1  jmcneill 	r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
   2790   1.1  jmcneill 	cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
   2791   1.1  jmcneill 	cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
   2792   1.1  jmcneill 	if (cq_size > MCX_MAX_CQE) {
   2793   1.1  jmcneill 		aprint_error_dev(self,
   2794   1.1  jmcneill 		    "command queue size overflow %u\n", cq_size);
   2795   1.1  jmcneill 		goto unmap;
   2796   1.1  jmcneill 	}
   2797   1.1  jmcneill 	if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
   2798   1.1  jmcneill 		aprint_error_dev(self,
   2799   1.1  jmcneill 		    "command queue entry size underflow %u\n", cq_stride);
   2800   1.1  jmcneill 		goto unmap;
   2801   1.1  jmcneill 	}
   2802   1.1  jmcneill 	if (cq_stride * cq_size > MCX_PAGE_SIZE) {
   2803   1.1  jmcneill 		aprint_error_dev(self, "command queue page overflow\n");
   2804   1.1  jmcneill 		goto unmap;
   2805   1.1  jmcneill 	}
   2806   1.1  jmcneill 
   2807  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE,
   2808   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2809   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate doorbell memory\n");
   2810   1.1  jmcneill 		goto unmap;
   2811   1.1  jmcneill 	}
   2812   1.1  jmcneill 
   2813   1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
   2814   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2815   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate command queue\n");
   2816   1.1  jmcneill 		goto dbfree;
   2817   1.1  jmcneill 	}
   2818   1.1  jmcneill 
   2819   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   2820  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t),
   2821  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   2822   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
   2823  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t),
   2824  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   2825   1.1  jmcneill 
   2826   1.1  jmcneill 	if (mcx_init_wait(sc) != 0) {
   2827   1.1  jmcneill 		aprint_error_dev(self, "timeout waiting for init\n");
   2828   1.1  jmcneill 		goto cqfree;
   2829   1.1  jmcneill 	}
   2830   1.1  jmcneill 
   2831   1.1  jmcneill 	sc->sc_cmdq_mask = cq_size - 1;
   2832   1.1  jmcneill 	sc->sc_cmdq_size = cq_stride;
   2833   1.1  jmcneill 
   2834   1.1  jmcneill 	if (mcx_enable_hca(sc) != 0) {
   2835   1.1  jmcneill 		/* error printed by mcx_enable_hca */
   2836   1.1  jmcneill 		goto cqfree;
   2837   1.1  jmcneill 	}
   2838   1.1  jmcneill 
   2839   1.1  jmcneill 	if (mcx_issi(sc) != 0) {
   2840   1.1  jmcneill 		/* error printed by mcx_issi */
   2841   1.1  jmcneill 		goto teardown;
   2842   1.1  jmcneill 	}
   2843   1.1  jmcneill 
   2844   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_boot_pages,
   2845   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
   2846   1.1  jmcneill 		/* error printed by mcx_pages */
   2847   1.1  jmcneill 		goto teardown;
   2848   1.1  jmcneill 	}
   2849   1.1  jmcneill 
   2850   1.1  jmcneill 	if (mcx_hca_max_caps(sc) != 0) {
   2851   1.1  jmcneill 		/* error printed by mcx_hca_max_caps */
   2852   1.1  jmcneill 		goto teardown;
   2853   1.1  jmcneill 	}
   2854   1.1  jmcneill 
   2855   1.1  jmcneill 	if (mcx_hca_set_caps(sc) != 0) {
   2856   1.1  jmcneill 		/* error printed by mcx_hca_set_caps */
   2857   1.1  jmcneill 		goto teardown;
   2858   1.1  jmcneill 	}
   2859   1.1  jmcneill 
   2860   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_init_pages,
   2861   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
   2862   1.1  jmcneill 		/* error printed by mcx_pages */
   2863   1.1  jmcneill 		goto teardown;
   2864   1.1  jmcneill 	}
   2865   1.1  jmcneill 
   2866   1.1  jmcneill 	if (mcx_init_hca(sc) != 0) {
   2867   1.1  jmcneill 		/* error printed by mcx_init_hca */
   2868   1.1  jmcneill 		goto teardown;
   2869   1.1  jmcneill 	}
   2870   1.1  jmcneill 
   2871   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_regular_pages,
   2872   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
   2873   1.1  jmcneill 		/* error printed by mcx_pages */
   2874   1.1  jmcneill 		goto teardown;
   2875   1.1  jmcneill 	}
   2876   1.1  jmcneill 
   2877   1.1  jmcneill 	/* apparently not necessary? */
   2878   1.1  jmcneill 	if (mcx_set_driver_version(sc) != 0) {
   2879   1.1  jmcneill 		/* error printed by mcx_set_driver_version */
   2880   1.1  jmcneill 		goto teardown;
   2881   1.1  jmcneill 	}
   2882   1.1  jmcneill 
   2883   1.1  jmcneill 	if (mcx_iff(sc) != 0) {	/* modify nic vport context */
   2884   1.1  jmcneill 		/* error printed by mcx_iff? */
   2885   1.1  jmcneill 		goto teardown;
   2886   1.1  jmcneill 	}
   2887   1.1  jmcneill 
   2888  1.15  jmcneill 	if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) {
   2889   1.1  jmcneill 		/* error printed by mcx_alloc_uar */
   2890   1.1  jmcneill 		goto teardown;
   2891   1.1  jmcneill 	}
   2892   1.1  jmcneill 
   2893   1.1  jmcneill 	if (mcx_alloc_pd(sc) != 0) {
   2894   1.1  jmcneill 		/* error printed by mcx_alloc_pd */
   2895   1.1  jmcneill 		goto teardown;
   2896   1.1  jmcneill 	}
   2897   1.1  jmcneill 
   2898   1.1  jmcneill 	if (mcx_alloc_tdomain(sc) != 0) {
   2899   1.1  jmcneill 		/* error printed by mcx_alloc_tdomain */
   2900   1.1  jmcneill 		goto teardown;
   2901   1.1  jmcneill 	}
   2902   1.1  jmcneill 
   2903   1.1  jmcneill 	/*
   2904   1.1  jmcneill 	 * PRM makes no mention of msi interrupts, just legacy and msi-x.
   2905   1.1  jmcneill 	 * mellanox support tells me legacy interrupts are not supported,
   2906   1.1  jmcneill 	 * so we're stuck with just msi-x.
   2907   1.1  jmcneill 	 */
   2908  1.15  jmcneill 	counts[PCI_INTR_TYPE_MSIX] = -1;
   2909   1.1  jmcneill 	counts[PCI_INTR_TYPE_MSI] = 0;
   2910   1.1  jmcneill 	counts[PCI_INTR_TYPE_INTX] = 0;
   2911   1.1  jmcneill 	if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) {
   2912   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate interrupt\n");
   2913   1.1  jmcneill 		goto teardown;
   2914   1.1  jmcneill 	}
   2915  1.15  jmcneill 	if (counts[PCI_INTR_TYPE_MSIX] < 2) {
   2916  1.15  jmcneill 		aprint_error_dev(self, "not enough MSI-X vectors\n");
   2917  1.15  jmcneill 		goto teardown;
   2918  1.15  jmcneill 	}
   2919   1.1  jmcneill 	KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX);
   2920  1.15  jmcneill 	snprintf(intrxname, sizeof(intrxname), "%s adminq", DEVNAME(sc));
   2921  1.15  jmcneill 	sc->sc_ihc = mcx_establish_intr(sc, 0, NULL, mcx_admin_intr, sc,
   2922  1.15  jmcneill 	    intrxname);
   2923  1.15  jmcneill 	if (sc->sc_ihc == NULL) {
   2924  1.15  jmcneill 		aprint_error_dev(self, "couldn't establish adminq interrupt\n");
   2925   1.1  jmcneill 		goto teardown;
   2926   1.1  jmcneill 	}
   2927   1.1  jmcneill 
   2928  1.15  jmcneill 	if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar,
   2929  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
   2930  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
   2931  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
   2932  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) {
   2933   1.1  jmcneill 		/* error printed by mcx_create_eq */
   2934   1.1  jmcneill 		goto teardown;
   2935   1.1  jmcneill 	}
   2936   1.1  jmcneill 
   2937   1.1  jmcneill 	if (mcx_query_nic_vport_context(sc, enaddr) != 0) {
   2938   1.1  jmcneill 		/* error printed by mcx_query_nic_vport_context */
   2939   1.1  jmcneill 		goto teardown;
   2940   1.1  jmcneill 	}
   2941   1.1  jmcneill 
   2942   1.1  jmcneill 	if (mcx_query_special_contexts(sc) != 0) {
   2943   1.1  jmcneill 		/* error printed by mcx_query_special_contexts */
   2944   1.1  jmcneill 		goto teardown;
   2945   1.1  jmcneill 	}
   2946   1.1  jmcneill 
   2947   1.1  jmcneill 	if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
   2948   1.1  jmcneill 		/* error printed by mcx_set_port_mtu */
   2949   1.1  jmcneill 		goto teardown;
   2950   1.1  jmcneill 	}
   2951   1.1  jmcneill 
   2952   1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
   2953   1.1  jmcneill 	    ether_sprintf(enaddr));
   2954   1.1  jmcneill 
   2955  1.15  jmcneill 	msix = counts[PCI_INTR_TYPE_MSIX];
   2956  1.15  jmcneill 	msix--; /* admin ops took one */
   2957  1.15  jmcneill 
   2958  1.15  jmcneill 	sc->sc_nqueues = uimin(MCX_MAX_QUEUES, msix);
   2959  1.15  jmcneill 	sc->sc_nqueues = uimin(sc->sc_nqueues, ncpu);
   2960  1.23    andvar 	/* Round down to a power of two.  */
   2961  1.23    andvar 	sc->sc_nqueues = 1U << ilog2(sc->sc_nqueues);
   2962  1.15  jmcneill 	sc->sc_queues = kmem_zalloc(sc->sc_nqueues * sizeof(*sc->sc_queues),
   2963  1.15  jmcneill 	    KM_SLEEP);
   2964  1.15  jmcneill 
   2965   1.1  jmcneill 	strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
   2966   1.1  jmcneill 	ifp->if_softc = sc;
   2967   1.1  jmcneill 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
   2968   1.1  jmcneill #ifdef MCX_MPSAFE
   2969   1.1  jmcneill 	ifp->if_extflags = IFEF_MPSAFE;
   2970   1.1  jmcneill #endif
   2971   1.1  jmcneill 	ifp->if_init = mcx_init;
   2972   1.1  jmcneill 	ifp->if_stop = mcx_stop;
   2973   1.1  jmcneill 	ifp->if_ioctl = mcx_ioctl;
   2974   1.1  jmcneill 	ifp->if_start = mcx_start;
   2975  1.16  jmcneill 	if (sc->sc_nqueues > 1) {
   2976  1.16  jmcneill 		ifp->if_transmit = mcx_transmit;
   2977  1.16  jmcneill 	}
   2978   1.1  jmcneill 	ifp->if_mtu = sc->sc_hardmtu;
   2979  1.15  jmcneill 	ifp->if_capabilities = IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
   2980  1.15  jmcneill 	    IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
   2981  1.15  jmcneill 	    IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx |
   2982  1.15  jmcneill 	    IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
   2983  1.15  jmcneill 	    IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx;
   2984   1.1  jmcneill 	IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
   2985   1.1  jmcneill 	IFQ_SET_READY(&ifp->if_snd);
   2986   1.1  jmcneill 
   2987  1.15  jmcneill 	sc->sc_ec.ec_capabilities = ETHERCAP_JUMBO_MTU |
   2988  1.15  jmcneill 	    ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING;
   2989  1.15  jmcneill 	sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;
   2990   1.1  jmcneill 
   2991   1.1  jmcneill 	sc->sc_ec.ec_ifmedia = &sc->sc_media;
   2992  1.12   thorpej 	ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, mcx_media_change,
   2993  1.12   thorpej 	    mcx_media_status, &sc->sc_media_mutex);
   2994   1.1  jmcneill 	mcx_media_add_types(sc);
   2995   1.1  jmcneill 	ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
   2996   1.1  jmcneill 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
   2997   1.1  jmcneill 
   2998   1.1  jmcneill 	if_attach(ifp);
   2999  1.22  jmcneill 
   3000  1.22  jmcneill 	/* Enable hardware offload by default */
   3001  1.22  jmcneill 	memset(&ifcr, 0, sizeof(ifcr));
   3002  1.22  jmcneill 	ifcr.ifcr_capenable = ifp->if_capabilities;
   3003  1.22  jmcneill 	ifioctl_common(ifp, SIOCSIFCAP, &ifcr);
   3004  1.22  jmcneill 
   3005   1.1  jmcneill 	if_deferred_start_init(ifp, NULL);
   3006   1.1  jmcneill 
   3007   1.1  jmcneill 	ether_ifattach(ifp, enaddr);
   3008   1.1  jmcneill 
   3009  1.15  jmcneill 	kcpuset_create(&affinity, false);
   3010  1.15  jmcneill 	kcpuset_set(affinity, 0);
   3011  1.15  jmcneill 
   3012  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   3013  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   3014  1.15  jmcneill 		struct mcx_rx *rx = &q->q_rx;
   3015  1.15  jmcneill 		struct mcx_tx *tx = &q->q_tx;
   3016  1.15  jmcneill 		int vec;
   3017  1.15  jmcneill 
   3018  1.15  jmcneill 		vec = i + 1;
   3019  1.15  jmcneill 		q->q_sc = sc;
   3020  1.15  jmcneill 		q->q_index = i;
   3021  1.15  jmcneill 
   3022  1.15  jmcneill 		if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
   3023  1.15  jmcneill 			aprint_error_dev(self, "unable to alloc uar %d\n", i);
   3024  1.15  jmcneill 			goto teardown;
   3025  1.15  jmcneill 		}
   3026  1.15  jmcneill 
   3027  1.15  jmcneill 		if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
   3028  1.15  jmcneill 			aprint_error_dev(self,
   3029  1.15  jmcneill 			    "unable to create event queue %d\n", i);
   3030  1.15  jmcneill 			goto teardown;
   3031  1.15  jmcneill 		}
   3032  1.15  jmcneill 
   3033  1.15  jmcneill 		rx->rx_softc = sc;
   3034  1.15  jmcneill 		callout_init(&rx->rx_refill, CALLOUT_FLAGS);
   3035  1.15  jmcneill 		callout_setfunc(&rx->rx_refill, mcx_refill, rx);
   3036  1.15  jmcneill 
   3037  1.15  jmcneill 		tx->tx_softc = sc;
   3038  1.16  jmcneill 		mutex_init(&tx->tx_lock, MUTEX_DEFAULT, IPL_NET);
   3039  1.16  jmcneill 		tx->tx_pcq = pcq_create(MCX_TXQ_NUM, KM_SLEEP);
   3040  1.16  jmcneill 		tx->tx_softint = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,
   3041  1.16  jmcneill 		    mcx_deferred_transmit, tx);
   3042  1.15  jmcneill 
   3043  1.15  jmcneill 		snprintf(intrxname, sizeof(intrxname), "%s queue %d",
   3044  1.15  jmcneill 		    DEVNAME(sc), i);
   3045  1.15  jmcneill 		q->q_ihc = mcx_establish_intr(sc, vec, affinity, mcx_cq_intr,
   3046  1.15  jmcneill 		    q, intrxname);
   3047  1.15  jmcneill 	}
   3048  1.15  jmcneill 
   3049   1.1  jmcneill 	callout_init(&sc->sc_calibrate, CALLOUT_FLAGS);
   3050   1.1  jmcneill 	callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc);
   3051   1.1  jmcneill 
   3052   1.1  jmcneill 	if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc,
   3053   1.1  jmcneill 	    PRI_NONE, IPL_NET, 0) != 0) {
   3054   1.1  jmcneill 		aprint_error_dev(self, "couldn't create port change workq\n");
   3055   1.1  jmcneill 		goto teardown;
   3056   1.1  jmcneill 	}
   3057   1.1  jmcneill 
   3058   1.1  jmcneill 	mcx_port_change(&sc->sc_port_change, sc);
   3059   1.1  jmcneill 
   3060  1.15  jmcneill 	sc->sc_mac_flow_table_id = -1;
   3061  1.15  jmcneill 	sc->sc_rss_flow_table_id = -1;
   3062  1.15  jmcneill 	sc->sc_rqt = -1;
   3063   1.1  jmcneill 	for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
   3064  1.15  jmcneill 		struct mcx_flow_group *mfg = &sc->sc_flow_group[i];
   3065  1.15  jmcneill 		mfg->g_id = -1;
   3066  1.15  jmcneill 		mfg->g_table = -1;
   3067  1.15  jmcneill 		mfg->g_size = 0;
   3068  1.15  jmcneill 		mfg->g_start = 0;
   3069   1.1  jmcneill 	}
   3070   1.1  jmcneill 	sc->sc_extra_mcast = 0;
   3071   1.1  jmcneill 	memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));
   3072  1.15  jmcneill 
   3073  1.15  jmcneill #if NKSTAT > 0
   3074  1.15  jmcneill 	mcx_kstat_attach(sc);
   3075  1.15  jmcneill #endif
   3076  1.15  jmcneill 	mcx_timecounter_attach(sc);
   3077   1.1  jmcneill 	return;
   3078   1.1  jmcneill 
   3079   1.1  jmcneill teardown:
   3080   1.1  jmcneill 	mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
   3081   1.1  jmcneill 	/* error printed by mcx_teardown_hca, and we're already unwinding */
   3082   1.1  jmcneill cqfree:
   3083   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   3084  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
   3085  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3086   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
   3087   1.1  jmcneill 	    MCX_CMDQ_INTERFACE_DISABLED);
   3088  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t),
   3089  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3090   1.1  jmcneill 
   3091   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
   3092  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
   3093  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3094   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
   3095   1.1  jmcneill 
   3096   1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
   3097   1.1  jmcneill dbfree:
   3098   1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
   3099   1.1  jmcneill unmap:
   3100   1.1  jmcneill 	bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
   3101   1.1  jmcneill 	sc->sc_mems = 0;
   3102   1.1  jmcneill }
   3103   1.1  jmcneill 
   3104  1.15  jmcneill static void *
   3105  1.15  jmcneill mcx_establish_intr(struct mcx_softc *sc, int index, kcpuset_t *affinity,
   3106  1.15  jmcneill     int (*func)(void *), void *arg, const char *xname)
   3107  1.15  jmcneill {
   3108  1.15  jmcneill 	char intrbuf[PCI_INTRSTR_LEN];
   3109  1.15  jmcneill 	const char *intrstr;
   3110  1.15  jmcneill 	void *ih;
   3111  1.15  jmcneill 
   3112  1.15  jmcneill 	pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[index], PCI_INTR_MPSAFE,
   3113  1.15  jmcneill 	    true);
   3114  1.15  jmcneill 
   3115  1.15  jmcneill 	intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[index], intrbuf,
   3116  1.15  jmcneill 	    sizeof(intrbuf));
   3117  1.15  jmcneill 	ih = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[index], IPL_NET,
   3118  1.15  jmcneill 	    func, arg, xname);
   3119  1.15  jmcneill 	if (ih == NULL) {
   3120  1.15  jmcneill 		aprint_error_dev(sc->sc_dev,
   3121  1.15  jmcneill 		    "unable to establish interrupt%s%s\n",
   3122  1.15  jmcneill 		    intrstr ? " at " : "",
   3123  1.15  jmcneill 		    intrstr ? intrstr : "");
   3124  1.15  jmcneill 		return NULL;
   3125  1.15  jmcneill 	}
   3126  1.15  jmcneill 
   3127  1.15  jmcneill 	if (affinity != NULL && index > 0) {
   3128  1.15  jmcneill 		/* Round-robin affinity */
   3129  1.15  jmcneill 		kcpuset_zero(affinity);
   3130  1.15  jmcneill 		kcpuset_set(affinity, (index - 1) % ncpu);
   3131  1.15  jmcneill 		interrupt_distribute(ih, affinity, NULL);
   3132  1.15  jmcneill 	}
   3133  1.15  jmcneill 
   3134  1.15  jmcneill 	return ih;
   3135  1.15  jmcneill }
   3136  1.15  jmcneill 
   3137   1.1  jmcneill static void
   3138   1.1  jmcneill mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm)
   3139   1.1  jmcneill {
   3140   1.1  jmcneill 	rxr->rxr_total = hwm;
   3141   1.1  jmcneill 	rxr->rxr_inuse = 0;
   3142   1.1  jmcneill }
   3143   1.1  jmcneill 
   3144   1.1  jmcneill static u_int
   3145   1.1  jmcneill mcx_rxr_get(struct mcx_rxring *rxr, u_int max)
   3146   1.1  jmcneill {
   3147   1.1  jmcneill 	const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse);
   3148   1.1  jmcneill 
   3149   1.1  jmcneill 	rxr->rxr_inuse += taken;
   3150   1.1  jmcneill 
   3151   1.1  jmcneill 	return taken;
   3152   1.1  jmcneill }
   3153   1.1  jmcneill 
   3154   1.1  jmcneill static void
   3155   1.1  jmcneill mcx_rxr_put(struct mcx_rxring *rxr, u_int n)
   3156   1.1  jmcneill {
   3157   1.1  jmcneill 	rxr->rxr_inuse -= n;
   3158   1.1  jmcneill }
   3159   1.1  jmcneill 
   3160   1.1  jmcneill static u_int
   3161   1.1  jmcneill mcx_rxr_inuse(struct mcx_rxring *rxr)
   3162   1.1  jmcneill {
   3163   1.1  jmcneill 	return rxr->rxr_inuse;
   3164   1.1  jmcneill }
   3165   1.1  jmcneill 
   3166   1.1  jmcneill static int
   3167   1.1  jmcneill mcx_version(struct mcx_softc *sc)
   3168   1.1  jmcneill {
   3169   1.1  jmcneill 	uint32_t fw0, fw1;
   3170   1.1  jmcneill 	uint16_t cmdif;
   3171   1.1  jmcneill 
   3172   1.1  jmcneill 	fw0 = mcx_rd(sc, MCX_FW_VER);
   3173   1.1  jmcneill 	fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
   3174   1.1  jmcneill 
   3175   1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0),
   3176   1.1  jmcneill 	    MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
   3177   1.1  jmcneill 
   3178   1.1  jmcneill 	cmdif = MCX_CMDIF(fw1);
   3179   1.1  jmcneill 	if (cmdif != MCX_CMD_IF_SUPPORTED) {
   3180   1.1  jmcneill 		aprint_error_dev(sc->sc_dev,
   3181   1.1  jmcneill 		    "unsupported command interface %u\n", cmdif);
   3182   1.1  jmcneill 		return (-1);
   3183   1.1  jmcneill 	}
   3184   1.1  jmcneill 
   3185   1.1  jmcneill 	return (0);
   3186   1.1  jmcneill }
   3187   1.1  jmcneill 
   3188   1.1  jmcneill static int
   3189   1.1  jmcneill mcx_init_wait(struct mcx_softc *sc)
   3190   1.1  jmcneill {
   3191   1.1  jmcneill 	unsigned int i;
   3192   1.1  jmcneill 	uint32_t r;
   3193   1.1  jmcneill 
   3194   1.1  jmcneill 	for (i = 0; i < 2000; i++) {
   3195   1.1  jmcneill 		r = mcx_rd(sc, MCX_STATE);
   3196   1.1  jmcneill 		if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
   3197   1.1  jmcneill 			return (0);
   3198   1.1  jmcneill 
   3199   1.1  jmcneill 		delay(1000);
   3200   1.1  jmcneill 		mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
   3201   1.1  jmcneill 		    BUS_SPACE_BARRIER_READ);
   3202   1.1  jmcneill 	}
   3203   1.1  jmcneill 
   3204   1.1  jmcneill 	return (-1);
   3205   1.1  jmcneill }
   3206   1.1  jmcneill 
   3207   1.1  jmcneill static uint8_t
   3208   1.1  jmcneill mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3209   1.1  jmcneill     unsigned int msec)
   3210   1.1  jmcneill {
   3211   1.1  jmcneill 	unsigned int i;
   3212   1.1  jmcneill 
   3213   1.1  jmcneill 	for (i = 0; i < msec; i++) {
   3214   1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   3215   1.1  jmcneill 		    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
   3216   1.1  jmcneill 
   3217   1.1  jmcneill 		if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
   3218  1.13  jmcneill 		    MCX_CQ_STATUS_OWN_SW)
   3219   1.1  jmcneill 			return (0);
   3220   1.1  jmcneill 
   3221   1.1  jmcneill 		delay(1000);
   3222   1.1  jmcneill 	}
   3223   1.1  jmcneill 
   3224   1.1  jmcneill 	return (ETIMEDOUT);
   3225   1.1  jmcneill }
   3226   1.1  jmcneill 
   3227   1.1  jmcneill static uint32_t
   3228   1.1  jmcneill mcx_mix_u64(uint32_t xor, uint64_t u64)
   3229   1.1  jmcneill {
   3230   1.1  jmcneill 	xor ^= u64 >> 32;
   3231   1.1  jmcneill 	xor ^= u64;
   3232   1.1  jmcneill 
   3233   1.1  jmcneill 	return (xor);
   3234   1.1  jmcneill }
   3235   1.1  jmcneill 
   3236   1.1  jmcneill static uint32_t
   3237   1.1  jmcneill mcx_mix_u32(uint32_t xor, uint32_t u32)
   3238   1.1  jmcneill {
   3239   1.1  jmcneill 	xor ^= u32;
   3240   1.1  jmcneill 
   3241   1.1  jmcneill 	return (xor);
   3242   1.1  jmcneill }
   3243   1.1  jmcneill 
   3244   1.1  jmcneill static uint32_t
   3245   1.1  jmcneill mcx_mix_u8(uint32_t xor, uint8_t u8)
   3246   1.1  jmcneill {
   3247   1.1  jmcneill 	xor ^= u8;
   3248   1.1  jmcneill 
   3249   1.1  jmcneill 	return (xor);
   3250   1.1  jmcneill }
   3251   1.1  jmcneill 
   3252   1.1  jmcneill static uint8_t
   3253   1.1  jmcneill mcx_mix_done(uint32_t xor)
   3254   1.1  jmcneill {
   3255   1.1  jmcneill 	xor ^= xor >> 16;
   3256   1.1  jmcneill 	xor ^= xor >> 8;
   3257   1.1  jmcneill 
   3258   1.1  jmcneill 	return (xor);
   3259   1.1  jmcneill }
   3260   1.1  jmcneill 
   3261   1.1  jmcneill static uint8_t
   3262   1.1  jmcneill mcx_xor(const void *buf, size_t len)
   3263   1.1  jmcneill {
   3264   1.1  jmcneill 	const uint32_t *dwords = buf;
   3265   1.1  jmcneill 	uint32_t xor = 0xff;
   3266   1.1  jmcneill 	size_t i;
   3267   1.1  jmcneill 
   3268   1.1  jmcneill 	len /= sizeof(*dwords);
   3269   1.1  jmcneill 
   3270   1.1  jmcneill 	for (i = 0; i < len; i++)
   3271   1.1  jmcneill 		xor ^= dwords[i];
   3272   1.1  jmcneill 
   3273   1.1  jmcneill 	return (mcx_mix_done(xor));
   3274   1.1  jmcneill }
   3275   1.1  jmcneill 
   3276   1.1  jmcneill static uint8_t
   3277   1.1  jmcneill mcx_cmdq_token(struct mcx_softc *sc)
   3278   1.1  jmcneill {
   3279   1.1  jmcneill 	uint8_t token;
   3280   1.1  jmcneill 
   3281   1.1  jmcneill 	do {
   3282   1.1  jmcneill 		token = ++sc->sc_cmdq_token;
   3283   1.1  jmcneill 	} while (token == 0);
   3284   1.1  jmcneill 
   3285   1.1  jmcneill 	return (token);
   3286   1.1  jmcneill }
   3287   1.1  jmcneill 
   3288   1.1  jmcneill static void
   3289   1.1  jmcneill mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3290   1.1  jmcneill     uint32_t ilen, uint32_t olen, uint8_t token)
   3291   1.1  jmcneill {
   3292   1.1  jmcneill 	memset(cqe, 0, sc->sc_cmdq_size);
   3293   1.1  jmcneill 
   3294   1.1  jmcneill 	cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
   3295   1.1  jmcneill 	be32enc(&cqe->cq_input_length, ilen);
   3296   1.1  jmcneill 	be32enc(&cqe->cq_output_length, olen);
   3297   1.1  jmcneill 	cqe->cq_token = token;
   3298   1.1  jmcneill 	cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
   3299   1.1  jmcneill }
   3300   1.1  jmcneill 
   3301   1.1  jmcneill static void
   3302   1.1  jmcneill mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
   3303   1.1  jmcneill {
   3304   1.1  jmcneill 	cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
   3305   1.1  jmcneill }
   3306   1.1  jmcneill 
   3307   1.1  jmcneill static int
   3308   1.1  jmcneill mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
   3309   1.1  jmcneill {
   3310   1.1  jmcneill 	/* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 :  0); */
   3311   1.1  jmcneill 	return (0);
   3312   1.1  jmcneill }
   3313   1.1  jmcneill 
   3314   1.1  jmcneill static void *
   3315   1.1  jmcneill mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
   3316   1.1  jmcneill {
   3317   1.1  jmcneill 	return (&cqe->cq_input_data);
   3318   1.1  jmcneill }
   3319   1.1  jmcneill 
   3320   1.1  jmcneill static void *
   3321   1.1  jmcneill mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
   3322   1.1  jmcneill {
   3323   1.1  jmcneill 	return (&cqe->cq_output_data);
   3324   1.1  jmcneill }
   3325   1.1  jmcneill 
   3326   1.1  jmcneill static void
   3327   1.1  jmcneill mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3328   1.1  jmcneill     unsigned int slot)
   3329   1.1  jmcneill {
   3330   1.1  jmcneill 	mcx_cmdq_sign(cqe);
   3331   1.1  jmcneill 
   3332   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   3333   1.1  jmcneill 	    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
   3334   1.1  jmcneill 
   3335   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
   3336  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t),
   3337  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3338   1.1  jmcneill }
   3339   1.1  jmcneill 
   3340   1.1  jmcneill static int
   3341   1.1  jmcneill mcx_enable_hca(struct mcx_softc *sc)
   3342   1.1  jmcneill {
   3343   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3344   1.1  jmcneill 	struct mcx_cmd_enable_hca_in *in;
   3345   1.1  jmcneill 	struct mcx_cmd_enable_hca_out *out;
   3346   1.1  jmcneill 	int error;
   3347   1.1  jmcneill 	uint8_t status;
   3348   1.1  jmcneill 
   3349   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3350   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3351   1.1  jmcneill 
   3352   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3353   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
   3354   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3355   1.1  jmcneill 	in->cmd_function_id = htobe16(0);
   3356   1.1  jmcneill 
   3357   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3358   1.1  jmcneill 
   3359   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3360   1.1  jmcneill 	if (error != 0) {
   3361   1.1  jmcneill 		printf(", hca enable timeout\n");
   3362   1.1  jmcneill 		return (-1);
   3363   1.1  jmcneill 	}
   3364   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3365   1.1  jmcneill 		printf(", hca enable command corrupt\n");
   3366   1.1  jmcneill 		return (-1);
   3367   1.1  jmcneill 	}
   3368   1.1  jmcneill 
   3369   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3370   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3371   1.1  jmcneill 		printf(", hca enable failed (%x)\n", status);
   3372   1.1  jmcneill 		return (-1);
   3373   1.1  jmcneill 	}
   3374   1.1  jmcneill 
   3375   1.1  jmcneill 	return (0);
   3376   1.1  jmcneill }
   3377   1.1  jmcneill 
   3378   1.1  jmcneill static int
   3379   1.1  jmcneill mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
   3380   1.1  jmcneill {
   3381   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3382   1.1  jmcneill 	struct mcx_cmd_teardown_hca_in *in;
   3383   1.1  jmcneill 	struct mcx_cmd_teardown_hca_out *out;
   3384   1.1  jmcneill 	int error;
   3385   1.1  jmcneill 	uint8_t status;
   3386   1.1  jmcneill 
   3387   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3388   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3389   1.1  jmcneill 
   3390   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3391   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
   3392   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3393   1.1  jmcneill 	in->cmd_profile = profile;
   3394   1.1  jmcneill 
   3395   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3396   1.1  jmcneill 
   3397   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3398   1.1  jmcneill 	if (error != 0) {
   3399   1.1  jmcneill 		printf(", hca teardown timeout\n");
   3400   1.1  jmcneill 		return (-1);
   3401   1.1  jmcneill 	}
   3402   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3403   1.1  jmcneill 		printf(", hca teardown command corrupt\n");
   3404   1.1  jmcneill 		return (-1);
   3405   1.1  jmcneill 	}
   3406   1.1  jmcneill 
   3407   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3408   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3409   1.1  jmcneill 		printf(", hca teardown failed (%x)\n", status);
   3410   1.1  jmcneill 		return (-1);
   3411   1.1  jmcneill 	}
   3412   1.1  jmcneill 
   3413   1.1  jmcneill 	return (0);
   3414   1.1  jmcneill }
   3415   1.1  jmcneill 
   3416   1.1  jmcneill static int
   3417   1.1  jmcneill mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   3418   1.1  jmcneill     unsigned int nmb, uint64_t *ptr, uint8_t token)
   3419   1.1  jmcneill {
   3420   1.1  jmcneill 	uint8_t *kva;
   3421   1.1  jmcneill 	uint64_t dva;
   3422   1.1  jmcneill 	int i;
   3423   1.1  jmcneill 	int error;
   3424   1.1  jmcneill 
   3425   1.1  jmcneill 	error = mcx_dmamem_alloc(sc, mxm,
   3426   1.1  jmcneill 	    nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
   3427   1.1  jmcneill 	if (error != 0)
   3428   1.1  jmcneill 		return (error);
   3429   1.1  jmcneill 
   3430   1.1  jmcneill 	mcx_dmamem_zero(mxm);
   3431   1.1  jmcneill 
   3432   1.1  jmcneill 	dva = MCX_DMA_DVA(mxm);
   3433   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3434   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3435   1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
   3436   1.1  jmcneill 
   3437   1.1  jmcneill 		/* patch the cqe or mbox pointing at this one */
   3438   1.1  jmcneill 		be64enc(ptr, dva);
   3439   1.1  jmcneill 
   3440   1.1  jmcneill 		/* fill in this mbox */
   3441   1.1  jmcneill 		be32enc(&mbox->mb_block_number, i);
   3442   1.1  jmcneill 		mbox->mb_token = token;
   3443   1.1  jmcneill 
   3444   1.1  jmcneill 		/* move to the next one */
   3445   1.1  jmcneill 		ptr = &mbox->mb_next_ptr;
   3446   1.1  jmcneill 
   3447   1.1  jmcneill 		dva += MCX_CMDQ_MAILBOX_SIZE;
   3448   1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   3449   1.1  jmcneill 	}
   3450   1.1  jmcneill 
   3451   1.1  jmcneill 	return (0);
   3452   1.1  jmcneill }
   3453   1.1  jmcneill 
   3454   1.1  jmcneill static uint32_t
   3455   1.1  jmcneill mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
   3456   1.1  jmcneill {
   3457   1.1  jmcneill 	uint32_t xor = 0xff;
   3458   1.1  jmcneill 
   3459   1.1  jmcneill 	/* only 3 fields get set, so mix them directly */
   3460   1.1  jmcneill 	xor = mcx_mix_u64(xor, mb->mb_next_ptr);
   3461   1.1  jmcneill 	xor = mcx_mix_u32(xor, mb->mb_block_number);
   3462   1.1  jmcneill 	xor = mcx_mix_u8(xor, mb->mb_token);
   3463   1.1  jmcneill 
   3464   1.1  jmcneill 	return (mcx_mix_done(xor));
   3465   1.1  jmcneill }
   3466   1.1  jmcneill 
   3467   1.1  jmcneill static void
   3468   1.1  jmcneill mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
   3469   1.1  jmcneill {
   3470   1.1  jmcneill 	uint8_t *kva;
   3471   1.1  jmcneill 	int i;
   3472   1.1  jmcneill 
   3473   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3474   1.1  jmcneill 
   3475   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3476   1.1  jmcneill 		struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
   3477   1.1  jmcneill 		uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
   3478   1.1  jmcneill 		mb->mb_ctrl_signature = sig;
   3479   1.1  jmcneill 		mb->mb_signature = sig ^
   3480   1.1  jmcneill 		    mcx_xor(mb->mb_data, sizeof(mb->mb_data));
   3481   1.1  jmcneill 
   3482   1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   3483   1.1  jmcneill 	}
   3484   1.1  jmcneill }
   3485   1.1  jmcneill 
   3486   1.1  jmcneill static void
   3487   1.1  jmcneill mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
   3488   1.1  jmcneill {
   3489   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
   3490   1.1  jmcneill 	    0, MCX_DMA_LEN(mxm), ops);
   3491   1.1  jmcneill }
   3492   1.1  jmcneill 
   3493   1.1  jmcneill static struct mcx_cmdq_mailbox *
   3494   1.1  jmcneill mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
   3495   1.1  jmcneill {
   3496   1.1  jmcneill 	uint8_t *kva;
   3497   1.1  jmcneill 
   3498   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3499   1.1  jmcneill 	kva += i * MCX_CMDQ_MAILBOX_SIZE;
   3500   1.1  jmcneill 
   3501   1.1  jmcneill 	return ((struct mcx_cmdq_mailbox *)kva);
   3502   1.1  jmcneill }
   3503   1.1  jmcneill 
   3504   1.1  jmcneill static inline void *
   3505   1.1  jmcneill mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
   3506   1.1  jmcneill {
   3507   1.1  jmcneill 	return (&mb->mb_data);
   3508   1.1  jmcneill }
   3509   1.1  jmcneill 
   3510   1.1  jmcneill static void
   3511   1.1  jmcneill mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
   3512   1.1  jmcneill     void *b, size_t len)
   3513   1.1  jmcneill {
   3514   1.1  jmcneill 	uint8_t *buf = b;
   3515   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3516   1.1  jmcneill 	int i;
   3517   1.1  jmcneill 
   3518   1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   3519   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3520   1.1  jmcneill 
   3521   1.1  jmcneill 		memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len));
   3522   1.1  jmcneill 
   3523   1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   3524   1.1  jmcneill 			break;
   3525   1.1  jmcneill 
   3526   1.1  jmcneill 		buf += sizeof(mb->mb_data);
   3527   1.1  jmcneill 		len -= sizeof(mb->mb_data);
   3528   1.1  jmcneill 		mb++;
   3529   1.1  jmcneill 	}
   3530   1.1  jmcneill }
   3531   1.1  jmcneill 
   3532   1.1  jmcneill static void
   3533  1.13  jmcneill mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages,
   3534  1.13  jmcneill     struct mcx_dmamem *buf)
   3535  1.13  jmcneill {
   3536  1.13  jmcneill 	uint64_t *pas;
   3537  1.13  jmcneill 	int mbox, mbox_pages, i;
   3538  1.13  jmcneill 
   3539  1.13  jmcneill 	mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE;
   3540  1.13  jmcneill 	offset %= MCX_CMDQ_MAILBOX_DATASIZE;
   3541  1.13  jmcneill 
   3542  1.13  jmcneill 	pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
   3543  1.13  jmcneill 	pas += (offset / sizeof(*pas));
   3544  1.13  jmcneill 	mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas);
   3545  1.13  jmcneill 	for (i = 0; i < npages; i++) {
   3546  1.13  jmcneill 		if (i == mbox_pages) {
   3547  1.13  jmcneill 			mbox++;
   3548  1.13  jmcneill 			pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
   3549  1.13  jmcneill 			mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas);
   3550  1.13  jmcneill 		}
   3551  1.13  jmcneill 		*pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE));
   3552  1.13  jmcneill 		pas++;
   3553  1.13  jmcneill 	}
   3554  1.13  jmcneill }
   3555  1.13  jmcneill 
   3556  1.13  jmcneill static void
   3557   1.1  jmcneill mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
   3558   1.1  jmcneill {
   3559   1.1  jmcneill 	uint8_t *buf = b;
   3560   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3561   1.1  jmcneill 	int i;
   3562   1.1  jmcneill 
   3563   1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   3564   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3565   1.1  jmcneill 		memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len));
   3566   1.1  jmcneill 
   3567   1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   3568   1.1  jmcneill 			break;
   3569   1.1  jmcneill 
   3570   1.1  jmcneill 		buf += sizeof(mb->mb_data);
   3571   1.1  jmcneill 		len -= sizeof(mb->mb_data);
   3572   1.1  jmcneill 		mb++;
   3573   1.1  jmcneill 	}
   3574   1.1  jmcneill }
   3575   1.1  jmcneill 
   3576   1.1  jmcneill static void
   3577   1.1  jmcneill mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   3578   1.1  jmcneill {
   3579   1.1  jmcneill 	mcx_dmamem_free(sc, mxm);
   3580   1.1  jmcneill }
   3581   1.1  jmcneill 
   3582   1.1  jmcneill #if 0
   3583   1.1  jmcneill static void
   3584   1.1  jmcneill mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
   3585   1.1  jmcneill {
   3586   1.1  jmcneill 	unsigned int i;
   3587   1.1  jmcneill 
   3588   1.1  jmcneill 	printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
   3589   1.1  jmcneill 	    be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr));
   3590   1.1  jmcneill 
   3591   1.1  jmcneill 	printf(", idata ");
   3592   1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_input_data); i++)
   3593   1.1  jmcneill 		printf("%02x", cqe->cq_input_data[i]);
   3594   1.1  jmcneill 
   3595   1.1  jmcneill 	printf(", odata ");
   3596   1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_output_data); i++)
   3597   1.1  jmcneill 		printf("%02x", cqe->cq_output_data[i]);
   3598   1.1  jmcneill 
   3599   1.1  jmcneill 	printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
   3600   1.1  jmcneill 	    be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length),
   3601   1.1  jmcneill 	    cqe->cq_token, cqe->cq_signature, cqe->cq_status);
   3602   1.1  jmcneill }
   3603   1.1  jmcneill 
   3604   1.1  jmcneill static void
   3605   1.1  jmcneill mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
   3606   1.1  jmcneill {
   3607   1.1  jmcneill 	int i, j;
   3608   1.1  jmcneill 	uint8_t *d;
   3609   1.1  jmcneill 
   3610   1.1  jmcneill 	for (i = 0; i < num; i++) {
   3611   1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox;
   3612   1.1  jmcneill 		mbox = mcx_cq_mbox(mboxes, i);
   3613   1.1  jmcneill 
   3614   1.1  jmcneill 		d = mcx_cq_mbox_data(mbox);
   3615   1.1  jmcneill 		for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
   3616   1.1  jmcneill 			if (j != 0 && (j % 16 == 0))
   3617   1.1  jmcneill 				printf("\n");
   3618   1.1  jmcneill 			printf("%.2x ", d[j]);
   3619   1.1  jmcneill 		}
   3620   1.1  jmcneill 	}
   3621   1.1  jmcneill }
   3622   1.1  jmcneill #endif
   3623   1.1  jmcneill 
   3624   1.1  jmcneill static int
   3625   1.1  jmcneill mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
   3626   1.1  jmcneill     int len)
   3627   1.1  jmcneill {
   3628   1.1  jmcneill 	struct mcx_dmamem mxm;
   3629   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3630   1.1  jmcneill 	struct mcx_cmd_access_reg_in *in;
   3631   1.1  jmcneill 	struct mcx_cmd_access_reg_out *out;
   3632   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3633   1.1  jmcneill 	int error, nmb;
   3634   1.1  jmcneill 
   3635   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3636   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
   3637   1.1  jmcneill 	    token);
   3638   1.1  jmcneill 
   3639   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3640   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
   3641   1.1  jmcneill 	in->cmd_op_mod = htobe16(op);
   3642   1.1  jmcneill 	in->cmd_register_id = htobe16(reg);
   3643   1.1  jmcneill 
   3644   1.1  jmcneill 	nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
   3645  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
   3646  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3647   1.1  jmcneill 		printf(", unable to allocate access reg mailboxen\n");
   3648   1.1  jmcneill 		return (-1);
   3649   1.1  jmcneill 	}
   3650   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   3651   1.1  jmcneill 	mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
   3652   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   3653   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   3654   1.1  jmcneill 
   3655   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3656   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3657   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   3658   1.1  jmcneill 
   3659   1.1  jmcneill 	if (error != 0) {
   3660   1.1  jmcneill 		printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
   3661   1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
   3662   1.1  jmcneill 		goto free;
   3663   1.1  jmcneill 	}
   3664   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3665   1.1  jmcneill 	if (error != 0) {
   3666   1.1  jmcneill 		printf("%s: access reg (%s %x) reply corrupt\n",
   3667   1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
   3668   1.1  jmcneill 		    reg);
   3669   1.1  jmcneill 		goto free;
   3670   1.1  jmcneill 	}
   3671   1.1  jmcneill 
   3672   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3673   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3674   1.1  jmcneill 		printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
   3675   1.1  jmcneill 		    DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
   3676  1.15  jmcneill 		    reg, out->cmd_status, be32toh(out->cmd_syndrome));
   3677   1.1  jmcneill 		error = -1;
   3678   1.1  jmcneill 		goto free;
   3679   1.1  jmcneill 	}
   3680   1.1  jmcneill 
   3681   1.1  jmcneill 	mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
   3682   1.1  jmcneill free:
   3683   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3684   1.1  jmcneill 
   3685   1.1  jmcneill 	return (error);
   3686   1.1  jmcneill }
   3687   1.1  jmcneill 
   3688   1.1  jmcneill static int
   3689  1.15  jmcneill mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3690  1.15  jmcneill     unsigned int slot)
   3691   1.1  jmcneill {
   3692   1.1  jmcneill 	struct mcx_cmd_set_issi_in *in;
   3693   1.1  jmcneill 	struct mcx_cmd_set_issi_out *out;
   3694   1.1  jmcneill 	uint8_t status;
   3695   1.1  jmcneill 
   3696   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3697   1.1  jmcneill 
   3698   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3699   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
   3700   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3701   1.1  jmcneill 	in->cmd_current_issi = htobe16(MCX_ISSI);
   3702   1.1  jmcneill 
   3703   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, slot);
   3704   1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
   3705   1.1  jmcneill 		return (-1);
   3706   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0)
   3707   1.1  jmcneill 		return (-1);
   3708   1.1  jmcneill 
   3709   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3710   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK)
   3711   1.1  jmcneill 		return (-1);
   3712   1.1  jmcneill 
   3713   1.1  jmcneill 	return (0);
   3714   1.1  jmcneill }
   3715   1.1  jmcneill 
   3716   1.1  jmcneill static int
   3717   1.1  jmcneill mcx_issi(struct mcx_softc *sc)
   3718   1.1  jmcneill {
   3719   1.1  jmcneill 	struct mcx_dmamem mxm;
   3720   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3721   1.1  jmcneill 	struct mcx_cmd_query_issi_in *in;
   3722   1.1  jmcneill 	struct mcx_cmd_query_issi_il_out *out;
   3723   1.1  jmcneill 	struct mcx_cmd_query_issi_mb_out *mb;
   3724   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3725   1.1  jmcneill 	uint8_t status;
   3726   1.1  jmcneill 	int error;
   3727   1.1  jmcneill 
   3728   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3729   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
   3730   1.1  jmcneill 
   3731   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3732   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
   3733   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3734   1.1  jmcneill 
   3735   1.1  jmcneill 	CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
   3736   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   3737   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3738   1.1  jmcneill 		printf(", unable to allocate query issi mailbox\n");
   3739   1.1  jmcneill 		return (-1);
   3740   1.1  jmcneill 	}
   3741   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   3742   1.1  jmcneill 
   3743   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3744   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3745   1.1  jmcneill 	if (error != 0) {
   3746   1.1  jmcneill 		printf(", query issi timeout\n");
   3747   1.1  jmcneill 		goto free;
   3748   1.1  jmcneill 	}
   3749   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3750   1.1  jmcneill 	if (error != 0) {
   3751   1.1  jmcneill 		printf(", query issi reply corrupt\n");
   3752   1.1  jmcneill 		goto free;
   3753   1.1  jmcneill 	}
   3754   1.1  jmcneill 
   3755   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3756   1.1  jmcneill 	switch (status) {
   3757   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   3758   1.1  jmcneill 		break;
   3759   1.1  jmcneill 	case MCX_CQ_STATUS_BAD_OPCODE:
   3760   1.1  jmcneill 		/* use ISSI 0 */
   3761   1.1  jmcneill 		goto free;
   3762   1.1  jmcneill 	default:
   3763   1.1  jmcneill 		printf(", query issi failed (%x)\n", status);
   3764   1.1  jmcneill 		error = -1;
   3765   1.1  jmcneill 		goto free;
   3766   1.1  jmcneill 	}
   3767   1.1  jmcneill 
   3768   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3769   1.1  jmcneill 	if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
   3770   1.1  jmcneill 		/* use ISSI 1 */
   3771   1.1  jmcneill 		goto free;
   3772   1.1  jmcneill 	}
   3773   1.1  jmcneill 
   3774   1.1  jmcneill 	/* don't need to read cqe anymore, can be used for SET ISSI */
   3775   1.1  jmcneill 
   3776   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   3777   1.1  jmcneill 	CTASSERT(MCX_ISSI < NBBY);
   3778   1.1  jmcneill 	 /* XXX math is hard */
   3779   1.1  jmcneill 	if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
   3780   1.1  jmcneill 		/* use ISSI 0 */
   3781   1.1  jmcneill 		goto free;
   3782   1.1  jmcneill 	}
   3783   1.1  jmcneill 
   3784   1.1  jmcneill 	if (mcx_set_issi(sc, cqe, 0) != 0) {
   3785   1.1  jmcneill 		/* ignore the error, just use ISSI 0 */
   3786   1.1  jmcneill 	} else {
   3787   1.1  jmcneill 		/* use ISSI 1 */
   3788   1.1  jmcneill 	}
   3789   1.1  jmcneill 
   3790   1.1  jmcneill free:
   3791   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   3792   1.1  jmcneill 	return (error);
   3793   1.1  jmcneill }
   3794   1.1  jmcneill 
   3795   1.1  jmcneill static int
   3796   1.1  jmcneill mcx_query_pages(struct mcx_softc *sc, uint16_t type,
   3797  1.15  jmcneill     int32_t *npages, uint16_t *func_id)
   3798   1.1  jmcneill {
   3799   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3800   1.1  jmcneill 	struct mcx_cmd_query_pages_in *in;
   3801   1.1  jmcneill 	struct mcx_cmd_query_pages_out *out;
   3802   1.1  jmcneill 
   3803   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3804   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3805   1.1  jmcneill 
   3806   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3807   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
   3808   1.1  jmcneill 	in->cmd_op_mod = type;
   3809   1.1  jmcneill 
   3810   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3811   1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
   3812   1.1  jmcneill 		printf(", query pages timeout\n");
   3813   1.1  jmcneill 		return (-1);
   3814   1.1  jmcneill 	}
   3815   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3816   1.1  jmcneill 		printf(", query pages reply corrupt\n");
   3817   1.1  jmcneill 		return (-1);
   3818   1.1  jmcneill 	}
   3819   1.1  jmcneill 
   3820   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3821   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3822   1.1  jmcneill 		printf(", query pages failed (%x)\n", out->cmd_status);
   3823   1.1  jmcneill 		return (-1);
   3824   1.1  jmcneill 	}
   3825   1.1  jmcneill 
   3826   1.1  jmcneill 	*func_id = out->cmd_func_id;
   3827   1.1  jmcneill 	*npages = be32dec(&out->cmd_num_pages);
   3828   1.1  jmcneill 
   3829   1.1  jmcneill 	return (0);
   3830   1.1  jmcneill }
   3831   1.1  jmcneill 
   3832   1.1  jmcneill struct bus_dma_iter {
   3833   1.1  jmcneill 	bus_dmamap_t		i_map;
   3834   1.1  jmcneill 	bus_size_t		i_offset;
   3835   1.1  jmcneill 	unsigned int		i_index;
   3836   1.1  jmcneill };
   3837   1.1  jmcneill 
   3838   1.1  jmcneill static void
   3839   1.1  jmcneill bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
   3840   1.1  jmcneill {
   3841   1.1  jmcneill 	i->i_map = map;
   3842   1.1  jmcneill 	i->i_offset = 0;
   3843   1.1  jmcneill 	i->i_index = 0;
   3844   1.1  jmcneill }
   3845   1.1  jmcneill 
   3846   1.1  jmcneill static bus_addr_t
   3847   1.1  jmcneill bus_dma_iter_addr(struct bus_dma_iter *i)
   3848   1.1  jmcneill {
   3849   1.1  jmcneill 	return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
   3850   1.1  jmcneill }
   3851   1.1  jmcneill 
   3852   1.1  jmcneill static void
   3853   1.1  jmcneill bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
   3854   1.1  jmcneill {
   3855   1.1  jmcneill 	bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
   3856   1.1  jmcneill 	bus_size_t diff;
   3857   1.1  jmcneill 
   3858   1.1  jmcneill 	do {
   3859   1.1  jmcneill 		diff = seg->ds_len - i->i_offset;
   3860   1.1  jmcneill 		if (size < diff)
   3861   1.1  jmcneill 			break;
   3862   1.1  jmcneill 
   3863   1.1  jmcneill 		size -= diff;
   3864   1.1  jmcneill 
   3865   1.1  jmcneill 		seg++;
   3866   1.1  jmcneill 
   3867   1.1  jmcneill 		i->i_offset = 0;
   3868   1.1  jmcneill 		i->i_index++;
   3869   1.1  jmcneill 	} while (size > 0);
   3870   1.1  jmcneill 
   3871   1.1  jmcneill 	i->i_offset += size;
   3872   1.1  jmcneill }
   3873   1.1  jmcneill 
   3874   1.1  jmcneill static int
   3875   1.1  jmcneill mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
   3876   1.1  jmcneill {
   3877   1.1  jmcneill 	struct mcx_dmamem mxm;
   3878   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3879   1.1  jmcneill 	struct mcx_cmd_manage_pages_in *in;
   3880   1.1  jmcneill 	struct mcx_cmd_manage_pages_out *out;
   3881   1.1  jmcneill 	unsigned int paslen, nmb, i, j, npages;
   3882   1.1  jmcneill 	struct bus_dma_iter iter;
   3883   1.1  jmcneill 	uint64_t *pas;
   3884   1.1  jmcneill 	uint8_t status;
   3885   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3886   1.1  jmcneill 	int error;
   3887   1.1  jmcneill 
   3888   1.1  jmcneill 	npages = mhm->mhm_npages;
   3889   1.1  jmcneill 
   3890   1.1  jmcneill 	paslen = sizeof(*pas) * npages;
   3891   1.1  jmcneill 	nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
   3892   1.1  jmcneill 
   3893   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3894   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
   3895   1.1  jmcneill 
   3896   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3897   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
   3898   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
   3899   1.1  jmcneill 	in->cmd_func_id = func_id;
   3900   1.1  jmcneill 	be32enc(&in->cmd_input_num_entries, npages);
   3901   1.1  jmcneill 
   3902   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
   3903   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   3904   1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   3905   1.1  jmcneill 		return (-1);
   3906   1.1  jmcneill 	}
   3907   1.1  jmcneill 
   3908   1.1  jmcneill 	bus_dma_iter_init(&iter, mhm->mhm_map);
   3909   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3910   1.1  jmcneill 		unsigned int lim;
   3911   1.1  jmcneill 
   3912   1.1  jmcneill 		pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
   3913   1.1  jmcneill 		lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
   3914   1.1  jmcneill 
   3915   1.1  jmcneill 		for (j = 0; j < lim; j++) {
   3916   1.1  jmcneill 			be64enc(&pas[j], bus_dma_iter_addr(&iter));
   3917   1.1  jmcneill 			bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
   3918   1.1  jmcneill 		}
   3919   1.1  jmcneill 
   3920   1.1  jmcneill 		npages -= lim;
   3921   1.1  jmcneill 	}
   3922   1.1  jmcneill 
   3923   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   3924   1.1  jmcneill 
   3925   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3926   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3927   1.1  jmcneill 	if (error != 0) {
   3928   1.1  jmcneill 		printf(", manage pages timeout\n");
   3929   1.1  jmcneill 		goto free;
   3930   1.1  jmcneill 	}
   3931   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3932   1.1  jmcneill 	if (error != 0) {
   3933   1.1  jmcneill 		printf(", manage pages reply corrupt\n");
   3934   1.1  jmcneill 		goto free;
   3935   1.1  jmcneill 	}
   3936   1.1  jmcneill 
   3937   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3938   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3939   1.1  jmcneill 		printf(", manage pages failed (%x)\n", status);
   3940   1.1  jmcneill 		error = -1;
   3941   1.1  jmcneill 		goto free;
   3942   1.1  jmcneill 	}
   3943   1.1  jmcneill 
   3944   1.1  jmcneill free:
   3945   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3946   1.1  jmcneill 
   3947   1.1  jmcneill 	return (error);
   3948   1.1  jmcneill }
   3949   1.1  jmcneill 
   3950   1.1  jmcneill static int
   3951   1.1  jmcneill mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
   3952   1.1  jmcneill {
   3953  1.15  jmcneill 	int32_t npages;
   3954   1.1  jmcneill 	uint16_t func_id;
   3955   1.1  jmcneill 
   3956   1.1  jmcneill 	if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
   3957   1.1  jmcneill 		/* error printed by mcx_query_pages */
   3958   1.1  jmcneill 		return (-1);
   3959   1.1  jmcneill 	}
   3960   1.1  jmcneill 
   3961  1.15  jmcneill 	if (npages < 1)
   3962   1.1  jmcneill 		return (0);
   3963   1.1  jmcneill 
   3964   1.1  jmcneill 	if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
   3965   1.1  jmcneill 		printf(", unable to allocate hwmem\n");
   3966   1.1  jmcneill 		return (-1);
   3967   1.1  jmcneill 	}
   3968   1.1  jmcneill 
   3969   1.1  jmcneill 	if (mcx_add_pages(sc, mhm, func_id) != 0) {
   3970   1.1  jmcneill 		printf(", unable to add hwmem\n");
   3971   1.1  jmcneill 		goto free;
   3972   1.1  jmcneill 	}
   3973   1.1  jmcneill 
   3974   1.1  jmcneill 	return (0);
   3975   1.1  jmcneill 
   3976   1.1  jmcneill free:
   3977   1.1  jmcneill 	mcx_hwmem_free(sc, mhm);
   3978   1.1  jmcneill 
   3979   1.1  jmcneill 	return (-1);
   3980   1.1  jmcneill }
   3981   1.1  jmcneill 
   3982   1.1  jmcneill static int
   3983   1.1  jmcneill mcx_hca_max_caps(struct mcx_softc *sc)
   3984   1.1  jmcneill {
   3985   1.1  jmcneill 	struct mcx_dmamem mxm;
   3986   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3987   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   3988   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   3989   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3990   1.1  jmcneill 	struct mcx_cap_device *hca;
   3991   1.1  jmcneill 	uint8_t status;
   3992   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3993   1.1  jmcneill 	int error;
   3994   1.1  jmcneill 
   3995   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3996   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   3997   1.1  jmcneill 	    token);
   3998   1.1  jmcneill 
   3999   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4000   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   4001   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
   4002   1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   4003   1.1  jmcneill 
   4004   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   4005   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4006   1.1  jmcneill 		printf(", unable to allocate query hca caps mailboxen\n");
   4007   1.1  jmcneill 		return (-1);
   4008   1.1  jmcneill 	}
   4009   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   4010   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   4011   1.1  jmcneill 
   4012   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4013   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4014   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   4015   1.1  jmcneill 
   4016   1.1  jmcneill 	if (error != 0) {
   4017   1.1  jmcneill 		printf(", query hca caps timeout\n");
   4018   1.1  jmcneill 		goto free;
   4019   1.1  jmcneill 	}
   4020   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   4021   1.1  jmcneill 	if (error != 0) {
   4022   1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   4023   1.1  jmcneill 		goto free;
   4024   1.1  jmcneill 	}
   4025   1.1  jmcneill 
   4026   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4027   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4028   1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   4029   1.1  jmcneill 		error = -1;
   4030   1.1  jmcneill 		goto free;
   4031   1.1  jmcneill 	}
   4032   1.1  jmcneill 
   4033   1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   4034   1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   4035   1.1  jmcneill 
   4036  1.15  jmcneill 	if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE)
   4037  1.15  jmcneill 	    != MCX_CAP_DEVICE_PORT_TYPE_ETH) {
   4038  1.15  jmcneill 		printf(", not in ethernet mode\n");
   4039  1.15  jmcneill 		error = -1;
   4040  1.15  jmcneill 		goto free;
   4041  1.15  jmcneill 	}
   4042   1.1  jmcneill 	if (hca->log_pg_sz > PAGE_SHIFT) {
   4043   1.1  jmcneill 		printf(", minimum system page shift %u is too large\n",
   4044   1.1  jmcneill 		    hca->log_pg_sz);
   4045   1.1  jmcneill 		error = -1;
   4046   1.1  jmcneill 		goto free;
   4047   1.1  jmcneill 	}
   4048   1.1  jmcneill 	/*
   4049   1.1  jmcneill 	 * blueflame register is split into two buffers, and we must alternate
   4050   1.1  jmcneill 	 * between the two of them.
   4051   1.1  jmcneill 	 */
   4052   1.1  jmcneill 	sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
   4053  1.15  jmcneill 	sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size);
   4054  1.24     skrll 
   4055  1.15  jmcneill 	if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG)
   4056  1.15  jmcneill 		sc->sc_mcam_reg = 1;
   4057  1.15  jmcneill 
   4058  1.15  jmcneill 	sc->sc_mhz = be32dec(&hca->device_frequency_mhz);
   4059  1.15  jmcneill 	sc->sc_khz = be32dec(&hca->device_frequency_khz);
   4060   1.1  jmcneill 
   4061   1.1  jmcneill free:
   4062   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4063   1.1  jmcneill 
   4064   1.1  jmcneill 	return (error);
   4065   1.1  jmcneill }
   4066   1.1  jmcneill 
   4067   1.1  jmcneill static int
   4068   1.1  jmcneill mcx_hca_set_caps(struct mcx_softc *sc)
   4069   1.1  jmcneill {
   4070   1.1  jmcneill 	struct mcx_dmamem mxm;
   4071   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4072   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   4073   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   4074   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   4075   1.1  jmcneill 	struct mcx_cap_device *hca;
   4076   1.1  jmcneill 	uint8_t status;
   4077   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   4078   1.1  jmcneill 	int error;
   4079   1.1  jmcneill 
   4080   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4081   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   4082   1.1  jmcneill 	    token);
   4083   1.1  jmcneill 
   4084   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4085   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   4086   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
   4087   1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   4088   1.1  jmcneill 
   4089   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   4090   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4091   1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   4092   1.1  jmcneill 		return (-1);
   4093   1.1  jmcneill 	}
   4094   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   4095   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   4096   1.1  jmcneill 
   4097   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4098   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4099   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   4100   1.1  jmcneill 
   4101   1.1  jmcneill 	if (error != 0) {
   4102   1.1  jmcneill 		printf(", query hca caps timeout\n");
   4103   1.1  jmcneill 		goto free;
   4104   1.1  jmcneill 	}
   4105   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   4106   1.1  jmcneill 	if (error != 0) {
   4107   1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   4108   1.1  jmcneill 		goto free;
   4109   1.1  jmcneill 	}
   4110   1.1  jmcneill 
   4111   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4112   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4113   1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   4114   1.1  jmcneill 		error = -1;
   4115   1.1  jmcneill 		goto free;
   4116   1.1  jmcneill 	}
   4117   1.1  jmcneill 
   4118   1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   4119   1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   4120   1.1  jmcneill 
   4121   1.1  jmcneill 	hca->log_pg_sz = PAGE_SHIFT;
   4122   1.1  jmcneill 
   4123   1.1  jmcneill free:
   4124   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4125   1.1  jmcneill 
   4126   1.1  jmcneill 	return (error);
   4127   1.1  jmcneill }
   4128   1.1  jmcneill 
   4129   1.1  jmcneill 
   4130   1.1  jmcneill static int
   4131   1.1  jmcneill mcx_init_hca(struct mcx_softc *sc)
   4132   1.1  jmcneill {
   4133   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4134   1.1  jmcneill 	struct mcx_cmd_init_hca_in *in;
   4135   1.1  jmcneill 	struct mcx_cmd_init_hca_out *out;
   4136   1.1  jmcneill 	int error;
   4137   1.1  jmcneill 	uint8_t status;
   4138   1.1  jmcneill 
   4139   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4140   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4141   1.1  jmcneill 
   4142   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4143   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
   4144   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4145   1.1  jmcneill 
   4146   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4147   1.1  jmcneill 
   4148   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4149   1.1  jmcneill 	if (error != 0) {
   4150   1.1  jmcneill 		printf(", hca init timeout\n");
   4151   1.1  jmcneill 		return (-1);
   4152   1.1  jmcneill 	}
   4153   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4154   1.1  jmcneill 		printf(", hca init command corrupt\n");
   4155   1.1  jmcneill 		return (-1);
   4156   1.1  jmcneill 	}
   4157   1.1  jmcneill 
   4158   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4159   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4160   1.1  jmcneill 		printf(", hca init failed (%x)\n", status);
   4161   1.1  jmcneill 		return (-1);
   4162   1.1  jmcneill 	}
   4163   1.1  jmcneill 
   4164   1.1  jmcneill 	return (0);
   4165   1.1  jmcneill }
   4166   1.1  jmcneill 
   4167   1.1  jmcneill static int
   4168   1.1  jmcneill mcx_set_driver_version(struct mcx_softc *sc)
   4169   1.1  jmcneill {
   4170   1.1  jmcneill 	struct mcx_dmamem mxm;
   4171   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4172   1.1  jmcneill 	struct mcx_cmd_set_driver_version_in *in;
   4173   1.1  jmcneill 	struct mcx_cmd_set_driver_version_out *out;
   4174   1.1  jmcneill 	int error;
   4175   1.1  jmcneill 	int token;
   4176   1.1  jmcneill 	uint8_t status;
   4177   1.1  jmcneill 
   4178   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4179   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4180   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) +
   4181   1.1  jmcneill 	    sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);
   4182   1.1  jmcneill 
   4183   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4184   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
   4185   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4186   1.1  jmcneill 
   4187   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4188   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4189   1.1  jmcneill 		printf(", unable to allocate set driver version mailboxen\n");
   4190   1.1  jmcneill 		return (-1);
   4191   1.1  jmcneill 	}
   4192   1.1  jmcneill 	strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
   4193   1.1  jmcneill 	    "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
   4194   1.1  jmcneill 
   4195   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4196   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4197   1.1  jmcneill 
   4198   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4199   1.1  jmcneill 	if (error != 0) {
   4200   1.1  jmcneill 		printf(", set driver version timeout\n");
   4201   1.1  jmcneill 		goto free;
   4202   1.1  jmcneill 	}
   4203   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4204   1.1  jmcneill 		printf(", set driver version command corrupt\n");
   4205   1.1  jmcneill 		goto free;
   4206   1.1  jmcneill 	}
   4207   1.1  jmcneill 
   4208   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4209   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4210   1.1  jmcneill 		printf(", set driver version failed (%x)\n", status);
   4211   1.1  jmcneill 		error = -1;
   4212   1.1  jmcneill 		goto free;
   4213   1.1  jmcneill 	}
   4214   1.1  jmcneill 
   4215   1.1  jmcneill free:
   4216   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4217   1.1  jmcneill 
   4218   1.1  jmcneill 	return (error);
   4219   1.1  jmcneill }
   4220   1.1  jmcneill 
   4221   1.1  jmcneill static int
   4222   1.1  jmcneill mcx_iff(struct mcx_softc *sc)
   4223   1.1  jmcneill {
   4224   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   4225   1.1  jmcneill 	struct mcx_dmamem mxm;
   4226   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4227   1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_in *in;
   4228   1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_out *out;
   4229   1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   4230   1.1  jmcneill 	int error;
   4231   1.1  jmcneill 	int token;
   4232   1.1  jmcneill 	int insize;
   4233  1.15  jmcneill 	uint32_t dest;
   4234  1.15  jmcneill 
   4235  1.15  jmcneill 	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   4236  1.15  jmcneill 	    sc->sc_rss_flow_table_id;
   4237   1.1  jmcneill 
   4238   1.1  jmcneill 	/* enable or disable the promisc flow */
   4239   1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_PROMISC)) {
   4240   1.1  jmcneill 		if (sc->sc_promisc_flow_enabled == 0) {
   4241  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc,
   4242  1.15  jmcneill 			    MCX_FLOW_GROUP_PROMISC, 0, NULL, dest);
   4243   1.1  jmcneill 			sc->sc_promisc_flow_enabled = 1;
   4244   1.1  jmcneill 		}
   4245   1.1  jmcneill 	} else if (sc->sc_promisc_flow_enabled != 0) {
   4246   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   4247   1.1  jmcneill 		sc->sc_promisc_flow_enabled = 0;
   4248   1.1  jmcneill 	}
   4249   1.1  jmcneill 
   4250   1.1  jmcneill 	/* enable or disable the all-multicast flow */
   4251   1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   4252   1.1  jmcneill 		if (sc->sc_allmulti_flow_enabled == 0) {
   4253   1.1  jmcneill 			uint8_t mcast[ETHER_ADDR_LEN];
   4254   1.1  jmcneill 
   4255   1.1  jmcneill 			memset(mcast, 0, sizeof(mcast));
   4256   1.1  jmcneill 			mcast[0] = 0x01;
   4257  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc,
   4258  1.15  jmcneill 			    MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest);
   4259   1.1  jmcneill 			sc->sc_allmulti_flow_enabled = 1;
   4260   1.1  jmcneill 		}
   4261   1.1  jmcneill 	} else if (sc->sc_allmulti_flow_enabled != 0) {
   4262   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   4263   1.1  jmcneill 		sc->sc_allmulti_flow_enabled = 0;
   4264   1.1  jmcneill 	}
   4265   1.1  jmcneill 
   4266   1.1  jmcneill 	insize = sizeof(struct mcx_nic_vport_ctx) + 240;
   4267   1.1  jmcneill 
   4268   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4269   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4270   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4271   1.1  jmcneill 
   4272   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4273   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
   4274   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4275   1.1  jmcneill 	in->cmd_field_select = htobe32(
   4276   1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
   4277   1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
   4278   1.1  jmcneill 
   4279   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4280  1.15  jmcneill 		printf(", unable to allocate modify "
   4281  1.15  jmcneill 		    "nic vport context mailboxen\n");
   4282   1.1  jmcneill 		return (-1);
   4283   1.1  jmcneill 	}
   4284   1.1  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)
   4285   1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
   4286   1.1  jmcneill 	ctx->vp_mtu = htobe32(sc->sc_hardmtu);
   4287   1.1  jmcneill 	/*
   4288  1.15  jmcneill          * always leave promisc-all enabled on the vport since we
   4289  1.15  jmcneill          * can't give it a vlan list, and we're already doing multicast
   4290  1.15  jmcneill          * filtering in the flow table.
   4291   1.1  jmcneill 	 */
   4292   1.1  jmcneill 	ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);
   4293   1.1  jmcneill 
   4294   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4295   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4296   1.1  jmcneill 
   4297   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4298   1.1  jmcneill 	if (error != 0) {
   4299   1.1  jmcneill 		printf(", modify nic vport context timeout\n");
   4300   1.1  jmcneill 		goto free;
   4301   1.1  jmcneill 	}
   4302   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4303   1.1  jmcneill 		printf(", modify nic vport context command corrupt\n");
   4304   1.1  jmcneill 		goto free;
   4305   1.1  jmcneill 	}
   4306   1.1  jmcneill 
   4307   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4308   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4309   1.1  jmcneill 		printf(", modify nic vport context failed (%x, %x)\n",
   4310  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4311   1.1  jmcneill 		error = -1;
   4312   1.1  jmcneill 		goto free;
   4313   1.1  jmcneill 	}
   4314   1.1  jmcneill 
   4315   1.1  jmcneill free:
   4316   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4317   1.1  jmcneill 
   4318   1.1  jmcneill 	return (error);
   4319   1.1  jmcneill }
   4320   1.1  jmcneill 
   4321   1.1  jmcneill static int
   4322  1.15  jmcneill mcx_alloc_uar(struct mcx_softc *sc, int *uar)
   4323   1.1  jmcneill {
   4324   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4325   1.1  jmcneill 	struct mcx_cmd_alloc_uar_in *in;
   4326   1.1  jmcneill 	struct mcx_cmd_alloc_uar_out *out;
   4327   1.1  jmcneill 	int error;
   4328   1.1  jmcneill 
   4329   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4330   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4331   1.1  jmcneill 
   4332   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4333   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
   4334   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4335   1.1  jmcneill 
   4336   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4337   1.1  jmcneill 
   4338   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4339   1.1  jmcneill 	if (error != 0) {
   4340   1.1  jmcneill 		printf(", alloc uar timeout\n");
   4341   1.1  jmcneill 		return (-1);
   4342   1.1  jmcneill 	}
   4343   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4344   1.1  jmcneill 		printf(", alloc uar command corrupt\n");
   4345   1.1  jmcneill 		return (-1);
   4346   1.1  jmcneill 	}
   4347   1.1  jmcneill 
   4348   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4349   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4350   1.1  jmcneill 		printf(", alloc uar failed (%x)\n", out->cmd_status);
   4351   1.1  jmcneill 		return (-1);
   4352   1.1  jmcneill 	}
   4353   1.1  jmcneill 
   4354  1.15  jmcneill 	*uar = mcx_get_id(out->cmd_uar);
   4355   1.1  jmcneill 	return (0);
   4356   1.1  jmcneill }
   4357   1.1  jmcneill 
   4358   1.1  jmcneill static int
   4359  1.15  jmcneill mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar,
   4360  1.15  jmcneill     uint64_t events, int vector)
   4361   1.1  jmcneill {
   4362   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4363   1.1  jmcneill 	struct mcx_dmamem mxm;
   4364   1.1  jmcneill 	struct mcx_cmd_create_eq_in *in;
   4365   1.1  jmcneill 	struct mcx_cmd_create_eq_mb_in *mbin;
   4366   1.1  jmcneill 	struct mcx_cmd_create_eq_out *out;
   4367   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   4368   1.1  jmcneill 	int error;
   4369   1.1  jmcneill 	uint64_t *pas;
   4370   1.1  jmcneill 	int insize, npages, paslen, i, token;
   4371   1.1  jmcneill 
   4372  1.15  jmcneill 	eq->eq_cons = 0;
   4373   1.1  jmcneill 
   4374   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
   4375   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4376   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4377   1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
   4378   1.1  jmcneill 
   4379  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE,
   4380   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4381   1.1  jmcneill 		printf(", unable to allocate event queue memory\n");
   4382   1.1  jmcneill 		return (-1);
   4383   1.1  jmcneill 	}
   4384   1.1  jmcneill 
   4385  1.15  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
   4386   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
   4387   1.1  jmcneill 		eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
   4388   1.1  jmcneill 	}
   4389   1.1  jmcneill 
   4390   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4391   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4392   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4393   1.1  jmcneill 
   4394   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4395   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
   4396   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4397   1.1  jmcneill 
   4398  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4399  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4400   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4401   1.1  jmcneill 		printf(", unable to allocate create eq mailboxen\n");
   4402  1.15  jmcneill 		goto free_eq;
   4403   1.1  jmcneill 	}
   4404   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4405   1.1  jmcneill 	mbin->cmd_eq_ctx.eq_uar_size = htobe32(
   4406  1.15  jmcneill 	    (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar);
   4407  1.15  jmcneill 	mbin->cmd_eq_ctx.eq_intr = vector;
   4408  1.15  jmcneill 	mbin->cmd_event_bitmask = htobe64(events);
   4409  1.15  jmcneill 
   4410  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   4411  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   4412   1.1  jmcneill 
   4413   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4414  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem);
   4415   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
   4416   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4417   1.1  jmcneill 
   4418   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4419   1.1  jmcneill 	if (error != 0) {
   4420   1.1  jmcneill 		printf(", create eq timeout\n");
   4421  1.15  jmcneill 		goto free_mxm;
   4422   1.1  jmcneill 	}
   4423   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4424   1.1  jmcneill 		printf(", create eq command corrupt\n");
   4425  1.15  jmcneill 		goto free_mxm;
   4426   1.1  jmcneill 	}
   4427   1.1  jmcneill 
   4428   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4429   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4430   1.1  jmcneill 		printf(", create eq failed (%x, %x)\n", out->cmd_status,
   4431   1.1  jmcneill 		    be32toh(out->cmd_syndrome));
   4432  1.15  jmcneill 		goto free_mxm;
   4433   1.1  jmcneill 	}
   4434   1.1  jmcneill 
   4435  1.15  jmcneill 	eq->eq_n = mcx_get_id(out->cmd_eqn);
   4436  1.15  jmcneill 
   4437  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4438  1.15  jmcneill 
   4439  1.15  jmcneill 	mcx_arm_eq(sc, eq, uar);
   4440  1.15  jmcneill 
   4441  1.15  jmcneill 	return (0);
   4442  1.15  jmcneill 
   4443  1.15  jmcneill free_mxm:
   4444  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   4445  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   4446   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4447  1.15  jmcneill free_eq:
   4448  1.15  jmcneill 	mcx_dmamem_free(sc, &eq->eq_mem);
   4449  1.15  jmcneill 	return (-1);
   4450   1.1  jmcneill }
   4451   1.1  jmcneill 
   4452   1.1  jmcneill static int
   4453   1.1  jmcneill mcx_alloc_pd(struct mcx_softc *sc)
   4454   1.1  jmcneill {
   4455   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4456   1.1  jmcneill 	struct mcx_cmd_alloc_pd_in *in;
   4457   1.1  jmcneill 	struct mcx_cmd_alloc_pd_out *out;
   4458   1.1  jmcneill 	int error;
   4459   1.1  jmcneill 
   4460   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4461   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4462   1.1  jmcneill 
   4463   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4464   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
   4465   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4466   1.1  jmcneill 
   4467   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4468   1.1  jmcneill 
   4469   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4470   1.1  jmcneill 	if (error != 0) {
   4471   1.1  jmcneill 		printf(", alloc pd timeout\n");
   4472   1.1  jmcneill 		return (-1);
   4473   1.1  jmcneill 	}
   4474   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4475   1.1  jmcneill 		printf(", alloc pd command corrupt\n");
   4476   1.1  jmcneill 		return (-1);
   4477   1.1  jmcneill 	}
   4478   1.1  jmcneill 
   4479   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4480   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4481   1.1  jmcneill 		printf(", alloc pd failed (%x)\n", out->cmd_status);
   4482   1.1  jmcneill 		return (-1);
   4483   1.1  jmcneill 	}
   4484   1.1  jmcneill 
   4485  1.13  jmcneill 	sc->sc_pd = mcx_get_id(out->cmd_pd);
   4486   1.1  jmcneill 	return (0);
   4487   1.1  jmcneill }
   4488   1.1  jmcneill 
   4489   1.1  jmcneill static int
   4490   1.1  jmcneill mcx_alloc_tdomain(struct mcx_softc *sc)
   4491   1.1  jmcneill {
   4492   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4493   1.1  jmcneill 	struct mcx_cmd_alloc_td_in *in;
   4494   1.1  jmcneill 	struct mcx_cmd_alloc_td_out *out;
   4495   1.1  jmcneill 	int error;
   4496   1.1  jmcneill 
   4497   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4498   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4499   1.1  jmcneill 
   4500   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4501   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
   4502   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4503   1.1  jmcneill 
   4504   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4505   1.1  jmcneill 
   4506   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4507   1.1  jmcneill 	if (error != 0) {
   4508   1.1  jmcneill 		printf(", alloc transport domain timeout\n");
   4509   1.1  jmcneill 		return (-1);
   4510   1.1  jmcneill 	}
   4511   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4512   1.1  jmcneill 		printf(", alloc transport domain command corrupt\n");
   4513   1.1  jmcneill 		return (-1);
   4514   1.1  jmcneill 	}
   4515   1.1  jmcneill 
   4516   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4517   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4518   1.1  jmcneill 		printf(", alloc transport domain failed (%x)\n",
   4519   1.1  jmcneill 		    out->cmd_status);
   4520   1.1  jmcneill 		return (-1);
   4521   1.1  jmcneill 	}
   4522   1.1  jmcneill 
   4523  1.13  jmcneill 	sc->sc_tdomain = mcx_get_id(out->cmd_tdomain);
   4524   1.1  jmcneill 	return (0);
   4525   1.1  jmcneill }
   4526   1.1  jmcneill 
   4527   1.1  jmcneill static int
   4528   1.1  jmcneill mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr)
   4529   1.1  jmcneill {
   4530   1.1  jmcneill 	struct mcx_dmamem mxm;
   4531   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4532   1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_in *in;
   4533   1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_out *out;
   4534   1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   4535   1.1  jmcneill 	uint8_t *addr;
   4536   1.1  jmcneill 	int error, token, i;
   4537   1.1  jmcneill 
   4538   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4539   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4540   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
   4541   1.1  jmcneill 
   4542   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4543   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
   4544   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4545   1.1  jmcneill 	in->cmd_allowed_list_type = 0;
   4546   1.1  jmcneill 
   4547  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4548  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4549  1.15  jmcneill 		printf(", unable to allocate "
   4550  1.15  jmcneill 		    "query nic vport context mailboxen\n");
   4551   1.1  jmcneill 		return (-1);
   4552   1.1  jmcneill 	}
   4553   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4554   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4555   1.1  jmcneill 
   4556   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4557   1.1  jmcneill 	if (error != 0) {
   4558   1.1  jmcneill 		printf(", query nic vport context timeout\n");
   4559   1.1  jmcneill 		goto free;
   4560   1.1  jmcneill 	}
   4561   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4562   1.1  jmcneill 		printf(", query nic vport context command corrupt\n");
   4563   1.1  jmcneill 		goto free;
   4564   1.1  jmcneill 	}
   4565   1.1  jmcneill 
   4566   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4567   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4568   1.1  jmcneill 		printf(", query nic vport context failed (%x, %x)\n",
   4569  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4570   1.1  jmcneill 		error = -1;
   4571   1.1  jmcneill 		goto free;
   4572   1.1  jmcneill 	}
   4573   1.1  jmcneill 
   4574  1.15  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)
   4575  1.15  jmcneill 	    mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4576   1.1  jmcneill 	addr = (uint8_t *)&ctx->vp_perm_addr;
   4577   1.1  jmcneill 	for (i = 0; i < ETHER_ADDR_LEN; i++) {
   4578   1.1  jmcneill 		enaddr[i] = addr[i + 2];
   4579   1.1  jmcneill 	}
   4580   1.1  jmcneill free:
   4581   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4582   1.1  jmcneill 
   4583   1.1  jmcneill 	return (error);
   4584   1.1  jmcneill }
   4585   1.1  jmcneill 
   4586   1.1  jmcneill static int
   4587   1.1  jmcneill mcx_query_special_contexts(struct mcx_softc *sc)
   4588   1.1  jmcneill {
   4589   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4590   1.1  jmcneill 	struct mcx_cmd_query_special_ctx_in *in;
   4591   1.1  jmcneill 	struct mcx_cmd_query_special_ctx_out *out;
   4592   1.1  jmcneill 	int error;
   4593   1.1  jmcneill 
   4594   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4595   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4596   1.1  jmcneill 
   4597   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4598   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
   4599   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4600   1.1  jmcneill 
   4601   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4602   1.1  jmcneill 
   4603   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4604   1.1  jmcneill 	if (error != 0) {
   4605   1.1  jmcneill 		printf(", query special contexts timeout\n");
   4606   1.1  jmcneill 		return (-1);
   4607   1.1  jmcneill 	}
   4608   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4609   1.1  jmcneill 		printf(", query special contexts command corrupt\n");
   4610   1.1  jmcneill 		return (-1);
   4611   1.1  jmcneill 	}
   4612   1.1  jmcneill 
   4613   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4614   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4615   1.1  jmcneill 		printf(", query special contexts failed (%x)\n",
   4616   1.1  jmcneill 		    out->cmd_status);
   4617   1.1  jmcneill 		return (-1);
   4618   1.1  jmcneill 	}
   4619   1.1  jmcneill 
   4620   1.1  jmcneill 	sc->sc_lkey = be32toh(out->cmd_resd_lkey);
   4621   1.1  jmcneill 	return (0);
   4622   1.1  jmcneill }
   4623   1.1  jmcneill 
   4624   1.1  jmcneill static int
   4625   1.1  jmcneill mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
   4626   1.1  jmcneill {
   4627   1.1  jmcneill 	struct mcx_reg_pmtu pmtu;
   4628   1.1  jmcneill 	int error;
   4629   1.1  jmcneill 
   4630   1.1  jmcneill 	/* read max mtu */
   4631   1.1  jmcneill 	memset(&pmtu, 0, sizeof(pmtu));
   4632   1.1  jmcneill 	pmtu.rp_local_port = 1;
   4633   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
   4634   1.1  jmcneill 	    sizeof(pmtu));
   4635   1.1  jmcneill 	if (error != 0) {
   4636   1.1  jmcneill 		printf(", unable to get port MTU\n");
   4637   1.1  jmcneill 		return error;
   4638   1.1  jmcneill 	}
   4639   1.1  jmcneill 
   4640   1.1  jmcneill 	mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu));
   4641   1.1  jmcneill 	pmtu.rp_admin_mtu = htobe16(mtu);
   4642   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
   4643   1.1  jmcneill 	    sizeof(pmtu));
   4644   1.1  jmcneill 	if (error != 0) {
   4645   1.1  jmcneill 		printf(", unable to set port MTU\n");
   4646   1.1  jmcneill 		return error;
   4647   1.1  jmcneill 	}
   4648   1.1  jmcneill 
   4649   1.1  jmcneill 	sc->sc_hardmtu = mtu;
   4650  1.15  jmcneill 	sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long));
   4651   1.1  jmcneill 	return 0;
   4652   1.1  jmcneill }
   4653   1.1  jmcneill 
   4654   1.1  jmcneill static int
   4655  1.15  jmcneill mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn)
   4656   1.1  jmcneill {
   4657   1.1  jmcneill 	struct mcx_cmdq_entry *cmde;
   4658   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   4659   1.1  jmcneill 	struct mcx_dmamem mxm;
   4660   1.1  jmcneill 	struct mcx_cmd_create_cq_in *in;
   4661   1.1  jmcneill 	struct mcx_cmd_create_cq_mb_in *mbin;
   4662   1.1  jmcneill 	struct mcx_cmd_create_cq_out *out;
   4663   1.1  jmcneill 	int error;
   4664   1.1  jmcneill 	uint64_t *pas;
   4665   1.1  jmcneill 	int insize, npages, paslen, i, token;
   4666   1.1  jmcneill 
   4667  1.15  jmcneill 	cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db);
   4668   1.1  jmcneill 
   4669   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
   4670   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4671   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4672   1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
   4673   1.1  jmcneill 
   4674   1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
   4675   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4676   1.1  jmcneill 		printf("%s: unable to allocate completion queue memory\n",
   4677   1.1  jmcneill 		    DEVNAME(sc));
   4678   1.1  jmcneill 		return (-1);
   4679   1.1  jmcneill 	}
   4680   1.1  jmcneill 	cqe = MCX_DMA_KVA(&cq->cq_mem);
   4681   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
   4682   1.1  jmcneill 		cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
   4683   1.1  jmcneill 	}
   4684   1.1  jmcneill 
   4685   1.1  jmcneill 	cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4686   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4687   1.1  jmcneill 	mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
   4688   1.1  jmcneill 
   4689   1.1  jmcneill 	in = mcx_cmdq_in(cmde);
   4690   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
   4691   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4692   1.1  jmcneill 
   4693  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4694  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4695   1.1  jmcneill 	    &cmde->cq_input_ptr, token) != 0) {
   4696  1.15  jmcneill 		printf("%s: unable to allocate create cq mailboxen\n",
   4697  1.15  jmcneill 		    DEVNAME(sc));
   4698  1.15  jmcneill 		goto free_cq;
   4699   1.1  jmcneill 	}
   4700   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4701   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_uar_size = htobe32(
   4702  1.15  jmcneill 	    (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar);
   4703   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
   4704   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
   4705   1.1  jmcneill 	    (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
   4706   1.1  jmcneill 	    MCX_CQ_MOD_COUNTER);
   4707   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_doorbell = htobe64(
   4708  1.15  jmcneill 	    MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell);
   4709  1.15  jmcneill 
   4710  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4711  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
   4712   1.1  jmcneill 
   4713   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4714  1.13  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem);
   4715   1.1  jmcneill 	mcx_cmdq_post(sc, cmde, 0);
   4716   1.1  jmcneill 
   4717   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cmde, 1000);
   4718   1.1  jmcneill 	if (error != 0) {
   4719   1.1  jmcneill 		printf("%s: create cq timeout\n", DEVNAME(sc));
   4720  1.15  jmcneill 		goto free_mxm;
   4721   1.1  jmcneill 	}
   4722   1.1  jmcneill 	if (mcx_cmdq_verify(cmde) != 0) {
   4723   1.1  jmcneill 		printf("%s: create cq command corrupt\n", DEVNAME(sc));
   4724  1.15  jmcneill 		goto free_mxm;
   4725   1.1  jmcneill 	}
   4726   1.1  jmcneill 
   4727   1.1  jmcneill 	out = mcx_cmdq_out(cmde);
   4728   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4729   1.1  jmcneill 		printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
   4730   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4731  1.15  jmcneill 		goto free_mxm;
   4732   1.1  jmcneill 	}
   4733   1.1  jmcneill 
   4734  1.13  jmcneill 	cq->cq_n = mcx_get_id(out->cmd_cqn);
   4735   1.1  jmcneill 	cq->cq_cons = 0;
   4736   1.1  jmcneill 	cq->cq_count = 0;
   4737   1.1  jmcneill 
   4738   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4739  1.15  jmcneill 
   4740  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4741  1.15  jmcneill 	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
   4742  1.15  jmcneill 	    BUS_DMASYNC_PREWRITE);
   4743  1.15  jmcneill 
   4744  1.15  jmcneill 	mcx_arm_cq(sc, cq, uar);
   4745  1.15  jmcneill 
   4746  1.15  jmcneill 	return (0);
   4747  1.15  jmcneill 
   4748  1.15  jmcneill free_mxm:
   4749  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4750  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   4751  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4752  1.15  jmcneill free_cq:
   4753  1.15  jmcneill 	mcx_dmamem_free(sc, &cq->cq_mem);
   4754  1.15  jmcneill 	return (-1);
   4755   1.1  jmcneill }
   4756   1.1  jmcneill 
   4757   1.1  jmcneill static int
   4758  1.15  jmcneill mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq)
   4759   1.1  jmcneill {
   4760   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4761   1.1  jmcneill 	struct mcx_cmd_destroy_cq_in *in;
   4762   1.1  jmcneill 	struct mcx_cmd_destroy_cq_out *out;
   4763   1.1  jmcneill 	int error;
   4764   1.1  jmcneill 	int token;
   4765   1.1  jmcneill 
   4766   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4767   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4768   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4769   1.1  jmcneill 
   4770   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4771   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
   4772   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4773  1.15  jmcneill 	in->cmd_cqn = htobe32(cq->cq_n);
   4774   1.1  jmcneill 
   4775   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4776   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4777   1.1  jmcneill 	if (error != 0) {
   4778   1.1  jmcneill 		printf("%s: destroy cq timeout\n", DEVNAME(sc));
   4779   1.1  jmcneill 		return error;
   4780   1.1  jmcneill 	}
   4781   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4782   1.1  jmcneill 		printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
   4783   1.1  jmcneill 		return error;
   4784   1.1  jmcneill 	}
   4785   1.1  jmcneill 
   4786   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4787   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4788   1.1  jmcneill 		printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
   4789   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4790   1.1  jmcneill 		return -1;
   4791   1.1  jmcneill 	}
   4792   1.1  jmcneill 
   4793  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4794  1.15  jmcneill 	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
   4795  1.15  jmcneill 	    BUS_DMASYNC_POSTWRITE);
   4796  1.15  jmcneill 
   4797  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4798  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   4799  1.15  jmcneill 	mcx_dmamem_free(sc, &cq->cq_mem);
   4800  1.15  jmcneill 
   4801  1.15  jmcneill 	cq->cq_n = 0;
   4802  1.15  jmcneill 	cq->cq_cons = 0;
   4803  1.15  jmcneill 	cq->cq_count = 0;
   4804   1.1  jmcneill 	return 0;
   4805   1.1  jmcneill }
   4806   1.1  jmcneill 
   4807   1.1  jmcneill static int
   4808  1.15  jmcneill mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn)
   4809   1.1  jmcneill {
   4810   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4811   1.1  jmcneill 	struct mcx_dmamem mxm;
   4812   1.1  jmcneill 	struct mcx_cmd_create_rq_in *in;
   4813   1.1  jmcneill 	struct mcx_cmd_create_rq_out *out;
   4814   1.1  jmcneill 	struct mcx_rq_ctx *mbin;
   4815   1.1  jmcneill 	int error;
   4816   1.1  jmcneill 	uint64_t *pas;
   4817  1.15  jmcneill 	uint32_t rq_flags;
   4818  1.13  jmcneill 	int insize, npages, paslen, token;
   4819   1.1  jmcneill 
   4820  1.15  jmcneill 	rx->rx_doorbell = MCX_WQ_DOORBELL_BASE +
   4821  1.15  jmcneill 	    (db * MCX_WQ_DOORBELL_STRIDE);
   4822  1.15  jmcneill 
   4823   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
   4824   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4825   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4826   1.1  jmcneill 	insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
   4827   1.1  jmcneill 
   4828  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE,
   4829   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4830   1.1  jmcneill 		printf("%s: unable to allocate receive queue memory\n",
   4831   1.1  jmcneill 		    DEVNAME(sc));
   4832   1.1  jmcneill 		return (-1);
   4833   1.1  jmcneill 	}
   4834   1.1  jmcneill 
   4835   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4836   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4837   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4838   1.1  jmcneill 
   4839   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4840   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
   4841   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4842   1.1  jmcneill 
   4843  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4844  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4845   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4846   1.1  jmcneill 		printf("%s: unable to allocate create rq mailboxen\n",
   4847   1.1  jmcneill 		    DEVNAME(sc));
   4848  1.15  jmcneill 		goto free_rq;
   4849  1.15  jmcneill 	}
   4850  1.15  jmcneill 	mbin = (struct mcx_rq_ctx *)
   4851  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   4852  1.15  jmcneill 	rq_flags = MCX_RQ_CTX_RLKEY;
   4853  1.15  jmcneill 	mbin->rq_flags = htobe32(rq_flags);
   4854   1.1  jmcneill 	mbin->rq_cqn = htobe32(cqn);
   4855   1.1  jmcneill 	mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   4856   1.1  jmcneill 	mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
   4857   1.1  jmcneill 	mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   4858  1.15  jmcneill 	    rx->rx_doorbell);
   4859   1.1  jmcneill 	mbin->rq_wq.wq_log_stride = htobe16(4);
   4860   1.1  jmcneill 	mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
   4861   1.1  jmcneill 
   4862  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   4863  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
   4864  1.15  jmcneill 
   4865   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4866  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem);
   4867   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4868   1.1  jmcneill 
   4869   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4870   1.1  jmcneill 	if (error != 0) {
   4871   1.1  jmcneill 		printf("%s: create rq timeout\n", DEVNAME(sc));
   4872  1.15  jmcneill 		goto free_mxm;
   4873   1.1  jmcneill 	}
   4874   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4875   1.1  jmcneill 		printf("%s: create rq command corrupt\n", DEVNAME(sc));
   4876  1.15  jmcneill 		goto free_mxm;
   4877   1.1  jmcneill 	}
   4878   1.1  jmcneill 
   4879   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4880   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4881   1.1  jmcneill 		printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
   4882   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4883  1.15  jmcneill 		goto free_mxm;
   4884   1.1  jmcneill 	}
   4885   1.1  jmcneill 
   4886  1.15  jmcneill 	rx->rx_rqn = mcx_get_id(out->cmd_rqn);
   4887   1.1  jmcneill 
   4888  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4889  1.15  jmcneill 
   4890  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4891  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   4892  1.15  jmcneill 
   4893  1.15  jmcneill 	return (0);
   4894   1.1  jmcneill 
   4895  1.15  jmcneill free_mxm:
   4896  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   4897  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   4898   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4899  1.15  jmcneill free_rq:
   4900  1.15  jmcneill 	mcx_dmamem_free(sc, &rx->rx_rq_mem);
   4901  1.15  jmcneill 	return (-1);
   4902   1.1  jmcneill }
   4903   1.1  jmcneill 
   4904   1.1  jmcneill static int
   4905  1.15  jmcneill mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx)
   4906   1.1  jmcneill {
   4907   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4908   1.1  jmcneill 	struct mcx_dmamem mxm;
   4909   1.1  jmcneill 	struct mcx_cmd_modify_rq_in *in;
   4910   1.1  jmcneill 	struct mcx_cmd_modify_rq_mb_in *mbin;
   4911   1.1  jmcneill 	struct mcx_cmd_modify_rq_out *out;
   4912   1.1  jmcneill 	int error;
   4913   1.1  jmcneill 	int token;
   4914   1.1  jmcneill 
   4915   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4916   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4917  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   4918  1.15  jmcneill 	    sizeof(*out), token);
   4919   1.1  jmcneill 
   4920   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4921   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
   4922   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4923  1.15  jmcneill 	in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn);
   4924   1.1  jmcneill 
   4925  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4926  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4927  1.15  jmcneill 		printf("%s: unable to allocate modify rq mailbox\n",
   4928  1.15  jmcneill 		    DEVNAME(sc));
   4929   1.1  jmcneill 		return (-1);
   4930   1.1  jmcneill 	}
   4931   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4932   1.1  jmcneill 	mbin->cmd_rq_ctx.rq_flags = htobe32(
   4933   1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
   4934   1.1  jmcneill 
   4935   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4936   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4937   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4938   1.1  jmcneill 	if (error != 0) {
   4939   1.1  jmcneill 		printf("%s: modify rq timeout\n", DEVNAME(sc));
   4940   1.1  jmcneill 		goto free;
   4941   1.1  jmcneill 	}
   4942   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4943   1.1  jmcneill 		printf("%s: modify rq command corrupt\n", DEVNAME(sc));
   4944   1.1  jmcneill 		goto free;
   4945   1.1  jmcneill 	}
   4946   1.1  jmcneill 
   4947   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4948   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4949   1.1  jmcneill 		printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
   4950   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4951   1.1  jmcneill 		error = -1;
   4952   1.1  jmcneill 		goto free;
   4953   1.1  jmcneill 	}
   4954   1.1  jmcneill 
   4955   1.1  jmcneill free:
   4956   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4957   1.1  jmcneill 	return (error);
   4958   1.1  jmcneill }
   4959   1.1  jmcneill 
   4960   1.1  jmcneill static int
   4961  1.15  jmcneill mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx)
   4962   1.1  jmcneill {
   4963   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4964   1.1  jmcneill 	struct mcx_cmd_destroy_rq_in *in;
   4965   1.1  jmcneill 	struct mcx_cmd_destroy_rq_out *out;
   4966   1.1  jmcneill 	int error;
   4967   1.1  jmcneill 	int token;
   4968   1.1  jmcneill 
   4969   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4970   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4971   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4972   1.1  jmcneill 
   4973   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4974   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
   4975   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4976  1.15  jmcneill 	in->cmd_rqn = htobe32(rx->rx_rqn);
   4977   1.1  jmcneill 
   4978   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4979   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4980   1.1  jmcneill 	if (error != 0) {
   4981   1.1  jmcneill 		printf("%s: destroy rq timeout\n", DEVNAME(sc));
   4982   1.1  jmcneill 		return error;
   4983   1.1  jmcneill 	}
   4984   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4985   1.1  jmcneill 		printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
   4986   1.1  jmcneill 		return error;
   4987   1.1  jmcneill 	}
   4988   1.1  jmcneill 
   4989   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4990   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4991   1.1  jmcneill 		printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
   4992   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4993   1.1  jmcneill 		return -1;
   4994   1.1  jmcneill 	}
   4995   1.1  jmcneill 
   4996  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4997  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   4998  1.15  jmcneill 
   4999  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   5000  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   5001  1.15  jmcneill 	mcx_dmamem_free(sc, &rx->rx_rq_mem);
   5002  1.15  jmcneill 
   5003  1.15  jmcneill 	rx->rx_rqn = 0;
   5004   1.1  jmcneill 	return 0;
   5005   1.1  jmcneill }
   5006   1.1  jmcneill 
   5007   1.1  jmcneill static int
   5008  1.15  jmcneill mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn)
   5009   1.1  jmcneill {
   5010   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5011   1.1  jmcneill 	struct mcx_dmamem mxm;
   5012   1.1  jmcneill 	struct mcx_cmd_create_tir_in *in;
   5013   1.1  jmcneill 	struct mcx_cmd_create_tir_mb_in *mbin;
   5014   1.1  jmcneill 	struct mcx_cmd_create_tir_out *out;
   5015   1.1  jmcneill 	int error;
   5016   1.1  jmcneill 	int token;
   5017   1.1  jmcneill 
   5018   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5019   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5020  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5021  1.15  jmcneill 	    sizeof(*out), token);
   5022   1.1  jmcneill 
   5023   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5024   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
   5025   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5026   1.1  jmcneill 
   5027  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5028  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5029   1.1  jmcneill 		printf("%s: unable to allocate create tir mailbox\n",
   5030   1.1  jmcneill 		    DEVNAME(sc));
   5031   1.1  jmcneill 		return (-1);
   5032   1.1  jmcneill 	}
   5033   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5034   1.1  jmcneill 	/* leave disp_type = 0, so packets get sent to the inline rqn */
   5035  1.15  jmcneill 	mbin->cmd_inline_rqn = htobe32(rx->rx_rqn);
   5036   1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   5037   1.1  jmcneill 
   5038   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5039   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5040   1.1  jmcneill 	if (error != 0) {
   5041   1.1  jmcneill 		printf("%s: create tir timeout\n", DEVNAME(sc));
   5042   1.1  jmcneill 		goto free;
   5043   1.1  jmcneill 	}
   5044   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5045   1.1  jmcneill 		printf("%s: create tir command corrupt\n", DEVNAME(sc));
   5046   1.1  jmcneill 		goto free;
   5047   1.1  jmcneill 	}
   5048   1.1  jmcneill 
   5049   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5050   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5051   1.1  jmcneill 		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
   5052   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5053   1.1  jmcneill 		error = -1;
   5054   1.1  jmcneill 		goto free;
   5055   1.1  jmcneill 	}
   5056   1.1  jmcneill 
   5057  1.15  jmcneill 	*tirn = mcx_get_id(out->cmd_tirn);
   5058  1.15  jmcneill free:
   5059  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5060  1.15  jmcneill 	return (error);
   5061  1.15  jmcneill }
   5062  1.15  jmcneill 
   5063  1.15  jmcneill static int
   5064  1.15  jmcneill mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel,
   5065  1.15  jmcneill     int *tirn)
   5066  1.15  jmcneill {
   5067  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5068  1.15  jmcneill 	struct mcx_dmamem mxm;
   5069  1.15  jmcneill 	struct mcx_cmd_create_tir_in *in;
   5070  1.15  jmcneill 	struct mcx_cmd_create_tir_mb_in *mbin;
   5071  1.15  jmcneill 	struct mcx_cmd_create_tir_out *out;
   5072  1.15  jmcneill 	int error;
   5073  1.15  jmcneill 	int token;
   5074  1.15  jmcneill 
   5075  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5076  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5077  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5078  1.15  jmcneill 	    sizeof(*out), token);
   5079  1.15  jmcneill 
   5080  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5081  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
   5082  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5083  1.15  jmcneill 
   5084  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5085  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5086  1.15  jmcneill 		printf("%s: unable to allocate create tir mailbox\n",
   5087  1.15  jmcneill 		    DEVNAME(sc));
   5088  1.15  jmcneill 		return (-1);
   5089  1.15  jmcneill 	}
   5090  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5091  1.15  jmcneill 	mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT
   5092  1.15  jmcneill 	    << MCX_TIR_CTX_DISP_TYPE_SHIFT);
   5093  1.15  jmcneill 	mbin->cmd_indir_table = htobe32(rqtn);
   5094  1.15  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain |
   5095  1.15  jmcneill 	    MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT);
   5096  1.15  jmcneill 	mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel);
   5097  1.15  jmcneill 	stoeplitz_to_key(&mbin->cmd_rx_hash_key,
   5098  1.15  jmcneill 	    sizeof(mbin->cmd_rx_hash_key));
   5099  1.15  jmcneill 
   5100  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5101  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5102  1.15  jmcneill 	if (error != 0) {
   5103  1.15  jmcneill 		printf("%s: create tir timeout\n", DEVNAME(sc));
   5104  1.15  jmcneill 		goto free;
   5105  1.15  jmcneill 	}
   5106  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5107  1.15  jmcneill 		printf("%s: create tir command corrupt\n", DEVNAME(sc));
   5108  1.15  jmcneill 		goto free;
   5109  1.15  jmcneill 	}
   5110  1.15  jmcneill 
   5111  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5112  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5113  1.15  jmcneill 		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
   5114  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5115  1.15  jmcneill 		error = -1;
   5116  1.15  jmcneill 		goto free;
   5117  1.15  jmcneill 	}
   5118  1.15  jmcneill 
   5119  1.15  jmcneill 	*tirn = mcx_get_id(out->cmd_tirn);
   5120   1.1  jmcneill free:
   5121   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5122   1.1  jmcneill 	return (error);
   5123   1.1  jmcneill }
   5124   1.1  jmcneill 
   5125   1.1  jmcneill static int
   5126  1.15  jmcneill mcx_destroy_tir(struct mcx_softc *sc, int tirn)
   5127   1.1  jmcneill {
   5128   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5129   1.1  jmcneill 	struct mcx_cmd_destroy_tir_in *in;
   5130   1.1  jmcneill 	struct mcx_cmd_destroy_tir_out *out;
   5131   1.1  jmcneill 	int error;
   5132   1.1  jmcneill 	int token;
   5133   1.1  jmcneill 
   5134   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5135   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5136   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5137   1.1  jmcneill 
   5138   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5139   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
   5140   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5141  1.15  jmcneill 	in->cmd_tirn = htobe32(tirn);
   5142   1.1  jmcneill 
   5143   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5144   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5145   1.1  jmcneill 	if (error != 0) {
   5146   1.1  jmcneill 		printf("%s: destroy tir timeout\n", DEVNAME(sc));
   5147   1.1  jmcneill 		return error;
   5148   1.1  jmcneill 	}
   5149   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5150   1.1  jmcneill 		printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
   5151   1.1  jmcneill 		return error;
   5152   1.1  jmcneill 	}
   5153   1.1  jmcneill 
   5154   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5155   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5156   1.1  jmcneill 		printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
   5157   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5158   1.1  jmcneill 		return -1;
   5159   1.1  jmcneill 	}
   5160   1.1  jmcneill 
   5161  1.15  jmcneill 	return (0);
   5162   1.1  jmcneill }
   5163   1.1  jmcneill 
   5164   1.1  jmcneill static int
   5165  1.15  jmcneill mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db,
   5166  1.15  jmcneill     int cqn)
   5167   1.1  jmcneill {
   5168   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5169   1.1  jmcneill 	struct mcx_dmamem mxm;
   5170   1.1  jmcneill 	struct mcx_cmd_create_sq_in *in;
   5171   1.1  jmcneill 	struct mcx_sq_ctx *mbin;
   5172   1.1  jmcneill 	struct mcx_cmd_create_sq_out *out;
   5173   1.1  jmcneill 	int error;
   5174   1.1  jmcneill 	uint64_t *pas;
   5175  1.13  jmcneill 	int insize, npages, paslen, token;
   5176   1.1  jmcneill 
   5177  1.15  jmcneill 	tx->tx_doorbell = MCX_WQ_DOORBELL_BASE +
   5178  1.15  jmcneill 	    (db * MCX_WQ_DOORBELL_STRIDE) + 4;
   5179  1.15  jmcneill 
   5180   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
   5181   1.1  jmcneill 	    MCX_PAGE_SIZE);
   5182   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   5183   1.1  jmcneill 	insize = sizeof(struct mcx_sq_ctx) + paslen;
   5184   1.1  jmcneill 
   5185  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE,
   5186   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   5187  1.15  jmcneill 		printf("%s: unable to allocate send queue memory\n",
   5188  1.15  jmcneill 		    DEVNAME(sc));
   5189   1.1  jmcneill 		return (-1);
   5190   1.1  jmcneill 	}
   5191   1.1  jmcneill 
   5192   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5193   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5194   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
   5195   1.1  jmcneill 	    token);
   5196   1.1  jmcneill 
   5197   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5198   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
   5199   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5200   1.1  jmcneill 
   5201  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   5202  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   5203   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5204  1.15  jmcneill 		printf("%s: unable to allocate create sq mailboxen\n",
   5205  1.15  jmcneill 		    DEVNAME(sc));
   5206  1.15  jmcneill 		goto free_sq;
   5207   1.1  jmcneill 	}
   5208  1.15  jmcneill 	mbin = (struct mcx_sq_ctx *)
   5209  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   5210   1.1  jmcneill 	mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
   5211   1.1  jmcneill 	    (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
   5212   1.1  jmcneill 	mbin->sq_cqn = htobe32(cqn);
   5213   1.1  jmcneill 	mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
   5214  1.15  jmcneill 	mbin->sq_tis_num = htobe32(sc->sc_tis);
   5215   1.1  jmcneill 	mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   5216   1.1  jmcneill 	mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
   5217  1.15  jmcneill 	mbin->sq_wq.wq_uar_page = htobe32(uar);
   5218   1.1  jmcneill 	mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   5219  1.15  jmcneill 	    tx->tx_doorbell);
   5220   1.1  jmcneill 	mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
   5221   1.1  jmcneill 	mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
   5222   1.1  jmcneill 
   5223  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5224  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
   5225  1.15  jmcneill 
   5226   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   5227  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10,
   5228  1.15  jmcneill 	    npages, &tx->tx_sq_mem);
   5229   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5230   1.1  jmcneill 
   5231   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5232   1.1  jmcneill 	if (error != 0) {
   5233   1.1  jmcneill 		printf("%s: create sq timeout\n", DEVNAME(sc));
   5234  1.15  jmcneill 		goto free_mxm;
   5235   1.1  jmcneill 	}
   5236   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5237   1.1  jmcneill 		printf("%s: create sq command corrupt\n", DEVNAME(sc));
   5238  1.15  jmcneill 		goto free_mxm;
   5239   1.1  jmcneill 	}
   5240   1.1  jmcneill 
   5241   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5242   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5243   1.1  jmcneill 		printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
   5244   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5245  1.15  jmcneill 		goto free_mxm;
   5246   1.1  jmcneill 	}
   5247   1.1  jmcneill 
   5248  1.15  jmcneill 	tx->tx_uar = uar;
   5249  1.15  jmcneill 	tx->tx_sqn = mcx_get_id(out->cmd_sqn);
   5250  1.15  jmcneill 
   5251  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5252  1.15  jmcneill 
   5253  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   5254  1.15  jmcneill 	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   5255  1.15  jmcneill 
   5256  1.15  jmcneill 	return (0);
   5257   1.1  jmcneill 
   5258  1.15  jmcneill free_mxm:
   5259  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5260  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   5261   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5262  1.15  jmcneill free_sq:
   5263  1.15  jmcneill 	mcx_dmamem_free(sc, &tx->tx_sq_mem);
   5264  1.15  jmcneill 	return (-1);
   5265   1.1  jmcneill }
   5266   1.1  jmcneill 
   5267   1.1  jmcneill static int
   5268  1.15  jmcneill mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx)
   5269   1.1  jmcneill {
   5270   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5271   1.1  jmcneill 	struct mcx_cmd_destroy_sq_in *in;
   5272   1.1  jmcneill 	struct mcx_cmd_destroy_sq_out *out;
   5273   1.1  jmcneill 	int error;
   5274   1.1  jmcneill 	int token;
   5275   1.1  jmcneill 
   5276   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5277   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5278   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5279   1.1  jmcneill 
   5280   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5281   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
   5282   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5283  1.15  jmcneill 	in->cmd_sqn = htobe32(tx->tx_sqn);
   5284   1.1  jmcneill 
   5285   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5286   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5287   1.1  jmcneill 	if (error != 0) {
   5288   1.1  jmcneill 		printf("%s: destroy sq timeout\n", DEVNAME(sc));
   5289   1.1  jmcneill 		return error;
   5290   1.1  jmcneill 	}
   5291   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5292   1.1  jmcneill 		printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
   5293   1.1  jmcneill 		return error;
   5294   1.1  jmcneill 	}
   5295   1.1  jmcneill 
   5296   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5297   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5298   1.1  jmcneill 		printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
   5299   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5300   1.1  jmcneill 		return -1;
   5301   1.1  jmcneill 	}
   5302   1.1  jmcneill 
   5303  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   5304  1.15  jmcneill 	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   5305  1.15  jmcneill 
   5306  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5307  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   5308  1.15  jmcneill 	mcx_dmamem_free(sc, &tx->tx_sq_mem);
   5309  1.15  jmcneill 
   5310  1.15  jmcneill 	tx->tx_sqn = 0;
   5311   1.1  jmcneill 	return 0;
   5312   1.1  jmcneill }
   5313   1.1  jmcneill 
   5314   1.1  jmcneill static int
   5315  1.15  jmcneill mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx)
   5316   1.1  jmcneill {
   5317   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5318   1.1  jmcneill 	struct mcx_dmamem mxm;
   5319   1.1  jmcneill 	struct mcx_cmd_modify_sq_in *in;
   5320   1.1  jmcneill 	struct mcx_cmd_modify_sq_mb_in *mbin;
   5321   1.1  jmcneill 	struct mcx_cmd_modify_sq_out *out;
   5322   1.1  jmcneill 	int error;
   5323   1.1  jmcneill 	int token;
   5324   1.1  jmcneill 
   5325   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5326   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5327  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5328  1.15  jmcneill 	    sizeof(*out), token);
   5329   1.1  jmcneill 
   5330   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5331   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
   5332   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5333  1.15  jmcneill 	in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn);
   5334   1.1  jmcneill 
   5335  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5336  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5337   1.1  jmcneill 		printf("%s: unable to allocate modify sq mailbox\n",
   5338   1.1  jmcneill 		    DEVNAME(sc));
   5339   1.1  jmcneill 		return (-1);
   5340   1.1  jmcneill 	}
   5341   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5342   1.1  jmcneill 	mbin->cmd_sq_ctx.sq_flags = htobe32(
   5343   1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
   5344   1.1  jmcneill 
   5345   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5346   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5347   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5348   1.1  jmcneill 	if (error != 0) {
   5349   1.1  jmcneill 		printf("%s: modify sq timeout\n", DEVNAME(sc));
   5350   1.1  jmcneill 		goto free;
   5351   1.1  jmcneill 	}
   5352   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5353   1.1  jmcneill 		printf("%s: modify sq command corrupt\n", DEVNAME(sc));
   5354   1.1  jmcneill 		goto free;
   5355   1.1  jmcneill 	}
   5356   1.1  jmcneill 
   5357   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5358   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5359   1.1  jmcneill 		printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
   5360   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5361   1.1  jmcneill 		error = -1;
   5362   1.1  jmcneill 		goto free;
   5363   1.1  jmcneill 	}
   5364   1.1  jmcneill 
   5365   1.1  jmcneill free:
   5366   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5367   1.1  jmcneill 	return (error);
   5368   1.1  jmcneill }
   5369   1.1  jmcneill 
   5370   1.1  jmcneill static int
   5371  1.15  jmcneill mcx_create_tis(struct mcx_softc *sc, int *tis)
   5372   1.1  jmcneill {
   5373   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5374   1.1  jmcneill 	struct mcx_dmamem mxm;
   5375   1.1  jmcneill 	struct mcx_cmd_create_tis_in *in;
   5376   1.1  jmcneill 	struct mcx_cmd_create_tis_mb_in *mbin;
   5377   1.1  jmcneill 	struct mcx_cmd_create_tis_out *out;
   5378   1.1  jmcneill 	int error;
   5379   1.1  jmcneill 	int token;
   5380   1.1  jmcneill 
   5381   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5382   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5383  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5384  1.15  jmcneill 	    sizeof(*out), token);
   5385   1.1  jmcneill 
   5386   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5387   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
   5388   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5389   1.1  jmcneill 
   5390  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5391  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5392  1.15  jmcneill 		printf("%s: unable to allocate create tis mailbox\n",
   5393  1.15  jmcneill 		    DEVNAME(sc));
   5394   1.1  jmcneill 		return (-1);
   5395   1.1  jmcneill 	}
   5396   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5397   1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   5398   1.1  jmcneill 
   5399   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5400   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5401   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5402   1.1  jmcneill 	if (error != 0) {
   5403   1.1  jmcneill 		printf("%s: create tis timeout\n", DEVNAME(sc));
   5404   1.1  jmcneill 		goto free;
   5405   1.1  jmcneill 	}
   5406   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5407   1.1  jmcneill 		printf("%s: create tis command corrupt\n", DEVNAME(sc));
   5408   1.1  jmcneill 		goto free;
   5409   1.1  jmcneill 	}
   5410   1.1  jmcneill 
   5411   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5412   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5413   1.1  jmcneill 		printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
   5414   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5415   1.1  jmcneill 		error = -1;
   5416   1.1  jmcneill 		goto free;
   5417   1.1  jmcneill 	}
   5418   1.1  jmcneill 
   5419  1.15  jmcneill 	*tis = mcx_get_id(out->cmd_tisn);
   5420   1.1  jmcneill free:
   5421   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5422   1.1  jmcneill 	return (error);
   5423   1.1  jmcneill }
   5424   1.1  jmcneill 
   5425   1.1  jmcneill static int
   5426  1.15  jmcneill mcx_destroy_tis(struct mcx_softc *sc, int tis)
   5427   1.1  jmcneill {
   5428   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5429   1.1  jmcneill 	struct mcx_cmd_destroy_tis_in *in;
   5430   1.1  jmcneill 	struct mcx_cmd_destroy_tis_out *out;
   5431   1.1  jmcneill 	int error;
   5432   1.1  jmcneill 	int token;
   5433   1.1  jmcneill 
   5434   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5435   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5436   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5437   1.1  jmcneill 
   5438   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5439   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
   5440   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5441  1.15  jmcneill 	in->cmd_tisn = htobe32(tis);
   5442   1.1  jmcneill 
   5443   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5444   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5445   1.1  jmcneill 	if (error != 0) {
   5446   1.1  jmcneill 		printf("%s: destroy tis timeout\n", DEVNAME(sc));
   5447   1.1  jmcneill 		return error;
   5448   1.1  jmcneill 	}
   5449   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5450   1.1  jmcneill 		printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
   5451   1.1  jmcneill 		return error;
   5452   1.1  jmcneill 	}
   5453   1.1  jmcneill 
   5454   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5455   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5456   1.1  jmcneill 		printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
   5457   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5458   1.1  jmcneill 		return -1;
   5459   1.1  jmcneill 	}
   5460   1.1  jmcneill 
   5461  1.15  jmcneill 	return 0;
   5462  1.15  jmcneill }
   5463  1.15  jmcneill 
   5464  1.15  jmcneill static int
   5465  1.15  jmcneill mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt)
   5466  1.15  jmcneill {
   5467  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5468  1.15  jmcneill 	struct mcx_dmamem mxm;
   5469  1.15  jmcneill 	struct mcx_cmd_create_rqt_in *in;
   5470  1.15  jmcneill 	struct mcx_cmd_create_rqt_mb_in *mbin;
   5471  1.15  jmcneill 	struct mcx_cmd_create_rqt_out *out;
   5472  1.15  jmcneill 	struct mcx_rqt_ctx *rqt_ctx;
   5473  1.15  jmcneill 	int *rqtn;
   5474  1.15  jmcneill 	int error;
   5475  1.15  jmcneill 	int token;
   5476  1.15  jmcneill 	int i;
   5477  1.15  jmcneill 
   5478  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5479  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5480  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) +
   5481  1.15  jmcneill 	    (size * sizeof(int)), sizeof(*out), token);
   5482  1.15  jmcneill 
   5483  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5484  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT);
   5485  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5486  1.15  jmcneill 
   5487  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5488  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5489  1.15  jmcneill 		printf("%s: unable to allocate create rqt mailbox\n",
   5490  1.15  jmcneill 		    DEVNAME(sc));
   5491  1.15  jmcneill 		return (-1);
   5492  1.15  jmcneill 	}
   5493  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5494  1.15  jmcneill 	rqt_ctx = &mbin->cmd_rqt;
   5495  1.15  jmcneill 	rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size);
   5496  1.15  jmcneill 	rqt_ctx->cmd_rqt_actual_size = htobe16(size);
   5497  1.15  jmcneill 
   5498  1.15  jmcneill 	/* rqt list follows the rqt context */
   5499  1.15  jmcneill 	rqtn = (int *)(rqt_ctx + 1);
   5500  1.15  jmcneill 	for (i = 0; i < size; i++) {
   5501  1.15  jmcneill 		rqtn[i] = htobe32(rqns[i]);
   5502  1.15  jmcneill 	}
   5503  1.15  jmcneill 
   5504  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5505  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5506  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5507  1.15  jmcneill 	if (error != 0) {
   5508  1.15  jmcneill 		printf("%s: create rqt timeout\n", DEVNAME(sc));
   5509  1.15  jmcneill 		goto free;
   5510  1.15  jmcneill 	}
   5511  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5512  1.15  jmcneill 		printf("%s: create rqt command corrupt\n", DEVNAME(sc));
   5513  1.15  jmcneill 		goto free;
   5514  1.15  jmcneill 	}
   5515  1.15  jmcneill 
   5516  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5517  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5518  1.15  jmcneill 		printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc),
   5519  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5520  1.15  jmcneill 		error = -1;
   5521  1.15  jmcneill 		goto free;
   5522  1.15  jmcneill 	}
   5523  1.15  jmcneill 
   5524  1.15  jmcneill 	*rqt = mcx_get_id(out->cmd_rqtn);
   5525  1.15  jmcneill 	return (0);
   5526  1.15  jmcneill free:
   5527  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5528  1.15  jmcneill 	return (error);
   5529  1.15  jmcneill }
   5530  1.15  jmcneill 
   5531  1.15  jmcneill static int
   5532  1.15  jmcneill mcx_destroy_rqt(struct mcx_softc *sc, int rqt)
   5533  1.15  jmcneill {
   5534  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5535  1.15  jmcneill 	struct mcx_cmd_destroy_rqt_in *in;
   5536  1.15  jmcneill 	struct mcx_cmd_destroy_rqt_out *out;
   5537  1.15  jmcneill 	int error;
   5538  1.15  jmcneill 	int token;
   5539  1.15  jmcneill 
   5540  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5541  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5542  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5543  1.15  jmcneill 
   5544  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5545  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT);
   5546  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5547  1.15  jmcneill 	in->cmd_rqtn = htobe32(rqt);
   5548  1.15  jmcneill 
   5549  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5550  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5551  1.15  jmcneill 	if (error != 0) {
   5552  1.15  jmcneill 		printf("%s: destroy rqt timeout\n", DEVNAME(sc));
   5553  1.15  jmcneill 		return error;
   5554  1.15  jmcneill 	}
   5555  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5556  1.15  jmcneill 		printf("%s: destroy rqt command corrupt\n", DEVNAME(sc));
   5557  1.15  jmcneill 		return error;
   5558  1.15  jmcneill 	}
   5559  1.15  jmcneill 
   5560  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5561  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5562  1.15  jmcneill 		printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc),
   5563  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5564  1.15  jmcneill 		return -1;
   5565  1.15  jmcneill 	}
   5566  1.15  jmcneill 
   5567   1.1  jmcneill 	return 0;
   5568   1.1  jmcneill }
   5569   1.1  jmcneill 
   5570   1.1  jmcneill #if 0
   5571   1.1  jmcneill static int
   5572   1.1  jmcneill mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
   5573   1.1  jmcneill {
   5574   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5575   1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_in *in;
   5576   1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_out *out;
   5577   1.1  jmcneill 	int error;
   5578   1.1  jmcneill 
   5579   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5580   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   5581   1.1  jmcneill 
   5582   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5583   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
   5584   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5585   1.1  jmcneill 
   5586   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5587   1.1  jmcneill 
   5588   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5589   1.1  jmcneill 	if (error != 0) {
   5590   1.1  jmcneill 		printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
   5591   1.1  jmcneill 		return (-1);
   5592   1.1  jmcneill 	}
   5593   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5594   1.1  jmcneill 		printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
   5595   1.1  jmcneill 		return (-1);
   5596   1.1  jmcneill 	}
   5597   1.1  jmcneill 
   5598   1.1  jmcneill 	out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
   5599   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5600   1.1  jmcneill 		printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
   5601   1.1  jmcneill 		    out->cmd_status);
   5602   1.1  jmcneill 		return (-1);
   5603   1.1  jmcneill 	}
   5604   1.1  jmcneill 
   5605   1.1  jmcneill 	sc->sc_flow_counter_id[i]  = be16toh(out->cmd_flow_counter_id);
   5606   1.1  jmcneill 	printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
   5607   1.1  jmcneill 
   5608   1.1  jmcneill 	return (0);
   5609   1.1  jmcneill }
   5610   1.1  jmcneill #endif
   5611   1.1  jmcneill 
   5612   1.1  jmcneill static int
   5613  1.15  jmcneill mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level,
   5614  1.15  jmcneill     int *flow_table_id)
   5615   1.1  jmcneill {
   5616   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5617   1.1  jmcneill 	struct mcx_dmamem mxm;
   5618   1.1  jmcneill 	struct mcx_cmd_create_flow_table_in *in;
   5619   1.1  jmcneill 	struct mcx_cmd_create_flow_table_mb_in *mbin;
   5620   1.1  jmcneill 	struct mcx_cmd_create_flow_table_out *out;
   5621   1.1  jmcneill 	int error;
   5622   1.1  jmcneill 	int token;
   5623   1.1  jmcneill 
   5624   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5625   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5626  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5627  1.15  jmcneill 	    sizeof(*out), token);
   5628   1.1  jmcneill 
   5629   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5630   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
   5631   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5632   1.1  jmcneill 
   5633  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5634  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5635   1.1  jmcneill 		printf("%s: unable to allocate create flow table mailbox\n",
   5636   1.1  jmcneill 		    DEVNAME(sc));
   5637   1.1  jmcneill 		return (-1);
   5638   1.1  jmcneill 	}
   5639   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5640   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5641   1.1  jmcneill 	mbin->cmd_ctx.ft_log_size = log_size;
   5642  1.15  jmcneill 	mbin->cmd_ctx.ft_level = level;
   5643   1.1  jmcneill 
   5644   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5645   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5646   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5647   1.1  jmcneill 	if (error != 0) {
   5648   1.1  jmcneill 		printf("%s: create flow table timeout\n", DEVNAME(sc));
   5649   1.1  jmcneill 		goto free;
   5650   1.1  jmcneill 	}
   5651   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5652   1.1  jmcneill 		printf("%s: create flow table command corrupt\n", DEVNAME(sc));
   5653   1.1  jmcneill 		goto free;
   5654   1.1  jmcneill 	}
   5655   1.1  jmcneill 
   5656   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5657   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5658   1.1  jmcneill 		printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
   5659   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5660   1.1  jmcneill 		error = -1;
   5661   1.1  jmcneill 		goto free;
   5662   1.1  jmcneill 	}
   5663   1.1  jmcneill 
   5664  1.15  jmcneill 	*flow_table_id = mcx_get_id(out->cmd_table_id);
   5665   1.1  jmcneill free:
   5666   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5667   1.1  jmcneill 	return (error);
   5668   1.1  jmcneill }
   5669   1.1  jmcneill 
   5670   1.1  jmcneill static int
   5671  1.15  jmcneill mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id)
   5672   1.1  jmcneill {
   5673   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5674   1.1  jmcneill 	struct mcx_dmamem mxm;
   5675   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_in *in;
   5676   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_mb_in *mbin;
   5677   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_out *out;
   5678   1.1  jmcneill 	int error;
   5679   1.1  jmcneill 	int token;
   5680   1.1  jmcneill 
   5681   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5682   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5683  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5684  1.15  jmcneill 	    sizeof(*out), token);
   5685   1.1  jmcneill 
   5686   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5687   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
   5688   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5689   1.1  jmcneill 
   5690  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5691  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5692   1.1  jmcneill 		printf("%s: unable to allocate set flow table root mailbox\n",
   5693   1.1  jmcneill 		    DEVNAME(sc));
   5694   1.1  jmcneill 		return (-1);
   5695   1.1  jmcneill 	}
   5696   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5697   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5698  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   5699   1.1  jmcneill 
   5700   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5701   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5702   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5703   1.1  jmcneill 	if (error != 0) {
   5704   1.1  jmcneill 		printf("%s: set flow table root timeout\n", DEVNAME(sc));
   5705   1.1  jmcneill 		goto free;
   5706   1.1  jmcneill 	}
   5707   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5708   1.1  jmcneill 		printf("%s: set flow table root command corrupt\n",
   5709   1.1  jmcneill 		    DEVNAME(sc));
   5710   1.1  jmcneill 		goto free;
   5711   1.1  jmcneill 	}
   5712   1.1  jmcneill 
   5713   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5714   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5715   1.1  jmcneill 		printf("%s: set flow table root failed (%x, %x)\n",
   5716   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5717   1.1  jmcneill 		error = -1;
   5718   1.1  jmcneill 		goto free;
   5719   1.1  jmcneill 	}
   5720   1.1  jmcneill 
   5721   1.1  jmcneill free:
   5722   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5723   1.1  jmcneill 	return (error);
   5724   1.1  jmcneill }
   5725   1.1  jmcneill 
   5726   1.1  jmcneill static int
   5727  1.15  jmcneill mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id)
   5728   1.1  jmcneill {
   5729   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5730   1.1  jmcneill 	struct mcx_dmamem mxm;
   5731   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_in *in;
   5732   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_mb_in *mb;
   5733   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_out *out;
   5734   1.1  jmcneill 	int error;
   5735   1.1  jmcneill 	int token;
   5736   1.1  jmcneill 
   5737   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5738   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5739   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   5740   1.1  jmcneill 
   5741   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5742   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
   5743   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5744   1.1  jmcneill 
   5745  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5746  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5747   1.1  jmcneill 		printf("%s: unable to allocate destroy flow table mailbox\n",
   5748   1.1  jmcneill 		    DEVNAME(sc));
   5749   1.1  jmcneill 		return (-1);
   5750   1.1  jmcneill 	}
   5751   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5752   1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5753  1.15  jmcneill 	mb->cmd_table_id = htobe32(flow_table_id);
   5754   1.1  jmcneill 
   5755   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5756   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5757   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5758   1.1  jmcneill 	if (error != 0) {
   5759   1.1  jmcneill 		printf("%s: destroy flow table timeout\n", DEVNAME(sc));
   5760   1.1  jmcneill 		goto free;
   5761   1.1  jmcneill 	}
   5762   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5763  1.15  jmcneill 		printf("%s: destroy flow table command corrupt\n",
   5764  1.15  jmcneill 		    DEVNAME(sc));
   5765   1.1  jmcneill 		goto free;
   5766   1.1  jmcneill 	}
   5767   1.1  jmcneill 
   5768   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5769   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5770   1.1  jmcneill 		printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
   5771   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5772   1.1  jmcneill 		error = -1;
   5773   1.1  jmcneill 		goto free;
   5774   1.1  jmcneill 	}
   5775   1.1  jmcneill 
   5776   1.1  jmcneill free:
   5777   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5778   1.1  jmcneill 	return (error);
   5779   1.1  jmcneill }
   5780   1.1  jmcneill 
   5781   1.1  jmcneill 
   5782   1.1  jmcneill static int
   5783  1.15  jmcneill mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group,
   5784  1.15  jmcneill     int start, int size, int match_enable, struct mcx_flow_match *match)
   5785   1.1  jmcneill {
   5786   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5787   1.1  jmcneill 	struct mcx_dmamem mxm;
   5788   1.1  jmcneill 	struct mcx_cmd_create_flow_group_in *in;
   5789   1.1  jmcneill 	struct mcx_cmd_create_flow_group_mb_in *mbin;
   5790   1.1  jmcneill 	struct mcx_cmd_create_flow_group_out *out;
   5791  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5792   1.1  jmcneill 	int error;
   5793   1.1  jmcneill 	int token;
   5794   1.1  jmcneill 
   5795   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5796   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5797   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   5798   1.1  jmcneill 	    token);
   5799   1.1  jmcneill 
   5800   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5801   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
   5802   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5803   1.1  jmcneill 
   5804   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   5805   1.1  jmcneill 	    != 0) {
   5806   1.1  jmcneill 		printf("%s: unable to allocate create flow group mailbox\n",
   5807   1.1  jmcneill 		    DEVNAME(sc));
   5808   1.1  jmcneill 		return (-1);
   5809   1.1  jmcneill 	}
   5810   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5811   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5812  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   5813   1.1  jmcneill 	mbin->cmd_start_flow_index = htobe32(start);
   5814   1.1  jmcneill 	mbin->cmd_end_flow_index = htobe32(start + (size - 1));
   5815   1.1  jmcneill 
   5816   1.1  jmcneill 	mbin->cmd_match_criteria_enable = match_enable;
   5817   1.1  jmcneill 	memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));
   5818   1.1  jmcneill 
   5819   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5820   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5821   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5822   1.1  jmcneill 	if (error != 0) {
   5823   1.1  jmcneill 		printf("%s: create flow group timeout\n", DEVNAME(sc));
   5824   1.1  jmcneill 		goto free;
   5825   1.1  jmcneill 	}
   5826   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5827   1.1  jmcneill 		printf("%s: create flow group command corrupt\n", DEVNAME(sc));
   5828   1.1  jmcneill 		goto free;
   5829   1.1  jmcneill 	}
   5830   1.1  jmcneill 
   5831   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5832   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5833   1.1  jmcneill 		printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
   5834   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5835   1.1  jmcneill 		error = -1;
   5836   1.1  jmcneill 		goto free;
   5837   1.1  jmcneill 	}
   5838   1.1  jmcneill 
   5839  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5840  1.15  jmcneill 	mfg->g_id = mcx_get_id(out->cmd_group_id);
   5841  1.15  jmcneill 	mfg->g_table = flow_table_id;
   5842  1.15  jmcneill 	mfg->g_start = start;
   5843  1.15  jmcneill 	mfg->g_size = size;
   5844   1.1  jmcneill 
   5845   1.1  jmcneill free:
   5846   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5847   1.1  jmcneill 	return (error);
   5848   1.1  jmcneill }
   5849   1.1  jmcneill 
   5850   1.1  jmcneill static int
   5851   1.1  jmcneill mcx_destroy_flow_group(struct mcx_softc *sc, int group)
   5852   1.1  jmcneill {
   5853   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5854   1.1  jmcneill 	struct mcx_dmamem mxm;
   5855   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_in *in;
   5856   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_mb_in *mb;
   5857   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_out *out;
   5858  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5859   1.1  jmcneill 	int error;
   5860   1.1  jmcneill 	int token;
   5861   1.1  jmcneill 
   5862   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5863   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5864   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   5865   1.1  jmcneill 
   5866   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5867   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
   5868   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5869   1.1  jmcneill 
   5870  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5871  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5872   1.1  jmcneill 		printf("%s: unable to allocate destroy flow group mailbox\n",
   5873   1.1  jmcneill 		    DEVNAME(sc));
   5874   1.1  jmcneill 		return (-1);
   5875   1.1  jmcneill 	}
   5876   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5877   1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5878  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5879  1.15  jmcneill 	mb->cmd_table_id = htobe32(mfg->g_table);
   5880  1.15  jmcneill 	mb->cmd_group_id = htobe32(mfg->g_id);
   5881   1.1  jmcneill 
   5882   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5883   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5884   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5885   1.1  jmcneill 	if (error != 0) {
   5886   1.1  jmcneill 		printf("%s: destroy flow group timeout\n", DEVNAME(sc));
   5887   1.1  jmcneill 		goto free;
   5888   1.1  jmcneill 	}
   5889   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5890   1.1  jmcneill 		printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
   5891   1.1  jmcneill 		goto free;
   5892   1.1  jmcneill 	}
   5893   1.1  jmcneill 
   5894   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5895   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5896   1.1  jmcneill 		printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
   5897   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5898   1.1  jmcneill 		error = -1;
   5899   1.1  jmcneill 		goto free;
   5900   1.1  jmcneill 	}
   5901   1.1  jmcneill 
   5902  1.15  jmcneill 	mfg->g_id = -1;
   5903  1.15  jmcneill 	mfg->g_table = -1;
   5904  1.15  jmcneill 	mfg->g_size = 0;
   5905  1.15  jmcneill 	mfg->g_start = 0;
   5906   1.1  jmcneill free:
   5907   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5908   1.1  jmcneill 	return (error);
   5909   1.1  jmcneill }
   5910   1.1  jmcneill 
   5911   1.1  jmcneill static int
   5912  1.15  jmcneill mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index,
   5913  1.15  jmcneill     const uint8_t *macaddr, uint32_t dest)
   5914   1.1  jmcneill {
   5915   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5916   1.1  jmcneill 	struct mcx_dmamem mxm;
   5917   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_in *in;
   5918   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
   5919   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_out *out;
   5920  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5921  1.15  jmcneill 	uint32_t *pdest;
   5922   1.1  jmcneill 	int error;
   5923   1.1  jmcneill 	int token;
   5924   1.1  jmcneill 
   5925   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5926   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5927  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
   5928   1.1  jmcneill 	    sizeof(*out), token);
   5929   1.1  jmcneill 
   5930   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5931   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
   5932   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5933   1.1  jmcneill 
   5934   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   5935   1.1  jmcneill 	    != 0) {
   5936   1.1  jmcneill 		printf("%s: unable to allocate set flow table entry mailbox\n",
   5937   1.1  jmcneill 		    DEVNAME(sc));
   5938   1.1  jmcneill 		return (-1);
   5939   1.1  jmcneill 	}
   5940  1.15  jmcneill 
   5941   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5942   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5943  1.15  jmcneill 
   5944  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5945  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   5946  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   5947  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
   5948   1.1  jmcneill 
   5949   1.1  jmcneill 	/* flow context ends at offset 0x330, 0x130 into the second mbox */
   5950  1.15  jmcneill 	pdest = (uint32_t *)
   5951   1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
   5952   1.1  jmcneill 	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
   5953   1.1  jmcneill 	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
   5954  1.15  jmcneill 	*pdest = htobe32(dest);
   5955   1.1  jmcneill 
   5956   1.1  jmcneill 	/* the only thing we match on at the moment is the dest mac address */
   5957   1.1  jmcneill 	if (macaddr != NULL) {
   5958   1.1  jmcneill 		memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
   5959   1.1  jmcneill 		    ETHER_ADDR_LEN);
   5960   1.1  jmcneill 	}
   5961   1.1  jmcneill 
   5962   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5963   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5964   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5965   1.1  jmcneill 	if (error != 0) {
   5966   1.1  jmcneill 		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
   5967   1.1  jmcneill 		goto free;
   5968   1.1  jmcneill 	}
   5969   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5970   1.1  jmcneill 		printf("%s: set flow table entry command corrupt\n",
   5971   1.1  jmcneill 		    DEVNAME(sc));
   5972   1.1  jmcneill 		goto free;
   5973   1.1  jmcneill 	}
   5974   1.1  jmcneill 
   5975   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5976   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5977   1.1  jmcneill 		printf("%s: set flow table entry failed (%x, %x)\n",
   5978   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5979   1.1  jmcneill 		error = -1;
   5980   1.1  jmcneill 		goto free;
   5981   1.1  jmcneill 	}
   5982   1.1  jmcneill 
   5983   1.1  jmcneill free:
   5984   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5985   1.1  jmcneill 	return (error);
   5986   1.1  jmcneill }
   5987   1.1  jmcneill 
   5988   1.1  jmcneill static int
   5989  1.15  jmcneill mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index,
   5990  1.15  jmcneill     int ethertype, int ip_proto, uint32_t dest)
   5991   1.1  jmcneill {
   5992   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5993   1.1  jmcneill 	struct mcx_dmamem mxm;
   5994  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_in *in;
   5995  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
   5996  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_out *out;
   5997  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5998  1.15  jmcneill 	uint32_t *pdest;
   5999   1.1  jmcneill 	int error;
   6000   1.1  jmcneill 	int token;
   6001   1.1  jmcneill 
   6002   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6003   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   6004  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
   6005  1.15  jmcneill 	    sizeof(*out), token);
   6006   1.1  jmcneill 
   6007   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6008  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
   6009   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6010   1.1  jmcneill 
   6011  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   6012  1.15  jmcneill 	    != 0) {
   6013  1.15  jmcneill 		printf("%s: unable to allocate set flow table entry mailbox\n",
   6014   1.1  jmcneill 		    DEVNAME(sc));
   6015   1.1  jmcneill 		return (-1);
   6016   1.1  jmcneill 	}
   6017  1.15  jmcneill 
   6018   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6019   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   6020  1.15  jmcneill 
   6021  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   6022  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   6023  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   6024  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
   6025  1.15  jmcneill 
   6026  1.15  jmcneill 	/* flow context ends at offset 0x330, 0x130 into the second mbox */
   6027  1.15  jmcneill 	pdest = (uint32_t *)
   6028  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
   6029  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
   6030  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
   6031  1.15  jmcneill 	*pdest = htobe32(dest);
   6032  1.15  jmcneill 
   6033  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype);
   6034  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto;
   6035   1.1  jmcneill 
   6036   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   6037   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6038   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6039   1.1  jmcneill 	if (error != 0) {
   6040  1.15  jmcneill 		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
   6041   1.1  jmcneill 		goto free;
   6042   1.1  jmcneill 	}
   6043   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6044  1.15  jmcneill 		printf("%s: set flow table entry command corrupt\n",
   6045   1.1  jmcneill 		    DEVNAME(sc));
   6046   1.1  jmcneill 		goto free;
   6047   1.1  jmcneill 	}
   6048   1.1  jmcneill 
   6049   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6050   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6051  1.15  jmcneill 		printf("%s: set flow table entry failed (%x, %x)\n",
   6052  1.15  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   6053   1.1  jmcneill 		error = -1;
   6054   1.1  jmcneill 		goto free;
   6055   1.1  jmcneill 	}
   6056   1.1  jmcneill 
   6057   1.1  jmcneill free:
   6058   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6059   1.1  jmcneill 	return (error);
   6060   1.1  jmcneill }
   6061   1.1  jmcneill 
   6062  1.15  jmcneill static int
   6063  1.15  jmcneill mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
   6064   1.1  jmcneill {
   6065  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6066   1.1  jmcneill 	struct mcx_dmamem mxm;
   6067  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_in *in;
   6068  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
   6069  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_out *out;
   6070  1.15  jmcneill 	struct mcx_flow_group *mfg;
   6071   1.1  jmcneill 	int error;
   6072  1.15  jmcneill 	int token;
   6073  1.15  jmcneill 
   6074  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6075  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6076  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   6077  1.15  jmcneill 	    token);
   6078  1.15  jmcneill 
   6079  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6080  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
   6081  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6082  1.15  jmcneill 
   6083  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6084  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   6085  1.15  jmcneill 		printf("%s: unable to allocate "
   6086  1.15  jmcneill 		    "delete flow table entry mailbox\n", DEVNAME(sc));
   6087  1.15  jmcneill 		return (-1);
   6088  1.15  jmcneill 	}
   6089  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6090  1.15  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   6091  1.15  jmcneill 
   6092  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   6093  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   6094  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   6095  1.15  jmcneill 
   6096  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   6097  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6098  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6099  1.15  jmcneill 	if (error != 0) {
   6100  1.15  jmcneill 		printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
   6101  1.15  jmcneill 		goto free;
   6102  1.15  jmcneill 	}
   6103  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6104  1.15  jmcneill 		printf("%s: delete flow table entry command corrupt\n",
   6105  1.15  jmcneill 		    DEVNAME(sc));
   6106  1.15  jmcneill 		goto free;
   6107  1.15  jmcneill 	}
   6108  1.15  jmcneill 
   6109  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6110  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6111  1.15  jmcneill 		printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
   6112  1.15  jmcneill 		    DEVNAME(sc), group, index, out->cmd_status,
   6113  1.15  jmcneill 		    be32toh(out->cmd_syndrome));
   6114  1.15  jmcneill 		error = -1;
   6115  1.15  jmcneill 		goto free;
   6116  1.15  jmcneill 	}
   6117  1.15  jmcneill 
   6118  1.15  jmcneill free:
   6119  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6120  1.15  jmcneill 	return (error);
   6121  1.15  jmcneill }
   6122  1.15  jmcneill 
   6123  1.15  jmcneill #if 0
   6124  1.15  jmcneill int
   6125  1.15  jmcneill mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id)
   6126  1.15  jmcneill {
   6127  1.15  jmcneill 	struct mcx_dmamem mxm;
   6128  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6129  1.15  jmcneill 	struct mcx_cmd_query_flow_table_in *in;
   6130  1.15  jmcneill 	struct mcx_cmd_query_flow_table_mb_in *mbin;
   6131  1.15  jmcneill 	struct mcx_cmd_query_flow_table_out *out;
   6132  1.15  jmcneill 	struct mcx_cmd_query_flow_table_mb_out *mbout;
   6133  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6134  1.15  jmcneill 	int error;
   6135  1.15  jmcneill 	int i;
   6136  1.15  jmcneill 	uint8_t *dump;
   6137   1.1  jmcneill 
   6138   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6139   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6140   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6141   1.1  jmcneill 
   6142   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6143   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
   6144   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6145   1.1  jmcneill 
   6146   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6147   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6148   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6149   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6150   1.1  jmcneill 		printf(", unable to allocate query flow table mailboxes\n");
   6151   1.1  jmcneill 		return (-1);
   6152   1.1  jmcneill 	}
   6153   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6154   1.1  jmcneill 
   6155   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6156   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6157  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6158   1.1  jmcneill 
   6159   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6160   1.1  jmcneill 
   6161   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6162   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6163   1.1  jmcneill 	if (error != 0) {
   6164   1.1  jmcneill 		printf("%s: query flow table timeout\n", DEVNAME(sc));
   6165   1.1  jmcneill 		goto free;
   6166   1.1  jmcneill 	}
   6167   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6168   1.1  jmcneill 	if (error != 0) {
   6169   1.1  jmcneill 		printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
   6170   1.1  jmcneill 		goto free;
   6171   1.1  jmcneill 	}
   6172   1.1  jmcneill 
   6173   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6174   1.1  jmcneill 	switch (out->cmd_status) {
   6175   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6176   1.1  jmcneill 		break;
   6177   1.1  jmcneill 	default:
   6178   1.1  jmcneill 		printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
   6179   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6180   1.1  jmcneill 		error = -1;
   6181   1.1  jmcneill 		goto free;
   6182   1.1  jmcneill 	}
   6183   1.1  jmcneill 
   6184   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_mb_out *)
   6185   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6186   1.1  jmcneill 	dump = (uint8_t *)mbout + 8;
   6187   1.1  jmcneill 	for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
   6188   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6189   1.1  jmcneill 		if (i % 16 == 15)
   6190   1.1  jmcneill 			printf("\n");
   6191   1.1  jmcneill 	}
   6192   1.1  jmcneill free:
   6193   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6194   1.1  jmcneill 	return (error);
   6195   1.1  jmcneill }
   6196   1.1  jmcneill int
   6197  1.15  jmcneill mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index)
   6198   1.1  jmcneill {
   6199   1.1  jmcneill 	struct mcx_dmamem mxm;
   6200   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6201   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_in *in;
   6202   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
   6203   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_out *out;
   6204   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
   6205   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6206   1.1  jmcneill 	int error;
   6207   1.1  jmcneill 	int i;
   6208   1.1  jmcneill 	uint8_t *dump;
   6209   1.1  jmcneill 
   6210   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6211   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6212   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6213   1.1  jmcneill 
   6214   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6215   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
   6216   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6217   1.1  jmcneill 
   6218   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6219   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6220   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6221   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6222  1.15  jmcneill 		printf(", unable to allocate "
   6223  1.15  jmcneill 		    "query flow table entry mailboxes\n");
   6224   1.1  jmcneill 		return (-1);
   6225   1.1  jmcneill 	}
   6226   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6227   1.1  jmcneill 
   6228   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6229   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6230  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6231   1.1  jmcneill 	mbin->cmd_flow_index = htobe32(index);
   6232   1.1  jmcneill 
   6233   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6234   1.1  jmcneill 
   6235   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6236   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6237   1.1  jmcneill 	if (error != 0) {
   6238   1.1  jmcneill 		printf("%s: query flow table entry timeout\n", DEVNAME(sc));
   6239   1.1  jmcneill 		goto free;
   6240   1.1  jmcneill 	}
   6241   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6242   1.1  jmcneill 	if (error != 0) {
   6243   1.1  jmcneill 		printf("%s: query flow table entry reply corrupt\n",
   6244   1.1  jmcneill 		    DEVNAME(sc));
   6245   1.1  jmcneill 		goto free;
   6246   1.1  jmcneill 	}
   6247   1.1  jmcneill 
   6248   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6249   1.1  jmcneill 	switch (out->cmd_status) {
   6250   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6251   1.1  jmcneill 		break;
   6252   1.1  jmcneill 	default:
   6253   1.1  jmcneill 		printf("%s: query flow table entry failed (%x/%x)\n",
   6254   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   6255   1.1  jmcneill 		error = -1;
   6256   1.1  jmcneill 		goto free;
   6257   1.1  jmcneill 	}
   6258   1.1  jmcneill 
   6259   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
   6260   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6261   1.1  jmcneill 	dump = (uint8_t *)mbout;
   6262   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6263   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6264   1.1  jmcneill 		if (i % 16 == 15)
   6265   1.1  jmcneill 			printf("\n");
   6266   1.1  jmcneill 	}
   6267   1.1  jmcneill 
   6268   1.1  jmcneill free:
   6269   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6270   1.1  jmcneill 	return (error);
   6271   1.1  jmcneill }
   6272   1.1  jmcneill 
   6273   1.1  jmcneill int
   6274  1.15  jmcneill mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id)
   6275   1.1  jmcneill {
   6276   1.1  jmcneill 	struct mcx_dmamem mxm;
   6277   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6278   1.1  jmcneill 	struct mcx_cmd_query_flow_group_in *in;
   6279   1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_in *mbin;
   6280   1.1  jmcneill 	struct mcx_cmd_query_flow_group_out *out;
   6281   1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_out *mbout;
   6282   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6283   1.1  jmcneill 	int error;
   6284   1.1  jmcneill 	int i;
   6285   1.1  jmcneill 	uint8_t *dump;
   6286   1.1  jmcneill 
   6287   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6288   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6289   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6290   1.1  jmcneill 
   6291   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6292   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
   6293   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6294   1.1  jmcneill 
   6295   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6296   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6297   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6298   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6299   1.1  jmcneill 		printf(", unable to allocate query flow group mailboxes\n");
   6300   1.1  jmcneill 		return (-1);
   6301   1.1  jmcneill 	}
   6302   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6303   1.1  jmcneill 
   6304   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6305   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6306  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6307   1.1  jmcneill 	mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
   6308   1.1  jmcneill 
   6309   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6310   1.1  jmcneill 
   6311   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6312   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6313   1.1  jmcneill 	if (error != 0) {
   6314   1.1  jmcneill 		printf("%s: query flow group timeout\n", DEVNAME(sc));
   6315   1.1  jmcneill 		goto free;
   6316   1.1  jmcneill 	}
   6317   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6318   1.1  jmcneill 	if (error != 0) {
   6319   1.1  jmcneill 		printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
   6320   1.1  jmcneill 		goto free;
   6321   1.1  jmcneill 	}
   6322   1.1  jmcneill 
   6323   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6324   1.1  jmcneill 	switch (out->cmd_status) {
   6325   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6326   1.1  jmcneill 		break;
   6327   1.1  jmcneill 	default:
   6328   1.1  jmcneill 		printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
   6329   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6330   1.1  jmcneill 		error = -1;
   6331   1.1  jmcneill 		goto free;
   6332   1.1  jmcneill 	}
   6333   1.1  jmcneill 
   6334   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_group_mb_out *)
   6335   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6336   1.1  jmcneill 	dump = (uint8_t *)mbout;
   6337   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6338   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6339   1.1  jmcneill 		if (i % 16 == 15)
   6340   1.1  jmcneill 			printf("\n");
   6341   1.1  jmcneill 	}
   6342   1.1  jmcneill 	dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
   6343   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6344   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6345   1.1  jmcneill 		if (i % 16 == 15)
   6346   1.1  jmcneill 			printf("\n");
   6347   1.1  jmcneill 	}
   6348   1.1  jmcneill 
   6349   1.1  jmcneill free:
   6350   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6351   1.1  jmcneill 	return (error);
   6352   1.1  jmcneill }
   6353   1.1  jmcneill 
   6354  1.15  jmcneill static int
   6355  1.15  jmcneill mcx_dump_counters(struct mcx_softc *sc)
   6356  1.15  jmcneill {
   6357  1.15  jmcneill 	struct mcx_dmamem mxm;
   6358  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6359  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_in *in;
   6360  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_mb_in *mbin;
   6361  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_out *out;
   6362  1.15  jmcneill 	struct mcx_nic_vport_counters *counters;
   6363  1.15  jmcneill 	int error, token;
   6364  1.15  jmcneill 
   6365  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6366  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6367  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6368  1.15  jmcneill 	    sizeof(*out) + sizeof(*counters), token);
   6369  1.15  jmcneill 
   6370  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6371  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
   6372  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6373  1.15  jmcneill 
   6374  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   6375  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6376  1.15  jmcneill 		printf(", unable to allocate "
   6377  1.15  jmcneill 		    "query nic vport counters mailboxen\n");
   6378  1.15  jmcneill 		return (-1);
   6379  1.15  jmcneill 	}
   6380  1.15  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6381  1.15  jmcneill 
   6382  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6383  1.15  jmcneill 	mbin->cmd_clear = 0x80;
   6384  1.15  jmcneill 
   6385  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6386  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6387  1.15  jmcneill 
   6388  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6389  1.15  jmcneill 	if (error != 0) {
   6390  1.15  jmcneill 		printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
   6391  1.15  jmcneill 		goto free;
   6392  1.15  jmcneill 	}
   6393  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6394  1.15  jmcneill 		printf("%s: query nic vport counters command corrupt\n",
   6395  1.15  jmcneill 		    DEVNAME(sc));
   6396  1.15  jmcneill 		goto free;
   6397  1.15  jmcneill 	}
   6398  1.15  jmcneill 
   6399  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6400  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6401  1.15  jmcneill 		printf("%s: query nic vport counters failed (%x, %x)\n",
   6402  1.15  jmcneill 		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
   6403  1.15  jmcneill 		error = -1;
   6404  1.15  jmcneill 		goto free;
   6405  1.15  jmcneill 	}
   6406  1.15  jmcneill 
   6407  1.15  jmcneill 	counters = (struct mcx_nic_vport_counters *)
   6408  1.15  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6409  1.15  jmcneill 	if (counters->rx_bcast.packets + counters->tx_bcast.packets +
   6410  1.15  jmcneill 	    counters->rx_ucast.packets + counters->tx_ucast.packets +
   6411  1.15  jmcneill 	    counters->rx_err.packets + counters->tx_err.packets)
   6412  1.15  jmcneill 		printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
   6413  1.15  jmcneill 		    DEVNAME(sc),
   6414  1.15  jmcneill 		    betoh64(counters->tx_err.packets),
   6415  1.15  jmcneill 		    betoh64(counters->rx_err.packets),
   6416  1.15  jmcneill 		    betoh64(counters->tx_ucast.packets),
   6417  1.15  jmcneill 		    betoh64(counters->rx_ucast.packets),
   6418  1.15  jmcneill 		    betoh64(counters->tx_bcast.packets),
   6419  1.15  jmcneill 		    betoh64(counters->rx_bcast.packets));
   6420  1.15  jmcneill free:
   6421  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6422  1.15  jmcneill 
   6423  1.15  jmcneill 	return (error);
   6424  1.15  jmcneill }
   6425  1.15  jmcneill 
   6426  1.15  jmcneill static int
   6427  1.15  jmcneill mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
   6428  1.15  jmcneill {
   6429  1.15  jmcneill 	struct mcx_dmamem mxm;
   6430  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6431  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_in *in;
   6432  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_mb_in *mbin;
   6433  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_out *out;
   6434  1.15  jmcneill 	struct mcx_counter *counters;
   6435  1.15  jmcneill 	int error, token;
   6436  1.15  jmcneill 
   6437  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6438  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6439  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
   6440  1.15  jmcneill 	    sizeof(*counters), token);
   6441  1.15  jmcneill 
   6442  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6443  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
   6444  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6445  1.15  jmcneill 
   6446  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   6447  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6448  1.15  jmcneill 		printf(", unable to allocate query flow counter mailboxen\n");
   6449  1.15  jmcneill 		return (-1);
   6450  1.15  jmcneill 	}
   6451  1.15  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6452  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6453  1.15  jmcneill 	mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
   6454  1.15  jmcneill 	mbin->cmd_clear = 0x80;
   6455  1.15  jmcneill 
   6456  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6457  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6458  1.15  jmcneill 
   6459  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6460  1.15  jmcneill 	if (error != 0) {
   6461  1.15  jmcneill 		printf("%s: query flow counter timeout\n", DEVNAME(sc));
   6462  1.15  jmcneill 		goto free;
   6463  1.15  jmcneill 	}
   6464  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6465  1.15  jmcneill 		printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
   6466  1.15  jmcneill 		goto free;
   6467  1.15  jmcneill 	}
   6468  1.15  jmcneill 
   6469  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6470  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6471  1.15  jmcneill 		printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
   6472  1.15  jmcneill 		    out->cmd_status, betoh32(out->cmd_syndrome));
   6473  1.15  jmcneill 		error = -1;
   6474  1.15  jmcneill 		goto free;
   6475  1.15  jmcneill 	}
   6476  1.15  jmcneill 
   6477  1.15  jmcneill 	counters = (struct mcx_counter *)
   6478  1.15  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6479  1.15  jmcneill 	if (counters->packets)
   6480  1.15  jmcneill 		printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
   6481  1.15  jmcneill 		    betoh64(counters->packets));
   6482  1.15  jmcneill free:
   6483  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6484  1.15  jmcneill 
   6485  1.15  jmcneill 	return (error);
   6486  1.15  jmcneill }
   6487  1.15  jmcneill 
   6488  1.15  jmcneill #endif
   6489  1.15  jmcneill 
   6490  1.15  jmcneill #if NKSTAT > 0
   6491  1.15  jmcneill 
   6492   1.1  jmcneill int
   6493  1.15  jmcneill mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx)
   6494   1.1  jmcneill {
   6495   1.1  jmcneill 	struct mcx_dmamem mxm;
   6496   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6497   1.1  jmcneill 	struct mcx_cmd_query_rq_in *in;
   6498   1.1  jmcneill 	struct mcx_cmd_query_rq_out *out;
   6499   1.1  jmcneill 	struct mcx_cmd_query_rq_mb_out *mbout;
   6500   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6501   1.1  jmcneill 	int error;
   6502   1.1  jmcneill 
   6503   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6504   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   6505   1.1  jmcneill 	    token);
   6506   1.1  jmcneill 
   6507   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6508   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
   6509   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6510  1.15  jmcneill 	in->cmd_rqn = htobe32(rx->rx_rqn);
   6511   1.1  jmcneill 
   6512   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6513   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6514   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6515  1.15  jmcneill 		printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc));
   6516   1.1  jmcneill 		return (-1);
   6517   1.1  jmcneill 	}
   6518   1.1  jmcneill 
   6519   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6520   1.1  jmcneill 
   6521   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6522   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6523   1.1  jmcneill 	if (error != 0) {
   6524   1.1  jmcneill 		printf("%s: query rq timeout\n", DEVNAME(sc));
   6525   1.1  jmcneill 		goto free;
   6526   1.1  jmcneill 	}
   6527   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6528   1.1  jmcneill 	if (error != 0) {
   6529   1.1  jmcneill 		printf("%s: query rq reply corrupt\n", DEVNAME(sc));
   6530   1.1  jmcneill 		goto free;
   6531   1.1  jmcneill 	}
   6532   1.1  jmcneill 
   6533   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6534   1.1  jmcneill 	switch (out->cmd_status) {
   6535   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6536   1.1  jmcneill 		break;
   6537   1.1  jmcneill 	default:
   6538   1.1  jmcneill 		printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
   6539   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6540   1.1  jmcneill 		error = -1;
   6541   1.1  jmcneill 		goto free;
   6542   1.1  jmcneill 	}
   6543   1.1  jmcneill 
   6544   1.1  jmcneill         mbout = (struct mcx_cmd_query_rq_mb_out *)
   6545   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6546  1.15  jmcneill 	memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx));
   6547   1.1  jmcneill 
   6548   1.1  jmcneill free:
   6549   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6550   1.1  jmcneill 	return (error);
   6551   1.1  jmcneill }
   6552   1.1  jmcneill 
   6553   1.1  jmcneill int
   6554  1.15  jmcneill mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx)
   6555   1.1  jmcneill {
   6556   1.1  jmcneill 	struct mcx_dmamem mxm;
   6557   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6558   1.1  jmcneill 	struct mcx_cmd_query_sq_in *in;
   6559   1.1  jmcneill 	struct mcx_cmd_query_sq_out *out;
   6560   1.1  jmcneill 	struct mcx_cmd_query_sq_mb_out *mbout;
   6561   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6562   1.1  jmcneill 	int error;
   6563   1.1  jmcneill 
   6564   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6565   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   6566   1.1  jmcneill 	    token);
   6567   1.1  jmcneill 
   6568   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6569   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
   6570   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6571  1.15  jmcneill 	in->cmd_sqn = htobe32(tx->tx_sqn);
   6572   1.1  jmcneill 
   6573   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6574   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6575   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6576  1.15  jmcneill 		printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc));
   6577   1.1  jmcneill 		return (-1);
   6578   1.1  jmcneill 	}
   6579   1.1  jmcneill 
   6580   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6581   1.1  jmcneill 
   6582   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6583   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6584   1.1  jmcneill 	if (error != 0) {
   6585   1.1  jmcneill 		printf("%s: query sq timeout\n", DEVNAME(sc));
   6586   1.1  jmcneill 		goto free;
   6587   1.1  jmcneill 	}
   6588   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6589   1.1  jmcneill 	if (error != 0) {
   6590   1.1  jmcneill 		printf("%s: query sq reply corrupt\n", DEVNAME(sc));
   6591   1.1  jmcneill 		goto free;
   6592   1.1  jmcneill 	}
   6593   1.1  jmcneill 
   6594   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6595   1.1  jmcneill 	switch (out->cmd_status) {
   6596   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6597   1.1  jmcneill 		break;
   6598   1.1  jmcneill 	default:
   6599   1.1  jmcneill 		printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
   6600   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6601   1.1  jmcneill 		error = -1;
   6602   1.1  jmcneill 		goto free;
   6603   1.1  jmcneill 	}
   6604   1.1  jmcneill 
   6605   1.1  jmcneill         mbout = (struct mcx_cmd_query_sq_mb_out *)
   6606   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6607  1.15  jmcneill 	memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx));
   6608   1.1  jmcneill 
   6609   1.1  jmcneill free:
   6610   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6611   1.1  jmcneill 	return (error);
   6612   1.1  jmcneill }
   6613   1.1  jmcneill 
   6614  1.15  jmcneill int
   6615  1.15  jmcneill mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx)
   6616   1.1  jmcneill {
   6617   1.1  jmcneill 	struct mcx_dmamem mxm;
   6618   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6619  1.15  jmcneill 	struct mcx_cmd_query_cq_in *in;
   6620  1.15  jmcneill 	struct mcx_cmd_query_cq_out *out;
   6621  1.15  jmcneill 	struct mcx_cq_ctx *ctx;
   6622  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6623  1.15  jmcneill 	int error;
   6624   1.1  jmcneill 
   6625   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6626  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
   6627  1.15  jmcneill 	    token);
   6628   1.1  jmcneill 
   6629   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6630  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ);
   6631   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6632  1.15  jmcneill 	in->cmd_cqn = htobe32(cq->cq_n);
   6633   1.1  jmcneill 
   6634  1.15  jmcneill 	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6635  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6636  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6637  1.15  jmcneill 		printf("%s: unable to allocate query cq mailboxes\n",
   6638  1.15  jmcneill 		    DEVNAME(sc));
   6639   1.1  jmcneill 		return (-1);
   6640   1.1  jmcneill 	}
   6641   1.1  jmcneill 
   6642  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6643   1.1  jmcneill 
   6644   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6645   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6646   1.1  jmcneill 	if (error != 0) {
   6647  1.15  jmcneill 		printf("%s: query cq timeout\n", DEVNAME(sc));
   6648   1.1  jmcneill 		goto free;
   6649   1.1  jmcneill 	}
   6650   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6651  1.15  jmcneill 		printf("%s: query cq reply corrupt\n", DEVNAME(sc));
   6652   1.1  jmcneill 		goto free;
   6653   1.1  jmcneill 	}
   6654   1.1  jmcneill 
   6655   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6656  1.15  jmcneill 	switch (out->cmd_status) {
   6657  1.15  jmcneill 	case MCX_CQ_STATUS_OK:
   6658  1.15  jmcneill 		break;
   6659  1.15  jmcneill 	default:
   6660  1.15  jmcneill 		printf("%s: query qc failed (%x/%x)\n", DEVNAME(sc),
   6661  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6662   1.1  jmcneill 		error = -1;
   6663   1.1  jmcneill 		goto free;
   6664   1.1  jmcneill 	}
   6665   1.1  jmcneill 
   6666  1.15  jmcneill 	ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6667  1.15  jmcneill 	memcpy(cq_ctx, ctx, sizeof(*cq_ctx));
   6668   1.1  jmcneill free:
   6669   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6670   1.1  jmcneill 	return (error);
   6671   1.1  jmcneill }
   6672   1.1  jmcneill 
   6673  1.15  jmcneill int
   6674  1.15  jmcneill mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx)
   6675   1.1  jmcneill {
   6676   1.1  jmcneill 	struct mcx_dmamem mxm;
   6677   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6678  1.15  jmcneill 	struct mcx_cmd_query_eq_in *in;
   6679  1.15  jmcneill 	struct mcx_cmd_query_eq_out *out;
   6680  1.15  jmcneill 	struct mcq_eq_ctx *ctx;
   6681  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6682  1.15  jmcneill 	int error;
   6683   1.1  jmcneill 
   6684   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6685  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
   6686  1.15  jmcneill 	    token);
   6687   1.1  jmcneill 
   6688   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6689  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ);
   6690   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6691  1.15  jmcneill 	in->cmd_eqn = htobe32(eq->eq_n);
   6692   1.1  jmcneill 
   6693  1.15  jmcneill 	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6694  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6695  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6696  1.15  jmcneill 		printf("%s: unable to allocate query eq mailboxes\n",
   6697  1.15  jmcneill 		    DEVNAME(sc));
   6698   1.1  jmcneill 		return (-1);
   6699   1.1  jmcneill 	}
   6700   1.1  jmcneill 
   6701   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6702  1.15  jmcneill 
   6703   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6704   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6705   1.1  jmcneill 	if (error != 0) {
   6706  1.15  jmcneill 		printf("%s: query eq timeout\n", DEVNAME(sc));
   6707   1.1  jmcneill 		goto free;
   6708   1.1  jmcneill 	}
   6709   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6710  1.15  jmcneill 		printf("%s: query eq reply corrupt\n", DEVNAME(sc));
   6711   1.1  jmcneill 		goto free;
   6712   1.1  jmcneill 	}
   6713   1.1  jmcneill 
   6714   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6715  1.15  jmcneill 	switch (out->cmd_status) {
   6716  1.15  jmcneill 	case MCX_CQ_STATUS_OK:
   6717  1.15  jmcneill 		break;
   6718  1.15  jmcneill 	default:
   6719  1.15  jmcneill 		printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc),
   6720  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6721   1.1  jmcneill 		error = -1;
   6722   1.1  jmcneill 		goto free;
   6723   1.1  jmcneill 	}
   6724   1.1  jmcneill 
   6725  1.15  jmcneill 	ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6726  1.15  jmcneill 	memcpy(eq_ctx, ctx, sizeof(*eq_ctx));
   6727   1.1  jmcneill free:
   6728   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6729   1.1  jmcneill 	return (error);
   6730   1.1  jmcneill }
   6731   1.1  jmcneill 
   6732  1.15  jmcneill #endif /* NKSTAT > 0 */
   6733  1.15  jmcneill 
   6734   1.1  jmcneill 
   6735  1.15  jmcneill static inline unsigned int
   6736  1.15  jmcneill mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots)
   6737   1.1  jmcneill {
   6738  1.15  jmcneill 	struct mcx_rq_entry *ring, *rqe;
   6739   1.1  jmcneill 	struct mcx_slot *ms;
   6740   1.1  jmcneill 	struct mbuf *m;
   6741   1.1  jmcneill 	uint slot, p, fills;
   6742   1.1  jmcneill 
   6743  1.15  jmcneill 	ring = MCX_DMA_KVA(&rx->rx_rq_mem);
   6744  1.15  jmcneill 	p = rx->rx_prod;
   6745  1.15  jmcneill 
   6746  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   6747  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   6748  1.15  jmcneill 
   6749   1.1  jmcneill 	slot = (p % (1 << MCX_LOG_RQ_SIZE));
   6750   1.1  jmcneill 	rqe = ring;
   6751   1.1  jmcneill 	for (fills = 0; fills < nslots; fills++) {
   6752  1.15  jmcneill 		slot = p % (1 << MCX_LOG_RQ_SIZE);
   6753  1.15  jmcneill 
   6754  1.15  jmcneill 		ms = &rx->rx_slots[slot];
   6755  1.15  jmcneill 		rqe = &ring[slot];
   6756  1.15  jmcneill 
   6757   1.1  jmcneill 		m = NULL;
   6758   1.1  jmcneill 		MGETHDR(m, M_DONTWAIT, MT_DATA);
   6759   1.1  jmcneill 		if (m == NULL)
   6760   1.1  jmcneill 			break;
   6761   1.1  jmcneill 
   6762   1.1  jmcneill 		MCLGET(m, M_DONTWAIT);
   6763   1.1  jmcneill 		if ((m->m_flags & M_EXT) == 0) {
   6764   1.1  jmcneill 			m_freem(m);
   6765   1.1  jmcneill 			break;
   6766   1.1  jmcneill 		}
   6767   1.1  jmcneill 
   6768  1.15  jmcneill 		m->m_len = m->m_pkthdr.len = sc->sc_hardmtu;
   6769  1.15  jmcneill 		m_adj(m, m->m_ext.ext_size - sc->sc_rxbufsz);
   6770  1.15  jmcneill 		m_adj(m, ETHER_ALIGN);
   6771  1.15  jmcneill 
   6772   1.1  jmcneill 		if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   6773   1.1  jmcneill 		    BUS_DMA_NOWAIT) != 0) {
   6774   1.1  jmcneill 			m_freem(m);
   6775   1.1  jmcneill 			break;
   6776   1.1  jmcneill 		}
   6777   1.3       tnn 		bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, BUS_DMASYNC_PREREAD);
   6778   1.1  jmcneill 		ms->ms_m = m;
   6779   1.1  jmcneill 
   6780  1.15  jmcneill 		be32enc(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len);
   6781  1.15  jmcneill 		be64enc(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr);
   6782  1.15  jmcneill 		be32enc(&rqe->rqe_lkey, sc->sc_lkey);
   6783   1.1  jmcneill 
   6784   1.1  jmcneill 		p++;
   6785   1.1  jmcneill 	}
   6786   1.1  jmcneill 
   6787  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   6788  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
   6789  1.15  jmcneill 
   6790  1.15  jmcneill 	rx->rx_prod = p;
   6791   1.1  jmcneill 
   6792  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   6793  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   6794  1.15  jmcneill 	be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell),
   6795  1.15  jmcneill 	    p & MCX_WQ_DOORBELL_MASK);
   6796  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   6797  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   6798   1.1  jmcneill 
   6799   1.1  jmcneill 	return (nslots - fills);
   6800   1.1  jmcneill }
   6801   1.1  jmcneill 
   6802   1.1  jmcneill static int
   6803  1.15  jmcneill mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx)
   6804   1.1  jmcneill {
   6805   1.1  jmcneill 	u_int slots;
   6806   1.1  jmcneill 
   6807  1.15  jmcneill 	slots = mcx_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE));
   6808   1.1  jmcneill 	if (slots == 0)
   6809   1.1  jmcneill 		return (1);
   6810   1.1  jmcneill 
   6811  1.15  jmcneill 	slots = mcx_rx_fill_slots(sc, rx, slots);
   6812  1.15  jmcneill 	mcx_rxr_put(&rx->rx_rxr, slots);
   6813   1.1  jmcneill 	return (0);
   6814   1.1  jmcneill }
   6815   1.1  jmcneill 
   6816   1.1  jmcneill void
   6817  1.15  jmcneill mcx_refill(void *xrx)
   6818   1.1  jmcneill {
   6819  1.15  jmcneill 	struct mcx_rx *rx = xrx;
   6820  1.15  jmcneill 	struct mcx_softc *sc = rx->rx_softc;
   6821   1.1  jmcneill 
   6822  1.15  jmcneill 	mcx_rx_fill(sc, rx);
   6823   1.1  jmcneill 
   6824  1.15  jmcneill 	if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
   6825  1.15  jmcneill 		callout_schedule(&rx->rx_refill, 1);
   6826   1.1  jmcneill }
   6827   1.1  jmcneill 
   6828  1.15  jmcneill static int
   6829  1.15  jmcneill mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx,
   6830  1.15  jmcneill     struct mcx_cq_entry *cqe)
   6831   1.1  jmcneill {
   6832   1.1  jmcneill 	struct mcx_slot *ms;
   6833   1.1  jmcneill 	bus_dmamap_t map;
   6834   1.1  jmcneill 	int slot, slots;
   6835   1.1  jmcneill 
   6836   1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
   6837   1.1  jmcneill 
   6838  1.15  jmcneill 	ms = &tx->tx_slots[slot];
   6839   1.1  jmcneill 	map = ms->ms_map;
   6840   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   6841   1.1  jmcneill 	    BUS_DMASYNC_POSTWRITE);
   6842   1.1  jmcneill 
   6843   1.1  jmcneill 	slots = 1;
   6844   1.1  jmcneill 	if (map->dm_nsegs > 1)
   6845   1.1  jmcneill 		slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;
   6846   1.1  jmcneill 
   6847   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, map);
   6848   1.1  jmcneill 	m_freem(ms->ms_m);
   6849   1.1  jmcneill 	ms->ms_m = NULL;
   6850  1.15  jmcneill 
   6851  1.15  jmcneill 	return (slots);
   6852   1.1  jmcneill }
   6853   1.1  jmcneill 
   6854   1.1  jmcneill static uint64_t
   6855   1.1  jmcneill mcx_uptime(void)
   6856   1.1  jmcneill {
   6857   1.1  jmcneill 	struct timespec ts;
   6858   1.1  jmcneill 
   6859   1.1  jmcneill 	nanouptime(&ts);
   6860   1.1  jmcneill 
   6861   1.1  jmcneill 	return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec);
   6862   1.1  jmcneill }
   6863   1.1  jmcneill 
   6864   1.1  jmcneill static void
   6865   1.1  jmcneill mcx_calibrate_first(struct mcx_softc *sc)
   6866   1.1  jmcneill {
   6867   1.1  jmcneill 	struct mcx_calibration *c = &sc->sc_calibration[0];
   6868  1.15  jmcneill 	int s;
   6869   1.1  jmcneill 
   6870   1.1  jmcneill 	sc->sc_calibration_gen = 0;
   6871   1.1  jmcneill 
   6872  1.15  jmcneill 	s = splhigh(); /* crit_enter? */
   6873   1.1  jmcneill 	c->c_ubase = mcx_uptime();
   6874   1.1  jmcneill 	c->c_tbase = mcx_timer(sc);
   6875  1.15  jmcneill 	splx(s);
   6876  1.15  jmcneill 	c->c_ratio = 0;
   6877   1.1  jmcneill 
   6878  1.15  jmcneill #if notyet
   6879   1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz);
   6880  1.15  jmcneill #endif
   6881   1.1  jmcneill }
   6882   1.1  jmcneill 
   6883  1.15  jmcneill #define MCX_TIMESTAMP_SHIFT 24
   6884   1.1  jmcneill 
   6885   1.1  jmcneill static void
   6886   1.1  jmcneill mcx_calibrate(void *arg)
   6887   1.1  jmcneill {
   6888   1.1  jmcneill 	struct mcx_softc *sc = arg;
   6889   1.1  jmcneill 	struct mcx_calibration *nc, *pc;
   6890  1.15  jmcneill 	uint64_t udiff, tdiff;
   6891   1.1  jmcneill 	unsigned int gen;
   6892  1.15  jmcneill 	int s;
   6893   1.1  jmcneill 
   6894   1.1  jmcneill 	if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING))
   6895   1.1  jmcneill 		return;
   6896   1.1  jmcneill 
   6897   1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz);
   6898   1.1  jmcneill 
   6899   1.1  jmcneill 	gen = sc->sc_calibration_gen;
   6900   1.1  jmcneill 	pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   6901   1.1  jmcneill 	gen++;
   6902   1.1  jmcneill 	nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   6903   1.1  jmcneill 
   6904   1.1  jmcneill 	nc->c_uptime = pc->c_ubase;
   6905   1.1  jmcneill 	nc->c_timestamp = pc->c_tbase;
   6906   1.1  jmcneill 
   6907  1.15  jmcneill 	s = splhigh(); /* crit_enter? */
   6908   1.1  jmcneill 	nc->c_ubase = mcx_uptime();
   6909   1.1  jmcneill 	nc->c_tbase = mcx_timer(sc);
   6910  1.15  jmcneill 	splx(s);
   6911   1.1  jmcneill 
   6912  1.15  jmcneill 	udiff = nc->c_ubase - nc->c_uptime;
   6913  1.15  jmcneill 	tdiff = nc->c_tbase - nc->c_timestamp;
   6914  1.15  jmcneill 
   6915  1.15  jmcneill 	/*
   6916  1.15  jmcneill 	 * udiff is the wall clock time between calibration ticks,
   6917  1.15  jmcneill 	 * which should be 32 seconds or 32 billion nanoseconds. if
   6918  1.15  jmcneill 	 * we squint, 1 billion nanoseconds is kind of like a 32 bit
   6919  1.15  jmcneill 	 * number, so 32 billion should still have a lot of high bits
   6920  1.15  jmcneill 	 * spare. we use this space by shifting the nanoseconds up
   6921  1.15  jmcneill 	 * 24 bits so we have a nice big number to divide by the
   6922  1.15  jmcneill 	 * number of mcx timer ticks.
   6923  1.15  jmcneill 	 */
   6924  1.15  jmcneill 	nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff;
   6925   1.1  jmcneill 
   6926   1.1  jmcneill 	membar_producer();
   6927   1.1  jmcneill 	sc->sc_calibration_gen = gen;
   6928   1.1  jmcneill }
   6929   1.1  jmcneill 
   6930   1.1  jmcneill static int
   6931  1.15  jmcneill mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx,
   6932  1.15  jmcneill     struct mcx_cq_entry *cqe, struct mcx_mbufq *mq,
   6933  1.15  jmcneill     const struct mcx_calibration *c)
   6934   1.1  jmcneill {
   6935  1.15  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   6936   1.1  jmcneill 	struct mcx_slot *ms;
   6937   1.1  jmcneill 	struct mbuf *m;
   6938  1.20  jmcneill 	uint32_t flags, len;
   6939   1.1  jmcneill 	int slot;
   6940   1.1  jmcneill 
   6941  1.20  jmcneill 	len = be32dec(&cqe->cq_byte_cnt);
   6942   1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
   6943   1.1  jmcneill 
   6944  1.15  jmcneill 	ms = &rx->rx_slots[slot];
   6945  1.20  jmcneill 	bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, len, BUS_DMASYNC_POSTREAD);
   6946   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
   6947   1.1  jmcneill 
   6948   1.1  jmcneill 	m = ms->ms_m;
   6949   1.1  jmcneill 	ms->ms_m = NULL;
   6950   1.1  jmcneill 
   6951   1.1  jmcneill 	m_set_rcvif(m, &sc->sc_ec.ec_if);
   6952  1.20  jmcneill 	m->m_pkthdr.len = m->m_len = len;
   6953   1.1  jmcneill 
   6954   1.1  jmcneill #if 0
   6955   1.1  jmcneill 	if (cqe->cq_rx_hash_type) {
   6956  1.15  jmcneill 		m->m_pkthdr.ph_flowid = be32toh(cqe->cq_rx_hash);
   6957  1.15  jmcneill 		m->m_pkthdr.csum_flags |= M_FLOWID;
   6958   1.1  jmcneill 	}
   6959   1.1  jmcneill #endif
   6960   1.1  jmcneill 
   6961  1.15  jmcneill 	flags = be32dec(&cqe->cq_flags);
   6962  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) {
   6963  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx)
   6964  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
   6965  1.15  jmcneill 	}
   6966  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) {
   6967  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx)
   6968  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
   6969  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx)
   6970  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_TCPv6;
   6971  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx)
   6972  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
   6973  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx)
   6974  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_UDPv6;
   6975  1.15  jmcneill 	}
   6976  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_CV) {
   6977  1.15  jmcneill 		vlan_set_tag(m, flags & MCX_CQ_ENTRY_FLAGS_VLAN_MASK);
   6978  1.15  jmcneill 	}
   6979  1.15  jmcneill 
   6980  1.15  jmcneill #if notyet
   6981  1.15  jmcneill 	if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_LINK0) && c->c_ratio) {
   6982  1.15  jmcneill 		uint64_t t = be64dec(&cqe->cq_timestamp);
   6983  1.15  jmcneill 		t -= c->c_timestamp;
   6984  1.15  jmcneill 		t *= c->c_ratio;
   6985  1.15  jmcneill 		t >>= MCX_TIMESTAMP_SHIFT;
   6986  1.15  jmcneill 		t += c->c_uptime;
   6987   1.1  jmcneill 
   6988  1.15  jmcneill 		m->m_pkthdr.ph_timestamp = t;
   6989   1.1  jmcneill 		SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
   6990   1.1  jmcneill 	}
   6991   1.1  jmcneill #endif
   6992   1.1  jmcneill 
   6993   1.1  jmcneill 	MBUFQ_ENQUEUE(mq, m);
   6994   1.1  jmcneill 
   6995   1.1  jmcneill 	return (1);
   6996   1.1  jmcneill }
   6997   1.1  jmcneill 
   6998   1.1  jmcneill static struct mcx_cq_entry *
   6999   1.1  jmcneill mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
   7000   1.1  jmcneill {
   7001   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   7002   1.1  jmcneill 	int next;
   7003   1.1  jmcneill 
   7004   1.1  jmcneill 	cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
   7005   1.1  jmcneill 	next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
   7006   1.1  jmcneill 
   7007   1.1  jmcneill 	if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
   7008   1.1  jmcneill 	    ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
   7009   1.1  jmcneill 		return (&cqe[next]);
   7010   1.1  jmcneill 	}
   7011   1.1  jmcneill 
   7012   1.1  jmcneill 	return (NULL);
   7013   1.1  jmcneill }
   7014   1.1  jmcneill 
   7015   1.1  jmcneill static void
   7016  1.15  jmcneill mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar)
   7017   1.1  jmcneill {
   7018  1.15  jmcneill 	struct mcx_cq_doorbell *db;
   7019   1.1  jmcneill 	bus_size_t offset;
   7020   1.1  jmcneill 	uint32_t val;
   7021   1.1  jmcneill 	uint64_t uval;
   7022   1.1  jmcneill 
   7023   1.1  jmcneill 	val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
   7024   1.1  jmcneill 	val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   7025   1.1  jmcneill 
   7026  1.15  jmcneill 	db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell);
   7027  1.15  jmcneill 
   7028  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7029  1.15  jmcneill 	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE);
   7030   1.1  jmcneill 
   7031  1.15  jmcneill 	be32enc(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   7032  1.15  jmcneill 	be32enc(&db->db_arm_ci, val);
   7033  1.15  jmcneill 
   7034  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7035  1.15  jmcneill 	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE);
   7036  1.15  jmcneill 
   7037  1.15  jmcneill 	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL;
   7038  1.15  jmcneill 
   7039  1.15  jmcneill 	uval = (uint64_t)val << 32;
   7040   1.1  jmcneill 	uval |= cq->cq_n;
   7041  1.15  jmcneill 
   7042  1.15  jmcneill 	bus_space_write_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval));
   7043  1.15  jmcneill 	mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE);
   7044   1.1  jmcneill }
   7045   1.1  jmcneill 
   7046   1.1  jmcneill void
   7047  1.15  jmcneill mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq)
   7048   1.1  jmcneill {
   7049  1.15  jmcneill 	struct mcx_rx *rx = &q->q_rx;
   7050  1.15  jmcneill 	struct mcx_tx *tx = &q->q_tx;
   7051   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   7052   1.1  jmcneill 	const struct mcx_calibration *c;
   7053   1.1  jmcneill 	unsigned int gen;
   7054   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   7055   1.1  jmcneill 	struct mcx_mbufq mq;
   7056   1.1  jmcneill 	struct mbuf *m;
   7057   1.1  jmcneill 	int rxfree, txfree;
   7058   1.1  jmcneill 
   7059   1.1  jmcneill 	MBUFQ_INIT(&mq);
   7060   1.1  jmcneill 
   7061   1.1  jmcneill 	gen = sc->sc_calibration_gen;
   7062   1.1  jmcneill 	membar_consumer();
   7063   1.1  jmcneill 	c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   7064   1.1  jmcneill 
   7065  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   7066  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   7067  1.15  jmcneill 
   7068   1.1  jmcneill 	rxfree = 0;
   7069   1.1  jmcneill 	txfree = 0;
   7070   1.1  jmcneill 	while ((cqe = mcx_next_cq_entry(sc, cq))) {
   7071   1.1  jmcneill 		uint8_t opcode;
   7072   1.1  jmcneill 		opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
   7073   1.1  jmcneill 		switch (opcode) {
   7074   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ:
   7075  1.15  jmcneill 			txfree += mcx_process_txeof(sc, tx, cqe);
   7076   1.1  jmcneill 			break;
   7077   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND:
   7078  1.15  jmcneill 			rxfree += mcx_process_rx(sc, rx, cqe, &mq, c);
   7079   1.1  jmcneill 			break;
   7080   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
   7081   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
   7082   1.1  jmcneill 			/* uint8_t *cqp = (uint8_t *)cqe; */
   7083  1.15  jmcneill 			/* printf("%s: cq completion error: %x\n",
   7084  1.15  jmcneill 			    DEVNAME(sc), cqp[0x37]); */
   7085   1.1  jmcneill 			break;
   7086   1.1  jmcneill 
   7087   1.1  jmcneill 		default:
   7088  1.15  jmcneill 			/* printf("%s: cq completion opcode %x??\n",
   7089  1.15  jmcneill 			    DEVNAME(sc), opcode); */
   7090   1.1  jmcneill 			break;
   7091   1.1  jmcneill 		}
   7092   1.1  jmcneill 
   7093   1.1  jmcneill 		cq->cq_cons++;
   7094   1.1  jmcneill 	}
   7095   1.1  jmcneill 
   7096  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   7097  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
   7098  1.15  jmcneill 
   7099   1.1  jmcneill 	if (rxfree > 0) {
   7100  1.15  jmcneill 		mcx_rxr_put(&rx->rx_rxr, rxfree);
   7101   1.1  jmcneill 		while (MBUFQ_FIRST(&mq) != NULL) {
   7102   1.1  jmcneill 			MBUFQ_DEQUEUE(&mq, m);
   7103   1.1  jmcneill 			if_percpuq_enqueue(ifp->if_percpuq, m);
   7104   1.1  jmcneill 		}
   7105   1.1  jmcneill 
   7106  1.15  jmcneill 		mcx_rx_fill(sc, rx);
   7107  1.15  jmcneill 		if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
   7108  1.15  jmcneill 			callout_schedule(&rx->rx_refill, 1);
   7109   1.1  jmcneill 	}
   7110  1.17  jmcneill 
   7111  1.17  jmcneill 	cq->cq_count++;
   7112  1.17  jmcneill 	mcx_arm_cq(sc, cq, q->q_uar);
   7113  1.17  jmcneill 
   7114   1.1  jmcneill 	if (txfree > 0) {
   7115  1.15  jmcneill 		tx->tx_cons += txfree;
   7116   1.1  jmcneill 		if_schedule_deferred_start(ifp);
   7117   1.1  jmcneill 	}
   7118   1.1  jmcneill }
   7119   1.1  jmcneill 
   7120  1.15  jmcneill 
   7121   1.1  jmcneill static void
   7122  1.15  jmcneill mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar)
   7123   1.1  jmcneill {
   7124   1.1  jmcneill 	bus_size_t offset;
   7125   1.1  jmcneill 	uint32_t val;
   7126   1.1  jmcneill 
   7127  1.15  jmcneill 	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM;
   7128  1.15  jmcneill 	val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff);
   7129   1.1  jmcneill 
   7130   1.1  jmcneill 	mcx_wr(sc, offset, val);
   7131  1.15  jmcneill 	mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE);
   7132   1.1  jmcneill }
   7133   1.1  jmcneill 
   7134   1.1  jmcneill static struct mcx_eq_entry *
   7135  1.15  jmcneill mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq)
   7136   1.1  jmcneill {
   7137   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   7138   1.1  jmcneill 	int next;
   7139   1.1  jmcneill 
   7140  1.15  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
   7141  1.15  jmcneill 	next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE);
   7142  1.15  jmcneill 	if ((eqe[next].eq_owner & 1) ==
   7143  1.15  jmcneill 	    ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
   7144  1.15  jmcneill 		eq->eq_cons++;
   7145   1.1  jmcneill 		return (&eqe[next]);
   7146   1.1  jmcneill 	}
   7147   1.1  jmcneill 	return (NULL);
   7148   1.1  jmcneill }
   7149   1.1  jmcneill 
   7150   1.1  jmcneill int
   7151  1.15  jmcneill mcx_admin_intr(void *xsc)
   7152   1.1  jmcneill {
   7153   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)xsc;
   7154  1.15  jmcneill 	struct mcx_eq *eq = &sc->sc_admin_eq;
   7155   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   7156   1.1  jmcneill 
   7157  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7158  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   7159  1.15  jmcneill 
   7160  1.15  jmcneill 	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
   7161   1.1  jmcneill 		switch (eqe->eq_event_type) {
   7162   1.1  jmcneill 		case MCX_EVENT_TYPE_LAST_WQE:
   7163   1.1  jmcneill 			/* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
   7164   1.1  jmcneill 			break;
   7165   1.1  jmcneill 
   7166   1.1  jmcneill 		case MCX_EVENT_TYPE_CQ_ERROR:
   7167   1.1  jmcneill 			/* printf("%s: cq error\n", DEVNAME(sc)); */
   7168   1.1  jmcneill 			break;
   7169   1.1  jmcneill 
   7170   1.1  jmcneill 		case MCX_EVENT_TYPE_CMD_COMPLETION:
   7171   1.1  jmcneill 			/* wakeup probably */
   7172   1.1  jmcneill 			break;
   7173   1.1  jmcneill 
   7174   1.1  jmcneill 		case MCX_EVENT_TYPE_PORT_CHANGE:
   7175   1.1  jmcneill 			workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL);
   7176   1.1  jmcneill 			break;
   7177   1.1  jmcneill 
   7178   1.1  jmcneill 		default:
   7179   1.1  jmcneill 			/* printf("%s: something happened\n", DEVNAME(sc)); */
   7180   1.1  jmcneill 			break;
   7181   1.1  jmcneill 		}
   7182   1.1  jmcneill 	}
   7183  1.15  jmcneill 
   7184  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7185  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   7186  1.15  jmcneill 
   7187  1.15  jmcneill 	mcx_arm_eq(sc, eq, sc->sc_uar);
   7188  1.15  jmcneill 
   7189  1.15  jmcneill 	return (1);
   7190  1.15  jmcneill }
   7191  1.15  jmcneill 
   7192  1.15  jmcneill int
   7193  1.15  jmcneill mcx_cq_intr(void *xq)
   7194  1.15  jmcneill {
   7195  1.15  jmcneill 	struct mcx_queues *q = (struct mcx_queues *)xq;
   7196  1.15  jmcneill 	struct mcx_softc *sc = q->q_sc;
   7197  1.15  jmcneill 	struct mcx_eq *eq = &q->q_eq;
   7198  1.15  jmcneill 	struct mcx_eq_entry *eqe;
   7199  1.15  jmcneill 	int cqn;
   7200  1.15  jmcneill 
   7201  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7202  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   7203  1.15  jmcneill 
   7204  1.15  jmcneill 	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
   7205  1.15  jmcneill 		switch (eqe->eq_event_type) {
   7206  1.15  jmcneill 		case MCX_EVENT_TYPE_COMPLETION:
   7207  1.15  jmcneill 			cqn = be32toh(eqe->eq_event_data[6]);
   7208  1.15  jmcneill 			if (cqn == q->q_cq.cq_n)
   7209  1.15  jmcneill 				mcx_process_cq(sc, q, &q->q_cq);
   7210  1.15  jmcneill 			break;
   7211  1.15  jmcneill 		}
   7212  1.15  jmcneill 	}
   7213  1.15  jmcneill 
   7214  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7215  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   7216  1.15  jmcneill 
   7217  1.15  jmcneill 	mcx_arm_eq(sc, eq, q->q_uar);
   7218  1.15  jmcneill 
   7219   1.1  jmcneill 	return (1);
   7220   1.1  jmcneill }
   7221   1.1  jmcneill 
   7222   1.1  jmcneill static void
   7223   1.1  jmcneill mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
   7224   1.1  jmcneill     int total)
   7225   1.1  jmcneill {
   7226   1.1  jmcneill 	struct mcx_slot *ms;
   7227   1.1  jmcneill 
   7228   1.1  jmcneill 	int i = allocated;
   7229   1.1  jmcneill 	while (i-- > 0) {
   7230   1.1  jmcneill 		ms = &slots[i];
   7231   1.1  jmcneill 		bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
   7232   1.1  jmcneill 		if (ms->ms_m != NULL)
   7233   1.1  jmcneill 			m_freem(ms->ms_m);
   7234   1.1  jmcneill 	}
   7235   1.1  jmcneill 	kmem_free(slots, total * sizeof(*ms));
   7236   1.1  jmcneill }
   7237   1.1  jmcneill 
   7238   1.1  jmcneill static int
   7239  1.15  jmcneill mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q)
   7240   1.1  jmcneill {
   7241  1.15  jmcneill 	struct mcx_rx *rx;
   7242  1.15  jmcneill 	struct mcx_tx *tx;
   7243   1.1  jmcneill 	struct mcx_slot *ms;
   7244  1.15  jmcneill 	int i;
   7245   1.1  jmcneill 
   7246  1.15  jmcneill 	rx = &q->q_rx;
   7247  1.15  jmcneill 	rx->rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE),
   7248   1.1  jmcneill 	    KM_SLEEP);
   7249   1.1  jmcneill 
   7250   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
   7251  1.15  jmcneill 		ms = &rx->rx_slots[i];
   7252   1.1  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
   7253   1.1  jmcneill 		    sc->sc_hardmtu, 0,
   7254   1.1  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   7255   1.1  jmcneill 		    &ms->ms_map) != 0) {
   7256   1.1  jmcneill 			printf("%s: failed to allocate rx dma maps\n",
   7257   1.1  jmcneill 			    DEVNAME(sc));
   7258   1.1  jmcneill 			goto destroy_rx_slots;
   7259   1.1  jmcneill 		}
   7260  1.15  jmcneill 	}
   7261  1.15  jmcneill 
   7262  1.15  jmcneill 	tx = &q->q_tx;
   7263  1.15  jmcneill 	tx->tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE),
   7264  1.15  jmcneill 	     KM_SLEEP);
   7265  1.15  jmcneill 
   7266  1.15  jmcneill 	for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
   7267  1.15  jmcneill 		ms = &tx->tx_slots[i];
   7268  1.15  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
   7269  1.15  jmcneill 		    MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
   7270  1.15  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   7271  1.15  jmcneill 		    &ms->ms_map) != 0) {
   7272  1.15  jmcneill 			printf("%s: failed to allocate tx dma maps\n",
   7273  1.15  jmcneill 			    DEVNAME(sc));
   7274  1.15  jmcneill 			goto destroy_tx_slots;
   7275  1.15  jmcneill 		}
   7276  1.15  jmcneill 	}
   7277  1.15  jmcneill 
   7278  1.15  jmcneill 	if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index,
   7279  1.15  jmcneill 	    q->q_eq.eq_n) != 0)
   7280  1.15  jmcneill 		goto destroy_tx_slots;
   7281  1.15  jmcneill 
   7282  1.15  jmcneill 	if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n)
   7283  1.15  jmcneill 	    != 0)
   7284  1.15  jmcneill 		goto destroy_cq;
   7285  1.15  jmcneill 
   7286  1.15  jmcneill 	if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0)
   7287  1.15  jmcneill 		goto destroy_sq;
   7288  1.15  jmcneill 
   7289  1.15  jmcneill 	return 0;
   7290  1.15  jmcneill 
   7291  1.15  jmcneill destroy_sq:
   7292  1.15  jmcneill 	mcx_destroy_sq(sc, tx);
   7293  1.15  jmcneill destroy_cq:
   7294  1.15  jmcneill 	mcx_destroy_cq(sc, &q->q_cq);
   7295  1.15  jmcneill destroy_tx_slots:
   7296  1.15  jmcneill 	mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
   7297  1.15  jmcneill 	tx->tx_slots = NULL;
   7298  1.15  jmcneill 
   7299  1.15  jmcneill 	i = (1 << MCX_LOG_RQ_SIZE);
   7300  1.15  jmcneill destroy_rx_slots:
   7301  1.15  jmcneill 	mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
   7302  1.15  jmcneill 	rx->rx_slots = NULL;
   7303  1.15  jmcneill 	return ENOMEM;
   7304  1.15  jmcneill }
   7305  1.15  jmcneill 
   7306  1.15  jmcneill static int
   7307  1.15  jmcneill mcx_rss_group_entry_count(struct mcx_softc *sc, int group)
   7308  1.15  jmcneill {
   7309  1.15  jmcneill 	int i;
   7310  1.15  jmcneill 	int count;
   7311  1.15  jmcneill 
   7312  1.15  jmcneill 	count = 0;
   7313  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7314  1.15  jmcneill 		if (mcx_rss_config[i].flow_group == group)
   7315  1.15  jmcneill 			count++;
   7316  1.15  jmcneill 	}
   7317  1.15  jmcneill 
   7318  1.15  jmcneill 	return count;
   7319  1.15  jmcneill }
   7320  1.15  jmcneill 
   7321  1.15  jmcneill static int
   7322  1.15  jmcneill mcx_init(struct ifnet *ifp)
   7323  1.15  jmcneill {
   7324  1.15  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7325  1.15  jmcneill 	struct mcx_rx *rx;
   7326  1.15  jmcneill 	struct mcx_tx *tx;
   7327  1.15  jmcneill 	int i, start, count, flow_group, flow_index;
   7328  1.15  jmcneill 	struct mcx_flow_match match_crit;
   7329  1.15  jmcneill 	struct mcx_rss_rule *rss;
   7330  1.15  jmcneill 	uint32_t dest;
   7331  1.15  jmcneill 	int rqns[MCX_MAX_QUEUES] = { 0 };
   7332  1.15  jmcneill 
   7333  1.15  jmcneill 	if (ISSET(ifp->if_flags, IFF_RUNNING))
   7334  1.15  jmcneill 		mcx_stop(ifp, 0);
   7335   1.1  jmcneill 
   7336  1.15  jmcneill 	if (mcx_create_tis(sc, &sc->sc_tis) != 0)
   7337  1.15  jmcneill 		goto down;
   7338   1.1  jmcneill 
   7339  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7340  1.15  jmcneill 		if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
   7341  1.15  jmcneill 			goto down;
   7342   1.1  jmcneill 		}
   7343   1.1  jmcneill 	}
   7344   1.1  jmcneill 
   7345  1.15  jmcneill 	/* RSS flow table and flow groups */
   7346  1.15  jmcneill 	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1,
   7347  1.15  jmcneill 	    &sc->sc_rss_flow_table_id) != 0)
   7348   1.1  jmcneill 		goto down;
   7349   1.1  jmcneill 
   7350  1.15  jmcneill 	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   7351  1.15  jmcneill 	    sc->sc_rss_flow_table_id;
   7352   1.1  jmcneill 
   7353  1.15  jmcneill 	/* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */
   7354  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7355  1.15  jmcneill 	match_crit.mc_ethertype = 0xffff;
   7356  1.15  jmcneill 	match_crit.mc_ip_proto = 0xff;
   7357  1.15  jmcneill 	match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG;
   7358  1.15  jmcneill 	start = 0;
   7359  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4);
   7360  1.15  jmcneill 	if (count != 0) {
   7361  1.15  jmcneill 		if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7362  1.15  jmcneill 		    MCX_FLOW_GROUP_RSS_L4, start, count,
   7363  1.15  jmcneill 		    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7364  1.15  jmcneill 			goto down;
   7365  1.15  jmcneill 		start += count;
   7366  1.15  jmcneill 	}
   7367   1.1  jmcneill 
   7368  1.15  jmcneill 	/* L3 RSS flow group (v4/v6, including fragments) */
   7369  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7370  1.15  jmcneill 	match_crit.mc_ethertype = 0xffff;
   7371  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3);
   7372  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7373  1.15  jmcneill 	    MCX_FLOW_GROUP_RSS_L3, start, count,
   7374  1.15  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7375   1.1  jmcneill 		goto down;
   7376  1.15  jmcneill 	start += count;
   7377   1.1  jmcneill 
   7378  1.15  jmcneill 	/* non-RSS flow group */
   7379  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE);
   7380  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7381  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7382  1.15  jmcneill 	    MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0)
   7383   1.1  jmcneill 		goto down;
   7384   1.1  jmcneill 
   7385  1.15  jmcneill 	/* Root flow table, matching packets based on mac address */
   7386  1.15  jmcneill 	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0,
   7387  1.15  jmcneill 	    &sc->sc_mac_flow_table_id) != 0)
   7388   1.1  jmcneill 		goto down;
   7389   1.1  jmcneill 
   7390   1.1  jmcneill 	/* promisc flow group */
   7391   1.1  jmcneill 	start = 0;
   7392   1.1  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7393  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7394  1.15  jmcneill 	    MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0)
   7395   1.1  jmcneill 		goto down;
   7396   1.1  jmcneill 	sc->sc_promisc_flow_enabled = 0;
   7397   1.1  jmcneill 	start++;
   7398   1.1  jmcneill 
   7399   1.1  jmcneill 	/* all multicast flow group */
   7400   1.1  jmcneill 	match_crit.mc_dest_mac[0] = 0x01;
   7401  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7402  1.15  jmcneill 	    MCX_FLOW_GROUP_ALLMULTI, start, 1,
   7403   1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7404   1.1  jmcneill 		goto down;
   7405   1.1  jmcneill 	sc->sc_allmulti_flow_enabled = 0;
   7406   1.1  jmcneill 	start++;
   7407   1.1  jmcneill 
   7408   1.1  jmcneill 	/* mac address matching flow group */
   7409   1.1  jmcneill 	memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
   7410  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7411  1.15  jmcneill 	    MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
   7412   1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7413   1.1  jmcneill 		goto down;
   7414   1.1  jmcneill 
   7415   1.1  jmcneill 	/* flow table entries for unicast and broadcast */
   7416   1.1  jmcneill 	start = 0;
   7417  1.15  jmcneill 	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
   7418  1.15  jmcneill 	    LLADDR(satosdl(ifp->if_dl->ifa_addr)), dest) != 0)
   7419   1.1  jmcneill 		goto down;
   7420   1.1  jmcneill 	start++;
   7421   1.1  jmcneill 
   7422  1.15  jmcneill 	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
   7423  1.15  jmcneill 	    etherbroadcastaddr, dest) != 0)
   7424   1.1  jmcneill 		goto down;
   7425   1.1  jmcneill 	start++;
   7426   1.1  jmcneill 
   7427   1.1  jmcneill 	/* multicast entries go after that */
   7428   1.1  jmcneill 	sc->sc_mcast_flow_base = start;
   7429   1.1  jmcneill 
   7430   1.1  jmcneill 	/* re-add any existing multicast flows */
   7431   1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7432   1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   7433  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC,
   7434   1.1  jmcneill 			    sc->sc_mcast_flow_base + i,
   7435  1.15  jmcneill 			    sc->sc_mcast_flows[i], dest);
   7436   1.1  jmcneill 		}
   7437   1.1  jmcneill 	}
   7438   1.1  jmcneill 
   7439  1.15  jmcneill 	if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0)
   7440   1.1  jmcneill 		goto down;
   7441   1.1  jmcneill 
   7442  1.15  jmcneill 	/*
   7443  1.15  jmcneill 	 * the RQT can be any size as long as it's a power of two.
   7444  1.15  jmcneill 	 * since we also restrict the number of queues to a power of two,
   7445  1.15  jmcneill 	 * we can just put each rx queue in once.
   7446  1.15  jmcneill 	 */
   7447  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++)
   7448  1.15  jmcneill 		rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;
   7449  1.15  jmcneill 
   7450  1.15  jmcneill 	if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0)
   7451   1.1  jmcneill 		goto down;
   7452   1.1  jmcneill 
   7453  1.15  jmcneill 	start = 0;
   7454  1.15  jmcneill 	flow_index = 0;
   7455  1.15  jmcneill 	flow_group = -1;
   7456  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7457  1.15  jmcneill 		rss = &mcx_rss_config[i];
   7458  1.15  jmcneill 		if (rss->flow_group != flow_group) {
   7459  1.15  jmcneill 			flow_group = rss->flow_group;
   7460  1.15  jmcneill 			flow_index = 0;
   7461  1.15  jmcneill 		}
   7462  1.15  jmcneill 
   7463  1.15  jmcneill 		if (rss->hash_sel == 0) {
   7464  1.15  jmcneill 			if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx,
   7465  1.15  jmcneill 			    &sc->sc_tir[i]) != 0)
   7466  1.15  jmcneill 				goto down;
   7467  1.15  jmcneill 		} else {
   7468  1.15  jmcneill 			if (mcx_create_tir_indirect(sc, sc->sc_rqt,
   7469  1.15  jmcneill 			    rss->hash_sel, &sc->sc_tir[i]) != 0)
   7470  1.15  jmcneill 				goto down;
   7471  1.15  jmcneill 		}
   7472  1.15  jmcneill 
   7473  1.15  jmcneill 		if (mcx_set_flow_table_entry_proto(sc, flow_group,
   7474  1.15  jmcneill 		    flow_index, rss->ethertype, rss->ip_proto,
   7475  1.15  jmcneill 		    MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0)
   7476  1.15  jmcneill 			goto down;
   7477  1.15  jmcneill 		flow_index++;
   7478  1.15  jmcneill 	}
   7479  1.15  jmcneill 
   7480  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7481  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   7482  1.15  jmcneill 		rx = &q->q_rx;
   7483  1.15  jmcneill 		tx = &q->q_tx;
   7484  1.15  jmcneill 
   7485  1.15  jmcneill 		/* start the queues */
   7486  1.15  jmcneill 		if (mcx_ready_sq(sc, tx) != 0)
   7487  1.15  jmcneill 			goto down;
   7488  1.15  jmcneill 
   7489  1.15  jmcneill 		if (mcx_ready_rq(sc, rx) != 0)
   7490  1.15  jmcneill 			goto down;
   7491  1.15  jmcneill 
   7492  1.15  jmcneill 		mcx_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
   7493  1.15  jmcneill 		rx->rx_prod = 0;
   7494  1.15  jmcneill 		mcx_rx_fill(sc, rx);
   7495   1.1  jmcneill 
   7496  1.15  jmcneill 		tx->tx_cons = 0;
   7497  1.15  jmcneill 		tx->tx_prod = 0;
   7498  1.15  jmcneill 	}
   7499   1.1  jmcneill 
   7500   1.1  jmcneill 	mcx_calibrate_first(sc);
   7501   1.1  jmcneill 
   7502   1.1  jmcneill 	SET(ifp->if_flags, IFF_RUNNING);
   7503   1.1  jmcneill 	CLR(ifp->if_flags, IFF_OACTIVE);
   7504   1.1  jmcneill 	if_schedule_deferred_start(ifp);
   7505   1.1  jmcneill 
   7506   1.1  jmcneill 	return 0;
   7507   1.1  jmcneill down:
   7508   1.1  jmcneill 	mcx_stop(ifp, 0);
   7509   1.1  jmcneill 	return EIO;
   7510   1.1  jmcneill }
   7511   1.1  jmcneill 
   7512   1.1  jmcneill static void
   7513   1.1  jmcneill mcx_stop(struct ifnet *ifp, int disable)
   7514   1.1  jmcneill {
   7515   1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7516  1.15  jmcneill 	struct mcx_rss_rule *rss;
   7517  1.15  jmcneill 	int group, i, flow_group, flow_index;
   7518   1.1  jmcneill 
   7519   1.1  jmcneill 	CLR(ifp->if_flags, IFF_RUNNING);
   7520   1.1  jmcneill 
   7521   1.1  jmcneill 	/*
   7522   1.1  jmcneill 	 * delete flow table entries first, so no packets can arrive
   7523   1.1  jmcneill 	 * after the barriers
   7524   1.1  jmcneill 	 */
   7525   1.1  jmcneill 	if (sc->sc_promisc_flow_enabled)
   7526   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   7527   1.1  jmcneill 	if (sc->sc_allmulti_flow_enabled)
   7528   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   7529   1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
   7530   1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
   7531   1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7532   1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   7533   1.1  jmcneill 			mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
   7534   1.1  jmcneill 			    sc->sc_mcast_flow_base + i);
   7535   1.1  jmcneill 		}
   7536   1.1  jmcneill 	}
   7537   1.1  jmcneill 
   7538  1.15  jmcneill 	flow_group = -1;
   7539  1.15  jmcneill 	flow_index = 0;
   7540  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7541  1.15  jmcneill 		rss = &mcx_rss_config[i];
   7542  1.15  jmcneill 		if (rss->flow_group != flow_group) {
   7543  1.15  jmcneill 			flow_group = rss->flow_group;
   7544  1.15  jmcneill 			flow_index = 0;
   7545  1.15  jmcneill 		}
   7546  1.15  jmcneill 
   7547  1.15  jmcneill 		mcx_delete_flow_table_entry(sc, flow_group, flow_index);
   7548  1.15  jmcneill 
   7549  1.15  jmcneill 		mcx_destroy_tir(sc, sc->sc_tir[i]);
   7550  1.15  jmcneill 		sc->sc_tir[i] = 0;
   7551  1.15  jmcneill 
   7552  1.15  jmcneill 		flow_index++;
   7553  1.15  jmcneill 	}
   7554  1.15  jmcneill 
   7555  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7556  1.15  jmcneill 		callout_halt(&sc->sc_queues[i].q_rx.rx_refill, NULL);
   7557  1.15  jmcneill 	}
   7558  1.15  jmcneill 
   7559   1.1  jmcneill 	callout_halt(&sc->sc_calibrate, NULL);
   7560   1.1  jmcneill 
   7561   1.1  jmcneill 	for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
   7562  1.15  jmcneill 		if (sc->sc_flow_group[group].g_id != -1)
   7563  1.15  jmcneill 			mcx_destroy_flow_group(sc, group);
   7564  1.15  jmcneill 	}
   7565  1.15  jmcneill 
   7566  1.15  jmcneill 	if (sc->sc_mac_flow_table_id != -1) {
   7567  1.15  jmcneill 		mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id);
   7568  1.15  jmcneill 		sc->sc_mac_flow_table_id = -1;
   7569  1.15  jmcneill 	}
   7570  1.15  jmcneill 	if (sc->sc_rss_flow_table_id != -1) {
   7571  1.15  jmcneill 		mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id);
   7572  1.15  jmcneill 		sc->sc_rss_flow_table_id = -1;
   7573  1.15  jmcneill 	}
   7574  1.15  jmcneill 	if (sc->sc_rqt != -1) {
   7575  1.15  jmcneill 		mcx_destroy_rqt(sc, sc->sc_rqt);
   7576  1.15  jmcneill 		sc->sc_rqt = -1;
   7577  1.15  jmcneill 	}
   7578  1.15  jmcneill 
   7579  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7580  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   7581  1.15  jmcneill 		struct mcx_rx *rx = &q->q_rx;
   7582  1.15  jmcneill 		struct mcx_tx *tx = &q->q_tx;
   7583  1.15  jmcneill 		struct mcx_cq *cq = &q->q_cq;
   7584  1.15  jmcneill 
   7585  1.15  jmcneill 		if (rx->rx_rqn != 0)
   7586  1.15  jmcneill 			mcx_destroy_rq(sc, rx);
   7587  1.15  jmcneill 
   7588  1.15  jmcneill 		if (tx->tx_sqn != 0)
   7589  1.15  jmcneill 			mcx_destroy_sq(sc, tx);
   7590  1.15  jmcneill 
   7591  1.15  jmcneill 		if (tx->tx_slots != NULL) {
   7592  1.15  jmcneill 			mcx_free_slots(sc, tx->tx_slots,
   7593  1.15  jmcneill 			    (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE));
   7594  1.15  jmcneill 			tx->tx_slots = NULL;
   7595  1.15  jmcneill 		}
   7596  1.15  jmcneill 		if (rx->rx_slots != NULL) {
   7597  1.15  jmcneill 			mcx_free_slots(sc, rx->rx_slots,
   7598  1.15  jmcneill 			    (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE));
   7599  1.15  jmcneill 			rx->rx_slots = NULL;
   7600  1.15  jmcneill 		}
   7601  1.15  jmcneill 
   7602  1.15  jmcneill 		if (cq->cq_n != 0)
   7603  1.15  jmcneill 			mcx_destroy_cq(sc, cq);
   7604  1.15  jmcneill 	}
   7605  1.15  jmcneill 	if (sc->sc_tis != 0) {
   7606  1.15  jmcneill 		mcx_destroy_tis(sc, sc->sc_tis);
   7607  1.15  jmcneill 		sc->sc_tis = 0;
   7608   1.1  jmcneill 	}
   7609   1.1  jmcneill }
   7610   1.1  jmcneill 
   7611   1.1  jmcneill static int
   7612   1.1  jmcneill mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data)
   7613   1.1  jmcneill {
   7614   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   7615   1.1  jmcneill 	struct ifreq *ifr = (struct ifreq *)data;
   7616   1.7  jmcneill 	struct ethercom *ec = &sc->sc_ec;
   7617   1.1  jmcneill 	uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
   7618   1.7  jmcneill 	struct ether_multi *enm;
   7619   1.7  jmcneill 	struct ether_multistep step;
   7620   1.7  jmcneill 	int s, i, flags, error = 0;
   7621  1.15  jmcneill 	uint32_t dest;
   7622   1.1  jmcneill 
   7623   1.1  jmcneill 	s = splnet();
   7624   1.1  jmcneill 	switch (cmd) {
   7625   1.1  jmcneill 
   7626   1.1  jmcneill 	case SIOCADDMULTI:
   7627   1.1  jmcneill 		if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   7628   1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   7629   1.7  jmcneill 			if (error != 0) {
   7630   1.7  jmcneill 				splx(s);
   7631   1.1  jmcneill 				return (error);
   7632   1.7  jmcneill 			}
   7633   1.1  jmcneill 
   7634  1.15  jmcneill 			dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   7635  1.15  jmcneill 			    sc->sc_rss_flow_table_id;
   7636  1.15  jmcneill 
   7637   1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7638   1.1  jmcneill 				if (sc->sc_mcast_flows[i][0] == 0) {
   7639   1.1  jmcneill 					memcpy(sc->sc_mcast_flows[i], addrlo,
   7640   1.1  jmcneill 					    ETHER_ADDR_LEN);
   7641   1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   7642  1.15  jmcneill 						mcx_set_flow_table_entry_mac(sc,
   7643   1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   7644   1.1  jmcneill 						    sc->sc_mcast_flow_base + i,
   7645  1.15  jmcneill 						    sc->sc_mcast_flows[i], dest);
   7646   1.1  jmcneill 					}
   7647   1.1  jmcneill 					break;
   7648   1.1  jmcneill 				}
   7649   1.1  jmcneill 			}
   7650   1.1  jmcneill 
   7651   1.1  jmcneill 			if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   7652   1.1  jmcneill 				if (i == MCX_NUM_MCAST_FLOWS) {
   7653   1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   7654   1.1  jmcneill 					sc->sc_extra_mcast++;
   7655   1.1  jmcneill 					error = ENETRESET;
   7656   1.1  jmcneill 				}
   7657   1.1  jmcneill 
   7658   1.7  jmcneill 				if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN)) {
   7659   1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   7660   1.1  jmcneill 					error = ENETRESET;
   7661   1.1  jmcneill 				}
   7662   1.1  jmcneill 			}
   7663   1.1  jmcneill 		}
   7664   1.1  jmcneill 		break;
   7665   1.1  jmcneill 
   7666   1.1  jmcneill 	case SIOCDELMULTI:
   7667   1.1  jmcneill 		if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   7668   1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   7669   1.7  jmcneill 			if (error != 0) {
   7670   1.7  jmcneill 				splx(s);
   7671   1.1  jmcneill 				return (error);
   7672   1.7  jmcneill 			}
   7673   1.1  jmcneill 
   7674   1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7675   1.1  jmcneill 				if (memcmp(sc->sc_mcast_flows[i], addrlo,
   7676   1.1  jmcneill 				    ETHER_ADDR_LEN) == 0) {
   7677   1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   7678   1.1  jmcneill 						mcx_delete_flow_table_entry(sc,
   7679   1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   7680   1.1  jmcneill 						    sc->sc_mcast_flow_base + i);
   7681   1.1  jmcneill 					}
   7682   1.1  jmcneill 					sc->sc_mcast_flows[i][0] = 0;
   7683   1.1  jmcneill 					break;
   7684   1.1  jmcneill 				}
   7685   1.1  jmcneill 			}
   7686   1.1  jmcneill 
   7687   1.1  jmcneill 			if (i == MCX_NUM_MCAST_FLOWS)
   7688   1.1  jmcneill 				sc->sc_extra_mcast--;
   7689   1.1  jmcneill 
   7690   1.1  jmcneill 			if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
   7691   1.7  jmcneill 			    sc->sc_extra_mcast == 0) {
   7692   1.7  jmcneill 				flags = 0;
   7693   1.7  jmcneill 				ETHER_LOCK(ec);
   7694   1.7  jmcneill 				ETHER_FIRST_MULTI(step, ec, enm);
   7695   1.7  jmcneill 				while (enm != NULL) {
   7696   1.7  jmcneill 					if (memcmp(enm->enm_addrlo,
   7697   1.7  jmcneill 					    enm->enm_addrhi, ETHER_ADDR_LEN)) {
   7698   1.7  jmcneill 						SET(flags, IFF_ALLMULTI);
   7699   1.7  jmcneill 						break;
   7700   1.7  jmcneill 					}
   7701   1.7  jmcneill 					ETHER_NEXT_MULTI(step, enm);
   7702   1.7  jmcneill 				}
   7703   1.7  jmcneill 				ETHER_UNLOCK(ec);
   7704   1.7  jmcneill 				if (!ISSET(flags, IFF_ALLMULTI)) {
   7705   1.7  jmcneill 					CLR(ifp->if_flags, IFF_ALLMULTI);
   7706   1.7  jmcneill 					error = ENETRESET;
   7707   1.7  jmcneill 				}
   7708   1.1  jmcneill 			}
   7709   1.1  jmcneill 		}
   7710   1.1  jmcneill 		break;
   7711   1.1  jmcneill 
   7712   1.1  jmcneill 	default:
   7713   1.1  jmcneill 		error = ether_ioctl(ifp, cmd, data);
   7714   1.1  jmcneill 	}
   7715   1.1  jmcneill 
   7716   1.1  jmcneill 	if (error == ENETRESET) {
   7717   1.1  jmcneill 		if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
   7718   1.1  jmcneill 		    (IFF_UP | IFF_RUNNING))
   7719   1.1  jmcneill 			mcx_iff(sc);
   7720   1.1  jmcneill 		error = 0;
   7721   1.1  jmcneill 	}
   7722   1.1  jmcneill 	splx(s);
   7723   1.1  jmcneill 
   7724   1.1  jmcneill 	return (error);
   7725   1.1  jmcneill }
   7726   1.1  jmcneill 
   7727   1.1  jmcneill #if 0
   7728   1.1  jmcneill static int
   7729   1.1  jmcneill mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
   7730   1.1  jmcneill {
   7731   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   7732   1.1  jmcneill 	struct mcx_reg_mcia mcia;
   7733   1.1  jmcneill 	struct mcx_reg_pmlp pmlp;
   7734   1.1  jmcneill 	int offset, error;
   7735   1.1  jmcneill 
   7736   1.1  jmcneill 	/* get module number */
   7737   1.1  jmcneill 	memset(&pmlp, 0, sizeof(pmlp));
   7738   1.1  jmcneill 	pmlp.rp_local_port = 1;
   7739   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
   7740   1.1  jmcneill 	    sizeof(pmlp));
   7741   1.1  jmcneill 	if (error != 0) {
   7742   1.1  jmcneill 		printf("%s: unable to get eeprom module number\n",
   7743   1.1  jmcneill 		    DEVNAME(sc));
   7744   1.1  jmcneill 		return error;
   7745   1.1  jmcneill 	}
   7746   1.1  jmcneill 
   7747   1.1  jmcneill 	for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
   7748   1.1  jmcneill 		memset(&mcia, 0, sizeof(mcia));
   7749   1.1  jmcneill 		mcia.rm_l = 0;
   7750   1.1  jmcneill 		mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) &
   7751   1.1  jmcneill 		    MCX_PMLP_MODULE_NUM_MASK;
   7752   1.1  jmcneill 		mcia.rm_i2c_addr = sff->sff_addr / 2;	/* apparently */
   7753   1.1  jmcneill 		mcia.rm_page_num = sff->sff_page;
   7754   1.1  jmcneill 		mcia.rm_dev_addr = htobe16(offset);
   7755   1.1  jmcneill 		mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);
   7756   1.1  jmcneill 
   7757   1.1  jmcneill 		error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
   7758   1.1  jmcneill 		    &mcia, sizeof(mcia));
   7759   1.1  jmcneill 		if (error != 0) {
   7760   1.1  jmcneill 			printf("%s: unable to read eeprom at %x\n",
   7761   1.1  jmcneill 			    DEVNAME(sc), offset);
   7762   1.1  jmcneill 			return error;
   7763   1.1  jmcneill 		}
   7764   1.1  jmcneill 
   7765   1.1  jmcneill 		memcpy(sff->sff_data + offset, mcia.rm_data,
   7766   1.1  jmcneill 		    MCX_MCIA_EEPROM_BYTES);
   7767   1.1  jmcneill 	}
   7768   1.1  jmcneill 
   7769   1.1  jmcneill 	return 0;
   7770   1.1  jmcneill }
   7771   1.1  jmcneill #endif
   7772   1.1  jmcneill 
   7773   1.1  jmcneill static int
   7774   1.1  jmcneill mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
   7775   1.1  jmcneill {
   7776   1.1  jmcneill 	switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   7777   1.1  jmcneill 	    BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
   7778   1.1  jmcneill 	case 0:
   7779   1.1  jmcneill 		break;
   7780   1.1  jmcneill 
   7781   1.1  jmcneill 	case EFBIG:
   7782   1.6    nonaka 		if (m_defrag(m, M_DONTWAIT) != NULL &&
   7783   1.1  jmcneill 		    bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   7784   1.1  jmcneill 		    BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
   7785   1.1  jmcneill 			break;
   7786   1.1  jmcneill 
   7787   1.1  jmcneill 		/* FALLTHROUGH */
   7788   1.1  jmcneill 	default:
   7789   1.1  jmcneill 		return (1);
   7790   1.1  jmcneill 	}
   7791   1.1  jmcneill 
   7792   1.1  jmcneill 	ms->ms_m = m;
   7793   1.1  jmcneill 	return (0);
   7794   1.1  jmcneill }
   7795   1.1  jmcneill 
   7796   1.1  jmcneill static void
   7797  1.16  jmcneill mcx_send_common_locked(struct ifnet *ifp, struct mcx_tx *tx, bool is_transmit)
   7798   1.1  jmcneill {
   7799   1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7800   1.1  jmcneill 	struct mcx_sq_entry *sq, *sqe;
   7801   1.1  jmcneill 	struct mcx_sq_entry_seg *sqs;
   7802   1.1  jmcneill 	struct mcx_slot *ms;
   7803   1.1  jmcneill 	bus_dmamap_t map;
   7804   1.1  jmcneill 	struct mbuf *m;
   7805   1.1  jmcneill 	u_int idx, free, used;
   7806   1.1  jmcneill 	uint64_t *bf;
   7807  1.15  jmcneill 	uint32_t csum;
   7808   1.1  jmcneill 	size_t bf_base;
   7809   1.1  jmcneill 	int i, seg, nseg;
   7810   1.1  jmcneill 
   7811  1.16  jmcneill 	KASSERT(mutex_owned(&tx->tx_lock));
   7812  1.16  jmcneill 
   7813  1.16  jmcneill 	if ((ifp->if_flags & IFF_RUNNING) == 0)
   7814  1.16  jmcneill 		return;
   7815  1.16  jmcneill 
   7816  1.15  jmcneill 	bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
   7817   1.1  jmcneill 
   7818  1.15  jmcneill 	idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE);
   7819  1.15  jmcneill 	free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod;
   7820   1.1  jmcneill 
   7821   1.1  jmcneill 	used = 0;
   7822   1.1  jmcneill 	bf = NULL;
   7823  1.15  jmcneill 
   7824  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   7825  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   7826  1.15  jmcneill 
   7827  1.15  jmcneill 	sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem);
   7828   1.1  jmcneill 
   7829   1.1  jmcneill 	for (;;) {
   7830   1.1  jmcneill 		if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
   7831   1.1  jmcneill 			SET(ifp->if_flags, IFF_OACTIVE);
   7832   1.1  jmcneill 			break;
   7833   1.1  jmcneill 		}
   7834   1.1  jmcneill 
   7835  1.16  jmcneill 		if (is_transmit) {
   7836  1.16  jmcneill 			m = pcq_get(tx->tx_pcq);
   7837  1.16  jmcneill 		} else {
   7838  1.16  jmcneill 			IFQ_DEQUEUE(&ifp->if_snd, m);
   7839  1.16  jmcneill 		}
   7840   1.1  jmcneill 		if (m == NULL) {
   7841   1.1  jmcneill 			break;
   7842   1.1  jmcneill 		}
   7843   1.1  jmcneill 
   7844   1.1  jmcneill 		sqe = sq + idx;
   7845  1.15  jmcneill 		ms = &tx->tx_slots[idx];
   7846   1.1  jmcneill 		memset(sqe, 0, sizeof(*sqe));
   7847   1.1  jmcneill 
   7848   1.1  jmcneill 		/* ctrl segment */
   7849   1.1  jmcneill 		sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
   7850  1.15  jmcneill 		    ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
   7851   1.1  jmcneill 		/* always generate a completion event */
   7852   1.1  jmcneill 		sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);
   7853   1.1  jmcneill 
   7854   1.1  jmcneill 		/* eth segment */
   7855  1.15  jmcneill 		csum = 0;
   7856  1.15  jmcneill 		if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
   7857  1.15  jmcneill 			csum |= MCX_SQE_L3_CSUM;
   7858  1.15  jmcneill 		if (m->m_pkthdr.csum_flags &
   7859  1.15  jmcneill 		    (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6))
   7860  1.15  jmcneill 			csum |= MCX_SQE_L4_CSUM;
   7861  1.15  jmcneill 		sqe->sqe_mss_csum = htobe32(csum);
   7862   1.1  jmcneill 		sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
   7863  1.15  jmcneill 		if (vlan_has_tag(m)) {
   7864  1.15  jmcneill 			struct ether_vlan_header *evh;
   7865  1.15  jmcneill 			evh = (struct ether_vlan_header *)
   7866  1.15  jmcneill 			    &sqe->sqe_inline_headers;
   7867  1.15  jmcneill 
   7868  1.15  jmcneill 			m_copydata(m, 0, ETHER_HDR_LEN, evh);
   7869  1.15  jmcneill 			evh->evl_proto = evh->evl_encap_proto;
   7870  1.15  jmcneill 			evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
   7871  1.15  jmcneill 			evh->evl_tag = htons(vlan_get_tag(m));
   7872  1.15  jmcneill 			m_adj(m, ETHER_HDR_LEN);
   7873  1.15  jmcneill 		} else {
   7874  1.15  jmcneill 			m_copydata(m, 0, MCX_SQ_INLINE_SIZE,
   7875  1.15  jmcneill 			    sqe->sqe_inline_headers);
   7876  1.15  jmcneill 			m_adj(m, MCX_SQ_INLINE_SIZE);
   7877  1.15  jmcneill 		}
   7878   1.1  jmcneill 
   7879   1.1  jmcneill 		if (mcx_load_mbuf(sc, ms, m) != 0) {
   7880   1.1  jmcneill 			m_freem(m);
   7881  1.10   thorpej 			if_statinc(ifp, if_oerrors);
   7882   1.1  jmcneill 			continue;
   7883   1.1  jmcneill 		}
   7884   1.1  jmcneill 		bf = (uint64_t *)sqe;
   7885   1.1  jmcneill 
   7886   1.1  jmcneill 		if (ifp->if_bpf != NULL)
   7887   1.1  jmcneill 			bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers,
   7888   1.1  jmcneill 			    MCX_SQ_INLINE_SIZE, m, BPF_D_OUT);
   7889   1.1  jmcneill 
   7890   1.1  jmcneill 		map = ms->ms_map;
   7891   1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   7892   1.1  jmcneill 		    BUS_DMASYNC_PREWRITE);
   7893   1.1  jmcneill 
   7894   1.1  jmcneill 		sqe->sqe_ds_sq_num =
   7895  1.15  jmcneill 		    htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) |
   7896   1.1  jmcneill 		    (map->dm_nsegs + 3));
   7897   1.1  jmcneill 
   7898   1.1  jmcneill 		/* data segment - first wqe has one segment */
   7899   1.1  jmcneill 		sqs = sqe->sqe_segs;
   7900   1.1  jmcneill 		seg = 0;
   7901   1.1  jmcneill 		nseg = 1;
   7902   1.1  jmcneill 		for (i = 0; i < map->dm_nsegs; i++) {
   7903   1.1  jmcneill 			if (seg == nseg) {
   7904   1.1  jmcneill 				/* next slot */
   7905   1.1  jmcneill 				idx++;
   7906   1.1  jmcneill 				if (idx == (1 << MCX_LOG_SQ_SIZE))
   7907   1.1  jmcneill 					idx = 0;
   7908  1.15  jmcneill 				tx->tx_prod++;
   7909   1.1  jmcneill 				used++;
   7910   1.1  jmcneill 
   7911   1.1  jmcneill 				sqs = (struct mcx_sq_entry_seg *)(sq + idx);
   7912   1.1  jmcneill 				seg = 0;
   7913   1.1  jmcneill 				nseg = MCX_SQ_SEGS_PER_SLOT;
   7914   1.1  jmcneill 			}
   7915   1.1  jmcneill 			sqs[seg].sqs_byte_count =
   7916   1.1  jmcneill 			    htobe32(map->dm_segs[i].ds_len);
   7917   1.1  jmcneill 			sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
   7918   1.1  jmcneill 			sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
   7919   1.1  jmcneill 			seg++;
   7920   1.1  jmcneill 		}
   7921   1.1  jmcneill 
   7922   1.1  jmcneill 		idx++;
   7923   1.1  jmcneill 		if (idx == (1 << MCX_LOG_SQ_SIZE))
   7924   1.1  jmcneill 			idx = 0;
   7925  1.15  jmcneill 		tx->tx_prod++;
   7926   1.1  jmcneill 		used++;
   7927   1.1  jmcneill 	}
   7928   1.1  jmcneill 
   7929  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   7930  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
   7931  1.15  jmcneill 
   7932   1.1  jmcneill 	if (used) {
   7933  1.15  jmcneill 		bus_size_t blueflame;
   7934   1.1  jmcneill 
   7935  1.15  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7936  1.15  jmcneill 		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   7937  1.15  jmcneill 		be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell),
   7938  1.15  jmcneill 		    tx->tx_prod & MCX_WQ_DOORBELL_MASK);
   7939  1.15  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7940  1.15  jmcneill 		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   7941   1.1  jmcneill 
   7942   1.1  jmcneill 		/*
   7943   1.1  jmcneill 		 * write the first 64 bits of the last sqe we produced
   7944   1.1  jmcneill 		 * to the blue flame buffer
   7945   1.1  jmcneill 		 */
   7946  1.15  jmcneill 
   7947  1.15  jmcneill 		blueflame = bf_base + tx->tx_bf_offset;
   7948   1.1  jmcneill 		bus_space_write_8(sc->sc_memt, sc->sc_memh,
   7949  1.15  jmcneill 		    blueflame, *bf);
   7950  1.15  jmcneill 		mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE);
   7951  1.15  jmcneill 
   7952   1.1  jmcneill 		/* next write goes to the other buffer */
   7953  1.15  jmcneill 		tx->tx_bf_offset ^= sc->sc_bf_size;
   7954   1.1  jmcneill 	}
   7955   1.1  jmcneill }
   7956   1.1  jmcneill 
   7957   1.1  jmcneill static void
   7958  1.16  jmcneill mcx_start(struct ifnet *ifp)
   7959  1.16  jmcneill {
   7960  1.16  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7961  1.16  jmcneill 	/* mcx_start() always uses TX ring[0] */
   7962  1.16  jmcneill 	struct mcx_tx *tx = &sc->sc_queues[0].q_tx;
   7963  1.16  jmcneill 
   7964  1.16  jmcneill 	mutex_enter(&tx->tx_lock);
   7965  1.16  jmcneill 	if (!ISSET(ifp->if_flags, IFF_OACTIVE)) {
   7966  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, false);
   7967  1.16  jmcneill 	}
   7968  1.16  jmcneill 	mutex_exit(&tx->tx_lock);
   7969  1.16  jmcneill }
   7970  1.16  jmcneill 
   7971  1.16  jmcneill static int
   7972  1.16  jmcneill mcx_transmit(struct ifnet *ifp, struct mbuf *m)
   7973  1.16  jmcneill {
   7974  1.16  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7975  1.16  jmcneill 	struct mcx_tx *tx;
   7976  1.16  jmcneill 
   7977  1.16  jmcneill 	tx = &sc->sc_queues[cpu_index(curcpu()) % sc->sc_nqueues].q_tx;
   7978  1.16  jmcneill 	if (__predict_false(!pcq_put(tx->tx_pcq, m))) {
   7979  1.16  jmcneill 		m_freem(m);
   7980  1.16  jmcneill 		return ENOBUFS;
   7981  1.16  jmcneill 	}
   7982  1.16  jmcneill 
   7983  1.16  jmcneill 	if (mutex_tryenter(&tx->tx_lock)) {
   7984  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, true);
   7985  1.16  jmcneill 		mutex_exit(&tx->tx_lock);
   7986  1.16  jmcneill 	} else {
   7987  1.16  jmcneill 		softint_schedule(tx->tx_softint);
   7988  1.16  jmcneill 	}
   7989  1.16  jmcneill 
   7990  1.16  jmcneill 	return 0;
   7991  1.16  jmcneill }
   7992  1.16  jmcneill 
   7993  1.16  jmcneill static void
   7994  1.16  jmcneill mcx_deferred_transmit(void *arg)
   7995  1.16  jmcneill {
   7996  1.16  jmcneill 	struct mcx_tx *tx = arg;
   7997  1.16  jmcneill 	struct mcx_softc *sc = tx->tx_softc;
   7998  1.16  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   7999  1.16  jmcneill 
   8000  1.16  jmcneill 	mutex_enter(&tx->tx_lock);
   8001  1.16  jmcneill 	if (pcq_peek(tx->tx_pcq) != NULL) {
   8002  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, true);
   8003  1.16  jmcneill 	}
   8004  1.16  jmcneill 	mutex_exit(&tx->tx_lock);
   8005  1.16  jmcneill }
   8006  1.16  jmcneill 
   8007   1.1  jmcneill 
   8008   1.1  jmcneill static void
   8009   1.1  jmcneill mcx_media_add_types(struct mcx_softc *sc)
   8010   1.1  jmcneill {
   8011   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8012   1.1  jmcneill 	int i;
   8013   1.1  jmcneill 	uint32_t proto_cap;
   8014   1.1  jmcneill 
   8015   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8016   1.1  jmcneill 	ptys.rp_local_port = 1;
   8017   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8018   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8019   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8020   1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   8021   1.1  jmcneill 		return;
   8022   1.1  jmcneill 	}
   8023   1.1  jmcneill 
   8024   1.1  jmcneill 	proto_cap = be32toh(ptys.rp_eth_proto_cap);
   8025   1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8026  1.13  jmcneill 		const struct mcx_eth_proto_capability *cap;
   8027  1.13  jmcneill 		if (!ISSET(proto_cap, 1U << i))
   8028  1.13  jmcneill 			continue;
   8029  1.13  jmcneill 
   8030  1.13  jmcneill 		cap = &mcx_eth_cap_map[i];
   8031  1.13  jmcneill 		if (cap->cap_media == 0)
   8032  1.13  jmcneill 			continue;
   8033  1.13  jmcneill 
   8034  1.13  jmcneill 		ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
   8035   1.1  jmcneill 	}
   8036   1.1  jmcneill }
   8037   1.1  jmcneill 
   8038   1.1  jmcneill static void
   8039   1.1  jmcneill mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
   8040   1.1  jmcneill {
   8041   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   8042   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8043   1.1  jmcneill 	int i;
   8044  1.13  jmcneill 	uint32_t proto_oper;
   8045   1.1  jmcneill 	uint64_t media_oper;
   8046   1.1  jmcneill 
   8047   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8048   1.1  jmcneill 	ptys.rp_local_port = 1;
   8049   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8050   1.1  jmcneill 
   8051   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8052   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8053   1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   8054   1.1  jmcneill 		return;
   8055   1.1  jmcneill 	}
   8056   1.1  jmcneill 
   8057   1.1  jmcneill 	proto_oper = be32toh(ptys.rp_eth_proto_oper);
   8058   1.1  jmcneill 
   8059   1.1  jmcneill 	media_oper = 0;
   8060  1.13  jmcneill 
   8061   1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8062  1.13  jmcneill 		const struct mcx_eth_proto_capability *cap;
   8063  1.13  jmcneill 		if (!ISSET(proto_oper, 1U << i))
   8064  1.13  jmcneill 			continue;
   8065  1.13  jmcneill 
   8066  1.13  jmcneill 		cap = &mcx_eth_cap_map[i];
   8067  1.13  jmcneill 
   8068  1.13  jmcneill 		if (cap->cap_media != 0)
   8069  1.13  jmcneill 			media_oper = cap->cap_media;
   8070   1.1  jmcneill 	}
   8071   1.1  jmcneill 
   8072   1.1  jmcneill 	ifmr->ifm_status = IFM_AVALID;
   8073   1.1  jmcneill 	if (proto_oper != 0) {
   8074   1.1  jmcneill 		ifmr->ifm_status |= IFM_ACTIVE;
   8075   1.1  jmcneill 		ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper;
   8076   1.1  jmcneill 		/* txpause, rxpause, duplex? */
   8077   1.1  jmcneill 	}
   8078   1.1  jmcneill }
   8079   1.1  jmcneill 
   8080   1.1  jmcneill static int
   8081   1.1  jmcneill mcx_media_change(struct ifnet *ifp)
   8082   1.1  jmcneill {
   8083   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   8084   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8085   1.1  jmcneill 	struct mcx_reg_paos paos;
   8086   1.1  jmcneill 	uint32_t media;
   8087   1.1  jmcneill 	int i, error;
   8088   1.1  jmcneill 
   8089   1.1  jmcneill 	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
   8090   1.1  jmcneill 		return EINVAL;
   8091   1.1  jmcneill 
   8092   1.1  jmcneill 	error = 0;
   8093   1.1  jmcneill 
   8094   1.1  jmcneill 	if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
   8095   1.1  jmcneill 		/* read ptys to get supported media */
   8096   1.1  jmcneill 		memset(&ptys, 0, sizeof(ptys));
   8097   1.1  jmcneill 		ptys.rp_local_port = 1;
   8098   1.1  jmcneill 		ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8099   1.1  jmcneill 		if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
   8100   1.1  jmcneill 		    &ptys, sizeof(ptys)) != 0) {
   8101   1.1  jmcneill 			printf("%s: unable to read port type/speed\n",
   8102   1.1  jmcneill 			    DEVNAME(sc));
   8103   1.1  jmcneill 			return EIO;
   8104   1.1  jmcneill 		}
   8105   1.1  jmcneill 
   8106   1.1  jmcneill 		media = be32toh(ptys.rp_eth_proto_cap);
   8107   1.1  jmcneill 	} else {
   8108   1.1  jmcneill 		/* map media type */
   8109   1.1  jmcneill 		media = 0;
   8110   1.1  jmcneill 		for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8111  1.13  jmcneill 			const struct mcx_eth_proto_capability *cap;
   8112  1.13  jmcneill 
   8113  1.13  jmcneill 			cap = &mcx_eth_cap_map[i];
   8114  1.13  jmcneill 			if (cap->cap_media ==
   8115   1.1  jmcneill 			    IFM_SUBTYPE(sc->sc_media.ifm_media)) {
   8116   1.1  jmcneill 				media = (1 << i);
   8117   1.1  jmcneill 				break;
   8118   1.1  jmcneill 			}
   8119   1.1  jmcneill 		}
   8120   1.1  jmcneill 	}
   8121   1.1  jmcneill 
   8122   1.1  jmcneill 	/* disable the port */
   8123   1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   8124   1.1  jmcneill 	paos.rp_local_port = 1;
   8125   1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
   8126   1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   8127   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   8128   1.1  jmcneill 	    sizeof(paos)) != 0) {
   8129   1.1  jmcneill 		printf("%s: unable to set port state to down\n", DEVNAME(sc));
   8130   1.1  jmcneill 		return EIO;
   8131   1.1  jmcneill 	}
   8132   1.1  jmcneill 
   8133   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8134   1.1  jmcneill 	ptys.rp_local_port = 1;
   8135   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8136   1.1  jmcneill 	ptys.rp_eth_proto_admin = htobe32(media);
   8137   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
   8138   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8139   1.1  jmcneill 		printf("%s: unable to set port media type/speed\n",
   8140   1.1  jmcneill 		    DEVNAME(sc));
   8141   1.1  jmcneill 		error = EIO;
   8142   1.1  jmcneill 	}
   8143   1.1  jmcneill 
   8144   1.1  jmcneill 	/* re-enable the port to start negotiation */
   8145   1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   8146   1.1  jmcneill 	paos.rp_local_port = 1;
   8147   1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
   8148   1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   8149   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   8150   1.1  jmcneill 	    sizeof(paos)) != 0) {
   8151   1.1  jmcneill 		printf("%s: unable to set port state to up\n", DEVNAME(sc));
   8152   1.1  jmcneill 		error = EIO;
   8153   1.1  jmcneill 	}
   8154   1.1  jmcneill 
   8155   1.1  jmcneill 	return error;
   8156   1.1  jmcneill }
   8157   1.1  jmcneill 
   8158   1.1  jmcneill static void
   8159   1.1  jmcneill mcx_port_change(struct work *wk, void *xsc)
   8160   1.1  jmcneill {
   8161   1.1  jmcneill 	struct mcx_softc *sc = xsc;
   8162   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   8163  1.13  jmcneill 	struct mcx_reg_ptys ptys = {
   8164  1.13  jmcneill 		.rp_local_port = 1,
   8165  1.13  jmcneill 		.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH,
   8166  1.13  jmcneill 	};
   8167   1.1  jmcneill 	int link_state = LINK_STATE_DOWN;
   8168   1.1  jmcneill 
   8169  1.13  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8170  1.13  jmcneill 	    sizeof(ptys)) == 0) {
   8171  1.13  jmcneill 		uint32_t proto_oper = be32toh(ptys.rp_eth_proto_oper);
   8172  1.13  jmcneill 		uint64_t baudrate = 0;
   8173  1.13  jmcneill 		unsigned int i;
   8174  1.13  jmcneill 
   8175  1.15  jmcneill 		if (proto_oper != 0)
   8176  1.15  jmcneill 			link_state = LINK_STATE_UP;
   8177  1.15  jmcneill 
   8178  1.13  jmcneill 		for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8179  1.13  jmcneill 			const struct mcx_eth_proto_capability *cap;
   8180  1.13  jmcneill 			if (!ISSET(proto_oper, 1U << i))
   8181  1.13  jmcneill 				continue;
   8182  1.13  jmcneill 
   8183  1.13  jmcneill 			cap = &mcx_eth_cap_map[i];
   8184  1.13  jmcneill 			if (cap->cap_baudrate == 0)
   8185  1.13  jmcneill 				continue;
   8186  1.13  jmcneill 
   8187  1.13  jmcneill 			baudrate = cap->cap_baudrate;
   8188  1.13  jmcneill 			break;
   8189  1.13  jmcneill 		}
   8190  1.13  jmcneill 
   8191  1.13  jmcneill 		ifp->if_baudrate = baudrate;
   8192   1.1  jmcneill 	}
   8193   1.1  jmcneill 
   8194   1.1  jmcneill 	if (link_state != ifp->if_link_state) {
   8195   1.1  jmcneill 		if_link_state_change(ifp, link_state);
   8196   1.1  jmcneill 	}
   8197   1.1  jmcneill }
   8198   1.1  jmcneill 
   8199   1.1  jmcneill 
   8200   1.1  jmcneill static inline uint32_t
   8201   1.1  jmcneill mcx_rd(struct mcx_softc *sc, bus_size_t r)
   8202   1.1  jmcneill {
   8203   1.1  jmcneill 	uint32_t word;
   8204   1.1  jmcneill 
   8205   1.1  jmcneill 	word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r);
   8206   1.1  jmcneill 
   8207   1.1  jmcneill 	return (be32toh(word));
   8208   1.1  jmcneill }
   8209   1.1  jmcneill 
   8210   1.1  jmcneill static inline void
   8211   1.1  jmcneill mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
   8212   1.1  jmcneill {
   8213   1.1  jmcneill 	bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
   8214   1.1  jmcneill }
   8215   1.1  jmcneill 
   8216   1.1  jmcneill static inline void
   8217   1.1  jmcneill mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
   8218   1.1  jmcneill {
   8219  1.21  jmcneill #ifndef __NetBSD__
   8220   1.1  jmcneill 	bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
   8221  1.21  jmcneill #endif
   8222   1.1  jmcneill }
   8223   1.1  jmcneill 
   8224   1.1  jmcneill static uint64_t
   8225   1.1  jmcneill mcx_timer(struct mcx_softc *sc)
   8226   1.1  jmcneill {
   8227   1.1  jmcneill 	uint32_t hi, lo, ni;
   8228   1.1  jmcneill 
   8229   1.1  jmcneill 	hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   8230   1.1  jmcneill 	for (;;) {
   8231   1.1  jmcneill 		lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
   8232   1.1  jmcneill 		mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
   8233   1.1  jmcneill 		ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   8234   1.1  jmcneill 
   8235   1.1  jmcneill 		if (ni == hi)
   8236   1.1  jmcneill 			break;
   8237   1.1  jmcneill 
   8238   1.1  jmcneill 		hi = ni;
   8239   1.1  jmcneill 	}
   8240   1.1  jmcneill 
   8241   1.1  jmcneill 	return (((uint64_t)hi << 32) | (uint64_t)lo);
   8242   1.1  jmcneill }
   8243   1.1  jmcneill 
   8244   1.1  jmcneill static int
   8245   1.1  jmcneill mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   8246   1.1  jmcneill     bus_size_t size, u_int align)
   8247   1.1  jmcneill {
   8248   1.1  jmcneill 	mxm->mxm_size = size;
   8249   1.1  jmcneill 
   8250   1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
   8251   1.1  jmcneill 	    mxm->mxm_size, 0,
   8252   1.1  jmcneill 	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   8253   1.1  jmcneill 	    &mxm->mxm_map) != 0)
   8254   1.1  jmcneill 		return (1);
   8255   1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
   8256   1.1  jmcneill 	    align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
   8257   1.1  jmcneill 	    BUS_DMA_WAITOK) != 0)
   8258   1.1  jmcneill 		goto destroy;
   8259   1.1  jmcneill 	if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
   8260  1.19  jmcneill 	    mxm->mxm_size, &mxm->mxm_kva,
   8261  1.19  jmcneill 	    BUS_DMA_WAITOK | BUS_DMA_COHERENT) != 0)
   8262   1.1  jmcneill 		goto free;
   8263   1.1  jmcneill 	if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
   8264   1.1  jmcneill 	    mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
   8265   1.1  jmcneill 		goto unmap;
   8266   1.1  jmcneill 
   8267   1.1  jmcneill 	mcx_dmamem_zero(mxm);
   8268   1.1  jmcneill 
   8269   1.1  jmcneill 	return (0);
   8270   1.1  jmcneill unmap:
   8271   1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   8272   1.1  jmcneill free:
   8273   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   8274   1.1  jmcneill destroy:
   8275   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   8276   1.1  jmcneill 	return (1);
   8277   1.1  jmcneill }
   8278   1.1  jmcneill 
   8279   1.1  jmcneill static void
   8280   1.1  jmcneill mcx_dmamem_zero(struct mcx_dmamem *mxm)
   8281   1.1  jmcneill {
   8282   1.1  jmcneill 	memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
   8283   1.1  jmcneill }
   8284   1.1  jmcneill 
   8285   1.1  jmcneill static void
   8286   1.1  jmcneill mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   8287   1.1  jmcneill {
   8288   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
   8289   1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   8290   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   8291   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   8292   1.1  jmcneill }
   8293   1.1  jmcneill 
   8294   1.1  jmcneill static int
   8295   1.1  jmcneill mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
   8296   1.1  jmcneill {
   8297   1.1  jmcneill 	bus_dma_segment_t *segs;
   8298   1.1  jmcneill 	bus_size_t len = pages * MCX_PAGE_SIZE;
   8299   1.1  jmcneill 	size_t seglen;
   8300   1.1  jmcneill 
   8301   1.1  jmcneill 	segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP);
   8302   1.1  jmcneill 	seglen = sizeof(*segs) * pages;
   8303   1.1  jmcneill 
   8304   1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
   8305   1.1  jmcneill 	    segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
   8306   1.1  jmcneill 		goto free_segs;
   8307   1.1  jmcneill 
   8308   1.1  jmcneill 	if (mhm->mhm_seg_count < pages) {
   8309   1.1  jmcneill 		size_t nseglen;
   8310   1.1  jmcneill 
   8311   1.1  jmcneill 		mhm->mhm_segs = kmem_alloc(
   8312   1.1  jmcneill 		    sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP);
   8313   1.1  jmcneill 
   8314   1.1  jmcneill 		nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
   8315   1.1  jmcneill 
   8316   1.1  jmcneill 		memcpy(mhm->mhm_segs, segs, nseglen);
   8317   1.1  jmcneill 
   8318   1.1  jmcneill 		kmem_free(segs, seglen);
   8319   1.1  jmcneill 
   8320   1.1  jmcneill 		segs = mhm->mhm_segs;
   8321   1.1  jmcneill 		seglen = nseglen;
   8322   1.1  jmcneill 	} else
   8323   1.1  jmcneill 		mhm->mhm_segs = segs;
   8324   1.1  jmcneill 
   8325   1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
   8326   1.1  jmcneill 	    MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
   8327   1.1  jmcneill 	    &mhm->mhm_map) != 0)
   8328   1.1  jmcneill 		goto free_dmamem;
   8329   1.1  jmcneill 
   8330   1.1  jmcneill 	if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
   8331   1.1  jmcneill 	    mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
   8332   1.1  jmcneill 		goto destroy;
   8333   1.1  jmcneill 
   8334   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   8335   1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
   8336   1.1  jmcneill 
   8337   1.1  jmcneill 	mhm->mhm_npages = pages;
   8338   1.1  jmcneill 
   8339   1.1  jmcneill 	return (0);
   8340   1.1  jmcneill 
   8341   1.1  jmcneill destroy:
   8342   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   8343   1.1  jmcneill free_dmamem:
   8344   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   8345   1.1  jmcneill free_segs:
   8346   1.1  jmcneill 	kmem_free(segs, seglen);
   8347   1.1  jmcneill 	mhm->mhm_segs = NULL;
   8348   1.1  jmcneill 
   8349   1.1  jmcneill 	return (-1);
   8350   1.1  jmcneill }
   8351   1.1  jmcneill 
   8352   1.1  jmcneill static void
   8353   1.1  jmcneill mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
   8354   1.1  jmcneill {
   8355   1.1  jmcneill 	if (mhm->mhm_npages == 0)
   8356   1.1  jmcneill 		return;
   8357   1.1  jmcneill 
   8358   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   8359   1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
   8360   1.1  jmcneill 
   8361   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
   8362   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   8363   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   8364   1.1  jmcneill 	kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
   8365   1.1  jmcneill 
   8366   1.1  jmcneill 	mhm->mhm_npages = 0;
   8367   1.1  jmcneill }
   8368  1.15  jmcneill 
   8369  1.15  jmcneill #if NKSTAT > 0
   8370  1.15  jmcneill struct mcx_ppcnt {
   8371  1.15  jmcneill 	char			 name[KSTAT_KV_NAMELEN];
   8372  1.15  jmcneill 	enum kstat_kv_unit	 unit;
   8373  1.15  jmcneill };
   8374  1.15  jmcneill 
   8375  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = {
   8376  1.15  jmcneill 	{ "Good Tx",		KSTAT_KV_U_PACKETS, },
   8377  1.15  jmcneill 	{ "Good Rx",		KSTAT_KV_U_PACKETS, },
   8378  1.15  jmcneill 	{ "FCS errs",		KSTAT_KV_U_PACKETS, },
   8379  1.15  jmcneill 	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
   8380  1.15  jmcneill 	{ "Good Tx",		KSTAT_KV_U_BYTES, },
   8381  1.15  jmcneill 	{ "Good Rx",		KSTAT_KV_U_BYTES, },
   8382  1.15  jmcneill 	{ "Multicast Tx",	KSTAT_KV_U_PACKETS, },
   8383  1.15  jmcneill 	{ "Broadcast Tx",	KSTAT_KV_U_PACKETS, },
   8384  1.15  jmcneill 	{ "Multicast Rx",	KSTAT_KV_U_PACKETS, },
   8385  1.15  jmcneill 	{ "Broadcast Rx",	KSTAT_KV_U_PACKETS, },
   8386  1.15  jmcneill 	{ "In Range Len",	KSTAT_KV_U_PACKETS, },
   8387  1.15  jmcneill 	{ "Out Of Range Len",	KSTAT_KV_U_PACKETS, },
   8388  1.15  jmcneill 	{ "Frame Too Long",	KSTAT_KV_U_PACKETS, },
   8389  1.15  jmcneill 	{ "Symbol Errs",	KSTAT_KV_U_PACKETS, },
   8390  1.15  jmcneill 	{ "MAC Ctrl Tx",	KSTAT_KV_U_PACKETS, },
   8391  1.15  jmcneill 	{ "MAC Ctrl Rx",	KSTAT_KV_U_PACKETS, },
   8392  1.15  jmcneill 	{ "MAC Ctrl Unsup",	KSTAT_KV_U_PACKETS, },
   8393  1.15  jmcneill 	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
   8394  1.15  jmcneill 	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
   8395  1.15  jmcneill };
   8396  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count);
   8397  1.15  jmcneill 
   8398  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = {
   8399  1.15  jmcneill 	{ "Rx Bytes",		KSTAT_KV_U_BYTES, },
   8400  1.15  jmcneill 	{ "Rx Unicast",		KSTAT_KV_U_PACKETS, },
   8401  1.15  jmcneill 	{ "Rx Discards",	KSTAT_KV_U_PACKETS, },
   8402  1.15  jmcneill 	{ "Rx Errors",		KSTAT_KV_U_PACKETS, },
   8403  1.15  jmcneill 	{ "Rx Unknown Proto",	KSTAT_KV_U_PACKETS, },
   8404  1.15  jmcneill 	{ "Tx Bytes",		KSTAT_KV_U_BYTES, },
   8405  1.15  jmcneill 	{ "Tx Unicast",		KSTAT_KV_U_PACKETS, },
   8406  1.15  jmcneill 	{ "Tx Discards",	KSTAT_KV_U_PACKETS, },
   8407  1.15  jmcneill 	{ "Tx Errors",		KSTAT_KV_U_PACKETS, },
   8408  1.15  jmcneill 	{ "Rx Multicast",	KSTAT_KV_U_PACKETS, },
   8409  1.15  jmcneill 	{ "Rx Broadcast",	KSTAT_KV_U_PACKETS, },
   8410  1.15  jmcneill 	{ "Tx Multicast",	KSTAT_KV_U_PACKETS, },
   8411  1.15  jmcneill 	{ "Tx Broadcast",	KSTAT_KV_U_PACKETS, },
   8412  1.15  jmcneill };
   8413  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count);
   8414  1.15  jmcneill 
   8415  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = {
   8416  1.15  jmcneill 	{ "Drop Events",	KSTAT_KV_U_PACKETS, },
   8417  1.15  jmcneill 	{ "Octets",		KSTAT_KV_U_BYTES, },
   8418  1.15  jmcneill 	{ "Packets",		KSTAT_KV_U_PACKETS, },
   8419  1.15  jmcneill 	{ "Broadcasts",		KSTAT_KV_U_PACKETS, },
   8420  1.15  jmcneill 	{ "Multicasts",		KSTAT_KV_U_PACKETS, },
   8421  1.15  jmcneill 	{ "CRC Align Errs",	KSTAT_KV_U_PACKETS, },
   8422  1.15  jmcneill 	{ "Undersize",		KSTAT_KV_U_PACKETS, },
   8423  1.15  jmcneill 	{ "Oversize",		KSTAT_KV_U_PACKETS, },
   8424  1.15  jmcneill 	{ "Fragments",		KSTAT_KV_U_PACKETS, },
   8425  1.15  jmcneill 	{ "Jabbers",		KSTAT_KV_U_PACKETS, },
   8426  1.15  jmcneill 	{ "Collisions",		KSTAT_KV_U_NONE, },
   8427  1.15  jmcneill 	{ "64B",		KSTAT_KV_U_PACKETS, },
   8428  1.15  jmcneill 	{ "65-127B",		KSTAT_KV_U_PACKETS, },
   8429  1.15  jmcneill 	{ "128-255B",		KSTAT_KV_U_PACKETS, },
   8430  1.15  jmcneill 	{ "256-511B",		KSTAT_KV_U_PACKETS, },
   8431  1.15  jmcneill 	{ "512-1023B",		KSTAT_KV_U_PACKETS, },
   8432  1.15  jmcneill 	{ "1024-1518B",		KSTAT_KV_U_PACKETS, },
   8433  1.15  jmcneill 	{ "1519-2047B",		KSTAT_KV_U_PACKETS, },
   8434  1.15  jmcneill 	{ "2048-4095B",		KSTAT_KV_U_PACKETS, },
   8435  1.15  jmcneill 	{ "4096-8191B",		KSTAT_KV_U_PACKETS, },
   8436  1.15  jmcneill 	{ "8192-10239B",	KSTAT_KV_U_PACKETS, },
   8437  1.15  jmcneill };
   8438  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count);
   8439  1.15  jmcneill 
   8440  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = {
   8441  1.15  jmcneill 	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
   8442  1.15  jmcneill 	{ "FCS Errs",		KSTAT_KV_U_PACKETS, },
   8443  1.15  jmcneill 	{ "Single Colls",	KSTAT_KV_U_PACKETS, },
   8444  1.15  jmcneill 	{ "Multiple Colls",	KSTAT_KV_U_PACKETS, },
   8445  1.15  jmcneill 	{ "SQE Test Errs",	KSTAT_KV_U_NONE, },
   8446  1.15  jmcneill 	{ "Deferred Tx",	KSTAT_KV_U_PACKETS, },
   8447  1.15  jmcneill 	{ "Late Colls",		KSTAT_KV_U_NONE, },
   8448  1.15  jmcneill 	{ "Exess Colls",	KSTAT_KV_U_NONE, },
   8449  1.15  jmcneill 	{ "Int MAC Tx Errs",	KSTAT_KV_U_PACKETS, },
   8450  1.15  jmcneill 	{ "CSM Sense Errs",	KSTAT_KV_U_NONE, },
   8451  1.15  jmcneill 	{ "Too Long",		KSTAT_KV_U_PACKETS, },
   8452  1.15  jmcneill 	{ "Int MAC Rx Errs",	KSTAT_KV_U_PACKETS, },
   8453  1.15  jmcneill 	{ "Symbol Errs",	KSTAT_KV_U_NONE, },
   8454  1.15  jmcneill 	{ "Unknown Control",	KSTAT_KV_U_PACKETS, },
   8455  1.15  jmcneill 	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
   8456  1.15  jmcneill 	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
   8457  1.15  jmcneill };
   8458  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count);
   8459  1.15  jmcneill 
   8460  1.15  jmcneill struct mcx_kstat_ppcnt {
   8461  1.15  jmcneill 	const char		*ksp_name;
   8462  1.15  jmcneill 	const struct mcx_ppcnt	*ksp_tpl;
   8463  1.15  jmcneill 	unsigned int		 ksp_n;
   8464  1.15  jmcneill 	uint8_t			 ksp_grp;
   8465  1.15  jmcneill };
   8466  1.15  jmcneill 
   8467  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = {
   8468  1.15  jmcneill 	.ksp_name =		"ieee802.3",
   8469  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_ieee8023_tpl,
   8470  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_ieee8023_tpl),
   8471  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_IEEE8023,
   8472  1.15  jmcneill };
   8473  1.15  jmcneill 
   8474  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = {
   8475  1.15  jmcneill 	.ksp_name =		"rfc2863",
   8476  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc2863_tpl,
   8477  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc2863_tpl),
   8478  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2863,
   8479  1.15  jmcneill };
   8480  1.15  jmcneill 
   8481  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = {
   8482  1.15  jmcneill 	.ksp_name =		"rfc2819",
   8483  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc2819_tpl,
   8484  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc2819_tpl),
   8485  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2819,
   8486  1.15  jmcneill };
   8487  1.15  jmcneill 
   8488  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = {
   8489  1.15  jmcneill 	.ksp_name =		"rfc3635",
   8490  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc3635_tpl,
   8491  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc3635_tpl),
   8492  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC3635,
   8493  1.15  jmcneill };
   8494  1.15  jmcneill 
   8495  1.15  jmcneill static int	mcx_kstat_ppcnt_read(struct kstat *);
   8496  1.15  jmcneill 
   8497  1.15  jmcneill static void	mcx_kstat_attach_tmps(struct mcx_softc *sc);
   8498  1.15  jmcneill static void	mcx_kstat_attach_queues(struct mcx_softc *sc);
   8499  1.15  jmcneill 
   8500  1.15  jmcneill static struct kstat *
   8501  1.15  jmcneill mcx_kstat_attach_ppcnt(struct mcx_softc *sc,
   8502  1.15  jmcneill     const struct mcx_kstat_ppcnt *ksp)
   8503  1.15  jmcneill {
   8504  1.15  jmcneill 	struct kstat *ks;
   8505  1.15  jmcneill 	struct kstat_kv *kvs;
   8506  1.15  jmcneill 	unsigned int i;
   8507  1.15  jmcneill 
   8508  1.15  jmcneill 	ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0);
   8509  1.15  jmcneill 	if (ks == NULL)
   8510  1.15  jmcneill 		return (NULL);
   8511  1.15  jmcneill 
   8512  1.15  jmcneill 	kvs = mallocarray(ksp->ksp_n, sizeof(*kvs),
   8513  1.15  jmcneill 	    M_DEVBUF, M_WAITOK);
   8514  1.15  jmcneill 
   8515  1.15  jmcneill 	for (i = 0; i < ksp->ksp_n; i++) {
   8516  1.15  jmcneill 		const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i];
   8517  1.15  jmcneill 
   8518  1.15  jmcneill 		kstat_kv_unit_init(&kvs[i], tpl->name,
   8519  1.15  jmcneill 		    KSTAT_KV_T_COUNTER64, tpl->unit);
   8520  1.15  jmcneill 	}
   8521  1.15  jmcneill 
   8522  1.15  jmcneill 	ks->ks_softc = sc;
   8523  1.15  jmcneill 	ks->ks_ptr = (void *)ksp;
   8524  1.15  jmcneill 	ks->ks_data = kvs;
   8525  1.15  jmcneill 	ks->ks_datalen = ksp->ksp_n * sizeof(*kvs);
   8526  1.15  jmcneill 	ks->ks_read = mcx_kstat_ppcnt_read;
   8527  1.15  jmcneill 
   8528  1.15  jmcneill 	kstat_install(ks);
   8529  1.15  jmcneill 
   8530  1.15  jmcneill 	return (ks);
   8531  1.15  jmcneill }
   8532  1.15  jmcneill 
   8533  1.15  jmcneill static void
   8534  1.15  jmcneill mcx_kstat_attach(struct mcx_softc *sc)
   8535  1.15  jmcneill {
   8536  1.15  jmcneill 	sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc,
   8537  1.15  jmcneill 	    &mcx_kstat_ppcnt_ieee8023);
   8538  1.15  jmcneill 	sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc,
   8539  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc2863);
   8540  1.15  jmcneill 	sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc,
   8541  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc2819);
   8542  1.15  jmcneill 	sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc,
   8543  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc3635);
   8544  1.15  jmcneill 
   8545  1.15  jmcneill 	mcx_kstat_attach_tmps(sc);
   8546  1.15  jmcneill 	mcx_kstat_attach_queues(sc);
   8547  1.15  jmcneill }
   8548  1.15  jmcneill 
   8549  1.15  jmcneill static int
   8550  1.15  jmcneill mcx_kstat_ppcnt_read(struct kstat *ks)
   8551  1.15  jmcneill {
   8552  1.15  jmcneill 	struct mcx_softc *sc = ks->ks_softc;
   8553  1.15  jmcneill 	struct mcx_kstat_ppcnt *ksp = ks->ks_ptr;
   8554  1.15  jmcneill 	struct mcx_reg_ppcnt ppcnt = {
   8555  1.15  jmcneill 		.ppcnt_grp = ksp->ksp_grp,
   8556  1.15  jmcneill 		.ppcnt_local_port = 1,
   8557  1.15  jmcneill 	};
   8558  1.15  jmcneill 	struct kstat_kv *kvs = ks->ks_data;
   8559  1.15  jmcneill 	uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set;
   8560  1.15  jmcneill 	unsigned int i;
   8561  1.15  jmcneill 	int rv;
   8562  1.15  jmcneill 
   8563  1.15  jmcneill 	KERNEL_LOCK(); /* XXX */
   8564  1.15  jmcneill 	rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ,
   8565  1.15  jmcneill 	    &ppcnt, sizeof(ppcnt));
   8566  1.15  jmcneill 	KERNEL_UNLOCK();
   8567  1.15  jmcneill 	if (rv != 0)
   8568  1.15  jmcneill 		return (EIO);
   8569  1.15  jmcneill 
   8570  1.15  jmcneill 	nanouptime(&ks->ks_updated);
   8571  1.15  jmcneill 
   8572  1.15  jmcneill 	for (i = 0; i < ksp->ksp_n; i++)
   8573  1.15  jmcneill 		kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]);
   8574  1.15  jmcneill 
   8575  1.15  jmcneill 	return (0);
   8576  1.15  jmcneill }
   8577  1.15  jmcneill 
   8578  1.15  jmcneill struct mcx_kstat_mtmp {
   8579  1.15  jmcneill 	struct kstat_kv		ktmp_name;
   8580  1.15  jmcneill 	struct kstat_kv		ktmp_temperature;
   8581  1.15  jmcneill 	struct kstat_kv		ktmp_threshold_lo;
   8582  1.15  jmcneill 	struct kstat_kv		ktmp_threshold_hi;
   8583  1.15  jmcneill };
   8584  1.15  jmcneill 
   8585  1.15  jmcneill static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = {
   8586  1.15  jmcneill 	KSTAT_KV_INITIALIZER("name",		KSTAT_KV_T_ISTR),
   8587  1.15  jmcneill 	KSTAT_KV_INITIALIZER("temperature",	KSTAT_KV_T_TEMP),
   8588  1.15  jmcneill 	KSTAT_KV_INITIALIZER("lo threshold",	KSTAT_KV_T_TEMP),
   8589  1.15  jmcneill 	KSTAT_KV_INITIALIZER("hi threshold",	KSTAT_KV_T_TEMP),
   8590  1.15  jmcneill };
   8591  1.15  jmcneill 
   8592  1.15  jmcneill static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 };
   8593  1.15  jmcneill 
   8594  1.15  jmcneill static int mcx_kstat_mtmp_read(struct kstat *);
   8595  1.15  jmcneill 
   8596  1.15  jmcneill static void
   8597  1.15  jmcneill mcx_kstat_attach_tmps(struct mcx_softc *sc)
   8598  1.15  jmcneill {
   8599  1.15  jmcneill 	struct kstat *ks;
   8600  1.15  jmcneill 	struct mcx_reg_mcam mcam;
   8601  1.15  jmcneill 	struct mcx_reg_mtcap mtcap;
   8602  1.15  jmcneill 	struct mcx_kstat_mtmp *ktmp;
   8603  1.15  jmcneill 	uint64_t map;
   8604  1.15  jmcneill 	unsigned int i, n;
   8605  1.15  jmcneill 
   8606  1.15  jmcneill 	memset(&mtcap, 0, sizeof(mtcap));
   8607  1.15  jmcneill 	memset(&mcam, 0, sizeof(mcam));
   8608  1.15  jmcneill 
   8609  1.15  jmcneill 	if (sc->sc_mcam_reg == 0) {
   8610  1.15  jmcneill 		/* no management capabilities */
   8611  1.15  jmcneill 		return;
   8612  1.15  jmcneill 	}
   8613  1.15  jmcneill 
   8614  1.15  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ,
   8615  1.15  jmcneill 	    &mcam, sizeof(mcam)) != 0) {
   8616  1.15  jmcneill 		/* unable to check management capabilities? */
   8617  1.15  jmcneill 		return;
   8618  1.15  jmcneill 	}
   8619  1.15  jmcneill 
   8620  1.15  jmcneill 	if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask,
   8621  1.15  jmcneill 	    MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) {
   8622  1.15  jmcneill 		/* no sensor map */
   8623  1.15  jmcneill 		return;
   8624  1.15  jmcneill 	}
   8625  1.15  jmcneill 
   8626  1.15  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ,
   8627  1.15  jmcneill 	    &mtcap, sizeof(mtcap)) != 0) {
   8628  1.15  jmcneill 		/* unable to find temperature sensors */
   8629  1.15  jmcneill 		return;
   8630  1.15  jmcneill 	}
   8631  1.15  jmcneill 
   8632  1.15  jmcneill 	sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count;
   8633  1.15  jmcneill 	sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count,
   8634  1.15  jmcneill 	    sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK);
   8635  1.15  jmcneill 
   8636  1.15  jmcneill 	n = 0;
   8637  1.15  jmcneill 	map = bemtoh64(&mtcap.mtcap_sensor_map);
   8638  1.15  jmcneill 	for (i = 0; i < sizeof(map) * NBBY; i++) {
   8639  1.15  jmcneill 		if (!ISSET(map, (1ULL << i)))
   8640  1.15  jmcneill 			continue;
   8641  1.15  jmcneill 
   8642  1.15  jmcneill 		ks = kstat_create(DEVNAME(sc), 0, "temperature", i,
   8643  1.15  jmcneill 		    KSTAT_T_KV, 0);
   8644  1.15  jmcneill 		if (ks == NULL) {
   8645  1.15  jmcneill 			/* unable to attach temperature sensor %u, i */
   8646  1.15  jmcneill 			continue;
   8647  1.15  jmcneill 		}
   8648  1.15  jmcneill 
   8649  1.15  jmcneill 		ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO);
   8650  1.15  jmcneill 		*ktmp = mcx_kstat_mtmp_tpl;
   8651  1.15  jmcneill 
   8652  1.15  jmcneill 		ks->ks_data = ktmp;
   8653  1.15  jmcneill 		ks->ks_datalen = sizeof(*ktmp);
   8654  1.15  jmcneill 		TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval);
   8655  1.15  jmcneill 		ks->ks_read = mcx_kstat_mtmp_read;
   8656  1.15  jmcneill 
   8657  1.15  jmcneill 		ks->ks_softc = sc;
   8658  1.15  jmcneill 		kstat_install(ks);
   8659  1.15  jmcneill 
   8660  1.15  jmcneill 		sc->sc_kstat_mtmp[n++] = ks;
   8661  1.15  jmcneill 		if (n >= sc->sc_kstat_mtmp_count)
   8662  1.15  jmcneill 			break;
   8663  1.15  jmcneill 	}
   8664  1.15  jmcneill }
   8665  1.15  jmcneill 
   8666  1.15  jmcneill static uint64_t
   8667  1.15  jmcneill mcx_tmp_to_uK(uint16_t *t)
   8668  1.15  jmcneill {
   8669  1.15  jmcneill 	int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */
   8670  1.15  jmcneill 	mt *= 1000000 / 8; /* convert to uC */
   8671  1.15  jmcneill 	mt += 273150000; /* convert to uK */
   8672  1.15  jmcneill 
   8673  1.15  jmcneill 	return (mt);
   8674  1.15  jmcneill }
   8675  1.15  jmcneill 
   8676  1.15  jmcneill static int
   8677  1.15  jmcneill mcx_kstat_mtmp_read(struct kstat *ks)
   8678  1.15  jmcneill {
   8679  1.15  jmcneill 	struct mcx_softc *sc = ks->ks_softc;
   8680  1.15  jmcneill 	struct mcx_kstat_mtmp *ktmp = ks->ks_data;
   8681  1.15  jmcneill 	struct mcx_reg_mtmp mtmp;
   8682  1.15  jmcneill 	int rv;
   8683  1.15  jmcneill 	struct timeval updated;
   8684  1.15  jmcneill 
   8685  1.15  jmcneill 	TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated);
   8686  1.15  jmcneill 
   8687  1.15  jmcneill 	if (!ratecheck(&updated, &mcx_kstat_mtmp_rate))
   8688  1.15  jmcneill 		return (0);
   8689  1.15  jmcneill 
   8690  1.15  jmcneill 	memset(&mtmp, 0, sizeof(mtmp));
   8691  1.15  jmcneill 	htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit);
   8692  1.15  jmcneill 
   8693  1.15  jmcneill 	KERNEL_LOCK(); /* XXX */
   8694  1.15  jmcneill 	rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ,
   8695  1.15  jmcneill 	    &mtmp, sizeof(mtmp));
   8696  1.15  jmcneill 	KERNEL_UNLOCK();
   8697  1.15  jmcneill 	if (rv != 0)
   8698  1.15  jmcneill 		return (EIO);
   8699  1.15  jmcneill 
   8700  1.15  jmcneill 	memset(kstat_kv_istr(&ktmp->ktmp_name), 0,
   8701  1.15  jmcneill 	    sizeof(kstat_kv_istr(&ktmp->ktmp_name)));
   8702  1.15  jmcneill 	memcpy(kstat_kv_istr(&ktmp->ktmp_name),
   8703  1.15  jmcneill 	    mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name));
   8704  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_temperature) =
   8705  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature);
   8706  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_threshold_lo) =
   8707  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo);
   8708  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_threshold_hi) =
   8709  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi);
   8710  1.15  jmcneill 
   8711  1.15  jmcneill 	TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated);
   8712  1.15  jmcneill 
   8713  1.15  jmcneill 	return (0);
   8714  1.15  jmcneill }
   8715  1.15  jmcneill 
   8716  1.15  jmcneill struct mcx_queuestat {
   8717  1.15  jmcneill 	char			 name[KSTAT_KV_NAMELEN];
   8718  1.15  jmcneill 	enum kstat_kv_type	 type;
   8719  1.15  jmcneill };
   8720  1.15  jmcneill 
   8721  1.15  jmcneill static const struct mcx_queuestat mcx_queue_kstat_tpl[] = {
   8722  1.15  jmcneill 	{ "RQ SW prod",		KSTAT_KV_T_COUNTER64 },
   8723  1.15  jmcneill 	{ "RQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8724  1.15  jmcneill 	{ "RQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8725  1.15  jmcneill 	{ "RQ HW state",	KSTAT_KV_T_ISTR },
   8726  1.15  jmcneill 
   8727  1.15  jmcneill 	{ "SQ SW prod",		KSTAT_KV_T_COUNTER64 },
   8728  1.15  jmcneill 	{ "SQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8729  1.15  jmcneill 	{ "SQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8730  1.15  jmcneill 	{ "SQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8731  1.15  jmcneill 	{ "SQ HW state",	KSTAT_KV_T_ISTR },
   8732  1.15  jmcneill 
   8733  1.15  jmcneill 	{ "CQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8734  1.15  jmcneill 	{ "CQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8735  1.15  jmcneill 	{ "CQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8736  1.15  jmcneill 	{ "CQ HW notify",	KSTAT_KV_T_COUNTER64 },
   8737  1.15  jmcneill 	{ "CQ HW solicit",	KSTAT_KV_T_COUNTER64 },
   8738  1.15  jmcneill 	{ "CQ HW status",	KSTAT_KV_T_ISTR },
   8739  1.15  jmcneill 	{ "CQ HW state",	KSTAT_KV_T_ISTR },
   8740  1.15  jmcneill 
   8741  1.15  jmcneill 	{ "EQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8742  1.15  jmcneill 	{ "EQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8743  1.15  jmcneill 	{ "EQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8744  1.15  jmcneill 	{ "EQ HW status",	KSTAT_KV_T_ISTR },
   8745  1.15  jmcneill 	{ "EQ HW state",	KSTAT_KV_T_ISTR },
   8746  1.15  jmcneill };
   8747  1.15  jmcneill 
   8748  1.15  jmcneill static int	mcx_kstat_queue_read(struct kstat *);
   8749  1.15  jmcneill 
   8750  1.15  jmcneill static void
   8751  1.15  jmcneill mcx_kstat_attach_queues(struct mcx_softc *sc)
   8752  1.15  jmcneill {
   8753  1.15  jmcneill 	struct kstat *ks;
   8754  1.15  jmcneill 	struct kstat_kv *kvs;
   8755  1.15  jmcneill 	int q, i;
   8756  1.15  jmcneill 
   8757  1.15  jmcneill 	for (q = 0; q < sc->sc_nqueues; q++) {
   8758  1.15  jmcneill 		ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
   8759  1.15  jmcneill 		    KSTAT_T_KV, 0);
   8760  1.15  jmcneill 		if (ks == NULL) {
   8761  1.15  jmcneill 			/* unable to attach queue stats %u, q */
   8762  1.15  jmcneill 			continue;
   8763  1.15  jmcneill 		}
   8764  1.15  jmcneill 
   8765  1.15  jmcneill 		kvs = mallocarray(nitems(mcx_queue_kstat_tpl),
   8766  1.15  jmcneill 		    sizeof(*kvs), M_DEVBUF, M_WAITOK);
   8767  1.15  jmcneill 
   8768  1.15  jmcneill 		for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) {
   8769  1.15  jmcneill 			const struct mcx_queuestat *tpl =
   8770  1.15  jmcneill 			    &mcx_queue_kstat_tpl[i];
   8771  1.15  jmcneill 
   8772  1.15  jmcneill 			kstat_kv_init(&kvs[i], tpl->name, tpl->type);
   8773  1.15  jmcneill 		}
   8774  1.15  jmcneill 
   8775  1.15  jmcneill 		ks->ks_softc = &sc->sc_queues[q];
   8776  1.15  jmcneill 		ks->ks_data = kvs;
   8777  1.15  jmcneill 		ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs);
   8778  1.15  jmcneill 		ks->ks_read = mcx_kstat_queue_read;
   8779  1.15  jmcneill 
   8780  1.15  jmcneill 		sc->sc_queues[q].q_kstat = ks;
   8781  1.15  jmcneill 		kstat_install(ks);
   8782  1.15  jmcneill 	}
   8783  1.15  jmcneill }
   8784  1.15  jmcneill 
   8785  1.15  jmcneill static int
   8786  1.15  jmcneill mcx_kstat_queue_read(struct kstat *ks)
   8787  1.15  jmcneill {
   8788  1.15  jmcneill 	struct mcx_queues *q = ks->ks_softc;
   8789  1.15  jmcneill 	struct mcx_softc *sc = q->q_sc;
   8790  1.15  jmcneill 	struct kstat_kv *kvs = ks->ks_data;
   8791  1.15  jmcneill 	union {
   8792  1.15  jmcneill 		struct mcx_rq_ctx rq;
   8793  1.15  jmcneill 		struct mcx_sq_ctx sq;
   8794  1.15  jmcneill 		struct mcx_cq_ctx cq;
   8795  1.15  jmcneill 		struct mcx_eq_ctx eq;
   8796  1.15  jmcneill 	} u;
   8797  1.15  jmcneill 	const char *text;
   8798  1.15  jmcneill 	int error = 0;
   8799  1.15  jmcneill 
   8800  1.15  jmcneill 	KERNEL_LOCK();
   8801  1.24     skrll 
   8802  1.15  jmcneill 	if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) {
   8803  1.15  jmcneill 		error = EIO;
   8804  1.15  jmcneill 		goto out;
   8805  1.15  jmcneill 	}
   8806  1.15  jmcneill 
   8807  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_rx.rx_prod;
   8808  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter);
   8809  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter);
   8810  1.15  jmcneill 	switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >>
   8811  1.15  jmcneill 	    MCX_RQ_CTX_STATE_SHIFT) {
   8812  1.15  jmcneill 	case MCX_RQ_CTX_STATE_RST:
   8813  1.15  jmcneill 		text = "RST";
   8814  1.15  jmcneill 		break;
   8815  1.15  jmcneill 	case MCX_RQ_CTX_STATE_RDY:
   8816  1.15  jmcneill 		text = "RDY";
   8817  1.15  jmcneill 		break;
   8818  1.15  jmcneill 	case MCX_RQ_CTX_STATE_ERR:
   8819  1.15  jmcneill 		text = "ERR";
   8820  1.15  jmcneill 		break;
   8821  1.15  jmcneill 	default:
   8822  1.15  jmcneill 		text = "unknown";
   8823  1.15  jmcneill 		break;
   8824  1.15  jmcneill 	}
   8825  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8826  1.15  jmcneill 	kvs++;
   8827  1.15  jmcneill 
   8828  1.15  jmcneill 	if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) {
   8829  1.15  jmcneill 		error = EIO;
   8830  1.15  jmcneill 		goto out;
   8831  1.15  jmcneill 	}
   8832  1.15  jmcneill 
   8833  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_tx.tx_prod;
   8834  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_tx.tx_cons;
   8835  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter);
   8836  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter);
   8837  1.15  jmcneill 	switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >>
   8838  1.15  jmcneill 	    MCX_SQ_CTX_STATE_SHIFT) {
   8839  1.15  jmcneill 	case MCX_SQ_CTX_STATE_RST:
   8840  1.15  jmcneill 		text = "RST";
   8841  1.15  jmcneill 		break;
   8842  1.15  jmcneill 	case MCX_SQ_CTX_STATE_RDY:
   8843  1.15  jmcneill 		text = "RDY";
   8844  1.15  jmcneill 		break;
   8845  1.15  jmcneill 	case MCX_SQ_CTX_STATE_ERR:
   8846  1.15  jmcneill 		text = "ERR";
   8847  1.15  jmcneill 		break;
   8848  1.15  jmcneill 	default:
   8849  1.15  jmcneill 		text = "unknown";
   8850  1.15  jmcneill 		break;
   8851  1.15  jmcneill 	}
   8852  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8853  1.15  jmcneill 	kvs++;
   8854  1.15  jmcneill 
   8855  1.15  jmcneill 	if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) {
   8856  1.15  jmcneill 		error = EIO;
   8857  1.15  jmcneill 		goto out;
   8858  1.15  jmcneill 	}
   8859  1.15  jmcneill 
   8860  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_cq.cq_cons;
   8861  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter);
   8862  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter);
   8863  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified);
   8864  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit);
   8865  1.15  jmcneill 
   8866  1.15  jmcneill 	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >>
   8867  1.15  jmcneill 	    MCX_CQ_CTX_STATUS_SHIFT) {
   8868  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_OK:
   8869  1.15  jmcneill 		text = "OK";
   8870  1.15  jmcneill 		break;
   8871  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_OVERFLOW:
   8872  1.15  jmcneill 		text = "overflow";
   8873  1.15  jmcneill 		break;
   8874  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_WRITE_FAIL:
   8875  1.15  jmcneill 		text = "write fail";
   8876  1.15  jmcneill 		break;
   8877  1.15  jmcneill 	default:
   8878  1.15  jmcneill 		text = "unknown";
   8879  1.15  jmcneill 		break;
   8880  1.15  jmcneill 	}
   8881  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8882  1.15  jmcneill 	kvs++;
   8883  1.15  jmcneill 
   8884  1.15  jmcneill 	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >>
   8885  1.15  jmcneill 	    MCX_CQ_CTX_STATE_SHIFT) {
   8886  1.15  jmcneill 	case MCX_CQ_CTX_STATE_SOLICITED:
   8887  1.15  jmcneill 		text = "solicited";
   8888  1.15  jmcneill 		break;
   8889  1.15  jmcneill 	case MCX_CQ_CTX_STATE_ARMED:
   8890  1.15  jmcneill 		text = "armed";
   8891  1.15  jmcneill 		break;
   8892  1.15  jmcneill 	case MCX_CQ_CTX_STATE_FIRED:
   8893  1.15  jmcneill 		text = "fired";
   8894  1.15  jmcneill 		break;
   8895  1.15  jmcneill 	default:
   8896  1.15  jmcneill 		text = "unknown";
   8897  1.15  jmcneill 		break;
   8898  1.15  jmcneill 	}
   8899  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8900  1.15  jmcneill 	kvs++;
   8901  1.15  jmcneill 
   8902  1.15  jmcneill 	if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) {
   8903  1.15  jmcneill 		error = EIO;
   8904  1.15  jmcneill 		goto out;
   8905  1.15  jmcneill 	}
   8906  1.15  jmcneill 
   8907  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_eq.eq_cons;
   8908  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter);
   8909  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter);
   8910  1.15  jmcneill 
   8911  1.15  jmcneill 	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >>
   8912  1.15  jmcneill 	    MCX_EQ_CTX_STATUS_SHIFT) {
   8913  1.15  jmcneill 	case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE:
   8914  1.15  jmcneill 		text = "write fail";
   8915  1.15  jmcneill 		break;
   8916  1.15  jmcneill 	case MCX_EQ_CTX_STATUS_OK:
   8917  1.15  jmcneill 		text = "OK";
   8918  1.15  jmcneill 		break;
   8919  1.15  jmcneill 	default:
   8920  1.15  jmcneill 		text = "unknown";
   8921  1.15  jmcneill 		break;
   8922  1.15  jmcneill 	}
   8923  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8924  1.15  jmcneill 	kvs++;
   8925  1.15  jmcneill 
   8926  1.15  jmcneill 	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >>
   8927  1.15  jmcneill 	    MCX_EQ_CTX_STATE_SHIFT) {
   8928  1.15  jmcneill 	case MCX_EQ_CTX_STATE_ARMED:
   8929  1.15  jmcneill 		text = "armed";
   8930  1.15  jmcneill 		break;
   8931  1.15  jmcneill 	case MCX_EQ_CTX_STATE_FIRED:
   8932  1.15  jmcneill 		text = "fired";
   8933  1.15  jmcneill 		break;
   8934  1.15  jmcneill 	default:
   8935  1.15  jmcneill 		text = "unknown";
   8936  1.15  jmcneill 		break;
   8937  1.15  jmcneill 	}
   8938  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8939  1.15  jmcneill 	kvs++;
   8940  1.15  jmcneill 
   8941  1.15  jmcneill 	nanouptime(&ks->ks_updated);
   8942  1.15  jmcneill out:
   8943  1.15  jmcneill 	KERNEL_UNLOCK();
   8944  1.15  jmcneill 	return (error);
   8945  1.15  jmcneill }
   8946  1.15  jmcneill 
   8947  1.15  jmcneill #endif /* NKSTAT > 0 */
   8948  1.15  jmcneill 
   8949  1.15  jmcneill static unsigned int
   8950  1.15  jmcneill mcx_timecounter_read(struct timecounter *tc)
   8951  1.15  jmcneill {
   8952  1.15  jmcneill 	struct mcx_softc *sc = tc->tc_priv;
   8953  1.15  jmcneill 
   8954  1.15  jmcneill 	return (mcx_rd(sc, MCX_INTERNAL_TIMER_L));
   8955  1.15  jmcneill }
   8956  1.15  jmcneill 
   8957  1.15  jmcneill static void
   8958  1.15  jmcneill mcx_timecounter_attach(struct mcx_softc *sc)
   8959  1.15  jmcneill {
   8960  1.15  jmcneill 	struct timecounter *tc = &sc->sc_timecounter;
   8961  1.15  jmcneill 
   8962  1.15  jmcneill 	tc->tc_get_timecount = mcx_timecounter_read;
   8963  1.15  jmcneill 	tc->tc_counter_mask = ~0U;
   8964  1.15  jmcneill 	tc->tc_frequency = sc->sc_khz * 1000;
   8965  1.15  jmcneill 	tc->tc_name = device_xname(sc->sc_dev);
   8966  1.15  jmcneill 	tc->tc_quality = -100;
   8967  1.15  jmcneill 	tc->tc_priv = sc;
   8968  1.15  jmcneill 
   8969  1.15  jmcneill 	tc_init(tc);
   8970  1.15  jmcneill }
   8971