Home | History | Annotate | Line # | Download | only in pci
if_mcx.c revision 1.29
      1  1.29   msaitoh /*	$NetBSD: if_mcx.c,v 1.29 2024/07/07 23:29:04 msaitoh Exp $ */
      2  1.20  jmcneill /*	$OpenBSD: if_mcx.c,v 1.101 2021/06/02 19:16:11 patrick Exp $ */
      3   1.1  jmcneill 
      4   1.1  jmcneill /*
      5   1.1  jmcneill  * Copyright (c) 2017 David Gwynne <dlg (at) openbsd.org>
      6   1.1  jmcneill  * Copyright (c) 2019 Jonathan Matthew <jmatthew (at) openbsd.org>
      7   1.1  jmcneill  *
      8   1.1  jmcneill  * Permission to use, copy, modify, and distribute this software for any
      9   1.1  jmcneill  * purpose with or without fee is hereby granted, provided that the above
     10   1.1  jmcneill  * copyright notice and this permission notice appear in all copies.
     11   1.1  jmcneill  *
     12   1.1  jmcneill  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     13   1.1  jmcneill  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     14   1.1  jmcneill  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     15   1.1  jmcneill  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     16   1.1  jmcneill  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     17   1.1  jmcneill  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     18   1.1  jmcneill  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     19   1.1  jmcneill  */
     20   1.1  jmcneill 
     21   1.1  jmcneill #ifdef _KERNEL_OPT
     22   1.1  jmcneill #include "opt_net_mpsafe.h"
     23   1.1  jmcneill #endif
     24   1.1  jmcneill 
     25  1.14  jmcneill #include <sys/cdefs.h>
     26  1.29   msaitoh __KERNEL_RCSID(0, "$NetBSD: if_mcx.c,v 1.29 2024/07/07 23:29:04 msaitoh Exp $");
     27  1.14  jmcneill 
     28   1.1  jmcneill #include <sys/param.h>
     29   1.1  jmcneill #include <sys/systm.h>
     30   1.1  jmcneill #include <sys/sockio.h>
     31   1.1  jmcneill #include <sys/mbuf.h>
     32   1.1  jmcneill #include <sys/kernel.h>
     33   1.1  jmcneill #include <sys/socket.h>
     34   1.1  jmcneill #include <sys/device.h>
     35   1.1  jmcneill #include <sys/pool.h>
     36   1.1  jmcneill #include <sys/queue.h>
     37   1.1  jmcneill #include <sys/callout.h>
     38   1.1  jmcneill #include <sys/workqueue.h>
     39   1.1  jmcneill #include <sys/atomic.h>
     40  1.15  jmcneill #include <sys/timetc.h>
     41   1.1  jmcneill #include <sys/kmem.h>
     42   1.1  jmcneill #include <sys/bus.h>
     43  1.15  jmcneill #include <sys/interrupt.h>
     44  1.16  jmcneill #include <sys/pcq.h>
     45  1.16  jmcneill #include <sys/cpu.h>
     46  1.23    andvar #include <sys/bitops.h>
     47   1.1  jmcneill 
     48   1.1  jmcneill #include <machine/intr.h>
     49   1.1  jmcneill 
     50   1.1  jmcneill #include <net/if.h>
     51   1.1  jmcneill #include <net/if_dl.h>
     52   1.1  jmcneill #include <net/if_ether.h>
     53   1.1  jmcneill #include <net/if_media.h>
     54  1.15  jmcneill #include <net/if_vlanvar.h>
     55  1.15  jmcneill #include <net/toeplitz.h>
     56   1.1  jmcneill 
     57   1.1  jmcneill #include <net/bpf.h>
     58   1.1  jmcneill 
     59   1.1  jmcneill #include <netinet/in.h>
     60   1.1  jmcneill 
     61   1.1  jmcneill #include <dev/pci/pcireg.h>
     62   1.1  jmcneill #include <dev/pci/pcivar.h>
     63   1.1  jmcneill #include <dev/pci/pcidevs.h>
     64   1.1  jmcneill 
     65  1.15  jmcneill /* TODO: Port kstat key/value stuff to evcnt/sysmon */
     66  1.15  jmcneill #define	NKSTAT		0
     67  1.15  jmcneill 
     68  1.12   thorpej /* XXX This driver is not yet MP-safe; don't claim to be! */
     69  1.12   thorpej /* #ifdef NET_MPSAFE */
     70  1.12   thorpej /* #define	MCX_MPSAFE	1 */
     71  1.12   thorpej /* #define	CALLOUT_FLAGS	CALLOUT_MPSAFE */
     72  1.12   thorpej /* #else */
     73   1.1  jmcneill #define	CALLOUT_FLAGS	0
     74  1.12   thorpej /* #endif */
     75   1.1  jmcneill 
     76  1.16  jmcneill #define	MCX_TXQ_NUM		2048
     77  1.16  jmcneill 
     78   1.1  jmcneill #define BUS_DMASYNC_PRERW	(BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
     79   1.1  jmcneill #define BUS_DMASYNC_POSTRW	(BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
     80   1.1  jmcneill 
     81   1.1  jmcneill #define MCX_HCA_BAR	PCI_MAPREG_START /* BAR 0 */
     82   1.1  jmcneill 
     83  1.15  jmcneill #define MCX_FW_VER			0x0000
     84  1.15  jmcneill #define  MCX_FW_VER_MAJOR(_v)			((_v) & 0xffff)
     85  1.15  jmcneill #define  MCX_FW_VER_MINOR(_v)			((_v) >> 16)
     86  1.15  jmcneill #define MCX_CMDIF_FW_SUBVER		0x0004
     87  1.15  jmcneill #define  MCX_FW_VER_SUBMINOR(_v)		((_v) & 0xffff)
     88  1.15  jmcneill #define  MCX_CMDIF(_v)				((_v) >> 16)
     89   1.1  jmcneill 
     90  1.15  jmcneill #define MCX_ISSI			1 /* as per the PRM */
     91  1.15  jmcneill #define MCX_CMD_IF_SUPPORTED		5
     92   1.1  jmcneill 
     93  1.15  jmcneill #define MCX_HARDMTU			9500
     94   1.1  jmcneill 
     95  1.15  jmcneill #define MCX_PAGE_SHIFT			12
     96  1.15  jmcneill #define MCX_PAGE_SIZE			(1 << MCX_PAGE_SHIFT)
     97   1.1  jmcneill 
     98   1.1  jmcneill /* queue sizes */
     99  1.15  jmcneill #define MCX_LOG_EQ_SIZE			7
    100  1.15  jmcneill #define MCX_LOG_CQ_SIZE			12
    101  1.15  jmcneill #define MCX_LOG_RQ_SIZE			10
    102  1.15  jmcneill #define MCX_LOG_SQ_SIZE			11
    103  1.15  jmcneill 
    104  1.15  jmcneill #define MCX_MAX_QUEUES			16
    105   1.1  jmcneill 
    106   1.1  jmcneill /* completion event moderation - about 10khz, or 90% of the cq */
    107  1.15  jmcneill #define MCX_CQ_MOD_PERIOD		50
    108  1.15  jmcneill #define MCX_CQ_MOD_COUNTER		\
    109  1.15  jmcneill 	(((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)
    110  1.15  jmcneill 
    111  1.15  jmcneill #define MCX_LOG_SQ_ENTRY_SIZE		6
    112  1.15  jmcneill #define MCX_SQ_ENTRY_MAX_SLOTS		4
    113  1.15  jmcneill #define MCX_SQ_SEGS_PER_SLOT		\
    114   1.1  jmcneill 	(sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
    115  1.15  jmcneill #define MCX_SQ_MAX_SEGMENTS		\
    116   1.1  jmcneill 	1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)
    117   1.1  jmcneill 
    118  1.15  jmcneill #define MCX_LOG_FLOW_TABLE_SIZE		5
    119  1.15  jmcneill #define MCX_NUM_STATIC_FLOWS		4 /* promisc, allmulti, ucast, bcast */
    120  1.15  jmcneill #define MCX_NUM_MCAST_FLOWS 		\
    121   1.1  jmcneill 	((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)
    122   1.1  jmcneill 
    123  1.15  jmcneill #define MCX_SQ_INLINE_SIZE		18
    124  1.15  jmcneill CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE);
    125   1.1  jmcneill 
    126   1.1  jmcneill /* doorbell offsets */
    127  1.15  jmcneill #define MCX_DOORBELL_AREA_SIZE		MCX_PAGE_SIZE
    128   1.1  jmcneill 
    129  1.15  jmcneill #define MCX_CQ_DOORBELL_BASE		0
    130  1.15  jmcneill #define MCX_CQ_DOORBELL_STRIDE		64
    131  1.15  jmcneill 
    132  1.15  jmcneill #define MCX_WQ_DOORBELL_BASE		MCX_PAGE_SIZE/2
    133  1.15  jmcneill #define MCX_WQ_DOORBELL_STRIDE		64
    134  1.15  jmcneill /* make sure the doorbells fit */
    135  1.15  jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE);
    136  1.15  jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE <
    137  1.15  jmcneill     MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE);
    138  1.15  jmcneill 
    139  1.15  jmcneill #define MCX_WQ_DOORBELL_MASK		0xffff
    140   1.1  jmcneill 
    141   1.1  jmcneill /* uar registers */
    142  1.15  jmcneill #define MCX_UAR_CQ_DOORBELL		0x20
    143  1.15  jmcneill #define MCX_UAR_EQ_DOORBELL_ARM		0x40
    144  1.15  jmcneill #define MCX_UAR_EQ_DOORBELL		0x48
    145  1.15  jmcneill #define MCX_UAR_BF			0x800
    146   1.1  jmcneill 
    147  1.15  jmcneill #define MCX_CMDQ_ADDR_HI		0x0010
    148  1.15  jmcneill #define MCX_CMDQ_ADDR_LO		0x0014
    149   1.1  jmcneill #define MCX_CMDQ_ADDR_NMASK		0xfff
    150   1.1  jmcneill #define MCX_CMDQ_LOG_SIZE(_v)		((_v) >> 4 & 0xf)
    151   1.1  jmcneill #define MCX_CMDQ_LOG_STRIDE(_v)		((_v) >> 0 & 0xf)
    152   1.1  jmcneill #define MCX_CMDQ_INTERFACE_MASK		(0x3 << 8)
    153   1.1  jmcneill #define MCX_CMDQ_INTERFACE_FULL_DRIVER	(0x0 << 8)
    154   1.1  jmcneill #define MCX_CMDQ_INTERFACE_DISABLED	(0x1 << 8)
    155   1.1  jmcneill 
    156   1.1  jmcneill #define MCX_CMDQ_DOORBELL		0x0018
    157   1.1  jmcneill 
    158  1.15  jmcneill #define MCX_STATE			0x01fc
    159  1.15  jmcneill #define MCX_STATE_MASK				(1U << 31)
    160  1.15  jmcneill #define MCX_STATE_INITIALIZING			(1 << 31)
    161  1.15  jmcneill #define MCX_STATE_READY				(0 << 31)
    162  1.15  jmcneill #define MCX_STATE_INTERFACE_MASK		(0x3 << 24)
    163  1.15  jmcneill #define MCX_STATE_INTERFACE_FULL_DRIVER		(0x0 << 24)
    164  1.15  jmcneill #define MCX_STATE_INTERFACE_DISABLED		(0x1 << 24)
    165  1.15  jmcneill 
    166  1.15  jmcneill #define MCX_INTERNAL_TIMER		0x1000
    167  1.15  jmcneill #define MCX_INTERNAL_TIMER_H		0x1000
    168  1.15  jmcneill #define MCX_INTERNAL_TIMER_L		0x1004
    169  1.15  jmcneill 
    170  1.15  jmcneill #define MCX_CLEAR_INT			0x100c
    171  1.15  jmcneill 
    172  1.15  jmcneill #define MCX_REG_OP_WRITE		0
    173  1.15  jmcneill #define MCX_REG_OP_READ			1
    174  1.15  jmcneill 
    175  1.15  jmcneill #define MCX_REG_PMLP			0x5002
    176  1.15  jmcneill #define MCX_REG_PMTU			0x5003
    177  1.15  jmcneill #define MCX_REG_PTYS			0x5004
    178  1.15  jmcneill #define MCX_REG_PAOS			0x5006
    179  1.15  jmcneill #define MCX_REG_PFCC			0x5007
    180  1.15  jmcneill #define MCX_REG_PPCNT			0x5008
    181  1.15  jmcneill #define MCX_REG_MTCAP			0x9009 /* mgmt temp capabilities */
    182  1.15  jmcneill #define MCX_REG_MTMP			0x900a /* mgmt temp */
    183  1.15  jmcneill #define MCX_REG_MCIA			0x9014
    184  1.15  jmcneill #define MCX_REG_MCAM			0x907f
    185  1.15  jmcneill 
    186  1.15  jmcneill #define MCX_ETHER_CAP_SGMII		0
    187  1.15  jmcneill #define MCX_ETHER_CAP_1000_KX		1
    188  1.15  jmcneill #define MCX_ETHER_CAP_10G_CX4		2
    189  1.15  jmcneill #define MCX_ETHER_CAP_10G_KX4		3
    190  1.15  jmcneill #define MCX_ETHER_CAP_10G_KR		4
    191  1.15  jmcneill #define MCX_ETHER_CAP_20G_KR2		5
    192  1.15  jmcneill #define MCX_ETHER_CAP_40G_CR4		6
    193  1.15  jmcneill #define MCX_ETHER_CAP_40G_KR4		7
    194  1.15  jmcneill #define MCX_ETHER_CAP_56G_R4		8
    195  1.15  jmcneill #define MCX_ETHER_CAP_10G_CR		12
    196  1.15  jmcneill #define MCX_ETHER_CAP_10G_SR		13
    197  1.15  jmcneill #define MCX_ETHER_CAP_10G_LR		14
    198  1.15  jmcneill #define MCX_ETHER_CAP_40G_SR4		15
    199  1.15  jmcneill #define MCX_ETHER_CAP_40G_LR4		16
    200  1.15  jmcneill #define MCX_ETHER_CAP_50G_SR2		18
    201  1.15  jmcneill #define MCX_ETHER_CAP_100G_CR4		20
    202  1.15  jmcneill #define MCX_ETHER_CAP_100G_SR4		21
    203  1.15  jmcneill #define MCX_ETHER_CAP_100G_KR4		22
    204  1.15  jmcneill #define MCX_ETHER_CAP_100G_LR4		23
    205  1.15  jmcneill #define MCX_ETHER_CAP_100_TX		24
    206  1.15  jmcneill #define MCX_ETHER_CAP_1000_T		25
    207  1.15  jmcneill #define MCX_ETHER_CAP_10G_T		26
    208  1.15  jmcneill #define MCX_ETHER_CAP_25G_CR		27
    209  1.15  jmcneill #define MCX_ETHER_CAP_25G_KR		28
    210  1.15  jmcneill #define MCX_ETHER_CAP_25G_SR		29
    211  1.15  jmcneill #define MCX_ETHER_CAP_50G_CR2		30
    212  1.15  jmcneill #define MCX_ETHER_CAP_50G_KR2		31
    213  1.15  jmcneill 
    214  1.15  jmcneill #define MCX_MAX_CQE			32
    215  1.15  jmcneill 
    216  1.15  jmcneill #define MCX_CMD_QUERY_HCA_CAP		0x100
    217  1.15  jmcneill #define MCX_CMD_QUERY_ADAPTER		0x101
    218  1.15  jmcneill #define MCX_CMD_INIT_HCA		0x102
    219  1.15  jmcneill #define MCX_CMD_TEARDOWN_HCA		0x103
    220  1.15  jmcneill #define MCX_CMD_ENABLE_HCA		0x104
    221  1.15  jmcneill #define MCX_CMD_DISABLE_HCA		0x105
    222  1.15  jmcneill #define MCX_CMD_QUERY_PAGES		0x107
    223  1.15  jmcneill #define MCX_CMD_MANAGE_PAGES		0x108
    224  1.15  jmcneill #define MCX_CMD_SET_HCA_CAP		0x109
    225  1.15  jmcneill #define MCX_CMD_QUERY_ISSI		0x10a
    226  1.15  jmcneill #define MCX_CMD_SET_ISSI		0x10b
    227  1.15  jmcneill #define MCX_CMD_SET_DRIVER_VERSION	0x10d
    228  1.15  jmcneill #define MCX_CMD_QUERY_SPECIAL_CONTEXTS	0x203
    229  1.15  jmcneill #define MCX_CMD_CREATE_EQ		0x301
    230  1.15  jmcneill #define MCX_CMD_DESTROY_EQ		0x302
    231  1.15  jmcneill #define MCX_CMD_QUERY_EQ		0x303
    232  1.15  jmcneill #define MCX_CMD_CREATE_CQ		0x400
    233  1.15  jmcneill #define MCX_CMD_DESTROY_CQ		0x401
    234  1.15  jmcneill #define MCX_CMD_QUERY_CQ		0x402
    235  1.15  jmcneill #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754
    236   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
    237  1.15  jmcneill 					0x755
    238  1.15  jmcneill #define MCX_CMD_QUERY_VPORT_COUNTERS 	0x770
    239  1.15  jmcneill #define MCX_CMD_ALLOC_PD		0x800
    240  1.15  jmcneill #define MCX_CMD_ALLOC_UAR		0x802
    241  1.15  jmcneill #define MCX_CMD_ACCESS_REG		0x805
    242  1.15  jmcneill #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN	0x816
    243  1.15  jmcneill #define MCX_CMD_CREATE_TIR		0x900
    244  1.15  jmcneill #define MCX_CMD_DESTROY_TIR		0x902
    245  1.15  jmcneill #define MCX_CMD_CREATE_SQ		0x904
    246  1.15  jmcneill #define MCX_CMD_MODIFY_SQ		0x905
    247  1.15  jmcneill #define MCX_CMD_DESTROY_SQ		0x906
    248  1.15  jmcneill #define MCX_CMD_QUERY_SQ		0x907
    249  1.15  jmcneill #define MCX_CMD_CREATE_RQ		0x908
    250  1.15  jmcneill #define MCX_CMD_MODIFY_RQ		0x909
    251  1.15  jmcneill #define MCX_CMD_DESTROY_RQ		0x90a
    252  1.15  jmcneill #define MCX_CMD_QUERY_RQ		0x90b
    253  1.15  jmcneill #define MCX_CMD_CREATE_TIS		0x912
    254  1.15  jmcneill #define MCX_CMD_DESTROY_TIS		0x914
    255  1.15  jmcneill #define MCX_CMD_CREATE_RQT		0x916
    256  1.15  jmcneill #define MCX_CMD_DESTROY_RQT		0x918
    257  1.15  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ROOT	0x92f
    258  1.15  jmcneill #define MCX_CMD_CREATE_FLOW_TABLE	0x930
    259  1.15  jmcneill #define MCX_CMD_DESTROY_FLOW_TABLE	0x931
    260  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE	0x932
    261  1.15  jmcneill #define MCX_CMD_CREATE_FLOW_GROUP	0x933
    262  1.15  jmcneill #define MCX_CMD_DESTROY_FLOW_GROUP	0x934
    263  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_GROUP	0x935
    264  1.15  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ENTRY	0x936
    265  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY	0x937
    266  1.15  jmcneill #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY	0x938
    267  1.15  jmcneill #define MCX_CMD_ALLOC_FLOW_COUNTER	0x939
    268  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_COUNTER	0x93b
    269  1.15  jmcneill 
    270  1.15  jmcneill #define MCX_QUEUE_STATE_RST		0
    271  1.15  jmcneill #define MCX_QUEUE_STATE_RDY		1
    272  1.15  jmcneill #define MCX_QUEUE_STATE_ERR		3
    273   1.1  jmcneill 
    274  1.15  jmcneill #define MCX_FLOW_TABLE_TYPE_RX		0
    275  1.15  jmcneill #define MCX_FLOW_TABLE_TYPE_TX		1
    276   1.1  jmcneill 
    277  1.15  jmcneill #define MCX_CMDQ_INLINE_DATASIZE	16
    278   1.1  jmcneill 
    279   1.1  jmcneill struct mcx_cmdq_entry {
    280   1.1  jmcneill 	uint8_t			cq_type;
    281   1.1  jmcneill #define MCX_CMDQ_TYPE_PCIE		0x7
    282   1.1  jmcneill 	uint8_t			cq_reserved0[3];
    283   1.1  jmcneill 
    284   1.1  jmcneill 	uint32_t		cq_input_length;
    285   1.1  jmcneill 	uint64_t		cq_input_ptr;
    286   1.1  jmcneill 	uint8_t			cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
    287   1.1  jmcneill 
    288   1.1  jmcneill 	uint8_t			cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
    289   1.1  jmcneill 	uint64_t		cq_output_ptr;
    290   1.1  jmcneill 	uint32_t		cq_output_length;
    291   1.1  jmcneill 
    292   1.1  jmcneill 	uint8_t			cq_token;
    293   1.1  jmcneill 	uint8_t			cq_signature;
    294   1.1  jmcneill 	uint8_t			cq_reserved1[1];
    295   1.1  jmcneill 	uint8_t			cq_status;
    296   1.1  jmcneill #define MCX_CQ_STATUS_SHIFT		1
    297   1.1  jmcneill #define MCX_CQ_STATUS_MASK		(0x7f << MCX_CQ_STATUS_SHIFT)
    298   1.1  jmcneill #define MCX_CQ_STATUS_OK		(0x00 << MCX_CQ_STATUS_SHIFT)
    299   1.1  jmcneill #define MCX_CQ_STATUS_INT_ERR		(0x01 << MCX_CQ_STATUS_SHIFT)
    300   1.1  jmcneill #define MCX_CQ_STATUS_BAD_OPCODE	(0x02 << MCX_CQ_STATUS_SHIFT)
    301   1.1  jmcneill #define MCX_CQ_STATUS_BAD_PARAM		(0x03 << MCX_CQ_STATUS_SHIFT)
    302   1.1  jmcneill #define MCX_CQ_STATUS_BAD_SYS_STATE	(0x04 << MCX_CQ_STATUS_SHIFT)
    303   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE	(0x05 << MCX_CQ_STATUS_SHIFT)
    304   1.1  jmcneill #define MCX_CQ_STATUS_RESOURCE_BUSY	(0x06 << MCX_CQ_STATUS_SHIFT)
    305   1.1  jmcneill #define MCX_CQ_STATUS_EXCEED_LIM	(0x08 << MCX_CQ_STATUS_SHIFT)
    306   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RES_STATE	(0x09 << MCX_CQ_STATUS_SHIFT)
    307   1.1  jmcneill #define MCX_CQ_STATUS_BAD_INDEX		(0x0a << MCX_CQ_STATUS_SHIFT)
    308   1.1  jmcneill #define MCX_CQ_STATUS_NO_RESOURCES	(0x0f << MCX_CQ_STATUS_SHIFT)
    309   1.1  jmcneill #define MCX_CQ_STATUS_BAD_INPUT_LEN	(0x50 << MCX_CQ_STATUS_SHIFT)
    310   1.1  jmcneill #define MCX_CQ_STATUS_BAD_OUTPUT_LEN	(0x51 << MCX_CQ_STATUS_SHIFT)
    311   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
    312   1.1  jmcneill 					(0x10 << MCX_CQ_STATUS_SHIFT)
    313   1.1  jmcneill #define MCX_CQ_STATUS_BAD_SIZE		(0x40 << MCX_CQ_STATUS_SHIFT)
    314   1.1  jmcneill #define MCX_CQ_STATUS_OWN_MASK		0x1
    315   1.1  jmcneill #define MCX_CQ_STATUS_OWN_SW		0x0
    316   1.1  jmcneill #define MCX_CQ_STATUS_OWN_HW		0x1
    317   1.1  jmcneill } __packed __aligned(8);
    318   1.1  jmcneill 
    319   1.1  jmcneill #define MCX_CMDQ_MAILBOX_DATASIZE	512
    320   1.1  jmcneill 
    321   1.1  jmcneill struct mcx_cmdq_mailbox {
    322   1.1  jmcneill 	uint8_t			mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
    323   1.1  jmcneill 	uint8_t			mb_reserved0[48];
    324   1.1  jmcneill 	uint64_t		mb_next_ptr;
    325   1.1  jmcneill 	uint32_t		mb_block_number;
    326   1.1  jmcneill 	uint8_t			mb_reserved1[1];
    327   1.1  jmcneill 	uint8_t			mb_token;
    328   1.1  jmcneill 	uint8_t			mb_ctrl_signature;
    329   1.1  jmcneill 	uint8_t			mb_signature;
    330   1.1  jmcneill } __packed __aligned(8);
    331   1.1  jmcneill 
    332   1.1  jmcneill #define MCX_CMDQ_MAILBOX_ALIGN	(1 << 10)
    333   1.1  jmcneill #define MCX_CMDQ_MAILBOX_SIZE	roundup(sizeof(struct mcx_cmdq_mailbox), \
    334   1.1  jmcneill 				    MCX_CMDQ_MAILBOX_ALIGN)
    335   1.1  jmcneill /*
    336  1.18    andvar  * command mailbox structures
    337   1.1  jmcneill  */
    338   1.1  jmcneill 
    339   1.1  jmcneill struct mcx_cmd_enable_hca_in {
    340   1.1  jmcneill 	uint16_t		cmd_opcode;
    341   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    342   1.1  jmcneill 	uint16_t		cmd_op_mod;
    343   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    344   1.1  jmcneill 	uint16_t		cmd_function_id;
    345   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    346   1.1  jmcneill } __packed __aligned(4);
    347   1.1  jmcneill 
    348   1.1  jmcneill struct mcx_cmd_enable_hca_out {
    349   1.1  jmcneill 	uint8_t			cmd_status;
    350   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    351   1.1  jmcneill 	uint32_t		cmd_syndrome;
    352   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    353   1.1  jmcneill } __packed __aligned(4);
    354   1.1  jmcneill 
    355   1.1  jmcneill struct mcx_cmd_init_hca_in {
    356   1.1  jmcneill 	uint16_t		cmd_opcode;
    357   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    358   1.1  jmcneill 	uint16_t		cmd_op_mod;
    359   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    360   1.1  jmcneill } __packed __aligned(4);
    361   1.1  jmcneill 
    362   1.1  jmcneill struct mcx_cmd_init_hca_out {
    363   1.1  jmcneill 	uint8_t			cmd_status;
    364   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    365   1.1  jmcneill 	uint32_t		cmd_syndrome;
    366   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    367   1.1  jmcneill } __packed __aligned(4);
    368   1.1  jmcneill 
    369   1.1  jmcneill struct mcx_cmd_teardown_hca_in {
    370   1.1  jmcneill 	uint16_t		cmd_opcode;
    371   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    372   1.1  jmcneill 	uint16_t		cmd_op_mod;
    373   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    374   1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_GRACEFUL	0x0
    375   1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_PANIC	0x1
    376   1.1  jmcneill 	uint16_t		cmd_profile;
    377   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    378   1.1  jmcneill } __packed __aligned(4);
    379   1.1  jmcneill 
    380   1.1  jmcneill struct mcx_cmd_teardown_hca_out {
    381   1.1  jmcneill 	uint8_t			cmd_status;
    382   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    383   1.1  jmcneill 	uint32_t		cmd_syndrome;
    384   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    385   1.1  jmcneill } __packed __aligned(4);
    386   1.1  jmcneill 
    387   1.1  jmcneill struct mcx_cmd_access_reg_in {
    388   1.1  jmcneill 	uint16_t		cmd_opcode;
    389   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    390   1.1  jmcneill 	uint16_t		cmd_op_mod;
    391   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    392   1.1  jmcneill 	uint16_t		cmd_register_id;
    393   1.1  jmcneill 	uint32_t		cmd_argument;
    394   1.1  jmcneill } __packed __aligned(4);
    395   1.1  jmcneill 
    396   1.1  jmcneill struct mcx_cmd_access_reg_out {
    397   1.1  jmcneill 	uint8_t			cmd_status;
    398   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    399   1.1  jmcneill 	uint32_t		cmd_syndrome;
    400   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    401   1.1  jmcneill } __packed __aligned(4);
    402   1.1  jmcneill 
    403   1.1  jmcneill struct mcx_reg_pmtu {
    404   1.1  jmcneill 	uint8_t			rp_reserved1;
    405   1.1  jmcneill 	uint8_t			rp_local_port;
    406   1.1  jmcneill 	uint8_t			rp_reserved2[2];
    407   1.1  jmcneill 	uint16_t		rp_max_mtu;
    408   1.1  jmcneill 	uint8_t			rp_reserved3[2];
    409   1.1  jmcneill 	uint16_t		rp_admin_mtu;
    410   1.1  jmcneill 	uint8_t			rp_reserved4[2];
    411   1.1  jmcneill 	uint16_t		rp_oper_mtu;
    412   1.1  jmcneill 	uint8_t			rp_reserved5[2];
    413   1.1  jmcneill } __packed __aligned(4);
    414   1.1  jmcneill 
    415   1.1  jmcneill struct mcx_reg_ptys {
    416   1.1  jmcneill 	uint8_t			rp_reserved1;
    417   1.1  jmcneill 	uint8_t			rp_local_port;
    418   1.1  jmcneill 	uint8_t			rp_reserved2;
    419   1.1  jmcneill 	uint8_t			rp_proto_mask;
    420   1.1  jmcneill #define MCX_REG_PTYS_PROTO_MASK_ETH		(1 << 2)
    421   1.1  jmcneill 	uint8_t			rp_reserved3[8];
    422   1.1  jmcneill 	uint32_t		rp_eth_proto_cap;
    423   1.1  jmcneill 	uint8_t			rp_reserved4[8];
    424   1.1  jmcneill 	uint32_t		rp_eth_proto_admin;
    425   1.1  jmcneill 	uint8_t			rp_reserved5[8];
    426   1.1  jmcneill 	uint32_t		rp_eth_proto_oper;
    427   1.1  jmcneill 	uint8_t			rp_reserved6[24];
    428   1.1  jmcneill } __packed __aligned(4);
    429   1.1  jmcneill 
    430   1.1  jmcneill struct mcx_reg_paos {
    431   1.1  jmcneill 	uint8_t			rp_reserved1;
    432   1.1  jmcneill 	uint8_t			rp_local_port;
    433   1.1  jmcneill 	uint8_t			rp_admin_status;
    434   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP		1
    435   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DOWN		2
    436   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE	3
    437   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED	4
    438   1.1  jmcneill 	uint8_t			rp_oper_status;
    439   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_UP		1
    440   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_DOWN		2
    441   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_FAILED		4
    442   1.1  jmcneill 	uint8_t			rp_admin_state_update;
    443   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN	(1 << 7)
    444   1.1  jmcneill 	uint8_t			rp_reserved2[11];
    445   1.1  jmcneill } __packed __aligned(4);
    446   1.1  jmcneill 
    447   1.1  jmcneill struct mcx_reg_pfcc {
    448   1.1  jmcneill 	uint8_t			rp_reserved1;
    449   1.1  jmcneill 	uint8_t			rp_local_port;
    450   1.1  jmcneill 	uint8_t			rp_reserved2[3];
    451   1.1  jmcneill 	uint8_t			rp_prio_mask_tx;
    452   1.1  jmcneill 	uint8_t			rp_reserved3;
    453   1.1  jmcneill 	uint8_t			rp_prio_mask_rx;
    454   1.1  jmcneill 	uint8_t			rp_pptx_aptx;
    455   1.1  jmcneill 	uint8_t			rp_pfctx;
    456   1.1  jmcneill 	uint8_t			rp_fctx_dis;
    457   1.1  jmcneill 	uint8_t			rp_reserved4;
    458   1.1  jmcneill 	uint8_t			rp_pprx_aprx;
    459   1.1  jmcneill 	uint8_t			rp_pfcrx;
    460   1.1  jmcneill 	uint8_t			rp_reserved5[2];
    461   1.1  jmcneill 	uint16_t		rp_dev_stall_min;
    462   1.1  jmcneill 	uint16_t		rp_dev_stall_crit;
    463   1.1  jmcneill 	uint8_t			rp_reserved6[12];
    464   1.1  jmcneill } __packed __aligned(4);
    465   1.1  jmcneill 
    466   1.1  jmcneill #define MCX_PMLP_MODULE_NUM_MASK	0xff
    467   1.1  jmcneill struct mcx_reg_pmlp {
    468   1.1  jmcneill 	uint8_t			rp_rxtx;
    469   1.1  jmcneill 	uint8_t			rp_local_port;
    470   1.1  jmcneill 	uint8_t			rp_reserved0;
    471   1.1  jmcneill 	uint8_t			rp_width;
    472   1.1  jmcneill 	uint32_t		rp_lane0_mapping;
    473   1.1  jmcneill 	uint32_t		rp_lane1_mapping;
    474   1.1  jmcneill 	uint32_t		rp_lane2_mapping;
    475   1.1  jmcneill 	uint32_t		rp_lane3_mapping;
    476   1.1  jmcneill 	uint8_t			rp_reserved1[44];
    477   1.1  jmcneill } __packed __aligned(4);
    478   1.1  jmcneill 
    479  1.15  jmcneill struct mcx_reg_ppcnt {
    480  1.15  jmcneill 	uint8_t			ppcnt_swid;
    481  1.15  jmcneill 	uint8_t			ppcnt_local_port;
    482  1.15  jmcneill 	uint8_t			ppcnt_pnat;
    483  1.15  jmcneill 	uint8_t			ppcnt_grp;
    484  1.15  jmcneill #define MCX_REG_PPCNT_GRP_IEEE8023		0x00
    485  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC2863		0x01
    486  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC2819		0x02
    487  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC3635		0x03
    488  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_PRIO		0x10
    489  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_TC		0x11
    490  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER		0x11
    491  1.15  jmcneill 
    492  1.15  jmcneill 	uint8_t			ppcnt_clr;
    493  1.15  jmcneill 	uint8_t			ppcnt_reserved1[2];
    494  1.15  jmcneill 	uint8_t			ppcnt_prio_tc;
    495  1.15  jmcneill #define MCX_REG_PPCNT_CLR			(1 << 7)
    496  1.15  jmcneill 
    497  1.15  jmcneill 	uint8_t			ppcnt_counter_set[248];
    498  1.15  jmcneill } __packed __aligned(8);
    499  1.15  jmcneill CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256);
    500  1.15  jmcneill CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) %
    501  1.15  jmcneill     sizeof(uint64_t)) == 0);
    502  1.15  jmcneill 
    503  1.15  jmcneill enum mcx_ppcnt_ieee8023 {
    504  1.15  jmcneill 	frames_transmitted_ok,
    505  1.15  jmcneill 	frames_received_ok,
    506  1.15  jmcneill 	frame_check_sequence_errors,
    507  1.15  jmcneill 	alignment_errors,
    508  1.15  jmcneill 	octets_transmitted_ok,
    509  1.15  jmcneill 	octets_received_ok,
    510  1.15  jmcneill 	multicast_frames_xmitted_ok,
    511  1.15  jmcneill 	broadcast_frames_xmitted_ok,
    512  1.15  jmcneill 	multicast_frames_received_ok,
    513  1.15  jmcneill 	broadcast_frames_received_ok,
    514  1.15  jmcneill 	in_range_length_errors,
    515  1.15  jmcneill 	out_of_range_length_field,
    516  1.15  jmcneill 	frame_too_long_errors,
    517  1.15  jmcneill 	symbol_error_during_carrier,
    518  1.15  jmcneill 	mac_control_frames_transmitted,
    519  1.15  jmcneill 	mac_control_frames_received,
    520  1.15  jmcneill 	unsupported_opcodes_received,
    521  1.15  jmcneill 	pause_mac_ctrl_frames_received,
    522  1.15  jmcneill 	pause_mac_ctrl_frames_transmitted,
    523  1.15  jmcneill 
    524  1.15  jmcneill 	mcx_ppcnt_ieee8023_count
    525  1.15  jmcneill };
    526  1.15  jmcneill CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98);
    527  1.15  jmcneill 
    528  1.15  jmcneill enum mcx_ppcnt_rfc2863 {
    529  1.15  jmcneill 	in_octets,
    530  1.15  jmcneill 	in_ucast_pkts,
    531  1.15  jmcneill 	in_discards,
    532  1.15  jmcneill 	in_errors,
    533  1.15  jmcneill 	in_unknown_protos,
    534  1.15  jmcneill 	out_octets,
    535  1.15  jmcneill 	out_ucast_pkts,
    536  1.15  jmcneill 	out_discards,
    537  1.15  jmcneill 	out_errors,
    538  1.15  jmcneill 	in_multicast_pkts,
    539  1.15  jmcneill 	in_broadcast_pkts,
    540  1.15  jmcneill 	out_multicast_pkts,
    541  1.15  jmcneill 	out_broadcast_pkts,
    542  1.15  jmcneill 
    543  1.15  jmcneill 	mcx_ppcnt_rfc2863_count
    544  1.15  jmcneill };
    545  1.15  jmcneill CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68);
    546  1.15  jmcneill 
    547  1.15  jmcneill enum mcx_ppcnt_rfc2819 {
    548  1.15  jmcneill 	drop_events,
    549  1.15  jmcneill 	octets,
    550  1.15  jmcneill 	pkts,
    551  1.15  jmcneill 	broadcast_pkts,
    552  1.15  jmcneill 	multicast_pkts,
    553  1.15  jmcneill 	crc_align_errors,
    554  1.15  jmcneill 	undersize_pkts,
    555  1.15  jmcneill 	oversize_pkts,
    556  1.15  jmcneill 	fragments,
    557  1.15  jmcneill 	jabbers,
    558  1.15  jmcneill 	collisions,
    559  1.15  jmcneill 	pkts64octets,
    560  1.15  jmcneill 	pkts65to127octets,
    561  1.15  jmcneill 	pkts128to255octets,
    562  1.15  jmcneill 	pkts256to511octets,
    563  1.15  jmcneill 	pkts512to1023octets,
    564  1.15  jmcneill 	pkts1024to1518octets,
    565  1.15  jmcneill 	pkts1519to2047octets,
    566  1.15  jmcneill 	pkts2048to4095octets,
    567  1.15  jmcneill 	pkts4096to8191octets,
    568  1.15  jmcneill 	pkts8192to10239octets,
    569  1.15  jmcneill 
    570  1.15  jmcneill 	mcx_ppcnt_rfc2819_count
    571  1.15  jmcneill };
    572  1.15  jmcneill CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8);
    573  1.15  jmcneill 
    574  1.15  jmcneill enum mcx_ppcnt_rfc3635 {
    575  1.15  jmcneill 	dot3stats_alignment_errors,
    576  1.15  jmcneill 	dot3stats_fcs_errors,
    577  1.15  jmcneill 	dot3stats_single_collision_frames,
    578  1.15  jmcneill 	dot3stats_multiple_collision_frames,
    579  1.15  jmcneill 	dot3stats_sqe_test_errors,
    580  1.15  jmcneill 	dot3stats_deferred_transmissions,
    581  1.15  jmcneill 	dot3stats_late_collisions,
    582  1.15  jmcneill 	dot3stats_excessive_collisions,
    583  1.15  jmcneill 	dot3stats_internal_mac_transmit_errors,
    584  1.15  jmcneill 	dot3stats_carrier_sense_errors,
    585  1.15  jmcneill 	dot3stats_frame_too_longs,
    586  1.15  jmcneill 	dot3stats_internal_mac_receive_errors,
    587  1.15  jmcneill 	dot3stats_symbol_errors,
    588  1.15  jmcneill 	dot3control_in_unknown_opcodes,
    589  1.15  jmcneill 	dot3in_pause_frames,
    590  1.15  jmcneill 	dot3out_pause_frames,
    591  1.15  jmcneill 
    592  1.15  jmcneill 	mcx_ppcnt_rfc3635_count
    593  1.15  jmcneill };
    594  1.15  jmcneill CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80);
    595  1.15  jmcneill 
    596  1.15  jmcneill struct mcx_reg_mcam {
    597  1.15  jmcneill 	uint8_t			_reserved1[1];
    598  1.15  jmcneill 	uint8_t			mcam_feature_group;
    599  1.15  jmcneill 	uint8_t			_reserved2[1];
    600  1.15  jmcneill 	uint8_t			mcam_access_reg_group;
    601  1.15  jmcneill 	uint8_t			_reserved3[4];
    602  1.15  jmcneill 	uint8_t			mcam_access_reg_cap_mask[16];
    603  1.15  jmcneill 	uint8_t			_reserved4[16];
    604  1.15  jmcneill 	uint8_t			mcam_feature_cap_mask[16];
    605  1.15  jmcneill 	uint8_t			_reserved5[16];
    606  1.15  jmcneill } __packed __aligned(4);
    607  1.15  jmcneill 
    608  1.15  jmcneill #define MCX_BITFIELD_BIT(bf, b)	(bf[(sizeof bf - 1) - (b / 8)] & (b % 8))
    609  1.15  jmcneill 
    610  1.15  jmcneill #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP	6
    611  1.15  jmcneill 
    612  1.15  jmcneill struct mcx_reg_mtcap {
    613  1.15  jmcneill 	uint8_t			_reserved1[3];
    614  1.15  jmcneill 	uint8_t			mtcap_sensor_count;
    615  1.15  jmcneill 	uint8_t			_reserved2[4];
    616  1.15  jmcneill 
    617  1.15  jmcneill 	uint64_t		mtcap_sensor_map;
    618  1.15  jmcneill };
    619  1.15  jmcneill 
    620  1.15  jmcneill struct mcx_reg_mtmp {
    621  1.15  jmcneill 	uint8_t			_reserved1[2];
    622  1.15  jmcneill 	uint16_t		mtmp_sensor_index;
    623  1.15  jmcneill 
    624  1.15  jmcneill 	uint8_t			_reserved2[2];
    625  1.15  jmcneill 	uint16_t		mtmp_temperature;
    626  1.15  jmcneill 
    627  1.15  jmcneill 	uint16_t		mtmp_mte_mtr;
    628  1.15  jmcneill #define MCX_REG_MTMP_MTE		(1 << 15)
    629  1.15  jmcneill #define MCX_REG_MTMP_MTR		(1 << 14)
    630  1.15  jmcneill 	uint16_t		mtmp_max_temperature;
    631  1.15  jmcneill 
    632  1.15  jmcneill 	uint16_t		mtmp_tee;
    633  1.15  jmcneill #define MCX_REG_MTMP_TEE_NOPE		(0 << 14)
    634  1.15  jmcneill #define MCX_REG_MTMP_TEE_GENERATE	(1 << 14)
    635  1.15  jmcneill #define MCX_REG_MTMP_TEE_GENERATE_ONE	(2 << 14)
    636  1.15  jmcneill 	uint16_t		mtmp_temperature_threshold_hi;
    637  1.15  jmcneill 
    638  1.15  jmcneill 	uint8_t			_reserved3[2];
    639  1.15  jmcneill 	uint16_t		mtmp_temperature_threshold_lo;
    640  1.15  jmcneill 
    641  1.15  jmcneill 	uint8_t			_reserved4[4];
    642  1.15  jmcneill 
    643  1.15  jmcneill 	uint8_t			mtmp_sensor_name[8];
    644  1.15  jmcneill };
    645  1.15  jmcneill CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20);
    646  1.15  jmcneill CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18);
    647  1.15  jmcneill 
    648   1.1  jmcneill #define MCX_MCIA_EEPROM_BYTES	32
    649   1.1  jmcneill struct mcx_reg_mcia {
    650   1.1  jmcneill 	uint8_t			rm_l;
    651   1.1  jmcneill 	uint8_t			rm_module;
    652   1.1  jmcneill 	uint8_t			rm_reserved0;
    653   1.1  jmcneill 	uint8_t			rm_status;
    654   1.1  jmcneill 	uint8_t			rm_i2c_addr;
    655   1.1  jmcneill 	uint8_t			rm_page_num;
    656   1.1  jmcneill 	uint16_t		rm_dev_addr;
    657   1.1  jmcneill 	uint16_t		rm_reserved1;
    658   1.1  jmcneill 	uint16_t		rm_size;
    659   1.1  jmcneill 	uint32_t		rm_reserved2;
    660   1.1  jmcneill 	uint8_t			rm_data[48];
    661   1.1  jmcneill } __packed __aligned(4);
    662   1.1  jmcneill 
    663   1.1  jmcneill struct mcx_cmd_query_issi_in {
    664   1.1  jmcneill 	uint16_t		cmd_opcode;
    665   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    666   1.1  jmcneill 	uint16_t		cmd_op_mod;
    667   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    668   1.1  jmcneill } __packed __aligned(4);
    669   1.1  jmcneill 
    670   1.1  jmcneill struct mcx_cmd_query_issi_il_out {
    671   1.1  jmcneill 	uint8_t			cmd_status;
    672   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    673   1.1  jmcneill 	uint32_t		cmd_syndrome;
    674   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    675   1.1  jmcneill 	uint16_t		cmd_current_issi;
    676   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    677   1.1  jmcneill } __packed __aligned(4);
    678   1.1  jmcneill 
    679   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
    680   1.1  jmcneill 
    681   1.1  jmcneill struct mcx_cmd_query_issi_mb_out {
    682   1.1  jmcneill 	uint8_t			cmd_reserved2[16];
    683   1.1  jmcneill 	uint8_t			cmd_supported_issi[80]; /* very big endian */
    684   1.1  jmcneill } __packed __aligned(4);
    685   1.1  jmcneill 
    686   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
    687   1.1  jmcneill 
    688   1.1  jmcneill struct mcx_cmd_set_issi_in {
    689   1.1  jmcneill 	uint16_t		cmd_opcode;
    690   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    691   1.1  jmcneill 	uint16_t		cmd_op_mod;
    692   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    693   1.1  jmcneill 	uint16_t		cmd_current_issi;
    694   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    695   1.1  jmcneill } __packed __aligned(4);
    696   1.1  jmcneill 
    697   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
    698   1.1  jmcneill 
    699   1.1  jmcneill struct mcx_cmd_set_issi_out {
    700   1.1  jmcneill 	uint8_t			cmd_status;
    701   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    702   1.1  jmcneill 	uint32_t		cmd_syndrome;
    703   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    704   1.1  jmcneill } __packed __aligned(4);
    705   1.1  jmcneill 
    706   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
    707   1.1  jmcneill 
    708   1.1  jmcneill struct mcx_cmd_query_pages_in {
    709   1.1  jmcneill 	uint16_t		cmd_opcode;
    710   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    711   1.1  jmcneill 	uint16_t		cmd_op_mod;
    712   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_BOOT	0x01
    713   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_INIT	0x02
    714   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_REGULAR	0x03
    715   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    716   1.1  jmcneill } __packed __aligned(4);
    717   1.1  jmcneill 
    718   1.1  jmcneill struct mcx_cmd_query_pages_out {
    719   1.1  jmcneill 	uint8_t			cmd_status;
    720   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    721   1.1  jmcneill 	uint32_t		cmd_syndrome;
    722   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    723   1.1  jmcneill 	uint16_t		cmd_func_id;
    724  1.15  jmcneill 	int32_t			cmd_num_pages;
    725   1.1  jmcneill } __packed __aligned(4);
    726   1.1  jmcneill 
    727   1.1  jmcneill struct mcx_cmd_manage_pages_in {
    728   1.1  jmcneill 	uint16_t		cmd_opcode;
    729   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    730   1.1  jmcneill 	uint16_t		cmd_op_mod;
    731   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
    732   1.1  jmcneill 					0x00
    733   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
    734   1.1  jmcneill 					0x01
    735   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
    736   1.1  jmcneill 					0x02
    737   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    738   1.1  jmcneill 	uint16_t		cmd_func_id;
    739   1.1  jmcneill 	uint32_t		cmd_input_num_entries;
    740   1.1  jmcneill } __packed __aligned(4);
    741   1.1  jmcneill 
    742   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
    743   1.1  jmcneill 
    744   1.1  jmcneill struct mcx_cmd_manage_pages_out {
    745   1.1  jmcneill 	uint8_t			cmd_status;
    746   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    747   1.1  jmcneill 	uint32_t		cmd_syndrome;
    748   1.1  jmcneill 	uint32_t		cmd_output_num_entries;
    749   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    750   1.1  jmcneill } __packed __aligned(4);
    751   1.1  jmcneill 
    752   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
    753   1.1  jmcneill 
    754   1.1  jmcneill struct mcx_cmd_query_hca_cap_in {
    755   1.1  jmcneill 	uint16_t		cmd_opcode;
    756   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    757   1.1  jmcneill 	uint16_t		cmd_op_mod;
    758   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_MAX	(0x0 << 0)
    759   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_CURRENT	(0x1 << 0)
    760   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_DEVICE	(0x0 << 1)
    761   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD	(0x1 << 1)
    762   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_FLOW	(0x7 << 1)
    763   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    764   1.1  jmcneill } __packed __aligned(4);
    765   1.1  jmcneill 
    766   1.1  jmcneill struct mcx_cmd_query_hca_cap_out {
    767   1.1  jmcneill 	uint8_t			cmd_status;
    768   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    769   1.1  jmcneill 	uint32_t		cmd_syndrome;
    770   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    771   1.1  jmcneill } __packed __aligned(4);
    772   1.1  jmcneill 
    773   1.1  jmcneill #define MCX_HCA_CAP_LEN			0x1000
    774   1.1  jmcneill #define MCX_HCA_CAP_NMAILBOXES		\
    775   1.1  jmcneill 	(MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
    776   1.1  jmcneill 
    777   1.1  jmcneill #if __GNUC_PREREQ__(4, 3)
    778   1.1  jmcneill #define __counter__		__COUNTER__
    779   1.1  jmcneill #else
    780   1.1  jmcneill #define __counter__		__LINE__
    781   1.1  jmcneill #endif
    782   1.1  jmcneill 
    783   1.1  jmcneill #define __token(_tok, _num)	_tok##_num
    784   1.1  jmcneill #define _token(_tok, _num)	__token(_tok, _num)
    785   1.1  jmcneill #define __reserved__		_token(__reserved, __counter__)
    786   1.1  jmcneill 
    787   1.1  jmcneill struct mcx_cap_device {
    788   1.1  jmcneill 	uint8_t			reserved0[16];
    789   1.1  jmcneill 
    790   1.1  jmcneill 	uint8_t			log_max_srq_sz;
    791   1.1  jmcneill 	uint8_t			log_max_qp_sz;
    792   1.1  jmcneill 	uint8_t			__reserved__[1];
    793   1.1  jmcneill 	uint8_t			log_max_qp; /* 5 bits */
    794   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_QP	0x1f
    795   1.1  jmcneill 
    796   1.1  jmcneill 	uint8_t			__reserved__[1];
    797   1.1  jmcneill 	uint8_t			log_max_srq; /* 5 bits */
    798   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SRQ	0x1f
    799   1.1  jmcneill 	uint8_t			__reserved__[2];
    800   1.1  jmcneill 
    801   1.1  jmcneill 	uint8_t			__reserved__[1];
    802   1.1  jmcneill 	uint8_t			log_max_cq_sz;
    803   1.1  jmcneill 	uint8_t			__reserved__[1];
    804   1.1  jmcneill 	uint8_t			log_max_cq; /* 5 bits */
    805   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_CQ	0x1f
    806   1.1  jmcneill 
    807   1.1  jmcneill 	uint8_t			log_max_eq_sz;
    808   1.1  jmcneill 	uint8_t			log_max_mkey; /* 6 bits */
    809   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MKEY	0x3f
    810   1.1  jmcneill 	uint8_t			__reserved__[1];
    811   1.1  jmcneill 	uint8_t			log_max_eq; /* 4 bits */
    812   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_EQ	0x0f
    813   1.1  jmcneill 
    814   1.1  jmcneill 	uint8_t			max_indirection;
    815   1.1  jmcneill 	uint8_t			log_max_mrw_sz; /* 7 bits */
    816   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ	0x7f
    817   1.1  jmcneill 	uint8_t			teardown_log_max_msf_list_size;
    818   1.1  jmcneill #define MCX_CAP_DEVICE_FORCE_TEARDOWN	0x80
    819   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
    820   1.1  jmcneill 					0x3f
    821   1.1  jmcneill 	uint8_t			log_max_klm_list_size; /* 6 bits */
    822   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
    823   1.1  jmcneill 					0x3f
    824   1.1  jmcneill 
    825   1.1  jmcneill 	uint8_t			__reserved__[1];
    826   1.1  jmcneill 	uint8_t			log_max_ra_req_dc; /* 6 bits */
    827   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC	0x3f
    828   1.1  jmcneill 	uint8_t			__reserved__[1];
    829   1.1  jmcneill 	uint8_t			log_max_ra_res_dc; /* 6 bits */
    830   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
    831   1.1  jmcneill 					0x3f
    832   1.1  jmcneill 
    833   1.1  jmcneill 	uint8_t			__reserved__[1];
    834   1.1  jmcneill 	uint8_t			log_max_ra_req_qp; /* 6 bits */
    835   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
    836   1.1  jmcneill 					0x3f
    837   1.1  jmcneill 	uint8_t			__reserved__[1];
    838   1.1  jmcneill 	uint8_t			log_max_ra_res_qp; /* 6 bits */
    839   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
    840   1.1  jmcneill 					0x3f
    841   1.1  jmcneill 
    842   1.1  jmcneill 	uint8_t			flags1;
    843   1.1  jmcneill #define MCX_CAP_DEVICE_END_PAD		0x80
    844   1.1  jmcneill #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED	0x40
    845   1.1  jmcneill #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
    846   1.1  jmcneill 					0x20
    847   1.1  jmcneill #define MCX_CAP_DEVICE_START_PAD	0x10
    848   1.1  jmcneill #define MCX_CAP_DEVICE_128BYTE_CACHELINE \
    849   1.1  jmcneill 					0x08
    850   1.1  jmcneill 	uint8_t			__reserved__[1];
    851   1.1  jmcneill 	uint16_t		gid_table_size;
    852   1.1  jmcneill 
    853   1.1  jmcneill 	uint16_t		flags2;
    854   1.1  jmcneill #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT	0x8000
    855   1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_COUNTERS	0x4000
    856   1.1  jmcneill #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
    857   1.1  jmcneill 					0x2000
    858   1.1  jmcneill #define MCX_CAP_DEVICE_DEBUG		0x1000
    859   1.1  jmcneill #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
    860   1.1  jmcneill 					0x8000
    861   1.1  jmcneill #define MCX_CAP_DEVICE_RQ_DELAY_DROP	0x4000
    862   1.1  jmcneill #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK	0x03ff
    863   1.1  jmcneill 	uint16_t		pkey_table_size;
    864   1.1  jmcneill 
    865   1.1  jmcneill 	uint8_t			flags3;
    866   1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
    867   1.1  jmcneill 					0x80
    868   1.1  jmcneill #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
    869   1.1  jmcneill 					0x40
    870   1.1  jmcneill #define MCX_CAP_DEVICE_IB_VIRTUAL	0x20
    871   1.1  jmcneill #define MCX_CAP_DEVICE_ETH_VIRTUAL	0x10
    872   1.1  jmcneill #define MCX_CAP_DEVICE_ETS		0x04
    873   1.1  jmcneill #define MCX_CAP_DEVICE_NIC_FLOW_TABLE	0x02
    874   1.1  jmcneill #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
    875   1.1  jmcneill 					0x01
    876   1.1  jmcneill 	uint8_t			local_ca_ack_delay; /* 5 bits */
    877   1.1  jmcneill #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
    878   1.1  jmcneill 					0x1f
    879  1.15  jmcneill #define MCX_CAP_DEVICE_MCAM_REG		0x40
    880   1.1  jmcneill 	uint8_t			port_type;
    881   1.1  jmcneill #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
    882   1.1  jmcneill 					0x80
    883   1.1  jmcneill #define MCX_CAP_DEVICE_PORT_TYPE	0x03
    884  1.15  jmcneill #define MCX_CAP_DEVICE_PORT_TYPE_ETH	0x01
    885   1.1  jmcneill 	uint8_t			num_ports;
    886   1.1  jmcneill 
    887   1.1  jmcneill 	uint8_t			snapshot_log_max_msg;
    888   1.1  jmcneill #define MCX_CAP_DEVICE_SNAPSHOT		0x80
    889   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSG	0x1f
    890   1.1  jmcneill 	uint8_t			max_tc; /* 4 bits */
    891   1.1  jmcneill #define MCX_CAP_DEVICE_MAX_TC		0x0f
    892   1.1  jmcneill 	uint8_t			flags4;
    893   1.1  jmcneill #define MCX_CAP_DEVICE_TEMP_WARN_EVENT	0x80
    894   1.1  jmcneill #define MCX_CAP_DEVICE_DCBX		0x40
    895   1.1  jmcneill #define MCX_CAP_DEVICE_ROL_S		0x02
    896   1.1  jmcneill #define MCX_CAP_DEVICE_ROL_G		0x01
    897   1.1  jmcneill 	uint8_t			wol;
    898   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_S		0x40
    899   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_G		0x20
    900   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_A		0x10
    901   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_B		0x08
    902   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_M		0x04
    903   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_U		0x02
    904   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_P		0x01
    905   1.1  jmcneill 
    906   1.1  jmcneill 	uint16_t		stat_rate_support;
    907   1.1  jmcneill 	uint8_t			__reserved__[1];
    908   1.1  jmcneill 	uint8_t			cqe_version; /* 4 bits */
    909   1.1  jmcneill #define MCX_CAP_DEVICE_CQE_VERSION	0x0f
    910   1.1  jmcneill 
    911   1.1  jmcneill 	uint32_t		flags5;
    912   1.1  jmcneill #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
    913   1.1  jmcneill 					0x80000000
    914   1.1  jmcneill #define MCX_CAP_DEVICE_STRIDING_RQ	0x40000000
    915   1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
    916   1.1  jmcneill 					0x10000000
    917   1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
    918   1.1  jmcneill 					0x08000000
    919   1.1  jmcneill #define MCX_CAP_DEVICE_DC_CONNECT_CP	0x00040000
    920   1.1  jmcneill #define MCX_CAP_DEVICE_DC_CNAK_DRACE	0x00020000
    921   1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    922   1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    923   1.1  jmcneill #define MCX_CAP_DEVICE_CMDIF_CHECKSUM	0x0000c000
    924   1.1  jmcneill #define MCX_CAP_DEVICE_SIGERR_QCE	0x00002000
    925   1.1  jmcneill #define MCX_CAP_DEVICE_WQ_SIGNATURE	0x00000800
    926   1.1  jmcneill #define MCX_CAP_DEVICE_SCTR_DATA_CQE	0x00000400
    927   1.1  jmcneill #define MCX_CAP_DEVICE_SHO		0x00000100
    928   1.1  jmcneill #define MCX_CAP_DEVICE_TPH		0x00000080
    929   1.1  jmcneill #define MCX_CAP_DEVICE_RF		0x00000040
    930   1.1  jmcneill #define MCX_CAP_DEVICE_DCT		0x00000020
    931   1.1  jmcneill #define MCX_CAP_DEVICE_QOS		0x00000010
    932   1.1  jmcneill #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS	0x00000008
    933   1.1  jmcneill #define MCX_CAP_DEVICE_ROCE		0x00000004
    934   1.1  jmcneill #define MCX_CAP_DEVICE_ATOMIC		0x00000002
    935   1.1  jmcneill 
    936   1.1  jmcneill 	uint32_t		flags6;
    937   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_OI		0x80000000
    938   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_RESIZE	0x40000000
    939   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_MODERATION	0x20000000
    940   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
    941   1.1  jmcneill 					0x10000000
    942   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_INVALIDATE	0x08000000
    943   1.1  jmcneill #define MCX_CAP_DEVICE_RESERVED_AT_255	0x04000000
    944   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_EQ_REMAP	0x02000000
    945   1.1  jmcneill #define MCX_CAP_DEVICE_PG		0x01000000
    946   1.1  jmcneill #define MCX_CAP_DEVICE_BLOCK_LB_MC	0x00800000
    947   1.1  jmcneill #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
    948   1.1  jmcneill 					0x00400000
    949   1.1  jmcneill #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
    950   1.1  jmcneill 					0x00200000
    951   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
    952   1.1  jmcneill 					0x00100000
    953   1.1  jmcneill #define MCX_CAP_DEVICE_CD		0x00080000
    954   1.1  jmcneill #define MCX_CAP_DEVICE_ATM		0x00040000
    955   1.1  jmcneill #define MCX_CAP_DEVICE_APM		0x00020000
    956   1.1  jmcneill #define MCX_CAP_DEVICE_IMAICL		0x00010000
    957   1.1  jmcneill #define MCX_CAP_DEVICE_QKV		0x00000200
    958   1.1  jmcneill #define MCX_CAP_DEVICE_PKV		0x00000100
    959   1.1  jmcneill #define MCX_CAP_DEVICE_SET_DETH_SQPN	0x00000080
    960   1.1  jmcneill #define MCX_CAP_DEVICE_XRC		0x00000008
    961   1.1  jmcneill #define MCX_CAP_DEVICE_UD		0x00000004
    962   1.1  jmcneill #define MCX_CAP_DEVICE_UC		0x00000002
    963   1.1  jmcneill #define MCX_CAP_DEVICE_RC		0x00000001
    964   1.1  jmcneill 
    965   1.1  jmcneill 	uint8_t			uar_flags;
    966   1.1  jmcneill #define MCX_CAP_DEVICE_UAR_4K		0x80
    967   1.1  jmcneill 	uint8_t			uar_sz;	/* 6 bits */
    968   1.1  jmcneill #define MCX_CAP_DEVICE_UAR_SZ		0x3f
    969   1.1  jmcneill 	uint8_t			__reserved__[1];
    970   1.1  jmcneill 	uint8_t			log_pg_sz;
    971   1.1  jmcneill 
    972   1.1  jmcneill 	uint8_t			flags7;
    973   1.1  jmcneill #define MCX_CAP_DEVICE_BF		0x80
    974   1.1  jmcneill #define MCX_CAP_DEVICE_DRIVER_VERSION	0x40
    975   1.1  jmcneill #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
    976   1.1  jmcneill 					0x20
    977   1.1  jmcneill 	uint8_t			log_bf_reg_size; /* 5 bits */
    978   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE	0x1f
    979   1.1  jmcneill 	uint8_t			__reserved__[2];
    980   1.1  jmcneill 
    981   1.1  jmcneill 	uint16_t		num_of_diagnostic_counters;
    982   1.1  jmcneill 	uint16_t		max_wqe_sz_sq;
    983   1.1  jmcneill 
    984   1.1  jmcneill 	uint8_t			__reserved__[2];
    985   1.1  jmcneill 	uint16_t		max_wqe_sz_rq;
    986   1.1  jmcneill 
    987   1.1  jmcneill 	uint8_t			__reserved__[2];
    988   1.1  jmcneill 	uint16_t		max_wqe_sz_sq_dc;
    989   1.1  jmcneill 
    990   1.1  jmcneill 	uint32_t		max_qp_mcg; /* 25 bits */
    991   1.1  jmcneill #define MCX_CAP_DEVICE_MAX_QP_MCG	0x1ffffff
    992   1.1  jmcneill 
    993   1.1  jmcneill 	uint8_t			__reserved__[3];
    994   1.1  jmcneill 	uint8_t			log_max_mcq;
    995   1.1  jmcneill 
    996   1.1  jmcneill 	uint8_t			log_max_transport_domain; /* 5 bits */
    997   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
    998   1.1  jmcneill 					0x1f
    999   1.1  jmcneill 	uint8_t			log_max_pd; /* 5 bits */
   1000   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_PD	0x1f
   1001   1.1  jmcneill 	uint8_t			__reserved__[1];
   1002   1.1  jmcneill 	uint8_t			log_max_xrcd; /* 5 bits */
   1003   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_XRCD	0x1f
   1004   1.1  jmcneill 
   1005   1.1  jmcneill 	uint8_t			__reserved__[2];
   1006   1.1  jmcneill 	uint16_t		max_flow_counter;
   1007   1.1  jmcneill 
   1008   1.1  jmcneill 	uint8_t			log_max_rq; /* 5 bits */
   1009   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQ	0x1f
   1010   1.1  jmcneill 	uint8_t			log_max_sq; /* 5 bits */
   1011   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SQ	0x1f
   1012   1.1  jmcneill 	uint8_t			log_max_tir; /* 5 bits */
   1013   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIR	0x1f
   1014   1.1  jmcneill 	uint8_t			log_max_tis; /* 5 bits */
   1015   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS	0x1f
   1016   1.1  jmcneill 
   1017   1.1  jmcneill 	uint8_t 		flags8;
   1018   1.1  jmcneill #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
   1019   1.1  jmcneill 					0x80
   1020   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RMP	0x1f
   1021   1.1  jmcneill 	uint8_t			log_max_rqt; /* 5 bits */
   1022   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT	0x1f
   1023   1.1  jmcneill 	uint8_t			log_max_rqt_size; /* 5 bits */
   1024   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE	0x1f
   1025   1.1  jmcneill 	uint8_t			log_max_tis_per_sq; /* 5 bits */
   1026   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
   1027   1.1  jmcneill 					0x1f
   1028  1.15  jmcneill 
   1029  1.15  jmcneill 	uint8_t			flags9;
   1030  1.15  jmcneill #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \
   1031  1.15  jmcneill 					0x80
   1032  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \
   1033  1.15  jmcneill 					0x1f
   1034  1.15  jmcneill 	uint8_t			log_min_stride_sz_rq; /* 5 bits */
   1035  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \
   1036  1.15  jmcneill 					0x1f
   1037  1.15  jmcneill 	uint8_t			log_max_stride_sz_sq; /* 5 bits */
   1038  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \
   1039  1.15  jmcneill 					0x1f
   1040  1.15  jmcneill 	uint8_t			log_min_stride_sz_sq; /* 5 bits */
   1041  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \
   1042  1.15  jmcneill 					0x1f
   1043  1.15  jmcneill 
   1044  1.15  jmcneill 	uint8_t			log_max_hairpin_queues;
   1045  1.15  jmcneill #define MXC_CAP_DEVICE_HAIRPIN		0x80
   1046  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \
   1047  1.15  jmcneill 					0x1f
   1048  1.15  jmcneill 	uint8_t			log_min_hairpin_queues;
   1049  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \
   1050  1.15  jmcneill 					0x1f
   1051  1.15  jmcneill 	uint8_t			log_max_hairpin_num_packets;
   1052  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \
   1053  1.15  jmcneill 					0x1f
   1054  1.15  jmcneill 	uint8_t			log_max_mq_sz;
   1055  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \
   1056  1.15  jmcneill 					0x1f
   1057  1.15  jmcneill 
   1058  1.15  jmcneill 	uint8_t			log_min_hairpin_wq_data_sz;
   1059  1.15  jmcneill #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \
   1060  1.15  jmcneill 					0x80
   1061  1.15  jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \
   1062  1.15  jmcneill 					0x40
   1063  1.15  jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \
   1064  1.15  jmcneill 					0x20
   1065  1.15  jmcneill #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \
   1066  1.15  jmcneill 					0x1f
   1067  1.15  jmcneill 	uint8_t			log_max_vlan_list;
   1068  1.15  jmcneill #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \
   1069  1.15  jmcneill 					0x80
   1070  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \
   1071  1.15  jmcneill 					0x1f
   1072  1.15  jmcneill 	uint8_t			log_max_current_mc_list;
   1073  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \
   1074  1.15  jmcneill 					0x1f
   1075  1.15  jmcneill 	uint8_t			log_max_current_uc_list;
   1076  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \
   1077  1.15  jmcneill 					0x1f
   1078  1.15  jmcneill 
   1079  1.15  jmcneill 	uint8_t			__reserved__[4];
   1080  1.15  jmcneill 
   1081  1.15  jmcneill 	uint32_t		create_qp_start_hint; /* 24 bits */
   1082  1.15  jmcneill 
   1083  1.15  jmcneill 	uint8_t			log_max_uctx; /* 5 bits */
   1084  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UCTX	0x1f
   1085  1.15  jmcneill 	uint8_t			log_max_umem; /* 5 bits */
   1086  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UMEM	0x1f
   1087  1.15  jmcneill 	uint16_t		max_num_eqs;
   1088  1.15  jmcneill 
   1089  1.15  jmcneill 	uint8_t			log_max_l2_table; /* 5 bits */
   1090  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE	0x1f
   1091  1.15  jmcneill 	uint8_t			__reserved__[1];
   1092  1.15  jmcneill 	uint16_t		log_uar_page_sz;
   1093  1.15  jmcneill 
   1094  1.15  jmcneill 	uint8_t			__reserved__[8];
   1095  1.15  jmcneill 
   1096  1.15  jmcneill 	uint32_t		device_frequency_mhz;
   1097  1.15  jmcneill 	uint32_t		device_frequency_khz;
   1098   1.1  jmcneill } __packed __aligned(8);
   1099   1.1  jmcneill 
   1100   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
   1101   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
   1102   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
   1103   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
   1104   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
   1105   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
   1106  1.15  jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98);
   1107  1.15  jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c);
   1108   1.1  jmcneill CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
   1109   1.1  jmcneill 
   1110   1.1  jmcneill struct mcx_cmd_set_driver_version_in {
   1111   1.1  jmcneill 	uint16_t		cmd_opcode;
   1112   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1113   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1114   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1115   1.1  jmcneill } __packed __aligned(4);
   1116   1.1  jmcneill 
   1117   1.1  jmcneill struct mcx_cmd_set_driver_version_out {
   1118   1.1  jmcneill 	uint8_t			cmd_status;
   1119   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1120   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1121   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1122   1.1  jmcneill } __packed __aligned(4);
   1123   1.1  jmcneill 
   1124   1.1  jmcneill struct mcx_cmd_set_driver_version {
   1125   1.1  jmcneill 	uint8_t			cmd_driver_version[64];
   1126   1.1  jmcneill } __packed __aligned(8);
   1127   1.1  jmcneill 
   1128   1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_in {
   1129   1.1  jmcneill 	uint16_t		cmd_opcode;
   1130   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1131   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1132   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1133   1.1  jmcneill 	uint32_t		cmd_field_select;
   1134   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR	0x04
   1135   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC	0x10
   1136   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU	0x40
   1137   1.1  jmcneill } __packed __aligned(4);
   1138   1.1  jmcneill 
   1139   1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_out {
   1140   1.1  jmcneill 	uint8_t			cmd_status;
   1141   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1142   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1143   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1144   1.1  jmcneill } __packed __aligned(4);
   1145   1.1  jmcneill 
   1146   1.1  jmcneill struct mcx_cmd_query_nic_vport_context_in {
   1147   1.1  jmcneill 	uint16_t		cmd_opcode;
   1148   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1149   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1150   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1151   1.1  jmcneill 	uint8_t			cmd_allowed_list_type;
   1152   1.1  jmcneill 	uint8_t			cmd_reserved2[3];
   1153   1.1  jmcneill } __packed __aligned(4);
   1154   1.1  jmcneill 
   1155   1.1  jmcneill struct mcx_cmd_query_nic_vport_context_out {
   1156   1.1  jmcneill 	uint8_t			cmd_status;
   1157   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1158   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1159   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1160   1.1  jmcneill } __packed __aligned(4);
   1161   1.1  jmcneill 
   1162   1.1  jmcneill struct mcx_nic_vport_ctx {
   1163   1.1  jmcneill 	uint32_t		vp_min_wqe_inline_mode;
   1164   1.1  jmcneill 	uint8_t			vp_reserved0[32];
   1165   1.1  jmcneill 	uint32_t		vp_mtu;
   1166   1.1  jmcneill 	uint8_t			vp_reserved1[200];
   1167   1.1  jmcneill 	uint16_t		vp_flags;
   1168   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_UC_MAC			(0)
   1169   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_MC_MAC			(1 << 24)
   1170   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_VLAN			(2 << 24)
   1171   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_ALL			(1 << 13)
   1172   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_MCAST			(1 << 14)
   1173   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_UCAST			(1 << 15)
   1174   1.1  jmcneill 	uint16_t		vp_allowed_list_size;
   1175   1.1  jmcneill 	uint64_t		vp_perm_addr;
   1176   1.1  jmcneill 	uint8_t			vp_reserved2[4];
   1177   1.1  jmcneill 	/* allowed list follows */
   1178   1.1  jmcneill } __packed __aligned(4);
   1179   1.1  jmcneill 
   1180   1.1  jmcneill struct mcx_counter {
   1181   1.1  jmcneill 	uint64_t		packets;
   1182   1.1  jmcneill 	uint64_t		octets;
   1183   1.1  jmcneill } __packed __aligned(4);
   1184   1.1  jmcneill 
   1185   1.1  jmcneill struct mcx_nic_vport_counters {
   1186   1.1  jmcneill 	struct mcx_counter	rx_err;
   1187   1.1  jmcneill 	struct mcx_counter	tx_err;
   1188   1.1  jmcneill 	uint8_t			reserved0[64]; /* 0x30 */
   1189   1.1  jmcneill 	struct mcx_counter	rx_bcast;
   1190   1.1  jmcneill 	struct mcx_counter	tx_bcast;
   1191   1.1  jmcneill 	struct mcx_counter	rx_ucast;
   1192   1.1  jmcneill 	struct mcx_counter	tx_ucast;
   1193   1.1  jmcneill 	struct mcx_counter	rx_mcast;
   1194   1.1  jmcneill 	struct mcx_counter	tx_mcast;
   1195   1.1  jmcneill 	uint8_t			reserved1[0x210 - 0xd0];
   1196   1.1  jmcneill } __packed __aligned(4);
   1197   1.1  jmcneill 
   1198   1.1  jmcneill struct mcx_cmd_query_vport_counters_in {
   1199   1.1  jmcneill 	uint16_t		cmd_opcode;
   1200   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1201   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1202   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1203   1.1  jmcneill } __packed __aligned(4);
   1204   1.1  jmcneill 
   1205   1.1  jmcneill struct mcx_cmd_query_vport_counters_mb_in {
   1206   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1207   1.1  jmcneill 	uint8_t			cmd_clear;
   1208   1.1  jmcneill 	uint8_t			cmd_reserved1[7];
   1209   1.1  jmcneill } __packed __aligned(4);
   1210   1.1  jmcneill 
   1211   1.1  jmcneill struct mcx_cmd_query_vport_counters_out {
   1212   1.1  jmcneill 	uint8_t			cmd_status;
   1213   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1214   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1215   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1216   1.1  jmcneill } __packed __aligned(4);
   1217   1.1  jmcneill 
   1218   1.1  jmcneill struct mcx_cmd_query_flow_counter_in {
   1219   1.1  jmcneill 	uint16_t		cmd_opcode;
   1220   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1221   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1222   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1223   1.1  jmcneill } __packed __aligned(4);
   1224   1.1  jmcneill 
   1225   1.1  jmcneill struct mcx_cmd_query_flow_counter_mb_in {
   1226   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1227   1.1  jmcneill 	uint8_t			cmd_clear;
   1228   1.1  jmcneill 	uint8_t			cmd_reserved1[5];
   1229   1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
   1230   1.1  jmcneill } __packed __aligned(4);
   1231   1.1  jmcneill 
   1232   1.1  jmcneill struct mcx_cmd_query_flow_counter_out {
   1233   1.1  jmcneill 	uint8_t			cmd_status;
   1234   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1235   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1236   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1237   1.1  jmcneill } __packed __aligned(4);
   1238   1.1  jmcneill 
   1239   1.1  jmcneill struct mcx_cmd_alloc_uar_in {
   1240   1.1  jmcneill 	uint16_t		cmd_opcode;
   1241   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1242   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1243   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1244   1.1  jmcneill } __packed __aligned(4);
   1245   1.1  jmcneill 
   1246   1.1  jmcneill struct mcx_cmd_alloc_uar_out {
   1247   1.1  jmcneill 	uint8_t			cmd_status;
   1248   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1249   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1250   1.1  jmcneill 	uint32_t		cmd_uar;
   1251   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1252   1.1  jmcneill } __packed __aligned(4);
   1253   1.1  jmcneill 
   1254   1.1  jmcneill struct mcx_cmd_query_special_ctx_in {
   1255   1.1  jmcneill 	uint16_t		cmd_opcode;
   1256   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1257   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1258   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1259   1.1  jmcneill } __packed __aligned(4);
   1260   1.1  jmcneill 
   1261   1.1  jmcneill struct mcx_cmd_query_special_ctx_out {
   1262   1.1  jmcneill 	uint8_t			cmd_status;
   1263   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1264   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1265   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1266   1.1  jmcneill 	uint32_t		cmd_resd_lkey;
   1267   1.1  jmcneill } __packed __aligned(4);
   1268   1.1  jmcneill 
   1269   1.1  jmcneill struct mcx_eq_ctx {
   1270   1.1  jmcneill 	uint32_t		eq_status;
   1271  1.15  jmcneill #define MCX_EQ_CTX_STATE_SHIFT		8
   1272  1.15  jmcneill #define MCX_EQ_CTX_STATE_MASK		(0xf << MCX_EQ_CTX_STATE_SHIFT)
   1273  1.15  jmcneill #define MCX_EQ_CTX_STATE_ARMED		0x9
   1274  1.15  jmcneill #define MCX_EQ_CTX_STATE_FIRED		0xa
   1275   1.1  jmcneill #define MCX_EQ_CTX_OI_SHIFT		17
   1276   1.1  jmcneill #define MCX_EQ_CTX_OI			(1 << MCX_EQ_CTX_OI_SHIFT)
   1277   1.1  jmcneill #define MCX_EQ_CTX_EC_SHIFT		18
   1278   1.1  jmcneill #define MCX_EQ_CTX_EC			(1 << MCX_EQ_CTX_EC_SHIFT)
   1279   1.1  jmcneill #define MCX_EQ_CTX_STATUS_SHIFT		28
   1280   1.1  jmcneill #define MCX_EQ_CTX_STATUS_MASK		(0xf << MCX_EQ_CTX_STATUS_SHIFT)
   1281  1.15  jmcneill #define MCX_EQ_CTX_STATUS_OK		0x0
   1282  1.15  jmcneill #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa
   1283   1.1  jmcneill 	uint32_t		eq_reserved1;
   1284   1.1  jmcneill 	uint32_t		eq_page_offset;
   1285   1.1  jmcneill #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT	5
   1286   1.1  jmcneill 	uint32_t		eq_uar_size;
   1287   1.1  jmcneill #define MCX_EQ_CTX_UAR_PAGE_MASK	0xffffff
   1288   1.1  jmcneill #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT	24
   1289   1.1  jmcneill 	uint32_t		eq_reserved2;
   1290   1.1  jmcneill 	uint8_t			eq_reserved3[3];
   1291   1.1  jmcneill 	uint8_t			eq_intr;
   1292   1.1  jmcneill 	uint32_t		eq_log_page_size;
   1293   1.1  jmcneill #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1294   1.1  jmcneill 	uint32_t		eq_reserved4[3];
   1295   1.1  jmcneill 	uint32_t		eq_consumer_counter;
   1296   1.1  jmcneill 	uint32_t		eq_producer_counter;
   1297   1.1  jmcneill #define MCX_EQ_CTX_COUNTER_MASK		0xffffff
   1298   1.1  jmcneill 	uint32_t		eq_reserved5[4];
   1299   1.1  jmcneill } __packed __aligned(4);
   1300   1.1  jmcneill 
   1301   1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
   1302   1.1  jmcneill 
   1303   1.1  jmcneill struct mcx_cmd_create_eq_in {
   1304   1.1  jmcneill 	uint16_t		cmd_opcode;
   1305   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1306   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1307   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1308   1.1  jmcneill } __packed __aligned(4);
   1309   1.1  jmcneill 
   1310   1.1  jmcneill struct mcx_cmd_create_eq_mb_in {
   1311   1.1  jmcneill 	struct mcx_eq_ctx	cmd_eq_ctx;
   1312   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1313   1.1  jmcneill 	uint64_t		cmd_event_bitmask;
   1314   1.1  jmcneill #define MCX_EVENT_TYPE_COMPLETION	0x00
   1315   1.1  jmcneill #define MCX_EVENT_TYPE_CQ_ERROR		0x04
   1316   1.1  jmcneill #define MCX_EVENT_TYPE_INTERNAL_ERROR	0x08
   1317   1.1  jmcneill #define MCX_EVENT_TYPE_PORT_CHANGE	0x09
   1318   1.1  jmcneill #define MCX_EVENT_TYPE_CMD_COMPLETION	0x0a
   1319   1.1  jmcneill #define MCX_EVENT_TYPE_PAGE_REQUEST	0x0b
   1320   1.1  jmcneill #define MCX_EVENT_TYPE_LAST_WQE		0x13
   1321   1.1  jmcneill 	uint8_t			cmd_reserved1[176];
   1322   1.1  jmcneill } __packed __aligned(4);
   1323   1.1  jmcneill 
   1324   1.1  jmcneill struct mcx_cmd_create_eq_out {
   1325   1.1  jmcneill 	uint8_t			cmd_status;
   1326   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1327   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1328   1.1  jmcneill 	uint32_t		cmd_eqn;
   1329   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1330   1.1  jmcneill } __packed __aligned(4);
   1331   1.1  jmcneill 
   1332  1.15  jmcneill struct mcx_cmd_query_eq_in {
   1333  1.15  jmcneill 	uint16_t		cmd_opcode;
   1334  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1335  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1336  1.15  jmcneill 	uint32_t		cmd_eqn;
   1337  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1338  1.15  jmcneill } __packed __aligned(4);
   1339  1.15  jmcneill 
   1340  1.15  jmcneill struct mcx_cmd_query_eq_out {
   1341  1.15  jmcneill 	uint8_t			cmd_status;
   1342  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1343  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1344  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1345  1.15  jmcneill } __packed __aligned(4);
   1346  1.15  jmcneill 
   1347   1.1  jmcneill struct mcx_eq_entry {
   1348   1.1  jmcneill 	uint8_t			eq_reserved1;
   1349   1.1  jmcneill 	uint8_t			eq_event_type;
   1350   1.1  jmcneill 	uint8_t			eq_reserved2;
   1351   1.1  jmcneill 	uint8_t			eq_event_sub_type;
   1352   1.1  jmcneill 
   1353   1.1  jmcneill 	uint8_t			eq_reserved3[28];
   1354   1.1  jmcneill 	uint32_t		eq_event_data[7];
   1355   1.1  jmcneill 	uint8_t			eq_reserved4[2];
   1356   1.1  jmcneill 	uint8_t			eq_signature;
   1357   1.1  jmcneill 	uint8_t			eq_owner;
   1358   1.1  jmcneill #define MCX_EQ_ENTRY_OWNER_INIT			1
   1359   1.1  jmcneill } __packed __aligned(4);
   1360   1.1  jmcneill 
   1361   1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_entry) == 64);
   1362   1.1  jmcneill 
   1363   1.1  jmcneill struct mcx_cmd_alloc_pd_in {
   1364   1.1  jmcneill 	uint16_t		cmd_opcode;
   1365   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1366   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1367   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1368   1.1  jmcneill } __packed __aligned(4);
   1369   1.1  jmcneill 
   1370   1.1  jmcneill struct mcx_cmd_alloc_pd_out {
   1371   1.1  jmcneill 	uint8_t			cmd_status;
   1372   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1373   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1374   1.1  jmcneill 	uint32_t		cmd_pd;
   1375   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1376   1.1  jmcneill } __packed __aligned(4);
   1377   1.1  jmcneill 
   1378   1.1  jmcneill struct mcx_cmd_alloc_td_in {
   1379   1.1  jmcneill 	uint16_t		cmd_opcode;
   1380   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1381   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1382   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1383   1.1  jmcneill } __packed __aligned(4);
   1384   1.1  jmcneill 
   1385   1.1  jmcneill struct mcx_cmd_alloc_td_out {
   1386   1.1  jmcneill 	uint8_t			cmd_status;
   1387   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1388   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1389   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1390   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1391   1.1  jmcneill } __packed __aligned(4);
   1392   1.1  jmcneill 
   1393   1.1  jmcneill struct mcx_cmd_create_tir_in {
   1394   1.1  jmcneill 	uint16_t		cmd_opcode;
   1395   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1396   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1397   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1398   1.1  jmcneill } __packed __aligned(4);
   1399   1.1  jmcneill 
   1400   1.1  jmcneill struct mcx_cmd_create_tir_mb_in {
   1401   1.1  jmcneill 	uint8_t			cmd_reserved0[20];
   1402   1.1  jmcneill 	uint32_t		cmd_disp_type;
   1403  1.15  jmcneill #define MCX_TIR_CTX_DISP_TYPE_DIRECT	0
   1404  1.15  jmcneill #define MCX_TIR_CTX_DISP_TYPE_INDIRECT	1
   1405   1.1  jmcneill #define MCX_TIR_CTX_DISP_TYPE_SHIFT	28
   1406   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1407   1.1  jmcneill 	uint32_t		cmd_lro;
   1408   1.1  jmcneill 	uint8_t			cmd_reserved2[8];
   1409   1.1  jmcneill 	uint32_t		cmd_inline_rqn;
   1410   1.1  jmcneill 	uint32_t		cmd_indir_table;
   1411   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1412  1.15  jmcneill #define MCX_TIR_CTX_HASH_TOEPLITZ	2
   1413  1.15  jmcneill #define MCX_TIR_CTX_HASH_SHIFT		28
   1414   1.1  jmcneill 	uint8_t			cmd_rx_hash_key[40];
   1415   1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_outer;
   1416  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_SRC_IP	(1 << 0)
   1417  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_DST_IP	(1 << 1)
   1418  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_SPORT	(1 << 2)
   1419  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_DPORT	(1 << 3)
   1420  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV4	(0 << 31)
   1421  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV6	(1 << 31)
   1422  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_TCP	(0 << 30)
   1423  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_UDP	(1 << 30)
   1424   1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_inner;
   1425   1.1  jmcneill 	uint8_t			cmd_reserved3[152];
   1426   1.1  jmcneill } __packed __aligned(4);
   1427   1.1  jmcneill 
   1428   1.1  jmcneill struct mcx_cmd_create_tir_out {
   1429   1.1  jmcneill 	uint8_t			cmd_status;
   1430   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1431   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1432   1.1  jmcneill 	uint32_t		cmd_tirn;
   1433   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1434   1.1  jmcneill } __packed __aligned(4);
   1435   1.1  jmcneill 
   1436   1.1  jmcneill struct mcx_cmd_destroy_tir_in {
   1437   1.1  jmcneill 	uint16_t		cmd_opcode;
   1438   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1439   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1440   1.1  jmcneill 	uint32_t		cmd_tirn;
   1441   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1442   1.1  jmcneill } __packed __aligned(4);
   1443   1.1  jmcneill 
   1444   1.1  jmcneill struct mcx_cmd_destroy_tir_out {
   1445   1.1  jmcneill 	uint8_t			cmd_status;
   1446   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1447   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1448   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1449   1.1  jmcneill } __packed __aligned(4);
   1450   1.1  jmcneill 
   1451   1.1  jmcneill struct mcx_cmd_create_tis_in {
   1452   1.1  jmcneill 	uint16_t		cmd_opcode;
   1453   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1454   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1455   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1456   1.1  jmcneill } __packed __aligned(4);
   1457   1.1  jmcneill 
   1458   1.1  jmcneill struct mcx_cmd_create_tis_mb_in {
   1459   1.1  jmcneill 	uint8_t			cmd_reserved[16];
   1460   1.1  jmcneill 	uint32_t		cmd_prio;
   1461   1.1  jmcneill 	uint8_t			cmd_reserved1[32];
   1462   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1463   1.1  jmcneill 	uint8_t			cmd_reserved2[120];
   1464   1.1  jmcneill } __packed __aligned(4);
   1465   1.1  jmcneill 
   1466   1.1  jmcneill struct mcx_cmd_create_tis_out {
   1467   1.1  jmcneill 	uint8_t			cmd_status;
   1468   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1469   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1470   1.1  jmcneill 	uint32_t		cmd_tisn;
   1471   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1472   1.1  jmcneill } __packed __aligned(4);
   1473   1.1  jmcneill 
   1474   1.1  jmcneill struct mcx_cmd_destroy_tis_in {
   1475   1.1  jmcneill 	uint16_t		cmd_opcode;
   1476   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1477   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1478   1.1  jmcneill 	uint32_t		cmd_tisn;
   1479   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1480   1.1  jmcneill } __packed __aligned(4);
   1481   1.1  jmcneill 
   1482   1.1  jmcneill struct mcx_cmd_destroy_tis_out {
   1483   1.1  jmcneill 	uint8_t			cmd_status;
   1484   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1485   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1486   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1487   1.1  jmcneill } __packed __aligned(4);
   1488   1.1  jmcneill 
   1489  1.15  jmcneill struct mcx_cmd_create_rqt_in {
   1490  1.15  jmcneill 	uint16_t		cmd_opcode;
   1491  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1492  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1493  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1494  1.15  jmcneill } __packed __aligned(4);
   1495  1.15  jmcneill 
   1496  1.15  jmcneill struct mcx_rqt_ctx {
   1497  1.15  jmcneill 	uint8_t			cmd_reserved0[20];
   1498  1.15  jmcneill 	uint16_t		cmd_reserved1;
   1499  1.15  jmcneill 	uint16_t		cmd_rqt_max_size;
   1500  1.15  jmcneill 	uint16_t		cmd_reserved2;
   1501  1.15  jmcneill 	uint16_t		cmd_rqt_actual_size;
   1502  1.15  jmcneill 	uint8_t			cmd_reserved3[212];
   1503  1.15  jmcneill } __packed __aligned(4);
   1504  1.15  jmcneill 
   1505  1.15  jmcneill struct mcx_cmd_create_rqt_mb_in {
   1506  1.15  jmcneill 	uint8_t			cmd_reserved0[16];
   1507  1.15  jmcneill 	struct mcx_rqt_ctx	cmd_rqt;
   1508  1.15  jmcneill } __packed __aligned(4);
   1509  1.15  jmcneill 
   1510  1.15  jmcneill struct mcx_cmd_create_rqt_out {
   1511  1.15  jmcneill 	uint8_t			cmd_status;
   1512  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1513  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1514  1.15  jmcneill 	uint32_t		cmd_rqtn;
   1515  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1516  1.15  jmcneill } __packed __aligned(4);
   1517  1.15  jmcneill 
   1518  1.15  jmcneill struct mcx_cmd_destroy_rqt_in {
   1519  1.15  jmcneill 	uint16_t		cmd_opcode;
   1520  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1521  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1522  1.15  jmcneill 	uint32_t		cmd_rqtn;
   1523  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1524  1.15  jmcneill } __packed __aligned(4);
   1525  1.15  jmcneill 
   1526  1.15  jmcneill struct mcx_cmd_destroy_rqt_out {
   1527  1.15  jmcneill 	uint8_t			cmd_status;
   1528  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1529  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1530  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1531  1.15  jmcneill } __packed __aligned(4);
   1532  1.15  jmcneill 
   1533   1.1  jmcneill struct mcx_cq_ctx {
   1534   1.1  jmcneill 	uint32_t		cq_status;
   1535  1.15  jmcneill #define MCX_CQ_CTX_STATUS_SHIFT		28
   1536  1.15  jmcneill #define MCX_CQ_CTX_STATUS_MASK		(0xf << MCX_CQ_CTX_STATUS_SHIFT)
   1537  1.15  jmcneill #define MCX_CQ_CTX_STATUS_OK		0x0
   1538  1.15  jmcneill #define MCX_CQ_CTX_STATUS_OVERFLOW	0x9
   1539  1.15  jmcneill #define MCX_CQ_CTX_STATUS_WRITE_FAIL	0xa
   1540  1.15  jmcneill #define MCX_CQ_CTX_STATE_SHIFT		8
   1541  1.15  jmcneill #define MCX_CQ_CTX_STATE_MASK		(0xf << MCX_CQ_CTX_STATE_SHIFT)
   1542  1.15  jmcneill #define MCX_CQ_CTX_STATE_SOLICITED	0x6
   1543  1.15  jmcneill #define MCX_CQ_CTX_STATE_ARMED		0x9
   1544  1.15  jmcneill #define MCX_CQ_CTX_STATE_FIRED		0xa
   1545   1.1  jmcneill 	uint32_t		cq_reserved1;
   1546   1.1  jmcneill 	uint32_t		cq_page_offset;
   1547   1.1  jmcneill 	uint32_t		cq_uar_size;
   1548   1.1  jmcneill #define MCX_CQ_CTX_UAR_PAGE_MASK	0xffffff
   1549   1.1  jmcneill #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT	24
   1550   1.1  jmcneill 	uint32_t		cq_period_max_count;
   1551   1.1  jmcneill #define MCX_CQ_CTX_PERIOD_SHIFT		16
   1552   1.1  jmcneill 	uint32_t		cq_eqn;
   1553   1.1  jmcneill 	uint32_t		cq_log_page_size;
   1554   1.1  jmcneill #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1555   1.1  jmcneill 	uint32_t		cq_reserved2;
   1556   1.1  jmcneill 	uint32_t		cq_last_notified;
   1557   1.1  jmcneill 	uint32_t		cq_last_solicit;
   1558   1.1  jmcneill 	uint32_t		cq_consumer_counter;
   1559   1.1  jmcneill 	uint32_t		cq_producer_counter;
   1560   1.1  jmcneill 	uint8_t			cq_reserved3[8];
   1561   1.1  jmcneill 	uint64_t		cq_doorbell;
   1562   1.1  jmcneill } __packed __aligned(4);
   1563   1.1  jmcneill 
   1564   1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
   1565   1.1  jmcneill 
   1566   1.1  jmcneill struct mcx_cmd_create_cq_in {
   1567   1.1  jmcneill 	uint16_t		cmd_opcode;
   1568   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1569   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1570   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1571   1.1  jmcneill } __packed __aligned(4);
   1572   1.1  jmcneill 
   1573   1.1  jmcneill struct mcx_cmd_create_cq_mb_in {
   1574   1.1  jmcneill 	struct mcx_cq_ctx	cmd_cq_ctx;
   1575   1.1  jmcneill 	uint8_t			cmd_reserved1[192];
   1576   1.1  jmcneill } __packed __aligned(4);
   1577   1.1  jmcneill 
   1578   1.1  jmcneill struct mcx_cmd_create_cq_out {
   1579   1.1  jmcneill 	uint8_t			cmd_status;
   1580   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1581   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1582   1.1  jmcneill 	uint32_t		cmd_cqn;
   1583   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1584   1.1  jmcneill } __packed __aligned(4);
   1585   1.1  jmcneill 
   1586   1.1  jmcneill struct mcx_cmd_destroy_cq_in {
   1587   1.1  jmcneill 	uint16_t		cmd_opcode;
   1588   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1589   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1590   1.1  jmcneill 	uint32_t		cmd_cqn;
   1591   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1592   1.1  jmcneill } __packed __aligned(4);
   1593   1.1  jmcneill 
   1594   1.1  jmcneill struct mcx_cmd_destroy_cq_out {
   1595   1.1  jmcneill 	uint8_t			cmd_status;
   1596   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1597   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1598   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1599   1.1  jmcneill } __packed __aligned(4);
   1600   1.1  jmcneill 
   1601  1.15  jmcneill struct mcx_cmd_query_cq_in {
   1602  1.15  jmcneill 	uint16_t		cmd_opcode;
   1603  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1604  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1605  1.15  jmcneill 	uint32_t		cmd_cqn;
   1606  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1607  1.15  jmcneill } __packed __aligned(4);
   1608  1.15  jmcneill 
   1609  1.15  jmcneill struct mcx_cmd_query_cq_out {
   1610  1.15  jmcneill 	uint8_t			cmd_status;
   1611  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1612  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1613  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1614  1.15  jmcneill } __packed __aligned(4);
   1615  1.15  jmcneill 
   1616   1.1  jmcneill struct mcx_cq_entry {
   1617   1.1  jmcneill 	uint32_t		__reserved__;
   1618   1.1  jmcneill 	uint32_t		cq_lro;
   1619   1.1  jmcneill 	uint32_t		cq_lro_ack_seq_num;
   1620   1.1  jmcneill 	uint32_t		cq_rx_hash;
   1621   1.1  jmcneill 	uint8_t			cq_rx_hash_type;
   1622   1.1  jmcneill 	uint8_t			cq_ml_path;
   1623   1.1  jmcneill 	uint16_t		__reserved__;
   1624   1.1  jmcneill 	uint32_t		cq_checksum;
   1625   1.1  jmcneill 	uint32_t		__reserved__;
   1626   1.1  jmcneill 	uint32_t		cq_flags;
   1627  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L4_OK		(1 << 26)
   1628  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L3_OK		(1 << 25)
   1629  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L2_OK		(1 << 24)
   1630  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_CV			(1 << 16)
   1631  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK		(0xffff)
   1632  1.15  jmcneill 
   1633   1.1  jmcneill 	uint32_t		cq_lro_srqn;
   1634   1.1  jmcneill 	uint32_t		__reserved__[2];
   1635   1.1  jmcneill 	uint32_t		cq_byte_cnt;
   1636   1.1  jmcneill 	uint64_t		cq_timestamp;
   1637   1.1  jmcneill 	uint8_t			cq_rx_drops;
   1638   1.1  jmcneill 	uint8_t			cq_flow_tag[3];
   1639   1.1  jmcneill 	uint16_t		cq_wqe_count;
   1640   1.1  jmcneill 	uint8_t			cq_signature;
   1641   1.1  jmcneill 	uint8_t			cq_opcode_owner;
   1642   1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_OWNER			(1 << 0)
   1643   1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_SE			(1 << 1)
   1644   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_SHIFT		2
   1645   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SHIFT		4
   1646   1.1  jmcneill 
   1647   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_NO_INLINE		0
   1648   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_32		1
   1649   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_64		2
   1650   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_COMPRESSED		3
   1651   1.1  jmcneill 
   1652   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ			0
   1653   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND		2
   1654   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ_ERR		13
   1655   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND_ERR		14
   1656   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_INVALID		15
   1657   1.1  jmcneill 
   1658   1.1  jmcneill } __packed __aligned(4);
   1659   1.1  jmcneill 
   1660   1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_entry) == 64);
   1661   1.1  jmcneill 
   1662   1.1  jmcneill struct mcx_cq_doorbell {
   1663   1.1  jmcneill 	uint32_t		 db_update_ci;
   1664   1.1  jmcneill 	uint32_t		 db_arm_ci;
   1665   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT	28
   1666   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD			(1 << 24)
   1667   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CI_MASK		(0xffffff)
   1668   1.1  jmcneill } __packed __aligned(8);
   1669   1.1  jmcneill 
   1670   1.1  jmcneill struct mcx_wq_ctx {
   1671   1.1  jmcneill 	uint8_t			 wq_type;
   1672   1.1  jmcneill #define MCX_WQ_CTX_TYPE_CYCLIC			(1 << 4)
   1673   1.1  jmcneill #define MCX_WQ_CTX_TYPE_SIGNATURE		(1 << 3)
   1674   1.1  jmcneill 	uint8_t			 wq_reserved0[5];
   1675   1.1  jmcneill 	uint16_t		 wq_lwm;
   1676   1.1  jmcneill 	uint32_t		 wq_pd;
   1677   1.1  jmcneill 	uint32_t		 wq_uar_page;
   1678   1.1  jmcneill 	uint64_t		 wq_doorbell;
   1679   1.1  jmcneill 	uint32_t		 wq_hw_counter;
   1680   1.1  jmcneill 	uint32_t		 wq_sw_counter;
   1681   1.1  jmcneill 	uint16_t		 wq_log_stride;
   1682   1.1  jmcneill 	uint8_t			 wq_log_page_sz;
   1683   1.1  jmcneill 	uint8_t			 wq_log_size;
   1684   1.1  jmcneill 	uint8_t			 wq_reserved1[156];
   1685   1.1  jmcneill } __packed __aligned(4);
   1686   1.1  jmcneill 
   1687   1.1  jmcneill CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
   1688   1.1  jmcneill 
   1689   1.1  jmcneill struct mcx_sq_ctx {
   1690   1.1  jmcneill 	uint32_t		sq_flags;
   1691   1.5   msaitoh #define MCX_SQ_CTX_RLKEY			(1U << 31)
   1692   1.1  jmcneill #define MCX_SQ_CTX_FRE_SHIFT			(1 << 29)
   1693   1.1  jmcneill #define MCX_SQ_CTX_FLUSH_IN_ERROR		(1 << 28)
   1694   1.1  jmcneill #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT		24
   1695   1.1  jmcneill #define MCX_SQ_CTX_STATE_SHIFT			20
   1696  1.15  jmcneill #define MCX_SQ_CTX_STATE_MASK			(0xf << 20)
   1697  1.15  jmcneill #define MCX_SQ_CTX_STATE_RST			0
   1698  1.15  jmcneill #define MCX_SQ_CTX_STATE_RDY			1
   1699  1.15  jmcneill #define MCX_SQ_CTX_STATE_ERR			3
   1700   1.1  jmcneill 	uint32_t		sq_user_index;
   1701   1.1  jmcneill 	uint32_t		sq_cqn;
   1702   1.1  jmcneill 	uint32_t		sq_reserved1[5];
   1703   1.1  jmcneill 	uint32_t		sq_tis_lst_sz;
   1704   1.1  jmcneill #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT		16
   1705   1.1  jmcneill 	uint32_t		sq_reserved2[2];
   1706   1.1  jmcneill 	uint32_t		sq_tis_num;
   1707   1.1  jmcneill 	struct mcx_wq_ctx	sq_wq;
   1708   1.1  jmcneill } __packed __aligned(4);
   1709   1.1  jmcneill 
   1710   1.1  jmcneill struct mcx_sq_entry_seg {
   1711   1.1  jmcneill 	uint32_t		sqs_byte_count;
   1712   1.1  jmcneill 	uint32_t		sqs_lkey;
   1713   1.1  jmcneill 	uint64_t		sqs_addr;
   1714   1.1  jmcneill } __packed __aligned(4);
   1715   1.1  jmcneill 
   1716   1.1  jmcneill struct mcx_sq_entry {
   1717   1.1  jmcneill 	/* control segment */
   1718   1.1  jmcneill 	uint32_t		sqe_opcode_index;
   1719   1.1  jmcneill #define MCX_SQE_WQE_INDEX_SHIFT			8
   1720   1.1  jmcneill #define MCX_SQE_WQE_OPCODE_NOP			0x00
   1721   1.1  jmcneill #define MCX_SQE_WQE_OPCODE_SEND			0x0a
   1722   1.1  jmcneill 	uint32_t		sqe_ds_sq_num;
   1723   1.1  jmcneill #define MCX_SQE_SQ_NUM_SHIFT			8
   1724   1.1  jmcneill 	uint32_t		sqe_signature;
   1725   1.1  jmcneill #define MCX_SQE_SIGNATURE_SHIFT			24
   1726   1.1  jmcneill #define MCX_SQE_SOLICITED_EVENT			0x02
   1727   1.1  jmcneill #define MCX_SQE_CE_CQE_ON_ERR			0x00
   1728   1.1  jmcneill #define MCX_SQE_CE_CQE_FIRST_ERR		0x04
   1729   1.1  jmcneill #define MCX_SQE_CE_CQE_ALWAYS			0x08
   1730   1.1  jmcneill #define MCX_SQE_CE_CQE_SOLICIT			0x0C
   1731   1.1  jmcneill #define MCX_SQE_FM_NO_FENCE			0x00
   1732   1.1  jmcneill #define MCX_SQE_FM_SMALL_FENCE			0x40
   1733   1.1  jmcneill 	uint32_t		sqe_mkey;
   1734   1.1  jmcneill 
   1735   1.1  jmcneill 	/* ethernet segment */
   1736   1.1  jmcneill 	uint32_t		sqe_reserved1;
   1737   1.1  jmcneill 	uint32_t		sqe_mss_csum;
   1738   1.1  jmcneill #define MCX_SQE_L4_CSUM				(1 << 31)
   1739   1.1  jmcneill #define MCX_SQE_L3_CSUM				(1 << 30)
   1740   1.1  jmcneill 	uint32_t		sqe_reserved2;
   1741   1.1  jmcneill 	uint16_t		sqe_inline_header_size;
   1742   1.1  jmcneill 	uint16_t		sqe_inline_headers[9];
   1743   1.1  jmcneill 
   1744   1.1  jmcneill 	/* data segment */
   1745   1.1  jmcneill 	struct mcx_sq_entry_seg sqe_segs[1];
   1746   1.1  jmcneill } __packed __aligned(64);
   1747   1.1  jmcneill 
   1748   1.1  jmcneill CTASSERT(sizeof(struct mcx_sq_entry) == 64);
   1749   1.1  jmcneill 
   1750   1.1  jmcneill struct mcx_cmd_create_sq_in {
   1751   1.1  jmcneill 	uint16_t		cmd_opcode;
   1752   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1753   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1754   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1755   1.1  jmcneill } __packed __aligned(4);
   1756   1.1  jmcneill 
   1757   1.1  jmcneill struct mcx_cmd_create_sq_out {
   1758   1.1  jmcneill 	uint8_t			cmd_status;
   1759   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1760   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1761   1.1  jmcneill 	uint32_t		cmd_sqn;
   1762   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1763   1.1  jmcneill } __packed __aligned(4);
   1764   1.1  jmcneill 
   1765   1.1  jmcneill struct mcx_cmd_modify_sq_in {
   1766   1.1  jmcneill 	uint16_t		cmd_opcode;
   1767   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1768   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1769   1.1  jmcneill 	uint32_t		cmd_sq_state;
   1770   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1771   1.1  jmcneill } __packed __aligned(4);
   1772   1.1  jmcneill 
   1773   1.1  jmcneill struct mcx_cmd_modify_sq_mb_in {
   1774   1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1775   1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1776   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1777   1.1  jmcneill 	struct mcx_sq_ctx	cmd_sq_ctx;
   1778   1.1  jmcneill } __packed __aligned(4);
   1779   1.1  jmcneill 
   1780   1.1  jmcneill struct mcx_cmd_modify_sq_out {
   1781   1.1  jmcneill 	uint8_t			cmd_status;
   1782   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1783   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1784   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1785   1.1  jmcneill } __packed __aligned(4);
   1786   1.1  jmcneill 
   1787   1.1  jmcneill struct mcx_cmd_destroy_sq_in {
   1788   1.1  jmcneill 	uint16_t		cmd_opcode;
   1789   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1790   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1791   1.1  jmcneill 	uint32_t		cmd_sqn;
   1792   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1793   1.1  jmcneill } __packed __aligned(4);
   1794   1.1  jmcneill 
   1795   1.1  jmcneill struct mcx_cmd_destroy_sq_out {
   1796   1.1  jmcneill 	uint8_t			cmd_status;
   1797   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1798   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1799   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1800   1.1  jmcneill } __packed __aligned(4);
   1801   1.1  jmcneill 
   1802   1.1  jmcneill 
   1803   1.1  jmcneill struct mcx_rq_ctx {
   1804   1.1  jmcneill 	uint32_t		rq_flags;
   1805   1.5   msaitoh #define MCX_RQ_CTX_RLKEY			(1U << 31)
   1806   1.1  jmcneill #define MCX_RQ_CTX_VLAN_STRIP_DIS		(1 << 28)
   1807   1.1  jmcneill #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT		24
   1808   1.1  jmcneill #define MCX_RQ_CTX_STATE_SHIFT			20
   1809  1.15  jmcneill #define MCX_RQ_CTX_STATE_MASK			(0xf << 20)
   1810  1.15  jmcneill #define MCX_RQ_CTX_STATE_RST			0
   1811  1.15  jmcneill #define MCX_RQ_CTX_STATE_RDY			1
   1812  1.15  jmcneill #define MCX_RQ_CTX_STATE_ERR			3
   1813   1.1  jmcneill #define MCX_RQ_CTX_FLUSH_IN_ERROR		(1 << 18)
   1814   1.1  jmcneill 	uint32_t		rq_user_index;
   1815   1.1  jmcneill 	uint32_t		rq_cqn;
   1816   1.1  jmcneill 	uint32_t		rq_reserved1;
   1817   1.1  jmcneill 	uint32_t		rq_rmpn;
   1818   1.1  jmcneill 	uint32_t		rq_reserved2[7];
   1819   1.1  jmcneill 	struct mcx_wq_ctx	rq_wq;
   1820   1.1  jmcneill } __packed __aligned(4);
   1821   1.1  jmcneill 
   1822   1.1  jmcneill struct mcx_rq_entry {
   1823   1.1  jmcneill 	uint32_t		rqe_byte_count;
   1824   1.1  jmcneill 	uint32_t		rqe_lkey;
   1825   1.1  jmcneill 	uint64_t		rqe_addr;
   1826   1.1  jmcneill } __packed __aligned(16);
   1827   1.1  jmcneill 
   1828   1.1  jmcneill struct mcx_cmd_create_rq_in {
   1829   1.1  jmcneill 	uint16_t		cmd_opcode;
   1830   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1831   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1832   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1833   1.1  jmcneill } __packed __aligned(4);
   1834   1.1  jmcneill 
   1835   1.1  jmcneill struct mcx_cmd_create_rq_out {
   1836   1.1  jmcneill 	uint8_t			cmd_status;
   1837   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1838   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1839   1.1  jmcneill 	uint32_t		cmd_rqn;
   1840   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1841   1.1  jmcneill } __packed __aligned(4);
   1842   1.1  jmcneill 
   1843   1.1  jmcneill struct mcx_cmd_modify_rq_in {
   1844   1.1  jmcneill 	uint16_t		cmd_opcode;
   1845   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1846   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1847   1.1  jmcneill 	uint32_t		cmd_rq_state;
   1848   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1849   1.1  jmcneill } __packed __aligned(4);
   1850   1.1  jmcneill 
   1851   1.1  jmcneill struct mcx_cmd_modify_rq_mb_in {
   1852   1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1853   1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1854   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1855   1.1  jmcneill 	struct mcx_rq_ctx	cmd_rq_ctx;
   1856   1.1  jmcneill } __packed __aligned(4);
   1857   1.1  jmcneill 
   1858   1.1  jmcneill struct mcx_cmd_modify_rq_out {
   1859   1.1  jmcneill 	uint8_t			cmd_status;
   1860   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1861   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1862   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1863   1.1  jmcneill } __packed __aligned(4);
   1864   1.1  jmcneill 
   1865   1.1  jmcneill struct mcx_cmd_destroy_rq_in {
   1866   1.1  jmcneill 	uint16_t		cmd_opcode;
   1867   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1868   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1869   1.1  jmcneill 	uint32_t		cmd_rqn;
   1870   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1871   1.1  jmcneill } __packed __aligned(4);
   1872   1.1  jmcneill 
   1873   1.1  jmcneill struct mcx_cmd_destroy_rq_out {
   1874   1.1  jmcneill 	uint8_t			cmd_status;
   1875   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1876   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1877   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1878   1.1  jmcneill } __packed __aligned(4);
   1879   1.1  jmcneill 
   1880   1.1  jmcneill struct mcx_cmd_create_flow_table_in {
   1881   1.1  jmcneill 	uint16_t		cmd_opcode;
   1882   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1883   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1884   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1885   1.1  jmcneill } __packed __aligned(4);
   1886   1.1  jmcneill 
   1887   1.1  jmcneill struct mcx_flow_table_ctx {
   1888   1.1  jmcneill 	uint8_t			ft_miss_action;
   1889   1.1  jmcneill 	uint8_t			ft_level;
   1890   1.1  jmcneill 	uint8_t			ft_reserved0;
   1891   1.1  jmcneill 	uint8_t			ft_log_size;
   1892   1.1  jmcneill 	uint32_t		ft_table_miss_id;
   1893   1.1  jmcneill 	uint8_t			ft_reserved1[28];
   1894   1.1  jmcneill } __packed __aligned(4);
   1895   1.1  jmcneill 
   1896   1.1  jmcneill struct mcx_cmd_create_flow_table_mb_in {
   1897   1.1  jmcneill 	uint8_t			cmd_table_type;
   1898   1.1  jmcneill 	uint8_t			cmd_reserved0[7];
   1899   1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   1900   1.1  jmcneill } __packed __aligned(4);
   1901   1.1  jmcneill 
   1902   1.1  jmcneill struct mcx_cmd_create_flow_table_out {
   1903   1.1  jmcneill 	uint8_t			cmd_status;
   1904   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1905   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1906   1.1  jmcneill 	uint32_t		cmd_table_id;
   1907   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1908   1.1  jmcneill } __packed __aligned(4);
   1909   1.1  jmcneill 
   1910   1.1  jmcneill struct mcx_cmd_destroy_flow_table_in {
   1911   1.1  jmcneill 	uint16_t		cmd_opcode;
   1912   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1913   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1914   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1915   1.1  jmcneill } __packed __aligned(4);
   1916   1.1  jmcneill 
   1917   1.1  jmcneill struct mcx_cmd_destroy_flow_table_mb_in {
   1918   1.1  jmcneill 	uint8_t			cmd_table_type;
   1919   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1920   1.1  jmcneill 	uint32_t		cmd_table_id;
   1921   1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   1922   1.1  jmcneill } __packed __aligned(4);
   1923   1.1  jmcneill 
   1924   1.1  jmcneill struct mcx_cmd_destroy_flow_table_out {
   1925   1.1  jmcneill 	uint8_t			cmd_status;
   1926   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1927   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1928   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1929   1.1  jmcneill } __packed __aligned(4);
   1930   1.1  jmcneill 
   1931   1.1  jmcneill struct mcx_cmd_set_flow_table_root_in {
   1932   1.1  jmcneill 	uint16_t		cmd_opcode;
   1933   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1934   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1935   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1936   1.1  jmcneill } __packed __aligned(4);
   1937   1.1  jmcneill 
   1938   1.1  jmcneill struct mcx_cmd_set_flow_table_root_mb_in {
   1939   1.1  jmcneill 	uint8_t			cmd_table_type;
   1940   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1941   1.1  jmcneill 	uint32_t		cmd_table_id;
   1942   1.1  jmcneill 	uint8_t			cmd_reserved1[56];
   1943   1.1  jmcneill } __packed __aligned(4);
   1944   1.1  jmcneill 
   1945   1.1  jmcneill struct mcx_cmd_set_flow_table_root_out {
   1946   1.1  jmcneill 	uint8_t			cmd_status;
   1947   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1948   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1949   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1950   1.1  jmcneill } __packed __aligned(4);
   1951   1.1  jmcneill 
   1952   1.1  jmcneill struct mcx_flow_match {
   1953   1.1  jmcneill 	/* outer headers */
   1954   1.1  jmcneill 	uint8_t			mc_src_mac[6];
   1955   1.1  jmcneill 	uint16_t		mc_ethertype;
   1956   1.1  jmcneill 	uint8_t			mc_dest_mac[6];
   1957   1.1  jmcneill 	uint16_t		mc_first_vlan;
   1958   1.1  jmcneill 	uint8_t			mc_ip_proto;
   1959   1.1  jmcneill 	uint8_t			mc_ip_dscp_ecn;
   1960   1.1  jmcneill 	uint8_t			mc_vlan_flags;
   1961  1.15  jmcneill #define MCX_FLOW_MATCH_IP_FRAG	(1 << 5)
   1962   1.1  jmcneill 	uint8_t			mc_tcp_flags;
   1963   1.1  jmcneill 	uint16_t		mc_tcp_sport;
   1964   1.1  jmcneill 	uint16_t		mc_tcp_dport;
   1965   1.1  jmcneill 	uint32_t		mc_reserved0;
   1966   1.1  jmcneill 	uint16_t		mc_udp_sport;
   1967   1.1  jmcneill 	uint16_t		mc_udp_dport;
   1968   1.1  jmcneill 	uint8_t			mc_src_ip[16];
   1969   1.1  jmcneill 	uint8_t			mc_dest_ip[16];
   1970   1.1  jmcneill 
   1971   1.1  jmcneill 	/* misc parameters */
   1972   1.1  jmcneill 	uint8_t			mc_reserved1[8];
   1973   1.1  jmcneill 	uint16_t		mc_second_vlan;
   1974   1.1  jmcneill 	uint8_t			mc_reserved2[2];
   1975   1.1  jmcneill 	uint8_t			mc_second_vlan_flags;
   1976   1.1  jmcneill 	uint8_t			mc_reserved3[15];
   1977   1.1  jmcneill 	uint32_t		mc_outer_ipv6_flow_label;
   1978   1.1  jmcneill 	uint8_t			mc_reserved4[32];
   1979   1.1  jmcneill 
   1980   1.1  jmcneill 	uint8_t			mc_reserved[384];
   1981   1.1  jmcneill } __packed __aligned(4);
   1982   1.1  jmcneill 
   1983   1.1  jmcneill CTASSERT(sizeof(struct mcx_flow_match) == 512);
   1984   1.1  jmcneill 
   1985   1.1  jmcneill struct mcx_cmd_create_flow_group_in {
   1986   1.1  jmcneill 	uint16_t		cmd_opcode;
   1987   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1988   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1989   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1990   1.1  jmcneill } __packed __aligned(4);
   1991   1.1  jmcneill 
   1992   1.1  jmcneill struct mcx_cmd_create_flow_group_mb_in {
   1993   1.1  jmcneill 	uint8_t			cmd_table_type;
   1994   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1995   1.1  jmcneill 	uint32_t		cmd_table_id;
   1996   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1997   1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   1998   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   1999   1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   2000   1.1  jmcneill 	uint8_t			cmd_reserved3[23];
   2001   1.1  jmcneill 	uint8_t			cmd_match_criteria_enable;
   2002   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER	(1 << 0)
   2003   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_MISC		(1 << 1)
   2004   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_INNER	(1 << 2)
   2005   1.1  jmcneill 	struct mcx_flow_match	cmd_match_criteria;
   2006   1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   2007   1.1  jmcneill } __packed __aligned(4);
   2008   1.1  jmcneill 
   2009   1.1  jmcneill struct mcx_cmd_create_flow_group_out {
   2010   1.1  jmcneill 	uint8_t			cmd_status;
   2011   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2012   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2013   1.1  jmcneill 	uint32_t		cmd_group_id;
   2014   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2015   1.1  jmcneill } __packed __aligned(4);
   2016   1.1  jmcneill 
   2017   1.1  jmcneill struct mcx_flow_ctx {
   2018   1.1  jmcneill 	uint8_t			fc_reserved0[4];
   2019   1.1  jmcneill 	uint32_t		fc_group_id;
   2020   1.1  jmcneill 	uint32_t		fc_flow_tag;
   2021   1.1  jmcneill 	uint32_t		fc_action;
   2022   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_ALLOW		(1 << 0)
   2023   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_DROP		(1 << 1)
   2024   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_FORWARD		(1 << 2)
   2025   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_COUNT		(1 << 3)
   2026   1.1  jmcneill 	uint32_t		fc_dest_list_size;
   2027   1.1  jmcneill 	uint32_t		fc_counter_list_size;
   2028   1.1  jmcneill 	uint8_t			fc_reserved1[40];
   2029   1.1  jmcneill 	struct mcx_flow_match	fc_match_value;
   2030   1.1  jmcneill 	uint8_t			fc_reserved2[192];
   2031   1.1  jmcneill } __packed __aligned(4);
   2032   1.1  jmcneill 
   2033   1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE	(1 << 24)
   2034   1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR		(2 << 24)
   2035   1.1  jmcneill 
   2036   1.1  jmcneill struct mcx_cmd_destroy_flow_group_in {
   2037   1.1  jmcneill 	uint16_t		cmd_opcode;
   2038   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2039   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2040   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2041   1.1  jmcneill } __packed __aligned(4);
   2042   1.1  jmcneill 
   2043   1.1  jmcneill struct mcx_cmd_destroy_flow_group_mb_in {
   2044   1.1  jmcneill 	uint8_t			cmd_table_type;
   2045   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2046   1.1  jmcneill 	uint32_t		cmd_table_id;
   2047   1.1  jmcneill 	uint32_t		cmd_group_id;
   2048   1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   2049   1.1  jmcneill } __packed __aligned(4);
   2050   1.1  jmcneill 
   2051   1.1  jmcneill struct mcx_cmd_destroy_flow_group_out {
   2052   1.1  jmcneill 	uint8_t			cmd_status;
   2053   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2054   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2055   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2056   1.1  jmcneill } __packed __aligned(4);
   2057   1.1  jmcneill 
   2058   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_in {
   2059   1.1  jmcneill 	uint16_t		cmd_opcode;
   2060   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2061   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2062   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2063   1.1  jmcneill } __packed __aligned(4);
   2064   1.1  jmcneill 
   2065   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_mb_in {
   2066   1.1  jmcneill 	uint8_t			cmd_table_type;
   2067   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2068   1.1  jmcneill 	uint32_t		cmd_table_id;
   2069   1.1  jmcneill 	uint32_t		cmd_modify_enable_mask;
   2070   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2071   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2072   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2073   1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   2074   1.1  jmcneill } __packed __aligned(4);
   2075   1.1  jmcneill 
   2076   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_out {
   2077   1.1  jmcneill 	uint8_t			cmd_status;
   2078   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2079   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2080   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2081   1.1  jmcneill } __packed __aligned(4);
   2082   1.1  jmcneill 
   2083   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_in {
   2084   1.1  jmcneill 	uint16_t		cmd_opcode;
   2085   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2086   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2087   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2088   1.1  jmcneill } __packed __aligned(4);
   2089   1.1  jmcneill 
   2090   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_in {
   2091   1.1  jmcneill 	uint8_t			cmd_table_type;
   2092   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2093   1.1  jmcneill 	uint32_t		cmd_table_id;
   2094   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2095   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2096   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2097   1.1  jmcneill } __packed __aligned(4);
   2098   1.1  jmcneill 
   2099   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_out {
   2100   1.1  jmcneill 	uint8_t			cmd_status;
   2101   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2102   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2103   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2104   1.1  jmcneill } __packed __aligned(4);
   2105   1.1  jmcneill 
   2106   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_out {
   2107   1.1  jmcneill 	uint8_t			cmd_reserved0[48];
   2108   1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   2109   1.1  jmcneill } __packed __aligned(4);
   2110   1.1  jmcneill 
   2111   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_in {
   2112   1.1  jmcneill 	uint16_t		cmd_opcode;
   2113   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2114   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2115   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2116   1.1  jmcneill } __packed __aligned(4);
   2117   1.1  jmcneill 
   2118   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in {
   2119   1.1  jmcneill 	uint8_t			cmd_table_type;
   2120   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2121   1.1  jmcneill 	uint32_t		cmd_table_id;
   2122   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2123   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2124   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2125   1.1  jmcneill } __packed __aligned(4);
   2126   1.1  jmcneill 
   2127   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_out {
   2128   1.1  jmcneill 	uint8_t			cmd_status;
   2129   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2130   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2131   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2132   1.1  jmcneill } __packed __aligned(4);
   2133   1.1  jmcneill 
   2134   1.1  jmcneill struct mcx_cmd_query_flow_group_in {
   2135   1.1  jmcneill 	uint16_t		cmd_opcode;
   2136   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2137   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2138   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2139   1.1  jmcneill } __packed __aligned(4);
   2140   1.1  jmcneill 
   2141   1.1  jmcneill struct mcx_cmd_query_flow_group_mb_in {
   2142   1.1  jmcneill 	uint8_t			cmd_table_type;
   2143   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2144   1.1  jmcneill 	uint32_t		cmd_table_id;
   2145   1.1  jmcneill 	uint32_t		cmd_group_id;
   2146   1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   2147   1.1  jmcneill } __packed __aligned(4);
   2148   1.1  jmcneill 
   2149   1.1  jmcneill struct mcx_cmd_query_flow_group_out {
   2150   1.1  jmcneill 	uint8_t			cmd_status;
   2151   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2152   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2153   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2154   1.1  jmcneill } __packed __aligned(4);
   2155   1.1  jmcneill 
   2156   1.1  jmcneill struct mcx_cmd_query_flow_group_mb_out {
   2157   1.1  jmcneill 	uint8_t			cmd_reserved0[12];
   2158   1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   2159   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2160   1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   2161   1.1  jmcneill 	uint8_t			cmd_reserved2[20];
   2162   1.1  jmcneill 	uint32_t		cmd_match_criteria_enable;
   2163   1.1  jmcneill 	uint8_t			cmd_match_criteria[512];
   2164   1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   2165   1.1  jmcneill } __packed __aligned(4);
   2166   1.1  jmcneill 
   2167   1.1  jmcneill struct mcx_cmd_query_flow_table_in {
   2168   1.1  jmcneill 	uint16_t		cmd_opcode;
   2169   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2170   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2171   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2172   1.1  jmcneill } __packed __aligned(4);
   2173   1.1  jmcneill 
   2174   1.1  jmcneill struct mcx_cmd_query_flow_table_mb_in {
   2175   1.1  jmcneill 	uint8_t			cmd_table_type;
   2176   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2177   1.1  jmcneill 	uint32_t		cmd_table_id;
   2178   1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   2179   1.1  jmcneill } __packed __aligned(4);
   2180   1.1  jmcneill 
   2181   1.1  jmcneill struct mcx_cmd_query_flow_table_out {
   2182   1.1  jmcneill 	uint8_t			cmd_status;
   2183   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2184   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2185   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2186   1.1  jmcneill } __packed __aligned(4);
   2187   1.1  jmcneill 
   2188   1.1  jmcneill struct mcx_cmd_query_flow_table_mb_out {
   2189   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2190   1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   2191   1.1  jmcneill } __packed __aligned(4);
   2192   1.1  jmcneill 
   2193   1.1  jmcneill struct mcx_cmd_alloc_flow_counter_in {
   2194   1.1  jmcneill 	uint16_t		cmd_opcode;
   2195   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2196   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2197   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2198   1.1  jmcneill } __packed __aligned(4);
   2199   1.1  jmcneill 
   2200   1.1  jmcneill struct mcx_cmd_query_rq_in {
   2201   1.1  jmcneill 	uint16_t		cmd_opcode;
   2202   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2203   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2204   1.1  jmcneill 	uint32_t		cmd_rqn;
   2205   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2206   1.1  jmcneill } __packed __aligned(4);
   2207   1.1  jmcneill 
   2208   1.1  jmcneill struct mcx_cmd_query_rq_out {
   2209   1.1  jmcneill 	uint8_t			cmd_status;
   2210   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2211   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2212   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2213   1.1  jmcneill } __packed __aligned(4);
   2214   1.1  jmcneill 
   2215   1.1  jmcneill struct mcx_cmd_query_rq_mb_out {
   2216   1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   2217   1.1  jmcneill 	struct mcx_rq_ctx	cmd_ctx;
   2218   1.1  jmcneill };
   2219   1.1  jmcneill 
   2220   1.1  jmcneill struct mcx_cmd_query_sq_in {
   2221   1.1  jmcneill 	uint16_t		cmd_opcode;
   2222   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2223   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2224   1.1  jmcneill 	uint32_t		cmd_sqn;
   2225   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2226   1.1  jmcneill } __packed __aligned(4);
   2227   1.1  jmcneill 
   2228   1.1  jmcneill struct mcx_cmd_query_sq_out {
   2229   1.1  jmcneill 	uint8_t			cmd_status;
   2230   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2231   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2232   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2233   1.1  jmcneill } __packed __aligned(4);
   2234   1.1  jmcneill 
   2235   1.1  jmcneill struct mcx_cmd_query_sq_mb_out {
   2236   1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   2237   1.1  jmcneill 	struct mcx_sq_ctx	cmd_ctx;
   2238   1.1  jmcneill };
   2239   1.1  jmcneill 
   2240   1.1  jmcneill struct mcx_cmd_alloc_flow_counter_out {
   2241   1.1  jmcneill 	uint8_t			cmd_status;
   2242   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2243   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2244   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
   2245   1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
   2246   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   2247   1.1  jmcneill } __packed __aligned(4);
   2248   1.1  jmcneill 
   2249   1.1  jmcneill struct mcx_wq_doorbell {
   2250   1.1  jmcneill 	uint32_t		 db_recv_counter;
   2251   1.1  jmcneill 	uint32_t		 db_send_counter;
   2252   1.1  jmcneill } __packed __aligned(8);
   2253   1.1  jmcneill 
   2254   1.1  jmcneill struct mcx_dmamem {
   2255   1.1  jmcneill 	bus_dmamap_t		 mxm_map;
   2256   1.1  jmcneill 	bus_dma_segment_t	 mxm_seg;
   2257   1.1  jmcneill 	int			 mxm_nsegs;
   2258   1.1  jmcneill 	size_t			 mxm_size;
   2259   1.1  jmcneill 	void			*mxm_kva;
   2260   1.1  jmcneill };
   2261   1.1  jmcneill #define MCX_DMA_MAP(_mxm)	((_mxm)->mxm_map)
   2262   1.1  jmcneill #define MCX_DMA_DVA(_mxm)	((_mxm)->mxm_map->dm_segs[0].ds_addr)
   2263   1.1  jmcneill #define MCX_DMA_KVA(_mxm)	((void *)(_mxm)->mxm_kva)
   2264  1.15  jmcneill #define MCX_DMA_OFF(_mxm, _off)	((void *)((char *)(_mxm)->mxm_kva + (_off)))
   2265   1.1  jmcneill #define MCX_DMA_LEN(_mxm)	((_mxm)->mxm_size)
   2266   1.1  jmcneill 
   2267   1.1  jmcneill struct mcx_hwmem {
   2268   1.1  jmcneill 	bus_dmamap_t		 mhm_map;
   2269   1.1  jmcneill 	bus_dma_segment_t	*mhm_segs;
   2270   1.1  jmcneill 	unsigned int		 mhm_seg_count;
   2271   1.1  jmcneill 	unsigned int		 mhm_npages;
   2272   1.1  jmcneill };
   2273   1.1  jmcneill 
   2274   1.1  jmcneill struct mcx_slot {
   2275   1.1  jmcneill 	bus_dmamap_t		 ms_map;
   2276   1.1  jmcneill 	struct mbuf		*ms_m;
   2277   1.1  jmcneill };
   2278   1.1  jmcneill 
   2279  1.15  jmcneill struct mcx_eq {
   2280  1.15  jmcneill 	int			 eq_n;
   2281  1.15  jmcneill 	uint32_t		 eq_cons;
   2282  1.15  jmcneill 	struct mcx_dmamem	 eq_mem;
   2283  1.15  jmcneill };
   2284  1.15  jmcneill 
   2285   1.1  jmcneill struct mcx_cq {
   2286   1.1  jmcneill 	int			 cq_n;
   2287   1.1  jmcneill 	struct mcx_dmamem	 cq_mem;
   2288  1.15  jmcneill 	bus_addr_t		 cq_doorbell;
   2289   1.1  jmcneill 	uint32_t		 cq_cons;
   2290   1.1  jmcneill 	uint32_t		 cq_count;
   2291   1.1  jmcneill };
   2292   1.1  jmcneill 
   2293   1.1  jmcneill struct mcx_calibration {
   2294   1.1  jmcneill 	uint64_t		 c_timestamp;	/* previous mcx chip time */
   2295   1.1  jmcneill 	uint64_t		 c_uptime;	/* previous kernel nanouptime */
   2296   1.1  jmcneill 	uint64_t		 c_tbase;	/* mcx chip time */
   2297   1.1  jmcneill 	uint64_t		 c_ubase;	/* kernel nanouptime */
   2298  1.15  jmcneill 	uint64_t		 c_ratio;
   2299   1.1  jmcneill };
   2300   1.1  jmcneill 
   2301   1.1  jmcneill #define MCX_CALIBRATE_FIRST    2
   2302  1.15  jmcneill #define MCX_CALIBRATE_NORMAL   32
   2303   1.1  jmcneill 
   2304   1.1  jmcneill struct mcx_rxring {
   2305   1.1  jmcneill 	u_int			 rxr_total;
   2306   1.1  jmcneill 	u_int			 rxr_inuse;
   2307   1.1  jmcneill };
   2308   1.1  jmcneill 
   2309   1.1  jmcneill MBUFQ_HEAD(mcx_mbufq);
   2310   1.1  jmcneill 
   2311  1.15  jmcneill struct mcx_rx {
   2312  1.15  jmcneill 	struct mcx_softc	*rx_softc;
   2313  1.15  jmcneill 
   2314  1.15  jmcneill 	int			 rx_rqn;
   2315  1.15  jmcneill 	struct mcx_dmamem	 rx_rq_mem;
   2316  1.15  jmcneill 	struct mcx_slot		*rx_slots;
   2317  1.15  jmcneill 	bus_addr_t		 rx_doorbell;
   2318  1.15  jmcneill 
   2319  1.15  jmcneill 	uint32_t		 rx_prod;
   2320  1.15  jmcneill 	callout_t		 rx_refill;
   2321  1.15  jmcneill 	struct mcx_rxring	 rx_rxr;
   2322  1.15  jmcneill } __aligned(64);
   2323  1.15  jmcneill 
   2324  1.15  jmcneill struct mcx_tx {
   2325  1.15  jmcneill 	struct mcx_softc	*tx_softc;
   2326  1.16  jmcneill 	kmutex_t		 tx_lock;
   2327  1.16  jmcneill 	pcq_t			*tx_pcq;
   2328  1.16  jmcneill 	void			*tx_softint;
   2329  1.15  jmcneill 
   2330  1.15  jmcneill 	int			 tx_uar;
   2331  1.15  jmcneill 	int			 tx_sqn;
   2332  1.15  jmcneill 	struct mcx_dmamem	 tx_sq_mem;
   2333  1.15  jmcneill 	struct mcx_slot		*tx_slots;
   2334  1.15  jmcneill 	bus_addr_t		 tx_doorbell;
   2335  1.15  jmcneill 	int			 tx_bf_offset;
   2336  1.15  jmcneill 
   2337  1.15  jmcneill 	uint32_t		 tx_cons;
   2338  1.15  jmcneill 	uint32_t		 tx_prod;
   2339  1.15  jmcneill } __aligned(64);
   2340  1.15  jmcneill 
   2341  1.15  jmcneill struct mcx_queues {
   2342  1.15  jmcneill 	void			*q_ihc;
   2343  1.15  jmcneill 	struct mcx_softc	*q_sc;
   2344  1.15  jmcneill 	int			 q_uar;
   2345  1.15  jmcneill 	int			 q_index;
   2346  1.15  jmcneill 	struct mcx_rx		 q_rx;
   2347  1.15  jmcneill 	struct mcx_tx		 q_tx;
   2348  1.15  jmcneill 	struct mcx_cq		 q_cq;
   2349  1.15  jmcneill 	struct mcx_eq		 q_eq;
   2350  1.15  jmcneill #if NKSTAT > 0
   2351  1.15  jmcneill 	struct kstat		*q_kstat;
   2352  1.15  jmcneill #endif
   2353  1.15  jmcneill };
   2354  1.15  jmcneill 
   2355  1.15  jmcneill struct mcx_flow_group {
   2356  1.15  jmcneill 	int			 g_id;
   2357  1.15  jmcneill 	int			 g_table;
   2358  1.15  jmcneill 	int			 g_start;
   2359  1.15  jmcneill 	int			 g_size;
   2360  1.15  jmcneill };
   2361  1.15  jmcneill 
   2362  1.15  jmcneill #define MCX_FLOW_GROUP_PROMISC		0
   2363  1.15  jmcneill #define MCX_FLOW_GROUP_ALLMULTI		1
   2364  1.15  jmcneill #define MCX_FLOW_GROUP_MAC		2
   2365  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_L4		3
   2366  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_L3		4
   2367  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_NONE		5
   2368  1.15  jmcneill #define	MCX_NUM_FLOW_GROUPS		6
   2369  1.15  jmcneill 
   2370  1.15  jmcneill #define MCX_HASH_SEL_L3		MCX_TIR_CTX_HASH_SEL_SRC_IP | \
   2371  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_DST_IP
   2372  1.15  jmcneill #define MCX_HASH_SEL_L4		MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \
   2373  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_DPORT
   2374  1.15  jmcneill 
   2375  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP  |\
   2376  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV4
   2377  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6_TCP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \
   2378  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV6
   2379  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
   2380  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV4
   2381  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
   2382  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV6
   2383  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4
   2384  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6
   2385  1.15  jmcneill 
   2386  1.15  jmcneill /*
   2387  1.15  jmcneill  * There are a few different pieces involved in configuring RSS.
   2388  1.15  jmcneill  * A Receive Queue Table (RQT) is the indirection table that maps packets to
   2389  1.15  jmcneill  * different rx queues based on a hash value.  We only create one, because
   2390  1.15  jmcneill  * we want to scatter any traffic we can apply RSS to across all our rx
   2391  1.15  jmcneill  * queues.  Anything else will only be delivered to the first rx queue,
   2392  1.15  jmcneill  * which doesn't require an RQT.
   2393  1.15  jmcneill  *
   2394  1.15  jmcneill  * A Transport Interface Receive (TIR) delivers packets to either a single rx
   2395  1.15  jmcneill  * queue or an RQT, and in the latter case, specifies the set of fields
   2396  1.15  jmcneill  * hashed, the hash function, and the hash key.  We need one of these for each
   2397  1.15  jmcneill  * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6,
   2398  1.15  jmcneill  * and one for non-RSS traffic.
   2399  1.15  jmcneill  *
   2400  1.15  jmcneill  * Flow tables hold flow table entries in sequence.  The first entry that
   2401  1.15  jmcneill  * matches a packet is applied, sending the packet to either another flow
   2402  1.15  jmcneill  * table or a TIR.  We use one flow table to select packets based on
   2403  1.15  jmcneill  * destination MAC address, and a second to apply RSS.  The entries in the
   2404  1.15  jmcneill  * first table send matching packets to the second, and the entries in the
   2405  1.15  jmcneill  * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR.
   2406  1.15  jmcneill  *
   2407  1.15  jmcneill  * The flow table entry that delivers packets to an RSS TIR must include match
   2408  1.15  jmcneill  * criteria that ensure packets delivered to the TIR include all the fields
   2409  1.15  jmcneill  * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must
   2410  1.15  jmcneill  * only accept v4 TCP packets.  Accordingly, we need flow table entries for
   2411  1.15  jmcneill  * each TIR.
   2412  1.15  jmcneill  *
   2413  1.15  jmcneill  * All of this is a lot more flexible than we need, and we can describe most
   2414  1.15  jmcneill  * of the stuff we need with a simple array.
   2415  1.15  jmcneill  *
   2416  1.15  jmcneill  * An RSS config creates a TIR with hashing enabled on a set of fields,
   2417  1.15  jmcneill  * pointing to either the first rx queue or the RQT containing all the rx
   2418  1.15  jmcneill  * queues, and a flow table entry that matches on an ether type and
   2419  1.15  jmcneill  * optionally an ip proto, that delivers packets to the TIR.
   2420  1.15  jmcneill  */
   2421  1.15  jmcneill static struct mcx_rss_rule {
   2422  1.15  jmcneill 	int			hash_sel;
   2423  1.15  jmcneill 	int			flow_group;
   2424  1.15  jmcneill 	int			ethertype;
   2425  1.15  jmcneill 	int			ip_proto;
   2426  1.15  jmcneill } mcx_rss_config[] = {
   2427  1.15  jmcneill 	/* udp and tcp for v4/v6 */
   2428  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4,
   2429  1.15  jmcneill 	  ETHERTYPE_IP, IPPROTO_TCP },
   2430  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4,
   2431  1.15  jmcneill 	  ETHERTYPE_IPV6, IPPROTO_TCP },
   2432  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4,
   2433  1.15  jmcneill 	  ETHERTYPE_IP, IPPROTO_UDP },
   2434  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4,
   2435  1.15  jmcneill 	  ETHERTYPE_IPV6, IPPROTO_UDP },
   2436  1.15  jmcneill 
   2437  1.15  jmcneill 	/* other v4/v6 */
   2438  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3,
   2439  1.15  jmcneill 	  ETHERTYPE_IP, 0 },
   2440  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3,
   2441  1.15  jmcneill 	  ETHERTYPE_IPV6, 0 },
   2442  1.15  jmcneill 
   2443  1.15  jmcneill 	/* non v4/v6 */
   2444  1.15  jmcneill 	{ 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 }
   2445  1.15  jmcneill };
   2446  1.15  jmcneill 
   2447   1.1  jmcneill struct mcx_softc {
   2448   1.1  jmcneill 	device_t		 sc_dev;
   2449   1.1  jmcneill 	struct ethercom		 sc_ec;
   2450   1.1  jmcneill 	struct ifmedia		 sc_media;
   2451   1.1  jmcneill 	uint64_t		 sc_media_status;
   2452   1.1  jmcneill 	uint64_t		 sc_media_active;
   2453  1.12   thorpej 	kmutex_t		 sc_media_mutex;
   2454   1.1  jmcneill 
   2455   1.1  jmcneill 	pci_chipset_tag_t	 sc_pc;
   2456   1.1  jmcneill 	pci_intr_handle_t	*sc_intrs;
   2457  1.15  jmcneill 	void			*sc_ihc;
   2458   1.1  jmcneill 	pcitag_t		 sc_tag;
   2459   1.1  jmcneill 
   2460   1.1  jmcneill 	bus_dma_tag_t		 sc_dmat;
   2461   1.1  jmcneill 	bus_space_tag_t		 sc_memt;
   2462   1.1  jmcneill 	bus_space_handle_t	 sc_memh;
   2463   1.1  jmcneill 	bus_size_t		 sc_mems;
   2464   1.1  jmcneill 
   2465   1.1  jmcneill 	struct mcx_dmamem	 sc_cmdq_mem;
   2466   1.1  jmcneill 	unsigned int		 sc_cmdq_mask;
   2467   1.1  jmcneill 	unsigned int		 sc_cmdq_size;
   2468   1.1  jmcneill 
   2469   1.1  jmcneill 	unsigned int		 sc_cmdq_token;
   2470   1.1  jmcneill 
   2471   1.1  jmcneill 	struct mcx_hwmem	 sc_boot_pages;
   2472   1.1  jmcneill 	struct mcx_hwmem	 sc_init_pages;
   2473   1.1  jmcneill 	struct mcx_hwmem	 sc_regular_pages;
   2474   1.1  jmcneill 
   2475   1.1  jmcneill 	int			 sc_uar;
   2476   1.1  jmcneill 	int			 sc_pd;
   2477   1.1  jmcneill 	int			 sc_tdomain;
   2478   1.1  jmcneill 	uint32_t		 sc_lkey;
   2479  1.15  jmcneill 	int			 sc_tis;
   2480  1.15  jmcneill 	int			 sc_tir[__arraycount(mcx_rss_config)];
   2481  1.15  jmcneill 	int			 sc_rqt;
   2482   1.1  jmcneill 
   2483   1.1  jmcneill 	struct mcx_dmamem	 sc_doorbell_mem;
   2484   1.1  jmcneill 
   2485  1.15  jmcneill 	struct mcx_eq		 sc_admin_eq;
   2486  1.15  jmcneill 	struct mcx_eq		 sc_queue_eq;
   2487  1.15  jmcneill 
   2488   1.1  jmcneill 	int			 sc_hardmtu;
   2489  1.15  jmcneill 	int			 sc_rxbufsz;
   2490  1.15  jmcneill 
   2491  1.15  jmcneill 	int			 sc_bf_size;
   2492  1.15  jmcneill 	int			 sc_max_rqt_size;
   2493   1.1  jmcneill 
   2494   1.1  jmcneill 	struct workqueue	*sc_workq;
   2495   1.1  jmcneill 	struct work		 sc_port_change;
   2496   1.1  jmcneill 
   2497  1.15  jmcneill 	int			 sc_mac_flow_table_id;
   2498  1.15  jmcneill 	int			 sc_rss_flow_table_id;
   2499  1.15  jmcneill 	struct mcx_flow_group	 sc_flow_group[MCX_NUM_FLOW_GROUPS];
   2500   1.1  jmcneill 	int			 sc_promisc_flow_enabled;
   2501   1.1  jmcneill 	int			 sc_allmulti_flow_enabled;
   2502   1.1  jmcneill 	int			 sc_mcast_flow_base;
   2503   1.1  jmcneill 	int			 sc_extra_mcast;
   2504   1.1  jmcneill 	uint8_t			 sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];
   2505   1.1  jmcneill 
   2506   1.1  jmcneill 	struct mcx_calibration	 sc_calibration[2];
   2507   1.1  jmcneill 	unsigned int		 sc_calibration_gen;
   2508   1.1  jmcneill 	callout_t		 sc_calibrate;
   2509  1.15  jmcneill 	uint32_t		 sc_mhz;
   2510  1.15  jmcneill 	uint32_t		 sc_khz;
   2511   1.1  jmcneill 
   2512  1.15  jmcneill 	struct mcx_queues	*sc_queues;
   2513  1.15  jmcneill 	unsigned int		 sc_nqueues;
   2514   1.1  jmcneill 
   2515  1.15  jmcneill 	int			 sc_mcam_reg;
   2516   1.1  jmcneill 
   2517  1.15  jmcneill #if NKSTAT > 0
   2518  1.15  jmcneill 	struct kstat		*sc_kstat_ieee8023;
   2519  1.15  jmcneill 	struct kstat		*sc_kstat_rfc2863;
   2520  1.15  jmcneill 	struct kstat		*sc_kstat_rfc2819;
   2521  1.15  jmcneill 	struct kstat		*sc_kstat_rfc3635;
   2522  1.15  jmcneill 	unsigned int		 sc_kstat_mtmp_count;
   2523  1.15  jmcneill 	struct kstat		**sc_kstat_mtmp;
   2524  1.15  jmcneill #endif
   2525   1.1  jmcneill 
   2526  1.15  jmcneill 	struct timecounter	 sc_timecounter;
   2527   1.1  jmcneill };
   2528   1.1  jmcneill #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
   2529   1.1  jmcneill 
   2530   1.1  jmcneill static int	mcx_match(device_t, cfdata_t, void *);
   2531   1.1  jmcneill static void	mcx_attach(device_t, device_t, void *);
   2532   1.1  jmcneill 
   2533  1.15  jmcneill static void *	mcx_establish_intr(struct mcx_softc *, int, kcpuset_t *,
   2534  1.15  jmcneill 		    int (*)(void *), void *, const char *);
   2535  1.15  jmcneill 
   2536   1.1  jmcneill static void	mcx_rxr_init(struct mcx_rxring *, u_int, u_int);
   2537   1.1  jmcneill static u_int	mcx_rxr_get(struct mcx_rxring *, u_int);
   2538   1.1  jmcneill static void	mcx_rxr_put(struct mcx_rxring *, u_int);
   2539   1.1  jmcneill static u_int	mcx_rxr_inuse(struct mcx_rxring *);
   2540   1.1  jmcneill 
   2541  1.15  jmcneill #if NKSTAT > 0
   2542  1.15  jmcneill static void	mcx_kstat_attach(struct mcx_softc *);
   2543  1.15  jmcneill #endif
   2544  1.15  jmcneill 
   2545  1.15  jmcneill static void	mcx_timecounter_attach(struct mcx_softc *);
   2546  1.15  jmcneill 
   2547   1.1  jmcneill static int	mcx_version(struct mcx_softc *);
   2548   1.1  jmcneill static int	mcx_init_wait(struct mcx_softc *);
   2549   1.1  jmcneill static int	mcx_enable_hca(struct mcx_softc *);
   2550   1.1  jmcneill static int	mcx_teardown_hca(struct mcx_softc *, uint16_t);
   2551   1.1  jmcneill static int	mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
   2552   1.1  jmcneill 		    int);
   2553   1.1  jmcneill static int	mcx_issi(struct mcx_softc *);
   2554   1.1  jmcneill static int	mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
   2555   1.1  jmcneill static int	mcx_hca_max_caps(struct mcx_softc *);
   2556   1.1  jmcneill static int	mcx_hca_set_caps(struct mcx_softc *);
   2557   1.1  jmcneill static int	mcx_init_hca(struct mcx_softc *);
   2558   1.1  jmcneill static int	mcx_set_driver_version(struct mcx_softc *);
   2559   1.1  jmcneill static int	mcx_iff(struct mcx_softc *);
   2560  1.15  jmcneill static int	mcx_alloc_uar(struct mcx_softc *, int *);
   2561   1.1  jmcneill static int	mcx_alloc_pd(struct mcx_softc *);
   2562   1.1  jmcneill static int	mcx_alloc_tdomain(struct mcx_softc *);
   2563  1.15  jmcneill static int	mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int,
   2564  1.15  jmcneill 		    uint64_t, int);
   2565   1.1  jmcneill static int	mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *);
   2566   1.1  jmcneill static int	mcx_query_special_contexts(struct mcx_softc *);
   2567   1.1  jmcneill static int	mcx_set_port_mtu(struct mcx_softc *, int);
   2568  1.15  jmcneill static int	mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int,
   2569  1.15  jmcneill 		    int);
   2570  1.15  jmcneill static int	mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *);
   2571  1.15  jmcneill static int	mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int,
   2572  1.15  jmcneill 		    int);
   2573  1.15  jmcneill static int	mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *);
   2574  1.15  jmcneill static int	mcx_ready_sq(struct mcx_softc *, struct mcx_tx *);
   2575  1.15  jmcneill static int	mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int);
   2576  1.15  jmcneill static int	mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *);
   2577  1.15  jmcneill static int	mcx_ready_rq(struct mcx_softc *, struct mcx_rx *);
   2578  1.15  jmcneill static int	mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *,
   2579  1.15  jmcneill 		    int *);
   2580  1.15  jmcneill static int	mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t,
   2581  1.15  jmcneill 		    int *);
   2582  1.15  jmcneill static int	mcx_destroy_tir(struct mcx_softc *, int);
   2583  1.15  jmcneill static int	mcx_create_tis(struct mcx_softc *, int *);
   2584  1.15  jmcneill static int	mcx_destroy_tis(struct mcx_softc *, int);
   2585  1.15  jmcneill static int	mcx_create_rqt(struct mcx_softc *, int, int *, int *);
   2586  1.15  jmcneill static int	mcx_destroy_rqt(struct mcx_softc *, int);
   2587  1.15  jmcneill static int	mcx_create_flow_table(struct mcx_softc *, int, int, int *);
   2588  1.15  jmcneill static int	mcx_set_flow_table_root(struct mcx_softc *, int);
   2589  1.15  jmcneill static int	mcx_destroy_flow_table(struct mcx_softc *, int);
   2590  1.15  jmcneill static int	mcx_create_flow_group(struct mcx_softc *, int, int, int,
   2591   1.1  jmcneill 		    int, int, struct mcx_flow_match *);
   2592   1.1  jmcneill static int	mcx_destroy_flow_group(struct mcx_softc *, int);
   2593  1.15  jmcneill static int	mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int,
   2594  1.15  jmcneill 		    const uint8_t *, uint32_t);
   2595  1.15  jmcneill static int	mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int,
   2596  1.15  jmcneill 		    int, int, uint32_t);
   2597   1.1  jmcneill static int	mcx_delete_flow_table_entry(struct mcx_softc *, int, int);
   2598   1.1  jmcneill 
   2599  1.15  jmcneill #if NKSTAT > 0
   2600  1.15  jmcneill static int	mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *);
   2601  1.15  jmcneill static int	mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *);
   2602  1.15  jmcneill static int	mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *);
   2603  1.15  jmcneill static int	mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *);
   2604  1.15  jmcneill #endif
   2605  1.15  jmcneill 
   2606   1.1  jmcneill #if 0
   2607  1.15  jmcneill static int	mcx_dump_flow_table(struct mcx_softc *, int);
   2608  1.15  jmcneill static int	mcx_dump_flow_table_entry(struct mcx_softc *, int, int);
   2609  1.15  jmcneill static int	mcx_dump_flow_group(struct mcx_softc *, int);
   2610   1.1  jmcneill #endif
   2611   1.1  jmcneill 
   2612   1.1  jmcneill 
   2613   1.1  jmcneill /*
   2614   1.1  jmcneill static void	mcx_cmdq_dump(const struct mcx_cmdq_entry *);
   2615   1.1  jmcneill static void	mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
   2616   1.1  jmcneill */
   2617   1.1  jmcneill static void	mcx_refill(void *);
   2618  1.15  jmcneill static int	mcx_process_rx(struct mcx_softc *, struct mcx_rx *,
   2619  1.15  jmcneill 		    struct mcx_cq_entry *, struct mcx_mbufq *,
   2620  1.15  jmcneill 		    const struct mcx_calibration *);
   2621  1.15  jmcneill static int	mcx_process_txeof(struct mcx_softc *, struct mcx_tx *,
   2622  1.15  jmcneill 		    struct mcx_cq_entry *);
   2623  1.15  jmcneill static void	mcx_process_cq(struct mcx_softc *, struct mcx_queues *,
   2624  1.15  jmcneill 		    struct mcx_cq *);
   2625  1.15  jmcneill 
   2626  1.15  jmcneill static void	mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int);
   2627  1.15  jmcneill static void	mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int);
   2628  1.15  jmcneill static int	mcx_admin_intr(void *);
   2629  1.15  jmcneill static int	mcx_cq_intr(void *);
   2630   1.1  jmcneill 
   2631   1.1  jmcneill static int	mcx_init(struct ifnet *);
   2632   1.1  jmcneill static void	mcx_stop(struct ifnet *, int);
   2633   1.1  jmcneill static int	mcx_ioctl(struct ifnet *, u_long, void *);
   2634   1.1  jmcneill static void	mcx_start(struct ifnet *);
   2635  1.16  jmcneill static int	mcx_transmit(struct ifnet *, struct mbuf *);
   2636  1.16  jmcneill static void	mcx_deferred_transmit(void *);
   2637   1.1  jmcneill static void	mcx_media_add_types(struct mcx_softc *);
   2638   1.1  jmcneill static void	mcx_media_status(struct ifnet *, struct ifmediareq *);
   2639   1.1  jmcneill static int	mcx_media_change(struct ifnet *);
   2640   1.1  jmcneill #if 0
   2641   1.1  jmcneill static int	mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
   2642   1.1  jmcneill #endif
   2643   1.1  jmcneill static void	mcx_port_change(struct work *, void *);
   2644   1.1  jmcneill 
   2645   1.1  jmcneill static void	mcx_calibrate_first(struct mcx_softc *);
   2646   1.1  jmcneill static void	mcx_calibrate(void *);
   2647   1.1  jmcneill 
   2648   1.1  jmcneill static inline uint32_t
   2649   1.1  jmcneill 		mcx_rd(struct mcx_softc *, bus_size_t);
   2650   1.1  jmcneill static inline void
   2651   1.1  jmcneill 		mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
   2652   1.1  jmcneill static inline void
   2653   1.1  jmcneill 		mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
   2654   1.1  jmcneill 
   2655   1.1  jmcneill static uint64_t	mcx_timer(struct mcx_softc *);
   2656   1.1  jmcneill 
   2657   1.1  jmcneill static int	mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
   2658   1.1  jmcneill 		    bus_size_t, u_int align);
   2659   1.1  jmcneill static void	mcx_dmamem_zero(struct mcx_dmamem *);
   2660   1.1  jmcneill static void	mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
   2661   1.1  jmcneill 
   2662   1.1  jmcneill static int	mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
   2663   1.1  jmcneill 		    unsigned int);
   2664   1.1  jmcneill static void	mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
   2665   1.1  jmcneill 
   2666   1.1  jmcneill CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL);
   2667   1.1  jmcneill 
   2668   1.1  jmcneill static const struct {
   2669   1.1  jmcneill 	pci_vendor_id_t		vendor;
   2670   1.1  jmcneill 	pci_product_id_t	product;
   2671   1.1  jmcneill } mcx_devices[] = {
   2672   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700 },
   2673  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700VF },
   2674   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710 },
   2675  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710VF },
   2676   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800 },
   2677  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800VF },
   2678   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800 },
   2679  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800VF },
   2680  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28908 },
   2681  1.29   msaitoh 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28908VF },
   2682  1.26   msaitoh 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT2892 },
   2683  1.26   msaitoh 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT2894 },
   2684   1.1  jmcneill };
   2685   1.1  jmcneill 
   2686  1.13  jmcneill struct mcx_eth_proto_capability {
   2687  1.13  jmcneill 	uint64_t	cap_media;
   2688  1.13  jmcneill 	uint64_t	cap_baudrate;
   2689  1.13  jmcneill };
   2690  1.13  jmcneill 
   2691  1.13  jmcneill static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = {
   2692  1.13  jmcneill 	[MCX_ETHER_CAP_SGMII]		= { IFM_1000_SGMII,	IF_Gbps(1) },
   2693  1.13  jmcneill 	[MCX_ETHER_CAP_1000_KX]		= { IFM_1000_KX,	IF_Gbps(1) },
   2694  1.13  jmcneill 	[MCX_ETHER_CAP_10G_CX4]		= { IFM_10G_CX4,	IF_Gbps(10) },
   2695  1.13  jmcneill 	[MCX_ETHER_CAP_10G_KX4]		= { IFM_10G_KX4,	IF_Gbps(10) },
   2696  1.13  jmcneill 	[MCX_ETHER_CAP_10G_KR]		= { IFM_10G_KR,		IF_Gbps(10) },
   2697  1.13  jmcneill 	[MCX_ETHER_CAP_20G_KR2]		= { IFM_20G_KR2,	IF_Gbps(20) },
   2698  1.13  jmcneill 	[MCX_ETHER_CAP_40G_CR4]		= { IFM_40G_CR4,	IF_Gbps(40) },
   2699  1.13  jmcneill 	[MCX_ETHER_CAP_40G_KR4]		= { IFM_40G_KR4,	IF_Gbps(40) },
   2700  1.13  jmcneill 	[MCX_ETHER_CAP_56G_R4]		= { IFM_56G_R4,		IF_Gbps(56) },
   2701  1.13  jmcneill 	[MCX_ETHER_CAP_10G_CR]		= { IFM_10G_CR1,	IF_Gbps(10) },
   2702  1.13  jmcneill 	[MCX_ETHER_CAP_10G_SR]		= { IFM_10G_SR,		IF_Gbps(10) },
   2703  1.13  jmcneill 	[MCX_ETHER_CAP_10G_LR]		= { IFM_10G_LR,		IF_Gbps(10) },
   2704  1.13  jmcneill 	[MCX_ETHER_CAP_40G_SR4]		= { IFM_40G_SR4,	IF_Gbps(40) },
   2705  1.13  jmcneill 	[MCX_ETHER_CAP_40G_LR4]		= { IFM_40G_LR4,	IF_Gbps(40) },
   2706  1.13  jmcneill 	[MCX_ETHER_CAP_50G_SR2]		= { IFM_50G_SR2,	IF_Gbps(50) },
   2707  1.13  jmcneill 	[MCX_ETHER_CAP_100G_CR4]	= { IFM_100G_CR4,	IF_Gbps(100) },
   2708  1.13  jmcneill 	[MCX_ETHER_CAP_100G_SR4]	= { IFM_100G_SR4,	IF_Gbps(100) },
   2709  1.13  jmcneill 	[MCX_ETHER_CAP_100G_KR4]	= { IFM_100G_KR4,	IF_Gbps(100) },
   2710  1.13  jmcneill 	[MCX_ETHER_CAP_100G_LR4]	= { IFM_100G_LR4,	IF_Gbps(100) },
   2711  1.13  jmcneill 	[MCX_ETHER_CAP_100_TX]		= { IFM_100_TX,		IF_Mbps(100) },
   2712  1.13  jmcneill 	[MCX_ETHER_CAP_1000_T]		= { IFM_1000_T,		IF_Gbps(1) },
   2713  1.13  jmcneill 	[MCX_ETHER_CAP_10G_T]		= { IFM_10G_T,		IF_Gbps(10) },
   2714  1.13  jmcneill 	[MCX_ETHER_CAP_25G_CR]		= { IFM_25G_CR,		IF_Gbps(25) },
   2715  1.13  jmcneill 	[MCX_ETHER_CAP_25G_KR]		= { IFM_25G_KR,		IF_Gbps(25) },
   2716  1.13  jmcneill 	[MCX_ETHER_CAP_25G_SR]		= { IFM_25G_SR,		IF_Gbps(25) },
   2717  1.13  jmcneill 	[MCX_ETHER_CAP_50G_CR2]		= { IFM_50G_CR2,	IF_Gbps(50) },
   2718  1.13  jmcneill 	[MCX_ETHER_CAP_50G_KR2]		= { IFM_50G_KR2,	IF_Gbps(50) },
   2719   1.1  jmcneill };
   2720   1.1  jmcneill 
   2721   1.1  jmcneill static int
   2722  1.13  jmcneill mcx_get_id(uint32_t val)
   2723  1.13  jmcneill {
   2724  1.13  jmcneill 	return be32toh(val) & 0x00ffffff;
   2725  1.13  jmcneill }
   2726  1.13  jmcneill 
   2727  1.13  jmcneill static int
   2728   1.1  jmcneill mcx_match(device_t parent, cfdata_t cf, void *aux)
   2729   1.1  jmcneill {
   2730   1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2731   1.1  jmcneill 	int n;
   2732   1.1  jmcneill 
   2733   1.1  jmcneill 	for (n = 0; n < __arraycount(mcx_devices); n++) {
   2734   1.1  jmcneill 		if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor &&
   2735   1.1  jmcneill 		    PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product)
   2736   1.1  jmcneill 			return 1;
   2737   1.1  jmcneill 	}
   2738   1.1  jmcneill 
   2739   1.1  jmcneill 	return 0;
   2740   1.1  jmcneill }
   2741   1.1  jmcneill 
   2742   1.1  jmcneill void
   2743   1.1  jmcneill mcx_attach(device_t parent, device_t self, void *aux)
   2744   1.1  jmcneill {
   2745   1.1  jmcneill 	struct mcx_softc *sc = device_private(self);
   2746   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   2747   1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2748  1.22  jmcneill 	struct ifcapreq ifcr;
   2749   1.1  jmcneill 	uint8_t enaddr[ETHER_ADDR_LEN];
   2750   1.1  jmcneill 	int counts[PCI_INTR_TYPE_SIZE];
   2751  1.15  jmcneill 	char intrxname[32];
   2752   1.1  jmcneill 	pcireg_t memtype;
   2753   1.1  jmcneill 	uint32_t r;
   2754   1.1  jmcneill 	unsigned int cq_stride;
   2755   1.1  jmcneill 	unsigned int cq_size;
   2756  1.15  jmcneill 	int i, msix;
   2757  1.15  jmcneill 	kcpuset_t *affinity;
   2758   1.1  jmcneill 
   2759   1.1  jmcneill 	sc->sc_dev = self;
   2760   1.1  jmcneill 	sc->sc_pc = pa->pa_pc;
   2761   1.1  jmcneill 	sc->sc_tag = pa->pa_tag;
   2762  1.11   thorpej 	if (pci_dma64_available(pa))
   2763  1.11   thorpej 		sc->sc_dmat = pa->pa_dmat64;
   2764  1.11   thorpej 	else
   2765  1.11   thorpej 		sc->sc_dmat = pa->pa_dmat;
   2766   1.1  jmcneill 
   2767   1.1  jmcneill 	/* Map the PCI memory space */
   2768   1.1  jmcneill 	memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
   2769   1.1  jmcneill 	if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
   2770  1.21  jmcneill #ifdef __NetBSD__
   2771  1.21  jmcneill 	    0,
   2772  1.21  jmcneill #else
   2773  1.21  jmcneill 	    BUS_SPACE_MAP_PREFETCHABLE,
   2774  1.21  jmcneill #endif
   2775  1.21  jmcneill 	    &sc->sc_memt, &sc->sc_memh,
   2776   1.1  jmcneill 	    NULL, &sc->sc_mems)) {
   2777   1.1  jmcneill 		aprint_error(": unable to map register memory\n");
   2778   1.1  jmcneill 		return;
   2779   1.1  jmcneill 	}
   2780   1.1  jmcneill 
   2781   1.1  jmcneill 	pci_aprint_devinfo(pa, "Ethernet controller");
   2782   1.1  jmcneill 
   2783  1.12   thorpej 	mutex_init(&sc->sc_media_mutex, MUTEX_DEFAULT, IPL_SOFTNET);
   2784  1.12   thorpej 
   2785   1.1  jmcneill 	if (mcx_version(sc) != 0) {
   2786   1.1  jmcneill 		/* error printed by mcx_version */
   2787   1.1  jmcneill 		goto unmap;
   2788   1.1  jmcneill 	}
   2789   1.1  jmcneill 
   2790   1.1  jmcneill 	r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
   2791   1.1  jmcneill 	cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
   2792   1.1  jmcneill 	cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
   2793   1.1  jmcneill 	if (cq_size > MCX_MAX_CQE) {
   2794   1.1  jmcneill 		aprint_error_dev(self,
   2795   1.1  jmcneill 		    "command queue size overflow %u\n", cq_size);
   2796   1.1  jmcneill 		goto unmap;
   2797   1.1  jmcneill 	}
   2798   1.1  jmcneill 	if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
   2799   1.1  jmcneill 		aprint_error_dev(self,
   2800   1.1  jmcneill 		    "command queue entry size underflow %u\n", cq_stride);
   2801   1.1  jmcneill 		goto unmap;
   2802   1.1  jmcneill 	}
   2803   1.1  jmcneill 	if (cq_stride * cq_size > MCX_PAGE_SIZE) {
   2804   1.1  jmcneill 		aprint_error_dev(self, "command queue page overflow\n");
   2805   1.1  jmcneill 		goto unmap;
   2806   1.1  jmcneill 	}
   2807   1.1  jmcneill 
   2808  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE,
   2809   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2810   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate doorbell memory\n");
   2811   1.1  jmcneill 		goto unmap;
   2812   1.1  jmcneill 	}
   2813   1.1  jmcneill 
   2814   1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
   2815   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2816   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate command queue\n");
   2817   1.1  jmcneill 		goto dbfree;
   2818   1.1  jmcneill 	}
   2819   1.1  jmcneill 
   2820   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   2821  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t),
   2822  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   2823   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
   2824  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t),
   2825  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   2826   1.1  jmcneill 
   2827   1.1  jmcneill 	if (mcx_init_wait(sc) != 0) {
   2828   1.1  jmcneill 		aprint_error_dev(self, "timeout waiting for init\n");
   2829   1.1  jmcneill 		goto cqfree;
   2830   1.1  jmcneill 	}
   2831   1.1  jmcneill 
   2832   1.1  jmcneill 	sc->sc_cmdq_mask = cq_size - 1;
   2833   1.1  jmcneill 	sc->sc_cmdq_size = cq_stride;
   2834   1.1  jmcneill 
   2835   1.1  jmcneill 	if (mcx_enable_hca(sc) != 0) {
   2836   1.1  jmcneill 		/* error printed by mcx_enable_hca */
   2837   1.1  jmcneill 		goto cqfree;
   2838   1.1  jmcneill 	}
   2839   1.1  jmcneill 
   2840   1.1  jmcneill 	if (mcx_issi(sc) != 0) {
   2841   1.1  jmcneill 		/* error printed by mcx_issi */
   2842   1.1  jmcneill 		goto teardown;
   2843   1.1  jmcneill 	}
   2844   1.1  jmcneill 
   2845   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_boot_pages,
   2846   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
   2847   1.1  jmcneill 		/* error printed by mcx_pages */
   2848   1.1  jmcneill 		goto teardown;
   2849   1.1  jmcneill 	}
   2850   1.1  jmcneill 
   2851   1.1  jmcneill 	if (mcx_hca_max_caps(sc) != 0) {
   2852   1.1  jmcneill 		/* error printed by mcx_hca_max_caps */
   2853   1.1  jmcneill 		goto teardown;
   2854   1.1  jmcneill 	}
   2855   1.1  jmcneill 
   2856   1.1  jmcneill 	if (mcx_hca_set_caps(sc) != 0) {
   2857   1.1  jmcneill 		/* error printed by mcx_hca_set_caps */
   2858   1.1  jmcneill 		goto teardown;
   2859   1.1  jmcneill 	}
   2860   1.1  jmcneill 
   2861   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_init_pages,
   2862   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
   2863   1.1  jmcneill 		/* error printed by mcx_pages */
   2864   1.1  jmcneill 		goto teardown;
   2865   1.1  jmcneill 	}
   2866   1.1  jmcneill 
   2867   1.1  jmcneill 	if (mcx_init_hca(sc) != 0) {
   2868   1.1  jmcneill 		/* error printed by mcx_init_hca */
   2869   1.1  jmcneill 		goto teardown;
   2870   1.1  jmcneill 	}
   2871   1.1  jmcneill 
   2872   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_regular_pages,
   2873   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
   2874   1.1  jmcneill 		/* error printed by mcx_pages */
   2875   1.1  jmcneill 		goto teardown;
   2876   1.1  jmcneill 	}
   2877   1.1  jmcneill 
   2878   1.1  jmcneill 	/* apparently not necessary? */
   2879   1.1  jmcneill 	if (mcx_set_driver_version(sc) != 0) {
   2880   1.1  jmcneill 		/* error printed by mcx_set_driver_version */
   2881   1.1  jmcneill 		goto teardown;
   2882   1.1  jmcneill 	}
   2883   1.1  jmcneill 
   2884   1.1  jmcneill 	if (mcx_iff(sc) != 0) {	/* modify nic vport context */
   2885   1.1  jmcneill 		/* error printed by mcx_iff? */
   2886   1.1  jmcneill 		goto teardown;
   2887   1.1  jmcneill 	}
   2888   1.1  jmcneill 
   2889  1.15  jmcneill 	if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) {
   2890   1.1  jmcneill 		/* error printed by mcx_alloc_uar */
   2891   1.1  jmcneill 		goto teardown;
   2892   1.1  jmcneill 	}
   2893   1.1  jmcneill 
   2894   1.1  jmcneill 	if (mcx_alloc_pd(sc) != 0) {
   2895   1.1  jmcneill 		/* error printed by mcx_alloc_pd */
   2896   1.1  jmcneill 		goto teardown;
   2897   1.1  jmcneill 	}
   2898   1.1  jmcneill 
   2899   1.1  jmcneill 	if (mcx_alloc_tdomain(sc) != 0) {
   2900   1.1  jmcneill 		/* error printed by mcx_alloc_tdomain */
   2901   1.1  jmcneill 		goto teardown;
   2902   1.1  jmcneill 	}
   2903   1.1  jmcneill 
   2904   1.1  jmcneill 	/*
   2905   1.1  jmcneill 	 * PRM makes no mention of msi interrupts, just legacy and msi-x.
   2906   1.1  jmcneill 	 * mellanox support tells me legacy interrupts are not supported,
   2907   1.1  jmcneill 	 * so we're stuck with just msi-x.
   2908   1.1  jmcneill 	 */
   2909  1.15  jmcneill 	counts[PCI_INTR_TYPE_MSIX] = -1;
   2910   1.1  jmcneill 	counts[PCI_INTR_TYPE_MSI] = 0;
   2911   1.1  jmcneill 	counts[PCI_INTR_TYPE_INTX] = 0;
   2912   1.1  jmcneill 	if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) {
   2913   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate interrupt\n");
   2914   1.1  jmcneill 		goto teardown;
   2915   1.1  jmcneill 	}
   2916  1.15  jmcneill 	if (counts[PCI_INTR_TYPE_MSIX] < 2) {
   2917  1.15  jmcneill 		aprint_error_dev(self, "not enough MSI-X vectors\n");
   2918  1.15  jmcneill 		goto teardown;
   2919  1.15  jmcneill 	}
   2920   1.1  jmcneill 	KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX);
   2921  1.15  jmcneill 	snprintf(intrxname, sizeof(intrxname), "%s adminq", DEVNAME(sc));
   2922  1.15  jmcneill 	sc->sc_ihc = mcx_establish_intr(sc, 0, NULL, mcx_admin_intr, sc,
   2923  1.15  jmcneill 	    intrxname);
   2924  1.15  jmcneill 	if (sc->sc_ihc == NULL) {
   2925  1.15  jmcneill 		aprint_error_dev(self, "couldn't establish adminq interrupt\n");
   2926   1.1  jmcneill 		goto teardown;
   2927   1.1  jmcneill 	}
   2928   1.1  jmcneill 
   2929  1.15  jmcneill 	if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar,
   2930  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
   2931  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
   2932  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
   2933  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) {
   2934   1.1  jmcneill 		/* error printed by mcx_create_eq */
   2935   1.1  jmcneill 		goto teardown;
   2936   1.1  jmcneill 	}
   2937   1.1  jmcneill 
   2938   1.1  jmcneill 	if (mcx_query_nic_vport_context(sc, enaddr) != 0) {
   2939   1.1  jmcneill 		/* error printed by mcx_query_nic_vport_context */
   2940   1.1  jmcneill 		goto teardown;
   2941   1.1  jmcneill 	}
   2942   1.1  jmcneill 
   2943   1.1  jmcneill 	if (mcx_query_special_contexts(sc) != 0) {
   2944   1.1  jmcneill 		/* error printed by mcx_query_special_contexts */
   2945   1.1  jmcneill 		goto teardown;
   2946   1.1  jmcneill 	}
   2947   1.1  jmcneill 
   2948   1.1  jmcneill 	if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
   2949   1.1  jmcneill 		/* error printed by mcx_set_port_mtu */
   2950   1.1  jmcneill 		goto teardown;
   2951   1.1  jmcneill 	}
   2952   1.1  jmcneill 
   2953   1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
   2954   1.1  jmcneill 	    ether_sprintf(enaddr));
   2955   1.1  jmcneill 
   2956  1.15  jmcneill 	msix = counts[PCI_INTR_TYPE_MSIX];
   2957  1.15  jmcneill 	msix--; /* admin ops took one */
   2958  1.15  jmcneill 
   2959  1.15  jmcneill 	sc->sc_nqueues = uimin(MCX_MAX_QUEUES, msix);
   2960  1.15  jmcneill 	sc->sc_nqueues = uimin(sc->sc_nqueues, ncpu);
   2961  1.23    andvar 	/* Round down to a power of two.  */
   2962  1.23    andvar 	sc->sc_nqueues = 1U << ilog2(sc->sc_nqueues);
   2963  1.15  jmcneill 	sc->sc_queues = kmem_zalloc(sc->sc_nqueues * sizeof(*sc->sc_queues),
   2964  1.15  jmcneill 	    KM_SLEEP);
   2965  1.15  jmcneill 
   2966   1.1  jmcneill 	strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
   2967   1.1  jmcneill 	ifp->if_softc = sc;
   2968   1.1  jmcneill 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
   2969   1.1  jmcneill #ifdef MCX_MPSAFE
   2970   1.1  jmcneill 	ifp->if_extflags = IFEF_MPSAFE;
   2971   1.1  jmcneill #endif
   2972   1.1  jmcneill 	ifp->if_init = mcx_init;
   2973   1.1  jmcneill 	ifp->if_stop = mcx_stop;
   2974   1.1  jmcneill 	ifp->if_ioctl = mcx_ioctl;
   2975   1.1  jmcneill 	ifp->if_start = mcx_start;
   2976  1.16  jmcneill 	if (sc->sc_nqueues > 1) {
   2977  1.16  jmcneill 		ifp->if_transmit = mcx_transmit;
   2978  1.16  jmcneill 	}
   2979   1.1  jmcneill 	ifp->if_mtu = sc->sc_hardmtu;
   2980  1.15  jmcneill 	ifp->if_capabilities = IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
   2981  1.15  jmcneill 	    IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
   2982  1.15  jmcneill 	    IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx |
   2983  1.15  jmcneill 	    IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
   2984  1.15  jmcneill 	    IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx;
   2985   1.1  jmcneill 	IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
   2986   1.1  jmcneill 	IFQ_SET_READY(&ifp->if_snd);
   2987   1.1  jmcneill 
   2988  1.15  jmcneill 	sc->sc_ec.ec_capabilities = ETHERCAP_JUMBO_MTU |
   2989  1.15  jmcneill 	    ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING;
   2990  1.15  jmcneill 	sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;
   2991   1.1  jmcneill 
   2992   1.1  jmcneill 	sc->sc_ec.ec_ifmedia = &sc->sc_media;
   2993  1.12   thorpej 	ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, mcx_media_change,
   2994  1.12   thorpej 	    mcx_media_status, &sc->sc_media_mutex);
   2995   1.1  jmcneill 	mcx_media_add_types(sc);
   2996   1.1  jmcneill 	ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
   2997   1.1  jmcneill 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
   2998   1.1  jmcneill 
   2999   1.1  jmcneill 	if_attach(ifp);
   3000  1.22  jmcneill 
   3001  1.22  jmcneill 	/* Enable hardware offload by default */
   3002  1.22  jmcneill 	memset(&ifcr, 0, sizeof(ifcr));
   3003  1.22  jmcneill 	ifcr.ifcr_capenable = ifp->if_capabilities;
   3004  1.22  jmcneill 	ifioctl_common(ifp, SIOCSIFCAP, &ifcr);
   3005  1.22  jmcneill 
   3006   1.1  jmcneill 	if_deferred_start_init(ifp, NULL);
   3007   1.1  jmcneill 
   3008   1.1  jmcneill 	ether_ifattach(ifp, enaddr);
   3009   1.1  jmcneill 
   3010  1.15  jmcneill 	kcpuset_create(&affinity, false);
   3011  1.15  jmcneill 	kcpuset_set(affinity, 0);
   3012  1.15  jmcneill 
   3013  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   3014  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   3015  1.15  jmcneill 		struct mcx_rx *rx = &q->q_rx;
   3016  1.15  jmcneill 		struct mcx_tx *tx = &q->q_tx;
   3017  1.15  jmcneill 		int vec;
   3018  1.15  jmcneill 
   3019  1.15  jmcneill 		vec = i + 1;
   3020  1.15  jmcneill 		q->q_sc = sc;
   3021  1.15  jmcneill 		q->q_index = i;
   3022  1.15  jmcneill 
   3023  1.15  jmcneill 		if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
   3024  1.15  jmcneill 			aprint_error_dev(self, "unable to alloc uar %d\n", i);
   3025  1.15  jmcneill 			goto teardown;
   3026  1.15  jmcneill 		}
   3027  1.15  jmcneill 
   3028  1.15  jmcneill 		if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
   3029  1.15  jmcneill 			aprint_error_dev(self,
   3030  1.15  jmcneill 			    "unable to create event queue %d\n", i);
   3031  1.15  jmcneill 			goto teardown;
   3032  1.15  jmcneill 		}
   3033  1.15  jmcneill 
   3034  1.15  jmcneill 		rx->rx_softc = sc;
   3035  1.15  jmcneill 		callout_init(&rx->rx_refill, CALLOUT_FLAGS);
   3036  1.15  jmcneill 		callout_setfunc(&rx->rx_refill, mcx_refill, rx);
   3037  1.15  jmcneill 
   3038  1.15  jmcneill 		tx->tx_softc = sc;
   3039  1.16  jmcneill 		mutex_init(&tx->tx_lock, MUTEX_DEFAULT, IPL_NET);
   3040  1.16  jmcneill 		tx->tx_pcq = pcq_create(MCX_TXQ_NUM, KM_SLEEP);
   3041  1.16  jmcneill 		tx->tx_softint = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,
   3042  1.16  jmcneill 		    mcx_deferred_transmit, tx);
   3043  1.15  jmcneill 
   3044  1.15  jmcneill 		snprintf(intrxname, sizeof(intrxname), "%s queue %d",
   3045  1.15  jmcneill 		    DEVNAME(sc), i);
   3046  1.15  jmcneill 		q->q_ihc = mcx_establish_intr(sc, vec, affinity, mcx_cq_intr,
   3047  1.15  jmcneill 		    q, intrxname);
   3048  1.15  jmcneill 	}
   3049  1.15  jmcneill 
   3050   1.1  jmcneill 	callout_init(&sc->sc_calibrate, CALLOUT_FLAGS);
   3051   1.1  jmcneill 	callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc);
   3052   1.1  jmcneill 
   3053   1.1  jmcneill 	if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc,
   3054   1.1  jmcneill 	    PRI_NONE, IPL_NET, 0) != 0) {
   3055   1.1  jmcneill 		aprint_error_dev(self, "couldn't create port change workq\n");
   3056   1.1  jmcneill 		goto teardown;
   3057   1.1  jmcneill 	}
   3058   1.1  jmcneill 
   3059   1.1  jmcneill 	mcx_port_change(&sc->sc_port_change, sc);
   3060   1.1  jmcneill 
   3061  1.15  jmcneill 	sc->sc_mac_flow_table_id = -1;
   3062  1.15  jmcneill 	sc->sc_rss_flow_table_id = -1;
   3063  1.15  jmcneill 	sc->sc_rqt = -1;
   3064   1.1  jmcneill 	for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
   3065  1.15  jmcneill 		struct mcx_flow_group *mfg = &sc->sc_flow_group[i];
   3066  1.15  jmcneill 		mfg->g_id = -1;
   3067  1.15  jmcneill 		mfg->g_table = -1;
   3068  1.15  jmcneill 		mfg->g_size = 0;
   3069  1.15  jmcneill 		mfg->g_start = 0;
   3070   1.1  jmcneill 	}
   3071   1.1  jmcneill 	sc->sc_extra_mcast = 0;
   3072   1.1  jmcneill 	memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));
   3073  1.15  jmcneill 
   3074  1.15  jmcneill #if NKSTAT > 0
   3075  1.15  jmcneill 	mcx_kstat_attach(sc);
   3076  1.15  jmcneill #endif
   3077  1.15  jmcneill 	mcx_timecounter_attach(sc);
   3078   1.1  jmcneill 	return;
   3079   1.1  jmcneill 
   3080   1.1  jmcneill teardown:
   3081   1.1  jmcneill 	mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
   3082   1.1  jmcneill 	/* error printed by mcx_teardown_hca, and we're already unwinding */
   3083   1.1  jmcneill cqfree:
   3084   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   3085  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
   3086  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3087   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
   3088   1.1  jmcneill 	    MCX_CMDQ_INTERFACE_DISABLED);
   3089  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t),
   3090  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3091   1.1  jmcneill 
   3092   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
   3093  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
   3094  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3095   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
   3096   1.1  jmcneill 
   3097   1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
   3098   1.1  jmcneill dbfree:
   3099   1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
   3100   1.1  jmcneill unmap:
   3101   1.1  jmcneill 	bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
   3102   1.1  jmcneill 	sc->sc_mems = 0;
   3103   1.1  jmcneill }
   3104   1.1  jmcneill 
   3105  1.15  jmcneill static void *
   3106  1.15  jmcneill mcx_establish_intr(struct mcx_softc *sc, int index, kcpuset_t *affinity,
   3107  1.15  jmcneill     int (*func)(void *), void *arg, const char *xname)
   3108  1.15  jmcneill {
   3109  1.15  jmcneill 	char intrbuf[PCI_INTRSTR_LEN];
   3110  1.15  jmcneill 	const char *intrstr;
   3111  1.15  jmcneill 	void *ih;
   3112  1.15  jmcneill 
   3113  1.15  jmcneill 	pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[index], PCI_INTR_MPSAFE,
   3114  1.15  jmcneill 	    true);
   3115  1.15  jmcneill 
   3116  1.15  jmcneill 	intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[index], intrbuf,
   3117  1.15  jmcneill 	    sizeof(intrbuf));
   3118  1.15  jmcneill 	ih = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[index], IPL_NET,
   3119  1.15  jmcneill 	    func, arg, xname);
   3120  1.15  jmcneill 	if (ih == NULL) {
   3121  1.15  jmcneill 		aprint_error_dev(sc->sc_dev,
   3122  1.15  jmcneill 		    "unable to establish interrupt%s%s\n",
   3123  1.15  jmcneill 		    intrstr ? " at " : "",
   3124  1.15  jmcneill 		    intrstr ? intrstr : "");
   3125  1.15  jmcneill 		return NULL;
   3126  1.15  jmcneill 	}
   3127  1.15  jmcneill 
   3128  1.15  jmcneill 	if (affinity != NULL && index > 0) {
   3129  1.15  jmcneill 		/* Round-robin affinity */
   3130  1.15  jmcneill 		kcpuset_zero(affinity);
   3131  1.15  jmcneill 		kcpuset_set(affinity, (index - 1) % ncpu);
   3132  1.15  jmcneill 		interrupt_distribute(ih, affinity, NULL);
   3133  1.15  jmcneill 	}
   3134  1.15  jmcneill 
   3135  1.15  jmcneill 	return ih;
   3136  1.15  jmcneill }
   3137  1.15  jmcneill 
   3138   1.1  jmcneill static void
   3139   1.1  jmcneill mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm)
   3140   1.1  jmcneill {
   3141   1.1  jmcneill 	rxr->rxr_total = hwm;
   3142   1.1  jmcneill 	rxr->rxr_inuse = 0;
   3143   1.1  jmcneill }
   3144   1.1  jmcneill 
   3145   1.1  jmcneill static u_int
   3146   1.1  jmcneill mcx_rxr_get(struct mcx_rxring *rxr, u_int max)
   3147   1.1  jmcneill {
   3148   1.1  jmcneill 	const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse);
   3149   1.1  jmcneill 
   3150   1.1  jmcneill 	rxr->rxr_inuse += taken;
   3151   1.1  jmcneill 
   3152   1.1  jmcneill 	return taken;
   3153   1.1  jmcneill }
   3154   1.1  jmcneill 
   3155   1.1  jmcneill static void
   3156   1.1  jmcneill mcx_rxr_put(struct mcx_rxring *rxr, u_int n)
   3157   1.1  jmcneill {
   3158   1.1  jmcneill 	rxr->rxr_inuse -= n;
   3159   1.1  jmcneill }
   3160   1.1  jmcneill 
   3161   1.1  jmcneill static u_int
   3162   1.1  jmcneill mcx_rxr_inuse(struct mcx_rxring *rxr)
   3163   1.1  jmcneill {
   3164   1.1  jmcneill 	return rxr->rxr_inuse;
   3165   1.1  jmcneill }
   3166   1.1  jmcneill 
   3167   1.1  jmcneill static int
   3168   1.1  jmcneill mcx_version(struct mcx_softc *sc)
   3169   1.1  jmcneill {
   3170   1.1  jmcneill 	uint32_t fw0, fw1;
   3171   1.1  jmcneill 	uint16_t cmdif;
   3172   1.1  jmcneill 
   3173   1.1  jmcneill 	fw0 = mcx_rd(sc, MCX_FW_VER);
   3174   1.1  jmcneill 	fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
   3175   1.1  jmcneill 
   3176   1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0),
   3177   1.1  jmcneill 	    MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
   3178   1.1  jmcneill 
   3179   1.1  jmcneill 	cmdif = MCX_CMDIF(fw1);
   3180   1.1  jmcneill 	if (cmdif != MCX_CMD_IF_SUPPORTED) {
   3181   1.1  jmcneill 		aprint_error_dev(sc->sc_dev,
   3182   1.1  jmcneill 		    "unsupported command interface %u\n", cmdif);
   3183   1.1  jmcneill 		return (-1);
   3184   1.1  jmcneill 	}
   3185   1.1  jmcneill 
   3186   1.1  jmcneill 	return (0);
   3187   1.1  jmcneill }
   3188   1.1  jmcneill 
   3189   1.1  jmcneill static int
   3190   1.1  jmcneill mcx_init_wait(struct mcx_softc *sc)
   3191   1.1  jmcneill {
   3192   1.1  jmcneill 	unsigned int i;
   3193   1.1  jmcneill 	uint32_t r;
   3194   1.1  jmcneill 
   3195   1.1  jmcneill 	for (i = 0; i < 2000; i++) {
   3196   1.1  jmcneill 		r = mcx_rd(sc, MCX_STATE);
   3197   1.1  jmcneill 		if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
   3198   1.1  jmcneill 			return (0);
   3199   1.1  jmcneill 
   3200   1.1  jmcneill 		delay(1000);
   3201   1.1  jmcneill 		mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
   3202   1.1  jmcneill 		    BUS_SPACE_BARRIER_READ);
   3203   1.1  jmcneill 	}
   3204   1.1  jmcneill 
   3205   1.1  jmcneill 	return (-1);
   3206   1.1  jmcneill }
   3207   1.1  jmcneill 
   3208   1.1  jmcneill static uint8_t
   3209   1.1  jmcneill mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3210   1.1  jmcneill     unsigned int msec)
   3211   1.1  jmcneill {
   3212   1.1  jmcneill 	unsigned int i;
   3213   1.1  jmcneill 
   3214   1.1  jmcneill 	for (i = 0; i < msec; i++) {
   3215   1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   3216   1.1  jmcneill 		    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
   3217   1.1  jmcneill 
   3218   1.1  jmcneill 		if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
   3219  1.13  jmcneill 		    MCX_CQ_STATUS_OWN_SW)
   3220   1.1  jmcneill 			return (0);
   3221   1.1  jmcneill 
   3222   1.1  jmcneill 		delay(1000);
   3223   1.1  jmcneill 	}
   3224   1.1  jmcneill 
   3225   1.1  jmcneill 	return (ETIMEDOUT);
   3226   1.1  jmcneill }
   3227   1.1  jmcneill 
   3228   1.1  jmcneill static uint32_t
   3229   1.1  jmcneill mcx_mix_u64(uint32_t xor, uint64_t u64)
   3230   1.1  jmcneill {
   3231   1.1  jmcneill 	xor ^= u64 >> 32;
   3232   1.1  jmcneill 	xor ^= u64;
   3233   1.1  jmcneill 
   3234   1.1  jmcneill 	return (xor);
   3235   1.1  jmcneill }
   3236   1.1  jmcneill 
   3237   1.1  jmcneill static uint32_t
   3238   1.1  jmcneill mcx_mix_u32(uint32_t xor, uint32_t u32)
   3239   1.1  jmcneill {
   3240   1.1  jmcneill 	xor ^= u32;
   3241   1.1  jmcneill 
   3242   1.1  jmcneill 	return (xor);
   3243   1.1  jmcneill }
   3244   1.1  jmcneill 
   3245   1.1  jmcneill static uint32_t
   3246   1.1  jmcneill mcx_mix_u8(uint32_t xor, uint8_t u8)
   3247   1.1  jmcneill {
   3248   1.1  jmcneill 	xor ^= u8;
   3249   1.1  jmcneill 
   3250   1.1  jmcneill 	return (xor);
   3251   1.1  jmcneill }
   3252   1.1  jmcneill 
   3253   1.1  jmcneill static uint8_t
   3254   1.1  jmcneill mcx_mix_done(uint32_t xor)
   3255   1.1  jmcneill {
   3256   1.1  jmcneill 	xor ^= xor >> 16;
   3257   1.1  jmcneill 	xor ^= xor >> 8;
   3258   1.1  jmcneill 
   3259   1.1  jmcneill 	return (xor);
   3260   1.1  jmcneill }
   3261   1.1  jmcneill 
   3262   1.1  jmcneill static uint8_t
   3263   1.1  jmcneill mcx_xor(const void *buf, size_t len)
   3264   1.1  jmcneill {
   3265   1.1  jmcneill 	const uint32_t *dwords = buf;
   3266   1.1  jmcneill 	uint32_t xor = 0xff;
   3267   1.1  jmcneill 	size_t i;
   3268   1.1  jmcneill 
   3269   1.1  jmcneill 	len /= sizeof(*dwords);
   3270   1.1  jmcneill 
   3271   1.1  jmcneill 	for (i = 0; i < len; i++)
   3272   1.1  jmcneill 		xor ^= dwords[i];
   3273   1.1  jmcneill 
   3274   1.1  jmcneill 	return (mcx_mix_done(xor));
   3275   1.1  jmcneill }
   3276   1.1  jmcneill 
   3277   1.1  jmcneill static uint8_t
   3278   1.1  jmcneill mcx_cmdq_token(struct mcx_softc *sc)
   3279   1.1  jmcneill {
   3280   1.1  jmcneill 	uint8_t token;
   3281   1.1  jmcneill 
   3282   1.1  jmcneill 	do {
   3283   1.1  jmcneill 		token = ++sc->sc_cmdq_token;
   3284   1.1  jmcneill 	} while (token == 0);
   3285   1.1  jmcneill 
   3286   1.1  jmcneill 	return (token);
   3287   1.1  jmcneill }
   3288   1.1  jmcneill 
   3289   1.1  jmcneill static void
   3290   1.1  jmcneill mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3291   1.1  jmcneill     uint32_t ilen, uint32_t olen, uint8_t token)
   3292   1.1  jmcneill {
   3293   1.1  jmcneill 	memset(cqe, 0, sc->sc_cmdq_size);
   3294   1.1  jmcneill 
   3295   1.1  jmcneill 	cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
   3296   1.1  jmcneill 	be32enc(&cqe->cq_input_length, ilen);
   3297   1.1  jmcneill 	be32enc(&cqe->cq_output_length, olen);
   3298   1.1  jmcneill 	cqe->cq_token = token;
   3299   1.1  jmcneill 	cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
   3300   1.1  jmcneill }
   3301   1.1  jmcneill 
   3302   1.1  jmcneill static void
   3303   1.1  jmcneill mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
   3304   1.1  jmcneill {
   3305   1.1  jmcneill 	cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
   3306   1.1  jmcneill }
   3307   1.1  jmcneill 
   3308   1.1  jmcneill static int
   3309   1.1  jmcneill mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
   3310   1.1  jmcneill {
   3311   1.1  jmcneill 	/* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 :  0); */
   3312   1.1  jmcneill 	return (0);
   3313   1.1  jmcneill }
   3314   1.1  jmcneill 
   3315   1.1  jmcneill static void *
   3316   1.1  jmcneill mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
   3317   1.1  jmcneill {
   3318   1.1  jmcneill 	return (&cqe->cq_input_data);
   3319   1.1  jmcneill }
   3320   1.1  jmcneill 
   3321   1.1  jmcneill static void *
   3322   1.1  jmcneill mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
   3323   1.1  jmcneill {
   3324   1.1  jmcneill 	return (&cqe->cq_output_data);
   3325   1.1  jmcneill }
   3326   1.1  jmcneill 
   3327   1.1  jmcneill static void
   3328   1.1  jmcneill mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3329   1.1  jmcneill     unsigned int slot)
   3330   1.1  jmcneill {
   3331   1.1  jmcneill 	mcx_cmdq_sign(cqe);
   3332   1.1  jmcneill 
   3333   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   3334   1.1  jmcneill 	    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
   3335   1.1  jmcneill 
   3336   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
   3337  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t),
   3338  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3339   1.1  jmcneill }
   3340   1.1  jmcneill 
   3341   1.1  jmcneill static int
   3342   1.1  jmcneill mcx_enable_hca(struct mcx_softc *sc)
   3343   1.1  jmcneill {
   3344   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3345   1.1  jmcneill 	struct mcx_cmd_enable_hca_in *in;
   3346   1.1  jmcneill 	struct mcx_cmd_enable_hca_out *out;
   3347   1.1  jmcneill 	int error;
   3348   1.1  jmcneill 	uint8_t status;
   3349   1.1  jmcneill 
   3350   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3351   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3352   1.1  jmcneill 
   3353   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3354   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
   3355   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3356   1.1  jmcneill 	in->cmd_function_id = htobe16(0);
   3357   1.1  jmcneill 
   3358   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3359   1.1  jmcneill 
   3360   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3361   1.1  jmcneill 	if (error != 0) {
   3362   1.1  jmcneill 		printf(", hca enable timeout\n");
   3363   1.1  jmcneill 		return (-1);
   3364   1.1  jmcneill 	}
   3365   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3366   1.1  jmcneill 		printf(", hca enable command corrupt\n");
   3367   1.1  jmcneill 		return (-1);
   3368   1.1  jmcneill 	}
   3369   1.1  jmcneill 
   3370   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3371   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3372   1.1  jmcneill 		printf(", hca enable failed (%x)\n", status);
   3373   1.1  jmcneill 		return (-1);
   3374   1.1  jmcneill 	}
   3375   1.1  jmcneill 
   3376   1.1  jmcneill 	return (0);
   3377   1.1  jmcneill }
   3378   1.1  jmcneill 
   3379   1.1  jmcneill static int
   3380   1.1  jmcneill mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
   3381   1.1  jmcneill {
   3382   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3383   1.1  jmcneill 	struct mcx_cmd_teardown_hca_in *in;
   3384   1.1  jmcneill 	struct mcx_cmd_teardown_hca_out *out;
   3385   1.1  jmcneill 	int error;
   3386   1.1  jmcneill 	uint8_t status;
   3387   1.1  jmcneill 
   3388   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3389   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3390   1.1  jmcneill 
   3391   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3392   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
   3393   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3394   1.1  jmcneill 	in->cmd_profile = profile;
   3395   1.1  jmcneill 
   3396   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3397   1.1  jmcneill 
   3398   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3399   1.1  jmcneill 	if (error != 0) {
   3400   1.1  jmcneill 		printf(", hca teardown timeout\n");
   3401   1.1  jmcneill 		return (-1);
   3402   1.1  jmcneill 	}
   3403   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3404   1.1  jmcneill 		printf(", hca teardown command corrupt\n");
   3405   1.1  jmcneill 		return (-1);
   3406   1.1  jmcneill 	}
   3407   1.1  jmcneill 
   3408   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3409   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3410   1.1  jmcneill 		printf(", hca teardown failed (%x)\n", status);
   3411   1.1  jmcneill 		return (-1);
   3412   1.1  jmcneill 	}
   3413   1.1  jmcneill 
   3414   1.1  jmcneill 	return (0);
   3415   1.1  jmcneill }
   3416   1.1  jmcneill 
   3417   1.1  jmcneill static int
   3418   1.1  jmcneill mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   3419   1.1  jmcneill     unsigned int nmb, uint64_t *ptr, uint8_t token)
   3420   1.1  jmcneill {
   3421   1.1  jmcneill 	uint8_t *kva;
   3422   1.1  jmcneill 	uint64_t dva;
   3423   1.1  jmcneill 	int i;
   3424   1.1  jmcneill 	int error;
   3425   1.1  jmcneill 
   3426   1.1  jmcneill 	error = mcx_dmamem_alloc(sc, mxm,
   3427   1.1  jmcneill 	    nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
   3428   1.1  jmcneill 	if (error != 0)
   3429   1.1  jmcneill 		return (error);
   3430   1.1  jmcneill 
   3431   1.1  jmcneill 	mcx_dmamem_zero(mxm);
   3432   1.1  jmcneill 
   3433   1.1  jmcneill 	dva = MCX_DMA_DVA(mxm);
   3434   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3435   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3436   1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
   3437   1.1  jmcneill 
   3438   1.1  jmcneill 		/* patch the cqe or mbox pointing at this one */
   3439   1.1  jmcneill 		be64enc(ptr, dva);
   3440   1.1  jmcneill 
   3441   1.1  jmcneill 		/* fill in this mbox */
   3442   1.1  jmcneill 		be32enc(&mbox->mb_block_number, i);
   3443   1.1  jmcneill 		mbox->mb_token = token;
   3444   1.1  jmcneill 
   3445   1.1  jmcneill 		/* move to the next one */
   3446   1.1  jmcneill 		ptr = &mbox->mb_next_ptr;
   3447   1.1  jmcneill 
   3448   1.1  jmcneill 		dva += MCX_CMDQ_MAILBOX_SIZE;
   3449   1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   3450   1.1  jmcneill 	}
   3451   1.1  jmcneill 
   3452   1.1  jmcneill 	return (0);
   3453   1.1  jmcneill }
   3454   1.1  jmcneill 
   3455   1.1  jmcneill static uint32_t
   3456   1.1  jmcneill mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
   3457   1.1  jmcneill {
   3458   1.1  jmcneill 	uint32_t xor = 0xff;
   3459   1.1  jmcneill 
   3460   1.1  jmcneill 	/* only 3 fields get set, so mix them directly */
   3461   1.1  jmcneill 	xor = mcx_mix_u64(xor, mb->mb_next_ptr);
   3462   1.1  jmcneill 	xor = mcx_mix_u32(xor, mb->mb_block_number);
   3463   1.1  jmcneill 	xor = mcx_mix_u8(xor, mb->mb_token);
   3464   1.1  jmcneill 
   3465   1.1  jmcneill 	return (mcx_mix_done(xor));
   3466   1.1  jmcneill }
   3467   1.1  jmcneill 
   3468   1.1  jmcneill static void
   3469   1.1  jmcneill mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
   3470   1.1  jmcneill {
   3471   1.1  jmcneill 	uint8_t *kva;
   3472   1.1  jmcneill 	int i;
   3473   1.1  jmcneill 
   3474   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3475   1.1  jmcneill 
   3476   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3477   1.1  jmcneill 		struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
   3478   1.1  jmcneill 		uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
   3479   1.1  jmcneill 		mb->mb_ctrl_signature = sig;
   3480   1.1  jmcneill 		mb->mb_signature = sig ^
   3481   1.1  jmcneill 		    mcx_xor(mb->mb_data, sizeof(mb->mb_data));
   3482   1.1  jmcneill 
   3483   1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   3484   1.1  jmcneill 	}
   3485   1.1  jmcneill }
   3486   1.1  jmcneill 
   3487   1.1  jmcneill static void
   3488   1.1  jmcneill mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
   3489   1.1  jmcneill {
   3490   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
   3491   1.1  jmcneill 	    0, MCX_DMA_LEN(mxm), ops);
   3492   1.1  jmcneill }
   3493   1.1  jmcneill 
   3494   1.1  jmcneill static struct mcx_cmdq_mailbox *
   3495   1.1  jmcneill mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
   3496   1.1  jmcneill {
   3497   1.1  jmcneill 	uint8_t *kva;
   3498   1.1  jmcneill 
   3499   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3500   1.1  jmcneill 	kva += i * MCX_CMDQ_MAILBOX_SIZE;
   3501   1.1  jmcneill 
   3502   1.1  jmcneill 	return ((struct mcx_cmdq_mailbox *)kva);
   3503   1.1  jmcneill }
   3504   1.1  jmcneill 
   3505   1.1  jmcneill static inline void *
   3506   1.1  jmcneill mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
   3507   1.1  jmcneill {
   3508   1.1  jmcneill 	return (&mb->mb_data);
   3509   1.1  jmcneill }
   3510   1.1  jmcneill 
   3511   1.1  jmcneill static void
   3512   1.1  jmcneill mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
   3513   1.1  jmcneill     void *b, size_t len)
   3514   1.1  jmcneill {
   3515   1.1  jmcneill 	uint8_t *buf = b;
   3516   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3517   1.1  jmcneill 	int i;
   3518   1.1  jmcneill 
   3519   1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   3520   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3521   1.1  jmcneill 
   3522   1.1  jmcneill 		memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len));
   3523   1.1  jmcneill 
   3524   1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   3525   1.1  jmcneill 			break;
   3526   1.1  jmcneill 
   3527   1.1  jmcneill 		buf += sizeof(mb->mb_data);
   3528   1.1  jmcneill 		len -= sizeof(mb->mb_data);
   3529   1.1  jmcneill 		mb++;
   3530   1.1  jmcneill 	}
   3531   1.1  jmcneill }
   3532   1.1  jmcneill 
   3533   1.1  jmcneill static void
   3534  1.13  jmcneill mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages,
   3535  1.13  jmcneill     struct mcx_dmamem *buf)
   3536  1.13  jmcneill {
   3537  1.13  jmcneill 	uint64_t *pas;
   3538  1.13  jmcneill 	int mbox, mbox_pages, i;
   3539  1.13  jmcneill 
   3540  1.13  jmcneill 	mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE;
   3541  1.13  jmcneill 	offset %= MCX_CMDQ_MAILBOX_DATASIZE;
   3542  1.13  jmcneill 
   3543  1.13  jmcneill 	pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
   3544  1.13  jmcneill 	pas += (offset / sizeof(*pas));
   3545  1.13  jmcneill 	mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas);
   3546  1.13  jmcneill 	for (i = 0; i < npages; i++) {
   3547  1.13  jmcneill 		if (i == mbox_pages) {
   3548  1.13  jmcneill 			mbox++;
   3549  1.13  jmcneill 			pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
   3550  1.13  jmcneill 			mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas);
   3551  1.13  jmcneill 		}
   3552  1.13  jmcneill 		*pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE));
   3553  1.13  jmcneill 		pas++;
   3554  1.13  jmcneill 	}
   3555  1.13  jmcneill }
   3556  1.13  jmcneill 
   3557  1.13  jmcneill static void
   3558   1.1  jmcneill mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
   3559   1.1  jmcneill {
   3560   1.1  jmcneill 	uint8_t *buf = b;
   3561   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3562   1.1  jmcneill 	int i;
   3563   1.1  jmcneill 
   3564   1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   3565   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3566   1.1  jmcneill 		memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len));
   3567   1.1  jmcneill 
   3568   1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   3569   1.1  jmcneill 			break;
   3570   1.1  jmcneill 
   3571   1.1  jmcneill 		buf += sizeof(mb->mb_data);
   3572   1.1  jmcneill 		len -= sizeof(mb->mb_data);
   3573   1.1  jmcneill 		mb++;
   3574   1.1  jmcneill 	}
   3575   1.1  jmcneill }
   3576   1.1  jmcneill 
   3577   1.1  jmcneill static void
   3578   1.1  jmcneill mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   3579   1.1  jmcneill {
   3580   1.1  jmcneill 	mcx_dmamem_free(sc, mxm);
   3581   1.1  jmcneill }
   3582   1.1  jmcneill 
   3583   1.1  jmcneill #if 0
   3584   1.1  jmcneill static void
   3585   1.1  jmcneill mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
   3586   1.1  jmcneill {
   3587   1.1  jmcneill 	unsigned int i;
   3588   1.1  jmcneill 
   3589   1.1  jmcneill 	printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
   3590   1.1  jmcneill 	    be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr));
   3591   1.1  jmcneill 
   3592   1.1  jmcneill 	printf(", idata ");
   3593   1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_input_data); i++)
   3594   1.1  jmcneill 		printf("%02x", cqe->cq_input_data[i]);
   3595   1.1  jmcneill 
   3596   1.1  jmcneill 	printf(", odata ");
   3597   1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_output_data); i++)
   3598   1.1  jmcneill 		printf("%02x", cqe->cq_output_data[i]);
   3599   1.1  jmcneill 
   3600   1.1  jmcneill 	printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
   3601   1.1  jmcneill 	    be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length),
   3602   1.1  jmcneill 	    cqe->cq_token, cqe->cq_signature, cqe->cq_status);
   3603   1.1  jmcneill }
   3604   1.1  jmcneill 
   3605   1.1  jmcneill static void
   3606   1.1  jmcneill mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
   3607   1.1  jmcneill {
   3608   1.1  jmcneill 	int i, j;
   3609   1.1  jmcneill 	uint8_t *d;
   3610   1.1  jmcneill 
   3611   1.1  jmcneill 	for (i = 0; i < num; i++) {
   3612   1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox;
   3613   1.1  jmcneill 		mbox = mcx_cq_mbox(mboxes, i);
   3614   1.1  jmcneill 
   3615   1.1  jmcneill 		d = mcx_cq_mbox_data(mbox);
   3616   1.1  jmcneill 		for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
   3617   1.1  jmcneill 			if (j != 0 && (j % 16 == 0))
   3618   1.1  jmcneill 				printf("\n");
   3619   1.1  jmcneill 			printf("%.2x ", d[j]);
   3620   1.1  jmcneill 		}
   3621   1.1  jmcneill 	}
   3622   1.1  jmcneill }
   3623   1.1  jmcneill #endif
   3624   1.1  jmcneill 
   3625   1.1  jmcneill static int
   3626   1.1  jmcneill mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
   3627   1.1  jmcneill     int len)
   3628   1.1  jmcneill {
   3629   1.1  jmcneill 	struct mcx_dmamem mxm;
   3630   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3631   1.1  jmcneill 	struct mcx_cmd_access_reg_in *in;
   3632   1.1  jmcneill 	struct mcx_cmd_access_reg_out *out;
   3633   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3634   1.1  jmcneill 	int error, nmb;
   3635   1.1  jmcneill 
   3636   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3637   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
   3638   1.1  jmcneill 	    token);
   3639   1.1  jmcneill 
   3640   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3641   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
   3642   1.1  jmcneill 	in->cmd_op_mod = htobe16(op);
   3643   1.1  jmcneill 	in->cmd_register_id = htobe16(reg);
   3644   1.1  jmcneill 
   3645   1.1  jmcneill 	nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
   3646  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
   3647  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3648   1.1  jmcneill 		printf(", unable to allocate access reg mailboxen\n");
   3649   1.1  jmcneill 		return (-1);
   3650   1.1  jmcneill 	}
   3651   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   3652   1.1  jmcneill 	mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
   3653   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   3654   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   3655   1.1  jmcneill 
   3656   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3657   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3658   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   3659   1.1  jmcneill 
   3660   1.1  jmcneill 	if (error != 0) {
   3661   1.1  jmcneill 		printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
   3662   1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
   3663   1.1  jmcneill 		goto free;
   3664   1.1  jmcneill 	}
   3665   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3666   1.1  jmcneill 	if (error != 0) {
   3667   1.1  jmcneill 		printf("%s: access reg (%s %x) reply corrupt\n",
   3668   1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
   3669   1.1  jmcneill 		    reg);
   3670   1.1  jmcneill 		goto free;
   3671   1.1  jmcneill 	}
   3672   1.1  jmcneill 
   3673   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3674   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3675   1.1  jmcneill 		printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
   3676   1.1  jmcneill 		    DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
   3677  1.15  jmcneill 		    reg, out->cmd_status, be32toh(out->cmd_syndrome));
   3678   1.1  jmcneill 		error = -1;
   3679   1.1  jmcneill 		goto free;
   3680   1.1  jmcneill 	}
   3681   1.1  jmcneill 
   3682   1.1  jmcneill 	mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
   3683   1.1  jmcneill free:
   3684   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3685   1.1  jmcneill 
   3686   1.1  jmcneill 	return (error);
   3687   1.1  jmcneill }
   3688   1.1  jmcneill 
   3689   1.1  jmcneill static int
   3690  1.15  jmcneill mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3691  1.15  jmcneill     unsigned int slot)
   3692   1.1  jmcneill {
   3693   1.1  jmcneill 	struct mcx_cmd_set_issi_in *in;
   3694   1.1  jmcneill 	struct mcx_cmd_set_issi_out *out;
   3695   1.1  jmcneill 	uint8_t status;
   3696   1.1  jmcneill 
   3697   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3698   1.1  jmcneill 
   3699   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3700   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
   3701   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3702   1.1  jmcneill 	in->cmd_current_issi = htobe16(MCX_ISSI);
   3703   1.1  jmcneill 
   3704   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, slot);
   3705   1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
   3706   1.1  jmcneill 		return (-1);
   3707   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0)
   3708   1.1  jmcneill 		return (-1);
   3709   1.1  jmcneill 
   3710   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3711   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK)
   3712   1.1  jmcneill 		return (-1);
   3713   1.1  jmcneill 
   3714   1.1  jmcneill 	return (0);
   3715   1.1  jmcneill }
   3716   1.1  jmcneill 
   3717   1.1  jmcneill static int
   3718   1.1  jmcneill mcx_issi(struct mcx_softc *sc)
   3719   1.1  jmcneill {
   3720   1.1  jmcneill 	struct mcx_dmamem mxm;
   3721   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3722   1.1  jmcneill 	struct mcx_cmd_query_issi_in *in;
   3723   1.1  jmcneill 	struct mcx_cmd_query_issi_il_out *out;
   3724   1.1  jmcneill 	struct mcx_cmd_query_issi_mb_out *mb;
   3725   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3726   1.1  jmcneill 	uint8_t status;
   3727   1.1  jmcneill 	int error;
   3728   1.1  jmcneill 
   3729   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3730   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
   3731   1.1  jmcneill 
   3732   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3733   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
   3734   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3735   1.1  jmcneill 
   3736   1.1  jmcneill 	CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
   3737   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   3738   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3739   1.1  jmcneill 		printf(", unable to allocate query issi mailbox\n");
   3740   1.1  jmcneill 		return (-1);
   3741   1.1  jmcneill 	}
   3742   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   3743   1.1  jmcneill 
   3744   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3745   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3746   1.1  jmcneill 	if (error != 0) {
   3747   1.1  jmcneill 		printf(", query issi timeout\n");
   3748   1.1  jmcneill 		goto free;
   3749   1.1  jmcneill 	}
   3750   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3751   1.1  jmcneill 	if (error != 0) {
   3752   1.1  jmcneill 		printf(", query issi reply corrupt\n");
   3753   1.1  jmcneill 		goto free;
   3754   1.1  jmcneill 	}
   3755   1.1  jmcneill 
   3756   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3757   1.1  jmcneill 	switch (status) {
   3758   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   3759   1.1  jmcneill 		break;
   3760   1.1  jmcneill 	case MCX_CQ_STATUS_BAD_OPCODE:
   3761   1.1  jmcneill 		/* use ISSI 0 */
   3762   1.1  jmcneill 		goto free;
   3763   1.1  jmcneill 	default:
   3764   1.1  jmcneill 		printf(", query issi failed (%x)\n", status);
   3765   1.1  jmcneill 		error = -1;
   3766   1.1  jmcneill 		goto free;
   3767   1.1  jmcneill 	}
   3768   1.1  jmcneill 
   3769   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3770   1.1  jmcneill 	if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
   3771   1.1  jmcneill 		/* use ISSI 1 */
   3772   1.1  jmcneill 		goto free;
   3773   1.1  jmcneill 	}
   3774   1.1  jmcneill 
   3775   1.1  jmcneill 	/* don't need to read cqe anymore, can be used for SET ISSI */
   3776   1.1  jmcneill 
   3777   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   3778   1.1  jmcneill 	CTASSERT(MCX_ISSI < NBBY);
   3779   1.1  jmcneill 	 /* XXX math is hard */
   3780   1.1  jmcneill 	if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
   3781   1.1  jmcneill 		/* use ISSI 0 */
   3782   1.1  jmcneill 		goto free;
   3783   1.1  jmcneill 	}
   3784   1.1  jmcneill 
   3785   1.1  jmcneill 	if (mcx_set_issi(sc, cqe, 0) != 0) {
   3786   1.1  jmcneill 		/* ignore the error, just use ISSI 0 */
   3787   1.1  jmcneill 	} else {
   3788   1.1  jmcneill 		/* use ISSI 1 */
   3789   1.1  jmcneill 	}
   3790   1.1  jmcneill 
   3791   1.1  jmcneill free:
   3792   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   3793   1.1  jmcneill 	return (error);
   3794   1.1  jmcneill }
   3795   1.1  jmcneill 
   3796   1.1  jmcneill static int
   3797   1.1  jmcneill mcx_query_pages(struct mcx_softc *sc, uint16_t type,
   3798  1.15  jmcneill     int32_t *npages, uint16_t *func_id)
   3799   1.1  jmcneill {
   3800   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3801   1.1  jmcneill 	struct mcx_cmd_query_pages_in *in;
   3802   1.1  jmcneill 	struct mcx_cmd_query_pages_out *out;
   3803   1.1  jmcneill 
   3804   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3805   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3806   1.1  jmcneill 
   3807   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3808   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
   3809   1.1  jmcneill 	in->cmd_op_mod = type;
   3810   1.1  jmcneill 
   3811   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3812   1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
   3813   1.1  jmcneill 		printf(", query pages timeout\n");
   3814   1.1  jmcneill 		return (-1);
   3815   1.1  jmcneill 	}
   3816   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3817   1.1  jmcneill 		printf(", query pages reply corrupt\n");
   3818   1.1  jmcneill 		return (-1);
   3819   1.1  jmcneill 	}
   3820   1.1  jmcneill 
   3821   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3822   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3823   1.1  jmcneill 		printf(", query pages failed (%x)\n", out->cmd_status);
   3824   1.1  jmcneill 		return (-1);
   3825   1.1  jmcneill 	}
   3826   1.1  jmcneill 
   3827   1.1  jmcneill 	*func_id = out->cmd_func_id;
   3828   1.1  jmcneill 	*npages = be32dec(&out->cmd_num_pages);
   3829   1.1  jmcneill 
   3830   1.1  jmcneill 	return (0);
   3831   1.1  jmcneill }
   3832   1.1  jmcneill 
   3833   1.1  jmcneill struct bus_dma_iter {
   3834   1.1  jmcneill 	bus_dmamap_t		i_map;
   3835   1.1  jmcneill 	bus_size_t		i_offset;
   3836   1.1  jmcneill 	unsigned int		i_index;
   3837   1.1  jmcneill };
   3838   1.1  jmcneill 
   3839   1.1  jmcneill static void
   3840   1.1  jmcneill bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
   3841   1.1  jmcneill {
   3842   1.1  jmcneill 	i->i_map = map;
   3843   1.1  jmcneill 	i->i_offset = 0;
   3844   1.1  jmcneill 	i->i_index = 0;
   3845   1.1  jmcneill }
   3846   1.1  jmcneill 
   3847   1.1  jmcneill static bus_addr_t
   3848   1.1  jmcneill bus_dma_iter_addr(struct bus_dma_iter *i)
   3849   1.1  jmcneill {
   3850   1.1  jmcneill 	return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
   3851   1.1  jmcneill }
   3852   1.1  jmcneill 
   3853   1.1  jmcneill static void
   3854   1.1  jmcneill bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
   3855   1.1  jmcneill {
   3856   1.1  jmcneill 	bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
   3857   1.1  jmcneill 	bus_size_t diff;
   3858   1.1  jmcneill 
   3859   1.1  jmcneill 	do {
   3860   1.1  jmcneill 		diff = seg->ds_len - i->i_offset;
   3861   1.1  jmcneill 		if (size < diff)
   3862   1.1  jmcneill 			break;
   3863   1.1  jmcneill 
   3864   1.1  jmcneill 		size -= diff;
   3865   1.1  jmcneill 
   3866   1.1  jmcneill 		seg++;
   3867   1.1  jmcneill 
   3868   1.1  jmcneill 		i->i_offset = 0;
   3869   1.1  jmcneill 		i->i_index++;
   3870   1.1  jmcneill 	} while (size > 0);
   3871   1.1  jmcneill 
   3872   1.1  jmcneill 	i->i_offset += size;
   3873   1.1  jmcneill }
   3874   1.1  jmcneill 
   3875   1.1  jmcneill static int
   3876   1.1  jmcneill mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
   3877   1.1  jmcneill {
   3878   1.1  jmcneill 	struct mcx_dmamem mxm;
   3879   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3880   1.1  jmcneill 	struct mcx_cmd_manage_pages_in *in;
   3881   1.1  jmcneill 	struct mcx_cmd_manage_pages_out *out;
   3882   1.1  jmcneill 	unsigned int paslen, nmb, i, j, npages;
   3883   1.1  jmcneill 	struct bus_dma_iter iter;
   3884   1.1  jmcneill 	uint64_t *pas;
   3885   1.1  jmcneill 	uint8_t status;
   3886   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3887   1.1  jmcneill 	int error;
   3888   1.1  jmcneill 
   3889   1.1  jmcneill 	npages = mhm->mhm_npages;
   3890   1.1  jmcneill 
   3891   1.1  jmcneill 	paslen = sizeof(*pas) * npages;
   3892   1.1  jmcneill 	nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
   3893   1.1  jmcneill 
   3894   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3895   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
   3896   1.1  jmcneill 
   3897   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3898   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
   3899   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
   3900   1.1  jmcneill 	in->cmd_func_id = func_id;
   3901   1.1  jmcneill 	be32enc(&in->cmd_input_num_entries, npages);
   3902   1.1  jmcneill 
   3903   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
   3904   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   3905   1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   3906   1.1  jmcneill 		return (-1);
   3907   1.1  jmcneill 	}
   3908   1.1  jmcneill 
   3909   1.1  jmcneill 	bus_dma_iter_init(&iter, mhm->mhm_map);
   3910   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3911   1.1  jmcneill 		unsigned int lim;
   3912   1.1  jmcneill 
   3913   1.1  jmcneill 		pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
   3914   1.1  jmcneill 		lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
   3915   1.1  jmcneill 
   3916   1.1  jmcneill 		for (j = 0; j < lim; j++) {
   3917   1.1  jmcneill 			be64enc(&pas[j], bus_dma_iter_addr(&iter));
   3918   1.1  jmcneill 			bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
   3919   1.1  jmcneill 		}
   3920   1.1  jmcneill 
   3921   1.1  jmcneill 		npages -= lim;
   3922   1.1  jmcneill 	}
   3923   1.1  jmcneill 
   3924   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   3925   1.1  jmcneill 
   3926   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3927   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3928   1.1  jmcneill 	if (error != 0) {
   3929   1.1  jmcneill 		printf(", manage pages timeout\n");
   3930   1.1  jmcneill 		goto free;
   3931   1.1  jmcneill 	}
   3932   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3933   1.1  jmcneill 	if (error != 0) {
   3934   1.1  jmcneill 		printf(", manage pages reply corrupt\n");
   3935   1.1  jmcneill 		goto free;
   3936   1.1  jmcneill 	}
   3937   1.1  jmcneill 
   3938   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3939   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3940   1.1  jmcneill 		printf(", manage pages failed (%x)\n", status);
   3941   1.1  jmcneill 		error = -1;
   3942   1.1  jmcneill 		goto free;
   3943   1.1  jmcneill 	}
   3944   1.1  jmcneill 
   3945   1.1  jmcneill free:
   3946   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3947   1.1  jmcneill 
   3948   1.1  jmcneill 	return (error);
   3949   1.1  jmcneill }
   3950   1.1  jmcneill 
   3951   1.1  jmcneill static int
   3952   1.1  jmcneill mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
   3953   1.1  jmcneill {
   3954  1.15  jmcneill 	int32_t npages;
   3955   1.1  jmcneill 	uint16_t func_id;
   3956   1.1  jmcneill 
   3957   1.1  jmcneill 	if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
   3958   1.1  jmcneill 		/* error printed by mcx_query_pages */
   3959   1.1  jmcneill 		return (-1);
   3960   1.1  jmcneill 	}
   3961   1.1  jmcneill 
   3962  1.15  jmcneill 	if (npages < 1)
   3963   1.1  jmcneill 		return (0);
   3964   1.1  jmcneill 
   3965   1.1  jmcneill 	if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
   3966   1.1  jmcneill 		printf(", unable to allocate hwmem\n");
   3967   1.1  jmcneill 		return (-1);
   3968   1.1  jmcneill 	}
   3969   1.1  jmcneill 
   3970   1.1  jmcneill 	if (mcx_add_pages(sc, mhm, func_id) != 0) {
   3971   1.1  jmcneill 		printf(", unable to add hwmem\n");
   3972   1.1  jmcneill 		goto free;
   3973   1.1  jmcneill 	}
   3974   1.1  jmcneill 
   3975   1.1  jmcneill 	return (0);
   3976   1.1  jmcneill 
   3977   1.1  jmcneill free:
   3978   1.1  jmcneill 	mcx_hwmem_free(sc, mhm);
   3979   1.1  jmcneill 
   3980   1.1  jmcneill 	return (-1);
   3981   1.1  jmcneill }
   3982   1.1  jmcneill 
   3983   1.1  jmcneill static int
   3984   1.1  jmcneill mcx_hca_max_caps(struct mcx_softc *sc)
   3985   1.1  jmcneill {
   3986   1.1  jmcneill 	struct mcx_dmamem mxm;
   3987   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3988   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   3989   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   3990   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3991   1.1  jmcneill 	struct mcx_cap_device *hca;
   3992   1.1  jmcneill 	uint8_t status;
   3993   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3994   1.1  jmcneill 	int error;
   3995   1.1  jmcneill 
   3996   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3997   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   3998   1.1  jmcneill 	    token);
   3999   1.1  jmcneill 
   4000   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4001   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   4002   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
   4003   1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   4004   1.1  jmcneill 
   4005   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   4006   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4007   1.1  jmcneill 		printf(", unable to allocate query hca caps mailboxen\n");
   4008   1.1  jmcneill 		return (-1);
   4009   1.1  jmcneill 	}
   4010   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   4011   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   4012   1.1  jmcneill 
   4013   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4014   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4015   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   4016   1.1  jmcneill 
   4017   1.1  jmcneill 	if (error != 0) {
   4018   1.1  jmcneill 		printf(", query hca caps timeout\n");
   4019   1.1  jmcneill 		goto free;
   4020   1.1  jmcneill 	}
   4021   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   4022   1.1  jmcneill 	if (error != 0) {
   4023   1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   4024   1.1  jmcneill 		goto free;
   4025   1.1  jmcneill 	}
   4026   1.1  jmcneill 
   4027   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4028   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4029   1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   4030   1.1  jmcneill 		error = -1;
   4031   1.1  jmcneill 		goto free;
   4032   1.1  jmcneill 	}
   4033   1.1  jmcneill 
   4034   1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   4035   1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   4036   1.1  jmcneill 
   4037  1.15  jmcneill 	if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE)
   4038  1.15  jmcneill 	    != MCX_CAP_DEVICE_PORT_TYPE_ETH) {
   4039  1.15  jmcneill 		printf(", not in ethernet mode\n");
   4040  1.15  jmcneill 		error = -1;
   4041  1.15  jmcneill 		goto free;
   4042  1.15  jmcneill 	}
   4043   1.1  jmcneill 	if (hca->log_pg_sz > PAGE_SHIFT) {
   4044   1.1  jmcneill 		printf(", minimum system page shift %u is too large\n",
   4045   1.1  jmcneill 		    hca->log_pg_sz);
   4046   1.1  jmcneill 		error = -1;
   4047   1.1  jmcneill 		goto free;
   4048   1.1  jmcneill 	}
   4049   1.1  jmcneill 	/*
   4050   1.1  jmcneill 	 * blueflame register is split into two buffers, and we must alternate
   4051   1.1  jmcneill 	 * between the two of them.
   4052   1.1  jmcneill 	 */
   4053   1.1  jmcneill 	sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
   4054  1.15  jmcneill 	sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size);
   4055  1.24     skrll 
   4056  1.15  jmcneill 	if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG)
   4057  1.15  jmcneill 		sc->sc_mcam_reg = 1;
   4058  1.15  jmcneill 
   4059  1.15  jmcneill 	sc->sc_mhz = be32dec(&hca->device_frequency_mhz);
   4060  1.15  jmcneill 	sc->sc_khz = be32dec(&hca->device_frequency_khz);
   4061   1.1  jmcneill 
   4062   1.1  jmcneill free:
   4063   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4064   1.1  jmcneill 
   4065   1.1  jmcneill 	return (error);
   4066   1.1  jmcneill }
   4067   1.1  jmcneill 
   4068   1.1  jmcneill static int
   4069   1.1  jmcneill mcx_hca_set_caps(struct mcx_softc *sc)
   4070   1.1  jmcneill {
   4071   1.1  jmcneill 	struct mcx_dmamem mxm;
   4072   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4073   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   4074   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   4075   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   4076   1.1  jmcneill 	struct mcx_cap_device *hca;
   4077   1.1  jmcneill 	uint8_t status;
   4078   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   4079   1.1  jmcneill 	int error;
   4080   1.1  jmcneill 
   4081   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4082   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   4083   1.1  jmcneill 	    token);
   4084   1.1  jmcneill 
   4085   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4086   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   4087   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
   4088   1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   4089   1.1  jmcneill 
   4090   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   4091   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4092   1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   4093   1.1  jmcneill 		return (-1);
   4094   1.1  jmcneill 	}
   4095   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   4096   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   4097   1.1  jmcneill 
   4098   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4099   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4100   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   4101   1.1  jmcneill 
   4102   1.1  jmcneill 	if (error != 0) {
   4103   1.1  jmcneill 		printf(", query hca caps timeout\n");
   4104   1.1  jmcneill 		goto free;
   4105   1.1  jmcneill 	}
   4106   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   4107   1.1  jmcneill 	if (error != 0) {
   4108   1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   4109   1.1  jmcneill 		goto free;
   4110   1.1  jmcneill 	}
   4111   1.1  jmcneill 
   4112   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4113   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4114   1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   4115   1.1  jmcneill 		error = -1;
   4116   1.1  jmcneill 		goto free;
   4117   1.1  jmcneill 	}
   4118   1.1  jmcneill 
   4119   1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   4120   1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   4121   1.1  jmcneill 
   4122   1.1  jmcneill 	hca->log_pg_sz = PAGE_SHIFT;
   4123   1.1  jmcneill 
   4124   1.1  jmcneill free:
   4125   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4126   1.1  jmcneill 
   4127   1.1  jmcneill 	return (error);
   4128   1.1  jmcneill }
   4129   1.1  jmcneill 
   4130   1.1  jmcneill 
   4131   1.1  jmcneill static int
   4132   1.1  jmcneill mcx_init_hca(struct mcx_softc *sc)
   4133   1.1  jmcneill {
   4134   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4135   1.1  jmcneill 	struct mcx_cmd_init_hca_in *in;
   4136   1.1  jmcneill 	struct mcx_cmd_init_hca_out *out;
   4137   1.1  jmcneill 	int error;
   4138   1.1  jmcneill 	uint8_t status;
   4139   1.1  jmcneill 
   4140   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4141   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4142   1.1  jmcneill 
   4143   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4144   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
   4145   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4146   1.1  jmcneill 
   4147   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4148   1.1  jmcneill 
   4149   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4150   1.1  jmcneill 	if (error != 0) {
   4151   1.1  jmcneill 		printf(", hca init timeout\n");
   4152   1.1  jmcneill 		return (-1);
   4153   1.1  jmcneill 	}
   4154   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4155   1.1  jmcneill 		printf(", hca init command corrupt\n");
   4156   1.1  jmcneill 		return (-1);
   4157   1.1  jmcneill 	}
   4158   1.1  jmcneill 
   4159   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4160   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4161   1.1  jmcneill 		printf(", hca init failed (%x)\n", status);
   4162   1.1  jmcneill 		return (-1);
   4163   1.1  jmcneill 	}
   4164   1.1  jmcneill 
   4165   1.1  jmcneill 	return (0);
   4166   1.1  jmcneill }
   4167   1.1  jmcneill 
   4168   1.1  jmcneill static int
   4169   1.1  jmcneill mcx_set_driver_version(struct mcx_softc *sc)
   4170   1.1  jmcneill {
   4171   1.1  jmcneill 	struct mcx_dmamem mxm;
   4172   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4173   1.1  jmcneill 	struct mcx_cmd_set_driver_version_in *in;
   4174   1.1  jmcneill 	struct mcx_cmd_set_driver_version_out *out;
   4175   1.1  jmcneill 	int error;
   4176   1.1  jmcneill 	int token;
   4177   1.1  jmcneill 	uint8_t status;
   4178   1.1  jmcneill 
   4179   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4180   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4181   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) +
   4182   1.1  jmcneill 	    sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);
   4183   1.1  jmcneill 
   4184   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4185   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
   4186   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4187   1.1  jmcneill 
   4188   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4189   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4190   1.1  jmcneill 		printf(", unable to allocate set driver version mailboxen\n");
   4191   1.1  jmcneill 		return (-1);
   4192   1.1  jmcneill 	}
   4193   1.1  jmcneill 	strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
   4194   1.1  jmcneill 	    "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
   4195   1.1  jmcneill 
   4196   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4197   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4198   1.1  jmcneill 
   4199   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4200   1.1  jmcneill 	if (error != 0) {
   4201   1.1  jmcneill 		printf(", set driver version timeout\n");
   4202   1.1  jmcneill 		goto free;
   4203   1.1  jmcneill 	}
   4204   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4205   1.1  jmcneill 		printf(", set driver version command corrupt\n");
   4206   1.1  jmcneill 		goto free;
   4207   1.1  jmcneill 	}
   4208   1.1  jmcneill 
   4209   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4210   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4211   1.1  jmcneill 		printf(", set driver version failed (%x)\n", status);
   4212   1.1  jmcneill 		error = -1;
   4213   1.1  jmcneill 		goto free;
   4214   1.1  jmcneill 	}
   4215   1.1  jmcneill 
   4216   1.1  jmcneill free:
   4217   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4218   1.1  jmcneill 
   4219   1.1  jmcneill 	return (error);
   4220   1.1  jmcneill }
   4221   1.1  jmcneill 
   4222   1.1  jmcneill static int
   4223   1.1  jmcneill mcx_iff(struct mcx_softc *sc)
   4224   1.1  jmcneill {
   4225   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   4226   1.1  jmcneill 	struct mcx_dmamem mxm;
   4227   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4228   1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_in *in;
   4229   1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_out *out;
   4230   1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   4231   1.1  jmcneill 	int error;
   4232   1.1  jmcneill 	int token;
   4233   1.1  jmcneill 	int insize;
   4234  1.15  jmcneill 	uint32_t dest;
   4235  1.15  jmcneill 
   4236  1.15  jmcneill 	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   4237  1.15  jmcneill 	    sc->sc_rss_flow_table_id;
   4238   1.1  jmcneill 
   4239   1.1  jmcneill 	/* enable or disable the promisc flow */
   4240   1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_PROMISC)) {
   4241   1.1  jmcneill 		if (sc->sc_promisc_flow_enabled == 0) {
   4242  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc,
   4243  1.15  jmcneill 			    MCX_FLOW_GROUP_PROMISC, 0, NULL, dest);
   4244   1.1  jmcneill 			sc->sc_promisc_flow_enabled = 1;
   4245   1.1  jmcneill 		}
   4246   1.1  jmcneill 	} else if (sc->sc_promisc_flow_enabled != 0) {
   4247   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   4248   1.1  jmcneill 		sc->sc_promisc_flow_enabled = 0;
   4249   1.1  jmcneill 	}
   4250   1.1  jmcneill 
   4251   1.1  jmcneill 	/* enable or disable the all-multicast flow */
   4252   1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   4253   1.1  jmcneill 		if (sc->sc_allmulti_flow_enabled == 0) {
   4254   1.1  jmcneill 			uint8_t mcast[ETHER_ADDR_LEN];
   4255   1.1  jmcneill 
   4256   1.1  jmcneill 			memset(mcast, 0, sizeof(mcast));
   4257   1.1  jmcneill 			mcast[0] = 0x01;
   4258  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc,
   4259  1.15  jmcneill 			    MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest);
   4260   1.1  jmcneill 			sc->sc_allmulti_flow_enabled = 1;
   4261   1.1  jmcneill 		}
   4262   1.1  jmcneill 	} else if (sc->sc_allmulti_flow_enabled != 0) {
   4263   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   4264   1.1  jmcneill 		sc->sc_allmulti_flow_enabled = 0;
   4265   1.1  jmcneill 	}
   4266   1.1  jmcneill 
   4267   1.1  jmcneill 	insize = sizeof(struct mcx_nic_vport_ctx) + 240;
   4268   1.1  jmcneill 
   4269   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4270   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4271   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4272   1.1  jmcneill 
   4273   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4274   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
   4275   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4276   1.1  jmcneill 	in->cmd_field_select = htobe32(
   4277   1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
   4278   1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
   4279   1.1  jmcneill 
   4280   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4281  1.15  jmcneill 		printf(", unable to allocate modify "
   4282  1.15  jmcneill 		    "nic vport context mailboxen\n");
   4283   1.1  jmcneill 		return (-1);
   4284   1.1  jmcneill 	}
   4285   1.1  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)
   4286   1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
   4287   1.1  jmcneill 	ctx->vp_mtu = htobe32(sc->sc_hardmtu);
   4288   1.1  jmcneill 	/*
   4289  1.15  jmcneill          * always leave promisc-all enabled on the vport since we
   4290  1.15  jmcneill          * can't give it a vlan list, and we're already doing multicast
   4291  1.15  jmcneill          * filtering in the flow table.
   4292   1.1  jmcneill 	 */
   4293   1.1  jmcneill 	ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);
   4294   1.1  jmcneill 
   4295   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4296   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4297   1.1  jmcneill 
   4298   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4299   1.1  jmcneill 	if (error != 0) {
   4300   1.1  jmcneill 		printf(", modify nic vport context timeout\n");
   4301   1.1  jmcneill 		goto free;
   4302   1.1  jmcneill 	}
   4303   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4304   1.1  jmcneill 		printf(", modify nic vport context command corrupt\n");
   4305   1.1  jmcneill 		goto free;
   4306   1.1  jmcneill 	}
   4307   1.1  jmcneill 
   4308   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4309   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4310   1.1  jmcneill 		printf(", modify nic vport context failed (%x, %x)\n",
   4311  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4312   1.1  jmcneill 		error = -1;
   4313   1.1  jmcneill 		goto free;
   4314   1.1  jmcneill 	}
   4315   1.1  jmcneill 
   4316   1.1  jmcneill free:
   4317   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4318   1.1  jmcneill 
   4319   1.1  jmcneill 	return (error);
   4320   1.1  jmcneill }
   4321   1.1  jmcneill 
   4322   1.1  jmcneill static int
   4323  1.15  jmcneill mcx_alloc_uar(struct mcx_softc *sc, int *uar)
   4324   1.1  jmcneill {
   4325   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4326   1.1  jmcneill 	struct mcx_cmd_alloc_uar_in *in;
   4327   1.1  jmcneill 	struct mcx_cmd_alloc_uar_out *out;
   4328   1.1  jmcneill 	int error;
   4329   1.1  jmcneill 
   4330   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4331   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4332   1.1  jmcneill 
   4333   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4334   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
   4335   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4336   1.1  jmcneill 
   4337   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4338   1.1  jmcneill 
   4339   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4340   1.1  jmcneill 	if (error != 0) {
   4341   1.1  jmcneill 		printf(", alloc uar timeout\n");
   4342   1.1  jmcneill 		return (-1);
   4343   1.1  jmcneill 	}
   4344   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4345   1.1  jmcneill 		printf(", alloc uar command corrupt\n");
   4346   1.1  jmcneill 		return (-1);
   4347   1.1  jmcneill 	}
   4348   1.1  jmcneill 
   4349   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4350   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4351   1.1  jmcneill 		printf(", alloc uar failed (%x)\n", out->cmd_status);
   4352   1.1  jmcneill 		return (-1);
   4353   1.1  jmcneill 	}
   4354   1.1  jmcneill 
   4355  1.15  jmcneill 	*uar = mcx_get_id(out->cmd_uar);
   4356   1.1  jmcneill 	return (0);
   4357   1.1  jmcneill }
   4358   1.1  jmcneill 
   4359   1.1  jmcneill static int
   4360  1.15  jmcneill mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar,
   4361  1.15  jmcneill     uint64_t events, int vector)
   4362   1.1  jmcneill {
   4363   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4364   1.1  jmcneill 	struct mcx_dmamem mxm;
   4365   1.1  jmcneill 	struct mcx_cmd_create_eq_in *in;
   4366   1.1  jmcneill 	struct mcx_cmd_create_eq_mb_in *mbin;
   4367   1.1  jmcneill 	struct mcx_cmd_create_eq_out *out;
   4368   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   4369   1.1  jmcneill 	int error;
   4370   1.1  jmcneill 	uint64_t *pas;
   4371   1.1  jmcneill 	int insize, npages, paslen, i, token;
   4372   1.1  jmcneill 
   4373  1.15  jmcneill 	eq->eq_cons = 0;
   4374   1.1  jmcneill 
   4375   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
   4376   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4377   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4378   1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
   4379   1.1  jmcneill 
   4380  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE,
   4381   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4382   1.1  jmcneill 		printf(", unable to allocate event queue memory\n");
   4383   1.1  jmcneill 		return (-1);
   4384   1.1  jmcneill 	}
   4385   1.1  jmcneill 
   4386  1.15  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
   4387   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
   4388   1.1  jmcneill 		eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
   4389   1.1  jmcneill 	}
   4390   1.1  jmcneill 
   4391   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4392   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4393   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4394   1.1  jmcneill 
   4395   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4396   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
   4397   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4398   1.1  jmcneill 
   4399  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4400  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4401   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4402   1.1  jmcneill 		printf(", unable to allocate create eq mailboxen\n");
   4403  1.15  jmcneill 		goto free_eq;
   4404   1.1  jmcneill 	}
   4405   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4406   1.1  jmcneill 	mbin->cmd_eq_ctx.eq_uar_size = htobe32(
   4407  1.15  jmcneill 	    (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar);
   4408  1.15  jmcneill 	mbin->cmd_eq_ctx.eq_intr = vector;
   4409  1.15  jmcneill 	mbin->cmd_event_bitmask = htobe64(events);
   4410  1.15  jmcneill 
   4411  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   4412  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   4413   1.1  jmcneill 
   4414   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4415  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem);
   4416   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
   4417   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4418   1.1  jmcneill 
   4419   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4420   1.1  jmcneill 	if (error != 0) {
   4421   1.1  jmcneill 		printf(", create eq timeout\n");
   4422  1.15  jmcneill 		goto free_mxm;
   4423   1.1  jmcneill 	}
   4424   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4425   1.1  jmcneill 		printf(", create eq command corrupt\n");
   4426  1.15  jmcneill 		goto free_mxm;
   4427   1.1  jmcneill 	}
   4428   1.1  jmcneill 
   4429   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4430   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4431   1.1  jmcneill 		printf(", create eq failed (%x, %x)\n", out->cmd_status,
   4432   1.1  jmcneill 		    be32toh(out->cmd_syndrome));
   4433  1.15  jmcneill 		goto free_mxm;
   4434   1.1  jmcneill 	}
   4435   1.1  jmcneill 
   4436  1.15  jmcneill 	eq->eq_n = mcx_get_id(out->cmd_eqn);
   4437  1.15  jmcneill 
   4438  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4439  1.15  jmcneill 
   4440  1.15  jmcneill 	mcx_arm_eq(sc, eq, uar);
   4441  1.15  jmcneill 
   4442  1.15  jmcneill 	return (0);
   4443  1.15  jmcneill 
   4444  1.15  jmcneill free_mxm:
   4445  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   4446  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   4447   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4448  1.15  jmcneill free_eq:
   4449  1.15  jmcneill 	mcx_dmamem_free(sc, &eq->eq_mem);
   4450  1.15  jmcneill 	return (-1);
   4451   1.1  jmcneill }
   4452   1.1  jmcneill 
   4453   1.1  jmcneill static int
   4454   1.1  jmcneill mcx_alloc_pd(struct mcx_softc *sc)
   4455   1.1  jmcneill {
   4456   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4457   1.1  jmcneill 	struct mcx_cmd_alloc_pd_in *in;
   4458   1.1  jmcneill 	struct mcx_cmd_alloc_pd_out *out;
   4459   1.1  jmcneill 	int error;
   4460   1.1  jmcneill 
   4461   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4462   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4463   1.1  jmcneill 
   4464   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4465   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
   4466   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4467   1.1  jmcneill 
   4468   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4469   1.1  jmcneill 
   4470   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4471   1.1  jmcneill 	if (error != 0) {
   4472   1.1  jmcneill 		printf(", alloc pd timeout\n");
   4473   1.1  jmcneill 		return (-1);
   4474   1.1  jmcneill 	}
   4475   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4476   1.1  jmcneill 		printf(", alloc pd command corrupt\n");
   4477   1.1  jmcneill 		return (-1);
   4478   1.1  jmcneill 	}
   4479   1.1  jmcneill 
   4480   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4481   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4482   1.1  jmcneill 		printf(", alloc pd failed (%x)\n", out->cmd_status);
   4483   1.1  jmcneill 		return (-1);
   4484   1.1  jmcneill 	}
   4485   1.1  jmcneill 
   4486  1.13  jmcneill 	sc->sc_pd = mcx_get_id(out->cmd_pd);
   4487   1.1  jmcneill 	return (0);
   4488   1.1  jmcneill }
   4489   1.1  jmcneill 
   4490   1.1  jmcneill static int
   4491   1.1  jmcneill mcx_alloc_tdomain(struct mcx_softc *sc)
   4492   1.1  jmcneill {
   4493   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4494   1.1  jmcneill 	struct mcx_cmd_alloc_td_in *in;
   4495   1.1  jmcneill 	struct mcx_cmd_alloc_td_out *out;
   4496   1.1  jmcneill 	int error;
   4497   1.1  jmcneill 
   4498   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4499   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4500   1.1  jmcneill 
   4501   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4502   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
   4503   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4504   1.1  jmcneill 
   4505   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4506   1.1  jmcneill 
   4507   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4508   1.1  jmcneill 	if (error != 0) {
   4509   1.1  jmcneill 		printf(", alloc transport domain timeout\n");
   4510   1.1  jmcneill 		return (-1);
   4511   1.1  jmcneill 	}
   4512   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4513   1.1  jmcneill 		printf(", alloc transport domain command corrupt\n");
   4514   1.1  jmcneill 		return (-1);
   4515   1.1  jmcneill 	}
   4516   1.1  jmcneill 
   4517   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4518   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4519   1.1  jmcneill 		printf(", alloc transport domain failed (%x)\n",
   4520   1.1  jmcneill 		    out->cmd_status);
   4521   1.1  jmcneill 		return (-1);
   4522   1.1  jmcneill 	}
   4523   1.1  jmcneill 
   4524  1.13  jmcneill 	sc->sc_tdomain = mcx_get_id(out->cmd_tdomain);
   4525   1.1  jmcneill 	return (0);
   4526   1.1  jmcneill }
   4527   1.1  jmcneill 
   4528   1.1  jmcneill static int
   4529   1.1  jmcneill mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr)
   4530   1.1  jmcneill {
   4531   1.1  jmcneill 	struct mcx_dmamem mxm;
   4532   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4533   1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_in *in;
   4534   1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_out *out;
   4535   1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   4536   1.1  jmcneill 	uint8_t *addr;
   4537   1.1  jmcneill 	int error, token, i;
   4538   1.1  jmcneill 
   4539   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4540   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4541   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
   4542   1.1  jmcneill 
   4543   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4544   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
   4545   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4546   1.1  jmcneill 	in->cmd_allowed_list_type = 0;
   4547   1.1  jmcneill 
   4548  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4549  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4550  1.15  jmcneill 		printf(", unable to allocate "
   4551  1.15  jmcneill 		    "query nic vport context mailboxen\n");
   4552   1.1  jmcneill 		return (-1);
   4553   1.1  jmcneill 	}
   4554   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4555   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4556   1.1  jmcneill 
   4557   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4558   1.1  jmcneill 	if (error != 0) {
   4559   1.1  jmcneill 		printf(", query nic vport context timeout\n");
   4560   1.1  jmcneill 		goto free;
   4561   1.1  jmcneill 	}
   4562   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4563   1.1  jmcneill 		printf(", query nic vport context command corrupt\n");
   4564   1.1  jmcneill 		goto free;
   4565   1.1  jmcneill 	}
   4566   1.1  jmcneill 
   4567   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4568   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4569   1.1  jmcneill 		printf(", query nic vport context failed (%x, %x)\n",
   4570  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4571   1.1  jmcneill 		error = -1;
   4572   1.1  jmcneill 		goto free;
   4573   1.1  jmcneill 	}
   4574   1.1  jmcneill 
   4575  1.15  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)
   4576  1.15  jmcneill 	    mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4577   1.1  jmcneill 	addr = (uint8_t *)&ctx->vp_perm_addr;
   4578   1.1  jmcneill 	for (i = 0; i < ETHER_ADDR_LEN; i++) {
   4579   1.1  jmcneill 		enaddr[i] = addr[i + 2];
   4580   1.1  jmcneill 	}
   4581   1.1  jmcneill free:
   4582   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4583   1.1  jmcneill 
   4584   1.1  jmcneill 	return (error);
   4585   1.1  jmcneill }
   4586   1.1  jmcneill 
   4587   1.1  jmcneill static int
   4588   1.1  jmcneill mcx_query_special_contexts(struct mcx_softc *sc)
   4589   1.1  jmcneill {
   4590   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4591   1.1  jmcneill 	struct mcx_cmd_query_special_ctx_in *in;
   4592   1.1  jmcneill 	struct mcx_cmd_query_special_ctx_out *out;
   4593   1.1  jmcneill 	int error;
   4594   1.1  jmcneill 
   4595   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4596   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4597   1.1  jmcneill 
   4598   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4599   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
   4600   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4601   1.1  jmcneill 
   4602   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4603   1.1  jmcneill 
   4604   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4605   1.1  jmcneill 	if (error != 0) {
   4606   1.1  jmcneill 		printf(", query special contexts timeout\n");
   4607   1.1  jmcneill 		return (-1);
   4608   1.1  jmcneill 	}
   4609   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4610   1.1  jmcneill 		printf(", query special contexts command corrupt\n");
   4611   1.1  jmcneill 		return (-1);
   4612   1.1  jmcneill 	}
   4613   1.1  jmcneill 
   4614   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4615   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4616   1.1  jmcneill 		printf(", query special contexts failed (%x)\n",
   4617   1.1  jmcneill 		    out->cmd_status);
   4618   1.1  jmcneill 		return (-1);
   4619   1.1  jmcneill 	}
   4620   1.1  jmcneill 
   4621   1.1  jmcneill 	sc->sc_lkey = be32toh(out->cmd_resd_lkey);
   4622   1.1  jmcneill 	return (0);
   4623   1.1  jmcneill }
   4624   1.1  jmcneill 
   4625   1.1  jmcneill static int
   4626   1.1  jmcneill mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
   4627   1.1  jmcneill {
   4628   1.1  jmcneill 	struct mcx_reg_pmtu pmtu;
   4629   1.1  jmcneill 	int error;
   4630   1.1  jmcneill 
   4631   1.1  jmcneill 	/* read max mtu */
   4632   1.1  jmcneill 	memset(&pmtu, 0, sizeof(pmtu));
   4633   1.1  jmcneill 	pmtu.rp_local_port = 1;
   4634   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
   4635   1.1  jmcneill 	    sizeof(pmtu));
   4636   1.1  jmcneill 	if (error != 0) {
   4637   1.1  jmcneill 		printf(", unable to get port MTU\n");
   4638   1.1  jmcneill 		return error;
   4639   1.1  jmcneill 	}
   4640   1.1  jmcneill 
   4641   1.1  jmcneill 	mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu));
   4642   1.1  jmcneill 	pmtu.rp_admin_mtu = htobe16(mtu);
   4643   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
   4644   1.1  jmcneill 	    sizeof(pmtu));
   4645   1.1  jmcneill 	if (error != 0) {
   4646   1.1  jmcneill 		printf(", unable to set port MTU\n");
   4647   1.1  jmcneill 		return error;
   4648   1.1  jmcneill 	}
   4649   1.1  jmcneill 
   4650   1.1  jmcneill 	sc->sc_hardmtu = mtu;
   4651  1.15  jmcneill 	sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long));
   4652   1.1  jmcneill 	return 0;
   4653   1.1  jmcneill }
   4654   1.1  jmcneill 
   4655   1.1  jmcneill static int
   4656  1.15  jmcneill mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn)
   4657   1.1  jmcneill {
   4658   1.1  jmcneill 	struct mcx_cmdq_entry *cmde;
   4659   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   4660   1.1  jmcneill 	struct mcx_dmamem mxm;
   4661   1.1  jmcneill 	struct mcx_cmd_create_cq_in *in;
   4662   1.1  jmcneill 	struct mcx_cmd_create_cq_mb_in *mbin;
   4663   1.1  jmcneill 	struct mcx_cmd_create_cq_out *out;
   4664   1.1  jmcneill 	int error;
   4665   1.1  jmcneill 	uint64_t *pas;
   4666   1.1  jmcneill 	int insize, npages, paslen, i, token;
   4667   1.1  jmcneill 
   4668  1.15  jmcneill 	cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db);
   4669   1.1  jmcneill 
   4670   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
   4671   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4672   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4673   1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
   4674   1.1  jmcneill 
   4675   1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
   4676   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4677   1.1  jmcneill 		printf("%s: unable to allocate completion queue memory\n",
   4678   1.1  jmcneill 		    DEVNAME(sc));
   4679   1.1  jmcneill 		return (-1);
   4680   1.1  jmcneill 	}
   4681   1.1  jmcneill 	cqe = MCX_DMA_KVA(&cq->cq_mem);
   4682   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
   4683   1.1  jmcneill 		cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
   4684   1.1  jmcneill 	}
   4685   1.1  jmcneill 
   4686   1.1  jmcneill 	cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4687   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4688   1.1  jmcneill 	mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
   4689   1.1  jmcneill 
   4690   1.1  jmcneill 	in = mcx_cmdq_in(cmde);
   4691   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
   4692   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4693   1.1  jmcneill 
   4694  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4695  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4696   1.1  jmcneill 	    &cmde->cq_input_ptr, token) != 0) {
   4697  1.15  jmcneill 		printf("%s: unable to allocate create cq mailboxen\n",
   4698  1.15  jmcneill 		    DEVNAME(sc));
   4699  1.15  jmcneill 		goto free_cq;
   4700   1.1  jmcneill 	}
   4701   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4702   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_uar_size = htobe32(
   4703  1.15  jmcneill 	    (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar);
   4704   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
   4705   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
   4706   1.1  jmcneill 	    (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
   4707   1.1  jmcneill 	    MCX_CQ_MOD_COUNTER);
   4708   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_doorbell = htobe64(
   4709  1.15  jmcneill 	    MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell);
   4710  1.15  jmcneill 
   4711  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4712  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
   4713   1.1  jmcneill 
   4714   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4715  1.13  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem);
   4716   1.1  jmcneill 	mcx_cmdq_post(sc, cmde, 0);
   4717   1.1  jmcneill 
   4718   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cmde, 1000);
   4719   1.1  jmcneill 	if (error != 0) {
   4720   1.1  jmcneill 		printf("%s: create cq timeout\n", DEVNAME(sc));
   4721  1.15  jmcneill 		goto free_mxm;
   4722   1.1  jmcneill 	}
   4723   1.1  jmcneill 	if (mcx_cmdq_verify(cmde) != 0) {
   4724   1.1  jmcneill 		printf("%s: create cq command corrupt\n", DEVNAME(sc));
   4725  1.15  jmcneill 		goto free_mxm;
   4726   1.1  jmcneill 	}
   4727   1.1  jmcneill 
   4728   1.1  jmcneill 	out = mcx_cmdq_out(cmde);
   4729   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4730   1.1  jmcneill 		printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
   4731   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4732  1.15  jmcneill 		goto free_mxm;
   4733   1.1  jmcneill 	}
   4734   1.1  jmcneill 
   4735  1.13  jmcneill 	cq->cq_n = mcx_get_id(out->cmd_cqn);
   4736   1.1  jmcneill 	cq->cq_cons = 0;
   4737   1.1  jmcneill 	cq->cq_count = 0;
   4738   1.1  jmcneill 
   4739   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4740  1.15  jmcneill 
   4741  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4742  1.15  jmcneill 	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
   4743  1.15  jmcneill 	    BUS_DMASYNC_PREWRITE);
   4744  1.15  jmcneill 
   4745  1.15  jmcneill 	mcx_arm_cq(sc, cq, uar);
   4746  1.15  jmcneill 
   4747  1.15  jmcneill 	return (0);
   4748  1.15  jmcneill 
   4749  1.15  jmcneill free_mxm:
   4750  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4751  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   4752  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4753  1.15  jmcneill free_cq:
   4754  1.15  jmcneill 	mcx_dmamem_free(sc, &cq->cq_mem);
   4755  1.15  jmcneill 	return (-1);
   4756   1.1  jmcneill }
   4757   1.1  jmcneill 
   4758   1.1  jmcneill static int
   4759  1.15  jmcneill mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq)
   4760   1.1  jmcneill {
   4761   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4762   1.1  jmcneill 	struct mcx_cmd_destroy_cq_in *in;
   4763   1.1  jmcneill 	struct mcx_cmd_destroy_cq_out *out;
   4764   1.1  jmcneill 	int error;
   4765   1.1  jmcneill 	int token;
   4766   1.1  jmcneill 
   4767   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4768   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4769   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4770   1.1  jmcneill 
   4771   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4772   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
   4773   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4774  1.15  jmcneill 	in->cmd_cqn = htobe32(cq->cq_n);
   4775   1.1  jmcneill 
   4776   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4777   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4778   1.1  jmcneill 	if (error != 0) {
   4779   1.1  jmcneill 		printf("%s: destroy cq timeout\n", DEVNAME(sc));
   4780   1.1  jmcneill 		return error;
   4781   1.1  jmcneill 	}
   4782   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4783   1.1  jmcneill 		printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
   4784   1.1  jmcneill 		return error;
   4785   1.1  jmcneill 	}
   4786   1.1  jmcneill 
   4787   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4788   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4789   1.1  jmcneill 		printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
   4790   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4791   1.1  jmcneill 		return -1;
   4792   1.1  jmcneill 	}
   4793   1.1  jmcneill 
   4794  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4795  1.15  jmcneill 	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
   4796  1.15  jmcneill 	    BUS_DMASYNC_POSTWRITE);
   4797  1.15  jmcneill 
   4798  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4799  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   4800  1.15  jmcneill 	mcx_dmamem_free(sc, &cq->cq_mem);
   4801  1.15  jmcneill 
   4802  1.15  jmcneill 	cq->cq_n = 0;
   4803  1.15  jmcneill 	cq->cq_cons = 0;
   4804  1.15  jmcneill 	cq->cq_count = 0;
   4805   1.1  jmcneill 	return 0;
   4806   1.1  jmcneill }
   4807   1.1  jmcneill 
   4808   1.1  jmcneill static int
   4809  1.15  jmcneill mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn)
   4810   1.1  jmcneill {
   4811   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4812   1.1  jmcneill 	struct mcx_dmamem mxm;
   4813   1.1  jmcneill 	struct mcx_cmd_create_rq_in *in;
   4814   1.1  jmcneill 	struct mcx_cmd_create_rq_out *out;
   4815   1.1  jmcneill 	struct mcx_rq_ctx *mbin;
   4816   1.1  jmcneill 	int error;
   4817   1.1  jmcneill 	uint64_t *pas;
   4818  1.15  jmcneill 	uint32_t rq_flags;
   4819  1.13  jmcneill 	int insize, npages, paslen, token;
   4820   1.1  jmcneill 
   4821  1.15  jmcneill 	rx->rx_doorbell = MCX_WQ_DOORBELL_BASE +
   4822  1.15  jmcneill 	    (db * MCX_WQ_DOORBELL_STRIDE);
   4823  1.15  jmcneill 
   4824   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
   4825   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4826   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4827   1.1  jmcneill 	insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
   4828   1.1  jmcneill 
   4829  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE,
   4830   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4831   1.1  jmcneill 		printf("%s: unable to allocate receive queue memory\n",
   4832   1.1  jmcneill 		    DEVNAME(sc));
   4833   1.1  jmcneill 		return (-1);
   4834   1.1  jmcneill 	}
   4835   1.1  jmcneill 
   4836   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4837   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4838   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4839   1.1  jmcneill 
   4840   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4841   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
   4842   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4843   1.1  jmcneill 
   4844  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4845  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4846   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4847   1.1  jmcneill 		printf("%s: unable to allocate create rq mailboxen\n",
   4848   1.1  jmcneill 		    DEVNAME(sc));
   4849  1.15  jmcneill 		goto free_rq;
   4850  1.15  jmcneill 	}
   4851  1.15  jmcneill 	mbin = (struct mcx_rq_ctx *)
   4852  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   4853  1.15  jmcneill 	rq_flags = MCX_RQ_CTX_RLKEY;
   4854  1.15  jmcneill 	mbin->rq_flags = htobe32(rq_flags);
   4855   1.1  jmcneill 	mbin->rq_cqn = htobe32(cqn);
   4856   1.1  jmcneill 	mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   4857   1.1  jmcneill 	mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
   4858   1.1  jmcneill 	mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   4859  1.15  jmcneill 	    rx->rx_doorbell);
   4860   1.1  jmcneill 	mbin->rq_wq.wq_log_stride = htobe16(4);
   4861   1.1  jmcneill 	mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
   4862   1.1  jmcneill 
   4863  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   4864  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
   4865  1.15  jmcneill 
   4866   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4867  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem);
   4868   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4869   1.1  jmcneill 
   4870   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4871   1.1  jmcneill 	if (error != 0) {
   4872   1.1  jmcneill 		printf("%s: create rq timeout\n", DEVNAME(sc));
   4873  1.15  jmcneill 		goto free_mxm;
   4874   1.1  jmcneill 	}
   4875   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4876   1.1  jmcneill 		printf("%s: create rq command corrupt\n", DEVNAME(sc));
   4877  1.15  jmcneill 		goto free_mxm;
   4878   1.1  jmcneill 	}
   4879   1.1  jmcneill 
   4880   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4881   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4882   1.1  jmcneill 		printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
   4883   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4884  1.15  jmcneill 		goto free_mxm;
   4885   1.1  jmcneill 	}
   4886   1.1  jmcneill 
   4887  1.15  jmcneill 	rx->rx_rqn = mcx_get_id(out->cmd_rqn);
   4888   1.1  jmcneill 
   4889  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4890  1.15  jmcneill 
   4891  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4892  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   4893  1.15  jmcneill 
   4894  1.15  jmcneill 	return (0);
   4895   1.1  jmcneill 
   4896  1.15  jmcneill free_mxm:
   4897  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   4898  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   4899   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4900  1.15  jmcneill free_rq:
   4901  1.15  jmcneill 	mcx_dmamem_free(sc, &rx->rx_rq_mem);
   4902  1.15  jmcneill 	return (-1);
   4903   1.1  jmcneill }
   4904   1.1  jmcneill 
   4905   1.1  jmcneill static int
   4906  1.15  jmcneill mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx)
   4907   1.1  jmcneill {
   4908   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4909   1.1  jmcneill 	struct mcx_dmamem mxm;
   4910   1.1  jmcneill 	struct mcx_cmd_modify_rq_in *in;
   4911   1.1  jmcneill 	struct mcx_cmd_modify_rq_mb_in *mbin;
   4912   1.1  jmcneill 	struct mcx_cmd_modify_rq_out *out;
   4913   1.1  jmcneill 	int error;
   4914   1.1  jmcneill 	int token;
   4915   1.1  jmcneill 
   4916   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4917   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4918  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   4919  1.15  jmcneill 	    sizeof(*out), token);
   4920   1.1  jmcneill 
   4921   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4922   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
   4923   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4924  1.15  jmcneill 	in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn);
   4925   1.1  jmcneill 
   4926  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4927  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4928  1.15  jmcneill 		printf("%s: unable to allocate modify rq mailbox\n",
   4929  1.15  jmcneill 		    DEVNAME(sc));
   4930   1.1  jmcneill 		return (-1);
   4931   1.1  jmcneill 	}
   4932   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4933   1.1  jmcneill 	mbin->cmd_rq_ctx.rq_flags = htobe32(
   4934   1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
   4935   1.1  jmcneill 
   4936   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4937   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4938   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4939   1.1  jmcneill 	if (error != 0) {
   4940   1.1  jmcneill 		printf("%s: modify rq timeout\n", DEVNAME(sc));
   4941   1.1  jmcneill 		goto free;
   4942   1.1  jmcneill 	}
   4943   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4944   1.1  jmcneill 		printf("%s: modify rq command corrupt\n", DEVNAME(sc));
   4945   1.1  jmcneill 		goto free;
   4946   1.1  jmcneill 	}
   4947   1.1  jmcneill 
   4948   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4949   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4950   1.1  jmcneill 		printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
   4951   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4952   1.1  jmcneill 		error = -1;
   4953   1.1  jmcneill 		goto free;
   4954   1.1  jmcneill 	}
   4955   1.1  jmcneill 
   4956   1.1  jmcneill free:
   4957   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4958   1.1  jmcneill 	return (error);
   4959   1.1  jmcneill }
   4960   1.1  jmcneill 
   4961   1.1  jmcneill static int
   4962  1.15  jmcneill mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx)
   4963   1.1  jmcneill {
   4964   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4965   1.1  jmcneill 	struct mcx_cmd_destroy_rq_in *in;
   4966   1.1  jmcneill 	struct mcx_cmd_destroy_rq_out *out;
   4967   1.1  jmcneill 	int error;
   4968   1.1  jmcneill 	int token;
   4969   1.1  jmcneill 
   4970   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4971   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4972   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4973   1.1  jmcneill 
   4974   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4975   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
   4976   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4977  1.15  jmcneill 	in->cmd_rqn = htobe32(rx->rx_rqn);
   4978   1.1  jmcneill 
   4979   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4980   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4981   1.1  jmcneill 	if (error != 0) {
   4982   1.1  jmcneill 		printf("%s: destroy rq timeout\n", DEVNAME(sc));
   4983   1.1  jmcneill 		return error;
   4984   1.1  jmcneill 	}
   4985   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4986   1.1  jmcneill 		printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
   4987   1.1  jmcneill 		return error;
   4988   1.1  jmcneill 	}
   4989   1.1  jmcneill 
   4990   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4991   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4992   1.1  jmcneill 		printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
   4993   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4994   1.1  jmcneill 		return -1;
   4995   1.1  jmcneill 	}
   4996   1.1  jmcneill 
   4997  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4998  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   4999  1.15  jmcneill 
   5000  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   5001  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   5002  1.15  jmcneill 	mcx_dmamem_free(sc, &rx->rx_rq_mem);
   5003  1.15  jmcneill 
   5004  1.15  jmcneill 	rx->rx_rqn = 0;
   5005   1.1  jmcneill 	return 0;
   5006   1.1  jmcneill }
   5007   1.1  jmcneill 
   5008   1.1  jmcneill static int
   5009  1.15  jmcneill mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn)
   5010   1.1  jmcneill {
   5011   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5012   1.1  jmcneill 	struct mcx_dmamem mxm;
   5013   1.1  jmcneill 	struct mcx_cmd_create_tir_in *in;
   5014   1.1  jmcneill 	struct mcx_cmd_create_tir_mb_in *mbin;
   5015   1.1  jmcneill 	struct mcx_cmd_create_tir_out *out;
   5016   1.1  jmcneill 	int error;
   5017   1.1  jmcneill 	int token;
   5018   1.1  jmcneill 
   5019   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5020   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5021  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5022  1.15  jmcneill 	    sizeof(*out), token);
   5023   1.1  jmcneill 
   5024   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5025   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
   5026   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5027   1.1  jmcneill 
   5028  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5029  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5030   1.1  jmcneill 		printf("%s: unable to allocate create tir mailbox\n",
   5031   1.1  jmcneill 		    DEVNAME(sc));
   5032   1.1  jmcneill 		return (-1);
   5033   1.1  jmcneill 	}
   5034   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5035   1.1  jmcneill 	/* leave disp_type = 0, so packets get sent to the inline rqn */
   5036  1.15  jmcneill 	mbin->cmd_inline_rqn = htobe32(rx->rx_rqn);
   5037   1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   5038   1.1  jmcneill 
   5039   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5040   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5041   1.1  jmcneill 	if (error != 0) {
   5042   1.1  jmcneill 		printf("%s: create tir timeout\n", DEVNAME(sc));
   5043   1.1  jmcneill 		goto free;
   5044   1.1  jmcneill 	}
   5045   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5046   1.1  jmcneill 		printf("%s: create tir command corrupt\n", DEVNAME(sc));
   5047   1.1  jmcneill 		goto free;
   5048   1.1  jmcneill 	}
   5049   1.1  jmcneill 
   5050   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5051   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5052   1.1  jmcneill 		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
   5053   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5054   1.1  jmcneill 		error = -1;
   5055   1.1  jmcneill 		goto free;
   5056   1.1  jmcneill 	}
   5057   1.1  jmcneill 
   5058  1.15  jmcneill 	*tirn = mcx_get_id(out->cmd_tirn);
   5059  1.15  jmcneill free:
   5060  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5061  1.15  jmcneill 	return (error);
   5062  1.15  jmcneill }
   5063  1.15  jmcneill 
   5064  1.15  jmcneill static int
   5065  1.15  jmcneill mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel,
   5066  1.15  jmcneill     int *tirn)
   5067  1.15  jmcneill {
   5068  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5069  1.15  jmcneill 	struct mcx_dmamem mxm;
   5070  1.15  jmcneill 	struct mcx_cmd_create_tir_in *in;
   5071  1.15  jmcneill 	struct mcx_cmd_create_tir_mb_in *mbin;
   5072  1.15  jmcneill 	struct mcx_cmd_create_tir_out *out;
   5073  1.15  jmcneill 	int error;
   5074  1.15  jmcneill 	int token;
   5075  1.15  jmcneill 
   5076  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5077  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5078  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5079  1.15  jmcneill 	    sizeof(*out), token);
   5080  1.15  jmcneill 
   5081  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5082  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
   5083  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5084  1.15  jmcneill 
   5085  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5086  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5087  1.15  jmcneill 		printf("%s: unable to allocate create tir mailbox\n",
   5088  1.15  jmcneill 		    DEVNAME(sc));
   5089  1.15  jmcneill 		return (-1);
   5090  1.15  jmcneill 	}
   5091  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5092  1.15  jmcneill 	mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT
   5093  1.15  jmcneill 	    << MCX_TIR_CTX_DISP_TYPE_SHIFT);
   5094  1.15  jmcneill 	mbin->cmd_indir_table = htobe32(rqtn);
   5095  1.15  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain |
   5096  1.15  jmcneill 	    MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT);
   5097  1.15  jmcneill 	mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel);
   5098  1.15  jmcneill 	stoeplitz_to_key(&mbin->cmd_rx_hash_key,
   5099  1.15  jmcneill 	    sizeof(mbin->cmd_rx_hash_key));
   5100  1.15  jmcneill 
   5101  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5102  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5103  1.15  jmcneill 	if (error != 0) {
   5104  1.15  jmcneill 		printf("%s: create tir timeout\n", DEVNAME(sc));
   5105  1.15  jmcneill 		goto free;
   5106  1.15  jmcneill 	}
   5107  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5108  1.15  jmcneill 		printf("%s: create tir command corrupt\n", DEVNAME(sc));
   5109  1.15  jmcneill 		goto free;
   5110  1.15  jmcneill 	}
   5111  1.15  jmcneill 
   5112  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5113  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5114  1.15  jmcneill 		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
   5115  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5116  1.15  jmcneill 		error = -1;
   5117  1.15  jmcneill 		goto free;
   5118  1.15  jmcneill 	}
   5119  1.15  jmcneill 
   5120  1.15  jmcneill 	*tirn = mcx_get_id(out->cmd_tirn);
   5121   1.1  jmcneill free:
   5122   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5123   1.1  jmcneill 	return (error);
   5124   1.1  jmcneill }
   5125   1.1  jmcneill 
   5126   1.1  jmcneill static int
   5127  1.15  jmcneill mcx_destroy_tir(struct mcx_softc *sc, int tirn)
   5128   1.1  jmcneill {
   5129   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5130   1.1  jmcneill 	struct mcx_cmd_destroy_tir_in *in;
   5131   1.1  jmcneill 	struct mcx_cmd_destroy_tir_out *out;
   5132   1.1  jmcneill 	int error;
   5133   1.1  jmcneill 	int token;
   5134   1.1  jmcneill 
   5135   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5136   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5137   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5138   1.1  jmcneill 
   5139   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5140   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
   5141   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5142  1.15  jmcneill 	in->cmd_tirn = htobe32(tirn);
   5143   1.1  jmcneill 
   5144   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5145   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5146   1.1  jmcneill 	if (error != 0) {
   5147   1.1  jmcneill 		printf("%s: destroy tir timeout\n", DEVNAME(sc));
   5148   1.1  jmcneill 		return error;
   5149   1.1  jmcneill 	}
   5150   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5151   1.1  jmcneill 		printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
   5152   1.1  jmcneill 		return error;
   5153   1.1  jmcneill 	}
   5154   1.1  jmcneill 
   5155   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5156   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5157   1.1  jmcneill 		printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
   5158   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5159   1.1  jmcneill 		return -1;
   5160   1.1  jmcneill 	}
   5161   1.1  jmcneill 
   5162  1.15  jmcneill 	return (0);
   5163   1.1  jmcneill }
   5164   1.1  jmcneill 
   5165   1.1  jmcneill static int
   5166  1.15  jmcneill mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db,
   5167  1.15  jmcneill     int cqn)
   5168   1.1  jmcneill {
   5169   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5170   1.1  jmcneill 	struct mcx_dmamem mxm;
   5171   1.1  jmcneill 	struct mcx_cmd_create_sq_in *in;
   5172   1.1  jmcneill 	struct mcx_sq_ctx *mbin;
   5173   1.1  jmcneill 	struct mcx_cmd_create_sq_out *out;
   5174   1.1  jmcneill 	int error;
   5175   1.1  jmcneill 	uint64_t *pas;
   5176  1.13  jmcneill 	int insize, npages, paslen, token;
   5177   1.1  jmcneill 
   5178  1.15  jmcneill 	tx->tx_doorbell = MCX_WQ_DOORBELL_BASE +
   5179  1.15  jmcneill 	    (db * MCX_WQ_DOORBELL_STRIDE) + 4;
   5180  1.15  jmcneill 
   5181   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
   5182   1.1  jmcneill 	    MCX_PAGE_SIZE);
   5183   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   5184   1.1  jmcneill 	insize = sizeof(struct mcx_sq_ctx) + paslen;
   5185   1.1  jmcneill 
   5186  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE,
   5187   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   5188  1.15  jmcneill 		printf("%s: unable to allocate send queue memory\n",
   5189  1.15  jmcneill 		    DEVNAME(sc));
   5190   1.1  jmcneill 		return (-1);
   5191   1.1  jmcneill 	}
   5192   1.1  jmcneill 
   5193   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5194   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5195   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
   5196   1.1  jmcneill 	    token);
   5197   1.1  jmcneill 
   5198   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5199   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
   5200   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5201   1.1  jmcneill 
   5202  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   5203  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   5204   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5205  1.15  jmcneill 		printf("%s: unable to allocate create sq mailboxen\n",
   5206  1.15  jmcneill 		    DEVNAME(sc));
   5207  1.15  jmcneill 		goto free_sq;
   5208   1.1  jmcneill 	}
   5209  1.15  jmcneill 	mbin = (struct mcx_sq_ctx *)
   5210  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   5211   1.1  jmcneill 	mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
   5212   1.1  jmcneill 	    (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
   5213   1.1  jmcneill 	mbin->sq_cqn = htobe32(cqn);
   5214   1.1  jmcneill 	mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
   5215  1.15  jmcneill 	mbin->sq_tis_num = htobe32(sc->sc_tis);
   5216   1.1  jmcneill 	mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   5217   1.1  jmcneill 	mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
   5218  1.15  jmcneill 	mbin->sq_wq.wq_uar_page = htobe32(uar);
   5219   1.1  jmcneill 	mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   5220  1.15  jmcneill 	    tx->tx_doorbell);
   5221   1.1  jmcneill 	mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
   5222   1.1  jmcneill 	mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
   5223   1.1  jmcneill 
   5224  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5225  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
   5226  1.15  jmcneill 
   5227   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   5228  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10,
   5229  1.15  jmcneill 	    npages, &tx->tx_sq_mem);
   5230   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5231   1.1  jmcneill 
   5232   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5233   1.1  jmcneill 	if (error != 0) {
   5234   1.1  jmcneill 		printf("%s: create sq timeout\n", DEVNAME(sc));
   5235  1.15  jmcneill 		goto free_mxm;
   5236   1.1  jmcneill 	}
   5237   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5238   1.1  jmcneill 		printf("%s: create sq command corrupt\n", DEVNAME(sc));
   5239  1.15  jmcneill 		goto free_mxm;
   5240   1.1  jmcneill 	}
   5241   1.1  jmcneill 
   5242   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5243   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5244   1.1  jmcneill 		printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
   5245   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5246  1.15  jmcneill 		goto free_mxm;
   5247   1.1  jmcneill 	}
   5248   1.1  jmcneill 
   5249  1.15  jmcneill 	tx->tx_uar = uar;
   5250  1.15  jmcneill 	tx->tx_sqn = mcx_get_id(out->cmd_sqn);
   5251  1.15  jmcneill 
   5252  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5253  1.15  jmcneill 
   5254  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   5255  1.15  jmcneill 	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   5256  1.15  jmcneill 
   5257  1.15  jmcneill 	return (0);
   5258   1.1  jmcneill 
   5259  1.15  jmcneill free_mxm:
   5260  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5261  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   5262   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5263  1.15  jmcneill free_sq:
   5264  1.15  jmcneill 	mcx_dmamem_free(sc, &tx->tx_sq_mem);
   5265  1.15  jmcneill 	return (-1);
   5266   1.1  jmcneill }
   5267   1.1  jmcneill 
   5268   1.1  jmcneill static int
   5269  1.15  jmcneill mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx)
   5270   1.1  jmcneill {
   5271   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5272   1.1  jmcneill 	struct mcx_cmd_destroy_sq_in *in;
   5273   1.1  jmcneill 	struct mcx_cmd_destroy_sq_out *out;
   5274   1.1  jmcneill 	int error;
   5275   1.1  jmcneill 	int token;
   5276   1.1  jmcneill 
   5277   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5278   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5279   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5280   1.1  jmcneill 
   5281   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5282   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
   5283   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5284  1.15  jmcneill 	in->cmd_sqn = htobe32(tx->tx_sqn);
   5285   1.1  jmcneill 
   5286   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5287   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5288   1.1  jmcneill 	if (error != 0) {
   5289   1.1  jmcneill 		printf("%s: destroy sq timeout\n", DEVNAME(sc));
   5290   1.1  jmcneill 		return error;
   5291   1.1  jmcneill 	}
   5292   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5293   1.1  jmcneill 		printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
   5294   1.1  jmcneill 		return error;
   5295   1.1  jmcneill 	}
   5296   1.1  jmcneill 
   5297   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5298   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5299   1.1  jmcneill 		printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
   5300   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5301   1.1  jmcneill 		return -1;
   5302   1.1  jmcneill 	}
   5303   1.1  jmcneill 
   5304  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   5305  1.15  jmcneill 	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   5306  1.15  jmcneill 
   5307  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5308  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   5309  1.15  jmcneill 	mcx_dmamem_free(sc, &tx->tx_sq_mem);
   5310  1.15  jmcneill 
   5311  1.15  jmcneill 	tx->tx_sqn = 0;
   5312   1.1  jmcneill 	return 0;
   5313   1.1  jmcneill }
   5314   1.1  jmcneill 
   5315   1.1  jmcneill static int
   5316  1.15  jmcneill mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx)
   5317   1.1  jmcneill {
   5318   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5319   1.1  jmcneill 	struct mcx_dmamem mxm;
   5320   1.1  jmcneill 	struct mcx_cmd_modify_sq_in *in;
   5321   1.1  jmcneill 	struct mcx_cmd_modify_sq_mb_in *mbin;
   5322   1.1  jmcneill 	struct mcx_cmd_modify_sq_out *out;
   5323   1.1  jmcneill 	int error;
   5324   1.1  jmcneill 	int token;
   5325   1.1  jmcneill 
   5326   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5327   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5328  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5329  1.15  jmcneill 	    sizeof(*out), token);
   5330   1.1  jmcneill 
   5331   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5332   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
   5333   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5334  1.15  jmcneill 	in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn);
   5335   1.1  jmcneill 
   5336  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5337  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5338   1.1  jmcneill 		printf("%s: unable to allocate modify sq mailbox\n",
   5339   1.1  jmcneill 		    DEVNAME(sc));
   5340   1.1  jmcneill 		return (-1);
   5341   1.1  jmcneill 	}
   5342   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5343   1.1  jmcneill 	mbin->cmd_sq_ctx.sq_flags = htobe32(
   5344   1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
   5345   1.1  jmcneill 
   5346   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5347   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5348   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5349   1.1  jmcneill 	if (error != 0) {
   5350   1.1  jmcneill 		printf("%s: modify sq timeout\n", DEVNAME(sc));
   5351   1.1  jmcneill 		goto free;
   5352   1.1  jmcneill 	}
   5353   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5354   1.1  jmcneill 		printf("%s: modify sq command corrupt\n", DEVNAME(sc));
   5355   1.1  jmcneill 		goto free;
   5356   1.1  jmcneill 	}
   5357   1.1  jmcneill 
   5358   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5359   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5360   1.1  jmcneill 		printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
   5361   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5362   1.1  jmcneill 		error = -1;
   5363   1.1  jmcneill 		goto free;
   5364   1.1  jmcneill 	}
   5365   1.1  jmcneill 
   5366   1.1  jmcneill free:
   5367   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5368   1.1  jmcneill 	return (error);
   5369   1.1  jmcneill }
   5370   1.1  jmcneill 
   5371   1.1  jmcneill static int
   5372  1.15  jmcneill mcx_create_tis(struct mcx_softc *sc, int *tis)
   5373   1.1  jmcneill {
   5374   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5375   1.1  jmcneill 	struct mcx_dmamem mxm;
   5376   1.1  jmcneill 	struct mcx_cmd_create_tis_in *in;
   5377   1.1  jmcneill 	struct mcx_cmd_create_tis_mb_in *mbin;
   5378   1.1  jmcneill 	struct mcx_cmd_create_tis_out *out;
   5379   1.1  jmcneill 	int error;
   5380   1.1  jmcneill 	int token;
   5381   1.1  jmcneill 
   5382   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5383   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5384  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5385  1.15  jmcneill 	    sizeof(*out), token);
   5386   1.1  jmcneill 
   5387   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5388   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
   5389   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5390   1.1  jmcneill 
   5391  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5392  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5393  1.15  jmcneill 		printf("%s: unable to allocate create tis mailbox\n",
   5394  1.15  jmcneill 		    DEVNAME(sc));
   5395   1.1  jmcneill 		return (-1);
   5396   1.1  jmcneill 	}
   5397   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5398   1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   5399   1.1  jmcneill 
   5400   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5401   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5402   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5403   1.1  jmcneill 	if (error != 0) {
   5404   1.1  jmcneill 		printf("%s: create tis timeout\n", DEVNAME(sc));
   5405   1.1  jmcneill 		goto free;
   5406   1.1  jmcneill 	}
   5407   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5408   1.1  jmcneill 		printf("%s: create tis command corrupt\n", DEVNAME(sc));
   5409   1.1  jmcneill 		goto free;
   5410   1.1  jmcneill 	}
   5411   1.1  jmcneill 
   5412   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5413   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5414   1.1  jmcneill 		printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
   5415   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5416   1.1  jmcneill 		error = -1;
   5417   1.1  jmcneill 		goto free;
   5418   1.1  jmcneill 	}
   5419   1.1  jmcneill 
   5420  1.15  jmcneill 	*tis = mcx_get_id(out->cmd_tisn);
   5421   1.1  jmcneill free:
   5422   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5423   1.1  jmcneill 	return (error);
   5424   1.1  jmcneill }
   5425   1.1  jmcneill 
   5426   1.1  jmcneill static int
   5427  1.15  jmcneill mcx_destroy_tis(struct mcx_softc *sc, int tis)
   5428   1.1  jmcneill {
   5429   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5430   1.1  jmcneill 	struct mcx_cmd_destroy_tis_in *in;
   5431   1.1  jmcneill 	struct mcx_cmd_destroy_tis_out *out;
   5432   1.1  jmcneill 	int error;
   5433   1.1  jmcneill 	int token;
   5434   1.1  jmcneill 
   5435   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5436   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5437   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5438   1.1  jmcneill 
   5439   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5440   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
   5441   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5442  1.15  jmcneill 	in->cmd_tisn = htobe32(tis);
   5443   1.1  jmcneill 
   5444   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5445   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5446   1.1  jmcneill 	if (error != 0) {
   5447   1.1  jmcneill 		printf("%s: destroy tis timeout\n", DEVNAME(sc));
   5448   1.1  jmcneill 		return error;
   5449   1.1  jmcneill 	}
   5450   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5451   1.1  jmcneill 		printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
   5452   1.1  jmcneill 		return error;
   5453   1.1  jmcneill 	}
   5454   1.1  jmcneill 
   5455   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5456   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5457   1.1  jmcneill 		printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
   5458   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5459   1.1  jmcneill 		return -1;
   5460   1.1  jmcneill 	}
   5461   1.1  jmcneill 
   5462  1.15  jmcneill 	return 0;
   5463  1.15  jmcneill }
   5464  1.15  jmcneill 
   5465  1.15  jmcneill static int
   5466  1.15  jmcneill mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt)
   5467  1.15  jmcneill {
   5468  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5469  1.15  jmcneill 	struct mcx_dmamem mxm;
   5470  1.15  jmcneill 	struct mcx_cmd_create_rqt_in *in;
   5471  1.15  jmcneill 	struct mcx_cmd_create_rqt_mb_in *mbin;
   5472  1.15  jmcneill 	struct mcx_cmd_create_rqt_out *out;
   5473  1.15  jmcneill 	struct mcx_rqt_ctx *rqt_ctx;
   5474  1.15  jmcneill 	int *rqtn;
   5475  1.15  jmcneill 	int error;
   5476  1.15  jmcneill 	int token;
   5477  1.15  jmcneill 	int i;
   5478  1.15  jmcneill 
   5479  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5480  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5481  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) +
   5482  1.15  jmcneill 	    (size * sizeof(int)), sizeof(*out), token);
   5483  1.15  jmcneill 
   5484  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5485  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT);
   5486  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5487  1.15  jmcneill 
   5488  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5489  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5490  1.15  jmcneill 		printf("%s: unable to allocate create rqt mailbox\n",
   5491  1.15  jmcneill 		    DEVNAME(sc));
   5492  1.15  jmcneill 		return (-1);
   5493  1.15  jmcneill 	}
   5494  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5495  1.15  jmcneill 	rqt_ctx = &mbin->cmd_rqt;
   5496  1.15  jmcneill 	rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size);
   5497  1.15  jmcneill 	rqt_ctx->cmd_rqt_actual_size = htobe16(size);
   5498  1.15  jmcneill 
   5499  1.15  jmcneill 	/* rqt list follows the rqt context */
   5500  1.15  jmcneill 	rqtn = (int *)(rqt_ctx + 1);
   5501  1.15  jmcneill 	for (i = 0; i < size; i++) {
   5502  1.15  jmcneill 		rqtn[i] = htobe32(rqns[i]);
   5503  1.15  jmcneill 	}
   5504  1.15  jmcneill 
   5505  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5506  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5507  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5508  1.15  jmcneill 	if (error != 0) {
   5509  1.15  jmcneill 		printf("%s: create rqt timeout\n", DEVNAME(sc));
   5510  1.15  jmcneill 		goto free;
   5511  1.15  jmcneill 	}
   5512  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5513  1.15  jmcneill 		printf("%s: create rqt command corrupt\n", DEVNAME(sc));
   5514  1.15  jmcneill 		goto free;
   5515  1.15  jmcneill 	}
   5516  1.15  jmcneill 
   5517  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5518  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5519  1.15  jmcneill 		printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc),
   5520  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5521  1.15  jmcneill 		error = -1;
   5522  1.15  jmcneill 		goto free;
   5523  1.15  jmcneill 	}
   5524  1.15  jmcneill 
   5525  1.15  jmcneill 	*rqt = mcx_get_id(out->cmd_rqtn);
   5526  1.15  jmcneill 	return (0);
   5527  1.15  jmcneill free:
   5528  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5529  1.15  jmcneill 	return (error);
   5530  1.15  jmcneill }
   5531  1.15  jmcneill 
   5532  1.15  jmcneill static int
   5533  1.15  jmcneill mcx_destroy_rqt(struct mcx_softc *sc, int rqt)
   5534  1.15  jmcneill {
   5535  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5536  1.15  jmcneill 	struct mcx_cmd_destroy_rqt_in *in;
   5537  1.15  jmcneill 	struct mcx_cmd_destroy_rqt_out *out;
   5538  1.15  jmcneill 	int error;
   5539  1.15  jmcneill 	int token;
   5540  1.15  jmcneill 
   5541  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5542  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5543  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5544  1.15  jmcneill 
   5545  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5546  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT);
   5547  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5548  1.15  jmcneill 	in->cmd_rqtn = htobe32(rqt);
   5549  1.15  jmcneill 
   5550  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5551  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5552  1.15  jmcneill 	if (error != 0) {
   5553  1.15  jmcneill 		printf("%s: destroy rqt timeout\n", DEVNAME(sc));
   5554  1.15  jmcneill 		return error;
   5555  1.15  jmcneill 	}
   5556  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5557  1.15  jmcneill 		printf("%s: destroy rqt command corrupt\n", DEVNAME(sc));
   5558  1.15  jmcneill 		return error;
   5559  1.15  jmcneill 	}
   5560  1.15  jmcneill 
   5561  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5562  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5563  1.15  jmcneill 		printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc),
   5564  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5565  1.15  jmcneill 		return -1;
   5566  1.15  jmcneill 	}
   5567  1.15  jmcneill 
   5568   1.1  jmcneill 	return 0;
   5569   1.1  jmcneill }
   5570   1.1  jmcneill 
   5571   1.1  jmcneill #if 0
   5572   1.1  jmcneill static int
   5573   1.1  jmcneill mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
   5574   1.1  jmcneill {
   5575   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5576   1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_in *in;
   5577   1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_out *out;
   5578   1.1  jmcneill 	int error;
   5579   1.1  jmcneill 
   5580   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5581   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   5582   1.1  jmcneill 
   5583   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5584   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
   5585   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5586   1.1  jmcneill 
   5587   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5588   1.1  jmcneill 
   5589   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5590   1.1  jmcneill 	if (error != 0) {
   5591   1.1  jmcneill 		printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
   5592   1.1  jmcneill 		return (-1);
   5593   1.1  jmcneill 	}
   5594   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5595   1.1  jmcneill 		printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
   5596   1.1  jmcneill 		return (-1);
   5597   1.1  jmcneill 	}
   5598   1.1  jmcneill 
   5599   1.1  jmcneill 	out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
   5600   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5601   1.1  jmcneill 		printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
   5602   1.1  jmcneill 		    out->cmd_status);
   5603   1.1  jmcneill 		return (-1);
   5604   1.1  jmcneill 	}
   5605   1.1  jmcneill 
   5606   1.1  jmcneill 	sc->sc_flow_counter_id[i]  = be16toh(out->cmd_flow_counter_id);
   5607   1.1  jmcneill 	printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
   5608   1.1  jmcneill 
   5609   1.1  jmcneill 	return (0);
   5610   1.1  jmcneill }
   5611   1.1  jmcneill #endif
   5612   1.1  jmcneill 
   5613   1.1  jmcneill static int
   5614  1.15  jmcneill mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level,
   5615  1.15  jmcneill     int *flow_table_id)
   5616   1.1  jmcneill {
   5617   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5618   1.1  jmcneill 	struct mcx_dmamem mxm;
   5619   1.1  jmcneill 	struct mcx_cmd_create_flow_table_in *in;
   5620   1.1  jmcneill 	struct mcx_cmd_create_flow_table_mb_in *mbin;
   5621   1.1  jmcneill 	struct mcx_cmd_create_flow_table_out *out;
   5622   1.1  jmcneill 	int error;
   5623   1.1  jmcneill 	int token;
   5624   1.1  jmcneill 
   5625   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5626   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5627  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5628  1.15  jmcneill 	    sizeof(*out), token);
   5629   1.1  jmcneill 
   5630   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5631   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
   5632   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5633   1.1  jmcneill 
   5634  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5635  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5636   1.1  jmcneill 		printf("%s: unable to allocate create flow table mailbox\n",
   5637   1.1  jmcneill 		    DEVNAME(sc));
   5638   1.1  jmcneill 		return (-1);
   5639   1.1  jmcneill 	}
   5640   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5641   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5642   1.1  jmcneill 	mbin->cmd_ctx.ft_log_size = log_size;
   5643  1.15  jmcneill 	mbin->cmd_ctx.ft_level = level;
   5644   1.1  jmcneill 
   5645   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5646   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5647   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5648   1.1  jmcneill 	if (error != 0) {
   5649   1.1  jmcneill 		printf("%s: create flow table timeout\n", DEVNAME(sc));
   5650   1.1  jmcneill 		goto free;
   5651   1.1  jmcneill 	}
   5652   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5653   1.1  jmcneill 		printf("%s: create flow table command corrupt\n", DEVNAME(sc));
   5654   1.1  jmcneill 		goto free;
   5655   1.1  jmcneill 	}
   5656   1.1  jmcneill 
   5657   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5658   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5659   1.1  jmcneill 		printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
   5660   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5661   1.1  jmcneill 		error = -1;
   5662   1.1  jmcneill 		goto free;
   5663   1.1  jmcneill 	}
   5664   1.1  jmcneill 
   5665  1.15  jmcneill 	*flow_table_id = mcx_get_id(out->cmd_table_id);
   5666   1.1  jmcneill free:
   5667   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5668   1.1  jmcneill 	return (error);
   5669   1.1  jmcneill }
   5670   1.1  jmcneill 
   5671   1.1  jmcneill static int
   5672  1.15  jmcneill mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id)
   5673   1.1  jmcneill {
   5674   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5675   1.1  jmcneill 	struct mcx_dmamem mxm;
   5676   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_in *in;
   5677   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_mb_in *mbin;
   5678   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_out *out;
   5679   1.1  jmcneill 	int error;
   5680   1.1  jmcneill 	int token;
   5681   1.1  jmcneill 
   5682   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5683   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5684  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5685  1.15  jmcneill 	    sizeof(*out), token);
   5686   1.1  jmcneill 
   5687   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5688   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
   5689   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5690   1.1  jmcneill 
   5691  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5692  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5693   1.1  jmcneill 		printf("%s: unable to allocate set flow table root mailbox\n",
   5694   1.1  jmcneill 		    DEVNAME(sc));
   5695   1.1  jmcneill 		return (-1);
   5696   1.1  jmcneill 	}
   5697   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5698   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5699  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   5700   1.1  jmcneill 
   5701   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5702   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5703   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5704   1.1  jmcneill 	if (error != 0) {
   5705   1.1  jmcneill 		printf("%s: set flow table root timeout\n", DEVNAME(sc));
   5706   1.1  jmcneill 		goto free;
   5707   1.1  jmcneill 	}
   5708   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5709   1.1  jmcneill 		printf("%s: set flow table root command corrupt\n",
   5710   1.1  jmcneill 		    DEVNAME(sc));
   5711   1.1  jmcneill 		goto free;
   5712   1.1  jmcneill 	}
   5713   1.1  jmcneill 
   5714   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5715   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5716   1.1  jmcneill 		printf("%s: set flow table root failed (%x, %x)\n",
   5717   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5718   1.1  jmcneill 		error = -1;
   5719   1.1  jmcneill 		goto free;
   5720   1.1  jmcneill 	}
   5721   1.1  jmcneill 
   5722   1.1  jmcneill free:
   5723   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5724   1.1  jmcneill 	return (error);
   5725   1.1  jmcneill }
   5726   1.1  jmcneill 
   5727   1.1  jmcneill static int
   5728  1.15  jmcneill mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id)
   5729   1.1  jmcneill {
   5730   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5731   1.1  jmcneill 	struct mcx_dmamem mxm;
   5732   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_in *in;
   5733   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_mb_in *mb;
   5734   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_out *out;
   5735   1.1  jmcneill 	int error;
   5736   1.1  jmcneill 	int token;
   5737   1.1  jmcneill 
   5738   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5739   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5740   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   5741   1.1  jmcneill 
   5742   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5743   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
   5744   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5745   1.1  jmcneill 
   5746  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5747  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5748   1.1  jmcneill 		printf("%s: unable to allocate destroy flow table mailbox\n",
   5749   1.1  jmcneill 		    DEVNAME(sc));
   5750   1.1  jmcneill 		return (-1);
   5751   1.1  jmcneill 	}
   5752   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5753   1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5754  1.15  jmcneill 	mb->cmd_table_id = htobe32(flow_table_id);
   5755   1.1  jmcneill 
   5756   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5757   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5758   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5759   1.1  jmcneill 	if (error != 0) {
   5760   1.1  jmcneill 		printf("%s: destroy flow table timeout\n", DEVNAME(sc));
   5761   1.1  jmcneill 		goto free;
   5762   1.1  jmcneill 	}
   5763   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5764  1.15  jmcneill 		printf("%s: destroy flow table command corrupt\n",
   5765  1.15  jmcneill 		    DEVNAME(sc));
   5766   1.1  jmcneill 		goto free;
   5767   1.1  jmcneill 	}
   5768   1.1  jmcneill 
   5769   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5770   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5771   1.1  jmcneill 		printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
   5772   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5773   1.1  jmcneill 		error = -1;
   5774   1.1  jmcneill 		goto free;
   5775   1.1  jmcneill 	}
   5776   1.1  jmcneill 
   5777   1.1  jmcneill free:
   5778   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5779   1.1  jmcneill 	return (error);
   5780   1.1  jmcneill }
   5781   1.1  jmcneill 
   5782   1.1  jmcneill 
   5783   1.1  jmcneill static int
   5784  1.15  jmcneill mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group,
   5785  1.15  jmcneill     int start, int size, int match_enable, struct mcx_flow_match *match)
   5786   1.1  jmcneill {
   5787   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5788   1.1  jmcneill 	struct mcx_dmamem mxm;
   5789   1.1  jmcneill 	struct mcx_cmd_create_flow_group_in *in;
   5790   1.1  jmcneill 	struct mcx_cmd_create_flow_group_mb_in *mbin;
   5791   1.1  jmcneill 	struct mcx_cmd_create_flow_group_out *out;
   5792  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5793   1.1  jmcneill 	int error;
   5794   1.1  jmcneill 	int token;
   5795   1.1  jmcneill 
   5796   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5797   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5798   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   5799   1.1  jmcneill 	    token);
   5800   1.1  jmcneill 
   5801   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5802   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
   5803   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5804   1.1  jmcneill 
   5805   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   5806   1.1  jmcneill 	    != 0) {
   5807   1.1  jmcneill 		printf("%s: unable to allocate create flow group mailbox\n",
   5808   1.1  jmcneill 		    DEVNAME(sc));
   5809   1.1  jmcneill 		return (-1);
   5810   1.1  jmcneill 	}
   5811   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5812   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5813  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   5814   1.1  jmcneill 	mbin->cmd_start_flow_index = htobe32(start);
   5815   1.1  jmcneill 	mbin->cmd_end_flow_index = htobe32(start + (size - 1));
   5816   1.1  jmcneill 
   5817   1.1  jmcneill 	mbin->cmd_match_criteria_enable = match_enable;
   5818   1.1  jmcneill 	memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));
   5819   1.1  jmcneill 
   5820   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5821   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5822   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5823   1.1  jmcneill 	if (error != 0) {
   5824   1.1  jmcneill 		printf("%s: create flow group timeout\n", DEVNAME(sc));
   5825   1.1  jmcneill 		goto free;
   5826   1.1  jmcneill 	}
   5827   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5828   1.1  jmcneill 		printf("%s: create flow group command corrupt\n", DEVNAME(sc));
   5829   1.1  jmcneill 		goto free;
   5830   1.1  jmcneill 	}
   5831   1.1  jmcneill 
   5832   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5833   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5834   1.1  jmcneill 		printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
   5835   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5836   1.1  jmcneill 		error = -1;
   5837   1.1  jmcneill 		goto free;
   5838   1.1  jmcneill 	}
   5839   1.1  jmcneill 
   5840  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5841  1.15  jmcneill 	mfg->g_id = mcx_get_id(out->cmd_group_id);
   5842  1.15  jmcneill 	mfg->g_table = flow_table_id;
   5843  1.15  jmcneill 	mfg->g_start = start;
   5844  1.15  jmcneill 	mfg->g_size = size;
   5845   1.1  jmcneill 
   5846   1.1  jmcneill free:
   5847   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5848   1.1  jmcneill 	return (error);
   5849   1.1  jmcneill }
   5850   1.1  jmcneill 
   5851   1.1  jmcneill static int
   5852   1.1  jmcneill mcx_destroy_flow_group(struct mcx_softc *sc, int group)
   5853   1.1  jmcneill {
   5854   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5855   1.1  jmcneill 	struct mcx_dmamem mxm;
   5856   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_in *in;
   5857   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_mb_in *mb;
   5858   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_out *out;
   5859  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5860   1.1  jmcneill 	int error;
   5861   1.1  jmcneill 	int token;
   5862   1.1  jmcneill 
   5863   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5864   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5865   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   5866   1.1  jmcneill 
   5867   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5868   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
   5869   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5870   1.1  jmcneill 
   5871  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5872  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5873   1.1  jmcneill 		printf("%s: unable to allocate destroy flow group mailbox\n",
   5874   1.1  jmcneill 		    DEVNAME(sc));
   5875   1.1  jmcneill 		return (-1);
   5876   1.1  jmcneill 	}
   5877   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5878   1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5879  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5880  1.15  jmcneill 	mb->cmd_table_id = htobe32(mfg->g_table);
   5881  1.15  jmcneill 	mb->cmd_group_id = htobe32(mfg->g_id);
   5882   1.1  jmcneill 
   5883   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5884   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5885   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5886   1.1  jmcneill 	if (error != 0) {
   5887   1.1  jmcneill 		printf("%s: destroy flow group timeout\n", DEVNAME(sc));
   5888   1.1  jmcneill 		goto free;
   5889   1.1  jmcneill 	}
   5890   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5891   1.1  jmcneill 		printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
   5892   1.1  jmcneill 		goto free;
   5893   1.1  jmcneill 	}
   5894   1.1  jmcneill 
   5895   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5896   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5897   1.1  jmcneill 		printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
   5898   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5899   1.1  jmcneill 		error = -1;
   5900   1.1  jmcneill 		goto free;
   5901   1.1  jmcneill 	}
   5902   1.1  jmcneill 
   5903  1.15  jmcneill 	mfg->g_id = -1;
   5904  1.15  jmcneill 	mfg->g_table = -1;
   5905  1.15  jmcneill 	mfg->g_size = 0;
   5906  1.15  jmcneill 	mfg->g_start = 0;
   5907   1.1  jmcneill free:
   5908   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5909   1.1  jmcneill 	return (error);
   5910   1.1  jmcneill }
   5911   1.1  jmcneill 
   5912   1.1  jmcneill static int
   5913  1.15  jmcneill mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index,
   5914  1.15  jmcneill     const uint8_t *macaddr, uint32_t dest)
   5915   1.1  jmcneill {
   5916   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5917   1.1  jmcneill 	struct mcx_dmamem mxm;
   5918   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_in *in;
   5919   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
   5920   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_out *out;
   5921  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5922  1.15  jmcneill 	uint32_t *pdest;
   5923   1.1  jmcneill 	int error;
   5924   1.1  jmcneill 	int token;
   5925   1.1  jmcneill 
   5926   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5927   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5928  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
   5929   1.1  jmcneill 	    sizeof(*out), token);
   5930   1.1  jmcneill 
   5931   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5932   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
   5933   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5934   1.1  jmcneill 
   5935   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   5936   1.1  jmcneill 	    != 0) {
   5937   1.1  jmcneill 		printf("%s: unable to allocate set flow table entry mailbox\n",
   5938   1.1  jmcneill 		    DEVNAME(sc));
   5939   1.1  jmcneill 		return (-1);
   5940   1.1  jmcneill 	}
   5941  1.15  jmcneill 
   5942   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5943   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5944  1.15  jmcneill 
   5945  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5946  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   5947  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   5948  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
   5949   1.1  jmcneill 
   5950   1.1  jmcneill 	/* flow context ends at offset 0x330, 0x130 into the second mbox */
   5951  1.15  jmcneill 	pdest = (uint32_t *)
   5952   1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
   5953   1.1  jmcneill 	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
   5954   1.1  jmcneill 	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
   5955  1.15  jmcneill 	*pdest = htobe32(dest);
   5956   1.1  jmcneill 
   5957   1.1  jmcneill 	/* the only thing we match on at the moment is the dest mac address */
   5958   1.1  jmcneill 	if (macaddr != NULL) {
   5959   1.1  jmcneill 		memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
   5960   1.1  jmcneill 		    ETHER_ADDR_LEN);
   5961   1.1  jmcneill 	}
   5962   1.1  jmcneill 
   5963   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5964   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5965   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5966   1.1  jmcneill 	if (error != 0) {
   5967   1.1  jmcneill 		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
   5968   1.1  jmcneill 		goto free;
   5969   1.1  jmcneill 	}
   5970   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5971   1.1  jmcneill 		printf("%s: set flow table entry command corrupt\n",
   5972   1.1  jmcneill 		    DEVNAME(sc));
   5973   1.1  jmcneill 		goto free;
   5974   1.1  jmcneill 	}
   5975   1.1  jmcneill 
   5976   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5977   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5978   1.1  jmcneill 		printf("%s: set flow table entry failed (%x, %x)\n",
   5979   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5980   1.1  jmcneill 		error = -1;
   5981   1.1  jmcneill 		goto free;
   5982   1.1  jmcneill 	}
   5983   1.1  jmcneill 
   5984   1.1  jmcneill free:
   5985   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5986   1.1  jmcneill 	return (error);
   5987   1.1  jmcneill }
   5988   1.1  jmcneill 
   5989   1.1  jmcneill static int
   5990  1.15  jmcneill mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index,
   5991  1.15  jmcneill     int ethertype, int ip_proto, uint32_t dest)
   5992   1.1  jmcneill {
   5993   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5994   1.1  jmcneill 	struct mcx_dmamem mxm;
   5995  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_in *in;
   5996  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
   5997  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_out *out;
   5998  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5999  1.15  jmcneill 	uint32_t *pdest;
   6000   1.1  jmcneill 	int error;
   6001   1.1  jmcneill 	int token;
   6002   1.1  jmcneill 
   6003   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6004   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   6005  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
   6006  1.15  jmcneill 	    sizeof(*out), token);
   6007   1.1  jmcneill 
   6008   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6009  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
   6010   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6011   1.1  jmcneill 
   6012  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   6013  1.15  jmcneill 	    != 0) {
   6014  1.15  jmcneill 		printf("%s: unable to allocate set flow table entry mailbox\n",
   6015   1.1  jmcneill 		    DEVNAME(sc));
   6016   1.1  jmcneill 		return (-1);
   6017   1.1  jmcneill 	}
   6018  1.15  jmcneill 
   6019   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6020   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   6021  1.15  jmcneill 
   6022  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   6023  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   6024  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   6025  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
   6026  1.15  jmcneill 
   6027  1.15  jmcneill 	/* flow context ends at offset 0x330, 0x130 into the second mbox */
   6028  1.15  jmcneill 	pdest = (uint32_t *)
   6029  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
   6030  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
   6031  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
   6032  1.15  jmcneill 	*pdest = htobe32(dest);
   6033  1.15  jmcneill 
   6034  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype);
   6035  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto;
   6036   1.1  jmcneill 
   6037   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   6038   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6039   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6040   1.1  jmcneill 	if (error != 0) {
   6041  1.15  jmcneill 		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
   6042   1.1  jmcneill 		goto free;
   6043   1.1  jmcneill 	}
   6044   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6045  1.15  jmcneill 		printf("%s: set flow table entry command corrupt\n",
   6046   1.1  jmcneill 		    DEVNAME(sc));
   6047   1.1  jmcneill 		goto free;
   6048   1.1  jmcneill 	}
   6049   1.1  jmcneill 
   6050   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6051   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6052  1.15  jmcneill 		printf("%s: set flow table entry failed (%x, %x)\n",
   6053  1.15  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   6054   1.1  jmcneill 		error = -1;
   6055   1.1  jmcneill 		goto free;
   6056   1.1  jmcneill 	}
   6057   1.1  jmcneill 
   6058   1.1  jmcneill free:
   6059   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6060   1.1  jmcneill 	return (error);
   6061   1.1  jmcneill }
   6062   1.1  jmcneill 
   6063  1.15  jmcneill static int
   6064  1.15  jmcneill mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
   6065   1.1  jmcneill {
   6066  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6067   1.1  jmcneill 	struct mcx_dmamem mxm;
   6068  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_in *in;
   6069  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
   6070  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_out *out;
   6071  1.15  jmcneill 	struct mcx_flow_group *mfg;
   6072   1.1  jmcneill 	int error;
   6073  1.15  jmcneill 	int token;
   6074  1.15  jmcneill 
   6075  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6076  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6077  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   6078  1.15  jmcneill 	    token);
   6079  1.15  jmcneill 
   6080  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6081  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
   6082  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6083  1.15  jmcneill 
   6084  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6085  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   6086  1.15  jmcneill 		printf("%s: unable to allocate "
   6087  1.15  jmcneill 		    "delete flow table entry mailbox\n", DEVNAME(sc));
   6088  1.15  jmcneill 		return (-1);
   6089  1.15  jmcneill 	}
   6090  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6091  1.15  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   6092  1.15  jmcneill 
   6093  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   6094  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   6095  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   6096  1.15  jmcneill 
   6097  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   6098  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6099  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6100  1.15  jmcneill 	if (error != 0) {
   6101  1.15  jmcneill 		printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
   6102  1.15  jmcneill 		goto free;
   6103  1.15  jmcneill 	}
   6104  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6105  1.15  jmcneill 		printf("%s: delete flow table entry command corrupt\n",
   6106  1.15  jmcneill 		    DEVNAME(sc));
   6107  1.15  jmcneill 		goto free;
   6108  1.15  jmcneill 	}
   6109  1.15  jmcneill 
   6110  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6111  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6112  1.15  jmcneill 		printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
   6113  1.15  jmcneill 		    DEVNAME(sc), group, index, out->cmd_status,
   6114  1.15  jmcneill 		    be32toh(out->cmd_syndrome));
   6115  1.15  jmcneill 		error = -1;
   6116  1.15  jmcneill 		goto free;
   6117  1.15  jmcneill 	}
   6118  1.15  jmcneill 
   6119  1.15  jmcneill free:
   6120  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6121  1.15  jmcneill 	return (error);
   6122  1.15  jmcneill }
   6123  1.15  jmcneill 
   6124  1.15  jmcneill #if 0
   6125  1.15  jmcneill int
   6126  1.15  jmcneill mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id)
   6127  1.15  jmcneill {
   6128  1.15  jmcneill 	struct mcx_dmamem mxm;
   6129  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6130  1.15  jmcneill 	struct mcx_cmd_query_flow_table_in *in;
   6131  1.15  jmcneill 	struct mcx_cmd_query_flow_table_mb_in *mbin;
   6132  1.15  jmcneill 	struct mcx_cmd_query_flow_table_out *out;
   6133  1.15  jmcneill 	struct mcx_cmd_query_flow_table_mb_out *mbout;
   6134  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6135  1.15  jmcneill 	int error;
   6136  1.15  jmcneill 	int i;
   6137  1.15  jmcneill 	uint8_t *dump;
   6138   1.1  jmcneill 
   6139   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6140   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6141   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6142   1.1  jmcneill 
   6143   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6144   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
   6145   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6146   1.1  jmcneill 
   6147   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6148   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6149   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6150   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6151   1.1  jmcneill 		printf(", unable to allocate query flow table mailboxes\n");
   6152   1.1  jmcneill 		return (-1);
   6153   1.1  jmcneill 	}
   6154   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6155   1.1  jmcneill 
   6156   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6157   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6158  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6159   1.1  jmcneill 
   6160   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6161   1.1  jmcneill 
   6162   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6163   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6164   1.1  jmcneill 	if (error != 0) {
   6165   1.1  jmcneill 		printf("%s: query flow table timeout\n", DEVNAME(sc));
   6166   1.1  jmcneill 		goto free;
   6167   1.1  jmcneill 	}
   6168   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6169   1.1  jmcneill 	if (error != 0) {
   6170   1.1  jmcneill 		printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
   6171   1.1  jmcneill 		goto free;
   6172   1.1  jmcneill 	}
   6173   1.1  jmcneill 
   6174   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6175   1.1  jmcneill 	switch (out->cmd_status) {
   6176   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6177   1.1  jmcneill 		break;
   6178   1.1  jmcneill 	default:
   6179   1.1  jmcneill 		printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
   6180   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6181   1.1  jmcneill 		error = -1;
   6182   1.1  jmcneill 		goto free;
   6183   1.1  jmcneill 	}
   6184   1.1  jmcneill 
   6185   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_mb_out *)
   6186   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6187   1.1  jmcneill 	dump = (uint8_t *)mbout + 8;
   6188   1.1  jmcneill 	for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
   6189   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6190   1.1  jmcneill 		if (i % 16 == 15)
   6191   1.1  jmcneill 			printf("\n");
   6192   1.1  jmcneill 	}
   6193   1.1  jmcneill free:
   6194   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6195   1.1  jmcneill 	return (error);
   6196   1.1  jmcneill }
   6197   1.1  jmcneill int
   6198  1.15  jmcneill mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index)
   6199   1.1  jmcneill {
   6200   1.1  jmcneill 	struct mcx_dmamem mxm;
   6201   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6202   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_in *in;
   6203   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
   6204   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_out *out;
   6205   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
   6206   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6207   1.1  jmcneill 	int error;
   6208   1.1  jmcneill 	int i;
   6209   1.1  jmcneill 	uint8_t *dump;
   6210   1.1  jmcneill 
   6211   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6212   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6213   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6214   1.1  jmcneill 
   6215   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6216   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
   6217   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6218   1.1  jmcneill 
   6219   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6220   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6221   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6222   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6223  1.15  jmcneill 		printf(", unable to allocate "
   6224  1.15  jmcneill 		    "query flow table entry mailboxes\n");
   6225   1.1  jmcneill 		return (-1);
   6226   1.1  jmcneill 	}
   6227   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6228   1.1  jmcneill 
   6229   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6230   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6231  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6232   1.1  jmcneill 	mbin->cmd_flow_index = htobe32(index);
   6233   1.1  jmcneill 
   6234   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6235   1.1  jmcneill 
   6236   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6237   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6238   1.1  jmcneill 	if (error != 0) {
   6239   1.1  jmcneill 		printf("%s: query flow table entry timeout\n", DEVNAME(sc));
   6240   1.1  jmcneill 		goto free;
   6241   1.1  jmcneill 	}
   6242   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6243   1.1  jmcneill 	if (error != 0) {
   6244   1.1  jmcneill 		printf("%s: query flow table entry reply corrupt\n",
   6245   1.1  jmcneill 		    DEVNAME(sc));
   6246   1.1  jmcneill 		goto free;
   6247   1.1  jmcneill 	}
   6248   1.1  jmcneill 
   6249   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6250   1.1  jmcneill 	switch (out->cmd_status) {
   6251   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6252   1.1  jmcneill 		break;
   6253   1.1  jmcneill 	default:
   6254   1.1  jmcneill 		printf("%s: query flow table entry failed (%x/%x)\n",
   6255   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   6256   1.1  jmcneill 		error = -1;
   6257   1.1  jmcneill 		goto free;
   6258   1.1  jmcneill 	}
   6259   1.1  jmcneill 
   6260   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
   6261   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6262   1.1  jmcneill 	dump = (uint8_t *)mbout;
   6263   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6264   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6265   1.1  jmcneill 		if (i % 16 == 15)
   6266   1.1  jmcneill 			printf("\n");
   6267   1.1  jmcneill 	}
   6268   1.1  jmcneill 
   6269   1.1  jmcneill free:
   6270   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6271   1.1  jmcneill 	return (error);
   6272   1.1  jmcneill }
   6273   1.1  jmcneill 
   6274   1.1  jmcneill int
   6275  1.15  jmcneill mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id)
   6276   1.1  jmcneill {
   6277   1.1  jmcneill 	struct mcx_dmamem mxm;
   6278   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6279   1.1  jmcneill 	struct mcx_cmd_query_flow_group_in *in;
   6280   1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_in *mbin;
   6281   1.1  jmcneill 	struct mcx_cmd_query_flow_group_out *out;
   6282   1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_out *mbout;
   6283   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6284   1.1  jmcneill 	int error;
   6285   1.1  jmcneill 	int i;
   6286   1.1  jmcneill 	uint8_t *dump;
   6287   1.1  jmcneill 
   6288   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6289   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6290   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6291   1.1  jmcneill 
   6292   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6293   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
   6294   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6295   1.1  jmcneill 
   6296   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6297   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6298   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6299   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6300   1.1  jmcneill 		printf(", unable to allocate query flow group mailboxes\n");
   6301   1.1  jmcneill 		return (-1);
   6302   1.1  jmcneill 	}
   6303   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6304   1.1  jmcneill 
   6305   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6306   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6307  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6308   1.1  jmcneill 	mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
   6309   1.1  jmcneill 
   6310   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6311   1.1  jmcneill 
   6312   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6313   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6314   1.1  jmcneill 	if (error != 0) {
   6315   1.1  jmcneill 		printf("%s: query flow group timeout\n", DEVNAME(sc));
   6316   1.1  jmcneill 		goto free;
   6317   1.1  jmcneill 	}
   6318   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6319   1.1  jmcneill 	if (error != 0) {
   6320   1.1  jmcneill 		printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
   6321   1.1  jmcneill 		goto free;
   6322   1.1  jmcneill 	}
   6323   1.1  jmcneill 
   6324   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6325   1.1  jmcneill 	switch (out->cmd_status) {
   6326   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6327   1.1  jmcneill 		break;
   6328   1.1  jmcneill 	default:
   6329   1.1  jmcneill 		printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
   6330   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6331   1.1  jmcneill 		error = -1;
   6332   1.1  jmcneill 		goto free;
   6333   1.1  jmcneill 	}
   6334   1.1  jmcneill 
   6335   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_group_mb_out *)
   6336   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6337   1.1  jmcneill 	dump = (uint8_t *)mbout;
   6338   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6339   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6340   1.1  jmcneill 		if (i % 16 == 15)
   6341   1.1  jmcneill 			printf("\n");
   6342   1.1  jmcneill 	}
   6343   1.1  jmcneill 	dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
   6344   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6345   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6346   1.1  jmcneill 		if (i % 16 == 15)
   6347   1.1  jmcneill 			printf("\n");
   6348   1.1  jmcneill 	}
   6349   1.1  jmcneill 
   6350   1.1  jmcneill free:
   6351   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6352   1.1  jmcneill 	return (error);
   6353   1.1  jmcneill }
   6354   1.1  jmcneill 
   6355  1.15  jmcneill static int
   6356  1.15  jmcneill mcx_dump_counters(struct mcx_softc *sc)
   6357  1.15  jmcneill {
   6358  1.15  jmcneill 	struct mcx_dmamem mxm;
   6359  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6360  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_in *in;
   6361  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_mb_in *mbin;
   6362  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_out *out;
   6363  1.15  jmcneill 	struct mcx_nic_vport_counters *counters;
   6364  1.15  jmcneill 	int error, token;
   6365  1.15  jmcneill 
   6366  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6367  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6368  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6369  1.15  jmcneill 	    sizeof(*out) + sizeof(*counters), token);
   6370  1.15  jmcneill 
   6371  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6372  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
   6373  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6374  1.15  jmcneill 
   6375  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   6376  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6377  1.15  jmcneill 		printf(", unable to allocate "
   6378  1.15  jmcneill 		    "query nic vport counters mailboxen\n");
   6379  1.15  jmcneill 		return (-1);
   6380  1.15  jmcneill 	}
   6381  1.15  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6382  1.15  jmcneill 
   6383  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6384  1.15  jmcneill 	mbin->cmd_clear = 0x80;
   6385  1.15  jmcneill 
   6386  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6387  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6388  1.15  jmcneill 
   6389  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6390  1.15  jmcneill 	if (error != 0) {
   6391  1.15  jmcneill 		printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
   6392  1.15  jmcneill 		goto free;
   6393  1.15  jmcneill 	}
   6394  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6395  1.15  jmcneill 		printf("%s: query nic vport counters command corrupt\n",
   6396  1.15  jmcneill 		    DEVNAME(sc));
   6397  1.15  jmcneill 		goto free;
   6398  1.15  jmcneill 	}
   6399  1.15  jmcneill 
   6400  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6401  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6402  1.15  jmcneill 		printf("%s: query nic vport counters failed (%x, %x)\n",
   6403  1.15  jmcneill 		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
   6404  1.15  jmcneill 		error = -1;
   6405  1.15  jmcneill 		goto free;
   6406  1.15  jmcneill 	}
   6407  1.15  jmcneill 
   6408  1.15  jmcneill 	counters = (struct mcx_nic_vport_counters *)
   6409  1.15  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6410  1.15  jmcneill 	if (counters->rx_bcast.packets + counters->tx_bcast.packets +
   6411  1.15  jmcneill 	    counters->rx_ucast.packets + counters->tx_ucast.packets +
   6412  1.15  jmcneill 	    counters->rx_err.packets + counters->tx_err.packets)
   6413  1.15  jmcneill 		printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
   6414  1.15  jmcneill 		    DEVNAME(sc),
   6415  1.15  jmcneill 		    betoh64(counters->tx_err.packets),
   6416  1.15  jmcneill 		    betoh64(counters->rx_err.packets),
   6417  1.15  jmcneill 		    betoh64(counters->tx_ucast.packets),
   6418  1.15  jmcneill 		    betoh64(counters->rx_ucast.packets),
   6419  1.15  jmcneill 		    betoh64(counters->tx_bcast.packets),
   6420  1.15  jmcneill 		    betoh64(counters->rx_bcast.packets));
   6421  1.15  jmcneill free:
   6422  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6423  1.15  jmcneill 
   6424  1.15  jmcneill 	return (error);
   6425  1.15  jmcneill }
   6426  1.15  jmcneill 
   6427  1.15  jmcneill static int
   6428  1.15  jmcneill mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
   6429  1.15  jmcneill {
   6430  1.15  jmcneill 	struct mcx_dmamem mxm;
   6431  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6432  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_in *in;
   6433  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_mb_in *mbin;
   6434  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_out *out;
   6435  1.15  jmcneill 	struct mcx_counter *counters;
   6436  1.15  jmcneill 	int error, token;
   6437  1.15  jmcneill 
   6438  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6439  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6440  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
   6441  1.15  jmcneill 	    sizeof(*counters), token);
   6442  1.15  jmcneill 
   6443  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6444  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
   6445  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6446  1.15  jmcneill 
   6447  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   6448  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6449  1.15  jmcneill 		printf(", unable to allocate query flow counter mailboxen\n");
   6450  1.15  jmcneill 		return (-1);
   6451  1.15  jmcneill 	}
   6452  1.15  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6453  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6454  1.15  jmcneill 	mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
   6455  1.15  jmcneill 	mbin->cmd_clear = 0x80;
   6456  1.15  jmcneill 
   6457  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6458  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6459  1.15  jmcneill 
   6460  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6461  1.15  jmcneill 	if (error != 0) {
   6462  1.15  jmcneill 		printf("%s: query flow counter timeout\n", DEVNAME(sc));
   6463  1.15  jmcneill 		goto free;
   6464  1.15  jmcneill 	}
   6465  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6466  1.15  jmcneill 		printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
   6467  1.15  jmcneill 		goto free;
   6468  1.15  jmcneill 	}
   6469  1.15  jmcneill 
   6470  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6471  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6472  1.15  jmcneill 		printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
   6473  1.15  jmcneill 		    out->cmd_status, betoh32(out->cmd_syndrome));
   6474  1.15  jmcneill 		error = -1;
   6475  1.15  jmcneill 		goto free;
   6476  1.15  jmcneill 	}
   6477  1.15  jmcneill 
   6478  1.15  jmcneill 	counters = (struct mcx_counter *)
   6479  1.15  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6480  1.15  jmcneill 	if (counters->packets)
   6481  1.15  jmcneill 		printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
   6482  1.15  jmcneill 		    betoh64(counters->packets));
   6483  1.15  jmcneill free:
   6484  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6485  1.15  jmcneill 
   6486  1.15  jmcneill 	return (error);
   6487  1.15  jmcneill }
   6488  1.15  jmcneill 
   6489  1.15  jmcneill #endif
   6490  1.15  jmcneill 
   6491  1.15  jmcneill #if NKSTAT > 0
   6492  1.15  jmcneill 
   6493   1.1  jmcneill int
   6494  1.15  jmcneill mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx)
   6495   1.1  jmcneill {
   6496   1.1  jmcneill 	struct mcx_dmamem mxm;
   6497   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6498   1.1  jmcneill 	struct mcx_cmd_query_rq_in *in;
   6499   1.1  jmcneill 	struct mcx_cmd_query_rq_out *out;
   6500   1.1  jmcneill 	struct mcx_cmd_query_rq_mb_out *mbout;
   6501   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6502   1.1  jmcneill 	int error;
   6503   1.1  jmcneill 
   6504   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6505   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   6506   1.1  jmcneill 	    token);
   6507   1.1  jmcneill 
   6508   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6509   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
   6510   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6511  1.15  jmcneill 	in->cmd_rqn = htobe32(rx->rx_rqn);
   6512   1.1  jmcneill 
   6513   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6514   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6515   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6516  1.15  jmcneill 		printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc));
   6517   1.1  jmcneill 		return (-1);
   6518   1.1  jmcneill 	}
   6519   1.1  jmcneill 
   6520   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6521   1.1  jmcneill 
   6522   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6523   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6524   1.1  jmcneill 	if (error != 0) {
   6525   1.1  jmcneill 		printf("%s: query rq timeout\n", DEVNAME(sc));
   6526   1.1  jmcneill 		goto free;
   6527   1.1  jmcneill 	}
   6528   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6529   1.1  jmcneill 	if (error != 0) {
   6530   1.1  jmcneill 		printf("%s: query rq reply corrupt\n", DEVNAME(sc));
   6531   1.1  jmcneill 		goto free;
   6532   1.1  jmcneill 	}
   6533   1.1  jmcneill 
   6534   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6535   1.1  jmcneill 	switch (out->cmd_status) {
   6536   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6537   1.1  jmcneill 		break;
   6538   1.1  jmcneill 	default:
   6539   1.1  jmcneill 		printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
   6540   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6541   1.1  jmcneill 		error = -1;
   6542   1.1  jmcneill 		goto free;
   6543   1.1  jmcneill 	}
   6544   1.1  jmcneill 
   6545   1.1  jmcneill         mbout = (struct mcx_cmd_query_rq_mb_out *)
   6546   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6547  1.15  jmcneill 	memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx));
   6548   1.1  jmcneill 
   6549   1.1  jmcneill free:
   6550   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6551   1.1  jmcneill 	return (error);
   6552   1.1  jmcneill }
   6553   1.1  jmcneill 
   6554   1.1  jmcneill int
   6555  1.15  jmcneill mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx)
   6556   1.1  jmcneill {
   6557   1.1  jmcneill 	struct mcx_dmamem mxm;
   6558   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6559   1.1  jmcneill 	struct mcx_cmd_query_sq_in *in;
   6560   1.1  jmcneill 	struct mcx_cmd_query_sq_out *out;
   6561   1.1  jmcneill 	struct mcx_cmd_query_sq_mb_out *mbout;
   6562   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6563   1.1  jmcneill 	int error;
   6564   1.1  jmcneill 
   6565   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6566   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   6567   1.1  jmcneill 	    token);
   6568   1.1  jmcneill 
   6569   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6570   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
   6571   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6572  1.15  jmcneill 	in->cmd_sqn = htobe32(tx->tx_sqn);
   6573   1.1  jmcneill 
   6574   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6575   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6576   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6577  1.15  jmcneill 		printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc));
   6578   1.1  jmcneill 		return (-1);
   6579   1.1  jmcneill 	}
   6580   1.1  jmcneill 
   6581   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6582   1.1  jmcneill 
   6583   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6584   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6585   1.1  jmcneill 	if (error != 0) {
   6586   1.1  jmcneill 		printf("%s: query sq timeout\n", DEVNAME(sc));
   6587   1.1  jmcneill 		goto free;
   6588   1.1  jmcneill 	}
   6589   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6590   1.1  jmcneill 	if (error != 0) {
   6591   1.1  jmcneill 		printf("%s: query sq reply corrupt\n", DEVNAME(sc));
   6592   1.1  jmcneill 		goto free;
   6593   1.1  jmcneill 	}
   6594   1.1  jmcneill 
   6595   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6596   1.1  jmcneill 	switch (out->cmd_status) {
   6597   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6598   1.1  jmcneill 		break;
   6599   1.1  jmcneill 	default:
   6600   1.1  jmcneill 		printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
   6601   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6602   1.1  jmcneill 		error = -1;
   6603   1.1  jmcneill 		goto free;
   6604   1.1  jmcneill 	}
   6605   1.1  jmcneill 
   6606   1.1  jmcneill         mbout = (struct mcx_cmd_query_sq_mb_out *)
   6607   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6608  1.15  jmcneill 	memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx));
   6609   1.1  jmcneill 
   6610   1.1  jmcneill free:
   6611   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6612   1.1  jmcneill 	return (error);
   6613   1.1  jmcneill }
   6614   1.1  jmcneill 
   6615  1.15  jmcneill int
   6616  1.15  jmcneill mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx)
   6617   1.1  jmcneill {
   6618   1.1  jmcneill 	struct mcx_dmamem mxm;
   6619   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6620  1.15  jmcneill 	struct mcx_cmd_query_cq_in *in;
   6621  1.15  jmcneill 	struct mcx_cmd_query_cq_out *out;
   6622  1.15  jmcneill 	struct mcx_cq_ctx *ctx;
   6623  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6624  1.15  jmcneill 	int error;
   6625   1.1  jmcneill 
   6626   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6627  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
   6628  1.15  jmcneill 	    token);
   6629   1.1  jmcneill 
   6630   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6631  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ);
   6632   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6633  1.15  jmcneill 	in->cmd_cqn = htobe32(cq->cq_n);
   6634   1.1  jmcneill 
   6635  1.15  jmcneill 	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6636  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6637  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6638  1.15  jmcneill 		printf("%s: unable to allocate query cq mailboxes\n",
   6639  1.15  jmcneill 		    DEVNAME(sc));
   6640   1.1  jmcneill 		return (-1);
   6641   1.1  jmcneill 	}
   6642   1.1  jmcneill 
   6643  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6644   1.1  jmcneill 
   6645   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6646   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6647   1.1  jmcneill 	if (error != 0) {
   6648  1.15  jmcneill 		printf("%s: query cq timeout\n", DEVNAME(sc));
   6649   1.1  jmcneill 		goto free;
   6650   1.1  jmcneill 	}
   6651   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6652  1.15  jmcneill 		printf("%s: query cq reply corrupt\n", DEVNAME(sc));
   6653   1.1  jmcneill 		goto free;
   6654   1.1  jmcneill 	}
   6655   1.1  jmcneill 
   6656   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6657  1.15  jmcneill 	switch (out->cmd_status) {
   6658  1.15  jmcneill 	case MCX_CQ_STATUS_OK:
   6659  1.15  jmcneill 		break;
   6660  1.15  jmcneill 	default:
   6661  1.15  jmcneill 		printf("%s: query qc failed (%x/%x)\n", DEVNAME(sc),
   6662  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6663   1.1  jmcneill 		error = -1;
   6664   1.1  jmcneill 		goto free;
   6665   1.1  jmcneill 	}
   6666   1.1  jmcneill 
   6667  1.15  jmcneill 	ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6668  1.15  jmcneill 	memcpy(cq_ctx, ctx, sizeof(*cq_ctx));
   6669   1.1  jmcneill free:
   6670   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6671   1.1  jmcneill 	return (error);
   6672   1.1  jmcneill }
   6673   1.1  jmcneill 
   6674  1.15  jmcneill int
   6675  1.15  jmcneill mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx)
   6676   1.1  jmcneill {
   6677   1.1  jmcneill 	struct mcx_dmamem mxm;
   6678   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6679  1.15  jmcneill 	struct mcx_cmd_query_eq_in *in;
   6680  1.15  jmcneill 	struct mcx_cmd_query_eq_out *out;
   6681  1.15  jmcneill 	struct mcq_eq_ctx *ctx;
   6682  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6683  1.15  jmcneill 	int error;
   6684   1.1  jmcneill 
   6685   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6686  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
   6687  1.15  jmcneill 	    token);
   6688   1.1  jmcneill 
   6689   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6690  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ);
   6691   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6692  1.15  jmcneill 	in->cmd_eqn = htobe32(eq->eq_n);
   6693   1.1  jmcneill 
   6694  1.15  jmcneill 	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6695  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6696  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6697  1.15  jmcneill 		printf("%s: unable to allocate query eq mailboxes\n",
   6698  1.15  jmcneill 		    DEVNAME(sc));
   6699   1.1  jmcneill 		return (-1);
   6700   1.1  jmcneill 	}
   6701   1.1  jmcneill 
   6702   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6703  1.15  jmcneill 
   6704   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6705   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6706   1.1  jmcneill 	if (error != 0) {
   6707  1.15  jmcneill 		printf("%s: query eq timeout\n", DEVNAME(sc));
   6708   1.1  jmcneill 		goto free;
   6709   1.1  jmcneill 	}
   6710   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6711  1.15  jmcneill 		printf("%s: query eq reply corrupt\n", DEVNAME(sc));
   6712   1.1  jmcneill 		goto free;
   6713   1.1  jmcneill 	}
   6714   1.1  jmcneill 
   6715   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6716  1.15  jmcneill 	switch (out->cmd_status) {
   6717  1.15  jmcneill 	case MCX_CQ_STATUS_OK:
   6718  1.15  jmcneill 		break;
   6719  1.15  jmcneill 	default:
   6720  1.15  jmcneill 		printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc),
   6721  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6722   1.1  jmcneill 		error = -1;
   6723   1.1  jmcneill 		goto free;
   6724   1.1  jmcneill 	}
   6725   1.1  jmcneill 
   6726  1.15  jmcneill 	ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6727  1.15  jmcneill 	memcpy(eq_ctx, ctx, sizeof(*eq_ctx));
   6728   1.1  jmcneill free:
   6729   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6730   1.1  jmcneill 	return (error);
   6731   1.1  jmcneill }
   6732   1.1  jmcneill 
   6733  1.15  jmcneill #endif /* NKSTAT > 0 */
   6734  1.15  jmcneill 
   6735   1.1  jmcneill 
   6736  1.15  jmcneill static inline unsigned int
   6737  1.15  jmcneill mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots)
   6738   1.1  jmcneill {
   6739  1.15  jmcneill 	struct mcx_rq_entry *ring, *rqe;
   6740   1.1  jmcneill 	struct mcx_slot *ms;
   6741   1.1  jmcneill 	struct mbuf *m;
   6742   1.1  jmcneill 	uint slot, p, fills;
   6743   1.1  jmcneill 
   6744  1.15  jmcneill 	ring = MCX_DMA_KVA(&rx->rx_rq_mem);
   6745  1.15  jmcneill 	p = rx->rx_prod;
   6746  1.15  jmcneill 
   6747  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   6748  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   6749  1.15  jmcneill 
   6750   1.1  jmcneill 	slot = (p % (1 << MCX_LOG_RQ_SIZE));
   6751   1.1  jmcneill 	rqe = ring;
   6752   1.1  jmcneill 	for (fills = 0; fills < nslots; fills++) {
   6753  1.15  jmcneill 		slot = p % (1 << MCX_LOG_RQ_SIZE);
   6754  1.15  jmcneill 
   6755  1.15  jmcneill 		ms = &rx->rx_slots[slot];
   6756  1.15  jmcneill 		rqe = &ring[slot];
   6757  1.15  jmcneill 
   6758   1.1  jmcneill 		m = NULL;
   6759   1.1  jmcneill 		MGETHDR(m, M_DONTWAIT, MT_DATA);
   6760   1.1  jmcneill 		if (m == NULL)
   6761   1.1  jmcneill 			break;
   6762   1.1  jmcneill 
   6763   1.1  jmcneill 		MCLGET(m, M_DONTWAIT);
   6764   1.1  jmcneill 		if ((m->m_flags & M_EXT) == 0) {
   6765   1.1  jmcneill 			m_freem(m);
   6766   1.1  jmcneill 			break;
   6767   1.1  jmcneill 		}
   6768   1.1  jmcneill 
   6769  1.15  jmcneill 		m->m_len = m->m_pkthdr.len = sc->sc_hardmtu;
   6770  1.15  jmcneill 		m_adj(m, m->m_ext.ext_size - sc->sc_rxbufsz);
   6771  1.15  jmcneill 		m_adj(m, ETHER_ALIGN);
   6772  1.15  jmcneill 
   6773   1.1  jmcneill 		if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   6774   1.1  jmcneill 		    BUS_DMA_NOWAIT) != 0) {
   6775   1.1  jmcneill 			m_freem(m);
   6776   1.1  jmcneill 			break;
   6777   1.1  jmcneill 		}
   6778   1.3       tnn 		bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, BUS_DMASYNC_PREREAD);
   6779   1.1  jmcneill 		ms->ms_m = m;
   6780   1.1  jmcneill 
   6781  1.15  jmcneill 		be32enc(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len);
   6782  1.15  jmcneill 		be64enc(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr);
   6783  1.15  jmcneill 		be32enc(&rqe->rqe_lkey, sc->sc_lkey);
   6784   1.1  jmcneill 
   6785   1.1  jmcneill 		p++;
   6786   1.1  jmcneill 	}
   6787   1.1  jmcneill 
   6788  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   6789  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
   6790  1.15  jmcneill 
   6791  1.15  jmcneill 	rx->rx_prod = p;
   6792   1.1  jmcneill 
   6793  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   6794  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   6795  1.15  jmcneill 	be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell),
   6796  1.15  jmcneill 	    p & MCX_WQ_DOORBELL_MASK);
   6797  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   6798  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   6799   1.1  jmcneill 
   6800   1.1  jmcneill 	return (nslots - fills);
   6801   1.1  jmcneill }
   6802   1.1  jmcneill 
   6803   1.1  jmcneill static int
   6804  1.15  jmcneill mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx)
   6805   1.1  jmcneill {
   6806   1.1  jmcneill 	u_int slots;
   6807   1.1  jmcneill 
   6808  1.15  jmcneill 	slots = mcx_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE));
   6809   1.1  jmcneill 	if (slots == 0)
   6810   1.1  jmcneill 		return (1);
   6811   1.1  jmcneill 
   6812  1.15  jmcneill 	slots = mcx_rx_fill_slots(sc, rx, slots);
   6813  1.15  jmcneill 	mcx_rxr_put(&rx->rx_rxr, slots);
   6814   1.1  jmcneill 	return (0);
   6815   1.1  jmcneill }
   6816   1.1  jmcneill 
   6817   1.1  jmcneill void
   6818  1.15  jmcneill mcx_refill(void *xrx)
   6819   1.1  jmcneill {
   6820  1.15  jmcneill 	struct mcx_rx *rx = xrx;
   6821  1.15  jmcneill 	struct mcx_softc *sc = rx->rx_softc;
   6822   1.1  jmcneill 
   6823  1.15  jmcneill 	mcx_rx_fill(sc, rx);
   6824   1.1  jmcneill 
   6825  1.15  jmcneill 	if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
   6826  1.15  jmcneill 		callout_schedule(&rx->rx_refill, 1);
   6827   1.1  jmcneill }
   6828   1.1  jmcneill 
   6829  1.15  jmcneill static int
   6830  1.15  jmcneill mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx,
   6831  1.15  jmcneill     struct mcx_cq_entry *cqe)
   6832   1.1  jmcneill {
   6833   1.1  jmcneill 	struct mcx_slot *ms;
   6834   1.1  jmcneill 	bus_dmamap_t map;
   6835   1.1  jmcneill 	int slot, slots;
   6836   1.1  jmcneill 
   6837   1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
   6838   1.1  jmcneill 
   6839  1.15  jmcneill 	ms = &tx->tx_slots[slot];
   6840   1.1  jmcneill 	map = ms->ms_map;
   6841   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   6842   1.1  jmcneill 	    BUS_DMASYNC_POSTWRITE);
   6843   1.1  jmcneill 
   6844   1.1  jmcneill 	slots = 1;
   6845   1.1  jmcneill 	if (map->dm_nsegs > 1)
   6846   1.1  jmcneill 		slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;
   6847   1.1  jmcneill 
   6848   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, map);
   6849   1.1  jmcneill 	m_freem(ms->ms_m);
   6850   1.1  jmcneill 	ms->ms_m = NULL;
   6851  1.15  jmcneill 
   6852  1.15  jmcneill 	return (slots);
   6853   1.1  jmcneill }
   6854   1.1  jmcneill 
   6855   1.1  jmcneill static uint64_t
   6856   1.1  jmcneill mcx_uptime(void)
   6857   1.1  jmcneill {
   6858   1.1  jmcneill 	struct timespec ts;
   6859   1.1  jmcneill 
   6860   1.1  jmcneill 	nanouptime(&ts);
   6861   1.1  jmcneill 
   6862   1.1  jmcneill 	return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec);
   6863   1.1  jmcneill }
   6864   1.1  jmcneill 
   6865   1.1  jmcneill static void
   6866   1.1  jmcneill mcx_calibrate_first(struct mcx_softc *sc)
   6867   1.1  jmcneill {
   6868   1.1  jmcneill 	struct mcx_calibration *c = &sc->sc_calibration[0];
   6869  1.15  jmcneill 	int s;
   6870   1.1  jmcneill 
   6871   1.1  jmcneill 	sc->sc_calibration_gen = 0;
   6872   1.1  jmcneill 
   6873  1.15  jmcneill 	s = splhigh(); /* crit_enter? */
   6874   1.1  jmcneill 	c->c_ubase = mcx_uptime();
   6875   1.1  jmcneill 	c->c_tbase = mcx_timer(sc);
   6876  1.15  jmcneill 	splx(s);
   6877  1.15  jmcneill 	c->c_ratio = 0;
   6878   1.1  jmcneill 
   6879  1.15  jmcneill #if notyet
   6880   1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz);
   6881  1.15  jmcneill #endif
   6882   1.1  jmcneill }
   6883   1.1  jmcneill 
   6884  1.15  jmcneill #define MCX_TIMESTAMP_SHIFT 24
   6885   1.1  jmcneill 
   6886   1.1  jmcneill static void
   6887   1.1  jmcneill mcx_calibrate(void *arg)
   6888   1.1  jmcneill {
   6889   1.1  jmcneill 	struct mcx_softc *sc = arg;
   6890   1.1  jmcneill 	struct mcx_calibration *nc, *pc;
   6891  1.15  jmcneill 	uint64_t udiff, tdiff;
   6892   1.1  jmcneill 	unsigned int gen;
   6893  1.15  jmcneill 	int s;
   6894   1.1  jmcneill 
   6895   1.1  jmcneill 	if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING))
   6896   1.1  jmcneill 		return;
   6897   1.1  jmcneill 
   6898   1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz);
   6899   1.1  jmcneill 
   6900   1.1  jmcneill 	gen = sc->sc_calibration_gen;
   6901   1.1  jmcneill 	pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   6902   1.1  jmcneill 	gen++;
   6903   1.1  jmcneill 	nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   6904   1.1  jmcneill 
   6905   1.1  jmcneill 	nc->c_uptime = pc->c_ubase;
   6906   1.1  jmcneill 	nc->c_timestamp = pc->c_tbase;
   6907   1.1  jmcneill 
   6908  1.15  jmcneill 	s = splhigh(); /* crit_enter? */
   6909   1.1  jmcneill 	nc->c_ubase = mcx_uptime();
   6910   1.1  jmcneill 	nc->c_tbase = mcx_timer(sc);
   6911  1.15  jmcneill 	splx(s);
   6912   1.1  jmcneill 
   6913  1.15  jmcneill 	udiff = nc->c_ubase - nc->c_uptime;
   6914  1.15  jmcneill 	tdiff = nc->c_tbase - nc->c_timestamp;
   6915  1.15  jmcneill 
   6916  1.15  jmcneill 	/*
   6917  1.15  jmcneill 	 * udiff is the wall clock time between calibration ticks,
   6918  1.15  jmcneill 	 * which should be 32 seconds or 32 billion nanoseconds. if
   6919  1.15  jmcneill 	 * we squint, 1 billion nanoseconds is kind of like a 32 bit
   6920  1.15  jmcneill 	 * number, so 32 billion should still have a lot of high bits
   6921  1.15  jmcneill 	 * spare. we use this space by shifting the nanoseconds up
   6922  1.15  jmcneill 	 * 24 bits so we have a nice big number to divide by the
   6923  1.15  jmcneill 	 * number of mcx timer ticks.
   6924  1.15  jmcneill 	 */
   6925  1.15  jmcneill 	nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff;
   6926   1.1  jmcneill 
   6927   1.1  jmcneill 	membar_producer();
   6928   1.1  jmcneill 	sc->sc_calibration_gen = gen;
   6929   1.1  jmcneill }
   6930   1.1  jmcneill 
   6931   1.1  jmcneill static int
   6932  1.15  jmcneill mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx,
   6933  1.15  jmcneill     struct mcx_cq_entry *cqe, struct mcx_mbufq *mq,
   6934  1.15  jmcneill     const struct mcx_calibration *c)
   6935   1.1  jmcneill {
   6936  1.15  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   6937   1.1  jmcneill 	struct mcx_slot *ms;
   6938   1.1  jmcneill 	struct mbuf *m;
   6939  1.20  jmcneill 	uint32_t flags, len;
   6940   1.1  jmcneill 	int slot;
   6941   1.1  jmcneill 
   6942  1.20  jmcneill 	len = be32dec(&cqe->cq_byte_cnt);
   6943   1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
   6944   1.1  jmcneill 
   6945  1.15  jmcneill 	ms = &rx->rx_slots[slot];
   6946  1.20  jmcneill 	bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, len, BUS_DMASYNC_POSTREAD);
   6947   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
   6948   1.1  jmcneill 
   6949   1.1  jmcneill 	m = ms->ms_m;
   6950   1.1  jmcneill 	ms->ms_m = NULL;
   6951   1.1  jmcneill 
   6952   1.1  jmcneill 	m_set_rcvif(m, &sc->sc_ec.ec_if);
   6953  1.20  jmcneill 	m->m_pkthdr.len = m->m_len = len;
   6954   1.1  jmcneill 
   6955   1.1  jmcneill #if 0
   6956   1.1  jmcneill 	if (cqe->cq_rx_hash_type) {
   6957  1.15  jmcneill 		m->m_pkthdr.ph_flowid = be32toh(cqe->cq_rx_hash);
   6958  1.15  jmcneill 		m->m_pkthdr.csum_flags |= M_FLOWID;
   6959   1.1  jmcneill 	}
   6960   1.1  jmcneill #endif
   6961   1.1  jmcneill 
   6962  1.15  jmcneill 	flags = be32dec(&cqe->cq_flags);
   6963  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) {
   6964  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx)
   6965  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
   6966  1.15  jmcneill 	}
   6967  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) {
   6968  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx)
   6969  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
   6970  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx)
   6971  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_TCPv6;
   6972  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx)
   6973  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
   6974  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx)
   6975  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_UDPv6;
   6976  1.15  jmcneill 	}
   6977  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_CV) {
   6978  1.15  jmcneill 		vlan_set_tag(m, flags & MCX_CQ_ENTRY_FLAGS_VLAN_MASK);
   6979  1.15  jmcneill 	}
   6980  1.15  jmcneill 
   6981  1.15  jmcneill #if notyet
   6982  1.15  jmcneill 	if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_LINK0) && c->c_ratio) {
   6983  1.15  jmcneill 		uint64_t t = be64dec(&cqe->cq_timestamp);
   6984  1.15  jmcneill 		t -= c->c_timestamp;
   6985  1.15  jmcneill 		t *= c->c_ratio;
   6986  1.15  jmcneill 		t >>= MCX_TIMESTAMP_SHIFT;
   6987  1.15  jmcneill 		t += c->c_uptime;
   6988   1.1  jmcneill 
   6989  1.15  jmcneill 		m->m_pkthdr.ph_timestamp = t;
   6990   1.1  jmcneill 		SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
   6991   1.1  jmcneill 	}
   6992   1.1  jmcneill #endif
   6993   1.1  jmcneill 
   6994   1.1  jmcneill 	MBUFQ_ENQUEUE(mq, m);
   6995   1.1  jmcneill 
   6996   1.1  jmcneill 	return (1);
   6997   1.1  jmcneill }
   6998   1.1  jmcneill 
   6999   1.1  jmcneill static struct mcx_cq_entry *
   7000   1.1  jmcneill mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
   7001   1.1  jmcneill {
   7002   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   7003   1.1  jmcneill 	int next;
   7004   1.1  jmcneill 
   7005   1.1  jmcneill 	cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
   7006   1.1  jmcneill 	next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
   7007   1.1  jmcneill 
   7008   1.1  jmcneill 	if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
   7009   1.1  jmcneill 	    ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
   7010   1.1  jmcneill 		return (&cqe[next]);
   7011   1.1  jmcneill 	}
   7012   1.1  jmcneill 
   7013   1.1  jmcneill 	return (NULL);
   7014   1.1  jmcneill }
   7015   1.1  jmcneill 
   7016   1.1  jmcneill static void
   7017  1.15  jmcneill mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar)
   7018   1.1  jmcneill {
   7019  1.15  jmcneill 	struct mcx_cq_doorbell *db;
   7020   1.1  jmcneill 	bus_size_t offset;
   7021   1.1  jmcneill 	uint32_t val;
   7022   1.1  jmcneill 	uint64_t uval;
   7023   1.1  jmcneill 
   7024   1.1  jmcneill 	val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
   7025   1.1  jmcneill 	val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   7026   1.1  jmcneill 
   7027  1.15  jmcneill 	db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell);
   7028  1.15  jmcneill 
   7029  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7030  1.15  jmcneill 	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE);
   7031   1.1  jmcneill 
   7032  1.15  jmcneill 	be32enc(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   7033  1.15  jmcneill 	be32enc(&db->db_arm_ci, val);
   7034  1.15  jmcneill 
   7035  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7036  1.15  jmcneill 	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE);
   7037  1.15  jmcneill 
   7038  1.15  jmcneill 	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL;
   7039  1.15  jmcneill 
   7040  1.15  jmcneill 	uval = (uint64_t)val << 32;
   7041   1.1  jmcneill 	uval |= cq->cq_n;
   7042  1.15  jmcneill 
   7043  1.15  jmcneill 	bus_space_write_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval));
   7044  1.15  jmcneill 	mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE);
   7045   1.1  jmcneill }
   7046   1.1  jmcneill 
   7047   1.1  jmcneill void
   7048  1.15  jmcneill mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq)
   7049   1.1  jmcneill {
   7050  1.15  jmcneill 	struct mcx_rx *rx = &q->q_rx;
   7051  1.15  jmcneill 	struct mcx_tx *tx = &q->q_tx;
   7052   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   7053   1.1  jmcneill 	const struct mcx_calibration *c;
   7054   1.1  jmcneill 	unsigned int gen;
   7055   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   7056   1.1  jmcneill 	struct mcx_mbufq mq;
   7057   1.1  jmcneill 	struct mbuf *m;
   7058   1.1  jmcneill 	int rxfree, txfree;
   7059   1.1  jmcneill 
   7060   1.1  jmcneill 	MBUFQ_INIT(&mq);
   7061   1.1  jmcneill 
   7062   1.1  jmcneill 	gen = sc->sc_calibration_gen;
   7063   1.1  jmcneill 	membar_consumer();
   7064   1.1  jmcneill 	c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   7065   1.1  jmcneill 
   7066  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   7067  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   7068  1.15  jmcneill 
   7069   1.1  jmcneill 	rxfree = 0;
   7070   1.1  jmcneill 	txfree = 0;
   7071   1.1  jmcneill 	while ((cqe = mcx_next_cq_entry(sc, cq))) {
   7072   1.1  jmcneill 		uint8_t opcode;
   7073   1.1  jmcneill 		opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
   7074   1.1  jmcneill 		switch (opcode) {
   7075   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ:
   7076  1.15  jmcneill 			txfree += mcx_process_txeof(sc, tx, cqe);
   7077   1.1  jmcneill 			break;
   7078   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND:
   7079  1.15  jmcneill 			rxfree += mcx_process_rx(sc, rx, cqe, &mq, c);
   7080   1.1  jmcneill 			break;
   7081   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
   7082   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
   7083   1.1  jmcneill 			/* uint8_t *cqp = (uint8_t *)cqe; */
   7084  1.15  jmcneill 			/* printf("%s: cq completion error: %x\n",
   7085  1.15  jmcneill 			    DEVNAME(sc), cqp[0x37]); */
   7086   1.1  jmcneill 			break;
   7087   1.1  jmcneill 
   7088   1.1  jmcneill 		default:
   7089  1.15  jmcneill 			/* printf("%s: cq completion opcode %x??\n",
   7090  1.15  jmcneill 			    DEVNAME(sc), opcode); */
   7091   1.1  jmcneill 			break;
   7092   1.1  jmcneill 		}
   7093   1.1  jmcneill 
   7094   1.1  jmcneill 		cq->cq_cons++;
   7095   1.1  jmcneill 	}
   7096   1.1  jmcneill 
   7097  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   7098  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
   7099  1.15  jmcneill 
   7100   1.1  jmcneill 	if (rxfree > 0) {
   7101  1.15  jmcneill 		mcx_rxr_put(&rx->rx_rxr, rxfree);
   7102   1.1  jmcneill 		while (MBUFQ_FIRST(&mq) != NULL) {
   7103   1.1  jmcneill 			MBUFQ_DEQUEUE(&mq, m);
   7104   1.1  jmcneill 			if_percpuq_enqueue(ifp->if_percpuq, m);
   7105   1.1  jmcneill 		}
   7106   1.1  jmcneill 
   7107  1.15  jmcneill 		mcx_rx_fill(sc, rx);
   7108  1.15  jmcneill 		if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
   7109  1.15  jmcneill 			callout_schedule(&rx->rx_refill, 1);
   7110   1.1  jmcneill 	}
   7111  1.17  jmcneill 
   7112  1.17  jmcneill 	cq->cq_count++;
   7113  1.17  jmcneill 	mcx_arm_cq(sc, cq, q->q_uar);
   7114  1.17  jmcneill 
   7115   1.1  jmcneill 	if (txfree > 0) {
   7116  1.15  jmcneill 		tx->tx_cons += txfree;
   7117   1.1  jmcneill 		if_schedule_deferred_start(ifp);
   7118   1.1  jmcneill 	}
   7119   1.1  jmcneill }
   7120   1.1  jmcneill 
   7121  1.15  jmcneill 
   7122   1.1  jmcneill static void
   7123  1.15  jmcneill mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar)
   7124   1.1  jmcneill {
   7125   1.1  jmcneill 	bus_size_t offset;
   7126   1.1  jmcneill 	uint32_t val;
   7127   1.1  jmcneill 
   7128  1.15  jmcneill 	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM;
   7129  1.15  jmcneill 	val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff);
   7130   1.1  jmcneill 
   7131   1.1  jmcneill 	mcx_wr(sc, offset, val);
   7132  1.15  jmcneill 	mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE);
   7133   1.1  jmcneill }
   7134   1.1  jmcneill 
   7135   1.1  jmcneill static struct mcx_eq_entry *
   7136  1.15  jmcneill mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq)
   7137   1.1  jmcneill {
   7138   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   7139   1.1  jmcneill 	int next;
   7140   1.1  jmcneill 
   7141  1.15  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
   7142  1.15  jmcneill 	next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE);
   7143  1.15  jmcneill 	if ((eqe[next].eq_owner & 1) ==
   7144  1.15  jmcneill 	    ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
   7145  1.15  jmcneill 		eq->eq_cons++;
   7146   1.1  jmcneill 		return (&eqe[next]);
   7147   1.1  jmcneill 	}
   7148   1.1  jmcneill 	return (NULL);
   7149   1.1  jmcneill }
   7150   1.1  jmcneill 
   7151   1.1  jmcneill int
   7152  1.15  jmcneill mcx_admin_intr(void *xsc)
   7153   1.1  jmcneill {
   7154   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)xsc;
   7155  1.15  jmcneill 	struct mcx_eq *eq = &sc->sc_admin_eq;
   7156   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   7157   1.1  jmcneill 
   7158  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7159  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   7160  1.15  jmcneill 
   7161  1.15  jmcneill 	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
   7162   1.1  jmcneill 		switch (eqe->eq_event_type) {
   7163   1.1  jmcneill 		case MCX_EVENT_TYPE_LAST_WQE:
   7164   1.1  jmcneill 			/* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
   7165   1.1  jmcneill 			break;
   7166   1.1  jmcneill 
   7167   1.1  jmcneill 		case MCX_EVENT_TYPE_CQ_ERROR:
   7168   1.1  jmcneill 			/* printf("%s: cq error\n", DEVNAME(sc)); */
   7169   1.1  jmcneill 			break;
   7170   1.1  jmcneill 
   7171   1.1  jmcneill 		case MCX_EVENT_TYPE_CMD_COMPLETION:
   7172   1.1  jmcneill 			/* wakeup probably */
   7173   1.1  jmcneill 			break;
   7174   1.1  jmcneill 
   7175   1.1  jmcneill 		case MCX_EVENT_TYPE_PORT_CHANGE:
   7176   1.1  jmcneill 			workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL);
   7177   1.1  jmcneill 			break;
   7178   1.1  jmcneill 
   7179   1.1  jmcneill 		default:
   7180   1.1  jmcneill 			/* printf("%s: something happened\n", DEVNAME(sc)); */
   7181   1.1  jmcneill 			break;
   7182   1.1  jmcneill 		}
   7183   1.1  jmcneill 	}
   7184  1.15  jmcneill 
   7185  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7186  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   7187  1.15  jmcneill 
   7188  1.15  jmcneill 	mcx_arm_eq(sc, eq, sc->sc_uar);
   7189  1.15  jmcneill 
   7190  1.15  jmcneill 	return (1);
   7191  1.15  jmcneill }
   7192  1.15  jmcneill 
   7193  1.15  jmcneill int
   7194  1.15  jmcneill mcx_cq_intr(void *xq)
   7195  1.15  jmcneill {
   7196  1.15  jmcneill 	struct mcx_queues *q = (struct mcx_queues *)xq;
   7197  1.15  jmcneill 	struct mcx_softc *sc = q->q_sc;
   7198  1.15  jmcneill 	struct mcx_eq *eq = &q->q_eq;
   7199  1.15  jmcneill 	struct mcx_eq_entry *eqe;
   7200  1.15  jmcneill 	int cqn;
   7201  1.15  jmcneill 
   7202  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7203  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   7204  1.15  jmcneill 
   7205  1.15  jmcneill 	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
   7206  1.15  jmcneill 		switch (eqe->eq_event_type) {
   7207  1.15  jmcneill 		case MCX_EVENT_TYPE_COMPLETION:
   7208  1.15  jmcneill 			cqn = be32toh(eqe->eq_event_data[6]);
   7209  1.15  jmcneill 			if (cqn == q->q_cq.cq_n)
   7210  1.15  jmcneill 				mcx_process_cq(sc, q, &q->q_cq);
   7211  1.15  jmcneill 			break;
   7212  1.15  jmcneill 		}
   7213  1.15  jmcneill 	}
   7214  1.15  jmcneill 
   7215  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7216  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   7217  1.15  jmcneill 
   7218  1.15  jmcneill 	mcx_arm_eq(sc, eq, q->q_uar);
   7219  1.15  jmcneill 
   7220   1.1  jmcneill 	return (1);
   7221   1.1  jmcneill }
   7222   1.1  jmcneill 
   7223   1.1  jmcneill static void
   7224   1.1  jmcneill mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
   7225   1.1  jmcneill     int total)
   7226   1.1  jmcneill {
   7227   1.1  jmcneill 	struct mcx_slot *ms;
   7228   1.1  jmcneill 
   7229   1.1  jmcneill 	int i = allocated;
   7230   1.1  jmcneill 	while (i-- > 0) {
   7231   1.1  jmcneill 		ms = &slots[i];
   7232   1.1  jmcneill 		bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
   7233  1.28       rin 		m_freem(ms->ms_m);
   7234   1.1  jmcneill 	}
   7235   1.1  jmcneill 	kmem_free(slots, total * sizeof(*ms));
   7236   1.1  jmcneill }
   7237   1.1  jmcneill 
   7238   1.1  jmcneill static int
   7239  1.15  jmcneill mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q)
   7240   1.1  jmcneill {
   7241  1.15  jmcneill 	struct mcx_rx *rx;
   7242  1.15  jmcneill 	struct mcx_tx *tx;
   7243   1.1  jmcneill 	struct mcx_slot *ms;
   7244  1.15  jmcneill 	int i;
   7245   1.1  jmcneill 
   7246  1.15  jmcneill 	rx = &q->q_rx;
   7247  1.15  jmcneill 	rx->rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE),
   7248   1.1  jmcneill 	    KM_SLEEP);
   7249   1.1  jmcneill 
   7250   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
   7251  1.15  jmcneill 		ms = &rx->rx_slots[i];
   7252   1.1  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
   7253   1.1  jmcneill 		    sc->sc_hardmtu, 0,
   7254   1.1  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   7255   1.1  jmcneill 		    &ms->ms_map) != 0) {
   7256   1.1  jmcneill 			printf("%s: failed to allocate rx dma maps\n",
   7257   1.1  jmcneill 			    DEVNAME(sc));
   7258   1.1  jmcneill 			goto destroy_rx_slots;
   7259   1.1  jmcneill 		}
   7260  1.15  jmcneill 	}
   7261  1.15  jmcneill 
   7262  1.15  jmcneill 	tx = &q->q_tx;
   7263  1.15  jmcneill 	tx->tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE),
   7264  1.15  jmcneill 	     KM_SLEEP);
   7265  1.15  jmcneill 
   7266  1.15  jmcneill 	for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
   7267  1.15  jmcneill 		ms = &tx->tx_slots[i];
   7268  1.15  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
   7269  1.15  jmcneill 		    MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
   7270  1.15  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   7271  1.15  jmcneill 		    &ms->ms_map) != 0) {
   7272  1.15  jmcneill 			printf("%s: failed to allocate tx dma maps\n",
   7273  1.15  jmcneill 			    DEVNAME(sc));
   7274  1.15  jmcneill 			goto destroy_tx_slots;
   7275  1.15  jmcneill 		}
   7276  1.15  jmcneill 	}
   7277  1.15  jmcneill 
   7278  1.15  jmcneill 	if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index,
   7279  1.15  jmcneill 	    q->q_eq.eq_n) != 0)
   7280  1.15  jmcneill 		goto destroy_tx_slots;
   7281  1.15  jmcneill 
   7282  1.15  jmcneill 	if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n)
   7283  1.15  jmcneill 	    != 0)
   7284  1.15  jmcneill 		goto destroy_cq;
   7285  1.15  jmcneill 
   7286  1.15  jmcneill 	if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0)
   7287  1.15  jmcneill 		goto destroy_sq;
   7288  1.15  jmcneill 
   7289  1.15  jmcneill 	return 0;
   7290  1.15  jmcneill 
   7291  1.15  jmcneill destroy_sq:
   7292  1.15  jmcneill 	mcx_destroy_sq(sc, tx);
   7293  1.15  jmcneill destroy_cq:
   7294  1.15  jmcneill 	mcx_destroy_cq(sc, &q->q_cq);
   7295  1.15  jmcneill destroy_tx_slots:
   7296  1.15  jmcneill 	mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
   7297  1.15  jmcneill 	tx->tx_slots = NULL;
   7298  1.15  jmcneill 
   7299  1.15  jmcneill 	i = (1 << MCX_LOG_RQ_SIZE);
   7300  1.15  jmcneill destroy_rx_slots:
   7301  1.15  jmcneill 	mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
   7302  1.15  jmcneill 	rx->rx_slots = NULL;
   7303  1.15  jmcneill 	return ENOMEM;
   7304  1.15  jmcneill }
   7305  1.15  jmcneill 
   7306  1.15  jmcneill static int
   7307  1.15  jmcneill mcx_rss_group_entry_count(struct mcx_softc *sc, int group)
   7308  1.15  jmcneill {
   7309  1.15  jmcneill 	int i;
   7310  1.15  jmcneill 	int count;
   7311  1.15  jmcneill 
   7312  1.15  jmcneill 	count = 0;
   7313  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7314  1.15  jmcneill 		if (mcx_rss_config[i].flow_group == group)
   7315  1.15  jmcneill 			count++;
   7316  1.15  jmcneill 	}
   7317  1.15  jmcneill 
   7318  1.15  jmcneill 	return count;
   7319  1.15  jmcneill }
   7320  1.15  jmcneill 
   7321  1.15  jmcneill static int
   7322  1.15  jmcneill mcx_init(struct ifnet *ifp)
   7323  1.15  jmcneill {
   7324  1.15  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7325  1.15  jmcneill 	struct mcx_rx *rx;
   7326  1.15  jmcneill 	struct mcx_tx *tx;
   7327  1.15  jmcneill 	int i, start, count, flow_group, flow_index;
   7328  1.15  jmcneill 	struct mcx_flow_match match_crit;
   7329  1.15  jmcneill 	struct mcx_rss_rule *rss;
   7330  1.15  jmcneill 	uint32_t dest;
   7331  1.15  jmcneill 	int rqns[MCX_MAX_QUEUES] = { 0 };
   7332  1.15  jmcneill 
   7333  1.15  jmcneill 	if (ISSET(ifp->if_flags, IFF_RUNNING))
   7334  1.15  jmcneill 		mcx_stop(ifp, 0);
   7335   1.1  jmcneill 
   7336  1.15  jmcneill 	if (mcx_create_tis(sc, &sc->sc_tis) != 0)
   7337  1.15  jmcneill 		goto down;
   7338   1.1  jmcneill 
   7339  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7340  1.15  jmcneill 		if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
   7341  1.15  jmcneill 			goto down;
   7342   1.1  jmcneill 		}
   7343   1.1  jmcneill 	}
   7344   1.1  jmcneill 
   7345  1.15  jmcneill 	/* RSS flow table and flow groups */
   7346  1.15  jmcneill 	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1,
   7347  1.15  jmcneill 	    &sc->sc_rss_flow_table_id) != 0)
   7348   1.1  jmcneill 		goto down;
   7349   1.1  jmcneill 
   7350  1.15  jmcneill 	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   7351  1.15  jmcneill 	    sc->sc_rss_flow_table_id;
   7352   1.1  jmcneill 
   7353  1.15  jmcneill 	/* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */
   7354  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7355  1.15  jmcneill 	match_crit.mc_ethertype = 0xffff;
   7356  1.15  jmcneill 	match_crit.mc_ip_proto = 0xff;
   7357  1.15  jmcneill 	match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG;
   7358  1.15  jmcneill 	start = 0;
   7359  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4);
   7360  1.15  jmcneill 	if (count != 0) {
   7361  1.15  jmcneill 		if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7362  1.15  jmcneill 		    MCX_FLOW_GROUP_RSS_L4, start, count,
   7363  1.15  jmcneill 		    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7364  1.15  jmcneill 			goto down;
   7365  1.15  jmcneill 		start += count;
   7366  1.15  jmcneill 	}
   7367   1.1  jmcneill 
   7368  1.15  jmcneill 	/* L3 RSS flow group (v4/v6, including fragments) */
   7369  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7370  1.15  jmcneill 	match_crit.mc_ethertype = 0xffff;
   7371  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3);
   7372  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7373  1.15  jmcneill 	    MCX_FLOW_GROUP_RSS_L3, start, count,
   7374  1.15  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7375   1.1  jmcneill 		goto down;
   7376  1.15  jmcneill 	start += count;
   7377   1.1  jmcneill 
   7378  1.15  jmcneill 	/* non-RSS flow group */
   7379  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE);
   7380  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7381  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7382  1.15  jmcneill 	    MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0)
   7383   1.1  jmcneill 		goto down;
   7384   1.1  jmcneill 
   7385  1.15  jmcneill 	/* Root flow table, matching packets based on mac address */
   7386  1.15  jmcneill 	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0,
   7387  1.15  jmcneill 	    &sc->sc_mac_flow_table_id) != 0)
   7388   1.1  jmcneill 		goto down;
   7389   1.1  jmcneill 
   7390   1.1  jmcneill 	/* promisc flow group */
   7391   1.1  jmcneill 	start = 0;
   7392   1.1  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7393  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7394  1.15  jmcneill 	    MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0)
   7395   1.1  jmcneill 		goto down;
   7396   1.1  jmcneill 	sc->sc_promisc_flow_enabled = 0;
   7397   1.1  jmcneill 	start++;
   7398   1.1  jmcneill 
   7399   1.1  jmcneill 	/* all multicast flow group */
   7400   1.1  jmcneill 	match_crit.mc_dest_mac[0] = 0x01;
   7401  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7402  1.15  jmcneill 	    MCX_FLOW_GROUP_ALLMULTI, start, 1,
   7403   1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7404   1.1  jmcneill 		goto down;
   7405   1.1  jmcneill 	sc->sc_allmulti_flow_enabled = 0;
   7406   1.1  jmcneill 	start++;
   7407   1.1  jmcneill 
   7408   1.1  jmcneill 	/* mac address matching flow group */
   7409   1.1  jmcneill 	memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
   7410  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7411  1.15  jmcneill 	    MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
   7412   1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7413   1.1  jmcneill 		goto down;
   7414   1.1  jmcneill 
   7415   1.1  jmcneill 	/* flow table entries for unicast and broadcast */
   7416   1.1  jmcneill 	start = 0;
   7417  1.15  jmcneill 	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
   7418  1.15  jmcneill 	    LLADDR(satosdl(ifp->if_dl->ifa_addr)), dest) != 0)
   7419   1.1  jmcneill 		goto down;
   7420   1.1  jmcneill 	start++;
   7421   1.1  jmcneill 
   7422  1.15  jmcneill 	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
   7423  1.15  jmcneill 	    etherbroadcastaddr, dest) != 0)
   7424   1.1  jmcneill 		goto down;
   7425   1.1  jmcneill 	start++;
   7426   1.1  jmcneill 
   7427   1.1  jmcneill 	/* multicast entries go after that */
   7428   1.1  jmcneill 	sc->sc_mcast_flow_base = start;
   7429   1.1  jmcneill 
   7430   1.1  jmcneill 	/* re-add any existing multicast flows */
   7431   1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7432   1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   7433  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC,
   7434   1.1  jmcneill 			    sc->sc_mcast_flow_base + i,
   7435  1.15  jmcneill 			    sc->sc_mcast_flows[i], dest);
   7436   1.1  jmcneill 		}
   7437   1.1  jmcneill 	}
   7438   1.1  jmcneill 
   7439  1.15  jmcneill 	if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0)
   7440   1.1  jmcneill 		goto down;
   7441   1.1  jmcneill 
   7442  1.15  jmcneill 	/*
   7443  1.15  jmcneill 	 * the RQT can be any size as long as it's a power of two.
   7444  1.15  jmcneill 	 * since we also restrict the number of queues to a power of two,
   7445  1.15  jmcneill 	 * we can just put each rx queue in once.
   7446  1.15  jmcneill 	 */
   7447  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++)
   7448  1.15  jmcneill 		rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;
   7449  1.15  jmcneill 
   7450  1.15  jmcneill 	if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0)
   7451   1.1  jmcneill 		goto down;
   7452   1.1  jmcneill 
   7453  1.15  jmcneill 	start = 0;
   7454  1.15  jmcneill 	flow_index = 0;
   7455  1.15  jmcneill 	flow_group = -1;
   7456  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7457  1.15  jmcneill 		rss = &mcx_rss_config[i];
   7458  1.15  jmcneill 		if (rss->flow_group != flow_group) {
   7459  1.15  jmcneill 			flow_group = rss->flow_group;
   7460  1.15  jmcneill 			flow_index = 0;
   7461  1.15  jmcneill 		}
   7462  1.15  jmcneill 
   7463  1.15  jmcneill 		if (rss->hash_sel == 0) {
   7464  1.15  jmcneill 			if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx,
   7465  1.15  jmcneill 			    &sc->sc_tir[i]) != 0)
   7466  1.15  jmcneill 				goto down;
   7467  1.15  jmcneill 		} else {
   7468  1.15  jmcneill 			if (mcx_create_tir_indirect(sc, sc->sc_rqt,
   7469  1.15  jmcneill 			    rss->hash_sel, &sc->sc_tir[i]) != 0)
   7470  1.15  jmcneill 				goto down;
   7471  1.15  jmcneill 		}
   7472  1.15  jmcneill 
   7473  1.15  jmcneill 		if (mcx_set_flow_table_entry_proto(sc, flow_group,
   7474  1.15  jmcneill 		    flow_index, rss->ethertype, rss->ip_proto,
   7475  1.15  jmcneill 		    MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0)
   7476  1.15  jmcneill 			goto down;
   7477  1.15  jmcneill 		flow_index++;
   7478  1.15  jmcneill 	}
   7479  1.15  jmcneill 
   7480  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7481  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   7482  1.15  jmcneill 		rx = &q->q_rx;
   7483  1.15  jmcneill 		tx = &q->q_tx;
   7484  1.15  jmcneill 
   7485  1.15  jmcneill 		/* start the queues */
   7486  1.15  jmcneill 		if (mcx_ready_sq(sc, tx) != 0)
   7487  1.15  jmcneill 			goto down;
   7488  1.15  jmcneill 
   7489  1.15  jmcneill 		if (mcx_ready_rq(sc, rx) != 0)
   7490  1.15  jmcneill 			goto down;
   7491  1.15  jmcneill 
   7492  1.15  jmcneill 		mcx_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
   7493  1.15  jmcneill 		rx->rx_prod = 0;
   7494  1.15  jmcneill 		mcx_rx_fill(sc, rx);
   7495   1.1  jmcneill 
   7496  1.15  jmcneill 		tx->tx_cons = 0;
   7497  1.15  jmcneill 		tx->tx_prod = 0;
   7498  1.15  jmcneill 	}
   7499   1.1  jmcneill 
   7500   1.1  jmcneill 	mcx_calibrate_first(sc);
   7501   1.1  jmcneill 
   7502   1.1  jmcneill 	SET(ifp->if_flags, IFF_RUNNING);
   7503   1.1  jmcneill 	CLR(ifp->if_flags, IFF_OACTIVE);
   7504   1.1  jmcneill 	if_schedule_deferred_start(ifp);
   7505   1.1  jmcneill 
   7506   1.1  jmcneill 	return 0;
   7507   1.1  jmcneill down:
   7508   1.1  jmcneill 	mcx_stop(ifp, 0);
   7509   1.1  jmcneill 	return EIO;
   7510   1.1  jmcneill }
   7511   1.1  jmcneill 
   7512   1.1  jmcneill static void
   7513   1.1  jmcneill mcx_stop(struct ifnet *ifp, int disable)
   7514   1.1  jmcneill {
   7515   1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7516  1.15  jmcneill 	struct mcx_rss_rule *rss;
   7517  1.15  jmcneill 	int group, i, flow_group, flow_index;
   7518   1.1  jmcneill 
   7519   1.1  jmcneill 	CLR(ifp->if_flags, IFF_RUNNING);
   7520   1.1  jmcneill 
   7521   1.1  jmcneill 	/*
   7522   1.1  jmcneill 	 * delete flow table entries first, so no packets can arrive
   7523   1.1  jmcneill 	 * after the barriers
   7524   1.1  jmcneill 	 */
   7525   1.1  jmcneill 	if (sc->sc_promisc_flow_enabled)
   7526   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   7527   1.1  jmcneill 	if (sc->sc_allmulti_flow_enabled)
   7528   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   7529   1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
   7530   1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
   7531   1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7532   1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   7533   1.1  jmcneill 			mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
   7534   1.1  jmcneill 			    sc->sc_mcast_flow_base + i);
   7535   1.1  jmcneill 		}
   7536   1.1  jmcneill 	}
   7537   1.1  jmcneill 
   7538  1.15  jmcneill 	flow_group = -1;
   7539  1.15  jmcneill 	flow_index = 0;
   7540  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7541  1.15  jmcneill 		rss = &mcx_rss_config[i];
   7542  1.15  jmcneill 		if (rss->flow_group != flow_group) {
   7543  1.15  jmcneill 			flow_group = rss->flow_group;
   7544  1.15  jmcneill 			flow_index = 0;
   7545  1.15  jmcneill 		}
   7546  1.15  jmcneill 
   7547  1.15  jmcneill 		mcx_delete_flow_table_entry(sc, flow_group, flow_index);
   7548  1.15  jmcneill 
   7549  1.15  jmcneill 		mcx_destroy_tir(sc, sc->sc_tir[i]);
   7550  1.15  jmcneill 		sc->sc_tir[i] = 0;
   7551  1.15  jmcneill 
   7552  1.15  jmcneill 		flow_index++;
   7553  1.15  jmcneill 	}
   7554  1.15  jmcneill 
   7555  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7556  1.15  jmcneill 		callout_halt(&sc->sc_queues[i].q_rx.rx_refill, NULL);
   7557  1.15  jmcneill 	}
   7558  1.15  jmcneill 
   7559   1.1  jmcneill 	callout_halt(&sc->sc_calibrate, NULL);
   7560   1.1  jmcneill 
   7561   1.1  jmcneill 	for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
   7562  1.15  jmcneill 		if (sc->sc_flow_group[group].g_id != -1)
   7563  1.15  jmcneill 			mcx_destroy_flow_group(sc, group);
   7564  1.15  jmcneill 	}
   7565  1.15  jmcneill 
   7566  1.15  jmcneill 	if (sc->sc_mac_flow_table_id != -1) {
   7567  1.15  jmcneill 		mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id);
   7568  1.15  jmcneill 		sc->sc_mac_flow_table_id = -1;
   7569  1.15  jmcneill 	}
   7570  1.15  jmcneill 	if (sc->sc_rss_flow_table_id != -1) {
   7571  1.15  jmcneill 		mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id);
   7572  1.15  jmcneill 		sc->sc_rss_flow_table_id = -1;
   7573  1.15  jmcneill 	}
   7574  1.15  jmcneill 	if (sc->sc_rqt != -1) {
   7575  1.15  jmcneill 		mcx_destroy_rqt(sc, sc->sc_rqt);
   7576  1.15  jmcneill 		sc->sc_rqt = -1;
   7577  1.15  jmcneill 	}
   7578  1.15  jmcneill 
   7579  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7580  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   7581  1.15  jmcneill 		struct mcx_rx *rx = &q->q_rx;
   7582  1.15  jmcneill 		struct mcx_tx *tx = &q->q_tx;
   7583  1.15  jmcneill 		struct mcx_cq *cq = &q->q_cq;
   7584  1.15  jmcneill 
   7585  1.15  jmcneill 		if (rx->rx_rqn != 0)
   7586  1.15  jmcneill 			mcx_destroy_rq(sc, rx);
   7587  1.15  jmcneill 
   7588  1.15  jmcneill 		if (tx->tx_sqn != 0)
   7589  1.15  jmcneill 			mcx_destroy_sq(sc, tx);
   7590  1.15  jmcneill 
   7591  1.15  jmcneill 		if (tx->tx_slots != NULL) {
   7592  1.15  jmcneill 			mcx_free_slots(sc, tx->tx_slots,
   7593  1.15  jmcneill 			    (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE));
   7594  1.15  jmcneill 			tx->tx_slots = NULL;
   7595  1.15  jmcneill 		}
   7596  1.15  jmcneill 		if (rx->rx_slots != NULL) {
   7597  1.15  jmcneill 			mcx_free_slots(sc, rx->rx_slots,
   7598  1.15  jmcneill 			    (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE));
   7599  1.15  jmcneill 			rx->rx_slots = NULL;
   7600  1.15  jmcneill 		}
   7601  1.15  jmcneill 
   7602  1.15  jmcneill 		if (cq->cq_n != 0)
   7603  1.15  jmcneill 			mcx_destroy_cq(sc, cq);
   7604  1.15  jmcneill 	}
   7605  1.15  jmcneill 	if (sc->sc_tis != 0) {
   7606  1.15  jmcneill 		mcx_destroy_tis(sc, sc->sc_tis);
   7607  1.15  jmcneill 		sc->sc_tis = 0;
   7608   1.1  jmcneill 	}
   7609   1.1  jmcneill }
   7610   1.1  jmcneill 
   7611   1.1  jmcneill static int
   7612   1.1  jmcneill mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data)
   7613   1.1  jmcneill {
   7614   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   7615   1.1  jmcneill 	struct ifreq *ifr = (struct ifreq *)data;
   7616   1.7  jmcneill 	struct ethercom *ec = &sc->sc_ec;
   7617   1.1  jmcneill 	uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
   7618   1.7  jmcneill 	struct ether_multi *enm;
   7619   1.7  jmcneill 	struct ether_multistep step;
   7620   1.7  jmcneill 	int s, i, flags, error = 0;
   7621  1.15  jmcneill 	uint32_t dest;
   7622   1.1  jmcneill 
   7623   1.1  jmcneill 	s = splnet();
   7624   1.1  jmcneill 	switch (cmd) {
   7625   1.1  jmcneill 
   7626   1.1  jmcneill 	case SIOCADDMULTI:
   7627   1.1  jmcneill 		if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   7628   1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   7629   1.7  jmcneill 			if (error != 0) {
   7630   1.7  jmcneill 				splx(s);
   7631   1.1  jmcneill 				return (error);
   7632   1.7  jmcneill 			}
   7633   1.1  jmcneill 
   7634  1.15  jmcneill 			dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   7635  1.15  jmcneill 			    sc->sc_rss_flow_table_id;
   7636  1.15  jmcneill 
   7637   1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7638   1.1  jmcneill 				if (sc->sc_mcast_flows[i][0] == 0) {
   7639   1.1  jmcneill 					memcpy(sc->sc_mcast_flows[i], addrlo,
   7640   1.1  jmcneill 					    ETHER_ADDR_LEN);
   7641   1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   7642  1.15  jmcneill 						mcx_set_flow_table_entry_mac(sc,
   7643   1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   7644   1.1  jmcneill 						    sc->sc_mcast_flow_base + i,
   7645  1.15  jmcneill 						    sc->sc_mcast_flows[i], dest);
   7646   1.1  jmcneill 					}
   7647   1.1  jmcneill 					break;
   7648   1.1  jmcneill 				}
   7649   1.1  jmcneill 			}
   7650   1.1  jmcneill 
   7651   1.1  jmcneill 			if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   7652   1.1  jmcneill 				if (i == MCX_NUM_MCAST_FLOWS) {
   7653   1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   7654   1.1  jmcneill 					sc->sc_extra_mcast++;
   7655   1.1  jmcneill 					error = ENETRESET;
   7656   1.1  jmcneill 				}
   7657   1.1  jmcneill 
   7658   1.7  jmcneill 				if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN)) {
   7659   1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   7660   1.1  jmcneill 					error = ENETRESET;
   7661   1.1  jmcneill 				}
   7662   1.1  jmcneill 			}
   7663   1.1  jmcneill 		}
   7664   1.1  jmcneill 		break;
   7665   1.1  jmcneill 
   7666   1.1  jmcneill 	case SIOCDELMULTI:
   7667   1.1  jmcneill 		if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   7668   1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   7669   1.7  jmcneill 			if (error != 0) {
   7670   1.7  jmcneill 				splx(s);
   7671   1.1  jmcneill 				return (error);
   7672   1.7  jmcneill 			}
   7673   1.1  jmcneill 
   7674   1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7675   1.1  jmcneill 				if (memcmp(sc->sc_mcast_flows[i], addrlo,
   7676   1.1  jmcneill 				    ETHER_ADDR_LEN) == 0) {
   7677   1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   7678   1.1  jmcneill 						mcx_delete_flow_table_entry(sc,
   7679   1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   7680   1.1  jmcneill 						    sc->sc_mcast_flow_base + i);
   7681   1.1  jmcneill 					}
   7682   1.1  jmcneill 					sc->sc_mcast_flows[i][0] = 0;
   7683   1.1  jmcneill 					break;
   7684   1.1  jmcneill 				}
   7685   1.1  jmcneill 			}
   7686   1.1  jmcneill 
   7687   1.1  jmcneill 			if (i == MCX_NUM_MCAST_FLOWS)
   7688   1.1  jmcneill 				sc->sc_extra_mcast--;
   7689   1.1  jmcneill 
   7690   1.1  jmcneill 			if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
   7691   1.7  jmcneill 			    sc->sc_extra_mcast == 0) {
   7692   1.7  jmcneill 				flags = 0;
   7693   1.7  jmcneill 				ETHER_LOCK(ec);
   7694   1.7  jmcneill 				ETHER_FIRST_MULTI(step, ec, enm);
   7695   1.7  jmcneill 				while (enm != NULL) {
   7696   1.7  jmcneill 					if (memcmp(enm->enm_addrlo,
   7697   1.7  jmcneill 					    enm->enm_addrhi, ETHER_ADDR_LEN)) {
   7698   1.7  jmcneill 						SET(flags, IFF_ALLMULTI);
   7699   1.7  jmcneill 						break;
   7700   1.7  jmcneill 					}
   7701   1.7  jmcneill 					ETHER_NEXT_MULTI(step, enm);
   7702   1.7  jmcneill 				}
   7703   1.7  jmcneill 				ETHER_UNLOCK(ec);
   7704   1.7  jmcneill 				if (!ISSET(flags, IFF_ALLMULTI)) {
   7705   1.7  jmcneill 					CLR(ifp->if_flags, IFF_ALLMULTI);
   7706   1.7  jmcneill 					error = ENETRESET;
   7707   1.7  jmcneill 				}
   7708   1.1  jmcneill 			}
   7709   1.1  jmcneill 		}
   7710   1.1  jmcneill 		break;
   7711   1.1  jmcneill 
   7712   1.1  jmcneill 	default:
   7713   1.1  jmcneill 		error = ether_ioctl(ifp, cmd, data);
   7714   1.1  jmcneill 	}
   7715   1.1  jmcneill 
   7716   1.1  jmcneill 	if (error == ENETRESET) {
   7717   1.1  jmcneill 		if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
   7718   1.1  jmcneill 		    (IFF_UP | IFF_RUNNING))
   7719   1.1  jmcneill 			mcx_iff(sc);
   7720   1.1  jmcneill 		error = 0;
   7721   1.1  jmcneill 	}
   7722   1.1  jmcneill 	splx(s);
   7723   1.1  jmcneill 
   7724   1.1  jmcneill 	return (error);
   7725   1.1  jmcneill }
   7726   1.1  jmcneill 
   7727   1.1  jmcneill #if 0
   7728   1.1  jmcneill static int
   7729   1.1  jmcneill mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
   7730   1.1  jmcneill {
   7731   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   7732   1.1  jmcneill 	struct mcx_reg_mcia mcia;
   7733   1.1  jmcneill 	struct mcx_reg_pmlp pmlp;
   7734   1.1  jmcneill 	int offset, error;
   7735   1.1  jmcneill 
   7736   1.1  jmcneill 	/* get module number */
   7737   1.1  jmcneill 	memset(&pmlp, 0, sizeof(pmlp));
   7738   1.1  jmcneill 	pmlp.rp_local_port = 1;
   7739   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
   7740   1.1  jmcneill 	    sizeof(pmlp));
   7741   1.1  jmcneill 	if (error != 0) {
   7742   1.1  jmcneill 		printf("%s: unable to get eeprom module number\n",
   7743   1.1  jmcneill 		    DEVNAME(sc));
   7744   1.1  jmcneill 		return error;
   7745   1.1  jmcneill 	}
   7746   1.1  jmcneill 
   7747   1.1  jmcneill 	for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
   7748   1.1  jmcneill 		memset(&mcia, 0, sizeof(mcia));
   7749   1.1  jmcneill 		mcia.rm_l = 0;
   7750   1.1  jmcneill 		mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) &
   7751   1.1  jmcneill 		    MCX_PMLP_MODULE_NUM_MASK;
   7752   1.1  jmcneill 		mcia.rm_i2c_addr = sff->sff_addr / 2;	/* apparently */
   7753   1.1  jmcneill 		mcia.rm_page_num = sff->sff_page;
   7754   1.1  jmcneill 		mcia.rm_dev_addr = htobe16(offset);
   7755   1.1  jmcneill 		mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);
   7756   1.1  jmcneill 
   7757   1.1  jmcneill 		error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
   7758   1.1  jmcneill 		    &mcia, sizeof(mcia));
   7759   1.1  jmcneill 		if (error != 0) {
   7760   1.1  jmcneill 			printf("%s: unable to read eeprom at %x\n",
   7761   1.1  jmcneill 			    DEVNAME(sc), offset);
   7762   1.1  jmcneill 			return error;
   7763   1.1  jmcneill 		}
   7764   1.1  jmcneill 
   7765   1.1  jmcneill 		memcpy(sff->sff_data + offset, mcia.rm_data,
   7766   1.1  jmcneill 		    MCX_MCIA_EEPROM_BYTES);
   7767   1.1  jmcneill 	}
   7768   1.1  jmcneill 
   7769   1.1  jmcneill 	return 0;
   7770   1.1  jmcneill }
   7771   1.1  jmcneill #endif
   7772   1.1  jmcneill 
   7773   1.1  jmcneill static int
   7774   1.1  jmcneill mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
   7775   1.1  jmcneill {
   7776   1.1  jmcneill 	switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   7777   1.1  jmcneill 	    BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
   7778   1.1  jmcneill 	case 0:
   7779   1.1  jmcneill 		break;
   7780   1.1  jmcneill 
   7781   1.1  jmcneill 	case EFBIG:
   7782   1.6    nonaka 		if (m_defrag(m, M_DONTWAIT) != NULL &&
   7783   1.1  jmcneill 		    bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   7784   1.1  jmcneill 		    BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
   7785   1.1  jmcneill 			break;
   7786   1.1  jmcneill 
   7787   1.1  jmcneill 		/* FALLTHROUGH */
   7788   1.1  jmcneill 	default:
   7789   1.1  jmcneill 		return (1);
   7790   1.1  jmcneill 	}
   7791   1.1  jmcneill 
   7792   1.1  jmcneill 	ms->ms_m = m;
   7793   1.1  jmcneill 	return (0);
   7794   1.1  jmcneill }
   7795   1.1  jmcneill 
   7796   1.1  jmcneill static void
   7797  1.16  jmcneill mcx_send_common_locked(struct ifnet *ifp, struct mcx_tx *tx, bool is_transmit)
   7798   1.1  jmcneill {
   7799   1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7800   1.1  jmcneill 	struct mcx_sq_entry *sq, *sqe;
   7801   1.1  jmcneill 	struct mcx_sq_entry_seg *sqs;
   7802   1.1  jmcneill 	struct mcx_slot *ms;
   7803   1.1  jmcneill 	bus_dmamap_t map;
   7804   1.1  jmcneill 	struct mbuf *m;
   7805   1.1  jmcneill 	u_int idx, free, used;
   7806   1.1  jmcneill 	uint64_t *bf;
   7807  1.15  jmcneill 	uint32_t csum;
   7808   1.1  jmcneill 	size_t bf_base;
   7809   1.1  jmcneill 	int i, seg, nseg;
   7810   1.1  jmcneill 
   7811  1.16  jmcneill 	KASSERT(mutex_owned(&tx->tx_lock));
   7812  1.16  jmcneill 
   7813  1.16  jmcneill 	if ((ifp->if_flags & IFF_RUNNING) == 0)
   7814  1.16  jmcneill 		return;
   7815  1.16  jmcneill 
   7816  1.15  jmcneill 	bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
   7817   1.1  jmcneill 
   7818  1.15  jmcneill 	idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE);
   7819  1.15  jmcneill 	free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod;
   7820   1.1  jmcneill 
   7821   1.1  jmcneill 	used = 0;
   7822   1.1  jmcneill 	bf = NULL;
   7823  1.15  jmcneill 
   7824  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   7825  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   7826  1.15  jmcneill 
   7827  1.15  jmcneill 	sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem);
   7828   1.1  jmcneill 
   7829   1.1  jmcneill 	for (;;) {
   7830   1.1  jmcneill 		if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
   7831   1.1  jmcneill 			SET(ifp->if_flags, IFF_OACTIVE);
   7832   1.1  jmcneill 			break;
   7833   1.1  jmcneill 		}
   7834   1.1  jmcneill 
   7835  1.16  jmcneill 		if (is_transmit) {
   7836  1.16  jmcneill 			m = pcq_get(tx->tx_pcq);
   7837  1.16  jmcneill 		} else {
   7838  1.16  jmcneill 			IFQ_DEQUEUE(&ifp->if_snd, m);
   7839  1.16  jmcneill 		}
   7840   1.1  jmcneill 		if (m == NULL) {
   7841   1.1  jmcneill 			break;
   7842   1.1  jmcneill 		}
   7843   1.1  jmcneill 
   7844   1.1  jmcneill 		sqe = sq + idx;
   7845  1.15  jmcneill 		ms = &tx->tx_slots[idx];
   7846   1.1  jmcneill 		memset(sqe, 0, sizeof(*sqe));
   7847   1.1  jmcneill 
   7848   1.1  jmcneill 		/* ctrl segment */
   7849   1.1  jmcneill 		sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
   7850  1.15  jmcneill 		    ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
   7851   1.1  jmcneill 		/* always generate a completion event */
   7852   1.1  jmcneill 		sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);
   7853   1.1  jmcneill 
   7854   1.1  jmcneill 		/* eth segment */
   7855  1.15  jmcneill 		csum = 0;
   7856  1.15  jmcneill 		if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
   7857  1.15  jmcneill 			csum |= MCX_SQE_L3_CSUM;
   7858  1.15  jmcneill 		if (m->m_pkthdr.csum_flags &
   7859  1.15  jmcneill 		    (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6))
   7860  1.15  jmcneill 			csum |= MCX_SQE_L4_CSUM;
   7861  1.15  jmcneill 		sqe->sqe_mss_csum = htobe32(csum);
   7862   1.1  jmcneill 		sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
   7863  1.15  jmcneill 		if (vlan_has_tag(m)) {
   7864  1.15  jmcneill 			struct ether_vlan_header *evh;
   7865  1.15  jmcneill 			evh = (struct ether_vlan_header *)
   7866  1.15  jmcneill 			    &sqe->sqe_inline_headers;
   7867  1.15  jmcneill 
   7868  1.15  jmcneill 			m_copydata(m, 0, ETHER_HDR_LEN, evh);
   7869  1.15  jmcneill 			evh->evl_proto = evh->evl_encap_proto;
   7870  1.15  jmcneill 			evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
   7871  1.15  jmcneill 			evh->evl_tag = htons(vlan_get_tag(m));
   7872  1.15  jmcneill 			m_adj(m, ETHER_HDR_LEN);
   7873  1.15  jmcneill 		} else {
   7874  1.15  jmcneill 			m_copydata(m, 0, MCX_SQ_INLINE_SIZE,
   7875  1.15  jmcneill 			    sqe->sqe_inline_headers);
   7876  1.15  jmcneill 			m_adj(m, MCX_SQ_INLINE_SIZE);
   7877  1.15  jmcneill 		}
   7878   1.1  jmcneill 
   7879   1.1  jmcneill 		if (mcx_load_mbuf(sc, ms, m) != 0) {
   7880   1.1  jmcneill 			m_freem(m);
   7881  1.10   thorpej 			if_statinc(ifp, if_oerrors);
   7882   1.1  jmcneill 			continue;
   7883   1.1  jmcneill 		}
   7884   1.1  jmcneill 		bf = (uint64_t *)sqe;
   7885   1.1  jmcneill 
   7886   1.1  jmcneill 		if (ifp->if_bpf != NULL)
   7887   1.1  jmcneill 			bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers,
   7888   1.1  jmcneill 			    MCX_SQ_INLINE_SIZE, m, BPF_D_OUT);
   7889   1.1  jmcneill 
   7890   1.1  jmcneill 		map = ms->ms_map;
   7891   1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   7892   1.1  jmcneill 		    BUS_DMASYNC_PREWRITE);
   7893   1.1  jmcneill 
   7894   1.1  jmcneill 		sqe->sqe_ds_sq_num =
   7895  1.15  jmcneill 		    htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) |
   7896   1.1  jmcneill 		    (map->dm_nsegs + 3));
   7897   1.1  jmcneill 
   7898   1.1  jmcneill 		/* data segment - first wqe has one segment */
   7899   1.1  jmcneill 		sqs = sqe->sqe_segs;
   7900   1.1  jmcneill 		seg = 0;
   7901   1.1  jmcneill 		nseg = 1;
   7902   1.1  jmcneill 		for (i = 0; i < map->dm_nsegs; i++) {
   7903   1.1  jmcneill 			if (seg == nseg) {
   7904   1.1  jmcneill 				/* next slot */
   7905   1.1  jmcneill 				idx++;
   7906   1.1  jmcneill 				if (idx == (1 << MCX_LOG_SQ_SIZE))
   7907   1.1  jmcneill 					idx = 0;
   7908  1.15  jmcneill 				tx->tx_prod++;
   7909   1.1  jmcneill 				used++;
   7910   1.1  jmcneill 
   7911   1.1  jmcneill 				sqs = (struct mcx_sq_entry_seg *)(sq + idx);
   7912   1.1  jmcneill 				seg = 0;
   7913   1.1  jmcneill 				nseg = MCX_SQ_SEGS_PER_SLOT;
   7914   1.1  jmcneill 			}
   7915   1.1  jmcneill 			sqs[seg].sqs_byte_count =
   7916   1.1  jmcneill 			    htobe32(map->dm_segs[i].ds_len);
   7917   1.1  jmcneill 			sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
   7918   1.1  jmcneill 			sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
   7919   1.1  jmcneill 			seg++;
   7920   1.1  jmcneill 		}
   7921   1.1  jmcneill 
   7922   1.1  jmcneill 		idx++;
   7923   1.1  jmcneill 		if (idx == (1 << MCX_LOG_SQ_SIZE))
   7924   1.1  jmcneill 			idx = 0;
   7925  1.15  jmcneill 		tx->tx_prod++;
   7926   1.1  jmcneill 		used++;
   7927   1.1  jmcneill 	}
   7928   1.1  jmcneill 
   7929  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   7930  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
   7931  1.15  jmcneill 
   7932   1.1  jmcneill 	if (used) {
   7933  1.15  jmcneill 		bus_size_t blueflame;
   7934   1.1  jmcneill 
   7935  1.15  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7936  1.15  jmcneill 		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   7937  1.15  jmcneill 		be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell),
   7938  1.15  jmcneill 		    tx->tx_prod & MCX_WQ_DOORBELL_MASK);
   7939  1.15  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7940  1.15  jmcneill 		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   7941   1.1  jmcneill 
   7942   1.1  jmcneill 		/*
   7943   1.1  jmcneill 		 * write the first 64 bits of the last sqe we produced
   7944   1.1  jmcneill 		 * to the blue flame buffer
   7945   1.1  jmcneill 		 */
   7946  1.15  jmcneill 
   7947  1.15  jmcneill 		blueflame = bf_base + tx->tx_bf_offset;
   7948   1.1  jmcneill 		bus_space_write_8(sc->sc_memt, sc->sc_memh,
   7949  1.15  jmcneill 		    blueflame, *bf);
   7950  1.15  jmcneill 		mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE);
   7951  1.15  jmcneill 
   7952   1.1  jmcneill 		/* next write goes to the other buffer */
   7953  1.15  jmcneill 		tx->tx_bf_offset ^= sc->sc_bf_size;
   7954   1.1  jmcneill 	}
   7955   1.1  jmcneill }
   7956   1.1  jmcneill 
   7957   1.1  jmcneill static void
   7958  1.16  jmcneill mcx_start(struct ifnet *ifp)
   7959  1.16  jmcneill {
   7960  1.16  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7961  1.16  jmcneill 	/* mcx_start() always uses TX ring[0] */
   7962  1.16  jmcneill 	struct mcx_tx *tx = &sc->sc_queues[0].q_tx;
   7963  1.16  jmcneill 
   7964  1.16  jmcneill 	mutex_enter(&tx->tx_lock);
   7965  1.16  jmcneill 	if (!ISSET(ifp->if_flags, IFF_OACTIVE)) {
   7966  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, false);
   7967  1.16  jmcneill 	}
   7968  1.16  jmcneill 	mutex_exit(&tx->tx_lock);
   7969  1.16  jmcneill }
   7970  1.16  jmcneill 
   7971  1.16  jmcneill static int
   7972  1.16  jmcneill mcx_transmit(struct ifnet *ifp, struct mbuf *m)
   7973  1.16  jmcneill {
   7974  1.16  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7975  1.16  jmcneill 	struct mcx_tx *tx;
   7976  1.16  jmcneill 
   7977  1.16  jmcneill 	tx = &sc->sc_queues[cpu_index(curcpu()) % sc->sc_nqueues].q_tx;
   7978  1.16  jmcneill 	if (__predict_false(!pcq_put(tx->tx_pcq, m))) {
   7979  1.16  jmcneill 		m_freem(m);
   7980  1.16  jmcneill 		return ENOBUFS;
   7981  1.16  jmcneill 	}
   7982  1.16  jmcneill 
   7983  1.16  jmcneill 	if (mutex_tryenter(&tx->tx_lock)) {
   7984  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, true);
   7985  1.16  jmcneill 		mutex_exit(&tx->tx_lock);
   7986  1.16  jmcneill 	} else {
   7987  1.16  jmcneill 		softint_schedule(tx->tx_softint);
   7988  1.16  jmcneill 	}
   7989  1.16  jmcneill 
   7990  1.16  jmcneill 	return 0;
   7991  1.16  jmcneill }
   7992  1.16  jmcneill 
   7993  1.16  jmcneill static void
   7994  1.16  jmcneill mcx_deferred_transmit(void *arg)
   7995  1.16  jmcneill {
   7996  1.16  jmcneill 	struct mcx_tx *tx = arg;
   7997  1.16  jmcneill 	struct mcx_softc *sc = tx->tx_softc;
   7998  1.16  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   7999  1.16  jmcneill 
   8000  1.16  jmcneill 	mutex_enter(&tx->tx_lock);
   8001  1.16  jmcneill 	if (pcq_peek(tx->tx_pcq) != NULL) {
   8002  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, true);
   8003  1.16  jmcneill 	}
   8004  1.16  jmcneill 	mutex_exit(&tx->tx_lock);
   8005  1.16  jmcneill }
   8006  1.16  jmcneill 
   8007   1.1  jmcneill 
   8008   1.1  jmcneill static void
   8009   1.1  jmcneill mcx_media_add_types(struct mcx_softc *sc)
   8010   1.1  jmcneill {
   8011   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8012   1.1  jmcneill 	int i;
   8013   1.1  jmcneill 	uint32_t proto_cap;
   8014   1.1  jmcneill 
   8015   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8016   1.1  jmcneill 	ptys.rp_local_port = 1;
   8017   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8018   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8019   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8020   1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   8021   1.1  jmcneill 		return;
   8022   1.1  jmcneill 	}
   8023   1.1  jmcneill 
   8024   1.1  jmcneill 	proto_cap = be32toh(ptys.rp_eth_proto_cap);
   8025   1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8026  1.13  jmcneill 		const struct mcx_eth_proto_capability *cap;
   8027  1.13  jmcneill 		if (!ISSET(proto_cap, 1U << i))
   8028  1.13  jmcneill 			continue;
   8029  1.13  jmcneill 
   8030  1.13  jmcneill 		cap = &mcx_eth_cap_map[i];
   8031  1.13  jmcneill 		if (cap->cap_media == 0)
   8032  1.13  jmcneill 			continue;
   8033  1.13  jmcneill 
   8034  1.13  jmcneill 		ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
   8035   1.1  jmcneill 	}
   8036   1.1  jmcneill }
   8037   1.1  jmcneill 
   8038   1.1  jmcneill static void
   8039   1.1  jmcneill mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
   8040   1.1  jmcneill {
   8041   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   8042   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8043   1.1  jmcneill 	int i;
   8044  1.13  jmcneill 	uint32_t proto_oper;
   8045   1.1  jmcneill 	uint64_t media_oper;
   8046   1.1  jmcneill 
   8047   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8048   1.1  jmcneill 	ptys.rp_local_port = 1;
   8049   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8050   1.1  jmcneill 
   8051   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8052   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8053   1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   8054   1.1  jmcneill 		return;
   8055   1.1  jmcneill 	}
   8056   1.1  jmcneill 
   8057   1.1  jmcneill 	proto_oper = be32toh(ptys.rp_eth_proto_oper);
   8058   1.1  jmcneill 
   8059   1.1  jmcneill 	media_oper = 0;
   8060  1.13  jmcneill 
   8061   1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8062  1.13  jmcneill 		const struct mcx_eth_proto_capability *cap;
   8063  1.13  jmcneill 		if (!ISSET(proto_oper, 1U << i))
   8064  1.13  jmcneill 			continue;
   8065  1.13  jmcneill 
   8066  1.13  jmcneill 		cap = &mcx_eth_cap_map[i];
   8067  1.13  jmcneill 
   8068  1.13  jmcneill 		if (cap->cap_media != 0)
   8069  1.13  jmcneill 			media_oper = cap->cap_media;
   8070   1.1  jmcneill 	}
   8071   1.1  jmcneill 
   8072   1.1  jmcneill 	ifmr->ifm_status = IFM_AVALID;
   8073   1.1  jmcneill 	if (proto_oper != 0) {
   8074   1.1  jmcneill 		ifmr->ifm_status |= IFM_ACTIVE;
   8075  1.27    andvar 		ifmr->ifm_active = IFM_ETHER | IFM_FDX | IFM_AUTO | media_oper;
   8076   1.1  jmcneill 		/* txpause, rxpause, duplex? */
   8077   1.1  jmcneill 	}
   8078   1.1  jmcneill }
   8079   1.1  jmcneill 
   8080   1.1  jmcneill static int
   8081   1.1  jmcneill mcx_media_change(struct ifnet *ifp)
   8082   1.1  jmcneill {
   8083   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   8084   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8085   1.1  jmcneill 	struct mcx_reg_paos paos;
   8086   1.1  jmcneill 	uint32_t media;
   8087   1.1  jmcneill 	int i, error;
   8088   1.1  jmcneill 
   8089   1.1  jmcneill 	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
   8090   1.1  jmcneill 		return EINVAL;
   8091   1.1  jmcneill 
   8092   1.1  jmcneill 	error = 0;
   8093   1.1  jmcneill 
   8094   1.1  jmcneill 	if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
   8095   1.1  jmcneill 		/* read ptys to get supported media */
   8096   1.1  jmcneill 		memset(&ptys, 0, sizeof(ptys));
   8097   1.1  jmcneill 		ptys.rp_local_port = 1;
   8098   1.1  jmcneill 		ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8099   1.1  jmcneill 		if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
   8100   1.1  jmcneill 		    &ptys, sizeof(ptys)) != 0) {
   8101   1.1  jmcneill 			printf("%s: unable to read port type/speed\n",
   8102   1.1  jmcneill 			    DEVNAME(sc));
   8103   1.1  jmcneill 			return EIO;
   8104   1.1  jmcneill 		}
   8105   1.1  jmcneill 
   8106   1.1  jmcneill 		media = be32toh(ptys.rp_eth_proto_cap);
   8107   1.1  jmcneill 	} else {
   8108   1.1  jmcneill 		/* map media type */
   8109   1.1  jmcneill 		media = 0;
   8110   1.1  jmcneill 		for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8111  1.13  jmcneill 			const struct mcx_eth_proto_capability *cap;
   8112  1.13  jmcneill 
   8113  1.13  jmcneill 			cap = &mcx_eth_cap_map[i];
   8114  1.13  jmcneill 			if (cap->cap_media ==
   8115   1.1  jmcneill 			    IFM_SUBTYPE(sc->sc_media.ifm_media)) {
   8116   1.1  jmcneill 				media = (1 << i);
   8117   1.1  jmcneill 				break;
   8118   1.1  jmcneill 			}
   8119   1.1  jmcneill 		}
   8120   1.1  jmcneill 	}
   8121   1.1  jmcneill 
   8122   1.1  jmcneill 	/* disable the port */
   8123   1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   8124   1.1  jmcneill 	paos.rp_local_port = 1;
   8125   1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
   8126   1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   8127   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   8128   1.1  jmcneill 	    sizeof(paos)) != 0) {
   8129   1.1  jmcneill 		printf("%s: unable to set port state to down\n", DEVNAME(sc));
   8130   1.1  jmcneill 		return EIO;
   8131   1.1  jmcneill 	}
   8132   1.1  jmcneill 
   8133   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8134   1.1  jmcneill 	ptys.rp_local_port = 1;
   8135   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8136   1.1  jmcneill 	ptys.rp_eth_proto_admin = htobe32(media);
   8137   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
   8138   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8139   1.1  jmcneill 		printf("%s: unable to set port media type/speed\n",
   8140   1.1  jmcneill 		    DEVNAME(sc));
   8141   1.1  jmcneill 		error = EIO;
   8142   1.1  jmcneill 	}
   8143   1.1  jmcneill 
   8144   1.1  jmcneill 	/* re-enable the port to start negotiation */
   8145   1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   8146   1.1  jmcneill 	paos.rp_local_port = 1;
   8147   1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
   8148   1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   8149   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   8150   1.1  jmcneill 	    sizeof(paos)) != 0) {
   8151   1.1  jmcneill 		printf("%s: unable to set port state to up\n", DEVNAME(sc));
   8152   1.1  jmcneill 		error = EIO;
   8153   1.1  jmcneill 	}
   8154   1.1  jmcneill 
   8155   1.1  jmcneill 	return error;
   8156   1.1  jmcneill }
   8157   1.1  jmcneill 
   8158   1.1  jmcneill static void
   8159   1.1  jmcneill mcx_port_change(struct work *wk, void *xsc)
   8160   1.1  jmcneill {
   8161   1.1  jmcneill 	struct mcx_softc *sc = xsc;
   8162   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   8163  1.13  jmcneill 	struct mcx_reg_ptys ptys = {
   8164  1.13  jmcneill 		.rp_local_port = 1,
   8165  1.13  jmcneill 		.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH,
   8166  1.13  jmcneill 	};
   8167   1.1  jmcneill 	int link_state = LINK_STATE_DOWN;
   8168   1.1  jmcneill 
   8169  1.13  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8170  1.13  jmcneill 	    sizeof(ptys)) == 0) {
   8171  1.13  jmcneill 		uint32_t proto_oper = be32toh(ptys.rp_eth_proto_oper);
   8172  1.13  jmcneill 		uint64_t baudrate = 0;
   8173  1.13  jmcneill 		unsigned int i;
   8174  1.13  jmcneill 
   8175  1.15  jmcneill 		if (proto_oper != 0)
   8176  1.15  jmcneill 			link_state = LINK_STATE_UP;
   8177  1.15  jmcneill 
   8178  1.13  jmcneill 		for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8179  1.13  jmcneill 			const struct mcx_eth_proto_capability *cap;
   8180  1.13  jmcneill 			if (!ISSET(proto_oper, 1U << i))
   8181  1.13  jmcneill 				continue;
   8182  1.13  jmcneill 
   8183  1.13  jmcneill 			cap = &mcx_eth_cap_map[i];
   8184  1.13  jmcneill 			if (cap->cap_baudrate == 0)
   8185  1.13  jmcneill 				continue;
   8186  1.13  jmcneill 
   8187  1.13  jmcneill 			baudrate = cap->cap_baudrate;
   8188  1.13  jmcneill 			break;
   8189  1.13  jmcneill 		}
   8190  1.13  jmcneill 
   8191  1.13  jmcneill 		ifp->if_baudrate = baudrate;
   8192   1.1  jmcneill 	}
   8193   1.1  jmcneill 
   8194   1.1  jmcneill 	if (link_state != ifp->if_link_state) {
   8195   1.1  jmcneill 		if_link_state_change(ifp, link_state);
   8196   1.1  jmcneill 	}
   8197   1.1  jmcneill }
   8198   1.1  jmcneill 
   8199   1.1  jmcneill 
   8200   1.1  jmcneill static inline uint32_t
   8201   1.1  jmcneill mcx_rd(struct mcx_softc *sc, bus_size_t r)
   8202   1.1  jmcneill {
   8203   1.1  jmcneill 	uint32_t word;
   8204   1.1  jmcneill 
   8205   1.1  jmcneill 	word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r);
   8206   1.1  jmcneill 
   8207   1.1  jmcneill 	return (be32toh(word));
   8208   1.1  jmcneill }
   8209   1.1  jmcneill 
   8210   1.1  jmcneill static inline void
   8211   1.1  jmcneill mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
   8212   1.1  jmcneill {
   8213   1.1  jmcneill 	bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
   8214   1.1  jmcneill }
   8215   1.1  jmcneill 
   8216   1.1  jmcneill static inline void
   8217   1.1  jmcneill mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
   8218   1.1  jmcneill {
   8219  1.21  jmcneill #ifndef __NetBSD__
   8220   1.1  jmcneill 	bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
   8221  1.21  jmcneill #endif
   8222   1.1  jmcneill }
   8223   1.1  jmcneill 
   8224   1.1  jmcneill static uint64_t
   8225   1.1  jmcneill mcx_timer(struct mcx_softc *sc)
   8226   1.1  jmcneill {
   8227   1.1  jmcneill 	uint32_t hi, lo, ni;
   8228   1.1  jmcneill 
   8229   1.1  jmcneill 	hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   8230   1.1  jmcneill 	for (;;) {
   8231   1.1  jmcneill 		lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
   8232   1.1  jmcneill 		mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
   8233   1.1  jmcneill 		ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   8234   1.1  jmcneill 
   8235   1.1  jmcneill 		if (ni == hi)
   8236   1.1  jmcneill 			break;
   8237   1.1  jmcneill 
   8238   1.1  jmcneill 		hi = ni;
   8239   1.1  jmcneill 	}
   8240   1.1  jmcneill 
   8241   1.1  jmcneill 	return (((uint64_t)hi << 32) | (uint64_t)lo);
   8242   1.1  jmcneill }
   8243   1.1  jmcneill 
   8244   1.1  jmcneill static int
   8245   1.1  jmcneill mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   8246   1.1  jmcneill     bus_size_t size, u_int align)
   8247   1.1  jmcneill {
   8248   1.1  jmcneill 	mxm->mxm_size = size;
   8249   1.1  jmcneill 
   8250   1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
   8251   1.1  jmcneill 	    mxm->mxm_size, 0,
   8252   1.1  jmcneill 	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   8253   1.1  jmcneill 	    &mxm->mxm_map) != 0)
   8254   1.1  jmcneill 		return (1);
   8255   1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
   8256   1.1  jmcneill 	    align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
   8257   1.1  jmcneill 	    BUS_DMA_WAITOK) != 0)
   8258   1.1  jmcneill 		goto destroy;
   8259   1.1  jmcneill 	if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
   8260  1.19  jmcneill 	    mxm->mxm_size, &mxm->mxm_kva,
   8261  1.19  jmcneill 	    BUS_DMA_WAITOK | BUS_DMA_COHERENT) != 0)
   8262   1.1  jmcneill 		goto free;
   8263   1.1  jmcneill 	if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
   8264   1.1  jmcneill 	    mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
   8265   1.1  jmcneill 		goto unmap;
   8266   1.1  jmcneill 
   8267   1.1  jmcneill 	mcx_dmamem_zero(mxm);
   8268   1.1  jmcneill 
   8269   1.1  jmcneill 	return (0);
   8270   1.1  jmcneill unmap:
   8271   1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   8272   1.1  jmcneill free:
   8273   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   8274   1.1  jmcneill destroy:
   8275   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   8276   1.1  jmcneill 	return (1);
   8277   1.1  jmcneill }
   8278   1.1  jmcneill 
   8279   1.1  jmcneill static void
   8280   1.1  jmcneill mcx_dmamem_zero(struct mcx_dmamem *mxm)
   8281   1.1  jmcneill {
   8282   1.1  jmcneill 	memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
   8283   1.1  jmcneill }
   8284   1.1  jmcneill 
   8285   1.1  jmcneill static void
   8286   1.1  jmcneill mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   8287   1.1  jmcneill {
   8288   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
   8289   1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   8290   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   8291   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   8292   1.1  jmcneill }
   8293   1.1  jmcneill 
   8294   1.1  jmcneill static int
   8295   1.1  jmcneill mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
   8296   1.1  jmcneill {
   8297   1.1  jmcneill 	bus_dma_segment_t *segs;
   8298   1.1  jmcneill 	bus_size_t len = pages * MCX_PAGE_SIZE;
   8299   1.1  jmcneill 	size_t seglen;
   8300   1.1  jmcneill 
   8301   1.1  jmcneill 	segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP);
   8302   1.1  jmcneill 	seglen = sizeof(*segs) * pages;
   8303   1.1  jmcneill 
   8304   1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
   8305   1.1  jmcneill 	    segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
   8306   1.1  jmcneill 		goto free_segs;
   8307   1.1  jmcneill 
   8308   1.1  jmcneill 	if (mhm->mhm_seg_count < pages) {
   8309   1.1  jmcneill 		size_t nseglen;
   8310   1.1  jmcneill 
   8311   1.1  jmcneill 		mhm->mhm_segs = kmem_alloc(
   8312   1.1  jmcneill 		    sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP);
   8313   1.1  jmcneill 
   8314   1.1  jmcneill 		nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
   8315   1.1  jmcneill 
   8316   1.1  jmcneill 		memcpy(mhm->mhm_segs, segs, nseglen);
   8317   1.1  jmcneill 
   8318   1.1  jmcneill 		kmem_free(segs, seglen);
   8319   1.1  jmcneill 
   8320   1.1  jmcneill 		segs = mhm->mhm_segs;
   8321   1.1  jmcneill 		seglen = nseglen;
   8322   1.1  jmcneill 	} else
   8323   1.1  jmcneill 		mhm->mhm_segs = segs;
   8324   1.1  jmcneill 
   8325   1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
   8326   1.1  jmcneill 	    MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
   8327   1.1  jmcneill 	    &mhm->mhm_map) != 0)
   8328   1.1  jmcneill 		goto free_dmamem;
   8329   1.1  jmcneill 
   8330   1.1  jmcneill 	if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
   8331   1.1  jmcneill 	    mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
   8332   1.1  jmcneill 		goto destroy;
   8333   1.1  jmcneill 
   8334   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   8335   1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
   8336   1.1  jmcneill 
   8337   1.1  jmcneill 	mhm->mhm_npages = pages;
   8338   1.1  jmcneill 
   8339   1.1  jmcneill 	return (0);
   8340   1.1  jmcneill 
   8341   1.1  jmcneill destroy:
   8342   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   8343   1.1  jmcneill free_dmamem:
   8344   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   8345   1.1  jmcneill free_segs:
   8346   1.1  jmcneill 	kmem_free(segs, seglen);
   8347   1.1  jmcneill 	mhm->mhm_segs = NULL;
   8348   1.1  jmcneill 
   8349   1.1  jmcneill 	return (-1);
   8350   1.1  jmcneill }
   8351   1.1  jmcneill 
   8352   1.1  jmcneill static void
   8353   1.1  jmcneill mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
   8354   1.1  jmcneill {
   8355   1.1  jmcneill 	if (mhm->mhm_npages == 0)
   8356   1.1  jmcneill 		return;
   8357   1.1  jmcneill 
   8358   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   8359   1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
   8360   1.1  jmcneill 
   8361   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
   8362   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   8363   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   8364   1.1  jmcneill 	kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
   8365   1.1  jmcneill 
   8366   1.1  jmcneill 	mhm->mhm_npages = 0;
   8367   1.1  jmcneill }
   8368  1.15  jmcneill 
   8369  1.15  jmcneill #if NKSTAT > 0
   8370  1.15  jmcneill struct mcx_ppcnt {
   8371  1.15  jmcneill 	char			 name[KSTAT_KV_NAMELEN];
   8372  1.15  jmcneill 	enum kstat_kv_unit	 unit;
   8373  1.15  jmcneill };
   8374  1.15  jmcneill 
   8375  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = {
   8376  1.15  jmcneill 	{ "Good Tx",		KSTAT_KV_U_PACKETS, },
   8377  1.15  jmcneill 	{ "Good Rx",		KSTAT_KV_U_PACKETS, },
   8378  1.15  jmcneill 	{ "FCS errs",		KSTAT_KV_U_PACKETS, },
   8379  1.15  jmcneill 	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
   8380  1.15  jmcneill 	{ "Good Tx",		KSTAT_KV_U_BYTES, },
   8381  1.15  jmcneill 	{ "Good Rx",		KSTAT_KV_U_BYTES, },
   8382  1.15  jmcneill 	{ "Multicast Tx",	KSTAT_KV_U_PACKETS, },
   8383  1.15  jmcneill 	{ "Broadcast Tx",	KSTAT_KV_U_PACKETS, },
   8384  1.15  jmcneill 	{ "Multicast Rx",	KSTAT_KV_U_PACKETS, },
   8385  1.15  jmcneill 	{ "Broadcast Rx",	KSTAT_KV_U_PACKETS, },
   8386  1.15  jmcneill 	{ "In Range Len",	KSTAT_KV_U_PACKETS, },
   8387  1.15  jmcneill 	{ "Out Of Range Len",	KSTAT_KV_U_PACKETS, },
   8388  1.15  jmcneill 	{ "Frame Too Long",	KSTAT_KV_U_PACKETS, },
   8389  1.15  jmcneill 	{ "Symbol Errs",	KSTAT_KV_U_PACKETS, },
   8390  1.15  jmcneill 	{ "MAC Ctrl Tx",	KSTAT_KV_U_PACKETS, },
   8391  1.15  jmcneill 	{ "MAC Ctrl Rx",	KSTAT_KV_U_PACKETS, },
   8392  1.15  jmcneill 	{ "MAC Ctrl Unsup",	KSTAT_KV_U_PACKETS, },
   8393  1.15  jmcneill 	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
   8394  1.15  jmcneill 	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
   8395  1.15  jmcneill };
   8396  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count);
   8397  1.15  jmcneill 
   8398  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = {
   8399  1.15  jmcneill 	{ "Rx Bytes",		KSTAT_KV_U_BYTES, },
   8400  1.15  jmcneill 	{ "Rx Unicast",		KSTAT_KV_U_PACKETS, },
   8401  1.15  jmcneill 	{ "Rx Discards",	KSTAT_KV_U_PACKETS, },
   8402  1.15  jmcneill 	{ "Rx Errors",		KSTAT_KV_U_PACKETS, },
   8403  1.15  jmcneill 	{ "Rx Unknown Proto",	KSTAT_KV_U_PACKETS, },
   8404  1.15  jmcneill 	{ "Tx Bytes",		KSTAT_KV_U_BYTES, },
   8405  1.15  jmcneill 	{ "Tx Unicast",		KSTAT_KV_U_PACKETS, },
   8406  1.15  jmcneill 	{ "Tx Discards",	KSTAT_KV_U_PACKETS, },
   8407  1.15  jmcneill 	{ "Tx Errors",		KSTAT_KV_U_PACKETS, },
   8408  1.15  jmcneill 	{ "Rx Multicast",	KSTAT_KV_U_PACKETS, },
   8409  1.15  jmcneill 	{ "Rx Broadcast",	KSTAT_KV_U_PACKETS, },
   8410  1.15  jmcneill 	{ "Tx Multicast",	KSTAT_KV_U_PACKETS, },
   8411  1.15  jmcneill 	{ "Tx Broadcast",	KSTAT_KV_U_PACKETS, },
   8412  1.15  jmcneill };
   8413  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count);
   8414  1.15  jmcneill 
   8415  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = {
   8416  1.15  jmcneill 	{ "Drop Events",	KSTAT_KV_U_PACKETS, },
   8417  1.15  jmcneill 	{ "Octets",		KSTAT_KV_U_BYTES, },
   8418  1.15  jmcneill 	{ "Packets",		KSTAT_KV_U_PACKETS, },
   8419  1.15  jmcneill 	{ "Broadcasts",		KSTAT_KV_U_PACKETS, },
   8420  1.15  jmcneill 	{ "Multicasts",		KSTAT_KV_U_PACKETS, },
   8421  1.15  jmcneill 	{ "CRC Align Errs",	KSTAT_KV_U_PACKETS, },
   8422  1.15  jmcneill 	{ "Undersize",		KSTAT_KV_U_PACKETS, },
   8423  1.15  jmcneill 	{ "Oversize",		KSTAT_KV_U_PACKETS, },
   8424  1.15  jmcneill 	{ "Fragments",		KSTAT_KV_U_PACKETS, },
   8425  1.15  jmcneill 	{ "Jabbers",		KSTAT_KV_U_PACKETS, },
   8426  1.15  jmcneill 	{ "Collisions",		KSTAT_KV_U_NONE, },
   8427  1.15  jmcneill 	{ "64B",		KSTAT_KV_U_PACKETS, },
   8428  1.15  jmcneill 	{ "65-127B",		KSTAT_KV_U_PACKETS, },
   8429  1.15  jmcneill 	{ "128-255B",		KSTAT_KV_U_PACKETS, },
   8430  1.15  jmcneill 	{ "256-511B",		KSTAT_KV_U_PACKETS, },
   8431  1.15  jmcneill 	{ "512-1023B",		KSTAT_KV_U_PACKETS, },
   8432  1.15  jmcneill 	{ "1024-1518B",		KSTAT_KV_U_PACKETS, },
   8433  1.15  jmcneill 	{ "1519-2047B",		KSTAT_KV_U_PACKETS, },
   8434  1.15  jmcneill 	{ "2048-4095B",		KSTAT_KV_U_PACKETS, },
   8435  1.15  jmcneill 	{ "4096-8191B",		KSTAT_KV_U_PACKETS, },
   8436  1.15  jmcneill 	{ "8192-10239B",	KSTAT_KV_U_PACKETS, },
   8437  1.15  jmcneill };
   8438  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count);
   8439  1.15  jmcneill 
   8440  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = {
   8441  1.15  jmcneill 	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
   8442  1.15  jmcneill 	{ "FCS Errs",		KSTAT_KV_U_PACKETS, },
   8443  1.15  jmcneill 	{ "Single Colls",	KSTAT_KV_U_PACKETS, },
   8444  1.15  jmcneill 	{ "Multiple Colls",	KSTAT_KV_U_PACKETS, },
   8445  1.15  jmcneill 	{ "SQE Test Errs",	KSTAT_KV_U_NONE, },
   8446  1.15  jmcneill 	{ "Deferred Tx",	KSTAT_KV_U_PACKETS, },
   8447  1.15  jmcneill 	{ "Late Colls",		KSTAT_KV_U_NONE, },
   8448  1.15  jmcneill 	{ "Exess Colls",	KSTAT_KV_U_NONE, },
   8449  1.15  jmcneill 	{ "Int MAC Tx Errs",	KSTAT_KV_U_PACKETS, },
   8450  1.15  jmcneill 	{ "CSM Sense Errs",	KSTAT_KV_U_NONE, },
   8451  1.15  jmcneill 	{ "Too Long",		KSTAT_KV_U_PACKETS, },
   8452  1.15  jmcneill 	{ "Int MAC Rx Errs",	KSTAT_KV_U_PACKETS, },
   8453  1.15  jmcneill 	{ "Symbol Errs",	KSTAT_KV_U_NONE, },
   8454  1.15  jmcneill 	{ "Unknown Control",	KSTAT_KV_U_PACKETS, },
   8455  1.15  jmcneill 	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
   8456  1.15  jmcneill 	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
   8457  1.15  jmcneill };
   8458  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count);
   8459  1.15  jmcneill 
   8460  1.15  jmcneill struct mcx_kstat_ppcnt {
   8461  1.15  jmcneill 	const char		*ksp_name;
   8462  1.15  jmcneill 	const struct mcx_ppcnt	*ksp_tpl;
   8463  1.15  jmcneill 	unsigned int		 ksp_n;
   8464  1.15  jmcneill 	uint8_t			 ksp_grp;
   8465  1.15  jmcneill };
   8466  1.15  jmcneill 
   8467  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = {
   8468  1.15  jmcneill 	.ksp_name =		"ieee802.3",
   8469  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_ieee8023_tpl,
   8470  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_ieee8023_tpl),
   8471  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_IEEE8023,
   8472  1.15  jmcneill };
   8473  1.15  jmcneill 
   8474  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = {
   8475  1.15  jmcneill 	.ksp_name =		"rfc2863",
   8476  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc2863_tpl,
   8477  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc2863_tpl),
   8478  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2863,
   8479  1.15  jmcneill };
   8480  1.15  jmcneill 
   8481  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = {
   8482  1.15  jmcneill 	.ksp_name =		"rfc2819",
   8483  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc2819_tpl,
   8484  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc2819_tpl),
   8485  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2819,
   8486  1.15  jmcneill };
   8487  1.15  jmcneill 
   8488  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = {
   8489  1.15  jmcneill 	.ksp_name =		"rfc3635",
   8490  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc3635_tpl,
   8491  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc3635_tpl),
   8492  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC3635,
   8493  1.15  jmcneill };
   8494  1.15  jmcneill 
   8495  1.15  jmcneill static int	mcx_kstat_ppcnt_read(struct kstat *);
   8496  1.15  jmcneill 
   8497  1.15  jmcneill static void	mcx_kstat_attach_tmps(struct mcx_softc *sc);
   8498  1.15  jmcneill static void	mcx_kstat_attach_queues(struct mcx_softc *sc);
   8499  1.15  jmcneill 
   8500  1.15  jmcneill static struct kstat *
   8501  1.15  jmcneill mcx_kstat_attach_ppcnt(struct mcx_softc *sc,
   8502  1.15  jmcneill     const struct mcx_kstat_ppcnt *ksp)
   8503  1.15  jmcneill {
   8504  1.15  jmcneill 	struct kstat *ks;
   8505  1.15  jmcneill 	struct kstat_kv *kvs;
   8506  1.15  jmcneill 	unsigned int i;
   8507  1.15  jmcneill 
   8508  1.15  jmcneill 	ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0);
   8509  1.15  jmcneill 	if (ks == NULL)
   8510  1.15  jmcneill 		return (NULL);
   8511  1.15  jmcneill 
   8512  1.15  jmcneill 	kvs = mallocarray(ksp->ksp_n, sizeof(*kvs),
   8513  1.15  jmcneill 	    M_DEVBUF, M_WAITOK);
   8514  1.15  jmcneill 
   8515  1.15  jmcneill 	for (i = 0; i < ksp->ksp_n; i++) {
   8516  1.15  jmcneill 		const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i];
   8517  1.15  jmcneill 
   8518  1.15  jmcneill 		kstat_kv_unit_init(&kvs[i], tpl->name,
   8519  1.15  jmcneill 		    KSTAT_KV_T_COUNTER64, tpl->unit);
   8520  1.15  jmcneill 	}
   8521  1.15  jmcneill 
   8522  1.15  jmcneill 	ks->ks_softc = sc;
   8523  1.15  jmcneill 	ks->ks_ptr = (void *)ksp;
   8524  1.15  jmcneill 	ks->ks_data = kvs;
   8525  1.15  jmcneill 	ks->ks_datalen = ksp->ksp_n * sizeof(*kvs);
   8526  1.15  jmcneill 	ks->ks_read = mcx_kstat_ppcnt_read;
   8527  1.15  jmcneill 
   8528  1.15  jmcneill 	kstat_install(ks);
   8529  1.15  jmcneill 
   8530  1.15  jmcneill 	return (ks);
   8531  1.15  jmcneill }
   8532  1.15  jmcneill 
   8533  1.15  jmcneill static void
   8534  1.15  jmcneill mcx_kstat_attach(struct mcx_softc *sc)
   8535  1.15  jmcneill {
   8536  1.15  jmcneill 	sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc,
   8537  1.15  jmcneill 	    &mcx_kstat_ppcnt_ieee8023);
   8538  1.15  jmcneill 	sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc,
   8539  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc2863);
   8540  1.15  jmcneill 	sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc,
   8541  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc2819);
   8542  1.15  jmcneill 	sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc,
   8543  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc3635);
   8544  1.15  jmcneill 
   8545  1.15  jmcneill 	mcx_kstat_attach_tmps(sc);
   8546  1.15  jmcneill 	mcx_kstat_attach_queues(sc);
   8547  1.15  jmcneill }
   8548  1.15  jmcneill 
   8549  1.15  jmcneill static int
   8550  1.15  jmcneill mcx_kstat_ppcnt_read(struct kstat *ks)
   8551  1.15  jmcneill {
   8552  1.15  jmcneill 	struct mcx_softc *sc = ks->ks_softc;
   8553  1.15  jmcneill 	struct mcx_kstat_ppcnt *ksp = ks->ks_ptr;
   8554  1.15  jmcneill 	struct mcx_reg_ppcnt ppcnt = {
   8555  1.15  jmcneill 		.ppcnt_grp = ksp->ksp_grp,
   8556  1.15  jmcneill 		.ppcnt_local_port = 1,
   8557  1.15  jmcneill 	};
   8558  1.15  jmcneill 	struct kstat_kv *kvs = ks->ks_data;
   8559  1.15  jmcneill 	uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set;
   8560  1.15  jmcneill 	unsigned int i;
   8561  1.15  jmcneill 	int rv;
   8562  1.15  jmcneill 
   8563  1.15  jmcneill 	KERNEL_LOCK(); /* XXX */
   8564  1.15  jmcneill 	rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ,
   8565  1.15  jmcneill 	    &ppcnt, sizeof(ppcnt));
   8566  1.15  jmcneill 	KERNEL_UNLOCK();
   8567  1.15  jmcneill 	if (rv != 0)
   8568  1.15  jmcneill 		return (EIO);
   8569  1.15  jmcneill 
   8570  1.15  jmcneill 	nanouptime(&ks->ks_updated);
   8571  1.15  jmcneill 
   8572  1.15  jmcneill 	for (i = 0; i < ksp->ksp_n; i++)
   8573  1.15  jmcneill 		kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]);
   8574  1.15  jmcneill 
   8575  1.15  jmcneill 	return (0);
   8576  1.15  jmcneill }
   8577  1.15  jmcneill 
   8578  1.15  jmcneill struct mcx_kstat_mtmp {
   8579  1.15  jmcneill 	struct kstat_kv		ktmp_name;
   8580  1.15  jmcneill 	struct kstat_kv		ktmp_temperature;
   8581  1.15  jmcneill 	struct kstat_kv		ktmp_threshold_lo;
   8582  1.15  jmcneill 	struct kstat_kv		ktmp_threshold_hi;
   8583  1.15  jmcneill };
   8584  1.15  jmcneill 
   8585  1.15  jmcneill static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = {
   8586  1.15  jmcneill 	KSTAT_KV_INITIALIZER("name",		KSTAT_KV_T_ISTR),
   8587  1.15  jmcneill 	KSTAT_KV_INITIALIZER("temperature",	KSTAT_KV_T_TEMP),
   8588  1.15  jmcneill 	KSTAT_KV_INITIALIZER("lo threshold",	KSTAT_KV_T_TEMP),
   8589  1.15  jmcneill 	KSTAT_KV_INITIALIZER("hi threshold",	KSTAT_KV_T_TEMP),
   8590  1.15  jmcneill };
   8591  1.15  jmcneill 
   8592  1.15  jmcneill static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 };
   8593  1.15  jmcneill 
   8594  1.15  jmcneill static int mcx_kstat_mtmp_read(struct kstat *);
   8595  1.15  jmcneill 
   8596  1.15  jmcneill static void
   8597  1.15  jmcneill mcx_kstat_attach_tmps(struct mcx_softc *sc)
   8598  1.15  jmcneill {
   8599  1.15  jmcneill 	struct kstat *ks;
   8600  1.15  jmcneill 	struct mcx_reg_mcam mcam;
   8601  1.15  jmcneill 	struct mcx_reg_mtcap mtcap;
   8602  1.15  jmcneill 	struct mcx_kstat_mtmp *ktmp;
   8603  1.15  jmcneill 	uint64_t map;
   8604  1.15  jmcneill 	unsigned int i, n;
   8605  1.15  jmcneill 
   8606  1.15  jmcneill 	memset(&mtcap, 0, sizeof(mtcap));
   8607  1.15  jmcneill 	memset(&mcam, 0, sizeof(mcam));
   8608  1.15  jmcneill 
   8609  1.15  jmcneill 	if (sc->sc_mcam_reg == 0) {
   8610  1.15  jmcneill 		/* no management capabilities */
   8611  1.15  jmcneill 		return;
   8612  1.15  jmcneill 	}
   8613  1.15  jmcneill 
   8614  1.15  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ,
   8615  1.15  jmcneill 	    &mcam, sizeof(mcam)) != 0) {
   8616  1.15  jmcneill 		/* unable to check management capabilities? */
   8617  1.15  jmcneill 		return;
   8618  1.15  jmcneill 	}
   8619  1.15  jmcneill 
   8620  1.15  jmcneill 	if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask,
   8621  1.15  jmcneill 	    MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) {
   8622  1.15  jmcneill 		/* no sensor map */
   8623  1.15  jmcneill 		return;
   8624  1.15  jmcneill 	}
   8625  1.15  jmcneill 
   8626  1.15  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ,
   8627  1.15  jmcneill 	    &mtcap, sizeof(mtcap)) != 0) {
   8628  1.15  jmcneill 		/* unable to find temperature sensors */
   8629  1.15  jmcneill 		return;
   8630  1.15  jmcneill 	}
   8631  1.15  jmcneill 
   8632  1.15  jmcneill 	sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count;
   8633  1.15  jmcneill 	sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count,
   8634  1.15  jmcneill 	    sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK);
   8635  1.15  jmcneill 
   8636  1.15  jmcneill 	n = 0;
   8637  1.15  jmcneill 	map = bemtoh64(&mtcap.mtcap_sensor_map);
   8638  1.15  jmcneill 	for (i = 0; i < sizeof(map) * NBBY; i++) {
   8639  1.15  jmcneill 		if (!ISSET(map, (1ULL << i)))
   8640  1.15  jmcneill 			continue;
   8641  1.15  jmcneill 
   8642  1.15  jmcneill 		ks = kstat_create(DEVNAME(sc), 0, "temperature", i,
   8643  1.15  jmcneill 		    KSTAT_T_KV, 0);
   8644  1.15  jmcneill 		if (ks == NULL) {
   8645  1.15  jmcneill 			/* unable to attach temperature sensor %u, i */
   8646  1.15  jmcneill 			continue;
   8647  1.15  jmcneill 		}
   8648  1.15  jmcneill 
   8649  1.15  jmcneill 		ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO);
   8650  1.15  jmcneill 		*ktmp = mcx_kstat_mtmp_tpl;
   8651  1.15  jmcneill 
   8652  1.15  jmcneill 		ks->ks_data = ktmp;
   8653  1.15  jmcneill 		ks->ks_datalen = sizeof(*ktmp);
   8654  1.15  jmcneill 		TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval);
   8655  1.15  jmcneill 		ks->ks_read = mcx_kstat_mtmp_read;
   8656  1.15  jmcneill 
   8657  1.15  jmcneill 		ks->ks_softc = sc;
   8658  1.15  jmcneill 		kstat_install(ks);
   8659  1.15  jmcneill 
   8660  1.15  jmcneill 		sc->sc_kstat_mtmp[n++] = ks;
   8661  1.15  jmcneill 		if (n >= sc->sc_kstat_mtmp_count)
   8662  1.15  jmcneill 			break;
   8663  1.15  jmcneill 	}
   8664  1.15  jmcneill }
   8665  1.15  jmcneill 
   8666  1.15  jmcneill static uint64_t
   8667  1.15  jmcneill mcx_tmp_to_uK(uint16_t *t)
   8668  1.15  jmcneill {
   8669  1.15  jmcneill 	int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */
   8670  1.15  jmcneill 	mt *= 1000000 / 8; /* convert to uC */
   8671  1.15  jmcneill 	mt += 273150000; /* convert to uK */
   8672  1.15  jmcneill 
   8673  1.15  jmcneill 	return (mt);
   8674  1.15  jmcneill }
   8675  1.15  jmcneill 
   8676  1.15  jmcneill static int
   8677  1.15  jmcneill mcx_kstat_mtmp_read(struct kstat *ks)
   8678  1.15  jmcneill {
   8679  1.15  jmcneill 	struct mcx_softc *sc = ks->ks_softc;
   8680  1.15  jmcneill 	struct mcx_kstat_mtmp *ktmp = ks->ks_data;
   8681  1.15  jmcneill 	struct mcx_reg_mtmp mtmp;
   8682  1.15  jmcneill 	int rv;
   8683  1.15  jmcneill 	struct timeval updated;
   8684  1.15  jmcneill 
   8685  1.15  jmcneill 	TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated);
   8686  1.15  jmcneill 
   8687  1.15  jmcneill 	if (!ratecheck(&updated, &mcx_kstat_mtmp_rate))
   8688  1.15  jmcneill 		return (0);
   8689  1.15  jmcneill 
   8690  1.15  jmcneill 	memset(&mtmp, 0, sizeof(mtmp));
   8691  1.15  jmcneill 	htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit);
   8692  1.15  jmcneill 
   8693  1.15  jmcneill 	KERNEL_LOCK(); /* XXX */
   8694  1.15  jmcneill 	rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ,
   8695  1.15  jmcneill 	    &mtmp, sizeof(mtmp));
   8696  1.15  jmcneill 	KERNEL_UNLOCK();
   8697  1.15  jmcneill 	if (rv != 0)
   8698  1.15  jmcneill 		return (EIO);
   8699  1.15  jmcneill 
   8700  1.15  jmcneill 	memset(kstat_kv_istr(&ktmp->ktmp_name), 0,
   8701  1.15  jmcneill 	    sizeof(kstat_kv_istr(&ktmp->ktmp_name)));
   8702  1.15  jmcneill 	memcpy(kstat_kv_istr(&ktmp->ktmp_name),
   8703  1.15  jmcneill 	    mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name));
   8704  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_temperature) =
   8705  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature);
   8706  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_threshold_lo) =
   8707  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo);
   8708  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_threshold_hi) =
   8709  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi);
   8710  1.15  jmcneill 
   8711  1.15  jmcneill 	TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated);
   8712  1.15  jmcneill 
   8713  1.15  jmcneill 	return (0);
   8714  1.15  jmcneill }
   8715  1.15  jmcneill 
   8716  1.15  jmcneill struct mcx_queuestat {
   8717  1.15  jmcneill 	char			 name[KSTAT_KV_NAMELEN];
   8718  1.15  jmcneill 	enum kstat_kv_type	 type;
   8719  1.15  jmcneill };
   8720  1.15  jmcneill 
   8721  1.15  jmcneill static const struct mcx_queuestat mcx_queue_kstat_tpl[] = {
   8722  1.15  jmcneill 	{ "RQ SW prod",		KSTAT_KV_T_COUNTER64 },
   8723  1.15  jmcneill 	{ "RQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8724  1.15  jmcneill 	{ "RQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8725  1.15  jmcneill 	{ "RQ HW state",	KSTAT_KV_T_ISTR },
   8726  1.15  jmcneill 
   8727  1.15  jmcneill 	{ "SQ SW prod",		KSTAT_KV_T_COUNTER64 },
   8728  1.15  jmcneill 	{ "SQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8729  1.15  jmcneill 	{ "SQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8730  1.15  jmcneill 	{ "SQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8731  1.15  jmcneill 	{ "SQ HW state",	KSTAT_KV_T_ISTR },
   8732  1.15  jmcneill 
   8733  1.15  jmcneill 	{ "CQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8734  1.15  jmcneill 	{ "CQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8735  1.15  jmcneill 	{ "CQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8736  1.15  jmcneill 	{ "CQ HW notify",	KSTAT_KV_T_COUNTER64 },
   8737  1.15  jmcneill 	{ "CQ HW solicit",	KSTAT_KV_T_COUNTER64 },
   8738  1.15  jmcneill 	{ "CQ HW status",	KSTAT_KV_T_ISTR },
   8739  1.15  jmcneill 	{ "CQ HW state",	KSTAT_KV_T_ISTR },
   8740  1.15  jmcneill 
   8741  1.15  jmcneill 	{ "EQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8742  1.15  jmcneill 	{ "EQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8743  1.15  jmcneill 	{ "EQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8744  1.15  jmcneill 	{ "EQ HW status",	KSTAT_KV_T_ISTR },
   8745  1.15  jmcneill 	{ "EQ HW state",	KSTAT_KV_T_ISTR },
   8746  1.15  jmcneill };
   8747  1.15  jmcneill 
   8748  1.15  jmcneill static int	mcx_kstat_queue_read(struct kstat *);
   8749  1.15  jmcneill 
   8750  1.15  jmcneill static void
   8751  1.15  jmcneill mcx_kstat_attach_queues(struct mcx_softc *sc)
   8752  1.15  jmcneill {
   8753  1.15  jmcneill 	struct kstat *ks;
   8754  1.15  jmcneill 	struct kstat_kv *kvs;
   8755  1.15  jmcneill 	int q, i;
   8756  1.15  jmcneill 
   8757  1.15  jmcneill 	for (q = 0; q < sc->sc_nqueues; q++) {
   8758  1.15  jmcneill 		ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
   8759  1.15  jmcneill 		    KSTAT_T_KV, 0);
   8760  1.15  jmcneill 		if (ks == NULL) {
   8761  1.15  jmcneill 			/* unable to attach queue stats %u, q */
   8762  1.15  jmcneill 			continue;
   8763  1.15  jmcneill 		}
   8764  1.15  jmcneill 
   8765  1.15  jmcneill 		kvs = mallocarray(nitems(mcx_queue_kstat_tpl),
   8766  1.15  jmcneill 		    sizeof(*kvs), M_DEVBUF, M_WAITOK);
   8767  1.15  jmcneill 
   8768  1.15  jmcneill 		for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) {
   8769  1.15  jmcneill 			const struct mcx_queuestat *tpl =
   8770  1.15  jmcneill 			    &mcx_queue_kstat_tpl[i];
   8771  1.15  jmcneill 
   8772  1.15  jmcneill 			kstat_kv_init(&kvs[i], tpl->name, tpl->type);
   8773  1.15  jmcneill 		}
   8774  1.15  jmcneill 
   8775  1.15  jmcneill 		ks->ks_softc = &sc->sc_queues[q];
   8776  1.15  jmcneill 		ks->ks_data = kvs;
   8777  1.15  jmcneill 		ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs);
   8778  1.15  jmcneill 		ks->ks_read = mcx_kstat_queue_read;
   8779  1.15  jmcneill 
   8780  1.15  jmcneill 		sc->sc_queues[q].q_kstat = ks;
   8781  1.15  jmcneill 		kstat_install(ks);
   8782  1.15  jmcneill 	}
   8783  1.15  jmcneill }
   8784  1.15  jmcneill 
   8785  1.15  jmcneill static int
   8786  1.15  jmcneill mcx_kstat_queue_read(struct kstat *ks)
   8787  1.15  jmcneill {
   8788  1.15  jmcneill 	struct mcx_queues *q = ks->ks_softc;
   8789  1.15  jmcneill 	struct mcx_softc *sc = q->q_sc;
   8790  1.15  jmcneill 	struct kstat_kv *kvs = ks->ks_data;
   8791  1.15  jmcneill 	union {
   8792  1.15  jmcneill 		struct mcx_rq_ctx rq;
   8793  1.15  jmcneill 		struct mcx_sq_ctx sq;
   8794  1.15  jmcneill 		struct mcx_cq_ctx cq;
   8795  1.15  jmcneill 		struct mcx_eq_ctx eq;
   8796  1.15  jmcneill 	} u;
   8797  1.15  jmcneill 	const char *text;
   8798  1.15  jmcneill 	int error = 0;
   8799  1.15  jmcneill 
   8800  1.15  jmcneill 	KERNEL_LOCK();
   8801  1.24     skrll 
   8802  1.15  jmcneill 	if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) {
   8803  1.15  jmcneill 		error = EIO;
   8804  1.15  jmcneill 		goto out;
   8805  1.15  jmcneill 	}
   8806  1.15  jmcneill 
   8807  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_rx.rx_prod;
   8808  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter);
   8809  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter);
   8810  1.15  jmcneill 	switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >>
   8811  1.15  jmcneill 	    MCX_RQ_CTX_STATE_SHIFT) {
   8812  1.15  jmcneill 	case MCX_RQ_CTX_STATE_RST:
   8813  1.15  jmcneill 		text = "RST";
   8814  1.15  jmcneill 		break;
   8815  1.15  jmcneill 	case MCX_RQ_CTX_STATE_RDY:
   8816  1.15  jmcneill 		text = "RDY";
   8817  1.15  jmcneill 		break;
   8818  1.15  jmcneill 	case MCX_RQ_CTX_STATE_ERR:
   8819  1.15  jmcneill 		text = "ERR";
   8820  1.15  jmcneill 		break;
   8821  1.15  jmcneill 	default:
   8822  1.15  jmcneill 		text = "unknown";
   8823  1.15  jmcneill 		break;
   8824  1.15  jmcneill 	}
   8825  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8826  1.15  jmcneill 	kvs++;
   8827  1.15  jmcneill 
   8828  1.15  jmcneill 	if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) {
   8829  1.15  jmcneill 		error = EIO;
   8830  1.15  jmcneill 		goto out;
   8831  1.15  jmcneill 	}
   8832  1.15  jmcneill 
   8833  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_tx.tx_prod;
   8834  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_tx.tx_cons;
   8835  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter);
   8836  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter);
   8837  1.15  jmcneill 	switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >>
   8838  1.15  jmcneill 	    MCX_SQ_CTX_STATE_SHIFT) {
   8839  1.15  jmcneill 	case MCX_SQ_CTX_STATE_RST:
   8840  1.15  jmcneill 		text = "RST";
   8841  1.15  jmcneill 		break;
   8842  1.15  jmcneill 	case MCX_SQ_CTX_STATE_RDY:
   8843  1.15  jmcneill 		text = "RDY";
   8844  1.15  jmcneill 		break;
   8845  1.15  jmcneill 	case MCX_SQ_CTX_STATE_ERR:
   8846  1.15  jmcneill 		text = "ERR";
   8847  1.15  jmcneill 		break;
   8848  1.15  jmcneill 	default:
   8849  1.15  jmcneill 		text = "unknown";
   8850  1.15  jmcneill 		break;
   8851  1.15  jmcneill 	}
   8852  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8853  1.15  jmcneill 	kvs++;
   8854  1.15  jmcneill 
   8855  1.15  jmcneill 	if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) {
   8856  1.15  jmcneill 		error = EIO;
   8857  1.15  jmcneill 		goto out;
   8858  1.15  jmcneill 	}
   8859  1.15  jmcneill 
   8860  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_cq.cq_cons;
   8861  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter);
   8862  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter);
   8863  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified);
   8864  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit);
   8865  1.15  jmcneill 
   8866  1.15  jmcneill 	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >>
   8867  1.15  jmcneill 	    MCX_CQ_CTX_STATUS_SHIFT) {
   8868  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_OK:
   8869  1.15  jmcneill 		text = "OK";
   8870  1.15  jmcneill 		break;
   8871  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_OVERFLOW:
   8872  1.15  jmcneill 		text = "overflow";
   8873  1.15  jmcneill 		break;
   8874  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_WRITE_FAIL:
   8875  1.15  jmcneill 		text = "write fail";
   8876  1.15  jmcneill 		break;
   8877  1.15  jmcneill 	default:
   8878  1.15  jmcneill 		text = "unknown";
   8879  1.15  jmcneill 		break;
   8880  1.15  jmcneill 	}
   8881  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8882  1.15  jmcneill 	kvs++;
   8883  1.15  jmcneill 
   8884  1.15  jmcneill 	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >>
   8885  1.15  jmcneill 	    MCX_CQ_CTX_STATE_SHIFT) {
   8886  1.15  jmcneill 	case MCX_CQ_CTX_STATE_SOLICITED:
   8887  1.15  jmcneill 		text = "solicited";
   8888  1.15  jmcneill 		break;
   8889  1.15  jmcneill 	case MCX_CQ_CTX_STATE_ARMED:
   8890  1.15  jmcneill 		text = "armed";
   8891  1.15  jmcneill 		break;
   8892  1.15  jmcneill 	case MCX_CQ_CTX_STATE_FIRED:
   8893  1.15  jmcneill 		text = "fired";
   8894  1.15  jmcneill 		break;
   8895  1.15  jmcneill 	default:
   8896  1.15  jmcneill 		text = "unknown";
   8897  1.15  jmcneill 		break;
   8898  1.15  jmcneill 	}
   8899  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8900  1.15  jmcneill 	kvs++;
   8901  1.15  jmcneill 
   8902  1.15  jmcneill 	if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) {
   8903  1.15  jmcneill 		error = EIO;
   8904  1.15  jmcneill 		goto out;
   8905  1.15  jmcneill 	}
   8906  1.15  jmcneill 
   8907  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_eq.eq_cons;
   8908  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter);
   8909  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter);
   8910  1.15  jmcneill 
   8911  1.15  jmcneill 	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >>
   8912  1.15  jmcneill 	    MCX_EQ_CTX_STATUS_SHIFT) {
   8913  1.15  jmcneill 	case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE:
   8914  1.15  jmcneill 		text = "write fail";
   8915  1.15  jmcneill 		break;
   8916  1.15  jmcneill 	case MCX_EQ_CTX_STATUS_OK:
   8917  1.15  jmcneill 		text = "OK";
   8918  1.15  jmcneill 		break;
   8919  1.15  jmcneill 	default:
   8920  1.15  jmcneill 		text = "unknown";
   8921  1.15  jmcneill 		break;
   8922  1.15  jmcneill 	}
   8923  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8924  1.15  jmcneill 	kvs++;
   8925  1.15  jmcneill 
   8926  1.15  jmcneill 	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >>
   8927  1.15  jmcneill 	    MCX_EQ_CTX_STATE_SHIFT) {
   8928  1.15  jmcneill 	case MCX_EQ_CTX_STATE_ARMED:
   8929  1.15  jmcneill 		text = "armed";
   8930  1.15  jmcneill 		break;
   8931  1.15  jmcneill 	case MCX_EQ_CTX_STATE_FIRED:
   8932  1.15  jmcneill 		text = "fired";
   8933  1.15  jmcneill 		break;
   8934  1.15  jmcneill 	default:
   8935  1.15  jmcneill 		text = "unknown";
   8936  1.15  jmcneill 		break;
   8937  1.15  jmcneill 	}
   8938  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8939  1.15  jmcneill 	kvs++;
   8940  1.15  jmcneill 
   8941  1.15  jmcneill 	nanouptime(&ks->ks_updated);
   8942  1.15  jmcneill out:
   8943  1.15  jmcneill 	KERNEL_UNLOCK();
   8944  1.15  jmcneill 	return (error);
   8945  1.15  jmcneill }
   8946  1.15  jmcneill 
   8947  1.15  jmcneill #endif /* NKSTAT > 0 */
   8948  1.15  jmcneill 
   8949  1.15  jmcneill static unsigned int
   8950  1.15  jmcneill mcx_timecounter_read(struct timecounter *tc)
   8951  1.15  jmcneill {
   8952  1.15  jmcneill 	struct mcx_softc *sc = tc->tc_priv;
   8953  1.15  jmcneill 
   8954  1.15  jmcneill 	return (mcx_rd(sc, MCX_INTERNAL_TIMER_L));
   8955  1.15  jmcneill }
   8956  1.15  jmcneill 
   8957  1.15  jmcneill static void
   8958  1.15  jmcneill mcx_timecounter_attach(struct mcx_softc *sc)
   8959  1.15  jmcneill {
   8960  1.15  jmcneill 	struct timecounter *tc = &sc->sc_timecounter;
   8961  1.15  jmcneill 
   8962  1.15  jmcneill 	tc->tc_get_timecount = mcx_timecounter_read;
   8963  1.15  jmcneill 	tc->tc_counter_mask = ~0U;
   8964  1.15  jmcneill 	tc->tc_frequency = sc->sc_khz * 1000;
   8965  1.15  jmcneill 	tc->tc_name = device_xname(sc->sc_dev);
   8966  1.15  jmcneill 	tc->tc_quality = -100;
   8967  1.15  jmcneill 	tc->tc_priv = sc;
   8968  1.15  jmcneill 
   8969  1.15  jmcneill 	tc_init(tc);
   8970  1.15  jmcneill }
   8971