Home | History | Annotate | Line # | Download | only in pci
if_mcx.c revision 1.17
      1  1.17  jmcneill /*	$NetBSD: if_mcx.c,v 1.17 2021/02/20 13:31:35 jmcneill Exp $ */
      2  1.17  jmcneill /*	$OpenBSD: if_mcx.c,v 1.99 2021/02/15 03:42:00 dlg Exp $ */
      3   1.1  jmcneill 
      4   1.1  jmcneill /*
      5   1.1  jmcneill  * Copyright (c) 2017 David Gwynne <dlg (at) openbsd.org>
      6   1.1  jmcneill  * Copyright (c) 2019 Jonathan Matthew <jmatthew (at) openbsd.org>
      7   1.1  jmcneill  *
      8   1.1  jmcneill  * Permission to use, copy, modify, and distribute this software for any
      9   1.1  jmcneill  * purpose with or without fee is hereby granted, provided that the above
     10   1.1  jmcneill  * copyright notice and this permission notice appear in all copies.
     11   1.1  jmcneill  *
     12   1.1  jmcneill  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     13   1.1  jmcneill  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     14   1.1  jmcneill  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     15   1.1  jmcneill  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     16   1.1  jmcneill  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     17   1.1  jmcneill  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     18   1.1  jmcneill  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     19   1.1  jmcneill  */
     20   1.1  jmcneill 
     21   1.1  jmcneill #ifdef _KERNEL_OPT
     22   1.1  jmcneill #include "opt_net_mpsafe.h"
     23   1.1  jmcneill #endif
     24   1.1  jmcneill 
     25  1.14  jmcneill #include <sys/cdefs.h>
     26  1.17  jmcneill __KERNEL_RCSID(0, "$NetBSD: if_mcx.c,v 1.17 2021/02/20 13:31:35 jmcneill Exp $");
     27  1.14  jmcneill 
     28   1.1  jmcneill #include <sys/param.h>
     29   1.1  jmcneill #include <sys/systm.h>
     30   1.1  jmcneill #include <sys/sockio.h>
     31   1.1  jmcneill #include <sys/mbuf.h>
     32   1.1  jmcneill #include <sys/kernel.h>
     33   1.1  jmcneill #include <sys/socket.h>
     34   1.1  jmcneill #include <sys/device.h>
     35   1.1  jmcneill #include <sys/pool.h>
     36   1.1  jmcneill #include <sys/queue.h>
     37   1.1  jmcneill #include <sys/callout.h>
     38   1.1  jmcneill #include <sys/workqueue.h>
     39   1.1  jmcneill #include <sys/atomic.h>
     40  1.15  jmcneill #include <sys/timetc.h>
     41   1.1  jmcneill #include <sys/kmem.h>
     42   1.1  jmcneill #include <sys/bus.h>
     43  1.15  jmcneill #include <sys/interrupt.h>
     44  1.16  jmcneill #include <sys/pcq.h>
     45  1.16  jmcneill #include <sys/cpu.h>
     46   1.1  jmcneill 
     47   1.1  jmcneill #include <machine/intr.h>
     48   1.1  jmcneill 
     49   1.1  jmcneill #include <net/if.h>
     50   1.1  jmcneill #include <net/if_dl.h>
     51   1.1  jmcneill #include <net/if_ether.h>
     52   1.1  jmcneill #include <net/if_media.h>
     53  1.15  jmcneill #include <net/if_vlanvar.h>
     54  1.15  jmcneill #include <net/toeplitz.h>
     55   1.1  jmcneill 
     56   1.1  jmcneill #include <net/bpf.h>
     57   1.1  jmcneill 
     58   1.1  jmcneill #include <netinet/in.h>
     59   1.1  jmcneill 
     60   1.1  jmcneill #include <dev/pci/pcireg.h>
     61   1.1  jmcneill #include <dev/pci/pcivar.h>
     62   1.1  jmcneill #include <dev/pci/pcidevs.h>
     63   1.1  jmcneill 
     64  1.15  jmcneill /* TODO: Port kstat key/value stuff to evcnt/sysmon */
     65  1.15  jmcneill #define	NKSTAT		0
     66  1.15  jmcneill 
     67  1.12   thorpej /* XXX This driver is not yet MP-safe; don't claim to be! */
     68  1.12   thorpej /* #ifdef NET_MPSAFE */
     69  1.12   thorpej /* #define	MCX_MPSAFE	1 */
     70  1.12   thorpej /* #define	CALLOUT_FLAGS	CALLOUT_MPSAFE */
     71  1.12   thorpej /* #else */
     72   1.1  jmcneill #define	CALLOUT_FLAGS	0
     73  1.12   thorpej /* #endif */
     74   1.1  jmcneill 
     75  1.16  jmcneill #define	MCX_TXQ_NUM		2048
     76  1.16  jmcneill 
     77   1.1  jmcneill #define BUS_DMASYNC_PRERW	(BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
     78   1.1  jmcneill #define BUS_DMASYNC_POSTRW	(BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
     79   1.1  jmcneill 
     80   1.1  jmcneill #define MCX_HCA_BAR	PCI_MAPREG_START /* BAR 0 */
     81   1.1  jmcneill 
     82  1.15  jmcneill #define MCX_FW_VER			0x0000
     83  1.15  jmcneill #define  MCX_FW_VER_MAJOR(_v)			((_v) & 0xffff)
     84  1.15  jmcneill #define  MCX_FW_VER_MINOR(_v)			((_v) >> 16)
     85  1.15  jmcneill #define MCX_CMDIF_FW_SUBVER		0x0004
     86  1.15  jmcneill #define  MCX_FW_VER_SUBMINOR(_v)		((_v) & 0xffff)
     87  1.15  jmcneill #define  MCX_CMDIF(_v)				((_v) >> 16)
     88   1.1  jmcneill 
     89  1.15  jmcneill #define MCX_ISSI			1 /* as per the PRM */
     90  1.15  jmcneill #define MCX_CMD_IF_SUPPORTED		5
     91   1.1  jmcneill 
     92  1.15  jmcneill #define MCX_HARDMTU			9500
     93   1.1  jmcneill 
     94  1.15  jmcneill #define MCX_PAGE_SHIFT			12
     95  1.15  jmcneill #define MCX_PAGE_SIZE			(1 << MCX_PAGE_SHIFT)
     96   1.1  jmcneill 
     97   1.1  jmcneill /* queue sizes */
     98  1.15  jmcneill #define MCX_LOG_EQ_SIZE			7
     99  1.15  jmcneill #define MCX_LOG_CQ_SIZE			12
    100  1.15  jmcneill #define MCX_LOG_RQ_SIZE			10
    101  1.15  jmcneill #define MCX_LOG_SQ_SIZE			11
    102  1.15  jmcneill 
    103  1.15  jmcneill #define MCX_MAX_QUEUES			16
    104   1.1  jmcneill 
    105   1.1  jmcneill /* completion event moderation - about 10khz, or 90% of the cq */
    106  1.15  jmcneill #define MCX_CQ_MOD_PERIOD		50
    107  1.15  jmcneill #define MCX_CQ_MOD_COUNTER		\
    108  1.15  jmcneill 	(((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)
    109  1.15  jmcneill 
    110  1.15  jmcneill #define MCX_LOG_SQ_ENTRY_SIZE		6
    111  1.15  jmcneill #define MCX_SQ_ENTRY_MAX_SLOTS		4
    112  1.15  jmcneill #define MCX_SQ_SEGS_PER_SLOT		\
    113   1.1  jmcneill 	(sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
    114  1.15  jmcneill #define MCX_SQ_MAX_SEGMENTS		\
    115   1.1  jmcneill 	1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)
    116   1.1  jmcneill 
    117  1.15  jmcneill #define MCX_LOG_FLOW_TABLE_SIZE		5
    118  1.15  jmcneill #define MCX_NUM_STATIC_FLOWS		4 /* promisc, allmulti, ucast, bcast */
    119  1.15  jmcneill #define MCX_NUM_MCAST_FLOWS 		\
    120   1.1  jmcneill 	((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)
    121   1.1  jmcneill 
    122  1.15  jmcneill #define MCX_SQ_INLINE_SIZE		18
    123  1.15  jmcneill CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE);
    124   1.1  jmcneill 
    125   1.1  jmcneill /* doorbell offsets */
    126  1.15  jmcneill #define MCX_DOORBELL_AREA_SIZE		MCX_PAGE_SIZE
    127   1.1  jmcneill 
    128  1.15  jmcneill #define MCX_CQ_DOORBELL_BASE		0
    129  1.15  jmcneill #define MCX_CQ_DOORBELL_STRIDE		64
    130  1.15  jmcneill 
    131  1.15  jmcneill #define MCX_WQ_DOORBELL_BASE		MCX_PAGE_SIZE/2
    132  1.15  jmcneill #define MCX_WQ_DOORBELL_STRIDE		64
    133  1.15  jmcneill /* make sure the doorbells fit */
    134  1.15  jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE);
    135  1.15  jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE <
    136  1.15  jmcneill     MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE);
    137  1.15  jmcneill 
    138  1.15  jmcneill #define MCX_WQ_DOORBELL_MASK		0xffff
    139   1.1  jmcneill 
    140   1.1  jmcneill /* uar registers */
    141  1.15  jmcneill #define MCX_UAR_CQ_DOORBELL		0x20
    142  1.15  jmcneill #define MCX_UAR_EQ_DOORBELL_ARM		0x40
    143  1.15  jmcneill #define MCX_UAR_EQ_DOORBELL		0x48
    144  1.15  jmcneill #define MCX_UAR_BF			0x800
    145   1.1  jmcneill 
    146  1.15  jmcneill #define MCX_CMDQ_ADDR_HI		0x0010
    147  1.15  jmcneill #define MCX_CMDQ_ADDR_LO		0x0014
    148   1.1  jmcneill #define MCX_CMDQ_ADDR_NMASK		0xfff
    149   1.1  jmcneill #define MCX_CMDQ_LOG_SIZE(_v)		((_v) >> 4 & 0xf)
    150   1.1  jmcneill #define MCX_CMDQ_LOG_STRIDE(_v)		((_v) >> 0 & 0xf)
    151   1.1  jmcneill #define MCX_CMDQ_INTERFACE_MASK		(0x3 << 8)
    152   1.1  jmcneill #define MCX_CMDQ_INTERFACE_FULL_DRIVER	(0x0 << 8)
    153   1.1  jmcneill #define MCX_CMDQ_INTERFACE_DISABLED	(0x1 << 8)
    154   1.1  jmcneill 
    155   1.1  jmcneill #define MCX_CMDQ_DOORBELL		0x0018
    156   1.1  jmcneill 
    157  1.15  jmcneill #define MCX_STATE			0x01fc
    158  1.15  jmcneill #define MCX_STATE_MASK				(1U << 31)
    159  1.15  jmcneill #define MCX_STATE_INITIALIZING			(1 << 31)
    160  1.15  jmcneill #define MCX_STATE_READY				(0 << 31)
    161  1.15  jmcneill #define MCX_STATE_INTERFACE_MASK		(0x3 << 24)
    162  1.15  jmcneill #define MCX_STATE_INTERFACE_FULL_DRIVER		(0x0 << 24)
    163  1.15  jmcneill #define MCX_STATE_INTERFACE_DISABLED		(0x1 << 24)
    164  1.15  jmcneill 
    165  1.15  jmcneill #define MCX_INTERNAL_TIMER		0x1000
    166  1.15  jmcneill #define MCX_INTERNAL_TIMER_H		0x1000
    167  1.15  jmcneill #define MCX_INTERNAL_TIMER_L		0x1004
    168  1.15  jmcneill 
    169  1.15  jmcneill #define MCX_CLEAR_INT			0x100c
    170  1.15  jmcneill 
    171  1.15  jmcneill #define MCX_REG_OP_WRITE		0
    172  1.15  jmcneill #define MCX_REG_OP_READ			1
    173  1.15  jmcneill 
    174  1.15  jmcneill #define MCX_REG_PMLP			0x5002
    175  1.15  jmcneill #define MCX_REG_PMTU			0x5003
    176  1.15  jmcneill #define MCX_REG_PTYS			0x5004
    177  1.15  jmcneill #define MCX_REG_PAOS			0x5006
    178  1.15  jmcneill #define MCX_REG_PFCC			0x5007
    179  1.15  jmcneill #define MCX_REG_PPCNT			0x5008
    180  1.15  jmcneill #define MCX_REG_MTCAP			0x9009 /* mgmt temp capabilities */
    181  1.15  jmcneill #define MCX_REG_MTMP			0x900a /* mgmt temp */
    182  1.15  jmcneill #define MCX_REG_MCIA			0x9014
    183  1.15  jmcneill #define MCX_REG_MCAM			0x907f
    184  1.15  jmcneill 
    185  1.15  jmcneill #define MCX_ETHER_CAP_SGMII		0
    186  1.15  jmcneill #define MCX_ETHER_CAP_1000_KX		1
    187  1.15  jmcneill #define MCX_ETHER_CAP_10G_CX4		2
    188  1.15  jmcneill #define MCX_ETHER_CAP_10G_KX4		3
    189  1.15  jmcneill #define MCX_ETHER_CAP_10G_KR		4
    190  1.15  jmcneill #define MCX_ETHER_CAP_20G_KR2		5
    191  1.15  jmcneill #define MCX_ETHER_CAP_40G_CR4		6
    192  1.15  jmcneill #define MCX_ETHER_CAP_40G_KR4		7
    193  1.15  jmcneill #define MCX_ETHER_CAP_56G_R4		8
    194  1.15  jmcneill #define MCX_ETHER_CAP_10G_CR		12
    195  1.15  jmcneill #define MCX_ETHER_CAP_10G_SR		13
    196  1.15  jmcneill #define MCX_ETHER_CAP_10G_LR		14
    197  1.15  jmcneill #define MCX_ETHER_CAP_40G_SR4		15
    198  1.15  jmcneill #define MCX_ETHER_CAP_40G_LR4		16
    199  1.15  jmcneill #define MCX_ETHER_CAP_50G_SR2		18
    200  1.15  jmcneill #define MCX_ETHER_CAP_100G_CR4		20
    201  1.15  jmcneill #define MCX_ETHER_CAP_100G_SR4		21
    202  1.15  jmcneill #define MCX_ETHER_CAP_100G_KR4		22
    203  1.15  jmcneill #define MCX_ETHER_CAP_100G_LR4		23
    204  1.15  jmcneill #define MCX_ETHER_CAP_100_TX		24
    205  1.15  jmcneill #define MCX_ETHER_CAP_1000_T		25
    206  1.15  jmcneill #define MCX_ETHER_CAP_10G_T		26
    207  1.15  jmcneill #define MCX_ETHER_CAP_25G_CR		27
    208  1.15  jmcneill #define MCX_ETHER_CAP_25G_KR		28
    209  1.15  jmcneill #define MCX_ETHER_CAP_25G_SR		29
    210  1.15  jmcneill #define MCX_ETHER_CAP_50G_CR2		30
    211  1.15  jmcneill #define MCX_ETHER_CAP_50G_KR2		31
    212  1.15  jmcneill 
    213  1.15  jmcneill #define MCX_MAX_CQE			32
    214  1.15  jmcneill 
    215  1.15  jmcneill #define MCX_CMD_QUERY_HCA_CAP		0x100
    216  1.15  jmcneill #define MCX_CMD_QUERY_ADAPTER		0x101
    217  1.15  jmcneill #define MCX_CMD_INIT_HCA		0x102
    218  1.15  jmcneill #define MCX_CMD_TEARDOWN_HCA		0x103
    219  1.15  jmcneill #define MCX_CMD_ENABLE_HCA		0x104
    220  1.15  jmcneill #define MCX_CMD_DISABLE_HCA		0x105
    221  1.15  jmcneill #define MCX_CMD_QUERY_PAGES		0x107
    222  1.15  jmcneill #define MCX_CMD_MANAGE_PAGES		0x108
    223  1.15  jmcneill #define MCX_CMD_SET_HCA_CAP		0x109
    224  1.15  jmcneill #define MCX_CMD_QUERY_ISSI		0x10a
    225  1.15  jmcneill #define MCX_CMD_SET_ISSI		0x10b
    226  1.15  jmcneill #define MCX_CMD_SET_DRIVER_VERSION	0x10d
    227  1.15  jmcneill #define MCX_CMD_QUERY_SPECIAL_CONTEXTS	0x203
    228  1.15  jmcneill #define MCX_CMD_CREATE_EQ		0x301
    229  1.15  jmcneill #define MCX_CMD_DESTROY_EQ		0x302
    230  1.15  jmcneill #define MCX_CMD_QUERY_EQ		0x303
    231  1.15  jmcneill #define MCX_CMD_CREATE_CQ		0x400
    232  1.15  jmcneill #define MCX_CMD_DESTROY_CQ		0x401
    233  1.15  jmcneill #define MCX_CMD_QUERY_CQ		0x402
    234  1.15  jmcneill #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754
    235   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
    236  1.15  jmcneill 					0x755
    237  1.15  jmcneill #define MCX_CMD_QUERY_VPORT_COUNTERS 	0x770
    238  1.15  jmcneill #define MCX_CMD_ALLOC_PD		0x800
    239  1.15  jmcneill #define MCX_CMD_ALLOC_UAR		0x802
    240  1.15  jmcneill #define MCX_CMD_ACCESS_REG		0x805
    241  1.15  jmcneill #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN	0x816
    242  1.15  jmcneill #define MCX_CMD_CREATE_TIR		0x900
    243  1.15  jmcneill #define MCX_CMD_DESTROY_TIR		0x902
    244  1.15  jmcneill #define MCX_CMD_CREATE_SQ		0x904
    245  1.15  jmcneill #define MCX_CMD_MODIFY_SQ		0x905
    246  1.15  jmcneill #define MCX_CMD_DESTROY_SQ		0x906
    247  1.15  jmcneill #define MCX_CMD_QUERY_SQ		0x907
    248  1.15  jmcneill #define MCX_CMD_CREATE_RQ		0x908
    249  1.15  jmcneill #define MCX_CMD_MODIFY_RQ		0x909
    250  1.15  jmcneill #define MCX_CMD_DESTROY_RQ		0x90a
    251  1.15  jmcneill #define MCX_CMD_QUERY_RQ		0x90b
    252  1.15  jmcneill #define MCX_CMD_CREATE_TIS		0x912
    253  1.15  jmcneill #define MCX_CMD_DESTROY_TIS		0x914
    254  1.15  jmcneill #define MCX_CMD_CREATE_RQT		0x916
    255  1.15  jmcneill #define MCX_CMD_DESTROY_RQT		0x918
    256  1.15  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ROOT	0x92f
    257  1.15  jmcneill #define MCX_CMD_CREATE_FLOW_TABLE	0x930
    258  1.15  jmcneill #define MCX_CMD_DESTROY_FLOW_TABLE	0x931
    259  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE	0x932
    260  1.15  jmcneill #define MCX_CMD_CREATE_FLOW_GROUP	0x933
    261  1.15  jmcneill #define MCX_CMD_DESTROY_FLOW_GROUP	0x934
    262  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_GROUP	0x935
    263  1.15  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ENTRY	0x936
    264  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY	0x937
    265  1.15  jmcneill #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY	0x938
    266  1.15  jmcneill #define MCX_CMD_ALLOC_FLOW_COUNTER	0x939
    267  1.15  jmcneill #define MCX_CMD_QUERY_FLOW_COUNTER	0x93b
    268  1.15  jmcneill 
    269  1.15  jmcneill #define MCX_QUEUE_STATE_RST		0
    270  1.15  jmcneill #define MCX_QUEUE_STATE_RDY		1
    271  1.15  jmcneill #define MCX_QUEUE_STATE_ERR		3
    272   1.1  jmcneill 
    273  1.15  jmcneill #define MCX_FLOW_TABLE_TYPE_RX		0
    274  1.15  jmcneill #define MCX_FLOW_TABLE_TYPE_TX		1
    275   1.1  jmcneill 
    276  1.15  jmcneill #define MCX_CMDQ_INLINE_DATASIZE	16
    277   1.1  jmcneill 
    278   1.1  jmcneill struct mcx_cmdq_entry {
    279   1.1  jmcneill 	uint8_t			cq_type;
    280   1.1  jmcneill #define MCX_CMDQ_TYPE_PCIE		0x7
    281   1.1  jmcneill 	uint8_t			cq_reserved0[3];
    282   1.1  jmcneill 
    283   1.1  jmcneill 	uint32_t		cq_input_length;
    284   1.1  jmcneill 	uint64_t		cq_input_ptr;
    285   1.1  jmcneill 	uint8_t			cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
    286   1.1  jmcneill 
    287   1.1  jmcneill 	uint8_t			cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
    288   1.1  jmcneill 	uint64_t		cq_output_ptr;
    289   1.1  jmcneill 	uint32_t		cq_output_length;
    290   1.1  jmcneill 
    291   1.1  jmcneill 	uint8_t			cq_token;
    292   1.1  jmcneill 	uint8_t			cq_signature;
    293   1.1  jmcneill 	uint8_t			cq_reserved1[1];
    294   1.1  jmcneill 	uint8_t			cq_status;
    295   1.1  jmcneill #define MCX_CQ_STATUS_SHIFT		1
    296   1.1  jmcneill #define MCX_CQ_STATUS_MASK		(0x7f << MCX_CQ_STATUS_SHIFT)
    297   1.1  jmcneill #define MCX_CQ_STATUS_OK		(0x00 << MCX_CQ_STATUS_SHIFT)
    298   1.1  jmcneill #define MCX_CQ_STATUS_INT_ERR		(0x01 << MCX_CQ_STATUS_SHIFT)
    299   1.1  jmcneill #define MCX_CQ_STATUS_BAD_OPCODE	(0x02 << MCX_CQ_STATUS_SHIFT)
    300   1.1  jmcneill #define MCX_CQ_STATUS_BAD_PARAM		(0x03 << MCX_CQ_STATUS_SHIFT)
    301   1.1  jmcneill #define MCX_CQ_STATUS_BAD_SYS_STATE	(0x04 << MCX_CQ_STATUS_SHIFT)
    302   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE	(0x05 << MCX_CQ_STATUS_SHIFT)
    303   1.1  jmcneill #define MCX_CQ_STATUS_RESOURCE_BUSY	(0x06 << MCX_CQ_STATUS_SHIFT)
    304   1.1  jmcneill #define MCX_CQ_STATUS_EXCEED_LIM	(0x08 << MCX_CQ_STATUS_SHIFT)
    305   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RES_STATE	(0x09 << MCX_CQ_STATUS_SHIFT)
    306   1.1  jmcneill #define MCX_CQ_STATUS_BAD_INDEX		(0x0a << MCX_CQ_STATUS_SHIFT)
    307   1.1  jmcneill #define MCX_CQ_STATUS_NO_RESOURCES	(0x0f << MCX_CQ_STATUS_SHIFT)
    308   1.1  jmcneill #define MCX_CQ_STATUS_BAD_INPUT_LEN	(0x50 << MCX_CQ_STATUS_SHIFT)
    309   1.1  jmcneill #define MCX_CQ_STATUS_BAD_OUTPUT_LEN	(0x51 << MCX_CQ_STATUS_SHIFT)
    310   1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
    311   1.1  jmcneill 					(0x10 << MCX_CQ_STATUS_SHIFT)
    312   1.1  jmcneill #define MCX_CQ_STATUS_BAD_SIZE		(0x40 << MCX_CQ_STATUS_SHIFT)
    313   1.1  jmcneill #define MCX_CQ_STATUS_OWN_MASK		0x1
    314   1.1  jmcneill #define MCX_CQ_STATUS_OWN_SW		0x0
    315   1.1  jmcneill #define MCX_CQ_STATUS_OWN_HW		0x1
    316   1.1  jmcneill } __packed __aligned(8);
    317   1.1  jmcneill 
    318   1.1  jmcneill #define MCX_CMDQ_MAILBOX_DATASIZE	512
    319   1.1  jmcneill 
    320   1.1  jmcneill struct mcx_cmdq_mailbox {
    321   1.1  jmcneill 	uint8_t			mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
    322   1.1  jmcneill 	uint8_t			mb_reserved0[48];
    323   1.1  jmcneill 	uint64_t		mb_next_ptr;
    324   1.1  jmcneill 	uint32_t		mb_block_number;
    325   1.1  jmcneill 	uint8_t			mb_reserved1[1];
    326   1.1  jmcneill 	uint8_t			mb_token;
    327   1.1  jmcneill 	uint8_t			mb_ctrl_signature;
    328   1.1  jmcneill 	uint8_t			mb_signature;
    329   1.1  jmcneill } __packed __aligned(8);
    330   1.1  jmcneill 
    331   1.1  jmcneill #define MCX_CMDQ_MAILBOX_ALIGN	(1 << 10)
    332   1.1  jmcneill #define MCX_CMDQ_MAILBOX_SIZE	roundup(sizeof(struct mcx_cmdq_mailbox), \
    333   1.1  jmcneill 				    MCX_CMDQ_MAILBOX_ALIGN)
    334   1.1  jmcneill /*
    335   1.1  jmcneill  * command mailbox structres
    336   1.1  jmcneill  */
    337   1.1  jmcneill 
    338   1.1  jmcneill struct mcx_cmd_enable_hca_in {
    339   1.1  jmcneill 	uint16_t		cmd_opcode;
    340   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    341   1.1  jmcneill 	uint16_t		cmd_op_mod;
    342   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    343   1.1  jmcneill 	uint16_t		cmd_function_id;
    344   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    345   1.1  jmcneill } __packed __aligned(4);
    346   1.1  jmcneill 
    347   1.1  jmcneill struct mcx_cmd_enable_hca_out {
    348   1.1  jmcneill 	uint8_t			cmd_status;
    349   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    350   1.1  jmcneill 	uint32_t		cmd_syndrome;
    351   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    352   1.1  jmcneill } __packed __aligned(4);
    353   1.1  jmcneill 
    354   1.1  jmcneill struct mcx_cmd_init_hca_in {
    355   1.1  jmcneill 	uint16_t		cmd_opcode;
    356   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    357   1.1  jmcneill 	uint16_t		cmd_op_mod;
    358   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    359   1.1  jmcneill } __packed __aligned(4);
    360   1.1  jmcneill 
    361   1.1  jmcneill struct mcx_cmd_init_hca_out {
    362   1.1  jmcneill 	uint8_t			cmd_status;
    363   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    364   1.1  jmcneill 	uint32_t		cmd_syndrome;
    365   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    366   1.1  jmcneill } __packed __aligned(4);
    367   1.1  jmcneill 
    368   1.1  jmcneill struct mcx_cmd_teardown_hca_in {
    369   1.1  jmcneill 	uint16_t		cmd_opcode;
    370   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    371   1.1  jmcneill 	uint16_t		cmd_op_mod;
    372   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    373   1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_GRACEFUL	0x0
    374   1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_PANIC	0x1
    375   1.1  jmcneill 	uint16_t		cmd_profile;
    376   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    377   1.1  jmcneill } __packed __aligned(4);
    378   1.1  jmcneill 
    379   1.1  jmcneill struct mcx_cmd_teardown_hca_out {
    380   1.1  jmcneill 	uint8_t			cmd_status;
    381   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    382   1.1  jmcneill 	uint32_t		cmd_syndrome;
    383   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    384   1.1  jmcneill } __packed __aligned(4);
    385   1.1  jmcneill 
    386   1.1  jmcneill struct mcx_cmd_access_reg_in {
    387   1.1  jmcneill 	uint16_t		cmd_opcode;
    388   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    389   1.1  jmcneill 	uint16_t		cmd_op_mod;
    390   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    391   1.1  jmcneill 	uint16_t		cmd_register_id;
    392   1.1  jmcneill 	uint32_t		cmd_argument;
    393   1.1  jmcneill } __packed __aligned(4);
    394   1.1  jmcneill 
    395   1.1  jmcneill struct mcx_cmd_access_reg_out {
    396   1.1  jmcneill 	uint8_t			cmd_status;
    397   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    398   1.1  jmcneill 	uint32_t		cmd_syndrome;
    399   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    400   1.1  jmcneill } __packed __aligned(4);
    401   1.1  jmcneill 
    402   1.1  jmcneill struct mcx_reg_pmtu {
    403   1.1  jmcneill 	uint8_t			rp_reserved1;
    404   1.1  jmcneill 	uint8_t			rp_local_port;
    405   1.1  jmcneill 	uint8_t			rp_reserved2[2];
    406   1.1  jmcneill 	uint16_t		rp_max_mtu;
    407   1.1  jmcneill 	uint8_t			rp_reserved3[2];
    408   1.1  jmcneill 	uint16_t		rp_admin_mtu;
    409   1.1  jmcneill 	uint8_t			rp_reserved4[2];
    410   1.1  jmcneill 	uint16_t		rp_oper_mtu;
    411   1.1  jmcneill 	uint8_t			rp_reserved5[2];
    412   1.1  jmcneill } __packed __aligned(4);
    413   1.1  jmcneill 
    414   1.1  jmcneill struct mcx_reg_ptys {
    415   1.1  jmcneill 	uint8_t			rp_reserved1;
    416   1.1  jmcneill 	uint8_t			rp_local_port;
    417   1.1  jmcneill 	uint8_t			rp_reserved2;
    418   1.1  jmcneill 	uint8_t			rp_proto_mask;
    419   1.1  jmcneill #define MCX_REG_PTYS_PROTO_MASK_ETH		(1 << 2)
    420   1.1  jmcneill 	uint8_t			rp_reserved3[8];
    421   1.1  jmcneill 	uint32_t		rp_eth_proto_cap;
    422   1.1  jmcneill 	uint8_t			rp_reserved4[8];
    423   1.1  jmcneill 	uint32_t		rp_eth_proto_admin;
    424   1.1  jmcneill 	uint8_t			rp_reserved5[8];
    425   1.1  jmcneill 	uint32_t		rp_eth_proto_oper;
    426   1.1  jmcneill 	uint8_t			rp_reserved6[24];
    427   1.1  jmcneill } __packed __aligned(4);
    428   1.1  jmcneill 
    429   1.1  jmcneill struct mcx_reg_paos {
    430   1.1  jmcneill 	uint8_t			rp_reserved1;
    431   1.1  jmcneill 	uint8_t			rp_local_port;
    432   1.1  jmcneill 	uint8_t			rp_admin_status;
    433   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP		1
    434   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DOWN		2
    435   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE	3
    436   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED	4
    437   1.1  jmcneill 	uint8_t			rp_oper_status;
    438   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_UP		1
    439   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_DOWN		2
    440   1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_FAILED		4
    441   1.1  jmcneill 	uint8_t			rp_admin_state_update;
    442   1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN	(1 << 7)
    443   1.1  jmcneill 	uint8_t			rp_reserved2[11];
    444   1.1  jmcneill } __packed __aligned(4);
    445   1.1  jmcneill 
    446   1.1  jmcneill struct mcx_reg_pfcc {
    447   1.1  jmcneill 	uint8_t			rp_reserved1;
    448   1.1  jmcneill 	uint8_t			rp_local_port;
    449   1.1  jmcneill 	uint8_t			rp_reserved2[3];
    450   1.1  jmcneill 	uint8_t			rp_prio_mask_tx;
    451   1.1  jmcneill 	uint8_t			rp_reserved3;
    452   1.1  jmcneill 	uint8_t			rp_prio_mask_rx;
    453   1.1  jmcneill 	uint8_t			rp_pptx_aptx;
    454   1.1  jmcneill 	uint8_t			rp_pfctx;
    455   1.1  jmcneill 	uint8_t			rp_fctx_dis;
    456   1.1  jmcneill 	uint8_t			rp_reserved4;
    457   1.1  jmcneill 	uint8_t			rp_pprx_aprx;
    458   1.1  jmcneill 	uint8_t			rp_pfcrx;
    459   1.1  jmcneill 	uint8_t			rp_reserved5[2];
    460   1.1  jmcneill 	uint16_t		rp_dev_stall_min;
    461   1.1  jmcneill 	uint16_t		rp_dev_stall_crit;
    462   1.1  jmcneill 	uint8_t			rp_reserved6[12];
    463   1.1  jmcneill } __packed __aligned(4);
    464   1.1  jmcneill 
    465   1.1  jmcneill #define MCX_PMLP_MODULE_NUM_MASK	0xff
    466   1.1  jmcneill struct mcx_reg_pmlp {
    467   1.1  jmcneill 	uint8_t			rp_rxtx;
    468   1.1  jmcneill 	uint8_t			rp_local_port;
    469   1.1  jmcneill 	uint8_t			rp_reserved0;
    470   1.1  jmcneill 	uint8_t			rp_width;
    471   1.1  jmcneill 	uint32_t		rp_lane0_mapping;
    472   1.1  jmcneill 	uint32_t		rp_lane1_mapping;
    473   1.1  jmcneill 	uint32_t		rp_lane2_mapping;
    474   1.1  jmcneill 	uint32_t		rp_lane3_mapping;
    475   1.1  jmcneill 	uint8_t			rp_reserved1[44];
    476   1.1  jmcneill } __packed __aligned(4);
    477   1.1  jmcneill 
    478  1.15  jmcneill struct mcx_reg_ppcnt {
    479  1.15  jmcneill 	uint8_t			ppcnt_swid;
    480  1.15  jmcneill 	uint8_t			ppcnt_local_port;
    481  1.15  jmcneill 	uint8_t			ppcnt_pnat;
    482  1.15  jmcneill 	uint8_t			ppcnt_grp;
    483  1.15  jmcneill #define MCX_REG_PPCNT_GRP_IEEE8023		0x00
    484  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC2863		0x01
    485  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC2819		0x02
    486  1.15  jmcneill #define MCX_REG_PPCNT_GRP_RFC3635		0x03
    487  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_PRIO		0x10
    488  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_TC		0x11
    489  1.15  jmcneill #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER		0x11
    490  1.15  jmcneill 
    491  1.15  jmcneill 	uint8_t			ppcnt_clr;
    492  1.15  jmcneill 	uint8_t			ppcnt_reserved1[2];
    493  1.15  jmcneill 	uint8_t			ppcnt_prio_tc;
    494  1.15  jmcneill #define MCX_REG_PPCNT_CLR			(1 << 7)
    495  1.15  jmcneill 
    496  1.15  jmcneill 	uint8_t			ppcnt_counter_set[248];
    497  1.15  jmcneill } __packed __aligned(8);
    498  1.15  jmcneill CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256);
    499  1.15  jmcneill CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) %
    500  1.15  jmcneill     sizeof(uint64_t)) == 0);
    501  1.15  jmcneill 
    502  1.15  jmcneill enum mcx_ppcnt_ieee8023 {
    503  1.15  jmcneill 	frames_transmitted_ok,
    504  1.15  jmcneill 	frames_received_ok,
    505  1.15  jmcneill 	frame_check_sequence_errors,
    506  1.15  jmcneill 	alignment_errors,
    507  1.15  jmcneill 	octets_transmitted_ok,
    508  1.15  jmcneill 	octets_received_ok,
    509  1.15  jmcneill 	multicast_frames_xmitted_ok,
    510  1.15  jmcneill 	broadcast_frames_xmitted_ok,
    511  1.15  jmcneill 	multicast_frames_received_ok,
    512  1.15  jmcneill 	broadcast_frames_received_ok,
    513  1.15  jmcneill 	in_range_length_errors,
    514  1.15  jmcneill 	out_of_range_length_field,
    515  1.15  jmcneill 	frame_too_long_errors,
    516  1.15  jmcneill 	symbol_error_during_carrier,
    517  1.15  jmcneill 	mac_control_frames_transmitted,
    518  1.15  jmcneill 	mac_control_frames_received,
    519  1.15  jmcneill 	unsupported_opcodes_received,
    520  1.15  jmcneill 	pause_mac_ctrl_frames_received,
    521  1.15  jmcneill 	pause_mac_ctrl_frames_transmitted,
    522  1.15  jmcneill 
    523  1.15  jmcneill 	mcx_ppcnt_ieee8023_count
    524  1.15  jmcneill };
    525  1.15  jmcneill CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98);
    526  1.15  jmcneill 
    527  1.15  jmcneill enum mcx_ppcnt_rfc2863 {
    528  1.15  jmcneill 	in_octets,
    529  1.15  jmcneill 	in_ucast_pkts,
    530  1.15  jmcneill 	in_discards,
    531  1.15  jmcneill 	in_errors,
    532  1.15  jmcneill 	in_unknown_protos,
    533  1.15  jmcneill 	out_octets,
    534  1.15  jmcneill 	out_ucast_pkts,
    535  1.15  jmcneill 	out_discards,
    536  1.15  jmcneill 	out_errors,
    537  1.15  jmcneill 	in_multicast_pkts,
    538  1.15  jmcneill 	in_broadcast_pkts,
    539  1.15  jmcneill 	out_multicast_pkts,
    540  1.15  jmcneill 	out_broadcast_pkts,
    541  1.15  jmcneill 
    542  1.15  jmcneill 	mcx_ppcnt_rfc2863_count
    543  1.15  jmcneill };
    544  1.15  jmcneill CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68);
    545  1.15  jmcneill 
    546  1.15  jmcneill enum mcx_ppcnt_rfc2819 {
    547  1.15  jmcneill 	drop_events,
    548  1.15  jmcneill 	octets,
    549  1.15  jmcneill 	pkts,
    550  1.15  jmcneill 	broadcast_pkts,
    551  1.15  jmcneill 	multicast_pkts,
    552  1.15  jmcneill 	crc_align_errors,
    553  1.15  jmcneill 	undersize_pkts,
    554  1.15  jmcneill 	oversize_pkts,
    555  1.15  jmcneill 	fragments,
    556  1.15  jmcneill 	jabbers,
    557  1.15  jmcneill 	collisions,
    558  1.15  jmcneill 	pkts64octets,
    559  1.15  jmcneill 	pkts65to127octets,
    560  1.15  jmcneill 	pkts128to255octets,
    561  1.15  jmcneill 	pkts256to511octets,
    562  1.15  jmcneill 	pkts512to1023octets,
    563  1.15  jmcneill 	pkts1024to1518octets,
    564  1.15  jmcneill 	pkts1519to2047octets,
    565  1.15  jmcneill 	pkts2048to4095octets,
    566  1.15  jmcneill 	pkts4096to8191octets,
    567  1.15  jmcneill 	pkts8192to10239octets,
    568  1.15  jmcneill 
    569  1.15  jmcneill 	mcx_ppcnt_rfc2819_count
    570  1.15  jmcneill };
    571  1.15  jmcneill CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8);
    572  1.15  jmcneill 
    573  1.15  jmcneill enum mcx_ppcnt_rfc3635 {
    574  1.15  jmcneill 	dot3stats_alignment_errors,
    575  1.15  jmcneill 	dot3stats_fcs_errors,
    576  1.15  jmcneill 	dot3stats_single_collision_frames,
    577  1.15  jmcneill 	dot3stats_multiple_collision_frames,
    578  1.15  jmcneill 	dot3stats_sqe_test_errors,
    579  1.15  jmcneill 	dot3stats_deferred_transmissions,
    580  1.15  jmcneill 	dot3stats_late_collisions,
    581  1.15  jmcneill 	dot3stats_excessive_collisions,
    582  1.15  jmcneill 	dot3stats_internal_mac_transmit_errors,
    583  1.15  jmcneill 	dot3stats_carrier_sense_errors,
    584  1.15  jmcneill 	dot3stats_frame_too_longs,
    585  1.15  jmcneill 	dot3stats_internal_mac_receive_errors,
    586  1.15  jmcneill 	dot3stats_symbol_errors,
    587  1.15  jmcneill 	dot3control_in_unknown_opcodes,
    588  1.15  jmcneill 	dot3in_pause_frames,
    589  1.15  jmcneill 	dot3out_pause_frames,
    590  1.15  jmcneill 
    591  1.15  jmcneill 	mcx_ppcnt_rfc3635_count
    592  1.15  jmcneill };
    593  1.15  jmcneill CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80);
    594  1.15  jmcneill 
    595  1.15  jmcneill struct mcx_reg_mcam {
    596  1.15  jmcneill 	uint8_t			_reserved1[1];
    597  1.15  jmcneill 	uint8_t			mcam_feature_group;
    598  1.15  jmcneill 	uint8_t			_reserved2[1];
    599  1.15  jmcneill 	uint8_t			mcam_access_reg_group;
    600  1.15  jmcneill 	uint8_t			_reserved3[4];
    601  1.15  jmcneill 	uint8_t			mcam_access_reg_cap_mask[16];
    602  1.15  jmcneill 	uint8_t			_reserved4[16];
    603  1.15  jmcneill 	uint8_t			mcam_feature_cap_mask[16];
    604  1.15  jmcneill 	uint8_t			_reserved5[16];
    605  1.15  jmcneill } __packed __aligned(4);
    606  1.15  jmcneill 
    607  1.15  jmcneill #define MCX_BITFIELD_BIT(bf, b)	(bf[(sizeof bf - 1) - (b / 8)] & (b % 8))
    608  1.15  jmcneill 
    609  1.15  jmcneill #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP	6
    610  1.15  jmcneill 
    611  1.15  jmcneill struct mcx_reg_mtcap {
    612  1.15  jmcneill 	uint8_t			_reserved1[3];
    613  1.15  jmcneill 	uint8_t			mtcap_sensor_count;
    614  1.15  jmcneill 	uint8_t			_reserved2[4];
    615  1.15  jmcneill 
    616  1.15  jmcneill 	uint64_t		mtcap_sensor_map;
    617  1.15  jmcneill };
    618  1.15  jmcneill 
    619  1.15  jmcneill struct mcx_reg_mtmp {
    620  1.15  jmcneill 	uint8_t			_reserved1[2];
    621  1.15  jmcneill 	uint16_t		mtmp_sensor_index;
    622  1.15  jmcneill 
    623  1.15  jmcneill 	uint8_t			_reserved2[2];
    624  1.15  jmcneill 	uint16_t		mtmp_temperature;
    625  1.15  jmcneill 
    626  1.15  jmcneill 	uint16_t		mtmp_mte_mtr;
    627  1.15  jmcneill #define MCX_REG_MTMP_MTE		(1 << 15)
    628  1.15  jmcneill #define MCX_REG_MTMP_MTR		(1 << 14)
    629  1.15  jmcneill 	uint16_t		mtmp_max_temperature;
    630  1.15  jmcneill 
    631  1.15  jmcneill 	uint16_t		mtmp_tee;
    632  1.15  jmcneill #define MCX_REG_MTMP_TEE_NOPE		(0 << 14)
    633  1.15  jmcneill #define MCX_REG_MTMP_TEE_GENERATE	(1 << 14)
    634  1.15  jmcneill #define MCX_REG_MTMP_TEE_GENERATE_ONE	(2 << 14)
    635  1.15  jmcneill 	uint16_t		mtmp_temperature_threshold_hi;
    636  1.15  jmcneill 
    637  1.15  jmcneill 	uint8_t			_reserved3[2];
    638  1.15  jmcneill 	uint16_t		mtmp_temperature_threshold_lo;
    639  1.15  jmcneill 
    640  1.15  jmcneill 	uint8_t			_reserved4[4];
    641  1.15  jmcneill 
    642  1.15  jmcneill 	uint8_t			mtmp_sensor_name[8];
    643  1.15  jmcneill };
    644  1.15  jmcneill CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20);
    645  1.15  jmcneill CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18);
    646  1.15  jmcneill 
    647   1.1  jmcneill #define MCX_MCIA_EEPROM_BYTES	32
    648   1.1  jmcneill struct mcx_reg_mcia {
    649   1.1  jmcneill 	uint8_t			rm_l;
    650   1.1  jmcneill 	uint8_t			rm_module;
    651   1.1  jmcneill 	uint8_t			rm_reserved0;
    652   1.1  jmcneill 	uint8_t			rm_status;
    653   1.1  jmcneill 	uint8_t			rm_i2c_addr;
    654   1.1  jmcneill 	uint8_t			rm_page_num;
    655   1.1  jmcneill 	uint16_t		rm_dev_addr;
    656   1.1  jmcneill 	uint16_t		rm_reserved1;
    657   1.1  jmcneill 	uint16_t		rm_size;
    658   1.1  jmcneill 	uint32_t		rm_reserved2;
    659   1.1  jmcneill 	uint8_t			rm_data[48];
    660   1.1  jmcneill } __packed __aligned(4);
    661   1.1  jmcneill 
    662   1.1  jmcneill struct mcx_cmd_query_issi_in {
    663   1.1  jmcneill 	uint16_t		cmd_opcode;
    664   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    665   1.1  jmcneill 	uint16_t		cmd_op_mod;
    666   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    667   1.1  jmcneill } __packed __aligned(4);
    668   1.1  jmcneill 
    669   1.1  jmcneill struct mcx_cmd_query_issi_il_out {
    670   1.1  jmcneill 	uint8_t			cmd_status;
    671   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    672   1.1  jmcneill 	uint32_t		cmd_syndrome;
    673   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    674   1.1  jmcneill 	uint16_t		cmd_current_issi;
    675   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    676   1.1  jmcneill } __packed __aligned(4);
    677   1.1  jmcneill 
    678   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
    679   1.1  jmcneill 
    680   1.1  jmcneill struct mcx_cmd_query_issi_mb_out {
    681   1.1  jmcneill 	uint8_t			cmd_reserved2[16];
    682   1.1  jmcneill 	uint8_t			cmd_supported_issi[80]; /* very big endian */
    683   1.1  jmcneill } __packed __aligned(4);
    684   1.1  jmcneill 
    685   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
    686   1.1  jmcneill 
    687   1.1  jmcneill struct mcx_cmd_set_issi_in {
    688   1.1  jmcneill 	uint16_t		cmd_opcode;
    689   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    690   1.1  jmcneill 	uint16_t		cmd_op_mod;
    691   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    692   1.1  jmcneill 	uint16_t		cmd_current_issi;
    693   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    694   1.1  jmcneill } __packed __aligned(4);
    695   1.1  jmcneill 
    696   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
    697   1.1  jmcneill 
    698   1.1  jmcneill struct mcx_cmd_set_issi_out {
    699   1.1  jmcneill 	uint8_t			cmd_status;
    700   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    701   1.1  jmcneill 	uint32_t		cmd_syndrome;
    702   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    703   1.1  jmcneill } __packed __aligned(4);
    704   1.1  jmcneill 
    705   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
    706   1.1  jmcneill 
    707   1.1  jmcneill struct mcx_cmd_query_pages_in {
    708   1.1  jmcneill 	uint16_t		cmd_opcode;
    709   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    710   1.1  jmcneill 	uint16_t		cmd_op_mod;
    711   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_BOOT	0x01
    712   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_INIT	0x02
    713   1.1  jmcneill #define MCX_CMD_QUERY_PAGES_REGULAR	0x03
    714   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    715   1.1  jmcneill } __packed __aligned(4);
    716   1.1  jmcneill 
    717   1.1  jmcneill struct mcx_cmd_query_pages_out {
    718   1.1  jmcneill 	uint8_t			cmd_status;
    719   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    720   1.1  jmcneill 	uint32_t		cmd_syndrome;
    721   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    722   1.1  jmcneill 	uint16_t		cmd_func_id;
    723  1.15  jmcneill 	int32_t			cmd_num_pages;
    724   1.1  jmcneill } __packed __aligned(4);
    725   1.1  jmcneill 
    726   1.1  jmcneill struct mcx_cmd_manage_pages_in {
    727   1.1  jmcneill 	uint16_t		cmd_opcode;
    728   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    729   1.1  jmcneill 	uint16_t		cmd_op_mod;
    730   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
    731   1.1  jmcneill 					0x00
    732   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
    733   1.1  jmcneill 					0x01
    734   1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
    735   1.1  jmcneill 					0x02
    736   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    737   1.1  jmcneill 	uint16_t		cmd_func_id;
    738   1.1  jmcneill 	uint32_t		cmd_input_num_entries;
    739   1.1  jmcneill } __packed __aligned(4);
    740   1.1  jmcneill 
    741   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
    742   1.1  jmcneill 
    743   1.1  jmcneill struct mcx_cmd_manage_pages_out {
    744   1.1  jmcneill 	uint8_t			cmd_status;
    745   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    746   1.1  jmcneill 	uint32_t		cmd_syndrome;
    747   1.1  jmcneill 	uint32_t		cmd_output_num_entries;
    748   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    749   1.1  jmcneill } __packed __aligned(4);
    750   1.1  jmcneill 
    751   1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
    752   1.1  jmcneill 
    753   1.1  jmcneill struct mcx_cmd_query_hca_cap_in {
    754   1.1  jmcneill 	uint16_t		cmd_opcode;
    755   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    756   1.1  jmcneill 	uint16_t		cmd_op_mod;
    757   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_MAX	(0x0 << 0)
    758   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_CURRENT	(0x1 << 0)
    759   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_DEVICE	(0x0 << 1)
    760   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD	(0x1 << 1)
    761   1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_FLOW	(0x7 << 1)
    762   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    763   1.1  jmcneill } __packed __aligned(4);
    764   1.1  jmcneill 
    765   1.1  jmcneill struct mcx_cmd_query_hca_cap_out {
    766   1.1  jmcneill 	uint8_t			cmd_status;
    767   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    768   1.1  jmcneill 	uint32_t		cmd_syndrome;
    769   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    770   1.1  jmcneill } __packed __aligned(4);
    771   1.1  jmcneill 
    772   1.1  jmcneill #define MCX_HCA_CAP_LEN			0x1000
    773   1.1  jmcneill #define MCX_HCA_CAP_NMAILBOXES		\
    774   1.1  jmcneill 	(MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
    775   1.1  jmcneill 
    776   1.1  jmcneill #if __GNUC_PREREQ__(4, 3)
    777   1.1  jmcneill #define __counter__		__COUNTER__
    778   1.1  jmcneill #else
    779   1.1  jmcneill #define __counter__		__LINE__
    780   1.1  jmcneill #endif
    781   1.1  jmcneill 
    782   1.1  jmcneill #define __token(_tok, _num)	_tok##_num
    783   1.1  jmcneill #define _token(_tok, _num)	__token(_tok, _num)
    784   1.1  jmcneill #define __reserved__		_token(__reserved, __counter__)
    785   1.1  jmcneill 
    786   1.1  jmcneill struct mcx_cap_device {
    787   1.1  jmcneill 	uint8_t			reserved0[16];
    788   1.1  jmcneill 
    789   1.1  jmcneill 	uint8_t			log_max_srq_sz;
    790   1.1  jmcneill 	uint8_t			log_max_qp_sz;
    791   1.1  jmcneill 	uint8_t			__reserved__[1];
    792   1.1  jmcneill 	uint8_t			log_max_qp; /* 5 bits */
    793   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_QP	0x1f
    794   1.1  jmcneill 
    795   1.1  jmcneill 	uint8_t			__reserved__[1];
    796   1.1  jmcneill 	uint8_t			log_max_srq; /* 5 bits */
    797   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SRQ	0x1f
    798   1.1  jmcneill 	uint8_t			__reserved__[2];
    799   1.1  jmcneill 
    800   1.1  jmcneill 	uint8_t			__reserved__[1];
    801   1.1  jmcneill 	uint8_t			log_max_cq_sz;
    802   1.1  jmcneill 	uint8_t			__reserved__[1];
    803   1.1  jmcneill 	uint8_t			log_max_cq; /* 5 bits */
    804   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_CQ	0x1f
    805   1.1  jmcneill 
    806   1.1  jmcneill 	uint8_t			log_max_eq_sz;
    807   1.1  jmcneill 	uint8_t			log_max_mkey; /* 6 bits */
    808   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MKEY	0x3f
    809   1.1  jmcneill 	uint8_t			__reserved__[1];
    810   1.1  jmcneill 	uint8_t			log_max_eq; /* 4 bits */
    811   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_EQ	0x0f
    812   1.1  jmcneill 
    813   1.1  jmcneill 	uint8_t			max_indirection;
    814   1.1  jmcneill 	uint8_t			log_max_mrw_sz; /* 7 bits */
    815   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ	0x7f
    816   1.1  jmcneill 	uint8_t			teardown_log_max_msf_list_size;
    817   1.1  jmcneill #define MCX_CAP_DEVICE_FORCE_TEARDOWN	0x80
    818   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
    819   1.1  jmcneill 					0x3f
    820   1.1  jmcneill 	uint8_t			log_max_klm_list_size; /* 6 bits */
    821   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
    822   1.1  jmcneill 					0x3f
    823   1.1  jmcneill 
    824   1.1  jmcneill 	uint8_t			__reserved__[1];
    825   1.1  jmcneill 	uint8_t			log_max_ra_req_dc; /* 6 bits */
    826   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC	0x3f
    827   1.1  jmcneill 	uint8_t			__reserved__[1];
    828   1.1  jmcneill 	uint8_t			log_max_ra_res_dc; /* 6 bits */
    829   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
    830   1.1  jmcneill 					0x3f
    831   1.1  jmcneill 
    832   1.1  jmcneill 	uint8_t			__reserved__[1];
    833   1.1  jmcneill 	uint8_t			log_max_ra_req_qp; /* 6 bits */
    834   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
    835   1.1  jmcneill 					0x3f
    836   1.1  jmcneill 	uint8_t			__reserved__[1];
    837   1.1  jmcneill 	uint8_t			log_max_ra_res_qp; /* 6 bits */
    838   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
    839   1.1  jmcneill 					0x3f
    840   1.1  jmcneill 
    841   1.1  jmcneill 	uint8_t			flags1;
    842   1.1  jmcneill #define MCX_CAP_DEVICE_END_PAD		0x80
    843   1.1  jmcneill #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED	0x40
    844   1.1  jmcneill #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
    845   1.1  jmcneill 					0x20
    846   1.1  jmcneill #define MCX_CAP_DEVICE_START_PAD	0x10
    847   1.1  jmcneill #define MCX_CAP_DEVICE_128BYTE_CACHELINE \
    848   1.1  jmcneill 					0x08
    849   1.1  jmcneill 	uint8_t			__reserved__[1];
    850   1.1  jmcneill 	uint16_t		gid_table_size;
    851   1.1  jmcneill 
    852   1.1  jmcneill 	uint16_t		flags2;
    853   1.1  jmcneill #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT	0x8000
    854   1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_COUNTERS	0x4000
    855   1.1  jmcneill #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
    856   1.1  jmcneill 					0x2000
    857   1.1  jmcneill #define MCX_CAP_DEVICE_DEBUG		0x1000
    858   1.1  jmcneill #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
    859   1.1  jmcneill 					0x8000
    860   1.1  jmcneill #define MCX_CAP_DEVICE_RQ_DELAY_DROP	0x4000
    861   1.1  jmcneill #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK	0x03ff
    862   1.1  jmcneill 	uint16_t		pkey_table_size;
    863   1.1  jmcneill 
    864   1.1  jmcneill 	uint8_t			flags3;
    865   1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
    866   1.1  jmcneill 					0x80
    867   1.1  jmcneill #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
    868   1.1  jmcneill 					0x40
    869   1.1  jmcneill #define MCX_CAP_DEVICE_IB_VIRTUAL	0x20
    870   1.1  jmcneill #define MCX_CAP_DEVICE_ETH_VIRTUAL	0x10
    871   1.1  jmcneill #define MCX_CAP_DEVICE_ETS		0x04
    872   1.1  jmcneill #define MCX_CAP_DEVICE_NIC_FLOW_TABLE	0x02
    873   1.1  jmcneill #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
    874   1.1  jmcneill 					0x01
    875   1.1  jmcneill 	uint8_t			local_ca_ack_delay; /* 5 bits */
    876   1.1  jmcneill #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
    877   1.1  jmcneill 					0x1f
    878  1.15  jmcneill #define MCX_CAP_DEVICE_MCAM_REG		0x40
    879   1.1  jmcneill 	uint8_t			port_type;
    880   1.1  jmcneill #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
    881   1.1  jmcneill 					0x80
    882   1.1  jmcneill #define MCX_CAP_DEVICE_PORT_TYPE	0x03
    883  1.15  jmcneill #define MCX_CAP_DEVICE_PORT_TYPE_ETH	0x01
    884   1.1  jmcneill 	uint8_t			num_ports;
    885   1.1  jmcneill 
    886   1.1  jmcneill 	uint8_t			snapshot_log_max_msg;
    887   1.1  jmcneill #define MCX_CAP_DEVICE_SNAPSHOT		0x80
    888   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSG	0x1f
    889   1.1  jmcneill 	uint8_t			max_tc; /* 4 bits */
    890   1.1  jmcneill #define MCX_CAP_DEVICE_MAX_TC		0x0f
    891   1.1  jmcneill 	uint8_t			flags4;
    892   1.1  jmcneill #define MCX_CAP_DEVICE_TEMP_WARN_EVENT	0x80
    893   1.1  jmcneill #define MCX_CAP_DEVICE_DCBX		0x40
    894   1.1  jmcneill #define MCX_CAP_DEVICE_ROL_S		0x02
    895   1.1  jmcneill #define MCX_CAP_DEVICE_ROL_G		0x01
    896   1.1  jmcneill 	uint8_t			wol;
    897   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_S		0x40
    898   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_G		0x20
    899   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_A		0x10
    900   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_B		0x08
    901   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_M		0x04
    902   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_U		0x02
    903   1.1  jmcneill #define MCX_CAP_DEVICE_WOL_P		0x01
    904   1.1  jmcneill 
    905   1.1  jmcneill 	uint16_t		stat_rate_support;
    906   1.1  jmcneill 	uint8_t			__reserved__[1];
    907   1.1  jmcneill 	uint8_t			cqe_version; /* 4 bits */
    908   1.1  jmcneill #define MCX_CAP_DEVICE_CQE_VERSION	0x0f
    909   1.1  jmcneill 
    910   1.1  jmcneill 	uint32_t		flags5;
    911   1.1  jmcneill #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
    912   1.1  jmcneill 					0x80000000
    913   1.1  jmcneill #define MCX_CAP_DEVICE_STRIDING_RQ	0x40000000
    914   1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
    915   1.1  jmcneill 					0x10000000
    916   1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
    917   1.1  jmcneill 					0x08000000
    918   1.1  jmcneill #define MCX_CAP_DEVICE_DC_CONNECT_CP	0x00040000
    919   1.1  jmcneill #define MCX_CAP_DEVICE_DC_CNAK_DRACE	0x00020000
    920   1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    921   1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    922   1.1  jmcneill #define MCX_CAP_DEVICE_CMDIF_CHECKSUM	0x0000c000
    923   1.1  jmcneill #define MCX_CAP_DEVICE_SIGERR_QCE	0x00002000
    924   1.1  jmcneill #define MCX_CAP_DEVICE_WQ_SIGNATURE	0x00000800
    925   1.1  jmcneill #define MCX_CAP_DEVICE_SCTR_DATA_CQE	0x00000400
    926   1.1  jmcneill #define MCX_CAP_DEVICE_SHO		0x00000100
    927   1.1  jmcneill #define MCX_CAP_DEVICE_TPH		0x00000080
    928   1.1  jmcneill #define MCX_CAP_DEVICE_RF		0x00000040
    929   1.1  jmcneill #define MCX_CAP_DEVICE_DCT		0x00000020
    930   1.1  jmcneill #define MCX_CAP_DEVICE_QOS		0x00000010
    931   1.1  jmcneill #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS	0x00000008
    932   1.1  jmcneill #define MCX_CAP_DEVICE_ROCE		0x00000004
    933   1.1  jmcneill #define MCX_CAP_DEVICE_ATOMIC		0x00000002
    934   1.1  jmcneill 
    935   1.1  jmcneill 	uint32_t		flags6;
    936   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_OI		0x80000000
    937   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_RESIZE	0x40000000
    938   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_MODERATION	0x20000000
    939   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
    940   1.1  jmcneill 					0x10000000
    941   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_INVALIDATE	0x08000000
    942   1.1  jmcneill #define MCX_CAP_DEVICE_RESERVED_AT_255	0x04000000
    943   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_EQ_REMAP	0x02000000
    944   1.1  jmcneill #define MCX_CAP_DEVICE_PG		0x01000000
    945   1.1  jmcneill #define MCX_CAP_DEVICE_BLOCK_LB_MC	0x00800000
    946   1.1  jmcneill #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
    947   1.1  jmcneill 					0x00400000
    948   1.1  jmcneill #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
    949   1.1  jmcneill 					0x00200000
    950   1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
    951   1.1  jmcneill 					0x00100000
    952   1.1  jmcneill #define MCX_CAP_DEVICE_CD		0x00080000
    953   1.1  jmcneill #define MCX_CAP_DEVICE_ATM		0x00040000
    954   1.1  jmcneill #define MCX_CAP_DEVICE_APM		0x00020000
    955   1.1  jmcneill #define MCX_CAP_DEVICE_IMAICL		0x00010000
    956   1.1  jmcneill #define MCX_CAP_DEVICE_QKV		0x00000200
    957   1.1  jmcneill #define MCX_CAP_DEVICE_PKV		0x00000100
    958   1.1  jmcneill #define MCX_CAP_DEVICE_SET_DETH_SQPN	0x00000080
    959   1.1  jmcneill #define MCX_CAP_DEVICE_XRC		0x00000008
    960   1.1  jmcneill #define MCX_CAP_DEVICE_UD		0x00000004
    961   1.1  jmcneill #define MCX_CAP_DEVICE_UC		0x00000002
    962   1.1  jmcneill #define MCX_CAP_DEVICE_RC		0x00000001
    963   1.1  jmcneill 
    964   1.1  jmcneill 	uint8_t			uar_flags;
    965   1.1  jmcneill #define MCX_CAP_DEVICE_UAR_4K		0x80
    966   1.1  jmcneill 	uint8_t			uar_sz;	/* 6 bits */
    967   1.1  jmcneill #define MCX_CAP_DEVICE_UAR_SZ		0x3f
    968   1.1  jmcneill 	uint8_t			__reserved__[1];
    969   1.1  jmcneill 	uint8_t			log_pg_sz;
    970   1.1  jmcneill 
    971   1.1  jmcneill 	uint8_t			flags7;
    972   1.1  jmcneill #define MCX_CAP_DEVICE_BF		0x80
    973   1.1  jmcneill #define MCX_CAP_DEVICE_DRIVER_VERSION	0x40
    974   1.1  jmcneill #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
    975   1.1  jmcneill 					0x20
    976   1.1  jmcneill 	uint8_t			log_bf_reg_size; /* 5 bits */
    977   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE	0x1f
    978   1.1  jmcneill 	uint8_t			__reserved__[2];
    979   1.1  jmcneill 
    980   1.1  jmcneill 	uint16_t		num_of_diagnostic_counters;
    981   1.1  jmcneill 	uint16_t		max_wqe_sz_sq;
    982   1.1  jmcneill 
    983   1.1  jmcneill 	uint8_t			__reserved__[2];
    984   1.1  jmcneill 	uint16_t		max_wqe_sz_rq;
    985   1.1  jmcneill 
    986   1.1  jmcneill 	uint8_t			__reserved__[2];
    987   1.1  jmcneill 	uint16_t		max_wqe_sz_sq_dc;
    988   1.1  jmcneill 
    989   1.1  jmcneill 	uint32_t		max_qp_mcg; /* 25 bits */
    990   1.1  jmcneill #define MCX_CAP_DEVICE_MAX_QP_MCG	0x1ffffff
    991   1.1  jmcneill 
    992   1.1  jmcneill 	uint8_t			__reserved__[3];
    993   1.1  jmcneill 	uint8_t			log_max_mcq;
    994   1.1  jmcneill 
    995   1.1  jmcneill 	uint8_t			log_max_transport_domain; /* 5 bits */
    996   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
    997   1.1  jmcneill 					0x1f
    998   1.1  jmcneill 	uint8_t			log_max_pd; /* 5 bits */
    999   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_PD	0x1f
   1000   1.1  jmcneill 	uint8_t			__reserved__[1];
   1001   1.1  jmcneill 	uint8_t			log_max_xrcd; /* 5 bits */
   1002   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_XRCD	0x1f
   1003   1.1  jmcneill 
   1004   1.1  jmcneill 	uint8_t			__reserved__[2];
   1005   1.1  jmcneill 	uint16_t		max_flow_counter;
   1006   1.1  jmcneill 
   1007   1.1  jmcneill 	uint8_t			log_max_rq; /* 5 bits */
   1008   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQ	0x1f
   1009   1.1  jmcneill 	uint8_t			log_max_sq; /* 5 bits */
   1010   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SQ	0x1f
   1011   1.1  jmcneill 	uint8_t			log_max_tir; /* 5 bits */
   1012   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIR	0x1f
   1013   1.1  jmcneill 	uint8_t			log_max_tis; /* 5 bits */
   1014   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS	0x1f
   1015   1.1  jmcneill 
   1016   1.1  jmcneill 	uint8_t 		flags8;
   1017   1.1  jmcneill #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
   1018   1.1  jmcneill 					0x80
   1019   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RMP	0x1f
   1020   1.1  jmcneill 	uint8_t			log_max_rqt; /* 5 bits */
   1021   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT	0x1f
   1022   1.1  jmcneill 	uint8_t			log_max_rqt_size; /* 5 bits */
   1023   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE	0x1f
   1024   1.1  jmcneill 	uint8_t			log_max_tis_per_sq; /* 5 bits */
   1025   1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
   1026   1.1  jmcneill 					0x1f
   1027  1.15  jmcneill 
   1028  1.15  jmcneill 	uint8_t			flags9;
   1029  1.15  jmcneill #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \
   1030  1.15  jmcneill 					0x80
   1031  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \
   1032  1.15  jmcneill 					0x1f
   1033  1.15  jmcneill 	uint8_t			log_min_stride_sz_rq; /* 5 bits */
   1034  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \
   1035  1.15  jmcneill 					0x1f
   1036  1.15  jmcneill 	uint8_t			log_max_stride_sz_sq; /* 5 bits */
   1037  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \
   1038  1.15  jmcneill 					0x1f
   1039  1.15  jmcneill 	uint8_t			log_min_stride_sz_sq; /* 5 bits */
   1040  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \
   1041  1.15  jmcneill 					0x1f
   1042  1.15  jmcneill 
   1043  1.15  jmcneill 	uint8_t			log_max_hairpin_queues;
   1044  1.15  jmcneill #define MXC_CAP_DEVICE_HAIRPIN		0x80
   1045  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \
   1046  1.15  jmcneill 					0x1f
   1047  1.15  jmcneill 	uint8_t			log_min_hairpin_queues;
   1048  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \
   1049  1.15  jmcneill 					0x1f
   1050  1.15  jmcneill 	uint8_t			log_max_hairpin_num_packets;
   1051  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \
   1052  1.15  jmcneill 					0x1f
   1053  1.15  jmcneill 	uint8_t			log_max_mq_sz;
   1054  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \
   1055  1.15  jmcneill 					0x1f
   1056  1.15  jmcneill 
   1057  1.15  jmcneill 	uint8_t			log_min_hairpin_wq_data_sz;
   1058  1.15  jmcneill #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \
   1059  1.15  jmcneill 					0x80
   1060  1.15  jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \
   1061  1.15  jmcneill 					0x40
   1062  1.15  jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \
   1063  1.15  jmcneill 					0x20
   1064  1.15  jmcneill #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \
   1065  1.15  jmcneill 					0x1f
   1066  1.15  jmcneill 	uint8_t			log_max_vlan_list;
   1067  1.15  jmcneill #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \
   1068  1.15  jmcneill 					0x80
   1069  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \
   1070  1.15  jmcneill 					0x1f
   1071  1.15  jmcneill 	uint8_t			log_max_current_mc_list;
   1072  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \
   1073  1.15  jmcneill 					0x1f
   1074  1.15  jmcneill 	uint8_t			log_max_current_uc_list;
   1075  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \
   1076  1.15  jmcneill 					0x1f
   1077  1.15  jmcneill 
   1078  1.15  jmcneill 	uint8_t			__reserved__[4];
   1079  1.15  jmcneill 
   1080  1.15  jmcneill 	uint32_t		create_qp_start_hint; /* 24 bits */
   1081  1.15  jmcneill 
   1082  1.15  jmcneill 	uint8_t			log_max_uctx; /* 5 bits */
   1083  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UCTX	0x1f
   1084  1.15  jmcneill 	uint8_t			log_max_umem; /* 5 bits */
   1085  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UMEM	0x1f
   1086  1.15  jmcneill 	uint16_t		max_num_eqs;
   1087  1.15  jmcneill 
   1088  1.15  jmcneill 	uint8_t			log_max_l2_table; /* 5 bits */
   1089  1.15  jmcneill #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE	0x1f
   1090  1.15  jmcneill 	uint8_t			__reserved__[1];
   1091  1.15  jmcneill 	uint16_t		log_uar_page_sz;
   1092  1.15  jmcneill 
   1093  1.15  jmcneill 	uint8_t			__reserved__[8];
   1094  1.15  jmcneill 
   1095  1.15  jmcneill 	uint32_t		device_frequency_mhz;
   1096  1.15  jmcneill 	uint32_t		device_frequency_khz;
   1097   1.1  jmcneill } __packed __aligned(8);
   1098   1.1  jmcneill 
   1099   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
   1100   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
   1101   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
   1102   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
   1103   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
   1104   1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
   1105  1.15  jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98);
   1106  1.15  jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c);
   1107   1.1  jmcneill CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
   1108   1.1  jmcneill 
   1109   1.1  jmcneill struct mcx_cmd_set_driver_version_in {
   1110   1.1  jmcneill 	uint16_t		cmd_opcode;
   1111   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1112   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1113   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1114   1.1  jmcneill } __packed __aligned(4);
   1115   1.1  jmcneill 
   1116   1.1  jmcneill struct mcx_cmd_set_driver_version_out {
   1117   1.1  jmcneill 	uint8_t			cmd_status;
   1118   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1119   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1120   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1121   1.1  jmcneill } __packed __aligned(4);
   1122   1.1  jmcneill 
   1123   1.1  jmcneill struct mcx_cmd_set_driver_version {
   1124   1.1  jmcneill 	uint8_t			cmd_driver_version[64];
   1125   1.1  jmcneill } __packed __aligned(8);
   1126   1.1  jmcneill 
   1127   1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_in {
   1128   1.1  jmcneill 	uint16_t		cmd_opcode;
   1129   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1130   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1131   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1132   1.1  jmcneill 	uint32_t		cmd_field_select;
   1133   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR	0x04
   1134   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC	0x10
   1135   1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU	0x40
   1136   1.1  jmcneill } __packed __aligned(4);
   1137   1.1  jmcneill 
   1138   1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_out {
   1139   1.1  jmcneill 	uint8_t			cmd_status;
   1140   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1141   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1142   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1143   1.1  jmcneill } __packed __aligned(4);
   1144   1.1  jmcneill 
   1145   1.1  jmcneill struct mcx_cmd_query_nic_vport_context_in {
   1146   1.1  jmcneill 	uint16_t		cmd_opcode;
   1147   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1148   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1149   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1150   1.1  jmcneill 	uint8_t			cmd_allowed_list_type;
   1151   1.1  jmcneill 	uint8_t			cmd_reserved2[3];
   1152   1.1  jmcneill } __packed __aligned(4);
   1153   1.1  jmcneill 
   1154   1.1  jmcneill struct mcx_cmd_query_nic_vport_context_out {
   1155   1.1  jmcneill 	uint8_t			cmd_status;
   1156   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1157   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1158   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1159   1.1  jmcneill } __packed __aligned(4);
   1160   1.1  jmcneill 
   1161   1.1  jmcneill struct mcx_nic_vport_ctx {
   1162   1.1  jmcneill 	uint32_t		vp_min_wqe_inline_mode;
   1163   1.1  jmcneill 	uint8_t			vp_reserved0[32];
   1164   1.1  jmcneill 	uint32_t		vp_mtu;
   1165   1.1  jmcneill 	uint8_t			vp_reserved1[200];
   1166   1.1  jmcneill 	uint16_t		vp_flags;
   1167   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_UC_MAC			(0)
   1168   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_MC_MAC			(1 << 24)
   1169   1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_VLAN			(2 << 24)
   1170   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_ALL			(1 << 13)
   1171   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_MCAST			(1 << 14)
   1172   1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_UCAST			(1 << 15)
   1173   1.1  jmcneill 	uint16_t		vp_allowed_list_size;
   1174   1.1  jmcneill 	uint64_t		vp_perm_addr;
   1175   1.1  jmcneill 	uint8_t			vp_reserved2[4];
   1176   1.1  jmcneill 	/* allowed list follows */
   1177   1.1  jmcneill } __packed __aligned(4);
   1178   1.1  jmcneill 
   1179   1.1  jmcneill struct mcx_counter {
   1180   1.1  jmcneill 	uint64_t		packets;
   1181   1.1  jmcneill 	uint64_t		octets;
   1182   1.1  jmcneill } __packed __aligned(4);
   1183   1.1  jmcneill 
   1184   1.1  jmcneill struct mcx_nic_vport_counters {
   1185   1.1  jmcneill 	struct mcx_counter	rx_err;
   1186   1.1  jmcneill 	struct mcx_counter	tx_err;
   1187   1.1  jmcneill 	uint8_t			reserved0[64]; /* 0x30 */
   1188   1.1  jmcneill 	struct mcx_counter	rx_bcast;
   1189   1.1  jmcneill 	struct mcx_counter	tx_bcast;
   1190   1.1  jmcneill 	struct mcx_counter	rx_ucast;
   1191   1.1  jmcneill 	struct mcx_counter	tx_ucast;
   1192   1.1  jmcneill 	struct mcx_counter	rx_mcast;
   1193   1.1  jmcneill 	struct mcx_counter	tx_mcast;
   1194   1.1  jmcneill 	uint8_t			reserved1[0x210 - 0xd0];
   1195   1.1  jmcneill } __packed __aligned(4);
   1196   1.1  jmcneill 
   1197   1.1  jmcneill struct mcx_cmd_query_vport_counters_in {
   1198   1.1  jmcneill 	uint16_t		cmd_opcode;
   1199   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1200   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1201   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1202   1.1  jmcneill } __packed __aligned(4);
   1203   1.1  jmcneill 
   1204   1.1  jmcneill struct mcx_cmd_query_vport_counters_mb_in {
   1205   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1206   1.1  jmcneill 	uint8_t			cmd_clear;
   1207   1.1  jmcneill 	uint8_t			cmd_reserved1[7];
   1208   1.1  jmcneill } __packed __aligned(4);
   1209   1.1  jmcneill 
   1210   1.1  jmcneill struct mcx_cmd_query_vport_counters_out {
   1211   1.1  jmcneill 	uint8_t			cmd_status;
   1212   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1213   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1214   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1215   1.1  jmcneill } __packed __aligned(4);
   1216   1.1  jmcneill 
   1217   1.1  jmcneill struct mcx_cmd_query_flow_counter_in {
   1218   1.1  jmcneill 	uint16_t		cmd_opcode;
   1219   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1220   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1221   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1222   1.1  jmcneill } __packed __aligned(4);
   1223   1.1  jmcneill 
   1224   1.1  jmcneill struct mcx_cmd_query_flow_counter_mb_in {
   1225   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1226   1.1  jmcneill 	uint8_t			cmd_clear;
   1227   1.1  jmcneill 	uint8_t			cmd_reserved1[5];
   1228   1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
   1229   1.1  jmcneill } __packed __aligned(4);
   1230   1.1  jmcneill 
   1231   1.1  jmcneill struct mcx_cmd_query_flow_counter_out {
   1232   1.1  jmcneill 	uint8_t			cmd_status;
   1233   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1234   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1235   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1236   1.1  jmcneill } __packed __aligned(4);
   1237   1.1  jmcneill 
   1238   1.1  jmcneill struct mcx_cmd_alloc_uar_in {
   1239   1.1  jmcneill 	uint16_t		cmd_opcode;
   1240   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1241   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1242   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1243   1.1  jmcneill } __packed __aligned(4);
   1244   1.1  jmcneill 
   1245   1.1  jmcneill struct mcx_cmd_alloc_uar_out {
   1246   1.1  jmcneill 	uint8_t			cmd_status;
   1247   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1248   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1249   1.1  jmcneill 	uint32_t		cmd_uar;
   1250   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1251   1.1  jmcneill } __packed __aligned(4);
   1252   1.1  jmcneill 
   1253   1.1  jmcneill struct mcx_cmd_query_special_ctx_in {
   1254   1.1  jmcneill 	uint16_t		cmd_opcode;
   1255   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1256   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1257   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1258   1.1  jmcneill } __packed __aligned(4);
   1259   1.1  jmcneill 
   1260   1.1  jmcneill struct mcx_cmd_query_special_ctx_out {
   1261   1.1  jmcneill 	uint8_t			cmd_status;
   1262   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1263   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1264   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1265   1.1  jmcneill 	uint32_t		cmd_resd_lkey;
   1266   1.1  jmcneill } __packed __aligned(4);
   1267   1.1  jmcneill 
   1268   1.1  jmcneill struct mcx_eq_ctx {
   1269   1.1  jmcneill 	uint32_t		eq_status;
   1270  1.15  jmcneill #define MCX_EQ_CTX_STATE_SHIFT		8
   1271  1.15  jmcneill #define MCX_EQ_CTX_STATE_MASK		(0xf << MCX_EQ_CTX_STATE_SHIFT)
   1272  1.15  jmcneill #define MCX_EQ_CTX_STATE_ARMED		0x9
   1273  1.15  jmcneill #define MCX_EQ_CTX_STATE_FIRED		0xa
   1274   1.1  jmcneill #define MCX_EQ_CTX_OI_SHIFT		17
   1275   1.1  jmcneill #define MCX_EQ_CTX_OI			(1 << MCX_EQ_CTX_OI_SHIFT)
   1276   1.1  jmcneill #define MCX_EQ_CTX_EC_SHIFT		18
   1277   1.1  jmcneill #define MCX_EQ_CTX_EC			(1 << MCX_EQ_CTX_EC_SHIFT)
   1278   1.1  jmcneill #define MCX_EQ_CTX_STATUS_SHIFT		28
   1279   1.1  jmcneill #define MCX_EQ_CTX_STATUS_MASK		(0xf << MCX_EQ_CTX_STATUS_SHIFT)
   1280  1.15  jmcneill #define MCX_EQ_CTX_STATUS_OK		0x0
   1281  1.15  jmcneill #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa
   1282   1.1  jmcneill 	uint32_t		eq_reserved1;
   1283   1.1  jmcneill 	uint32_t		eq_page_offset;
   1284   1.1  jmcneill #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT	5
   1285   1.1  jmcneill 	uint32_t		eq_uar_size;
   1286   1.1  jmcneill #define MCX_EQ_CTX_UAR_PAGE_MASK	0xffffff
   1287   1.1  jmcneill #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT	24
   1288   1.1  jmcneill 	uint32_t		eq_reserved2;
   1289   1.1  jmcneill 	uint8_t			eq_reserved3[3];
   1290   1.1  jmcneill 	uint8_t			eq_intr;
   1291   1.1  jmcneill 	uint32_t		eq_log_page_size;
   1292   1.1  jmcneill #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1293   1.1  jmcneill 	uint32_t		eq_reserved4[3];
   1294   1.1  jmcneill 	uint32_t		eq_consumer_counter;
   1295   1.1  jmcneill 	uint32_t		eq_producer_counter;
   1296   1.1  jmcneill #define MCX_EQ_CTX_COUNTER_MASK		0xffffff
   1297   1.1  jmcneill 	uint32_t		eq_reserved5[4];
   1298   1.1  jmcneill } __packed __aligned(4);
   1299   1.1  jmcneill 
   1300   1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
   1301   1.1  jmcneill 
   1302   1.1  jmcneill struct mcx_cmd_create_eq_in {
   1303   1.1  jmcneill 	uint16_t		cmd_opcode;
   1304   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1305   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1306   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1307   1.1  jmcneill } __packed __aligned(4);
   1308   1.1  jmcneill 
   1309   1.1  jmcneill struct mcx_cmd_create_eq_mb_in {
   1310   1.1  jmcneill 	struct mcx_eq_ctx	cmd_eq_ctx;
   1311   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1312   1.1  jmcneill 	uint64_t		cmd_event_bitmask;
   1313   1.1  jmcneill #define MCX_EVENT_TYPE_COMPLETION	0x00
   1314   1.1  jmcneill #define MCX_EVENT_TYPE_CQ_ERROR		0x04
   1315   1.1  jmcneill #define MCX_EVENT_TYPE_INTERNAL_ERROR	0x08
   1316   1.1  jmcneill #define MCX_EVENT_TYPE_PORT_CHANGE	0x09
   1317   1.1  jmcneill #define MCX_EVENT_TYPE_CMD_COMPLETION	0x0a
   1318   1.1  jmcneill #define MCX_EVENT_TYPE_PAGE_REQUEST	0x0b
   1319   1.1  jmcneill #define MCX_EVENT_TYPE_LAST_WQE		0x13
   1320   1.1  jmcneill 	uint8_t			cmd_reserved1[176];
   1321   1.1  jmcneill } __packed __aligned(4);
   1322   1.1  jmcneill 
   1323   1.1  jmcneill struct mcx_cmd_create_eq_out {
   1324   1.1  jmcneill 	uint8_t			cmd_status;
   1325   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1326   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1327   1.1  jmcneill 	uint32_t		cmd_eqn;
   1328   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1329   1.1  jmcneill } __packed __aligned(4);
   1330   1.1  jmcneill 
   1331  1.15  jmcneill struct mcx_cmd_query_eq_in {
   1332  1.15  jmcneill 	uint16_t		cmd_opcode;
   1333  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1334  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1335  1.15  jmcneill 	uint32_t		cmd_eqn;
   1336  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1337  1.15  jmcneill } __packed __aligned(4);
   1338  1.15  jmcneill 
   1339  1.15  jmcneill struct mcx_cmd_query_eq_out {
   1340  1.15  jmcneill 	uint8_t			cmd_status;
   1341  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1342  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1343  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1344  1.15  jmcneill } __packed __aligned(4);
   1345  1.15  jmcneill 
   1346   1.1  jmcneill struct mcx_eq_entry {
   1347   1.1  jmcneill 	uint8_t			eq_reserved1;
   1348   1.1  jmcneill 	uint8_t			eq_event_type;
   1349   1.1  jmcneill 	uint8_t			eq_reserved2;
   1350   1.1  jmcneill 	uint8_t			eq_event_sub_type;
   1351   1.1  jmcneill 
   1352   1.1  jmcneill 	uint8_t			eq_reserved3[28];
   1353   1.1  jmcneill 	uint32_t		eq_event_data[7];
   1354   1.1  jmcneill 	uint8_t			eq_reserved4[2];
   1355   1.1  jmcneill 	uint8_t			eq_signature;
   1356   1.1  jmcneill 	uint8_t			eq_owner;
   1357   1.1  jmcneill #define MCX_EQ_ENTRY_OWNER_INIT			1
   1358   1.1  jmcneill } __packed __aligned(4);
   1359   1.1  jmcneill 
   1360   1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_entry) == 64);
   1361   1.1  jmcneill 
   1362   1.1  jmcneill struct mcx_cmd_alloc_pd_in {
   1363   1.1  jmcneill 	uint16_t		cmd_opcode;
   1364   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1365   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1366   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1367   1.1  jmcneill } __packed __aligned(4);
   1368   1.1  jmcneill 
   1369   1.1  jmcneill struct mcx_cmd_alloc_pd_out {
   1370   1.1  jmcneill 	uint8_t			cmd_status;
   1371   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1372   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1373   1.1  jmcneill 	uint32_t		cmd_pd;
   1374   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1375   1.1  jmcneill } __packed __aligned(4);
   1376   1.1  jmcneill 
   1377   1.1  jmcneill struct mcx_cmd_alloc_td_in {
   1378   1.1  jmcneill 	uint16_t		cmd_opcode;
   1379   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1380   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1381   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1382   1.1  jmcneill } __packed __aligned(4);
   1383   1.1  jmcneill 
   1384   1.1  jmcneill struct mcx_cmd_alloc_td_out {
   1385   1.1  jmcneill 	uint8_t			cmd_status;
   1386   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1387   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1388   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1389   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1390   1.1  jmcneill } __packed __aligned(4);
   1391   1.1  jmcneill 
   1392   1.1  jmcneill struct mcx_cmd_create_tir_in {
   1393   1.1  jmcneill 	uint16_t		cmd_opcode;
   1394   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1395   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1396   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1397   1.1  jmcneill } __packed __aligned(4);
   1398   1.1  jmcneill 
   1399   1.1  jmcneill struct mcx_cmd_create_tir_mb_in {
   1400   1.1  jmcneill 	uint8_t			cmd_reserved0[20];
   1401   1.1  jmcneill 	uint32_t		cmd_disp_type;
   1402  1.15  jmcneill #define MCX_TIR_CTX_DISP_TYPE_DIRECT	0
   1403  1.15  jmcneill #define MCX_TIR_CTX_DISP_TYPE_INDIRECT	1
   1404   1.1  jmcneill #define MCX_TIR_CTX_DISP_TYPE_SHIFT	28
   1405   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1406   1.1  jmcneill 	uint32_t		cmd_lro;
   1407   1.1  jmcneill 	uint8_t			cmd_reserved2[8];
   1408   1.1  jmcneill 	uint32_t		cmd_inline_rqn;
   1409   1.1  jmcneill 	uint32_t		cmd_indir_table;
   1410   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1411  1.15  jmcneill #define MCX_TIR_CTX_HASH_TOEPLITZ	2
   1412  1.15  jmcneill #define MCX_TIR_CTX_HASH_SHIFT		28
   1413   1.1  jmcneill 	uint8_t			cmd_rx_hash_key[40];
   1414   1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_outer;
   1415  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_SRC_IP	(1 << 0)
   1416  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_DST_IP	(1 << 1)
   1417  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_SPORT	(1 << 2)
   1418  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_DPORT	(1 << 3)
   1419  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV4	(0 << 31)
   1420  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV6	(1 << 31)
   1421  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_TCP	(0 << 30)
   1422  1.15  jmcneill #define MCX_TIR_CTX_HASH_SEL_UDP	(1 << 30)
   1423   1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_inner;
   1424   1.1  jmcneill 	uint8_t			cmd_reserved3[152];
   1425   1.1  jmcneill } __packed __aligned(4);
   1426   1.1  jmcneill 
   1427   1.1  jmcneill struct mcx_cmd_create_tir_out {
   1428   1.1  jmcneill 	uint8_t			cmd_status;
   1429   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1430   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1431   1.1  jmcneill 	uint32_t		cmd_tirn;
   1432   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1433   1.1  jmcneill } __packed __aligned(4);
   1434   1.1  jmcneill 
   1435   1.1  jmcneill struct mcx_cmd_destroy_tir_in {
   1436   1.1  jmcneill 	uint16_t		cmd_opcode;
   1437   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1438   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1439   1.1  jmcneill 	uint32_t		cmd_tirn;
   1440   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1441   1.1  jmcneill } __packed __aligned(4);
   1442   1.1  jmcneill 
   1443   1.1  jmcneill struct mcx_cmd_destroy_tir_out {
   1444   1.1  jmcneill 	uint8_t			cmd_status;
   1445   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1446   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1447   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1448   1.1  jmcneill } __packed __aligned(4);
   1449   1.1  jmcneill 
   1450   1.1  jmcneill struct mcx_cmd_create_tis_in {
   1451   1.1  jmcneill 	uint16_t		cmd_opcode;
   1452   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1453   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1454   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1455   1.1  jmcneill } __packed __aligned(4);
   1456   1.1  jmcneill 
   1457   1.1  jmcneill struct mcx_cmd_create_tis_mb_in {
   1458   1.1  jmcneill 	uint8_t			cmd_reserved[16];
   1459   1.1  jmcneill 	uint32_t		cmd_prio;
   1460   1.1  jmcneill 	uint8_t			cmd_reserved1[32];
   1461   1.1  jmcneill 	uint32_t		cmd_tdomain;
   1462   1.1  jmcneill 	uint8_t			cmd_reserved2[120];
   1463   1.1  jmcneill } __packed __aligned(4);
   1464   1.1  jmcneill 
   1465   1.1  jmcneill struct mcx_cmd_create_tis_out {
   1466   1.1  jmcneill 	uint8_t			cmd_status;
   1467   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1468   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1469   1.1  jmcneill 	uint32_t		cmd_tisn;
   1470   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1471   1.1  jmcneill } __packed __aligned(4);
   1472   1.1  jmcneill 
   1473   1.1  jmcneill struct mcx_cmd_destroy_tis_in {
   1474   1.1  jmcneill 	uint16_t		cmd_opcode;
   1475   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1476   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1477   1.1  jmcneill 	uint32_t		cmd_tisn;
   1478   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1479   1.1  jmcneill } __packed __aligned(4);
   1480   1.1  jmcneill 
   1481   1.1  jmcneill struct mcx_cmd_destroy_tis_out {
   1482   1.1  jmcneill 	uint8_t			cmd_status;
   1483   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1484   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1485   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1486   1.1  jmcneill } __packed __aligned(4);
   1487   1.1  jmcneill 
   1488  1.15  jmcneill struct mcx_cmd_create_rqt_in {
   1489  1.15  jmcneill 	uint16_t		cmd_opcode;
   1490  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1491  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1492  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1493  1.15  jmcneill } __packed __aligned(4);
   1494  1.15  jmcneill 
   1495  1.15  jmcneill struct mcx_rqt_ctx {
   1496  1.15  jmcneill 	uint8_t			cmd_reserved0[20];
   1497  1.15  jmcneill 	uint16_t		cmd_reserved1;
   1498  1.15  jmcneill 	uint16_t		cmd_rqt_max_size;
   1499  1.15  jmcneill 	uint16_t		cmd_reserved2;
   1500  1.15  jmcneill 	uint16_t		cmd_rqt_actual_size;
   1501  1.15  jmcneill 	uint8_t			cmd_reserved3[212];
   1502  1.15  jmcneill } __packed __aligned(4);
   1503  1.15  jmcneill 
   1504  1.15  jmcneill struct mcx_cmd_create_rqt_mb_in {
   1505  1.15  jmcneill 	uint8_t			cmd_reserved0[16];
   1506  1.15  jmcneill 	struct mcx_rqt_ctx	cmd_rqt;
   1507  1.15  jmcneill } __packed __aligned(4);
   1508  1.15  jmcneill 
   1509  1.15  jmcneill struct mcx_cmd_create_rqt_out {
   1510  1.15  jmcneill 	uint8_t			cmd_status;
   1511  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1512  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1513  1.15  jmcneill 	uint32_t		cmd_rqtn;
   1514  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1515  1.15  jmcneill } __packed __aligned(4);
   1516  1.15  jmcneill 
   1517  1.15  jmcneill struct mcx_cmd_destroy_rqt_in {
   1518  1.15  jmcneill 	uint16_t		cmd_opcode;
   1519  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1520  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1521  1.15  jmcneill 	uint32_t		cmd_rqtn;
   1522  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1523  1.15  jmcneill } __packed __aligned(4);
   1524  1.15  jmcneill 
   1525  1.15  jmcneill struct mcx_cmd_destroy_rqt_out {
   1526  1.15  jmcneill 	uint8_t			cmd_status;
   1527  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1528  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1529  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1530  1.15  jmcneill } __packed __aligned(4);
   1531  1.15  jmcneill 
   1532   1.1  jmcneill struct mcx_cq_ctx {
   1533   1.1  jmcneill 	uint32_t		cq_status;
   1534  1.15  jmcneill #define MCX_CQ_CTX_STATUS_SHIFT		28
   1535  1.15  jmcneill #define MCX_CQ_CTX_STATUS_MASK		(0xf << MCX_CQ_CTX_STATUS_SHIFT)
   1536  1.15  jmcneill #define MCX_CQ_CTX_STATUS_OK		0x0
   1537  1.15  jmcneill #define MCX_CQ_CTX_STATUS_OVERFLOW	0x9
   1538  1.15  jmcneill #define MCX_CQ_CTX_STATUS_WRITE_FAIL	0xa
   1539  1.15  jmcneill #define MCX_CQ_CTX_STATE_SHIFT		8
   1540  1.15  jmcneill #define MCX_CQ_CTX_STATE_MASK		(0xf << MCX_CQ_CTX_STATE_SHIFT)
   1541  1.15  jmcneill #define MCX_CQ_CTX_STATE_SOLICITED	0x6
   1542  1.15  jmcneill #define MCX_CQ_CTX_STATE_ARMED		0x9
   1543  1.15  jmcneill #define MCX_CQ_CTX_STATE_FIRED		0xa
   1544   1.1  jmcneill 	uint32_t		cq_reserved1;
   1545   1.1  jmcneill 	uint32_t		cq_page_offset;
   1546   1.1  jmcneill 	uint32_t		cq_uar_size;
   1547   1.1  jmcneill #define MCX_CQ_CTX_UAR_PAGE_MASK	0xffffff
   1548   1.1  jmcneill #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT	24
   1549   1.1  jmcneill 	uint32_t		cq_period_max_count;
   1550   1.1  jmcneill #define MCX_CQ_CTX_PERIOD_SHIFT		16
   1551   1.1  jmcneill 	uint32_t		cq_eqn;
   1552   1.1  jmcneill 	uint32_t		cq_log_page_size;
   1553   1.1  jmcneill #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1554   1.1  jmcneill 	uint32_t		cq_reserved2;
   1555   1.1  jmcneill 	uint32_t		cq_last_notified;
   1556   1.1  jmcneill 	uint32_t		cq_last_solicit;
   1557   1.1  jmcneill 	uint32_t		cq_consumer_counter;
   1558   1.1  jmcneill 	uint32_t		cq_producer_counter;
   1559   1.1  jmcneill 	uint8_t			cq_reserved3[8];
   1560   1.1  jmcneill 	uint64_t		cq_doorbell;
   1561   1.1  jmcneill } __packed __aligned(4);
   1562   1.1  jmcneill 
   1563   1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
   1564   1.1  jmcneill 
   1565   1.1  jmcneill struct mcx_cmd_create_cq_in {
   1566   1.1  jmcneill 	uint16_t		cmd_opcode;
   1567   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1568   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1569   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1570   1.1  jmcneill } __packed __aligned(4);
   1571   1.1  jmcneill 
   1572   1.1  jmcneill struct mcx_cmd_create_cq_mb_in {
   1573   1.1  jmcneill 	struct mcx_cq_ctx	cmd_cq_ctx;
   1574   1.1  jmcneill 	uint8_t			cmd_reserved1[192];
   1575   1.1  jmcneill } __packed __aligned(4);
   1576   1.1  jmcneill 
   1577   1.1  jmcneill struct mcx_cmd_create_cq_out {
   1578   1.1  jmcneill 	uint8_t			cmd_status;
   1579   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1580   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1581   1.1  jmcneill 	uint32_t		cmd_cqn;
   1582   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1583   1.1  jmcneill } __packed __aligned(4);
   1584   1.1  jmcneill 
   1585   1.1  jmcneill struct mcx_cmd_destroy_cq_in {
   1586   1.1  jmcneill 	uint16_t		cmd_opcode;
   1587   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1588   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1589   1.1  jmcneill 	uint32_t		cmd_cqn;
   1590   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1591   1.1  jmcneill } __packed __aligned(4);
   1592   1.1  jmcneill 
   1593   1.1  jmcneill struct mcx_cmd_destroy_cq_out {
   1594   1.1  jmcneill 	uint8_t			cmd_status;
   1595   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1596   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1597   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1598   1.1  jmcneill } __packed __aligned(4);
   1599   1.1  jmcneill 
   1600  1.15  jmcneill struct mcx_cmd_query_cq_in {
   1601  1.15  jmcneill 	uint16_t		cmd_opcode;
   1602  1.15  jmcneill 	uint8_t			cmd_reserved0[4];
   1603  1.15  jmcneill 	uint16_t		cmd_op_mod;
   1604  1.15  jmcneill 	uint32_t		cmd_cqn;
   1605  1.15  jmcneill 	uint8_t			cmd_reserved1[4];
   1606  1.15  jmcneill } __packed __aligned(4);
   1607  1.15  jmcneill 
   1608  1.15  jmcneill struct mcx_cmd_query_cq_out {
   1609  1.15  jmcneill 	uint8_t			cmd_status;
   1610  1.15  jmcneill 	uint8_t			cmd_reserved0[3];
   1611  1.15  jmcneill 	uint32_t		cmd_syndrome;
   1612  1.15  jmcneill 	uint8_t			cmd_reserved1[8];
   1613  1.15  jmcneill } __packed __aligned(4);
   1614  1.15  jmcneill 
   1615   1.1  jmcneill struct mcx_cq_entry {
   1616   1.1  jmcneill 	uint32_t		__reserved__;
   1617   1.1  jmcneill 	uint32_t		cq_lro;
   1618   1.1  jmcneill 	uint32_t		cq_lro_ack_seq_num;
   1619   1.1  jmcneill 	uint32_t		cq_rx_hash;
   1620   1.1  jmcneill 	uint8_t			cq_rx_hash_type;
   1621   1.1  jmcneill 	uint8_t			cq_ml_path;
   1622   1.1  jmcneill 	uint16_t		__reserved__;
   1623   1.1  jmcneill 	uint32_t		cq_checksum;
   1624   1.1  jmcneill 	uint32_t		__reserved__;
   1625   1.1  jmcneill 	uint32_t		cq_flags;
   1626  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L4_OK		(1 << 26)
   1627  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L3_OK		(1 << 25)
   1628  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_L2_OK		(1 << 24)
   1629  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_CV			(1 << 16)
   1630  1.15  jmcneill #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK		(0xffff)
   1631  1.15  jmcneill 
   1632   1.1  jmcneill 	uint32_t		cq_lro_srqn;
   1633   1.1  jmcneill 	uint32_t		__reserved__[2];
   1634   1.1  jmcneill 	uint32_t		cq_byte_cnt;
   1635   1.1  jmcneill 	uint64_t		cq_timestamp;
   1636   1.1  jmcneill 	uint8_t			cq_rx_drops;
   1637   1.1  jmcneill 	uint8_t			cq_flow_tag[3];
   1638   1.1  jmcneill 	uint16_t		cq_wqe_count;
   1639   1.1  jmcneill 	uint8_t			cq_signature;
   1640   1.1  jmcneill 	uint8_t			cq_opcode_owner;
   1641   1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_OWNER			(1 << 0)
   1642   1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_SE			(1 << 1)
   1643   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_SHIFT		2
   1644   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SHIFT		4
   1645   1.1  jmcneill 
   1646   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_NO_INLINE		0
   1647   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_32		1
   1648   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_64		2
   1649   1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_COMPRESSED		3
   1650   1.1  jmcneill 
   1651   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ			0
   1652   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND		2
   1653   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ_ERR		13
   1654   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND_ERR		14
   1655   1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_INVALID		15
   1656   1.1  jmcneill 
   1657   1.1  jmcneill } __packed __aligned(4);
   1658   1.1  jmcneill 
   1659   1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_entry) == 64);
   1660   1.1  jmcneill 
   1661   1.1  jmcneill struct mcx_cq_doorbell {
   1662   1.1  jmcneill 	uint32_t		 db_update_ci;
   1663   1.1  jmcneill 	uint32_t		 db_arm_ci;
   1664   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT	28
   1665   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD			(1 << 24)
   1666   1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CI_MASK		(0xffffff)
   1667   1.1  jmcneill } __packed __aligned(8);
   1668   1.1  jmcneill 
   1669   1.1  jmcneill struct mcx_wq_ctx {
   1670   1.1  jmcneill 	uint8_t			 wq_type;
   1671   1.1  jmcneill #define MCX_WQ_CTX_TYPE_CYCLIC			(1 << 4)
   1672   1.1  jmcneill #define MCX_WQ_CTX_TYPE_SIGNATURE		(1 << 3)
   1673   1.1  jmcneill 	uint8_t			 wq_reserved0[5];
   1674   1.1  jmcneill 	uint16_t		 wq_lwm;
   1675   1.1  jmcneill 	uint32_t		 wq_pd;
   1676   1.1  jmcneill 	uint32_t		 wq_uar_page;
   1677   1.1  jmcneill 	uint64_t		 wq_doorbell;
   1678   1.1  jmcneill 	uint32_t		 wq_hw_counter;
   1679   1.1  jmcneill 	uint32_t		 wq_sw_counter;
   1680   1.1  jmcneill 	uint16_t		 wq_log_stride;
   1681   1.1  jmcneill 	uint8_t			 wq_log_page_sz;
   1682   1.1  jmcneill 	uint8_t			 wq_log_size;
   1683   1.1  jmcneill 	uint8_t			 wq_reserved1[156];
   1684   1.1  jmcneill } __packed __aligned(4);
   1685   1.1  jmcneill 
   1686   1.1  jmcneill CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
   1687   1.1  jmcneill 
   1688   1.1  jmcneill struct mcx_sq_ctx {
   1689   1.1  jmcneill 	uint32_t		sq_flags;
   1690   1.5   msaitoh #define MCX_SQ_CTX_RLKEY			(1U << 31)
   1691   1.1  jmcneill #define MCX_SQ_CTX_FRE_SHIFT			(1 << 29)
   1692   1.1  jmcneill #define MCX_SQ_CTX_FLUSH_IN_ERROR		(1 << 28)
   1693   1.1  jmcneill #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT		24
   1694   1.1  jmcneill #define MCX_SQ_CTX_STATE_SHIFT			20
   1695  1.15  jmcneill #define MCX_SQ_CTX_STATE_MASK			(0xf << 20)
   1696  1.15  jmcneill #define MCX_SQ_CTX_STATE_RST			0
   1697  1.15  jmcneill #define MCX_SQ_CTX_STATE_RDY			1
   1698  1.15  jmcneill #define MCX_SQ_CTX_STATE_ERR			3
   1699   1.1  jmcneill 	uint32_t		sq_user_index;
   1700   1.1  jmcneill 	uint32_t		sq_cqn;
   1701   1.1  jmcneill 	uint32_t		sq_reserved1[5];
   1702   1.1  jmcneill 	uint32_t		sq_tis_lst_sz;
   1703   1.1  jmcneill #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT		16
   1704   1.1  jmcneill 	uint32_t		sq_reserved2[2];
   1705   1.1  jmcneill 	uint32_t		sq_tis_num;
   1706   1.1  jmcneill 	struct mcx_wq_ctx	sq_wq;
   1707   1.1  jmcneill } __packed __aligned(4);
   1708   1.1  jmcneill 
   1709   1.1  jmcneill struct mcx_sq_entry_seg {
   1710   1.1  jmcneill 	uint32_t		sqs_byte_count;
   1711   1.1  jmcneill 	uint32_t		sqs_lkey;
   1712   1.1  jmcneill 	uint64_t		sqs_addr;
   1713   1.1  jmcneill } __packed __aligned(4);
   1714   1.1  jmcneill 
   1715   1.1  jmcneill struct mcx_sq_entry {
   1716   1.1  jmcneill 	/* control segment */
   1717   1.1  jmcneill 	uint32_t		sqe_opcode_index;
   1718   1.1  jmcneill #define MCX_SQE_WQE_INDEX_SHIFT			8
   1719   1.1  jmcneill #define MCX_SQE_WQE_OPCODE_NOP			0x00
   1720   1.1  jmcneill #define MCX_SQE_WQE_OPCODE_SEND			0x0a
   1721   1.1  jmcneill 	uint32_t		sqe_ds_sq_num;
   1722   1.1  jmcneill #define MCX_SQE_SQ_NUM_SHIFT			8
   1723   1.1  jmcneill 	uint32_t		sqe_signature;
   1724   1.1  jmcneill #define MCX_SQE_SIGNATURE_SHIFT			24
   1725   1.1  jmcneill #define MCX_SQE_SOLICITED_EVENT			0x02
   1726   1.1  jmcneill #define MCX_SQE_CE_CQE_ON_ERR			0x00
   1727   1.1  jmcneill #define MCX_SQE_CE_CQE_FIRST_ERR		0x04
   1728   1.1  jmcneill #define MCX_SQE_CE_CQE_ALWAYS			0x08
   1729   1.1  jmcneill #define MCX_SQE_CE_CQE_SOLICIT			0x0C
   1730   1.1  jmcneill #define MCX_SQE_FM_NO_FENCE			0x00
   1731   1.1  jmcneill #define MCX_SQE_FM_SMALL_FENCE			0x40
   1732   1.1  jmcneill 	uint32_t		sqe_mkey;
   1733   1.1  jmcneill 
   1734   1.1  jmcneill 	/* ethernet segment */
   1735   1.1  jmcneill 	uint32_t		sqe_reserved1;
   1736   1.1  jmcneill 	uint32_t		sqe_mss_csum;
   1737   1.1  jmcneill #define MCX_SQE_L4_CSUM				(1 << 31)
   1738   1.1  jmcneill #define MCX_SQE_L3_CSUM				(1 << 30)
   1739   1.1  jmcneill 	uint32_t		sqe_reserved2;
   1740   1.1  jmcneill 	uint16_t		sqe_inline_header_size;
   1741   1.1  jmcneill 	uint16_t		sqe_inline_headers[9];
   1742   1.1  jmcneill 
   1743   1.1  jmcneill 	/* data segment */
   1744   1.1  jmcneill 	struct mcx_sq_entry_seg sqe_segs[1];
   1745   1.1  jmcneill } __packed __aligned(64);
   1746   1.1  jmcneill 
   1747   1.1  jmcneill CTASSERT(sizeof(struct mcx_sq_entry) == 64);
   1748   1.1  jmcneill 
   1749   1.1  jmcneill struct mcx_cmd_create_sq_in {
   1750   1.1  jmcneill 	uint16_t		cmd_opcode;
   1751   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1752   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1753   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1754   1.1  jmcneill } __packed __aligned(4);
   1755   1.1  jmcneill 
   1756   1.1  jmcneill struct mcx_cmd_create_sq_out {
   1757   1.1  jmcneill 	uint8_t			cmd_status;
   1758   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1759   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1760   1.1  jmcneill 	uint32_t		cmd_sqn;
   1761   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1762   1.1  jmcneill } __packed __aligned(4);
   1763   1.1  jmcneill 
   1764   1.1  jmcneill struct mcx_cmd_modify_sq_in {
   1765   1.1  jmcneill 	uint16_t		cmd_opcode;
   1766   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1767   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1768   1.1  jmcneill 	uint32_t		cmd_sq_state;
   1769   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1770   1.1  jmcneill } __packed __aligned(4);
   1771   1.1  jmcneill 
   1772   1.1  jmcneill struct mcx_cmd_modify_sq_mb_in {
   1773   1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1774   1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1775   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1776   1.1  jmcneill 	struct mcx_sq_ctx	cmd_sq_ctx;
   1777   1.1  jmcneill } __packed __aligned(4);
   1778   1.1  jmcneill 
   1779   1.1  jmcneill struct mcx_cmd_modify_sq_out {
   1780   1.1  jmcneill 	uint8_t			cmd_status;
   1781   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1782   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1783   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1784   1.1  jmcneill } __packed __aligned(4);
   1785   1.1  jmcneill 
   1786   1.1  jmcneill struct mcx_cmd_destroy_sq_in {
   1787   1.1  jmcneill 	uint16_t		cmd_opcode;
   1788   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1789   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1790   1.1  jmcneill 	uint32_t		cmd_sqn;
   1791   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1792   1.1  jmcneill } __packed __aligned(4);
   1793   1.1  jmcneill 
   1794   1.1  jmcneill struct mcx_cmd_destroy_sq_out {
   1795   1.1  jmcneill 	uint8_t			cmd_status;
   1796   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1797   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1798   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1799   1.1  jmcneill } __packed __aligned(4);
   1800   1.1  jmcneill 
   1801   1.1  jmcneill 
   1802   1.1  jmcneill struct mcx_rq_ctx {
   1803   1.1  jmcneill 	uint32_t		rq_flags;
   1804   1.5   msaitoh #define MCX_RQ_CTX_RLKEY			(1U << 31)
   1805   1.1  jmcneill #define MCX_RQ_CTX_VLAN_STRIP_DIS		(1 << 28)
   1806   1.1  jmcneill #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT		24
   1807   1.1  jmcneill #define MCX_RQ_CTX_STATE_SHIFT			20
   1808  1.15  jmcneill #define MCX_RQ_CTX_STATE_MASK			(0xf << 20)
   1809  1.15  jmcneill #define MCX_RQ_CTX_STATE_RST			0
   1810  1.15  jmcneill #define MCX_RQ_CTX_STATE_RDY			1
   1811  1.15  jmcneill #define MCX_RQ_CTX_STATE_ERR			3
   1812   1.1  jmcneill #define MCX_RQ_CTX_FLUSH_IN_ERROR		(1 << 18)
   1813   1.1  jmcneill 	uint32_t		rq_user_index;
   1814   1.1  jmcneill 	uint32_t		rq_cqn;
   1815   1.1  jmcneill 	uint32_t		rq_reserved1;
   1816   1.1  jmcneill 	uint32_t		rq_rmpn;
   1817   1.1  jmcneill 	uint32_t		rq_reserved2[7];
   1818   1.1  jmcneill 	struct mcx_wq_ctx	rq_wq;
   1819   1.1  jmcneill } __packed __aligned(4);
   1820   1.1  jmcneill 
   1821   1.1  jmcneill struct mcx_rq_entry {
   1822   1.1  jmcneill 	uint32_t		rqe_byte_count;
   1823   1.1  jmcneill 	uint32_t		rqe_lkey;
   1824   1.1  jmcneill 	uint64_t		rqe_addr;
   1825   1.1  jmcneill } __packed __aligned(16);
   1826   1.1  jmcneill 
   1827   1.1  jmcneill struct mcx_cmd_create_rq_in {
   1828   1.1  jmcneill 	uint16_t		cmd_opcode;
   1829   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1830   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1831   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1832   1.1  jmcneill } __packed __aligned(4);
   1833   1.1  jmcneill 
   1834   1.1  jmcneill struct mcx_cmd_create_rq_out {
   1835   1.1  jmcneill 	uint8_t			cmd_status;
   1836   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1837   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1838   1.1  jmcneill 	uint32_t		cmd_rqn;
   1839   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1840   1.1  jmcneill } __packed __aligned(4);
   1841   1.1  jmcneill 
   1842   1.1  jmcneill struct mcx_cmd_modify_rq_in {
   1843   1.1  jmcneill 	uint16_t		cmd_opcode;
   1844   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1845   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1846   1.1  jmcneill 	uint32_t		cmd_rq_state;
   1847   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1848   1.1  jmcneill } __packed __aligned(4);
   1849   1.1  jmcneill 
   1850   1.1  jmcneill struct mcx_cmd_modify_rq_mb_in {
   1851   1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1852   1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1853   1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1854   1.1  jmcneill 	struct mcx_rq_ctx	cmd_rq_ctx;
   1855   1.1  jmcneill } __packed __aligned(4);
   1856   1.1  jmcneill 
   1857   1.1  jmcneill struct mcx_cmd_modify_rq_out {
   1858   1.1  jmcneill 	uint8_t			cmd_status;
   1859   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1860   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1861   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1862   1.1  jmcneill } __packed __aligned(4);
   1863   1.1  jmcneill 
   1864   1.1  jmcneill struct mcx_cmd_destroy_rq_in {
   1865   1.1  jmcneill 	uint16_t		cmd_opcode;
   1866   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1867   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1868   1.1  jmcneill 	uint32_t		cmd_rqn;
   1869   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1870   1.1  jmcneill } __packed __aligned(4);
   1871   1.1  jmcneill 
   1872   1.1  jmcneill struct mcx_cmd_destroy_rq_out {
   1873   1.1  jmcneill 	uint8_t			cmd_status;
   1874   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1875   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1876   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1877   1.1  jmcneill } __packed __aligned(4);
   1878   1.1  jmcneill 
   1879   1.1  jmcneill struct mcx_cmd_create_flow_table_in {
   1880   1.1  jmcneill 	uint16_t		cmd_opcode;
   1881   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1882   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1883   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1884   1.1  jmcneill } __packed __aligned(4);
   1885   1.1  jmcneill 
   1886   1.1  jmcneill struct mcx_flow_table_ctx {
   1887   1.1  jmcneill 	uint8_t			ft_miss_action;
   1888   1.1  jmcneill 	uint8_t			ft_level;
   1889   1.1  jmcneill 	uint8_t			ft_reserved0;
   1890   1.1  jmcneill 	uint8_t			ft_log_size;
   1891   1.1  jmcneill 	uint32_t		ft_table_miss_id;
   1892   1.1  jmcneill 	uint8_t			ft_reserved1[28];
   1893   1.1  jmcneill } __packed __aligned(4);
   1894   1.1  jmcneill 
   1895   1.1  jmcneill struct mcx_cmd_create_flow_table_mb_in {
   1896   1.1  jmcneill 	uint8_t			cmd_table_type;
   1897   1.1  jmcneill 	uint8_t			cmd_reserved0[7];
   1898   1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   1899   1.1  jmcneill } __packed __aligned(4);
   1900   1.1  jmcneill 
   1901   1.1  jmcneill struct mcx_cmd_create_flow_table_out {
   1902   1.1  jmcneill 	uint8_t			cmd_status;
   1903   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1904   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1905   1.1  jmcneill 	uint32_t		cmd_table_id;
   1906   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1907   1.1  jmcneill } __packed __aligned(4);
   1908   1.1  jmcneill 
   1909   1.1  jmcneill struct mcx_cmd_destroy_flow_table_in {
   1910   1.1  jmcneill 	uint16_t		cmd_opcode;
   1911   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1912   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1913   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1914   1.1  jmcneill } __packed __aligned(4);
   1915   1.1  jmcneill 
   1916   1.1  jmcneill struct mcx_cmd_destroy_flow_table_mb_in {
   1917   1.1  jmcneill 	uint8_t			cmd_table_type;
   1918   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1919   1.1  jmcneill 	uint32_t		cmd_table_id;
   1920   1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   1921   1.1  jmcneill } __packed __aligned(4);
   1922   1.1  jmcneill 
   1923   1.1  jmcneill struct mcx_cmd_destroy_flow_table_out {
   1924   1.1  jmcneill 	uint8_t			cmd_status;
   1925   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1926   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1927   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1928   1.1  jmcneill } __packed __aligned(4);
   1929   1.1  jmcneill 
   1930   1.1  jmcneill struct mcx_cmd_set_flow_table_root_in {
   1931   1.1  jmcneill 	uint16_t		cmd_opcode;
   1932   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1933   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1934   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1935   1.1  jmcneill } __packed __aligned(4);
   1936   1.1  jmcneill 
   1937   1.1  jmcneill struct mcx_cmd_set_flow_table_root_mb_in {
   1938   1.1  jmcneill 	uint8_t			cmd_table_type;
   1939   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1940   1.1  jmcneill 	uint32_t		cmd_table_id;
   1941   1.1  jmcneill 	uint8_t			cmd_reserved1[56];
   1942   1.1  jmcneill } __packed __aligned(4);
   1943   1.1  jmcneill 
   1944   1.1  jmcneill struct mcx_cmd_set_flow_table_root_out {
   1945   1.1  jmcneill 	uint8_t			cmd_status;
   1946   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1947   1.1  jmcneill 	uint32_t		cmd_syndrome;
   1948   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1949   1.1  jmcneill } __packed __aligned(4);
   1950   1.1  jmcneill 
   1951   1.1  jmcneill struct mcx_flow_match {
   1952   1.1  jmcneill 	/* outer headers */
   1953   1.1  jmcneill 	uint8_t			mc_src_mac[6];
   1954   1.1  jmcneill 	uint16_t		mc_ethertype;
   1955   1.1  jmcneill 	uint8_t			mc_dest_mac[6];
   1956   1.1  jmcneill 	uint16_t		mc_first_vlan;
   1957   1.1  jmcneill 	uint8_t			mc_ip_proto;
   1958   1.1  jmcneill 	uint8_t			mc_ip_dscp_ecn;
   1959   1.1  jmcneill 	uint8_t			mc_vlan_flags;
   1960  1.15  jmcneill #define MCX_FLOW_MATCH_IP_FRAG	(1 << 5)
   1961   1.1  jmcneill 	uint8_t			mc_tcp_flags;
   1962   1.1  jmcneill 	uint16_t		mc_tcp_sport;
   1963   1.1  jmcneill 	uint16_t		mc_tcp_dport;
   1964   1.1  jmcneill 	uint32_t		mc_reserved0;
   1965   1.1  jmcneill 	uint16_t		mc_udp_sport;
   1966   1.1  jmcneill 	uint16_t		mc_udp_dport;
   1967   1.1  jmcneill 	uint8_t			mc_src_ip[16];
   1968   1.1  jmcneill 	uint8_t			mc_dest_ip[16];
   1969   1.1  jmcneill 
   1970   1.1  jmcneill 	/* misc parameters */
   1971   1.1  jmcneill 	uint8_t			mc_reserved1[8];
   1972   1.1  jmcneill 	uint16_t		mc_second_vlan;
   1973   1.1  jmcneill 	uint8_t			mc_reserved2[2];
   1974   1.1  jmcneill 	uint8_t			mc_second_vlan_flags;
   1975   1.1  jmcneill 	uint8_t			mc_reserved3[15];
   1976   1.1  jmcneill 	uint32_t		mc_outer_ipv6_flow_label;
   1977   1.1  jmcneill 	uint8_t			mc_reserved4[32];
   1978   1.1  jmcneill 
   1979   1.1  jmcneill 	uint8_t			mc_reserved[384];
   1980   1.1  jmcneill } __packed __aligned(4);
   1981   1.1  jmcneill 
   1982   1.1  jmcneill CTASSERT(sizeof(struct mcx_flow_match) == 512);
   1983   1.1  jmcneill 
   1984   1.1  jmcneill struct mcx_cmd_create_flow_group_in {
   1985   1.1  jmcneill 	uint16_t		cmd_opcode;
   1986   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1987   1.1  jmcneill 	uint16_t		cmd_op_mod;
   1988   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1989   1.1  jmcneill } __packed __aligned(4);
   1990   1.1  jmcneill 
   1991   1.1  jmcneill struct mcx_cmd_create_flow_group_mb_in {
   1992   1.1  jmcneill 	uint8_t			cmd_table_type;
   1993   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1994   1.1  jmcneill 	uint32_t		cmd_table_id;
   1995   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1996   1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   1997   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   1998   1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   1999   1.1  jmcneill 	uint8_t			cmd_reserved3[23];
   2000   1.1  jmcneill 	uint8_t			cmd_match_criteria_enable;
   2001   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER	(1 << 0)
   2002   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_MISC		(1 << 1)
   2003   1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_INNER	(1 << 2)
   2004   1.1  jmcneill 	struct mcx_flow_match	cmd_match_criteria;
   2005   1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   2006   1.1  jmcneill } __packed __aligned(4);
   2007   1.1  jmcneill 
   2008   1.1  jmcneill struct mcx_cmd_create_flow_group_out {
   2009   1.1  jmcneill 	uint8_t			cmd_status;
   2010   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2011   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2012   1.1  jmcneill 	uint32_t		cmd_group_id;
   2013   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2014   1.1  jmcneill } __packed __aligned(4);
   2015   1.1  jmcneill 
   2016   1.1  jmcneill struct mcx_flow_ctx {
   2017   1.1  jmcneill 	uint8_t			fc_reserved0[4];
   2018   1.1  jmcneill 	uint32_t		fc_group_id;
   2019   1.1  jmcneill 	uint32_t		fc_flow_tag;
   2020   1.1  jmcneill 	uint32_t		fc_action;
   2021   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_ALLOW		(1 << 0)
   2022   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_DROP		(1 << 1)
   2023   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_FORWARD		(1 << 2)
   2024   1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_COUNT		(1 << 3)
   2025   1.1  jmcneill 	uint32_t		fc_dest_list_size;
   2026   1.1  jmcneill 	uint32_t		fc_counter_list_size;
   2027   1.1  jmcneill 	uint8_t			fc_reserved1[40];
   2028   1.1  jmcneill 	struct mcx_flow_match	fc_match_value;
   2029   1.1  jmcneill 	uint8_t			fc_reserved2[192];
   2030   1.1  jmcneill } __packed __aligned(4);
   2031   1.1  jmcneill 
   2032   1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE	(1 << 24)
   2033   1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR		(2 << 24)
   2034   1.1  jmcneill 
   2035   1.1  jmcneill struct mcx_cmd_destroy_flow_group_in {
   2036   1.1  jmcneill 	uint16_t		cmd_opcode;
   2037   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2038   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2039   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2040   1.1  jmcneill } __packed __aligned(4);
   2041   1.1  jmcneill 
   2042   1.1  jmcneill struct mcx_cmd_destroy_flow_group_mb_in {
   2043   1.1  jmcneill 	uint8_t			cmd_table_type;
   2044   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2045   1.1  jmcneill 	uint32_t		cmd_table_id;
   2046   1.1  jmcneill 	uint32_t		cmd_group_id;
   2047   1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   2048   1.1  jmcneill } __packed __aligned(4);
   2049   1.1  jmcneill 
   2050   1.1  jmcneill struct mcx_cmd_destroy_flow_group_out {
   2051   1.1  jmcneill 	uint8_t			cmd_status;
   2052   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2053   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2054   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2055   1.1  jmcneill } __packed __aligned(4);
   2056   1.1  jmcneill 
   2057   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_in {
   2058   1.1  jmcneill 	uint16_t		cmd_opcode;
   2059   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2060   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2061   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2062   1.1  jmcneill } __packed __aligned(4);
   2063   1.1  jmcneill 
   2064   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_mb_in {
   2065   1.1  jmcneill 	uint8_t			cmd_table_type;
   2066   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2067   1.1  jmcneill 	uint32_t		cmd_table_id;
   2068   1.1  jmcneill 	uint32_t		cmd_modify_enable_mask;
   2069   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2070   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2071   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2072   1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   2073   1.1  jmcneill } __packed __aligned(4);
   2074   1.1  jmcneill 
   2075   1.1  jmcneill struct mcx_cmd_set_flow_table_entry_out {
   2076   1.1  jmcneill 	uint8_t			cmd_status;
   2077   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2078   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2079   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2080   1.1  jmcneill } __packed __aligned(4);
   2081   1.1  jmcneill 
   2082   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_in {
   2083   1.1  jmcneill 	uint16_t		cmd_opcode;
   2084   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2085   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2086   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2087   1.1  jmcneill } __packed __aligned(4);
   2088   1.1  jmcneill 
   2089   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_in {
   2090   1.1  jmcneill 	uint8_t			cmd_table_type;
   2091   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2092   1.1  jmcneill 	uint32_t		cmd_table_id;
   2093   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2094   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2095   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2096   1.1  jmcneill } __packed __aligned(4);
   2097   1.1  jmcneill 
   2098   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_out {
   2099   1.1  jmcneill 	uint8_t			cmd_status;
   2100   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2101   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2102   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2103   1.1  jmcneill } __packed __aligned(4);
   2104   1.1  jmcneill 
   2105   1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_out {
   2106   1.1  jmcneill 	uint8_t			cmd_reserved0[48];
   2107   1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   2108   1.1  jmcneill } __packed __aligned(4);
   2109   1.1  jmcneill 
   2110   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_in {
   2111   1.1  jmcneill 	uint16_t		cmd_opcode;
   2112   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2113   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2114   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2115   1.1  jmcneill } __packed __aligned(4);
   2116   1.1  jmcneill 
   2117   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in {
   2118   1.1  jmcneill 	uint8_t			cmd_table_type;
   2119   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2120   1.1  jmcneill 	uint32_t		cmd_table_id;
   2121   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2122   1.1  jmcneill 	uint32_t		cmd_flow_index;
   2123   1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   2124   1.1  jmcneill } __packed __aligned(4);
   2125   1.1  jmcneill 
   2126   1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_out {
   2127   1.1  jmcneill 	uint8_t			cmd_status;
   2128   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2129   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2130   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2131   1.1  jmcneill } __packed __aligned(4);
   2132   1.1  jmcneill 
   2133   1.1  jmcneill struct mcx_cmd_query_flow_group_in {
   2134   1.1  jmcneill 	uint16_t		cmd_opcode;
   2135   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2136   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2137   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2138   1.1  jmcneill } __packed __aligned(4);
   2139   1.1  jmcneill 
   2140   1.1  jmcneill struct mcx_cmd_query_flow_group_mb_in {
   2141   1.1  jmcneill 	uint8_t			cmd_table_type;
   2142   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2143   1.1  jmcneill 	uint32_t		cmd_table_id;
   2144   1.1  jmcneill 	uint32_t		cmd_group_id;
   2145   1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   2146   1.1  jmcneill } __packed __aligned(4);
   2147   1.1  jmcneill 
   2148   1.1  jmcneill struct mcx_cmd_query_flow_group_out {
   2149   1.1  jmcneill 	uint8_t			cmd_status;
   2150   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2151   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2152   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2153   1.1  jmcneill } __packed __aligned(4);
   2154   1.1  jmcneill 
   2155   1.1  jmcneill struct mcx_cmd_query_flow_group_mb_out {
   2156   1.1  jmcneill 	uint8_t			cmd_reserved0[12];
   2157   1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   2158   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2159   1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   2160   1.1  jmcneill 	uint8_t			cmd_reserved2[20];
   2161   1.1  jmcneill 	uint32_t		cmd_match_criteria_enable;
   2162   1.1  jmcneill 	uint8_t			cmd_match_criteria[512];
   2163   1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   2164   1.1  jmcneill } __packed __aligned(4);
   2165   1.1  jmcneill 
   2166   1.1  jmcneill struct mcx_cmd_query_flow_table_in {
   2167   1.1  jmcneill 	uint16_t		cmd_opcode;
   2168   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2169   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2170   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2171   1.1  jmcneill } __packed __aligned(4);
   2172   1.1  jmcneill 
   2173   1.1  jmcneill struct mcx_cmd_query_flow_table_mb_in {
   2174   1.1  jmcneill 	uint8_t			cmd_table_type;
   2175   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2176   1.1  jmcneill 	uint32_t		cmd_table_id;
   2177   1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   2178   1.1  jmcneill } __packed __aligned(4);
   2179   1.1  jmcneill 
   2180   1.1  jmcneill struct mcx_cmd_query_flow_table_out {
   2181   1.1  jmcneill 	uint8_t			cmd_status;
   2182   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2183   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2184   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2185   1.1  jmcneill } __packed __aligned(4);
   2186   1.1  jmcneill 
   2187   1.1  jmcneill struct mcx_cmd_query_flow_table_mb_out {
   2188   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2189   1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   2190   1.1  jmcneill } __packed __aligned(4);
   2191   1.1  jmcneill 
   2192   1.1  jmcneill struct mcx_cmd_alloc_flow_counter_in {
   2193   1.1  jmcneill 	uint16_t		cmd_opcode;
   2194   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2195   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2196   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2197   1.1  jmcneill } __packed __aligned(4);
   2198   1.1  jmcneill 
   2199   1.1  jmcneill struct mcx_cmd_query_rq_in {
   2200   1.1  jmcneill 	uint16_t		cmd_opcode;
   2201   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2202   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2203   1.1  jmcneill 	uint32_t		cmd_rqn;
   2204   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2205   1.1  jmcneill } __packed __aligned(4);
   2206   1.1  jmcneill 
   2207   1.1  jmcneill struct mcx_cmd_query_rq_out {
   2208   1.1  jmcneill 	uint8_t			cmd_status;
   2209   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2210   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2211   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2212   1.1  jmcneill } __packed __aligned(4);
   2213   1.1  jmcneill 
   2214   1.1  jmcneill struct mcx_cmd_query_rq_mb_out {
   2215   1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   2216   1.1  jmcneill 	struct mcx_rq_ctx	cmd_ctx;
   2217   1.1  jmcneill };
   2218   1.1  jmcneill 
   2219   1.1  jmcneill struct mcx_cmd_query_sq_in {
   2220   1.1  jmcneill 	uint16_t		cmd_opcode;
   2221   1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   2222   1.1  jmcneill 	uint16_t		cmd_op_mod;
   2223   1.1  jmcneill 	uint32_t		cmd_sqn;
   2224   1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   2225   1.1  jmcneill } __packed __aligned(4);
   2226   1.1  jmcneill 
   2227   1.1  jmcneill struct mcx_cmd_query_sq_out {
   2228   1.1  jmcneill 	uint8_t			cmd_status;
   2229   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2230   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2231   1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   2232   1.1  jmcneill } __packed __aligned(4);
   2233   1.1  jmcneill 
   2234   1.1  jmcneill struct mcx_cmd_query_sq_mb_out {
   2235   1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   2236   1.1  jmcneill 	struct mcx_sq_ctx	cmd_ctx;
   2237   1.1  jmcneill };
   2238   1.1  jmcneill 
   2239   1.1  jmcneill struct mcx_cmd_alloc_flow_counter_out {
   2240   1.1  jmcneill 	uint8_t			cmd_status;
   2241   1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   2242   1.1  jmcneill 	uint32_t		cmd_syndrome;
   2243   1.1  jmcneill 	uint8_t			cmd_reserved1[2];
   2244   1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
   2245   1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   2246   1.1  jmcneill } __packed __aligned(4);
   2247   1.1  jmcneill 
   2248   1.1  jmcneill struct mcx_wq_doorbell {
   2249   1.1  jmcneill 	uint32_t		 db_recv_counter;
   2250   1.1  jmcneill 	uint32_t		 db_send_counter;
   2251   1.1  jmcneill } __packed __aligned(8);
   2252   1.1  jmcneill 
   2253   1.1  jmcneill struct mcx_dmamem {
   2254   1.1  jmcneill 	bus_dmamap_t		 mxm_map;
   2255   1.1  jmcneill 	bus_dma_segment_t	 mxm_seg;
   2256   1.1  jmcneill 	int			 mxm_nsegs;
   2257   1.1  jmcneill 	size_t			 mxm_size;
   2258   1.1  jmcneill 	void			*mxm_kva;
   2259   1.1  jmcneill };
   2260   1.1  jmcneill #define MCX_DMA_MAP(_mxm)	((_mxm)->mxm_map)
   2261   1.1  jmcneill #define MCX_DMA_DVA(_mxm)	((_mxm)->mxm_map->dm_segs[0].ds_addr)
   2262   1.1  jmcneill #define MCX_DMA_KVA(_mxm)	((void *)(_mxm)->mxm_kva)
   2263  1.15  jmcneill #define MCX_DMA_OFF(_mxm, _off)	((void *)((char *)(_mxm)->mxm_kva + (_off)))
   2264   1.1  jmcneill #define MCX_DMA_LEN(_mxm)	((_mxm)->mxm_size)
   2265   1.1  jmcneill 
   2266   1.1  jmcneill struct mcx_hwmem {
   2267   1.1  jmcneill 	bus_dmamap_t		 mhm_map;
   2268   1.1  jmcneill 	bus_dma_segment_t	*mhm_segs;
   2269   1.1  jmcneill 	unsigned int		 mhm_seg_count;
   2270   1.1  jmcneill 	unsigned int		 mhm_npages;
   2271   1.1  jmcneill };
   2272   1.1  jmcneill 
   2273   1.1  jmcneill struct mcx_slot {
   2274   1.1  jmcneill 	bus_dmamap_t		 ms_map;
   2275   1.1  jmcneill 	struct mbuf		*ms_m;
   2276   1.1  jmcneill };
   2277   1.1  jmcneill 
   2278  1.15  jmcneill struct mcx_eq {
   2279  1.15  jmcneill 	int			 eq_n;
   2280  1.15  jmcneill 	uint32_t		 eq_cons;
   2281  1.15  jmcneill 	struct mcx_dmamem	 eq_mem;
   2282  1.15  jmcneill };
   2283  1.15  jmcneill 
   2284   1.1  jmcneill struct mcx_cq {
   2285   1.1  jmcneill 	int			 cq_n;
   2286   1.1  jmcneill 	struct mcx_dmamem	 cq_mem;
   2287  1.15  jmcneill 	bus_addr_t		 cq_doorbell;
   2288   1.1  jmcneill 	uint32_t		 cq_cons;
   2289   1.1  jmcneill 	uint32_t		 cq_count;
   2290   1.1  jmcneill };
   2291   1.1  jmcneill 
   2292   1.1  jmcneill struct mcx_calibration {
   2293   1.1  jmcneill 	uint64_t		 c_timestamp;	/* previous mcx chip time */
   2294   1.1  jmcneill 	uint64_t		 c_uptime;	/* previous kernel nanouptime */
   2295   1.1  jmcneill 	uint64_t		 c_tbase;	/* mcx chip time */
   2296   1.1  jmcneill 	uint64_t		 c_ubase;	/* kernel nanouptime */
   2297  1.15  jmcneill 	uint64_t		 c_ratio;
   2298   1.1  jmcneill };
   2299   1.1  jmcneill 
   2300   1.1  jmcneill #define MCX_CALIBRATE_FIRST    2
   2301  1.15  jmcneill #define MCX_CALIBRATE_NORMAL   32
   2302   1.1  jmcneill 
   2303   1.1  jmcneill struct mcx_rxring {
   2304   1.1  jmcneill 	u_int			 rxr_total;
   2305   1.1  jmcneill 	u_int			 rxr_inuse;
   2306   1.1  jmcneill };
   2307   1.1  jmcneill 
   2308   1.1  jmcneill MBUFQ_HEAD(mcx_mbufq);
   2309   1.1  jmcneill 
   2310  1.15  jmcneill struct mcx_rx {
   2311  1.15  jmcneill 	struct mcx_softc	*rx_softc;
   2312  1.15  jmcneill 
   2313  1.15  jmcneill 	int			 rx_rqn;
   2314  1.15  jmcneill 	struct mcx_dmamem	 rx_rq_mem;
   2315  1.15  jmcneill 	struct mcx_slot		*rx_slots;
   2316  1.15  jmcneill 	bus_addr_t		 rx_doorbell;
   2317  1.15  jmcneill 
   2318  1.15  jmcneill 	uint32_t		 rx_prod;
   2319  1.15  jmcneill 	callout_t		 rx_refill;
   2320  1.15  jmcneill 	struct mcx_rxring	 rx_rxr;
   2321  1.15  jmcneill } __aligned(64);
   2322  1.15  jmcneill 
   2323  1.15  jmcneill struct mcx_tx {
   2324  1.15  jmcneill 	struct mcx_softc	*tx_softc;
   2325  1.16  jmcneill 	kmutex_t		 tx_lock;
   2326  1.16  jmcneill 	pcq_t			*tx_pcq;
   2327  1.16  jmcneill 	void			*tx_softint;
   2328  1.15  jmcneill 
   2329  1.15  jmcneill 	int			 tx_uar;
   2330  1.15  jmcneill 	int			 tx_sqn;
   2331  1.15  jmcneill 	struct mcx_dmamem	 tx_sq_mem;
   2332  1.15  jmcneill 	struct mcx_slot		*tx_slots;
   2333  1.15  jmcneill 	bus_addr_t		 tx_doorbell;
   2334  1.15  jmcneill 	int			 tx_bf_offset;
   2335  1.15  jmcneill 
   2336  1.15  jmcneill 	uint32_t		 tx_cons;
   2337  1.15  jmcneill 	uint32_t		 tx_prod;
   2338  1.15  jmcneill } __aligned(64);
   2339  1.15  jmcneill 
   2340  1.15  jmcneill struct mcx_queues {
   2341  1.15  jmcneill 	void			*q_ihc;
   2342  1.15  jmcneill 	struct mcx_softc	*q_sc;
   2343  1.15  jmcneill 	int			 q_uar;
   2344  1.15  jmcneill 	int			 q_index;
   2345  1.15  jmcneill 	struct mcx_rx		 q_rx;
   2346  1.15  jmcneill 	struct mcx_tx		 q_tx;
   2347  1.15  jmcneill 	struct mcx_cq		 q_cq;
   2348  1.15  jmcneill 	struct mcx_eq		 q_eq;
   2349  1.15  jmcneill #if NKSTAT > 0
   2350  1.15  jmcneill 	struct kstat		*q_kstat;
   2351  1.15  jmcneill #endif
   2352  1.15  jmcneill };
   2353  1.15  jmcneill 
   2354  1.15  jmcneill struct mcx_flow_group {
   2355  1.15  jmcneill 	int			 g_id;
   2356  1.15  jmcneill 	int			 g_table;
   2357  1.15  jmcneill 	int			 g_start;
   2358  1.15  jmcneill 	int			 g_size;
   2359  1.15  jmcneill };
   2360  1.15  jmcneill 
   2361  1.15  jmcneill #define MCX_FLOW_GROUP_PROMISC		0
   2362  1.15  jmcneill #define MCX_FLOW_GROUP_ALLMULTI		1
   2363  1.15  jmcneill #define MCX_FLOW_GROUP_MAC		2
   2364  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_L4		3
   2365  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_L3		4
   2366  1.15  jmcneill #define MCX_FLOW_GROUP_RSS_NONE		5
   2367  1.15  jmcneill #define	MCX_NUM_FLOW_GROUPS		6
   2368  1.15  jmcneill 
   2369  1.15  jmcneill #define MCX_HASH_SEL_L3		MCX_TIR_CTX_HASH_SEL_SRC_IP | \
   2370  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_DST_IP
   2371  1.15  jmcneill #define MCX_HASH_SEL_L4		MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \
   2372  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_DPORT
   2373  1.15  jmcneill 
   2374  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP  |\
   2375  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV4
   2376  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6_TCP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \
   2377  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV6
   2378  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
   2379  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV4
   2380  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6_UDP	MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
   2381  1.15  jmcneill 				MCX_TIR_CTX_HASH_SEL_IPV6
   2382  1.15  jmcneill #define MCX_RSS_HASH_SEL_V4	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4
   2383  1.15  jmcneill #define MCX_RSS_HASH_SEL_V6	MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6
   2384  1.15  jmcneill 
   2385  1.15  jmcneill /*
   2386  1.15  jmcneill  * There are a few different pieces involved in configuring RSS.
   2387  1.15  jmcneill  * A Receive Queue Table (RQT) is the indirection table that maps packets to
   2388  1.15  jmcneill  * different rx queues based on a hash value.  We only create one, because
   2389  1.15  jmcneill  * we want to scatter any traffic we can apply RSS to across all our rx
   2390  1.15  jmcneill  * queues.  Anything else will only be delivered to the first rx queue,
   2391  1.15  jmcneill  * which doesn't require an RQT.
   2392  1.15  jmcneill  *
   2393  1.15  jmcneill  * A Transport Interface Receive (TIR) delivers packets to either a single rx
   2394  1.15  jmcneill  * queue or an RQT, and in the latter case, specifies the set of fields
   2395  1.15  jmcneill  * hashed, the hash function, and the hash key.  We need one of these for each
   2396  1.15  jmcneill  * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6,
   2397  1.15  jmcneill  * and one for non-RSS traffic.
   2398  1.15  jmcneill  *
   2399  1.15  jmcneill  * Flow tables hold flow table entries in sequence.  The first entry that
   2400  1.15  jmcneill  * matches a packet is applied, sending the packet to either another flow
   2401  1.15  jmcneill  * table or a TIR.  We use one flow table to select packets based on
   2402  1.15  jmcneill  * destination MAC address, and a second to apply RSS.  The entries in the
   2403  1.15  jmcneill  * first table send matching packets to the second, and the entries in the
   2404  1.15  jmcneill  * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR.
   2405  1.15  jmcneill  *
   2406  1.15  jmcneill  * The flow table entry that delivers packets to an RSS TIR must include match
   2407  1.15  jmcneill  * criteria that ensure packets delivered to the TIR include all the fields
   2408  1.15  jmcneill  * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must
   2409  1.15  jmcneill  * only accept v4 TCP packets.  Accordingly, we need flow table entries for
   2410  1.15  jmcneill  * each TIR.
   2411  1.15  jmcneill  *
   2412  1.15  jmcneill  * All of this is a lot more flexible than we need, and we can describe most
   2413  1.15  jmcneill  * of the stuff we need with a simple array.
   2414  1.15  jmcneill  *
   2415  1.15  jmcneill  * An RSS config creates a TIR with hashing enabled on a set of fields,
   2416  1.15  jmcneill  * pointing to either the first rx queue or the RQT containing all the rx
   2417  1.15  jmcneill  * queues, and a flow table entry that matches on an ether type and
   2418  1.15  jmcneill  * optionally an ip proto, that delivers packets to the TIR.
   2419  1.15  jmcneill  */
   2420  1.15  jmcneill static struct mcx_rss_rule {
   2421  1.15  jmcneill 	int			hash_sel;
   2422  1.15  jmcneill 	int			flow_group;
   2423  1.15  jmcneill 	int			ethertype;
   2424  1.15  jmcneill 	int			ip_proto;
   2425  1.15  jmcneill } mcx_rss_config[] = {
   2426  1.15  jmcneill 	/* udp and tcp for v4/v6 */
   2427  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4,
   2428  1.15  jmcneill 	  ETHERTYPE_IP, IPPROTO_TCP },
   2429  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4,
   2430  1.15  jmcneill 	  ETHERTYPE_IPV6, IPPROTO_TCP },
   2431  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4,
   2432  1.15  jmcneill 	  ETHERTYPE_IP, IPPROTO_UDP },
   2433  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4,
   2434  1.15  jmcneill 	  ETHERTYPE_IPV6, IPPROTO_UDP },
   2435  1.15  jmcneill 
   2436  1.15  jmcneill 	/* other v4/v6 */
   2437  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3,
   2438  1.15  jmcneill 	  ETHERTYPE_IP, 0 },
   2439  1.15  jmcneill 	{ MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3,
   2440  1.15  jmcneill 	  ETHERTYPE_IPV6, 0 },
   2441  1.15  jmcneill 
   2442  1.15  jmcneill 	/* non v4/v6 */
   2443  1.15  jmcneill 	{ 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 }
   2444  1.15  jmcneill };
   2445  1.15  jmcneill 
   2446   1.1  jmcneill struct mcx_softc {
   2447   1.1  jmcneill 	device_t		 sc_dev;
   2448   1.1  jmcneill 	struct ethercom		 sc_ec;
   2449   1.1  jmcneill 	struct ifmedia		 sc_media;
   2450   1.1  jmcneill 	uint64_t		 sc_media_status;
   2451   1.1  jmcneill 	uint64_t		 sc_media_active;
   2452  1.12   thorpej 	kmutex_t		 sc_media_mutex;
   2453   1.1  jmcneill 
   2454   1.1  jmcneill 	pci_chipset_tag_t	 sc_pc;
   2455   1.1  jmcneill 	pci_intr_handle_t	*sc_intrs;
   2456  1.15  jmcneill 	void			*sc_ihc;
   2457   1.1  jmcneill 	pcitag_t		 sc_tag;
   2458   1.1  jmcneill 
   2459   1.1  jmcneill 	bus_dma_tag_t		 sc_dmat;
   2460   1.1  jmcneill 	bus_space_tag_t		 sc_memt;
   2461   1.1  jmcneill 	bus_space_handle_t	 sc_memh;
   2462   1.1  jmcneill 	bus_size_t		 sc_mems;
   2463   1.1  jmcneill 
   2464   1.1  jmcneill 	struct mcx_dmamem	 sc_cmdq_mem;
   2465   1.1  jmcneill 	unsigned int		 sc_cmdq_mask;
   2466   1.1  jmcneill 	unsigned int		 sc_cmdq_size;
   2467   1.1  jmcneill 
   2468   1.1  jmcneill 	unsigned int		 sc_cmdq_token;
   2469   1.1  jmcneill 
   2470   1.1  jmcneill 	struct mcx_hwmem	 sc_boot_pages;
   2471   1.1  jmcneill 	struct mcx_hwmem	 sc_init_pages;
   2472   1.1  jmcneill 	struct mcx_hwmem	 sc_regular_pages;
   2473   1.1  jmcneill 
   2474   1.1  jmcneill 	int			 sc_uar;
   2475   1.1  jmcneill 	int			 sc_pd;
   2476   1.1  jmcneill 	int			 sc_tdomain;
   2477   1.1  jmcneill 	uint32_t		 sc_lkey;
   2478  1.15  jmcneill 	int			 sc_tis;
   2479  1.15  jmcneill 	int			 sc_tir[__arraycount(mcx_rss_config)];
   2480  1.15  jmcneill 	int			 sc_rqt;
   2481   1.1  jmcneill 
   2482   1.1  jmcneill 	struct mcx_dmamem	 sc_doorbell_mem;
   2483   1.1  jmcneill 
   2484  1.15  jmcneill 	struct mcx_eq		 sc_admin_eq;
   2485  1.15  jmcneill 	struct mcx_eq		 sc_queue_eq;
   2486  1.15  jmcneill 
   2487   1.1  jmcneill 	int			 sc_hardmtu;
   2488  1.15  jmcneill 	int			 sc_rxbufsz;
   2489  1.15  jmcneill 
   2490  1.15  jmcneill 	int			 sc_bf_size;
   2491  1.15  jmcneill 	int			 sc_max_rqt_size;
   2492   1.1  jmcneill 
   2493   1.1  jmcneill 	struct workqueue	*sc_workq;
   2494   1.1  jmcneill 	struct work		 sc_port_change;
   2495   1.1  jmcneill 
   2496  1.15  jmcneill 	int			 sc_mac_flow_table_id;
   2497  1.15  jmcneill 	int			 sc_rss_flow_table_id;
   2498  1.15  jmcneill 	struct mcx_flow_group	 sc_flow_group[MCX_NUM_FLOW_GROUPS];
   2499   1.1  jmcneill 	int			 sc_promisc_flow_enabled;
   2500   1.1  jmcneill 	int			 sc_allmulti_flow_enabled;
   2501   1.1  jmcneill 	int			 sc_mcast_flow_base;
   2502   1.1  jmcneill 	int			 sc_extra_mcast;
   2503   1.1  jmcneill 	uint8_t			 sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];
   2504   1.1  jmcneill 
   2505   1.1  jmcneill 	struct mcx_calibration	 sc_calibration[2];
   2506   1.1  jmcneill 	unsigned int		 sc_calibration_gen;
   2507   1.1  jmcneill 	callout_t		 sc_calibrate;
   2508  1.15  jmcneill 	uint32_t		 sc_mhz;
   2509  1.15  jmcneill 	uint32_t		 sc_khz;
   2510   1.1  jmcneill 
   2511  1.15  jmcneill 	struct mcx_queues	*sc_queues;
   2512  1.15  jmcneill 	unsigned int		 sc_nqueues;
   2513   1.1  jmcneill 
   2514  1.15  jmcneill 	int			 sc_mcam_reg;
   2515   1.1  jmcneill 
   2516  1.15  jmcneill #if NKSTAT > 0
   2517  1.15  jmcneill 	struct kstat		*sc_kstat_ieee8023;
   2518  1.15  jmcneill 	struct kstat		*sc_kstat_rfc2863;
   2519  1.15  jmcneill 	struct kstat		*sc_kstat_rfc2819;
   2520  1.15  jmcneill 	struct kstat		*sc_kstat_rfc3635;
   2521  1.15  jmcneill 	unsigned int		 sc_kstat_mtmp_count;
   2522  1.15  jmcneill 	struct kstat		**sc_kstat_mtmp;
   2523  1.15  jmcneill #endif
   2524   1.1  jmcneill 
   2525  1.15  jmcneill 	struct timecounter	 sc_timecounter;
   2526   1.1  jmcneill };
   2527   1.1  jmcneill #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
   2528   1.1  jmcneill 
   2529   1.1  jmcneill static int	mcx_match(device_t, cfdata_t, void *);
   2530   1.1  jmcneill static void	mcx_attach(device_t, device_t, void *);
   2531   1.1  jmcneill 
   2532  1.15  jmcneill static void *	mcx_establish_intr(struct mcx_softc *, int, kcpuset_t *,
   2533  1.15  jmcneill 		    int (*)(void *), void *, const char *);
   2534  1.15  jmcneill 
   2535   1.1  jmcneill static void	mcx_rxr_init(struct mcx_rxring *, u_int, u_int);
   2536   1.1  jmcneill static u_int	mcx_rxr_get(struct mcx_rxring *, u_int);
   2537   1.1  jmcneill static void	mcx_rxr_put(struct mcx_rxring *, u_int);
   2538   1.1  jmcneill static u_int	mcx_rxr_inuse(struct mcx_rxring *);
   2539   1.1  jmcneill 
   2540  1.15  jmcneill #if NKSTAT > 0
   2541  1.15  jmcneill static void	mcx_kstat_attach(struct mcx_softc *);
   2542  1.15  jmcneill #endif
   2543  1.15  jmcneill 
   2544  1.15  jmcneill static void	mcx_timecounter_attach(struct mcx_softc *);
   2545  1.15  jmcneill 
   2546   1.1  jmcneill static int	mcx_version(struct mcx_softc *);
   2547   1.1  jmcneill static int	mcx_init_wait(struct mcx_softc *);
   2548   1.1  jmcneill static int	mcx_enable_hca(struct mcx_softc *);
   2549   1.1  jmcneill static int	mcx_teardown_hca(struct mcx_softc *, uint16_t);
   2550   1.1  jmcneill static int	mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
   2551   1.1  jmcneill 		    int);
   2552   1.1  jmcneill static int	mcx_issi(struct mcx_softc *);
   2553   1.1  jmcneill static int	mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
   2554   1.1  jmcneill static int	mcx_hca_max_caps(struct mcx_softc *);
   2555   1.1  jmcneill static int	mcx_hca_set_caps(struct mcx_softc *);
   2556   1.1  jmcneill static int	mcx_init_hca(struct mcx_softc *);
   2557   1.1  jmcneill static int	mcx_set_driver_version(struct mcx_softc *);
   2558   1.1  jmcneill static int	mcx_iff(struct mcx_softc *);
   2559  1.15  jmcneill static int	mcx_alloc_uar(struct mcx_softc *, int *);
   2560   1.1  jmcneill static int	mcx_alloc_pd(struct mcx_softc *);
   2561   1.1  jmcneill static int	mcx_alloc_tdomain(struct mcx_softc *);
   2562  1.15  jmcneill static int	mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int,
   2563  1.15  jmcneill 		    uint64_t, int);
   2564   1.1  jmcneill static int	mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *);
   2565   1.1  jmcneill static int	mcx_query_special_contexts(struct mcx_softc *);
   2566   1.1  jmcneill static int	mcx_set_port_mtu(struct mcx_softc *, int);
   2567  1.15  jmcneill static int	mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int,
   2568  1.15  jmcneill 		    int);
   2569  1.15  jmcneill static int	mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *);
   2570  1.15  jmcneill static int	mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int,
   2571  1.15  jmcneill 		    int);
   2572  1.15  jmcneill static int	mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *);
   2573  1.15  jmcneill static int	mcx_ready_sq(struct mcx_softc *, struct mcx_tx *);
   2574  1.15  jmcneill static int	mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int);
   2575  1.15  jmcneill static int	mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *);
   2576  1.15  jmcneill static int	mcx_ready_rq(struct mcx_softc *, struct mcx_rx *);
   2577  1.15  jmcneill static int	mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *,
   2578  1.15  jmcneill 		    int *);
   2579  1.15  jmcneill static int	mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t,
   2580  1.15  jmcneill 		    int *);
   2581  1.15  jmcneill static int	mcx_destroy_tir(struct mcx_softc *, int);
   2582  1.15  jmcneill static int	mcx_create_tis(struct mcx_softc *, int *);
   2583  1.15  jmcneill static int	mcx_destroy_tis(struct mcx_softc *, int);
   2584  1.15  jmcneill static int	mcx_create_rqt(struct mcx_softc *, int, int *, int *);
   2585  1.15  jmcneill static int	mcx_destroy_rqt(struct mcx_softc *, int);
   2586  1.15  jmcneill static int	mcx_create_flow_table(struct mcx_softc *, int, int, int *);
   2587  1.15  jmcneill static int	mcx_set_flow_table_root(struct mcx_softc *, int);
   2588  1.15  jmcneill static int	mcx_destroy_flow_table(struct mcx_softc *, int);
   2589  1.15  jmcneill static int	mcx_create_flow_group(struct mcx_softc *, int, int, int,
   2590   1.1  jmcneill 		    int, int, struct mcx_flow_match *);
   2591   1.1  jmcneill static int	mcx_destroy_flow_group(struct mcx_softc *, int);
   2592  1.15  jmcneill static int	mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int,
   2593  1.15  jmcneill 		    const uint8_t *, uint32_t);
   2594  1.15  jmcneill static int	mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int,
   2595  1.15  jmcneill 		    int, int, uint32_t);
   2596   1.1  jmcneill static int	mcx_delete_flow_table_entry(struct mcx_softc *, int, int);
   2597   1.1  jmcneill 
   2598  1.15  jmcneill #if NKSTAT > 0
   2599  1.15  jmcneill static int	mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *);
   2600  1.15  jmcneill static int	mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *);
   2601  1.15  jmcneill static int	mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *);
   2602  1.15  jmcneill static int	mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *);
   2603  1.15  jmcneill #endif
   2604  1.15  jmcneill 
   2605   1.1  jmcneill #if 0
   2606  1.15  jmcneill static int	mcx_dump_flow_table(struct mcx_softc *, int);
   2607  1.15  jmcneill static int	mcx_dump_flow_table_entry(struct mcx_softc *, int, int);
   2608  1.15  jmcneill static int	mcx_dump_flow_group(struct mcx_softc *, int);
   2609   1.1  jmcneill #endif
   2610   1.1  jmcneill 
   2611   1.1  jmcneill 
   2612   1.1  jmcneill /*
   2613   1.1  jmcneill static void	mcx_cmdq_dump(const struct mcx_cmdq_entry *);
   2614   1.1  jmcneill static void	mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
   2615   1.1  jmcneill */
   2616   1.1  jmcneill static void	mcx_refill(void *);
   2617  1.15  jmcneill static int	mcx_process_rx(struct mcx_softc *, struct mcx_rx *,
   2618  1.15  jmcneill 		    struct mcx_cq_entry *, struct mcx_mbufq *,
   2619  1.15  jmcneill 		    const struct mcx_calibration *);
   2620  1.15  jmcneill static int	mcx_process_txeof(struct mcx_softc *, struct mcx_tx *,
   2621  1.15  jmcneill 		    struct mcx_cq_entry *);
   2622  1.15  jmcneill static void	mcx_process_cq(struct mcx_softc *, struct mcx_queues *,
   2623  1.15  jmcneill 		    struct mcx_cq *);
   2624  1.15  jmcneill 
   2625  1.15  jmcneill static void	mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int);
   2626  1.15  jmcneill static void	mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int);
   2627  1.15  jmcneill static int	mcx_admin_intr(void *);
   2628  1.15  jmcneill static int	mcx_cq_intr(void *);
   2629   1.1  jmcneill 
   2630   1.1  jmcneill static int	mcx_init(struct ifnet *);
   2631   1.1  jmcneill static void	mcx_stop(struct ifnet *, int);
   2632   1.1  jmcneill static int	mcx_ioctl(struct ifnet *, u_long, void *);
   2633   1.1  jmcneill static void	mcx_start(struct ifnet *);
   2634  1.16  jmcneill static int	mcx_transmit(struct ifnet *, struct mbuf *);
   2635  1.16  jmcneill static void	mcx_deferred_transmit(void *);
   2636   1.1  jmcneill static void	mcx_watchdog(struct ifnet *);
   2637   1.1  jmcneill static void	mcx_media_add_types(struct mcx_softc *);
   2638   1.1  jmcneill static void	mcx_media_status(struct ifnet *, struct ifmediareq *);
   2639   1.1  jmcneill static int	mcx_media_change(struct ifnet *);
   2640   1.1  jmcneill #if 0
   2641   1.1  jmcneill static int	mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
   2642   1.1  jmcneill #endif
   2643   1.1  jmcneill static void	mcx_port_change(struct work *, void *);
   2644   1.1  jmcneill 
   2645   1.1  jmcneill static void	mcx_calibrate_first(struct mcx_softc *);
   2646   1.1  jmcneill static void	mcx_calibrate(void *);
   2647   1.1  jmcneill 
   2648   1.1  jmcneill static inline uint32_t
   2649   1.1  jmcneill 		mcx_rd(struct mcx_softc *, bus_size_t);
   2650   1.1  jmcneill static inline void
   2651   1.1  jmcneill 		mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
   2652   1.1  jmcneill static inline void
   2653   1.1  jmcneill 		mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
   2654   1.1  jmcneill 
   2655   1.1  jmcneill static uint64_t	mcx_timer(struct mcx_softc *);
   2656   1.1  jmcneill 
   2657   1.1  jmcneill static int	mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
   2658   1.1  jmcneill 		    bus_size_t, u_int align);
   2659   1.1  jmcneill static void	mcx_dmamem_zero(struct mcx_dmamem *);
   2660   1.1  jmcneill static void	mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
   2661   1.1  jmcneill 
   2662   1.1  jmcneill static int	mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
   2663   1.1  jmcneill 		    unsigned int);
   2664   1.1  jmcneill static void	mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
   2665   1.1  jmcneill 
   2666   1.1  jmcneill CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL);
   2667   1.1  jmcneill 
   2668   1.1  jmcneill static const struct {
   2669   1.1  jmcneill 	pci_vendor_id_t		vendor;
   2670   1.1  jmcneill 	pci_product_id_t	product;
   2671   1.1  jmcneill } mcx_devices[] = {
   2672   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700 },
   2673  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700VF },
   2674   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710 },
   2675  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710VF },
   2676   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800 },
   2677  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800VF },
   2678   1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800 },
   2679  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800VF },
   2680  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28908 },
   2681  1.15  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT2892  },
   2682   1.1  jmcneill };
   2683   1.1  jmcneill 
   2684  1.13  jmcneill struct mcx_eth_proto_capability {
   2685  1.13  jmcneill 	uint64_t	cap_media;
   2686  1.13  jmcneill 	uint64_t	cap_baudrate;
   2687  1.13  jmcneill };
   2688  1.13  jmcneill 
   2689  1.13  jmcneill static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = {
   2690  1.13  jmcneill 	[MCX_ETHER_CAP_SGMII]		= { IFM_1000_SGMII,	IF_Gbps(1) },
   2691  1.13  jmcneill 	[MCX_ETHER_CAP_1000_KX]		= { IFM_1000_KX,	IF_Gbps(1) },
   2692  1.13  jmcneill 	[MCX_ETHER_CAP_10G_CX4]		= { IFM_10G_CX4,	IF_Gbps(10) },
   2693  1.13  jmcneill 	[MCX_ETHER_CAP_10G_KX4]		= { IFM_10G_KX4,	IF_Gbps(10) },
   2694  1.13  jmcneill 	[MCX_ETHER_CAP_10G_KR]		= { IFM_10G_KR,		IF_Gbps(10) },
   2695  1.13  jmcneill 	[MCX_ETHER_CAP_20G_KR2]		= { IFM_20G_KR2,	IF_Gbps(20) },
   2696  1.13  jmcneill 	[MCX_ETHER_CAP_40G_CR4]		= { IFM_40G_CR4,	IF_Gbps(40) },
   2697  1.13  jmcneill 	[MCX_ETHER_CAP_40G_KR4]		= { IFM_40G_KR4,	IF_Gbps(40) },
   2698  1.13  jmcneill 	[MCX_ETHER_CAP_56G_R4]		= { IFM_56G_R4,		IF_Gbps(56) },
   2699  1.13  jmcneill 	[MCX_ETHER_CAP_10G_CR]		= { IFM_10G_CR1,	IF_Gbps(10) },
   2700  1.13  jmcneill 	[MCX_ETHER_CAP_10G_SR]		= { IFM_10G_SR,		IF_Gbps(10) },
   2701  1.13  jmcneill 	[MCX_ETHER_CAP_10G_LR]		= { IFM_10G_LR,		IF_Gbps(10) },
   2702  1.13  jmcneill 	[MCX_ETHER_CAP_40G_SR4]		= { IFM_40G_SR4,	IF_Gbps(40) },
   2703  1.13  jmcneill 	[MCX_ETHER_CAP_40G_LR4]		= { IFM_40G_LR4,	IF_Gbps(40) },
   2704  1.13  jmcneill 	[MCX_ETHER_CAP_50G_SR2]		= { IFM_50G_SR2,	IF_Gbps(50) },
   2705  1.13  jmcneill 	[MCX_ETHER_CAP_100G_CR4]	= { IFM_100G_CR4,	IF_Gbps(100) },
   2706  1.13  jmcneill 	[MCX_ETHER_CAP_100G_SR4]	= { IFM_100G_SR4,	IF_Gbps(100) },
   2707  1.13  jmcneill 	[MCX_ETHER_CAP_100G_KR4]	= { IFM_100G_KR4,	IF_Gbps(100) },
   2708  1.13  jmcneill 	[MCX_ETHER_CAP_100G_LR4]	= { IFM_100G_LR4,	IF_Gbps(100) },
   2709  1.13  jmcneill 	[MCX_ETHER_CAP_100_TX]		= { IFM_100_TX,		IF_Mbps(100) },
   2710  1.13  jmcneill 	[MCX_ETHER_CAP_1000_T]		= { IFM_1000_T,		IF_Gbps(1) },
   2711  1.13  jmcneill 	[MCX_ETHER_CAP_10G_T]		= { IFM_10G_T,		IF_Gbps(10) },
   2712  1.13  jmcneill 	[MCX_ETHER_CAP_25G_CR]		= { IFM_25G_CR,		IF_Gbps(25) },
   2713  1.13  jmcneill 	[MCX_ETHER_CAP_25G_KR]		= { IFM_25G_KR,		IF_Gbps(25) },
   2714  1.13  jmcneill 	[MCX_ETHER_CAP_25G_SR]		= { IFM_25G_SR,		IF_Gbps(25) },
   2715  1.13  jmcneill 	[MCX_ETHER_CAP_50G_CR2]		= { IFM_50G_CR2,	IF_Gbps(50) },
   2716  1.13  jmcneill 	[MCX_ETHER_CAP_50G_KR2]		= { IFM_50G_KR2,	IF_Gbps(50) },
   2717   1.1  jmcneill };
   2718   1.1  jmcneill 
   2719   1.1  jmcneill static int
   2720  1.13  jmcneill mcx_get_id(uint32_t val)
   2721  1.13  jmcneill {
   2722  1.13  jmcneill 	return be32toh(val) & 0x00ffffff;
   2723  1.13  jmcneill }
   2724  1.13  jmcneill 
   2725  1.13  jmcneill static int
   2726   1.1  jmcneill mcx_match(device_t parent, cfdata_t cf, void *aux)
   2727   1.1  jmcneill {
   2728   1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2729   1.1  jmcneill 	int n;
   2730   1.1  jmcneill 
   2731   1.1  jmcneill 	for (n = 0; n < __arraycount(mcx_devices); n++) {
   2732   1.1  jmcneill 		if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor &&
   2733   1.1  jmcneill 		    PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product)
   2734   1.1  jmcneill 			return 1;
   2735   1.1  jmcneill 	}
   2736   1.1  jmcneill 
   2737   1.1  jmcneill 	return 0;
   2738   1.1  jmcneill }
   2739   1.1  jmcneill 
   2740   1.1  jmcneill void
   2741   1.1  jmcneill mcx_attach(device_t parent, device_t self, void *aux)
   2742   1.1  jmcneill {
   2743   1.1  jmcneill 	struct mcx_softc *sc = device_private(self);
   2744   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   2745   1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2746   1.1  jmcneill 	uint8_t enaddr[ETHER_ADDR_LEN];
   2747   1.1  jmcneill 	int counts[PCI_INTR_TYPE_SIZE];
   2748  1.15  jmcneill 	char intrxname[32];
   2749   1.1  jmcneill 	pcireg_t memtype;
   2750   1.1  jmcneill 	uint32_t r;
   2751   1.1  jmcneill 	unsigned int cq_stride;
   2752   1.1  jmcneill 	unsigned int cq_size;
   2753  1.15  jmcneill 	int i, msix;
   2754  1.15  jmcneill 	kcpuset_t *affinity;
   2755   1.1  jmcneill 
   2756   1.1  jmcneill 	sc->sc_dev = self;
   2757   1.1  jmcneill 	sc->sc_pc = pa->pa_pc;
   2758   1.1  jmcneill 	sc->sc_tag = pa->pa_tag;
   2759  1.11   thorpej 	if (pci_dma64_available(pa))
   2760  1.11   thorpej 		sc->sc_dmat = pa->pa_dmat64;
   2761  1.11   thorpej 	else
   2762  1.11   thorpej 		sc->sc_dmat = pa->pa_dmat;
   2763   1.1  jmcneill 
   2764   1.1  jmcneill 	/* Map the PCI memory space */
   2765   1.1  jmcneill 	memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
   2766   1.1  jmcneill 	if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
   2767   1.1  jmcneill 	    0 /*BUS_SPACE_MAP_PREFETCHABLE*/, &sc->sc_memt, &sc->sc_memh,
   2768   1.1  jmcneill 	    NULL, &sc->sc_mems)) {
   2769   1.1  jmcneill 		aprint_error(": unable to map register memory\n");
   2770   1.1  jmcneill 		return;
   2771   1.1  jmcneill 	}
   2772   1.1  jmcneill 
   2773   1.1  jmcneill 	pci_aprint_devinfo(pa, "Ethernet controller");
   2774   1.1  jmcneill 
   2775  1.12   thorpej 	mutex_init(&sc->sc_media_mutex, MUTEX_DEFAULT, IPL_SOFTNET);
   2776  1.12   thorpej 
   2777   1.1  jmcneill 	if (mcx_version(sc) != 0) {
   2778   1.1  jmcneill 		/* error printed by mcx_version */
   2779   1.1  jmcneill 		goto unmap;
   2780   1.1  jmcneill 	}
   2781   1.1  jmcneill 
   2782   1.1  jmcneill 	r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
   2783   1.1  jmcneill 	cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
   2784   1.1  jmcneill 	cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
   2785   1.1  jmcneill 	if (cq_size > MCX_MAX_CQE) {
   2786   1.1  jmcneill 		aprint_error_dev(self,
   2787   1.1  jmcneill 		    "command queue size overflow %u\n", cq_size);
   2788   1.1  jmcneill 		goto unmap;
   2789   1.1  jmcneill 	}
   2790   1.1  jmcneill 	if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
   2791   1.1  jmcneill 		aprint_error_dev(self,
   2792   1.1  jmcneill 		    "command queue entry size underflow %u\n", cq_stride);
   2793   1.1  jmcneill 		goto unmap;
   2794   1.1  jmcneill 	}
   2795   1.1  jmcneill 	if (cq_stride * cq_size > MCX_PAGE_SIZE) {
   2796   1.1  jmcneill 		aprint_error_dev(self, "command queue page overflow\n");
   2797   1.1  jmcneill 		goto unmap;
   2798   1.1  jmcneill 	}
   2799   1.1  jmcneill 
   2800  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE,
   2801   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2802   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate doorbell memory\n");
   2803   1.1  jmcneill 		goto unmap;
   2804   1.1  jmcneill 	}
   2805   1.1  jmcneill 
   2806   1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
   2807   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2808   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate command queue\n");
   2809   1.1  jmcneill 		goto dbfree;
   2810   1.1  jmcneill 	}
   2811   1.1  jmcneill 
   2812   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   2813  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t),
   2814  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   2815   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
   2816  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t),
   2817  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   2818   1.1  jmcneill 
   2819   1.1  jmcneill 	if (mcx_init_wait(sc) != 0) {
   2820   1.1  jmcneill 		aprint_error_dev(self, "timeout waiting for init\n");
   2821   1.1  jmcneill 		goto cqfree;
   2822   1.1  jmcneill 	}
   2823   1.1  jmcneill 
   2824   1.1  jmcneill 	sc->sc_cmdq_mask = cq_size - 1;
   2825   1.1  jmcneill 	sc->sc_cmdq_size = cq_stride;
   2826   1.1  jmcneill 
   2827   1.1  jmcneill 	if (mcx_enable_hca(sc) != 0) {
   2828   1.1  jmcneill 		/* error printed by mcx_enable_hca */
   2829   1.1  jmcneill 		goto cqfree;
   2830   1.1  jmcneill 	}
   2831   1.1  jmcneill 
   2832   1.1  jmcneill 	if (mcx_issi(sc) != 0) {
   2833   1.1  jmcneill 		/* error printed by mcx_issi */
   2834   1.1  jmcneill 		goto teardown;
   2835   1.1  jmcneill 	}
   2836   1.1  jmcneill 
   2837   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_boot_pages,
   2838   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
   2839   1.1  jmcneill 		/* error printed by mcx_pages */
   2840   1.1  jmcneill 		goto teardown;
   2841   1.1  jmcneill 	}
   2842   1.1  jmcneill 
   2843   1.1  jmcneill 	if (mcx_hca_max_caps(sc) != 0) {
   2844   1.1  jmcneill 		/* error printed by mcx_hca_max_caps */
   2845   1.1  jmcneill 		goto teardown;
   2846   1.1  jmcneill 	}
   2847   1.1  jmcneill 
   2848   1.1  jmcneill 	if (mcx_hca_set_caps(sc) != 0) {
   2849   1.1  jmcneill 		/* error printed by mcx_hca_set_caps */
   2850   1.1  jmcneill 		goto teardown;
   2851   1.1  jmcneill 	}
   2852   1.1  jmcneill 
   2853   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_init_pages,
   2854   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
   2855   1.1  jmcneill 		/* error printed by mcx_pages */
   2856   1.1  jmcneill 		goto teardown;
   2857   1.1  jmcneill 	}
   2858   1.1  jmcneill 
   2859   1.1  jmcneill 	if (mcx_init_hca(sc) != 0) {
   2860   1.1  jmcneill 		/* error printed by mcx_init_hca */
   2861   1.1  jmcneill 		goto teardown;
   2862   1.1  jmcneill 	}
   2863   1.1  jmcneill 
   2864   1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_regular_pages,
   2865   1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
   2866   1.1  jmcneill 		/* error printed by mcx_pages */
   2867   1.1  jmcneill 		goto teardown;
   2868   1.1  jmcneill 	}
   2869   1.1  jmcneill 
   2870   1.1  jmcneill 	/* apparently not necessary? */
   2871   1.1  jmcneill 	if (mcx_set_driver_version(sc) != 0) {
   2872   1.1  jmcneill 		/* error printed by mcx_set_driver_version */
   2873   1.1  jmcneill 		goto teardown;
   2874   1.1  jmcneill 	}
   2875   1.1  jmcneill 
   2876   1.1  jmcneill 	if (mcx_iff(sc) != 0) {	/* modify nic vport context */
   2877   1.1  jmcneill 		/* error printed by mcx_iff? */
   2878   1.1  jmcneill 		goto teardown;
   2879   1.1  jmcneill 	}
   2880   1.1  jmcneill 
   2881  1.15  jmcneill 	if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) {
   2882   1.1  jmcneill 		/* error printed by mcx_alloc_uar */
   2883   1.1  jmcneill 		goto teardown;
   2884   1.1  jmcneill 	}
   2885   1.1  jmcneill 
   2886   1.1  jmcneill 	if (mcx_alloc_pd(sc) != 0) {
   2887   1.1  jmcneill 		/* error printed by mcx_alloc_pd */
   2888   1.1  jmcneill 		goto teardown;
   2889   1.1  jmcneill 	}
   2890   1.1  jmcneill 
   2891   1.1  jmcneill 	if (mcx_alloc_tdomain(sc) != 0) {
   2892   1.1  jmcneill 		/* error printed by mcx_alloc_tdomain */
   2893   1.1  jmcneill 		goto teardown;
   2894   1.1  jmcneill 	}
   2895   1.1  jmcneill 
   2896   1.1  jmcneill 	/*
   2897   1.1  jmcneill 	 * PRM makes no mention of msi interrupts, just legacy and msi-x.
   2898   1.1  jmcneill 	 * mellanox support tells me legacy interrupts are not supported,
   2899   1.1  jmcneill 	 * so we're stuck with just msi-x.
   2900   1.1  jmcneill 	 */
   2901  1.15  jmcneill 	counts[PCI_INTR_TYPE_MSIX] = -1;
   2902   1.1  jmcneill 	counts[PCI_INTR_TYPE_MSI] = 0;
   2903   1.1  jmcneill 	counts[PCI_INTR_TYPE_INTX] = 0;
   2904   1.1  jmcneill 	if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) {
   2905   1.1  jmcneill 		aprint_error_dev(self, "unable to allocate interrupt\n");
   2906   1.1  jmcneill 		goto teardown;
   2907   1.1  jmcneill 	}
   2908  1.15  jmcneill 	if (counts[PCI_INTR_TYPE_MSIX] < 2) {
   2909  1.15  jmcneill 		aprint_error_dev(self, "not enough MSI-X vectors\n");
   2910  1.15  jmcneill 		goto teardown;
   2911  1.15  jmcneill 	}
   2912   1.1  jmcneill 	KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX);
   2913  1.15  jmcneill 	snprintf(intrxname, sizeof(intrxname), "%s adminq", DEVNAME(sc));
   2914  1.15  jmcneill 	sc->sc_ihc = mcx_establish_intr(sc, 0, NULL, mcx_admin_intr, sc,
   2915  1.15  jmcneill 	    intrxname);
   2916  1.15  jmcneill 	if (sc->sc_ihc == NULL) {
   2917  1.15  jmcneill 		aprint_error_dev(self, "couldn't establish adminq interrupt\n");
   2918   1.1  jmcneill 		goto teardown;
   2919   1.1  jmcneill 	}
   2920   1.1  jmcneill 
   2921  1.15  jmcneill 	if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar,
   2922  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
   2923  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
   2924  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
   2925  1.15  jmcneill 	    (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) {
   2926   1.1  jmcneill 		/* error printed by mcx_create_eq */
   2927   1.1  jmcneill 		goto teardown;
   2928   1.1  jmcneill 	}
   2929   1.1  jmcneill 
   2930   1.1  jmcneill 	if (mcx_query_nic_vport_context(sc, enaddr) != 0) {
   2931   1.1  jmcneill 		/* error printed by mcx_query_nic_vport_context */
   2932   1.1  jmcneill 		goto teardown;
   2933   1.1  jmcneill 	}
   2934   1.1  jmcneill 
   2935   1.1  jmcneill 	if (mcx_query_special_contexts(sc) != 0) {
   2936   1.1  jmcneill 		/* error printed by mcx_query_special_contexts */
   2937   1.1  jmcneill 		goto teardown;
   2938   1.1  jmcneill 	}
   2939   1.1  jmcneill 
   2940   1.1  jmcneill 	if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
   2941   1.1  jmcneill 		/* error printed by mcx_set_port_mtu */
   2942   1.1  jmcneill 		goto teardown;
   2943   1.1  jmcneill 	}
   2944   1.1  jmcneill 
   2945   1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
   2946   1.1  jmcneill 	    ether_sprintf(enaddr));
   2947   1.1  jmcneill 
   2948  1.15  jmcneill 	msix = counts[PCI_INTR_TYPE_MSIX];
   2949  1.15  jmcneill 	msix--; /* admin ops took one */
   2950  1.15  jmcneill 
   2951  1.15  jmcneill 	sc->sc_nqueues = uimin(MCX_MAX_QUEUES, msix);
   2952  1.15  jmcneill 	sc->sc_nqueues = uimin(sc->sc_nqueues, ncpu);
   2953  1.15  jmcneill 	sc->sc_queues = kmem_zalloc(sc->sc_nqueues * sizeof(*sc->sc_queues),
   2954  1.15  jmcneill 	    KM_SLEEP);
   2955  1.15  jmcneill 
   2956   1.1  jmcneill 	strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
   2957   1.1  jmcneill 	ifp->if_softc = sc;
   2958   1.1  jmcneill 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
   2959   1.1  jmcneill #ifdef MCX_MPSAFE
   2960   1.1  jmcneill 	ifp->if_extflags = IFEF_MPSAFE;
   2961   1.1  jmcneill #endif
   2962   1.1  jmcneill 	ifp->if_init = mcx_init;
   2963   1.1  jmcneill 	ifp->if_stop = mcx_stop;
   2964   1.1  jmcneill 	ifp->if_ioctl = mcx_ioctl;
   2965   1.1  jmcneill 	ifp->if_start = mcx_start;
   2966  1.16  jmcneill 	if (sc->sc_nqueues > 1) {
   2967  1.16  jmcneill 		ifp->if_transmit = mcx_transmit;
   2968  1.16  jmcneill 	}
   2969   1.1  jmcneill 	ifp->if_watchdog = mcx_watchdog;
   2970   1.1  jmcneill 	ifp->if_mtu = sc->sc_hardmtu;
   2971  1.15  jmcneill 	ifp->if_capabilities = IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
   2972  1.15  jmcneill 	    IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
   2973  1.15  jmcneill 	    IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx |
   2974  1.15  jmcneill 	    IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
   2975  1.15  jmcneill 	    IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx;
   2976   1.1  jmcneill 	IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
   2977   1.1  jmcneill 	IFQ_SET_READY(&ifp->if_snd);
   2978   1.1  jmcneill 
   2979  1.15  jmcneill 	sc->sc_ec.ec_capabilities = ETHERCAP_JUMBO_MTU |
   2980  1.15  jmcneill 	    ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING;
   2981  1.15  jmcneill 	sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;
   2982   1.1  jmcneill 
   2983   1.1  jmcneill 	sc->sc_ec.ec_ifmedia = &sc->sc_media;
   2984  1.12   thorpej 	ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, mcx_media_change,
   2985  1.12   thorpej 	    mcx_media_status, &sc->sc_media_mutex);
   2986   1.1  jmcneill 	mcx_media_add_types(sc);
   2987   1.1  jmcneill 	ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
   2988   1.1  jmcneill 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
   2989   1.1  jmcneill 
   2990   1.1  jmcneill 	if_attach(ifp);
   2991   1.1  jmcneill 	if_deferred_start_init(ifp, NULL);
   2992   1.1  jmcneill 
   2993   1.1  jmcneill 	ether_ifattach(ifp, enaddr);
   2994   1.1  jmcneill 
   2995  1.15  jmcneill 	kcpuset_create(&affinity, false);
   2996  1.15  jmcneill 	kcpuset_set(affinity, 0);
   2997  1.15  jmcneill 
   2998  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   2999  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   3000  1.15  jmcneill 		struct mcx_rx *rx = &q->q_rx;
   3001  1.15  jmcneill 		struct mcx_tx *tx = &q->q_tx;
   3002  1.15  jmcneill 		int vec;
   3003  1.15  jmcneill 
   3004  1.15  jmcneill 		vec = i + 1;
   3005  1.15  jmcneill 		q->q_sc = sc;
   3006  1.15  jmcneill 		q->q_index = i;
   3007  1.15  jmcneill 
   3008  1.15  jmcneill 		if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
   3009  1.15  jmcneill 			aprint_error_dev(self, "unable to alloc uar %d\n", i);
   3010  1.15  jmcneill 			goto teardown;
   3011  1.15  jmcneill 		}
   3012  1.15  jmcneill 
   3013  1.15  jmcneill 		if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
   3014  1.15  jmcneill 			aprint_error_dev(self,
   3015  1.15  jmcneill 			    "unable to create event queue %d\n", i);
   3016  1.15  jmcneill 			goto teardown;
   3017  1.15  jmcneill 		}
   3018  1.15  jmcneill 
   3019  1.15  jmcneill 		rx->rx_softc = sc;
   3020  1.15  jmcneill 		callout_init(&rx->rx_refill, CALLOUT_FLAGS);
   3021  1.15  jmcneill 		callout_setfunc(&rx->rx_refill, mcx_refill, rx);
   3022  1.15  jmcneill 
   3023  1.15  jmcneill 		tx->tx_softc = sc;
   3024  1.16  jmcneill 		mutex_init(&tx->tx_lock, MUTEX_DEFAULT, IPL_NET);
   3025  1.16  jmcneill 		tx->tx_pcq = pcq_create(MCX_TXQ_NUM, KM_SLEEP);
   3026  1.16  jmcneill 		tx->tx_softint = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,
   3027  1.16  jmcneill 		    mcx_deferred_transmit, tx);
   3028  1.15  jmcneill 
   3029  1.15  jmcneill 		snprintf(intrxname, sizeof(intrxname), "%s queue %d",
   3030  1.15  jmcneill 		    DEVNAME(sc), i);
   3031  1.15  jmcneill 		q->q_ihc = mcx_establish_intr(sc, vec, affinity, mcx_cq_intr,
   3032  1.15  jmcneill 		    q, intrxname);
   3033  1.15  jmcneill 	}
   3034  1.15  jmcneill 
   3035   1.1  jmcneill 	callout_init(&sc->sc_calibrate, CALLOUT_FLAGS);
   3036   1.1  jmcneill 	callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc);
   3037   1.1  jmcneill 
   3038   1.1  jmcneill 	if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc,
   3039   1.1  jmcneill 	    PRI_NONE, IPL_NET, 0) != 0) {
   3040   1.1  jmcneill 		aprint_error_dev(self, "couldn't create port change workq\n");
   3041   1.1  jmcneill 		goto teardown;
   3042   1.1  jmcneill 	}
   3043   1.1  jmcneill 
   3044   1.1  jmcneill 	mcx_port_change(&sc->sc_port_change, sc);
   3045   1.1  jmcneill 
   3046  1.15  jmcneill 	sc->sc_mac_flow_table_id = -1;
   3047  1.15  jmcneill 	sc->sc_rss_flow_table_id = -1;
   3048  1.15  jmcneill 	sc->sc_rqt = -1;
   3049   1.1  jmcneill 	for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
   3050  1.15  jmcneill 		struct mcx_flow_group *mfg = &sc->sc_flow_group[i];
   3051  1.15  jmcneill 		mfg->g_id = -1;
   3052  1.15  jmcneill 		mfg->g_table = -1;
   3053  1.15  jmcneill 		mfg->g_size = 0;
   3054  1.15  jmcneill 		mfg->g_start = 0;
   3055   1.1  jmcneill 	}
   3056   1.1  jmcneill 	sc->sc_extra_mcast = 0;
   3057   1.1  jmcneill 	memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));
   3058  1.15  jmcneill 
   3059  1.15  jmcneill #if NKSTAT > 0
   3060  1.15  jmcneill 	mcx_kstat_attach(sc);
   3061  1.15  jmcneill #endif
   3062  1.15  jmcneill 	mcx_timecounter_attach(sc);
   3063   1.1  jmcneill 	return;
   3064   1.1  jmcneill 
   3065   1.1  jmcneill teardown:
   3066   1.1  jmcneill 	mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
   3067   1.1  jmcneill 	/* error printed by mcx_teardown_hca, and we're already unwinding */
   3068   1.1  jmcneill cqfree:
   3069   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   3070  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
   3071  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3072   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
   3073   1.1  jmcneill 	    MCX_CMDQ_INTERFACE_DISABLED);
   3074  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t),
   3075  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3076   1.1  jmcneill 
   3077   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
   3078  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
   3079  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3080   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
   3081   1.1  jmcneill 
   3082   1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
   3083   1.1  jmcneill dbfree:
   3084   1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
   3085   1.1  jmcneill unmap:
   3086   1.1  jmcneill 	bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
   3087   1.1  jmcneill 	sc->sc_mems = 0;
   3088   1.1  jmcneill }
   3089   1.1  jmcneill 
   3090  1.15  jmcneill static void *
   3091  1.15  jmcneill mcx_establish_intr(struct mcx_softc *sc, int index, kcpuset_t *affinity,
   3092  1.15  jmcneill     int (*func)(void *), void *arg, const char *xname)
   3093  1.15  jmcneill {
   3094  1.15  jmcneill 	char intrbuf[PCI_INTRSTR_LEN];
   3095  1.15  jmcneill 	const char *intrstr;
   3096  1.15  jmcneill 	void *ih;
   3097  1.15  jmcneill 
   3098  1.15  jmcneill 	pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[index], PCI_INTR_MPSAFE,
   3099  1.15  jmcneill 	    true);
   3100  1.15  jmcneill 
   3101  1.15  jmcneill 	intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[index], intrbuf,
   3102  1.15  jmcneill 	    sizeof(intrbuf));
   3103  1.15  jmcneill 	ih = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[index], IPL_NET,
   3104  1.15  jmcneill 	    func, arg, xname);
   3105  1.15  jmcneill 	if (ih == NULL) {
   3106  1.15  jmcneill 		aprint_error_dev(sc->sc_dev,
   3107  1.15  jmcneill 		    "unable to establish interrupt%s%s\n",
   3108  1.15  jmcneill 		    intrstr ? " at " : "",
   3109  1.15  jmcneill 		    intrstr ? intrstr : "");
   3110  1.15  jmcneill 		return NULL;
   3111  1.15  jmcneill 	}
   3112  1.15  jmcneill 
   3113  1.15  jmcneill 	if (affinity != NULL && index > 0) {
   3114  1.15  jmcneill 		/* Round-robin affinity */
   3115  1.15  jmcneill 		kcpuset_zero(affinity);
   3116  1.15  jmcneill 		kcpuset_set(affinity, (index - 1) % ncpu);
   3117  1.15  jmcneill 		interrupt_distribute(ih, affinity, NULL);
   3118  1.15  jmcneill 	}
   3119  1.15  jmcneill 
   3120  1.15  jmcneill 	return ih;
   3121  1.15  jmcneill }
   3122  1.15  jmcneill 
   3123   1.1  jmcneill static void
   3124   1.1  jmcneill mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm)
   3125   1.1  jmcneill {
   3126   1.1  jmcneill 	rxr->rxr_total = hwm;
   3127   1.1  jmcneill 	rxr->rxr_inuse = 0;
   3128   1.1  jmcneill }
   3129   1.1  jmcneill 
   3130   1.1  jmcneill static u_int
   3131   1.1  jmcneill mcx_rxr_get(struct mcx_rxring *rxr, u_int max)
   3132   1.1  jmcneill {
   3133   1.1  jmcneill 	const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse);
   3134   1.1  jmcneill 
   3135   1.1  jmcneill 	rxr->rxr_inuse += taken;
   3136   1.1  jmcneill 
   3137   1.1  jmcneill 	return taken;
   3138   1.1  jmcneill }
   3139   1.1  jmcneill 
   3140   1.1  jmcneill static void
   3141   1.1  jmcneill mcx_rxr_put(struct mcx_rxring *rxr, u_int n)
   3142   1.1  jmcneill {
   3143   1.1  jmcneill 	rxr->rxr_inuse -= n;
   3144   1.1  jmcneill }
   3145   1.1  jmcneill 
   3146   1.1  jmcneill static u_int
   3147   1.1  jmcneill mcx_rxr_inuse(struct mcx_rxring *rxr)
   3148   1.1  jmcneill {
   3149   1.1  jmcneill 	return rxr->rxr_inuse;
   3150   1.1  jmcneill }
   3151   1.1  jmcneill 
   3152   1.1  jmcneill static int
   3153   1.1  jmcneill mcx_version(struct mcx_softc *sc)
   3154   1.1  jmcneill {
   3155   1.1  jmcneill 	uint32_t fw0, fw1;
   3156   1.1  jmcneill 	uint16_t cmdif;
   3157   1.1  jmcneill 
   3158   1.1  jmcneill 	fw0 = mcx_rd(sc, MCX_FW_VER);
   3159   1.1  jmcneill 	fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
   3160   1.1  jmcneill 
   3161   1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0),
   3162   1.1  jmcneill 	    MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
   3163   1.1  jmcneill 
   3164   1.1  jmcneill 	cmdif = MCX_CMDIF(fw1);
   3165   1.1  jmcneill 	if (cmdif != MCX_CMD_IF_SUPPORTED) {
   3166   1.1  jmcneill 		aprint_error_dev(sc->sc_dev,
   3167   1.1  jmcneill 		    "unsupported command interface %u\n", cmdif);
   3168   1.1  jmcneill 		return (-1);
   3169   1.1  jmcneill 	}
   3170   1.1  jmcneill 
   3171   1.1  jmcneill 	return (0);
   3172   1.1  jmcneill }
   3173   1.1  jmcneill 
   3174   1.1  jmcneill static int
   3175   1.1  jmcneill mcx_init_wait(struct mcx_softc *sc)
   3176   1.1  jmcneill {
   3177   1.1  jmcneill 	unsigned int i;
   3178   1.1  jmcneill 	uint32_t r;
   3179   1.1  jmcneill 
   3180   1.1  jmcneill 	for (i = 0; i < 2000; i++) {
   3181   1.1  jmcneill 		r = mcx_rd(sc, MCX_STATE);
   3182   1.1  jmcneill 		if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
   3183   1.1  jmcneill 			return (0);
   3184   1.1  jmcneill 
   3185   1.1  jmcneill 		delay(1000);
   3186   1.1  jmcneill 		mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
   3187   1.1  jmcneill 		    BUS_SPACE_BARRIER_READ);
   3188   1.1  jmcneill 	}
   3189   1.1  jmcneill 
   3190   1.1  jmcneill 	return (-1);
   3191   1.1  jmcneill }
   3192   1.1  jmcneill 
   3193   1.1  jmcneill static uint8_t
   3194   1.1  jmcneill mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3195   1.1  jmcneill     unsigned int msec)
   3196   1.1  jmcneill {
   3197   1.1  jmcneill 	unsigned int i;
   3198   1.1  jmcneill 
   3199   1.1  jmcneill 	for (i = 0; i < msec; i++) {
   3200   1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   3201   1.1  jmcneill 		    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
   3202   1.1  jmcneill 
   3203   1.1  jmcneill 		if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
   3204  1.13  jmcneill 		    MCX_CQ_STATUS_OWN_SW)
   3205   1.1  jmcneill 			return (0);
   3206   1.1  jmcneill 
   3207   1.1  jmcneill 		delay(1000);
   3208   1.1  jmcneill 	}
   3209   1.1  jmcneill 
   3210   1.1  jmcneill 	return (ETIMEDOUT);
   3211   1.1  jmcneill }
   3212   1.1  jmcneill 
   3213   1.1  jmcneill static uint32_t
   3214   1.1  jmcneill mcx_mix_u64(uint32_t xor, uint64_t u64)
   3215   1.1  jmcneill {
   3216   1.1  jmcneill 	xor ^= u64 >> 32;
   3217   1.1  jmcneill 	xor ^= u64;
   3218   1.1  jmcneill 
   3219   1.1  jmcneill 	return (xor);
   3220   1.1  jmcneill }
   3221   1.1  jmcneill 
   3222   1.1  jmcneill static uint32_t
   3223   1.1  jmcneill mcx_mix_u32(uint32_t xor, uint32_t u32)
   3224   1.1  jmcneill {
   3225   1.1  jmcneill 	xor ^= u32;
   3226   1.1  jmcneill 
   3227   1.1  jmcneill 	return (xor);
   3228   1.1  jmcneill }
   3229   1.1  jmcneill 
   3230   1.1  jmcneill static uint32_t
   3231   1.1  jmcneill mcx_mix_u8(uint32_t xor, uint8_t u8)
   3232   1.1  jmcneill {
   3233   1.1  jmcneill 	xor ^= u8;
   3234   1.1  jmcneill 
   3235   1.1  jmcneill 	return (xor);
   3236   1.1  jmcneill }
   3237   1.1  jmcneill 
   3238   1.1  jmcneill static uint8_t
   3239   1.1  jmcneill mcx_mix_done(uint32_t xor)
   3240   1.1  jmcneill {
   3241   1.1  jmcneill 	xor ^= xor >> 16;
   3242   1.1  jmcneill 	xor ^= xor >> 8;
   3243   1.1  jmcneill 
   3244   1.1  jmcneill 	return (xor);
   3245   1.1  jmcneill }
   3246   1.1  jmcneill 
   3247   1.1  jmcneill static uint8_t
   3248   1.1  jmcneill mcx_xor(const void *buf, size_t len)
   3249   1.1  jmcneill {
   3250   1.1  jmcneill 	const uint32_t *dwords = buf;
   3251   1.1  jmcneill 	uint32_t xor = 0xff;
   3252   1.1  jmcneill 	size_t i;
   3253   1.1  jmcneill 
   3254   1.1  jmcneill 	len /= sizeof(*dwords);
   3255   1.1  jmcneill 
   3256   1.1  jmcneill 	for (i = 0; i < len; i++)
   3257   1.1  jmcneill 		xor ^= dwords[i];
   3258   1.1  jmcneill 
   3259   1.1  jmcneill 	return (mcx_mix_done(xor));
   3260   1.1  jmcneill }
   3261   1.1  jmcneill 
   3262   1.1  jmcneill static uint8_t
   3263   1.1  jmcneill mcx_cmdq_token(struct mcx_softc *sc)
   3264   1.1  jmcneill {
   3265   1.1  jmcneill 	uint8_t token;
   3266   1.1  jmcneill 
   3267   1.1  jmcneill 	do {
   3268   1.1  jmcneill 		token = ++sc->sc_cmdq_token;
   3269   1.1  jmcneill 	} while (token == 0);
   3270   1.1  jmcneill 
   3271   1.1  jmcneill 	return (token);
   3272   1.1  jmcneill }
   3273   1.1  jmcneill 
   3274   1.1  jmcneill static void
   3275   1.1  jmcneill mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3276   1.1  jmcneill     uint32_t ilen, uint32_t olen, uint8_t token)
   3277   1.1  jmcneill {
   3278   1.1  jmcneill 	memset(cqe, 0, sc->sc_cmdq_size);
   3279   1.1  jmcneill 
   3280   1.1  jmcneill 	cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
   3281   1.1  jmcneill 	be32enc(&cqe->cq_input_length, ilen);
   3282   1.1  jmcneill 	be32enc(&cqe->cq_output_length, olen);
   3283   1.1  jmcneill 	cqe->cq_token = token;
   3284   1.1  jmcneill 	cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
   3285   1.1  jmcneill }
   3286   1.1  jmcneill 
   3287   1.1  jmcneill static void
   3288   1.1  jmcneill mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
   3289   1.1  jmcneill {
   3290   1.1  jmcneill 	cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
   3291   1.1  jmcneill }
   3292   1.1  jmcneill 
   3293   1.1  jmcneill static int
   3294   1.1  jmcneill mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
   3295   1.1  jmcneill {
   3296   1.1  jmcneill 	/* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 :  0); */
   3297   1.1  jmcneill 	return (0);
   3298   1.1  jmcneill }
   3299   1.1  jmcneill 
   3300   1.1  jmcneill static void *
   3301   1.1  jmcneill mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
   3302   1.1  jmcneill {
   3303   1.1  jmcneill 	return (&cqe->cq_input_data);
   3304   1.1  jmcneill }
   3305   1.1  jmcneill 
   3306   1.1  jmcneill static void *
   3307   1.1  jmcneill mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
   3308   1.1  jmcneill {
   3309   1.1  jmcneill 	return (&cqe->cq_output_data);
   3310   1.1  jmcneill }
   3311   1.1  jmcneill 
   3312   1.1  jmcneill static void
   3313   1.1  jmcneill mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3314   1.1  jmcneill     unsigned int slot)
   3315   1.1  jmcneill {
   3316   1.1  jmcneill 	mcx_cmdq_sign(cqe);
   3317   1.1  jmcneill 
   3318   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   3319   1.1  jmcneill 	    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
   3320   1.1  jmcneill 
   3321   1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
   3322  1.15  jmcneill 	mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t),
   3323  1.15  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   3324   1.1  jmcneill }
   3325   1.1  jmcneill 
   3326   1.1  jmcneill static int
   3327   1.1  jmcneill mcx_enable_hca(struct mcx_softc *sc)
   3328   1.1  jmcneill {
   3329   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3330   1.1  jmcneill 	struct mcx_cmd_enable_hca_in *in;
   3331   1.1  jmcneill 	struct mcx_cmd_enable_hca_out *out;
   3332   1.1  jmcneill 	int error;
   3333   1.1  jmcneill 	uint8_t status;
   3334   1.1  jmcneill 
   3335   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3336   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3337   1.1  jmcneill 
   3338   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3339   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
   3340   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3341   1.1  jmcneill 	in->cmd_function_id = htobe16(0);
   3342   1.1  jmcneill 
   3343   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3344   1.1  jmcneill 
   3345   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3346   1.1  jmcneill 	if (error != 0) {
   3347   1.1  jmcneill 		printf(", hca enable timeout\n");
   3348   1.1  jmcneill 		return (-1);
   3349   1.1  jmcneill 	}
   3350   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3351   1.1  jmcneill 		printf(", hca enable command corrupt\n");
   3352   1.1  jmcneill 		return (-1);
   3353   1.1  jmcneill 	}
   3354   1.1  jmcneill 
   3355   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3356   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3357   1.1  jmcneill 		printf(", hca enable failed (%x)\n", status);
   3358   1.1  jmcneill 		return (-1);
   3359   1.1  jmcneill 	}
   3360   1.1  jmcneill 
   3361   1.1  jmcneill 	return (0);
   3362   1.1  jmcneill }
   3363   1.1  jmcneill 
   3364   1.1  jmcneill static int
   3365   1.1  jmcneill mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
   3366   1.1  jmcneill {
   3367   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3368   1.1  jmcneill 	struct mcx_cmd_teardown_hca_in *in;
   3369   1.1  jmcneill 	struct mcx_cmd_teardown_hca_out *out;
   3370   1.1  jmcneill 	int error;
   3371   1.1  jmcneill 	uint8_t status;
   3372   1.1  jmcneill 
   3373   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3374   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3375   1.1  jmcneill 
   3376   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3377   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
   3378   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3379   1.1  jmcneill 	in->cmd_profile = profile;
   3380   1.1  jmcneill 
   3381   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3382   1.1  jmcneill 
   3383   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3384   1.1  jmcneill 	if (error != 0) {
   3385   1.1  jmcneill 		printf(", hca teardown timeout\n");
   3386   1.1  jmcneill 		return (-1);
   3387   1.1  jmcneill 	}
   3388   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3389   1.1  jmcneill 		printf(", hca teardown command corrupt\n");
   3390   1.1  jmcneill 		return (-1);
   3391   1.1  jmcneill 	}
   3392   1.1  jmcneill 
   3393   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3394   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3395   1.1  jmcneill 		printf(", hca teardown failed (%x)\n", status);
   3396   1.1  jmcneill 		return (-1);
   3397   1.1  jmcneill 	}
   3398   1.1  jmcneill 
   3399   1.1  jmcneill 	return (0);
   3400   1.1  jmcneill }
   3401   1.1  jmcneill 
   3402   1.1  jmcneill static int
   3403   1.1  jmcneill mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   3404   1.1  jmcneill     unsigned int nmb, uint64_t *ptr, uint8_t token)
   3405   1.1  jmcneill {
   3406   1.1  jmcneill 	uint8_t *kva;
   3407   1.1  jmcneill 	uint64_t dva;
   3408   1.1  jmcneill 	int i;
   3409   1.1  jmcneill 	int error;
   3410   1.1  jmcneill 
   3411   1.1  jmcneill 	error = mcx_dmamem_alloc(sc, mxm,
   3412   1.1  jmcneill 	    nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
   3413   1.1  jmcneill 	if (error != 0)
   3414   1.1  jmcneill 		return (error);
   3415   1.1  jmcneill 
   3416   1.1  jmcneill 	mcx_dmamem_zero(mxm);
   3417   1.1  jmcneill 
   3418   1.1  jmcneill 	dva = MCX_DMA_DVA(mxm);
   3419   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3420   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3421   1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
   3422   1.1  jmcneill 
   3423   1.1  jmcneill 		/* patch the cqe or mbox pointing at this one */
   3424   1.1  jmcneill 		be64enc(ptr, dva);
   3425   1.1  jmcneill 
   3426   1.1  jmcneill 		/* fill in this mbox */
   3427   1.1  jmcneill 		be32enc(&mbox->mb_block_number, i);
   3428   1.1  jmcneill 		mbox->mb_token = token;
   3429   1.1  jmcneill 
   3430   1.1  jmcneill 		/* move to the next one */
   3431   1.1  jmcneill 		ptr = &mbox->mb_next_ptr;
   3432   1.1  jmcneill 
   3433   1.1  jmcneill 		dva += MCX_CMDQ_MAILBOX_SIZE;
   3434   1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   3435   1.1  jmcneill 	}
   3436   1.1  jmcneill 
   3437   1.1  jmcneill 	return (0);
   3438   1.1  jmcneill }
   3439   1.1  jmcneill 
   3440   1.1  jmcneill static uint32_t
   3441   1.1  jmcneill mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
   3442   1.1  jmcneill {
   3443   1.1  jmcneill 	uint32_t xor = 0xff;
   3444   1.1  jmcneill 
   3445   1.1  jmcneill 	/* only 3 fields get set, so mix them directly */
   3446   1.1  jmcneill 	xor = mcx_mix_u64(xor, mb->mb_next_ptr);
   3447   1.1  jmcneill 	xor = mcx_mix_u32(xor, mb->mb_block_number);
   3448   1.1  jmcneill 	xor = mcx_mix_u8(xor, mb->mb_token);
   3449   1.1  jmcneill 
   3450   1.1  jmcneill 	return (mcx_mix_done(xor));
   3451   1.1  jmcneill }
   3452   1.1  jmcneill 
   3453   1.1  jmcneill static void
   3454   1.1  jmcneill mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
   3455   1.1  jmcneill {
   3456   1.1  jmcneill 	uint8_t *kva;
   3457   1.1  jmcneill 	int i;
   3458   1.1  jmcneill 
   3459   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3460   1.1  jmcneill 
   3461   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3462   1.1  jmcneill 		struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
   3463   1.1  jmcneill 		uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
   3464   1.1  jmcneill 		mb->mb_ctrl_signature = sig;
   3465   1.1  jmcneill 		mb->mb_signature = sig ^
   3466   1.1  jmcneill 		    mcx_xor(mb->mb_data, sizeof(mb->mb_data));
   3467   1.1  jmcneill 
   3468   1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   3469   1.1  jmcneill 	}
   3470   1.1  jmcneill }
   3471   1.1  jmcneill 
   3472   1.1  jmcneill static void
   3473   1.1  jmcneill mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
   3474   1.1  jmcneill {
   3475   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
   3476   1.1  jmcneill 	    0, MCX_DMA_LEN(mxm), ops);
   3477   1.1  jmcneill }
   3478   1.1  jmcneill 
   3479   1.1  jmcneill static struct mcx_cmdq_mailbox *
   3480   1.1  jmcneill mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
   3481   1.1  jmcneill {
   3482   1.1  jmcneill 	uint8_t *kva;
   3483   1.1  jmcneill 
   3484   1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   3485   1.1  jmcneill 	kva += i * MCX_CMDQ_MAILBOX_SIZE;
   3486   1.1  jmcneill 
   3487   1.1  jmcneill 	return ((struct mcx_cmdq_mailbox *)kva);
   3488   1.1  jmcneill }
   3489   1.1  jmcneill 
   3490   1.1  jmcneill static inline void *
   3491   1.1  jmcneill mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
   3492   1.1  jmcneill {
   3493   1.1  jmcneill 	return (&mb->mb_data);
   3494   1.1  jmcneill }
   3495   1.1  jmcneill 
   3496   1.1  jmcneill static void
   3497   1.1  jmcneill mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
   3498   1.1  jmcneill     void *b, size_t len)
   3499   1.1  jmcneill {
   3500   1.1  jmcneill 	uint8_t *buf = b;
   3501   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3502   1.1  jmcneill 	int i;
   3503   1.1  jmcneill 
   3504   1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   3505   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3506   1.1  jmcneill 
   3507   1.1  jmcneill 		memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len));
   3508   1.1  jmcneill 
   3509   1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   3510   1.1  jmcneill 			break;
   3511   1.1  jmcneill 
   3512   1.1  jmcneill 		buf += sizeof(mb->mb_data);
   3513   1.1  jmcneill 		len -= sizeof(mb->mb_data);
   3514   1.1  jmcneill 		mb++;
   3515   1.1  jmcneill 	}
   3516   1.1  jmcneill }
   3517   1.1  jmcneill 
   3518   1.1  jmcneill static void
   3519  1.13  jmcneill mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages,
   3520  1.13  jmcneill     struct mcx_dmamem *buf)
   3521  1.13  jmcneill {
   3522  1.13  jmcneill 	uint64_t *pas;
   3523  1.13  jmcneill 	int mbox, mbox_pages, i;
   3524  1.13  jmcneill 
   3525  1.13  jmcneill 	mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE;
   3526  1.13  jmcneill 	offset %= MCX_CMDQ_MAILBOX_DATASIZE;
   3527  1.13  jmcneill 
   3528  1.13  jmcneill 	pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
   3529  1.13  jmcneill 	pas += (offset / sizeof(*pas));
   3530  1.13  jmcneill 	mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas);
   3531  1.13  jmcneill 	for (i = 0; i < npages; i++) {
   3532  1.13  jmcneill 		if (i == mbox_pages) {
   3533  1.13  jmcneill 			mbox++;
   3534  1.13  jmcneill 			pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
   3535  1.13  jmcneill 			mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas);
   3536  1.13  jmcneill 		}
   3537  1.13  jmcneill 		*pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE));
   3538  1.13  jmcneill 		pas++;
   3539  1.13  jmcneill 	}
   3540  1.13  jmcneill }
   3541  1.13  jmcneill 
   3542  1.13  jmcneill static void
   3543   1.1  jmcneill mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
   3544   1.1  jmcneill {
   3545   1.1  jmcneill 	uint8_t *buf = b;
   3546   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3547   1.1  jmcneill 	int i;
   3548   1.1  jmcneill 
   3549   1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   3550   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3551   1.1  jmcneill 		memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len));
   3552   1.1  jmcneill 
   3553   1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   3554   1.1  jmcneill 			break;
   3555   1.1  jmcneill 
   3556   1.1  jmcneill 		buf += sizeof(mb->mb_data);
   3557   1.1  jmcneill 		len -= sizeof(mb->mb_data);
   3558   1.1  jmcneill 		mb++;
   3559   1.1  jmcneill 	}
   3560   1.1  jmcneill }
   3561   1.1  jmcneill 
   3562   1.1  jmcneill static void
   3563   1.1  jmcneill mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   3564   1.1  jmcneill {
   3565   1.1  jmcneill 	mcx_dmamem_free(sc, mxm);
   3566   1.1  jmcneill }
   3567   1.1  jmcneill 
   3568   1.1  jmcneill #if 0
   3569   1.1  jmcneill static void
   3570   1.1  jmcneill mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
   3571   1.1  jmcneill {
   3572   1.1  jmcneill 	unsigned int i;
   3573   1.1  jmcneill 
   3574   1.1  jmcneill 	printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
   3575   1.1  jmcneill 	    be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr));
   3576   1.1  jmcneill 
   3577   1.1  jmcneill 	printf(", idata ");
   3578   1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_input_data); i++)
   3579   1.1  jmcneill 		printf("%02x", cqe->cq_input_data[i]);
   3580   1.1  jmcneill 
   3581   1.1  jmcneill 	printf(", odata ");
   3582   1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_output_data); i++)
   3583   1.1  jmcneill 		printf("%02x", cqe->cq_output_data[i]);
   3584   1.1  jmcneill 
   3585   1.1  jmcneill 	printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
   3586   1.1  jmcneill 	    be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length),
   3587   1.1  jmcneill 	    cqe->cq_token, cqe->cq_signature, cqe->cq_status);
   3588   1.1  jmcneill }
   3589   1.1  jmcneill 
   3590   1.1  jmcneill static void
   3591   1.1  jmcneill mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
   3592   1.1  jmcneill {
   3593   1.1  jmcneill 	int i, j;
   3594   1.1  jmcneill 	uint8_t *d;
   3595   1.1  jmcneill 
   3596   1.1  jmcneill 	for (i = 0; i < num; i++) {
   3597   1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox;
   3598   1.1  jmcneill 		mbox = mcx_cq_mbox(mboxes, i);
   3599   1.1  jmcneill 
   3600   1.1  jmcneill 		d = mcx_cq_mbox_data(mbox);
   3601   1.1  jmcneill 		for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
   3602   1.1  jmcneill 			if (j != 0 && (j % 16 == 0))
   3603   1.1  jmcneill 				printf("\n");
   3604   1.1  jmcneill 			printf("%.2x ", d[j]);
   3605   1.1  jmcneill 		}
   3606   1.1  jmcneill 	}
   3607   1.1  jmcneill }
   3608   1.1  jmcneill #endif
   3609   1.1  jmcneill 
   3610   1.1  jmcneill static int
   3611   1.1  jmcneill mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
   3612   1.1  jmcneill     int len)
   3613   1.1  jmcneill {
   3614   1.1  jmcneill 	struct mcx_dmamem mxm;
   3615   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3616   1.1  jmcneill 	struct mcx_cmd_access_reg_in *in;
   3617   1.1  jmcneill 	struct mcx_cmd_access_reg_out *out;
   3618   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3619   1.1  jmcneill 	int error, nmb;
   3620   1.1  jmcneill 
   3621   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3622   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
   3623   1.1  jmcneill 	    token);
   3624   1.1  jmcneill 
   3625   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3626   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
   3627   1.1  jmcneill 	in->cmd_op_mod = htobe16(op);
   3628   1.1  jmcneill 	in->cmd_register_id = htobe16(reg);
   3629   1.1  jmcneill 
   3630   1.1  jmcneill 	nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
   3631  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
   3632  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3633   1.1  jmcneill 		printf(", unable to allocate access reg mailboxen\n");
   3634   1.1  jmcneill 		return (-1);
   3635   1.1  jmcneill 	}
   3636   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   3637   1.1  jmcneill 	mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
   3638   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   3639   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   3640   1.1  jmcneill 
   3641   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3642   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3643   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   3644   1.1  jmcneill 
   3645   1.1  jmcneill 	if (error != 0) {
   3646   1.1  jmcneill 		printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
   3647   1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
   3648   1.1  jmcneill 		goto free;
   3649   1.1  jmcneill 	}
   3650   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3651   1.1  jmcneill 	if (error != 0) {
   3652   1.1  jmcneill 		printf("%s: access reg (%s %x) reply corrupt\n",
   3653   1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
   3654   1.1  jmcneill 		    reg);
   3655   1.1  jmcneill 		goto free;
   3656   1.1  jmcneill 	}
   3657   1.1  jmcneill 
   3658   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3659   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3660   1.1  jmcneill 		printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
   3661   1.1  jmcneill 		    DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
   3662  1.15  jmcneill 		    reg, out->cmd_status, be32toh(out->cmd_syndrome));
   3663   1.1  jmcneill 		error = -1;
   3664   1.1  jmcneill 		goto free;
   3665   1.1  jmcneill 	}
   3666   1.1  jmcneill 
   3667   1.1  jmcneill 	mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
   3668   1.1  jmcneill free:
   3669   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3670   1.1  jmcneill 
   3671   1.1  jmcneill 	return (error);
   3672   1.1  jmcneill }
   3673   1.1  jmcneill 
   3674   1.1  jmcneill static int
   3675  1.15  jmcneill mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   3676  1.15  jmcneill     unsigned int slot)
   3677   1.1  jmcneill {
   3678   1.1  jmcneill 	struct mcx_cmd_set_issi_in *in;
   3679   1.1  jmcneill 	struct mcx_cmd_set_issi_out *out;
   3680   1.1  jmcneill 	uint8_t status;
   3681   1.1  jmcneill 
   3682   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3683   1.1  jmcneill 
   3684   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3685   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
   3686   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3687   1.1  jmcneill 	in->cmd_current_issi = htobe16(MCX_ISSI);
   3688   1.1  jmcneill 
   3689   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, slot);
   3690   1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
   3691   1.1  jmcneill 		return (-1);
   3692   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0)
   3693   1.1  jmcneill 		return (-1);
   3694   1.1  jmcneill 
   3695   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3696   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK)
   3697   1.1  jmcneill 		return (-1);
   3698   1.1  jmcneill 
   3699   1.1  jmcneill 	return (0);
   3700   1.1  jmcneill }
   3701   1.1  jmcneill 
   3702   1.1  jmcneill static int
   3703   1.1  jmcneill mcx_issi(struct mcx_softc *sc)
   3704   1.1  jmcneill {
   3705   1.1  jmcneill 	struct mcx_dmamem mxm;
   3706   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3707   1.1  jmcneill 	struct mcx_cmd_query_issi_in *in;
   3708   1.1  jmcneill 	struct mcx_cmd_query_issi_il_out *out;
   3709   1.1  jmcneill 	struct mcx_cmd_query_issi_mb_out *mb;
   3710   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3711   1.1  jmcneill 	uint8_t status;
   3712   1.1  jmcneill 	int error;
   3713   1.1  jmcneill 
   3714   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3715   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
   3716   1.1  jmcneill 
   3717   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3718   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
   3719   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3720   1.1  jmcneill 
   3721   1.1  jmcneill 	CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
   3722   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   3723   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3724   1.1  jmcneill 		printf(", unable to allocate query issi mailbox\n");
   3725   1.1  jmcneill 		return (-1);
   3726   1.1  jmcneill 	}
   3727   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   3728   1.1  jmcneill 
   3729   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3730   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3731   1.1  jmcneill 	if (error != 0) {
   3732   1.1  jmcneill 		printf(", query issi timeout\n");
   3733   1.1  jmcneill 		goto free;
   3734   1.1  jmcneill 	}
   3735   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3736   1.1  jmcneill 	if (error != 0) {
   3737   1.1  jmcneill 		printf(", query issi reply corrupt\n");
   3738   1.1  jmcneill 		goto free;
   3739   1.1  jmcneill 	}
   3740   1.1  jmcneill 
   3741   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3742   1.1  jmcneill 	switch (status) {
   3743   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   3744   1.1  jmcneill 		break;
   3745   1.1  jmcneill 	case MCX_CQ_STATUS_BAD_OPCODE:
   3746   1.1  jmcneill 		/* use ISSI 0 */
   3747   1.1  jmcneill 		goto free;
   3748   1.1  jmcneill 	default:
   3749   1.1  jmcneill 		printf(", query issi failed (%x)\n", status);
   3750   1.1  jmcneill 		error = -1;
   3751   1.1  jmcneill 		goto free;
   3752   1.1  jmcneill 	}
   3753   1.1  jmcneill 
   3754   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3755   1.1  jmcneill 	if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
   3756   1.1  jmcneill 		/* use ISSI 1 */
   3757   1.1  jmcneill 		goto free;
   3758   1.1  jmcneill 	}
   3759   1.1  jmcneill 
   3760   1.1  jmcneill 	/* don't need to read cqe anymore, can be used for SET ISSI */
   3761   1.1  jmcneill 
   3762   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   3763   1.1  jmcneill 	CTASSERT(MCX_ISSI < NBBY);
   3764   1.1  jmcneill 	 /* XXX math is hard */
   3765   1.1  jmcneill 	if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
   3766   1.1  jmcneill 		/* use ISSI 0 */
   3767   1.1  jmcneill 		goto free;
   3768   1.1  jmcneill 	}
   3769   1.1  jmcneill 
   3770   1.1  jmcneill 	if (mcx_set_issi(sc, cqe, 0) != 0) {
   3771   1.1  jmcneill 		/* ignore the error, just use ISSI 0 */
   3772   1.1  jmcneill 	} else {
   3773   1.1  jmcneill 		/* use ISSI 1 */
   3774   1.1  jmcneill 	}
   3775   1.1  jmcneill 
   3776   1.1  jmcneill free:
   3777   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   3778   1.1  jmcneill 	return (error);
   3779   1.1  jmcneill }
   3780   1.1  jmcneill 
   3781   1.1  jmcneill static int
   3782   1.1  jmcneill mcx_query_pages(struct mcx_softc *sc, uint16_t type,
   3783  1.15  jmcneill     int32_t *npages, uint16_t *func_id)
   3784   1.1  jmcneill {
   3785   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3786   1.1  jmcneill 	struct mcx_cmd_query_pages_in *in;
   3787   1.1  jmcneill 	struct mcx_cmd_query_pages_out *out;
   3788   1.1  jmcneill 
   3789   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3790   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3791   1.1  jmcneill 
   3792   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3793   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
   3794   1.1  jmcneill 	in->cmd_op_mod = type;
   3795   1.1  jmcneill 
   3796   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3797   1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
   3798   1.1  jmcneill 		printf(", query pages timeout\n");
   3799   1.1  jmcneill 		return (-1);
   3800   1.1  jmcneill 	}
   3801   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3802   1.1  jmcneill 		printf(", query pages reply corrupt\n");
   3803   1.1  jmcneill 		return (-1);
   3804   1.1  jmcneill 	}
   3805   1.1  jmcneill 
   3806   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3807   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3808   1.1  jmcneill 		printf(", query pages failed (%x)\n", out->cmd_status);
   3809   1.1  jmcneill 		return (-1);
   3810   1.1  jmcneill 	}
   3811   1.1  jmcneill 
   3812   1.1  jmcneill 	*func_id = out->cmd_func_id;
   3813   1.1  jmcneill 	*npages = be32dec(&out->cmd_num_pages);
   3814   1.1  jmcneill 
   3815   1.1  jmcneill 	return (0);
   3816   1.1  jmcneill }
   3817   1.1  jmcneill 
   3818   1.1  jmcneill struct bus_dma_iter {
   3819   1.1  jmcneill 	bus_dmamap_t		i_map;
   3820   1.1  jmcneill 	bus_size_t		i_offset;
   3821   1.1  jmcneill 	unsigned int		i_index;
   3822   1.1  jmcneill };
   3823   1.1  jmcneill 
   3824   1.1  jmcneill static void
   3825   1.1  jmcneill bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
   3826   1.1  jmcneill {
   3827   1.1  jmcneill 	i->i_map = map;
   3828   1.1  jmcneill 	i->i_offset = 0;
   3829   1.1  jmcneill 	i->i_index = 0;
   3830   1.1  jmcneill }
   3831   1.1  jmcneill 
   3832   1.1  jmcneill static bus_addr_t
   3833   1.1  jmcneill bus_dma_iter_addr(struct bus_dma_iter *i)
   3834   1.1  jmcneill {
   3835   1.1  jmcneill 	return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
   3836   1.1  jmcneill }
   3837   1.1  jmcneill 
   3838   1.1  jmcneill static void
   3839   1.1  jmcneill bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
   3840   1.1  jmcneill {
   3841   1.1  jmcneill 	bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
   3842   1.1  jmcneill 	bus_size_t diff;
   3843   1.1  jmcneill 
   3844   1.1  jmcneill 	do {
   3845   1.1  jmcneill 		diff = seg->ds_len - i->i_offset;
   3846   1.1  jmcneill 		if (size < diff)
   3847   1.1  jmcneill 			break;
   3848   1.1  jmcneill 
   3849   1.1  jmcneill 		size -= diff;
   3850   1.1  jmcneill 
   3851   1.1  jmcneill 		seg++;
   3852   1.1  jmcneill 
   3853   1.1  jmcneill 		i->i_offset = 0;
   3854   1.1  jmcneill 		i->i_index++;
   3855   1.1  jmcneill 	} while (size > 0);
   3856   1.1  jmcneill 
   3857   1.1  jmcneill 	i->i_offset += size;
   3858   1.1  jmcneill }
   3859   1.1  jmcneill 
   3860   1.1  jmcneill static int
   3861   1.1  jmcneill mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
   3862   1.1  jmcneill {
   3863   1.1  jmcneill 	struct mcx_dmamem mxm;
   3864   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3865   1.1  jmcneill 	struct mcx_cmd_manage_pages_in *in;
   3866   1.1  jmcneill 	struct mcx_cmd_manage_pages_out *out;
   3867   1.1  jmcneill 	unsigned int paslen, nmb, i, j, npages;
   3868   1.1  jmcneill 	struct bus_dma_iter iter;
   3869   1.1  jmcneill 	uint64_t *pas;
   3870   1.1  jmcneill 	uint8_t status;
   3871   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3872   1.1  jmcneill 	int error;
   3873   1.1  jmcneill 
   3874   1.1  jmcneill 	npages = mhm->mhm_npages;
   3875   1.1  jmcneill 
   3876   1.1  jmcneill 	paslen = sizeof(*pas) * npages;
   3877   1.1  jmcneill 	nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
   3878   1.1  jmcneill 
   3879   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3880   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
   3881   1.1  jmcneill 
   3882   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3883   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
   3884   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
   3885   1.1  jmcneill 	in->cmd_func_id = func_id;
   3886   1.1  jmcneill 	be32enc(&in->cmd_input_num_entries, npages);
   3887   1.1  jmcneill 
   3888   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
   3889   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   3890   1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   3891   1.1  jmcneill 		return (-1);
   3892   1.1  jmcneill 	}
   3893   1.1  jmcneill 
   3894   1.1  jmcneill 	bus_dma_iter_init(&iter, mhm->mhm_map);
   3895   1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3896   1.1  jmcneill 		unsigned int lim;
   3897   1.1  jmcneill 
   3898   1.1  jmcneill 		pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
   3899   1.1  jmcneill 		lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
   3900   1.1  jmcneill 
   3901   1.1  jmcneill 		for (j = 0; j < lim; j++) {
   3902   1.1  jmcneill 			be64enc(&pas[j], bus_dma_iter_addr(&iter));
   3903   1.1  jmcneill 			bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
   3904   1.1  jmcneill 		}
   3905   1.1  jmcneill 
   3906   1.1  jmcneill 		npages -= lim;
   3907   1.1  jmcneill 	}
   3908   1.1  jmcneill 
   3909   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   3910   1.1  jmcneill 
   3911   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3912   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3913   1.1  jmcneill 	if (error != 0) {
   3914   1.1  jmcneill 		printf(", manage pages timeout\n");
   3915   1.1  jmcneill 		goto free;
   3916   1.1  jmcneill 	}
   3917   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3918   1.1  jmcneill 	if (error != 0) {
   3919   1.1  jmcneill 		printf(", manage pages reply corrupt\n");
   3920   1.1  jmcneill 		goto free;
   3921   1.1  jmcneill 	}
   3922   1.1  jmcneill 
   3923   1.1  jmcneill 	status = cqe->cq_output_data[0];
   3924   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3925   1.1  jmcneill 		printf(", manage pages failed (%x)\n", status);
   3926   1.1  jmcneill 		error = -1;
   3927   1.1  jmcneill 		goto free;
   3928   1.1  jmcneill 	}
   3929   1.1  jmcneill 
   3930   1.1  jmcneill free:
   3931   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3932   1.1  jmcneill 
   3933   1.1  jmcneill 	return (error);
   3934   1.1  jmcneill }
   3935   1.1  jmcneill 
   3936   1.1  jmcneill static int
   3937   1.1  jmcneill mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
   3938   1.1  jmcneill {
   3939  1.15  jmcneill 	int32_t npages;
   3940   1.1  jmcneill 	uint16_t func_id;
   3941   1.1  jmcneill 
   3942   1.1  jmcneill 	if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
   3943   1.1  jmcneill 		/* error printed by mcx_query_pages */
   3944   1.1  jmcneill 		return (-1);
   3945   1.1  jmcneill 	}
   3946   1.1  jmcneill 
   3947  1.15  jmcneill 	if (npages < 1)
   3948   1.1  jmcneill 		return (0);
   3949   1.1  jmcneill 
   3950   1.1  jmcneill 	if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
   3951   1.1  jmcneill 		printf(", unable to allocate hwmem\n");
   3952   1.1  jmcneill 		return (-1);
   3953   1.1  jmcneill 	}
   3954   1.1  jmcneill 
   3955   1.1  jmcneill 	if (mcx_add_pages(sc, mhm, func_id) != 0) {
   3956   1.1  jmcneill 		printf(", unable to add hwmem\n");
   3957   1.1  jmcneill 		goto free;
   3958   1.1  jmcneill 	}
   3959   1.1  jmcneill 
   3960   1.1  jmcneill 	return (0);
   3961   1.1  jmcneill 
   3962   1.1  jmcneill free:
   3963   1.1  jmcneill 	mcx_hwmem_free(sc, mhm);
   3964   1.1  jmcneill 
   3965   1.1  jmcneill 	return (-1);
   3966   1.1  jmcneill }
   3967   1.1  jmcneill 
   3968   1.1  jmcneill static int
   3969   1.1  jmcneill mcx_hca_max_caps(struct mcx_softc *sc)
   3970   1.1  jmcneill {
   3971   1.1  jmcneill 	struct mcx_dmamem mxm;
   3972   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3973   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   3974   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   3975   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3976   1.1  jmcneill 	struct mcx_cap_device *hca;
   3977   1.1  jmcneill 	uint8_t status;
   3978   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3979   1.1  jmcneill 	int error;
   3980   1.1  jmcneill 
   3981   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3982   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   3983   1.1  jmcneill 	    token);
   3984   1.1  jmcneill 
   3985   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3986   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   3987   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
   3988   1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   3989   1.1  jmcneill 
   3990   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   3991   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3992   1.1  jmcneill 		printf(", unable to allocate query hca caps mailboxen\n");
   3993   1.1  jmcneill 		return (-1);
   3994   1.1  jmcneill 	}
   3995   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   3996   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   3997   1.1  jmcneill 
   3998   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3999   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4000   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   4001   1.1  jmcneill 
   4002   1.1  jmcneill 	if (error != 0) {
   4003   1.1  jmcneill 		printf(", query hca caps timeout\n");
   4004   1.1  jmcneill 		goto free;
   4005   1.1  jmcneill 	}
   4006   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   4007   1.1  jmcneill 	if (error != 0) {
   4008   1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   4009   1.1  jmcneill 		goto free;
   4010   1.1  jmcneill 	}
   4011   1.1  jmcneill 
   4012   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4013   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4014   1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   4015   1.1  jmcneill 		error = -1;
   4016   1.1  jmcneill 		goto free;
   4017   1.1  jmcneill 	}
   4018   1.1  jmcneill 
   4019   1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   4020   1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   4021   1.1  jmcneill 
   4022  1.15  jmcneill 	if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE)
   4023  1.15  jmcneill 	    != MCX_CAP_DEVICE_PORT_TYPE_ETH) {
   4024  1.15  jmcneill 		printf(", not in ethernet mode\n");
   4025  1.15  jmcneill 		error = -1;
   4026  1.15  jmcneill 		goto free;
   4027  1.15  jmcneill 	}
   4028   1.1  jmcneill 	if (hca->log_pg_sz > PAGE_SHIFT) {
   4029   1.1  jmcneill 		printf(", minimum system page shift %u is too large\n",
   4030   1.1  jmcneill 		    hca->log_pg_sz);
   4031   1.1  jmcneill 		error = -1;
   4032   1.1  jmcneill 		goto free;
   4033   1.1  jmcneill 	}
   4034   1.1  jmcneill 	/*
   4035   1.1  jmcneill 	 * blueflame register is split into two buffers, and we must alternate
   4036   1.1  jmcneill 	 * between the two of them.
   4037   1.1  jmcneill 	 */
   4038   1.1  jmcneill 	sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
   4039  1.15  jmcneill 	sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size);
   4040  1.15  jmcneill 
   4041  1.15  jmcneill 	if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG)
   4042  1.15  jmcneill 		sc->sc_mcam_reg = 1;
   4043  1.15  jmcneill 
   4044  1.15  jmcneill 	sc->sc_mhz = be32dec(&hca->device_frequency_mhz);
   4045  1.15  jmcneill 	sc->sc_khz = be32dec(&hca->device_frequency_khz);
   4046   1.1  jmcneill 
   4047   1.1  jmcneill free:
   4048   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4049   1.1  jmcneill 
   4050   1.1  jmcneill 	return (error);
   4051   1.1  jmcneill }
   4052   1.1  jmcneill 
   4053   1.1  jmcneill static int
   4054   1.1  jmcneill mcx_hca_set_caps(struct mcx_softc *sc)
   4055   1.1  jmcneill {
   4056   1.1  jmcneill 	struct mcx_dmamem mxm;
   4057   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4058   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   4059   1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   4060   1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   4061   1.1  jmcneill 	struct mcx_cap_device *hca;
   4062   1.1  jmcneill 	uint8_t status;
   4063   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   4064   1.1  jmcneill 	int error;
   4065   1.1  jmcneill 
   4066   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4067   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   4068   1.1  jmcneill 	    token);
   4069   1.1  jmcneill 
   4070   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4071   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   4072   1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
   4073   1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   4074   1.1  jmcneill 
   4075   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   4076   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4077   1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   4078   1.1  jmcneill 		return (-1);
   4079   1.1  jmcneill 	}
   4080   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   4081   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   4082   1.1  jmcneill 
   4083   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4084   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4085   1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   4086   1.1  jmcneill 
   4087   1.1  jmcneill 	if (error != 0) {
   4088   1.1  jmcneill 		printf(", query hca caps timeout\n");
   4089   1.1  jmcneill 		goto free;
   4090   1.1  jmcneill 	}
   4091   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   4092   1.1  jmcneill 	if (error != 0) {
   4093   1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   4094   1.1  jmcneill 		goto free;
   4095   1.1  jmcneill 	}
   4096   1.1  jmcneill 
   4097   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4098   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4099   1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   4100   1.1  jmcneill 		error = -1;
   4101   1.1  jmcneill 		goto free;
   4102   1.1  jmcneill 	}
   4103   1.1  jmcneill 
   4104   1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   4105   1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   4106   1.1  jmcneill 
   4107   1.1  jmcneill 	hca->log_pg_sz = PAGE_SHIFT;
   4108   1.1  jmcneill 
   4109   1.1  jmcneill free:
   4110   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4111   1.1  jmcneill 
   4112   1.1  jmcneill 	return (error);
   4113   1.1  jmcneill }
   4114   1.1  jmcneill 
   4115   1.1  jmcneill 
   4116   1.1  jmcneill static int
   4117   1.1  jmcneill mcx_init_hca(struct mcx_softc *sc)
   4118   1.1  jmcneill {
   4119   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4120   1.1  jmcneill 	struct mcx_cmd_init_hca_in *in;
   4121   1.1  jmcneill 	struct mcx_cmd_init_hca_out *out;
   4122   1.1  jmcneill 	int error;
   4123   1.1  jmcneill 	uint8_t status;
   4124   1.1  jmcneill 
   4125   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4126   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4127   1.1  jmcneill 
   4128   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4129   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
   4130   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4131   1.1  jmcneill 
   4132   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4133   1.1  jmcneill 
   4134   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4135   1.1  jmcneill 	if (error != 0) {
   4136   1.1  jmcneill 		printf(", hca init timeout\n");
   4137   1.1  jmcneill 		return (-1);
   4138   1.1  jmcneill 	}
   4139   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4140   1.1  jmcneill 		printf(", hca init command corrupt\n");
   4141   1.1  jmcneill 		return (-1);
   4142   1.1  jmcneill 	}
   4143   1.1  jmcneill 
   4144   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4145   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4146   1.1  jmcneill 		printf(", hca init failed (%x)\n", status);
   4147   1.1  jmcneill 		return (-1);
   4148   1.1  jmcneill 	}
   4149   1.1  jmcneill 
   4150   1.1  jmcneill 	return (0);
   4151   1.1  jmcneill }
   4152   1.1  jmcneill 
   4153   1.1  jmcneill static int
   4154   1.1  jmcneill mcx_set_driver_version(struct mcx_softc *sc)
   4155   1.1  jmcneill {
   4156   1.1  jmcneill 	struct mcx_dmamem mxm;
   4157   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4158   1.1  jmcneill 	struct mcx_cmd_set_driver_version_in *in;
   4159   1.1  jmcneill 	struct mcx_cmd_set_driver_version_out *out;
   4160   1.1  jmcneill 	int error;
   4161   1.1  jmcneill 	int token;
   4162   1.1  jmcneill 	uint8_t status;
   4163   1.1  jmcneill 
   4164   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4165   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4166   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) +
   4167   1.1  jmcneill 	    sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);
   4168   1.1  jmcneill 
   4169   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4170   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
   4171   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4172   1.1  jmcneill 
   4173   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4174   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4175   1.1  jmcneill 		printf(", unable to allocate set driver version mailboxen\n");
   4176   1.1  jmcneill 		return (-1);
   4177   1.1  jmcneill 	}
   4178   1.1  jmcneill 	strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
   4179   1.1  jmcneill 	    "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
   4180   1.1  jmcneill 
   4181   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4182   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4183   1.1  jmcneill 
   4184   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4185   1.1  jmcneill 	if (error != 0) {
   4186   1.1  jmcneill 		printf(", set driver version timeout\n");
   4187   1.1  jmcneill 		goto free;
   4188   1.1  jmcneill 	}
   4189   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4190   1.1  jmcneill 		printf(", set driver version command corrupt\n");
   4191   1.1  jmcneill 		goto free;
   4192   1.1  jmcneill 	}
   4193   1.1  jmcneill 
   4194   1.1  jmcneill 	status = cqe->cq_output_data[0];
   4195   1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   4196   1.1  jmcneill 		printf(", set driver version failed (%x)\n", status);
   4197   1.1  jmcneill 		error = -1;
   4198   1.1  jmcneill 		goto free;
   4199   1.1  jmcneill 	}
   4200   1.1  jmcneill 
   4201   1.1  jmcneill free:
   4202   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4203   1.1  jmcneill 
   4204   1.1  jmcneill 	return (error);
   4205   1.1  jmcneill }
   4206   1.1  jmcneill 
   4207   1.1  jmcneill static int
   4208   1.1  jmcneill mcx_iff(struct mcx_softc *sc)
   4209   1.1  jmcneill {
   4210   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   4211   1.1  jmcneill 	struct mcx_dmamem mxm;
   4212   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4213   1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_in *in;
   4214   1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_out *out;
   4215   1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   4216   1.1  jmcneill 	int error;
   4217   1.1  jmcneill 	int token;
   4218   1.1  jmcneill 	int insize;
   4219  1.15  jmcneill 	uint32_t dest;
   4220  1.15  jmcneill 
   4221  1.15  jmcneill 	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   4222  1.15  jmcneill 	    sc->sc_rss_flow_table_id;
   4223   1.1  jmcneill 
   4224   1.1  jmcneill 	/* enable or disable the promisc flow */
   4225   1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_PROMISC)) {
   4226   1.1  jmcneill 		if (sc->sc_promisc_flow_enabled == 0) {
   4227  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc,
   4228  1.15  jmcneill 			    MCX_FLOW_GROUP_PROMISC, 0, NULL, dest);
   4229   1.1  jmcneill 			sc->sc_promisc_flow_enabled = 1;
   4230   1.1  jmcneill 		}
   4231   1.1  jmcneill 	} else if (sc->sc_promisc_flow_enabled != 0) {
   4232   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   4233   1.1  jmcneill 		sc->sc_promisc_flow_enabled = 0;
   4234   1.1  jmcneill 	}
   4235   1.1  jmcneill 
   4236   1.1  jmcneill 	/* enable or disable the all-multicast flow */
   4237   1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   4238   1.1  jmcneill 		if (sc->sc_allmulti_flow_enabled == 0) {
   4239   1.1  jmcneill 			uint8_t mcast[ETHER_ADDR_LEN];
   4240   1.1  jmcneill 
   4241   1.1  jmcneill 			memset(mcast, 0, sizeof(mcast));
   4242   1.1  jmcneill 			mcast[0] = 0x01;
   4243  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc,
   4244  1.15  jmcneill 			    MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest);
   4245   1.1  jmcneill 			sc->sc_allmulti_flow_enabled = 1;
   4246   1.1  jmcneill 		}
   4247   1.1  jmcneill 	} else if (sc->sc_allmulti_flow_enabled != 0) {
   4248   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   4249   1.1  jmcneill 		sc->sc_allmulti_flow_enabled = 0;
   4250   1.1  jmcneill 	}
   4251   1.1  jmcneill 
   4252   1.1  jmcneill 	insize = sizeof(struct mcx_nic_vport_ctx) + 240;
   4253   1.1  jmcneill 
   4254   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4255   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4256   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4257   1.1  jmcneill 
   4258   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4259   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
   4260   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4261   1.1  jmcneill 	in->cmd_field_select = htobe32(
   4262   1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
   4263   1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
   4264   1.1  jmcneill 
   4265   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4266  1.15  jmcneill 		printf(", unable to allocate modify "
   4267  1.15  jmcneill 		    "nic vport context mailboxen\n");
   4268   1.1  jmcneill 		return (-1);
   4269   1.1  jmcneill 	}
   4270   1.1  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)
   4271   1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
   4272   1.1  jmcneill 	ctx->vp_mtu = htobe32(sc->sc_hardmtu);
   4273   1.1  jmcneill 	/*
   4274  1.15  jmcneill          * always leave promisc-all enabled on the vport since we
   4275  1.15  jmcneill          * can't give it a vlan list, and we're already doing multicast
   4276  1.15  jmcneill          * filtering in the flow table.
   4277   1.1  jmcneill 	 */
   4278   1.1  jmcneill 	ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);
   4279   1.1  jmcneill 
   4280   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4281   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4282   1.1  jmcneill 
   4283   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4284   1.1  jmcneill 	if (error != 0) {
   4285   1.1  jmcneill 		printf(", modify nic vport context timeout\n");
   4286   1.1  jmcneill 		goto free;
   4287   1.1  jmcneill 	}
   4288   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4289   1.1  jmcneill 		printf(", modify nic vport context command corrupt\n");
   4290   1.1  jmcneill 		goto free;
   4291   1.1  jmcneill 	}
   4292   1.1  jmcneill 
   4293   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4294   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4295   1.1  jmcneill 		printf(", modify nic vport context failed (%x, %x)\n",
   4296  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4297   1.1  jmcneill 		error = -1;
   4298   1.1  jmcneill 		goto free;
   4299   1.1  jmcneill 	}
   4300   1.1  jmcneill 
   4301   1.1  jmcneill free:
   4302   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4303   1.1  jmcneill 
   4304   1.1  jmcneill 	return (error);
   4305   1.1  jmcneill }
   4306   1.1  jmcneill 
   4307   1.1  jmcneill static int
   4308  1.15  jmcneill mcx_alloc_uar(struct mcx_softc *sc, int *uar)
   4309   1.1  jmcneill {
   4310   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4311   1.1  jmcneill 	struct mcx_cmd_alloc_uar_in *in;
   4312   1.1  jmcneill 	struct mcx_cmd_alloc_uar_out *out;
   4313   1.1  jmcneill 	int error;
   4314   1.1  jmcneill 
   4315   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4316   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4317   1.1  jmcneill 
   4318   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4319   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
   4320   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4321   1.1  jmcneill 
   4322   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4323   1.1  jmcneill 
   4324   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4325   1.1  jmcneill 	if (error != 0) {
   4326   1.1  jmcneill 		printf(", alloc uar timeout\n");
   4327   1.1  jmcneill 		return (-1);
   4328   1.1  jmcneill 	}
   4329   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4330   1.1  jmcneill 		printf(", alloc uar command corrupt\n");
   4331   1.1  jmcneill 		return (-1);
   4332   1.1  jmcneill 	}
   4333   1.1  jmcneill 
   4334   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4335   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4336   1.1  jmcneill 		printf(", alloc uar failed (%x)\n", out->cmd_status);
   4337   1.1  jmcneill 		return (-1);
   4338   1.1  jmcneill 	}
   4339   1.1  jmcneill 
   4340  1.15  jmcneill 	*uar = mcx_get_id(out->cmd_uar);
   4341   1.1  jmcneill 	return (0);
   4342   1.1  jmcneill }
   4343   1.1  jmcneill 
   4344   1.1  jmcneill static int
   4345  1.15  jmcneill mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar,
   4346  1.15  jmcneill     uint64_t events, int vector)
   4347   1.1  jmcneill {
   4348   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4349   1.1  jmcneill 	struct mcx_dmamem mxm;
   4350   1.1  jmcneill 	struct mcx_cmd_create_eq_in *in;
   4351   1.1  jmcneill 	struct mcx_cmd_create_eq_mb_in *mbin;
   4352   1.1  jmcneill 	struct mcx_cmd_create_eq_out *out;
   4353   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   4354   1.1  jmcneill 	int error;
   4355   1.1  jmcneill 	uint64_t *pas;
   4356   1.1  jmcneill 	int insize, npages, paslen, i, token;
   4357   1.1  jmcneill 
   4358  1.15  jmcneill 	eq->eq_cons = 0;
   4359   1.1  jmcneill 
   4360   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
   4361   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4362   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4363   1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
   4364   1.1  jmcneill 
   4365  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE,
   4366   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4367   1.1  jmcneill 		printf(", unable to allocate event queue memory\n");
   4368   1.1  jmcneill 		return (-1);
   4369   1.1  jmcneill 	}
   4370   1.1  jmcneill 
   4371  1.15  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
   4372   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
   4373   1.1  jmcneill 		eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
   4374   1.1  jmcneill 	}
   4375   1.1  jmcneill 
   4376   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4377   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4378   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4379   1.1  jmcneill 
   4380   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4381   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
   4382   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4383   1.1  jmcneill 
   4384  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4385  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4386   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4387   1.1  jmcneill 		printf(", unable to allocate create eq mailboxen\n");
   4388  1.15  jmcneill 		goto free_eq;
   4389   1.1  jmcneill 	}
   4390   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4391   1.1  jmcneill 	mbin->cmd_eq_ctx.eq_uar_size = htobe32(
   4392  1.15  jmcneill 	    (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar);
   4393  1.15  jmcneill 	mbin->cmd_eq_ctx.eq_intr = vector;
   4394  1.15  jmcneill 	mbin->cmd_event_bitmask = htobe64(events);
   4395  1.15  jmcneill 
   4396  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   4397  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   4398   1.1  jmcneill 
   4399   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4400  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem);
   4401   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
   4402   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4403   1.1  jmcneill 
   4404   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4405   1.1  jmcneill 	if (error != 0) {
   4406   1.1  jmcneill 		printf(", create eq timeout\n");
   4407  1.15  jmcneill 		goto free_mxm;
   4408   1.1  jmcneill 	}
   4409   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4410   1.1  jmcneill 		printf(", create eq command corrupt\n");
   4411  1.15  jmcneill 		goto free_mxm;
   4412   1.1  jmcneill 	}
   4413   1.1  jmcneill 
   4414   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4415   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4416   1.1  jmcneill 		printf(", create eq failed (%x, %x)\n", out->cmd_status,
   4417   1.1  jmcneill 		    be32toh(out->cmd_syndrome));
   4418  1.15  jmcneill 		goto free_mxm;
   4419   1.1  jmcneill 	}
   4420   1.1  jmcneill 
   4421  1.15  jmcneill 	eq->eq_n = mcx_get_id(out->cmd_eqn);
   4422  1.15  jmcneill 
   4423  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4424  1.15  jmcneill 
   4425  1.15  jmcneill 	mcx_arm_eq(sc, eq, uar);
   4426  1.15  jmcneill 
   4427  1.15  jmcneill 	return (0);
   4428  1.15  jmcneill 
   4429  1.15  jmcneill free_mxm:
   4430  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   4431  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   4432   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4433  1.15  jmcneill free_eq:
   4434  1.15  jmcneill 	mcx_dmamem_free(sc, &eq->eq_mem);
   4435  1.15  jmcneill 	return (-1);
   4436   1.1  jmcneill }
   4437   1.1  jmcneill 
   4438   1.1  jmcneill static int
   4439   1.1  jmcneill mcx_alloc_pd(struct mcx_softc *sc)
   4440   1.1  jmcneill {
   4441   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4442   1.1  jmcneill 	struct mcx_cmd_alloc_pd_in *in;
   4443   1.1  jmcneill 	struct mcx_cmd_alloc_pd_out *out;
   4444   1.1  jmcneill 	int error;
   4445   1.1  jmcneill 
   4446   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4447   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4448   1.1  jmcneill 
   4449   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4450   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
   4451   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4452   1.1  jmcneill 
   4453   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4454   1.1  jmcneill 
   4455   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4456   1.1  jmcneill 	if (error != 0) {
   4457   1.1  jmcneill 		printf(", alloc pd timeout\n");
   4458   1.1  jmcneill 		return (-1);
   4459   1.1  jmcneill 	}
   4460   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4461   1.1  jmcneill 		printf(", alloc pd command corrupt\n");
   4462   1.1  jmcneill 		return (-1);
   4463   1.1  jmcneill 	}
   4464   1.1  jmcneill 
   4465   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4466   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4467   1.1  jmcneill 		printf(", alloc pd failed (%x)\n", out->cmd_status);
   4468   1.1  jmcneill 		return (-1);
   4469   1.1  jmcneill 	}
   4470   1.1  jmcneill 
   4471  1.13  jmcneill 	sc->sc_pd = mcx_get_id(out->cmd_pd);
   4472   1.1  jmcneill 	return (0);
   4473   1.1  jmcneill }
   4474   1.1  jmcneill 
   4475   1.1  jmcneill static int
   4476   1.1  jmcneill mcx_alloc_tdomain(struct mcx_softc *sc)
   4477   1.1  jmcneill {
   4478   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4479   1.1  jmcneill 	struct mcx_cmd_alloc_td_in *in;
   4480   1.1  jmcneill 	struct mcx_cmd_alloc_td_out *out;
   4481   1.1  jmcneill 	int error;
   4482   1.1  jmcneill 
   4483   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4484   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4485   1.1  jmcneill 
   4486   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4487   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
   4488   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4489   1.1  jmcneill 
   4490   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4491   1.1  jmcneill 
   4492   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4493   1.1  jmcneill 	if (error != 0) {
   4494   1.1  jmcneill 		printf(", alloc transport domain timeout\n");
   4495   1.1  jmcneill 		return (-1);
   4496   1.1  jmcneill 	}
   4497   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4498   1.1  jmcneill 		printf(", alloc transport domain command corrupt\n");
   4499   1.1  jmcneill 		return (-1);
   4500   1.1  jmcneill 	}
   4501   1.1  jmcneill 
   4502   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4503   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4504   1.1  jmcneill 		printf(", alloc transport domain failed (%x)\n",
   4505   1.1  jmcneill 		    out->cmd_status);
   4506   1.1  jmcneill 		return (-1);
   4507   1.1  jmcneill 	}
   4508   1.1  jmcneill 
   4509  1.13  jmcneill 	sc->sc_tdomain = mcx_get_id(out->cmd_tdomain);
   4510   1.1  jmcneill 	return (0);
   4511   1.1  jmcneill }
   4512   1.1  jmcneill 
   4513   1.1  jmcneill static int
   4514   1.1  jmcneill mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr)
   4515   1.1  jmcneill {
   4516   1.1  jmcneill 	struct mcx_dmamem mxm;
   4517   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4518   1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_in *in;
   4519   1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_out *out;
   4520   1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   4521   1.1  jmcneill 	uint8_t *addr;
   4522   1.1  jmcneill 	int error, token, i;
   4523   1.1  jmcneill 
   4524   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4525   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4526   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
   4527   1.1  jmcneill 
   4528   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4529   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
   4530   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4531   1.1  jmcneill 	in->cmd_allowed_list_type = 0;
   4532   1.1  jmcneill 
   4533  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4534  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   4535  1.15  jmcneill 		printf(", unable to allocate "
   4536  1.15  jmcneill 		    "query nic vport context mailboxen\n");
   4537   1.1  jmcneill 		return (-1);
   4538   1.1  jmcneill 	}
   4539   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4540   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4541   1.1  jmcneill 
   4542   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4543   1.1  jmcneill 	if (error != 0) {
   4544   1.1  jmcneill 		printf(", query nic vport context timeout\n");
   4545   1.1  jmcneill 		goto free;
   4546   1.1  jmcneill 	}
   4547   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4548   1.1  jmcneill 		printf(", query nic vport context command corrupt\n");
   4549   1.1  jmcneill 		goto free;
   4550   1.1  jmcneill 	}
   4551   1.1  jmcneill 
   4552   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4553   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4554   1.1  jmcneill 		printf(", query nic vport context failed (%x, %x)\n",
   4555  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4556   1.1  jmcneill 		error = -1;
   4557   1.1  jmcneill 		goto free;
   4558   1.1  jmcneill 	}
   4559   1.1  jmcneill 
   4560  1.15  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)
   4561  1.15  jmcneill 	    mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4562   1.1  jmcneill 	addr = (uint8_t *)&ctx->vp_perm_addr;
   4563   1.1  jmcneill 	for (i = 0; i < ETHER_ADDR_LEN; i++) {
   4564   1.1  jmcneill 		enaddr[i] = addr[i + 2];
   4565   1.1  jmcneill 	}
   4566   1.1  jmcneill free:
   4567   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4568   1.1  jmcneill 
   4569   1.1  jmcneill 	return (error);
   4570   1.1  jmcneill }
   4571   1.1  jmcneill 
   4572   1.1  jmcneill static int
   4573   1.1  jmcneill mcx_query_special_contexts(struct mcx_softc *sc)
   4574   1.1  jmcneill {
   4575   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4576   1.1  jmcneill 	struct mcx_cmd_query_special_ctx_in *in;
   4577   1.1  jmcneill 	struct mcx_cmd_query_special_ctx_out *out;
   4578   1.1  jmcneill 	int error;
   4579   1.1  jmcneill 
   4580   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4581   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4582   1.1  jmcneill 
   4583   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4584   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
   4585   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4586   1.1  jmcneill 
   4587   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4588   1.1  jmcneill 
   4589   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4590   1.1  jmcneill 	if (error != 0) {
   4591   1.1  jmcneill 		printf(", query special contexts timeout\n");
   4592   1.1  jmcneill 		return (-1);
   4593   1.1  jmcneill 	}
   4594   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4595   1.1  jmcneill 		printf(", query special contexts command corrupt\n");
   4596   1.1  jmcneill 		return (-1);
   4597   1.1  jmcneill 	}
   4598   1.1  jmcneill 
   4599   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4600   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4601   1.1  jmcneill 		printf(", query special contexts failed (%x)\n",
   4602   1.1  jmcneill 		    out->cmd_status);
   4603   1.1  jmcneill 		return (-1);
   4604   1.1  jmcneill 	}
   4605   1.1  jmcneill 
   4606   1.1  jmcneill 	sc->sc_lkey = be32toh(out->cmd_resd_lkey);
   4607   1.1  jmcneill 	return (0);
   4608   1.1  jmcneill }
   4609   1.1  jmcneill 
   4610   1.1  jmcneill static int
   4611   1.1  jmcneill mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
   4612   1.1  jmcneill {
   4613   1.1  jmcneill 	struct mcx_reg_pmtu pmtu;
   4614   1.1  jmcneill 	int error;
   4615   1.1  jmcneill 
   4616   1.1  jmcneill 	/* read max mtu */
   4617   1.1  jmcneill 	memset(&pmtu, 0, sizeof(pmtu));
   4618   1.1  jmcneill 	pmtu.rp_local_port = 1;
   4619   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
   4620   1.1  jmcneill 	    sizeof(pmtu));
   4621   1.1  jmcneill 	if (error != 0) {
   4622   1.1  jmcneill 		printf(", unable to get port MTU\n");
   4623   1.1  jmcneill 		return error;
   4624   1.1  jmcneill 	}
   4625   1.1  jmcneill 
   4626   1.1  jmcneill 	mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu));
   4627   1.1  jmcneill 	pmtu.rp_admin_mtu = htobe16(mtu);
   4628   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
   4629   1.1  jmcneill 	    sizeof(pmtu));
   4630   1.1  jmcneill 	if (error != 0) {
   4631   1.1  jmcneill 		printf(", unable to set port MTU\n");
   4632   1.1  jmcneill 		return error;
   4633   1.1  jmcneill 	}
   4634   1.1  jmcneill 
   4635   1.1  jmcneill 	sc->sc_hardmtu = mtu;
   4636  1.15  jmcneill 	sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long));
   4637   1.1  jmcneill 	return 0;
   4638   1.1  jmcneill }
   4639   1.1  jmcneill 
   4640   1.1  jmcneill static int
   4641  1.15  jmcneill mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn)
   4642   1.1  jmcneill {
   4643   1.1  jmcneill 	struct mcx_cmdq_entry *cmde;
   4644   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   4645   1.1  jmcneill 	struct mcx_dmamem mxm;
   4646   1.1  jmcneill 	struct mcx_cmd_create_cq_in *in;
   4647   1.1  jmcneill 	struct mcx_cmd_create_cq_mb_in *mbin;
   4648   1.1  jmcneill 	struct mcx_cmd_create_cq_out *out;
   4649   1.1  jmcneill 	int error;
   4650   1.1  jmcneill 	uint64_t *pas;
   4651   1.1  jmcneill 	int insize, npages, paslen, i, token;
   4652   1.1  jmcneill 
   4653  1.15  jmcneill 	cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db);
   4654   1.1  jmcneill 
   4655   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
   4656   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4657   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4658   1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
   4659   1.1  jmcneill 
   4660   1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
   4661   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4662   1.1  jmcneill 		printf("%s: unable to allocate completion queue memory\n",
   4663   1.1  jmcneill 		    DEVNAME(sc));
   4664   1.1  jmcneill 		return (-1);
   4665   1.1  jmcneill 	}
   4666   1.1  jmcneill 	cqe = MCX_DMA_KVA(&cq->cq_mem);
   4667   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
   4668   1.1  jmcneill 		cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
   4669   1.1  jmcneill 	}
   4670   1.1  jmcneill 
   4671   1.1  jmcneill 	cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4672   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4673   1.1  jmcneill 	mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
   4674   1.1  jmcneill 
   4675   1.1  jmcneill 	in = mcx_cmdq_in(cmde);
   4676   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
   4677   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4678   1.1  jmcneill 
   4679  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4680  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4681   1.1  jmcneill 	    &cmde->cq_input_ptr, token) != 0) {
   4682  1.15  jmcneill 		printf("%s: unable to allocate create cq mailboxen\n",
   4683  1.15  jmcneill 		    DEVNAME(sc));
   4684  1.15  jmcneill 		goto free_cq;
   4685   1.1  jmcneill 	}
   4686   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4687   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_uar_size = htobe32(
   4688  1.15  jmcneill 	    (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar);
   4689   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
   4690   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
   4691   1.1  jmcneill 	    (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
   4692   1.1  jmcneill 	    MCX_CQ_MOD_COUNTER);
   4693   1.1  jmcneill 	mbin->cmd_cq_ctx.cq_doorbell = htobe64(
   4694  1.15  jmcneill 	    MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell);
   4695  1.15  jmcneill 
   4696  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4697  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
   4698   1.1  jmcneill 
   4699   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4700  1.13  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem);
   4701   1.1  jmcneill 	mcx_cmdq_post(sc, cmde, 0);
   4702   1.1  jmcneill 
   4703   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cmde, 1000);
   4704   1.1  jmcneill 	if (error != 0) {
   4705   1.1  jmcneill 		printf("%s: create cq timeout\n", DEVNAME(sc));
   4706  1.15  jmcneill 		goto free_mxm;
   4707   1.1  jmcneill 	}
   4708   1.1  jmcneill 	if (mcx_cmdq_verify(cmde) != 0) {
   4709   1.1  jmcneill 		printf("%s: create cq command corrupt\n", DEVNAME(sc));
   4710  1.15  jmcneill 		goto free_mxm;
   4711   1.1  jmcneill 	}
   4712   1.1  jmcneill 
   4713   1.1  jmcneill 	out = mcx_cmdq_out(cmde);
   4714   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4715   1.1  jmcneill 		printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
   4716   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4717  1.15  jmcneill 		goto free_mxm;
   4718   1.1  jmcneill 	}
   4719   1.1  jmcneill 
   4720  1.13  jmcneill 	cq->cq_n = mcx_get_id(out->cmd_cqn);
   4721   1.1  jmcneill 	cq->cq_cons = 0;
   4722   1.1  jmcneill 	cq->cq_count = 0;
   4723   1.1  jmcneill 
   4724   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4725  1.15  jmcneill 
   4726  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4727  1.15  jmcneill 	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
   4728  1.15  jmcneill 	    BUS_DMASYNC_PREWRITE);
   4729  1.15  jmcneill 
   4730  1.15  jmcneill 	mcx_arm_cq(sc, cq, uar);
   4731  1.15  jmcneill 
   4732  1.15  jmcneill 	return (0);
   4733  1.15  jmcneill 
   4734  1.15  jmcneill free_mxm:
   4735  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4736  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   4737  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4738  1.15  jmcneill free_cq:
   4739  1.15  jmcneill 	mcx_dmamem_free(sc, &cq->cq_mem);
   4740  1.15  jmcneill 	return (-1);
   4741   1.1  jmcneill }
   4742   1.1  jmcneill 
   4743   1.1  jmcneill static int
   4744  1.15  jmcneill mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq)
   4745   1.1  jmcneill {
   4746   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4747   1.1  jmcneill 	struct mcx_cmd_destroy_cq_in *in;
   4748   1.1  jmcneill 	struct mcx_cmd_destroy_cq_out *out;
   4749   1.1  jmcneill 	int error;
   4750   1.1  jmcneill 	int token;
   4751   1.1  jmcneill 
   4752   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4753   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4754   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4755   1.1  jmcneill 
   4756   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4757   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
   4758   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4759  1.15  jmcneill 	in->cmd_cqn = htobe32(cq->cq_n);
   4760   1.1  jmcneill 
   4761   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4762   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4763   1.1  jmcneill 	if (error != 0) {
   4764   1.1  jmcneill 		printf("%s: destroy cq timeout\n", DEVNAME(sc));
   4765   1.1  jmcneill 		return error;
   4766   1.1  jmcneill 	}
   4767   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4768   1.1  jmcneill 		printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
   4769   1.1  jmcneill 		return error;
   4770   1.1  jmcneill 	}
   4771   1.1  jmcneill 
   4772   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4773   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4774   1.1  jmcneill 		printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
   4775   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4776   1.1  jmcneill 		return -1;
   4777   1.1  jmcneill 	}
   4778   1.1  jmcneill 
   4779  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4780  1.15  jmcneill 	    cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
   4781  1.15  jmcneill 	    BUS_DMASYNC_POSTWRITE);
   4782  1.15  jmcneill 
   4783  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   4784  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   4785  1.15  jmcneill 	mcx_dmamem_free(sc, &cq->cq_mem);
   4786  1.15  jmcneill 
   4787  1.15  jmcneill 	cq->cq_n = 0;
   4788  1.15  jmcneill 	cq->cq_cons = 0;
   4789  1.15  jmcneill 	cq->cq_count = 0;
   4790   1.1  jmcneill 	return 0;
   4791   1.1  jmcneill }
   4792   1.1  jmcneill 
   4793   1.1  jmcneill static int
   4794  1.15  jmcneill mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn)
   4795   1.1  jmcneill {
   4796   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4797   1.1  jmcneill 	struct mcx_dmamem mxm;
   4798   1.1  jmcneill 	struct mcx_cmd_create_rq_in *in;
   4799   1.1  jmcneill 	struct mcx_cmd_create_rq_out *out;
   4800   1.1  jmcneill 	struct mcx_rq_ctx *mbin;
   4801   1.1  jmcneill 	int error;
   4802   1.1  jmcneill 	uint64_t *pas;
   4803  1.15  jmcneill 	uint32_t rq_flags;
   4804  1.13  jmcneill 	int insize, npages, paslen, token;
   4805   1.1  jmcneill 
   4806  1.15  jmcneill 	rx->rx_doorbell = MCX_WQ_DOORBELL_BASE +
   4807  1.15  jmcneill 	    (db * MCX_WQ_DOORBELL_STRIDE);
   4808  1.15  jmcneill 
   4809   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
   4810   1.1  jmcneill 	    MCX_PAGE_SIZE);
   4811   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4812   1.1  jmcneill 	insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
   4813   1.1  jmcneill 
   4814  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE,
   4815   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4816   1.1  jmcneill 		printf("%s: unable to allocate receive queue memory\n",
   4817   1.1  jmcneill 		    DEVNAME(sc));
   4818   1.1  jmcneill 		return (-1);
   4819   1.1  jmcneill 	}
   4820   1.1  jmcneill 
   4821   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4822   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4823   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4824   1.1  jmcneill 
   4825   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4826   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
   4827   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4828   1.1  jmcneill 
   4829  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   4830  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4831   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4832   1.1  jmcneill 		printf("%s: unable to allocate create rq mailboxen\n",
   4833   1.1  jmcneill 		    DEVNAME(sc));
   4834  1.15  jmcneill 		goto free_rq;
   4835  1.15  jmcneill 	}
   4836  1.15  jmcneill 	mbin = (struct mcx_rq_ctx *)
   4837  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   4838  1.15  jmcneill 	rq_flags = MCX_RQ_CTX_RLKEY;
   4839  1.15  jmcneill 	mbin->rq_flags = htobe32(rq_flags);
   4840   1.1  jmcneill 	mbin->rq_cqn = htobe32(cqn);
   4841   1.1  jmcneill 	mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   4842   1.1  jmcneill 	mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
   4843   1.1  jmcneill 	mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   4844  1.15  jmcneill 	    rx->rx_doorbell);
   4845   1.1  jmcneill 	mbin->rq_wq.wq_log_stride = htobe16(4);
   4846   1.1  jmcneill 	mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
   4847   1.1  jmcneill 
   4848  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   4849  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
   4850  1.15  jmcneill 
   4851   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4852  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem);
   4853   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4854   1.1  jmcneill 
   4855   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4856   1.1  jmcneill 	if (error != 0) {
   4857   1.1  jmcneill 		printf("%s: create rq timeout\n", DEVNAME(sc));
   4858  1.15  jmcneill 		goto free_mxm;
   4859   1.1  jmcneill 	}
   4860   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4861   1.1  jmcneill 		printf("%s: create rq command corrupt\n", DEVNAME(sc));
   4862  1.15  jmcneill 		goto free_mxm;
   4863   1.1  jmcneill 	}
   4864   1.1  jmcneill 
   4865   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4866   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4867   1.1  jmcneill 		printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
   4868   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4869  1.15  jmcneill 		goto free_mxm;
   4870   1.1  jmcneill 	}
   4871   1.1  jmcneill 
   4872  1.15  jmcneill 	rx->rx_rqn = mcx_get_id(out->cmd_rqn);
   4873   1.1  jmcneill 
   4874  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4875  1.15  jmcneill 
   4876  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4877  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   4878  1.15  jmcneill 
   4879  1.15  jmcneill 	return (0);
   4880   1.1  jmcneill 
   4881  1.15  jmcneill free_mxm:
   4882  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   4883  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   4884   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4885  1.15  jmcneill free_rq:
   4886  1.15  jmcneill 	mcx_dmamem_free(sc, &rx->rx_rq_mem);
   4887  1.15  jmcneill 	return (-1);
   4888   1.1  jmcneill }
   4889   1.1  jmcneill 
   4890   1.1  jmcneill static int
   4891  1.15  jmcneill mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx)
   4892   1.1  jmcneill {
   4893   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4894   1.1  jmcneill 	struct mcx_dmamem mxm;
   4895   1.1  jmcneill 	struct mcx_cmd_modify_rq_in *in;
   4896   1.1  jmcneill 	struct mcx_cmd_modify_rq_mb_in *mbin;
   4897   1.1  jmcneill 	struct mcx_cmd_modify_rq_out *out;
   4898   1.1  jmcneill 	int error;
   4899   1.1  jmcneill 	int token;
   4900   1.1  jmcneill 
   4901   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4902   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4903  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   4904  1.15  jmcneill 	    sizeof(*out), token);
   4905   1.1  jmcneill 
   4906   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4907   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
   4908   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4909  1.15  jmcneill 	in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn);
   4910   1.1  jmcneill 
   4911  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   4912  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4913  1.15  jmcneill 		printf("%s: unable to allocate modify rq mailbox\n",
   4914  1.15  jmcneill 		    DEVNAME(sc));
   4915   1.1  jmcneill 		return (-1);
   4916   1.1  jmcneill 	}
   4917   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4918   1.1  jmcneill 	mbin->cmd_rq_ctx.rq_flags = htobe32(
   4919   1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
   4920   1.1  jmcneill 
   4921   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4922   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4923   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4924   1.1  jmcneill 	if (error != 0) {
   4925   1.1  jmcneill 		printf("%s: modify rq timeout\n", DEVNAME(sc));
   4926   1.1  jmcneill 		goto free;
   4927   1.1  jmcneill 	}
   4928   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4929   1.1  jmcneill 		printf("%s: modify rq command corrupt\n", DEVNAME(sc));
   4930   1.1  jmcneill 		goto free;
   4931   1.1  jmcneill 	}
   4932   1.1  jmcneill 
   4933   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4934   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4935   1.1  jmcneill 		printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
   4936   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4937   1.1  jmcneill 		error = -1;
   4938   1.1  jmcneill 		goto free;
   4939   1.1  jmcneill 	}
   4940   1.1  jmcneill 
   4941   1.1  jmcneill free:
   4942   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4943   1.1  jmcneill 	return (error);
   4944   1.1  jmcneill }
   4945   1.1  jmcneill 
   4946   1.1  jmcneill static int
   4947  1.15  jmcneill mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx)
   4948   1.1  jmcneill {
   4949   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4950   1.1  jmcneill 	struct mcx_cmd_destroy_rq_in *in;
   4951   1.1  jmcneill 	struct mcx_cmd_destroy_rq_out *out;
   4952   1.1  jmcneill 	int error;
   4953   1.1  jmcneill 	int token;
   4954   1.1  jmcneill 
   4955   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4956   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4957   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4958   1.1  jmcneill 
   4959   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4960   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
   4961   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4962  1.15  jmcneill 	in->cmd_rqn = htobe32(rx->rx_rqn);
   4963   1.1  jmcneill 
   4964   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4965   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4966   1.1  jmcneill 	if (error != 0) {
   4967   1.1  jmcneill 		printf("%s: destroy rq timeout\n", DEVNAME(sc));
   4968   1.1  jmcneill 		return error;
   4969   1.1  jmcneill 	}
   4970   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4971   1.1  jmcneill 		printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
   4972   1.1  jmcneill 		return error;
   4973   1.1  jmcneill 	}
   4974   1.1  jmcneill 
   4975   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4976   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4977   1.1  jmcneill 		printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
   4978   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4979   1.1  jmcneill 		return -1;
   4980   1.1  jmcneill 	}
   4981   1.1  jmcneill 
   4982  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   4983  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   4984  1.15  jmcneill 
   4985  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   4986  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   4987  1.15  jmcneill 	mcx_dmamem_free(sc, &rx->rx_rq_mem);
   4988  1.15  jmcneill 
   4989  1.15  jmcneill 	rx->rx_rqn = 0;
   4990   1.1  jmcneill 	return 0;
   4991   1.1  jmcneill }
   4992   1.1  jmcneill 
   4993   1.1  jmcneill static int
   4994  1.15  jmcneill mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn)
   4995   1.1  jmcneill {
   4996   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4997   1.1  jmcneill 	struct mcx_dmamem mxm;
   4998   1.1  jmcneill 	struct mcx_cmd_create_tir_in *in;
   4999   1.1  jmcneill 	struct mcx_cmd_create_tir_mb_in *mbin;
   5000   1.1  jmcneill 	struct mcx_cmd_create_tir_out *out;
   5001   1.1  jmcneill 	int error;
   5002   1.1  jmcneill 	int token;
   5003   1.1  jmcneill 
   5004   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5005   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5006  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5007  1.15  jmcneill 	    sizeof(*out), token);
   5008   1.1  jmcneill 
   5009   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5010   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
   5011   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5012   1.1  jmcneill 
   5013  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5014  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5015   1.1  jmcneill 		printf("%s: unable to allocate create tir mailbox\n",
   5016   1.1  jmcneill 		    DEVNAME(sc));
   5017   1.1  jmcneill 		return (-1);
   5018   1.1  jmcneill 	}
   5019   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5020   1.1  jmcneill 	/* leave disp_type = 0, so packets get sent to the inline rqn */
   5021  1.15  jmcneill 	mbin->cmd_inline_rqn = htobe32(rx->rx_rqn);
   5022   1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   5023   1.1  jmcneill 
   5024   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5025   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5026   1.1  jmcneill 	if (error != 0) {
   5027   1.1  jmcneill 		printf("%s: create tir timeout\n", DEVNAME(sc));
   5028   1.1  jmcneill 		goto free;
   5029   1.1  jmcneill 	}
   5030   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5031   1.1  jmcneill 		printf("%s: create tir command corrupt\n", DEVNAME(sc));
   5032   1.1  jmcneill 		goto free;
   5033   1.1  jmcneill 	}
   5034   1.1  jmcneill 
   5035   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5036   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5037   1.1  jmcneill 		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
   5038   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5039   1.1  jmcneill 		error = -1;
   5040   1.1  jmcneill 		goto free;
   5041   1.1  jmcneill 	}
   5042   1.1  jmcneill 
   5043  1.15  jmcneill 	*tirn = mcx_get_id(out->cmd_tirn);
   5044  1.15  jmcneill free:
   5045  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5046  1.15  jmcneill 	return (error);
   5047  1.15  jmcneill }
   5048  1.15  jmcneill 
   5049  1.15  jmcneill static int
   5050  1.15  jmcneill mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel,
   5051  1.15  jmcneill     int *tirn)
   5052  1.15  jmcneill {
   5053  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5054  1.15  jmcneill 	struct mcx_dmamem mxm;
   5055  1.15  jmcneill 	struct mcx_cmd_create_tir_in *in;
   5056  1.15  jmcneill 	struct mcx_cmd_create_tir_mb_in *mbin;
   5057  1.15  jmcneill 	struct mcx_cmd_create_tir_out *out;
   5058  1.15  jmcneill 	int error;
   5059  1.15  jmcneill 	int token;
   5060  1.15  jmcneill 
   5061  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5062  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5063  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5064  1.15  jmcneill 	    sizeof(*out), token);
   5065  1.15  jmcneill 
   5066  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5067  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
   5068  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5069  1.15  jmcneill 
   5070  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5071  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5072  1.15  jmcneill 		printf("%s: unable to allocate create tir mailbox\n",
   5073  1.15  jmcneill 		    DEVNAME(sc));
   5074  1.15  jmcneill 		return (-1);
   5075  1.15  jmcneill 	}
   5076  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5077  1.15  jmcneill 	mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT
   5078  1.15  jmcneill 	    << MCX_TIR_CTX_DISP_TYPE_SHIFT);
   5079  1.15  jmcneill 	mbin->cmd_indir_table = htobe32(rqtn);
   5080  1.15  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain |
   5081  1.15  jmcneill 	    MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT);
   5082  1.15  jmcneill 	mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel);
   5083  1.15  jmcneill 	stoeplitz_to_key(&mbin->cmd_rx_hash_key,
   5084  1.15  jmcneill 	    sizeof(mbin->cmd_rx_hash_key));
   5085  1.15  jmcneill 
   5086  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5087  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5088  1.15  jmcneill 	if (error != 0) {
   5089  1.15  jmcneill 		printf("%s: create tir timeout\n", DEVNAME(sc));
   5090  1.15  jmcneill 		goto free;
   5091  1.15  jmcneill 	}
   5092  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5093  1.15  jmcneill 		printf("%s: create tir command corrupt\n", DEVNAME(sc));
   5094  1.15  jmcneill 		goto free;
   5095  1.15  jmcneill 	}
   5096  1.15  jmcneill 
   5097  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5098  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5099  1.15  jmcneill 		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
   5100  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5101  1.15  jmcneill 		error = -1;
   5102  1.15  jmcneill 		goto free;
   5103  1.15  jmcneill 	}
   5104  1.15  jmcneill 
   5105  1.15  jmcneill 	*tirn = mcx_get_id(out->cmd_tirn);
   5106   1.1  jmcneill free:
   5107   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5108   1.1  jmcneill 	return (error);
   5109   1.1  jmcneill }
   5110   1.1  jmcneill 
   5111   1.1  jmcneill static int
   5112  1.15  jmcneill mcx_destroy_tir(struct mcx_softc *sc, int tirn)
   5113   1.1  jmcneill {
   5114   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5115   1.1  jmcneill 	struct mcx_cmd_destroy_tir_in *in;
   5116   1.1  jmcneill 	struct mcx_cmd_destroy_tir_out *out;
   5117   1.1  jmcneill 	int error;
   5118   1.1  jmcneill 	int token;
   5119   1.1  jmcneill 
   5120   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5121   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5122   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5123   1.1  jmcneill 
   5124   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5125   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
   5126   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5127  1.15  jmcneill 	in->cmd_tirn = htobe32(tirn);
   5128   1.1  jmcneill 
   5129   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5130   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5131   1.1  jmcneill 	if (error != 0) {
   5132   1.1  jmcneill 		printf("%s: destroy tir timeout\n", DEVNAME(sc));
   5133   1.1  jmcneill 		return error;
   5134   1.1  jmcneill 	}
   5135   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5136   1.1  jmcneill 		printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
   5137   1.1  jmcneill 		return error;
   5138   1.1  jmcneill 	}
   5139   1.1  jmcneill 
   5140   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5141   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5142   1.1  jmcneill 		printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
   5143   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5144   1.1  jmcneill 		return -1;
   5145   1.1  jmcneill 	}
   5146   1.1  jmcneill 
   5147  1.15  jmcneill 	return (0);
   5148   1.1  jmcneill }
   5149   1.1  jmcneill 
   5150   1.1  jmcneill static int
   5151  1.15  jmcneill mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db,
   5152  1.15  jmcneill     int cqn)
   5153   1.1  jmcneill {
   5154   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5155   1.1  jmcneill 	struct mcx_dmamem mxm;
   5156   1.1  jmcneill 	struct mcx_cmd_create_sq_in *in;
   5157   1.1  jmcneill 	struct mcx_sq_ctx *mbin;
   5158   1.1  jmcneill 	struct mcx_cmd_create_sq_out *out;
   5159   1.1  jmcneill 	int error;
   5160   1.1  jmcneill 	uint64_t *pas;
   5161  1.13  jmcneill 	int insize, npages, paslen, token;
   5162   1.1  jmcneill 
   5163  1.15  jmcneill 	tx->tx_doorbell = MCX_WQ_DOORBELL_BASE +
   5164  1.15  jmcneill 	    (db * MCX_WQ_DOORBELL_STRIDE) + 4;
   5165  1.15  jmcneill 
   5166   1.1  jmcneill 	npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
   5167   1.1  jmcneill 	    MCX_PAGE_SIZE);
   5168   1.1  jmcneill 	paslen = npages * sizeof(*pas);
   5169   1.1  jmcneill 	insize = sizeof(struct mcx_sq_ctx) + paslen;
   5170   1.1  jmcneill 
   5171  1.15  jmcneill 	if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE,
   5172   1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   5173  1.15  jmcneill 		printf("%s: unable to allocate send queue memory\n",
   5174  1.15  jmcneill 		    DEVNAME(sc));
   5175   1.1  jmcneill 		return (-1);
   5176   1.1  jmcneill 	}
   5177   1.1  jmcneill 
   5178   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5179   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5180   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
   5181   1.1  jmcneill 	    token);
   5182   1.1  jmcneill 
   5183   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5184   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
   5185   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5186   1.1  jmcneill 
   5187  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm,
   5188  1.15  jmcneill 	    howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   5189   1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5190  1.15  jmcneill 		printf("%s: unable to allocate create sq mailboxen\n",
   5191  1.15  jmcneill 		    DEVNAME(sc));
   5192  1.15  jmcneill 		goto free_sq;
   5193   1.1  jmcneill 	}
   5194  1.15  jmcneill 	mbin = (struct mcx_sq_ctx *)
   5195  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   5196   1.1  jmcneill 	mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
   5197   1.1  jmcneill 	    (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
   5198   1.1  jmcneill 	mbin->sq_cqn = htobe32(cqn);
   5199   1.1  jmcneill 	mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
   5200  1.15  jmcneill 	mbin->sq_tis_num = htobe32(sc->sc_tis);
   5201   1.1  jmcneill 	mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   5202   1.1  jmcneill 	mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
   5203  1.15  jmcneill 	mbin->sq_wq.wq_uar_page = htobe32(uar);
   5204   1.1  jmcneill 	mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   5205  1.15  jmcneill 	    tx->tx_doorbell);
   5206   1.1  jmcneill 	mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
   5207   1.1  jmcneill 	mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
   5208   1.1  jmcneill 
   5209  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5210  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
   5211  1.15  jmcneill 
   5212   1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   5213  1.15  jmcneill 	mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10,
   5214  1.15  jmcneill 	    npages, &tx->tx_sq_mem);
   5215   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5216   1.1  jmcneill 
   5217   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5218   1.1  jmcneill 	if (error != 0) {
   5219   1.1  jmcneill 		printf("%s: create sq timeout\n", DEVNAME(sc));
   5220  1.15  jmcneill 		goto free_mxm;
   5221   1.1  jmcneill 	}
   5222   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5223   1.1  jmcneill 		printf("%s: create sq command corrupt\n", DEVNAME(sc));
   5224  1.15  jmcneill 		goto free_mxm;
   5225   1.1  jmcneill 	}
   5226   1.1  jmcneill 
   5227   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5228   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5229   1.1  jmcneill 		printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
   5230   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5231  1.15  jmcneill 		goto free_mxm;
   5232   1.1  jmcneill 	}
   5233   1.1  jmcneill 
   5234  1.15  jmcneill 	tx->tx_uar = uar;
   5235  1.15  jmcneill 	tx->tx_sqn = mcx_get_id(out->cmd_sqn);
   5236  1.15  jmcneill 
   5237  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5238  1.15  jmcneill 
   5239  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   5240  1.15  jmcneill 	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   5241  1.15  jmcneill 
   5242  1.15  jmcneill 	return (0);
   5243   1.1  jmcneill 
   5244  1.15  jmcneill free_mxm:
   5245  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5246  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   5247   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5248  1.15  jmcneill free_sq:
   5249  1.15  jmcneill 	mcx_dmamem_free(sc, &tx->tx_sq_mem);
   5250  1.15  jmcneill 	return (-1);
   5251   1.1  jmcneill }
   5252   1.1  jmcneill 
   5253   1.1  jmcneill static int
   5254  1.15  jmcneill mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx)
   5255   1.1  jmcneill {
   5256   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5257   1.1  jmcneill 	struct mcx_cmd_destroy_sq_in *in;
   5258   1.1  jmcneill 	struct mcx_cmd_destroy_sq_out *out;
   5259   1.1  jmcneill 	int error;
   5260   1.1  jmcneill 	int token;
   5261   1.1  jmcneill 
   5262   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5263   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5264   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5265   1.1  jmcneill 
   5266   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5267   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
   5268   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5269  1.15  jmcneill 	in->cmd_sqn = htobe32(tx->tx_sqn);
   5270   1.1  jmcneill 
   5271   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5272   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5273   1.1  jmcneill 	if (error != 0) {
   5274   1.1  jmcneill 		printf("%s: destroy sq timeout\n", DEVNAME(sc));
   5275   1.1  jmcneill 		return error;
   5276   1.1  jmcneill 	}
   5277   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5278   1.1  jmcneill 		printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
   5279   1.1  jmcneill 		return error;
   5280   1.1  jmcneill 	}
   5281   1.1  jmcneill 
   5282   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5283   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5284   1.1  jmcneill 		printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
   5285   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5286   1.1  jmcneill 		return -1;
   5287   1.1  jmcneill 	}
   5288   1.1  jmcneill 
   5289  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   5290  1.15  jmcneill 	    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   5291  1.15  jmcneill 
   5292  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   5293  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   5294  1.15  jmcneill 	mcx_dmamem_free(sc, &tx->tx_sq_mem);
   5295  1.15  jmcneill 
   5296  1.15  jmcneill 	tx->tx_sqn = 0;
   5297   1.1  jmcneill 	return 0;
   5298   1.1  jmcneill }
   5299   1.1  jmcneill 
   5300   1.1  jmcneill static int
   5301  1.15  jmcneill mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx)
   5302   1.1  jmcneill {
   5303   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5304   1.1  jmcneill 	struct mcx_dmamem mxm;
   5305   1.1  jmcneill 	struct mcx_cmd_modify_sq_in *in;
   5306   1.1  jmcneill 	struct mcx_cmd_modify_sq_mb_in *mbin;
   5307   1.1  jmcneill 	struct mcx_cmd_modify_sq_out *out;
   5308   1.1  jmcneill 	int error;
   5309   1.1  jmcneill 	int token;
   5310   1.1  jmcneill 
   5311   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5312   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5313  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5314  1.15  jmcneill 	    sizeof(*out), token);
   5315   1.1  jmcneill 
   5316   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5317   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
   5318   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5319  1.15  jmcneill 	in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn);
   5320   1.1  jmcneill 
   5321  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5322  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5323   1.1  jmcneill 		printf("%s: unable to allocate modify sq mailbox\n",
   5324   1.1  jmcneill 		    DEVNAME(sc));
   5325   1.1  jmcneill 		return (-1);
   5326   1.1  jmcneill 	}
   5327   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5328   1.1  jmcneill 	mbin->cmd_sq_ctx.sq_flags = htobe32(
   5329   1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
   5330   1.1  jmcneill 
   5331   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5332   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5333   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5334   1.1  jmcneill 	if (error != 0) {
   5335   1.1  jmcneill 		printf("%s: modify sq timeout\n", DEVNAME(sc));
   5336   1.1  jmcneill 		goto free;
   5337   1.1  jmcneill 	}
   5338   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5339   1.1  jmcneill 		printf("%s: modify sq command corrupt\n", DEVNAME(sc));
   5340   1.1  jmcneill 		goto free;
   5341   1.1  jmcneill 	}
   5342   1.1  jmcneill 
   5343   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5344   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5345   1.1  jmcneill 		printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
   5346   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5347   1.1  jmcneill 		error = -1;
   5348   1.1  jmcneill 		goto free;
   5349   1.1  jmcneill 	}
   5350   1.1  jmcneill 
   5351   1.1  jmcneill free:
   5352   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5353   1.1  jmcneill 	return (error);
   5354   1.1  jmcneill }
   5355   1.1  jmcneill 
   5356   1.1  jmcneill static int
   5357  1.15  jmcneill mcx_create_tis(struct mcx_softc *sc, int *tis)
   5358   1.1  jmcneill {
   5359   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5360   1.1  jmcneill 	struct mcx_dmamem mxm;
   5361   1.1  jmcneill 	struct mcx_cmd_create_tis_in *in;
   5362   1.1  jmcneill 	struct mcx_cmd_create_tis_mb_in *mbin;
   5363   1.1  jmcneill 	struct mcx_cmd_create_tis_out *out;
   5364   1.1  jmcneill 	int error;
   5365   1.1  jmcneill 	int token;
   5366   1.1  jmcneill 
   5367   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5368   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5369  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5370  1.15  jmcneill 	    sizeof(*out), token);
   5371   1.1  jmcneill 
   5372   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5373   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
   5374   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5375   1.1  jmcneill 
   5376  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5377  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5378  1.15  jmcneill 		printf("%s: unable to allocate create tis mailbox\n",
   5379  1.15  jmcneill 		    DEVNAME(sc));
   5380   1.1  jmcneill 		return (-1);
   5381   1.1  jmcneill 	}
   5382   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5383   1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   5384   1.1  jmcneill 
   5385   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5386   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5387   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5388   1.1  jmcneill 	if (error != 0) {
   5389   1.1  jmcneill 		printf("%s: create tis timeout\n", DEVNAME(sc));
   5390   1.1  jmcneill 		goto free;
   5391   1.1  jmcneill 	}
   5392   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5393   1.1  jmcneill 		printf("%s: create tis command corrupt\n", DEVNAME(sc));
   5394   1.1  jmcneill 		goto free;
   5395   1.1  jmcneill 	}
   5396   1.1  jmcneill 
   5397   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5398   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5399   1.1  jmcneill 		printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
   5400   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5401   1.1  jmcneill 		error = -1;
   5402   1.1  jmcneill 		goto free;
   5403   1.1  jmcneill 	}
   5404   1.1  jmcneill 
   5405  1.15  jmcneill 	*tis = mcx_get_id(out->cmd_tisn);
   5406   1.1  jmcneill free:
   5407   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5408   1.1  jmcneill 	return (error);
   5409   1.1  jmcneill }
   5410   1.1  jmcneill 
   5411   1.1  jmcneill static int
   5412  1.15  jmcneill mcx_destroy_tis(struct mcx_softc *sc, int tis)
   5413   1.1  jmcneill {
   5414   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5415   1.1  jmcneill 	struct mcx_cmd_destroy_tis_in *in;
   5416   1.1  jmcneill 	struct mcx_cmd_destroy_tis_out *out;
   5417   1.1  jmcneill 	int error;
   5418   1.1  jmcneill 	int token;
   5419   1.1  jmcneill 
   5420   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5421   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5422   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5423   1.1  jmcneill 
   5424   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5425   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
   5426   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5427  1.15  jmcneill 	in->cmd_tisn = htobe32(tis);
   5428   1.1  jmcneill 
   5429   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5430   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5431   1.1  jmcneill 	if (error != 0) {
   5432   1.1  jmcneill 		printf("%s: destroy tis timeout\n", DEVNAME(sc));
   5433   1.1  jmcneill 		return error;
   5434   1.1  jmcneill 	}
   5435   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5436   1.1  jmcneill 		printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
   5437   1.1  jmcneill 		return error;
   5438   1.1  jmcneill 	}
   5439   1.1  jmcneill 
   5440   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5441   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5442   1.1  jmcneill 		printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
   5443   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5444   1.1  jmcneill 		return -1;
   5445   1.1  jmcneill 	}
   5446   1.1  jmcneill 
   5447  1.15  jmcneill 	return 0;
   5448  1.15  jmcneill }
   5449  1.15  jmcneill 
   5450  1.15  jmcneill static int
   5451  1.15  jmcneill mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt)
   5452  1.15  jmcneill {
   5453  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5454  1.15  jmcneill 	struct mcx_dmamem mxm;
   5455  1.15  jmcneill 	struct mcx_cmd_create_rqt_in *in;
   5456  1.15  jmcneill 	struct mcx_cmd_create_rqt_mb_in *mbin;
   5457  1.15  jmcneill 	struct mcx_cmd_create_rqt_out *out;
   5458  1.15  jmcneill 	struct mcx_rqt_ctx *rqt_ctx;
   5459  1.15  jmcneill 	int *rqtn;
   5460  1.15  jmcneill 	int error;
   5461  1.15  jmcneill 	int token;
   5462  1.15  jmcneill 	int i;
   5463  1.15  jmcneill 
   5464  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5465  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5466  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) +
   5467  1.15  jmcneill 	    (size * sizeof(int)), sizeof(*out), token);
   5468  1.15  jmcneill 
   5469  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5470  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT);
   5471  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5472  1.15  jmcneill 
   5473  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5474  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5475  1.15  jmcneill 		printf("%s: unable to allocate create rqt mailbox\n",
   5476  1.15  jmcneill 		    DEVNAME(sc));
   5477  1.15  jmcneill 		return (-1);
   5478  1.15  jmcneill 	}
   5479  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5480  1.15  jmcneill 	rqt_ctx = &mbin->cmd_rqt;
   5481  1.15  jmcneill 	rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size);
   5482  1.15  jmcneill 	rqt_ctx->cmd_rqt_actual_size = htobe16(size);
   5483  1.15  jmcneill 
   5484  1.15  jmcneill 	/* rqt list follows the rqt context */
   5485  1.15  jmcneill 	rqtn = (int *)(rqt_ctx + 1);
   5486  1.15  jmcneill 	for (i = 0; i < size; i++) {
   5487  1.15  jmcneill 		rqtn[i] = htobe32(rqns[i]);
   5488  1.15  jmcneill 	}
   5489  1.15  jmcneill 
   5490  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5491  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5492  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5493  1.15  jmcneill 	if (error != 0) {
   5494  1.15  jmcneill 		printf("%s: create rqt timeout\n", DEVNAME(sc));
   5495  1.15  jmcneill 		goto free;
   5496  1.15  jmcneill 	}
   5497  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5498  1.15  jmcneill 		printf("%s: create rqt command corrupt\n", DEVNAME(sc));
   5499  1.15  jmcneill 		goto free;
   5500  1.15  jmcneill 	}
   5501  1.15  jmcneill 
   5502  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5503  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5504  1.15  jmcneill 		printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc),
   5505  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5506  1.15  jmcneill 		error = -1;
   5507  1.15  jmcneill 		goto free;
   5508  1.15  jmcneill 	}
   5509  1.15  jmcneill 
   5510  1.15  jmcneill 	*rqt = mcx_get_id(out->cmd_rqtn);
   5511  1.15  jmcneill 	return (0);
   5512  1.15  jmcneill free:
   5513  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5514  1.15  jmcneill 	return (error);
   5515  1.15  jmcneill }
   5516  1.15  jmcneill 
   5517  1.15  jmcneill static int
   5518  1.15  jmcneill mcx_destroy_rqt(struct mcx_softc *sc, int rqt)
   5519  1.15  jmcneill {
   5520  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   5521  1.15  jmcneill 	struct mcx_cmd_destroy_rqt_in *in;
   5522  1.15  jmcneill 	struct mcx_cmd_destroy_rqt_out *out;
   5523  1.15  jmcneill 	int error;
   5524  1.15  jmcneill 	int token;
   5525  1.15  jmcneill 
   5526  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5527  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   5528  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   5529  1.15  jmcneill 
   5530  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   5531  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT);
   5532  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   5533  1.15  jmcneill 	in->cmd_rqtn = htobe32(rqt);
   5534  1.15  jmcneill 
   5535  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5536  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5537  1.15  jmcneill 	if (error != 0) {
   5538  1.15  jmcneill 		printf("%s: destroy rqt timeout\n", DEVNAME(sc));
   5539  1.15  jmcneill 		return error;
   5540  1.15  jmcneill 	}
   5541  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5542  1.15  jmcneill 		printf("%s: destroy rqt command corrupt\n", DEVNAME(sc));
   5543  1.15  jmcneill 		return error;
   5544  1.15  jmcneill 	}
   5545  1.15  jmcneill 
   5546  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   5547  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5548  1.15  jmcneill 		printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc),
   5549  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5550  1.15  jmcneill 		return -1;
   5551  1.15  jmcneill 	}
   5552  1.15  jmcneill 
   5553   1.1  jmcneill 	return 0;
   5554   1.1  jmcneill }
   5555   1.1  jmcneill 
   5556   1.1  jmcneill #if 0
   5557   1.1  jmcneill static int
   5558   1.1  jmcneill mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
   5559   1.1  jmcneill {
   5560   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5561   1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_in *in;
   5562   1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_out *out;
   5563   1.1  jmcneill 	int error;
   5564   1.1  jmcneill 
   5565   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5566   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   5567   1.1  jmcneill 
   5568   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5569   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
   5570   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5571   1.1  jmcneill 
   5572   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5573   1.1  jmcneill 
   5574   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5575   1.1  jmcneill 	if (error != 0) {
   5576   1.1  jmcneill 		printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
   5577   1.1  jmcneill 		return (-1);
   5578   1.1  jmcneill 	}
   5579   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5580   1.1  jmcneill 		printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
   5581   1.1  jmcneill 		return (-1);
   5582   1.1  jmcneill 	}
   5583   1.1  jmcneill 
   5584   1.1  jmcneill 	out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
   5585   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5586   1.1  jmcneill 		printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
   5587   1.1  jmcneill 		    out->cmd_status);
   5588   1.1  jmcneill 		return (-1);
   5589   1.1  jmcneill 	}
   5590   1.1  jmcneill 
   5591   1.1  jmcneill 	sc->sc_flow_counter_id[i]  = be16toh(out->cmd_flow_counter_id);
   5592   1.1  jmcneill 	printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
   5593   1.1  jmcneill 
   5594   1.1  jmcneill 	return (0);
   5595   1.1  jmcneill }
   5596   1.1  jmcneill #endif
   5597   1.1  jmcneill 
   5598   1.1  jmcneill static int
   5599  1.15  jmcneill mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level,
   5600  1.15  jmcneill     int *flow_table_id)
   5601   1.1  jmcneill {
   5602   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5603   1.1  jmcneill 	struct mcx_dmamem mxm;
   5604   1.1  jmcneill 	struct mcx_cmd_create_flow_table_in *in;
   5605   1.1  jmcneill 	struct mcx_cmd_create_flow_table_mb_in *mbin;
   5606   1.1  jmcneill 	struct mcx_cmd_create_flow_table_out *out;
   5607   1.1  jmcneill 	int error;
   5608   1.1  jmcneill 	int token;
   5609   1.1  jmcneill 
   5610   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5611   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5612  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5613  1.15  jmcneill 	    sizeof(*out), token);
   5614   1.1  jmcneill 
   5615   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5616   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
   5617   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5618   1.1  jmcneill 
   5619  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5620  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5621   1.1  jmcneill 		printf("%s: unable to allocate create flow table mailbox\n",
   5622   1.1  jmcneill 		    DEVNAME(sc));
   5623   1.1  jmcneill 		return (-1);
   5624   1.1  jmcneill 	}
   5625   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5626   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5627   1.1  jmcneill 	mbin->cmd_ctx.ft_log_size = log_size;
   5628  1.15  jmcneill 	mbin->cmd_ctx.ft_level = level;
   5629   1.1  jmcneill 
   5630   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5631   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5632   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5633   1.1  jmcneill 	if (error != 0) {
   5634   1.1  jmcneill 		printf("%s: create flow table timeout\n", DEVNAME(sc));
   5635   1.1  jmcneill 		goto free;
   5636   1.1  jmcneill 	}
   5637   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5638   1.1  jmcneill 		printf("%s: create flow table command corrupt\n", DEVNAME(sc));
   5639   1.1  jmcneill 		goto free;
   5640   1.1  jmcneill 	}
   5641   1.1  jmcneill 
   5642   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5643   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5644   1.1  jmcneill 		printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
   5645   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5646   1.1  jmcneill 		error = -1;
   5647   1.1  jmcneill 		goto free;
   5648   1.1  jmcneill 	}
   5649   1.1  jmcneill 
   5650  1.15  jmcneill 	*flow_table_id = mcx_get_id(out->cmd_table_id);
   5651   1.1  jmcneill free:
   5652   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5653   1.1  jmcneill 	return (error);
   5654   1.1  jmcneill }
   5655   1.1  jmcneill 
   5656   1.1  jmcneill static int
   5657  1.15  jmcneill mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id)
   5658   1.1  jmcneill {
   5659   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5660   1.1  jmcneill 	struct mcx_dmamem mxm;
   5661   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_in *in;
   5662   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_mb_in *mbin;
   5663   1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_out *out;
   5664   1.1  jmcneill 	int error;
   5665   1.1  jmcneill 	int token;
   5666   1.1  jmcneill 
   5667   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5668   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5669  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5670  1.15  jmcneill 	    sizeof(*out), token);
   5671   1.1  jmcneill 
   5672   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5673   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
   5674   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5675   1.1  jmcneill 
   5676  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5677  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5678   1.1  jmcneill 		printf("%s: unable to allocate set flow table root mailbox\n",
   5679   1.1  jmcneill 		    DEVNAME(sc));
   5680   1.1  jmcneill 		return (-1);
   5681   1.1  jmcneill 	}
   5682   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5683   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5684  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   5685   1.1  jmcneill 
   5686   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5687   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5688   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5689   1.1  jmcneill 	if (error != 0) {
   5690   1.1  jmcneill 		printf("%s: set flow table root timeout\n", DEVNAME(sc));
   5691   1.1  jmcneill 		goto free;
   5692   1.1  jmcneill 	}
   5693   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5694   1.1  jmcneill 		printf("%s: set flow table root command corrupt\n",
   5695   1.1  jmcneill 		    DEVNAME(sc));
   5696   1.1  jmcneill 		goto free;
   5697   1.1  jmcneill 	}
   5698   1.1  jmcneill 
   5699   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5700   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5701   1.1  jmcneill 		printf("%s: set flow table root failed (%x, %x)\n",
   5702   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5703   1.1  jmcneill 		error = -1;
   5704   1.1  jmcneill 		goto free;
   5705   1.1  jmcneill 	}
   5706   1.1  jmcneill 
   5707   1.1  jmcneill free:
   5708   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5709   1.1  jmcneill 	return (error);
   5710   1.1  jmcneill }
   5711   1.1  jmcneill 
   5712   1.1  jmcneill static int
   5713  1.15  jmcneill mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id)
   5714   1.1  jmcneill {
   5715   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5716   1.1  jmcneill 	struct mcx_dmamem mxm;
   5717   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_in *in;
   5718   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_mb_in *mb;
   5719   1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_out *out;
   5720   1.1  jmcneill 	int error;
   5721   1.1  jmcneill 	int token;
   5722   1.1  jmcneill 
   5723   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5724   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5725   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   5726   1.1  jmcneill 
   5727   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5728   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
   5729   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5730   1.1  jmcneill 
   5731  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   5732  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5733   1.1  jmcneill 		printf("%s: unable to allocate destroy flow table mailbox\n",
   5734   1.1  jmcneill 		    DEVNAME(sc));
   5735   1.1  jmcneill 		return (-1);
   5736   1.1  jmcneill 	}
   5737   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5738   1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5739  1.15  jmcneill 	mb->cmd_table_id = htobe32(flow_table_id);
   5740   1.1  jmcneill 
   5741   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5742   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5743   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5744   1.1  jmcneill 	if (error != 0) {
   5745   1.1  jmcneill 		printf("%s: destroy flow table timeout\n", DEVNAME(sc));
   5746   1.1  jmcneill 		goto free;
   5747   1.1  jmcneill 	}
   5748   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5749  1.15  jmcneill 		printf("%s: destroy flow table command corrupt\n",
   5750  1.15  jmcneill 		    DEVNAME(sc));
   5751   1.1  jmcneill 		goto free;
   5752   1.1  jmcneill 	}
   5753   1.1  jmcneill 
   5754   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5755   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5756   1.1  jmcneill 		printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
   5757   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5758   1.1  jmcneill 		error = -1;
   5759   1.1  jmcneill 		goto free;
   5760   1.1  jmcneill 	}
   5761   1.1  jmcneill 
   5762   1.1  jmcneill free:
   5763   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5764   1.1  jmcneill 	return (error);
   5765   1.1  jmcneill }
   5766   1.1  jmcneill 
   5767   1.1  jmcneill 
   5768   1.1  jmcneill static int
   5769  1.15  jmcneill mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group,
   5770  1.15  jmcneill     int start, int size, int match_enable, struct mcx_flow_match *match)
   5771   1.1  jmcneill {
   5772   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5773   1.1  jmcneill 	struct mcx_dmamem mxm;
   5774   1.1  jmcneill 	struct mcx_cmd_create_flow_group_in *in;
   5775   1.1  jmcneill 	struct mcx_cmd_create_flow_group_mb_in *mbin;
   5776   1.1  jmcneill 	struct mcx_cmd_create_flow_group_out *out;
   5777  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5778   1.1  jmcneill 	int error;
   5779   1.1  jmcneill 	int token;
   5780   1.1  jmcneill 
   5781   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5782   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5783   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   5784   1.1  jmcneill 	    token);
   5785   1.1  jmcneill 
   5786   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5787   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
   5788   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5789   1.1  jmcneill 
   5790   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   5791   1.1  jmcneill 	    != 0) {
   5792   1.1  jmcneill 		printf("%s: unable to allocate create flow group mailbox\n",
   5793   1.1  jmcneill 		    DEVNAME(sc));
   5794   1.1  jmcneill 		return (-1);
   5795   1.1  jmcneill 	}
   5796   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5797   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5798  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   5799   1.1  jmcneill 	mbin->cmd_start_flow_index = htobe32(start);
   5800   1.1  jmcneill 	mbin->cmd_end_flow_index = htobe32(start + (size - 1));
   5801   1.1  jmcneill 
   5802   1.1  jmcneill 	mbin->cmd_match_criteria_enable = match_enable;
   5803   1.1  jmcneill 	memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));
   5804   1.1  jmcneill 
   5805   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5806   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5807   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5808   1.1  jmcneill 	if (error != 0) {
   5809   1.1  jmcneill 		printf("%s: create flow group timeout\n", DEVNAME(sc));
   5810   1.1  jmcneill 		goto free;
   5811   1.1  jmcneill 	}
   5812   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5813   1.1  jmcneill 		printf("%s: create flow group command corrupt\n", DEVNAME(sc));
   5814   1.1  jmcneill 		goto free;
   5815   1.1  jmcneill 	}
   5816   1.1  jmcneill 
   5817   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5818   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5819   1.1  jmcneill 		printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
   5820   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5821   1.1  jmcneill 		error = -1;
   5822   1.1  jmcneill 		goto free;
   5823   1.1  jmcneill 	}
   5824   1.1  jmcneill 
   5825  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5826  1.15  jmcneill 	mfg->g_id = mcx_get_id(out->cmd_group_id);
   5827  1.15  jmcneill 	mfg->g_table = flow_table_id;
   5828  1.15  jmcneill 	mfg->g_start = start;
   5829  1.15  jmcneill 	mfg->g_size = size;
   5830   1.1  jmcneill 
   5831   1.1  jmcneill free:
   5832   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5833   1.1  jmcneill 	return (error);
   5834   1.1  jmcneill }
   5835   1.1  jmcneill 
   5836   1.1  jmcneill static int
   5837   1.1  jmcneill mcx_destroy_flow_group(struct mcx_softc *sc, int group)
   5838   1.1  jmcneill {
   5839   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5840   1.1  jmcneill 	struct mcx_dmamem mxm;
   5841   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_in *in;
   5842   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_mb_in *mb;
   5843   1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_out *out;
   5844  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5845   1.1  jmcneill 	int error;
   5846   1.1  jmcneill 	int token;
   5847   1.1  jmcneill 
   5848   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5849   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5850   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   5851   1.1  jmcneill 
   5852   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5853   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
   5854   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5855   1.1  jmcneill 
   5856  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5857  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   5858   1.1  jmcneill 		printf("%s: unable to allocate destroy flow group mailbox\n",
   5859   1.1  jmcneill 		    DEVNAME(sc));
   5860   1.1  jmcneill 		return (-1);
   5861   1.1  jmcneill 	}
   5862   1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5863   1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5864  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5865  1.15  jmcneill 	mb->cmd_table_id = htobe32(mfg->g_table);
   5866  1.15  jmcneill 	mb->cmd_group_id = htobe32(mfg->g_id);
   5867   1.1  jmcneill 
   5868   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5869   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5870   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5871   1.1  jmcneill 	if (error != 0) {
   5872   1.1  jmcneill 		printf("%s: destroy flow group timeout\n", DEVNAME(sc));
   5873   1.1  jmcneill 		goto free;
   5874   1.1  jmcneill 	}
   5875   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5876   1.1  jmcneill 		printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
   5877   1.1  jmcneill 		goto free;
   5878   1.1  jmcneill 	}
   5879   1.1  jmcneill 
   5880   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5881   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5882   1.1  jmcneill 		printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
   5883   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5884   1.1  jmcneill 		error = -1;
   5885   1.1  jmcneill 		goto free;
   5886   1.1  jmcneill 	}
   5887   1.1  jmcneill 
   5888  1.15  jmcneill 	mfg->g_id = -1;
   5889  1.15  jmcneill 	mfg->g_table = -1;
   5890  1.15  jmcneill 	mfg->g_size = 0;
   5891  1.15  jmcneill 	mfg->g_start = 0;
   5892   1.1  jmcneill free:
   5893   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5894   1.1  jmcneill 	return (error);
   5895   1.1  jmcneill }
   5896   1.1  jmcneill 
   5897   1.1  jmcneill static int
   5898  1.15  jmcneill mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index,
   5899  1.15  jmcneill     const uint8_t *macaddr, uint32_t dest)
   5900   1.1  jmcneill {
   5901   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5902   1.1  jmcneill 	struct mcx_dmamem mxm;
   5903   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_in *in;
   5904   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
   5905   1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_out *out;
   5906  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5907  1.15  jmcneill 	uint32_t *pdest;
   5908   1.1  jmcneill 	int error;
   5909   1.1  jmcneill 	int token;
   5910   1.1  jmcneill 
   5911   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5912   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5913  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
   5914   1.1  jmcneill 	    sizeof(*out), token);
   5915   1.1  jmcneill 
   5916   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5917   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
   5918   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5919   1.1  jmcneill 
   5920   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   5921   1.1  jmcneill 	    != 0) {
   5922   1.1  jmcneill 		printf("%s: unable to allocate set flow table entry mailbox\n",
   5923   1.1  jmcneill 		    DEVNAME(sc));
   5924   1.1  jmcneill 		return (-1);
   5925   1.1  jmcneill 	}
   5926  1.15  jmcneill 
   5927   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5928   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5929  1.15  jmcneill 
   5930  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   5931  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   5932  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   5933  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
   5934   1.1  jmcneill 
   5935   1.1  jmcneill 	/* flow context ends at offset 0x330, 0x130 into the second mbox */
   5936  1.15  jmcneill 	pdest = (uint32_t *)
   5937   1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
   5938   1.1  jmcneill 	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
   5939   1.1  jmcneill 	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
   5940  1.15  jmcneill 	*pdest = htobe32(dest);
   5941   1.1  jmcneill 
   5942   1.1  jmcneill 	/* the only thing we match on at the moment is the dest mac address */
   5943   1.1  jmcneill 	if (macaddr != NULL) {
   5944   1.1  jmcneill 		memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
   5945   1.1  jmcneill 		    ETHER_ADDR_LEN);
   5946   1.1  jmcneill 	}
   5947   1.1  jmcneill 
   5948   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5949   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5950   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5951   1.1  jmcneill 	if (error != 0) {
   5952   1.1  jmcneill 		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
   5953   1.1  jmcneill 		goto free;
   5954   1.1  jmcneill 	}
   5955   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5956   1.1  jmcneill 		printf("%s: set flow table entry command corrupt\n",
   5957   1.1  jmcneill 		    DEVNAME(sc));
   5958   1.1  jmcneill 		goto free;
   5959   1.1  jmcneill 	}
   5960   1.1  jmcneill 
   5961   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5962   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5963   1.1  jmcneill 		printf("%s: set flow table entry failed (%x, %x)\n",
   5964   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5965   1.1  jmcneill 		error = -1;
   5966   1.1  jmcneill 		goto free;
   5967   1.1  jmcneill 	}
   5968   1.1  jmcneill 
   5969   1.1  jmcneill free:
   5970   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5971   1.1  jmcneill 	return (error);
   5972   1.1  jmcneill }
   5973   1.1  jmcneill 
   5974   1.1  jmcneill static int
   5975  1.15  jmcneill mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index,
   5976  1.15  jmcneill     int ethertype, int ip_proto, uint32_t dest)
   5977   1.1  jmcneill {
   5978   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5979   1.1  jmcneill 	struct mcx_dmamem mxm;
   5980  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_in *in;
   5981  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
   5982  1.15  jmcneill 	struct mcx_cmd_set_flow_table_entry_out *out;
   5983  1.15  jmcneill 	struct mcx_flow_group *mfg;
   5984  1.15  jmcneill 	uint32_t *pdest;
   5985   1.1  jmcneill 	int error;
   5986   1.1  jmcneill 	int token;
   5987   1.1  jmcneill 
   5988   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5989   1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5990  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
   5991  1.15  jmcneill 	    sizeof(*out), token);
   5992   1.1  jmcneill 
   5993   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5994  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
   5995   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5996   1.1  jmcneill 
   5997  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   5998  1.15  jmcneill 	    != 0) {
   5999  1.15  jmcneill 		printf("%s: unable to allocate set flow table entry mailbox\n",
   6000   1.1  jmcneill 		    DEVNAME(sc));
   6001   1.1  jmcneill 		return (-1);
   6002   1.1  jmcneill 	}
   6003  1.15  jmcneill 
   6004   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6005   1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   6006  1.15  jmcneill 
   6007  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   6008  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   6009  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   6010  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
   6011  1.15  jmcneill 
   6012  1.15  jmcneill 	/* flow context ends at offset 0x330, 0x130 into the second mbox */
   6013  1.15  jmcneill 	pdest = (uint32_t *)
   6014  1.15  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
   6015  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
   6016  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
   6017  1.15  jmcneill 	*pdest = htobe32(dest);
   6018  1.15  jmcneill 
   6019  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype);
   6020  1.15  jmcneill 	mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto;
   6021   1.1  jmcneill 
   6022   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   6023   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6024   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6025   1.1  jmcneill 	if (error != 0) {
   6026  1.15  jmcneill 		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
   6027   1.1  jmcneill 		goto free;
   6028   1.1  jmcneill 	}
   6029   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6030  1.15  jmcneill 		printf("%s: set flow table entry command corrupt\n",
   6031   1.1  jmcneill 		    DEVNAME(sc));
   6032   1.1  jmcneill 		goto free;
   6033   1.1  jmcneill 	}
   6034   1.1  jmcneill 
   6035   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6036   1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6037  1.15  jmcneill 		printf("%s: set flow table entry failed (%x, %x)\n",
   6038  1.15  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   6039   1.1  jmcneill 		error = -1;
   6040   1.1  jmcneill 		goto free;
   6041   1.1  jmcneill 	}
   6042   1.1  jmcneill 
   6043   1.1  jmcneill free:
   6044   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6045   1.1  jmcneill 	return (error);
   6046   1.1  jmcneill }
   6047   1.1  jmcneill 
   6048  1.15  jmcneill static int
   6049  1.15  jmcneill mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
   6050   1.1  jmcneill {
   6051  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6052   1.1  jmcneill 	struct mcx_dmamem mxm;
   6053  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_in *in;
   6054  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
   6055  1.15  jmcneill 	struct mcx_cmd_delete_flow_table_entry_out *out;
   6056  1.15  jmcneill 	struct mcx_flow_group *mfg;
   6057   1.1  jmcneill 	int error;
   6058  1.15  jmcneill 	int token;
   6059  1.15  jmcneill 
   6060  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6061  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6062  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   6063  1.15  jmcneill 	    token);
   6064  1.15  jmcneill 
   6065  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6066  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
   6067  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6068  1.15  jmcneill 
   6069  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6070  1.15  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   6071  1.15  jmcneill 		printf("%s: unable to allocate "
   6072  1.15  jmcneill 		    "delete flow table entry mailbox\n", DEVNAME(sc));
   6073  1.15  jmcneill 		return (-1);
   6074  1.15  jmcneill 	}
   6075  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6076  1.15  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   6077  1.15  jmcneill 
   6078  1.15  jmcneill 	mfg = &sc->sc_flow_group[group];
   6079  1.15  jmcneill 	mbin->cmd_table_id = htobe32(mfg->g_table);
   6080  1.15  jmcneill 	mbin->cmd_flow_index = htobe32(mfg->g_start + index);
   6081  1.15  jmcneill 
   6082  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   6083  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6084  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6085  1.15  jmcneill 	if (error != 0) {
   6086  1.15  jmcneill 		printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
   6087  1.15  jmcneill 		goto free;
   6088  1.15  jmcneill 	}
   6089  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6090  1.15  jmcneill 		printf("%s: delete flow table entry command corrupt\n",
   6091  1.15  jmcneill 		    DEVNAME(sc));
   6092  1.15  jmcneill 		goto free;
   6093  1.15  jmcneill 	}
   6094  1.15  jmcneill 
   6095  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6096  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6097  1.15  jmcneill 		printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
   6098  1.15  jmcneill 		    DEVNAME(sc), group, index, out->cmd_status,
   6099  1.15  jmcneill 		    be32toh(out->cmd_syndrome));
   6100  1.15  jmcneill 		error = -1;
   6101  1.15  jmcneill 		goto free;
   6102  1.15  jmcneill 	}
   6103  1.15  jmcneill 
   6104  1.15  jmcneill free:
   6105  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6106  1.15  jmcneill 	return (error);
   6107  1.15  jmcneill }
   6108  1.15  jmcneill 
   6109  1.15  jmcneill #if 0
   6110  1.15  jmcneill int
   6111  1.15  jmcneill mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id)
   6112  1.15  jmcneill {
   6113  1.15  jmcneill 	struct mcx_dmamem mxm;
   6114  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6115  1.15  jmcneill 	struct mcx_cmd_query_flow_table_in *in;
   6116  1.15  jmcneill 	struct mcx_cmd_query_flow_table_mb_in *mbin;
   6117  1.15  jmcneill 	struct mcx_cmd_query_flow_table_out *out;
   6118  1.15  jmcneill 	struct mcx_cmd_query_flow_table_mb_out *mbout;
   6119  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6120  1.15  jmcneill 	int error;
   6121  1.15  jmcneill 	int i;
   6122  1.15  jmcneill 	uint8_t *dump;
   6123   1.1  jmcneill 
   6124   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6125   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6126   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6127   1.1  jmcneill 
   6128   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6129   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
   6130   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6131   1.1  jmcneill 
   6132   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6133   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6134   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6135   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6136   1.1  jmcneill 		printf(", unable to allocate query flow table mailboxes\n");
   6137   1.1  jmcneill 		return (-1);
   6138   1.1  jmcneill 	}
   6139   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6140   1.1  jmcneill 
   6141   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6142   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6143  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6144   1.1  jmcneill 
   6145   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6146   1.1  jmcneill 
   6147   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6148   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6149   1.1  jmcneill 	if (error != 0) {
   6150   1.1  jmcneill 		printf("%s: query flow table timeout\n", DEVNAME(sc));
   6151   1.1  jmcneill 		goto free;
   6152   1.1  jmcneill 	}
   6153   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6154   1.1  jmcneill 	if (error != 0) {
   6155   1.1  jmcneill 		printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
   6156   1.1  jmcneill 		goto free;
   6157   1.1  jmcneill 	}
   6158   1.1  jmcneill 
   6159   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6160   1.1  jmcneill 	switch (out->cmd_status) {
   6161   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6162   1.1  jmcneill 		break;
   6163   1.1  jmcneill 	default:
   6164   1.1  jmcneill 		printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
   6165   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6166   1.1  jmcneill 		error = -1;
   6167   1.1  jmcneill 		goto free;
   6168   1.1  jmcneill 	}
   6169   1.1  jmcneill 
   6170   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_mb_out *)
   6171   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6172   1.1  jmcneill 	dump = (uint8_t *)mbout + 8;
   6173   1.1  jmcneill 	for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
   6174   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6175   1.1  jmcneill 		if (i % 16 == 15)
   6176   1.1  jmcneill 			printf("\n");
   6177   1.1  jmcneill 	}
   6178   1.1  jmcneill free:
   6179   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6180   1.1  jmcneill 	return (error);
   6181   1.1  jmcneill }
   6182   1.1  jmcneill int
   6183  1.15  jmcneill mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index)
   6184   1.1  jmcneill {
   6185   1.1  jmcneill 	struct mcx_dmamem mxm;
   6186   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6187   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_in *in;
   6188   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
   6189   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_out *out;
   6190   1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
   6191   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6192   1.1  jmcneill 	int error;
   6193   1.1  jmcneill 	int i;
   6194   1.1  jmcneill 	uint8_t *dump;
   6195   1.1  jmcneill 
   6196   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6197   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6198   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6199   1.1  jmcneill 
   6200   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6201   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
   6202   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6203   1.1  jmcneill 
   6204   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6205   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6206   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6207   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6208  1.15  jmcneill 		printf(", unable to allocate "
   6209  1.15  jmcneill 		    "query flow table entry mailboxes\n");
   6210   1.1  jmcneill 		return (-1);
   6211   1.1  jmcneill 	}
   6212   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6213   1.1  jmcneill 
   6214   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6215   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6216  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6217   1.1  jmcneill 	mbin->cmd_flow_index = htobe32(index);
   6218   1.1  jmcneill 
   6219   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6220   1.1  jmcneill 
   6221   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6222   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6223   1.1  jmcneill 	if (error != 0) {
   6224   1.1  jmcneill 		printf("%s: query flow table entry timeout\n", DEVNAME(sc));
   6225   1.1  jmcneill 		goto free;
   6226   1.1  jmcneill 	}
   6227   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6228   1.1  jmcneill 	if (error != 0) {
   6229   1.1  jmcneill 		printf("%s: query flow table entry reply corrupt\n",
   6230   1.1  jmcneill 		    DEVNAME(sc));
   6231   1.1  jmcneill 		goto free;
   6232   1.1  jmcneill 	}
   6233   1.1  jmcneill 
   6234   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6235   1.1  jmcneill 	switch (out->cmd_status) {
   6236   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6237   1.1  jmcneill 		break;
   6238   1.1  jmcneill 	default:
   6239   1.1  jmcneill 		printf("%s: query flow table entry failed (%x/%x)\n",
   6240   1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   6241   1.1  jmcneill 		error = -1;
   6242   1.1  jmcneill 		goto free;
   6243   1.1  jmcneill 	}
   6244   1.1  jmcneill 
   6245   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
   6246   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6247   1.1  jmcneill 	dump = (uint8_t *)mbout;
   6248   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6249   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6250   1.1  jmcneill 		if (i % 16 == 15)
   6251   1.1  jmcneill 			printf("\n");
   6252   1.1  jmcneill 	}
   6253   1.1  jmcneill 
   6254   1.1  jmcneill free:
   6255   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6256   1.1  jmcneill 	return (error);
   6257   1.1  jmcneill }
   6258   1.1  jmcneill 
   6259   1.1  jmcneill int
   6260  1.15  jmcneill mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id)
   6261   1.1  jmcneill {
   6262   1.1  jmcneill 	struct mcx_dmamem mxm;
   6263   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6264   1.1  jmcneill 	struct mcx_cmd_query_flow_group_in *in;
   6265   1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_in *mbin;
   6266   1.1  jmcneill 	struct mcx_cmd_query_flow_group_out *out;
   6267   1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_out *mbout;
   6268   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6269   1.1  jmcneill 	int error;
   6270   1.1  jmcneill 	int i;
   6271   1.1  jmcneill 	uint8_t *dump;
   6272   1.1  jmcneill 
   6273   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6274   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6275   1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   6276   1.1  jmcneill 
   6277   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6278   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
   6279   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6280   1.1  jmcneill 
   6281   1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   6282   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6283   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6284   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6285   1.1  jmcneill 		printf(", unable to allocate query flow group mailboxes\n");
   6286   1.1  jmcneill 		return (-1);
   6287   1.1  jmcneill 	}
   6288   1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6289   1.1  jmcneill 
   6290   1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6291   1.1  jmcneill 	mbin->cmd_table_type = 0;
   6292  1.15  jmcneill 	mbin->cmd_table_id = htobe32(flow_table_id);
   6293   1.1  jmcneill 	mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
   6294   1.1  jmcneill 
   6295   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6296   1.1  jmcneill 
   6297   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6298   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6299   1.1  jmcneill 	if (error != 0) {
   6300   1.1  jmcneill 		printf("%s: query flow group timeout\n", DEVNAME(sc));
   6301   1.1  jmcneill 		goto free;
   6302   1.1  jmcneill 	}
   6303   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6304   1.1  jmcneill 	if (error != 0) {
   6305   1.1  jmcneill 		printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
   6306   1.1  jmcneill 		goto free;
   6307   1.1  jmcneill 	}
   6308   1.1  jmcneill 
   6309   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6310   1.1  jmcneill 	switch (out->cmd_status) {
   6311   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6312   1.1  jmcneill 		break;
   6313   1.1  jmcneill 	default:
   6314   1.1  jmcneill 		printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
   6315   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6316   1.1  jmcneill 		error = -1;
   6317   1.1  jmcneill 		goto free;
   6318   1.1  jmcneill 	}
   6319   1.1  jmcneill 
   6320   1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_group_mb_out *)
   6321   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6322   1.1  jmcneill 	dump = (uint8_t *)mbout;
   6323   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6324   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6325   1.1  jmcneill 		if (i % 16 == 15)
   6326   1.1  jmcneill 			printf("\n");
   6327   1.1  jmcneill 	}
   6328   1.1  jmcneill 	dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
   6329   1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   6330   1.1  jmcneill 		printf("%.2x ", dump[i]);
   6331   1.1  jmcneill 		if (i % 16 == 15)
   6332   1.1  jmcneill 			printf("\n");
   6333   1.1  jmcneill 	}
   6334   1.1  jmcneill 
   6335   1.1  jmcneill free:
   6336   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6337   1.1  jmcneill 	return (error);
   6338   1.1  jmcneill }
   6339   1.1  jmcneill 
   6340  1.15  jmcneill static int
   6341  1.15  jmcneill mcx_dump_counters(struct mcx_softc *sc)
   6342  1.15  jmcneill {
   6343  1.15  jmcneill 	struct mcx_dmamem mxm;
   6344  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6345  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_in *in;
   6346  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_mb_in *mbin;
   6347  1.15  jmcneill 	struct mcx_cmd_query_vport_counters_out *out;
   6348  1.15  jmcneill 	struct mcx_nic_vport_counters *counters;
   6349  1.15  jmcneill 	int error, token;
   6350  1.15  jmcneill 
   6351  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6352  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6353  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   6354  1.15  jmcneill 	    sizeof(*out) + sizeof(*counters), token);
   6355  1.15  jmcneill 
   6356  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6357  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
   6358  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6359  1.15  jmcneill 
   6360  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   6361  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6362  1.15  jmcneill 		printf(", unable to allocate "
   6363  1.15  jmcneill 		    "query nic vport counters mailboxen\n");
   6364  1.15  jmcneill 		return (-1);
   6365  1.15  jmcneill 	}
   6366  1.15  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6367  1.15  jmcneill 
   6368  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6369  1.15  jmcneill 	mbin->cmd_clear = 0x80;
   6370  1.15  jmcneill 
   6371  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6372  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6373  1.15  jmcneill 
   6374  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6375  1.15  jmcneill 	if (error != 0) {
   6376  1.15  jmcneill 		printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
   6377  1.15  jmcneill 		goto free;
   6378  1.15  jmcneill 	}
   6379  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6380  1.15  jmcneill 		printf("%s: query nic vport counters command corrupt\n",
   6381  1.15  jmcneill 		    DEVNAME(sc));
   6382  1.15  jmcneill 		goto free;
   6383  1.15  jmcneill 	}
   6384  1.15  jmcneill 
   6385  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6386  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6387  1.15  jmcneill 		printf("%s: query nic vport counters failed (%x, %x)\n",
   6388  1.15  jmcneill 		    DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
   6389  1.15  jmcneill 		error = -1;
   6390  1.15  jmcneill 		goto free;
   6391  1.15  jmcneill 	}
   6392  1.15  jmcneill 
   6393  1.15  jmcneill 	counters = (struct mcx_nic_vport_counters *)
   6394  1.15  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6395  1.15  jmcneill 	if (counters->rx_bcast.packets + counters->tx_bcast.packets +
   6396  1.15  jmcneill 	    counters->rx_ucast.packets + counters->tx_ucast.packets +
   6397  1.15  jmcneill 	    counters->rx_err.packets + counters->tx_err.packets)
   6398  1.15  jmcneill 		printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
   6399  1.15  jmcneill 		    DEVNAME(sc),
   6400  1.15  jmcneill 		    betoh64(counters->tx_err.packets),
   6401  1.15  jmcneill 		    betoh64(counters->rx_err.packets),
   6402  1.15  jmcneill 		    betoh64(counters->tx_ucast.packets),
   6403  1.15  jmcneill 		    betoh64(counters->rx_ucast.packets),
   6404  1.15  jmcneill 		    betoh64(counters->tx_bcast.packets),
   6405  1.15  jmcneill 		    betoh64(counters->rx_bcast.packets));
   6406  1.15  jmcneill free:
   6407  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6408  1.15  jmcneill 
   6409  1.15  jmcneill 	return (error);
   6410  1.15  jmcneill }
   6411  1.15  jmcneill 
   6412  1.15  jmcneill static int
   6413  1.15  jmcneill mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
   6414  1.15  jmcneill {
   6415  1.15  jmcneill 	struct mcx_dmamem mxm;
   6416  1.15  jmcneill 	struct mcx_cmdq_entry *cqe;
   6417  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_in *in;
   6418  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_mb_in *mbin;
   6419  1.15  jmcneill 	struct mcx_cmd_query_flow_counter_out *out;
   6420  1.15  jmcneill 	struct mcx_counter *counters;
   6421  1.15  jmcneill 	int error, token;
   6422  1.15  jmcneill 
   6423  1.15  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6424  1.15  jmcneill 	token = mcx_cmdq_token(sc);
   6425  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
   6426  1.15  jmcneill 	    sizeof(*counters), token);
   6427  1.15  jmcneill 
   6428  1.15  jmcneill 	in = mcx_cmdq_in(cqe);
   6429  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
   6430  1.15  jmcneill 	in->cmd_op_mod = htobe16(0);
   6431  1.15  jmcneill 
   6432  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   6433  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6434  1.15  jmcneill 		printf(", unable to allocate query flow counter mailboxen\n");
   6435  1.15  jmcneill 		return (-1);
   6436  1.15  jmcneill 	}
   6437  1.15  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   6438  1.15  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   6439  1.15  jmcneill 	mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
   6440  1.15  jmcneill 	mbin->cmd_clear = 0x80;
   6441  1.15  jmcneill 
   6442  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6443  1.15  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6444  1.15  jmcneill 
   6445  1.15  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6446  1.15  jmcneill 	if (error != 0) {
   6447  1.15  jmcneill 		printf("%s: query flow counter timeout\n", DEVNAME(sc));
   6448  1.15  jmcneill 		goto free;
   6449  1.15  jmcneill 	}
   6450  1.15  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6451  1.15  jmcneill 		printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
   6452  1.15  jmcneill 		goto free;
   6453  1.15  jmcneill 	}
   6454  1.15  jmcneill 
   6455  1.15  jmcneill 	out = mcx_cmdq_out(cqe);
   6456  1.15  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   6457  1.15  jmcneill 		printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
   6458  1.15  jmcneill 		    out->cmd_status, betoh32(out->cmd_syndrome));
   6459  1.15  jmcneill 		error = -1;
   6460  1.15  jmcneill 		goto free;
   6461  1.15  jmcneill 	}
   6462  1.15  jmcneill 
   6463  1.15  jmcneill 	counters = (struct mcx_counter *)
   6464  1.15  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6465  1.15  jmcneill 	if (counters->packets)
   6466  1.15  jmcneill 		printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
   6467  1.15  jmcneill 		    betoh64(counters->packets));
   6468  1.15  jmcneill free:
   6469  1.15  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6470  1.15  jmcneill 
   6471  1.15  jmcneill 	return (error);
   6472  1.15  jmcneill }
   6473  1.15  jmcneill 
   6474  1.15  jmcneill #endif
   6475  1.15  jmcneill 
   6476  1.15  jmcneill #if NKSTAT > 0
   6477  1.15  jmcneill 
   6478   1.1  jmcneill int
   6479  1.15  jmcneill mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx)
   6480   1.1  jmcneill {
   6481   1.1  jmcneill 	struct mcx_dmamem mxm;
   6482   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6483   1.1  jmcneill 	struct mcx_cmd_query_rq_in *in;
   6484   1.1  jmcneill 	struct mcx_cmd_query_rq_out *out;
   6485   1.1  jmcneill 	struct mcx_cmd_query_rq_mb_out *mbout;
   6486   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6487   1.1  jmcneill 	int error;
   6488   1.1  jmcneill 
   6489   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6490   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   6491   1.1  jmcneill 	    token);
   6492   1.1  jmcneill 
   6493   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6494   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
   6495   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6496  1.15  jmcneill 	in->cmd_rqn = htobe32(rx->rx_rqn);
   6497   1.1  jmcneill 
   6498   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6499   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6500   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6501  1.15  jmcneill 		printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc));
   6502   1.1  jmcneill 		return (-1);
   6503   1.1  jmcneill 	}
   6504   1.1  jmcneill 
   6505   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6506   1.1  jmcneill 
   6507   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6508   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6509   1.1  jmcneill 	if (error != 0) {
   6510   1.1  jmcneill 		printf("%s: query rq timeout\n", DEVNAME(sc));
   6511   1.1  jmcneill 		goto free;
   6512   1.1  jmcneill 	}
   6513   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6514   1.1  jmcneill 	if (error != 0) {
   6515   1.1  jmcneill 		printf("%s: query rq reply corrupt\n", DEVNAME(sc));
   6516   1.1  jmcneill 		goto free;
   6517   1.1  jmcneill 	}
   6518   1.1  jmcneill 
   6519   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6520   1.1  jmcneill 	switch (out->cmd_status) {
   6521   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6522   1.1  jmcneill 		break;
   6523   1.1  jmcneill 	default:
   6524   1.1  jmcneill 		printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
   6525   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6526   1.1  jmcneill 		error = -1;
   6527   1.1  jmcneill 		goto free;
   6528   1.1  jmcneill 	}
   6529   1.1  jmcneill 
   6530   1.1  jmcneill         mbout = (struct mcx_cmd_query_rq_mb_out *)
   6531   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6532  1.15  jmcneill 	memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx));
   6533   1.1  jmcneill 
   6534   1.1  jmcneill free:
   6535   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6536   1.1  jmcneill 	return (error);
   6537   1.1  jmcneill }
   6538   1.1  jmcneill 
   6539   1.1  jmcneill int
   6540  1.15  jmcneill mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx)
   6541   1.1  jmcneill {
   6542   1.1  jmcneill 	struct mcx_dmamem mxm;
   6543   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6544   1.1  jmcneill 	struct mcx_cmd_query_sq_in *in;
   6545   1.1  jmcneill 	struct mcx_cmd_query_sq_out *out;
   6546   1.1  jmcneill 	struct mcx_cmd_query_sq_mb_out *mbout;
   6547   1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6548   1.1  jmcneill 	int error;
   6549   1.1  jmcneill 
   6550   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6551   1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   6552   1.1  jmcneill 	    token);
   6553   1.1  jmcneill 
   6554   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6555   1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
   6556   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6557  1.15  jmcneill 	in->cmd_sqn = htobe32(tx->tx_sqn);
   6558   1.1  jmcneill 
   6559   1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6560   1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6561   1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6562  1.15  jmcneill 		printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc));
   6563   1.1  jmcneill 		return (-1);
   6564   1.1  jmcneill 	}
   6565   1.1  jmcneill 
   6566   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6567   1.1  jmcneill 
   6568   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6569   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6570   1.1  jmcneill 	if (error != 0) {
   6571   1.1  jmcneill 		printf("%s: query sq timeout\n", DEVNAME(sc));
   6572   1.1  jmcneill 		goto free;
   6573   1.1  jmcneill 	}
   6574   1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   6575   1.1  jmcneill 	if (error != 0) {
   6576   1.1  jmcneill 		printf("%s: query sq reply corrupt\n", DEVNAME(sc));
   6577   1.1  jmcneill 		goto free;
   6578   1.1  jmcneill 	}
   6579   1.1  jmcneill 
   6580   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6581   1.1  jmcneill 	switch (out->cmd_status) {
   6582   1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   6583   1.1  jmcneill 		break;
   6584   1.1  jmcneill 	default:
   6585   1.1  jmcneill 		printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
   6586   1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6587   1.1  jmcneill 		error = -1;
   6588   1.1  jmcneill 		goto free;
   6589   1.1  jmcneill 	}
   6590   1.1  jmcneill 
   6591   1.1  jmcneill         mbout = (struct mcx_cmd_query_sq_mb_out *)
   6592   1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6593  1.15  jmcneill 	memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx));
   6594   1.1  jmcneill 
   6595   1.1  jmcneill free:
   6596   1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   6597   1.1  jmcneill 	return (error);
   6598   1.1  jmcneill }
   6599   1.1  jmcneill 
   6600  1.15  jmcneill int
   6601  1.15  jmcneill mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx)
   6602   1.1  jmcneill {
   6603   1.1  jmcneill 	struct mcx_dmamem mxm;
   6604   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6605  1.15  jmcneill 	struct mcx_cmd_query_cq_in *in;
   6606  1.15  jmcneill 	struct mcx_cmd_query_cq_out *out;
   6607  1.15  jmcneill 	struct mcx_cq_ctx *ctx;
   6608  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6609  1.15  jmcneill 	int error;
   6610   1.1  jmcneill 
   6611   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6612  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
   6613  1.15  jmcneill 	    token);
   6614   1.1  jmcneill 
   6615   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6616  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ);
   6617   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6618  1.15  jmcneill 	in->cmd_cqn = htobe32(cq->cq_n);
   6619   1.1  jmcneill 
   6620  1.15  jmcneill 	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6621  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6622  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6623  1.15  jmcneill 		printf("%s: unable to allocate query cq mailboxes\n",
   6624  1.15  jmcneill 		    DEVNAME(sc));
   6625   1.1  jmcneill 		return (-1);
   6626   1.1  jmcneill 	}
   6627   1.1  jmcneill 
   6628  1.15  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6629   1.1  jmcneill 
   6630   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6631   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6632   1.1  jmcneill 	if (error != 0) {
   6633  1.15  jmcneill 		printf("%s: query cq timeout\n", DEVNAME(sc));
   6634   1.1  jmcneill 		goto free;
   6635   1.1  jmcneill 	}
   6636   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6637  1.15  jmcneill 		printf("%s: query cq reply corrupt\n", DEVNAME(sc));
   6638   1.1  jmcneill 		goto free;
   6639   1.1  jmcneill 	}
   6640   1.1  jmcneill 
   6641   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6642  1.15  jmcneill 	switch (out->cmd_status) {
   6643  1.15  jmcneill 	case MCX_CQ_STATUS_OK:
   6644  1.15  jmcneill 		break;
   6645  1.15  jmcneill 	default:
   6646  1.15  jmcneill 		printf("%s: query qc failed (%x/%x)\n", DEVNAME(sc),
   6647  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6648   1.1  jmcneill 		error = -1;
   6649   1.1  jmcneill 		goto free;
   6650   1.1  jmcneill 	}
   6651   1.1  jmcneill 
   6652  1.15  jmcneill 	ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6653  1.15  jmcneill 	memcpy(cq_ctx, ctx, sizeof(*cq_ctx));
   6654   1.1  jmcneill free:
   6655   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6656   1.1  jmcneill 	return (error);
   6657   1.1  jmcneill }
   6658   1.1  jmcneill 
   6659  1.15  jmcneill int
   6660  1.15  jmcneill mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx)
   6661   1.1  jmcneill {
   6662   1.1  jmcneill 	struct mcx_dmamem mxm;
   6663   1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   6664  1.15  jmcneill 	struct mcx_cmd_query_eq_in *in;
   6665  1.15  jmcneill 	struct mcx_cmd_query_eq_out *out;
   6666  1.15  jmcneill 	struct mcq_eq_ctx *ctx;
   6667  1.15  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   6668  1.15  jmcneill 	int error;
   6669   1.1  jmcneill 
   6670   1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   6671  1.15  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
   6672  1.15  jmcneill 	    token);
   6673   1.1  jmcneill 
   6674   1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   6675  1.15  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ);
   6676   1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   6677  1.15  jmcneill 	in->cmd_eqn = htobe32(eq->eq_n);
   6678   1.1  jmcneill 
   6679  1.15  jmcneill 	CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   6680  1.15  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   6681  1.15  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   6682  1.15  jmcneill 		printf("%s: unable to allocate query eq mailboxes\n",
   6683  1.15  jmcneill 		    DEVNAME(sc));
   6684   1.1  jmcneill 		return (-1);
   6685   1.1  jmcneill 	}
   6686   1.1  jmcneill 
   6687   1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   6688  1.15  jmcneill 
   6689   1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   6690   1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   6691   1.1  jmcneill 	if (error != 0) {
   6692  1.15  jmcneill 		printf("%s: query eq timeout\n", DEVNAME(sc));
   6693   1.1  jmcneill 		goto free;
   6694   1.1  jmcneill 	}
   6695   1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   6696  1.15  jmcneill 		printf("%s: query eq reply corrupt\n", DEVNAME(sc));
   6697   1.1  jmcneill 		goto free;
   6698   1.1  jmcneill 	}
   6699   1.1  jmcneill 
   6700   1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   6701  1.15  jmcneill 	switch (out->cmd_status) {
   6702  1.15  jmcneill 	case MCX_CQ_STATUS_OK:
   6703  1.15  jmcneill 		break;
   6704  1.15  jmcneill 	default:
   6705  1.15  jmcneill 		printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc),
   6706  1.15  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   6707   1.1  jmcneill 		error = -1;
   6708   1.1  jmcneill 		goto free;
   6709   1.1  jmcneill 	}
   6710   1.1  jmcneill 
   6711  1.15  jmcneill 	ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   6712  1.15  jmcneill 	memcpy(eq_ctx, ctx, sizeof(*eq_ctx));
   6713   1.1  jmcneill free:
   6714   1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   6715   1.1  jmcneill 	return (error);
   6716   1.1  jmcneill }
   6717   1.1  jmcneill 
   6718  1.15  jmcneill #endif /* NKSTAT > 0 */
   6719  1.15  jmcneill 
   6720   1.1  jmcneill 
   6721  1.15  jmcneill static inline unsigned int
   6722  1.15  jmcneill mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots)
   6723   1.1  jmcneill {
   6724  1.15  jmcneill 	struct mcx_rq_entry *ring, *rqe;
   6725   1.1  jmcneill 	struct mcx_slot *ms;
   6726   1.1  jmcneill 	struct mbuf *m;
   6727   1.1  jmcneill 	uint slot, p, fills;
   6728   1.1  jmcneill 
   6729  1.15  jmcneill 	ring = MCX_DMA_KVA(&rx->rx_rq_mem);
   6730  1.15  jmcneill 	p = rx->rx_prod;
   6731  1.15  jmcneill 
   6732  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   6733  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
   6734  1.15  jmcneill 
   6735   1.1  jmcneill 	slot = (p % (1 << MCX_LOG_RQ_SIZE));
   6736   1.1  jmcneill 	rqe = ring;
   6737   1.1  jmcneill 	for (fills = 0; fills < nslots; fills++) {
   6738  1.15  jmcneill 		slot = p % (1 << MCX_LOG_RQ_SIZE);
   6739  1.15  jmcneill 
   6740  1.15  jmcneill 		ms = &rx->rx_slots[slot];
   6741  1.15  jmcneill 		rqe = &ring[slot];
   6742  1.15  jmcneill 
   6743   1.1  jmcneill 		m = NULL;
   6744   1.1  jmcneill 		MGETHDR(m, M_DONTWAIT, MT_DATA);
   6745   1.1  jmcneill 		if (m == NULL)
   6746   1.1  jmcneill 			break;
   6747   1.1  jmcneill 
   6748   1.1  jmcneill 		MCLGET(m, M_DONTWAIT);
   6749   1.1  jmcneill 		if ((m->m_flags & M_EXT) == 0) {
   6750   1.1  jmcneill 			m_freem(m);
   6751   1.1  jmcneill 			break;
   6752   1.1  jmcneill 		}
   6753   1.1  jmcneill 
   6754  1.15  jmcneill 		m->m_len = m->m_pkthdr.len = sc->sc_hardmtu;
   6755  1.15  jmcneill 		m_adj(m, m->m_ext.ext_size - sc->sc_rxbufsz);
   6756  1.15  jmcneill 		m_adj(m, ETHER_ALIGN);
   6757  1.15  jmcneill 
   6758   1.1  jmcneill 		if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   6759   1.1  jmcneill 		    BUS_DMA_NOWAIT) != 0) {
   6760   1.1  jmcneill 			m_freem(m);
   6761   1.1  jmcneill 			break;
   6762   1.1  jmcneill 		}
   6763   1.3       tnn 		bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, BUS_DMASYNC_PREREAD);
   6764   1.1  jmcneill 		ms->ms_m = m;
   6765   1.1  jmcneill 
   6766  1.15  jmcneill 		be32enc(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len);
   6767  1.15  jmcneill 		be64enc(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr);
   6768  1.15  jmcneill 		be32enc(&rqe->rqe_lkey, sc->sc_lkey);
   6769   1.1  jmcneill 
   6770   1.1  jmcneill 		p++;
   6771   1.1  jmcneill 	}
   6772   1.1  jmcneill 
   6773  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
   6774  1.15  jmcneill 	    0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
   6775  1.15  jmcneill 
   6776  1.15  jmcneill 	rx->rx_prod = p;
   6777   1.1  jmcneill 
   6778  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   6779  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   6780  1.15  jmcneill 	be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell),
   6781  1.15  jmcneill 	    p & MCX_WQ_DOORBELL_MASK);
   6782  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   6783  1.15  jmcneill 	    rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   6784   1.1  jmcneill 
   6785   1.1  jmcneill 	return (nslots - fills);
   6786   1.1  jmcneill }
   6787   1.1  jmcneill 
   6788   1.1  jmcneill static int
   6789  1.15  jmcneill mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx)
   6790   1.1  jmcneill {
   6791   1.1  jmcneill 	u_int slots;
   6792   1.1  jmcneill 
   6793  1.15  jmcneill 	slots = mcx_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE));
   6794   1.1  jmcneill 	if (slots == 0)
   6795   1.1  jmcneill 		return (1);
   6796   1.1  jmcneill 
   6797  1.15  jmcneill 	slots = mcx_rx_fill_slots(sc, rx, slots);
   6798  1.15  jmcneill 	mcx_rxr_put(&rx->rx_rxr, slots);
   6799   1.1  jmcneill 	return (0);
   6800   1.1  jmcneill }
   6801   1.1  jmcneill 
   6802   1.1  jmcneill void
   6803  1.15  jmcneill mcx_refill(void *xrx)
   6804   1.1  jmcneill {
   6805  1.15  jmcneill 	struct mcx_rx *rx = xrx;
   6806  1.15  jmcneill 	struct mcx_softc *sc = rx->rx_softc;
   6807   1.1  jmcneill 
   6808  1.15  jmcneill 	mcx_rx_fill(sc, rx);
   6809   1.1  jmcneill 
   6810  1.15  jmcneill 	if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
   6811  1.15  jmcneill 		callout_schedule(&rx->rx_refill, 1);
   6812   1.1  jmcneill }
   6813   1.1  jmcneill 
   6814  1.15  jmcneill static int
   6815  1.15  jmcneill mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx,
   6816  1.15  jmcneill     struct mcx_cq_entry *cqe)
   6817   1.1  jmcneill {
   6818   1.1  jmcneill 	struct mcx_slot *ms;
   6819   1.1  jmcneill 	bus_dmamap_t map;
   6820   1.1  jmcneill 	int slot, slots;
   6821   1.1  jmcneill 
   6822   1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
   6823   1.1  jmcneill 
   6824  1.15  jmcneill 	ms = &tx->tx_slots[slot];
   6825   1.1  jmcneill 	map = ms->ms_map;
   6826   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   6827   1.1  jmcneill 	    BUS_DMASYNC_POSTWRITE);
   6828   1.1  jmcneill 
   6829   1.1  jmcneill 	slots = 1;
   6830   1.1  jmcneill 	if (map->dm_nsegs > 1)
   6831   1.1  jmcneill 		slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;
   6832   1.1  jmcneill 
   6833   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, map);
   6834   1.1  jmcneill 	m_freem(ms->ms_m);
   6835   1.1  jmcneill 	ms->ms_m = NULL;
   6836  1.15  jmcneill 
   6837  1.15  jmcneill 	return (slots);
   6838   1.1  jmcneill }
   6839   1.1  jmcneill 
   6840   1.1  jmcneill static uint64_t
   6841   1.1  jmcneill mcx_uptime(void)
   6842   1.1  jmcneill {
   6843   1.1  jmcneill 	struct timespec ts;
   6844   1.1  jmcneill 
   6845   1.1  jmcneill 	nanouptime(&ts);
   6846   1.1  jmcneill 
   6847   1.1  jmcneill 	return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec);
   6848   1.1  jmcneill }
   6849   1.1  jmcneill 
   6850   1.1  jmcneill static void
   6851   1.1  jmcneill mcx_calibrate_first(struct mcx_softc *sc)
   6852   1.1  jmcneill {
   6853   1.1  jmcneill 	struct mcx_calibration *c = &sc->sc_calibration[0];
   6854  1.15  jmcneill 	int s;
   6855   1.1  jmcneill 
   6856   1.1  jmcneill 	sc->sc_calibration_gen = 0;
   6857   1.1  jmcneill 
   6858  1.15  jmcneill 	s = splhigh(); /* crit_enter? */
   6859   1.1  jmcneill 	c->c_ubase = mcx_uptime();
   6860   1.1  jmcneill 	c->c_tbase = mcx_timer(sc);
   6861  1.15  jmcneill 	splx(s);
   6862  1.15  jmcneill 	c->c_ratio = 0;
   6863   1.1  jmcneill 
   6864  1.15  jmcneill #if notyet
   6865   1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz);
   6866  1.15  jmcneill #endif
   6867   1.1  jmcneill }
   6868   1.1  jmcneill 
   6869  1.15  jmcneill #define MCX_TIMESTAMP_SHIFT 24
   6870   1.1  jmcneill 
   6871   1.1  jmcneill static void
   6872   1.1  jmcneill mcx_calibrate(void *arg)
   6873   1.1  jmcneill {
   6874   1.1  jmcneill 	struct mcx_softc *sc = arg;
   6875   1.1  jmcneill 	struct mcx_calibration *nc, *pc;
   6876  1.15  jmcneill 	uint64_t udiff, tdiff;
   6877   1.1  jmcneill 	unsigned int gen;
   6878  1.15  jmcneill 	int s;
   6879   1.1  jmcneill 
   6880   1.1  jmcneill 	if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING))
   6881   1.1  jmcneill 		return;
   6882   1.1  jmcneill 
   6883   1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz);
   6884   1.1  jmcneill 
   6885   1.1  jmcneill 	gen = sc->sc_calibration_gen;
   6886   1.1  jmcneill 	pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   6887   1.1  jmcneill 	gen++;
   6888   1.1  jmcneill 	nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   6889   1.1  jmcneill 
   6890   1.1  jmcneill 	nc->c_uptime = pc->c_ubase;
   6891   1.1  jmcneill 	nc->c_timestamp = pc->c_tbase;
   6892   1.1  jmcneill 
   6893  1.15  jmcneill 	s = splhigh(); /* crit_enter? */
   6894   1.1  jmcneill 	nc->c_ubase = mcx_uptime();
   6895   1.1  jmcneill 	nc->c_tbase = mcx_timer(sc);
   6896  1.15  jmcneill 	splx(s);
   6897   1.1  jmcneill 
   6898  1.15  jmcneill 	udiff = nc->c_ubase - nc->c_uptime;
   6899  1.15  jmcneill 	tdiff = nc->c_tbase - nc->c_timestamp;
   6900  1.15  jmcneill 
   6901  1.15  jmcneill 	/*
   6902  1.15  jmcneill 	 * udiff is the wall clock time between calibration ticks,
   6903  1.15  jmcneill 	 * which should be 32 seconds or 32 billion nanoseconds. if
   6904  1.15  jmcneill 	 * we squint, 1 billion nanoseconds is kind of like a 32 bit
   6905  1.15  jmcneill 	 * number, so 32 billion should still have a lot of high bits
   6906  1.15  jmcneill 	 * spare. we use this space by shifting the nanoseconds up
   6907  1.15  jmcneill 	 * 24 bits so we have a nice big number to divide by the
   6908  1.15  jmcneill 	 * number of mcx timer ticks.
   6909  1.15  jmcneill 	 */
   6910  1.15  jmcneill 	nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff;
   6911   1.1  jmcneill 
   6912   1.1  jmcneill 	membar_producer();
   6913   1.1  jmcneill 	sc->sc_calibration_gen = gen;
   6914   1.1  jmcneill }
   6915   1.1  jmcneill 
   6916   1.1  jmcneill static int
   6917  1.15  jmcneill mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx,
   6918  1.15  jmcneill     struct mcx_cq_entry *cqe, struct mcx_mbufq *mq,
   6919  1.15  jmcneill     const struct mcx_calibration *c)
   6920   1.1  jmcneill {
   6921  1.15  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   6922   1.1  jmcneill 	struct mcx_slot *ms;
   6923   1.1  jmcneill 	struct mbuf *m;
   6924  1.15  jmcneill 	uint32_t flags;
   6925   1.1  jmcneill 	int slot;
   6926   1.1  jmcneill 
   6927   1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
   6928   1.1  jmcneill 
   6929  1.15  jmcneill 	ms = &rx->rx_slots[slot];
   6930   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize,
   6931   1.1  jmcneill 	    BUS_DMASYNC_POSTREAD);
   6932   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
   6933   1.1  jmcneill 
   6934   1.1  jmcneill 	m = ms->ms_m;
   6935   1.1  jmcneill 	ms->ms_m = NULL;
   6936   1.1  jmcneill 
   6937   1.1  jmcneill 	m_set_rcvif(m, &sc->sc_ec.ec_if);
   6938   1.1  jmcneill 	m->m_pkthdr.len = m->m_len = be32dec(&cqe->cq_byte_cnt);
   6939   1.1  jmcneill 
   6940   1.1  jmcneill #if 0
   6941   1.1  jmcneill 	if (cqe->cq_rx_hash_type) {
   6942  1.15  jmcneill 		m->m_pkthdr.ph_flowid = be32toh(cqe->cq_rx_hash);
   6943  1.15  jmcneill 		m->m_pkthdr.csum_flags |= M_FLOWID;
   6944   1.1  jmcneill 	}
   6945   1.1  jmcneill #endif
   6946   1.1  jmcneill 
   6947  1.15  jmcneill 	flags = be32dec(&cqe->cq_flags);
   6948  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) {
   6949  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx)
   6950  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
   6951  1.15  jmcneill 	}
   6952  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) {
   6953  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx)
   6954  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
   6955  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx)
   6956  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_TCPv6;
   6957  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx)
   6958  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
   6959  1.15  jmcneill 		if (ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx)
   6960  1.15  jmcneill 			m->m_pkthdr.csum_flags |= M_CSUM_UDPv6;
   6961  1.15  jmcneill 	}
   6962  1.15  jmcneill 	if (flags & MCX_CQ_ENTRY_FLAGS_CV) {
   6963  1.15  jmcneill 		vlan_set_tag(m, flags & MCX_CQ_ENTRY_FLAGS_VLAN_MASK);
   6964  1.15  jmcneill 	}
   6965  1.15  jmcneill 
   6966  1.15  jmcneill #if notyet
   6967  1.15  jmcneill 	if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_LINK0) && c->c_ratio) {
   6968  1.15  jmcneill 		uint64_t t = be64dec(&cqe->cq_timestamp);
   6969  1.15  jmcneill 		t -= c->c_timestamp;
   6970  1.15  jmcneill 		t *= c->c_ratio;
   6971  1.15  jmcneill 		t >>= MCX_TIMESTAMP_SHIFT;
   6972  1.15  jmcneill 		t += c->c_uptime;
   6973   1.1  jmcneill 
   6974  1.15  jmcneill 		m->m_pkthdr.ph_timestamp = t;
   6975   1.1  jmcneill 		SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
   6976   1.1  jmcneill 	}
   6977   1.1  jmcneill #endif
   6978   1.1  jmcneill 
   6979   1.1  jmcneill 	MBUFQ_ENQUEUE(mq, m);
   6980   1.1  jmcneill 
   6981   1.1  jmcneill 	return (1);
   6982   1.1  jmcneill }
   6983   1.1  jmcneill 
   6984   1.1  jmcneill static struct mcx_cq_entry *
   6985   1.1  jmcneill mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
   6986   1.1  jmcneill {
   6987   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   6988   1.1  jmcneill 	int next;
   6989   1.1  jmcneill 
   6990   1.1  jmcneill 	cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
   6991   1.1  jmcneill 	next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
   6992   1.1  jmcneill 
   6993   1.1  jmcneill 	if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
   6994   1.1  jmcneill 	    ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
   6995   1.1  jmcneill 		return (&cqe[next]);
   6996   1.1  jmcneill 	}
   6997   1.1  jmcneill 
   6998   1.1  jmcneill 	return (NULL);
   6999   1.1  jmcneill }
   7000   1.1  jmcneill 
   7001   1.1  jmcneill static void
   7002  1.15  jmcneill mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar)
   7003   1.1  jmcneill {
   7004  1.15  jmcneill 	struct mcx_cq_doorbell *db;
   7005   1.1  jmcneill 	bus_size_t offset;
   7006   1.1  jmcneill 	uint32_t val;
   7007   1.1  jmcneill 	uint64_t uval;
   7008   1.1  jmcneill 
   7009   1.1  jmcneill 	val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
   7010   1.1  jmcneill 	val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   7011   1.1  jmcneill 
   7012  1.15  jmcneill 	db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell);
   7013  1.15  jmcneill 
   7014  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7015  1.15  jmcneill 	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE);
   7016   1.1  jmcneill 
   7017  1.15  jmcneill 	be32enc(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   7018  1.15  jmcneill 	be32enc(&db->db_arm_ci, val);
   7019  1.15  jmcneill 
   7020  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7021  1.15  jmcneill 	    cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE);
   7022  1.15  jmcneill 
   7023  1.15  jmcneill 	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL;
   7024  1.15  jmcneill 
   7025  1.15  jmcneill 	uval = (uint64_t)val << 32;
   7026   1.1  jmcneill 	uval |= cq->cq_n;
   7027  1.15  jmcneill 
   7028  1.15  jmcneill 	bus_space_write_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval));
   7029  1.15  jmcneill 	mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE);
   7030   1.1  jmcneill }
   7031   1.1  jmcneill 
   7032   1.1  jmcneill void
   7033  1.15  jmcneill mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq)
   7034   1.1  jmcneill {
   7035  1.15  jmcneill 	struct mcx_rx *rx = &q->q_rx;
   7036  1.15  jmcneill 	struct mcx_tx *tx = &q->q_tx;
   7037   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   7038   1.1  jmcneill 	const struct mcx_calibration *c;
   7039   1.1  jmcneill 	unsigned int gen;
   7040   1.1  jmcneill 	struct mcx_cq_entry *cqe;
   7041   1.1  jmcneill 	struct mcx_mbufq mq;
   7042   1.1  jmcneill 	struct mbuf *m;
   7043   1.1  jmcneill 	int rxfree, txfree;
   7044   1.1  jmcneill 
   7045   1.1  jmcneill 	MBUFQ_INIT(&mq);
   7046   1.1  jmcneill 
   7047   1.1  jmcneill 	gen = sc->sc_calibration_gen;
   7048   1.1  jmcneill 	membar_consumer();
   7049   1.1  jmcneill 	c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   7050   1.1  jmcneill 
   7051  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   7052  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
   7053  1.15  jmcneill 
   7054   1.1  jmcneill 	rxfree = 0;
   7055   1.1  jmcneill 	txfree = 0;
   7056   1.1  jmcneill 	while ((cqe = mcx_next_cq_entry(sc, cq))) {
   7057   1.1  jmcneill 		uint8_t opcode;
   7058   1.1  jmcneill 		opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
   7059   1.1  jmcneill 		switch (opcode) {
   7060   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ:
   7061  1.15  jmcneill 			txfree += mcx_process_txeof(sc, tx, cqe);
   7062   1.1  jmcneill 			break;
   7063   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND:
   7064  1.15  jmcneill 			rxfree += mcx_process_rx(sc, rx, cqe, &mq, c);
   7065   1.1  jmcneill 			break;
   7066   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
   7067   1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
   7068   1.1  jmcneill 			/* uint8_t *cqp = (uint8_t *)cqe; */
   7069  1.15  jmcneill 			/* printf("%s: cq completion error: %x\n",
   7070  1.15  jmcneill 			    DEVNAME(sc), cqp[0x37]); */
   7071   1.1  jmcneill 			break;
   7072   1.1  jmcneill 
   7073   1.1  jmcneill 		default:
   7074  1.15  jmcneill 			/* printf("%s: cq completion opcode %x??\n",
   7075  1.15  jmcneill 			    DEVNAME(sc), opcode); */
   7076   1.1  jmcneill 			break;
   7077   1.1  jmcneill 		}
   7078   1.1  jmcneill 
   7079   1.1  jmcneill 		cq->cq_cons++;
   7080   1.1  jmcneill 	}
   7081   1.1  jmcneill 
   7082  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
   7083  1.15  jmcneill 	    0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
   7084  1.15  jmcneill 
   7085   1.1  jmcneill 	if (rxfree > 0) {
   7086  1.15  jmcneill 		mcx_rxr_put(&rx->rx_rxr, rxfree);
   7087   1.1  jmcneill 		while (MBUFQ_FIRST(&mq) != NULL) {
   7088   1.1  jmcneill 			MBUFQ_DEQUEUE(&mq, m);
   7089   1.1  jmcneill 			if_percpuq_enqueue(ifp->if_percpuq, m);
   7090   1.1  jmcneill 		}
   7091   1.1  jmcneill 
   7092  1.15  jmcneill 		mcx_rx_fill(sc, rx);
   7093  1.15  jmcneill 		if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
   7094  1.15  jmcneill 			callout_schedule(&rx->rx_refill, 1);
   7095   1.1  jmcneill 	}
   7096  1.17  jmcneill 
   7097  1.17  jmcneill 	cq->cq_count++;
   7098  1.17  jmcneill 	mcx_arm_cq(sc, cq, q->q_uar);
   7099  1.17  jmcneill 
   7100   1.1  jmcneill 	if (txfree > 0) {
   7101  1.15  jmcneill 		tx->tx_cons += txfree;
   7102   1.1  jmcneill 		if_schedule_deferred_start(ifp);
   7103   1.1  jmcneill 	}
   7104   1.1  jmcneill }
   7105   1.1  jmcneill 
   7106  1.15  jmcneill 
   7107   1.1  jmcneill static void
   7108  1.15  jmcneill mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar)
   7109   1.1  jmcneill {
   7110   1.1  jmcneill 	bus_size_t offset;
   7111   1.1  jmcneill 	uint32_t val;
   7112   1.1  jmcneill 
   7113  1.15  jmcneill 	offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM;
   7114  1.15  jmcneill 	val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff);
   7115   1.1  jmcneill 
   7116   1.1  jmcneill 	mcx_wr(sc, offset, val);
   7117  1.15  jmcneill 	mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE);
   7118   1.1  jmcneill }
   7119   1.1  jmcneill 
   7120   1.1  jmcneill static struct mcx_eq_entry *
   7121  1.15  jmcneill mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq)
   7122   1.1  jmcneill {
   7123   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   7124   1.1  jmcneill 	int next;
   7125   1.1  jmcneill 
   7126  1.15  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
   7127  1.15  jmcneill 	next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE);
   7128  1.15  jmcneill 	if ((eqe[next].eq_owner & 1) ==
   7129  1.15  jmcneill 	    ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
   7130  1.15  jmcneill 		eq->eq_cons++;
   7131   1.1  jmcneill 		return (&eqe[next]);
   7132   1.1  jmcneill 	}
   7133   1.1  jmcneill 	return (NULL);
   7134   1.1  jmcneill }
   7135   1.1  jmcneill 
   7136   1.1  jmcneill int
   7137  1.15  jmcneill mcx_admin_intr(void *xsc)
   7138   1.1  jmcneill {
   7139   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)xsc;
   7140  1.15  jmcneill 	struct mcx_eq *eq = &sc->sc_admin_eq;
   7141   1.1  jmcneill 	struct mcx_eq_entry *eqe;
   7142   1.1  jmcneill 
   7143  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7144  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   7145  1.15  jmcneill 
   7146  1.15  jmcneill 	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
   7147   1.1  jmcneill 		switch (eqe->eq_event_type) {
   7148   1.1  jmcneill 		case MCX_EVENT_TYPE_LAST_WQE:
   7149   1.1  jmcneill 			/* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
   7150   1.1  jmcneill 			break;
   7151   1.1  jmcneill 
   7152   1.1  jmcneill 		case MCX_EVENT_TYPE_CQ_ERROR:
   7153   1.1  jmcneill 			/* printf("%s: cq error\n", DEVNAME(sc)); */
   7154   1.1  jmcneill 			break;
   7155   1.1  jmcneill 
   7156   1.1  jmcneill 		case MCX_EVENT_TYPE_CMD_COMPLETION:
   7157   1.1  jmcneill 			/* wakeup probably */
   7158   1.1  jmcneill 			break;
   7159   1.1  jmcneill 
   7160   1.1  jmcneill 		case MCX_EVENT_TYPE_PORT_CHANGE:
   7161   1.1  jmcneill 			workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL);
   7162   1.1  jmcneill 			break;
   7163   1.1  jmcneill 
   7164   1.1  jmcneill 		default:
   7165   1.1  jmcneill 			/* printf("%s: something happened\n", DEVNAME(sc)); */
   7166   1.1  jmcneill 			break;
   7167   1.1  jmcneill 		}
   7168   1.1  jmcneill 	}
   7169  1.15  jmcneill 
   7170  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7171  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   7172  1.15  jmcneill 
   7173  1.15  jmcneill 	mcx_arm_eq(sc, eq, sc->sc_uar);
   7174  1.15  jmcneill 
   7175  1.15  jmcneill 	return (1);
   7176  1.15  jmcneill }
   7177  1.15  jmcneill 
   7178  1.15  jmcneill int
   7179  1.15  jmcneill mcx_cq_intr(void *xq)
   7180  1.15  jmcneill {
   7181  1.15  jmcneill 	struct mcx_queues *q = (struct mcx_queues *)xq;
   7182  1.15  jmcneill 	struct mcx_softc *sc = q->q_sc;
   7183  1.15  jmcneill 	struct mcx_eq *eq = &q->q_eq;
   7184  1.15  jmcneill 	struct mcx_eq_entry *eqe;
   7185  1.15  jmcneill 	int cqn;
   7186  1.15  jmcneill 
   7187  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7188  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
   7189  1.15  jmcneill 
   7190  1.15  jmcneill 	while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
   7191  1.15  jmcneill 		switch (eqe->eq_event_type) {
   7192  1.15  jmcneill 		case MCX_EVENT_TYPE_COMPLETION:
   7193  1.15  jmcneill 			cqn = be32toh(eqe->eq_event_data[6]);
   7194  1.15  jmcneill 			if (cqn == q->q_cq.cq_n)
   7195  1.15  jmcneill 				mcx_process_cq(sc, q, &q->q_cq);
   7196  1.15  jmcneill 			break;
   7197  1.15  jmcneill 		}
   7198  1.15  jmcneill 	}
   7199  1.15  jmcneill 
   7200  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
   7201  1.15  jmcneill 	    0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
   7202  1.15  jmcneill 
   7203  1.15  jmcneill 	mcx_arm_eq(sc, eq, q->q_uar);
   7204  1.15  jmcneill 
   7205   1.1  jmcneill 	return (1);
   7206   1.1  jmcneill }
   7207   1.1  jmcneill 
   7208   1.1  jmcneill static void
   7209   1.1  jmcneill mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
   7210   1.1  jmcneill     int total)
   7211   1.1  jmcneill {
   7212   1.1  jmcneill 	struct mcx_slot *ms;
   7213   1.1  jmcneill 
   7214   1.1  jmcneill 	int i = allocated;
   7215   1.1  jmcneill 	while (i-- > 0) {
   7216   1.1  jmcneill 		ms = &slots[i];
   7217   1.1  jmcneill 		bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
   7218   1.1  jmcneill 		if (ms->ms_m != NULL)
   7219   1.1  jmcneill 			m_freem(ms->ms_m);
   7220   1.1  jmcneill 	}
   7221   1.1  jmcneill 	kmem_free(slots, total * sizeof(*ms));
   7222   1.1  jmcneill }
   7223   1.1  jmcneill 
   7224   1.1  jmcneill static int
   7225  1.15  jmcneill mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q)
   7226   1.1  jmcneill {
   7227  1.15  jmcneill 	struct mcx_rx *rx;
   7228  1.15  jmcneill 	struct mcx_tx *tx;
   7229   1.1  jmcneill 	struct mcx_slot *ms;
   7230  1.15  jmcneill 	int i;
   7231   1.1  jmcneill 
   7232  1.15  jmcneill 	rx = &q->q_rx;
   7233  1.15  jmcneill 	rx->rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE),
   7234   1.1  jmcneill 	    KM_SLEEP);
   7235   1.1  jmcneill 
   7236   1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
   7237  1.15  jmcneill 		ms = &rx->rx_slots[i];
   7238   1.1  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
   7239   1.1  jmcneill 		    sc->sc_hardmtu, 0,
   7240   1.1  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   7241   1.1  jmcneill 		    &ms->ms_map) != 0) {
   7242   1.1  jmcneill 			printf("%s: failed to allocate rx dma maps\n",
   7243   1.1  jmcneill 			    DEVNAME(sc));
   7244   1.1  jmcneill 			goto destroy_rx_slots;
   7245   1.1  jmcneill 		}
   7246  1.15  jmcneill 	}
   7247  1.15  jmcneill 
   7248  1.15  jmcneill 	tx = &q->q_tx;
   7249  1.15  jmcneill 	tx->tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE),
   7250  1.15  jmcneill 	     KM_SLEEP);
   7251  1.15  jmcneill 
   7252  1.15  jmcneill 	for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
   7253  1.15  jmcneill 		ms = &tx->tx_slots[i];
   7254  1.15  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
   7255  1.15  jmcneill 		    MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
   7256  1.15  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   7257  1.15  jmcneill 		    &ms->ms_map) != 0) {
   7258  1.15  jmcneill 			printf("%s: failed to allocate tx dma maps\n",
   7259  1.15  jmcneill 			    DEVNAME(sc));
   7260  1.15  jmcneill 			goto destroy_tx_slots;
   7261  1.15  jmcneill 		}
   7262  1.15  jmcneill 	}
   7263  1.15  jmcneill 
   7264  1.15  jmcneill 	if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index,
   7265  1.15  jmcneill 	    q->q_eq.eq_n) != 0)
   7266  1.15  jmcneill 		goto destroy_tx_slots;
   7267  1.15  jmcneill 
   7268  1.15  jmcneill 	if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n)
   7269  1.15  jmcneill 	    != 0)
   7270  1.15  jmcneill 		goto destroy_cq;
   7271  1.15  jmcneill 
   7272  1.15  jmcneill 	if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0)
   7273  1.15  jmcneill 		goto destroy_sq;
   7274  1.15  jmcneill 
   7275  1.15  jmcneill 	return 0;
   7276  1.15  jmcneill 
   7277  1.15  jmcneill destroy_sq:
   7278  1.15  jmcneill 	mcx_destroy_sq(sc, tx);
   7279  1.15  jmcneill destroy_cq:
   7280  1.15  jmcneill 	mcx_destroy_cq(sc, &q->q_cq);
   7281  1.15  jmcneill destroy_tx_slots:
   7282  1.15  jmcneill 	mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
   7283  1.15  jmcneill 	tx->tx_slots = NULL;
   7284  1.15  jmcneill 
   7285  1.15  jmcneill 	i = (1 << MCX_LOG_RQ_SIZE);
   7286  1.15  jmcneill destroy_rx_slots:
   7287  1.15  jmcneill 	mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
   7288  1.15  jmcneill 	rx->rx_slots = NULL;
   7289  1.15  jmcneill 	return ENOMEM;
   7290  1.15  jmcneill }
   7291  1.15  jmcneill 
   7292  1.15  jmcneill static int
   7293  1.15  jmcneill mcx_rss_group_entry_count(struct mcx_softc *sc, int group)
   7294  1.15  jmcneill {
   7295  1.15  jmcneill 	int i;
   7296  1.15  jmcneill 	int count;
   7297  1.15  jmcneill 
   7298  1.15  jmcneill 	count = 0;
   7299  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7300  1.15  jmcneill 		if (mcx_rss_config[i].flow_group == group)
   7301  1.15  jmcneill 			count++;
   7302  1.15  jmcneill 	}
   7303  1.15  jmcneill 
   7304  1.15  jmcneill 	return count;
   7305  1.15  jmcneill }
   7306  1.15  jmcneill 
   7307  1.15  jmcneill static int
   7308  1.15  jmcneill mcx_init(struct ifnet *ifp)
   7309  1.15  jmcneill {
   7310  1.15  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7311  1.15  jmcneill 	struct mcx_rx *rx;
   7312  1.15  jmcneill 	struct mcx_tx *tx;
   7313  1.15  jmcneill 	int i, start, count, flow_group, flow_index;
   7314  1.15  jmcneill 	struct mcx_flow_match match_crit;
   7315  1.15  jmcneill 	struct mcx_rss_rule *rss;
   7316  1.15  jmcneill 	uint32_t dest;
   7317  1.15  jmcneill 	int rqns[MCX_MAX_QUEUES] = { 0 };
   7318  1.15  jmcneill 
   7319  1.15  jmcneill 	if (ISSET(ifp->if_flags, IFF_RUNNING))
   7320  1.15  jmcneill 		mcx_stop(ifp, 0);
   7321   1.1  jmcneill 
   7322  1.15  jmcneill 	if (mcx_create_tis(sc, &sc->sc_tis) != 0)
   7323  1.15  jmcneill 		goto down;
   7324   1.1  jmcneill 
   7325  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7326  1.15  jmcneill 		if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
   7327  1.15  jmcneill 			goto down;
   7328   1.1  jmcneill 		}
   7329   1.1  jmcneill 	}
   7330   1.1  jmcneill 
   7331  1.15  jmcneill 	/* RSS flow table and flow groups */
   7332  1.15  jmcneill 	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1,
   7333  1.15  jmcneill 	    &sc->sc_rss_flow_table_id) != 0)
   7334   1.1  jmcneill 		goto down;
   7335   1.1  jmcneill 
   7336  1.15  jmcneill 	dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   7337  1.15  jmcneill 	    sc->sc_rss_flow_table_id;
   7338   1.1  jmcneill 
   7339  1.15  jmcneill 	/* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */
   7340  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7341  1.15  jmcneill 	match_crit.mc_ethertype = 0xffff;
   7342  1.15  jmcneill 	match_crit.mc_ip_proto = 0xff;
   7343  1.15  jmcneill 	match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG;
   7344  1.15  jmcneill 	start = 0;
   7345  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4);
   7346  1.15  jmcneill 	if (count != 0) {
   7347  1.15  jmcneill 		if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7348  1.15  jmcneill 		    MCX_FLOW_GROUP_RSS_L4, start, count,
   7349  1.15  jmcneill 		    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7350  1.15  jmcneill 			goto down;
   7351  1.15  jmcneill 		start += count;
   7352  1.15  jmcneill 	}
   7353   1.1  jmcneill 
   7354  1.15  jmcneill 	/* L3 RSS flow group (v4/v6, including fragments) */
   7355  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7356  1.15  jmcneill 	match_crit.mc_ethertype = 0xffff;
   7357  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3);
   7358  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7359  1.15  jmcneill 	    MCX_FLOW_GROUP_RSS_L3, start, count,
   7360  1.15  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7361   1.1  jmcneill 		goto down;
   7362  1.15  jmcneill 	start += count;
   7363   1.1  jmcneill 
   7364  1.15  jmcneill 	/* non-RSS flow group */
   7365  1.15  jmcneill 	count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE);
   7366  1.15  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7367  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
   7368  1.15  jmcneill 	    MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0)
   7369   1.1  jmcneill 		goto down;
   7370   1.1  jmcneill 
   7371  1.15  jmcneill 	/* Root flow table, matching packets based on mac address */
   7372  1.15  jmcneill 	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0,
   7373  1.15  jmcneill 	    &sc->sc_mac_flow_table_id) != 0)
   7374   1.1  jmcneill 		goto down;
   7375   1.1  jmcneill 
   7376   1.1  jmcneill 	/* promisc flow group */
   7377   1.1  jmcneill 	start = 0;
   7378   1.1  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   7379  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7380  1.15  jmcneill 	    MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0)
   7381   1.1  jmcneill 		goto down;
   7382   1.1  jmcneill 	sc->sc_promisc_flow_enabled = 0;
   7383   1.1  jmcneill 	start++;
   7384   1.1  jmcneill 
   7385   1.1  jmcneill 	/* all multicast flow group */
   7386   1.1  jmcneill 	match_crit.mc_dest_mac[0] = 0x01;
   7387  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7388  1.15  jmcneill 	    MCX_FLOW_GROUP_ALLMULTI, start, 1,
   7389   1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7390   1.1  jmcneill 		goto down;
   7391   1.1  jmcneill 	sc->sc_allmulti_flow_enabled = 0;
   7392   1.1  jmcneill 	start++;
   7393   1.1  jmcneill 
   7394   1.1  jmcneill 	/* mac address matching flow group */
   7395   1.1  jmcneill 	memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
   7396  1.15  jmcneill 	if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
   7397  1.15  jmcneill 	    MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
   7398   1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   7399   1.1  jmcneill 		goto down;
   7400   1.1  jmcneill 
   7401   1.1  jmcneill 	/* flow table entries for unicast and broadcast */
   7402   1.1  jmcneill 	start = 0;
   7403  1.15  jmcneill 	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
   7404  1.15  jmcneill 	    LLADDR(satosdl(ifp->if_dl->ifa_addr)), dest) != 0)
   7405   1.1  jmcneill 		goto down;
   7406   1.1  jmcneill 	start++;
   7407   1.1  jmcneill 
   7408  1.15  jmcneill 	if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
   7409  1.15  jmcneill 	    etherbroadcastaddr, dest) != 0)
   7410   1.1  jmcneill 		goto down;
   7411   1.1  jmcneill 	start++;
   7412   1.1  jmcneill 
   7413   1.1  jmcneill 	/* multicast entries go after that */
   7414   1.1  jmcneill 	sc->sc_mcast_flow_base = start;
   7415   1.1  jmcneill 
   7416   1.1  jmcneill 	/* re-add any existing multicast flows */
   7417   1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7418   1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   7419  1.15  jmcneill 			mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC,
   7420   1.1  jmcneill 			    sc->sc_mcast_flow_base + i,
   7421  1.15  jmcneill 			    sc->sc_mcast_flows[i], dest);
   7422   1.1  jmcneill 		}
   7423   1.1  jmcneill 	}
   7424   1.1  jmcneill 
   7425  1.15  jmcneill 	if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0)
   7426   1.1  jmcneill 		goto down;
   7427   1.1  jmcneill 
   7428  1.15  jmcneill 	/*
   7429  1.15  jmcneill 	 * the RQT can be any size as long as it's a power of two.
   7430  1.15  jmcneill 	 * since we also restrict the number of queues to a power of two,
   7431  1.15  jmcneill 	 * we can just put each rx queue in once.
   7432  1.15  jmcneill 	 */
   7433  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++)
   7434  1.15  jmcneill 		rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;
   7435  1.15  jmcneill 
   7436  1.15  jmcneill 	if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0)
   7437   1.1  jmcneill 		goto down;
   7438   1.1  jmcneill 
   7439  1.15  jmcneill 	start = 0;
   7440  1.15  jmcneill 	flow_index = 0;
   7441  1.15  jmcneill 	flow_group = -1;
   7442  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7443  1.15  jmcneill 		rss = &mcx_rss_config[i];
   7444  1.15  jmcneill 		if (rss->flow_group != flow_group) {
   7445  1.15  jmcneill 			flow_group = rss->flow_group;
   7446  1.15  jmcneill 			flow_index = 0;
   7447  1.15  jmcneill 		}
   7448  1.15  jmcneill 
   7449  1.15  jmcneill 		if (rss->hash_sel == 0) {
   7450  1.15  jmcneill 			if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx,
   7451  1.15  jmcneill 			    &sc->sc_tir[i]) != 0)
   7452  1.15  jmcneill 				goto down;
   7453  1.15  jmcneill 		} else {
   7454  1.15  jmcneill 			if (mcx_create_tir_indirect(sc, sc->sc_rqt,
   7455  1.15  jmcneill 			    rss->hash_sel, &sc->sc_tir[i]) != 0)
   7456  1.15  jmcneill 				goto down;
   7457  1.15  jmcneill 		}
   7458  1.15  jmcneill 
   7459  1.15  jmcneill 		if (mcx_set_flow_table_entry_proto(sc, flow_group,
   7460  1.15  jmcneill 		    flow_index, rss->ethertype, rss->ip_proto,
   7461  1.15  jmcneill 		    MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0)
   7462  1.15  jmcneill 			goto down;
   7463  1.15  jmcneill 		flow_index++;
   7464  1.15  jmcneill 	}
   7465  1.15  jmcneill 
   7466  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7467  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   7468  1.15  jmcneill 		rx = &q->q_rx;
   7469  1.15  jmcneill 		tx = &q->q_tx;
   7470  1.15  jmcneill 
   7471  1.15  jmcneill 		/* start the queues */
   7472  1.15  jmcneill 		if (mcx_ready_sq(sc, tx) != 0)
   7473  1.15  jmcneill 			goto down;
   7474  1.15  jmcneill 
   7475  1.15  jmcneill 		if (mcx_ready_rq(sc, rx) != 0)
   7476  1.15  jmcneill 			goto down;
   7477  1.15  jmcneill 
   7478  1.15  jmcneill 		mcx_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
   7479  1.15  jmcneill 		rx->rx_prod = 0;
   7480  1.15  jmcneill 		mcx_rx_fill(sc, rx);
   7481   1.1  jmcneill 
   7482  1.15  jmcneill 		tx->tx_cons = 0;
   7483  1.15  jmcneill 		tx->tx_prod = 0;
   7484  1.15  jmcneill 	}
   7485   1.1  jmcneill 
   7486   1.1  jmcneill 	mcx_calibrate_first(sc);
   7487   1.1  jmcneill 
   7488   1.1  jmcneill 	SET(ifp->if_flags, IFF_RUNNING);
   7489   1.1  jmcneill 	CLR(ifp->if_flags, IFF_OACTIVE);
   7490   1.1  jmcneill 	if_schedule_deferred_start(ifp);
   7491   1.1  jmcneill 
   7492   1.1  jmcneill 	return 0;
   7493   1.1  jmcneill down:
   7494   1.1  jmcneill 	mcx_stop(ifp, 0);
   7495   1.1  jmcneill 	return EIO;
   7496   1.1  jmcneill }
   7497   1.1  jmcneill 
   7498   1.1  jmcneill static void
   7499   1.1  jmcneill mcx_stop(struct ifnet *ifp, int disable)
   7500   1.1  jmcneill {
   7501   1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7502  1.15  jmcneill 	struct mcx_rss_rule *rss;
   7503  1.15  jmcneill 	int group, i, flow_group, flow_index;
   7504   1.1  jmcneill 
   7505   1.1  jmcneill 	CLR(ifp->if_flags, IFF_RUNNING);
   7506   1.1  jmcneill 
   7507   1.1  jmcneill 	/*
   7508   1.1  jmcneill 	 * delete flow table entries first, so no packets can arrive
   7509   1.1  jmcneill 	 * after the barriers
   7510   1.1  jmcneill 	 */
   7511   1.1  jmcneill 	if (sc->sc_promisc_flow_enabled)
   7512   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   7513   1.1  jmcneill 	if (sc->sc_allmulti_flow_enabled)
   7514   1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   7515   1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
   7516   1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
   7517   1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7518   1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   7519   1.1  jmcneill 			mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
   7520   1.1  jmcneill 			    sc->sc_mcast_flow_base + i);
   7521   1.1  jmcneill 		}
   7522   1.1  jmcneill 	}
   7523   1.1  jmcneill 
   7524  1.15  jmcneill 	flow_group = -1;
   7525  1.15  jmcneill 	flow_index = 0;
   7526  1.15  jmcneill 	for (i = 0; i < __arraycount(mcx_rss_config); i++) {
   7527  1.15  jmcneill 		rss = &mcx_rss_config[i];
   7528  1.15  jmcneill 		if (rss->flow_group != flow_group) {
   7529  1.15  jmcneill 			flow_group = rss->flow_group;
   7530  1.15  jmcneill 			flow_index = 0;
   7531  1.15  jmcneill 		}
   7532  1.15  jmcneill 
   7533  1.15  jmcneill 		mcx_delete_flow_table_entry(sc, flow_group, flow_index);
   7534  1.15  jmcneill 
   7535  1.15  jmcneill 		mcx_destroy_tir(sc, sc->sc_tir[i]);
   7536  1.15  jmcneill 		sc->sc_tir[i] = 0;
   7537  1.15  jmcneill 
   7538  1.15  jmcneill 		flow_index++;
   7539  1.15  jmcneill 	}
   7540  1.15  jmcneill 
   7541  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7542  1.15  jmcneill 		callout_halt(&sc->sc_queues[i].q_rx.rx_refill, NULL);
   7543  1.15  jmcneill 	}
   7544  1.15  jmcneill 
   7545   1.1  jmcneill 	callout_halt(&sc->sc_calibrate, NULL);
   7546   1.1  jmcneill 
   7547   1.1  jmcneill 	for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
   7548  1.15  jmcneill 		if (sc->sc_flow_group[group].g_id != -1)
   7549  1.15  jmcneill 			mcx_destroy_flow_group(sc, group);
   7550  1.15  jmcneill 	}
   7551  1.15  jmcneill 
   7552  1.15  jmcneill 	if (sc->sc_mac_flow_table_id != -1) {
   7553  1.15  jmcneill 		mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id);
   7554  1.15  jmcneill 		sc->sc_mac_flow_table_id = -1;
   7555  1.15  jmcneill 	}
   7556  1.15  jmcneill 	if (sc->sc_rss_flow_table_id != -1) {
   7557  1.15  jmcneill 		mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id);
   7558  1.15  jmcneill 		sc->sc_rss_flow_table_id = -1;
   7559  1.15  jmcneill 	}
   7560  1.15  jmcneill 	if (sc->sc_rqt != -1) {
   7561  1.15  jmcneill 		mcx_destroy_rqt(sc, sc->sc_rqt);
   7562  1.15  jmcneill 		sc->sc_rqt = -1;
   7563  1.15  jmcneill 	}
   7564  1.15  jmcneill 
   7565  1.15  jmcneill 	for (i = 0; i < sc->sc_nqueues; i++) {
   7566  1.15  jmcneill 		struct mcx_queues *q = &sc->sc_queues[i];
   7567  1.15  jmcneill 		struct mcx_rx *rx = &q->q_rx;
   7568  1.15  jmcneill 		struct mcx_tx *tx = &q->q_tx;
   7569  1.15  jmcneill 		struct mcx_cq *cq = &q->q_cq;
   7570  1.15  jmcneill 
   7571  1.15  jmcneill 		if (rx->rx_rqn != 0)
   7572  1.15  jmcneill 			mcx_destroy_rq(sc, rx);
   7573  1.15  jmcneill 
   7574  1.15  jmcneill 		if (tx->tx_sqn != 0)
   7575  1.15  jmcneill 			mcx_destroy_sq(sc, tx);
   7576  1.15  jmcneill 
   7577  1.15  jmcneill 		if (tx->tx_slots != NULL) {
   7578  1.15  jmcneill 			mcx_free_slots(sc, tx->tx_slots,
   7579  1.15  jmcneill 			    (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE));
   7580  1.15  jmcneill 			tx->tx_slots = NULL;
   7581  1.15  jmcneill 		}
   7582  1.15  jmcneill 		if (rx->rx_slots != NULL) {
   7583  1.15  jmcneill 			mcx_free_slots(sc, rx->rx_slots,
   7584  1.15  jmcneill 			    (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE));
   7585  1.15  jmcneill 			rx->rx_slots = NULL;
   7586  1.15  jmcneill 		}
   7587  1.15  jmcneill 
   7588  1.15  jmcneill 		if (cq->cq_n != 0)
   7589  1.15  jmcneill 			mcx_destroy_cq(sc, cq);
   7590  1.15  jmcneill 	}
   7591  1.15  jmcneill 	if (sc->sc_tis != 0) {
   7592  1.15  jmcneill 		mcx_destroy_tis(sc, sc->sc_tis);
   7593  1.15  jmcneill 		sc->sc_tis = 0;
   7594   1.1  jmcneill 	}
   7595   1.1  jmcneill }
   7596   1.1  jmcneill 
   7597   1.1  jmcneill static int
   7598   1.1  jmcneill mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data)
   7599   1.1  jmcneill {
   7600   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   7601   1.1  jmcneill 	struct ifreq *ifr = (struct ifreq *)data;
   7602   1.7  jmcneill 	struct ethercom *ec = &sc->sc_ec;
   7603   1.1  jmcneill 	uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
   7604   1.7  jmcneill 	struct ether_multi *enm;
   7605   1.7  jmcneill 	struct ether_multistep step;
   7606   1.7  jmcneill 	int s, i, flags, error = 0;
   7607  1.15  jmcneill 	uint32_t dest;
   7608   1.1  jmcneill 
   7609   1.1  jmcneill 	s = splnet();
   7610   1.1  jmcneill 	switch (cmd) {
   7611   1.1  jmcneill 
   7612   1.1  jmcneill 	case SIOCADDMULTI:
   7613   1.1  jmcneill 		if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   7614   1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   7615   1.7  jmcneill 			if (error != 0) {
   7616   1.7  jmcneill 				splx(s);
   7617   1.1  jmcneill 				return (error);
   7618   1.7  jmcneill 			}
   7619   1.1  jmcneill 
   7620  1.15  jmcneill 			dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
   7621  1.15  jmcneill 			    sc->sc_rss_flow_table_id;
   7622  1.15  jmcneill 
   7623   1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7624   1.1  jmcneill 				if (sc->sc_mcast_flows[i][0] == 0) {
   7625   1.1  jmcneill 					memcpy(sc->sc_mcast_flows[i], addrlo,
   7626   1.1  jmcneill 					    ETHER_ADDR_LEN);
   7627   1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   7628  1.15  jmcneill 						mcx_set_flow_table_entry_mac(sc,
   7629   1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   7630   1.1  jmcneill 						    sc->sc_mcast_flow_base + i,
   7631  1.15  jmcneill 						    sc->sc_mcast_flows[i], dest);
   7632   1.1  jmcneill 					}
   7633   1.1  jmcneill 					break;
   7634   1.1  jmcneill 				}
   7635   1.1  jmcneill 			}
   7636   1.1  jmcneill 
   7637   1.1  jmcneill 			if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   7638   1.1  jmcneill 				if (i == MCX_NUM_MCAST_FLOWS) {
   7639   1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   7640   1.1  jmcneill 					sc->sc_extra_mcast++;
   7641   1.1  jmcneill 					error = ENETRESET;
   7642   1.1  jmcneill 				}
   7643   1.1  jmcneill 
   7644   1.7  jmcneill 				if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN)) {
   7645   1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   7646   1.1  jmcneill 					error = ENETRESET;
   7647   1.1  jmcneill 				}
   7648   1.1  jmcneill 			}
   7649   1.1  jmcneill 		}
   7650   1.1  jmcneill 		break;
   7651   1.1  jmcneill 
   7652   1.1  jmcneill 	case SIOCDELMULTI:
   7653   1.1  jmcneill 		if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   7654   1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   7655   1.7  jmcneill 			if (error != 0) {
   7656   1.7  jmcneill 				splx(s);
   7657   1.1  jmcneill 				return (error);
   7658   1.7  jmcneill 			}
   7659   1.1  jmcneill 
   7660   1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   7661   1.1  jmcneill 				if (memcmp(sc->sc_mcast_flows[i], addrlo,
   7662   1.1  jmcneill 				    ETHER_ADDR_LEN) == 0) {
   7663   1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   7664   1.1  jmcneill 						mcx_delete_flow_table_entry(sc,
   7665   1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   7666   1.1  jmcneill 						    sc->sc_mcast_flow_base + i);
   7667   1.1  jmcneill 					}
   7668   1.1  jmcneill 					sc->sc_mcast_flows[i][0] = 0;
   7669   1.1  jmcneill 					break;
   7670   1.1  jmcneill 				}
   7671   1.1  jmcneill 			}
   7672   1.1  jmcneill 
   7673   1.1  jmcneill 			if (i == MCX_NUM_MCAST_FLOWS)
   7674   1.1  jmcneill 				sc->sc_extra_mcast--;
   7675   1.1  jmcneill 
   7676   1.1  jmcneill 			if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
   7677   1.7  jmcneill 			    sc->sc_extra_mcast == 0) {
   7678   1.7  jmcneill 				flags = 0;
   7679   1.7  jmcneill 				ETHER_LOCK(ec);
   7680   1.7  jmcneill 				ETHER_FIRST_MULTI(step, ec, enm);
   7681   1.7  jmcneill 				while (enm != NULL) {
   7682   1.7  jmcneill 					if (memcmp(enm->enm_addrlo,
   7683   1.7  jmcneill 					    enm->enm_addrhi, ETHER_ADDR_LEN)) {
   7684   1.7  jmcneill 						SET(flags, IFF_ALLMULTI);
   7685   1.7  jmcneill 						break;
   7686   1.7  jmcneill 					}
   7687   1.7  jmcneill 					ETHER_NEXT_MULTI(step, enm);
   7688   1.7  jmcneill 				}
   7689   1.7  jmcneill 				ETHER_UNLOCK(ec);
   7690   1.7  jmcneill 				if (!ISSET(flags, IFF_ALLMULTI)) {
   7691   1.7  jmcneill 					CLR(ifp->if_flags, IFF_ALLMULTI);
   7692   1.7  jmcneill 					error = ENETRESET;
   7693   1.7  jmcneill 				}
   7694   1.1  jmcneill 			}
   7695   1.1  jmcneill 		}
   7696   1.1  jmcneill 		break;
   7697   1.1  jmcneill 
   7698   1.1  jmcneill 	default:
   7699   1.1  jmcneill 		error = ether_ioctl(ifp, cmd, data);
   7700   1.1  jmcneill 	}
   7701   1.1  jmcneill 
   7702   1.1  jmcneill 	if (error == ENETRESET) {
   7703   1.1  jmcneill 		if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
   7704   1.1  jmcneill 		    (IFF_UP | IFF_RUNNING))
   7705   1.1  jmcneill 			mcx_iff(sc);
   7706   1.1  jmcneill 		error = 0;
   7707   1.1  jmcneill 	}
   7708   1.1  jmcneill 	splx(s);
   7709   1.1  jmcneill 
   7710   1.1  jmcneill 	return (error);
   7711   1.1  jmcneill }
   7712   1.1  jmcneill 
   7713   1.1  jmcneill #if 0
   7714   1.1  jmcneill static int
   7715   1.1  jmcneill mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
   7716   1.1  jmcneill {
   7717   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   7718   1.1  jmcneill 	struct mcx_reg_mcia mcia;
   7719   1.1  jmcneill 	struct mcx_reg_pmlp pmlp;
   7720   1.1  jmcneill 	int offset, error;
   7721   1.1  jmcneill 
   7722   1.1  jmcneill 	/* get module number */
   7723   1.1  jmcneill 	memset(&pmlp, 0, sizeof(pmlp));
   7724   1.1  jmcneill 	pmlp.rp_local_port = 1;
   7725   1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
   7726   1.1  jmcneill 	    sizeof(pmlp));
   7727   1.1  jmcneill 	if (error != 0) {
   7728   1.1  jmcneill 		printf("%s: unable to get eeprom module number\n",
   7729   1.1  jmcneill 		    DEVNAME(sc));
   7730   1.1  jmcneill 		return error;
   7731   1.1  jmcneill 	}
   7732   1.1  jmcneill 
   7733   1.1  jmcneill 	for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
   7734   1.1  jmcneill 		memset(&mcia, 0, sizeof(mcia));
   7735   1.1  jmcneill 		mcia.rm_l = 0;
   7736   1.1  jmcneill 		mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) &
   7737   1.1  jmcneill 		    MCX_PMLP_MODULE_NUM_MASK;
   7738   1.1  jmcneill 		mcia.rm_i2c_addr = sff->sff_addr / 2;	/* apparently */
   7739   1.1  jmcneill 		mcia.rm_page_num = sff->sff_page;
   7740   1.1  jmcneill 		mcia.rm_dev_addr = htobe16(offset);
   7741   1.1  jmcneill 		mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);
   7742   1.1  jmcneill 
   7743   1.1  jmcneill 		error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
   7744   1.1  jmcneill 		    &mcia, sizeof(mcia));
   7745   1.1  jmcneill 		if (error != 0) {
   7746   1.1  jmcneill 			printf("%s: unable to read eeprom at %x\n",
   7747   1.1  jmcneill 			    DEVNAME(sc), offset);
   7748   1.1  jmcneill 			return error;
   7749   1.1  jmcneill 		}
   7750   1.1  jmcneill 
   7751   1.1  jmcneill 		memcpy(sff->sff_data + offset, mcia.rm_data,
   7752   1.1  jmcneill 		    MCX_MCIA_EEPROM_BYTES);
   7753   1.1  jmcneill 	}
   7754   1.1  jmcneill 
   7755   1.1  jmcneill 	return 0;
   7756   1.1  jmcneill }
   7757   1.1  jmcneill #endif
   7758   1.1  jmcneill 
   7759   1.1  jmcneill static int
   7760   1.1  jmcneill mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
   7761   1.1  jmcneill {
   7762   1.1  jmcneill 	switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   7763   1.1  jmcneill 	    BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
   7764   1.1  jmcneill 	case 0:
   7765   1.1  jmcneill 		break;
   7766   1.1  jmcneill 
   7767   1.1  jmcneill 	case EFBIG:
   7768   1.6    nonaka 		if (m_defrag(m, M_DONTWAIT) != NULL &&
   7769   1.1  jmcneill 		    bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   7770   1.1  jmcneill 		    BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
   7771   1.1  jmcneill 			break;
   7772   1.1  jmcneill 
   7773   1.1  jmcneill 		/* FALLTHROUGH */
   7774   1.1  jmcneill 	default:
   7775   1.1  jmcneill 		return (1);
   7776   1.1  jmcneill 	}
   7777   1.1  jmcneill 
   7778   1.1  jmcneill 	ms->ms_m = m;
   7779   1.1  jmcneill 	return (0);
   7780   1.1  jmcneill }
   7781   1.1  jmcneill 
   7782   1.1  jmcneill static void
   7783  1.16  jmcneill mcx_send_common_locked(struct ifnet *ifp, struct mcx_tx *tx, bool is_transmit)
   7784   1.1  jmcneill {
   7785   1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7786   1.1  jmcneill 	struct mcx_sq_entry *sq, *sqe;
   7787   1.1  jmcneill 	struct mcx_sq_entry_seg *sqs;
   7788   1.1  jmcneill 	struct mcx_slot *ms;
   7789   1.1  jmcneill 	bus_dmamap_t map;
   7790   1.1  jmcneill 	struct mbuf *m;
   7791   1.1  jmcneill 	u_int idx, free, used;
   7792   1.1  jmcneill 	uint64_t *bf;
   7793  1.15  jmcneill 	uint32_t csum;
   7794   1.1  jmcneill 	size_t bf_base;
   7795   1.1  jmcneill 	int i, seg, nseg;
   7796   1.1  jmcneill 
   7797  1.16  jmcneill 	KASSERT(mutex_owned(&tx->tx_lock));
   7798  1.16  jmcneill 
   7799  1.16  jmcneill 	if ((ifp->if_flags & IFF_RUNNING) == 0)
   7800  1.16  jmcneill 		return;
   7801  1.16  jmcneill 
   7802  1.15  jmcneill 	bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
   7803   1.1  jmcneill 
   7804  1.15  jmcneill 	idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE);
   7805  1.15  jmcneill 	free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod;
   7806   1.1  jmcneill 
   7807   1.1  jmcneill 	used = 0;
   7808   1.1  jmcneill 	bf = NULL;
   7809  1.15  jmcneill 
   7810  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   7811  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
   7812  1.15  jmcneill 
   7813  1.15  jmcneill 	sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem);
   7814   1.1  jmcneill 
   7815   1.1  jmcneill 	for (;;) {
   7816   1.1  jmcneill 		if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
   7817   1.1  jmcneill 			SET(ifp->if_flags, IFF_OACTIVE);
   7818   1.1  jmcneill 			break;
   7819   1.1  jmcneill 		}
   7820   1.1  jmcneill 
   7821  1.16  jmcneill 		if (is_transmit) {
   7822  1.16  jmcneill 			m = pcq_get(tx->tx_pcq);
   7823  1.16  jmcneill 		} else {
   7824  1.16  jmcneill 			IFQ_DEQUEUE(&ifp->if_snd, m);
   7825  1.16  jmcneill 		}
   7826   1.1  jmcneill 		if (m == NULL) {
   7827   1.1  jmcneill 			break;
   7828   1.1  jmcneill 		}
   7829   1.1  jmcneill 
   7830   1.1  jmcneill 		sqe = sq + idx;
   7831  1.15  jmcneill 		ms = &tx->tx_slots[idx];
   7832   1.1  jmcneill 		memset(sqe, 0, sizeof(*sqe));
   7833   1.1  jmcneill 
   7834   1.1  jmcneill 		/* ctrl segment */
   7835   1.1  jmcneill 		sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
   7836  1.15  jmcneill 		    ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
   7837   1.1  jmcneill 		/* always generate a completion event */
   7838   1.1  jmcneill 		sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);
   7839   1.1  jmcneill 
   7840   1.1  jmcneill 		/* eth segment */
   7841  1.15  jmcneill 		csum = 0;
   7842  1.15  jmcneill 		if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
   7843  1.15  jmcneill 			csum |= MCX_SQE_L3_CSUM;
   7844  1.15  jmcneill 		if (m->m_pkthdr.csum_flags &
   7845  1.15  jmcneill 		    (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6))
   7846  1.15  jmcneill 			csum |= MCX_SQE_L4_CSUM;
   7847  1.15  jmcneill 		sqe->sqe_mss_csum = htobe32(csum);
   7848   1.1  jmcneill 		sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
   7849  1.15  jmcneill 		if (vlan_has_tag(m)) {
   7850  1.15  jmcneill 			struct ether_vlan_header *evh;
   7851  1.15  jmcneill 			evh = (struct ether_vlan_header *)
   7852  1.15  jmcneill 			    &sqe->sqe_inline_headers;
   7853  1.15  jmcneill 
   7854  1.15  jmcneill 			m_copydata(m, 0, ETHER_HDR_LEN, evh);
   7855  1.15  jmcneill 			evh->evl_proto = evh->evl_encap_proto;
   7856  1.15  jmcneill 			evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
   7857  1.15  jmcneill 			evh->evl_tag = htons(vlan_get_tag(m));
   7858  1.15  jmcneill 			m_adj(m, ETHER_HDR_LEN);
   7859  1.15  jmcneill 		} else {
   7860  1.15  jmcneill 			m_copydata(m, 0, MCX_SQ_INLINE_SIZE,
   7861  1.15  jmcneill 			    sqe->sqe_inline_headers);
   7862  1.15  jmcneill 			m_adj(m, MCX_SQ_INLINE_SIZE);
   7863  1.15  jmcneill 		}
   7864   1.1  jmcneill 
   7865   1.1  jmcneill 		if (mcx_load_mbuf(sc, ms, m) != 0) {
   7866   1.1  jmcneill 			m_freem(m);
   7867  1.10   thorpej 			if_statinc(ifp, if_oerrors);
   7868   1.1  jmcneill 			continue;
   7869   1.1  jmcneill 		}
   7870   1.1  jmcneill 		bf = (uint64_t *)sqe;
   7871   1.1  jmcneill 
   7872   1.1  jmcneill 		if (ifp->if_bpf != NULL)
   7873   1.1  jmcneill 			bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers,
   7874   1.1  jmcneill 			    MCX_SQ_INLINE_SIZE, m, BPF_D_OUT);
   7875   1.1  jmcneill 
   7876   1.1  jmcneill 		map = ms->ms_map;
   7877   1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   7878   1.1  jmcneill 		    BUS_DMASYNC_PREWRITE);
   7879   1.1  jmcneill 
   7880   1.1  jmcneill 		sqe->sqe_ds_sq_num =
   7881  1.15  jmcneill 		    htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) |
   7882   1.1  jmcneill 		    (map->dm_nsegs + 3));
   7883   1.1  jmcneill 
   7884   1.1  jmcneill 		/* data segment - first wqe has one segment */
   7885   1.1  jmcneill 		sqs = sqe->sqe_segs;
   7886   1.1  jmcneill 		seg = 0;
   7887   1.1  jmcneill 		nseg = 1;
   7888   1.1  jmcneill 		for (i = 0; i < map->dm_nsegs; i++) {
   7889   1.1  jmcneill 			if (seg == nseg) {
   7890   1.1  jmcneill 				/* next slot */
   7891   1.1  jmcneill 				idx++;
   7892   1.1  jmcneill 				if (idx == (1 << MCX_LOG_SQ_SIZE))
   7893   1.1  jmcneill 					idx = 0;
   7894  1.15  jmcneill 				tx->tx_prod++;
   7895   1.1  jmcneill 				used++;
   7896   1.1  jmcneill 
   7897   1.1  jmcneill 				sqs = (struct mcx_sq_entry_seg *)(sq + idx);
   7898   1.1  jmcneill 				seg = 0;
   7899   1.1  jmcneill 				nseg = MCX_SQ_SEGS_PER_SLOT;
   7900   1.1  jmcneill 			}
   7901   1.1  jmcneill 			sqs[seg].sqs_byte_count =
   7902   1.1  jmcneill 			    htobe32(map->dm_segs[i].ds_len);
   7903   1.1  jmcneill 			sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
   7904   1.1  jmcneill 			sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
   7905   1.1  jmcneill 			seg++;
   7906   1.1  jmcneill 		}
   7907   1.1  jmcneill 
   7908   1.1  jmcneill 		idx++;
   7909   1.1  jmcneill 		if (idx == (1 << MCX_LOG_SQ_SIZE))
   7910   1.1  jmcneill 			idx = 0;
   7911  1.15  jmcneill 		tx->tx_prod++;
   7912   1.1  jmcneill 		used++;
   7913   1.1  jmcneill 	}
   7914   1.1  jmcneill 
   7915  1.15  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
   7916  1.15  jmcneill 	    0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
   7917  1.15  jmcneill 
   7918   1.1  jmcneill 	if (used) {
   7919  1.15  jmcneill 		bus_size_t blueflame;
   7920   1.1  jmcneill 
   7921  1.15  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7922  1.15  jmcneill 		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
   7923  1.15  jmcneill 		be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell),
   7924  1.15  jmcneill 		    tx->tx_prod & MCX_WQ_DOORBELL_MASK);
   7925  1.15  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
   7926  1.15  jmcneill 		    tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
   7927   1.1  jmcneill 
   7928   1.1  jmcneill 		/*
   7929   1.1  jmcneill 		 * write the first 64 bits of the last sqe we produced
   7930   1.1  jmcneill 		 * to the blue flame buffer
   7931   1.1  jmcneill 		 */
   7932  1.15  jmcneill 
   7933  1.15  jmcneill 		blueflame = bf_base + tx->tx_bf_offset;
   7934   1.1  jmcneill 		bus_space_write_8(sc->sc_memt, sc->sc_memh,
   7935  1.15  jmcneill 		    blueflame, *bf);
   7936  1.15  jmcneill 		mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE);
   7937  1.15  jmcneill 
   7938   1.1  jmcneill 		/* next write goes to the other buffer */
   7939  1.15  jmcneill 		tx->tx_bf_offset ^= sc->sc_bf_size;
   7940   1.1  jmcneill 	}
   7941   1.1  jmcneill }
   7942   1.1  jmcneill 
   7943   1.1  jmcneill static void
   7944  1.16  jmcneill mcx_start(struct ifnet *ifp)
   7945  1.16  jmcneill {
   7946  1.16  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7947  1.16  jmcneill 	/* mcx_start() always uses TX ring[0] */
   7948  1.16  jmcneill 	struct mcx_tx *tx = &sc->sc_queues[0].q_tx;
   7949  1.16  jmcneill 
   7950  1.16  jmcneill 	mutex_enter(&tx->tx_lock);
   7951  1.16  jmcneill 	if (!ISSET(ifp->if_flags, IFF_OACTIVE)) {
   7952  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, false);
   7953  1.16  jmcneill 	}
   7954  1.16  jmcneill 	mutex_exit(&tx->tx_lock);
   7955  1.16  jmcneill }
   7956  1.16  jmcneill 
   7957  1.16  jmcneill static int
   7958  1.16  jmcneill mcx_transmit(struct ifnet *ifp, struct mbuf *m)
   7959  1.16  jmcneill {
   7960  1.16  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   7961  1.16  jmcneill 	struct mcx_tx *tx;
   7962  1.16  jmcneill 
   7963  1.16  jmcneill 	tx = &sc->sc_queues[cpu_index(curcpu()) % sc->sc_nqueues].q_tx;
   7964  1.16  jmcneill 	if (__predict_false(!pcq_put(tx->tx_pcq, m))) {
   7965  1.16  jmcneill 		m_freem(m);
   7966  1.16  jmcneill 		return ENOBUFS;
   7967  1.16  jmcneill 	}
   7968  1.16  jmcneill 
   7969  1.16  jmcneill 	if (mutex_tryenter(&tx->tx_lock)) {
   7970  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, true);
   7971  1.16  jmcneill 		mutex_exit(&tx->tx_lock);
   7972  1.16  jmcneill 	} else {
   7973  1.16  jmcneill 		softint_schedule(tx->tx_softint);
   7974  1.16  jmcneill 	}
   7975  1.16  jmcneill 
   7976  1.16  jmcneill 	return 0;
   7977  1.16  jmcneill }
   7978  1.16  jmcneill 
   7979  1.16  jmcneill static void
   7980  1.16  jmcneill mcx_deferred_transmit(void *arg)
   7981  1.16  jmcneill {
   7982  1.16  jmcneill 	struct mcx_tx *tx = arg;
   7983  1.16  jmcneill 	struct mcx_softc *sc = tx->tx_softc;
   7984  1.16  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   7985  1.16  jmcneill 
   7986  1.16  jmcneill 	mutex_enter(&tx->tx_lock);
   7987  1.16  jmcneill 	if (pcq_peek(tx->tx_pcq) != NULL) {
   7988  1.16  jmcneill 		mcx_send_common_locked(ifp, tx, true);
   7989  1.16  jmcneill 	}
   7990  1.16  jmcneill 	mutex_exit(&tx->tx_lock);
   7991  1.16  jmcneill }
   7992  1.16  jmcneill 
   7993  1.16  jmcneill static void
   7994   1.1  jmcneill mcx_watchdog(struct ifnet *ifp)
   7995   1.1  jmcneill {
   7996   1.1  jmcneill }
   7997   1.1  jmcneill 
   7998   1.1  jmcneill static void
   7999   1.1  jmcneill mcx_media_add_types(struct mcx_softc *sc)
   8000   1.1  jmcneill {
   8001   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8002   1.1  jmcneill 	int i;
   8003   1.1  jmcneill 	uint32_t proto_cap;
   8004   1.1  jmcneill 
   8005   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8006   1.1  jmcneill 	ptys.rp_local_port = 1;
   8007   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8008   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8009   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8010   1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   8011   1.1  jmcneill 		return;
   8012   1.1  jmcneill 	}
   8013   1.1  jmcneill 
   8014   1.1  jmcneill 	proto_cap = be32toh(ptys.rp_eth_proto_cap);
   8015   1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8016  1.13  jmcneill 		const struct mcx_eth_proto_capability *cap;
   8017  1.13  jmcneill 		if (!ISSET(proto_cap, 1U << i))
   8018  1.13  jmcneill 			continue;
   8019  1.13  jmcneill 
   8020  1.13  jmcneill 		cap = &mcx_eth_cap_map[i];
   8021  1.13  jmcneill 		if (cap->cap_media == 0)
   8022  1.13  jmcneill 			continue;
   8023  1.13  jmcneill 
   8024  1.13  jmcneill 		ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
   8025   1.1  jmcneill 	}
   8026   1.1  jmcneill }
   8027   1.1  jmcneill 
   8028   1.1  jmcneill static void
   8029   1.1  jmcneill mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
   8030   1.1  jmcneill {
   8031   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   8032   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8033   1.1  jmcneill 	int i;
   8034  1.13  jmcneill 	uint32_t proto_oper;
   8035   1.1  jmcneill 	uint64_t media_oper;
   8036   1.1  jmcneill 
   8037   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8038   1.1  jmcneill 	ptys.rp_local_port = 1;
   8039   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8040   1.1  jmcneill 
   8041   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8042   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8043   1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   8044   1.1  jmcneill 		return;
   8045   1.1  jmcneill 	}
   8046   1.1  jmcneill 
   8047   1.1  jmcneill 	proto_oper = be32toh(ptys.rp_eth_proto_oper);
   8048   1.1  jmcneill 
   8049   1.1  jmcneill 	media_oper = 0;
   8050  1.13  jmcneill 
   8051   1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8052  1.13  jmcneill 		const struct mcx_eth_proto_capability *cap;
   8053  1.13  jmcneill 		if (!ISSET(proto_oper, 1U << i))
   8054  1.13  jmcneill 			continue;
   8055  1.13  jmcneill 
   8056  1.13  jmcneill 		cap = &mcx_eth_cap_map[i];
   8057  1.13  jmcneill 
   8058  1.13  jmcneill 		if (cap->cap_media != 0)
   8059  1.13  jmcneill 			media_oper = cap->cap_media;
   8060   1.1  jmcneill 	}
   8061   1.1  jmcneill 
   8062   1.1  jmcneill 	ifmr->ifm_status = IFM_AVALID;
   8063   1.1  jmcneill 	if (proto_oper != 0) {
   8064   1.1  jmcneill 		ifmr->ifm_status |= IFM_ACTIVE;
   8065   1.1  jmcneill 		ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper;
   8066   1.1  jmcneill 		/* txpause, rxpause, duplex? */
   8067   1.1  jmcneill 	}
   8068   1.1  jmcneill }
   8069   1.1  jmcneill 
   8070   1.1  jmcneill static int
   8071   1.1  jmcneill mcx_media_change(struct ifnet *ifp)
   8072   1.1  jmcneill {
   8073   1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   8074   1.1  jmcneill 	struct mcx_reg_ptys ptys;
   8075   1.1  jmcneill 	struct mcx_reg_paos paos;
   8076   1.1  jmcneill 	uint32_t media;
   8077   1.1  jmcneill 	int i, error;
   8078   1.1  jmcneill 
   8079   1.1  jmcneill 	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
   8080   1.1  jmcneill 		return EINVAL;
   8081   1.1  jmcneill 
   8082   1.1  jmcneill 	error = 0;
   8083   1.1  jmcneill 
   8084   1.1  jmcneill 	if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
   8085   1.1  jmcneill 		/* read ptys to get supported media */
   8086   1.1  jmcneill 		memset(&ptys, 0, sizeof(ptys));
   8087   1.1  jmcneill 		ptys.rp_local_port = 1;
   8088   1.1  jmcneill 		ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8089   1.1  jmcneill 		if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
   8090   1.1  jmcneill 		    &ptys, sizeof(ptys)) != 0) {
   8091   1.1  jmcneill 			printf("%s: unable to read port type/speed\n",
   8092   1.1  jmcneill 			    DEVNAME(sc));
   8093   1.1  jmcneill 			return EIO;
   8094   1.1  jmcneill 		}
   8095   1.1  jmcneill 
   8096   1.1  jmcneill 		media = be32toh(ptys.rp_eth_proto_cap);
   8097   1.1  jmcneill 	} else {
   8098   1.1  jmcneill 		/* map media type */
   8099   1.1  jmcneill 		media = 0;
   8100   1.1  jmcneill 		for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8101  1.13  jmcneill 			const struct mcx_eth_proto_capability *cap;
   8102  1.13  jmcneill 
   8103  1.13  jmcneill 			cap = &mcx_eth_cap_map[i];
   8104  1.13  jmcneill 			if (cap->cap_media ==
   8105   1.1  jmcneill 			    IFM_SUBTYPE(sc->sc_media.ifm_media)) {
   8106   1.1  jmcneill 				media = (1 << i);
   8107   1.1  jmcneill 				break;
   8108   1.1  jmcneill 			}
   8109   1.1  jmcneill 		}
   8110   1.1  jmcneill 	}
   8111   1.1  jmcneill 
   8112   1.1  jmcneill 	/* disable the port */
   8113   1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   8114   1.1  jmcneill 	paos.rp_local_port = 1;
   8115   1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
   8116   1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   8117   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   8118   1.1  jmcneill 	    sizeof(paos)) != 0) {
   8119   1.1  jmcneill 		printf("%s: unable to set port state to down\n", DEVNAME(sc));
   8120   1.1  jmcneill 		return EIO;
   8121   1.1  jmcneill 	}
   8122   1.1  jmcneill 
   8123   1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   8124   1.1  jmcneill 	ptys.rp_local_port = 1;
   8125   1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   8126   1.1  jmcneill 	ptys.rp_eth_proto_admin = htobe32(media);
   8127   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
   8128   1.1  jmcneill 	    sizeof(ptys)) != 0) {
   8129   1.1  jmcneill 		printf("%s: unable to set port media type/speed\n",
   8130   1.1  jmcneill 		    DEVNAME(sc));
   8131   1.1  jmcneill 		error = EIO;
   8132   1.1  jmcneill 	}
   8133   1.1  jmcneill 
   8134   1.1  jmcneill 	/* re-enable the port to start negotiation */
   8135   1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   8136   1.1  jmcneill 	paos.rp_local_port = 1;
   8137   1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
   8138   1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   8139   1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   8140   1.1  jmcneill 	    sizeof(paos)) != 0) {
   8141   1.1  jmcneill 		printf("%s: unable to set port state to up\n", DEVNAME(sc));
   8142   1.1  jmcneill 		error = EIO;
   8143   1.1  jmcneill 	}
   8144   1.1  jmcneill 
   8145   1.1  jmcneill 	return error;
   8146   1.1  jmcneill }
   8147   1.1  jmcneill 
   8148   1.1  jmcneill static void
   8149   1.1  jmcneill mcx_port_change(struct work *wk, void *xsc)
   8150   1.1  jmcneill {
   8151   1.1  jmcneill 	struct mcx_softc *sc = xsc;
   8152   1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   8153  1.13  jmcneill 	struct mcx_reg_ptys ptys = {
   8154  1.13  jmcneill 		.rp_local_port = 1,
   8155  1.13  jmcneill 		.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH,
   8156  1.13  jmcneill 	};
   8157   1.1  jmcneill 	int link_state = LINK_STATE_DOWN;
   8158   1.1  jmcneill 
   8159  1.13  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   8160  1.13  jmcneill 	    sizeof(ptys)) == 0) {
   8161  1.13  jmcneill 		uint32_t proto_oper = be32toh(ptys.rp_eth_proto_oper);
   8162  1.13  jmcneill 		uint64_t baudrate = 0;
   8163  1.13  jmcneill 		unsigned int i;
   8164  1.13  jmcneill 
   8165  1.15  jmcneill 		if (proto_oper != 0)
   8166  1.15  jmcneill 			link_state = LINK_STATE_UP;
   8167  1.15  jmcneill 
   8168  1.13  jmcneill 		for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   8169  1.13  jmcneill 			const struct mcx_eth_proto_capability *cap;
   8170  1.13  jmcneill 			if (!ISSET(proto_oper, 1U << i))
   8171  1.13  jmcneill 				continue;
   8172  1.13  jmcneill 
   8173  1.13  jmcneill 			cap = &mcx_eth_cap_map[i];
   8174  1.13  jmcneill 			if (cap->cap_baudrate == 0)
   8175  1.13  jmcneill 				continue;
   8176  1.13  jmcneill 
   8177  1.13  jmcneill 			baudrate = cap->cap_baudrate;
   8178  1.13  jmcneill 			break;
   8179  1.13  jmcneill 		}
   8180  1.13  jmcneill 
   8181  1.13  jmcneill 		ifp->if_baudrate = baudrate;
   8182   1.1  jmcneill 	}
   8183   1.1  jmcneill 
   8184   1.1  jmcneill 	if (link_state != ifp->if_link_state) {
   8185   1.1  jmcneill 		if_link_state_change(ifp, link_state);
   8186   1.1  jmcneill 	}
   8187   1.1  jmcneill }
   8188   1.1  jmcneill 
   8189   1.1  jmcneill 
   8190   1.1  jmcneill static inline uint32_t
   8191   1.1  jmcneill mcx_rd(struct mcx_softc *sc, bus_size_t r)
   8192   1.1  jmcneill {
   8193   1.1  jmcneill 	uint32_t word;
   8194   1.1  jmcneill 
   8195   1.1  jmcneill 	word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r);
   8196   1.1  jmcneill 
   8197   1.1  jmcneill 	return (be32toh(word));
   8198   1.1  jmcneill }
   8199   1.1  jmcneill 
   8200   1.1  jmcneill static inline void
   8201   1.1  jmcneill mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
   8202   1.1  jmcneill {
   8203   1.1  jmcneill 	bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
   8204   1.1  jmcneill }
   8205   1.1  jmcneill 
   8206   1.1  jmcneill static inline void
   8207   1.1  jmcneill mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
   8208   1.1  jmcneill {
   8209   1.1  jmcneill 	bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
   8210   1.1  jmcneill }
   8211   1.1  jmcneill 
   8212   1.1  jmcneill static uint64_t
   8213   1.1  jmcneill mcx_timer(struct mcx_softc *sc)
   8214   1.1  jmcneill {
   8215   1.1  jmcneill 	uint32_t hi, lo, ni;
   8216   1.1  jmcneill 
   8217   1.1  jmcneill 	hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   8218   1.1  jmcneill 	for (;;) {
   8219   1.1  jmcneill 		lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
   8220   1.1  jmcneill 		mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
   8221   1.1  jmcneill 		ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   8222   1.1  jmcneill 
   8223   1.1  jmcneill 		if (ni == hi)
   8224   1.1  jmcneill 			break;
   8225   1.1  jmcneill 
   8226   1.1  jmcneill 		hi = ni;
   8227   1.1  jmcneill 	}
   8228   1.1  jmcneill 
   8229   1.1  jmcneill 	return (((uint64_t)hi << 32) | (uint64_t)lo);
   8230   1.1  jmcneill }
   8231   1.1  jmcneill 
   8232   1.1  jmcneill static int
   8233   1.1  jmcneill mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   8234   1.1  jmcneill     bus_size_t size, u_int align)
   8235   1.1  jmcneill {
   8236   1.1  jmcneill 	mxm->mxm_size = size;
   8237   1.1  jmcneill 
   8238   1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
   8239   1.1  jmcneill 	    mxm->mxm_size, 0,
   8240   1.1  jmcneill 	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   8241   1.1  jmcneill 	    &mxm->mxm_map) != 0)
   8242   1.1  jmcneill 		return (1);
   8243   1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
   8244   1.1  jmcneill 	    align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
   8245   1.1  jmcneill 	    BUS_DMA_WAITOK) != 0)
   8246   1.1  jmcneill 		goto destroy;
   8247   1.1  jmcneill 	if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
   8248   1.1  jmcneill 	    mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0)
   8249   1.1  jmcneill 		goto free;
   8250   1.1  jmcneill 	if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
   8251   1.1  jmcneill 	    mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
   8252   1.1  jmcneill 		goto unmap;
   8253   1.1  jmcneill 
   8254   1.1  jmcneill 	mcx_dmamem_zero(mxm);
   8255   1.1  jmcneill 
   8256   1.1  jmcneill 	return (0);
   8257   1.1  jmcneill unmap:
   8258   1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   8259   1.1  jmcneill free:
   8260   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   8261   1.1  jmcneill destroy:
   8262   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   8263   1.1  jmcneill 	return (1);
   8264   1.1  jmcneill }
   8265   1.1  jmcneill 
   8266   1.1  jmcneill static void
   8267   1.1  jmcneill mcx_dmamem_zero(struct mcx_dmamem *mxm)
   8268   1.1  jmcneill {
   8269   1.1  jmcneill 	memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
   8270   1.1  jmcneill }
   8271   1.1  jmcneill 
   8272   1.1  jmcneill static void
   8273   1.1  jmcneill mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   8274   1.1  jmcneill {
   8275   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
   8276   1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   8277   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   8278   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   8279   1.1  jmcneill }
   8280   1.1  jmcneill 
   8281   1.1  jmcneill static int
   8282   1.1  jmcneill mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
   8283   1.1  jmcneill {
   8284   1.1  jmcneill 	bus_dma_segment_t *segs;
   8285   1.1  jmcneill 	bus_size_t len = pages * MCX_PAGE_SIZE;
   8286   1.1  jmcneill 	size_t seglen;
   8287   1.1  jmcneill 
   8288   1.1  jmcneill 	segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP);
   8289   1.1  jmcneill 	seglen = sizeof(*segs) * pages;
   8290   1.1  jmcneill 
   8291   1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
   8292   1.1  jmcneill 	    segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
   8293   1.1  jmcneill 		goto free_segs;
   8294   1.1  jmcneill 
   8295   1.1  jmcneill 	if (mhm->mhm_seg_count < pages) {
   8296   1.1  jmcneill 		size_t nseglen;
   8297   1.1  jmcneill 
   8298   1.1  jmcneill 		mhm->mhm_segs = kmem_alloc(
   8299   1.1  jmcneill 		    sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP);
   8300   1.1  jmcneill 
   8301   1.1  jmcneill 		nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
   8302   1.1  jmcneill 
   8303   1.1  jmcneill 		memcpy(mhm->mhm_segs, segs, nseglen);
   8304   1.1  jmcneill 
   8305   1.1  jmcneill 		kmem_free(segs, seglen);
   8306   1.1  jmcneill 
   8307   1.1  jmcneill 		segs = mhm->mhm_segs;
   8308   1.1  jmcneill 		seglen = nseglen;
   8309   1.1  jmcneill 	} else
   8310   1.1  jmcneill 		mhm->mhm_segs = segs;
   8311   1.1  jmcneill 
   8312   1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
   8313   1.1  jmcneill 	    MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
   8314   1.1  jmcneill 	    &mhm->mhm_map) != 0)
   8315   1.1  jmcneill 		goto free_dmamem;
   8316   1.1  jmcneill 
   8317   1.1  jmcneill 	if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
   8318   1.1  jmcneill 	    mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
   8319   1.1  jmcneill 		goto destroy;
   8320   1.1  jmcneill 
   8321   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   8322   1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
   8323   1.1  jmcneill 
   8324   1.1  jmcneill 	mhm->mhm_npages = pages;
   8325   1.1  jmcneill 
   8326   1.1  jmcneill 	return (0);
   8327   1.1  jmcneill 
   8328   1.1  jmcneill destroy:
   8329   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   8330   1.1  jmcneill free_dmamem:
   8331   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   8332   1.1  jmcneill free_segs:
   8333   1.1  jmcneill 	kmem_free(segs, seglen);
   8334   1.1  jmcneill 	mhm->mhm_segs = NULL;
   8335   1.1  jmcneill 
   8336   1.1  jmcneill 	return (-1);
   8337   1.1  jmcneill }
   8338   1.1  jmcneill 
   8339   1.1  jmcneill static void
   8340   1.1  jmcneill mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
   8341   1.1  jmcneill {
   8342   1.1  jmcneill 	if (mhm->mhm_npages == 0)
   8343   1.1  jmcneill 		return;
   8344   1.1  jmcneill 
   8345   1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   8346   1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
   8347   1.1  jmcneill 
   8348   1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
   8349   1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   8350   1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   8351   1.1  jmcneill 	kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
   8352   1.1  jmcneill 
   8353   1.1  jmcneill 	mhm->mhm_npages = 0;
   8354   1.1  jmcneill }
   8355  1.15  jmcneill 
   8356  1.15  jmcneill #if NKSTAT > 0
   8357  1.15  jmcneill struct mcx_ppcnt {
   8358  1.15  jmcneill 	char			 name[KSTAT_KV_NAMELEN];
   8359  1.15  jmcneill 	enum kstat_kv_unit	 unit;
   8360  1.15  jmcneill };
   8361  1.15  jmcneill 
   8362  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = {
   8363  1.15  jmcneill 	{ "Good Tx",		KSTAT_KV_U_PACKETS, },
   8364  1.15  jmcneill 	{ "Good Rx",		KSTAT_KV_U_PACKETS, },
   8365  1.15  jmcneill 	{ "FCS errs",		KSTAT_KV_U_PACKETS, },
   8366  1.15  jmcneill 	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
   8367  1.15  jmcneill 	{ "Good Tx",		KSTAT_KV_U_BYTES, },
   8368  1.15  jmcneill 	{ "Good Rx",		KSTAT_KV_U_BYTES, },
   8369  1.15  jmcneill 	{ "Multicast Tx",	KSTAT_KV_U_PACKETS, },
   8370  1.15  jmcneill 	{ "Broadcast Tx",	KSTAT_KV_U_PACKETS, },
   8371  1.15  jmcneill 	{ "Multicast Rx",	KSTAT_KV_U_PACKETS, },
   8372  1.15  jmcneill 	{ "Broadcast Rx",	KSTAT_KV_U_PACKETS, },
   8373  1.15  jmcneill 	{ "In Range Len",	KSTAT_KV_U_PACKETS, },
   8374  1.15  jmcneill 	{ "Out Of Range Len",	KSTAT_KV_U_PACKETS, },
   8375  1.15  jmcneill 	{ "Frame Too Long",	KSTAT_KV_U_PACKETS, },
   8376  1.15  jmcneill 	{ "Symbol Errs",	KSTAT_KV_U_PACKETS, },
   8377  1.15  jmcneill 	{ "MAC Ctrl Tx",	KSTAT_KV_U_PACKETS, },
   8378  1.15  jmcneill 	{ "MAC Ctrl Rx",	KSTAT_KV_U_PACKETS, },
   8379  1.15  jmcneill 	{ "MAC Ctrl Unsup",	KSTAT_KV_U_PACKETS, },
   8380  1.15  jmcneill 	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
   8381  1.15  jmcneill 	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
   8382  1.15  jmcneill };
   8383  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count);
   8384  1.15  jmcneill 
   8385  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = {
   8386  1.15  jmcneill 	{ "Rx Bytes",		KSTAT_KV_U_BYTES, },
   8387  1.15  jmcneill 	{ "Rx Unicast",		KSTAT_KV_U_PACKETS, },
   8388  1.15  jmcneill 	{ "Rx Discards",	KSTAT_KV_U_PACKETS, },
   8389  1.15  jmcneill 	{ "Rx Errors",		KSTAT_KV_U_PACKETS, },
   8390  1.15  jmcneill 	{ "Rx Unknown Proto",	KSTAT_KV_U_PACKETS, },
   8391  1.15  jmcneill 	{ "Tx Bytes",		KSTAT_KV_U_BYTES, },
   8392  1.15  jmcneill 	{ "Tx Unicast",		KSTAT_KV_U_PACKETS, },
   8393  1.15  jmcneill 	{ "Tx Discards",	KSTAT_KV_U_PACKETS, },
   8394  1.15  jmcneill 	{ "Tx Errors",		KSTAT_KV_U_PACKETS, },
   8395  1.15  jmcneill 	{ "Rx Multicast",	KSTAT_KV_U_PACKETS, },
   8396  1.15  jmcneill 	{ "Rx Broadcast",	KSTAT_KV_U_PACKETS, },
   8397  1.15  jmcneill 	{ "Tx Multicast",	KSTAT_KV_U_PACKETS, },
   8398  1.15  jmcneill 	{ "Tx Broadcast",	KSTAT_KV_U_PACKETS, },
   8399  1.15  jmcneill };
   8400  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count);
   8401  1.15  jmcneill 
   8402  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = {
   8403  1.15  jmcneill 	{ "Drop Events",	KSTAT_KV_U_PACKETS, },
   8404  1.15  jmcneill 	{ "Octets",		KSTAT_KV_U_BYTES, },
   8405  1.15  jmcneill 	{ "Packets",		KSTAT_KV_U_PACKETS, },
   8406  1.15  jmcneill 	{ "Broadcasts",		KSTAT_KV_U_PACKETS, },
   8407  1.15  jmcneill 	{ "Multicasts",		KSTAT_KV_U_PACKETS, },
   8408  1.15  jmcneill 	{ "CRC Align Errs",	KSTAT_KV_U_PACKETS, },
   8409  1.15  jmcneill 	{ "Undersize",		KSTAT_KV_U_PACKETS, },
   8410  1.15  jmcneill 	{ "Oversize",		KSTAT_KV_U_PACKETS, },
   8411  1.15  jmcneill 	{ "Fragments",		KSTAT_KV_U_PACKETS, },
   8412  1.15  jmcneill 	{ "Jabbers",		KSTAT_KV_U_PACKETS, },
   8413  1.15  jmcneill 	{ "Collisions",		KSTAT_KV_U_NONE, },
   8414  1.15  jmcneill 	{ "64B",		KSTAT_KV_U_PACKETS, },
   8415  1.15  jmcneill 	{ "65-127B",		KSTAT_KV_U_PACKETS, },
   8416  1.15  jmcneill 	{ "128-255B",		KSTAT_KV_U_PACKETS, },
   8417  1.15  jmcneill 	{ "256-511B",		KSTAT_KV_U_PACKETS, },
   8418  1.15  jmcneill 	{ "512-1023B",		KSTAT_KV_U_PACKETS, },
   8419  1.15  jmcneill 	{ "1024-1518B",		KSTAT_KV_U_PACKETS, },
   8420  1.15  jmcneill 	{ "1519-2047B",		KSTAT_KV_U_PACKETS, },
   8421  1.15  jmcneill 	{ "2048-4095B",		KSTAT_KV_U_PACKETS, },
   8422  1.15  jmcneill 	{ "4096-8191B",		KSTAT_KV_U_PACKETS, },
   8423  1.15  jmcneill 	{ "8192-10239B",	KSTAT_KV_U_PACKETS, },
   8424  1.15  jmcneill };
   8425  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count);
   8426  1.15  jmcneill 
   8427  1.15  jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = {
   8428  1.15  jmcneill 	{ "Alignment Errs",	KSTAT_KV_U_PACKETS, },
   8429  1.15  jmcneill 	{ "FCS Errs",		KSTAT_KV_U_PACKETS, },
   8430  1.15  jmcneill 	{ "Single Colls",	KSTAT_KV_U_PACKETS, },
   8431  1.15  jmcneill 	{ "Multiple Colls",	KSTAT_KV_U_PACKETS, },
   8432  1.15  jmcneill 	{ "SQE Test Errs",	KSTAT_KV_U_NONE, },
   8433  1.15  jmcneill 	{ "Deferred Tx",	KSTAT_KV_U_PACKETS, },
   8434  1.15  jmcneill 	{ "Late Colls",		KSTAT_KV_U_NONE, },
   8435  1.15  jmcneill 	{ "Exess Colls",	KSTAT_KV_U_NONE, },
   8436  1.15  jmcneill 	{ "Int MAC Tx Errs",	KSTAT_KV_U_PACKETS, },
   8437  1.15  jmcneill 	{ "CSM Sense Errs",	KSTAT_KV_U_NONE, },
   8438  1.15  jmcneill 	{ "Too Long",		KSTAT_KV_U_PACKETS, },
   8439  1.15  jmcneill 	{ "Int MAC Rx Errs",	KSTAT_KV_U_PACKETS, },
   8440  1.15  jmcneill 	{ "Symbol Errs",	KSTAT_KV_U_NONE, },
   8441  1.15  jmcneill 	{ "Unknown Control",	KSTAT_KV_U_PACKETS, },
   8442  1.15  jmcneill 	{ "Pause Rx",		KSTAT_KV_U_PACKETS, },
   8443  1.15  jmcneill 	{ "Pause Tx",		KSTAT_KV_U_PACKETS, },
   8444  1.15  jmcneill };
   8445  1.15  jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count);
   8446  1.15  jmcneill 
   8447  1.15  jmcneill struct mcx_kstat_ppcnt {
   8448  1.15  jmcneill 	const char		*ksp_name;
   8449  1.15  jmcneill 	const struct mcx_ppcnt	*ksp_tpl;
   8450  1.15  jmcneill 	unsigned int		 ksp_n;
   8451  1.15  jmcneill 	uint8_t			 ksp_grp;
   8452  1.15  jmcneill };
   8453  1.15  jmcneill 
   8454  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = {
   8455  1.15  jmcneill 	.ksp_name =		"ieee802.3",
   8456  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_ieee8023_tpl,
   8457  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_ieee8023_tpl),
   8458  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_IEEE8023,
   8459  1.15  jmcneill };
   8460  1.15  jmcneill 
   8461  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = {
   8462  1.15  jmcneill 	.ksp_name =		"rfc2863",
   8463  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc2863_tpl,
   8464  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc2863_tpl),
   8465  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2863,
   8466  1.15  jmcneill };
   8467  1.15  jmcneill 
   8468  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = {
   8469  1.15  jmcneill 	.ksp_name =		"rfc2819",
   8470  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc2819_tpl,
   8471  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc2819_tpl),
   8472  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC2819,
   8473  1.15  jmcneill };
   8474  1.15  jmcneill 
   8475  1.15  jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = {
   8476  1.15  jmcneill 	.ksp_name =		"rfc3635",
   8477  1.15  jmcneill 	.ksp_tpl =		mcx_ppcnt_rfc3635_tpl,
   8478  1.15  jmcneill 	.ksp_n =		__arraycount(mcx_ppcnt_rfc3635_tpl),
   8479  1.15  jmcneill 	.ksp_grp =		MCX_REG_PPCNT_GRP_RFC3635,
   8480  1.15  jmcneill };
   8481  1.15  jmcneill 
   8482  1.15  jmcneill static int	mcx_kstat_ppcnt_read(struct kstat *);
   8483  1.15  jmcneill 
   8484  1.15  jmcneill static void	mcx_kstat_attach_tmps(struct mcx_softc *sc);
   8485  1.15  jmcneill static void	mcx_kstat_attach_queues(struct mcx_softc *sc);
   8486  1.15  jmcneill 
   8487  1.15  jmcneill static struct kstat *
   8488  1.15  jmcneill mcx_kstat_attach_ppcnt(struct mcx_softc *sc,
   8489  1.15  jmcneill     const struct mcx_kstat_ppcnt *ksp)
   8490  1.15  jmcneill {
   8491  1.15  jmcneill 	struct kstat *ks;
   8492  1.15  jmcneill 	struct kstat_kv *kvs;
   8493  1.15  jmcneill 	unsigned int i;
   8494  1.15  jmcneill 
   8495  1.15  jmcneill 	ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0);
   8496  1.15  jmcneill 	if (ks == NULL)
   8497  1.15  jmcneill 		return (NULL);
   8498  1.15  jmcneill 
   8499  1.15  jmcneill 	kvs = mallocarray(ksp->ksp_n, sizeof(*kvs),
   8500  1.15  jmcneill 	    M_DEVBUF, M_WAITOK);
   8501  1.15  jmcneill 
   8502  1.15  jmcneill 	for (i = 0; i < ksp->ksp_n; i++) {
   8503  1.15  jmcneill 		const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i];
   8504  1.15  jmcneill 
   8505  1.15  jmcneill 		kstat_kv_unit_init(&kvs[i], tpl->name,
   8506  1.15  jmcneill 		    KSTAT_KV_T_COUNTER64, tpl->unit);
   8507  1.15  jmcneill 	}
   8508  1.15  jmcneill 
   8509  1.15  jmcneill 	ks->ks_softc = sc;
   8510  1.15  jmcneill 	ks->ks_ptr = (void *)ksp;
   8511  1.15  jmcneill 	ks->ks_data = kvs;
   8512  1.15  jmcneill 	ks->ks_datalen = ksp->ksp_n * sizeof(*kvs);
   8513  1.15  jmcneill 	ks->ks_read = mcx_kstat_ppcnt_read;
   8514  1.15  jmcneill 
   8515  1.15  jmcneill 	kstat_install(ks);
   8516  1.15  jmcneill 
   8517  1.15  jmcneill 	return (ks);
   8518  1.15  jmcneill }
   8519  1.15  jmcneill 
   8520  1.15  jmcneill static void
   8521  1.15  jmcneill mcx_kstat_attach(struct mcx_softc *sc)
   8522  1.15  jmcneill {
   8523  1.15  jmcneill 	sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc,
   8524  1.15  jmcneill 	    &mcx_kstat_ppcnt_ieee8023);
   8525  1.15  jmcneill 	sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc,
   8526  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc2863);
   8527  1.15  jmcneill 	sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc,
   8528  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc2819);
   8529  1.15  jmcneill 	sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc,
   8530  1.15  jmcneill 	    &mcx_kstat_ppcnt_rfc3635);
   8531  1.15  jmcneill 
   8532  1.15  jmcneill 	mcx_kstat_attach_tmps(sc);
   8533  1.15  jmcneill 	mcx_kstat_attach_queues(sc);
   8534  1.15  jmcneill }
   8535  1.15  jmcneill 
   8536  1.15  jmcneill static int
   8537  1.15  jmcneill mcx_kstat_ppcnt_read(struct kstat *ks)
   8538  1.15  jmcneill {
   8539  1.15  jmcneill 	struct mcx_softc *sc = ks->ks_softc;
   8540  1.15  jmcneill 	struct mcx_kstat_ppcnt *ksp = ks->ks_ptr;
   8541  1.15  jmcneill 	struct mcx_reg_ppcnt ppcnt = {
   8542  1.15  jmcneill 		.ppcnt_grp = ksp->ksp_grp,
   8543  1.15  jmcneill 		.ppcnt_local_port = 1,
   8544  1.15  jmcneill 	};
   8545  1.15  jmcneill 	struct kstat_kv *kvs = ks->ks_data;
   8546  1.15  jmcneill 	uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set;
   8547  1.15  jmcneill 	unsigned int i;
   8548  1.15  jmcneill 	int rv;
   8549  1.15  jmcneill 
   8550  1.15  jmcneill 	KERNEL_LOCK(); /* XXX */
   8551  1.15  jmcneill 	rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ,
   8552  1.15  jmcneill 	    &ppcnt, sizeof(ppcnt));
   8553  1.15  jmcneill 	KERNEL_UNLOCK();
   8554  1.15  jmcneill 	if (rv != 0)
   8555  1.15  jmcneill 		return (EIO);
   8556  1.15  jmcneill 
   8557  1.15  jmcneill 	nanouptime(&ks->ks_updated);
   8558  1.15  jmcneill 
   8559  1.15  jmcneill 	for (i = 0; i < ksp->ksp_n; i++)
   8560  1.15  jmcneill 		kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]);
   8561  1.15  jmcneill 
   8562  1.15  jmcneill 	return (0);
   8563  1.15  jmcneill }
   8564  1.15  jmcneill 
   8565  1.15  jmcneill struct mcx_kstat_mtmp {
   8566  1.15  jmcneill 	struct kstat_kv		ktmp_name;
   8567  1.15  jmcneill 	struct kstat_kv		ktmp_temperature;
   8568  1.15  jmcneill 	struct kstat_kv		ktmp_threshold_lo;
   8569  1.15  jmcneill 	struct kstat_kv		ktmp_threshold_hi;
   8570  1.15  jmcneill };
   8571  1.15  jmcneill 
   8572  1.15  jmcneill static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = {
   8573  1.15  jmcneill 	KSTAT_KV_INITIALIZER("name",		KSTAT_KV_T_ISTR),
   8574  1.15  jmcneill 	KSTAT_KV_INITIALIZER("temperature",	KSTAT_KV_T_TEMP),
   8575  1.15  jmcneill 	KSTAT_KV_INITIALIZER("lo threshold",	KSTAT_KV_T_TEMP),
   8576  1.15  jmcneill 	KSTAT_KV_INITIALIZER("hi threshold",	KSTAT_KV_T_TEMP),
   8577  1.15  jmcneill };
   8578  1.15  jmcneill 
   8579  1.15  jmcneill static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 };
   8580  1.15  jmcneill 
   8581  1.15  jmcneill static int mcx_kstat_mtmp_read(struct kstat *);
   8582  1.15  jmcneill 
   8583  1.15  jmcneill static void
   8584  1.15  jmcneill mcx_kstat_attach_tmps(struct mcx_softc *sc)
   8585  1.15  jmcneill {
   8586  1.15  jmcneill 	struct kstat *ks;
   8587  1.15  jmcneill 	struct mcx_reg_mcam mcam;
   8588  1.15  jmcneill 	struct mcx_reg_mtcap mtcap;
   8589  1.15  jmcneill 	struct mcx_kstat_mtmp *ktmp;
   8590  1.15  jmcneill 	uint64_t map;
   8591  1.15  jmcneill 	unsigned int i, n;
   8592  1.15  jmcneill 
   8593  1.15  jmcneill 	memset(&mtcap, 0, sizeof(mtcap));
   8594  1.15  jmcneill 	memset(&mcam, 0, sizeof(mcam));
   8595  1.15  jmcneill 
   8596  1.15  jmcneill 	if (sc->sc_mcam_reg == 0) {
   8597  1.15  jmcneill 		/* no management capabilities */
   8598  1.15  jmcneill 		return;
   8599  1.15  jmcneill 	}
   8600  1.15  jmcneill 
   8601  1.15  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ,
   8602  1.15  jmcneill 	    &mcam, sizeof(mcam)) != 0) {
   8603  1.15  jmcneill 		/* unable to check management capabilities? */
   8604  1.15  jmcneill 		return;
   8605  1.15  jmcneill 	}
   8606  1.15  jmcneill 
   8607  1.15  jmcneill 	if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask,
   8608  1.15  jmcneill 	    MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) {
   8609  1.15  jmcneill 		/* no sensor map */
   8610  1.15  jmcneill 		return;
   8611  1.15  jmcneill 	}
   8612  1.15  jmcneill 
   8613  1.15  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ,
   8614  1.15  jmcneill 	    &mtcap, sizeof(mtcap)) != 0) {
   8615  1.15  jmcneill 		/* unable to find temperature sensors */
   8616  1.15  jmcneill 		return;
   8617  1.15  jmcneill 	}
   8618  1.15  jmcneill 
   8619  1.15  jmcneill 	sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count;
   8620  1.15  jmcneill 	sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count,
   8621  1.15  jmcneill 	    sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK);
   8622  1.15  jmcneill 
   8623  1.15  jmcneill 	n = 0;
   8624  1.15  jmcneill 	map = bemtoh64(&mtcap.mtcap_sensor_map);
   8625  1.15  jmcneill 	for (i = 0; i < sizeof(map) * NBBY; i++) {
   8626  1.15  jmcneill 		if (!ISSET(map, (1ULL << i)))
   8627  1.15  jmcneill 			continue;
   8628  1.15  jmcneill 
   8629  1.15  jmcneill 		ks = kstat_create(DEVNAME(sc), 0, "temperature", i,
   8630  1.15  jmcneill 		    KSTAT_T_KV, 0);
   8631  1.15  jmcneill 		if (ks == NULL) {
   8632  1.15  jmcneill 			/* unable to attach temperature sensor %u, i */
   8633  1.15  jmcneill 			continue;
   8634  1.15  jmcneill 		}
   8635  1.15  jmcneill 
   8636  1.15  jmcneill 		ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO);
   8637  1.15  jmcneill 		*ktmp = mcx_kstat_mtmp_tpl;
   8638  1.15  jmcneill 
   8639  1.15  jmcneill 		ks->ks_data = ktmp;
   8640  1.15  jmcneill 		ks->ks_datalen = sizeof(*ktmp);
   8641  1.15  jmcneill 		TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval);
   8642  1.15  jmcneill 		ks->ks_read = mcx_kstat_mtmp_read;
   8643  1.15  jmcneill 
   8644  1.15  jmcneill 		ks->ks_softc = sc;
   8645  1.15  jmcneill 		kstat_install(ks);
   8646  1.15  jmcneill 
   8647  1.15  jmcneill 		sc->sc_kstat_mtmp[n++] = ks;
   8648  1.15  jmcneill 		if (n >= sc->sc_kstat_mtmp_count)
   8649  1.15  jmcneill 			break;
   8650  1.15  jmcneill 	}
   8651  1.15  jmcneill }
   8652  1.15  jmcneill 
   8653  1.15  jmcneill static uint64_t
   8654  1.15  jmcneill mcx_tmp_to_uK(uint16_t *t)
   8655  1.15  jmcneill {
   8656  1.15  jmcneill 	int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */
   8657  1.15  jmcneill 	mt *= 1000000 / 8; /* convert to uC */
   8658  1.15  jmcneill 	mt += 273150000; /* convert to uK */
   8659  1.15  jmcneill 
   8660  1.15  jmcneill 	return (mt);
   8661  1.15  jmcneill }
   8662  1.15  jmcneill 
   8663  1.15  jmcneill static int
   8664  1.15  jmcneill mcx_kstat_mtmp_read(struct kstat *ks)
   8665  1.15  jmcneill {
   8666  1.15  jmcneill 	struct mcx_softc *sc = ks->ks_softc;
   8667  1.15  jmcneill 	struct mcx_kstat_mtmp *ktmp = ks->ks_data;
   8668  1.15  jmcneill 	struct mcx_reg_mtmp mtmp;
   8669  1.15  jmcneill 	int rv;
   8670  1.15  jmcneill 	struct timeval updated;
   8671  1.15  jmcneill 
   8672  1.15  jmcneill 	TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated);
   8673  1.15  jmcneill 
   8674  1.15  jmcneill 	if (!ratecheck(&updated, &mcx_kstat_mtmp_rate))
   8675  1.15  jmcneill 		return (0);
   8676  1.15  jmcneill 
   8677  1.15  jmcneill 	memset(&mtmp, 0, sizeof(mtmp));
   8678  1.15  jmcneill 	htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit);
   8679  1.15  jmcneill 
   8680  1.15  jmcneill 	KERNEL_LOCK(); /* XXX */
   8681  1.15  jmcneill 	rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ,
   8682  1.15  jmcneill 	    &mtmp, sizeof(mtmp));
   8683  1.15  jmcneill 	KERNEL_UNLOCK();
   8684  1.15  jmcneill 	if (rv != 0)
   8685  1.15  jmcneill 		return (EIO);
   8686  1.15  jmcneill 
   8687  1.15  jmcneill 	memset(kstat_kv_istr(&ktmp->ktmp_name), 0,
   8688  1.15  jmcneill 	    sizeof(kstat_kv_istr(&ktmp->ktmp_name)));
   8689  1.15  jmcneill 	memcpy(kstat_kv_istr(&ktmp->ktmp_name),
   8690  1.15  jmcneill 	    mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name));
   8691  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_temperature) =
   8692  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature);
   8693  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_threshold_lo) =
   8694  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo);
   8695  1.15  jmcneill 	kstat_kv_temp(&ktmp->ktmp_threshold_hi) =
   8696  1.15  jmcneill 	    mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi);
   8697  1.15  jmcneill 
   8698  1.15  jmcneill 	TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated);
   8699  1.15  jmcneill 
   8700  1.15  jmcneill 	return (0);
   8701  1.15  jmcneill }
   8702  1.15  jmcneill 
   8703  1.15  jmcneill struct mcx_queuestat {
   8704  1.15  jmcneill 	char			 name[KSTAT_KV_NAMELEN];
   8705  1.15  jmcneill 	enum kstat_kv_type	 type;
   8706  1.15  jmcneill };
   8707  1.15  jmcneill 
   8708  1.15  jmcneill static const struct mcx_queuestat mcx_queue_kstat_tpl[] = {
   8709  1.15  jmcneill 	{ "RQ SW prod",		KSTAT_KV_T_COUNTER64 },
   8710  1.15  jmcneill 	{ "RQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8711  1.15  jmcneill 	{ "RQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8712  1.15  jmcneill 	{ "RQ HW state",	KSTAT_KV_T_ISTR },
   8713  1.15  jmcneill 
   8714  1.15  jmcneill 	{ "SQ SW prod",		KSTAT_KV_T_COUNTER64 },
   8715  1.15  jmcneill 	{ "SQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8716  1.15  jmcneill 	{ "SQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8717  1.15  jmcneill 	{ "SQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8718  1.15  jmcneill 	{ "SQ HW state",	KSTAT_KV_T_ISTR },
   8719  1.15  jmcneill 
   8720  1.15  jmcneill 	{ "CQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8721  1.15  jmcneill 	{ "CQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8722  1.15  jmcneill 	{ "CQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8723  1.15  jmcneill 	{ "CQ HW notify",	KSTAT_KV_T_COUNTER64 },
   8724  1.15  jmcneill 	{ "CQ HW solicit",	KSTAT_KV_T_COUNTER64 },
   8725  1.15  jmcneill 	{ "CQ HW status",	KSTAT_KV_T_ISTR },
   8726  1.15  jmcneill 	{ "CQ HW state",	KSTAT_KV_T_ISTR },
   8727  1.15  jmcneill 
   8728  1.15  jmcneill 	{ "EQ SW cons",		KSTAT_KV_T_COUNTER64 },
   8729  1.15  jmcneill 	{ "EQ HW prod",		KSTAT_KV_T_COUNTER64 },
   8730  1.15  jmcneill 	{ "EQ HW cons",		KSTAT_KV_T_COUNTER64 },
   8731  1.15  jmcneill 	{ "EQ HW status",	KSTAT_KV_T_ISTR },
   8732  1.15  jmcneill 	{ "EQ HW state",	KSTAT_KV_T_ISTR },
   8733  1.15  jmcneill };
   8734  1.15  jmcneill 
   8735  1.15  jmcneill static int	mcx_kstat_queue_read(struct kstat *);
   8736  1.15  jmcneill 
   8737  1.15  jmcneill static void
   8738  1.15  jmcneill mcx_kstat_attach_queues(struct mcx_softc *sc)
   8739  1.15  jmcneill {
   8740  1.15  jmcneill 	struct kstat *ks;
   8741  1.15  jmcneill 	struct kstat_kv *kvs;
   8742  1.15  jmcneill 	int q, i;
   8743  1.15  jmcneill 
   8744  1.15  jmcneill 	for (q = 0; q < sc->sc_nqueues; q++) {
   8745  1.15  jmcneill 		ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
   8746  1.15  jmcneill 		    KSTAT_T_KV, 0);
   8747  1.15  jmcneill 		if (ks == NULL) {
   8748  1.15  jmcneill 			/* unable to attach queue stats %u, q */
   8749  1.15  jmcneill 			continue;
   8750  1.15  jmcneill 		}
   8751  1.15  jmcneill 
   8752  1.15  jmcneill 		kvs = mallocarray(nitems(mcx_queue_kstat_tpl),
   8753  1.15  jmcneill 		    sizeof(*kvs), M_DEVBUF, M_WAITOK);
   8754  1.15  jmcneill 
   8755  1.15  jmcneill 		for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) {
   8756  1.15  jmcneill 			const struct mcx_queuestat *tpl =
   8757  1.15  jmcneill 			    &mcx_queue_kstat_tpl[i];
   8758  1.15  jmcneill 
   8759  1.15  jmcneill 			kstat_kv_init(&kvs[i], tpl->name, tpl->type);
   8760  1.15  jmcneill 		}
   8761  1.15  jmcneill 
   8762  1.15  jmcneill 		ks->ks_softc = &sc->sc_queues[q];
   8763  1.15  jmcneill 		ks->ks_data = kvs;
   8764  1.15  jmcneill 		ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs);
   8765  1.15  jmcneill 		ks->ks_read = mcx_kstat_queue_read;
   8766  1.15  jmcneill 
   8767  1.15  jmcneill 		sc->sc_queues[q].q_kstat = ks;
   8768  1.15  jmcneill 		kstat_install(ks);
   8769  1.15  jmcneill 	}
   8770  1.15  jmcneill }
   8771  1.15  jmcneill 
   8772  1.15  jmcneill static int
   8773  1.15  jmcneill mcx_kstat_queue_read(struct kstat *ks)
   8774  1.15  jmcneill {
   8775  1.15  jmcneill 	struct mcx_queues *q = ks->ks_softc;
   8776  1.15  jmcneill 	struct mcx_softc *sc = q->q_sc;
   8777  1.15  jmcneill 	struct kstat_kv *kvs = ks->ks_data;
   8778  1.15  jmcneill 	union {
   8779  1.15  jmcneill 		struct mcx_rq_ctx rq;
   8780  1.15  jmcneill 		struct mcx_sq_ctx sq;
   8781  1.15  jmcneill 		struct mcx_cq_ctx cq;
   8782  1.15  jmcneill 		struct mcx_eq_ctx eq;
   8783  1.15  jmcneill 	} u;
   8784  1.15  jmcneill 	const char *text;
   8785  1.15  jmcneill 	int error = 0;
   8786  1.15  jmcneill 
   8787  1.15  jmcneill 	KERNEL_LOCK();
   8788  1.15  jmcneill 
   8789  1.15  jmcneill 	if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) {
   8790  1.15  jmcneill 		error = EIO;
   8791  1.15  jmcneill 		goto out;
   8792  1.15  jmcneill 	}
   8793  1.15  jmcneill 
   8794  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_rx.rx_prod;
   8795  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter);
   8796  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter);
   8797  1.15  jmcneill 	switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >>
   8798  1.15  jmcneill 	    MCX_RQ_CTX_STATE_SHIFT) {
   8799  1.15  jmcneill 	case MCX_RQ_CTX_STATE_RST:
   8800  1.15  jmcneill 		text = "RST";
   8801  1.15  jmcneill 		break;
   8802  1.15  jmcneill 	case MCX_RQ_CTX_STATE_RDY:
   8803  1.15  jmcneill 		text = "RDY";
   8804  1.15  jmcneill 		break;
   8805  1.15  jmcneill 	case MCX_RQ_CTX_STATE_ERR:
   8806  1.15  jmcneill 		text = "ERR";
   8807  1.15  jmcneill 		break;
   8808  1.15  jmcneill 	default:
   8809  1.15  jmcneill 		text = "unknown";
   8810  1.15  jmcneill 		break;
   8811  1.15  jmcneill 	}
   8812  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8813  1.15  jmcneill 	kvs++;
   8814  1.15  jmcneill 
   8815  1.15  jmcneill 	if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) {
   8816  1.15  jmcneill 		error = EIO;
   8817  1.15  jmcneill 		goto out;
   8818  1.15  jmcneill 	}
   8819  1.15  jmcneill 
   8820  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_tx.tx_prod;
   8821  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_tx.tx_cons;
   8822  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter);
   8823  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter);
   8824  1.15  jmcneill 	switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >>
   8825  1.15  jmcneill 	    MCX_SQ_CTX_STATE_SHIFT) {
   8826  1.15  jmcneill 	case MCX_SQ_CTX_STATE_RST:
   8827  1.15  jmcneill 		text = "RST";
   8828  1.15  jmcneill 		break;
   8829  1.15  jmcneill 	case MCX_SQ_CTX_STATE_RDY:
   8830  1.15  jmcneill 		text = "RDY";
   8831  1.15  jmcneill 		break;
   8832  1.15  jmcneill 	case MCX_SQ_CTX_STATE_ERR:
   8833  1.15  jmcneill 		text = "ERR";
   8834  1.15  jmcneill 		break;
   8835  1.15  jmcneill 	default:
   8836  1.15  jmcneill 		text = "unknown";
   8837  1.15  jmcneill 		break;
   8838  1.15  jmcneill 	}
   8839  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8840  1.15  jmcneill 	kvs++;
   8841  1.15  jmcneill 
   8842  1.15  jmcneill 	if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) {
   8843  1.15  jmcneill 		error = EIO;
   8844  1.15  jmcneill 		goto out;
   8845  1.15  jmcneill 	}
   8846  1.15  jmcneill 
   8847  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_cq.cq_cons;
   8848  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter);
   8849  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter);
   8850  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified);
   8851  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit);
   8852  1.15  jmcneill 
   8853  1.15  jmcneill 	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >>
   8854  1.15  jmcneill 	    MCX_CQ_CTX_STATUS_SHIFT) {
   8855  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_OK:
   8856  1.15  jmcneill 		text = "OK";
   8857  1.15  jmcneill 		break;
   8858  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_OVERFLOW:
   8859  1.15  jmcneill 		text = "overflow";
   8860  1.15  jmcneill 		break;
   8861  1.15  jmcneill 	case MCX_CQ_CTX_STATUS_WRITE_FAIL:
   8862  1.15  jmcneill 		text = "write fail";
   8863  1.15  jmcneill 		break;
   8864  1.15  jmcneill 	default:
   8865  1.15  jmcneill 		text = "unknown";
   8866  1.15  jmcneill 		break;
   8867  1.15  jmcneill 	}
   8868  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8869  1.15  jmcneill 	kvs++;
   8870  1.15  jmcneill 
   8871  1.15  jmcneill 	switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >>
   8872  1.15  jmcneill 	    MCX_CQ_CTX_STATE_SHIFT) {
   8873  1.15  jmcneill 	case MCX_CQ_CTX_STATE_SOLICITED:
   8874  1.15  jmcneill 		text = "solicited";
   8875  1.15  jmcneill 		break;
   8876  1.15  jmcneill 	case MCX_CQ_CTX_STATE_ARMED:
   8877  1.15  jmcneill 		text = "armed";
   8878  1.15  jmcneill 		break;
   8879  1.15  jmcneill 	case MCX_CQ_CTX_STATE_FIRED:
   8880  1.15  jmcneill 		text = "fired";
   8881  1.15  jmcneill 		break;
   8882  1.15  jmcneill 	default:
   8883  1.15  jmcneill 		text = "unknown";
   8884  1.15  jmcneill 		break;
   8885  1.15  jmcneill 	}
   8886  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8887  1.15  jmcneill 	kvs++;
   8888  1.15  jmcneill 
   8889  1.15  jmcneill 	if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) {
   8890  1.15  jmcneill 		error = EIO;
   8891  1.15  jmcneill 		goto out;
   8892  1.15  jmcneill 	}
   8893  1.15  jmcneill 
   8894  1.15  jmcneill 	kstat_kv_u64(kvs++) = q->q_eq.eq_cons;
   8895  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter);
   8896  1.15  jmcneill 	kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter);
   8897  1.15  jmcneill 
   8898  1.15  jmcneill 	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >>
   8899  1.15  jmcneill 	    MCX_EQ_CTX_STATUS_SHIFT) {
   8900  1.15  jmcneill 	case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE:
   8901  1.15  jmcneill 		text = "write fail";
   8902  1.15  jmcneill 		break;
   8903  1.15  jmcneill 	case MCX_EQ_CTX_STATUS_OK:
   8904  1.15  jmcneill 		text = "OK";
   8905  1.15  jmcneill 		break;
   8906  1.15  jmcneill 	default:
   8907  1.15  jmcneill 		text = "unknown";
   8908  1.15  jmcneill 		break;
   8909  1.15  jmcneill 	}
   8910  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8911  1.15  jmcneill 	kvs++;
   8912  1.15  jmcneill 
   8913  1.15  jmcneill 	switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >>
   8914  1.15  jmcneill 	    MCX_EQ_CTX_STATE_SHIFT) {
   8915  1.15  jmcneill 	case MCX_EQ_CTX_STATE_ARMED:
   8916  1.15  jmcneill 		text = "armed";
   8917  1.15  jmcneill 		break;
   8918  1.15  jmcneill 	case MCX_EQ_CTX_STATE_FIRED:
   8919  1.15  jmcneill 		text = "fired";
   8920  1.15  jmcneill 		break;
   8921  1.15  jmcneill 	default:
   8922  1.15  jmcneill 		text = "unknown";
   8923  1.15  jmcneill 		break;
   8924  1.15  jmcneill 	}
   8925  1.15  jmcneill 	strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
   8926  1.15  jmcneill 	kvs++;
   8927  1.15  jmcneill 
   8928  1.15  jmcneill 	nanouptime(&ks->ks_updated);
   8929  1.15  jmcneill out:
   8930  1.15  jmcneill 	KERNEL_UNLOCK();
   8931  1.15  jmcneill 	return (error);
   8932  1.15  jmcneill }
   8933  1.15  jmcneill 
   8934  1.15  jmcneill #endif /* NKSTAT > 0 */
   8935  1.15  jmcneill 
   8936  1.15  jmcneill static unsigned int
   8937  1.15  jmcneill mcx_timecounter_read(struct timecounter *tc)
   8938  1.15  jmcneill {
   8939  1.15  jmcneill 	struct mcx_softc *sc = tc->tc_priv;
   8940  1.15  jmcneill 
   8941  1.15  jmcneill 	return (mcx_rd(sc, MCX_INTERNAL_TIMER_L));
   8942  1.15  jmcneill }
   8943  1.15  jmcneill 
   8944  1.15  jmcneill static void
   8945  1.15  jmcneill mcx_timecounter_attach(struct mcx_softc *sc)
   8946  1.15  jmcneill {
   8947  1.15  jmcneill 	struct timecounter *tc = &sc->sc_timecounter;
   8948  1.15  jmcneill 
   8949  1.15  jmcneill 	tc->tc_get_timecount = mcx_timecounter_read;
   8950  1.15  jmcneill 	tc->tc_counter_mask = ~0U;
   8951  1.15  jmcneill 	tc->tc_frequency = sc->sc_khz * 1000;
   8952  1.15  jmcneill 	tc->tc_name = device_xname(sc->sc_dev);
   8953  1.15  jmcneill 	tc->tc_quality = -100;
   8954  1.15  jmcneill 	tc->tc_priv = sc;
   8955  1.15  jmcneill 
   8956  1.15  jmcneill 	tc_init(tc);
   8957  1.15  jmcneill }
   8958