Home | History | Annotate | Line # | Download | only in pci
if_mcx.c revision 1.1
      1  1.1  jmcneill /*	$NetBSD: if_mcx.c,v 1.1 2019/09/21 14:04:12 jmcneill Exp $ */
      2  1.1  jmcneill /*	$OpenBSD: if_mcx.c,v 1.33 2019/09/12 04:23:59 jmatthew Exp $ */
      3  1.1  jmcneill 
      4  1.1  jmcneill /*
      5  1.1  jmcneill  * Copyright (c) 2017 David Gwynne <dlg (at) openbsd.org>
      6  1.1  jmcneill  * Copyright (c) 2019 Jonathan Matthew <jmatthew (at) openbsd.org>
      7  1.1  jmcneill  *
      8  1.1  jmcneill  * Permission to use, copy, modify, and distribute this software for any
      9  1.1  jmcneill  * purpose with or without fee is hereby granted, provided that the above
     10  1.1  jmcneill  * copyright notice and this permission notice appear in all copies.
     11  1.1  jmcneill  *
     12  1.1  jmcneill  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     13  1.1  jmcneill  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     14  1.1  jmcneill  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     15  1.1  jmcneill  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     16  1.1  jmcneill  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     17  1.1  jmcneill  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     18  1.1  jmcneill  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     19  1.1  jmcneill  */
     20  1.1  jmcneill 
     21  1.1  jmcneill #ifdef _KERNEL_OPT
     22  1.1  jmcneill #include "opt_net_mpsafe.h"
     23  1.1  jmcneill #endif
     24  1.1  jmcneill 
     25  1.1  jmcneill #include <sys/param.h>
     26  1.1  jmcneill #include <sys/systm.h>
     27  1.1  jmcneill #include <sys/sockio.h>
     28  1.1  jmcneill #include <sys/mbuf.h>
     29  1.1  jmcneill #include <sys/kernel.h>
     30  1.1  jmcneill #include <sys/socket.h>
     31  1.1  jmcneill #include <sys/device.h>
     32  1.1  jmcneill #include <sys/pool.h>
     33  1.1  jmcneill #include <sys/queue.h>
     34  1.1  jmcneill #include <sys/callout.h>
     35  1.1  jmcneill #include <sys/workqueue.h>
     36  1.1  jmcneill #include <sys/atomic.h>
     37  1.1  jmcneill #include <sys/kmem.h>
     38  1.1  jmcneill #include <sys/bus.h>
     39  1.1  jmcneill 
     40  1.1  jmcneill #include <machine/intr.h>
     41  1.1  jmcneill 
     42  1.1  jmcneill #include <net/if.h>
     43  1.1  jmcneill #include <net/if_dl.h>
     44  1.1  jmcneill #include <net/if_ether.h>
     45  1.1  jmcneill #include <net/if_media.h>
     46  1.1  jmcneill 
     47  1.1  jmcneill #include <net/bpf.h>
     48  1.1  jmcneill 
     49  1.1  jmcneill #include <netinet/in.h>
     50  1.1  jmcneill 
     51  1.1  jmcneill #include <dev/pci/pcireg.h>
     52  1.1  jmcneill #include <dev/pci/pcivar.h>
     53  1.1  jmcneill #include <dev/pci/pcidevs.h>
     54  1.1  jmcneill 
     55  1.1  jmcneill #ifdef NET_MPSAFE
     56  1.1  jmcneill #define	MCX_MPSAFE	1
     57  1.1  jmcneill #define	CALLOUT_FLAGS	CALLOUT_MPSAFE
     58  1.1  jmcneill #else
     59  1.1  jmcneill #define	CALLOUT_FLAGS	0
     60  1.1  jmcneill #endif
     61  1.1  jmcneill 
     62  1.1  jmcneill #define	MCX_MAX_NINTR	1
     63  1.1  jmcneill 
     64  1.1  jmcneill #define BUS_DMASYNC_PRERW	(BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
     65  1.1  jmcneill #define BUS_DMASYNC_POSTRW	(BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
     66  1.1  jmcneill 
     67  1.1  jmcneill #define MCX_HCA_BAR	PCI_MAPREG_START /* BAR 0 */
     68  1.1  jmcneill 
     69  1.1  jmcneill #define MCX_FW_VER	 	  0x0000
     70  1.1  jmcneill #define  MCX_FW_VER_MAJOR(_v)		((_v) & 0xffff)
     71  1.1  jmcneill #define  MCX_FW_VER_MINOR(_v)		((_v) >> 16)
     72  1.1  jmcneill #define MCX_CMDIF_FW_SUBVER	  0x0004
     73  1.1  jmcneill #define  MCX_FW_VER_SUBMINOR(_v)	((_v) & 0xffff)
     74  1.1  jmcneill #define  MCX_CMDIF(_v)			((_v) >> 16)
     75  1.1  jmcneill 
     76  1.1  jmcneill #define MCX_ISSI		 1 /* as per the PRM */
     77  1.1  jmcneill #define MCX_CMD_IF_SUPPORTED	 5
     78  1.1  jmcneill 
     79  1.1  jmcneill #define MCX_HARDMTU		 9500
     80  1.1  jmcneill 
     81  1.1  jmcneill #define MCX_MAX_CQS		 2		/* rq, sq */
     82  1.1  jmcneill 
     83  1.1  jmcneill /* queue sizes */
     84  1.1  jmcneill #define MCX_LOG_EQ_SIZE		 6		/* one page */
     85  1.1  jmcneill #define MCX_LOG_CQ_SIZE		 11
     86  1.1  jmcneill #define MCX_LOG_RQ_SIZE		 10
     87  1.1  jmcneill #define MCX_LOG_SQ_SIZE		 11
     88  1.1  jmcneill 
     89  1.1  jmcneill /* completion event moderation - about 10khz, or 90% of the cq */
     90  1.1  jmcneill #define MCX_CQ_MOD_PERIOD	50
     91  1.1  jmcneill #define MCX_CQ_MOD_COUNTER	(((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)
     92  1.1  jmcneill 
     93  1.1  jmcneill #define MCX_LOG_SQ_ENTRY_SIZE	 6
     94  1.1  jmcneill #define MCX_SQ_ENTRY_MAX_SLOTS	 4
     95  1.1  jmcneill #define MCX_SQ_SEGS_PER_SLOT	 \
     96  1.1  jmcneill 	(sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
     97  1.1  jmcneill #define MCX_SQ_MAX_SEGMENTS	 \
     98  1.1  jmcneill 	1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)
     99  1.1  jmcneill 
    100  1.1  jmcneill #define MCX_LOG_FLOW_TABLE_SIZE	 5
    101  1.1  jmcneill #define MCX_NUM_STATIC_FLOWS	 4	/* promisc, allmulti, ucast, bcast */
    102  1.1  jmcneill #define MCX_NUM_MCAST_FLOWS 	\
    103  1.1  jmcneill 	((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)
    104  1.1  jmcneill 
    105  1.1  jmcneill #define MCX_SQ_INLINE_SIZE	 18
    106  1.1  jmcneill 
    107  1.1  jmcneill /* doorbell offsets */
    108  1.1  jmcneill #define MCX_CQ_DOORBELL_OFFSET	 0
    109  1.1  jmcneill #define MCX_CQ_DOORBELL_SIZE	 16
    110  1.1  jmcneill #define MCX_RQ_DOORBELL_OFFSET	 64
    111  1.1  jmcneill #define MCX_SQ_DOORBELL_OFFSET	 64
    112  1.1  jmcneill 
    113  1.1  jmcneill #define MCX_WQ_DOORBELL_MASK	 0xffff
    114  1.1  jmcneill 
    115  1.1  jmcneill /* uar registers */
    116  1.1  jmcneill #define MCX_UAR_CQ_DOORBELL	 0x20
    117  1.1  jmcneill #define MCX_UAR_EQ_DOORBELL_ARM	 0x40
    118  1.1  jmcneill #define MCX_UAR_EQ_DOORBELL	 0x48
    119  1.1  jmcneill #define MCX_UAR_BF		 0x800
    120  1.1  jmcneill 
    121  1.1  jmcneill #define MCX_CMDQ_ADDR_HI		 0x0010
    122  1.1  jmcneill #define MCX_CMDQ_ADDR_LO		 0x0014
    123  1.1  jmcneill #define MCX_CMDQ_ADDR_NMASK		0xfff
    124  1.1  jmcneill #define MCX_CMDQ_LOG_SIZE(_v)		((_v) >> 4 & 0xf)
    125  1.1  jmcneill #define MCX_CMDQ_LOG_STRIDE(_v)		((_v) >> 0 & 0xf)
    126  1.1  jmcneill #define MCX_CMDQ_INTERFACE_MASK		(0x3 << 8)
    127  1.1  jmcneill #define MCX_CMDQ_INTERFACE_FULL_DRIVER	(0x0 << 8)
    128  1.1  jmcneill #define MCX_CMDQ_INTERFACE_DISABLED	(0x1 << 8)
    129  1.1  jmcneill 
    130  1.1  jmcneill #define MCX_CMDQ_DOORBELL		0x0018
    131  1.1  jmcneill 
    132  1.1  jmcneill #define MCX_STATE		0x01fc
    133  1.1  jmcneill #define MCX_STATE_MASK			(1 << 31)
    134  1.1  jmcneill #define MCX_STATE_INITIALIZING		(1 << 31)
    135  1.1  jmcneill #define MCX_STATE_READY			(0 << 31)
    136  1.1  jmcneill #define MCX_STATE_INTERFACE_MASK	(0x3 << 24)
    137  1.1  jmcneill #define MCX_STATE_INTERFACE_FULL_DRIVER	(0x0 << 24)
    138  1.1  jmcneill #define MCX_STATE_INTERFACE_DISABLED	(0x1 << 24)
    139  1.1  jmcneill 
    140  1.1  jmcneill #define MCX_INTERNAL_TIMER	0x1000
    141  1.1  jmcneill #define MCX_INTERNAL_TIMER_H	0x1000
    142  1.1  jmcneill #define MCX_INTERNAL_TIMER_L	0x1004
    143  1.1  jmcneill 
    144  1.1  jmcneill #define MCX_CLEAR_INT		0x100c
    145  1.1  jmcneill 
    146  1.1  jmcneill #define MCX_REG_OP_WRITE	0
    147  1.1  jmcneill #define MCX_REG_OP_READ		1
    148  1.1  jmcneill 
    149  1.1  jmcneill #define MCX_REG_PMLP		0x5002
    150  1.1  jmcneill #define MCX_REG_PMTU		0x5003
    151  1.1  jmcneill #define MCX_REG_PTYS		0x5004
    152  1.1  jmcneill #define MCX_REG_PAOS		0x5006
    153  1.1  jmcneill #define MCX_REG_PFCC		0x5007
    154  1.1  jmcneill #define MCX_REG_PPCNT		0x5008
    155  1.1  jmcneill #define MCX_REG_MCIA		0x9014
    156  1.1  jmcneill 
    157  1.1  jmcneill #define MCX_ETHER_CAP_SGMII	(1 << 0)
    158  1.1  jmcneill #define MCX_ETHER_CAP_1000_KX	(1 << 1)
    159  1.1  jmcneill #define MCX_ETHER_CAP_10G_CX4	(1 << 2)
    160  1.1  jmcneill #define MCX_ETHER_CAP_10G_KX4	(1 << 3)
    161  1.1  jmcneill #define MCX_ETHER_CAP_10G_KR	(1 << 4)
    162  1.1  jmcneill #define MCX_ETHER_CAP_40G_CR4	(1 << 6)
    163  1.1  jmcneill #define MCX_ETHER_CAP_40G_KR4	(1 << 7)
    164  1.1  jmcneill #define MCX_ETHER_CAP_10G_CR	(1 << 12)
    165  1.1  jmcneill #define MCX_ETHER_CAP_10G_SR	(1 << 13)
    166  1.1  jmcneill #define MCX_ETHER_CAP_10G_LR	(1 << 14)
    167  1.1  jmcneill #define MCX_ETHER_CAP_40G_SR4	(1 << 15)
    168  1.1  jmcneill #define MCX_ETHER_CAP_40G_LR4	(1 << 16)
    169  1.1  jmcneill #define MCX_ETHER_CAP_50G_SR2	(1 << 18)
    170  1.1  jmcneill #define MCX_ETHER_CAP_100G_CR4	(1 << 20)
    171  1.1  jmcneill #define MCX_ETHER_CAP_100G_SR4	(1 << 21)
    172  1.1  jmcneill #define MCX_ETHER_CAP_100G_KR4	(1 << 22)
    173  1.1  jmcneill #define MCX_ETHER_CAP_25G_CR	(1 << 27)
    174  1.1  jmcneill #define MCX_ETHER_CAP_25G_KR	(1 << 28)
    175  1.1  jmcneill #define MCX_ETHER_CAP_25G_SR	(1 << 29)
    176  1.1  jmcneill #define MCX_ETHER_CAP_50G_CR2	(1 << 30)
    177  1.1  jmcneill #define MCX_ETHER_CAP_50G_KR2	(1 << 31)
    178  1.1  jmcneill 
    179  1.1  jmcneill #define MCX_PAGE_SHIFT		12
    180  1.1  jmcneill #define MCX_PAGE_SIZE		(1 << MCX_PAGE_SHIFT)
    181  1.1  jmcneill #define MCX_MAX_CQE		32
    182  1.1  jmcneill 
    183  1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP	0x100
    184  1.1  jmcneill #define MCX_CMD_QUERY_ADAPTER	0x101
    185  1.1  jmcneill #define MCX_CMD_INIT_HCA	0x102
    186  1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA	0x103
    187  1.1  jmcneill #define MCX_CMD_ENABLE_HCA	0x104
    188  1.1  jmcneill #define MCX_CMD_DISABLE_HCA	0x105
    189  1.1  jmcneill #define MCX_CMD_QUERY_PAGES	0x107
    190  1.1  jmcneill #define MCX_CMD_MANAGE_PAGES	0x108
    191  1.1  jmcneill #define MCX_CMD_SET_HCA_CAP	0x109
    192  1.1  jmcneill #define MCX_CMD_QUERY_ISSI	0x10a
    193  1.1  jmcneill #define MCX_CMD_SET_ISSI	0x10b
    194  1.1  jmcneill #define MCX_CMD_SET_DRIVER_VERSION \
    195  1.1  jmcneill 				0x10d
    196  1.1  jmcneill #define MCX_CMD_QUERY_SPECIAL_CONTEXTS \
    197  1.1  jmcneill 				0x203
    198  1.1  jmcneill #define MCX_CMD_CREATE_EQ	0x301
    199  1.1  jmcneill #define MCX_CMD_DESTROY_EQ	0x302
    200  1.1  jmcneill #define MCX_CMD_CREATE_CQ	0x400
    201  1.1  jmcneill #define MCX_CMD_DESTROY_CQ	0x401
    202  1.1  jmcneill #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT \
    203  1.1  jmcneill 				0x754
    204  1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
    205  1.1  jmcneill 				0x755
    206  1.1  jmcneill #define MCX_CMD_QUERY_VPORT_COUNTERS \
    207  1.1  jmcneill 				0x770
    208  1.1  jmcneill #define MCX_CMD_ALLOC_PD	0x800
    209  1.1  jmcneill #define MCX_CMD_ALLOC_UAR	0x802
    210  1.1  jmcneill #define MCX_CMD_ACCESS_REG	0x805
    211  1.1  jmcneill #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN \
    212  1.1  jmcneill 				0x816
    213  1.1  jmcneill #define MCX_CMD_CREATE_TIR	0x900
    214  1.1  jmcneill #define MCX_CMD_DESTROY_TIR	0x902
    215  1.1  jmcneill #define MCX_CMD_CREATE_SQ	0x904
    216  1.1  jmcneill #define MCX_CMD_MODIFY_SQ	0x905
    217  1.1  jmcneill #define MCX_CMD_DESTROY_SQ	0x906
    218  1.1  jmcneill #define MCX_CMD_QUERY_SQ	0x907
    219  1.1  jmcneill #define MCX_CMD_CREATE_RQ	0x908
    220  1.1  jmcneill #define MCX_CMD_MODIFY_RQ	0x909
    221  1.1  jmcneill #define MCX_CMD_DESTROY_RQ	0x90a
    222  1.1  jmcneill #define MCX_CMD_QUERY_RQ	0x90b
    223  1.1  jmcneill #define MCX_CMD_CREATE_TIS	0x912
    224  1.1  jmcneill #define MCX_CMD_DESTROY_TIS	0x914
    225  1.1  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ROOT \
    226  1.1  jmcneill 				0x92f
    227  1.1  jmcneill #define MCX_CMD_CREATE_FLOW_TABLE \
    228  1.1  jmcneill 				0x930
    229  1.1  jmcneill #define MCX_CMD_DESTROY_FLOW_TABLE \
    230  1.1  jmcneill 				0x931
    231  1.1  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE \
    232  1.1  jmcneill 				0x932
    233  1.1  jmcneill #define MCX_CMD_CREATE_FLOW_GROUP \
    234  1.1  jmcneill 				0x933
    235  1.1  jmcneill #define MCX_CMD_DESTROY_FLOW_GROUP \
    236  1.1  jmcneill 				0x934
    237  1.1  jmcneill #define MCX_CMD_QUERY_FLOW_GROUP \
    238  1.1  jmcneill 				0x935
    239  1.1  jmcneill #define MCX_CMD_SET_FLOW_TABLE_ENTRY \
    240  1.1  jmcneill 				0x936
    241  1.1  jmcneill #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY \
    242  1.1  jmcneill 				0x937
    243  1.1  jmcneill #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY \
    244  1.1  jmcneill 				0x938
    245  1.1  jmcneill #define MCX_CMD_ALLOC_FLOW_COUNTER \
    246  1.1  jmcneill 				0x939
    247  1.1  jmcneill #define MCX_CMD_QUERY_FLOW_COUNTER \
    248  1.1  jmcneill 				0x93b
    249  1.1  jmcneill 
    250  1.1  jmcneill #define MCX_QUEUE_STATE_RST	0
    251  1.1  jmcneill #define MCX_QUEUE_STATE_RDY	1
    252  1.1  jmcneill #define MCX_QUEUE_STATE_ERR	3
    253  1.1  jmcneill 
    254  1.1  jmcneill #define MCX_FLOW_TABLE_TYPE_RX	0
    255  1.1  jmcneill #define MCX_FLOW_TABLE_TYPE_TX	1
    256  1.1  jmcneill 
    257  1.1  jmcneill #define MCX_CMDQ_INLINE_DATASIZE 16
    258  1.1  jmcneill 
    259  1.1  jmcneill struct mcx_cmdq_entry {
    260  1.1  jmcneill 	uint8_t			cq_type;
    261  1.1  jmcneill #define MCX_CMDQ_TYPE_PCIE		0x7
    262  1.1  jmcneill 	uint8_t			cq_reserved0[3];
    263  1.1  jmcneill 
    264  1.1  jmcneill 	uint32_t		cq_input_length;
    265  1.1  jmcneill 	uint64_t		cq_input_ptr;
    266  1.1  jmcneill 	uint8_t			cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
    267  1.1  jmcneill 
    268  1.1  jmcneill 	uint8_t			cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
    269  1.1  jmcneill 	uint64_t		cq_output_ptr;
    270  1.1  jmcneill 	uint32_t		cq_output_length;
    271  1.1  jmcneill 
    272  1.1  jmcneill 	uint8_t			cq_token;
    273  1.1  jmcneill 	uint8_t			cq_signature;
    274  1.1  jmcneill 	uint8_t			cq_reserved1[1];
    275  1.1  jmcneill 	uint8_t			cq_status;
    276  1.1  jmcneill #define MCX_CQ_STATUS_SHIFT		1
    277  1.1  jmcneill #define MCX_CQ_STATUS_MASK		(0x7f << MCX_CQ_STATUS_SHIFT)
    278  1.1  jmcneill #define MCX_CQ_STATUS_OK		(0x00 << MCX_CQ_STATUS_SHIFT)
    279  1.1  jmcneill #define MCX_CQ_STATUS_INT_ERR		(0x01 << MCX_CQ_STATUS_SHIFT)
    280  1.1  jmcneill #define MCX_CQ_STATUS_BAD_OPCODE	(0x02 << MCX_CQ_STATUS_SHIFT)
    281  1.1  jmcneill #define MCX_CQ_STATUS_BAD_PARAM		(0x03 << MCX_CQ_STATUS_SHIFT)
    282  1.1  jmcneill #define MCX_CQ_STATUS_BAD_SYS_STATE	(0x04 << MCX_CQ_STATUS_SHIFT)
    283  1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE	(0x05 << MCX_CQ_STATUS_SHIFT)
    284  1.1  jmcneill #define MCX_CQ_STATUS_RESOURCE_BUSY	(0x06 << MCX_CQ_STATUS_SHIFT)
    285  1.1  jmcneill #define MCX_CQ_STATUS_EXCEED_LIM	(0x08 << MCX_CQ_STATUS_SHIFT)
    286  1.1  jmcneill #define MCX_CQ_STATUS_BAD_RES_STATE	(0x09 << MCX_CQ_STATUS_SHIFT)
    287  1.1  jmcneill #define MCX_CQ_STATUS_BAD_INDEX		(0x0a << MCX_CQ_STATUS_SHIFT)
    288  1.1  jmcneill #define MCX_CQ_STATUS_NO_RESOURCES	(0x0f << MCX_CQ_STATUS_SHIFT)
    289  1.1  jmcneill #define MCX_CQ_STATUS_BAD_INPUT_LEN	(0x50 << MCX_CQ_STATUS_SHIFT)
    290  1.1  jmcneill #define MCX_CQ_STATUS_BAD_OUTPUT_LEN	(0x51 << MCX_CQ_STATUS_SHIFT)
    291  1.1  jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
    292  1.1  jmcneill 					(0x10 << MCX_CQ_STATUS_SHIFT)
    293  1.1  jmcneill #define MCX_CQ_STATUS_BAD_SIZE		(0x40 << MCX_CQ_STATUS_SHIFT)
    294  1.1  jmcneill #define MCX_CQ_STATUS_OWN_MASK		0x1
    295  1.1  jmcneill #define MCX_CQ_STATUS_OWN_SW		0x0
    296  1.1  jmcneill #define MCX_CQ_STATUS_OWN_HW		0x1
    297  1.1  jmcneill } __packed __aligned(8);
    298  1.1  jmcneill 
    299  1.1  jmcneill #define MCX_CMDQ_MAILBOX_DATASIZE	512
    300  1.1  jmcneill 
    301  1.1  jmcneill struct mcx_cmdq_mailbox {
    302  1.1  jmcneill 	uint8_t			mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
    303  1.1  jmcneill 	uint8_t			mb_reserved0[48];
    304  1.1  jmcneill 	uint64_t		mb_next_ptr;
    305  1.1  jmcneill 	uint32_t		mb_block_number;
    306  1.1  jmcneill 	uint8_t			mb_reserved1[1];
    307  1.1  jmcneill 	uint8_t			mb_token;
    308  1.1  jmcneill 	uint8_t			mb_ctrl_signature;
    309  1.1  jmcneill 	uint8_t			mb_signature;
    310  1.1  jmcneill } __packed __aligned(8);
    311  1.1  jmcneill 
    312  1.1  jmcneill #define MCX_CMDQ_MAILBOX_ALIGN	(1 << 10)
    313  1.1  jmcneill #define MCX_CMDQ_MAILBOX_SIZE	roundup(sizeof(struct mcx_cmdq_mailbox), \
    314  1.1  jmcneill 				    MCX_CMDQ_MAILBOX_ALIGN)
    315  1.1  jmcneill /*
    316  1.1  jmcneill  * command mailbox structres
    317  1.1  jmcneill  */
    318  1.1  jmcneill 
    319  1.1  jmcneill struct mcx_cmd_enable_hca_in {
    320  1.1  jmcneill 	uint16_t		cmd_opcode;
    321  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    322  1.1  jmcneill 	uint16_t		cmd_op_mod;
    323  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    324  1.1  jmcneill 	uint16_t		cmd_function_id;
    325  1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    326  1.1  jmcneill } __packed __aligned(4);
    327  1.1  jmcneill 
    328  1.1  jmcneill struct mcx_cmd_enable_hca_out {
    329  1.1  jmcneill 	uint8_t			cmd_status;
    330  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    331  1.1  jmcneill 	uint32_t		cmd_syndrome;
    332  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    333  1.1  jmcneill } __packed __aligned(4);
    334  1.1  jmcneill 
    335  1.1  jmcneill struct mcx_cmd_init_hca_in {
    336  1.1  jmcneill 	uint16_t		cmd_opcode;
    337  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    338  1.1  jmcneill 	uint16_t		cmd_op_mod;
    339  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    340  1.1  jmcneill } __packed __aligned(4);
    341  1.1  jmcneill 
    342  1.1  jmcneill struct mcx_cmd_init_hca_out {
    343  1.1  jmcneill 	uint8_t			cmd_status;
    344  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    345  1.1  jmcneill 	uint32_t		cmd_syndrome;
    346  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    347  1.1  jmcneill } __packed __aligned(4);
    348  1.1  jmcneill 
    349  1.1  jmcneill struct mcx_cmd_teardown_hca_in {
    350  1.1  jmcneill 	uint16_t		cmd_opcode;
    351  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    352  1.1  jmcneill 	uint16_t		cmd_op_mod;
    353  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    354  1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_GRACEFUL	0x0
    355  1.1  jmcneill #define MCX_CMD_TEARDOWN_HCA_PANIC	0x1
    356  1.1  jmcneill 	uint16_t		cmd_profile;
    357  1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    358  1.1  jmcneill } __packed __aligned(4);
    359  1.1  jmcneill 
    360  1.1  jmcneill struct mcx_cmd_teardown_hca_out {
    361  1.1  jmcneill 	uint8_t			cmd_status;
    362  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    363  1.1  jmcneill 	uint32_t		cmd_syndrome;
    364  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    365  1.1  jmcneill } __packed __aligned(4);
    366  1.1  jmcneill 
    367  1.1  jmcneill struct mcx_cmd_access_reg_in {
    368  1.1  jmcneill 	uint16_t		cmd_opcode;
    369  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    370  1.1  jmcneill 	uint16_t		cmd_op_mod;
    371  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    372  1.1  jmcneill 	uint16_t		cmd_register_id;
    373  1.1  jmcneill 	uint32_t		cmd_argument;
    374  1.1  jmcneill } __packed __aligned(4);
    375  1.1  jmcneill 
    376  1.1  jmcneill struct mcx_cmd_access_reg_out {
    377  1.1  jmcneill 	uint8_t			cmd_status;
    378  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    379  1.1  jmcneill 	uint32_t		cmd_syndrome;
    380  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    381  1.1  jmcneill } __packed __aligned(4);
    382  1.1  jmcneill 
    383  1.1  jmcneill struct mcx_reg_pmtu {
    384  1.1  jmcneill 	uint8_t			rp_reserved1;
    385  1.1  jmcneill 	uint8_t			rp_local_port;
    386  1.1  jmcneill 	uint8_t			rp_reserved2[2];
    387  1.1  jmcneill 	uint16_t		rp_max_mtu;
    388  1.1  jmcneill 	uint8_t			rp_reserved3[2];
    389  1.1  jmcneill 	uint16_t		rp_admin_mtu;
    390  1.1  jmcneill 	uint8_t			rp_reserved4[2];
    391  1.1  jmcneill 	uint16_t		rp_oper_mtu;
    392  1.1  jmcneill 	uint8_t			rp_reserved5[2];
    393  1.1  jmcneill } __packed __aligned(4);
    394  1.1  jmcneill 
    395  1.1  jmcneill struct mcx_reg_ptys {
    396  1.1  jmcneill 	uint8_t			rp_reserved1;
    397  1.1  jmcneill 	uint8_t			rp_local_port;
    398  1.1  jmcneill 	uint8_t			rp_reserved2;
    399  1.1  jmcneill 	uint8_t			rp_proto_mask;
    400  1.1  jmcneill #define MCX_REG_PTYS_PROTO_MASK_ETH		(1 << 2)
    401  1.1  jmcneill 	uint8_t			rp_reserved3[8];
    402  1.1  jmcneill 	uint32_t		rp_eth_proto_cap;
    403  1.1  jmcneill 	uint8_t			rp_reserved4[8];
    404  1.1  jmcneill 	uint32_t		rp_eth_proto_admin;
    405  1.1  jmcneill 	uint8_t			rp_reserved5[8];
    406  1.1  jmcneill 	uint32_t		rp_eth_proto_oper;
    407  1.1  jmcneill 	uint8_t			rp_reserved6[24];
    408  1.1  jmcneill } __packed __aligned(4);
    409  1.1  jmcneill 
    410  1.1  jmcneill struct mcx_reg_paos {
    411  1.1  jmcneill 	uint8_t			rp_reserved1;
    412  1.1  jmcneill 	uint8_t			rp_local_port;
    413  1.1  jmcneill 	uint8_t			rp_admin_status;
    414  1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP		1
    415  1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DOWN		2
    416  1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE	3
    417  1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED	4
    418  1.1  jmcneill 	uint8_t			rp_oper_status;
    419  1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_UP		1
    420  1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_DOWN		2
    421  1.1  jmcneill #define MCX_REG_PAOS_OPER_STATUS_FAILED		4
    422  1.1  jmcneill 	uint8_t			rp_admin_state_update;
    423  1.1  jmcneill #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN	(1 << 7)
    424  1.1  jmcneill 	uint8_t			rp_reserved2[11];
    425  1.1  jmcneill } __packed __aligned(4);
    426  1.1  jmcneill 
    427  1.1  jmcneill struct mcx_reg_pfcc {
    428  1.1  jmcneill 	uint8_t			rp_reserved1;
    429  1.1  jmcneill 	uint8_t			rp_local_port;
    430  1.1  jmcneill 	uint8_t			rp_reserved2[3];
    431  1.1  jmcneill 	uint8_t			rp_prio_mask_tx;
    432  1.1  jmcneill 	uint8_t			rp_reserved3;
    433  1.1  jmcneill 	uint8_t			rp_prio_mask_rx;
    434  1.1  jmcneill 	uint8_t			rp_pptx_aptx;
    435  1.1  jmcneill 	uint8_t			rp_pfctx;
    436  1.1  jmcneill 	uint8_t			rp_fctx_dis;
    437  1.1  jmcneill 	uint8_t			rp_reserved4;
    438  1.1  jmcneill 	uint8_t			rp_pprx_aprx;
    439  1.1  jmcneill 	uint8_t			rp_pfcrx;
    440  1.1  jmcneill 	uint8_t			rp_reserved5[2];
    441  1.1  jmcneill 	uint16_t		rp_dev_stall_min;
    442  1.1  jmcneill 	uint16_t		rp_dev_stall_crit;
    443  1.1  jmcneill 	uint8_t			rp_reserved6[12];
    444  1.1  jmcneill } __packed __aligned(4);
    445  1.1  jmcneill 
    446  1.1  jmcneill #define MCX_PMLP_MODULE_NUM_MASK	0xff
    447  1.1  jmcneill struct mcx_reg_pmlp {
    448  1.1  jmcneill 	uint8_t			rp_rxtx;
    449  1.1  jmcneill 	uint8_t			rp_local_port;
    450  1.1  jmcneill 	uint8_t			rp_reserved0;
    451  1.1  jmcneill 	uint8_t			rp_width;
    452  1.1  jmcneill 	uint32_t		rp_lane0_mapping;
    453  1.1  jmcneill 	uint32_t		rp_lane1_mapping;
    454  1.1  jmcneill 	uint32_t		rp_lane2_mapping;
    455  1.1  jmcneill 	uint32_t		rp_lane3_mapping;
    456  1.1  jmcneill 	uint8_t			rp_reserved1[44];
    457  1.1  jmcneill } __packed __aligned(4);
    458  1.1  jmcneill 
    459  1.1  jmcneill #define MCX_MCIA_EEPROM_BYTES	32
    460  1.1  jmcneill struct mcx_reg_mcia {
    461  1.1  jmcneill 	uint8_t			rm_l;
    462  1.1  jmcneill 	uint8_t			rm_module;
    463  1.1  jmcneill 	uint8_t			rm_reserved0;
    464  1.1  jmcneill 	uint8_t			rm_status;
    465  1.1  jmcneill 	uint8_t			rm_i2c_addr;
    466  1.1  jmcneill 	uint8_t			rm_page_num;
    467  1.1  jmcneill 	uint16_t		rm_dev_addr;
    468  1.1  jmcneill 	uint16_t		rm_reserved1;
    469  1.1  jmcneill 	uint16_t		rm_size;
    470  1.1  jmcneill 	uint32_t		rm_reserved2;
    471  1.1  jmcneill 	uint8_t			rm_data[48];
    472  1.1  jmcneill } __packed __aligned(4);
    473  1.1  jmcneill 
    474  1.1  jmcneill struct mcx_cmd_query_issi_in {
    475  1.1  jmcneill 	uint16_t		cmd_opcode;
    476  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    477  1.1  jmcneill 	uint16_t		cmd_op_mod;
    478  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    479  1.1  jmcneill } __packed __aligned(4);
    480  1.1  jmcneill 
    481  1.1  jmcneill struct mcx_cmd_query_issi_il_out {
    482  1.1  jmcneill 	uint8_t			cmd_status;
    483  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    484  1.1  jmcneill 	uint32_t		cmd_syndrome;
    485  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    486  1.1  jmcneill 	uint16_t		cmd_current_issi;
    487  1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    488  1.1  jmcneill } __packed __aligned(4);
    489  1.1  jmcneill 
    490  1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
    491  1.1  jmcneill 
    492  1.1  jmcneill struct mcx_cmd_query_issi_mb_out {
    493  1.1  jmcneill 	uint8_t			cmd_reserved2[16];
    494  1.1  jmcneill 	uint8_t			cmd_supported_issi[80]; /* very big endian */
    495  1.1  jmcneill } __packed __aligned(4);
    496  1.1  jmcneill 
    497  1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
    498  1.1  jmcneill 
    499  1.1  jmcneill struct mcx_cmd_set_issi_in {
    500  1.1  jmcneill 	uint16_t		cmd_opcode;
    501  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    502  1.1  jmcneill 	uint16_t		cmd_op_mod;
    503  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    504  1.1  jmcneill 	uint16_t		cmd_current_issi;
    505  1.1  jmcneill 	uint8_t			cmd_reserved2[4];
    506  1.1  jmcneill } __packed __aligned(4);
    507  1.1  jmcneill 
    508  1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
    509  1.1  jmcneill 
    510  1.1  jmcneill struct mcx_cmd_set_issi_out {
    511  1.1  jmcneill 	uint8_t			cmd_status;
    512  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    513  1.1  jmcneill 	uint32_t		cmd_syndrome;
    514  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    515  1.1  jmcneill } __packed __aligned(4);
    516  1.1  jmcneill 
    517  1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
    518  1.1  jmcneill 
    519  1.1  jmcneill struct mcx_cmd_query_pages_in {
    520  1.1  jmcneill 	uint16_t		cmd_opcode;
    521  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    522  1.1  jmcneill 	uint16_t		cmd_op_mod;
    523  1.1  jmcneill #define MCX_CMD_QUERY_PAGES_BOOT	0x01
    524  1.1  jmcneill #define MCX_CMD_QUERY_PAGES_INIT	0x02
    525  1.1  jmcneill #define MCX_CMD_QUERY_PAGES_REGULAR	0x03
    526  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    527  1.1  jmcneill } __packed __aligned(4);
    528  1.1  jmcneill 
    529  1.1  jmcneill struct mcx_cmd_query_pages_out {
    530  1.1  jmcneill 	uint8_t			cmd_status;
    531  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    532  1.1  jmcneill 	uint32_t		cmd_syndrome;
    533  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    534  1.1  jmcneill 	uint16_t		cmd_func_id;
    535  1.1  jmcneill 	uint32_t		cmd_num_pages;
    536  1.1  jmcneill } __packed __aligned(4);
    537  1.1  jmcneill 
    538  1.1  jmcneill struct mcx_cmd_manage_pages_in {
    539  1.1  jmcneill 	uint16_t		cmd_opcode;
    540  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    541  1.1  jmcneill 	uint16_t		cmd_op_mod;
    542  1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
    543  1.1  jmcneill 					0x00
    544  1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
    545  1.1  jmcneill 					0x01
    546  1.1  jmcneill #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
    547  1.1  jmcneill 					0x02
    548  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
    549  1.1  jmcneill 	uint16_t		cmd_func_id;
    550  1.1  jmcneill 	uint32_t		cmd_input_num_entries;
    551  1.1  jmcneill } __packed __aligned(4);
    552  1.1  jmcneill 
    553  1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
    554  1.1  jmcneill 
    555  1.1  jmcneill struct mcx_cmd_manage_pages_out {
    556  1.1  jmcneill 	uint8_t			cmd_status;
    557  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    558  1.1  jmcneill 	uint32_t		cmd_syndrome;
    559  1.1  jmcneill 	uint32_t		cmd_output_num_entries;
    560  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    561  1.1  jmcneill } __packed __aligned(4);
    562  1.1  jmcneill 
    563  1.1  jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
    564  1.1  jmcneill 
    565  1.1  jmcneill struct mcx_cmd_query_hca_cap_in {
    566  1.1  jmcneill 	uint16_t		cmd_opcode;
    567  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    568  1.1  jmcneill 	uint16_t		cmd_op_mod;
    569  1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_MAX	(0x0 << 0)
    570  1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_CURRENT	(0x1 << 0)
    571  1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_DEVICE	(0x0 << 1)
    572  1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD	(0x1 << 1)
    573  1.1  jmcneill #define MCX_CMD_QUERY_HCA_CAP_FLOW	(0x7 << 1)
    574  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    575  1.1  jmcneill } __packed __aligned(4);
    576  1.1  jmcneill 
    577  1.1  jmcneill struct mcx_cmd_query_hca_cap_out {
    578  1.1  jmcneill 	uint8_t			cmd_status;
    579  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    580  1.1  jmcneill 	uint32_t		cmd_syndrome;
    581  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    582  1.1  jmcneill } __packed __aligned(4);
    583  1.1  jmcneill 
    584  1.1  jmcneill #define MCX_HCA_CAP_LEN			0x1000
    585  1.1  jmcneill #define MCX_HCA_CAP_NMAILBOXES		\
    586  1.1  jmcneill 	(MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
    587  1.1  jmcneill 
    588  1.1  jmcneill #if __GNUC_PREREQ__(4, 3)
    589  1.1  jmcneill #define __counter__		__COUNTER__
    590  1.1  jmcneill #else
    591  1.1  jmcneill #define __counter__		__LINE__
    592  1.1  jmcneill #endif
    593  1.1  jmcneill 
    594  1.1  jmcneill #define __token(_tok, _num)	_tok##_num
    595  1.1  jmcneill #define _token(_tok, _num)	__token(_tok, _num)
    596  1.1  jmcneill #define __reserved__		_token(__reserved, __counter__)
    597  1.1  jmcneill 
    598  1.1  jmcneill struct mcx_cap_device {
    599  1.1  jmcneill 	uint8_t			reserved0[16];
    600  1.1  jmcneill 
    601  1.1  jmcneill 	uint8_t			log_max_srq_sz;
    602  1.1  jmcneill 	uint8_t			log_max_qp_sz;
    603  1.1  jmcneill 	uint8_t			__reserved__[1];
    604  1.1  jmcneill 	uint8_t			log_max_qp; /* 5 bits */
    605  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_QP	0x1f
    606  1.1  jmcneill 
    607  1.1  jmcneill 	uint8_t			__reserved__[1];
    608  1.1  jmcneill 	uint8_t			log_max_srq; /* 5 bits */
    609  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SRQ	0x1f
    610  1.1  jmcneill 	uint8_t			__reserved__[2];
    611  1.1  jmcneill 
    612  1.1  jmcneill 	uint8_t			__reserved__[1];
    613  1.1  jmcneill 	uint8_t			log_max_cq_sz;
    614  1.1  jmcneill 	uint8_t			__reserved__[1];
    615  1.1  jmcneill 	uint8_t			log_max_cq; /* 5 bits */
    616  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_CQ	0x1f
    617  1.1  jmcneill 
    618  1.1  jmcneill 	uint8_t			log_max_eq_sz;
    619  1.1  jmcneill 	uint8_t			log_max_mkey; /* 6 bits */
    620  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MKEY	0x3f
    621  1.1  jmcneill 	uint8_t			__reserved__[1];
    622  1.1  jmcneill 	uint8_t			log_max_eq; /* 4 bits */
    623  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_EQ	0x0f
    624  1.1  jmcneill 
    625  1.1  jmcneill 	uint8_t			max_indirection;
    626  1.1  jmcneill 	uint8_t			log_max_mrw_sz; /* 7 bits */
    627  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ	0x7f
    628  1.1  jmcneill 	uint8_t			teardown_log_max_msf_list_size;
    629  1.1  jmcneill #define MCX_CAP_DEVICE_FORCE_TEARDOWN	0x80
    630  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
    631  1.1  jmcneill 					0x3f
    632  1.1  jmcneill 	uint8_t			log_max_klm_list_size; /* 6 bits */
    633  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
    634  1.1  jmcneill 					0x3f
    635  1.1  jmcneill 
    636  1.1  jmcneill 	uint8_t			__reserved__[1];
    637  1.1  jmcneill 	uint8_t			log_max_ra_req_dc; /* 6 bits */
    638  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC	0x3f
    639  1.1  jmcneill 	uint8_t			__reserved__[1];
    640  1.1  jmcneill 	uint8_t			log_max_ra_res_dc; /* 6 bits */
    641  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
    642  1.1  jmcneill 					0x3f
    643  1.1  jmcneill 
    644  1.1  jmcneill 	uint8_t			__reserved__[1];
    645  1.1  jmcneill 	uint8_t			log_max_ra_req_qp; /* 6 bits */
    646  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
    647  1.1  jmcneill 					0x3f
    648  1.1  jmcneill 	uint8_t			__reserved__[1];
    649  1.1  jmcneill 	uint8_t			log_max_ra_res_qp; /* 6 bits */
    650  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
    651  1.1  jmcneill 					0x3f
    652  1.1  jmcneill 
    653  1.1  jmcneill 	uint8_t			flags1;
    654  1.1  jmcneill #define MCX_CAP_DEVICE_END_PAD		0x80
    655  1.1  jmcneill #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED	0x40
    656  1.1  jmcneill #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
    657  1.1  jmcneill 					0x20
    658  1.1  jmcneill #define MCX_CAP_DEVICE_START_PAD	0x10
    659  1.1  jmcneill #define MCX_CAP_DEVICE_128BYTE_CACHELINE \
    660  1.1  jmcneill 					0x08
    661  1.1  jmcneill 	uint8_t			__reserved__[1];
    662  1.1  jmcneill 	uint16_t		gid_table_size;
    663  1.1  jmcneill 
    664  1.1  jmcneill 	uint16_t		flags2;
    665  1.1  jmcneill #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT	0x8000
    666  1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_COUNTERS	0x4000
    667  1.1  jmcneill #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
    668  1.1  jmcneill 					0x2000
    669  1.1  jmcneill #define MCX_CAP_DEVICE_DEBUG		0x1000
    670  1.1  jmcneill #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
    671  1.1  jmcneill 					0x8000
    672  1.1  jmcneill #define MCX_CAP_DEVICE_RQ_DELAY_DROP	0x4000
    673  1.1  jmcneill #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK	0x03ff
    674  1.1  jmcneill 	uint16_t		pkey_table_size;
    675  1.1  jmcneill 
    676  1.1  jmcneill 	uint8_t			flags3;
    677  1.1  jmcneill #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
    678  1.1  jmcneill 					0x80
    679  1.1  jmcneill #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
    680  1.1  jmcneill 					0x40
    681  1.1  jmcneill #define MCX_CAP_DEVICE_IB_VIRTUAL	0x20
    682  1.1  jmcneill #define MCX_CAP_DEVICE_ETH_VIRTUAL	0x10
    683  1.1  jmcneill #define MCX_CAP_DEVICE_ETS		0x04
    684  1.1  jmcneill #define MCX_CAP_DEVICE_NIC_FLOW_TABLE	0x02
    685  1.1  jmcneill #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
    686  1.1  jmcneill 					0x01
    687  1.1  jmcneill 	uint8_t			local_ca_ack_delay; /* 5 bits */
    688  1.1  jmcneill #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
    689  1.1  jmcneill 					0x1f
    690  1.1  jmcneill 	uint8_t			port_type;
    691  1.1  jmcneill #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
    692  1.1  jmcneill 					0x80
    693  1.1  jmcneill #define MCX_CAP_DEVICE_PORT_TYPE	0x03
    694  1.1  jmcneill 	uint8_t			num_ports;
    695  1.1  jmcneill 
    696  1.1  jmcneill 	uint8_t			snapshot_log_max_msg;
    697  1.1  jmcneill #define MCX_CAP_DEVICE_SNAPSHOT		0x80
    698  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSG	0x1f
    699  1.1  jmcneill 	uint8_t			max_tc; /* 4 bits */
    700  1.1  jmcneill #define MCX_CAP_DEVICE_MAX_TC		0x0f
    701  1.1  jmcneill 	uint8_t			flags4;
    702  1.1  jmcneill #define MCX_CAP_DEVICE_TEMP_WARN_EVENT	0x80
    703  1.1  jmcneill #define MCX_CAP_DEVICE_DCBX		0x40
    704  1.1  jmcneill #define MCX_CAP_DEVICE_ROL_S		0x02
    705  1.1  jmcneill #define MCX_CAP_DEVICE_ROL_G		0x01
    706  1.1  jmcneill 	uint8_t			wol;
    707  1.1  jmcneill #define MCX_CAP_DEVICE_WOL_S		0x40
    708  1.1  jmcneill #define MCX_CAP_DEVICE_WOL_G		0x20
    709  1.1  jmcneill #define MCX_CAP_DEVICE_WOL_A		0x10
    710  1.1  jmcneill #define MCX_CAP_DEVICE_WOL_B		0x08
    711  1.1  jmcneill #define MCX_CAP_DEVICE_WOL_M		0x04
    712  1.1  jmcneill #define MCX_CAP_DEVICE_WOL_U		0x02
    713  1.1  jmcneill #define MCX_CAP_DEVICE_WOL_P		0x01
    714  1.1  jmcneill 
    715  1.1  jmcneill 	uint16_t		stat_rate_support;
    716  1.1  jmcneill 	uint8_t			__reserved__[1];
    717  1.1  jmcneill 	uint8_t			cqe_version; /* 4 bits */
    718  1.1  jmcneill #define MCX_CAP_DEVICE_CQE_VERSION	0x0f
    719  1.1  jmcneill 
    720  1.1  jmcneill 	uint32_t		flags5;
    721  1.1  jmcneill #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
    722  1.1  jmcneill 					0x80000000
    723  1.1  jmcneill #define MCX_CAP_DEVICE_STRIDING_RQ	0x40000000
    724  1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
    725  1.1  jmcneill 					0x10000000
    726  1.1  jmcneill #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
    727  1.1  jmcneill 					0x08000000
    728  1.1  jmcneill #define MCX_CAP_DEVICE_DC_CONNECT_CP	0x00040000
    729  1.1  jmcneill #define MCX_CAP_DEVICE_DC_CNAK_DRACE	0x00020000
    730  1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    731  1.1  jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR	0x00010000
    732  1.1  jmcneill #define MCX_CAP_DEVICE_CMDIF_CHECKSUM	0x0000c000
    733  1.1  jmcneill #define MCX_CAP_DEVICE_SIGERR_QCE	0x00002000
    734  1.1  jmcneill #define MCX_CAP_DEVICE_WQ_SIGNATURE	0x00000800
    735  1.1  jmcneill #define MCX_CAP_DEVICE_SCTR_DATA_CQE	0x00000400
    736  1.1  jmcneill #define MCX_CAP_DEVICE_SHO		0x00000100
    737  1.1  jmcneill #define MCX_CAP_DEVICE_TPH		0x00000080
    738  1.1  jmcneill #define MCX_CAP_DEVICE_RF		0x00000040
    739  1.1  jmcneill #define MCX_CAP_DEVICE_DCT		0x00000020
    740  1.1  jmcneill #define MCX_CAP_DEVICE_QOS		0x00000010
    741  1.1  jmcneill #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS	0x00000008
    742  1.1  jmcneill #define MCX_CAP_DEVICE_ROCE		0x00000004
    743  1.1  jmcneill #define MCX_CAP_DEVICE_ATOMIC		0x00000002
    744  1.1  jmcneill 
    745  1.1  jmcneill 	uint32_t		flags6;
    746  1.1  jmcneill #define MCX_CAP_DEVICE_CQ_OI		0x80000000
    747  1.1  jmcneill #define MCX_CAP_DEVICE_CQ_RESIZE	0x40000000
    748  1.1  jmcneill #define MCX_CAP_DEVICE_CQ_MODERATION	0x20000000
    749  1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
    750  1.1  jmcneill 					0x10000000
    751  1.1  jmcneill #define MCX_CAP_DEVICE_CQ_INVALIDATE	0x08000000
    752  1.1  jmcneill #define MCX_CAP_DEVICE_RESERVED_AT_255	0x04000000
    753  1.1  jmcneill #define MCX_CAP_DEVICE_CQ_EQ_REMAP	0x02000000
    754  1.1  jmcneill #define MCX_CAP_DEVICE_PG		0x01000000
    755  1.1  jmcneill #define MCX_CAP_DEVICE_BLOCK_LB_MC	0x00800000
    756  1.1  jmcneill #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
    757  1.1  jmcneill 					0x00400000
    758  1.1  jmcneill #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
    759  1.1  jmcneill 					0x00200000
    760  1.1  jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
    761  1.1  jmcneill 					0x00100000
    762  1.1  jmcneill #define MCX_CAP_DEVICE_CD		0x00080000
    763  1.1  jmcneill #define MCX_CAP_DEVICE_ATM		0x00040000
    764  1.1  jmcneill #define MCX_CAP_DEVICE_APM		0x00020000
    765  1.1  jmcneill #define MCX_CAP_DEVICE_IMAICL		0x00010000
    766  1.1  jmcneill #define MCX_CAP_DEVICE_QKV		0x00000200
    767  1.1  jmcneill #define MCX_CAP_DEVICE_PKV		0x00000100
    768  1.1  jmcneill #define MCX_CAP_DEVICE_SET_DETH_SQPN	0x00000080
    769  1.1  jmcneill #define MCX_CAP_DEVICE_XRC		0x00000008
    770  1.1  jmcneill #define MCX_CAP_DEVICE_UD		0x00000004
    771  1.1  jmcneill #define MCX_CAP_DEVICE_UC		0x00000002
    772  1.1  jmcneill #define MCX_CAP_DEVICE_RC		0x00000001
    773  1.1  jmcneill 
    774  1.1  jmcneill 	uint8_t			uar_flags;
    775  1.1  jmcneill #define MCX_CAP_DEVICE_UAR_4K		0x80
    776  1.1  jmcneill 	uint8_t			uar_sz;	/* 6 bits */
    777  1.1  jmcneill #define MCX_CAP_DEVICE_UAR_SZ		0x3f
    778  1.1  jmcneill 	uint8_t			__reserved__[1];
    779  1.1  jmcneill 	uint8_t			log_pg_sz;
    780  1.1  jmcneill 
    781  1.1  jmcneill 	uint8_t			flags7;
    782  1.1  jmcneill #define MCX_CAP_DEVICE_BF		0x80
    783  1.1  jmcneill #define MCX_CAP_DEVICE_DRIVER_VERSION	0x40
    784  1.1  jmcneill #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
    785  1.1  jmcneill 					0x20
    786  1.1  jmcneill 	uint8_t			log_bf_reg_size; /* 5 bits */
    787  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE	0x1f
    788  1.1  jmcneill 	uint8_t			__reserved__[2];
    789  1.1  jmcneill 
    790  1.1  jmcneill 	uint16_t		num_of_diagnostic_counters;
    791  1.1  jmcneill 	uint16_t		max_wqe_sz_sq;
    792  1.1  jmcneill 
    793  1.1  jmcneill 	uint8_t			__reserved__[2];
    794  1.1  jmcneill 	uint16_t		max_wqe_sz_rq;
    795  1.1  jmcneill 
    796  1.1  jmcneill 	uint8_t			__reserved__[2];
    797  1.1  jmcneill 	uint16_t		max_wqe_sz_sq_dc;
    798  1.1  jmcneill 
    799  1.1  jmcneill 	uint32_t		max_qp_mcg; /* 25 bits */
    800  1.1  jmcneill #define MCX_CAP_DEVICE_MAX_QP_MCG	0x1ffffff
    801  1.1  jmcneill 
    802  1.1  jmcneill 	uint8_t			__reserved__[3];
    803  1.1  jmcneill 	uint8_t			log_max_mcq;
    804  1.1  jmcneill 
    805  1.1  jmcneill 	uint8_t			log_max_transport_domain; /* 5 bits */
    806  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
    807  1.1  jmcneill 					0x1f
    808  1.1  jmcneill 	uint8_t			log_max_pd; /* 5 bits */
    809  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_PD	0x1f
    810  1.1  jmcneill 	uint8_t			__reserved__[1];
    811  1.1  jmcneill 	uint8_t			log_max_xrcd; /* 5 bits */
    812  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_XRCD	0x1f
    813  1.1  jmcneill 
    814  1.1  jmcneill 	uint8_t			__reserved__[2];
    815  1.1  jmcneill 	uint16_t		max_flow_counter;
    816  1.1  jmcneill 
    817  1.1  jmcneill 	uint8_t			log_max_rq; /* 5 bits */
    818  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQ	0x1f
    819  1.1  jmcneill 	uint8_t			log_max_sq; /* 5 bits */
    820  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SQ	0x1f
    821  1.1  jmcneill 	uint8_t			log_max_tir; /* 5 bits */
    822  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIR	0x1f
    823  1.1  jmcneill 	uint8_t			log_max_tis; /* 5 bits */
    824  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS	0x1f
    825  1.1  jmcneill 
    826  1.1  jmcneill 	uint8_t 		flags8;
    827  1.1  jmcneill #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
    828  1.1  jmcneill 					0x80
    829  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RMP	0x1f
    830  1.1  jmcneill 	uint8_t			log_max_rqt; /* 5 bits */
    831  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT	0x1f
    832  1.1  jmcneill 	uint8_t			log_max_rqt_size; /* 5 bits */
    833  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE	0x1f
    834  1.1  jmcneill 	uint8_t			log_max_tis_per_sq; /* 5 bits */
    835  1.1  jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
    836  1.1  jmcneill 					0x1f
    837  1.1  jmcneill } __packed __aligned(8);
    838  1.1  jmcneill 
    839  1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
    840  1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
    841  1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
    842  1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
    843  1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
    844  1.1  jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
    845  1.1  jmcneill CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
    846  1.1  jmcneill 
    847  1.1  jmcneill struct mcx_cmd_set_driver_version_in {
    848  1.1  jmcneill 	uint16_t		cmd_opcode;
    849  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    850  1.1  jmcneill 	uint16_t		cmd_op_mod;
    851  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    852  1.1  jmcneill } __packed __aligned(4);
    853  1.1  jmcneill 
    854  1.1  jmcneill struct mcx_cmd_set_driver_version_out {
    855  1.1  jmcneill 	uint8_t			cmd_status;
    856  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    857  1.1  jmcneill 	uint32_t		cmd_syndrome;
    858  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    859  1.1  jmcneill } __packed __aligned(4);
    860  1.1  jmcneill 
    861  1.1  jmcneill struct mcx_cmd_set_driver_version {
    862  1.1  jmcneill 	uint8_t			cmd_driver_version[64];
    863  1.1  jmcneill } __packed __aligned(8);
    864  1.1  jmcneill 
    865  1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_in {
    866  1.1  jmcneill 	uint16_t		cmd_opcode;
    867  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    868  1.1  jmcneill 	uint16_t		cmd_op_mod;
    869  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    870  1.1  jmcneill 	uint32_t		cmd_field_select;
    871  1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR	0x04
    872  1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC	0x10
    873  1.1  jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU	0x40
    874  1.1  jmcneill } __packed __aligned(4);
    875  1.1  jmcneill 
    876  1.1  jmcneill struct mcx_cmd_modify_nic_vport_context_out {
    877  1.1  jmcneill 	uint8_t			cmd_status;
    878  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    879  1.1  jmcneill 	uint32_t		cmd_syndrome;
    880  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    881  1.1  jmcneill } __packed __aligned(4);
    882  1.1  jmcneill 
    883  1.1  jmcneill struct mcx_cmd_query_nic_vport_context_in {
    884  1.1  jmcneill 	uint16_t		cmd_opcode;
    885  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    886  1.1  jmcneill 	uint16_t		cmd_op_mod;
    887  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    888  1.1  jmcneill 	uint8_t			cmd_allowed_list_type;
    889  1.1  jmcneill 	uint8_t			cmd_reserved2[3];
    890  1.1  jmcneill } __packed __aligned(4);
    891  1.1  jmcneill 
    892  1.1  jmcneill struct mcx_cmd_query_nic_vport_context_out {
    893  1.1  jmcneill 	uint8_t			cmd_status;
    894  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    895  1.1  jmcneill 	uint32_t		cmd_syndrome;
    896  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    897  1.1  jmcneill } __packed __aligned(4);
    898  1.1  jmcneill 
    899  1.1  jmcneill struct mcx_nic_vport_ctx {
    900  1.1  jmcneill 	uint32_t		vp_min_wqe_inline_mode;
    901  1.1  jmcneill 	uint8_t			vp_reserved0[32];
    902  1.1  jmcneill 	uint32_t		vp_mtu;
    903  1.1  jmcneill 	uint8_t			vp_reserved1[200];
    904  1.1  jmcneill 	uint16_t		vp_flags;
    905  1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_UC_MAC			(0)
    906  1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_MC_MAC			(1 << 24)
    907  1.1  jmcneill #define MCX_NIC_VPORT_CTX_LIST_VLAN			(2 << 24)
    908  1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_ALL			(1 << 13)
    909  1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_MCAST			(1 << 14)
    910  1.1  jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_UCAST			(1 << 15)
    911  1.1  jmcneill 	uint16_t		vp_allowed_list_size;
    912  1.1  jmcneill 	uint64_t		vp_perm_addr;
    913  1.1  jmcneill 	uint8_t			vp_reserved2[4];
    914  1.1  jmcneill 	/* allowed list follows */
    915  1.1  jmcneill } __packed __aligned(4);
    916  1.1  jmcneill 
    917  1.1  jmcneill struct mcx_counter {
    918  1.1  jmcneill 	uint64_t		packets;
    919  1.1  jmcneill 	uint64_t		octets;
    920  1.1  jmcneill } __packed __aligned(4);
    921  1.1  jmcneill 
    922  1.1  jmcneill struct mcx_nic_vport_counters {
    923  1.1  jmcneill 	struct mcx_counter	rx_err;
    924  1.1  jmcneill 	struct mcx_counter	tx_err;
    925  1.1  jmcneill 	uint8_t			reserved0[64]; /* 0x30 */
    926  1.1  jmcneill 	struct mcx_counter	rx_bcast;
    927  1.1  jmcneill 	struct mcx_counter	tx_bcast;
    928  1.1  jmcneill 	struct mcx_counter	rx_ucast;
    929  1.1  jmcneill 	struct mcx_counter	tx_ucast;
    930  1.1  jmcneill 	struct mcx_counter	rx_mcast;
    931  1.1  jmcneill 	struct mcx_counter	tx_mcast;
    932  1.1  jmcneill 	uint8_t			reserved1[0x210 - 0xd0];
    933  1.1  jmcneill } __packed __aligned(4);
    934  1.1  jmcneill 
    935  1.1  jmcneill struct mcx_cmd_query_vport_counters_in {
    936  1.1  jmcneill 	uint16_t		cmd_opcode;
    937  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    938  1.1  jmcneill 	uint16_t		cmd_op_mod;
    939  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    940  1.1  jmcneill } __packed __aligned(4);
    941  1.1  jmcneill 
    942  1.1  jmcneill struct mcx_cmd_query_vport_counters_mb_in {
    943  1.1  jmcneill 	uint8_t			cmd_reserved0[8];
    944  1.1  jmcneill 	uint8_t			cmd_clear;
    945  1.1  jmcneill 	uint8_t			cmd_reserved1[7];
    946  1.1  jmcneill } __packed __aligned(4);
    947  1.1  jmcneill 
    948  1.1  jmcneill struct mcx_cmd_query_vport_counters_out {
    949  1.1  jmcneill 	uint8_t			cmd_status;
    950  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    951  1.1  jmcneill 	uint32_t		cmd_syndrome;
    952  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    953  1.1  jmcneill } __packed __aligned(4);
    954  1.1  jmcneill 
    955  1.1  jmcneill struct mcx_cmd_query_flow_counter_in {
    956  1.1  jmcneill 	uint16_t		cmd_opcode;
    957  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    958  1.1  jmcneill 	uint16_t		cmd_op_mod;
    959  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    960  1.1  jmcneill } __packed __aligned(4);
    961  1.1  jmcneill 
    962  1.1  jmcneill struct mcx_cmd_query_flow_counter_mb_in {
    963  1.1  jmcneill 	uint8_t			cmd_reserved0[8];
    964  1.1  jmcneill 	uint8_t			cmd_clear;
    965  1.1  jmcneill 	uint8_t			cmd_reserved1[5];
    966  1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
    967  1.1  jmcneill } __packed __aligned(4);
    968  1.1  jmcneill 
    969  1.1  jmcneill struct mcx_cmd_query_flow_counter_out {
    970  1.1  jmcneill 	uint8_t			cmd_status;
    971  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    972  1.1  jmcneill 	uint32_t		cmd_syndrome;
    973  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    974  1.1  jmcneill } __packed __aligned(4);
    975  1.1  jmcneill 
    976  1.1  jmcneill struct mcx_cmd_alloc_uar_in {
    977  1.1  jmcneill 	uint16_t		cmd_opcode;
    978  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    979  1.1  jmcneill 	uint16_t		cmd_op_mod;
    980  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    981  1.1  jmcneill } __packed __aligned(4);
    982  1.1  jmcneill 
    983  1.1  jmcneill struct mcx_cmd_alloc_uar_out {
    984  1.1  jmcneill 	uint8_t			cmd_status;
    985  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
    986  1.1  jmcneill 	uint32_t		cmd_syndrome;
    987  1.1  jmcneill 	uint32_t		cmd_uar;
    988  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
    989  1.1  jmcneill } __packed __aligned(4);
    990  1.1  jmcneill 
    991  1.1  jmcneill struct mcx_cmd_query_special_ctx_in {
    992  1.1  jmcneill 	uint16_t		cmd_opcode;
    993  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
    994  1.1  jmcneill 	uint16_t		cmd_op_mod;
    995  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
    996  1.1  jmcneill } __packed __aligned(4);
    997  1.1  jmcneill 
    998  1.1  jmcneill struct mcx_cmd_query_special_ctx_out {
    999  1.1  jmcneill 	uint8_t			cmd_status;
   1000  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1001  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1002  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1003  1.1  jmcneill 	uint32_t		cmd_resd_lkey;
   1004  1.1  jmcneill } __packed __aligned(4);
   1005  1.1  jmcneill 
   1006  1.1  jmcneill struct mcx_eq_ctx {
   1007  1.1  jmcneill 	uint32_t		eq_status;
   1008  1.1  jmcneill #define MCX_EQ_CTX_ST_SHIFT		8
   1009  1.1  jmcneill #define MCX_EQ_CTX_ST_MASK		(0xf << MCX_EQ_CTX_ST_SHIFT)
   1010  1.1  jmcneill #define MCX_EQ_CTX_ST_ARMED		(0x9 << MCX_EQ_CTX_ST_SHIFT)
   1011  1.1  jmcneill #define MCX_EQ_CTX_ST_FIRED		(0xa << MCX_EQ_CTX_ST_SHIFT)
   1012  1.1  jmcneill #define MCX_EQ_CTX_OI_SHIFT		17
   1013  1.1  jmcneill #define MCX_EQ_CTX_OI			(1 << MCX_EQ_CTX_OI_SHIFT)
   1014  1.1  jmcneill #define MCX_EQ_CTX_EC_SHIFT		18
   1015  1.1  jmcneill #define MCX_EQ_CTX_EC			(1 << MCX_EQ_CTX_EC_SHIFT)
   1016  1.1  jmcneill #define MCX_EQ_CTX_STATUS_SHIFT		28
   1017  1.1  jmcneill #define MCX_EQ_CTX_STATUS_MASK		(0xf << MCX_EQ_CTX_STATUS_SHIFT)
   1018  1.1  jmcneill #define MCX_EQ_CTX_STATUS_OK		(0x0 << MCX_EQ_CTX_STATUS_SHIFT)
   1019  1.1  jmcneill #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE \
   1020  1.1  jmcneill 					(0xa << MCX_EQ_CTX_STATUS_SHIFT)
   1021  1.1  jmcneill 	uint32_t		eq_reserved1;
   1022  1.1  jmcneill 	uint32_t		eq_page_offset;
   1023  1.1  jmcneill #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT	5
   1024  1.1  jmcneill 	uint32_t		eq_uar_size;
   1025  1.1  jmcneill #define MCX_EQ_CTX_UAR_PAGE_MASK	0xffffff
   1026  1.1  jmcneill #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT	24
   1027  1.1  jmcneill 	uint32_t		eq_reserved2;
   1028  1.1  jmcneill 	uint8_t			eq_reserved3[3];
   1029  1.1  jmcneill 	uint8_t			eq_intr;
   1030  1.1  jmcneill 	uint32_t		eq_log_page_size;
   1031  1.1  jmcneill #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1032  1.1  jmcneill 	uint32_t		eq_reserved4[3];
   1033  1.1  jmcneill 	uint32_t		eq_consumer_counter;
   1034  1.1  jmcneill 	uint32_t		eq_producer_counter;
   1035  1.1  jmcneill #define MCX_EQ_CTX_COUNTER_MASK		0xffffff
   1036  1.1  jmcneill 	uint32_t		eq_reserved5[4];
   1037  1.1  jmcneill } __packed __aligned(4);
   1038  1.1  jmcneill 
   1039  1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
   1040  1.1  jmcneill 
   1041  1.1  jmcneill struct mcx_cmd_create_eq_in {
   1042  1.1  jmcneill 	uint16_t		cmd_opcode;
   1043  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1044  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1045  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1046  1.1  jmcneill } __packed __aligned(4);
   1047  1.1  jmcneill 
   1048  1.1  jmcneill struct mcx_cmd_create_eq_mb_in {
   1049  1.1  jmcneill 	struct mcx_eq_ctx	cmd_eq_ctx;
   1050  1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1051  1.1  jmcneill 	uint64_t		cmd_event_bitmask;
   1052  1.1  jmcneill #define MCX_EVENT_TYPE_COMPLETION	0x00
   1053  1.1  jmcneill #define MCX_EVENT_TYPE_CQ_ERROR		0x04
   1054  1.1  jmcneill #define MCX_EVENT_TYPE_INTERNAL_ERROR	0x08
   1055  1.1  jmcneill #define MCX_EVENT_TYPE_PORT_CHANGE	0x09
   1056  1.1  jmcneill #define MCX_EVENT_TYPE_CMD_COMPLETION	0x0a
   1057  1.1  jmcneill #define MCX_EVENT_TYPE_PAGE_REQUEST	0x0b
   1058  1.1  jmcneill #define MCX_EVENT_TYPE_LAST_WQE		0x13
   1059  1.1  jmcneill 	uint8_t			cmd_reserved1[176];
   1060  1.1  jmcneill } __packed __aligned(4);
   1061  1.1  jmcneill 
   1062  1.1  jmcneill struct mcx_cmd_create_eq_out {
   1063  1.1  jmcneill 	uint8_t			cmd_status;
   1064  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1065  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1066  1.1  jmcneill 	uint32_t		cmd_eqn;
   1067  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1068  1.1  jmcneill } __packed __aligned(4);
   1069  1.1  jmcneill 
   1070  1.1  jmcneill struct mcx_eq_entry {
   1071  1.1  jmcneill 	uint8_t			eq_reserved1;
   1072  1.1  jmcneill 	uint8_t			eq_event_type;
   1073  1.1  jmcneill 	uint8_t			eq_reserved2;
   1074  1.1  jmcneill 	uint8_t			eq_event_sub_type;
   1075  1.1  jmcneill 
   1076  1.1  jmcneill 	uint8_t			eq_reserved3[28];
   1077  1.1  jmcneill 	uint32_t		eq_event_data[7];
   1078  1.1  jmcneill 	uint8_t			eq_reserved4[2];
   1079  1.1  jmcneill 	uint8_t			eq_signature;
   1080  1.1  jmcneill 	uint8_t			eq_owner;
   1081  1.1  jmcneill #define MCX_EQ_ENTRY_OWNER_INIT			1
   1082  1.1  jmcneill } __packed __aligned(4);
   1083  1.1  jmcneill 
   1084  1.1  jmcneill CTASSERT(sizeof(struct mcx_eq_entry) == 64);
   1085  1.1  jmcneill 
   1086  1.1  jmcneill struct mcx_cmd_alloc_pd_in {
   1087  1.1  jmcneill 	uint16_t		cmd_opcode;
   1088  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1089  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1090  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1091  1.1  jmcneill } __packed __aligned(4);
   1092  1.1  jmcneill 
   1093  1.1  jmcneill struct mcx_cmd_alloc_pd_out {
   1094  1.1  jmcneill 	uint8_t			cmd_status;
   1095  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1096  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1097  1.1  jmcneill 	uint32_t		cmd_pd;
   1098  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1099  1.1  jmcneill } __packed __aligned(4);
   1100  1.1  jmcneill 
   1101  1.1  jmcneill struct mcx_cmd_alloc_td_in {
   1102  1.1  jmcneill 	uint16_t		cmd_opcode;
   1103  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1104  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1105  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1106  1.1  jmcneill } __packed __aligned(4);
   1107  1.1  jmcneill 
   1108  1.1  jmcneill struct mcx_cmd_alloc_td_out {
   1109  1.1  jmcneill 	uint8_t			cmd_status;
   1110  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1111  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1112  1.1  jmcneill 	uint32_t		cmd_tdomain;
   1113  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1114  1.1  jmcneill } __packed __aligned(4);
   1115  1.1  jmcneill 
   1116  1.1  jmcneill struct mcx_cmd_create_tir_in {
   1117  1.1  jmcneill 	uint16_t		cmd_opcode;
   1118  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1119  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1120  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1121  1.1  jmcneill } __packed __aligned(4);
   1122  1.1  jmcneill 
   1123  1.1  jmcneill struct mcx_cmd_create_tir_mb_in {
   1124  1.1  jmcneill 	uint8_t			cmd_reserved0[20];
   1125  1.1  jmcneill 	uint32_t		cmd_disp_type;
   1126  1.1  jmcneill #define MCX_TIR_CTX_DISP_TYPE_SHIFT	28
   1127  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1128  1.1  jmcneill 	uint32_t		cmd_lro;
   1129  1.1  jmcneill 	uint8_t			cmd_reserved2[8];
   1130  1.1  jmcneill 	uint32_t		cmd_inline_rqn;
   1131  1.1  jmcneill 	uint32_t		cmd_indir_table;
   1132  1.1  jmcneill 	uint32_t		cmd_tdomain;
   1133  1.1  jmcneill 	uint8_t			cmd_rx_hash_key[40];
   1134  1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_outer;
   1135  1.1  jmcneill 	uint32_t		cmd_rx_hash_sel_inner;
   1136  1.1  jmcneill 	uint8_t			cmd_reserved3[152];
   1137  1.1  jmcneill } __packed __aligned(4);
   1138  1.1  jmcneill 
   1139  1.1  jmcneill struct mcx_cmd_create_tir_out {
   1140  1.1  jmcneill 	uint8_t			cmd_status;
   1141  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1142  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1143  1.1  jmcneill 	uint32_t		cmd_tirn;
   1144  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1145  1.1  jmcneill } __packed __aligned(4);
   1146  1.1  jmcneill 
   1147  1.1  jmcneill struct mcx_cmd_destroy_tir_in {
   1148  1.1  jmcneill 	uint16_t		cmd_opcode;
   1149  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1150  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1151  1.1  jmcneill 	uint32_t		cmd_tirn;
   1152  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1153  1.1  jmcneill } __packed __aligned(4);
   1154  1.1  jmcneill 
   1155  1.1  jmcneill struct mcx_cmd_destroy_tir_out {
   1156  1.1  jmcneill 	uint8_t			cmd_status;
   1157  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1158  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1159  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1160  1.1  jmcneill } __packed __aligned(4);
   1161  1.1  jmcneill 
   1162  1.1  jmcneill struct mcx_cmd_create_tis_in {
   1163  1.1  jmcneill 	uint16_t		cmd_opcode;
   1164  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1165  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1166  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1167  1.1  jmcneill } __packed __aligned(4);
   1168  1.1  jmcneill 
   1169  1.1  jmcneill struct mcx_cmd_create_tis_mb_in {
   1170  1.1  jmcneill 	uint8_t			cmd_reserved[16];
   1171  1.1  jmcneill 	uint32_t		cmd_prio;
   1172  1.1  jmcneill 	uint8_t			cmd_reserved1[32];
   1173  1.1  jmcneill 	uint32_t		cmd_tdomain;
   1174  1.1  jmcneill 	uint8_t			cmd_reserved2[120];
   1175  1.1  jmcneill } __packed __aligned(4);
   1176  1.1  jmcneill 
   1177  1.1  jmcneill struct mcx_cmd_create_tis_out {
   1178  1.1  jmcneill 	uint8_t			cmd_status;
   1179  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1180  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1181  1.1  jmcneill 	uint32_t		cmd_tisn;
   1182  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1183  1.1  jmcneill } __packed __aligned(4);
   1184  1.1  jmcneill 
   1185  1.1  jmcneill struct mcx_cmd_destroy_tis_in {
   1186  1.1  jmcneill 	uint16_t		cmd_opcode;
   1187  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1188  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1189  1.1  jmcneill 	uint32_t		cmd_tisn;
   1190  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1191  1.1  jmcneill } __packed __aligned(4);
   1192  1.1  jmcneill 
   1193  1.1  jmcneill struct mcx_cmd_destroy_tis_out {
   1194  1.1  jmcneill 	uint8_t			cmd_status;
   1195  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1196  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1197  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1198  1.1  jmcneill } __packed __aligned(4);
   1199  1.1  jmcneill 
   1200  1.1  jmcneill struct mcx_cq_ctx {
   1201  1.1  jmcneill 	uint32_t		cq_status;
   1202  1.1  jmcneill 	uint32_t		cq_reserved1;
   1203  1.1  jmcneill 	uint32_t		cq_page_offset;
   1204  1.1  jmcneill 	uint32_t		cq_uar_size;
   1205  1.1  jmcneill #define MCX_CQ_CTX_UAR_PAGE_MASK	0xffffff
   1206  1.1  jmcneill #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT	24
   1207  1.1  jmcneill 	uint32_t		cq_period_max_count;
   1208  1.1  jmcneill #define MCX_CQ_CTX_PERIOD_SHIFT		16
   1209  1.1  jmcneill 	uint32_t		cq_eqn;
   1210  1.1  jmcneill 	uint32_t		cq_log_page_size;
   1211  1.1  jmcneill #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT	24
   1212  1.1  jmcneill 	uint32_t		cq_reserved2;
   1213  1.1  jmcneill 	uint32_t		cq_last_notified;
   1214  1.1  jmcneill 	uint32_t		cq_last_solicit;
   1215  1.1  jmcneill 	uint32_t		cq_consumer_counter;
   1216  1.1  jmcneill 	uint32_t		cq_producer_counter;
   1217  1.1  jmcneill 	uint8_t			cq_reserved3[8];
   1218  1.1  jmcneill 	uint64_t		cq_doorbell;
   1219  1.1  jmcneill } __packed __aligned(4);
   1220  1.1  jmcneill 
   1221  1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
   1222  1.1  jmcneill 
   1223  1.1  jmcneill struct mcx_cmd_create_cq_in {
   1224  1.1  jmcneill 	uint16_t		cmd_opcode;
   1225  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1226  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1227  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1228  1.1  jmcneill } __packed __aligned(4);
   1229  1.1  jmcneill 
   1230  1.1  jmcneill struct mcx_cmd_create_cq_mb_in {
   1231  1.1  jmcneill 	struct mcx_cq_ctx	cmd_cq_ctx;
   1232  1.1  jmcneill 	uint8_t			cmd_reserved1[192];
   1233  1.1  jmcneill } __packed __aligned(4);
   1234  1.1  jmcneill 
   1235  1.1  jmcneill struct mcx_cmd_create_cq_out {
   1236  1.1  jmcneill 	uint8_t			cmd_status;
   1237  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1238  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1239  1.1  jmcneill 	uint32_t		cmd_cqn;
   1240  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1241  1.1  jmcneill } __packed __aligned(4);
   1242  1.1  jmcneill 
   1243  1.1  jmcneill struct mcx_cmd_destroy_cq_in {
   1244  1.1  jmcneill 	uint16_t		cmd_opcode;
   1245  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1246  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1247  1.1  jmcneill 	uint32_t		cmd_cqn;
   1248  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1249  1.1  jmcneill } __packed __aligned(4);
   1250  1.1  jmcneill 
   1251  1.1  jmcneill struct mcx_cmd_destroy_cq_out {
   1252  1.1  jmcneill 	uint8_t			cmd_status;
   1253  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1254  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1255  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1256  1.1  jmcneill } __packed __aligned(4);
   1257  1.1  jmcneill 
   1258  1.1  jmcneill struct mcx_cq_entry {
   1259  1.1  jmcneill 	uint32_t		__reserved__;
   1260  1.1  jmcneill 	uint32_t		cq_lro;
   1261  1.1  jmcneill 	uint32_t		cq_lro_ack_seq_num;
   1262  1.1  jmcneill 	uint32_t		cq_rx_hash;
   1263  1.1  jmcneill 	uint8_t			cq_rx_hash_type;
   1264  1.1  jmcneill 	uint8_t			cq_ml_path;
   1265  1.1  jmcneill 	uint16_t		__reserved__;
   1266  1.1  jmcneill 	uint32_t		cq_checksum;
   1267  1.1  jmcneill 	uint32_t		__reserved__;
   1268  1.1  jmcneill 	uint32_t		cq_flags;
   1269  1.1  jmcneill 	uint32_t		cq_lro_srqn;
   1270  1.1  jmcneill 	uint32_t		__reserved__[2];
   1271  1.1  jmcneill 	uint32_t		cq_byte_cnt;
   1272  1.1  jmcneill 	uint64_t		cq_timestamp;
   1273  1.1  jmcneill 	uint8_t			cq_rx_drops;
   1274  1.1  jmcneill 	uint8_t			cq_flow_tag[3];
   1275  1.1  jmcneill 	uint16_t		cq_wqe_count;
   1276  1.1  jmcneill 	uint8_t			cq_signature;
   1277  1.1  jmcneill 	uint8_t			cq_opcode_owner;
   1278  1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_OWNER			(1 << 0)
   1279  1.1  jmcneill #define MCX_CQ_ENTRY_FLAG_SE			(1 << 1)
   1280  1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_SHIFT		2
   1281  1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SHIFT		4
   1282  1.1  jmcneill 
   1283  1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_NO_INLINE		0
   1284  1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_32		1
   1285  1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_64		2
   1286  1.1  jmcneill #define MCX_CQ_ENTRY_FORMAT_COMPRESSED		3
   1287  1.1  jmcneill 
   1288  1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ			0
   1289  1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND		2
   1290  1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ_ERR		13
   1291  1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND_ERR		14
   1292  1.1  jmcneill #define MCX_CQ_ENTRY_OPCODE_INVALID		15
   1293  1.1  jmcneill 
   1294  1.1  jmcneill } __packed __aligned(4);
   1295  1.1  jmcneill 
   1296  1.1  jmcneill CTASSERT(sizeof(struct mcx_cq_entry) == 64);
   1297  1.1  jmcneill 
   1298  1.1  jmcneill struct mcx_cq_doorbell {
   1299  1.1  jmcneill 	uint32_t		 db_update_ci;
   1300  1.1  jmcneill 	uint32_t		 db_arm_ci;
   1301  1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT	28
   1302  1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CMD			(1 << 24)
   1303  1.1  jmcneill #define MCX_CQ_DOORBELL_ARM_CI_MASK		(0xffffff)
   1304  1.1  jmcneill } __packed __aligned(8);
   1305  1.1  jmcneill 
   1306  1.1  jmcneill struct mcx_wq_ctx {
   1307  1.1  jmcneill 	uint8_t			 wq_type;
   1308  1.1  jmcneill #define MCX_WQ_CTX_TYPE_CYCLIC			(1 << 4)
   1309  1.1  jmcneill #define MCX_WQ_CTX_TYPE_SIGNATURE		(1 << 3)
   1310  1.1  jmcneill 	uint8_t			 wq_reserved0[5];
   1311  1.1  jmcneill 	uint16_t		 wq_lwm;
   1312  1.1  jmcneill 	uint32_t		 wq_pd;
   1313  1.1  jmcneill 	uint32_t		 wq_uar_page;
   1314  1.1  jmcneill 	uint64_t		 wq_doorbell;
   1315  1.1  jmcneill 	uint32_t		 wq_hw_counter;
   1316  1.1  jmcneill 	uint32_t		 wq_sw_counter;
   1317  1.1  jmcneill 	uint16_t		 wq_log_stride;
   1318  1.1  jmcneill 	uint8_t			 wq_log_page_sz;
   1319  1.1  jmcneill 	uint8_t			 wq_log_size;
   1320  1.1  jmcneill 	uint8_t			 wq_reserved1[156];
   1321  1.1  jmcneill } __packed __aligned(4);
   1322  1.1  jmcneill 
   1323  1.1  jmcneill CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
   1324  1.1  jmcneill 
   1325  1.1  jmcneill struct mcx_sq_ctx {
   1326  1.1  jmcneill 	uint32_t		sq_flags;
   1327  1.1  jmcneill #define MCX_SQ_CTX_RLKEY			(1 << 31)
   1328  1.1  jmcneill #define MCX_SQ_CTX_FRE_SHIFT			(1 << 29)
   1329  1.1  jmcneill #define MCX_SQ_CTX_FLUSH_IN_ERROR		(1 << 28)
   1330  1.1  jmcneill #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT		24
   1331  1.1  jmcneill #define MCX_SQ_CTX_STATE_SHIFT			20
   1332  1.1  jmcneill 	uint32_t		sq_user_index;
   1333  1.1  jmcneill 	uint32_t		sq_cqn;
   1334  1.1  jmcneill 	uint32_t		sq_reserved1[5];
   1335  1.1  jmcneill 	uint32_t		sq_tis_lst_sz;
   1336  1.1  jmcneill #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT		16
   1337  1.1  jmcneill 	uint32_t		sq_reserved2[2];
   1338  1.1  jmcneill 	uint32_t		sq_tis_num;
   1339  1.1  jmcneill 	struct mcx_wq_ctx	sq_wq;
   1340  1.1  jmcneill } __packed __aligned(4);
   1341  1.1  jmcneill 
   1342  1.1  jmcneill struct mcx_sq_entry_seg {
   1343  1.1  jmcneill 	uint32_t		sqs_byte_count;
   1344  1.1  jmcneill 	uint32_t		sqs_lkey;
   1345  1.1  jmcneill 	uint64_t		sqs_addr;
   1346  1.1  jmcneill } __packed __aligned(4);
   1347  1.1  jmcneill 
   1348  1.1  jmcneill struct mcx_sq_entry {
   1349  1.1  jmcneill 	/* control segment */
   1350  1.1  jmcneill 	uint32_t		sqe_opcode_index;
   1351  1.1  jmcneill #define MCX_SQE_WQE_INDEX_SHIFT			8
   1352  1.1  jmcneill #define MCX_SQE_WQE_OPCODE_NOP			0x00
   1353  1.1  jmcneill #define MCX_SQE_WQE_OPCODE_SEND			0x0a
   1354  1.1  jmcneill 	uint32_t		sqe_ds_sq_num;
   1355  1.1  jmcneill #define MCX_SQE_SQ_NUM_SHIFT			8
   1356  1.1  jmcneill 	uint32_t		sqe_signature;
   1357  1.1  jmcneill #define MCX_SQE_SIGNATURE_SHIFT			24
   1358  1.1  jmcneill #define MCX_SQE_SOLICITED_EVENT			0x02
   1359  1.1  jmcneill #define MCX_SQE_CE_CQE_ON_ERR			0x00
   1360  1.1  jmcneill #define MCX_SQE_CE_CQE_FIRST_ERR		0x04
   1361  1.1  jmcneill #define MCX_SQE_CE_CQE_ALWAYS			0x08
   1362  1.1  jmcneill #define MCX_SQE_CE_CQE_SOLICIT			0x0C
   1363  1.1  jmcneill #define MCX_SQE_FM_NO_FENCE			0x00
   1364  1.1  jmcneill #define MCX_SQE_FM_SMALL_FENCE			0x40
   1365  1.1  jmcneill 	uint32_t		sqe_mkey;
   1366  1.1  jmcneill 
   1367  1.1  jmcneill 	/* ethernet segment */
   1368  1.1  jmcneill 	uint32_t		sqe_reserved1;
   1369  1.1  jmcneill 	uint32_t		sqe_mss_csum;
   1370  1.1  jmcneill #define MCX_SQE_L4_CSUM				(1 << 31)
   1371  1.1  jmcneill #define MCX_SQE_L3_CSUM				(1 << 30)
   1372  1.1  jmcneill 	uint32_t		sqe_reserved2;
   1373  1.1  jmcneill 	uint16_t		sqe_inline_header_size;
   1374  1.1  jmcneill 	uint16_t		sqe_inline_headers[9];
   1375  1.1  jmcneill 
   1376  1.1  jmcneill 	/* data segment */
   1377  1.1  jmcneill 	struct mcx_sq_entry_seg sqe_segs[1];
   1378  1.1  jmcneill } __packed __aligned(64);
   1379  1.1  jmcneill 
   1380  1.1  jmcneill CTASSERT(sizeof(struct mcx_sq_entry) == 64);
   1381  1.1  jmcneill 
   1382  1.1  jmcneill struct mcx_cmd_create_sq_in {
   1383  1.1  jmcneill 	uint16_t		cmd_opcode;
   1384  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1385  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1386  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1387  1.1  jmcneill } __packed __aligned(4);
   1388  1.1  jmcneill 
   1389  1.1  jmcneill struct mcx_cmd_create_sq_out {
   1390  1.1  jmcneill 	uint8_t			cmd_status;
   1391  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1392  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1393  1.1  jmcneill 	uint32_t		cmd_sqn;
   1394  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1395  1.1  jmcneill } __packed __aligned(4);
   1396  1.1  jmcneill 
   1397  1.1  jmcneill struct mcx_cmd_modify_sq_in {
   1398  1.1  jmcneill 	uint16_t		cmd_opcode;
   1399  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1400  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1401  1.1  jmcneill 	uint32_t		cmd_sq_state;
   1402  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1403  1.1  jmcneill } __packed __aligned(4);
   1404  1.1  jmcneill 
   1405  1.1  jmcneill struct mcx_cmd_modify_sq_mb_in {
   1406  1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1407  1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1408  1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1409  1.1  jmcneill 	struct mcx_sq_ctx	cmd_sq_ctx;
   1410  1.1  jmcneill } __packed __aligned(4);
   1411  1.1  jmcneill 
   1412  1.1  jmcneill struct mcx_cmd_modify_sq_out {
   1413  1.1  jmcneill 	uint8_t			cmd_status;
   1414  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1415  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1416  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1417  1.1  jmcneill } __packed __aligned(4);
   1418  1.1  jmcneill 
   1419  1.1  jmcneill struct mcx_cmd_destroy_sq_in {
   1420  1.1  jmcneill 	uint16_t		cmd_opcode;
   1421  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1422  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1423  1.1  jmcneill 	uint32_t		cmd_sqn;
   1424  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1425  1.1  jmcneill } __packed __aligned(4);
   1426  1.1  jmcneill 
   1427  1.1  jmcneill struct mcx_cmd_destroy_sq_out {
   1428  1.1  jmcneill 	uint8_t			cmd_status;
   1429  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1430  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1431  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1432  1.1  jmcneill } __packed __aligned(4);
   1433  1.1  jmcneill 
   1434  1.1  jmcneill 
   1435  1.1  jmcneill struct mcx_rq_ctx {
   1436  1.1  jmcneill 	uint32_t		rq_flags;
   1437  1.1  jmcneill #define MCX_RQ_CTX_RLKEY			(1 << 31)
   1438  1.1  jmcneill #define MCX_RQ_CTX_VLAN_STRIP_DIS		(1 << 28)
   1439  1.1  jmcneill #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT		24
   1440  1.1  jmcneill #define MCX_RQ_CTX_STATE_SHIFT			20
   1441  1.1  jmcneill #define MCX_RQ_CTX_FLUSH_IN_ERROR		(1 << 18)
   1442  1.1  jmcneill 	uint32_t		rq_user_index;
   1443  1.1  jmcneill 	uint32_t		rq_cqn;
   1444  1.1  jmcneill 	uint32_t		rq_reserved1;
   1445  1.1  jmcneill 	uint32_t		rq_rmpn;
   1446  1.1  jmcneill 	uint32_t		rq_reserved2[7];
   1447  1.1  jmcneill 	struct mcx_wq_ctx	rq_wq;
   1448  1.1  jmcneill } __packed __aligned(4);
   1449  1.1  jmcneill 
   1450  1.1  jmcneill struct mcx_rq_entry {
   1451  1.1  jmcneill 	uint32_t		rqe_byte_count;
   1452  1.1  jmcneill 	uint32_t		rqe_lkey;
   1453  1.1  jmcneill 	uint64_t		rqe_addr;
   1454  1.1  jmcneill } __packed __aligned(16);
   1455  1.1  jmcneill 
   1456  1.1  jmcneill struct mcx_cmd_create_rq_in {
   1457  1.1  jmcneill 	uint16_t		cmd_opcode;
   1458  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1459  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1460  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1461  1.1  jmcneill } __packed __aligned(4);
   1462  1.1  jmcneill 
   1463  1.1  jmcneill struct mcx_cmd_create_rq_out {
   1464  1.1  jmcneill 	uint8_t			cmd_status;
   1465  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1466  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1467  1.1  jmcneill 	uint32_t		cmd_rqn;
   1468  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1469  1.1  jmcneill } __packed __aligned(4);
   1470  1.1  jmcneill 
   1471  1.1  jmcneill struct mcx_cmd_modify_rq_in {
   1472  1.1  jmcneill 	uint16_t		cmd_opcode;
   1473  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1474  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1475  1.1  jmcneill 	uint32_t		cmd_rq_state;
   1476  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1477  1.1  jmcneill } __packed __aligned(4);
   1478  1.1  jmcneill 
   1479  1.1  jmcneill struct mcx_cmd_modify_rq_mb_in {
   1480  1.1  jmcneill 	uint32_t		cmd_modify_hi;
   1481  1.1  jmcneill 	uint32_t		cmd_modify_lo;
   1482  1.1  jmcneill 	uint8_t			cmd_reserved0[8];
   1483  1.1  jmcneill 	struct mcx_rq_ctx	cmd_rq_ctx;
   1484  1.1  jmcneill } __packed __aligned(4);
   1485  1.1  jmcneill 
   1486  1.1  jmcneill struct mcx_cmd_modify_rq_out {
   1487  1.1  jmcneill 	uint8_t			cmd_status;
   1488  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1489  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1490  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1491  1.1  jmcneill } __packed __aligned(4);
   1492  1.1  jmcneill 
   1493  1.1  jmcneill struct mcx_cmd_destroy_rq_in {
   1494  1.1  jmcneill 	uint16_t		cmd_opcode;
   1495  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1496  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1497  1.1  jmcneill 	uint32_t		cmd_rqn;
   1498  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1499  1.1  jmcneill } __packed __aligned(4);
   1500  1.1  jmcneill 
   1501  1.1  jmcneill struct mcx_cmd_destroy_rq_out {
   1502  1.1  jmcneill 	uint8_t			cmd_status;
   1503  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1504  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1505  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1506  1.1  jmcneill } __packed __aligned(4);
   1507  1.1  jmcneill 
   1508  1.1  jmcneill struct mcx_cmd_create_flow_table_in {
   1509  1.1  jmcneill 	uint16_t		cmd_opcode;
   1510  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1511  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1512  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1513  1.1  jmcneill } __packed __aligned(4);
   1514  1.1  jmcneill 
   1515  1.1  jmcneill struct mcx_flow_table_ctx {
   1516  1.1  jmcneill 	uint8_t			ft_miss_action;
   1517  1.1  jmcneill 	uint8_t			ft_level;
   1518  1.1  jmcneill 	uint8_t			ft_reserved0;
   1519  1.1  jmcneill 	uint8_t			ft_log_size;
   1520  1.1  jmcneill 	uint32_t		ft_table_miss_id;
   1521  1.1  jmcneill 	uint8_t			ft_reserved1[28];
   1522  1.1  jmcneill } __packed __aligned(4);
   1523  1.1  jmcneill 
   1524  1.1  jmcneill struct mcx_cmd_create_flow_table_mb_in {
   1525  1.1  jmcneill 	uint8_t			cmd_table_type;
   1526  1.1  jmcneill 	uint8_t			cmd_reserved0[7];
   1527  1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   1528  1.1  jmcneill } __packed __aligned(4);
   1529  1.1  jmcneill 
   1530  1.1  jmcneill struct mcx_cmd_create_flow_table_out {
   1531  1.1  jmcneill 	uint8_t			cmd_status;
   1532  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1533  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1534  1.1  jmcneill 	uint32_t		cmd_table_id;
   1535  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1536  1.1  jmcneill } __packed __aligned(4);
   1537  1.1  jmcneill 
   1538  1.1  jmcneill struct mcx_cmd_destroy_flow_table_in {
   1539  1.1  jmcneill 	uint16_t		cmd_opcode;
   1540  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1541  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1542  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1543  1.1  jmcneill } __packed __aligned(4);
   1544  1.1  jmcneill 
   1545  1.1  jmcneill struct mcx_cmd_destroy_flow_table_mb_in {
   1546  1.1  jmcneill 	uint8_t			cmd_table_type;
   1547  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1548  1.1  jmcneill 	uint32_t		cmd_table_id;
   1549  1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   1550  1.1  jmcneill } __packed __aligned(4);
   1551  1.1  jmcneill 
   1552  1.1  jmcneill struct mcx_cmd_destroy_flow_table_out {
   1553  1.1  jmcneill 	uint8_t			cmd_status;
   1554  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1555  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1556  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1557  1.1  jmcneill } __packed __aligned(4);
   1558  1.1  jmcneill 
   1559  1.1  jmcneill struct mcx_cmd_set_flow_table_root_in {
   1560  1.1  jmcneill 	uint16_t		cmd_opcode;
   1561  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1562  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1563  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1564  1.1  jmcneill } __packed __aligned(4);
   1565  1.1  jmcneill 
   1566  1.1  jmcneill struct mcx_cmd_set_flow_table_root_mb_in {
   1567  1.1  jmcneill 	uint8_t			cmd_table_type;
   1568  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1569  1.1  jmcneill 	uint32_t		cmd_table_id;
   1570  1.1  jmcneill 	uint8_t			cmd_reserved1[56];
   1571  1.1  jmcneill } __packed __aligned(4);
   1572  1.1  jmcneill 
   1573  1.1  jmcneill struct mcx_cmd_set_flow_table_root_out {
   1574  1.1  jmcneill 	uint8_t			cmd_status;
   1575  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1576  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1577  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1578  1.1  jmcneill } __packed __aligned(4);
   1579  1.1  jmcneill 
   1580  1.1  jmcneill struct mcx_flow_match {
   1581  1.1  jmcneill 	/* outer headers */
   1582  1.1  jmcneill 	uint8_t			mc_src_mac[6];
   1583  1.1  jmcneill 	uint16_t		mc_ethertype;
   1584  1.1  jmcneill 	uint8_t			mc_dest_mac[6];
   1585  1.1  jmcneill 	uint16_t		mc_first_vlan;
   1586  1.1  jmcneill 	uint8_t			mc_ip_proto;
   1587  1.1  jmcneill 	uint8_t			mc_ip_dscp_ecn;
   1588  1.1  jmcneill 	uint8_t			mc_vlan_flags;
   1589  1.1  jmcneill 	uint8_t			mc_tcp_flags;
   1590  1.1  jmcneill 	uint16_t		mc_tcp_sport;
   1591  1.1  jmcneill 	uint16_t		mc_tcp_dport;
   1592  1.1  jmcneill 	uint32_t		mc_reserved0;
   1593  1.1  jmcneill 	uint16_t		mc_udp_sport;
   1594  1.1  jmcneill 	uint16_t		mc_udp_dport;
   1595  1.1  jmcneill 	uint8_t			mc_src_ip[16];
   1596  1.1  jmcneill 	uint8_t			mc_dest_ip[16];
   1597  1.1  jmcneill 
   1598  1.1  jmcneill 	/* misc parameters */
   1599  1.1  jmcneill 	uint8_t			mc_reserved1[8];
   1600  1.1  jmcneill 	uint16_t		mc_second_vlan;
   1601  1.1  jmcneill 	uint8_t			mc_reserved2[2];
   1602  1.1  jmcneill 	uint8_t			mc_second_vlan_flags;
   1603  1.1  jmcneill 	uint8_t			mc_reserved3[15];
   1604  1.1  jmcneill 	uint32_t		mc_outer_ipv6_flow_label;
   1605  1.1  jmcneill 	uint8_t			mc_reserved4[32];
   1606  1.1  jmcneill 
   1607  1.1  jmcneill 	uint8_t			mc_reserved[384];
   1608  1.1  jmcneill } __packed __aligned(4);
   1609  1.1  jmcneill 
   1610  1.1  jmcneill CTASSERT(sizeof(struct mcx_flow_match) == 512);
   1611  1.1  jmcneill 
   1612  1.1  jmcneill struct mcx_cmd_create_flow_group_in {
   1613  1.1  jmcneill 	uint16_t		cmd_opcode;
   1614  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1615  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1616  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1617  1.1  jmcneill } __packed __aligned(4);
   1618  1.1  jmcneill 
   1619  1.1  jmcneill struct mcx_cmd_create_flow_group_mb_in {
   1620  1.1  jmcneill 	uint8_t			cmd_table_type;
   1621  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1622  1.1  jmcneill 	uint32_t		cmd_table_id;
   1623  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1624  1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   1625  1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   1626  1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   1627  1.1  jmcneill 	uint8_t			cmd_reserved3[23];
   1628  1.1  jmcneill 	uint8_t			cmd_match_criteria_enable;
   1629  1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER	(1 << 0)
   1630  1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_MISC		(1 << 1)
   1631  1.1  jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_INNER	(1 << 2)
   1632  1.1  jmcneill 	struct mcx_flow_match	cmd_match_criteria;
   1633  1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   1634  1.1  jmcneill } __packed __aligned(4);
   1635  1.1  jmcneill 
   1636  1.1  jmcneill struct mcx_cmd_create_flow_group_out {
   1637  1.1  jmcneill 	uint8_t			cmd_status;
   1638  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1639  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1640  1.1  jmcneill 	uint32_t		cmd_group_id;
   1641  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1642  1.1  jmcneill } __packed __aligned(4);
   1643  1.1  jmcneill 
   1644  1.1  jmcneill struct mcx_flow_ctx {
   1645  1.1  jmcneill 	uint8_t			fc_reserved0[4];
   1646  1.1  jmcneill 	uint32_t		fc_group_id;
   1647  1.1  jmcneill 	uint32_t		fc_flow_tag;
   1648  1.1  jmcneill 	uint32_t		fc_action;
   1649  1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_ALLOW		(1 << 0)
   1650  1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_DROP		(1 << 1)
   1651  1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_FORWARD		(1 << 2)
   1652  1.1  jmcneill #define MCX_FLOW_CONTEXT_ACTION_COUNT		(1 << 3)
   1653  1.1  jmcneill 	uint32_t		fc_dest_list_size;
   1654  1.1  jmcneill 	uint32_t		fc_counter_list_size;
   1655  1.1  jmcneill 	uint8_t			fc_reserved1[40];
   1656  1.1  jmcneill 	struct mcx_flow_match	fc_match_value;
   1657  1.1  jmcneill 	uint8_t			fc_reserved2[192];
   1658  1.1  jmcneill } __packed __aligned(4);
   1659  1.1  jmcneill 
   1660  1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE	(1 << 24)
   1661  1.1  jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR		(2 << 24)
   1662  1.1  jmcneill 
   1663  1.1  jmcneill struct mcx_cmd_destroy_flow_group_in {
   1664  1.1  jmcneill 	uint16_t		cmd_opcode;
   1665  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1666  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1667  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1668  1.1  jmcneill } __packed __aligned(4);
   1669  1.1  jmcneill 
   1670  1.1  jmcneill struct mcx_cmd_destroy_flow_group_mb_in {
   1671  1.1  jmcneill 	uint8_t			cmd_table_type;
   1672  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1673  1.1  jmcneill 	uint32_t		cmd_table_id;
   1674  1.1  jmcneill 	uint32_t		cmd_group_id;
   1675  1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   1676  1.1  jmcneill } __packed __aligned(4);
   1677  1.1  jmcneill 
   1678  1.1  jmcneill struct mcx_cmd_destroy_flow_group_out {
   1679  1.1  jmcneill 	uint8_t			cmd_status;
   1680  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1681  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1682  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1683  1.1  jmcneill } __packed __aligned(4);
   1684  1.1  jmcneill 
   1685  1.1  jmcneill struct mcx_cmd_set_flow_table_entry_in {
   1686  1.1  jmcneill 	uint16_t		cmd_opcode;
   1687  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1688  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1689  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1690  1.1  jmcneill } __packed __aligned(4);
   1691  1.1  jmcneill 
   1692  1.1  jmcneill struct mcx_cmd_set_flow_table_entry_mb_in {
   1693  1.1  jmcneill 	uint8_t			cmd_table_type;
   1694  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1695  1.1  jmcneill 	uint32_t		cmd_table_id;
   1696  1.1  jmcneill 	uint32_t		cmd_modify_enable_mask;
   1697  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1698  1.1  jmcneill 	uint32_t		cmd_flow_index;
   1699  1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   1700  1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   1701  1.1  jmcneill } __packed __aligned(4);
   1702  1.1  jmcneill 
   1703  1.1  jmcneill struct mcx_cmd_set_flow_table_entry_out {
   1704  1.1  jmcneill 	uint8_t			cmd_status;
   1705  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1706  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1707  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1708  1.1  jmcneill } __packed __aligned(4);
   1709  1.1  jmcneill 
   1710  1.1  jmcneill struct mcx_cmd_query_flow_table_entry_in {
   1711  1.1  jmcneill 	uint16_t		cmd_opcode;
   1712  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1713  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1714  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1715  1.1  jmcneill } __packed __aligned(4);
   1716  1.1  jmcneill 
   1717  1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_in {
   1718  1.1  jmcneill 	uint8_t			cmd_table_type;
   1719  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1720  1.1  jmcneill 	uint32_t		cmd_table_id;
   1721  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1722  1.1  jmcneill 	uint32_t		cmd_flow_index;
   1723  1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   1724  1.1  jmcneill } __packed __aligned(4);
   1725  1.1  jmcneill 
   1726  1.1  jmcneill struct mcx_cmd_query_flow_table_entry_out {
   1727  1.1  jmcneill 	uint8_t			cmd_status;
   1728  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1729  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1730  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1731  1.1  jmcneill } __packed __aligned(4);
   1732  1.1  jmcneill 
   1733  1.1  jmcneill struct mcx_cmd_query_flow_table_entry_mb_out {
   1734  1.1  jmcneill 	uint8_t			cmd_reserved0[48];
   1735  1.1  jmcneill 	struct mcx_flow_ctx	cmd_flow_ctx;
   1736  1.1  jmcneill } __packed __aligned(4);
   1737  1.1  jmcneill 
   1738  1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_in {
   1739  1.1  jmcneill 	uint16_t		cmd_opcode;
   1740  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1741  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1742  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1743  1.1  jmcneill } __packed __aligned(4);
   1744  1.1  jmcneill 
   1745  1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in {
   1746  1.1  jmcneill 	uint8_t			cmd_table_type;
   1747  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1748  1.1  jmcneill 	uint32_t		cmd_table_id;
   1749  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1750  1.1  jmcneill 	uint32_t		cmd_flow_index;
   1751  1.1  jmcneill 	uint8_t			cmd_reserved2[28];
   1752  1.1  jmcneill } __packed __aligned(4);
   1753  1.1  jmcneill 
   1754  1.1  jmcneill struct mcx_cmd_delete_flow_table_entry_out {
   1755  1.1  jmcneill 	uint8_t			cmd_status;
   1756  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1757  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1758  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1759  1.1  jmcneill } __packed __aligned(4);
   1760  1.1  jmcneill 
   1761  1.1  jmcneill struct mcx_cmd_query_flow_group_in {
   1762  1.1  jmcneill 	uint16_t		cmd_opcode;
   1763  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1764  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1765  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1766  1.1  jmcneill } __packed __aligned(4);
   1767  1.1  jmcneill 
   1768  1.1  jmcneill struct mcx_cmd_query_flow_group_mb_in {
   1769  1.1  jmcneill 	uint8_t			cmd_table_type;
   1770  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1771  1.1  jmcneill 	uint32_t		cmd_table_id;
   1772  1.1  jmcneill 	uint32_t		cmd_group_id;
   1773  1.1  jmcneill 	uint8_t			cmd_reserved1[36];
   1774  1.1  jmcneill } __packed __aligned(4);
   1775  1.1  jmcneill 
   1776  1.1  jmcneill struct mcx_cmd_query_flow_group_out {
   1777  1.1  jmcneill 	uint8_t			cmd_status;
   1778  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1779  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1780  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1781  1.1  jmcneill } __packed __aligned(4);
   1782  1.1  jmcneill 
   1783  1.1  jmcneill struct mcx_cmd_query_flow_group_mb_out {
   1784  1.1  jmcneill 	uint8_t			cmd_reserved0[12];
   1785  1.1  jmcneill 	uint32_t		cmd_start_flow_index;
   1786  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1787  1.1  jmcneill 	uint32_t		cmd_end_flow_index;
   1788  1.1  jmcneill 	uint8_t			cmd_reserved2[20];
   1789  1.1  jmcneill 	uint32_t		cmd_match_criteria_enable;
   1790  1.1  jmcneill 	uint8_t			cmd_match_criteria[512];
   1791  1.1  jmcneill 	uint8_t			cmd_reserved4[448];
   1792  1.1  jmcneill } __packed __aligned(4);
   1793  1.1  jmcneill 
   1794  1.1  jmcneill struct mcx_cmd_query_flow_table_in {
   1795  1.1  jmcneill 	uint16_t		cmd_opcode;
   1796  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1797  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1798  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1799  1.1  jmcneill } __packed __aligned(4);
   1800  1.1  jmcneill 
   1801  1.1  jmcneill struct mcx_cmd_query_flow_table_mb_in {
   1802  1.1  jmcneill 	uint8_t			cmd_table_type;
   1803  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1804  1.1  jmcneill 	uint32_t		cmd_table_id;
   1805  1.1  jmcneill 	uint8_t			cmd_reserved1[40];
   1806  1.1  jmcneill } __packed __aligned(4);
   1807  1.1  jmcneill 
   1808  1.1  jmcneill struct mcx_cmd_query_flow_table_out {
   1809  1.1  jmcneill 	uint8_t			cmd_status;
   1810  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1811  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1812  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1813  1.1  jmcneill } __packed __aligned(4);
   1814  1.1  jmcneill 
   1815  1.1  jmcneill struct mcx_cmd_query_flow_table_mb_out {
   1816  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1817  1.1  jmcneill 	struct mcx_flow_table_ctx cmd_ctx;
   1818  1.1  jmcneill } __packed __aligned(4);
   1819  1.1  jmcneill 
   1820  1.1  jmcneill struct mcx_cmd_alloc_flow_counter_in {
   1821  1.1  jmcneill 	uint16_t		cmd_opcode;
   1822  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1823  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1824  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1825  1.1  jmcneill } __packed __aligned(4);
   1826  1.1  jmcneill 
   1827  1.1  jmcneill struct mcx_cmd_query_rq_in {
   1828  1.1  jmcneill 	uint16_t		cmd_opcode;
   1829  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1830  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1831  1.1  jmcneill 	uint32_t		cmd_rqn;
   1832  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1833  1.1  jmcneill } __packed __aligned(4);
   1834  1.1  jmcneill 
   1835  1.1  jmcneill struct mcx_cmd_query_rq_out {
   1836  1.1  jmcneill 	uint8_t			cmd_status;
   1837  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1838  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1839  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1840  1.1  jmcneill } __packed __aligned(4);
   1841  1.1  jmcneill 
   1842  1.1  jmcneill struct mcx_cmd_query_rq_mb_out {
   1843  1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   1844  1.1  jmcneill 	struct mcx_rq_ctx	cmd_ctx;
   1845  1.1  jmcneill };
   1846  1.1  jmcneill 
   1847  1.1  jmcneill struct mcx_cmd_query_sq_in {
   1848  1.1  jmcneill 	uint16_t		cmd_opcode;
   1849  1.1  jmcneill 	uint8_t			cmd_reserved0[4];
   1850  1.1  jmcneill 	uint16_t		cmd_op_mod;
   1851  1.1  jmcneill 	uint32_t		cmd_sqn;
   1852  1.1  jmcneill 	uint8_t			cmd_reserved1[4];
   1853  1.1  jmcneill } __packed __aligned(4);
   1854  1.1  jmcneill 
   1855  1.1  jmcneill struct mcx_cmd_query_sq_out {
   1856  1.1  jmcneill 	uint8_t			cmd_status;
   1857  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1858  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1859  1.1  jmcneill 	uint8_t			cmd_reserved1[8];
   1860  1.1  jmcneill } __packed __aligned(4);
   1861  1.1  jmcneill 
   1862  1.1  jmcneill struct mcx_cmd_query_sq_mb_out {
   1863  1.1  jmcneill 	uint8_t			cmd_reserved0[16];
   1864  1.1  jmcneill 	struct mcx_sq_ctx	cmd_ctx;
   1865  1.1  jmcneill };
   1866  1.1  jmcneill 
   1867  1.1  jmcneill struct mcx_cmd_alloc_flow_counter_out {
   1868  1.1  jmcneill 	uint8_t			cmd_status;
   1869  1.1  jmcneill 	uint8_t			cmd_reserved0[3];
   1870  1.1  jmcneill 	uint32_t		cmd_syndrome;
   1871  1.1  jmcneill 	uint8_t			cmd_reserved1[2];
   1872  1.1  jmcneill 	uint16_t		cmd_flow_counter_id;
   1873  1.1  jmcneill 	uint8_t			cmd_reserved2[4];
   1874  1.1  jmcneill } __packed __aligned(4);
   1875  1.1  jmcneill 
   1876  1.1  jmcneill struct mcx_wq_doorbell {
   1877  1.1  jmcneill 	uint32_t		 db_recv_counter;
   1878  1.1  jmcneill 	uint32_t		 db_send_counter;
   1879  1.1  jmcneill } __packed __aligned(8);
   1880  1.1  jmcneill 
   1881  1.1  jmcneill struct mcx_dmamem {
   1882  1.1  jmcneill 	bus_dmamap_t		 mxm_map;
   1883  1.1  jmcneill 	bus_dma_segment_t	 mxm_seg;
   1884  1.1  jmcneill 	int			 mxm_nsegs;
   1885  1.1  jmcneill 	size_t			 mxm_size;
   1886  1.1  jmcneill 	void			*mxm_kva;
   1887  1.1  jmcneill };
   1888  1.1  jmcneill #define MCX_DMA_MAP(_mxm)	((_mxm)->mxm_map)
   1889  1.1  jmcneill #define MCX_DMA_DVA(_mxm)	((_mxm)->mxm_map->dm_segs[0].ds_addr)
   1890  1.1  jmcneill #define MCX_DMA_KVA(_mxm)	((void *)(_mxm)->mxm_kva)
   1891  1.1  jmcneill #define MCX_DMA_LEN(_mxm)	((_mxm)->mxm_size)
   1892  1.1  jmcneill 
   1893  1.1  jmcneill struct mcx_hwmem {
   1894  1.1  jmcneill 	bus_dmamap_t		 mhm_map;
   1895  1.1  jmcneill 	bus_dma_segment_t	*mhm_segs;
   1896  1.1  jmcneill 	unsigned int		 mhm_seg_count;
   1897  1.1  jmcneill 	unsigned int		 mhm_npages;
   1898  1.1  jmcneill };
   1899  1.1  jmcneill 
   1900  1.1  jmcneill struct mcx_slot {
   1901  1.1  jmcneill 	bus_dmamap_t		 ms_map;
   1902  1.1  jmcneill 	struct mbuf		*ms_m;
   1903  1.1  jmcneill };
   1904  1.1  jmcneill 
   1905  1.1  jmcneill struct mcx_cq {
   1906  1.1  jmcneill 	int			 cq_n;
   1907  1.1  jmcneill 	struct mcx_dmamem	 cq_mem;
   1908  1.1  jmcneill 	uint32_t		*cq_doorbell;
   1909  1.1  jmcneill 	uint32_t		 cq_cons;
   1910  1.1  jmcneill 	uint32_t		 cq_count;
   1911  1.1  jmcneill };
   1912  1.1  jmcneill 
   1913  1.1  jmcneill struct mcx_calibration {
   1914  1.1  jmcneill 	uint64_t		 c_timestamp;	/* previous mcx chip time */
   1915  1.1  jmcneill 	uint64_t		 c_uptime;	/* previous kernel nanouptime */
   1916  1.1  jmcneill 	uint64_t		 c_tbase;	/* mcx chip time */
   1917  1.1  jmcneill 	uint64_t		 c_ubase;	/* kernel nanouptime */
   1918  1.1  jmcneill 	uint64_t		 c_tdiff;
   1919  1.1  jmcneill 	uint64_t		 c_udiff;
   1920  1.1  jmcneill };
   1921  1.1  jmcneill 
   1922  1.1  jmcneill #define MCX_CALIBRATE_FIRST    2
   1923  1.1  jmcneill #define MCX_CALIBRATE_NORMAL   30
   1924  1.1  jmcneill 
   1925  1.1  jmcneill struct mcx_rxring {
   1926  1.1  jmcneill 	u_int			 rxr_total;
   1927  1.1  jmcneill 	u_int			 rxr_inuse;
   1928  1.1  jmcneill };
   1929  1.1  jmcneill 
   1930  1.1  jmcneill MBUFQ_HEAD(mcx_mbufq);
   1931  1.1  jmcneill 
   1932  1.1  jmcneill struct mcx_softc {
   1933  1.1  jmcneill 	device_t		 sc_dev;
   1934  1.1  jmcneill 	struct ethercom		 sc_ec;
   1935  1.1  jmcneill 	struct ifmedia		 sc_media;
   1936  1.1  jmcneill 	uint64_t		 sc_media_status;
   1937  1.1  jmcneill 	uint64_t		 sc_media_active;
   1938  1.1  jmcneill 
   1939  1.1  jmcneill 	pci_chipset_tag_t	 sc_pc;
   1940  1.1  jmcneill 	pci_intr_handle_t	*sc_intrs;
   1941  1.1  jmcneill 	void			*sc_ihs[MCX_MAX_NINTR];
   1942  1.1  jmcneill 	pcitag_t		 sc_tag;
   1943  1.1  jmcneill 
   1944  1.1  jmcneill 	bus_dma_tag_t		 sc_dmat;
   1945  1.1  jmcneill 	bus_space_tag_t		 sc_memt;
   1946  1.1  jmcneill 	bus_space_handle_t	 sc_memh;
   1947  1.1  jmcneill 	bus_size_t		 sc_mems;
   1948  1.1  jmcneill 
   1949  1.1  jmcneill 	struct mcx_dmamem	 sc_cmdq_mem;
   1950  1.1  jmcneill 	unsigned int		 sc_cmdq_mask;
   1951  1.1  jmcneill 	unsigned int		 sc_cmdq_size;
   1952  1.1  jmcneill 
   1953  1.1  jmcneill 	unsigned int		 sc_cmdq_token;
   1954  1.1  jmcneill 
   1955  1.1  jmcneill 	struct mcx_hwmem	 sc_boot_pages;
   1956  1.1  jmcneill 	struct mcx_hwmem	 sc_init_pages;
   1957  1.1  jmcneill 	struct mcx_hwmem	 sc_regular_pages;
   1958  1.1  jmcneill 
   1959  1.1  jmcneill 	int			 sc_uar;
   1960  1.1  jmcneill 	int			 sc_pd;
   1961  1.1  jmcneill 	int			 sc_tdomain;
   1962  1.1  jmcneill 	uint32_t		 sc_lkey;
   1963  1.1  jmcneill 
   1964  1.1  jmcneill 	struct mcx_dmamem	 sc_doorbell_mem;
   1965  1.1  jmcneill 
   1966  1.1  jmcneill 	int			 sc_eqn;
   1967  1.1  jmcneill 	int			 sc_eq_cons;
   1968  1.1  jmcneill 	struct mcx_dmamem	 sc_eq_mem;
   1969  1.1  jmcneill 	int			 sc_hardmtu;
   1970  1.1  jmcneill 
   1971  1.1  jmcneill 	struct workqueue	*sc_workq;
   1972  1.1  jmcneill 	struct work		 sc_port_change;
   1973  1.1  jmcneill 
   1974  1.1  jmcneill 	int			 sc_flow_table_id;
   1975  1.1  jmcneill #define MCX_FLOW_GROUP_PROMISC	 0
   1976  1.1  jmcneill #define MCX_FLOW_GROUP_ALLMULTI	 1
   1977  1.1  jmcneill #define MCX_FLOW_GROUP_MAC	 2
   1978  1.1  jmcneill #define MCX_NUM_FLOW_GROUPS	 3
   1979  1.1  jmcneill 	int			 sc_flow_group_id[MCX_NUM_FLOW_GROUPS];
   1980  1.1  jmcneill 	int			 sc_flow_group_size[MCX_NUM_FLOW_GROUPS];
   1981  1.1  jmcneill 	int			 sc_flow_group_start[MCX_NUM_FLOW_GROUPS];
   1982  1.1  jmcneill 	int			 sc_promisc_flow_enabled;
   1983  1.1  jmcneill 	int			 sc_allmulti_flow_enabled;
   1984  1.1  jmcneill 	int			 sc_mcast_flow_base;
   1985  1.1  jmcneill 	int			 sc_extra_mcast;
   1986  1.1  jmcneill 	uint8_t			 sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];
   1987  1.1  jmcneill 
   1988  1.1  jmcneill 	struct mcx_calibration	 sc_calibration[2];
   1989  1.1  jmcneill 	unsigned int		 sc_calibration_gen;
   1990  1.1  jmcneill 	callout_t		 sc_calibrate;
   1991  1.1  jmcneill 
   1992  1.1  jmcneill 	struct mcx_cq		 sc_cq[MCX_MAX_CQS];
   1993  1.1  jmcneill 	int			 sc_num_cq;
   1994  1.1  jmcneill 
   1995  1.1  jmcneill 	/* rx */
   1996  1.1  jmcneill 	int			 sc_tirn;
   1997  1.1  jmcneill 	int			 sc_rqn;
   1998  1.1  jmcneill 	struct mcx_dmamem	 sc_rq_mem;
   1999  1.1  jmcneill 	struct mcx_slot		*sc_rx_slots;
   2000  1.1  jmcneill 	uint32_t		*sc_rx_doorbell;
   2001  1.1  jmcneill 
   2002  1.1  jmcneill 	uint32_t		 sc_rx_prod;
   2003  1.1  jmcneill 	callout_t		 sc_rx_refill;
   2004  1.1  jmcneill 	struct mcx_rxring	 sc_rxr;
   2005  1.1  jmcneill 
   2006  1.1  jmcneill 	/* tx */
   2007  1.1  jmcneill 	int			 sc_tisn;
   2008  1.1  jmcneill 	int			 sc_sqn;
   2009  1.1  jmcneill 	struct mcx_dmamem	 sc_sq_mem;
   2010  1.1  jmcneill 	struct mcx_slot		*sc_tx_slots;
   2011  1.1  jmcneill 	uint32_t		*sc_tx_doorbell;
   2012  1.1  jmcneill 	int			 sc_bf_size;
   2013  1.1  jmcneill 	int			 sc_bf_offset;
   2014  1.1  jmcneill 
   2015  1.1  jmcneill 	uint32_t		 sc_tx_cons;
   2016  1.1  jmcneill 	uint32_t		 sc_tx_prod;
   2017  1.1  jmcneill 
   2018  1.1  jmcneill 	uint64_t		 sc_last_cq_db;
   2019  1.1  jmcneill 	uint64_t		 sc_last_srq_db;
   2020  1.1  jmcneill };
   2021  1.1  jmcneill #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
   2022  1.1  jmcneill 
   2023  1.1  jmcneill static int	mcx_match(device_t, cfdata_t, void *);
   2024  1.1  jmcneill static void	mcx_attach(device_t, device_t, void *);
   2025  1.1  jmcneill 
   2026  1.1  jmcneill static void	mcx_rxr_init(struct mcx_rxring *, u_int, u_int);
   2027  1.1  jmcneill static u_int	mcx_rxr_get(struct mcx_rxring *, u_int);
   2028  1.1  jmcneill static void	mcx_rxr_put(struct mcx_rxring *, u_int);
   2029  1.1  jmcneill static u_int	mcx_rxr_inuse(struct mcx_rxring *);
   2030  1.1  jmcneill 
   2031  1.1  jmcneill static int	mcx_version(struct mcx_softc *);
   2032  1.1  jmcneill static int	mcx_init_wait(struct mcx_softc *);
   2033  1.1  jmcneill static int	mcx_enable_hca(struct mcx_softc *);
   2034  1.1  jmcneill static int	mcx_teardown_hca(struct mcx_softc *, uint16_t);
   2035  1.1  jmcneill static int	mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
   2036  1.1  jmcneill 		    int);
   2037  1.1  jmcneill static int	mcx_issi(struct mcx_softc *);
   2038  1.1  jmcneill static int	mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
   2039  1.1  jmcneill static int	mcx_hca_max_caps(struct mcx_softc *);
   2040  1.1  jmcneill static int	mcx_hca_set_caps(struct mcx_softc *);
   2041  1.1  jmcneill static int	mcx_init_hca(struct mcx_softc *);
   2042  1.1  jmcneill static int	mcx_set_driver_version(struct mcx_softc *);
   2043  1.1  jmcneill static int	mcx_iff(struct mcx_softc *);
   2044  1.1  jmcneill static int	mcx_alloc_uar(struct mcx_softc *);
   2045  1.1  jmcneill static int	mcx_alloc_pd(struct mcx_softc *);
   2046  1.1  jmcneill static int	mcx_alloc_tdomain(struct mcx_softc *);
   2047  1.1  jmcneill static int	mcx_create_eq(struct mcx_softc *);
   2048  1.1  jmcneill static int	mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *);
   2049  1.1  jmcneill static int	mcx_query_special_contexts(struct mcx_softc *);
   2050  1.1  jmcneill static int	mcx_set_port_mtu(struct mcx_softc *, int);
   2051  1.1  jmcneill static int	mcx_create_cq(struct mcx_softc *, int);
   2052  1.1  jmcneill static int	mcx_destroy_cq(struct mcx_softc *, int);
   2053  1.1  jmcneill static int	mcx_create_sq(struct mcx_softc *, int);
   2054  1.1  jmcneill static int	mcx_destroy_sq(struct mcx_softc *);
   2055  1.1  jmcneill static int	mcx_ready_sq(struct mcx_softc *);
   2056  1.1  jmcneill static int	mcx_create_rq(struct mcx_softc *, int);
   2057  1.1  jmcneill static int	mcx_destroy_rq(struct mcx_softc *);
   2058  1.1  jmcneill static int	mcx_ready_rq(struct mcx_softc *);
   2059  1.1  jmcneill static int	mcx_create_tir(struct mcx_softc *);
   2060  1.1  jmcneill static int	mcx_destroy_tir(struct mcx_softc *);
   2061  1.1  jmcneill static int	mcx_create_tis(struct mcx_softc *);
   2062  1.1  jmcneill static int	mcx_destroy_tis(struct mcx_softc *);
   2063  1.1  jmcneill static int	mcx_create_flow_table(struct mcx_softc *, int);
   2064  1.1  jmcneill static int	mcx_set_flow_table_root(struct mcx_softc *);
   2065  1.1  jmcneill static int	mcx_destroy_flow_table(struct mcx_softc *);
   2066  1.1  jmcneill static int	mcx_create_flow_group(struct mcx_softc *, int, int,
   2067  1.1  jmcneill 		    int, int, struct mcx_flow_match *);
   2068  1.1  jmcneill static int	mcx_destroy_flow_group(struct mcx_softc *, int);
   2069  1.1  jmcneill static int	mcx_set_flow_table_entry(struct mcx_softc *, int, int,
   2070  1.1  jmcneill 		    const uint8_t *);
   2071  1.1  jmcneill static int	mcx_delete_flow_table_entry(struct mcx_softc *, int, int);
   2072  1.1  jmcneill 
   2073  1.1  jmcneill #if 0
   2074  1.1  jmcneill static int	mcx_dump_flow_table(struct mcx_softc *);
   2075  1.1  jmcneill static int	mcx_dump_flow_table_entry(struct mcx_softc *, int);
   2076  1.1  jmcneill static int	mcx_dump_flow_group(struct mcx_softc *);
   2077  1.1  jmcneill static int	mcx_dump_rq(struct mcx_softc *);
   2078  1.1  jmcneill static int	mcx_dump_sq(struct mcx_softc *);
   2079  1.1  jmcneill #endif
   2080  1.1  jmcneill 
   2081  1.1  jmcneill 
   2082  1.1  jmcneill /*
   2083  1.1  jmcneill static void	mcx_cmdq_dump(const struct mcx_cmdq_entry *);
   2084  1.1  jmcneill static void	mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
   2085  1.1  jmcneill */
   2086  1.1  jmcneill static void	mcx_refill(void *);
   2087  1.1  jmcneill static int	mcx_process_rx(struct mcx_softc *, struct mcx_cq_entry *,
   2088  1.1  jmcneill 		    struct mcx_mbufq *, const struct mcx_calibration *);
   2089  1.1  jmcneill static void	mcx_process_txeof(struct mcx_softc *, struct mcx_cq_entry *,
   2090  1.1  jmcneill 		    int *);
   2091  1.1  jmcneill static void	mcx_process_cq(struct mcx_softc *, struct mcx_cq *);
   2092  1.1  jmcneill 
   2093  1.1  jmcneill static void	mcx_arm_cq(struct mcx_softc *, struct mcx_cq *);
   2094  1.1  jmcneill static void	mcx_arm_eq(struct mcx_softc *);
   2095  1.1  jmcneill static int	mcx_intr(void *);
   2096  1.1  jmcneill 
   2097  1.1  jmcneill static int	mcx_init(struct ifnet *);
   2098  1.1  jmcneill static void	mcx_stop(struct ifnet *, int);
   2099  1.1  jmcneill static int	mcx_ioctl(struct ifnet *, u_long, void *);
   2100  1.1  jmcneill static void	mcx_start(struct ifnet *);
   2101  1.1  jmcneill static void	mcx_watchdog(struct ifnet *);
   2102  1.1  jmcneill static void	mcx_media_add_types(struct mcx_softc *);
   2103  1.1  jmcneill static void	mcx_media_status(struct ifnet *, struct ifmediareq *);
   2104  1.1  jmcneill static int	mcx_media_change(struct ifnet *);
   2105  1.1  jmcneill #if 0
   2106  1.1  jmcneill static int	mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
   2107  1.1  jmcneill #endif
   2108  1.1  jmcneill static void	mcx_port_change(struct work *, void *);
   2109  1.1  jmcneill 
   2110  1.1  jmcneill static void	mcx_calibrate_first(struct mcx_softc *);
   2111  1.1  jmcneill static void	mcx_calibrate(void *);
   2112  1.1  jmcneill 
   2113  1.1  jmcneill static inline uint32_t
   2114  1.1  jmcneill 		mcx_rd(struct mcx_softc *, bus_size_t);
   2115  1.1  jmcneill static inline void
   2116  1.1  jmcneill 		mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
   2117  1.1  jmcneill static inline void
   2118  1.1  jmcneill 		mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
   2119  1.1  jmcneill 
   2120  1.1  jmcneill static uint64_t	mcx_timer(struct mcx_softc *);
   2121  1.1  jmcneill 
   2122  1.1  jmcneill static int	mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
   2123  1.1  jmcneill 		    bus_size_t, u_int align);
   2124  1.1  jmcneill static void	mcx_dmamem_zero(struct mcx_dmamem *);
   2125  1.1  jmcneill static void	mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
   2126  1.1  jmcneill 
   2127  1.1  jmcneill static int	mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
   2128  1.1  jmcneill 		    unsigned int);
   2129  1.1  jmcneill static void	mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
   2130  1.1  jmcneill 
   2131  1.1  jmcneill CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL);
   2132  1.1  jmcneill 
   2133  1.1  jmcneill static const struct {
   2134  1.1  jmcneill 	pci_vendor_id_t		vendor;
   2135  1.1  jmcneill 	pci_product_id_t	product;
   2136  1.1  jmcneill } mcx_devices[] = {
   2137  1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27700 },
   2138  1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27710 },
   2139  1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT27800 },
   2140  1.1  jmcneill 	{ PCI_VENDOR_MELLANOX,	PCI_PRODUCT_MELLANOX_MT28800 },
   2141  1.1  jmcneill };
   2142  1.1  jmcneill 
   2143  1.1  jmcneill static const uint64_t mcx_eth_cap_map[] = {
   2144  1.1  jmcneill 	IFM_1000_SGMII,
   2145  1.1  jmcneill 	IFM_1000_KX,
   2146  1.1  jmcneill 	IFM_10G_CX4,
   2147  1.1  jmcneill 	IFM_10G_KX4,
   2148  1.1  jmcneill 	IFM_10G_KR,
   2149  1.1  jmcneill 	0,
   2150  1.1  jmcneill 	IFM_40G_CR4,
   2151  1.1  jmcneill 	IFM_40G_KR4,
   2152  1.1  jmcneill 	0,
   2153  1.1  jmcneill 	0,
   2154  1.1  jmcneill 	0,
   2155  1.1  jmcneill 	0,
   2156  1.1  jmcneill 	IFM_10G_T,
   2157  1.1  jmcneill 	IFM_10G_SR,
   2158  1.1  jmcneill 	IFM_10G_LR,
   2159  1.1  jmcneill 	IFM_40G_SR4,
   2160  1.1  jmcneill 	IFM_40G_LR4,
   2161  1.1  jmcneill 	0,
   2162  1.1  jmcneill 	0, /* IFM_50G_SR2 */
   2163  1.1  jmcneill 	0,
   2164  1.1  jmcneill 	IFM_100G_CR4,
   2165  1.1  jmcneill 	IFM_100G_SR4,
   2166  1.1  jmcneill 	IFM_100G_KR4,
   2167  1.1  jmcneill 	0,
   2168  1.1  jmcneill 	0,
   2169  1.1  jmcneill 	0,
   2170  1.1  jmcneill 	0,
   2171  1.1  jmcneill 	IFM_25G_CR,
   2172  1.1  jmcneill 	IFM_25G_KR,
   2173  1.1  jmcneill 	IFM_25G_SR,
   2174  1.1  jmcneill 	IFM_50G_CR2,
   2175  1.1  jmcneill 	IFM_50G_KR2
   2176  1.1  jmcneill };
   2177  1.1  jmcneill 
   2178  1.1  jmcneill static int
   2179  1.1  jmcneill mcx_match(device_t parent, cfdata_t cf, void *aux)
   2180  1.1  jmcneill {
   2181  1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2182  1.1  jmcneill 	int n;
   2183  1.1  jmcneill 
   2184  1.1  jmcneill 	for (n = 0; n < __arraycount(mcx_devices); n++) {
   2185  1.1  jmcneill 		if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor &&
   2186  1.1  jmcneill 		    PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product)
   2187  1.1  jmcneill 			return 1;
   2188  1.1  jmcneill 	}
   2189  1.1  jmcneill 
   2190  1.1  jmcneill 	return 0;
   2191  1.1  jmcneill }
   2192  1.1  jmcneill 
   2193  1.1  jmcneill void
   2194  1.1  jmcneill mcx_attach(device_t parent, device_t self, void *aux)
   2195  1.1  jmcneill {
   2196  1.1  jmcneill 	struct mcx_softc *sc = device_private(self);
   2197  1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   2198  1.1  jmcneill 	struct pci_attach_args *pa = aux;
   2199  1.1  jmcneill 	uint8_t enaddr[ETHER_ADDR_LEN];
   2200  1.1  jmcneill 	int counts[PCI_INTR_TYPE_SIZE];
   2201  1.1  jmcneill 	char intrbuf[PCI_INTRSTR_LEN];
   2202  1.1  jmcneill 	pcireg_t memtype;
   2203  1.1  jmcneill 	uint32_t r;
   2204  1.1  jmcneill 	unsigned int cq_stride;
   2205  1.1  jmcneill 	unsigned int cq_size;
   2206  1.1  jmcneill 	const char *intrstr;
   2207  1.1  jmcneill 	int i;
   2208  1.1  jmcneill 
   2209  1.1  jmcneill 	sc->sc_dev = self;
   2210  1.1  jmcneill 	sc->sc_pc = pa->pa_pc;
   2211  1.1  jmcneill 	sc->sc_tag = pa->pa_tag;
   2212  1.1  jmcneill 	sc->sc_dmat = pa->pa_dmat;
   2213  1.1  jmcneill 
   2214  1.1  jmcneill 	/* Map the PCI memory space */
   2215  1.1  jmcneill 	memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
   2216  1.1  jmcneill 	if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
   2217  1.1  jmcneill 	    0 /*BUS_SPACE_MAP_PREFETCHABLE*/, &sc->sc_memt, &sc->sc_memh,
   2218  1.1  jmcneill 	    NULL, &sc->sc_mems)) {
   2219  1.1  jmcneill 		aprint_error(": unable to map register memory\n");
   2220  1.1  jmcneill 		return;
   2221  1.1  jmcneill 	}
   2222  1.1  jmcneill 
   2223  1.1  jmcneill 	pci_aprint_devinfo(pa, "Ethernet controller");
   2224  1.1  jmcneill 
   2225  1.1  jmcneill 	if (mcx_version(sc) != 0) {
   2226  1.1  jmcneill 		/* error printed by mcx_version */
   2227  1.1  jmcneill 		goto unmap;
   2228  1.1  jmcneill 	}
   2229  1.1  jmcneill 
   2230  1.1  jmcneill 	r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
   2231  1.1  jmcneill 	cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
   2232  1.1  jmcneill 	cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
   2233  1.1  jmcneill 	if (cq_size > MCX_MAX_CQE) {
   2234  1.1  jmcneill 		aprint_error_dev(self,
   2235  1.1  jmcneill 		    "command queue size overflow %u\n", cq_size);
   2236  1.1  jmcneill 		goto unmap;
   2237  1.1  jmcneill 	}
   2238  1.1  jmcneill 	if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
   2239  1.1  jmcneill 		aprint_error_dev(self,
   2240  1.1  jmcneill 		    "command queue entry size underflow %u\n", cq_stride);
   2241  1.1  jmcneill 		goto unmap;
   2242  1.1  jmcneill 	}
   2243  1.1  jmcneill 	if (cq_stride * cq_size > MCX_PAGE_SIZE) {
   2244  1.1  jmcneill 		aprint_error_dev(self, "command queue page overflow\n");
   2245  1.1  jmcneill 		goto unmap;
   2246  1.1  jmcneill 	}
   2247  1.1  jmcneill 
   2248  1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_PAGE_SIZE,
   2249  1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2250  1.1  jmcneill 		aprint_error_dev(self, "unable to allocate doorbell memory\n");
   2251  1.1  jmcneill 		goto unmap;
   2252  1.1  jmcneill 	}
   2253  1.1  jmcneill 
   2254  1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
   2255  1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   2256  1.1  jmcneill 		aprint_error_dev(self, "unable to allocate command queue\n");
   2257  1.1  jmcneill 		goto dbfree;
   2258  1.1  jmcneill 	}
   2259  1.1  jmcneill 
   2260  1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   2261  1.1  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), BUS_SPACE_BARRIER_WRITE);
   2262  1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
   2263  1.1  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), BUS_SPACE_BARRIER_WRITE);
   2264  1.1  jmcneill 
   2265  1.1  jmcneill 	if (mcx_init_wait(sc) != 0) {
   2266  1.1  jmcneill 		aprint_error_dev(self, "timeout waiting for init\n");
   2267  1.1  jmcneill 		goto cqfree;
   2268  1.1  jmcneill 	}
   2269  1.1  jmcneill 
   2270  1.1  jmcneill 	sc->sc_cmdq_mask = cq_size - 1;
   2271  1.1  jmcneill 	sc->sc_cmdq_size = cq_stride;
   2272  1.1  jmcneill 
   2273  1.1  jmcneill 	if (mcx_enable_hca(sc) != 0) {
   2274  1.1  jmcneill 		/* error printed by mcx_enable_hca */
   2275  1.1  jmcneill 		goto cqfree;
   2276  1.1  jmcneill 	}
   2277  1.1  jmcneill 
   2278  1.1  jmcneill 	if (mcx_issi(sc) != 0) {
   2279  1.1  jmcneill 		/* error printed by mcx_issi */
   2280  1.1  jmcneill 		goto teardown;
   2281  1.1  jmcneill 	}
   2282  1.1  jmcneill 
   2283  1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_boot_pages,
   2284  1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
   2285  1.1  jmcneill 		/* error printed by mcx_pages */
   2286  1.1  jmcneill 		goto teardown;
   2287  1.1  jmcneill 	}
   2288  1.1  jmcneill 
   2289  1.1  jmcneill 	if (mcx_hca_max_caps(sc) != 0) {
   2290  1.1  jmcneill 		/* error printed by mcx_hca_max_caps */
   2291  1.1  jmcneill 		goto teardown;
   2292  1.1  jmcneill 	}
   2293  1.1  jmcneill 
   2294  1.1  jmcneill 	if (mcx_hca_set_caps(sc) != 0) {
   2295  1.1  jmcneill 		/* error printed by mcx_hca_set_caps */
   2296  1.1  jmcneill 		goto teardown;
   2297  1.1  jmcneill 	}
   2298  1.1  jmcneill 
   2299  1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_init_pages,
   2300  1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
   2301  1.1  jmcneill 		/* error printed by mcx_pages */
   2302  1.1  jmcneill 		goto teardown;
   2303  1.1  jmcneill 	}
   2304  1.1  jmcneill 
   2305  1.1  jmcneill 	if (mcx_init_hca(sc) != 0) {
   2306  1.1  jmcneill 		/* error printed by mcx_init_hca */
   2307  1.1  jmcneill 		goto teardown;
   2308  1.1  jmcneill 	}
   2309  1.1  jmcneill 
   2310  1.1  jmcneill 	if (mcx_pages(sc, &sc->sc_regular_pages,
   2311  1.1  jmcneill 	    htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
   2312  1.1  jmcneill 		/* error printed by mcx_pages */
   2313  1.1  jmcneill 		goto teardown;
   2314  1.1  jmcneill 	}
   2315  1.1  jmcneill 
   2316  1.1  jmcneill 	/* apparently not necessary? */
   2317  1.1  jmcneill 	if (mcx_set_driver_version(sc) != 0) {
   2318  1.1  jmcneill 		/* error printed by mcx_set_driver_version */
   2319  1.1  jmcneill 		goto teardown;
   2320  1.1  jmcneill 	}
   2321  1.1  jmcneill 
   2322  1.1  jmcneill 	if (mcx_iff(sc) != 0) {	/* modify nic vport context */
   2323  1.1  jmcneill 		/* error printed by mcx_iff? */
   2324  1.1  jmcneill 		goto teardown;
   2325  1.1  jmcneill 	}
   2326  1.1  jmcneill 
   2327  1.1  jmcneill 	if (mcx_alloc_uar(sc) != 0) {
   2328  1.1  jmcneill 		/* error printed by mcx_alloc_uar */
   2329  1.1  jmcneill 		goto teardown;
   2330  1.1  jmcneill 	}
   2331  1.1  jmcneill 
   2332  1.1  jmcneill 	if (mcx_alloc_pd(sc) != 0) {
   2333  1.1  jmcneill 		/* error printed by mcx_alloc_pd */
   2334  1.1  jmcneill 		goto teardown;
   2335  1.1  jmcneill 	}
   2336  1.1  jmcneill 
   2337  1.1  jmcneill 	if (mcx_alloc_tdomain(sc) != 0) {
   2338  1.1  jmcneill 		/* error printed by mcx_alloc_tdomain */
   2339  1.1  jmcneill 		goto teardown;
   2340  1.1  jmcneill 	}
   2341  1.1  jmcneill 
   2342  1.1  jmcneill 	/*
   2343  1.1  jmcneill 	 * PRM makes no mention of msi interrupts, just legacy and msi-x.
   2344  1.1  jmcneill 	 * mellanox support tells me legacy interrupts are not supported,
   2345  1.1  jmcneill 	 * so we're stuck with just msi-x.
   2346  1.1  jmcneill 	 */
   2347  1.1  jmcneill 	counts[PCI_INTR_TYPE_MSIX] = 1;
   2348  1.1  jmcneill 	counts[PCI_INTR_TYPE_MSI] = 0;
   2349  1.1  jmcneill 	counts[PCI_INTR_TYPE_INTX] = 0;
   2350  1.1  jmcneill 	if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) {
   2351  1.1  jmcneill 		aprint_error_dev(self, "unable to allocate interrupt\n");
   2352  1.1  jmcneill 		goto teardown;
   2353  1.1  jmcneill 	}
   2354  1.1  jmcneill 	KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX);
   2355  1.1  jmcneill 
   2356  1.1  jmcneill #ifdef MCX_MPSAFE
   2357  1.1  jmcneill 	pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[0], PCI_INTR_MPSAFE, true);
   2358  1.1  jmcneill #endif
   2359  1.1  jmcneill 
   2360  1.1  jmcneill 	intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[0], intrbuf,
   2361  1.1  jmcneill 	    sizeof(intrbuf));
   2362  1.1  jmcneill 	sc->sc_ihs[0] = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[0],
   2363  1.1  jmcneill 	    IPL_NET, mcx_intr, sc, DEVNAME(sc));
   2364  1.1  jmcneill 	if (sc->sc_ihs[0] == NULL) {
   2365  1.1  jmcneill 		aprint_error_dev(self, "unable to establish interrupt%s%s\n",
   2366  1.1  jmcneill 		    intrstr ? " at " : "",
   2367  1.1  jmcneill 		    intrstr ? intrstr : "");
   2368  1.1  jmcneill 		goto teardown;
   2369  1.1  jmcneill 	}
   2370  1.1  jmcneill 
   2371  1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr);
   2372  1.1  jmcneill 
   2373  1.1  jmcneill 	if (mcx_create_eq(sc) != 0) {
   2374  1.1  jmcneill 		/* error printed by mcx_create_eq */
   2375  1.1  jmcneill 		goto teardown;
   2376  1.1  jmcneill 	}
   2377  1.1  jmcneill 
   2378  1.1  jmcneill 	if (mcx_query_nic_vport_context(sc, enaddr) != 0) {
   2379  1.1  jmcneill 		/* error printed by mcx_query_nic_vport_context */
   2380  1.1  jmcneill 		goto teardown;
   2381  1.1  jmcneill 	}
   2382  1.1  jmcneill 
   2383  1.1  jmcneill 	if (mcx_query_special_contexts(sc) != 0) {
   2384  1.1  jmcneill 		/* error printed by mcx_query_special_contexts */
   2385  1.1  jmcneill 		goto teardown;
   2386  1.1  jmcneill 	}
   2387  1.1  jmcneill 
   2388  1.1  jmcneill 	if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
   2389  1.1  jmcneill 		/* error printed by mcx_set_port_mtu */
   2390  1.1  jmcneill 		goto teardown;
   2391  1.1  jmcneill 	}
   2392  1.1  jmcneill 
   2393  1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
   2394  1.1  jmcneill 	    ether_sprintf(enaddr));
   2395  1.1  jmcneill 
   2396  1.1  jmcneill 	strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
   2397  1.1  jmcneill 	ifp->if_softc = sc;
   2398  1.1  jmcneill 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
   2399  1.1  jmcneill #ifdef MCX_MPSAFE
   2400  1.1  jmcneill 	ifp->if_extflags = IFEF_MPSAFE;
   2401  1.1  jmcneill #endif
   2402  1.1  jmcneill 	ifp->if_init = mcx_init;
   2403  1.1  jmcneill 	ifp->if_stop = mcx_stop;
   2404  1.1  jmcneill 	ifp->if_ioctl = mcx_ioctl;
   2405  1.1  jmcneill 	ifp->if_start = mcx_start;
   2406  1.1  jmcneill 	ifp->if_watchdog = mcx_watchdog;
   2407  1.1  jmcneill 	ifp->if_mtu = sc->sc_hardmtu;
   2408  1.1  jmcneill 	IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
   2409  1.1  jmcneill 	IFQ_SET_READY(&ifp->if_snd);
   2410  1.1  jmcneill 
   2411  1.1  jmcneill 	sc->sc_ec.ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU;
   2412  1.1  jmcneill 
   2413  1.1  jmcneill 	sc->sc_ec.ec_ifmedia = &sc->sc_media;
   2414  1.1  jmcneill 	ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change,
   2415  1.1  jmcneill 	    mcx_media_status);
   2416  1.1  jmcneill 	mcx_media_add_types(sc);
   2417  1.1  jmcneill 	ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
   2418  1.1  jmcneill 	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
   2419  1.1  jmcneill 
   2420  1.1  jmcneill 	if_attach(ifp);
   2421  1.1  jmcneill 	if_deferred_start_init(ifp, NULL);
   2422  1.1  jmcneill 
   2423  1.1  jmcneill 	ether_ifattach(ifp, enaddr);
   2424  1.1  jmcneill 
   2425  1.1  jmcneill 	callout_init(&sc->sc_rx_refill, CALLOUT_FLAGS);
   2426  1.1  jmcneill 	callout_setfunc(&sc->sc_rx_refill, mcx_refill, sc);
   2427  1.1  jmcneill 	callout_init(&sc->sc_calibrate, CALLOUT_FLAGS);
   2428  1.1  jmcneill 	callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc);
   2429  1.1  jmcneill 
   2430  1.1  jmcneill 	if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc,
   2431  1.1  jmcneill 	    PRI_NONE, IPL_NET, 0) != 0) {
   2432  1.1  jmcneill 		aprint_error_dev(self, "couldn't create port change workq\n");
   2433  1.1  jmcneill 		goto teardown;
   2434  1.1  jmcneill 	}
   2435  1.1  jmcneill 
   2436  1.1  jmcneill 	mcx_port_change(&sc->sc_port_change, sc);
   2437  1.1  jmcneill 
   2438  1.1  jmcneill 	sc->sc_flow_table_id = -1;
   2439  1.1  jmcneill 	for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
   2440  1.1  jmcneill 		sc->sc_flow_group_id[i] = -1;
   2441  1.1  jmcneill 		sc->sc_flow_group_size[i] = 0;
   2442  1.1  jmcneill 		sc->sc_flow_group_start[i] = 0;
   2443  1.1  jmcneill 	}
   2444  1.1  jmcneill 	sc->sc_extra_mcast = 0;
   2445  1.1  jmcneill 	memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));
   2446  1.1  jmcneill 	return;
   2447  1.1  jmcneill 
   2448  1.1  jmcneill teardown:
   2449  1.1  jmcneill 	mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
   2450  1.1  jmcneill 	/* error printed by mcx_teardown_hca, and we're already unwinding */
   2451  1.1  jmcneill cqfree:
   2452  1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
   2453  1.1  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE);
   2454  1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
   2455  1.1  jmcneill 	    MCX_CMDQ_INTERFACE_DISABLED);
   2456  1.1  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE);
   2457  1.1  jmcneill 
   2458  1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
   2459  1.1  jmcneill 	mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), BUS_SPACE_BARRIER_WRITE);
   2460  1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
   2461  1.1  jmcneill 
   2462  1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
   2463  1.1  jmcneill dbfree:
   2464  1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
   2465  1.1  jmcneill unmap:
   2466  1.1  jmcneill 	bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
   2467  1.1  jmcneill 	sc->sc_mems = 0;
   2468  1.1  jmcneill }
   2469  1.1  jmcneill 
   2470  1.1  jmcneill static void
   2471  1.1  jmcneill mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm)
   2472  1.1  jmcneill {
   2473  1.1  jmcneill 	rxr->rxr_total = hwm;
   2474  1.1  jmcneill 	rxr->rxr_inuse = 0;
   2475  1.1  jmcneill }
   2476  1.1  jmcneill 
   2477  1.1  jmcneill static u_int
   2478  1.1  jmcneill mcx_rxr_get(struct mcx_rxring *rxr, u_int max)
   2479  1.1  jmcneill {
   2480  1.1  jmcneill 	const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse);
   2481  1.1  jmcneill 
   2482  1.1  jmcneill 	rxr->rxr_inuse += taken;
   2483  1.1  jmcneill 
   2484  1.1  jmcneill 	return taken;
   2485  1.1  jmcneill }
   2486  1.1  jmcneill 
   2487  1.1  jmcneill static void
   2488  1.1  jmcneill mcx_rxr_put(struct mcx_rxring *rxr, u_int n)
   2489  1.1  jmcneill {
   2490  1.1  jmcneill 	rxr->rxr_inuse -= n;
   2491  1.1  jmcneill }
   2492  1.1  jmcneill 
   2493  1.1  jmcneill static u_int
   2494  1.1  jmcneill mcx_rxr_inuse(struct mcx_rxring *rxr)
   2495  1.1  jmcneill {
   2496  1.1  jmcneill 	return rxr->rxr_inuse;
   2497  1.1  jmcneill }
   2498  1.1  jmcneill 
   2499  1.1  jmcneill static int
   2500  1.1  jmcneill mcx_version(struct mcx_softc *sc)
   2501  1.1  jmcneill {
   2502  1.1  jmcneill 	uint32_t fw0, fw1;
   2503  1.1  jmcneill 	uint16_t cmdif;
   2504  1.1  jmcneill 
   2505  1.1  jmcneill 	fw0 = mcx_rd(sc, MCX_FW_VER);
   2506  1.1  jmcneill 	fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
   2507  1.1  jmcneill 
   2508  1.1  jmcneill 	aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0),
   2509  1.1  jmcneill 	    MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
   2510  1.1  jmcneill 
   2511  1.1  jmcneill 	cmdif = MCX_CMDIF(fw1);
   2512  1.1  jmcneill 	if (cmdif != MCX_CMD_IF_SUPPORTED) {
   2513  1.1  jmcneill 		aprint_error_dev(sc->sc_dev,
   2514  1.1  jmcneill 		    "unsupported command interface %u\n", cmdif);
   2515  1.1  jmcneill 		return (-1);
   2516  1.1  jmcneill 	}
   2517  1.1  jmcneill 
   2518  1.1  jmcneill 	return (0);
   2519  1.1  jmcneill }
   2520  1.1  jmcneill 
   2521  1.1  jmcneill static int
   2522  1.1  jmcneill mcx_init_wait(struct mcx_softc *sc)
   2523  1.1  jmcneill {
   2524  1.1  jmcneill 	unsigned int i;
   2525  1.1  jmcneill 	uint32_t r;
   2526  1.1  jmcneill 
   2527  1.1  jmcneill 	for (i = 0; i < 2000; i++) {
   2528  1.1  jmcneill 		r = mcx_rd(sc, MCX_STATE);
   2529  1.1  jmcneill 		if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
   2530  1.1  jmcneill 			return (0);
   2531  1.1  jmcneill 
   2532  1.1  jmcneill 		delay(1000);
   2533  1.1  jmcneill 		mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
   2534  1.1  jmcneill 		    BUS_SPACE_BARRIER_READ);
   2535  1.1  jmcneill 	}
   2536  1.1  jmcneill 
   2537  1.1  jmcneill 	return (-1);
   2538  1.1  jmcneill }
   2539  1.1  jmcneill 
   2540  1.1  jmcneill static uint8_t
   2541  1.1  jmcneill mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   2542  1.1  jmcneill     unsigned int msec)
   2543  1.1  jmcneill {
   2544  1.1  jmcneill 	unsigned int i;
   2545  1.1  jmcneill 
   2546  1.1  jmcneill 	for (i = 0; i < msec; i++) {
   2547  1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   2548  1.1  jmcneill 		    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
   2549  1.1  jmcneill 
   2550  1.1  jmcneill 		if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
   2551  1.1  jmcneill 		    MCX_CQ_STATUS_OWN_SW) {
   2552  1.1  jmcneill 			if (sc->sc_eqn != 0)
   2553  1.1  jmcneill 				mcx_intr(sc);
   2554  1.1  jmcneill 			return (0);
   2555  1.1  jmcneill 		}
   2556  1.1  jmcneill 
   2557  1.1  jmcneill 		delay(1000);
   2558  1.1  jmcneill 	}
   2559  1.1  jmcneill 
   2560  1.1  jmcneill 	return (ETIMEDOUT);
   2561  1.1  jmcneill }
   2562  1.1  jmcneill 
   2563  1.1  jmcneill static uint32_t
   2564  1.1  jmcneill mcx_mix_u64(uint32_t xor, uint64_t u64)
   2565  1.1  jmcneill {
   2566  1.1  jmcneill 	xor ^= u64 >> 32;
   2567  1.1  jmcneill 	xor ^= u64;
   2568  1.1  jmcneill 
   2569  1.1  jmcneill 	return (xor);
   2570  1.1  jmcneill }
   2571  1.1  jmcneill 
   2572  1.1  jmcneill static uint32_t
   2573  1.1  jmcneill mcx_mix_u32(uint32_t xor, uint32_t u32)
   2574  1.1  jmcneill {
   2575  1.1  jmcneill 	xor ^= u32;
   2576  1.1  jmcneill 
   2577  1.1  jmcneill 	return (xor);
   2578  1.1  jmcneill }
   2579  1.1  jmcneill 
   2580  1.1  jmcneill static uint32_t
   2581  1.1  jmcneill mcx_mix_u8(uint32_t xor, uint8_t u8)
   2582  1.1  jmcneill {
   2583  1.1  jmcneill 	xor ^= u8;
   2584  1.1  jmcneill 
   2585  1.1  jmcneill 	return (xor);
   2586  1.1  jmcneill }
   2587  1.1  jmcneill 
   2588  1.1  jmcneill static uint8_t
   2589  1.1  jmcneill mcx_mix_done(uint32_t xor)
   2590  1.1  jmcneill {
   2591  1.1  jmcneill 	xor ^= xor >> 16;
   2592  1.1  jmcneill 	xor ^= xor >> 8;
   2593  1.1  jmcneill 
   2594  1.1  jmcneill 	return (xor);
   2595  1.1  jmcneill }
   2596  1.1  jmcneill 
   2597  1.1  jmcneill static uint8_t
   2598  1.1  jmcneill mcx_xor(const void *buf, size_t len)
   2599  1.1  jmcneill {
   2600  1.1  jmcneill 	const uint32_t *dwords = buf;
   2601  1.1  jmcneill 	uint32_t xor = 0xff;
   2602  1.1  jmcneill 	size_t i;
   2603  1.1  jmcneill 
   2604  1.1  jmcneill 	len /= sizeof(*dwords);
   2605  1.1  jmcneill 
   2606  1.1  jmcneill 	for (i = 0; i < len; i++)
   2607  1.1  jmcneill 		xor ^= dwords[i];
   2608  1.1  jmcneill 
   2609  1.1  jmcneill 	return (mcx_mix_done(xor));
   2610  1.1  jmcneill }
   2611  1.1  jmcneill 
   2612  1.1  jmcneill static uint8_t
   2613  1.1  jmcneill mcx_cmdq_token(struct mcx_softc *sc)
   2614  1.1  jmcneill {
   2615  1.1  jmcneill 	uint8_t token;
   2616  1.1  jmcneill 
   2617  1.1  jmcneill 	do {
   2618  1.1  jmcneill 		token = ++sc->sc_cmdq_token;
   2619  1.1  jmcneill 	} while (token == 0);
   2620  1.1  jmcneill 
   2621  1.1  jmcneill 	return (token);
   2622  1.1  jmcneill }
   2623  1.1  jmcneill 
   2624  1.1  jmcneill static void
   2625  1.1  jmcneill mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   2626  1.1  jmcneill     uint32_t ilen, uint32_t olen, uint8_t token)
   2627  1.1  jmcneill {
   2628  1.1  jmcneill 	memset(cqe, 0, sc->sc_cmdq_size);
   2629  1.1  jmcneill 
   2630  1.1  jmcneill 	cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
   2631  1.1  jmcneill 	be32enc(&cqe->cq_input_length, ilen);
   2632  1.1  jmcneill 	be32enc(&cqe->cq_output_length, olen);
   2633  1.1  jmcneill 	cqe->cq_token = token;
   2634  1.1  jmcneill 	cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
   2635  1.1  jmcneill }
   2636  1.1  jmcneill 
   2637  1.1  jmcneill static void
   2638  1.1  jmcneill mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
   2639  1.1  jmcneill {
   2640  1.1  jmcneill 	cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
   2641  1.1  jmcneill }
   2642  1.1  jmcneill 
   2643  1.1  jmcneill static int
   2644  1.1  jmcneill mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
   2645  1.1  jmcneill {
   2646  1.1  jmcneill 	/* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 :  0); */
   2647  1.1  jmcneill 	return (0);
   2648  1.1  jmcneill }
   2649  1.1  jmcneill 
   2650  1.1  jmcneill static void *
   2651  1.1  jmcneill mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
   2652  1.1  jmcneill {
   2653  1.1  jmcneill 	return (&cqe->cq_input_data);
   2654  1.1  jmcneill }
   2655  1.1  jmcneill 
   2656  1.1  jmcneill static void *
   2657  1.1  jmcneill mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
   2658  1.1  jmcneill {
   2659  1.1  jmcneill 	return (&cqe->cq_output_data);
   2660  1.1  jmcneill }
   2661  1.1  jmcneill 
   2662  1.1  jmcneill static void
   2663  1.1  jmcneill mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
   2664  1.1  jmcneill     unsigned int slot)
   2665  1.1  jmcneill {
   2666  1.1  jmcneill 	mcx_cmdq_sign(cqe);
   2667  1.1  jmcneill 
   2668  1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
   2669  1.1  jmcneill 	    0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
   2670  1.1  jmcneill 
   2671  1.1  jmcneill 	mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
   2672  1.1  jmcneill }
   2673  1.1  jmcneill 
   2674  1.1  jmcneill static int
   2675  1.1  jmcneill mcx_enable_hca(struct mcx_softc *sc)
   2676  1.1  jmcneill {
   2677  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   2678  1.1  jmcneill 	struct mcx_cmd_enable_hca_in *in;
   2679  1.1  jmcneill 	struct mcx_cmd_enable_hca_out *out;
   2680  1.1  jmcneill 	int error;
   2681  1.1  jmcneill 	uint8_t status;
   2682  1.1  jmcneill 
   2683  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   2684  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   2685  1.1  jmcneill 
   2686  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   2687  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
   2688  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   2689  1.1  jmcneill 	in->cmd_function_id = htobe16(0);
   2690  1.1  jmcneill 
   2691  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   2692  1.1  jmcneill 
   2693  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   2694  1.1  jmcneill 	if (error != 0) {
   2695  1.1  jmcneill 		printf(", hca enable timeout\n");
   2696  1.1  jmcneill 		return (-1);
   2697  1.1  jmcneill 	}
   2698  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   2699  1.1  jmcneill 		printf(", hca enable command corrupt\n");
   2700  1.1  jmcneill 		return (-1);
   2701  1.1  jmcneill 	}
   2702  1.1  jmcneill 
   2703  1.1  jmcneill 	status = cqe->cq_output_data[0];
   2704  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   2705  1.1  jmcneill 		printf(", hca enable failed (%x)\n", status);
   2706  1.1  jmcneill 		return (-1);
   2707  1.1  jmcneill 	}
   2708  1.1  jmcneill 
   2709  1.1  jmcneill 	return (0);
   2710  1.1  jmcneill }
   2711  1.1  jmcneill 
   2712  1.1  jmcneill static int
   2713  1.1  jmcneill mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
   2714  1.1  jmcneill {
   2715  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   2716  1.1  jmcneill 	struct mcx_cmd_teardown_hca_in *in;
   2717  1.1  jmcneill 	struct mcx_cmd_teardown_hca_out *out;
   2718  1.1  jmcneill 	int error;
   2719  1.1  jmcneill 	uint8_t status;
   2720  1.1  jmcneill 
   2721  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   2722  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   2723  1.1  jmcneill 
   2724  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   2725  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
   2726  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   2727  1.1  jmcneill 	in->cmd_profile = profile;
   2728  1.1  jmcneill 
   2729  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   2730  1.1  jmcneill 
   2731  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   2732  1.1  jmcneill 	if (error != 0) {
   2733  1.1  jmcneill 		printf(", hca teardown timeout\n");
   2734  1.1  jmcneill 		return (-1);
   2735  1.1  jmcneill 	}
   2736  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   2737  1.1  jmcneill 		printf(", hca teardown command corrupt\n");
   2738  1.1  jmcneill 		return (-1);
   2739  1.1  jmcneill 	}
   2740  1.1  jmcneill 
   2741  1.1  jmcneill 	status = cqe->cq_output_data[0];
   2742  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   2743  1.1  jmcneill 		printf(", hca teardown failed (%x)\n", status);
   2744  1.1  jmcneill 		return (-1);
   2745  1.1  jmcneill 	}
   2746  1.1  jmcneill 
   2747  1.1  jmcneill 	return (0);
   2748  1.1  jmcneill }
   2749  1.1  jmcneill 
   2750  1.1  jmcneill static int
   2751  1.1  jmcneill mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   2752  1.1  jmcneill     unsigned int nmb, uint64_t *ptr, uint8_t token)
   2753  1.1  jmcneill {
   2754  1.1  jmcneill 	uint8_t *kva;
   2755  1.1  jmcneill 	uint64_t dva;
   2756  1.1  jmcneill 	int i;
   2757  1.1  jmcneill 	int error;
   2758  1.1  jmcneill 
   2759  1.1  jmcneill 	error = mcx_dmamem_alloc(sc, mxm,
   2760  1.1  jmcneill 	    nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
   2761  1.1  jmcneill 	if (error != 0)
   2762  1.1  jmcneill 		return (error);
   2763  1.1  jmcneill 
   2764  1.1  jmcneill 	mcx_dmamem_zero(mxm);
   2765  1.1  jmcneill 
   2766  1.1  jmcneill 	dva = MCX_DMA_DVA(mxm);
   2767  1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   2768  1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   2769  1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
   2770  1.1  jmcneill 
   2771  1.1  jmcneill 		/* patch the cqe or mbox pointing at this one */
   2772  1.1  jmcneill 		be64enc(ptr, dva);
   2773  1.1  jmcneill 
   2774  1.1  jmcneill 		/* fill in this mbox */
   2775  1.1  jmcneill 		be32enc(&mbox->mb_block_number, i);
   2776  1.1  jmcneill 		mbox->mb_token = token;
   2777  1.1  jmcneill 
   2778  1.1  jmcneill 		/* move to the next one */
   2779  1.1  jmcneill 		ptr = &mbox->mb_next_ptr;
   2780  1.1  jmcneill 
   2781  1.1  jmcneill 		dva += MCX_CMDQ_MAILBOX_SIZE;
   2782  1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   2783  1.1  jmcneill 	}
   2784  1.1  jmcneill 
   2785  1.1  jmcneill 	return (0);
   2786  1.1  jmcneill }
   2787  1.1  jmcneill 
   2788  1.1  jmcneill static uint32_t
   2789  1.1  jmcneill mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
   2790  1.1  jmcneill {
   2791  1.1  jmcneill 	uint32_t xor = 0xff;
   2792  1.1  jmcneill 
   2793  1.1  jmcneill 	/* only 3 fields get set, so mix them directly */
   2794  1.1  jmcneill 	xor = mcx_mix_u64(xor, mb->mb_next_ptr);
   2795  1.1  jmcneill 	xor = mcx_mix_u32(xor, mb->mb_block_number);
   2796  1.1  jmcneill 	xor = mcx_mix_u8(xor, mb->mb_token);
   2797  1.1  jmcneill 
   2798  1.1  jmcneill 	return (mcx_mix_done(xor));
   2799  1.1  jmcneill }
   2800  1.1  jmcneill 
   2801  1.1  jmcneill static void
   2802  1.1  jmcneill mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
   2803  1.1  jmcneill {
   2804  1.1  jmcneill 	uint8_t *kva;
   2805  1.1  jmcneill 	int i;
   2806  1.1  jmcneill 
   2807  1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   2808  1.1  jmcneill 
   2809  1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   2810  1.1  jmcneill 		struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
   2811  1.1  jmcneill 		uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
   2812  1.1  jmcneill 		mb->mb_ctrl_signature = sig;
   2813  1.1  jmcneill 		mb->mb_signature = sig ^
   2814  1.1  jmcneill 		    mcx_xor(mb->mb_data, sizeof(mb->mb_data));
   2815  1.1  jmcneill 
   2816  1.1  jmcneill 		kva += MCX_CMDQ_MAILBOX_SIZE;
   2817  1.1  jmcneill 	}
   2818  1.1  jmcneill }
   2819  1.1  jmcneill 
   2820  1.1  jmcneill static void
   2821  1.1  jmcneill mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
   2822  1.1  jmcneill {
   2823  1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
   2824  1.1  jmcneill 	    0, MCX_DMA_LEN(mxm), ops);
   2825  1.1  jmcneill }
   2826  1.1  jmcneill 
   2827  1.1  jmcneill static struct mcx_cmdq_mailbox *
   2828  1.1  jmcneill mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
   2829  1.1  jmcneill {
   2830  1.1  jmcneill 	uint8_t *kva;
   2831  1.1  jmcneill 
   2832  1.1  jmcneill 	kva = MCX_DMA_KVA(mxm);
   2833  1.1  jmcneill 	kva += i * MCX_CMDQ_MAILBOX_SIZE;
   2834  1.1  jmcneill 
   2835  1.1  jmcneill 	return ((struct mcx_cmdq_mailbox *)kva);
   2836  1.1  jmcneill }
   2837  1.1  jmcneill 
   2838  1.1  jmcneill static inline void *
   2839  1.1  jmcneill mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
   2840  1.1  jmcneill {
   2841  1.1  jmcneill 	return (&mb->mb_data);
   2842  1.1  jmcneill }
   2843  1.1  jmcneill 
   2844  1.1  jmcneill static void
   2845  1.1  jmcneill mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
   2846  1.1  jmcneill     void *b, size_t len)
   2847  1.1  jmcneill {
   2848  1.1  jmcneill 	uint8_t *buf = b;
   2849  1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   2850  1.1  jmcneill 	int i;
   2851  1.1  jmcneill 
   2852  1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   2853  1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   2854  1.1  jmcneill 
   2855  1.1  jmcneill 		memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len));
   2856  1.1  jmcneill 
   2857  1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   2858  1.1  jmcneill 			break;
   2859  1.1  jmcneill 
   2860  1.1  jmcneill 		buf += sizeof(mb->mb_data);
   2861  1.1  jmcneill 		len -= sizeof(mb->mb_data);
   2862  1.1  jmcneill 		mb++;
   2863  1.1  jmcneill 	}
   2864  1.1  jmcneill }
   2865  1.1  jmcneill 
   2866  1.1  jmcneill static void
   2867  1.1  jmcneill mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
   2868  1.1  jmcneill {
   2869  1.1  jmcneill 	uint8_t *buf = b;
   2870  1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   2871  1.1  jmcneill 	int i;
   2872  1.1  jmcneill 
   2873  1.1  jmcneill 	mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
   2874  1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   2875  1.1  jmcneill 		memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len));
   2876  1.1  jmcneill 
   2877  1.1  jmcneill 		if (sizeof(mb->mb_data) >= len)
   2878  1.1  jmcneill 			break;
   2879  1.1  jmcneill 
   2880  1.1  jmcneill 		buf += sizeof(mb->mb_data);
   2881  1.1  jmcneill 		len -= sizeof(mb->mb_data);
   2882  1.1  jmcneill 		mb++;
   2883  1.1  jmcneill 	}
   2884  1.1  jmcneill }
   2885  1.1  jmcneill 
   2886  1.1  jmcneill static void
   2887  1.1  jmcneill mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   2888  1.1  jmcneill {
   2889  1.1  jmcneill 	mcx_dmamem_free(sc, mxm);
   2890  1.1  jmcneill }
   2891  1.1  jmcneill 
   2892  1.1  jmcneill #if 0
   2893  1.1  jmcneill static void
   2894  1.1  jmcneill mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
   2895  1.1  jmcneill {
   2896  1.1  jmcneill 	unsigned int i;
   2897  1.1  jmcneill 
   2898  1.1  jmcneill 	printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
   2899  1.1  jmcneill 	    be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr));
   2900  1.1  jmcneill 
   2901  1.1  jmcneill 	printf(", idata ");
   2902  1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_input_data); i++)
   2903  1.1  jmcneill 		printf("%02x", cqe->cq_input_data[i]);
   2904  1.1  jmcneill 
   2905  1.1  jmcneill 	printf(", odata ");
   2906  1.1  jmcneill 	for (i = 0; i < sizeof(cqe->cq_output_data); i++)
   2907  1.1  jmcneill 		printf("%02x", cqe->cq_output_data[i]);
   2908  1.1  jmcneill 
   2909  1.1  jmcneill 	printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
   2910  1.1  jmcneill 	    be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length),
   2911  1.1  jmcneill 	    cqe->cq_token, cqe->cq_signature, cqe->cq_status);
   2912  1.1  jmcneill }
   2913  1.1  jmcneill 
   2914  1.1  jmcneill static void
   2915  1.1  jmcneill mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
   2916  1.1  jmcneill {
   2917  1.1  jmcneill 	int i, j;
   2918  1.1  jmcneill 	uint8_t *d;
   2919  1.1  jmcneill 
   2920  1.1  jmcneill 	for (i = 0; i < num; i++) {
   2921  1.1  jmcneill 		struct mcx_cmdq_mailbox *mbox;
   2922  1.1  jmcneill 		mbox = mcx_cq_mbox(mboxes, i);
   2923  1.1  jmcneill 
   2924  1.1  jmcneill 		d = mcx_cq_mbox_data(mbox);
   2925  1.1  jmcneill 		for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
   2926  1.1  jmcneill 			if (j != 0 && (j % 16 == 0))
   2927  1.1  jmcneill 				printf("\n");
   2928  1.1  jmcneill 			printf("%.2x ", d[j]);
   2929  1.1  jmcneill 		}
   2930  1.1  jmcneill 	}
   2931  1.1  jmcneill }
   2932  1.1  jmcneill #endif
   2933  1.1  jmcneill 
   2934  1.1  jmcneill static int
   2935  1.1  jmcneill mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
   2936  1.1  jmcneill     int len)
   2937  1.1  jmcneill {
   2938  1.1  jmcneill 	struct mcx_dmamem mxm;
   2939  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   2940  1.1  jmcneill 	struct mcx_cmd_access_reg_in *in;
   2941  1.1  jmcneill 	struct mcx_cmd_access_reg_out *out;
   2942  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   2943  1.1  jmcneill 	int error, nmb;
   2944  1.1  jmcneill 
   2945  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   2946  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
   2947  1.1  jmcneill 	    token);
   2948  1.1  jmcneill 
   2949  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   2950  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
   2951  1.1  jmcneill 	in->cmd_op_mod = htobe16(op);
   2952  1.1  jmcneill 	in->cmd_register_id = htobe16(reg);
   2953  1.1  jmcneill 
   2954  1.1  jmcneill 	nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
   2955  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, &cqe->cq_output_ptr, token) != 0) {
   2956  1.1  jmcneill 		printf(", unable to allocate access reg mailboxen\n");
   2957  1.1  jmcneill 		return (-1);
   2958  1.1  jmcneill 	}
   2959  1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   2960  1.1  jmcneill 	mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
   2961  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   2962  1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   2963  1.1  jmcneill 
   2964  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   2965  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   2966  1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   2967  1.1  jmcneill 
   2968  1.1  jmcneill 	if (error != 0) {
   2969  1.1  jmcneill 		printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
   2970  1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
   2971  1.1  jmcneill 		goto free;
   2972  1.1  jmcneill 	}
   2973  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   2974  1.1  jmcneill 	if (error != 0) {
   2975  1.1  jmcneill 		printf("%s: access reg (%s %x) reply corrupt\n",
   2976  1.1  jmcneill 		    (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
   2977  1.1  jmcneill 		    reg);
   2978  1.1  jmcneill 		goto free;
   2979  1.1  jmcneill 	}
   2980  1.1  jmcneill 
   2981  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   2982  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   2983  1.1  jmcneill 		printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
   2984  1.1  jmcneill 		    DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
   2985  1.1  jmcneill 		    reg, out->cmd_status, out->cmd_syndrome);
   2986  1.1  jmcneill 		error = -1;
   2987  1.1  jmcneill 		goto free;
   2988  1.1  jmcneill 	}
   2989  1.1  jmcneill 
   2990  1.1  jmcneill 	mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
   2991  1.1  jmcneill free:
   2992  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   2993  1.1  jmcneill 
   2994  1.1  jmcneill 	return (error);
   2995  1.1  jmcneill }
   2996  1.1  jmcneill 
   2997  1.1  jmcneill static int
   2998  1.1  jmcneill mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, unsigned int slot)
   2999  1.1  jmcneill {
   3000  1.1  jmcneill 	struct mcx_cmd_set_issi_in *in;
   3001  1.1  jmcneill 	struct mcx_cmd_set_issi_out *out;
   3002  1.1  jmcneill 	uint8_t status;
   3003  1.1  jmcneill 
   3004  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3005  1.1  jmcneill 
   3006  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3007  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
   3008  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3009  1.1  jmcneill 	in->cmd_current_issi = htobe16(MCX_ISSI);
   3010  1.1  jmcneill 
   3011  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, slot);
   3012  1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
   3013  1.1  jmcneill 		return (-1);
   3014  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0)
   3015  1.1  jmcneill 		return (-1);
   3016  1.1  jmcneill 
   3017  1.1  jmcneill 	status = cqe->cq_output_data[0];
   3018  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK)
   3019  1.1  jmcneill 		return (-1);
   3020  1.1  jmcneill 
   3021  1.1  jmcneill 	return (0);
   3022  1.1  jmcneill }
   3023  1.1  jmcneill 
   3024  1.1  jmcneill static int
   3025  1.1  jmcneill mcx_issi(struct mcx_softc *sc)
   3026  1.1  jmcneill {
   3027  1.1  jmcneill 	struct mcx_dmamem mxm;
   3028  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3029  1.1  jmcneill 	struct mcx_cmd_query_issi_in *in;
   3030  1.1  jmcneill 	struct mcx_cmd_query_issi_il_out *out;
   3031  1.1  jmcneill 	struct mcx_cmd_query_issi_mb_out *mb;
   3032  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3033  1.1  jmcneill 	uint8_t status;
   3034  1.1  jmcneill 	int error;
   3035  1.1  jmcneill 
   3036  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3037  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
   3038  1.1  jmcneill 
   3039  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3040  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
   3041  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3042  1.1  jmcneill 
   3043  1.1  jmcneill 	CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
   3044  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   3045  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3046  1.1  jmcneill 		printf(", unable to allocate query issi mailbox\n");
   3047  1.1  jmcneill 		return (-1);
   3048  1.1  jmcneill 	}
   3049  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   3050  1.1  jmcneill 
   3051  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3052  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3053  1.1  jmcneill 	if (error != 0) {
   3054  1.1  jmcneill 		printf(", query issi timeout\n");
   3055  1.1  jmcneill 		goto free;
   3056  1.1  jmcneill 	}
   3057  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3058  1.1  jmcneill 	if (error != 0) {
   3059  1.1  jmcneill 		printf(", query issi reply corrupt\n");
   3060  1.1  jmcneill 		goto free;
   3061  1.1  jmcneill 	}
   3062  1.1  jmcneill 
   3063  1.1  jmcneill 	status = cqe->cq_output_data[0];
   3064  1.1  jmcneill 	switch (status) {
   3065  1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   3066  1.1  jmcneill 		break;
   3067  1.1  jmcneill 	case MCX_CQ_STATUS_BAD_OPCODE:
   3068  1.1  jmcneill 		/* use ISSI 0 */
   3069  1.1  jmcneill 		goto free;
   3070  1.1  jmcneill 	default:
   3071  1.1  jmcneill 		printf(", query issi failed (%x)\n", status);
   3072  1.1  jmcneill 		error = -1;
   3073  1.1  jmcneill 		goto free;
   3074  1.1  jmcneill 	}
   3075  1.1  jmcneill 
   3076  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3077  1.1  jmcneill 	if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
   3078  1.1  jmcneill 		/* use ISSI 1 */
   3079  1.1  jmcneill 		goto free;
   3080  1.1  jmcneill 	}
   3081  1.1  jmcneill 
   3082  1.1  jmcneill 	/* don't need to read cqe anymore, can be used for SET ISSI */
   3083  1.1  jmcneill 
   3084  1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   3085  1.1  jmcneill 	CTASSERT(MCX_ISSI < NBBY);
   3086  1.1  jmcneill 	 /* XXX math is hard */
   3087  1.1  jmcneill 	if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
   3088  1.1  jmcneill 		/* use ISSI 0 */
   3089  1.1  jmcneill 		goto free;
   3090  1.1  jmcneill 	}
   3091  1.1  jmcneill 
   3092  1.1  jmcneill 	if (mcx_set_issi(sc, cqe, 0) != 0) {
   3093  1.1  jmcneill 		/* ignore the error, just use ISSI 0 */
   3094  1.1  jmcneill 	} else {
   3095  1.1  jmcneill 		/* use ISSI 1 */
   3096  1.1  jmcneill 	}
   3097  1.1  jmcneill 
   3098  1.1  jmcneill free:
   3099  1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   3100  1.1  jmcneill 	return (error);
   3101  1.1  jmcneill }
   3102  1.1  jmcneill 
   3103  1.1  jmcneill static int
   3104  1.1  jmcneill mcx_query_pages(struct mcx_softc *sc, uint16_t type,
   3105  1.1  jmcneill     uint32_t *npages, uint16_t *func_id)
   3106  1.1  jmcneill {
   3107  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3108  1.1  jmcneill 	struct mcx_cmd_query_pages_in *in;
   3109  1.1  jmcneill 	struct mcx_cmd_query_pages_out *out;
   3110  1.1  jmcneill 
   3111  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3112  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3113  1.1  jmcneill 
   3114  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3115  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
   3116  1.1  jmcneill 	in->cmd_op_mod = type;
   3117  1.1  jmcneill 
   3118  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3119  1.1  jmcneill 	if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
   3120  1.1  jmcneill 		printf(", query pages timeout\n");
   3121  1.1  jmcneill 		return (-1);
   3122  1.1  jmcneill 	}
   3123  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3124  1.1  jmcneill 		printf(", query pages reply corrupt\n");
   3125  1.1  jmcneill 		return (-1);
   3126  1.1  jmcneill 	}
   3127  1.1  jmcneill 
   3128  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3129  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3130  1.1  jmcneill 		printf(", query pages failed (%x)\n", out->cmd_status);
   3131  1.1  jmcneill 		return (-1);
   3132  1.1  jmcneill 	}
   3133  1.1  jmcneill 
   3134  1.1  jmcneill 	*func_id = out->cmd_func_id;
   3135  1.1  jmcneill 	*npages = be32dec(&out->cmd_num_pages);
   3136  1.1  jmcneill 
   3137  1.1  jmcneill 	return (0);
   3138  1.1  jmcneill }
   3139  1.1  jmcneill 
   3140  1.1  jmcneill struct bus_dma_iter {
   3141  1.1  jmcneill 	bus_dmamap_t		i_map;
   3142  1.1  jmcneill 	bus_size_t		i_offset;
   3143  1.1  jmcneill 	unsigned int		i_index;
   3144  1.1  jmcneill };
   3145  1.1  jmcneill 
   3146  1.1  jmcneill static void
   3147  1.1  jmcneill bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
   3148  1.1  jmcneill {
   3149  1.1  jmcneill 	i->i_map = map;
   3150  1.1  jmcneill 	i->i_offset = 0;
   3151  1.1  jmcneill 	i->i_index = 0;
   3152  1.1  jmcneill }
   3153  1.1  jmcneill 
   3154  1.1  jmcneill static bus_addr_t
   3155  1.1  jmcneill bus_dma_iter_addr(struct bus_dma_iter *i)
   3156  1.1  jmcneill {
   3157  1.1  jmcneill 	return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
   3158  1.1  jmcneill }
   3159  1.1  jmcneill 
   3160  1.1  jmcneill static void
   3161  1.1  jmcneill bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
   3162  1.1  jmcneill {
   3163  1.1  jmcneill 	bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
   3164  1.1  jmcneill 	bus_size_t diff;
   3165  1.1  jmcneill 
   3166  1.1  jmcneill 	do {
   3167  1.1  jmcneill 		diff = seg->ds_len - i->i_offset;
   3168  1.1  jmcneill 		if (size < diff)
   3169  1.1  jmcneill 			break;
   3170  1.1  jmcneill 
   3171  1.1  jmcneill 		size -= diff;
   3172  1.1  jmcneill 
   3173  1.1  jmcneill 		seg++;
   3174  1.1  jmcneill 
   3175  1.1  jmcneill 		i->i_offset = 0;
   3176  1.1  jmcneill 		i->i_index++;
   3177  1.1  jmcneill 	} while (size > 0);
   3178  1.1  jmcneill 
   3179  1.1  jmcneill 	i->i_offset += size;
   3180  1.1  jmcneill }
   3181  1.1  jmcneill 
   3182  1.1  jmcneill static int
   3183  1.1  jmcneill mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
   3184  1.1  jmcneill {
   3185  1.1  jmcneill 	struct mcx_dmamem mxm;
   3186  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3187  1.1  jmcneill 	struct mcx_cmd_manage_pages_in *in;
   3188  1.1  jmcneill 	struct mcx_cmd_manage_pages_out *out;
   3189  1.1  jmcneill 	unsigned int paslen, nmb, i, j, npages;
   3190  1.1  jmcneill 	struct bus_dma_iter iter;
   3191  1.1  jmcneill 	uint64_t *pas;
   3192  1.1  jmcneill 	uint8_t status;
   3193  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3194  1.1  jmcneill 	int error;
   3195  1.1  jmcneill 
   3196  1.1  jmcneill 	npages = mhm->mhm_npages;
   3197  1.1  jmcneill 
   3198  1.1  jmcneill 	paslen = sizeof(*pas) * npages;
   3199  1.1  jmcneill 	nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
   3200  1.1  jmcneill 
   3201  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3202  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
   3203  1.1  jmcneill 
   3204  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3205  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
   3206  1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
   3207  1.1  jmcneill 	in->cmd_func_id = func_id;
   3208  1.1  jmcneill 	be32enc(&in->cmd_input_num_entries, npages);
   3209  1.1  jmcneill 
   3210  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
   3211  1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   3212  1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   3213  1.1  jmcneill 		return (-1);
   3214  1.1  jmcneill 	}
   3215  1.1  jmcneill 
   3216  1.1  jmcneill 	bus_dma_iter_init(&iter, mhm->mhm_map);
   3217  1.1  jmcneill 	for (i = 0; i < nmb; i++) {
   3218  1.1  jmcneill 		unsigned int lim;
   3219  1.1  jmcneill 
   3220  1.1  jmcneill 		pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
   3221  1.1  jmcneill 		lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
   3222  1.1  jmcneill 
   3223  1.1  jmcneill 		for (j = 0; j < lim; j++) {
   3224  1.1  jmcneill 			be64enc(&pas[j], bus_dma_iter_addr(&iter));
   3225  1.1  jmcneill 			bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
   3226  1.1  jmcneill 		}
   3227  1.1  jmcneill 
   3228  1.1  jmcneill 		npages -= lim;
   3229  1.1  jmcneill 	}
   3230  1.1  jmcneill 
   3231  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, nmb);
   3232  1.1  jmcneill 
   3233  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3234  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3235  1.1  jmcneill 	if (error != 0) {
   3236  1.1  jmcneill 		printf(", manage pages timeout\n");
   3237  1.1  jmcneill 		goto free;
   3238  1.1  jmcneill 	}
   3239  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3240  1.1  jmcneill 	if (error != 0) {
   3241  1.1  jmcneill 		printf(", manage pages reply corrupt\n");
   3242  1.1  jmcneill 		goto free;
   3243  1.1  jmcneill 	}
   3244  1.1  jmcneill 
   3245  1.1  jmcneill 	status = cqe->cq_output_data[0];
   3246  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3247  1.1  jmcneill 		printf(", manage pages failed (%x)\n", status);
   3248  1.1  jmcneill 		error = -1;
   3249  1.1  jmcneill 		goto free;
   3250  1.1  jmcneill 	}
   3251  1.1  jmcneill 
   3252  1.1  jmcneill free:
   3253  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3254  1.1  jmcneill 
   3255  1.1  jmcneill 	return (error);
   3256  1.1  jmcneill }
   3257  1.1  jmcneill 
   3258  1.1  jmcneill static int
   3259  1.1  jmcneill mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
   3260  1.1  jmcneill {
   3261  1.1  jmcneill 	uint32_t npages;
   3262  1.1  jmcneill 	uint16_t func_id;
   3263  1.1  jmcneill 
   3264  1.1  jmcneill 	if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
   3265  1.1  jmcneill 		/* error printed by mcx_query_pages */
   3266  1.1  jmcneill 		return (-1);
   3267  1.1  jmcneill 	}
   3268  1.1  jmcneill 
   3269  1.1  jmcneill 	if (npages == 0)
   3270  1.1  jmcneill 		return (0);
   3271  1.1  jmcneill 
   3272  1.1  jmcneill 	if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
   3273  1.1  jmcneill 		printf(", unable to allocate hwmem\n");
   3274  1.1  jmcneill 		return (-1);
   3275  1.1  jmcneill 	}
   3276  1.1  jmcneill 
   3277  1.1  jmcneill 	if (mcx_add_pages(sc, mhm, func_id) != 0) {
   3278  1.1  jmcneill 		printf(", unable to add hwmem\n");
   3279  1.1  jmcneill 		goto free;
   3280  1.1  jmcneill 	}
   3281  1.1  jmcneill 
   3282  1.1  jmcneill 	return (0);
   3283  1.1  jmcneill 
   3284  1.1  jmcneill free:
   3285  1.1  jmcneill 	mcx_hwmem_free(sc, mhm);
   3286  1.1  jmcneill 
   3287  1.1  jmcneill 	return (-1);
   3288  1.1  jmcneill }
   3289  1.1  jmcneill 
   3290  1.1  jmcneill static int
   3291  1.1  jmcneill mcx_hca_max_caps(struct mcx_softc *sc)
   3292  1.1  jmcneill {
   3293  1.1  jmcneill 	struct mcx_dmamem mxm;
   3294  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3295  1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   3296  1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   3297  1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3298  1.1  jmcneill 	struct mcx_cap_device *hca;
   3299  1.1  jmcneill 	uint8_t status;
   3300  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3301  1.1  jmcneill 	int error;
   3302  1.1  jmcneill 
   3303  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3304  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   3305  1.1  jmcneill 	    token);
   3306  1.1  jmcneill 
   3307  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3308  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   3309  1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
   3310  1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   3311  1.1  jmcneill 
   3312  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   3313  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3314  1.1  jmcneill 		printf(", unable to allocate query hca caps mailboxen\n");
   3315  1.1  jmcneill 		return (-1);
   3316  1.1  jmcneill 	}
   3317  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   3318  1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   3319  1.1  jmcneill 
   3320  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3321  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3322  1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   3323  1.1  jmcneill 
   3324  1.1  jmcneill 	if (error != 0) {
   3325  1.1  jmcneill 		printf(", query hca caps timeout\n");
   3326  1.1  jmcneill 		goto free;
   3327  1.1  jmcneill 	}
   3328  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3329  1.1  jmcneill 	if (error != 0) {
   3330  1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   3331  1.1  jmcneill 		goto free;
   3332  1.1  jmcneill 	}
   3333  1.1  jmcneill 
   3334  1.1  jmcneill 	status = cqe->cq_output_data[0];
   3335  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3336  1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   3337  1.1  jmcneill 		error = -1;
   3338  1.1  jmcneill 		goto free;
   3339  1.1  jmcneill 	}
   3340  1.1  jmcneill 
   3341  1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   3342  1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   3343  1.1  jmcneill 
   3344  1.1  jmcneill 	if (hca->log_pg_sz > PAGE_SHIFT) {
   3345  1.1  jmcneill 		printf(", minimum system page shift %u is too large\n",
   3346  1.1  jmcneill 		    hca->log_pg_sz);
   3347  1.1  jmcneill 		error = -1;
   3348  1.1  jmcneill 		goto free;
   3349  1.1  jmcneill 	}
   3350  1.1  jmcneill 	/*
   3351  1.1  jmcneill 	 * blueflame register is split into two buffers, and we must alternate
   3352  1.1  jmcneill 	 * between the two of them.
   3353  1.1  jmcneill 	 */
   3354  1.1  jmcneill 	sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
   3355  1.1  jmcneill 
   3356  1.1  jmcneill free:
   3357  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3358  1.1  jmcneill 
   3359  1.1  jmcneill 	return (error);
   3360  1.1  jmcneill }
   3361  1.1  jmcneill 
   3362  1.1  jmcneill static int
   3363  1.1  jmcneill mcx_hca_set_caps(struct mcx_softc *sc)
   3364  1.1  jmcneill {
   3365  1.1  jmcneill 	struct mcx_dmamem mxm;
   3366  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3367  1.1  jmcneill 	struct mcx_cmd_query_hca_cap_in *in;
   3368  1.1  jmcneill 	struct mcx_cmd_query_hca_cap_out *out;
   3369  1.1  jmcneill 	struct mcx_cmdq_mailbox *mb;
   3370  1.1  jmcneill 	struct mcx_cap_device *hca;
   3371  1.1  jmcneill 	uint8_t status;
   3372  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   3373  1.1  jmcneill 	int error;
   3374  1.1  jmcneill 
   3375  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3376  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
   3377  1.1  jmcneill 	    token);
   3378  1.1  jmcneill 
   3379  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3380  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
   3381  1.1  jmcneill 	in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
   3382  1.1  jmcneill 	    MCX_CMD_QUERY_HCA_CAP_DEVICE);
   3383  1.1  jmcneill 
   3384  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
   3385  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   3386  1.1  jmcneill 		printf(", unable to allocate manage pages mailboxen\n");
   3387  1.1  jmcneill 		return (-1);
   3388  1.1  jmcneill 	}
   3389  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
   3390  1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
   3391  1.1  jmcneill 
   3392  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3393  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3394  1.1  jmcneill 	mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
   3395  1.1  jmcneill 
   3396  1.1  jmcneill 	if (error != 0) {
   3397  1.1  jmcneill 		printf(", query hca caps timeout\n");
   3398  1.1  jmcneill 		goto free;
   3399  1.1  jmcneill 	}
   3400  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   3401  1.1  jmcneill 	if (error != 0) {
   3402  1.1  jmcneill 		printf(", query hca caps reply corrupt\n");
   3403  1.1  jmcneill 		goto free;
   3404  1.1  jmcneill 	}
   3405  1.1  jmcneill 
   3406  1.1  jmcneill 	status = cqe->cq_output_data[0];
   3407  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3408  1.1  jmcneill 		printf(", query hca caps failed (%x)\n", status);
   3409  1.1  jmcneill 		error = -1;
   3410  1.1  jmcneill 		goto free;
   3411  1.1  jmcneill 	}
   3412  1.1  jmcneill 
   3413  1.1  jmcneill 	mb = mcx_cq_mbox(&mxm, 0);
   3414  1.1  jmcneill 	hca = mcx_cq_mbox_data(mb);
   3415  1.1  jmcneill 
   3416  1.1  jmcneill 	hca->log_pg_sz = PAGE_SHIFT;
   3417  1.1  jmcneill 
   3418  1.1  jmcneill free:
   3419  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3420  1.1  jmcneill 
   3421  1.1  jmcneill 	return (error);
   3422  1.1  jmcneill }
   3423  1.1  jmcneill 
   3424  1.1  jmcneill 
   3425  1.1  jmcneill static int
   3426  1.1  jmcneill mcx_init_hca(struct mcx_softc *sc)
   3427  1.1  jmcneill {
   3428  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3429  1.1  jmcneill 	struct mcx_cmd_init_hca_in *in;
   3430  1.1  jmcneill 	struct mcx_cmd_init_hca_out *out;
   3431  1.1  jmcneill 	int error;
   3432  1.1  jmcneill 	uint8_t status;
   3433  1.1  jmcneill 
   3434  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3435  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3436  1.1  jmcneill 
   3437  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3438  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
   3439  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3440  1.1  jmcneill 
   3441  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3442  1.1  jmcneill 
   3443  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3444  1.1  jmcneill 	if (error != 0) {
   3445  1.1  jmcneill 		printf(", hca init timeout\n");
   3446  1.1  jmcneill 		return (-1);
   3447  1.1  jmcneill 	}
   3448  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3449  1.1  jmcneill 		printf(", hca init command corrupt\n");
   3450  1.1  jmcneill 		return (-1);
   3451  1.1  jmcneill 	}
   3452  1.1  jmcneill 
   3453  1.1  jmcneill 	status = cqe->cq_output_data[0];
   3454  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3455  1.1  jmcneill 		printf(", hca init failed (%x)\n", status);
   3456  1.1  jmcneill 		return (-1);
   3457  1.1  jmcneill 	}
   3458  1.1  jmcneill 
   3459  1.1  jmcneill 	return (0);
   3460  1.1  jmcneill }
   3461  1.1  jmcneill 
   3462  1.1  jmcneill static int
   3463  1.1  jmcneill mcx_set_driver_version(struct mcx_softc *sc)
   3464  1.1  jmcneill {
   3465  1.1  jmcneill 	struct mcx_dmamem mxm;
   3466  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3467  1.1  jmcneill 	struct mcx_cmd_set_driver_version_in *in;
   3468  1.1  jmcneill 	struct mcx_cmd_set_driver_version_out *out;
   3469  1.1  jmcneill 	int error;
   3470  1.1  jmcneill 	int token;
   3471  1.1  jmcneill 	uint8_t status;
   3472  1.1  jmcneill 
   3473  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3474  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   3475  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) +
   3476  1.1  jmcneill 	    sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);
   3477  1.1  jmcneill 
   3478  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3479  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
   3480  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3481  1.1  jmcneill 
   3482  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
   3483  1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   3484  1.1  jmcneill 		printf(", unable to allocate set driver version mailboxen\n");
   3485  1.1  jmcneill 		return (-1);
   3486  1.1  jmcneill 	}
   3487  1.1  jmcneill 	strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
   3488  1.1  jmcneill 	    "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
   3489  1.1  jmcneill 
   3490  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   3491  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3492  1.1  jmcneill 
   3493  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3494  1.1  jmcneill 	if (error != 0) {
   3495  1.1  jmcneill 		printf(", set driver version timeout\n");
   3496  1.1  jmcneill 		goto free;
   3497  1.1  jmcneill 	}
   3498  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3499  1.1  jmcneill 		printf(", set driver version command corrupt\n");
   3500  1.1  jmcneill 		goto free;
   3501  1.1  jmcneill 	}
   3502  1.1  jmcneill 
   3503  1.1  jmcneill 	status = cqe->cq_output_data[0];
   3504  1.1  jmcneill 	if (status != MCX_CQ_STATUS_OK) {
   3505  1.1  jmcneill 		printf(", set driver version failed (%x)\n", status);
   3506  1.1  jmcneill 		error = -1;
   3507  1.1  jmcneill 		goto free;
   3508  1.1  jmcneill 	}
   3509  1.1  jmcneill 
   3510  1.1  jmcneill free:
   3511  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3512  1.1  jmcneill 
   3513  1.1  jmcneill 	return (error);
   3514  1.1  jmcneill }
   3515  1.1  jmcneill 
   3516  1.1  jmcneill static int
   3517  1.1  jmcneill mcx_iff(struct mcx_softc *sc)
   3518  1.1  jmcneill {
   3519  1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   3520  1.1  jmcneill 	struct mcx_dmamem mxm;
   3521  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3522  1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_in *in;
   3523  1.1  jmcneill 	struct mcx_cmd_modify_nic_vport_context_out *out;
   3524  1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   3525  1.1  jmcneill 	int error;
   3526  1.1  jmcneill 	int token;
   3527  1.1  jmcneill 	int insize;
   3528  1.1  jmcneill 
   3529  1.1  jmcneill 	/* enable or disable the promisc flow */
   3530  1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_PROMISC)) {
   3531  1.1  jmcneill 		if (sc->sc_promisc_flow_enabled == 0) {
   3532  1.1  jmcneill 			mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC,
   3533  1.1  jmcneill 			    0, NULL);
   3534  1.1  jmcneill 			sc->sc_promisc_flow_enabled = 1;
   3535  1.1  jmcneill 		}
   3536  1.1  jmcneill 	} else if (sc->sc_promisc_flow_enabled != 0) {
   3537  1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   3538  1.1  jmcneill 		sc->sc_promisc_flow_enabled = 0;
   3539  1.1  jmcneill 	}
   3540  1.1  jmcneill 
   3541  1.1  jmcneill 	/* enable or disable the all-multicast flow */
   3542  1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   3543  1.1  jmcneill 		if (sc->sc_allmulti_flow_enabled == 0) {
   3544  1.1  jmcneill 			uint8_t mcast[ETHER_ADDR_LEN];
   3545  1.1  jmcneill 
   3546  1.1  jmcneill 			memset(mcast, 0, sizeof(mcast));
   3547  1.1  jmcneill 			mcast[0] = 0x01;
   3548  1.1  jmcneill 			mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI,
   3549  1.1  jmcneill 			    0, mcast);
   3550  1.1  jmcneill 			sc->sc_allmulti_flow_enabled = 1;
   3551  1.1  jmcneill 		}
   3552  1.1  jmcneill 	} else if (sc->sc_allmulti_flow_enabled != 0) {
   3553  1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   3554  1.1  jmcneill 		sc->sc_allmulti_flow_enabled = 0;
   3555  1.1  jmcneill 	}
   3556  1.1  jmcneill 
   3557  1.1  jmcneill 	insize = sizeof(struct mcx_nic_vport_ctx) + 240;
   3558  1.1  jmcneill 
   3559  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3560  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   3561  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   3562  1.1  jmcneill 
   3563  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3564  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
   3565  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3566  1.1  jmcneill 	in->cmd_field_select = htobe32(
   3567  1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
   3568  1.1  jmcneill 	    MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
   3569  1.1  jmcneill 
   3570  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   3571  1.1  jmcneill 		printf(", unable to allocate modify nic vport context mailboxen\n");
   3572  1.1  jmcneill 		return (-1);
   3573  1.1  jmcneill 	}
   3574  1.1  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)
   3575  1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
   3576  1.1  jmcneill 	ctx->vp_mtu = htobe32(sc->sc_hardmtu);
   3577  1.1  jmcneill 	/*
   3578  1.1  jmcneill 	 * always leave promisc-all enabled on the vport since we can't give it
   3579  1.1  jmcneill 	 * a vlan list, and we're already doing multicast filtering in the flow
   3580  1.1  jmcneill 	 * table.
   3581  1.1  jmcneill 	 */
   3582  1.1  jmcneill 	ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);
   3583  1.1  jmcneill 
   3584  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   3585  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3586  1.1  jmcneill 
   3587  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3588  1.1  jmcneill 	if (error != 0) {
   3589  1.1  jmcneill 		printf(", modify nic vport context timeout\n");
   3590  1.1  jmcneill 		goto free;
   3591  1.1  jmcneill 	}
   3592  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3593  1.1  jmcneill 		printf(", modify nic vport context command corrupt\n");
   3594  1.1  jmcneill 		goto free;
   3595  1.1  jmcneill 	}
   3596  1.1  jmcneill 
   3597  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3598  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3599  1.1  jmcneill 		printf(", modify nic vport context failed (%x, %x)\n",
   3600  1.1  jmcneill 		    out->cmd_status, out->cmd_syndrome);
   3601  1.1  jmcneill 		error = -1;
   3602  1.1  jmcneill 		goto free;
   3603  1.1  jmcneill 	}
   3604  1.1  jmcneill 
   3605  1.1  jmcneill free:
   3606  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3607  1.1  jmcneill 
   3608  1.1  jmcneill 	return (error);
   3609  1.1  jmcneill }
   3610  1.1  jmcneill 
   3611  1.1  jmcneill static int
   3612  1.1  jmcneill mcx_alloc_uar(struct mcx_softc *sc)
   3613  1.1  jmcneill {
   3614  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3615  1.1  jmcneill 	struct mcx_cmd_alloc_uar_in *in;
   3616  1.1  jmcneill 	struct mcx_cmd_alloc_uar_out *out;
   3617  1.1  jmcneill 	int error;
   3618  1.1  jmcneill 
   3619  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3620  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3621  1.1  jmcneill 
   3622  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3623  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
   3624  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3625  1.1  jmcneill 
   3626  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3627  1.1  jmcneill 
   3628  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3629  1.1  jmcneill 	if (error != 0) {
   3630  1.1  jmcneill 		printf(", alloc uar timeout\n");
   3631  1.1  jmcneill 		return (-1);
   3632  1.1  jmcneill 	}
   3633  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3634  1.1  jmcneill 		printf(", alloc uar command corrupt\n");
   3635  1.1  jmcneill 		return (-1);
   3636  1.1  jmcneill 	}
   3637  1.1  jmcneill 
   3638  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3639  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3640  1.1  jmcneill 		printf(", alloc uar failed (%x)\n", out->cmd_status);
   3641  1.1  jmcneill 		return (-1);
   3642  1.1  jmcneill 	}
   3643  1.1  jmcneill 
   3644  1.1  jmcneill 	sc->sc_uar = be32toh(out->cmd_uar);
   3645  1.1  jmcneill 
   3646  1.1  jmcneill 	return (0);
   3647  1.1  jmcneill }
   3648  1.1  jmcneill 
   3649  1.1  jmcneill static int
   3650  1.1  jmcneill mcx_create_eq(struct mcx_softc *sc)
   3651  1.1  jmcneill {
   3652  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3653  1.1  jmcneill 	struct mcx_dmamem mxm;
   3654  1.1  jmcneill 	struct mcx_cmd_create_eq_in *in;
   3655  1.1  jmcneill 	struct mcx_cmd_create_eq_mb_in *mbin;
   3656  1.1  jmcneill 	struct mcx_cmd_create_eq_out *out;
   3657  1.1  jmcneill 	struct mcx_eq_entry *eqe;
   3658  1.1  jmcneill 	int error;
   3659  1.1  jmcneill 	uint64_t *pas;
   3660  1.1  jmcneill 	int insize, npages, paslen, i, token;
   3661  1.1  jmcneill 
   3662  1.1  jmcneill 	sc->sc_eq_cons = 0;
   3663  1.1  jmcneill 
   3664  1.1  jmcneill 	npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
   3665  1.1  jmcneill 	    MCX_PAGE_SIZE);
   3666  1.1  jmcneill 	paslen = npages * sizeof(*pas);
   3667  1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
   3668  1.1  jmcneill 
   3669  1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_eq_mem, npages * MCX_PAGE_SIZE,
   3670  1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   3671  1.1  jmcneill 		printf(", unable to allocate event queue memory\n");
   3672  1.1  jmcneill 		return (-1);
   3673  1.1  jmcneill 	}
   3674  1.1  jmcneill 
   3675  1.1  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&sc->sc_eq_mem);
   3676  1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
   3677  1.1  jmcneill 		eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
   3678  1.1  jmcneill 	}
   3679  1.1  jmcneill 
   3680  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3681  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   3682  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   3683  1.1  jmcneill 
   3684  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3685  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
   3686  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3687  1.1  jmcneill 
   3688  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   3689  1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   3690  1.1  jmcneill 		printf(", unable to allocate create eq mailboxen\n");
   3691  1.1  jmcneill 		return (-1);
   3692  1.1  jmcneill 	}
   3693  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   3694  1.1  jmcneill 	mbin->cmd_eq_ctx.eq_uar_size = htobe32(
   3695  1.1  jmcneill 	    (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | sc->sc_uar);
   3696  1.1  jmcneill 	mbin->cmd_event_bitmask = htobe64(
   3697  1.1  jmcneill 	    (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
   3698  1.1  jmcneill 	    (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
   3699  1.1  jmcneill 	    (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
   3700  1.1  jmcneill 	    (1ull << MCX_EVENT_TYPE_PAGE_REQUEST));
   3701  1.1  jmcneill 
   3702  1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   3703  1.1  jmcneill 	pas = (uint64_t *)(mbin + 1);
   3704  1.1  jmcneill 	for (i = 0; i < npages; i++) {
   3705  1.1  jmcneill 		pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_eq_mem) +
   3706  1.1  jmcneill 		    (i * MCX_PAGE_SIZE));
   3707  1.1  jmcneill 	}
   3708  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
   3709  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3710  1.1  jmcneill 
   3711  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3712  1.1  jmcneill 	if (error != 0) {
   3713  1.1  jmcneill 		printf(", create eq timeout\n");
   3714  1.1  jmcneill 		goto free;
   3715  1.1  jmcneill 	}
   3716  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3717  1.1  jmcneill 		printf(", create eq command corrupt\n");
   3718  1.1  jmcneill 		goto free;
   3719  1.1  jmcneill 	}
   3720  1.1  jmcneill 
   3721  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3722  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3723  1.1  jmcneill 		printf(", create eq failed (%x, %x)\n", out->cmd_status,
   3724  1.1  jmcneill 		    be32toh(out->cmd_syndrome));
   3725  1.1  jmcneill 		error = -1;
   3726  1.1  jmcneill 		goto free;
   3727  1.1  jmcneill 	}
   3728  1.1  jmcneill 
   3729  1.1  jmcneill 	sc->sc_eqn = be32toh(out->cmd_eqn);
   3730  1.1  jmcneill 	mcx_arm_eq(sc);
   3731  1.1  jmcneill free:
   3732  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3733  1.1  jmcneill 	return (error);
   3734  1.1  jmcneill }
   3735  1.1  jmcneill 
   3736  1.1  jmcneill static int
   3737  1.1  jmcneill mcx_alloc_pd(struct mcx_softc *sc)
   3738  1.1  jmcneill {
   3739  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3740  1.1  jmcneill 	struct mcx_cmd_alloc_pd_in *in;
   3741  1.1  jmcneill 	struct mcx_cmd_alloc_pd_out *out;
   3742  1.1  jmcneill 	int error;
   3743  1.1  jmcneill 
   3744  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3745  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3746  1.1  jmcneill 
   3747  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3748  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
   3749  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3750  1.1  jmcneill 
   3751  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3752  1.1  jmcneill 
   3753  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3754  1.1  jmcneill 	if (error != 0) {
   3755  1.1  jmcneill 		printf(", alloc pd timeout\n");
   3756  1.1  jmcneill 		return (-1);
   3757  1.1  jmcneill 	}
   3758  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3759  1.1  jmcneill 		printf(", alloc pd command corrupt\n");
   3760  1.1  jmcneill 		return (-1);
   3761  1.1  jmcneill 	}
   3762  1.1  jmcneill 
   3763  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3764  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3765  1.1  jmcneill 		printf(", alloc pd failed (%x)\n", out->cmd_status);
   3766  1.1  jmcneill 		return (-1);
   3767  1.1  jmcneill 	}
   3768  1.1  jmcneill 
   3769  1.1  jmcneill 	sc->sc_pd = be32toh(out->cmd_pd);
   3770  1.1  jmcneill 	return (0);
   3771  1.1  jmcneill }
   3772  1.1  jmcneill 
   3773  1.1  jmcneill static int
   3774  1.1  jmcneill mcx_alloc_tdomain(struct mcx_softc *sc)
   3775  1.1  jmcneill {
   3776  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3777  1.1  jmcneill 	struct mcx_cmd_alloc_td_in *in;
   3778  1.1  jmcneill 	struct mcx_cmd_alloc_td_out *out;
   3779  1.1  jmcneill 	int error;
   3780  1.1  jmcneill 
   3781  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3782  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3783  1.1  jmcneill 
   3784  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3785  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
   3786  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3787  1.1  jmcneill 
   3788  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3789  1.1  jmcneill 
   3790  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3791  1.1  jmcneill 	if (error != 0) {
   3792  1.1  jmcneill 		printf(", alloc transport domain timeout\n");
   3793  1.1  jmcneill 		return (-1);
   3794  1.1  jmcneill 	}
   3795  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3796  1.1  jmcneill 		printf(", alloc transport domain command corrupt\n");
   3797  1.1  jmcneill 		return (-1);
   3798  1.1  jmcneill 	}
   3799  1.1  jmcneill 
   3800  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3801  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3802  1.1  jmcneill 		printf(", alloc transport domain failed (%x)\n",
   3803  1.1  jmcneill 		    out->cmd_status);
   3804  1.1  jmcneill 		return (-1);
   3805  1.1  jmcneill 	}
   3806  1.1  jmcneill 
   3807  1.1  jmcneill 	sc->sc_tdomain = be32toh(out->cmd_tdomain);
   3808  1.1  jmcneill 	return (0);
   3809  1.1  jmcneill }
   3810  1.1  jmcneill 
   3811  1.1  jmcneill static int
   3812  1.1  jmcneill mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr)
   3813  1.1  jmcneill {
   3814  1.1  jmcneill 	struct mcx_dmamem mxm;
   3815  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3816  1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_in *in;
   3817  1.1  jmcneill 	struct mcx_cmd_query_nic_vport_context_out *out;
   3818  1.1  jmcneill 	struct mcx_nic_vport_ctx *ctx;
   3819  1.1  jmcneill 	uint8_t *addr;
   3820  1.1  jmcneill 	int error, token, i;
   3821  1.1  jmcneill 
   3822  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3823  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   3824  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
   3825  1.1  jmcneill 
   3826  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3827  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
   3828  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3829  1.1  jmcneill 	in->cmd_allowed_list_type = 0;
   3830  1.1  jmcneill 
   3831  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) {
   3832  1.1  jmcneill 		printf(", unable to allocate query nic vport context mailboxen\n");
   3833  1.1  jmcneill 		return (-1);
   3834  1.1  jmcneill 	}
   3835  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   3836  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3837  1.1  jmcneill 
   3838  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3839  1.1  jmcneill 	if (error != 0) {
   3840  1.1  jmcneill 		printf(", query nic vport context timeout\n");
   3841  1.1  jmcneill 		goto free;
   3842  1.1  jmcneill 	}
   3843  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3844  1.1  jmcneill 		printf(", query nic vport context command corrupt\n");
   3845  1.1  jmcneill 		goto free;
   3846  1.1  jmcneill 	}
   3847  1.1  jmcneill 
   3848  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3849  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3850  1.1  jmcneill 		printf(", query nic vport context failed (%x, %x)\n",
   3851  1.1  jmcneill 		    out->cmd_status, out->cmd_syndrome);
   3852  1.1  jmcneill 		error = -1;
   3853  1.1  jmcneill 		goto free;
   3854  1.1  jmcneill 	}
   3855  1.1  jmcneill 
   3856  1.1  jmcneill 	ctx = (struct mcx_nic_vport_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   3857  1.1  jmcneill 	addr = (uint8_t *)&ctx->vp_perm_addr;
   3858  1.1  jmcneill 	for (i = 0; i < ETHER_ADDR_LEN; i++) {
   3859  1.1  jmcneill 		enaddr[i] = addr[i + 2];
   3860  1.1  jmcneill 	}
   3861  1.1  jmcneill free:
   3862  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   3863  1.1  jmcneill 
   3864  1.1  jmcneill 	return (error);
   3865  1.1  jmcneill }
   3866  1.1  jmcneill 
   3867  1.1  jmcneill static int
   3868  1.1  jmcneill mcx_query_special_contexts(struct mcx_softc *sc)
   3869  1.1  jmcneill {
   3870  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   3871  1.1  jmcneill 	struct mcx_cmd_query_special_ctx_in *in;
   3872  1.1  jmcneill 	struct mcx_cmd_query_special_ctx_out *out;
   3873  1.1  jmcneill 	int error;
   3874  1.1  jmcneill 
   3875  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3876  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   3877  1.1  jmcneill 
   3878  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   3879  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
   3880  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3881  1.1  jmcneill 
   3882  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   3883  1.1  jmcneill 
   3884  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   3885  1.1  jmcneill 	if (error != 0) {
   3886  1.1  jmcneill 		printf(", query special contexts timeout\n");
   3887  1.1  jmcneill 		return (-1);
   3888  1.1  jmcneill 	}
   3889  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   3890  1.1  jmcneill 		printf(", query special contexts command corrupt\n");
   3891  1.1  jmcneill 		return (-1);
   3892  1.1  jmcneill 	}
   3893  1.1  jmcneill 
   3894  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   3895  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   3896  1.1  jmcneill 		printf(", query special contexts failed (%x)\n",
   3897  1.1  jmcneill 		    out->cmd_status);
   3898  1.1  jmcneill 		return (-1);
   3899  1.1  jmcneill 	}
   3900  1.1  jmcneill 
   3901  1.1  jmcneill 	sc->sc_lkey = be32toh(out->cmd_resd_lkey);
   3902  1.1  jmcneill 	return (0);
   3903  1.1  jmcneill }
   3904  1.1  jmcneill 
   3905  1.1  jmcneill static int
   3906  1.1  jmcneill mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
   3907  1.1  jmcneill {
   3908  1.1  jmcneill 	struct mcx_reg_pmtu pmtu;
   3909  1.1  jmcneill 	int error;
   3910  1.1  jmcneill 
   3911  1.1  jmcneill 	/* read max mtu */
   3912  1.1  jmcneill 	memset(&pmtu, 0, sizeof(pmtu));
   3913  1.1  jmcneill 	pmtu.rp_local_port = 1;
   3914  1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
   3915  1.1  jmcneill 	    sizeof(pmtu));
   3916  1.1  jmcneill 	if (error != 0) {
   3917  1.1  jmcneill 		printf(", unable to get port MTU\n");
   3918  1.1  jmcneill 		return error;
   3919  1.1  jmcneill 	}
   3920  1.1  jmcneill 
   3921  1.1  jmcneill 	mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu));
   3922  1.1  jmcneill 	pmtu.rp_admin_mtu = htobe16(mtu);
   3923  1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
   3924  1.1  jmcneill 	    sizeof(pmtu));
   3925  1.1  jmcneill 	if (error != 0) {
   3926  1.1  jmcneill 		printf(", unable to set port MTU\n");
   3927  1.1  jmcneill 		return error;
   3928  1.1  jmcneill 	}
   3929  1.1  jmcneill 
   3930  1.1  jmcneill 	sc->sc_hardmtu = mtu;
   3931  1.1  jmcneill 	return 0;
   3932  1.1  jmcneill }
   3933  1.1  jmcneill 
   3934  1.1  jmcneill static int
   3935  1.1  jmcneill mcx_create_cq(struct mcx_softc *sc, int eqn)
   3936  1.1  jmcneill {
   3937  1.1  jmcneill 	struct mcx_cmdq_entry *cmde;
   3938  1.1  jmcneill 	struct mcx_cq_entry *cqe;
   3939  1.1  jmcneill 	struct mcx_cq *cq;
   3940  1.1  jmcneill 	struct mcx_dmamem mxm;
   3941  1.1  jmcneill 	struct mcx_cmd_create_cq_in *in;
   3942  1.1  jmcneill 	struct mcx_cmd_create_cq_mb_in *mbin;
   3943  1.1  jmcneill 	struct mcx_cmd_create_cq_out *out;
   3944  1.1  jmcneill 	int error;
   3945  1.1  jmcneill 	uint64_t *pas;
   3946  1.1  jmcneill 	int insize, npages, paslen, i, token;
   3947  1.1  jmcneill 
   3948  1.1  jmcneill 	if (sc->sc_num_cq >= MCX_MAX_CQS) {
   3949  1.1  jmcneill 		printf("%s: tried to create too many cqs\n", DEVNAME(sc));
   3950  1.1  jmcneill 		return (-1);
   3951  1.1  jmcneill 	}
   3952  1.1  jmcneill 	cq = &sc->sc_cq[sc->sc_num_cq];
   3953  1.1  jmcneill 
   3954  1.1  jmcneill 	npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
   3955  1.1  jmcneill 	    MCX_PAGE_SIZE);
   3956  1.1  jmcneill 	paslen = npages * sizeof(*pas);
   3957  1.1  jmcneill 	insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
   3958  1.1  jmcneill 
   3959  1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
   3960  1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   3961  1.1  jmcneill 		printf("%s: unable to allocate completion queue memory\n",
   3962  1.1  jmcneill 		    DEVNAME(sc));
   3963  1.1  jmcneill 		return (-1);
   3964  1.1  jmcneill 	}
   3965  1.1  jmcneill 	cqe = MCX_DMA_KVA(&cq->cq_mem);
   3966  1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
   3967  1.1  jmcneill 		cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
   3968  1.1  jmcneill 	}
   3969  1.1  jmcneill 
   3970  1.1  jmcneill 	cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   3971  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   3972  1.1  jmcneill 	mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
   3973  1.1  jmcneill 
   3974  1.1  jmcneill 	in = mcx_cmdq_in(cmde);
   3975  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
   3976  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   3977  1.1  jmcneill 
   3978  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   3979  1.1  jmcneill 	    &cmde->cq_input_ptr, token) != 0) {
   3980  1.1  jmcneill 		printf("%s: unable to allocate create cq mailboxen\n", DEVNAME(sc));
   3981  1.1  jmcneill 		error = -1;
   3982  1.1  jmcneill 		goto free;
   3983  1.1  jmcneill 	}
   3984  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   3985  1.1  jmcneill 	mbin->cmd_cq_ctx.cq_uar_size = htobe32(
   3986  1.1  jmcneill 	    (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | sc->sc_uar);
   3987  1.1  jmcneill 	mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
   3988  1.1  jmcneill 	mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
   3989  1.1  jmcneill 	    (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
   3990  1.1  jmcneill 	    MCX_CQ_MOD_COUNTER);
   3991  1.1  jmcneill 	mbin->cmd_cq_ctx.cq_doorbell = htobe64(
   3992  1.1  jmcneill 	    MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   3993  1.1  jmcneill 	    MCX_CQ_DOORBELL_OFFSET + (MCX_CQ_DOORBELL_SIZE * sc->sc_num_cq));
   3994  1.1  jmcneill 
   3995  1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   3996  1.1  jmcneill 	pas = (uint64_t *)(mbin + 1);
   3997  1.1  jmcneill 	for (i = 0; i < npages; i++) {
   3998  1.1  jmcneill 		pas[i] = htobe64(MCX_DMA_DVA(&cq->cq_mem) + (i * MCX_PAGE_SIZE));
   3999  1.1  jmcneill 	}
   4000  1.1  jmcneill 	mcx_cmdq_post(sc, cmde, 0);
   4001  1.1  jmcneill 
   4002  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cmde, 1000);
   4003  1.1  jmcneill 	if (error != 0) {
   4004  1.1  jmcneill 		printf("%s: create cq timeout\n", DEVNAME(sc));
   4005  1.1  jmcneill 		goto free;
   4006  1.1  jmcneill 	}
   4007  1.1  jmcneill 	if (mcx_cmdq_verify(cmde) != 0) {
   4008  1.1  jmcneill 		printf("%s: create cq command corrupt\n", DEVNAME(sc));
   4009  1.1  jmcneill 		goto free;
   4010  1.1  jmcneill 	}
   4011  1.1  jmcneill 
   4012  1.1  jmcneill 	out = mcx_cmdq_out(cmde);
   4013  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4014  1.1  jmcneill 		printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
   4015  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4016  1.1  jmcneill 		error = -1;
   4017  1.1  jmcneill 		goto free;
   4018  1.1  jmcneill 	}
   4019  1.1  jmcneill 
   4020  1.1  jmcneill 	cq->cq_n = be32toh(out->cmd_cqn);
   4021  1.1  jmcneill 	cq->cq_cons = 0;
   4022  1.1  jmcneill 	cq->cq_count = 0;
   4023  1.1  jmcneill 	cq->cq_doorbell = (void *)((uint8_t *)MCX_DMA_KVA(&sc->sc_doorbell_mem) +
   4024  1.1  jmcneill 	    MCX_CQ_DOORBELL_OFFSET + (MCX_CQ_DOORBELL_SIZE * sc->sc_num_cq));
   4025  1.1  jmcneill 	mcx_arm_cq(sc, cq);
   4026  1.1  jmcneill 	sc->sc_num_cq++;
   4027  1.1  jmcneill 
   4028  1.1  jmcneill free:
   4029  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4030  1.1  jmcneill 	return (error);
   4031  1.1  jmcneill }
   4032  1.1  jmcneill 
   4033  1.1  jmcneill static int
   4034  1.1  jmcneill mcx_destroy_cq(struct mcx_softc *sc, int index)
   4035  1.1  jmcneill {
   4036  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4037  1.1  jmcneill 	struct mcx_cmd_destroy_cq_in *in;
   4038  1.1  jmcneill 	struct mcx_cmd_destroy_cq_out *out;
   4039  1.1  jmcneill 	int error;
   4040  1.1  jmcneill 	int token;
   4041  1.1  jmcneill 
   4042  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4043  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4044  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4045  1.1  jmcneill 
   4046  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4047  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
   4048  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4049  1.1  jmcneill 	in->cmd_cqn = htobe32(sc->sc_cq[index].cq_n);
   4050  1.1  jmcneill 
   4051  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4052  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4053  1.1  jmcneill 	if (error != 0) {
   4054  1.1  jmcneill 		printf("%s: destroy cq timeout\n", DEVNAME(sc));
   4055  1.1  jmcneill 		return error;
   4056  1.1  jmcneill 	}
   4057  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4058  1.1  jmcneill 		printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
   4059  1.1  jmcneill 		return error;
   4060  1.1  jmcneill 	}
   4061  1.1  jmcneill 
   4062  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4063  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4064  1.1  jmcneill 		printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
   4065  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4066  1.1  jmcneill 		return -1;
   4067  1.1  jmcneill 	}
   4068  1.1  jmcneill 
   4069  1.1  jmcneill 	sc->sc_cq[index].cq_n = 0;
   4070  1.1  jmcneill 	mcx_dmamem_free(sc, &sc->sc_cq[index].cq_mem);
   4071  1.1  jmcneill 	sc->sc_cq[index].cq_cons = 0;
   4072  1.1  jmcneill 	sc->sc_cq[index].cq_count = 0;
   4073  1.1  jmcneill 	return 0;
   4074  1.1  jmcneill }
   4075  1.1  jmcneill 
   4076  1.1  jmcneill static int
   4077  1.1  jmcneill mcx_create_rq(struct mcx_softc *sc, int cqn)
   4078  1.1  jmcneill {
   4079  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4080  1.1  jmcneill 	struct mcx_dmamem mxm;
   4081  1.1  jmcneill 	struct mcx_cmd_create_rq_in *in;
   4082  1.1  jmcneill 	struct mcx_cmd_create_rq_out *out;
   4083  1.1  jmcneill 	struct mcx_rq_ctx *mbin;
   4084  1.1  jmcneill 	int error;
   4085  1.1  jmcneill 	uint64_t *pas;
   4086  1.1  jmcneill 	uint8_t *doorbell;
   4087  1.1  jmcneill 	int insize, npages, paslen, i, token;
   4088  1.1  jmcneill 
   4089  1.1  jmcneill 	npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
   4090  1.1  jmcneill 	    MCX_PAGE_SIZE);
   4091  1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4092  1.1  jmcneill 	insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
   4093  1.1  jmcneill 
   4094  1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_rq_mem, npages * MCX_PAGE_SIZE,
   4095  1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4096  1.1  jmcneill 		printf("%s: unable to allocate receive queue memory\n",
   4097  1.1  jmcneill 		    DEVNAME(sc));
   4098  1.1  jmcneill 		return (-1);
   4099  1.1  jmcneill 	}
   4100  1.1  jmcneill 
   4101  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4102  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4103  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
   4104  1.1  jmcneill 
   4105  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4106  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
   4107  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4108  1.1  jmcneill 
   4109  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4110  1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4111  1.1  jmcneill 		printf("%s: unable to allocate create rq mailboxen\n",
   4112  1.1  jmcneill 		    DEVNAME(sc));
   4113  1.1  jmcneill 		error = -1;
   4114  1.1  jmcneill 		goto free;
   4115  1.1  jmcneill 	}
   4116  1.1  jmcneill 	mbin = (struct mcx_rq_ctx *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   4117  1.1  jmcneill 	mbin->rq_flags = htobe32(MCX_RQ_CTX_RLKEY | MCX_RQ_CTX_VLAN_STRIP_DIS);
   4118  1.1  jmcneill 	mbin->rq_cqn = htobe32(cqn);
   4119  1.1  jmcneill 	mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   4120  1.1  jmcneill 	mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
   4121  1.1  jmcneill 	mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   4122  1.1  jmcneill 	    MCX_RQ_DOORBELL_OFFSET);
   4123  1.1  jmcneill 	mbin->rq_wq.wq_log_stride = htobe16(4);
   4124  1.1  jmcneill 	mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
   4125  1.1  jmcneill 
   4126  1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4127  1.1  jmcneill 	pas = (uint64_t *)(mbin + 1);
   4128  1.1  jmcneill 	for (i = 0; i < npages; i++) {
   4129  1.1  jmcneill 		pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_rq_mem) +
   4130  1.1  jmcneill 		    (i * MCX_PAGE_SIZE));
   4131  1.1  jmcneill 	}
   4132  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4133  1.1  jmcneill 
   4134  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4135  1.1  jmcneill 	if (error != 0) {
   4136  1.1  jmcneill 		printf("%s: create rq timeout\n", DEVNAME(sc));
   4137  1.1  jmcneill 		goto free;
   4138  1.1  jmcneill 	}
   4139  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4140  1.1  jmcneill 		printf("%s: create rq command corrupt\n", DEVNAME(sc));
   4141  1.1  jmcneill 		goto free;
   4142  1.1  jmcneill 	}
   4143  1.1  jmcneill 
   4144  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4145  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4146  1.1  jmcneill 		printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
   4147  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4148  1.1  jmcneill 		error = -1;
   4149  1.1  jmcneill 		goto free;
   4150  1.1  jmcneill 	}
   4151  1.1  jmcneill 
   4152  1.1  jmcneill 	sc->sc_rqn = be32toh(out->cmd_rqn);
   4153  1.1  jmcneill 
   4154  1.1  jmcneill 	doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem);
   4155  1.1  jmcneill 	sc->sc_rx_doorbell = (uint32_t *)(doorbell + MCX_RQ_DOORBELL_OFFSET);
   4156  1.1  jmcneill 
   4157  1.1  jmcneill free:
   4158  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4159  1.1  jmcneill 	return (error);
   4160  1.1  jmcneill }
   4161  1.1  jmcneill 
   4162  1.1  jmcneill static int
   4163  1.1  jmcneill mcx_ready_rq(struct mcx_softc *sc)
   4164  1.1  jmcneill {
   4165  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4166  1.1  jmcneill 	struct mcx_dmamem mxm;
   4167  1.1  jmcneill 	struct mcx_cmd_modify_rq_in *in;
   4168  1.1  jmcneill 	struct mcx_cmd_modify_rq_mb_in *mbin;
   4169  1.1  jmcneill 	struct mcx_cmd_modify_rq_out *out;
   4170  1.1  jmcneill 	int error;
   4171  1.1  jmcneill 	int token;
   4172  1.1  jmcneill 
   4173  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4174  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4175  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
   4176  1.1  jmcneill 
   4177  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4178  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
   4179  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4180  1.1  jmcneill 	in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | sc->sc_rqn);
   4181  1.1  jmcneill 
   4182  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4183  1.1  jmcneill 		printf("%s: unable to allocate modify rq mailbox\n", DEVNAME(sc));
   4184  1.1  jmcneill 		return (-1);
   4185  1.1  jmcneill 	}
   4186  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4187  1.1  jmcneill 	mbin->cmd_rq_ctx.rq_flags = htobe32(
   4188  1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
   4189  1.1  jmcneill 
   4190  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4191  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4192  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4193  1.1  jmcneill 	if (error != 0) {
   4194  1.1  jmcneill 		printf("%s: modify rq timeout\n", DEVNAME(sc));
   4195  1.1  jmcneill 		goto free;
   4196  1.1  jmcneill 	}
   4197  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4198  1.1  jmcneill 		printf("%s: modify rq command corrupt\n", DEVNAME(sc));
   4199  1.1  jmcneill 		goto free;
   4200  1.1  jmcneill 	}
   4201  1.1  jmcneill 
   4202  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4203  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4204  1.1  jmcneill 		printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
   4205  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4206  1.1  jmcneill 		error = -1;
   4207  1.1  jmcneill 		goto free;
   4208  1.1  jmcneill 	}
   4209  1.1  jmcneill 
   4210  1.1  jmcneill free:
   4211  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4212  1.1  jmcneill 	return (error);
   4213  1.1  jmcneill }
   4214  1.1  jmcneill 
   4215  1.1  jmcneill static int
   4216  1.1  jmcneill mcx_destroy_rq(struct mcx_softc *sc)
   4217  1.1  jmcneill {
   4218  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4219  1.1  jmcneill 	struct mcx_cmd_destroy_rq_in *in;
   4220  1.1  jmcneill 	struct mcx_cmd_destroy_rq_out *out;
   4221  1.1  jmcneill 	int error;
   4222  1.1  jmcneill 	int token;
   4223  1.1  jmcneill 
   4224  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4225  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4226  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4227  1.1  jmcneill 
   4228  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4229  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
   4230  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4231  1.1  jmcneill 	in->cmd_rqn = htobe32(sc->sc_rqn);
   4232  1.1  jmcneill 
   4233  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4234  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4235  1.1  jmcneill 	if (error != 0) {
   4236  1.1  jmcneill 		printf("%s: destroy rq timeout\n", DEVNAME(sc));
   4237  1.1  jmcneill 		return error;
   4238  1.1  jmcneill 	}
   4239  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4240  1.1  jmcneill 		printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
   4241  1.1  jmcneill 		return error;
   4242  1.1  jmcneill 	}
   4243  1.1  jmcneill 
   4244  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4245  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4246  1.1  jmcneill 		printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
   4247  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4248  1.1  jmcneill 		return -1;
   4249  1.1  jmcneill 	}
   4250  1.1  jmcneill 
   4251  1.1  jmcneill 	sc->sc_rqn = 0;
   4252  1.1  jmcneill 	return 0;
   4253  1.1  jmcneill }
   4254  1.1  jmcneill 
   4255  1.1  jmcneill static int
   4256  1.1  jmcneill mcx_create_tir(struct mcx_softc *sc)
   4257  1.1  jmcneill {
   4258  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4259  1.1  jmcneill 	struct mcx_dmamem mxm;
   4260  1.1  jmcneill 	struct mcx_cmd_create_tir_in *in;
   4261  1.1  jmcneill 	struct mcx_cmd_create_tir_mb_in *mbin;
   4262  1.1  jmcneill 	struct mcx_cmd_create_tir_out *out;
   4263  1.1  jmcneill 	int error;
   4264  1.1  jmcneill 	int token;
   4265  1.1  jmcneill 
   4266  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4267  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4268  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
   4269  1.1  jmcneill 
   4270  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4271  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
   4272  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4273  1.1  jmcneill 
   4274  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4275  1.1  jmcneill 		printf("%s: unable to allocate create tir mailbox\n",
   4276  1.1  jmcneill 		    DEVNAME(sc));
   4277  1.1  jmcneill 		return (-1);
   4278  1.1  jmcneill 	}
   4279  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4280  1.1  jmcneill 	/* leave disp_type = 0, so packets get sent to the inline rqn */
   4281  1.1  jmcneill 	mbin->cmd_inline_rqn = htobe32(sc->sc_rqn);
   4282  1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   4283  1.1  jmcneill 
   4284  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4285  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4286  1.1  jmcneill 	if (error != 0) {
   4287  1.1  jmcneill 		printf("%s: create tir timeout\n", DEVNAME(sc));
   4288  1.1  jmcneill 		goto free;
   4289  1.1  jmcneill 	}
   4290  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4291  1.1  jmcneill 		printf("%s: create tir command corrupt\n", DEVNAME(sc));
   4292  1.1  jmcneill 		goto free;
   4293  1.1  jmcneill 	}
   4294  1.1  jmcneill 
   4295  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4296  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4297  1.1  jmcneill 		printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
   4298  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4299  1.1  jmcneill 		error = -1;
   4300  1.1  jmcneill 		goto free;
   4301  1.1  jmcneill 	}
   4302  1.1  jmcneill 
   4303  1.1  jmcneill 	sc->sc_tirn = be32toh(out->cmd_tirn);
   4304  1.1  jmcneill free:
   4305  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4306  1.1  jmcneill 	return (error);
   4307  1.1  jmcneill }
   4308  1.1  jmcneill 
   4309  1.1  jmcneill static int
   4310  1.1  jmcneill mcx_destroy_tir(struct mcx_softc *sc)
   4311  1.1  jmcneill {
   4312  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4313  1.1  jmcneill 	struct mcx_cmd_destroy_tir_in *in;
   4314  1.1  jmcneill 	struct mcx_cmd_destroy_tir_out *out;
   4315  1.1  jmcneill 	int error;
   4316  1.1  jmcneill 	int token;
   4317  1.1  jmcneill 
   4318  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4319  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4320  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4321  1.1  jmcneill 
   4322  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4323  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
   4324  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4325  1.1  jmcneill 	in->cmd_tirn = htobe32(sc->sc_tirn);
   4326  1.1  jmcneill 
   4327  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4328  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4329  1.1  jmcneill 	if (error != 0) {
   4330  1.1  jmcneill 		printf("%s: destroy tir timeout\n", DEVNAME(sc));
   4331  1.1  jmcneill 		return error;
   4332  1.1  jmcneill 	}
   4333  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4334  1.1  jmcneill 		printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
   4335  1.1  jmcneill 		return error;
   4336  1.1  jmcneill 	}
   4337  1.1  jmcneill 
   4338  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4339  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4340  1.1  jmcneill 		printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
   4341  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4342  1.1  jmcneill 		return -1;
   4343  1.1  jmcneill 	}
   4344  1.1  jmcneill 
   4345  1.1  jmcneill 	sc->sc_tirn = 0;
   4346  1.1  jmcneill 	return 0;
   4347  1.1  jmcneill }
   4348  1.1  jmcneill 
   4349  1.1  jmcneill static int
   4350  1.1  jmcneill mcx_create_sq(struct mcx_softc *sc, int cqn)
   4351  1.1  jmcneill {
   4352  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4353  1.1  jmcneill 	struct mcx_dmamem mxm;
   4354  1.1  jmcneill 	struct mcx_cmd_create_sq_in *in;
   4355  1.1  jmcneill 	struct mcx_sq_ctx *mbin;
   4356  1.1  jmcneill 	struct mcx_cmd_create_sq_out *out;
   4357  1.1  jmcneill 	int error;
   4358  1.1  jmcneill 	uint64_t *pas;
   4359  1.1  jmcneill 	uint8_t *doorbell;
   4360  1.1  jmcneill 	int insize, npages, paslen, i, token;
   4361  1.1  jmcneill 
   4362  1.1  jmcneill 	npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
   4363  1.1  jmcneill 	    MCX_PAGE_SIZE);
   4364  1.1  jmcneill 	paslen = npages * sizeof(*pas);
   4365  1.1  jmcneill 	insize = sizeof(struct mcx_sq_ctx) + paslen;
   4366  1.1  jmcneill 
   4367  1.1  jmcneill 	if (mcx_dmamem_alloc(sc, &sc->sc_sq_mem, npages * MCX_PAGE_SIZE,
   4368  1.1  jmcneill 	    MCX_PAGE_SIZE) != 0) {
   4369  1.1  jmcneill 		printf("%s: unable to allocate send queue memory\n", DEVNAME(sc));
   4370  1.1  jmcneill 		return (-1);
   4371  1.1  jmcneill 	}
   4372  1.1  jmcneill 
   4373  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4374  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4375  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
   4376  1.1  jmcneill 	    token);
   4377  1.1  jmcneill 
   4378  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4379  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
   4380  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4381  1.1  jmcneill 
   4382  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
   4383  1.1  jmcneill 	    &cqe->cq_input_ptr, token) != 0) {
   4384  1.1  jmcneill 		printf("%s: unable to allocate create sq mailboxen\n", DEVNAME(sc));
   4385  1.1  jmcneill 		error = -1;
   4386  1.1  jmcneill 		goto free;
   4387  1.1  jmcneill 	}
   4388  1.1  jmcneill 	mbin = (struct mcx_sq_ctx *)(((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
   4389  1.1  jmcneill 	mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
   4390  1.1  jmcneill 	    (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
   4391  1.1  jmcneill 	mbin->sq_cqn = htobe32(cqn);
   4392  1.1  jmcneill 	mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
   4393  1.1  jmcneill 	mbin->sq_tis_num = htobe32(sc->sc_tisn);
   4394  1.1  jmcneill 	mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
   4395  1.1  jmcneill 	mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
   4396  1.1  jmcneill 	mbin->sq_wq.wq_uar_page = htobe32(sc->sc_uar);
   4397  1.1  jmcneill 	mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
   4398  1.1  jmcneill 	    MCX_SQ_DOORBELL_OFFSET);
   4399  1.1  jmcneill 	mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
   4400  1.1  jmcneill 	mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
   4401  1.1  jmcneill 
   4402  1.1  jmcneill 	/* physical addresses follow the mailbox in data */
   4403  1.1  jmcneill 	pas = (uint64_t *)(mbin + 1);
   4404  1.1  jmcneill 	for (i = 0; i < npages; i++) {
   4405  1.1  jmcneill 		pas[i] = htobe64(MCX_DMA_DVA(&sc->sc_sq_mem) +
   4406  1.1  jmcneill 		    (i * MCX_PAGE_SIZE));
   4407  1.1  jmcneill 	}
   4408  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4409  1.1  jmcneill 
   4410  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4411  1.1  jmcneill 	if (error != 0) {
   4412  1.1  jmcneill 		printf("%s: create sq timeout\n", DEVNAME(sc));
   4413  1.1  jmcneill 		goto free;
   4414  1.1  jmcneill 	}
   4415  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4416  1.1  jmcneill 		printf("%s: create sq command corrupt\n", DEVNAME(sc));
   4417  1.1  jmcneill 		goto free;
   4418  1.1  jmcneill 	}
   4419  1.1  jmcneill 
   4420  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4421  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4422  1.1  jmcneill 		printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
   4423  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4424  1.1  jmcneill 		error = -1;
   4425  1.1  jmcneill 		goto free;
   4426  1.1  jmcneill 	}
   4427  1.1  jmcneill 
   4428  1.1  jmcneill 	sc->sc_sqn = be32toh(out->cmd_sqn);
   4429  1.1  jmcneill 
   4430  1.1  jmcneill 	doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem);
   4431  1.1  jmcneill 	sc->sc_tx_doorbell = (uint32_t *)(doorbell + MCX_SQ_DOORBELL_OFFSET + 4);
   4432  1.1  jmcneill free:
   4433  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4434  1.1  jmcneill 	return (error);
   4435  1.1  jmcneill }
   4436  1.1  jmcneill 
   4437  1.1  jmcneill static int
   4438  1.1  jmcneill mcx_destroy_sq(struct mcx_softc *sc)
   4439  1.1  jmcneill {
   4440  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4441  1.1  jmcneill 	struct mcx_cmd_destroy_sq_in *in;
   4442  1.1  jmcneill 	struct mcx_cmd_destroy_sq_out *out;
   4443  1.1  jmcneill 	int error;
   4444  1.1  jmcneill 	int token;
   4445  1.1  jmcneill 
   4446  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4447  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4448  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4449  1.1  jmcneill 
   4450  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4451  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
   4452  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4453  1.1  jmcneill 	in->cmd_sqn = htobe32(sc->sc_sqn);
   4454  1.1  jmcneill 
   4455  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4456  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4457  1.1  jmcneill 	if (error != 0) {
   4458  1.1  jmcneill 		printf("%s: destroy sq timeout\n", DEVNAME(sc));
   4459  1.1  jmcneill 		return error;
   4460  1.1  jmcneill 	}
   4461  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4462  1.1  jmcneill 		printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
   4463  1.1  jmcneill 		return error;
   4464  1.1  jmcneill 	}
   4465  1.1  jmcneill 
   4466  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4467  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4468  1.1  jmcneill 		printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
   4469  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4470  1.1  jmcneill 		return -1;
   4471  1.1  jmcneill 	}
   4472  1.1  jmcneill 
   4473  1.1  jmcneill 	sc->sc_sqn = 0;
   4474  1.1  jmcneill 	return 0;
   4475  1.1  jmcneill }
   4476  1.1  jmcneill 
   4477  1.1  jmcneill static int
   4478  1.1  jmcneill mcx_ready_sq(struct mcx_softc *sc)
   4479  1.1  jmcneill {
   4480  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4481  1.1  jmcneill 	struct mcx_dmamem mxm;
   4482  1.1  jmcneill 	struct mcx_cmd_modify_sq_in *in;
   4483  1.1  jmcneill 	struct mcx_cmd_modify_sq_mb_in *mbin;
   4484  1.1  jmcneill 	struct mcx_cmd_modify_sq_out *out;
   4485  1.1  jmcneill 	int error;
   4486  1.1  jmcneill 	int token;
   4487  1.1  jmcneill 
   4488  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4489  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4490  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
   4491  1.1  jmcneill 
   4492  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4493  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
   4494  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4495  1.1  jmcneill 	in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | sc->sc_sqn);
   4496  1.1  jmcneill 
   4497  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4498  1.1  jmcneill 		printf("%s: unable to allocate modify sq mailbox\n",
   4499  1.1  jmcneill 		    DEVNAME(sc));
   4500  1.1  jmcneill 		return (-1);
   4501  1.1  jmcneill 	}
   4502  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4503  1.1  jmcneill 	mbin->cmd_sq_ctx.sq_flags = htobe32(
   4504  1.1  jmcneill 	    MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
   4505  1.1  jmcneill 
   4506  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4507  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4508  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4509  1.1  jmcneill 	if (error != 0) {
   4510  1.1  jmcneill 		printf("%s: modify sq timeout\n", DEVNAME(sc));
   4511  1.1  jmcneill 		goto free;
   4512  1.1  jmcneill 	}
   4513  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4514  1.1  jmcneill 		printf("%s: modify sq command corrupt\n", DEVNAME(sc));
   4515  1.1  jmcneill 		goto free;
   4516  1.1  jmcneill 	}
   4517  1.1  jmcneill 
   4518  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4519  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4520  1.1  jmcneill 		printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
   4521  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4522  1.1  jmcneill 		error = -1;
   4523  1.1  jmcneill 		goto free;
   4524  1.1  jmcneill 	}
   4525  1.1  jmcneill 
   4526  1.1  jmcneill free:
   4527  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4528  1.1  jmcneill 	return (error);
   4529  1.1  jmcneill }
   4530  1.1  jmcneill 
   4531  1.1  jmcneill static int
   4532  1.1  jmcneill mcx_create_tis(struct mcx_softc *sc)
   4533  1.1  jmcneill {
   4534  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4535  1.1  jmcneill 	struct mcx_dmamem mxm;
   4536  1.1  jmcneill 	struct mcx_cmd_create_tis_in *in;
   4537  1.1  jmcneill 	struct mcx_cmd_create_tis_mb_in *mbin;
   4538  1.1  jmcneill 	struct mcx_cmd_create_tis_out *out;
   4539  1.1  jmcneill 	int error;
   4540  1.1  jmcneill 	int token;
   4541  1.1  jmcneill 
   4542  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4543  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4544  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
   4545  1.1  jmcneill 
   4546  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4547  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
   4548  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4549  1.1  jmcneill 
   4550  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4551  1.1  jmcneill 		printf("%s: unable to allocate create tis mailbox\n", DEVNAME(sc));
   4552  1.1  jmcneill 		return (-1);
   4553  1.1  jmcneill 	}
   4554  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4555  1.1  jmcneill 	mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
   4556  1.1  jmcneill 
   4557  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4558  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4559  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4560  1.1  jmcneill 	if (error != 0) {
   4561  1.1  jmcneill 		printf("%s: create tis timeout\n", DEVNAME(sc));
   4562  1.1  jmcneill 		goto free;
   4563  1.1  jmcneill 	}
   4564  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4565  1.1  jmcneill 		printf("%s: create tis command corrupt\n", DEVNAME(sc));
   4566  1.1  jmcneill 		goto free;
   4567  1.1  jmcneill 	}
   4568  1.1  jmcneill 
   4569  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4570  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4571  1.1  jmcneill 		printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
   4572  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4573  1.1  jmcneill 		error = -1;
   4574  1.1  jmcneill 		goto free;
   4575  1.1  jmcneill 	}
   4576  1.1  jmcneill 
   4577  1.1  jmcneill 	sc->sc_tisn = be32toh(out->cmd_tisn);
   4578  1.1  jmcneill free:
   4579  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4580  1.1  jmcneill 	return (error);
   4581  1.1  jmcneill }
   4582  1.1  jmcneill 
   4583  1.1  jmcneill static int
   4584  1.1  jmcneill mcx_destroy_tis(struct mcx_softc *sc)
   4585  1.1  jmcneill {
   4586  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4587  1.1  jmcneill 	struct mcx_cmd_destroy_tis_in *in;
   4588  1.1  jmcneill 	struct mcx_cmd_destroy_tis_out *out;
   4589  1.1  jmcneill 	int error;
   4590  1.1  jmcneill 	int token;
   4591  1.1  jmcneill 
   4592  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4593  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4594  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
   4595  1.1  jmcneill 
   4596  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4597  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
   4598  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4599  1.1  jmcneill 	in->cmd_tisn = htobe32(sc->sc_tisn);
   4600  1.1  jmcneill 
   4601  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4602  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4603  1.1  jmcneill 	if (error != 0) {
   4604  1.1  jmcneill 		printf("%s: destroy tis timeout\n", DEVNAME(sc));
   4605  1.1  jmcneill 		return error;
   4606  1.1  jmcneill 	}
   4607  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4608  1.1  jmcneill 		printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
   4609  1.1  jmcneill 		return error;
   4610  1.1  jmcneill 	}
   4611  1.1  jmcneill 
   4612  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4613  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4614  1.1  jmcneill 		printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
   4615  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4616  1.1  jmcneill 		return -1;
   4617  1.1  jmcneill 	}
   4618  1.1  jmcneill 
   4619  1.1  jmcneill 	sc->sc_tirn = 0;
   4620  1.1  jmcneill 	return 0;
   4621  1.1  jmcneill }
   4622  1.1  jmcneill 
   4623  1.1  jmcneill #if 0
   4624  1.1  jmcneill static int
   4625  1.1  jmcneill mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
   4626  1.1  jmcneill {
   4627  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4628  1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_in *in;
   4629  1.1  jmcneill 	struct mcx_cmd_alloc_flow_counter_out *out;
   4630  1.1  jmcneill 	int error;
   4631  1.1  jmcneill 
   4632  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4633  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
   4634  1.1  jmcneill 
   4635  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4636  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
   4637  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4638  1.1  jmcneill 
   4639  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4640  1.1  jmcneill 
   4641  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4642  1.1  jmcneill 	if (error != 0) {
   4643  1.1  jmcneill 		printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
   4644  1.1  jmcneill 		return (-1);
   4645  1.1  jmcneill 	}
   4646  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4647  1.1  jmcneill 		printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
   4648  1.1  jmcneill 		return (-1);
   4649  1.1  jmcneill 	}
   4650  1.1  jmcneill 
   4651  1.1  jmcneill 	out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
   4652  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4653  1.1  jmcneill 		printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
   4654  1.1  jmcneill 		    out->cmd_status);
   4655  1.1  jmcneill 		return (-1);
   4656  1.1  jmcneill 	}
   4657  1.1  jmcneill 
   4658  1.1  jmcneill 	sc->sc_flow_counter_id[i]  = be16toh(out->cmd_flow_counter_id);
   4659  1.1  jmcneill 	printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
   4660  1.1  jmcneill 
   4661  1.1  jmcneill 	return (0);
   4662  1.1  jmcneill }
   4663  1.1  jmcneill #endif
   4664  1.1  jmcneill 
   4665  1.1  jmcneill static int
   4666  1.1  jmcneill mcx_create_flow_table(struct mcx_softc *sc, int log_size)
   4667  1.1  jmcneill {
   4668  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4669  1.1  jmcneill 	struct mcx_dmamem mxm;
   4670  1.1  jmcneill 	struct mcx_cmd_create_flow_table_in *in;
   4671  1.1  jmcneill 	struct mcx_cmd_create_flow_table_mb_in *mbin;
   4672  1.1  jmcneill 	struct mcx_cmd_create_flow_table_out *out;
   4673  1.1  jmcneill 	int error;
   4674  1.1  jmcneill 	int token;
   4675  1.1  jmcneill 
   4676  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4677  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4678  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
   4679  1.1  jmcneill 
   4680  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4681  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
   4682  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4683  1.1  jmcneill 
   4684  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4685  1.1  jmcneill 		printf("%s: unable to allocate create flow table mailbox\n",
   4686  1.1  jmcneill 		    DEVNAME(sc));
   4687  1.1  jmcneill 		return (-1);
   4688  1.1  jmcneill 	}
   4689  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4690  1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   4691  1.1  jmcneill 	mbin->cmd_ctx.ft_log_size = log_size;
   4692  1.1  jmcneill 
   4693  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4694  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4695  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4696  1.1  jmcneill 	if (error != 0) {
   4697  1.1  jmcneill 		printf("%s: create flow table timeout\n", DEVNAME(sc));
   4698  1.1  jmcneill 		goto free;
   4699  1.1  jmcneill 	}
   4700  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4701  1.1  jmcneill 		printf("%s: create flow table command corrupt\n", DEVNAME(sc));
   4702  1.1  jmcneill 		goto free;
   4703  1.1  jmcneill 	}
   4704  1.1  jmcneill 
   4705  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4706  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4707  1.1  jmcneill 		printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
   4708  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4709  1.1  jmcneill 		error = -1;
   4710  1.1  jmcneill 		goto free;
   4711  1.1  jmcneill 	}
   4712  1.1  jmcneill 
   4713  1.1  jmcneill 	sc->sc_flow_table_id = be32toh(out->cmd_table_id);
   4714  1.1  jmcneill free:
   4715  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4716  1.1  jmcneill 	return (error);
   4717  1.1  jmcneill }
   4718  1.1  jmcneill 
   4719  1.1  jmcneill static int
   4720  1.1  jmcneill mcx_set_flow_table_root(struct mcx_softc *sc)
   4721  1.1  jmcneill {
   4722  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4723  1.1  jmcneill 	struct mcx_dmamem mxm;
   4724  1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_in *in;
   4725  1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_mb_in *mbin;
   4726  1.1  jmcneill 	struct mcx_cmd_set_flow_table_root_out *out;
   4727  1.1  jmcneill 	int error;
   4728  1.1  jmcneill 	int token;
   4729  1.1  jmcneill 
   4730  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4731  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4732  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), token);
   4733  1.1  jmcneill 
   4734  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4735  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
   4736  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4737  1.1  jmcneill 
   4738  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4739  1.1  jmcneill 		printf("%s: unable to allocate set flow table root mailbox\n",
   4740  1.1  jmcneill 		    DEVNAME(sc));
   4741  1.1  jmcneill 		return (-1);
   4742  1.1  jmcneill 	}
   4743  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4744  1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   4745  1.1  jmcneill 	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
   4746  1.1  jmcneill 
   4747  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4748  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4749  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4750  1.1  jmcneill 	if (error != 0) {
   4751  1.1  jmcneill 		printf("%s: set flow table root timeout\n", DEVNAME(sc));
   4752  1.1  jmcneill 		goto free;
   4753  1.1  jmcneill 	}
   4754  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4755  1.1  jmcneill 		printf("%s: set flow table root command corrupt\n",
   4756  1.1  jmcneill 		    DEVNAME(sc));
   4757  1.1  jmcneill 		goto free;
   4758  1.1  jmcneill 	}
   4759  1.1  jmcneill 
   4760  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4761  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4762  1.1  jmcneill 		printf("%s: set flow table root failed (%x, %x)\n",
   4763  1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   4764  1.1  jmcneill 		error = -1;
   4765  1.1  jmcneill 		goto free;
   4766  1.1  jmcneill 	}
   4767  1.1  jmcneill 
   4768  1.1  jmcneill free:
   4769  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4770  1.1  jmcneill 	return (error);
   4771  1.1  jmcneill }
   4772  1.1  jmcneill 
   4773  1.1  jmcneill static int
   4774  1.1  jmcneill mcx_destroy_flow_table(struct mcx_softc *sc)
   4775  1.1  jmcneill {
   4776  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4777  1.1  jmcneill 	struct mcx_dmamem mxm;
   4778  1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_in *in;
   4779  1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_mb_in *mb;
   4780  1.1  jmcneill 	struct mcx_cmd_destroy_flow_table_out *out;
   4781  1.1  jmcneill 	int error;
   4782  1.1  jmcneill 	int token;
   4783  1.1  jmcneill 
   4784  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4785  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4786  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   4787  1.1  jmcneill 
   4788  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4789  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
   4790  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4791  1.1  jmcneill 
   4792  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
   4793  1.1  jmcneill 		printf("%s: unable to allocate destroy flow table mailbox\n",
   4794  1.1  jmcneill 		    DEVNAME(sc));
   4795  1.1  jmcneill 		return (-1);
   4796  1.1  jmcneill 	}
   4797  1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4798  1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   4799  1.1  jmcneill 	mb->cmd_table_id = htobe32(sc->sc_flow_table_id);
   4800  1.1  jmcneill 
   4801  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   4802  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4803  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4804  1.1  jmcneill 	if (error != 0) {
   4805  1.1  jmcneill 		printf("%s: destroy flow table timeout\n", DEVNAME(sc));
   4806  1.1  jmcneill 		goto free;
   4807  1.1  jmcneill 	}
   4808  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4809  1.1  jmcneill 		printf("%s: destroy flow table command corrupt\n", DEVNAME(sc));
   4810  1.1  jmcneill 		goto free;
   4811  1.1  jmcneill 	}
   4812  1.1  jmcneill 
   4813  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4814  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4815  1.1  jmcneill 		printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
   4816  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4817  1.1  jmcneill 		error = -1;
   4818  1.1  jmcneill 		goto free;
   4819  1.1  jmcneill 	}
   4820  1.1  jmcneill 
   4821  1.1  jmcneill 	sc->sc_flow_table_id = -1;
   4822  1.1  jmcneill free:
   4823  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4824  1.1  jmcneill 	return (error);
   4825  1.1  jmcneill }
   4826  1.1  jmcneill 
   4827  1.1  jmcneill 
   4828  1.1  jmcneill static int
   4829  1.1  jmcneill mcx_create_flow_group(struct mcx_softc *sc, int group, int start, int size,
   4830  1.1  jmcneill     int match_enable, struct mcx_flow_match *match)
   4831  1.1  jmcneill {
   4832  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4833  1.1  jmcneill 	struct mcx_dmamem mxm;
   4834  1.1  jmcneill 	struct mcx_cmd_create_flow_group_in *in;
   4835  1.1  jmcneill 	struct mcx_cmd_create_flow_group_mb_in *mbin;
   4836  1.1  jmcneill 	struct mcx_cmd_create_flow_group_out *out;
   4837  1.1  jmcneill 	int error;
   4838  1.1  jmcneill 	int token;
   4839  1.1  jmcneill 
   4840  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4841  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4842  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   4843  1.1  jmcneill 	    token);
   4844  1.1  jmcneill 
   4845  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4846  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
   4847  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4848  1.1  jmcneill 
   4849  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   4850  1.1  jmcneill 	    != 0) {
   4851  1.1  jmcneill 		printf("%s: unable to allocate create flow group mailbox\n",
   4852  1.1  jmcneill 		    DEVNAME(sc));
   4853  1.1  jmcneill 		return (-1);
   4854  1.1  jmcneill 	}
   4855  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4856  1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   4857  1.1  jmcneill 	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
   4858  1.1  jmcneill 	mbin->cmd_start_flow_index = htobe32(start);
   4859  1.1  jmcneill 	mbin->cmd_end_flow_index = htobe32(start + (size - 1));
   4860  1.1  jmcneill 
   4861  1.1  jmcneill 	mbin->cmd_match_criteria_enable = match_enable;
   4862  1.1  jmcneill 	memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));
   4863  1.1  jmcneill 
   4864  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   4865  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4866  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4867  1.1  jmcneill 	if (error != 0) {
   4868  1.1  jmcneill 		printf("%s: create flow group timeout\n", DEVNAME(sc));
   4869  1.1  jmcneill 		goto free;
   4870  1.1  jmcneill 	}
   4871  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4872  1.1  jmcneill 		printf("%s: create flow group command corrupt\n", DEVNAME(sc));
   4873  1.1  jmcneill 		goto free;
   4874  1.1  jmcneill 	}
   4875  1.1  jmcneill 
   4876  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4877  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4878  1.1  jmcneill 		printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
   4879  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4880  1.1  jmcneill 		error = -1;
   4881  1.1  jmcneill 		goto free;
   4882  1.1  jmcneill 	}
   4883  1.1  jmcneill 
   4884  1.1  jmcneill 	sc->sc_flow_group_id[group] = be32toh(out->cmd_group_id);
   4885  1.1  jmcneill 	sc->sc_flow_group_size[group] = size;
   4886  1.1  jmcneill 	sc->sc_flow_group_start[group] = start;
   4887  1.1  jmcneill 
   4888  1.1  jmcneill free:
   4889  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4890  1.1  jmcneill 	return (error);
   4891  1.1  jmcneill }
   4892  1.1  jmcneill 
   4893  1.1  jmcneill static int
   4894  1.1  jmcneill mcx_destroy_flow_group(struct mcx_softc *sc, int group)
   4895  1.1  jmcneill {
   4896  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4897  1.1  jmcneill 	struct mcx_dmamem mxm;
   4898  1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_in *in;
   4899  1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_mb_in *mb;
   4900  1.1  jmcneill 	struct mcx_cmd_destroy_flow_group_out *out;
   4901  1.1  jmcneill 	int error;
   4902  1.1  jmcneill 	int token;
   4903  1.1  jmcneill 
   4904  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4905  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4906  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
   4907  1.1  jmcneill 
   4908  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4909  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
   4910  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4911  1.1  jmcneill 
   4912  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) != 0) {
   4913  1.1  jmcneill 		printf("%s: unable to allocate destroy flow group mailbox\n",
   4914  1.1  jmcneill 		    DEVNAME(sc));
   4915  1.1  jmcneill 		return (-1);
   4916  1.1  jmcneill 	}
   4917  1.1  jmcneill 	mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4918  1.1  jmcneill 	mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   4919  1.1  jmcneill 	mb->cmd_table_id = htobe32(sc->sc_flow_table_id);
   4920  1.1  jmcneill 	mb->cmd_group_id = htobe32(sc->sc_flow_group_id[group]);
   4921  1.1  jmcneill 
   4922  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   4923  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4924  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4925  1.1  jmcneill 	if (error != 0) {
   4926  1.1  jmcneill 		printf("%s: destroy flow group timeout\n", DEVNAME(sc));
   4927  1.1  jmcneill 		goto free;
   4928  1.1  jmcneill 	}
   4929  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   4930  1.1  jmcneill 		printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
   4931  1.1  jmcneill 		goto free;
   4932  1.1  jmcneill 	}
   4933  1.1  jmcneill 
   4934  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   4935  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   4936  1.1  jmcneill 		printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
   4937  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   4938  1.1  jmcneill 		error = -1;
   4939  1.1  jmcneill 		goto free;
   4940  1.1  jmcneill 	}
   4941  1.1  jmcneill 
   4942  1.1  jmcneill 	sc->sc_flow_group_id[group] = -1;
   4943  1.1  jmcneill 	sc->sc_flow_group_size[group] = 0;
   4944  1.1  jmcneill free:
   4945  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   4946  1.1  jmcneill 	return (error);
   4947  1.1  jmcneill }
   4948  1.1  jmcneill 
   4949  1.1  jmcneill static int
   4950  1.1  jmcneill mcx_set_flow_table_entry(struct mcx_softc *sc, int group, int index,
   4951  1.1  jmcneill     const uint8_t *macaddr)
   4952  1.1  jmcneill {
   4953  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   4954  1.1  jmcneill 	struct mcx_dmamem mxm;
   4955  1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_in *in;
   4956  1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
   4957  1.1  jmcneill 	struct mcx_cmd_set_flow_table_entry_out *out;
   4958  1.1  jmcneill 	uint32_t *dest;
   4959  1.1  jmcneill 	int error;
   4960  1.1  jmcneill 	int token;
   4961  1.1  jmcneill 
   4962  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   4963  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   4964  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*dest),
   4965  1.1  jmcneill 	    sizeof(*out), token);
   4966  1.1  jmcneill 
   4967  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   4968  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
   4969  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   4970  1.1  jmcneill 
   4971  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
   4972  1.1  jmcneill 	    != 0) {
   4973  1.1  jmcneill 		printf("%s: unable to allocate set flow table entry mailbox\n",
   4974  1.1  jmcneill 		    DEVNAME(sc));
   4975  1.1  jmcneill 		return (-1);
   4976  1.1  jmcneill 	}
   4977  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   4978  1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   4979  1.1  jmcneill 	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
   4980  1.1  jmcneill 	mbin->cmd_flow_index = htobe32(sc->sc_flow_group_start[group] + index);
   4981  1.1  jmcneill 	mbin->cmd_flow_ctx.fc_group_id = htobe32(sc->sc_flow_group_id[group]);
   4982  1.1  jmcneill 
   4983  1.1  jmcneill 	/* flow context ends at offset 0x330, 0x130 into the second mbox */
   4984  1.1  jmcneill 	dest = (uint32_t *)
   4985  1.1  jmcneill 	    (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
   4986  1.1  jmcneill 	mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
   4987  1.1  jmcneill 	mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
   4988  1.1  jmcneill 	*dest = htobe32(sc->sc_tirn | MCX_FLOW_CONTEXT_DEST_TYPE_TIR);
   4989  1.1  jmcneill 
   4990  1.1  jmcneill 	/* the only thing we match on at the moment is the dest mac address */
   4991  1.1  jmcneill 	if (macaddr != NULL) {
   4992  1.1  jmcneill 		memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
   4993  1.1  jmcneill 		    ETHER_ADDR_LEN);
   4994  1.1  jmcneill 	}
   4995  1.1  jmcneill 
   4996  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   4997  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   4998  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   4999  1.1  jmcneill 	if (error != 0) {
   5000  1.1  jmcneill 		printf("%s: set flow table entry timeout\n", DEVNAME(sc));
   5001  1.1  jmcneill 		goto free;
   5002  1.1  jmcneill 	}
   5003  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5004  1.1  jmcneill 		printf("%s: set flow table entry command corrupt\n",
   5005  1.1  jmcneill 		    DEVNAME(sc));
   5006  1.1  jmcneill 		goto free;
   5007  1.1  jmcneill 	}
   5008  1.1  jmcneill 
   5009  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5010  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5011  1.1  jmcneill 		printf("%s: set flow table entry failed (%x, %x)\n",
   5012  1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5013  1.1  jmcneill 		error = -1;
   5014  1.1  jmcneill 		goto free;
   5015  1.1  jmcneill 	}
   5016  1.1  jmcneill 
   5017  1.1  jmcneill free:
   5018  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5019  1.1  jmcneill 	return (error);
   5020  1.1  jmcneill }
   5021  1.1  jmcneill 
   5022  1.1  jmcneill static int
   5023  1.1  jmcneill mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
   5024  1.1  jmcneill {
   5025  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5026  1.1  jmcneill 	struct mcx_dmamem mxm;
   5027  1.1  jmcneill 	struct mcx_cmd_delete_flow_table_entry_in *in;
   5028  1.1  jmcneill 	struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
   5029  1.1  jmcneill 	struct mcx_cmd_delete_flow_table_entry_out *out;
   5030  1.1  jmcneill 	int error;
   5031  1.1  jmcneill 	int token;
   5032  1.1  jmcneill 
   5033  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5034  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5035  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
   5036  1.1  jmcneill 	    token);
   5037  1.1  jmcneill 
   5038  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5039  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
   5040  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5041  1.1  jmcneill 
   5042  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) != 0) {
   5043  1.1  jmcneill 		printf("%s: unable to allocate delete flow table entry mailbox\n",
   5044  1.1  jmcneill 		    DEVNAME(sc));
   5045  1.1  jmcneill 		return (-1);
   5046  1.1  jmcneill 	}
   5047  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5048  1.1  jmcneill 	mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
   5049  1.1  jmcneill 	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
   5050  1.1  jmcneill 	mbin->cmd_flow_index = htobe32(sc->sc_flow_group_start[group] + index);
   5051  1.1  jmcneill 
   5052  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 2);
   5053  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5054  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5055  1.1  jmcneill 	if (error != 0) {
   5056  1.1  jmcneill 		printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
   5057  1.1  jmcneill 		goto free;
   5058  1.1  jmcneill 	}
   5059  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5060  1.1  jmcneill 		printf("%s: delete flow table entry command corrupt\n",
   5061  1.1  jmcneill 		    DEVNAME(sc));
   5062  1.1  jmcneill 		goto free;
   5063  1.1  jmcneill 	}
   5064  1.1  jmcneill 
   5065  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5066  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5067  1.1  jmcneill 		printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
   5068  1.1  jmcneill 		    DEVNAME(sc), group, index, out->cmd_status,
   5069  1.1  jmcneill 		    be32toh(out->cmd_syndrome));
   5070  1.1  jmcneill 		error = -1;
   5071  1.1  jmcneill 		goto free;
   5072  1.1  jmcneill 	}
   5073  1.1  jmcneill 
   5074  1.1  jmcneill free:
   5075  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5076  1.1  jmcneill 	return (error);
   5077  1.1  jmcneill }
   5078  1.1  jmcneill 
   5079  1.1  jmcneill #if 0
   5080  1.1  jmcneill int
   5081  1.1  jmcneill mcx_dump_flow_table(struct mcx_softc *sc)
   5082  1.1  jmcneill {
   5083  1.1  jmcneill 	struct mcx_dmamem mxm;
   5084  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5085  1.1  jmcneill 	struct mcx_cmd_query_flow_table_in *in;
   5086  1.1  jmcneill 	struct mcx_cmd_query_flow_table_mb_in *mbin;
   5087  1.1  jmcneill 	struct mcx_cmd_query_flow_table_out *out;
   5088  1.1  jmcneill 	struct mcx_cmd_query_flow_table_mb_out *mbout;
   5089  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   5090  1.1  jmcneill 	int error;
   5091  1.1  jmcneill 	int i;
   5092  1.1  jmcneill 	uint8_t *dump;
   5093  1.1  jmcneill 
   5094  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5095  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5096  1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   5097  1.1  jmcneill 
   5098  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5099  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
   5100  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5101  1.1  jmcneill 
   5102  1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   5103  1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
   5104  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5105  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   5106  1.1  jmcneill 		printf(", unable to allocate query flow table mailboxes\n");
   5107  1.1  jmcneill 		return (-1);
   5108  1.1  jmcneill 	}
   5109  1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   5110  1.1  jmcneill 
   5111  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5112  1.1  jmcneill 	mbin->cmd_table_type = 0;
   5113  1.1  jmcneill 	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
   5114  1.1  jmcneill 
   5115  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5116  1.1  jmcneill 
   5117  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5118  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5119  1.1  jmcneill 	if (error != 0) {
   5120  1.1  jmcneill 		printf("%s: query flow table timeout\n", DEVNAME(sc));
   5121  1.1  jmcneill 		goto free;
   5122  1.1  jmcneill 	}
   5123  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   5124  1.1  jmcneill 	if (error != 0) {
   5125  1.1  jmcneill 		printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
   5126  1.1  jmcneill 		goto free;
   5127  1.1  jmcneill 	}
   5128  1.1  jmcneill 
   5129  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5130  1.1  jmcneill 	switch (out->cmd_status) {
   5131  1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   5132  1.1  jmcneill 		break;
   5133  1.1  jmcneill 	default:
   5134  1.1  jmcneill 		printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
   5135  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5136  1.1  jmcneill 		error = -1;
   5137  1.1  jmcneill 		goto free;
   5138  1.1  jmcneill 	}
   5139  1.1  jmcneill 
   5140  1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_mb_out *)
   5141  1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   5142  1.1  jmcneill 	dump = (uint8_t *)mbout + 8;
   5143  1.1  jmcneill 	for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
   5144  1.1  jmcneill 		printf("%.2x ", dump[i]);
   5145  1.1  jmcneill 		if (i % 16 == 15)
   5146  1.1  jmcneill 			printf("\n");
   5147  1.1  jmcneill 	}
   5148  1.1  jmcneill free:
   5149  1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   5150  1.1  jmcneill 	return (error);
   5151  1.1  jmcneill }
   5152  1.1  jmcneill int
   5153  1.1  jmcneill mcx_dump_flow_table_entry(struct mcx_softc *sc, int index)
   5154  1.1  jmcneill {
   5155  1.1  jmcneill 	struct mcx_dmamem mxm;
   5156  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5157  1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_in *in;
   5158  1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
   5159  1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_out *out;
   5160  1.1  jmcneill 	struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
   5161  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   5162  1.1  jmcneill 	int error;
   5163  1.1  jmcneill 	int i;
   5164  1.1  jmcneill 	uint8_t *dump;
   5165  1.1  jmcneill 
   5166  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5167  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5168  1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   5169  1.1  jmcneill 
   5170  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5171  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
   5172  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5173  1.1  jmcneill 
   5174  1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   5175  1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   5176  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5177  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   5178  1.1  jmcneill 		printf(", unable to allocate query flow table entry mailboxes\n");
   5179  1.1  jmcneill 		return (-1);
   5180  1.1  jmcneill 	}
   5181  1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   5182  1.1  jmcneill 
   5183  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5184  1.1  jmcneill 	mbin->cmd_table_type = 0;
   5185  1.1  jmcneill 	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
   5186  1.1  jmcneill 	mbin->cmd_flow_index = htobe32(index);
   5187  1.1  jmcneill 
   5188  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5189  1.1  jmcneill 
   5190  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5191  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5192  1.1  jmcneill 	if (error != 0) {
   5193  1.1  jmcneill 		printf("%s: query flow table entry timeout\n", DEVNAME(sc));
   5194  1.1  jmcneill 		goto free;
   5195  1.1  jmcneill 	}
   5196  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   5197  1.1  jmcneill 	if (error != 0) {
   5198  1.1  jmcneill 		printf("%s: query flow table entry reply corrupt\n",
   5199  1.1  jmcneill 		    DEVNAME(sc));
   5200  1.1  jmcneill 		goto free;
   5201  1.1  jmcneill 	}
   5202  1.1  jmcneill 
   5203  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5204  1.1  jmcneill 	switch (out->cmd_status) {
   5205  1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   5206  1.1  jmcneill 		break;
   5207  1.1  jmcneill 	default:
   5208  1.1  jmcneill 		printf("%s: query flow table entry failed (%x/%x)\n",
   5209  1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
   5210  1.1  jmcneill 		error = -1;
   5211  1.1  jmcneill 		goto free;
   5212  1.1  jmcneill 	}
   5213  1.1  jmcneill 
   5214  1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
   5215  1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   5216  1.1  jmcneill 	dump = (uint8_t *)mbout;
   5217  1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   5218  1.1  jmcneill 		printf("%.2x ", dump[i]);
   5219  1.1  jmcneill 		if (i % 16 == 15)
   5220  1.1  jmcneill 			printf("\n");
   5221  1.1  jmcneill 	}
   5222  1.1  jmcneill 
   5223  1.1  jmcneill free:
   5224  1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   5225  1.1  jmcneill 	return (error);
   5226  1.1  jmcneill }
   5227  1.1  jmcneill 
   5228  1.1  jmcneill int
   5229  1.1  jmcneill mcx_dump_flow_group(struct mcx_softc *sc)
   5230  1.1  jmcneill {
   5231  1.1  jmcneill 	struct mcx_dmamem mxm;
   5232  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5233  1.1  jmcneill 	struct mcx_cmd_query_flow_group_in *in;
   5234  1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_in *mbin;
   5235  1.1  jmcneill 	struct mcx_cmd_query_flow_group_out *out;
   5236  1.1  jmcneill 	struct mcx_cmd_query_flow_group_mb_out *mbout;
   5237  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   5238  1.1  jmcneill 	int error;
   5239  1.1  jmcneill 	int i;
   5240  1.1  jmcneill 	uint8_t *dump;
   5241  1.1  jmcneill 
   5242  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5243  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5244  1.1  jmcneill 	    sizeof(*out) + sizeof(*mbout) + 16, token);
   5245  1.1  jmcneill 
   5246  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5247  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
   5248  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5249  1.1  jmcneill 
   5250  1.1  jmcneill 	CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
   5251  1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   5252  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5253  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   5254  1.1  jmcneill 		printf(", unable to allocate query flow group mailboxes\n");
   5255  1.1  jmcneill 		return (-1);
   5256  1.1  jmcneill 	}
   5257  1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   5258  1.1  jmcneill 
   5259  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5260  1.1  jmcneill 	mbin->cmd_table_type = 0;
   5261  1.1  jmcneill 	mbin->cmd_table_id = htobe32(sc->sc_flow_table_id);
   5262  1.1  jmcneill 	mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
   5263  1.1  jmcneill 
   5264  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5265  1.1  jmcneill 
   5266  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5267  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5268  1.1  jmcneill 	if (error != 0) {
   5269  1.1  jmcneill 		printf("%s: query flow group timeout\n", DEVNAME(sc));
   5270  1.1  jmcneill 		goto free;
   5271  1.1  jmcneill 	}
   5272  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   5273  1.1  jmcneill 	if (error != 0) {
   5274  1.1  jmcneill 		printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
   5275  1.1  jmcneill 		goto free;
   5276  1.1  jmcneill 	}
   5277  1.1  jmcneill 
   5278  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5279  1.1  jmcneill 	switch (out->cmd_status) {
   5280  1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   5281  1.1  jmcneill 		break;
   5282  1.1  jmcneill 	default:
   5283  1.1  jmcneill 		printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
   5284  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5285  1.1  jmcneill 		error = -1;
   5286  1.1  jmcneill 		goto free;
   5287  1.1  jmcneill 	}
   5288  1.1  jmcneill 
   5289  1.1  jmcneill         mbout = (struct mcx_cmd_query_flow_group_mb_out *)
   5290  1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   5291  1.1  jmcneill 	dump = (uint8_t *)mbout;
   5292  1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   5293  1.1  jmcneill 		printf("%.2x ", dump[i]);
   5294  1.1  jmcneill 		if (i % 16 == 15)
   5295  1.1  jmcneill 			printf("\n");
   5296  1.1  jmcneill 	}
   5297  1.1  jmcneill 	dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
   5298  1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   5299  1.1  jmcneill 		printf("%.2x ", dump[i]);
   5300  1.1  jmcneill 		if (i % 16 == 15)
   5301  1.1  jmcneill 			printf("\n");
   5302  1.1  jmcneill 	}
   5303  1.1  jmcneill 
   5304  1.1  jmcneill free:
   5305  1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   5306  1.1  jmcneill 	return (error);
   5307  1.1  jmcneill }
   5308  1.1  jmcneill 
   5309  1.1  jmcneill int
   5310  1.1  jmcneill mcx_dump_rq(struct mcx_softc *sc)
   5311  1.1  jmcneill {
   5312  1.1  jmcneill 	struct mcx_dmamem mxm;
   5313  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5314  1.1  jmcneill 	struct mcx_cmd_query_rq_in *in;
   5315  1.1  jmcneill 	struct mcx_cmd_query_rq_out *out;
   5316  1.1  jmcneill 	struct mcx_cmd_query_rq_mb_out *mbout;
   5317  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   5318  1.1  jmcneill 	int error;
   5319  1.1  jmcneill 
   5320  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5321  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   5322  1.1  jmcneill 	    token);
   5323  1.1  jmcneill 
   5324  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5325  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
   5326  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5327  1.1  jmcneill 	in->cmd_rqn = htobe32(sc->sc_rqn);
   5328  1.1  jmcneill 
   5329  1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   5330  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5331  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   5332  1.1  jmcneill 		printf(", unable to allocate query flow group mailboxes\n");
   5333  1.1  jmcneill 		return (-1);
   5334  1.1  jmcneill 	}
   5335  1.1  jmcneill 
   5336  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5337  1.1  jmcneill 
   5338  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5339  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5340  1.1  jmcneill 	if (error != 0) {
   5341  1.1  jmcneill 		printf("%s: query rq timeout\n", DEVNAME(sc));
   5342  1.1  jmcneill 		goto free;
   5343  1.1  jmcneill 	}
   5344  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   5345  1.1  jmcneill 	if (error != 0) {
   5346  1.1  jmcneill 		printf("%s: query rq reply corrupt\n", DEVNAME(sc));
   5347  1.1  jmcneill 		goto free;
   5348  1.1  jmcneill 	}
   5349  1.1  jmcneill 
   5350  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5351  1.1  jmcneill 	switch (out->cmd_status) {
   5352  1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   5353  1.1  jmcneill 		break;
   5354  1.1  jmcneill 	default:
   5355  1.1  jmcneill 		printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
   5356  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5357  1.1  jmcneill 		error = -1;
   5358  1.1  jmcneill 		goto free;
   5359  1.1  jmcneill 	}
   5360  1.1  jmcneill 
   5361  1.1  jmcneill         mbout = (struct mcx_cmd_query_rq_mb_out *)
   5362  1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   5363  1.1  jmcneill 	printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n",
   5364  1.1  jmcneill 	    DEVNAME(sc),
   5365  1.1  jmcneill 	    (be32toh(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f,
   5366  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_user_index),
   5367  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_cqn),
   5368  1.1  jmcneill 	    be16toh(mbout->cmd_ctx.rq_wq.wq_log_stride),
   5369  1.1  jmcneill 	    mbout->cmd_ctx.rq_wq.wq_log_page_sz,
   5370  1.1  jmcneill 	    mbout->cmd_ctx.rq_wq.wq_log_size,
   5371  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_wq.wq_hw_counter),
   5372  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_wq.wq_sw_counter));
   5373  1.1  jmcneill 
   5374  1.1  jmcneill free:
   5375  1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   5376  1.1  jmcneill 	return (error);
   5377  1.1  jmcneill }
   5378  1.1  jmcneill 
   5379  1.1  jmcneill int
   5380  1.1  jmcneill mcx_dump_sq(struct mcx_softc *sc)
   5381  1.1  jmcneill {
   5382  1.1  jmcneill 	struct mcx_dmamem mxm;
   5383  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5384  1.1  jmcneill 	struct mcx_cmd_query_sq_in *in;
   5385  1.1  jmcneill 	struct mcx_cmd_query_sq_out *out;
   5386  1.1  jmcneill 	struct mcx_cmd_query_sq_mb_out *mbout;
   5387  1.1  jmcneill 	uint8_t token = mcx_cmdq_token(sc);
   5388  1.1  jmcneill 	int error;
   5389  1.1  jmcneill 	int i;
   5390  1.1  jmcneill 	uint8_t *dump;
   5391  1.1  jmcneill 
   5392  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5393  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
   5394  1.1  jmcneill 	    token);
   5395  1.1  jmcneill 
   5396  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5397  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
   5398  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5399  1.1  jmcneill 	in->cmd_sqn = htobe32(sc->sc_sqn);
   5400  1.1  jmcneill 
   5401  1.1  jmcneill 	CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
   5402  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
   5403  1.1  jmcneill 	    &cqe->cq_output_ptr, token) != 0) {
   5404  1.1  jmcneill 		printf(", unable to allocate query sq mailboxes\n");
   5405  1.1  jmcneill 		return (-1);
   5406  1.1  jmcneill 	}
   5407  1.1  jmcneill 
   5408  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5409  1.1  jmcneill 
   5410  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5411  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5412  1.1  jmcneill 	if (error != 0) {
   5413  1.1  jmcneill 		printf("%s: query sq timeout\n", DEVNAME(sc));
   5414  1.1  jmcneill 		goto free;
   5415  1.1  jmcneill 	}
   5416  1.1  jmcneill 	error = mcx_cmdq_verify(cqe);
   5417  1.1  jmcneill 	if (error != 0) {
   5418  1.1  jmcneill 		printf("%s: query sq reply corrupt\n", DEVNAME(sc));
   5419  1.1  jmcneill 		goto free;
   5420  1.1  jmcneill 	}
   5421  1.1  jmcneill 
   5422  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5423  1.1  jmcneill 	switch (out->cmd_status) {
   5424  1.1  jmcneill 	case MCX_CQ_STATUS_OK:
   5425  1.1  jmcneill 		break;
   5426  1.1  jmcneill 	default:
   5427  1.1  jmcneill 		printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
   5428  1.1  jmcneill 		    out->cmd_status, be32toh(out->cmd_syndrome));
   5429  1.1  jmcneill 		error = -1;
   5430  1.1  jmcneill 		goto free;
   5431  1.1  jmcneill 	}
   5432  1.1  jmcneill 
   5433  1.1  jmcneill         mbout = (struct mcx_cmd_query_sq_mb_out *)
   5434  1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   5435  1.1  jmcneill /*
   5436  1.1  jmcneill 	printf("%s: rq: state %d, ui %d, cqn %d, s/s %d/%d/%d, hw %d, sw %d\n",
   5437  1.1  jmcneill 	    DEVNAME(sc),
   5438  1.1  jmcneill 	    (be32toh(mbout->cmd_ctx.rq_flags) >> MCX_RQ_CTX_STATE_SHIFT) & 0x0f,
   5439  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_user_index),
   5440  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_cqn),
   5441  1.1  jmcneill 	    be16toh(mbout->cmd_ctx.rq_wq.wq_log_stride),
   5442  1.1  jmcneill 	    mbout->cmd_ctx.rq_wq.wq_log_page_sz,
   5443  1.1  jmcneill 	    mbout->cmd_ctx.rq_wq.wq_log_size,
   5444  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_wq.wq_hw_counter),
   5445  1.1  jmcneill 	    be32toh(mbout->cmd_ctx.rq_wq.wq_sw_counter));
   5446  1.1  jmcneill */
   5447  1.1  jmcneill 	dump = (uint8_t *)mbout;
   5448  1.1  jmcneill 	for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
   5449  1.1  jmcneill 		printf("%.2x ", dump[i]);
   5450  1.1  jmcneill 		if (i % 16 == 15)
   5451  1.1  jmcneill 			printf("\n");
   5452  1.1  jmcneill 	}
   5453  1.1  jmcneill 
   5454  1.1  jmcneill free:
   5455  1.1  jmcneill 	mcx_cq_mboxes_free(sc, &mxm);
   5456  1.1  jmcneill 	return (error);
   5457  1.1  jmcneill }
   5458  1.1  jmcneill 
   5459  1.1  jmcneill static int
   5460  1.1  jmcneill mcx_dump_counters(struct mcx_softc *sc)
   5461  1.1  jmcneill {
   5462  1.1  jmcneill 	struct mcx_dmamem mxm;
   5463  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5464  1.1  jmcneill 	struct mcx_cmd_query_vport_counters_in *in;
   5465  1.1  jmcneill 	struct mcx_cmd_query_vport_counters_mb_in *mbin;
   5466  1.1  jmcneill 	struct mcx_cmd_query_vport_counters_out *out;
   5467  1.1  jmcneill 	struct mcx_nic_vport_counters *counters;
   5468  1.1  jmcneill 	int error, token;
   5469  1.1  jmcneill 
   5470  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5471  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5472  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
   5473  1.1  jmcneill 	    sizeof(*out) + sizeof(*counters), token);
   5474  1.1  jmcneill 
   5475  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5476  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
   5477  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5478  1.1  jmcneill 
   5479  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) {
   5480  1.1  jmcneill 		printf(", unable to allocate query nic vport counters mailboxen\n");
   5481  1.1  jmcneill 		return (-1);
   5482  1.1  jmcneill 	}
   5483  1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   5484  1.1  jmcneill 
   5485  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5486  1.1  jmcneill 	mbin->cmd_clear = 0x80;
   5487  1.1  jmcneill 
   5488  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5489  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5490  1.1  jmcneill 
   5491  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5492  1.1  jmcneill 	if (error != 0) {
   5493  1.1  jmcneill 		printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
   5494  1.1  jmcneill 		goto free;
   5495  1.1  jmcneill 	}
   5496  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5497  1.1  jmcneill 		printf("%s: query nic vport counters command corrupt\n",
   5498  1.1  jmcneill 		    DEVNAME(sc));
   5499  1.1  jmcneill 		goto free;
   5500  1.1  jmcneill 	}
   5501  1.1  jmcneill 
   5502  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5503  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5504  1.1  jmcneill 		printf("%s: query nic vport counters failed (%x, %x)\n",
   5505  1.1  jmcneill 		    DEVNAME(sc), out->cmd_status, out->cmd_syndrome);
   5506  1.1  jmcneill 		error = -1;
   5507  1.1  jmcneill 		goto free;
   5508  1.1  jmcneill 	}
   5509  1.1  jmcneill 
   5510  1.1  jmcneill 	counters = (struct mcx_nic_vport_counters *)
   5511  1.1  jmcneill 	    (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   5512  1.1  jmcneill 	if (counters->rx_bcast.packets + counters->tx_bcast.packets +
   5513  1.1  jmcneill 	    counters->rx_ucast.packets + counters->tx_ucast.packets +
   5514  1.1  jmcneill 	    counters->rx_err.packets + counters->tx_err.packets)
   5515  1.1  jmcneill 		printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
   5516  1.1  jmcneill 		    DEVNAME(sc),
   5517  1.1  jmcneill 		    be64toh(counters->tx_err.packets),
   5518  1.1  jmcneill 		    be64toh(counters->rx_err.packets),
   5519  1.1  jmcneill 		    be64toh(counters->tx_ucast.packets),
   5520  1.1  jmcneill 		    be64toh(counters->rx_ucast.packets),
   5521  1.1  jmcneill 		    be64toh(counters->tx_bcast.packets),
   5522  1.1  jmcneill 		    be64toh(counters->rx_bcast.packets));
   5523  1.1  jmcneill free:
   5524  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5525  1.1  jmcneill 
   5526  1.1  jmcneill 	return (error);
   5527  1.1  jmcneill }
   5528  1.1  jmcneill 
   5529  1.1  jmcneill static int
   5530  1.1  jmcneill mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
   5531  1.1  jmcneill {
   5532  1.1  jmcneill 	struct mcx_dmamem mxm;
   5533  1.1  jmcneill 	struct mcx_cmdq_entry *cqe;
   5534  1.1  jmcneill 	struct mcx_cmd_query_flow_counter_in *in;
   5535  1.1  jmcneill 	struct mcx_cmd_query_flow_counter_mb_in *mbin;
   5536  1.1  jmcneill 	struct mcx_cmd_query_flow_counter_out *out;
   5537  1.1  jmcneill 	struct mcx_counter *counters;
   5538  1.1  jmcneill 	int error, token;
   5539  1.1  jmcneill 
   5540  1.1  jmcneill 	cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
   5541  1.1  jmcneill 	token = mcx_cmdq_token(sc);
   5542  1.1  jmcneill 	mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
   5543  1.1  jmcneill 	    sizeof(*counters), token);
   5544  1.1  jmcneill 
   5545  1.1  jmcneill 	in = mcx_cmdq_in(cqe);
   5546  1.1  jmcneill 	in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
   5547  1.1  jmcneill 	in->cmd_op_mod = htobe16(0);
   5548  1.1  jmcneill 
   5549  1.1  jmcneill 	if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_output_ptr, token) != 0) {
   5550  1.1  jmcneill 		printf(", unable to allocate query flow counter mailboxen\n");
   5551  1.1  jmcneill 		return (-1);
   5552  1.1  jmcneill 	}
   5553  1.1  jmcneill 	cqe->cq_input_ptr = cqe->cq_output_ptr;
   5554  1.1  jmcneill 	mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
   5555  1.1  jmcneill 	mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
   5556  1.1  jmcneill 	mbin->cmd_clear = 0x80;
   5557  1.1  jmcneill 
   5558  1.1  jmcneill 	mcx_cmdq_mboxes_sign(&mxm, 1);
   5559  1.1  jmcneill 	mcx_cmdq_post(sc, cqe, 0);
   5560  1.1  jmcneill 
   5561  1.1  jmcneill 	error = mcx_cmdq_poll(sc, cqe, 1000);
   5562  1.1  jmcneill 	if (error != 0) {
   5563  1.1  jmcneill 		printf("%s: query flow counter timeout\n", DEVNAME(sc));
   5564  1.1  jmcneill 		goto free;
   5565  1.1  jmcneill 	}
   5566  1.1  jmcneill 	if (mcx_cmdq_verify(cqe) != 0) {
   5567  1.1  jmcneill 		printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
   5568  1.1  jmcneill 		goto free;
   5569  1.1  jmcneill 	}
   5570  1.1  jmcneill 
   5571  1.1  jmcneill 	out = mcx_cmdq_out(cqe);
   5572  1.1  jmcneill 	if (out->cmd_status != MCX_CQ_STATUS_OK) {
   5573  1.1  jmcneill 		printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
   5574  1.1  jmcneill 		    out->cmd_status, out->cmd_syndrome);
   5575  1.1  jmcneill 		error = -1;
   5576  1.1  jmcneill 		goto free;
   5577  1.1  jmcneill 	}
   5578  1.1  jmcneill 
   5579  1.1  jmcneill 	counters = (struct mcx_counter *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
   5580  1.1  jmcneill 	if (counters->packets)
   5581  1.1  jmcneill 		printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
   5582  1.1  jmcneill 		    be64toh(counters->packets));
   5583  1.1  jmcneill free:
   5584  1.1  jmcneill 	mcx_dmamem_free(sc, &mxm);
   5585  1.1  jmcneill 
   5586  1.1  jmcneill 	return (error);
   5587  1.1  jmcneill }
   5588  1.1  jmcneill 
   5589  1.1  jmcneill #endif
   5590  1.1  jmcneill 
   5591  1.1  jmcneill static int
   5592  1.1  jmcneill mcx_rx_fill_slots(struct mcx_softc *sc, void *ring, struct mcx_slot *slots,
   5593  1.1  jmcneill     uint *prod, int bufsize, uint nslots)
   5594  1.1  jmcneill {
   5595  1.1  jmcneill 	struct mcx_rq_entry *rqe;
   5596  1.1  jmcneill 	struct mcx_slot *ms;
   5597  1.1  jmcneill 	struct mbuf *m;
   5598  1.1  jmcneill 	uint slot, p, fills;
   5599  1.1  jmcneill 
   5600  1.1  jmcneill 	p = *prod;
   5601  1.1  jmcneill 	slot = (p % (1 << MCX_LOG_RQ_SIZE));
   5602  1.1  jmcneill 	rqe = ring;
   5603  1.1  jmcneill 	for (fills = 0; fills < nslots; fills++) {
   5604  1.1  jmcneill 		ms = &slots[slot];
   5605  1.1  jmcneill #if 0
   5606  1.1  jmcneill 		m = MCLGETI(NULL, M_DONTWAIT, NULL, bufsize + ETHER_ALIGN);
   5607  1.1  jmcneill 		if (m == NULL)
   5608  1.1  jmcneill 			break;
   5609  1.1  jmcneill #else
   5610  1.1  jmcneill 		m = NULL;
   5611  1.1  jmcneill 		MGETHDR(m, M_DONTWAIT, MT_DATA);
   5612  1.1  jmcneill 		if (m == NULL)
   5613  1.1  jmcneill 			break;
   5614  1.1  jmcneill 
   5615  1.1  jmcneill 		MCLGET(m, M_DONTWAIT);
   5616  1.1  jmcneill 		if ((m->m_flags & M_EXT) == 0) {
   5617  1.1  jmcneill 			m_freem(m);
   5618  1.1  jmcneill 			break;
   5619  1.1  jmcneill 		}
   5620  1.1  jmcneill #endif
   5621  1.1  jmcneill 
   5622  1.1  jmcneill 		m->m_data += ETHER_ALIGN;
   5623  1.1  jmcneill 		m->m_len = m->m_pkthdr.len = bufsize;
   5624  1.1  jmcneill 		if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   5625  1.1  jmcneill 		    BUS_DMA_NOWAIT) != 0) {
   5626  1.1  jmcneill 			m_freem(m);
   5627  1.1  jmcneill 			break;
   5628  1.1  jmcneill 		}
   5629  1.1  jmcneill 		ms->ms_m = m;
   5630  1.1  jmcneill 
   5631  1.1  jmcneill 		rqe[slot].rqe_byte_count = htobe32(bufsize);
   5632  1.1  jmcneill 		rqe[slot].rqe_addr = htobe64(ms->ms_map->dm_segs[0].ds_addr);
   5633  1.1  jmcneill 		rqe[slot].rqe_lkey = htobe32(sc->sc_lkey);
   5634  1.1  jmcneill 
   5635  1.1  jmcneill 		p++;
   5636  1.1  jmcneill 		slot++;
   5637  1.1  jmcneill 		if (slot == (1 << MCX_LOG_RQ_SIZE))
   5638  1.1  jmcneill 			slot = 0;
   5639  1.1  jmcneill 	}
   5640  1.1  jmcneill 
   5641  1.1  jmcneill 	if (fills != 0) {
   5642  1.1  jmcneill 		*sc->sc_rx_doorbell = htobe32(p & MCX_WQ_DOORBELL_MASK);
   5643  1.1  jmcneill 		/* barrier? */
   5644  1.1  jmcneill 	}
   5645  1.1  jmcneill 
   5646  1.1  jmcneill 	*prod = p;
   5647  1.1  jmcneill 
   5648  1.1  jmcneill 	return (nslots - fills);
   5649  1.1  jmcneill }
   5650  1.1  jmcneill 
   5651  1.1  jmcneill static int
   5652  1.1  jmcneill mcx_rx_fill(struct mcx_softc *sc)
   5653  1.1  jmcneill {
   5654  1.1  jmcneill 	u_int slots;
   5655  1.1  jmcneill 
   5656  1.1  jmcneill 	slots = mcx_rxr_get(&sc->sc_rxr, (1 << MCX_LOG_RQ_SIZE));
   5657  1.1  jmcneill 	if (slots == 0)
   5658  1.1  jmcneill 		return (1);
   5659  1.1  jmcneill 
   5660  1.1  jmcneill 	slots = mcx_rx_fill_slots(sc, MCX_DMA_KVA(&sc->sc_rq_mem),
   5661  1.1  jmcneill 	    sc->sc_rx_slots, &sc->sc_rx_prod, sc->sc_hardmtu, slots);
   5662  1.1  jmcneill 	mcx_rxr_put(&sc->sc_rxr, slots);
   5663  1.1  jmcneill 	return (0);
   5664  1.1  jmcneill }
   5665  1.1  jmcneill 
   5666  1.1  jmcneill void
   5667  1.1  jmcneill mcx_refill(void *xsc)
   5668  1.1  jmcneill {
   5669  1.1  jmcneill 	struct mcx_softc *sc = xsc;
   5670  1.1  jmcneill 
   5671  1.1  jmcneill 	mcx_rx_fill(sc);
   5672  1.1  jmcneill 
   5673  1.1  jmcneill 	if (mcx_rxr_inuse(&sc->sc_rxr) == 0)
   5674  1.1  jmcneill 		callout_schedule(&sc->sc_rx_refill, 1);
   5675  1.1  jmcneill }
   5676  1.1  jmcneill 
   5677  1.1  jmcneill void
   5678  1.1  jmcneill mcx_process_txeof(struct mcx_softc *sc, struct mcx_cq_entry *cqe, int *txfree)
   5679  1.1  jmcneill {
   5680  1.1  jmcneill 	struct mcx_slot *ms;
   5681  1.1  jmcneill 	bus_dmamap_t map;
   5682  1.1  jmcneill 	int slot, slots;
   5683  1.1  jmcneill 
   5684  1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
   5685  1.1  jmcneill 
   5686  1.1  jmcneill 	ms = &sc->sc_tx_slots[slot];
   5687  1.1  jmcneill 	map = ms->ms_map;
   5688  1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   5689  1.1  jmcneill 	    BUS_DMASYNC_POSTWRITE);
   5690  1.1  jmcneill 
   5691  1.1  jmcneill 	slots = 1;
   5692  1.1  jmcneill 	if (map->dm_nsegs > 1)
   5693  1.1  jmcneill 		slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;
   5694  1.1  jmcneill 
   5695  1.1  jmcneill 	(*txfree) += slots;
   5696  1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, map);
   5697  1.1  jmcneill 	m_freem(ms->ms_m);
   5698  1.1  jmcneill 	ms->ms_m = NULL;
   5699  1.1  jmcneill }
   5700  1.1  jmcneill 
   5701  1.1  jmcneill static uint64_t
   5702  1.1  jmcneill mcx_uptime(void)
   5703  1.1  jmcneill {
   5704  1.1  jmcneill 	struct timespec ts;
   5705  1.1  jmcneill 
   5706  1.1  jmcneill 	nanouptime(&ts);
   5707  1.1  jmcneill 
   5708  1.1  jmcneill 	return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec);
   5709  1.1  jmcneill }
   5710  1.1  jmcneill 
   5711  1.1  jmcneill static void
   5712  1.1  jmcneill mcx_calibrate_first(struct mcx_softc *sc)
   5713  1.1  jmcneill {
   5714  1.1  jmcneill 	struct mcx_calibration *c = &sc->sc_calibration[0];
   5715  1.1  jmcneill 
   5716  1.1  jmcneill 	sc->sc_calibration_gen = 0;
   5717  1.1  jmcneill 
   5718  1.1  jmcneill 	c->c_ubase = mcx_uptime();
   5719  1.1  jmcneill 	c->c_tbase = mcx_timer(sc);
   5720  1.1  jmcneill 	c->c_tdiff = 0;
   5721  1.1  jmcneill 
   5722  1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz);
   5723  1.1  jmcneill }
   5724  1.1  jmcneill 
   5725  1.1  jmcneill #define MCX_TIMESTAMP_SHIFT 10
   5726  1.1  jmcneill 
   5727  1.1  jmcneill static void
   5728  1.1  jmcneill mcx_calibrate(void *arg)
   5729  1.1  jmcneill {
   5730  1.1  jmcneill 	struct mcx_softc *sc = arg;
   5731  1.1  jmcneill 	struct mcx_calibration *nc, *pc;
   5732  1.1  jmcneill 	unsigned int gen;
   5733  1.1  jmcneill 
   5734  1.1  jmcneill 	if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING))
   5735  1.1  jmcneill 		return;
   5736  1.1  jmcneill 
   5737  1.1  jmcneill 	callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz);
   5738  1.1  jmcneill 
   5739  1.1  jmcneill 	gen = sc->sc_calibration_gen;
   5740  1.1  jmcneill 	pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   5741  1.1  jmcneill 	gen++;
   5742  1.1  jmcneill 	nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   5743  1.1  jmcneill 
   5744  1.1  jmcneill 	nc->c_uptime = pc->c_ubase;
   5745  1.1  jmcneill 	nc->c_timestamp = pc->c_tbase;
   5746  1.1  jmcneill 
   5747  1.1  jmcneill 	nc->c_ubase = mcx_uptime();
   5748  1.1  jmcneill 	nc->c_tbase = mcx_timer(sc);
   5749  1.1  jmcneill 
   5750  1.1  jmcneill 	nc->c_udiff = (nc->c_ubase - nc->c_uptime) >> MCX_TIMESTAMP_SHIFT;
   5751  1.1  jmcneill 	nc->c_tdiff = (nc->c_tbase - nc->c_timestamp) >> MCX_TIMESTAMP_SHIFT;
   5752  1.1  jmcneill 
   5753  1.1  jmcneill 	membar_producer();
   5754  1.1  jmcneill 	sc->sc_calibration_gen = gen;
   5755  1.1  jmcneill }
   5756  1.1  jmcneill 
   5757  1.1  jmcneill static int
   5758  1.1  jmcneill mcx_process_rx(struct mcx_softc *sc, struct mcx_cq_entry *cqe,
   5759  1.1  jmcneill     struct mcx_mbufq *mq, const struct mcx_calibration *c)
   5760  1.1  jmcneill {
   5761  1.1  jmcneill 	struct mcx_slot *ms;
   5762  1.1  jmcneill 	struct mbuf *m;
   5763  1.1  jmcneill 	int slot;
   5764  1.1  jmcneill 
   5765  1.1  jmcneill 	slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
   5766  1.1  jmcneill 
   5767  1.1  jmcneill 	ms = &sc->sc_rx_slots[slot];
   5768  1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize,
   5769  1.1  jmcneill 	    BUS_DMASYNC_POSTREAD);
   5770  1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
   5771  1.1  jmcneill 
   5772  1.1  jmcneill 	m = ms->ms_m;
   5773  1.1  jmcneill 	ms->ms_m = NULL;
   5774  1.1  jmcneill 
   5775  1.1  jmcneill 	m_set_rcvif(m, &sc->sc_ec.ec_if);
   5776  1.1  jmcneill 	m->m_pkthdr.len = m->m_len = be32dec(&cqe->cq_byte_cnt);
   5777  1.1  jmcneill 
   5778  1.1  jmcneill #if 0
   5779  1.1  jmcneill 	if (cqe->cq_rx_hash_type) {
   5780  1.1  jmcneill 		m->m_pkthdr.ph_flowid = M_FLOWID_VALID |
   5781  1.1  jmcneill 		    be32toh(cqe->cq_rx_hash);
   5782  1.1  jmcneill 	}
   5783  1.1  jmcneill #endif
   5784  1.1  jmcneill 
   5785  1.1  jmcneill #if 0
   5786  1.1  jmcneill 	if (c->c_tdiff) {
   5787  1.1  jmcneill 		uint64_t t = be64dec(&cqe->cq_timestamp) - c->c_timestamp;
   5788  1.1  jmcneill 		t *= c->c_udiff;
   5789  1.1  jmcneill 		t /= c->c_tdiff;
   5790  1.1  jmcneill 
   5791  1.1  jmcneill 		m->m_pkthdr.ph_timestamp = c->c_uptime + t;
   5792  1.1  jmcneill 		SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
   5793  1.1  jmcneill 	}
   5794  1.1  jmcneill #endif
   5795  1.1  jmcneill 
   5796  1.1  jmcneill 	MBUFQ_ENQUEUE(mq, m);
   5797  1.1  jmcneill 
   5798  1.1  jmcneill 	return (1);
   5799  1.1  jmcneill }
   5800  1.1  jmcneill 
   5801  1.1  jmcneill static struct mcx_cq_entry *
   5802  1.1  jmcneill mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
   5803  1.1  jmcneill {
   5804  1.1  jmcneill 	struct mcx_cq_entry *cqe;
   5805  1.1  jmcneill 	int next;
   5806  1.1  jmcneill 
   5807  1.1  jmcneill 	cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
   5808  1.1  jmcneill 	next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
   5809  1.1  jmcneill 
   5810  1.1  jmcneill 	if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
   5811  1.1  jmcneill 	    ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
   5812  1.1  jmcneill 		return (&cqe[next]);
   5813  1.1  jmcneill 	}
   5814  1.1  jmcneill 
   5815  1.1  jmcneill 	return (NULL);
   5816  1.1  jmcneill }
   5817  1.1  jmcneill 
   5818  1.1  jmcneill static void
   5819  1.1  jmcneill mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq)
   5820  1.1  jmcneill {
   5821  1.1  jmcneill 	bus_size_t offset;
   5822  1.1  jmcneill 	uint32_t val;
   5823  1.1  jmcneill 	uint64_t uval;
   5824  1.1  jmcneill 
   5825  1.1  jmcneill 	/* different uar per cq? */
   5826  1.1  jmcneill 	offset = (MCX_PAGE_SIZE * sc->sc_uar);
   5827  1.1  jmcneill 	val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
   5828  1.1  jmcneill 	val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   5829  1.1  jmcneill 
   5830  1.1  jmcneill 	cq->cq_doorbell[0] = htobe32(cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
   5831  1.1  jmcneill 	cq->cq_doorbell[1] = htobe32(val);
   5832  1.1  jmcneill 
   5833  1.1  jmcneill 	uval = val;
   5834  1.1  jmcneill 	uval <<= 32;
   5835  1.1  jmcneill 	uval |= cq->cq_n;
   5836  1.1  jmcneill 	bus_space_write_8(sc->sc_memt, sc->sc_memh,
   5837  1.1  jmcneill 	    offset + MCX_UAR_CQ_DOORBELL, htobe64(uval));
   5838  1.1  jmcneill 	mcx_bar(sc, offset + MCX_UAR_CQ_DOORBELL, sizeof(uint64_t),
   5839  1.1  jmcneill 	    BUS_SPACE_BARRIER_WRITE);
   5840  1.1  jmcneill }
   5841  1.1  jmcneill 
   5842  1.1  jmcneill void
   5843  1.1  jmcneill mcx_process_cq(struct mcx_softc *sc, struct mcx_cq *cq)
   5844  1.1  jmcneill {
   5845  1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   5846  1.1  jmcneill 	const struct mcx_calibration *c;
   5847  1.1  jmcneill 	unsigned int gen;
   5848  1.1  jmcneill 	struct mcx_cq_entry *cqe;
   5849  1.1  jmcneill 	struct mcx_mbufq mq;
   5850  1.1  jmcneill 	struct mbuf *m;
   5851  1.1  jmcneill 	int rxfree, txfree;
   5852  1.1  jmcneill 
   5853  1.1  jmcneill 	MBUFQ_INIT(&mq);
   5854  1.1  jmcneill 
   5855  1.1  jmcneill 	gen = sc->sc_calibration_gen;
   5856  1.1  jmcneill 	membar_consumer();
   5857  1.1  jmcneill 	c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
   5858  1.1  jmcneill 
   5859  1.1  jmcneill 	rxfree = 0;
   5860  1.1  jmcneill 	txfree = 0;
   5861  1.1  jmcneill 	while ((cqe = mcx_next_cq_entry(sc, cq))) {
   5862  1.1  jmcneill 		uint8_t opcode;
   5863  1.1  jmcneill 		opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
   5864  1.1  jmcneill 		switch (opcode) {
   5865  1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ:
   5866  1.1  jmcneill 			mcx_process_txeof(sc, cqe, &txfree);
   5867  1.1  jmcneill 			break;
   5868  1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND:
   5869  1.1  jmcneill 			rxfree += mcx_process_rx(sc, cqe, &mq, c);
   5870  1.1  jmcneill 			break;
   5871  1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
   5872  1.1  jmcneill 		case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
   5873  1.1  jmcneill 			/* uint8_t *cqp = (uint8_t *)cqe; */
   5874  1.1  jmcneill 			/* printf("%s: cq completion error: %x\n", DEVNAME(sc), cqp[0x37]); */
   5875  1.1  jmcneill 			break;
   5876  1.1  jmcneill 
   5877  1.1  jmcneill 		default:
   5878  1.1  jmcneill 			/* printf("%s: cq completion opcode %x??\n", DEVNAME(sc), opcode); */
   5879  1.1  jmcneill 			break;
   5880  1.1  jmcneill 		}
   5881  1.1  jmcneill 
   5882  1.1  jmcneill 		cq->cq_cons++;
   5883  1.1  jmcneill 	}
   5884  1.1  jmcneill 
   5885  1.1  jmcneill 	cq->cq_count++;
   5886  1.1  jmcneill 	mcx_arm_cq(sc, cq);
   5887  1.1  jmcneill 
   5888  1.1  jmcneill 	if (rxfree > 0) {
   5889  1.1  jmcneill 		mcx_rxr_put(&sc->sc_rxr, rxfree);
   5890  1.1  jmcneill 		while (MBUFQ_FIRST(&mq) != NULL) {
   5891  1.1  jmcneill 			MBUFQ_DEQUEUE(&mq, m);
   5892  1.1  jmcneill 			if_percpuq_enqueue(ifp->if_percpuq, m);
   5893  1.1  jmcneill 		}
   5894  1.1  jmcneill 
   5895  1.1  jmcneill 		mcx_rx_fill(sc);
   5896  1.1  jmcneill 
   5897  1.1  jmcneill 		if (mcx_rxr_inuse(&sc->sc_rxr) == 0)
   5898  1.1  jmcneill 			callout_schedule(&sc->sc_rx_refill, 1);
   5899  1.1  jmcneill 	}
   5900  1.1  jmcneill 	if (txfree > 0) {
   5901  1.1  jmcneill 		sc->sc_tx_cons += txfree;
   5902  1.1  jmcneill 		if_schedule_deferred_start(ifp);
   5903  1.1  jmcneill 	}
   5904  1.1  jmcneill }
   5905  1.1  jmcneill 
   5906  1.1  jmcneill static void
   5907  1.1  jmcneill mcx_arm_eq(struct mcx_softc *sc)
   5908  1.1  jmcneill {
   5909  1.1  jmcneill 	bus_size_t offset;
   5910  1.1  jmcneill 	uint32_t val;
   5911  1.1  jmcneill 
   5912  1.1  jmcneill 	offset = (MCX_PAGE_SIZE * sc->sc_uar) + MCX_UAR_EQ_DOORBELL_ARM;
   5913  1.1  jmcneill 	val = (sc->sc_eqn << 24) | (sc->sc_eq_cons & 0xffffff);
   5914  1.1  jmcneill 
   5915  1.1  jmcneill 	mcx_wr(sc, offset, val);
   5916  1.1  jmcneill 	/* barrier? */
   5917  1.1  jmcneill }
   5918  1.1  jmcneill 
   5919  1.1  jmcneill static struct mcx_eq_entry *
   5920  1.1  jmcneill mcx_next_eq_entry(struct mcx_softc *sc)
   5921  1.1  jmcneill {
   5922  1.1  jmcneill 	struct mcx_eq_entry *eqe;
   5923  1.1  jmcneill 	int next;
   5924  1.1  jmcneill 
   5925  1.1  jmcneill 	eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&sc->sc_eq_mem);
   5926  1.1  jmcneill 	next = sc->sc_eq_cons % (1 << MCX_LOG_EQ_SIZE);
   5927  1.1  jmcneill 	if ((eqe[next].eq_owner & 1) == ((sc->sc_eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
   5928  1.1  jmcneill 		sc->sc_eq_cons++;
   5929  1.1  jmcneill 		return (&eqe[next]);
   5930  1.1  jmcneill 	}
   5931  1.1  jmcneill 	return (NULL);
   5932  1.1  jmcneill }
   5933  1.1  jmcneill 
   5934  1.1  jmcneill int
   5935  1.1  jmcneill mcx_intr(void *xsc)
   5936  1.1  jmcneill {
   5937  1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)xsc;
   5938  1.1  jmcneill 	struct mcx_eq_entry *eqe;
   5939  1.1  jmcneill 	int i, cq;
   5940  1.1  jmcneill 
   5941  1.1  jmcneill 	while ((eqe = mcx_next_eq_entry(sc))) {
   5942  1.1  jmcneill 		switch (eqe->eq_event_type) {
   5943  1.1  jmcneill 		case MCX_EVENT_TYPE_COMPLETION:
   5944  1.1  jmcneill 			cq = be32toh(eqe->eq_event_data[6]);
   5945  1.1  jmcneill 			for (i = 0; i < sc->sc_num_cq; i++) {
   5946  1.1  jmcneill 				if (sc->sc_cq[i].cq_n == cq) {
   5947  1.1  jmcneill 					mcx_process_cq(sc, &sc->sc_cq[i]);
   5948  1.1  jmcneill 					break;
   5949  1.1  jmcneill 				}
   5950  1.1  jmcneill 			}
   5951  1.1  jmcneill 			break;
   5952  1.1  jmcneill 
   5953  1.1  jmcneill 		case MCX_EVENT_TYPE_LAST_WQE:
   5954  1.1  jmcneill 			/* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
   5955  1.1  jmcneill 			break;
   5956  1.1  jmcneill 
   5957  1.1  jmcneill 		case MCX_EVENT_TYPE_CQ_ERROR:
   5958  1.1  jmcneill 			/* printf("%s: cq error\n", DEVNAME(sc)); */
   5959  1.1  jmcneill 			break;
   5960  1.1  jmcneill 
   5961  1.1  jmcneill 		case MCX_EVENT_TYPE_CMD_COMPLETION:
   5962  1.1  jmcneill 			/* wakeup probably */
   5963  1.1  jmcneill 			break;
   5964  1.1  jmcneill 
   5965  1.1  jmcneill 		case MCX_EVENT_TYPE_PORT_CHANGE:
   5966  1.1  jmcneill 			workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL);
   5967  1.1  jmcneill 			break;
   5968  1.1  jmcneill 
   5969  1.1  jmcneill 		default:
   5970  1.1  jmcneill 			/* printf("%s: something happened\n", DEVNAME(sc)); */
   5971  1.1  jmcneill 			break;
   5972  1.1  jmcneill 		}
   5973  1.1  jmcneill 	}
   5974  1.1  jmcneill 	mcx_arm_eq(sc);
   5975  1.1  jmcneill 	return (1);
   5976  1.1  jmcneill }
   5977  1.1  jmcneill 
   5978  1.1  jmcneill static void
   5979  1.1  jmcneill mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
   5980  1.1  jmcneill     int total)
   5981  1.1  jmcneill {
   5982  1.1  jmcneill 	struct mcx_slot *ms;
   5983  1.1  jmcneill 
   5984  1.1  jmcneill 	int i = allocated;
   5985  1.1  jmcneill 	while (i-- > 0) {
   5986  1.1  jmcneill 		ms = &slots[i];
   5987  1.1  jmcneill 		bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
   5988  1.1  jmcneill 		if (ms->ms_m != NULL)
   5989  1.1  jmcneill 			m_freem(ms->ms_m);
   5990  1.1  jmcneill 	}
   5991  1.1  jmcneill 	kmem_free(slots, total * sizeof(*ms));
   5992  1.1  jmcneill }
   5993  1.1  jmcneill 
   5994  1.1  jmcneill static int
   5995  1.1  jmcneill mcx_init(struct ifnet *ifp)
   5996  1.1  jmcneill {
   5997  1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   5998  1.1  jmcneill 	struct mcx_slot *ms;
   5999  1.1  jmcneill 	int i, start;
   6000  1.1  jmcneill 	struct mcx_flow_match match_crit;
   6001  1.1  jmcneill 
   6002  1.1  jmcneill 	if (ISSET(ifp->if_flags, IFF_RUNNING))
   6003  1.1  jmcneill 		mcx_stop(ifp, 0);
   6004  1.1  jmcneill 
   6005  1.1  jmcneill 	sc->sc_rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE),
   6006  1.1  jmcneill 	    KM_SLEEP);
   6007  1.1  jmcneill 
   6008  1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
   6009  1.1  jmcneill 		ms = &sc->sc_rx_slots[i];
   6010  1.1  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
   6011  1.1  jmcneill 		    sc->sc_hardmtu, 0,
   6012  1.1  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   6013  1.1  jmcneill 		    &ms->ms_map) != 0) {
   6014  1.1  jmcneill 			printf("%s: failed to allocate rx dma maps\n",
   6015  1.1  jmcneill 			    DEVNAME(sc));
   6016  1.1  jmcneill 			goto destroy_rx_slots;
   6017  1.1  jmcneill 		}
   6018  1.1  jmcneill 	}
   6019  1.1  jmcneill 
   6020  1.1  jmcneill 	sc->sc_tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE),
   6021  1.1  jmcneill 	     KM_SLEEP);
   6022  1.1  jmcneill 
   6023  1.1  jmcneill 	for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
   6024  1.1  jmcneill 		ms = &sc->sc_tx_slots[i];
   6025  1.1  jmcneill 		if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
   6026  1.1  jmcneill 		    MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
   6027  1.1  jmcneill 		    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   6028  1.1  jmcneill 		    &ms->ms_map) != 0) {
   6029  1.1  jmcneill 			printf("%s: failed to allocate tx dma maps\n",
   6030  1.1  jmcneill 			    DEVNAME(sc));
   6031  1.1  jmcneill 			goto destroy_tx_slots;
   6032  1.1  jmcneill 		}
   6033  1.1  jmcneill 	}
   6034  1.1  jmcneill 
   6035  1.1  jmcneill 	if (mcx_create_cq(sc, sc->sc_eqn) != 0)
   6036  1.1  jmcneill 		goto down;
   6037  1.1  jmcneill 
   6038  1.1  jmcneill 	/* send queue */
   6039  1.1  jmcneill 	if (mcx_create_tis(sc) != 0)
   6040  1.1  jmcneill 		goto down;
   6041  1.1  jmcneill 
   6042  1.1  jmcneill 	if (mcx_create_sq(sc, sc->sc_cq[0].cq_n) != 0)
   6043  1.1  jmcneill 		goto down;
   6044  1.1  jmcneill 
   6045  1.1  jmcneill 	/* receive queue */
   6046  1.1  jmcneill 	if (mcx_create_rq(sc, sc->sc_cq[0].cq_n) != 0)
   6047  1.1  jmcneill 		goto down;
   6048  1.1  jmcneill 
   6049  1.1  jmcneill 	if (mcx_create_tir(sc) != 0)
   6050  1.1  jmcneill 		goto down;
   6051  1.1  jmcneill 
   6052  1.1  jmcneill 	if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE) != 0)
   6053  1.1  jmcneill 		goto down;
   6054  1.1  jmcneill 
   6055  1.1  jmcneill 	/* promisc flow group */
   6056  1.1  jmcneill 	start = 0;
   6057  1.1  jmcneill 	memset(&match_crit, 0, sizeof(match_crit));
   6058  1.1  jmcneill 	if (mcx_create_flow_group(sc, MCX_FLOW_GROUP_PROMISC, start, 1,
   6059  1.1  jmcneill 	    0, &match_crit) != 0)
   6060  1.1  jmcneill 		goto down;
   6061  1.1  jmcneill 	sc->sc_promisc_flow_enabled = 0;
   6062  1.1  jmcneill 	start++;
   6063  1.1  jmcneill 
   6064  1.1  jmcneill 	/* all multicast flow group */
   6065  1.1  jmcneill 	match_crit.mc_dest_mac[0] = 0x01;
   6066  1.1  jmcneill 	if (mcx_create_flow_group(sc, MCX_FLOW_GROUP_ALLMULTI, start, 1,
   6067  1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   6068  1.1  jmcneill 		goto down;
   6069  1.1  jmcneill 	sc->sc_allmulti_flow_enabled = 0;
   6070  1.1  jmcneill 	start++;
   6071  1.1  jmcneill 
   6072  1.1  jmcneill 	/* mac address matching flow group */
   6073  1.1  jmcneill 	memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
   6074  1.1  jmcneill 	if (mcx_create_flow_group(sc, MCX_FLOW_GROUP_MAC, start,
   6075  1.1  jmcneill 	    (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
   6076  1.1  jmcneill 	    MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
   6077  1.1  jmcneill 		goto down;
   6078  1.1  jmcneill 
   6079  1.1  jmcneill 	/* flow table entries for unicast and broadcast */
   6080  1.1  jmcneill 	start = 0;
   6081  1.1  jmcneill 	if (mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, start,
   6082  1.1  jmcneill 	    LLADDR(satosdl(ifp->if_dl->ifa_addr))) != 0)
   6083  1.1  jmcneill 		goto down;
   6084  1.1  jmcneill 	start++;
   6085  1.1  jmcneill 
   6086  1.1  jmcneill 	if (mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, start,
   6087  1.1  jmcneill 	    etherbroadcastaddr) != 0)
   6088  1.1  jmcneill 		goto down;
   6089  1.1  jmcneill 	start++;
   6090  1.1  jmcneill 
   6091  1.1  jmcneill 	/* multicast entries go after that */
   6092  1.1  jmcneill 	sc->sc_mcast_flow_base = start;
   6093  1.1  jmcneill 
   6094  1.1  jmcneill 	/* re-add any existing multicast flows */
   6095  1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   6096  1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   6097  1.1  jmcneill 			mcx_set_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
   6098  1.1  jmcneill 			    sc->sc_mcast_flow_base + i,
   6099  1.1  jmcneill 			    sc->sc_mcast_flows[i]);
   6100  1.1  jmcneill 		}
   6101  1.1  jmcneill 	}
   6102  1.1  jmcneill 
   6103  1.1  jmcneill 	if (mcx_set_flow_table_root(sc) != 0)
   6104  1.1  jmcneill 		goto down;
   6105  1.1  jmcneill 
   6106  1.1  jmcneill 	/* start the queues */
   6107  1.1  jmcneill 	if (mcx_ready_sq(sc) != 0)
   6108  1.1  jmcneill 		goto down;
   6109  1.1  jmcneill 
   6110  1.1  jmcneill 	if (mcx_ready_rq(sc) != 0)
   6111  1.1  jmcneill 		goto down;
   6112  1.1  jmcneill 
   6113  1.1  jmcneill 	mcx_rxr_init(&sc->sc_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
   6114  1.1  jmcneill 	sc->sc_rx_prod = 0;
   6115  1.1  jmcneill 	mcx_rx_fill(sc);
   6116  1.1  jmcneill 
   6117  1.1  jmcneill 	mcx_calibrate_first(sc);
   6118  1.1  jmcneill 
   6119  1.1  jmcneill 	SET(ifp->if_flags, IFF_RUNNING);
   6120  1.1  jmcneill 
   6121  1.1  jmcneill 	sc->sc_tx_cons = 0;
   6122  1.1  jmcneill 	sc->sc_tx_prod = 0;
   6123  1.1  jmcneill 	CLR(ifp->if_flags, IFF_OACTIVE);
   6124  1.1  jmcneill 	if_schedule_deferred_start(ifp);
   6125  1.1  jmcneill 
   6126  1.1  jmcneill 	return 0;
   6127  1.1  jmcneill destroy_tx_slots:
   6128  1.1  jmcneill 	mcx_free_slots(sc, sc->sc_tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
   6129  1.1  jmcneill 	sc->sc_rx_slots = NULL;
   6130  1.1  jmcneill 
   6131  1.1  jmcneill 	i = (1 << MCX_LOG_RQ_SIZE);
   6132  1.1  jmcneill destroy_rx_slots:
   6133  1.1  jmcneill 	mcx_free_slots(sc, sc->sc_rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
   6134  1.1  jmcneill 	sc->sc_rx_slots = NULL;
   6135  1.1  jmcneill down:
   6136  1.1  jmcneill 	mcx_stop(ifp, 0);
   6137  1.1  jmcneill 	return EIO;
   6138  1.1  jmcneill }
   6139  1.1  jmcneill 
   6140  1.1  jmcneill static void
   6141  1.1  jmcneill mcx_stop(struct ifnet *ifp, int disable)
   6142  1.1  jmcneill {
   6143  1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   6144  1.1  jmcneill 	int group, i;
   6145  1.1  jmcneill 
   6146  1.1  jmcneill 	CLR(ifp->if_flags, IFF_RUNNING);
   6147  1.1  jmcneill 
   6148  1.1  jmcneill 	/*
   6149  1.1  jmcneill 	 * delete flow table entries first, so no packets can arrive
   6150  1.1  jmcneill 	 * after the barriers
   6151  1.1  jmcneill 	 */
   6152  1.1  jmcneill 	if (sc->sc_promisc_flow_enabled)
   6153  1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
   6154  1.1  jmcneill 	if (sc->sc_allmulti_flow_enabled)
   6155  1.1  jmcneill 		mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
   6156  1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
   6157  1.1  jmcneill 	mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
   6158  1.1  jmcneill 	for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   6159  1.1  jmcneill 		if (sc->sc_mcast_flows[i][0] != 0) {
   6160  1.1  jmcneill 			mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
   6161  1.1  jmcneill 			    sc->sc_mcast_flow_base + i);
   6162  1.1  jmcneill 		}
   6163  1.1  jmcneill 	}
   6164  1.1  jmcneill 
   6165  1.1  jmcneill 	callout_halt(&sc->sc_calibrate, NULL);
   6166  1.1  jmcneill 
   6167  1.1  jmcneill 	for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
   6168  1.1  jmcneill 		if (sc->sc_flow_group_id[group] != -1)
   6169  1.1  jmcneill 			mcx_destroy_flow_group(sc,
   6170  1.1  jmcneill 			    sc->sc_flow_group_id[group]);
   6171  1.1  jmcneill 	}
   6172  1.1  jmcneill 
   6173  1.1  jmcneill 	if (sc->sc_flow_table_id != -1)
   6174  1.1  jmcneill 		mcx_destroy_flow_table(sc);
   6175  1.1  jmcneill 
   6176  1.1  jmcneill 	if (sc->sc_tirn != 0)
   6177  1.1  jmcneill 		mcx_destroy_tir(sc);
   6178  1.1  jmcneill 	if (sc->sc_rqn != 0)
   6179  1.1  jmcneill 		mcx_destroy_rq(sc);
   6180  1.1  jmcneill 
   6181  1.1  jmcneill 	if (sc->sc_sqn != 0)
   6182  1.1  jmcneill 		mcx_destroy_sq(sc);
   6183  1.1  jmcneill 	if (sc->sc_tisn != 0)
   6184  1.1  jmcneill 		mcx_destroy_tis(sc);
   6185  1.1  jmcneill 
   6186  1.1  jmcneill 	for (i = 0; i < sc->sc_num_cq; i++)
   6187  1.1  jmcneill 		mcx_destroy_cq(sc, i);
   6188  1.1  jmcneill 	sc->sc_num_cq = 0;
   6189  1.1  jmcneill 
   6190  1.1  jmcneill 	if (sc->sc_tx_slots != NULL) {
   6191  1.1  jmcneill 		mcx_free_slots(sc, sc->sc_tx_slots, (1 << MCX_LOG_SQ_SIZE),
   6192  1.1  jmcneill 		    (1 << MCX_LOG_SQ_SIZE));
   6193  1.1  jmcneill 		sc->sc_tx_slots = NULL;
   6194  1.1  jmcneill 	}
   6195  1.1  jmcneill 	if (sc->sc_rx_slots != NULL) {
   6196  1.1  jmcneill 		mcx_free_slots(sc, sc->sc_rx_slots, (1 << MCX_LOG_RQ_SIZE),
   6197  1.1  jmcneill 		    (1 << MCX_LOG_RQ_SIZE));
   6198  1.1  jmcneill 		sc->sc_rx_slots = NULL;
   6199  1.1  jmcneill 	}
   6200  1.1  jmcneill }
   6201  1.1  jmcneill 
   6202  1.1  jmcneill static int
   6203  1.1  jmcneill mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data)
   6204  1.1  jmcneill {
   6205  1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   6206  1.1  jmcneill 	struct ifreq *ifr = (struct ifreq *)data;
   6207  1.1  jmcneill 	uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
   6208  1.1  jmcneill 	int s, i, error = 0;
   6209  1.1  jmcneill 
   6210  1.1  jmcneill 	s = splnet();
   6211  1.1  jmcneill 	switch (cmd) {
   6212  1.1  jmcneill 
   6213  1.1  jmcneill 	case SIOCADDMULTI:
   6214  1.1  jmcneill 		if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   6215  1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   6216  1.1  jmcneill 			if (error != 0)
   6217  1.1  jmcneill 				return (error);
   6218  1.1  jmcneill 
   6219  1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   6220  1.1  jmcneill 				if (sc->sc_mcast_flows[i][0] == 0) {
   6221  1.1  jmcneill 					memcpy(sc->sc_mcast_flows[i], addrlo,
   6222  1.1  jmcneill 					    ETHER_ADDR_LEN);
   6223  1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   6224  1.1  jmcneill 						mcx_set_flow_table_entry(sc,
   6225  1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   6226  1.1  jmcneill 						    sc->sc_mcast_flow_base + i,
   6227  1.1  jmcneill 						    sc->sc_mcast_flows[i]);
   6228  1.1  jmcneill 					}
   6229  1.1  jmcneill 					break;
   6230  1.1  jmcneill 				}
   6231  1.1  jmcneill 			}
   6232  1.1  jmcneill 
   6233  1.1  jmcneill 			if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
   6234  1.1  jmcneill 				if (i == MCX_NUM_MCAST_FLOWS) {
   6235  1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   6236  1.1  jmcneill 					sc->sc_extra_mcast++;
   6237  1.1  jmcneill 					error = ENETRESET;
   6238  1.1  jmcneill 				}
   6239  1.1  jmcneill 
   6240  1.1  jmcneill 				if (sc->sc_ec.ec_multicnt > 0) {
   6241  1.1  jmcneill 					SET(ifp->if_flags, IFF_ALLMULTI);
   6242  1.1  jmcneill 					error = ENETRESET;
   6243  1.1  jmcneill 				}
   6244  1.1  jmcneill 			}
   6245  1.1  jmcneill 		}
   6246  1.1  jmcneill 		break;
   6247  1.1  jmcneill 
   6248  1.1  jmcneill 	case SIOCDELMULTI:
   6249  1.1  jmcneill 		if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
   6250  1.1  jmcneill 			error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
   6251  1.1  jmcneill 			if (error != 0)
   6252  1.1  jmcneill 				return (error);
   6253  1.1  jmcneill 
   6254  1.1  jmcneill 			for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
   6255  1.1  jmcneill 				if (memcmp(sc->sc_mcast_flows[i], addrlo,
   6256  1.1  jmcneill 				    ETHER_ADDR_LEN) == 0) {
   6257  1.1  jmcneill 					if (ISSET(ifp->if_flags, IFF_RUNNING)) {
   6258  1.1  jmcneill 						mcx_delete_flow_table_entry(sc,
   6259  1.1  jmcneill 						    MCX_FLOW_GROUP_MAC,
   6260  1.1  jmcneill 						    sc->sc_mcast_flow_base + i);
   6261  1.1  jmcneill 					}
   6262  1.1  jmcneill 					sc->sc_mcast_flows[i][0] = 0;
   6263  1.1  jmcneill 					break;
   6264  1.1  jmcneill 				}
   6265  1.1  jmcneill 			}
   6266  1.1  jmcneill 
   6267  1.1  jmcneill 			if (i == MCX_NUM_MCAST_FLOWS)
   6268  1.1  jmcneill 				sc->sc_extra_mcast--;
   6269  1.1  jmcneill 
   6270  1.1  jmcneill 			if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
   6271  1.1  jmcneill 			    (sc->sc_extra_mcast == 0) &&
   6272  1.1  jmcneill 			    (sc->sc_ec.ec_multicnt == 0)) {
   6273  1.1  jmcneill 				CLR(ifp->if_flags, IFF_ALLMULTI);
   6274  1.1  jmcneill 				error = ENETRESET;
   6275  1.1  jmcneill 			}
   6276  1.1  jmcneill 		}
   6277  1.1  jmcneill 		break;
   6278  1.1  jmcneill 
   6279  1.1  jmcneill 	default:
   6280  1.1  jmcneill 		error = ether_ioctl(ifp, cmd, data);
   6281  1.1  jmcneill 	}
   6282  1.1  jmcneill 
   6283  1.1  jmcneill 	if (error == ENETRESET) {
   6284  1.1  jmcneill 		if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
   6285  1.1  jmcneill 		    (IFF_UP | IFF_RUNNING))
   6286  1.1  jmcneill 			mcx_iff(sc);
   6287  1.1  jmcneill 		error = 0;
   6288  1.1  jmcneill 	}
   6289  1.1  jmcneill 	splx(s);
   6290  1.1  jmcneill 
   6291  1.1  jmcneill 	return (error);
   6292  1.1  jmcneill }
   6293  1.1  jmcneill 
   6294  1.1  jmcneill #if 0
   6295  1.1  jmcneill static int
   6296  1.1  jmcneill mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
   6297  1.1  jmcneill {
   6298  1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   6299  1.1  jmcneill 	struct mcx_reg_mcia mcia;
   6300  1.1  jmcneill 	struct mcx_reg_pmlp pmlp;
   6301  1.1  jmcneill 	int offset, error;
   6302  1.1  jmcneill 
   6303  1.1  jmcneill 	/* get module number */
   6304  1.1  jmcneill 	memset(&pmlp, 0, sizeof(pmlp));
   6305  1.1  jmcneill 	pmlp.rp_local_port = 1;
   6306  1.1  jmcneill 	error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
   6307  1.1  jmcneill 	    sizeof(pmlp));
   6308  1.1  jmcneill 	if (error != 0) {
   6309  1.1  jmcneill 		printf("%s: unable to get eeprom module number\n",
   6310  1.1  jmcneill 		    DEVNAME(sc));
   6311  1.1  jmcneill 		return error;
   6312  1.1  jmcneill 	}
   6313  1.1  jmcneill 
   6314  1.1  jmcneill 	for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
   6315  1.1  jmcneill 		memset(&mcia, 0, sizeof(mcia));
   6316  1.1  jmcneill 		mcia.rm_l = 0;
   6317  1.1  jmcneill 		mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) &
   6318  1.1  jmcneill 		    MCX_PMLP_MODULE_NUM_MASK;
   6319  1.1  jmcneill 		mcia.rm_i2c_addr = sff->sff_addr / 2;	/* apparently */
   6320  1.1  jmcneill 		mcia.rm_page_num = sff->sff_page;
   6321  1.1  jmcneill 		mcia.rm_dev_addr = htobe16(offset);
   6322  1.1  jmcneill 		mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);
   6323  1.1  jmcneill 
   6324  1.1  jmcneill 		error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
   6325  1.1  jmcneill 		    &mcia, sizeof(mcia));
   6326  1.1  jmcneill 		if (error != 0) {
   6327  1.1  jmcneill 			printf("%s: unable to read eeprom at %x\n",
   6328  1.1  jmcneill 			    DEVNAME(sc), offset);
   6329  1.1  jmcneill 			return error;
   6330  1.1  jmcneill 		}
   6331  1.1  jmcneill 
   6332  1.1  jmcneill 		memcpy(sff->sff_data + offset, mcia.rm_data,
   6333  1.1  jmcneill 		    MCX_MCIA_EEPROM_BYTES);
   6334  1.1  jmcneill 	}
   6335  1.1  jmcneill 
   6336  1.1  jmcneill 	return 0;
   6337  1.1  jmcneill }
   6338  1.1  jmcneill #endif
   6339  1.1  jmcneill 
   6340  1.1  jmcneill static int
   6341  1.1  jmcneill mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
   6342  1.1  jmcneill {
   6343  1.1  jmcneill 	switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   6344  1.1  jmcneill 	    BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
   6345  1.1  jmcneill 	case 0:
   6346  1.1  jmcneill 		break;
   6347  1.1  jmcneill 
   6348  1.1  jmcneill 	case EFBIG:
   6349  1.1  jmcneill 		if (m_defrag(m, M_DONTWAIT) == 0 &&
   6350  1.1  jmcneill 		    bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
   6351  1.1  jmcneill 		    BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
   6352  1.1  jmcneill 			break;
   6353  1.1  jmcneill 
   6354  1.1  jmcneill 		/* FALLTHROUGH */
   6355  1.1  jmcneill 	default:
   6356  1.1  jmcneill 		return (1);
   6357  1.1  jmcneill 	}
   6358  1.1  jmcneill 
   6359  1.1  jmcneill 	ms->ms_m = m;
   6360  1.1  jmcneill 	return (0);
   6361  1.1  jmcneill }
   6362  1.1  jmcneill 
   6363  1.1  jmcneill static void
   6364  1.1  jmcneill mcx_start(struct ifnet *ifp)
   6365  1.1  jmcneill {
   6366  1.1  jmcneill 	struct mcx_softc *sc = ifp->if_softc;
   6367  1.1  jmcneill 	struct mcx_sq_entry *sq, *sqe;
   6368  1.1  jmcneill 	struct mcx_sq_entry_seg *sqs;
   6369  1.1  jmcneill 	struct mcx_slot *ms;
   6370  1.1  jmcneill 	bus_dmamap_t map;
   6371  1.1  jmcneill 	struct mbuf *m;
   6372  1.1  jmcneill 	u_int idx, free, used;
   6373  1.1  jmcneill 	uint64_t *bf;
   6374  1.1  jmcneill 	size_t bf_base;
   6375  1.1  jmcneill 	int i, seg, nseg;
   6376  1.1  jmcneill 
   6377  1.1  jmcneill 	bf_base = (sc->sc_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
   6378  1.1  jmcneill 
   6379  1.1  jmcneill 	idx = sc->sc_tx_prod % (1 << MCX_LOG_SQ_SIZE);
   6380  1.1  jmcneill 	free = (sc->sc_tx_cons + (1 << MCX_LOG_SQ_SIZE)) - sc->sc_tx_prod;
   6381  1.1  jmcneill 
   6382  1.1  jmcneill 	used = 0;
   6383  1.1  jmcneill 	bf = NULL;
   6384  1.1  jmcneill 	sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&sc->sc_sq_mem);
   6385  1.1  jmcneill 
   6386  1.1  jmcneill 	for (;;) {
   6387  1.1  jmcneill 		if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
   6388  1.1  jmcneill 			SET(ifp->if_flags, IFF_OACTIVE);
   6389  1.1  jmcneill 			break;
   6390  1.1  jmcneill 		}
   6391  1.1  jmcneill 
   6392  1.1  jmcneill 		IFQ_DEQUEUE(&ifp->if_snd, m);
   6393  1.1  jmcneill 		if (m == NULL) {
   6394  1.1  jmcneill 			break;
   6395  1.1  jmcneill 		}
   6396  1.1  jmcneill 
   6397  1.1  jmcneill 		sqe = sq + idx;
   6398  1.1  jmcneill 		ms = &sc->sc_tx_slots[idx];
   6399  1.1  jmcneill 		memset(sqe, 0, sizeof(*sqe));
   6400  1.1  jmcneill 
   6401  1.1  jmcneill 		/* ctrl segment */
   6402  1.1  jmcneill 		sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
   6403  1.1  jmcneill 		    ((sc->sc_tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
   6404  1.1  jmcneill 		/* always generate a completion event */
   6405  1.1  jmcneill 		sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);
   6406  1.1  jmcneill 
   6407  1.1  jmcneill 		/* eth segment */
   6408  1.1  jmcneill 		sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
   6409  1.1  jmcneill 		m_copydata(m, 0, MCX_SQ_INLINE_SIZE, sqe->sqe_inline_headers);
   6410  1.1  jmcneill 		m_adj(m, MCX_SQ_INLINE_SIZE);
   6411  1.1  jmcneill 
   6412  1.1  jmcneill 		if (mcx_load_mbuf(sc, ms, m) != 0) {
   6413  1.1  jmcneill 			m_freem(m);
   6414  1.1  jmcneill 			ifp->if_oerrors++;
   6415  1.1  jmcneill 			continue;
   6416  1.1  jmcneill 		}
   6417  1.1  jmcneill 		bf = (uint64_t *)sqe;
   6418  1.1  jmcneill 
   6419  1.1  jmcneill 		if (ifp->if_bpf != NULL)
   6420  1.1  jmcneill 			bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers,
   6421  1.1  jmcneill 			    MCX_SQ_INLINE_SIZE, m, BPF_D_OUT);
   6422  1.1  jmcneill 
   6423  1.1  jmcneill 		map = ms->ms_map;
   6424  1.1  jmcneill 		bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
   6425  1.1  jmcneill 		    BUS_DMASYNC_PREWRITE);
   6426  1.1  jmcneill 
   6427  1.1  jmcneill 		sqe->sqe_ds_sq_num =
   6428  1.1  jmcneill 		    htobe32((sc->sc_sqn << MCX_SQE_SQ_NUM_SHIFT) |
   6429  1.1  jmcneill 		    (map->dm_nsegs + 3));
   6430  1.1  jmcneill 
   6431  1.1  jmcneill 		/* data segment - first wqe has one segment */
   6432  1.1  jmcneill 		sqs = sqe->sqe_segs;
   6433  1.1  jmcneill 		seg = 0;
   6434  1.1  jmcneill 		nseg = 1;
   6435  1.1  jmcneill 		for (i = 0; i < map->dm_nsegs; i++) {
   6436  1.1  jmcneill 			if (seg == nseg) {
   6437  1.1  jmcneill 				/* next slot */
   6438  1.1  jmcneill 				idx++;
   6439  1.1  jmcneill 				if (idx == (1 << MCX_LOG_SQ_SIZE))
   6440  1.1  jmcneill 					idx = 0;
   6441  1.1  jmcneill 				sc->sc_tx_prod++;
   6442  1.1  jmcneill 				used++;
   6443  1.1  jmcneill 
   6444  1.1  jmcneill 				sqs = (struct mcx_sq_entry_seg *)(sq + idx);
   6445  1.1  jmcneill 				seg = 0;
   6446  1.1  jmcneill 				nseg = MCX_SQ_SEGS_PER_SLOT;
   6447  1.1  jmcneill 			}
   6448  1.1  jmcneill 			sqs[seg].sqs_byte_count =
   6449  1.1  jmcneill 			    htobe32(map->dm_segs[i].ds_len);
   6450  1.1  jmcneill 			sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
   6451  1.1  jmcneill 			sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
   6452  1.1  jmcneill 			seg++;
   6453  1.1  jmcneill 		}
   6454  1.1  jmcneill 
   6455  1.1  jmcneill 		idx++;
   6456  1.1  jmcneill 		if (idx == (1 << MCX_LOG_SQ_SIZE))
   6457  1.1  jmcneill 			idx = 0;
   6458  1.1  jmcneill 		sc->sc_tx_prod++;
   6459  1.1  jmcneill 		used++;
   6460  1.1  jmcneill 	}
   6461  1.1  jmcneill 
   6462  1.1  jmcneill 	if (used) {
   6463  1.1  jmcneill 		*sc->sc_tx_doorbell = htobe32(sc->sc_tx_prod & MCX_WQ_DOORBELL_MASK);
   6464  1.1  jmcneill 
   6465  1.1  jmcneill 		membar_sync();
   6466  1.1  jmcneill 
   6467  1.1  jmcneill 		/*
   6468  1.1  jmcneill 		 * write the first 64 bits of the last sqe we produced
   6469  1.1  jmcneill 		 * to the blue flame buffer
   6470  1.1  jmcneill 		 */
   6471  1.1  jmcneill 		bus_space_write_8(sc->sc_memt, sc->sc_memh,
   6472  1.1  jmcneill 		    bf_base + sc->sc_bf_offset, *bf);
   6473  1.1  jmcneill 		/* next write goes to the other buffer */
   6474  1.1  jmcneill 		sc->sc_bf_offset ^= sc->sc_bf_size;
   6475  1.1  jmcneill 
   6476  1.1  jmcneill 		membar_sync();
   6477  1.1  jmcneill 	}
   6478  1.1  jmcneill }
   6479  1.1  jmcneill 
   6480  1.1  jmcneill static void
   6481  1.1  jmcneill mcx_watchdog(struct ifnet *ifp)
   6482  1.1  jmcneill {
   6483  1.1  jmcneill }
   6484  1.1  jmcneill 
   6485  1.1  jmcneill static void
   6486  1.1  jmcneill mcx_media_add_types(struct mcx_softc *sc)
   6487  1.1  jmcneill {
   6488  1.1  jmcneill 	struct mcx_reg_ptys ptys;
   6489  1.1  jmcneill 	int i;
   6490  1.1  jmcneill 	uint32_t proto_cap;
   6491  1.1  jmcneill 
   6492  1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   6493  1.1  jmcneill 	ptys.rp_local_port = 1;
   6494  1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   6495  1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   6496  1.1  jmcneill 	    sizeof(ptys)) != 0) {
   6497  1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   6498  1.1  jmcneill 		return;
   6499  1.1  jmcneill 	}
   6500  1.1  jmcneill 
   6501  1.1  jmcneill 	proto_cap = be32toh(ptys.rp_eth_proto_cap);
   6502  1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   6503  1.1  jmcneill 		if ((proto_cap & (1 << i)) && (mcx_eth_cap_map[i] != 0))
   6504  1.1  jmcneill 			ifmedia_add(&sc->sc_media, IFM_ETHER |
   6505  1.1  jmcneill 			    mcx_eth_cap_map[i], 0, NULL);
   6506  1.1  jmcneill 	}
   6507  1.1  jmcneill }
   6508  1.1  jmcneill 
   6509  1.1  jmcneill static void
   6510  1.1  jmcneill mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
   6511  1.1  jmcneill {
   6512  1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   6513  1.1  jmcneill 	struct mcx_reg_ptys ptys;
   6514  1.1  jmcneill 	int i;
   6515  1.1  jmcneill 	uint32_t /* proto_cap, */ proto_oper;
   6516  1.1  jmcneill 	uint64_t media_oper;
   6517  1.1  jmcneill 
   6518  1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   6519  1.1  jmcneill 	ptys.rp_local_port = 1;
   6520  1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   6521  1.1  jmcneill 
   6522  1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
   6523  1.1  jmcneill 	    sizeof(ptys)) != 0) {
   6524  1.1  jmcneill 		printf("%s: unable to read port type/speed\n", DEVNAME(sc));
   6525  1.1  jmcneill 		return;
   6526  1.1  jmcneill 	}
   6527  1.1  jmcneill 
   6528  1.1  jmcneill 	/* proto_cap = be32toh(ptys.rp_eth_proto_cap); */
   6529  1.1  jmcneill 	proto_oper = be32toh(ptys.rp_eth_proto_oper);
   6530  1.1  jmcneill 
   6531  1.1  jmcneill 	media_oper = 0;
   6532  1.1  jmcneill 	for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   6533  1.1  jmcneill 		if (proto_oper & (1 << i)) {
   6534  1.1  jmcneill 			media_oper = mcx_eth_cap_map[i];
   6535  1.1  jmcneill 		}
   6536  1.1  jmcneill 	}
   6537  1.1  jmcneill 
   6538  1.1  jmcneill 	ifmr->ifm_status = IFM_AVALID;
   6539  1.1  jmcneill 	/* not sure if this is the right thing to check, maybe paos? */
   6540  1.1  jmcneill 	if (proto_oper != 0) {
   6541  1.1  jmcneill 		ifmr->ifm_status |= IFM_ACTIVE;
   6542  1.1  jmcneill 		ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper;
   6543  1.1  jmcneill 		/* txpause, rxpause, duplex? */
   6544  1.1  jmcneill 	}
   6545  1.1  jmcneill }
   6546  1.1  jmcneill 
   6547  1.1  jmcneill static int
   6548  1.1  jmcneill mcx_media_change(struct ifnet *ifp)
   6549  1.1  jmcneill {
   6550  1.1  jmcneill 	struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
   6551  1.1  jmcneill 	struct mcx_reg_ptys ptys;
   6552  1.1  jmcneill 	struct mcx_reg_paos paos;
   6553  1.1  jmcneill 	uint32_t media;
   6554  1.1  jmcneill 	int i, error;
   6555  1.1  jmcneill 
   6556  1.1  jmcneill 	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
   6557  1.1  jmcneill 		return EINVAL;
   6558  1.1  jmcneill 
   6559  1.1  jmcneill 	error = 0;
   6560  1.1  jmcneill 
   6561  1.1  jmcneill 	if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
   6562  1.1  jmcneill 		/* read ptys to get supported media */
   6563  1.1  jmcneill 		memset(&ptys, 0, sizeof(ptys));
   6564  1.1  jmcneill 		ptys.rp_local_port = 1;
   6565  1.1  jmcneill 		ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   6566  1.1  jmcneill 		if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
   6567  1.1  jmcneill 		    &ptys, sizeof(ptys)) != 0) {
   6568  1.1  jmcneill 			printf("%s: unable to read port type/speed\n",
   6569  1.1  jmcneill 			    DEVNAME(sc));
   6570  1.1  jmcneill 			return EIO;
   6571  1.1  jmcneill 		}
   6572  1.1  jmcneill 
   6573  1.1  jmcneill 		media = be32toh(ptys.rp_eth_proto_cap);
   6574  1.1  jmcneill 	} else {
   6575  1.1  jmcneill 		/* map media type */
   6576  1.1  jmcneill 		media = 0;
   6577  1.1  jmcneill 		for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
   6578  1.1  jmcneill 			if (mcx_eth_cap_map[i] ==
   6579  1.1  jmcneill 			    IFM_SUBTYPE(sc->sc_media.ifm_media)) {
   6580  1.1  jmcneill 				media = (1 << i);
   6581  1.1  jmcneill 				break;
   6582  1.1  jmcneill 			}
   6583  1.1  jmcneill 		}
   6584  1.1  jmcneill 	}
   6585  1.1  jmcneill 
   6586  1.1  jmcneill 	/* disable the port */
   6587  1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   6588  1.1  jmcneill 	paos.rp_local_port = 1;
   6589  1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
   6590  1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   6591  1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   6592  1.1  jmcneill 	    sizeof(paos)) != 0) {
   6593  1.1  jmcneill 		printf("%s: unable to set port state to down\n", DEVNAME(sc));
   6594  1.1  jmcneill 		return EIO;
   6595  1.1  jmcneill 	}
   6596  1.1  jmcneill 
   6597  1.1  jmcneill 	memset(&ptys, 0, sizeof(ptys));
   6598  1.1  jmcneill 	ptys.rp_local_port = 1;
   6599  1.1  jmcneill 	ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
   6600  1.1  jmcneill 	ptys.rp_eth_proto_admin = htobe32(media);
   6601  1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
   6602  1.1  jmcneill 	    sizeof(ptys)) != 0) {
   6603  1.1  jmcneill 		printf("%s: unable to set port media type/speed\n",
   6604  1.1  jmcneill 		    DEVNAME(sc));
   6605  1.1  jmcneill 		error = EIO;
   6606  1.1  jmcneill 	}
   6607  1.1  jmcneill 
   6608  1.1  jmcneill 	/* re-enable the port to start negotiation */
   6609  1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   6610  1.1  jmcneill 	paos.rp_local_port = 1;
   6611  1.1  jmcneill 	paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
   6612  1.1  jmcneill 	paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
   6613  1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
   6614  1.1  jmcneill 	    sizeof(paos)) != 0) {
   6615  1.1  jmcneill 		printf("%s: unable to set port state to up\n", DEVNAME(sc));
   6616  1.1  jmcneill 		error = EIO;
   6617  1.1  jmcneill 	}
   6618  1.1  jmcneill 
   6619  1.1  jmcneill 	return error;
   6620  1.1  jmcneill }
   6621  1.1  jmcneill 
   6622  1.1  jmcneill static void
   6623  1.1  jmcneill mcx_port_change(struct work *wk, void *xsc)
   6624  1.1  jmcneill {
   6625  1.1  jmcneill 	struct mcx_softc *sc = xsc;
   6626  1.1  jmcneill 	struct ifnet *ifp = &sc->sc_ec.ec_if;
   6627  1.1  jmcneill 	struct mcx_reg_paos paos;
   6628  1.1  jmcneill 	int link_state = LINK_STATE_DOWN;
   6629  1.1  jmcneill 
   6630  1.1  jmcneill 	memset(&paos, 0, sizeof(paos));
   6631  1.1  jmcneill 	paos.rp_local_port = 1;
   6632  1.1  jmcneill 	if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_READ, &paos,
   6633  1.1  jmcneill 	    sizeof(paos)) == 0) {
   6634  1.1  jmcneill 		if (paos.rp_oper_status == MCX_REG_PAOS_OPER_STATUS_UP)
   6635  1.1  jmcneill 			link_state = LINK_STATE_UP;
   6636  1.1  jmcneill 	}
   6637  1.1  jmcneill 
   6638  1.1  jmcneill 	if (link_state != ifp->if_link_state) {
   6639  1.1  jmcneill 		if_link_state_change(ifp, link_state);
   6640  1.1  jmcneill 	}
   6641  1.1  jmcneill }
   6642  1.1  jmcneill 
   6643  1.1  jmcneill 
   6644  1.1  jmcneill static inline uint32_t
   6645  1.1  jmcneill mcx_rd(struct mcx_softc *sc, bus_size_t r)
   6646  1.1  jmcneill {
   6647  1.1  jmcneill 	uint32_t word;
   6648  1.1  jmcneill 
   6649  1.1  jmcneill 	word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r);
   6650  1.1  jmcneill 
   6651  1.1  jmcneill 	return (be32toh(word));
   6652  1.1  jmcneill }
   6653  1.1  jmcneill 
   6654  1.1  jmcneill static inline void
   6655  1.1  jmcneill mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
   6656  1.1  jmcneill {
   6657  1.1  jmcneill 	bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
   6658  1.1  jmcneill }
   6659  1.1  jmcneill 
   6660  1.1  jmcneill static inline void
   6661  1.1  jmcneill mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
   6662  1.1  jmcneill {
   6663  1.1  jmcneill 	bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
   6664  1.1  jmcneill }
   6665  1.1  jmcneill 
   6666  1.1  jmcneill static uint64_t
   6667  1.1  jmcneill mcx_timer(struct mcx_softc *sc)
   6668  1.1  jmcneill {
   6669  1.1  jmcneill 	uint32_t hi, lo, ni;
   6670  1.1  jmcneill 
   6671  1.1  jmcneill 	hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   6672  1.1  jmcneill 	for (;;) {
   6673  1.1  jmcneill 		lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
   6674  1.1  jmcneill 		mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
   6675  1.1  jmcneill 		ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
   6676  1.1  jmcneill 
   6677  1.1  jmcneill 		if (ni == hi)
   6678  1.1  jmcneill 			break;
   6679  1.1  jmcneill 
   6680  1.1  jmcneill 		hi = ni;
   6681  1.1  jmcneill 	}
   6682  1.1  jmcneill 
   6683  1.1  jmcneill 	return (((uint64_t)hi << 32) | (uint64_t)lo);
   6684  1.1  jmcneill }
   6685  1.1  jmcneill 
   6686  1.1  jmcneill static int
   6687  1.1  jmcneill mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
   6688  1.1  jmcneill     bus_size_t size, u_int align)
   6689  1.1  jmcneill {
   6690  1.1  jmcneill 	mxm->mxm_size = size;
   6691  1.1  jmcneill 
   6692  1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
   6693  1.1  jmcneill 	    mxm->mxm_size, 0,
   6694  1.1  jmcneill 	    BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
   6695  1.1  jmcneill 	    &mxm->mxm_map) != 0)
   6696  1.1  jmcneill 		return (1);
   6697  1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
   6698  1.1  jmcneill 	    align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
   6699  1.1  jmcneill 	    BUS_DMA_WAITOK) != 0)
   6700  1.1  jmcneill 		goto destroy;
   6701  1.1  jmcneill 	if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
   6702  1.1  jmcneill 	    mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0)
   6703  1.1  jmcneill 		goto free;
   6704  1.1  jmcneill 	if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
   6705  1.1  jmcneill 	    mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
   6706  1.1  jmcneill 		goto unmap;
   6707  1.1  jmcneill 
   6708  1.1  jmcneill 	mcx_dmamem_zero(mxm);
   6709  1.1  jmcneill 
   6710  1.1  jmcneill 	return (0);
   6711  1.1  jmcneill unmap:
   6712  1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   6713  1.1  jmcneill free:
   6714  1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   6715  1.1  jmcneill destroy:
   6716  1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   6717  1.1  jmcneill 	return (1);
   6718  1.1  jmcneill }
   6719  1.1  jmcneill 
   6720  1.1  jmcneill static void
   6721  1.1  jmcneill mcx_dmamem_zero(struct mcx_dmamem *mxm)
   6722  1.1  jmcneill {
   6723  1.1  jmcneill 	memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
   6724  1.1  jmcneill }
   6725  1.1  jmcneill 
   6726  1.1  jmcneill static void
   6727  1.1  jmcneill mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
   6728  1.1  jmcneill {
   6729  1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
   6730  1.1  jmcneill 	bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
   6731  1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
   6732  1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
   6733  1.1  jmcneill }
   6734  1.1  jmcneill 
   6735  1.1  jmcneill static int
   6736  1.1  jmcneill mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
   6737  1.1  jmcneill {
   6738  1.1  jmcneill 	bus_dma_segment_t *segs;
   6739  1.1  jmcneill 	bus_size_t len = pages * MCX_PAGE_SIZE;
   6740  1.1  jmcneill 	size_t seglen;
   6741  1.1  jmcneill 
   6742  1.1  jmcneill 	segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP);
   6743  1.1  jmcneill 	seglen = sizeof(*segs) * pages;
   6744  1.1  jmcneill 
   6745  1.1  jmcneill 	if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
   6746  1.1  jmcneill 	    segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
   6747  1.1  jmcneill 		goto free_segs;
   6748  1.1  jmcneill 
   6749  1.1  jmcneill 	if (mhm->mhm_seg_count < pages) {
   6750  1.1  jmcneill 		size_t nseglen;
   6751  1.1  jmcneill 
   6752  1.1  jmcneill 		mhm->mhm_segs = kmem_alloc(
   6753  1.1  jmcneill 		    sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP);
   6754  1.1  jmcneill 
   6755  1.1  jmcneill 		nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
   6756  1.1  jmcneill 
   6757  1.1  jmcneill 		memcpy(mhm->mhm_segs, segs, nseglen);
   6758  1.1  jmcneill 
   6759  1.1  jmcneill 		kmem_free(segs, seglen);
   6760  1.1  jmcneill 
   6761  1.1  jmcneill 		segs = mhm->mhm_segs;
   6762  1.1  jmcneill 		seglen = nseglen;
   6763  1.1  jmcneill 	} else
   6764  1.1  jmcneill 		mhm->mhm_segs = segs;
   6765  1.1  jmcneill 
   6766  1.1  jmcneill 	if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
   6767  1.1  jmcneill 	    MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
   6768  1.1  jmcneill 	    &mhm->mhm_map) != 0)
   6769  1.1  jmcneill 		goto free_dmamem;
   6770  1.1  jmcneill 
   6771  1.1  jmcneill 	if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
   6772  1.1  jmcneill 	    mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
   6773  1.1  jmcneill 		goto destroy;
   6774  1.1  jmcneill 
   6775  1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   6776  1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
   6777  1.1  jmcneill 
   6778  1.1  jmcneill 	mhm->mhm_npages = pages;
   6779  1.1  jmcneill 
   6780  1.1  jmcneill 	return (0);
   6781  1.1  jmcneill 
   6782  1.1  jmcneill destroy:
   6783  1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   6784  1.1  jmcneill free_dmamem:
   6785  1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   6786  1.1  jmcneill free_segs:
   6787  1.1  jmcneill 	kmem_free(segs, seglen);
   6788  1.1  jmcneill 	mhm->mhm_segs = NULL;
   6789  1.1  jmcneill 
   6790  1.1  jmcneill 	return (-1);
   6791  1.1  jmcneill }
   6792  1.1  jmcneill 
   6793  1.1  jmcneill static void
   6794  1.1  jmcneill mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
   6795  1.1  jmcneill {
   6796  1.1  jmcneill 	if (mhm->mhm_npages == 0)
   6797  1.1  jmcneill 		return;
   6798  1.1  jmcneill 
   6799  1.1  jmcneill 	bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
   6800  1.1  jmcneill 	    0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
   6801  1.1  jmcneill 
   6802  1.1  jmcneill 	bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
   6803  1.1  jmcneill 	bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
   6804  1.1  jmcneill 	bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
   6805  1.1  jmcneill 	kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
   6806  1.1  jmcneill 
   6807  1.1  jmcneill 	mhm->mhm_npages = 0;
   6808  1.1  jmcneill }
   6809