if_mcx.c revision 1.29 1 1.29 msaitoh /* $NetBSD: if_mcx.c,v 1.29 2024/07/07 23:29:04 msaitoh Exp $ */
2 1.20 jmcneill /* $OpenBSD: if_mcx.c,v 1.101 2021/06/02 19:16:11 patrick Exp $ */
3 1.1 jmcneill
4 1.1 jmcneill /*
5 1.1 jmcneill * Copyright (c) 2017 David Gwynne <dlg (at) openbsd.org>
6 1.1 jmcneill * Copyright (c) 2019 Jonathan Matthew <jmatthew (at) openbsd.org>
7 1.1 jmcneill *
8 1.1 jmcneill * Permission to use, copy, modify, and distribute this software for any
9 1.1 jmcneill * purpose with or without fee is hereby granted, provided that the above
10 1.1 jmcneill * copyright notice and this permission notice appear in all copies.
11 1.1 jmcneill *
12 1.1 jmcneill * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 1.1 jmcneill * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 1.1 jmcneill * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 1.1 jmcneill * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 1.1 jmcneill * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 1.1 jmcneill * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 1.1 jmcneill * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 1.1 jmcneill */
20 1.1 jmcneill
21 1.1 jmcneill #ifdef _KERNEL_OPT
22 1.1 jmcneill #include "opt_net_mpsafe.h"
23 1.1 jmcneill #endif
24 1.1 jmcneill
25 1.14 jmcneill #include <sys/cdefs.h>
26 1.29 msaitoh __KERNEL_RCSID(0, "$NetBSD: if_mcx.c,v 1.29 2024/07/07 23:29:04 msaitoh Exp $");
27 1.14 jmcneill
28 1.1 jmcneill #include <sys/param.h>
29 1.1 jmcneill #include <sys/systm.h>
30 1.1 jmcneill #include <sys/sockio.h>
31 1.1 jmcneill #include <sys/mbuf.h>
32 1.1 jmcneill #include <sys/kernel.h>
33 1.1 jmcneill #include <sys/socket.h>
34 1.1 jmcneill #include <sys/device.h>
35 1.1 jmcneill #include <sys/pool.h>
36 1.1 jmcneill #include <sys/queue.h>
37 1.1 jmcneill #include <sys/callout.h>
38 1.1 jmcneill #include <sys/workqueue.h>
39 1.1 jmcneill #include <sys/atomic.h>
40 1.15 jmcneill #include <sys/timetc.h>
41 1.1 jmcneill #include <sys/kmem.h>
42 1.1 jmcneill #include <sys/bus.h>
43 1.15 jmcneill #include <sys/interrupt.h>
44 1.16 jmcneill #include <sys/pcq.h>
45 1.16 jmcneill #include <sys/cpu.h>
46 1.23 andvar #include <sys/bitops.h>
47 1.1 jmcneill
48 1.1 jmcneill #include <machine/intr.h>
49 1.1 jmcneill
50 1.1 jmcneill #include <net/if.h>
51 1.1 jmcneill #include <net/if_dl.h>
52 1.1 jmcneill #include <net/if_ether.h>
53 1.1 jmcneill #include <net/if_media.h>
54 1.15 jmcneill #include <net/if_vlanvar.h>
55 1.15 jmcneill #include <net/toeplitz.h>
56 1.1 jmcneill
57 1.1 jmcneill #include <net/bpf.h>
58 1.1 jmcneill
59 1.1 jmcneill #include <netinet/in.h>
60 1.1 jmcneill
61 1.1 jmcneill #include <dev/pci/pcireg.h>
62 1.1 jmcneill #include <dev/pci/pcivar.h>
63 1.1 jmcneill #include <dev/pci/pcidevs.h>
64 1.1 jmcneill
65 1.15 jmcneill /* TODO: Port kstat key/value stuff to evcnt/sysmon */
66 1.15 jmcneill #define NKSTAT 0
67 1.15 jmcneill
68 1.12 thorpej /* XXX This driver is not yet MP-safe; don't claim to be! */
69 1.12 thorpej /* #ifdef NET_MPSAFE */
70 1.12 thorpej /* #define MCX_MPSAFE 1 */
71 1.12 thorpej /* #define CALLOUT_FLAGS CALLOUT_MPSAFE */
72 1.12 thorpej /* #else */
73 1.1 jmcneill #define CALLOUT_FLAGS 0
74 1.12 thorpej /* #endif */
75 1.1 jmcneill
76 1.16 jmcneill #define MCX_TXQ_NUM 2048
77 1.16 jmcneill
78 1.1 jmcneill #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
79 1.1 jmcneill #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
80 1.1 jmcneill
81 1.1 jmcneill #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */
82 1.1 jmcneill
83 1.15 jmcneill #define MCX_FW_VER 0x0000
84 1.15 jmcneill #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff)
85 1.15 jmcneill #define MCX_FW_VER_MINOR(_v) ((_v) >> 16)
86 1.15 jmcneill #define MCX_CMDIF_FW_SUBVER 0x0004
87 1.15 jmcneill #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff)
88 1.15 jmcneill #define MCX_CMDIF(_v) ((_v) >> 16)
89 1.1 jmcneill
90 1.15 jmcneill #define MCX_ISSI 1 /* as per the PRM */
91 1.15 jmcneill #define MCX_CMD_IF_SUPPORTED 5
92 1.1 jmcneill
93 1.15 jmcneill #define MCX_HARDMTU 9500
94 1.1 jmcneill
95 1.15 jmcneill #define MCX_PAGE_SHIFT 12
96 1.15 jmcneill #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT)
97 1.1 jmcneill
98 1.1 jmcneill /* queue sizes */
99 1.15 jmcneill #define MCX_LOG_EQ_SIZE 7
100 1.15 jmcneill #define MCX_LOG_CQ_SIZE 12
101 1.15 jmcneill #define MCX_LOG_RQ_SIZE 10
102 1.15 jmcneill #define MCX_LOG_SQ_SIZE 11
103 1.15 jmcneill
104 1.15 jmcneill #define MCX_MAX_QUEUES 16
105 1.1 jmcneill
106 1.1 jmcneill /* completion event moderation - about 10khz, or 90% of the cq */
107 1.15 jmcneill #define MCX_CQ_MOD_PERIOD 50
108 1.15 jmcneill #define MCX_CQ_MOD_COUNTER \
109 1.15 jmcneill (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)
110 1.15 jmcneill
111 1.15 jmcneill #define MCX_LOG_SQ_ENTRY_SIZE 6
112 1.15 jmcneill #define MCX_SQ_ENTRY_MAX_SLOTS 4
113 1.15 jmcneill #define MCX_SQ_SEGS_PER_SLOT \
114 1.1 jmcneill (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
115 1.15 jmcneill #define MCX_SQ_MAX_SEGMENTS \
116 1.1 jmcneill 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)
117 1.1 jmcneill
118 1.15 jmcneill #define MCX_LOG_FLOW_TABLE_SIZE 5
119 1.15 jmcneill #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */
120 1.15 jmcneill #define MCX_NUM_MCAST_FLOWS \
121 1.1 jmcneill ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)
122 1.1 jmcneill
123 1.15 jmcneill #define MCX_SQ_INLINE_SIZE 18
124 1.15 jmcneill CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE);
125 1.1 jmcneill
126 1.1 jmcneill /* doorbell offsets */
127 1.15 jmcneill #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE
128 1.1 jmcneill
129 1.15 jmcneill #define MCX_CQ_DOORBELL_BASE 0
130 1.15 jmcneill #define MCX_CQ_DOORBELL_STRIDE 64
131 1.15 jmcneill
132 1.15 jmcneill #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2
133 1.15 jmcneill #define MCX_WQ_DOORBELL_STRIDE 64
134 1.15 jmcneill /* make sure the doorbells fit */
135 1.15 jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE);
136 1.15 jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE <
137 1.15 jmcneill MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE);
138 1.15 jmcneill
139 1.15 jmcneill #define MCX_WQ_DOORBELL_MASK 0xffff
140 1.1 jmcneill
141 1.1 jmcneill /* uar registers */
142 1.15 jmcneill #define MCX_UAR_CQ_DOORBELL 0x20
143 1.15 jmcneill #define MCX_UAR_EQ_DOORBELL_ARM 0x40
144 1.15 jmcneill #define MCX_UAR_EQ_DOORBELL 0x48
145 1.15 jmcneill #define MCX_UAR_BF 0x800
146 1.1 jmcneill
147 1.15 jmcneill #define MCX_CMDQ_ADDR_HI 0x0010
148 1.15 jmcneill #define MCX_CMDQ_ADDR_LO 0x0014
149 1.1 jmcneill #define MCX_CMDQ_ADDR_NMASK 0xfff
150 1.1 jmcneill #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf)
151 1.1 jmcneill #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf)
152 1.1 jmcneill #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8)
153 1.1 jmcneill #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8)
154 1.1 jmcneill #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8)
155 1.1 jmcneill
156 1.1 jmcneill #define MCX_CMDQ_DOORBELL 0x0018
157 1.1 jmcneill
158 1.15 jmcneill #define MCX_STATE 0x01fc
159 1.15 jmcneill #define MCX_STATE_MASK (1U << 31)
160 1.15 jmcneill #define MCX_STATE_INITIALIZING (1 << 31)
161 1.15 jmcneill #define MCX_STATE_READY (0 << 31)
162 1.15 jmcneill #define MCX_STATE_INTERFACE_MASK (0x3 << 24)
163 1.15 jmcneill #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24)
164 1.15 jmcneill #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24)
165 1.15 jmcneill
166 1.15 jmcneill #define MCX_INTERNAL_TIMER 0x1000
167 1.15 jmcneill #define MCX_INTERNAL_TIMER_H 0x1000
168 1.15 jmcneill #define MCX_INTERNAL_TIMER_L 0x1004
169 1.15 jmcneill
170 1.15 jmcneill #define MCX_CLEAR_INT 0x100c
171 1.15 jmcneill
172 1.15 jmcneill #define MCX_REG_OP_WRITE 0
173 1.15 jmcneill #define MCX_REG_OP_READ 1
174 1.15 jmcneill
175 1.15 jmcneill #define MCX_REG_PMLP 0x5002
176 1.15 jmcneill #define MCX_REG_PMTU 0x5003
177 1.15 jmcneill #define MCX_REG_PTYS 0x5004
178 1.15 jmcneill #define MCX_REG_PAOS 0x5006
179 1.15 jmcneill #define MCX_REG_PFCC 0x5007
180 1.15 jmcneill #define MCX_REG_PPCNT 0x5008
181 1.15 jmcneill #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */
182 1.15 jmcneill #define MCX_REG_MTMP 0x900a /* mgmt temp */
183 1.15 jmcneill #define MCX_REG_MCIA 0x9014
184 1.15 jmcneill #define MCX_REG_MCAM 0x907f
185 1.15 jmcneill
186 1.15 jmcneill #define MCX_ETHER_CAP_SGMII 0
187 1.15 jmcneill #define MCX_ETHER_CAP_1000_KX 1
188 1.15 jmcneill #define MCX_ETHER_CAP_10G_CX4 2
189 1.15 jmcneill #define MCX_ETHER_CAP_10G_KX4 3
190 1.15 jmcneill #define MCX_ETHER_CAP_10G_KR 4
191 1.15 jmcneill #define MCX_ETHER_CAP_20G_KR2 5
192 1.15 jmcneill #define MCX_ETHER_CAP_40G_CR4 6
193 1.15 jmcneill #define MCX_ETHER_CAP_40G_KR4 7
194 1.15 jmcneill #define MCX_ETHER_CAP_56G_R4 8
195 1.15 jmcneill #define MCX_ETHER_CAP_10G_CR 12
196 1.15 jmcneill #define MCX_ETHER_CAP_10G_SR 13
197 1.15 jmcneill #define MCX_ETHER_CAP_10G_LR 14
198 1.15 jmcneill #define MCX_ETHER_CAP_40G_SR4 15
199 1.15 jmcneill #define MCX_ETHER_CAP_40G_LR4 16
200 1.15 jmcneill #define MCX_ETHER_CAP_50G_SR2 18
201 1.15 jmcneill #define MCX_ETHER_CAP_100G_CR4 20
202 1.15 jmcneill #define MCX_ETHER_CAP_100G_SR4 21
203 1.15 jmcneill #define MCX_ETHER_CAP_100G_KR4 22
204 1.15 jmcneill #define MCX_ETHER_CAP_100G_LR4 23
205 1.15 jmcneill #define MCX_ETHER_CAP_100_TX 24
206 1.15 jmcneill #define MCX_ETHER_CAP_1000_T 25
207 1.15 jmcneill #define MCX_ETHER_CAP_10G_T 26
208 1.15 jmcneill #define MCX_ETHER_CAP_25G_CR 27
209 1.15 jmcneill #define MCX_ETHER_CAP_25G_KR 28
210 1.15 jmcneill #define MCX_ETHER_CAP_25G_SR 29
211 1.15 jmcneill #define MCX_ETHER_CAP_50G_CR2 30
212 1.15 jmcneill #define MCX_ETHER_CAP_50G_KR2 31
213 1.15 jmcneill
214 1.15 jmcneill #define MCX_MAX_CQE 32
215 1.15 jmcneill
216 1.15 jmcneill #define MCX_CMD_QUERY_HCA_CAP 0x100
217 1.15 jmcneill #define MCX_CMD_QUERY_ADAPTER 0x101
218 1.15 jmcneill #define MCX_CMD_INIT_HCA 0x102
219 1.15 jmcneill #define MCX_CMD_TEARDOWN_HCA 0x103
220 1.15 jmcneill #define MCX_CMD_ENABLE_HCA 0x104
221 1.15 jmcneill #define MCX_CMD_DISABLE_HCA 0x105
222 1.15 jmcneill #define MCX_CMD_QUERY_PAGES 0x107
223 1.15 jmcneill #define MCX_CMD_MANAGE_PAGES 0x108
224 1.15 jmcneill #define MCX_CMD_SET_HCA_CAP 0x109
225 1.15 jmcneill #define MCX_CMD_QUERY_ISSI 0x10a
226 1.15 jmcneill #define MCX_CMD_SET_ISSI 0x10b
227 1.15 jmcneill #define MCX_CMD_SET_DRIVER_VERSION 0x10d
228 1.15 jmcneill #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203
229 1.15 jmcneill #define MCX_CMD_CREATE_EQ 0x301
230 1.15 jmcneill #define MCX_CMD_DESTROY_EQ 0x302
231 1.15 jmcneill #define MCX_CMD_QUERY_EQ 0x303
232 1.15 jmcneill #define MCX_CMD_CREATE_CQ 0x400
233 1.15 jmcneill #define MCX_CMD_DESTROY_CQ 0x401
234 1.15 jmcneill #define MCX_CMD_QUERY_CQ 0x402
235 1.15 jmcneill #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754
236 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
237 1.15 jmcneill 0x755
238 1.15 jmcneill #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770
239 1.15 jmcneill #define MCX_CMD_ALLOC_PD 0x800
240 1.15 jmcneill #define MCX_CMD_ALLOC_UAR 0x802
241 1.15 jmcneill #define MCX_CMD_ACCESS_REG 0x805
242 1.15 jmcneill #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816
243 1.15 jmcneill #define MCX_CMD_CREATE_TIR 0x900
244 1.15 jmcneill #define MCX_CMD_DESTROY_TIR 0x902
245 1.15 jmcneill #define MCX_CMD_CREATE_SQ 0x904
246 1.15 jmcneill #define MCX_CMD_MODIFY_SQ 0x905
247 1.15 jmcneill #define MCX_CMD_DESTROY_SQ 0x906
248 1.15 jmcneill #define MCX_CMD_QUERY_SQ 0x907
249 1.15 jmcneill #define MCX_CMD_CREATE_RQ 0x908
250 1.15 jmcneill #define MCX_CMD_MODIFY_RQ 0x909
251 1.15 jmcneill #define MCX_CMD_DESTROY_RQ 0x90a
252 1.15 jmcneill #define MCX_CMD_QUERY_RQ 0x90b
253 1.15 jmcneill #define MCX_CMD_CREATE_TIS 0x912
254 1.15 jmcneill #define MCX_CMD_DESTROY_TIS 0x914
255 1.15 jmcneill #define MCX_CMD_CREATE_RQT 0x916
256 1.15 jmcneill #define MCX_CMD_DESTROY_RQT 0x918
257 1.15 jmcneill #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f
258 1.15 jmcneill #define MCX_CMD_CREATE_FLOW_TABLE 0x930
259 1.15 jmcneill #define MCX_CMD_DESTROY_FLOW_TABLE 0x931
260 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_TABLE 0x932
261 1.15 jmcneill #define MCX_CMD_CREATE_FLOW_GROUP 0x933
262 1.15 jmcneill #define MCX_CMD_DESTROY_FLOW_GROUP 0x934
263 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_GROUP 0x935
264 1.15 jmcneill #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936
265 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937
266 1.15 jmcneill #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938
267 1.15 jmcneill #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939
268 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b
269 1.15 jmcneill
270 1.15 jmcneill #define MCX_QUEUE_STATE_RST 0
271 1.15 jmcneill #define MCX_QUEUE_STATE_RDY 1
272 1.15 jmcneill #define MCX_QUEUE_STATE_ERR 3
273 1.1 jmcneill
274 1.15 jmcneill #define MCX_FLOW_TABLE_TYPE_RX 0
275 1.15 jmcneill #define MCX_FLOW_TABLE_TYPE_TX 1
276 1.1 jmcneill
277 1.15 jmcneill #define MCX_CMDQ_INLINE_DATASIZE 16
278 1.1 jmcneill
279 1.1 jmcneill struct mcx_cmdq_entry {
280 1.1 jmcneill uint8_t cq_type;
281 1.1 jmcneill #define MCX_CMDQ_TYPE_PCIE 0x7
282 1.1 jmcneill uint8_t cq_reserved0[3];
283 1.1 jmcneill
284 1.1 jmcneill uint32_t cq_input_length;
285 1.1 jmcneill uint64_t cq_input_ptr;
286 1.1 jmcneill uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
287 1.1 jmcneill
288 1.1 jmcneill uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
289 1.1 jmcneill uint64_t cq_output_ptr;
290 1.1 jmcneill uint32_t cq_output_length;
291 1.1 jmcneill
292 1.1 jmcneill uint8_t cq_token;
293 1.1 jmcneill uint8_t cq_signature;
294 1.1 jmcneill uint8_t cq_reserved1[1];
295 1.1 jmcneill uint8_t cq_status;
296 1.1 jmcneill #define MCX_CQ_STATUS_SHIFT 1
297 1.1 jmcneill #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT)
298 1.1 jmcneill #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT)
299 1.1 jmcneill #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT)
300 1.1 jmcneill #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT)
301 1.1 jmcneill #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT)
302 1.1 jmcneill #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT)
303 1.1 jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT)
304 1.1 jmcneill #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT)
305 1.1 jmcneill #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT)
306 1.1 jmcneill #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT)
307 1.1 jmcneill #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT)
308 1.1 jmcneill #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT)
309 1.1 jmcneill #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT)
310 1.1 jmcneill #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT)
311 1.1 jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
312 1.1 jmcneill (0x10 << MCX_CQ_STATUS_SHIFT)
313 1.1 jmcneill #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT)
314 1.1 jmcneill #define MCX_CQ_STATUS_OWN_MASK 0x1
315 1.1 jmcneill #define MCX_CQ_STATUS_OWN_SW 0x0
316 1.1 jmcneill #define MCX_CQ_STATUS_OWN_HW 0x1
317 1.1 jmcneill } __packed __aligned(8);
318 1.1 jmcneill
319 1.1 jmcneill #define MCX_CMDQ_MAILBOX_DATASIZE 512
320 1.1 jmcneill
321 1.1 jmcneill struct mcx_cmdq_mailbox {
322 1.1 jmcneill uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
323 1.1 jmcneill uint8_t mb_reserved0[48];
324 1.1 jmcneill uint64_t mb_next_ptr;
325 1.1 jmcneill uint32_t mb_block_number;
326 1.1 jmcneill uint8_t mb_reserved1[1];
327 1.1 jmcneill uint8_t mb_token;
328 1.1 jmcneill uint8_t mb_ctrl_signature;
329 1.1 jmcneill uint8_t mb_signature;
330 1.1 jmcneill } __packed __aligned(8);
331 1.1 jmcneill
332 1.1 jmcneill #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10)
333 1.1 jmcneill #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \
334 1.1 jmcneill MCX_CMDQ_MAILBOX_ALIGN)
335 1.1 jmcneill /*
336 1.18 andvar * command mailbox structures
337 1.1 jmcneill */
338 1.1 jmcneill
339 1.1 jmcneill struct mcx_cmd_enable_hca_in {
340 1.1 jmcneill uint16_t cmd_opcode;
341 1.1 jmcneill uint8_t cmd_reserved0[4];
342 1.1 jmcneill uint16_t cmd_op_mod;
343 1.1 jmcneill uint8_t cmd_reserved1[2];
344 1.1 jmcneill uint16_t cmd_function_id;
345 1.1 jmcneill uint8_t cmd_reserved2[4];
346 1.1 jmcneill } __packed __aligned(4);
347 1.1 jmcneill
348 1.1 jmcneill struct mcx_cmd_enable_hca_out {
349 1.1 jmcneill uint8_t cmd_status;
350 1.1 jmcneill uint8_t cmd_reserved0[3];
351 1.1 jmcneill uint32_t cmd_syndrome;
352 1.1 jmcneill uint8_t cmd_reserved1[4];
353 1.1 jmcneill } __packed __aligned(4);
354 1.1 jmcneill
355 1.1 jmcneill struct mcx_cmd_init_hca_in {
356 1.1 jmcneill uint16_t cmd_opcode;
357 1.1 jmcneill uint8_t cmd_reserved0[4];
358 1.1 jmcneill uint16_t cmd_op_mod;
359 1.1 jmcneill uint8_t cmd_reserved1[8];
360 1.1 jmcneill } __packed __aligned(4);
361 1.1 jmcneill
362 1.1 jmcneill struct mcx_cmd_init_hca_out {
363 1.1 jmcneill uint8_t cmd_status;
364 1.1 jmcneill uint8_t cmd_reserved0[3];
365 1.1 jmcneill uint32_t cmd_syndrome;
366 1.1 jmcneill uint8_t cmd_reserved1[8];
367 1.1 jmcneill } __packed __aligned(4);
368 1.1 jmcneill
369 1.1 jmcneill struct mcx_cmd_teardown_hca_in {
370 1.1 jmcneill uint16_t cmd_opcode;
371 1.1 jmcneill uint8_t cmd_reserved0[4];
372 1.1 jmcneill uint16_t cmd_op_mod;
373 1.1 jmcneill uint8_t cmd_reserved1[2];
374 1.1 jmcneill #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0
375 1.1 jmcneill #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1
376 1.1 jmcneill uint16_t cmd_profile;
377 1.1 jmcneill uint8_t cmd_reserved2[4];
378 1.1 jmcneill } __packed __aligned(4);
379 1.1 jmcneill
380 1.1 jmcneill struct mcx_cmd_teardown_hca_out {
381 1.1 jmcneill uint8_t cmd_status;
382 1.1 jmcneill uint8_t cmd_reserved0[3];
383 1.1 jmcneill uint32_t cmd_syndrome;
384 1.1 jmcneill uint8_t cmd_reserved1[8];
385 1.1 jmcneill } __packed __aligned(4);
386 1.1 jmcneill
387 1.1 jmcneill struct mcx_cmd_access_reg_in {
388 1.1 jmcneill uint16_t cmd_opcode;
389 1.1 jmcneill uint8_t cmd_reserved0[4];
390 1.1 jmcneill uint16_t cmd_op_mod;
391 1.1 jmcneill uint8_t cmd_reserved1[2];
392 1.1 jmcneill uint16_t cmd_register_id;
393 1.1 jmcneill uint32_t cmd_argument;
394 1.1 jmcneill } __packed __aligned(4);
395 1.1 jmcneill
396 1.1 jmcneill struct mcx_cmd_access_reg_out {
397 1.1 jmcneill uint8_t cmd_status;
398 1.1 jmcneill uint8_t cmd_reserved0[3];
399 1.1 jmcneill uint32_t cmd_syndrome;
400 1.1 jmcneill uint8_t cmd_reserved1[8];
401 1.1 jmcneill } __packed __aligned(4);
402 1.1 jmcneill
403 1.1 jmcneill struct mcx_reg_pmtu {
404 1.1 jmcneill uint8_t rp_reserved1;
405 1.1 jmcneill uint8_t rp_local_port;
406 1.1 jmcneill uint8_t rp_reserved2[2];
407 1.1 jmcneill uint16_t rp_max_mtu;
408 1.1 jmcneill uint8_t rp_reserved3[2];
409 1.1 jmcneill uint16_t rp_admin_mtu;
410 1.1 jmcneill uint8_t rp_reserved4[2];
411 1.1 jmcneill uint16_t rp_oper_mtu;
412 1.1 jmcneill uint8_t rp_reserved5[2];
413 1.1 jmcneill } __packed __aligned(4);
414 1.1 jmcneill
415 1.1 jmcneill struct mcx_reg_ptys {
416 1.1 jmcneill uint8_t rp_reserved1;
417 1.1 jmcneill uint8_t rp_local_port;
418 1.1 jmcneill uint8_t rp_reserved2;
419 1.1 jmcneill uint8_t rp_proto_mask;
420 1.1 jmcneill #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2)
421 1.1 jmcneill uint8_t rp_reserved3[8];
422 1.1 jmcneill uint32_t rp_eth_proto_cap;
423 1.1 jmcneill uint8_t rp_reserved4[8];
424 1.1 jmcneill uint32_t rp_eth_proto_admin;
425 1.1 jmcneill uint8_t rp_reserved5[8];
426 1.1 jmcneill uint32_t rp_eth_proto_oper;
427 1.1 jmcneill uint8_t rp_reserved6[24];
428 1.1 jmcneill } __packed __aligned(4);
429 1.1 jmcneill
430 1.1 jmcneill struct mcx_reg_paos {
431 1.1 jmcneill uint8_t rp_reserved1;
432 1.1 jmcneill uint8_t rp_local_port;
433 1.1 jmcneill uint8_t rp_admin_status;
434 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP 1
435 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2
436 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3
437 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4
438 1.1 jmcneill uint8_t rp_oper_status;
439 1.1 jmcneill #define MCX_REG_PAOS_OPER_STATUS_UP 1
440 1.1 jmcneill #define MCX_REG_PAOS_OPER_STATUS_DOWN 2
441 1.1 jmcneill #define MCX_REG_PAOS_OPER_STATUS_FAILED 4
442 1.1 jmcneill uint8_t rp_admin_state_update;
443 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7)
444 1.1 jmcneill uint8_t rp_reserved2[11];
445 1.1 jmcneill } __packed __aligned(4);
446 1.1 jmcneill
447 1.1 jmcneill struct mcx_reg_pfcc {
448 1.1 jmcneill uint8_t rp_reserved1;
449 1.1 jmcneill uint8_t rp_local_port;
450 1.1 jmcneill uint8_t rp_reserved2[3];
451 1.1 jmcneill uint8_t rp_prio_mask_tx;
452 1.1 jmcneill uint8_t rp_reserved3;
453 1.1 jmcneill uint8_t rp_prio_mask_rx;
454 1.1 jmcneill uint8_t rp_pptx_aptx;
455 1.1 jmcneill uint8_t rp_pfctx;
456 1.1 jmcneill uint8_t rp_fctx_dis;
457 1.1 jmcneill uint8_t rp_reserved4;
458 1.1 jmcneill uint8_t rp_pprx_aprx;
459 1.1 jmcneill uint8_t rp_pfcrx;
460 1.1 jmcneill uint8_t rp_reserved5[2];
461 1.1 jmcneill uint16_t rp_dev_stall_min;
462 1.1 jmcneill uint16_t rp_dev_stall_crit;
463 1.1 jmcneill uint8_t rp_reserved6[12];
464 1.1 jmcneill } __packed __aligned(4);
465 1.1 jmcneill
466 1.1 jmcneill #define MCX_PMLP_MODULE_NUM_MASK 0xff
467 1.1 jmcneill struct mcx_reg_pmlp {
468 1.1 jmcneill uint8_t rp_rxtx;
469 1.1 jmcneill uint8_t rp_local_port;
470 1.1 jmcneill uint8_t rp_reserved0;
471 1.1 jmcneill uint8_t rp_width;
472 1.1 jmcneill uint32_t rp_lane0_mapping;
473 1.1 jmcneill uint32_t rp_lane1_mapping;
474 1.1 jmcneill uint32_t rp_lane2_mapping;
475 1.1 jmcneill uint32_t rp_lane3_mapping;
476 1.1 jmcneill uint8_t rp_reserved1[44];
477 1.1 jmcneill } __packed __aligned(4);
478 1.1 jmcneill
479 1.15 jmcneill struct mcx_reg_ppcnt {
480 1.15 jmcneill uint8_t ppcnt_swid;
481 1.15 jmcneill uint8_t ppcnt_local_port;
482 1.15 jmcneill uint8_t ppcnt_pnat;
483 1.15 jmcneill uint8_t ppcnt_grp;
484 1.15 jmcneill #define MCX_REG_PPCNT_GRP_IEEE8023 0x00
485 1.15 jmcneill #define MCX_REG_PPCNT_GRP_RFC2863 0x01
486 1.15 jmcneill #define MCX_REG_PPCNT_GRP_RFC2819 0x02
487 1.15 jmcneill #define MCX_REG_PPCNT_GRP_RFC3635 0x03
488 1.15 jmcneill #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10
489 1.15 jmcneill #define MCX_REG_PPCNT_GRP_PER_TC 0x11
490 1.15 jmcneill #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11
491 1.15 jmcneill
492 1.15 jmcneill uint8_t ppcnt_clr;
493 1.15 jmcneill uint8_t ppcnt_reserved1[2];
494 1.15 jmcneill uint8_t ppcnt_prio_tc;
495 1.15 jmcneill #define MCX_REG_PPCNT_CLR (1 << 7)
496 1.15 jmcneill
497 1.15 jmcneill uint8_t ppcnt_counter_set[248];
498 1.15 jmcneill } __packed __aligned(8);
499 1.15 jmcneill CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256);
500 1.15 jmcneill CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) %
501 1.15 jmcneill sizeof(uint64_t)) == 0);
502 1.15 jmcneill
503 1.15 jmcneill enum mcx_ppcnt_ieee8023 {
504 1.15 jmcneill frames_transmitted_ok,
505 1.15 jmcneill frames_received_ok,
506 1.15 jmcneill frame_check_sequence_errors,
507 1.15 jmcneill alignment_errors,
508 1.15 jmcneill octets_transmitted_ok,
509 1.15 jmcneill octets_received_ok,
510 1.15 jmcneill multicast_frames_xmitted_ok,
511 1.15 jmcneill broadcast_frames_xmitted_ok,
512 1.15 jmcneill multicast_frames_received_ok,
513 1.15 jmcneill broadcast_frames_received_ok,
514 1.15 jmcneill in_range_length_errors,
515 1.15 jmcneill out_of_range_length_field,
516 1.15 jmcneill frame_too_long_errors,
517 1.15 jmcneill symbol_error_during_carrier,
518 1.15 jmcneill mac_control_frames_transmitted,
519 1.15 jmcneill mac_control_frames_received,
520 1.15 jmcneill unsupported_opcodes_received,
521 1.15 jmcneill pause_mac_ctrl_frames_received,
522 1.15 jmcneill pause_mac_ctrl_frames_transmitted,
523 1.15 jmcneill
524 1.15 jmcneill mcx_ppcnt_ieee8023_count
525 1.15 jmcneill };
526 1.15 jmcneill CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98);
527 1.15 jmcneill
528 1.15 jmcneill enum mcx_ppcnt_rfc2863 {
529 1.15 jmcneill in_octets,
530 1.15 jmcneill in_ucast_pkts,
531 1.15 jmcneill in_discards,
532 1.15 jmcneill in_errors,
533 1.15 jmcneill in_unknown_protos,
534 1.15 jmcneill out_octets,
535 1.15 jmcneill out_ucast_pkts,
536 1.15 jmcneill out_discards,
537 1.15 jmcneill out_errors,
538 1.15 jmcneill in_multicast_pkts,
539 1.15 jmcneill in_broadcast_pkts,
540 1.15 jmcneill out_multicast_pkts,
541 1.15 jmcneill out_broadcast_pkts,
542 1.15 jmcneill
543 1.15 jmcneill mcx_ppcnt_rfc2863_count
544 1.15 jmcneill };
545 1.15 jmcneill CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68);
546 1.15 jmcneill
547 1.15 jmcneill enum mcx_ppcnt_rfc2819 {
548 1.15 jmcneill drop_events,
549 1.15 jmcneill octets,
550 1.15 jmcneill pkts,
551 1.15 jmcneill broadcast_pkts,
552 1.15 jmcneill multicast_pkts,
553 1.15 jmcneill crc_align_errors,
554 1.15 jmcneill undersize_pkts,
555 1.15 jmcneill oversize_pkts,
556 1.15 jmcneill fragments,
557 1.15 jmcneill jabbers,
558 1.15 jmcneill collisions,
559 1.15 jmcneill pkts64octets,
560 1.15 jmcneill pkts65to127octets,
561 1.15 jmcneill pkts128to255octets,
562 1.15 jmcneill pkts256to511octets,
563 1.15 jmcneill pkts512to1023octets,
564 1.15 jmcneill pkts1024to1518octets,
565 1.15 jmcneill pkts1519to2047octets,
566 1.15 jmcneill pkts2048to4095octets,
567 1.15 jmcneill pkts4096to8191octets,
568 1.15 jmcneill pkts8192to10239octets,
569 1.15 jmcneill
570 1.15 jmcneill mcx_ppcnt_rfc2819_count
571 1.15 jmcneill };
572 1.15 jmcneill CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8);
573 1.15 jmcneill
574 1.15 jmcneill enum mcx_ppcnt_rfc3635 {
575 1.15 jmcneill dot3stats_alignment_errors,
576 1.15 jmcneill dot3stats_fcs_errors,
577 1.15 jmcneill dot3stats_single_collision_frames,
578 1.15 jmcneill dot3stats_multiple_collision_frames,
579 1.15 jmcneill dot3stats_sqe_test_errors,
580 1.15 jmcneill dot3stats_deferred_transmissions,
581 1.15 jmcneill dot3stats_late_collisions,
582 1.15 jmcneill dot3stats_excessive_collisions,
583 1.15 jmcneill dot3stats_internal_mac_transmit_errors,
584 1.15 jmcneill dot3stats_carrier_sense_errors,
585 1.15 jmcneill dot3stats_frame_too_longs,
586 1.15 jmcneill dot3stats_internal_mac_receive_errors,
587 1.15 jmcneill dot3stats_symbol_errors,
588 1.15 jmcneill dot3control_in_unknown_opcodes,
589 1.15 jmcneill dot3in_pause_frames,
590 1.15 jmcneill dot3out_pause_frames,
591 1.15 jmcneill
592 1.15 jmcneill mcx_ppcnt_rfc3635_count
593 1.15 jmcneill };
594 1.15 jmcneill CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80);
595 1.15 jmcneill
596 1.15 jmcneill struct mcx_reg_mcam {
597 1.15 jmcneill uint8_t _reserved1[1];
598 1.15 jmcneill uint8_t mcam_feature_group;
599 1.15 jmcneill uint8_t _reserved2[1];
600 1.15 jmcneill uint8_t mcam_access_reg_group;
601 1.15 jmcneill uint8_t _reserved3[4];
602 1.15 jmcneill uint8_t mcam_access_reg_cap_mask[16];
603 1.15 jmcneill uint8_t _reserved4[16];
604 1.15 jmcneill uint8_t mcam_feature_cap_mask[16];
605 1.15 jmcneill uint8_t _reserved5[16];
606 1.15 jmcneill } __packed __aligned(4);
607 1.15 jmcneill
608 1.15 jmcneill #define MCX_BITFIELD_BIT(bf, b) (bf[(sizeof bf - 1) - (b / 8)] & (b % 8))
609 1.15 jmcneill
610 1.15 jmcneill #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP 6
611 1.15 jmcneill
612 1.15 jmcneill struct mcx_reg_mtcap {
613 1.15 jmcneill uint8_t _reserved1[3];
614 1.15 jmcneill uint8_t mtcap_sensor_count;
615 1.15 jmcneill uint8_t _reserved2[4];
616 1.15 jmcneill
617 1.15 jmcneill uint64_t mtcap_sensor_map;
618 1.15 jmcneill };
619 1.15 jmcneill
620 1.15 jmcneill struct mcx_reg_mtmp {
621 1.15 jmcneill uint8_t _reserved1[2];
622 1.15 jmcneill uint16_t mtmp_sensor_index;
623 1.15 jmcneill
624 1.15 jmcneill uint8_t _reserved2[2];
625 1.15 jmcneill uint16_t mtmp_temperature;
626 1.15 jmcneill
627 1.15 jmcneill uint16_t mtmp_mte_mtr;
628 1.15 jmcneill #define MCX_REG_MTMP_MTE (1 << 15)
629 1.15 jmcneill #define MCX_REG_MTMP_MTR (1 << 14)
630 1.15 jmcneill uint16_t mtmp_max_temperature;
631 1.15 jmcneill
632 1.15 jmcneill uint16_t mtmp_tee;
633 1.15 jmcneill #define MCX_REG_MTMP_TEE_NOPE (0 << 14)
634 1.15 jmcneill #define MCX_REG_MTMP_TEE_GENERATE (1 << 14)
635 1.15 jmcneill #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14)
636 1.15 jmcneill uint16_t mtmp_temperature_threshold_hi;
637 1.15 jmcneill
638 1.15 jmcneill uint8_t _reserved3[2];
639 1.15 jmcneill uint16_t mtmp_temperature_threshold_lo;
640 1.15 jmcneill
641 1.15 jmcneill uint8_t _reserved4[4];
642 1.15 jmcneill
643 1.15 jmcneill uint8_t mtmp_sensor_name[8];
644 1.15 jmcneill };
645 1.15 jmcneill CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20);
646 1.15 jmcneill CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18);
647 1.15 jmcneill
648 1.1 jmcneill #define MCX_MCIA_EEPROM_BYTES 32
649 1.1 jmcneill struct mcx_reg_mcia {
650 1.1 jmcneill uint8_t rm_l;
651 1.1 jmcneill uint8_t rm_module;
652 1.1 jmcneill uint8_t rm_reserved0;
653 1.1 jmcneill uint8_t rm_status;
654 1.1 jmcneill uint8_t rm_i2c_addr;
655 1.1 jmcneill uint8_t rm_page_num;
656 1.1 jmcneill uint16_t rm_dev_addr;
657 1.1 jmcneill uint16_t rm_reserved1;
658 1.1 jmcneill uint16_t rm_size;
659 1.1 jmcneill uint32_t rm_reserved2;
660 1.1 jmcneill uint8_t rm_data[48];
661 1.1 jmcneill } __packed __aligned(4);
662 1.1 jmcneill
663 1.1 jmcneill struct mcx_cmd_query_issi_in {
664 1.1 jmcneill uint16_t cmd_opcode;
665 1.1 jmcneill uint8_t cmd_reserved0[4];
666 1.1 jmcneill uint16_t cmd_op_mod;
667 1.1 jmcneill uint8_t cmd_reserved1[8];
668 1.1 jmcneill } __packed __aligned(4);
669 1.1 jmcneill
670 1.1 jmcneill struct mcx_cmd_query_issi_il_out {
671 1.1 jmcneill uint8_t cmd_status;
672 1.1 jmcneill uint8_t cmd_reserved0[3];
673 1.1 jmcneill uint32_t cmd_syndrome;
674 1.1 jmcneill uint8_t cmd_reserved1[2];
675 1.1 jmcneill uint16_t cmd_current_issi;
676 1.1 jmcneill uint8_t cmd_reserved2[4];
677 1.1 jmcneill } __packed __aligned(4);
678 1.1 jmcneill
679 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
680 1.1 jmcneill
681 1.1 jmcneill struct mcx_cmd_query_issi_mb_out {
682 1.1 jmcneill uint8_t cmd_reserved2[16];
683 1.1 jmcneill uint8_t cmd_supported_issi[80]; /* very big endian */
684 1.1 jmcneill } __packed __aligned(4);
685 1.1 jmcneill
686 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
687 1.1 jmcneill
688 1.1 jmcneill struct mcx_cmd_set_issi_in {
689 1.1 jmcneill uint16_t cmd_opcode;
690 1.1 jmcneill uint8_t cmd_reserved0[4];
691 1.1 jmcneill uint16_t cmd_op_mod;
692 1.1 jmcneill uint8_t cmd_reserved1[2];
693 1.1 jmcneill uint16_t cmd_current_issi;
694 1.1 jmcneill uint8_t cmd_reserved2[4];
695 1.1 jmcneill } __packed __aligned(4);
696 1.1 jmcneill
697 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
698 1.1 jmcneill
699 1.1 jmcneill struct mcx_cmd_set_issi_out {
700 1.1 jmcneill uint8_t cmd_status;
701 1.1 jmcneill uint8_t cmd_reserved0[3];
702 1.1 jmcneill uint32_t cmd_syndrome;
703 1.1 jmcneill uint8_t cmd_reserved1[8];
704 1.1 jmcneill } __packed __aligned(4);
705 1.1 jmcneill
706 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
707 1.1 jmcneill
708 1.1 jmcneill struct mcx_cmd_query_pages_in {
709 1.1 jmcneill uint16_t cmd_opcode;
710 1.1 jmcneill uint8_t cmd_reserved0[4];
711 1.1 jmcneill uint16_t cmd_op_mod;
712 1.1 jmcneill #define MCX_CMD_QUERY_PAGES_BOOT 0x01
713 1.1 jmcneill #define MCX_CMD_QUERY_PAGES_INIT 0x02
714 1.1 jmcneill #define MCX_CMD_QUERY_PAGES_REGULAR 0x03
715 1.1 jmcneill uint8_t cmd_reserved1[8];
716 1.1 jmcneill } __packed __aligned(4);
717 1.1 jmcneill
718 1.1 jmcneill struct mcx_cmd_query_pages_out {
719 1.1 jmcneill uint8_t cmd_status;
720 1.1 jmcneill uint8_t cmd_reserved0[3];
721 1.1 jmcneill uint32_t cmd_syndrome;
722 1.1 jmcneill uint8_t cmd_reserved1[2];
723 1.1 jmcneill uint16_t cmd_func_id;
724 1.15 jmcneill int32_t cmd_num_pages;
725 1.1 jmcneill } __packed __aligned(4);
726 1.1 jmcneill
727 1.1 jmcneill struct mcx_cmd_manage_pages_in {
728 1.1 jmcneill uint16_t cmd_opcode;
729 1.1 jmcneill uint8_t cmd_reserved0[4];
730 1.1 jmcneill uint16_t cmd_op_mod;
731 1.1 jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
732 1.1 jmcneill 0x00
733 1.1 jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
734 1.1 jmcneill 0x01
735 1.1 jmcneill #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
736 1.1 jmcneill 0x02
737 1.1 jmcneill uint8_t cmd_reserved1[2];
738 1.1 jmcneill uint16_t cmd_func_id;
739 1.1 jmcneill uint32_t cmd_input_num_entries;
740 1.1 jmcneill } __packed __aligned(4);
741 1.1 jmcneill
742 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
743 1.1 jmcneill
744 1.1 jmcneill struct mcx_cmd_manage_pages_out {
745 1.1 jmcneill uint8_t cmd_status;
746 1.1 jmcneill uint8_t cmd_reserved0[3];
747 1.1 jmcneill uint32_t cmd_syndrome;
748 1.1 jmcneill uint32_t cmd_output_num_entries;
749 1.1 jmcneill uint8_t cmd_reserved1[4];
750 1.1 jmcneill } __packed __aligned(4);
751 1.1 jmcneill
752 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
753 1.1 jmcneill
754 1.1 jmcneill struct mcx_cmd_query_hca_cap_in {
755 1.1 jmcneill uint16_t cmd_opcode;
756 1.1 jmcneill uint8_t cmd_reserved0[4];
757 1.1 jmcneill uint16_t cmd_op_mod;
758 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0)
759 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0)
760 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1)
761 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1)
762 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1)
763 1.1 jmcneill uint8_t cmd_reserved1[8];
764 1.1 jmcneill } __packed __aligned(4);
765 1.1 jmcneill
766 1.1 jmcneill struct mcx_cmd_query_hca_cap_out {
767 1.1 jmcneill uint8_t cmd_status;
768 1.1 jmcneill uint8_t cmd_reserved0[3];
769 1.1 jmcneill uint32_t cmd_syndrome;
770 1.1 jmcneill uint8_t cmd_reserved1[8];
771 1.1 jmcneill } __packed __aligned(4);
772 1.1 jmcneill
773 1.1 jmcneill #define MCX_HCA_CAP_LEN 0x1000
774 1.1 jmcneill #define MCX_HCA_CAP_NMAILBOXES \
775 1.1 jmcneill (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
776 1.1 jmcneill
777 1.1 jmcneill #if __GNUC_PREREQ__(4, 3)
778 1.1 jmcneill #define __counter__ __COUNTER__
779 1.1 jmcneill #else
780 1.1 jmcneill #define __counter__ __LINE__
781 1.1 jmcneill #endif
782 1.1 jmcneill
783 1.1 jmcneill #define __token(_tok, _num) _tok##_num
784 1.1 jmcneill #define _token(_tok, _num) __token(_tok, _num)
785 1.1 jmcneill #define __reserved__ _token(__reserved, __counter__)
786 1.1 jmcneill
787 1.1 jmcneill struct mcx_cap_device {
788 1.1 jmcneill uint8_t reserved0[16];
789 1.1 jmcneill
790 1.1 jmcneill uint8_t log_max_srq_sz;
791 1.1 jmcneill uint8_t log_max_qp_sz;
792 1.1 jmcneill uint8_t __reserved__[1];
793 1.1 jmcneill uint8_t log_max_qp; /* 5 bits */
794 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f
795 1.1 jmcneill
796 1.1 jmcneill uint8_t __reserved__[1];
797 1.1 jmcneill uint8_t log_max_srq; /* 5 bits */
798 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f
799 1.1 jmcneill uint8_t __reserved__[2];
800 1.1 jmcneill
801 1.1 jmcneill uint8_t __reserved__[1];
802 1.1 jmcneill uint8_t log_max_cq_sz;
803 1.1 jmcneill uint8_t __reserved__[1];
804 1.1 jmcneill uint8_t log_max_cq; /* 5 bits */
805 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f
806 1.1 jmcneill
807 1.1 jmcneill uint8_t log_max_eq_sz;
808 1.1 jmcneill uint8_t log_max_mkey; /* 6 bits */
809 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f
810 1.1 jmcneill uint8_t __reserved__[1];
811 1.1 jmcneill uint8_t log_max_eq; /* 4 bits */
812 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f
813 1.1 jmcneill
814 1.1 jmcneill uint8_t max_indirection;
815 1.1 jmcneill uint8_t log_max_mrw_sz; /* 7 bits */
816 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f
817 1.1 jmcneill uint8_t teardown_log_max_msf_list_size;
818 1.1 jmcneill #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80
819 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
820 1.1 jmcneill 0x3f
821 1.1 jmcneill uint8_t log_max_klm_list_size; /* 6 bits */
822 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
823 1.1 jmcneill 0x3f
824 1.1 jmcneill
825 1.1 jmcneill uint8_t __reserved__[1];
826 1.1 jmcneill uint8_t log_max_ra_req_dc; /* 6 bits */
827 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f
828 1.1 jmcneill uint8_t __reserved__[1];
829 1.1 jmcneill uint8_t log_max_ra_res_dc; /* 6 bits */
830 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
831 1.1 jmcneill 0x3f
832 1.1 jmcneill
833 1.1 jmcneill uint8_t __reserved__[1];
834 1.1 jmcneill uint8_t log_max_ra_req_qp; /* 6 bits */
835 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
836 1.1 jmcneill 0x3f
837 1.1 jmcneill uint8_t __reserved__[1];
838 1.1 jmcneill uint8_t log_max_ra_res_qp; /* 6 bits */
839 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
840 1.1 jmcneill 0x3f
841 1.1 jmcneill
842 1.1 jmcneill uint8_t flags1;
843 1.1 jmcneill #define MCX_CAP_DEVICE_END_PAD 0x80
844 1.1 jmcneill #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40
845 1.1 jmcneill #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
846 1.1 jmcneill 0x20
847 1.1 jmcneill #define MCX_CAP_DEVICE_START_PAD 0x10
848 1.1 jmcneill #define MCX_CAP_DEVICE_128BYTE_CACHELINE \
849 1.1 jmcneill 0x08
850 1.1 jmcneill uint8_t __reserved__[1];
851 1.1 jmcneill uint16_t gid_table_size;
852 1.1 jmcneill
853 1.1 jmcneill uint16_t flags2;
854 1.1 jmcneill #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000
855 1.1 jmcneill #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000
856 1.1 jmcneill #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
857 1.1 jmcneill 0x2000
858 1.1 jmcneill #define MCX_CAP_DEVICE_DEBUG 0x1000
859 1.1 jmcneill #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
860 1.1 jmcneill 0x8000
861 1.1 jmcneill #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000
862 1.1 jmcneill #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff
863 1.1 jmcneill uint16_t pkey_table_size;
864 1.1 jmcneill
865 1.1 jmcneill uint8_t flags3;
866 1.1 jmcneill #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
867 1.1 jmcneill 0x80
868 1.1 jmcneill #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
869 1.1 jmcneill 0x40
870 1.1 jmcneill #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20
871 1.1 jmcneill #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10
872 1.1 jmcneill #define MCX_CAP_DEVICE_ETS 0x04
873 1.1 jmcneill #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02
874 1.1 jmcneill #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
875 1.1 jmcneill 0x01
876 1.1 jmcneill uint8_t local_ca_ack_delay; /* 5 bits */
877 1.1 jmcneill #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
878 1.1 jmcneill 0x1f
879 1.15 jmcneill #define MCX_CAP_DEVICE_MCAM_REG 0x40
880 1.1 jmcneill uint8_t port_type;
881 1.1 jmcneill #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
882 1.1 jmcneill 0x80
883 1.1 jmcneill #define MCX_CAP_DEVICE_PORT_TYPE 0x03
884 1.15 jmcneill #define MCX_CAP_DEVICE_PORT_TYPE_ETH 0x01
885 1.1 jmcneill uint8_t num_ports;
886 1.1 jmcneill
887 1.1 jmcneill uint8_t snapshot_log_max_msg;
888 1.1 jmcneill #define MCX_CAP_DEVICE_SNAPSHOT 0x80
889 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f
890 1.1 jmcneill uint8_t max_tc; /* 4 bits */
891 1.1 jmcneill #define MCX_CAP_DEVICE_MAX_TC 0x0f
892 1.1 jmcneill uint8_t flags4;
893 1.1 jmcneill #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80
894 1.1 jmcneill #define MCX_CAP_DEVICE_DCBX 0x40
895 1.1 jmcneill #define MCX_CAP_DEVICE_ROL_S 0x02
896 1.1 jmcneill #define MCX_CAP_DEVICE_ROL_G 0x01
897 1.1 jmcneill uint8_t wol;
898 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_S 0x40
899 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_G 0x20
900 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_A 0x10
901 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_B 0x08
902 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_M 0x04
903 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_U 0x02
904 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_P 0x01
905 1.1 jmcneill
906 1.1 jmcneill uint16_t stat_rate_support;
907 1.1 jmcneill uint8_t __reserved__[1];
908 1.1 jmcneill uint8_t cqe_version; /* 4 bits */
909 1.1 jmcneill #define MCX_CAP_DEVICE_CQE_VERSION 0x0f
910 1.1 jmcneill
911 1.1 jmcneill uint32_t flags5;
912 1.1 jmcneill #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
913 1.1 jmcneill 0x80000000
914 1.1 jmcneill #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000
915 1.1 jmcneill #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
916 1.1 jmcneill 0x10000000
917 1.1 jmcneill #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
918 1.1 jmcneill 0x08000000
919 1.1 jmcneill #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000
920 1.1 jmcneill #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000
921 1.1 jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000
922 1.1 jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000
923 1.1 jmcneill #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000
924 1.1 jmcneill #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000
925 1.1 jmcneill #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800
926 1.1 jmcneill #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400
927 1.1 jmcneill #define MCX_CAP_DEVICE_SHO 0x00000100
928 1.1 jmcneill #define MCX_CAP_DEVICE_TPH 0x00000080
929 1.1 jmcneill #define MCX_CAP_DEVICE_RF 0x00000040
930 1.1 jmcneill #define MCX_CAP_DEVICE_DCT 0x00000020
931 1.1 jmcneill #define MCX_CAP_DEVICE_QOS 0x00000010
932 1.1 jmcneill #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008
933 1.1 jmcneill #define MCX_CAP_DEVICE_ROCE 0x00000004
934 1.1 jmcneill #define MCX_CAP_DEVICE_ATOMIC 0x00000002
935 1.1 jmcneill
936 1.1 jmcneill uint32_t flags6;
937 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_OI 0x80000000
938 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000
939 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000
940 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
941 1.1 jmcneill 0x10000000
942 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000
943 1.1 jmcneill #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000
944 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000
945 1.1 jmcneill #define MCX_CAP_DEVICE_PG 0x01000000
946 1.1 jmcneill #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000
947 1.1 jmcneill #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
948 1.1 jmcneill 0x00400000
949 1.1 jmcneill #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
950 1.1 jmcneill 0x00200000
951 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
952 1.1 jmcneill 0x00100000
953 1.1 jmcneill #define MCX_CAP_DEVICE_CD 0x00080000
954 1.1 jmcneill #define MCX_CAP_DEVICE_ATM 0x00040000
955 1.1 jmcneill #define MCX_CAP_DEVICE_APM 0x00020000
956 1.1 jmcneill #define MCX_CAP_DEVICE_IMAICL 0x00010000
957 1.1 jmcneill #define MCX_CAP_DEVICE_QKV 0x00000200
958 1.1 jmcneill #define MCX_CAP_DEVICE_PKV 0x00000100
959 1.1 jmcneill #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080
960 1.1 jmcneill #define MCX_CAP_DEVICE_XRC 0x00000008
961 1.1 jmcneill #define MCX_CAP_DEVICE_UD 0x00000004
962 1.1 jmcneill #define MCX_CAP_DEVICE_UC 0x00000002
963 1.1 jmcneill #define MCX_CAP_DEVICE_RC 0x00000001
964 1.1 jmcneill
965 1.1 jmcneill uint8_t uar_flags;
966 1.1 jmcneill #define MCX_CAP_DEVICE_UAR_4K 0x80
967 1.1 jmcneill uint8_t uar_sz; /* 6 bits */
968 1.1 jmcneill #define MCX_CAP_DEVICE_UAR_SZ 0x3f
969 1.1 jmcneill uint8_t __reserved__[1];
970 1.1 jmcneill uint8_t log_pg_sz;
971 1.1 jmcneill
972 1.1 jmcneill uint8_t flags7;
973 1.1 jmcneill #define MCX_CAP_DEVICE_BF 0x80
974 1.1 jmcneill #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40
975 1.1 jmcneill #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
976 1.1 jmcneill 0x20
977 1.1 jmcneill uint8_t log_bf_reg_size; /* 5 bits */
978 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f
979 1.1 jmcneill uint8_t __reserved__[2];
980 1.1 jmcneill
981 1.1 jmcneill uint16_t num_of_diagnostic_counters;
982 1.1 jmcneill uint16_t max_wqe_sz_sq;
983 1.1 jmcneill
984 1.1 jmcneill uint8_t __reserved__[2];
985 1.1 jmcneill uint16_t max_wqe_sz_rq;
986 1.1 jmcneill
987 1.1 jmcneill uint8_t __reserved__[2];
988 1.1 jmcneill uint16_t max_wqe_sz_sq_dc;
989 1.1 jmcneill
990 1.1 jmcneill uint32_t max_qp_mcg; /* 25 bits */
991 1.1 jmcneill #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff
992 1.1 jmcneill
993 1.1 jmcneill uint8_t __reserved__[3];
994 1.1 jmcneill uint8_t log_max_mcq;
995 1.1 jmcneill
996 1.1 jmcneill uint8_t log_max_transport_domain; /* 5 bits */
997 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
998 1.1 jmcneill 0x1f
999 1.1 jmcneill uint8_t log_max_pd; /* 5 bits */
1000 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f
1001 1.1 jmcneill uint8_t __reserved__[1];
1002 1.1 jmcneill uint8_t log_max_xrcd; /* 5 bits */
1003 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f
1004 1.1 jmcneill
1005 1.1 jmcneill uint8_t __reserved__[2];
1006 1.1 jmcneill uint16_t max_flow_counter;
1007 1.1 jmcneill
1008 1.1 jmcneill uint8_t log_max_rq; /* 5 bits */
1009 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f
1010 1.1 jmcneill uint8_t log_max_sq; /* 5 bits */
1011 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f
1012 1.1 jmcneill uint8_t log_max_tir; /* 5 bits */
1013 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f
1014 1.1 jmcneill uint8_t log_max_tis; /* 5 bits */
1015 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f
1016 1.1 jmcneill
1017 1.1 jmcneill uint8_t flags8;
1018 1.1 jmcneill #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
1019 1.1 jmcneill 0x80
1020 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f
1021 1.1 jmcneill uint8_t log_max_rqt; /* 5 bits */
1022 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f
1023 1.1 jmcneill uint8_t log_max_rqt_size; /* 5 bits */
1024 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f
1025 1.1 jmcneill uint8_t log_max_tis_per_sq; /* 5 bits */
1026 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
1027 1.1 jmcneill 0x1f
1028 1.15 jmcneill
1029 1.15 jmcneill uint8_t flags9;
1030 1.15 jmcneill #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \
1031 1.15 jmcneill 0x80
1032 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \
1033 1.15 jmcneill 0x1f
1034 1.15 jmcneill uint8_t log_min_stride_sz_rq; /* 5 bits */
1035 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \
1036 1.15 jmcneill 0x1f
1037 1.15 jmcneill uint8_t log_max_stride_sz_sq; /* 5 bits */
1038 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \
1039 1.15 jmcneill 0x1f
1040 1.15 jmcneill uint8_t log_min_stride_sz_sq; /* 5 bits */
1041 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \
1042 1.15 jmcneill 0x1f
1043 1.15 jmcneill
1044 1.15 jmcneill uint8_t log_max_hairpin_queues;
1045 1.15 jmcneill #define MXC_CAP_DEVICE_HAIRPIN 0x80
1046 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \
1047 1.15 jmcneill 0x1f
1048 1.15 jmcneill uint8_t log_min_hairpin_queues;
1049 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \
1050 1.15 jmcneill 0x1f
1051 1.15 jmcneill uint8_t log_max_hairpin_num_packets;
1052 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \
1053 1.15 jmcneill 0x1f
1054 1.15 jmcneill uint8_t log_max_mq_sz;
1055 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \
1056 1.15 jmcneill 0x1f
1057 1.15 jmcneill
1058 1.15 jmcneill uint8_t log_min_hairpin_wq_data_sz;
1059 1.15 jmcneill #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \
1060 1.15 jmcneill 0x80
1061 1.15 jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \
1062 1.15 jmcneill 0x40
1063 1.15 jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \
1064 1.15 jmcneill 0x20
1065 1.15 jmcneill #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \
1066 1.15 jmcneill 0x1f
1067 1.15 jmcneill uint8_t log_max_vlan_list;
1068 1.15 jmcneill #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \
1069 1.15 jmcneill 0x80
1070 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \
1071 1.15 jmcneill 0x1f
1072 1.15 jmcneill uint8_t log_max_current_mc_list;
1073 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \
1074 1.15 jmcneill 0x1f
1075 1.15 jmcneill uint8_t log_max_current_uc_list;
1076 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \
1077 1.15 jmcneill 0x1f
1078 1.15 jmcneill
1079 1.15 jmcneill uint8_t __reserved__[4];
1080 1.15 jmcneill
1081 1.15 jmcneill uint32_t create_qp_start_hint; /* 24 bits */
1082 1.15 jmcneill
1083 1.15 jmcneill uint8_t log_max_uctx; /* 5 bits */
1084 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UCTX 0x1f
1085 1.15 jmcneill uint8_t log_max_umem; /* 5 bits */
1086 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UMEM 0x1f
1087 1.15 jmcneill uint16_t max_num_eqs;
1088 1.15 jmcneill
1089 1.15 jmcneill uint8_t log_max_l2_table; /* 5 bits */
1090 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE 0x1f
1091 1.15 jmcneill uint8_t __reserved__[1];
1092 1.15 jmcneill uint16_t log_uar_page_sz;
1093 1.15 jmcneill
1094 1.15 jmcneill uint8_t __reserved__[8];
1095 1.15 jmcneill
1096 1.15 jmcneill uint32_t device_frequency_mhz;
1097 1.15 jmcneill uint32_t device_frequency_khz;
1098 1.1 jmcneill } __packed __aligned(8);
1099 1.1 jmcneill
1100 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
1101 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
1102 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
1103 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
1104 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
1105 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
1106 1.15 jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98);
1107 1.15 jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c);
1108 1.1 jmcneill CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
1109 1.1 jmcneill
1110 1.1 jmcneill struct mcx_cmd_set_driver_version_in {
1111 1.1 jmcneill uint16_t cmd_opcode;
1112 1.1 jmcneill uint8_t cmd_reserved0[4];
1113 1.1 jmcneill uint16_t cmd_op_mod;
1114 1.1 jmcneill uint8_t cmd_reserved1[8];
1115 1.1 jmcneill } __packed __aligned(4);
1116 1.1 jmcneill
1117 1.1 jmcneill struct mcx_cmd_set_driver_version_out {
1118 1.1 jmcneill uint8_t cmd_status;
1119 1.1 jmcneill uint8_t cmd_reserved0[3];
1120 1.1 jmcneill uint32_t cmd_syndrome;
1121 1.1 jmcneill uint8_t cmd_reserved1[8];
1122 1.1 jmcneill } __packed __aligned(4);
1123 1.1 jmcneill
1124 1.1 jmcneill struct mcx_cmd_set_driver_version {
1125 1.1 jmcneill uint8_t cmd_driver_version[64];
1126 1.1 jmcneill } __packed __aligned(8);
1127 1.1 jmcneill
1128 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_in {
1129 1.1 jmcneill uint16_t cmd_opcode;
1130 1.1 jmcneill uint8_t cmd_reserved0[4];
1131 1.1 jmcneill uint16_t cmd_op_mod;
1132 1.1 jmcneill uint8_t cmd_reserved1[4];
1133 1.1 jmcneill uint32_t cmd_field_select;
1134 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04
1135 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10
1136 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40
1137 1.1 jmcneill } __packed __aligned(4);
1138 1.1 jmcneill
1139 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_out {
1140 1.1 jmcneill uint8_t cmd_status;
1141 1.1 jmcneill uint8_t cmd_reserved0[3];
1142 1.1 jmcneill uint32_t cmd_syndrome;
1143 1.1 jmcneill uint8_t cmd_reserved1[8];
1144 1.1 jmcneill } __packed __aligned(4);
1145 1.1 jmcneill
1146 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_in {
1147 1.1 jmcneill uint16_t cmd_opcode;
1148 1.1 jmcneill uint8_t cmd_reserved0[4];
1149 1.1 jmcneill uint16_t cmd_op_mod;
1150 1.1 jmcneill uint8_t cmd_reserved1[4];
1151 1.1 jmcneill uint8_t cmd_allowed_list_type;
1152 1.1 jmcneill uint8_t cmd_reserved2[3];
1153 1.1 jmcneill } __packed __aligned(4);
1154 1.1 jmcneill
1155 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_out {
1156 1.1 jmcneill uint8_t cmd_status;
1157 1.1 jmcneill uint8_t cmd_reserved0[3];
1158 1.1 jmcneill uint32_t cmd_syndrome;
1159 1.1 jmcneill uint8_t cmd_reserved1[8];
1160 1.1 jmcneill } __packed __aligned(4);
1161 1.1 jmcneill
1162 1.1 jmcneill struct mcx_nic_vport_ctx {
1163 1.1 jmcneill uint32_t vp_min_wqe_inline_mode;
1164 1.1 jmcneill uint8_t vp_reserved0[32];
1165 1.1 jmcneill uint32_t vp_mtu;
1166 1.1 jmcneill uint8_t vp_reserved1[200];
1167 1.1 jmcneill uint16_t vp_flags;
1168 1.1 jmcneill #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0)
1169 1.1 jmcneill #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24)
1170 1.1 jmcneill #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24)
1171 1.1 jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13)
1172 1.1 jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14)
1173 1.1 jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15)
1174 1.1 jmcneill uint16_t vp_allowed_list_size;
1175 1.1 jmcneill uint64_t vp_perm_addr;
1176 1.1 jmcneill uint8_t vp_reserved2[4];
1177 1.1 jmcneill /* allowed list follows */
1178 1.1 jmcneill } __packed __aligned(4);
1179 1.1 jmcneill
1180 1.1 jmcneill struct mcx_counter {
1181 1.1 jmcneill uint64_t packets;
1182 1.1 jmcneill uint64_t octets;
1183 1.1 jmcneill } __packed __aligned(4);
1184 1.1 jmcneill
1185 1.1 jmcneill struct mcx_nic_vport_counters {
1186 1.1 jmcneill struct mcx_counter rx_err;
1187 1.1 jmcneill struct mcx_counter tx_err;
1188 1.1 jmcneill uint8_t reserved0[64]; /* 0x30 */
1189 1.1 jmcneill struct mcx_counter rx_bcast;
1190 1.1 jmcneill struct mcx_counter tx_bcast;
1191 1.1 jmcneill struct mcx_counter rx_ucast;
1192 1.1 jmcneill struct mcx_counter tx_ucast;
1193 1.1 jmcneill struct mcx_counter rx_mcast;
1194 1.1 jmcneill struct mcx_counter tx_mcast;
1195 1.1 jmcneill uint8_t reserved1[0x210 - 0xd0];
1196 1.1 jmcneill } __packed __aligned(4);
1197 1.1 jmcneill
1198 1.1 jmcneill struct mcx_cmd_query_vport_counters_in {
1199 1.1 jmcneill uint16_t cmd_opcode;
1200 1.1 jmcneill uint8_t cmd_reserved0[4];
1201 1.1 jmcneill uint16_t cmd_op_mod;
1202 1.1 jmcneill uint8_t cmd_reserved1[8];
1203 1.1 jmcneill } __packed __aligned(4);
1204 1.1 jmcneill
1205 1.1 jmcneill struct mcx_cmd_query_vport_counters_mb_in {
1206 1.1 jmcneill uint8_t cmd_reserved0[8];
1207 1.1 jmcneill uint8_t cmd_clear;
1208 1.1 jmcneill uint8_t cmd_reserved1[7];
1209 1.1 jmcneill } __packed __aligned(4);
1210 1.1 jmcneill
1211 1.1 jmcneill struct mcx_cmd_query_vport_counters_out {
1212 1.1 jmcneill uint8_t cmd_status;
1213 1.1 jmcneill uint8_t cmd_reserved0[3];
1214 1.1 jmcneill uint32_t cmd_syndrome;
1215 1.1 jmcneill uint8_t cmd_reserved1[8];
1216 1.1 jmcneill } __packed __aligned(4);
1217 1.1 jmcneill
1218 1.1 jmcneill struct mcx_cmd_query_flow_counter_in {
1219 1.1 jmcneill uint16_t cmd_opcode;
1220 1.1 jmcneill uint8_t cmd_reserved0[4];
1221 1.1 jmcneill uint16_t cmd_op_mod;
1222 1.1 jmcneill uint8_t cmd_reserved1[8];
1223 1.1 jmcneill } __packed __aligned(4);
1224 1.1 jmcneill
1225 1.1 jmcneill struct mcx_cmd_query_flow_counter_mb_in {
1226 1.1 jmcneill uint8_t cmd_reserved0[8];
1227 1.1 jmcneill uint8_t cmd_clear;
1228 1.1 jmcneill uint8_t cmd_reserved1[5];
1229 1.1 jmcneill uint16_t cmd_flow_counter_id;
1230 1.1 jmcneill } __packed __aligned(4);
1231 1.1 jmcneill
1232 1.1 jmcneill struct mcx_cmd_query_flow_counter_out {
1233 1.1 jmcneill uint8_t cmd_status;
1234 1.1 jmcneill uint8_t cmd_reserved0[3];
1235 1.1 jmcneill uint32_t cmd_syndrome;
1236 1.1 jmcneill uint8_t cmd_reserved1[8];
1237 1.1 jmcneill } __packed __aligned(4);
1238 1.1 jmcneill
1239 1.1 jmcneill struct mcx_cmd_alloc_uar_in {
1240 1.1 jmcneill uint16_t cmd_opcode;
1241 1.1 jmcneill uint8_t cmd_reserved0[4];
1242 1.1 jmcneill uint16_t cmd_op_mod;
1243 1.1 jmcneill uint8_t cmd_reserved1[8];
1244 1.1 jmcneill } __packed __aligned(4);
1245 1.1 jmcneill
1246 1.1 jmcneill struct mcx_cmd_alloc_uar_out {
1247 1.1 jmcneill uint8_t cmd_status;
1248 1.1 jmcneill uint8_t cmd_reserved0[3];
1249 1.1 jmcneill uint32_t cmd_syndrome;
1250 1.1 jmcneill uint32_t cmd_uar;
1251 1.1 jmcneill uint8_t cmd_reserved1[4];
1252 1.1 jmcneill } __packed __aligned(4);
1253 1.1 jmcneill
1254 1.1 jmcneill struct mcx_cmd_query_special_ctx_in {
1255 1.1 jmcneill uint16_t cmd_opcode;
1256 1.1 jmcneill uint8_t cmd_reserved0[4];
1257 1.1 jmcneill uint16_t cmd_op_mod;
1258 1.1 jmcneill uint8_t cmd_reserved1[8];
1259 1.1 jmcneill } __packed __aligned(4);
1260 1.1 jmcneill
1261 1.1 jmcneill struct mcx_cmd_query_special_ctx_out {
1262 1.1 jmcneill uint8_t cmd_status;
1263 1.1 jmcneill uint8_t cmd_reserved0[3];
1264 1.1 jmcneill uint32_t cmd_syndrome;
1265 1.1 jmcneill uint8_t cmd_reserved1[4];
1266 1.1 jmcneill uint32_t cmd_resd_lkey;
1267 1.1 jmcneill } __packed __aligned(4);
1268 1.1 jmcneill
1269 1.1 jmcneill struct mcx_eq_ctx {
1270 1.1 jmcneill uint32_t eq_status;
1271 1.15 jmcneill #define MCX_EQ_CTX_STATE_SHIFT 8
1272 1.15 jmcneill #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT)
1273 1.15 jmcneill #define MCX_EQ_CTX_STATE_ARMED 0x9
1274 1.15 jmcneill #define MCX_EQ_CTX_STATE_FIRED 0xa
1275 1.1 jmcneill #define MCX_EQ_CTX_OI_SHIFT 17
1276 1.1 jmcneill #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT)
1277 1.1 jmcneill #define MCX_EQ_CTX_EC_SHIFT 18
1278 1.1 jmcneill #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT)
1279 1.1 jmcneill #define MCX_EQ_CTX_STATUS_SHIFT 28
1280 1.1 jmcneill #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT)
1281 1.15 jmcneill #define MCX_EQ_CTX_STATUS_OK 0x0
1282 1.15 jmcneill #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa
1283 1.1 jmcneill uint32_t eq_reserved1;
1284 1.1 jmcneill uint32_t eq_page_offset;
1285 1.1 jmcneill #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5
1286 1.1 jmcneill uint32_t eq_uar_size;
1287 1.1 jmcneill #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff
1288 1.1 jmcneill #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24
1289 1.1 jmcneill uint32_t eq_reserved2;
1290 1.1 jmcneill uint8_t eq_reserved3[3];
1291 1.1 jmcneill uint8_t eq_intr;
1292 1.1 jmcneill uint32_t eq_log_page_size;
1293 1.1 jmcneill #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24
1294 1.1 jmcneill uint32_t eq_reserved4[3];
1295 1.1 jmcneill uint32_t eq_consumer_counter;
1296 1.1 jmcneill uint32_t eq_producer_counter;
1297 1.1 jmcneill #define MCX_EQ_CTX_COUNTER_MASK 0xffffff
1298 1.1 jmcneill uint32_t eq_reserved5[4];
1299 1.1 jmcneill } __packed __aligned(4);
1300 1.1 jmcneill
1301 1.1 jmcneill CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
1302 1.1 jmcneill
1303 1.1 jmcneill struct mcx_cmd_create_eq_in {
1304 1.1 jmcneill uint16_t cmd_opcode;
1305 1.1 jmcneill uint8_t cmd_reserved0[4];
1306 1.1 jmcneill uint16_t cmd_op_mod;
1307 1.1 jmcneill uint8_t cmd_reserved1[8];
1308 1.1 jmcneill } __packed __aligned(4);
1309 1.1 jmcneill
1310 1.1 jmcneill struct mcx_cmd_create_eq_mb_in {
1311 1.1 jmcneill struct mcx_eq_ctx cmd_eq_ctx;
1312 1.1 jmcneill uint8_t cmd_reserved0[8];
1313 1.1 jmcneill uint64_t cmd_event_bitmask;
1314 1.1 jmcneill #define MCX_EVENT_TYPE_COMPLETION 0x00
1315 1.1 jmcneill #define MCX_EVENT_TYPE_CQ_ERROR 0x04
1316 1.1 jmcneill #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08
1317 1.1 jmcneill #define MCX_EVENT_TYPE_PORT_CHANGE 0x09
1318 1.1 jmcneill #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a
1319 1.1 jmcneill #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b
1320 1.1 jmcneill #define MCX_EVENT_TYPE_LAST_WQE 0x13
1321 1.1 jmcneill uint8_t cmd_reserved1[176];
1322 1.1 jmcneill } __packed __aligned(4);
1323 1.1 jmcneill
1324 1.1 jmcneill struct mcx_cmd_create_eq_out {
1325 1.1 jmcneill uint8_t cmd_status;
1326 1.1 jmcneill uint8_t cmd_reserved0[3];
1327 1.1 jmcneill uint32_t cmd_syndrome;
1328 1.1 jmcneill uint32_t cmd_eqn;
1329 1.1 jmcneill uint8_t cmd_reserved1[4];
1330 1.1 jmcneill } __packed __aligned(4);
1331 1.1 jmcneill
1332 1.15 jmcneill struct mcx_cmd_query_eq_in {
1333 1.15 jmcneill uint16_t cmd_opcode;
1334 1.15 jmcneill uint8_t cmd_reserved0[4];
1335 1.15 jmcneill uint16_t cmd_op_mod;
1336 1.15 jmcneill uint32_t cmd_eqn;
1337 1.15 jmcneill uint8_t cmd_reserved1[4];
1338 1.15 jmcneill } __packed __aligned(4);
1339 1.15 jmcneill
1340 1.15 jmcneill struct mcx_cmd_query_eq_out {
1341 1.15 jmcneill uint8_t cmd_status;
1342 1.15 jmcneill uint8_t cmd_reserved0[3];
1343 1.15 jmcneill uint32_t cmd_syndrome;
1344 1.15 jmcneill uint8_t cmd_reserved1[8];
1345 1.15 jmcneill } __packed __aligned(4);
1346 1.15 jmcneill
1347 1.1 jmcneill struct mcx_eq_entry {
1348 1.1 jmcneill uint8_t eq_reserved1;
1349 1.1 jmcneill uint8_t eq_event_type;
1350 1.1 jmcneill uint8_t eq_reserved2;
1351 1.1 jmcneill uint8_t eq_event_sub_type;
1352 1.1 jmcneill
1353 1.1 jmcneill uint8_t eq_reserved3[28];
1354 1.1 jmcneill uint32_t eq_event_data[7];
1355 1.1 jmcneill uint8_t eq_reserved4[2];
1356 1.1 jmcneill uint8_t eq_signature;
1357 1.1 jmcneill uint8_t eq_owner;
1358 1.1 jmcneill #define MCX_EQ_ENTRY_OWNER_INIT 1
1359 1.1 jmcneill } __packed __aligned(4);
1360 1.1 jmcneill
1361 1.1 jmcneill CTASSERT(sizeof(struct mcx_eq_entry) == 64);
1362 1.1 jmcneill
1363 1.1 jmcneill struct mcx_cmd_alloc_pd_in {
1364 1.1 jmcneill uint16_t cmd_opcode;
1365 1.1 jmcneill uint8_t cmd_reserved0[4];
1366 1.1 jmcneill uint16_t cmd_op_mod;
1367 1.1 jmcneill uint8_t cmd_reserved1[8];
1368 1.1 jmcneill } __packed __aligned(4);
1369 1.1 jmcneill
1370 1.1 jmcneill struct mcx_cmd_alloc_pd_out {
1371 1.1 jmcneill uint8_t cmd_status;
1372 1.1 jmcneill uint8_t cmd_reserved0[3];
1373 1.1 jmcneill uint32_t cmd_syndrome;
1374 1.1 jmcneill uint32_t cmd_pd;
1375 1.1 jmcneill uint8_t cmd_reserved1[4];
1376 1.1 jmcneill } __packed __aligned(4);
1377 1.1 jmcneill
1378 1.1 jmcneill struct mcx_cmd_alloc_td_in {
1379 1.1 jmcneill uint16_t cmd_opcode;
1380 1.1 jmcneill uint8_t cmd_reserved0[4];
1381 1.1 jmcneill uint16_t cmd_op_mod;
1382 1.1 jmcneill uint8_t cmd_reserved1[8];
1383 1.1 jmcneill } __packed __aligned(4);
1384 1.1 jmcneill
1385 1.1 jmcneill struct mcx_cmd_alloc_td_out {
1386 1.1 jmcneill uint8_t cmd_status;
1387 1.1 jmcneill uint8_t cmd_reserved0[3];
1388 1.1 jmcneill uint32_t cmd_syndrome;
1389 1.1 jmcneill uint32_t cmd_tdomain;
1390 1.1 jmcneill uint8_t cmd_reserved1[4];
1391 1.1 jmcneill } __packed __aligned(4);
1392 1.1 jmcneill
1393 1.1 jmcneill struct mcx_cmd_create_tir_in {
1394 1.1 jmcneill uint16_t cmd_opcode;
1395 1.1 jmcneill uint8_t cmd_reserved0[4];
1396 1.1 jmcneill uint16_t cmd_op_mod;
1397 1.1 jmcneill uint8_t cmd_reserved1[8];
1398 1.1 jmcneill } __packed __aligned(4);
1399 1.1 jmcneill
1400 1.1 jmcneill struct mcx_cmd_create_tir_mb_in {
1401 1.1 jmcneill uint8_t cmd_reserved0[20];
1402 1.1 jmcneill uint32_t cmd_disp_type;
1403 1.15 jmcneill #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0
1404 1.15 jmcneill #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1
1405 1.1 jmcneill #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28
1406 1.1 jmcneill uint8_t cmd_reserved1[8];
1407 1.1 jmcneill uint32_t cmd_lro;
1408 1.1 jmcneill uint8_t cmd_reserved2[8];
1409 1.1 jmcneill uint32_t cmd_inline_rqn;
1410 1.1 jmcneill uint32_t cmd_indir_table;
1411 1.1 jmcneill uint32_t cmd_tdomain;
1412 1.15 jmcneill #define MCX_TIR_CTX_HASH_TOEPLITZ 2
1413 1.15 jmcneill #define MCX_TIR_CTX_HASH_SHIFT 28
1414 1.1 jmcneill uint8_t cmd_rx_hash_key[40];
1415 1.1 jmcneill uint32_t cmd_rx_hash_sel_outer;
1416 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0)
1417 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1)
1418 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2)
1419 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3)
1420 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31)
1421 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31)
1422 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30)
1423 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30)
1424 1.1 jmcneill uint32_t cmd_rx_hash_sel_inner;
1425 1.1 jmcneill uint8_t cmd_reserved3[152];
1426 1.1 jmcneill } __packed __aligned(4);
1427 1.1 jmcneill
1428 1.1 jmcneill struct mcx_cmd_create_tir_out {
1429 1.1 jmcneill uint8_t cmd_status;
1430 1.1 jmcneill uint8_t cmd_reserved0[3];
1431 1.1 jmcneill uint32_t cmd_syndrome;
1432 1.1 jmcneill uint32_t cmd_tirn;
1433 1.1 jmcneill uint8_t cmd_reserved1[4];
1434 1.1 jmcneill } __packed __aligned(4);
1435 1.1 jmcneill
1436 1.1 jmcneill struct mcx_cmd_destroy_tir_in {
1437 1.1 jmcneill uint16_t cmd_opcode;
1438 1.1 jmcneill uint8_t cmd_reserved0[4];
1439 1.1 jmcneill uint16_t cmd_op_mod;
1440 1.1 jmcneill uint32_t cmd_tirn;
1441 1.1 jmcneill uint8_t cmd_reserved1[4];
1442 1.1 jmcneill } __packed __aligned(4);
1443 1.1 jmcneill
1444 1.1 jmcneill struct mcx_cmd_destroy_tir_out {
1445 1.1 jmcneill uint8_t cmd_status;
1446 1.1 jmcneill uint8_t cmd_reserved0[3];
1447 1.1 jmcneill uint32_t cmd_syndrome;
1448 1.1 jmcneill uint8_t cmd_reserved1[8];
1449 1.1 jmcneill } __packed __aligned(4);
1450 1.1 jmcneill
1451 1.1 jmcneill struct mcx_cmd_create_tis_in {
1452 1.1 jmcneill uint16_t cmd_opcode;
1453 1.1 jmcneill uint8_t cmd_reserved0[4];
1454 1.1 jmcneill uint16_t cmd_op_mod;
1455 1.1 jmcneill uint8_t cmd_reserved1[8];
1456 1.1 jmcneill } __packed __aligned(4);
1457 1.1 jmcneill
1458 1.1 jmcneill struct mcx_cmd_create_tis_mb_in {
1459 1.1 jmcneill uint8_t cmd_reserved[16];
1460 1.1 jmcneill uint32_t cmd_prio;
1461 1.1 jmcneill uint8_t cmd_reserved1[32];
1462 1.1 jmcneill uint32_t cmd_tdomain;
1463 1.1 jmcneill uint8_t cmd_reserved2[120];
1464 1.1 jmcneill } __packed __aligned(4);
1465 1.1 jmcneill
1466 1.1 jmcneill struct mcx_cmd_create_tis_out {
1467 1.1 jmcneill uint8_t cmd_status;
1468 1.1 jmcneill uint8_t cmd_reserved0[3];
1469 1.1 jmcneill uint32_t cmd_syndrome;
1470 1.1 jmcneill uint32_t cmd_tisn;
1471 1.1 jmcneill uint8_t cmd_reserved1[4];
1472 1.1 jmcneill } __packed __aligned(4);
1473 1.1 jmcneill
1474 1.1 jmcneill struct mcx_cmd_destroy_tis_in {
1475 1.1 jmcneill uint16_t cmd_opcode;
1476 1.1 jmcneill uint8_t cmd_reserved0[4];
1477 1.1 jmcneill uint16_t cmd_op_mod;
1478 1.1 jmcneill uint32_t cmd_tisn;
1479 1.1 jmcneill uint8_t cmd_reserved1[4];
1480 1.1 jmcneill } __packed __aligned(4);
1481 1.1 jmcneill
1482 1.1 jmcneill struct mcx_cmd_destroy_tis_out {
1483 1.1 jmcneill uint8_t cmd_status;
1484 1.1 jmcneill uint8_t cmd_reserved0[3];
1485 1.1 jmcneill uint32_t cmd_syndrome;
1486 1.1 jmcneill uint8_t cmd_reserved1[8];
1487 1.1 jmcneill } __packed __aligned(4);
1488 1.1 jmcneill
1489 1.15 jmcneill struct mcx_cmd_create_rqt_in {
1490 1.15 jmcneill uint16_t cmd_opcode;
1491 1.15 jmcneill uint8_t cmd_reserved0[4];
1492 1.15 jmcneill uint16_t cmd_op_mod;
1493 1.15 jmcneill uint8_t cmd_reserved1[8];
1494 1.15 jmcneill } __packed __aligned(4);
1495 1.15 jmcneill
1496 1.15 jmcneill struct mcx_rqt_ctx {
1497 1.15 jmcneill uint8_t cmd_reserved0[20];
1498 1.15 jmcneill uint16_t cmd_reserved1;
1499 1.15 jmcneill uint16_t cmd_rqt_max_size;
1500 1.15 jmcneill uint16_t cmd_reserved2;
1501 1.15 jmcneill uint16_t cmd_rqt_actual_size;
1502 1.15 jmcneill uint8_t cmd_reserved3[212];
1503 1.15 jmcneill } __packed __aligned(4);
1504 1.15 jmcneill
1505 1.15 jmcneill struct mcx_cmd_create_rqt_mb_in {
1506 1.15 jmcneill uint8_t cmd_reserved0[16];
1507 1.15 jmcneill struct mcx_rqt_ctx cmd_rqt;
1508 1.15 jmcneill } __packed __aligned(4);
1509 1.15 jmcneill
1510 1.15 jmcneill struct mcx_cmd_create_rqt_out {
1511 1.15 jmcneill uint8_t cmd_status;
1512 1.15 jmcneill uint8_t cmd_reserved0[3];
1513 1.15 jmcneill uint32_t cmd_syndrome;
1514 1.15 jmcneill uint32_t cmd_rqtn;
1515 1.15 jmcneill uint8_t cmd_reserved1[4];
1516 1.15 jmcneill } __packed __aligned(4);
1517 1.15 jmcneill
1518 1.15 jmcneill struct mcx_cmd_destroy_rqt_in {
1519 1.15 jmcneill uint16_t cmd_opcode;
1520 1.15 jmcneill uint8_t cmd_reserved0[4];
1521 1.15 jmcneill uint16_t cmd_op_mod;
1522 1.15 jmcneill uint32_t cmd_rqtn;
1523 1.15 jmcneill uint8_t cmd_reserved1[4];
1524 1.15 jmcneill } __packed __aligned(4);
1525 1.15 jmcneill
1526 1.15 jmcneill struct mcx_cmd_destroy_rqt_out {
1527 1.15 jmcneill uint8_t cmd_status;
1528 1.15 jmcneill uint8_t cmd_reserved0[3];
1529 1.15 jmcneill uint32_t cmd_syndrome;
1530 1.15 jmcneill uint8_t cmd_reserved1[8];
1531 1.15 jmcneill } __packed __aligned(4);
1532 1.15 jmcneill
1533 1.1 jmcneill struct mcx_cq_ctx {
1534 1.1 jmcneill uint32_t cq_status;
1535 1.15 jmcneill #define MCX_CQ_CTX_STATUS_SHIFT 28
1536 1.15 jmcneill #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT)
1537 1.15 jmcneill #define MCX_CQ_CTX_STATUS_OK 0x0
1538 1.15 jmcneill #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9
1539 1.15 jmcneill #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa
1540 1.15 jmcneill #define MCX_CQ_CTX_STATE_SHIFT 8
1541 1.15 jmcneill #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT)
1542 1.15 jmcneill #define MCX_CQ_CTX_STATE_SOLICITED 0x6
1543 1.15 jmcneill #define MCX_CQ_CTX_STATE_ARMED 0x9
1544 1.15 jmcneill #define MCX_CQ_CTX_STATE_FIRED 0xa
1545 1.1 jmcneill uint32_t cq_reserved1;
1546 1.1 jmcneill uint32_t cq_page_offset;
1547 1.1 jmcneill uint32_t cq_uar_size;
1548 1.1 jmcneill #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff
1549 1.1 jmcneill #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24
1550 1.1 jmcneill uint32_t cq_period_max_count;
1551 1.1 jmcneill #define MCX_CQ_CTX_PERIOD_SHIFT 16
1552 1.1 jmcneill uint32_t cq_eqn;
1553 1.1 jmcneill uint32_t cq_log_page_size;
1554 1.1 jmcneill #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24
1555 1.1 jmcneill uint32_t cq_reserved2;
1556 1.1 jmcneill uint32_t cq_last_notified;
1557 1.1 jmcneill uint32_t cq_last_solicit;
1558 1.1 jmcneill uint32_t cq_consumer_counter;
1559 1.1 jmcneill uint32_t cq_producer_counter;
1560 1.1 jmcneill uint8_t cq_reserved3[8];
1561 1.1 jmcneill uint64_t cq_doorbell;
1562 1.1 jmcneill } __packed __aligned(4);
1563 1.1 jmcneill
1564 1.1 jmcneill CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
1565 1.1 jmcneill
1566 1.1 jmcneill struct mcx_cmd_create_cq_in {
1567 1.1 jmcneill uint16_t cmd_opcode;
1568 1.1 jmcneill uint8_t cmd_reserved0[4];
1569 1.1 jmcneill uint16_t cmd_op_mod;
1570 1.1 jmcneill uint8_t cmd_reserved1[8];
1571 1.1 jmcneill } __packed __aligned(4);
1572 1.1 jmcneill
1573 1.1 jmcneill struct mcx_cmd_create_cq_mb_in {
1574 1.1 jmcneill struct mcx_cq_ctx cmd_cq_ctx;
1575 1.1 jmcneill uint8_t cmd_reserved1[192];
1576 1.1 jmcneill } __packed __aligned(4);
1577 1.1 jmcneill
1578 1.1 jmcneill struct mcx_cmd_create_cq_out {
1579 1.1 jmcneill uint8_t cmd_status;
1580 1.1 jmcneill uint8_t cmd_reserved0[3];
1581 1.1 jmcneill uint32_t cmd_syndrome;
1582 1.1 jmcneill uint32_t cmd_cqn;
1583 1.1 jmcneill uint8_t cmd_reserved1[4];
1584 1.1 jmcneill } __packed __aligned(4);
1585 1.1 jmcneill
1586 1.1 jmcneill struct mcx_cmd_destroy_cq_in {
1587 1.1 jmcneill uint16_t cmd_opcode;
1588 1.1 jmcneill uint8_t cmd_reserved0[4];
1589 1.1 jmcneill uint16_t cmd_op_mod;
1590 1.1 jmcneill uint32_t cmd_cqn;
1591 1.1 jmcneill uint8_t cmd_reserved1[4];
1592 1.1 jmcneill } __packed __aligned(4);
1593 1.1 jmcneill
1594 1.1 jmcneill struct mcx_cmd_destroy_cq_out {
1595 1.1 jmcneill uint8_t cmd_status;
1596 1.1 jmcneill uint8_t cmd_reserved0[3];
1597 1.1 jmcneill uint32_t cmd_syndrome;
1598 1.1 jmcneill uint8_t cmd_reserved1[8];
1599 1.1 jmcneill } __packed __aligned(4);
1600 1.1 jmcneill
1601 1.15 jmcneill struct mcx_cmd_query_cq_in {
1602 1.15 jmcneill uint16_t cmd_opcode;
1603 1.15 jmcneill uint8_t cmd_reserved0[4];
1604 1.15 jmcneill uint16_t cmd_op_mod;
1605 1.15 jmcneill uint32_t cmd_cqn;
1606 1.15 jmcneill uint8_t cmd_reserved1[4];
1607 1.15 jmcneill } __packed __aligned(4);
1608 1.15 jmcneill
1609 1.15 jmcneill struct mcx_cmd_query_cq_out {
1610 1.15 jmcneill uint8_t cmd_status;
1611 1.15 jmcneill uint8_t cmd_reserved0[3];
1612 1.15 jmcneill uint32_t cmd_syndrome;
1613 1.15 jmcneill uint8_t cmd_reserved1[8];
1614 1.15 jmcneill } __packed __aligned(4);
1615 1.15 jmcneill
1616 1.1 jmcneill struct mcx_cq_entry {
1617 1.1 jmcneill uint32_t __reserved__;
1618 1.1 jmcneill uint32_t cq_lro;
1619 1.1 jmcneill uint32_t cq_lro_ack_seq_num;
1620 1.1 jmcneill uint32_t cq_rx_hash;
1621 1.1 jmcneill uint8_t cq_rx_hash_type;
1622 1.1 jmcneill uint8_t cq_ml_path;
1623 1.1 jmcneill uint16_t __reserved__;
1624 1.1 jmcneill uint32_t cq_checksum;
1625 1.1 jmcneill uint32_t __reserved__;
1626 1.1 jmcneill uint32_t cq_flags;
1627 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26)
1628 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25)
1629 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24)
1630 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16)
1631 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff)
1632 1.15 jmcneill
1633 1.1 jmcneill uint32_t cq_lro_srqn;
1634 1.1 jmcneill uint32_t __reserved__[2];
1635 1.1 jmcneill uint32_t cq_byte_cnt;
1636 1.1 jmcneill uint64_t cq_timestamp;
1637 1.1 jmcneill uint8_t cq_rx_drops;
1638 1.1 jmcneill uint8_t cq_flow_tag[3];
1639 1.1 jmcneill uint16_t cq_wqe_count;
1640 1.1 jmcneill uint8_t cq_signature;
1641 1.1 jmcneill uint8_t cq_opcode_owner;
1642 1.1 jmcneill #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0)
1643 1.1 jmcneill #define MCX_CQ_ENTRY_FLAG_SE (1 << 1)
1644 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_SHIFT 2
1645 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_SHIFT 4
1646 1.1 jmcneill
1647 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0
1648 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1
1649 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2
1650 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3
1651 1.1 jmcneill
1652 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ 0
1653 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND 2
1654 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13
1655 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14
1656 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_INVALID 15
1657 1.1 jmcneill
1658 1.1 jmcneill } __packed __aligned(4);
1659 1.1 jmcneill
1660 1.1 jmcneill CTASSERT(sizeof(struct mcx_cq_entry) == 64);
1661 1.1 jmcneill
1662 1.1 jmcneill struct mcx_cq_doorbell {
1663 1.1 jmcneill uint32_t db_update_ci;
1664 1.1 jmcneill uint32_t db_arm_ci;
1665 1.1 jmcneill #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28
1666 1.1 jmcneill #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24)
1667 1.1 jmcneill #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff)
1668 1.1 jmcneill } __packed __aligned(8);
1669 1.1 jmcneill
1670 1.1 jmcneill struct mcx_wq_ctx {
1671 1.1 jmcneill uint8_t wq_type;
1672 1.1 jmcneill #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4)
1673 1.1 jmcneill #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3)
1674 1.1 jmcneill uint8_t wq_reserved0[5];
1675 1.1 jmcneill uint16_t wq_lwm;
1676 1.1 jmcneill uint32_t wq_pd;
1677 1.1 jmcneill uint32_t wq_uar_page;
1678 1.1 jmcneill uint64_t wq_doorbell;
1679 1.1 jmcneill uint32_t wq_hw_counter;
1680 1.1 jmcneill uint32_t wq_sw_counter;
1681 1.1 jmcneill uint16_t wq_log_stride;
1682 1.1 jmcneill uint8_t wq_log_page_sz;
1683 1.1 jmcneill uint8_t wq_log_size;
1684 1.1 jmcneill uint8_t wq_reserved1[156];
1685 1.1 jmcneill } __packed __aligned(4);
1686 1.1 jmcneill
1687 1.1 jmcneill CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
1688 1.1 jmcneill
1689 1.1 jmcneill struct mcx_sq_ctx {
1690 1.1 jmcneill uint32_t sq_flags;
1691 1.5 msaitoh #define MCX_SQ_CTX_RLKEY (1U << 31)
1692 1.1 jmcneill #define MCX_SQ_CTX_FRE_SHIFT (1 << 29)
1693 1.1 jmcneill #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28)
1694 1.1 jmcneill #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24
1695 1.1 jmcneill #define MCX_SQ_CTX_STATE_SHIFT 20
1696 1.15 jmcneill #define MCX_SQ_CTX_STATE_MASK (0xf << 20)
1697 1.15 jmcneill #define MCX_SQ_CTX_STATE_RST 0
1698 1.15 jmcneill #define MCX_SQ_CTX_STATE_RDY 1
1699 1.15 jmcneill #define MCX_SQ_CTX_STATE_ERR 3
1700 1.1 jmcneill uint32_t sq_user_index;
1701 1.1 jmcneill uint32_t sq_cqn;
1702 1.1 jmcneill uint32_t sq_reserved1[5];
1703 1.1 jmcneill uint32_t sq_tis_lst_sz;
1704 1.1 jmcneill #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16
1705 1.1 jmcneill uint32_t sq_reserved2[2];
1706 1.1 jmcneill uint32_t sq_tis_num;
1707 1.1 jmcneill struct mcx_wq_ctx sq_wq;
1708 1.1 jmcneill } __packed __aligned(4);
1709 1.1 jmcneill
1710 1.1 jmcneill struct mcx_sq_entry_seg {
1711 1.1 jmcneill uint32_t sqs_byte_count;
1712 1.1 jmcneill uint32_t sqs_lkey;
1713 1.1 jmcneill uint64_t sqs_addr;
1714 1.1 jmcneill } __packed __aligned(4);
1715 1.1 jmcneill
1716 1.1 jmcneill struct mcx_sq_entry {
1717 1.1 jmcneill /* control segment */
1718 1.1 jmcneill uint32_t sqe_opcode_index;
1719 1.1 jmcneill #define MCX_SQE_WQE_INDEX_SHIFT 8
1720 1.1 jmcneill #define MCX_SQE_WQE_OPCODE_NOP 0x00
1721 1.1 jmcneill #define MCX_SQE_WQE_OPCODE_SEND 0x0a
1722 1.1 jmcneill uint32_t sqe_ds_sq_num;
1723 1.1 jmcneill #define MCX_SQE_SQ_NUM_SHIFT 8
1724 1.1 jmcneill uint32_t sqe_signature;
1725 1.1 jmcneill #define MCX_SQE_SIGNATURE_SHIFT 24
1726 1.1 jmcneill #define MCX_SQE_SOLICITED_EVENT 0x02
1727 1.1 jmcneill #define MCX_SQE_CE_CQE_ON_ERR 0x00
1728 1.1 jmcneill #define MCX_SQE_CE_CQE_FIRST_ERR 0x04
1729 1.1 jmcneill #define MCX_SQE_CE_CQE_ALWAYS 0x08
1730 1.1 jmcneill #define MCX_SQE_CE_CQE_SOLICIT 0x0C
1731 1.1 jmcneill #define MCX_SQE_FM_NO_FENCE 0x00
1732 1.1 jmcneill #define MCX_SQE_FM_SMALL_FENCE 0x40
1733 1.1 jmcneill uint32_t sqe_mkey;
1734 1.1 jmcneill
1735 1.1 jmcneill /* ethernet segment */
1736 1.1 jmcneill uint32_t sqe_reserved1;
1737 1.1 jmcneill uint32_t sqe_mss_csum;
1738 1.1 jmcneill #define MCX_SQE_L4_CSUM (1 << 31)
1739 1.1 jmcneill #define MCX_SQE_L3_CSUM (1 << 30)
1740 1.1 jmcneill uint32_t sqe_reserved2;
1741 1.1 jmcneill uint16_t sqe_inline_header_size;
1742 1.1 jmcneill uint16_t sqe_inline_headers[9];
1743 1.1 jmcneill
1744 1.1 jmcneill /* data segment */
1745 1.1 jmcneill struct mcx_sq_entry_seg sqe_segs[1];
1746 1.1 jmcneill } __packed __aligned(64);
1747 1.1 jmcneill
1748 1.1 jmcneill CTASSERT(sizeof(struct mcx_sq_entry) == 64);
1749 1.1 jmcneill
1750 1.1 jmcneill struct mcx_cmd_create_sq_in {
1751 1.1 jmcneill uint16_t cmd_opcode;
1752 1.1 jmcneill uint8_t cmd_reserved0[4];
1753 1.1 jmcneill uint16_t cmd_op_mod;
1754 1.1 jmcneill uint8_t cmd_reserved1[8];
1755 1.1 jmcneill } __packed __aligned(4);
1756 1.1 jmcneill
1757 1.1 jmcneill struct mcx_cmd_create_sq_out {
1758 1.1 jmcneill uint8_t cmd_status;
1759 1.1 jmcneill uint8_t cmd_reserved0[3];
1760 1.1 jmcneill uint32_t cmd_syndrome;
1761 1.1 jmcneill uint32_t cmd_sqn;
1762 1.1 jmcneill uint8_t cmd_reserved1[4];
1763 1.1 jmcneill } __packed __aligned(4);
1764 1.1 jmcneill
1765 1.1 jmcneill struct mcx_cmd_modify_sq_in {
1766 1.1 jmcneill uint16_t cmd_opcode;
1767 1.1 jmcneill uint8_t cmd_reserved0[4];
1768 1.1 jmcneill uint16_t cmd_op_mod;
1769 1.1 jmcneill uint32_t cmd_sq_state;
1770 1.1 jmcneill uint8_t cmd_reserved1[4];
1771 1.1 jmcneill } __packed __aligned(4);
1772 1.1 jmcneill
1773 1.1 jmcneill struct mcx_cmd_modify_sq_mb_in {
1774 1.1 jmcneill uint32_t cmd_modify_hi;
1775 1.1 jmcneill uint32_t cmd_modify_lo;
1776 1.1 jmcneill uint8_t cmd_reserved0[8];
1777 1.1 jmcneill struct mcx_sq_ctx cmd_sq_ctx;
1778 1.1 jmcneill } __packed __aligned(4);
1779 1.1 jmcneill
1780 1.1 jmcneill struct mcx_cmd_modify_sq_out {
1781 1.1 jmcneill uint8_t cmd_status;
1782 1.1 jmcneill uint8_t cmd_reserved0[3];
1783 1.1 jmcneill uint32_t cmd_syndrome;
1784 1.1 jmcneill uint8_t cmd_reserved1[8];
1785 1.1 jmcneill } __packed __aligned(4);
1786 1.1 jmcneill
1787 1.1 jmcneill struct mcx_cmd_destroy_sq_in {
1788 1.1 jmcneill uint16_t cmd_opcode;
1789 1.1 jmcneill uint8_t cmd_reserved0[4];
1790 1.1 jmcneill uint16_t cmd_op_mod;
1791 1.1 jmcneill uint32_t cmd_sqn;
1792 1.1 jmcneill uint8_t cmd_reserved1[4];
1793 1.1 jmcneill } __packed __aligned(4);
1794 1.1 jmcneill
1795 1.1 jmcneill struct mcx_cmd_destroy_sq_out {
1796 1.1 jmcneill uint8_t cmd_status;
1797 1.1 jmcneill uint8_t cmd_reserved0[3];
1798 1.1 jmcneill uint32_t cmd_syndrome;
1799 1.1 jmcneill uint8_t cmd_reserved1[8];
1800 1.1 jmcneill } __packed __aligned(4);
1801 1.1 jmcneill
1802 1.1 jmcneill
1803 1.1 jmcneill struct mcx_rq_ctx {
1804 1.1 jmcneill uint32_t rq_flags;
1805 1.5 msaitoh #define MCX_RQ_CTX_RLKEY (1U << 31)
1806 1.1 jmcneill #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28)
1807 1.1 jmcneill #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24
1808 1.1 jmcneill #define MCX_RQ_CTX_STATE_SHIFT 20
1809 1.15 jmcneill #define MCX_RQ_CTX_STATE_MASK (0xf << 20)
1810 1.15 jmcneill #define MCX_RQ_CTX_STATE_RST 0
1811 1.15 jmcneill #define MCX_RQ_CTX_STATE_RDY 1
1812 1.15 jmcneill #define MCX_RQ_CTX_STATE_ERR 3
1813 1.1 jmcneill #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18)
1814 1.1 jmcneill uint32_t rq_user_index;
1815 1.1 jmcneill uint32_t rq_cqn;
1816 1.1 jmcneill uint32_t rq_reserved1;
1817 1.1 jmcneill uint32_t rq_rmpn;
1818 1.1 jmcneill uint32_t rq_reserved2[7];
1819 1.1 jmcneill struct mcx_wq_ctx rq_wq;
1820 1.1 jmcneill } __packed __aligned(4);
1821 1.1 jmcneill
1822 1.1 jmcneill struct mcx_rq_entry {
1823 1.1 jmcneill uint32_t rqe_byte_count;
1824 1.1 jmcneill uint32_t rqe_lkey;
1825 1.1 jmcneill uint64_t rqe_addr;
1826 1.1 jmcneill } __packed __aligned(16);
1827 1.1 jmcneill
1828 1.1 jmcneill struct mcx_cmd_create_rq_in {
1829 1.1 jmcneill uint16_t cmd_opcode;
1830 1.1 jmcneill uint8_t cmd_reserved0[4];
1831 1.1 jmcneill uint16_t cmd_op_mod;
1832 1.1 jmcneill uint8_t cmd_reserved1[8];
1833 1.1 jmcneill } __packed __aligned(4);
1834 1.1 jmcneill
1835 1.1 jmcneill struct mcx_cmd_create_rq_out {
1836 1.1 jmcneill uint8_t cmd_status;
1837 1.1 jmcneill uint8_t cmd_reserved0[3];
1838 1.1 jmcneill uint32_t cmd_syndrome;
1839 1.1 jmcneill uint32_t cmd_rqn;
1840 1.1 jmcneill uint8_t cmd_reserved1[4];
1841 1.1 jmcneill } __packed __aligned(4);
1842 1.1 jmcneill
1843 1.1 jmcneill struct mcx_cmd_modify_rq_in {
1844 1.1 jmcneill uint16_t cmd_opcode;
1845 1.1 jmcneill uint8_t cmd_reserved0[4];
1846 1.1 jmcneill uint16_t cmd_op_mod;
1847 1.1 jmcneill uint32_t cmd_rq_state;
1848 1.1 jmcneill uint8_t cmd_reserved1[4];
1849 1.1 jmcneill } __packed __aligned(4);
1850 1.1 jmcneill
1851 1.1 jmcneill struct mcx_cmd_modify_rq_mb_in {
1852 1.1 jmcneill uint32_t cmd_modify_hi;
1853 1.1 jmcneill uint32_t cmd_modify_lo;
1854 1.1 jmcneill uint8_t cmd_reserved0[8];
1855 1.1 jmcneill struct mcx_rq_ctx cmd_rq_ctx;
1856 1.1 jmcneill } __packed __aligned(4);
1857 1.1 jmcneill
1858 1.1 jmcneill struct mcx_cmd_modify_rq_out {
1859 1.1 jmcneill uint8_t cmd_status;
1860 1.1 jmcneill uint8_t cmd_reserved0[3];
1861 1.1 jmcneill uint32_t cmd_syndrome;
1862 1.1 jmcneill uint8_t cmd_reserved1[8];
1863 1.1 jmcneill } __packed __aligned(4);
1864 1.1 jmcneill
1865 1.1 jmcneill struct mcx_cmd_destroy_rq_in {
1866 1.1 jmcneill uint16_t cmd_opcode;
1867 1.1 jmcneill uint8_t cmd_reserved0[4];
1868 1.1 jmcneill uint16_t cmd_op_mod;
1869 1.1 jmcneill uint32_t cmd_rqn;
1870 1.1 jmcneill uint8_t cmd_reserved1[4];
1871 1.1 jmcneill } __packed __aligned(4);
1872 1.1 jmcneill
1873 1.1 jmcneill struct mcx_cmd_destroy_rq_out {
1874 1.1 jmcneill uint8_t cmd_status;
1875 1.1 jmcneill uint8_t cmd_reserved0[3];
1876 1.1 jmcneill uint32_t cmd_syndrome;
1877 1.1 jmcneill uint8_t cmd_reserved1[8];
1878 1.1 jmcneill } __packed __aligned(4);
1879 1.1 jmcneill
1880 1.1 jmcneill struct mcx_cmd_create_flow_table_in {
1881 1.1 jmcneill uint16_t cmd_opcode;
1882 1.1 jmcneill uint8_t cmd_reserved0[4];
1883 1.1 jmcneill uint16_t cmd_op_mod;
1884 1.1 jmcneill uint8_t cmd_reserved1[8];
1885 1.1 jmcneill } __packed __aligned(4);
1886 1.1 jmcneill
1887 1.1 jmcneill struct mcx_flow_table_ctx {
1888 1.1 jmcneill uint8_t ft_miss_action;
1889 1.1 jmcneill uint8_t ft_level;
1890 1.1 jmcneill uint8_t ft_reserved0;
1891 1.1 jmcneill uint8_t ft_log_size;
1892 1.1 jmcneill uint32_t ft_table_miss_id;
1893 1.1 jmcneill uint8_t ft_reserved1[28];
1894 1.1 jmcneill } __packed __aligned(4);
1895 1.1 jmcneill
1896 1.1 jmcneill struct mcx_cmd_create_flow_table_mb_in {
1897 1.1 jmcneill uint8_t cmd_table_type;
1898 1.1 jmcneill uint8_t cmd_reserved0[7];
1899 1.1 jmcneill struct mcx_flow_table_ctx cmd_ctx;
1900 1.1 jmcneill } __packed __aligned(4);
1901 1.1 jmcneill
1902 1.1 jmcneill struct mcx_cmd_create_flow_table_out {
1903 1.1 jmcneill uint8_t cmd_status;
1904 1.1 jmcneill uint8_t cmd_reserved0[3];
1905 1.1 jmcneill uint32_t cmd_syndrome;
1906 1.1 jmcneill uint32_t cmd_table_id;
1907 1.1 jmcneill uint8_t cmd_reserved1[4];
1908 1.1 jmcneill } __packed __aligned(4);
1909 1.1 jmcneill
1910 1.1 jmcneill struct mcx_cmd_destroy_flow_table_in {
1911 1.1 jmcneill uint16_t cmd_opcode;
1912 1.1 jmcneill uint8_t cmd_reserved0[4];
1913 1.1 jmcneill uint16_t cmd_op_mod;
1914 1.1 jmcneill uint8_t cmd_reserved1[8];
1915 1.1 jmcneill } __packed __aligned(4);
1916 1.1 jmcneill
1917 1.1 jmcneill struct mcx_cmd_destroy_flow_table_mb_in {
1918 1.1 jmcneill uint8_t cmd_table_type;
1919 1.1 jmcneill uint8_t cmd_reserved0[3];
1920 1.1 jmcneill uint32_t cmd_table_id;
1921 1.1 jmcneill uint8_t cmd_reserved1[40];
1922 1.1 jmcneill } __packed __aligned(4);
1923 1.1 jmcneill
1924 1.1 jmcneill struct mcx_cmd_destroy_flow_table_out {
1925 1.1 jmcneill uint8_t cmd_status;
1926 1.1 jmcneill uint8_t cmd_reserved0[3];
1927 1.1 jmcneill uint32_t cmd_syndrome;
1928 1.1 jmcneill uint8_t cmd_reserved1[8];
1929 1.1 jmcneill } __packed __aligned(4);
1930 1.1 jmcneill
1931 1.1 jmcneill struct mcx_cmd_set_flow_table_root_in {
1932 1.1 jmcneill uint16_t cmd_opcode;
1933 1.1 jmcneill uint8_t cmd_reserved0[4];
1934 1.1 jmcneill uint16_t cmd_op_mod;
1935 1.1 jmcneill uint8_t cmd_reserved1[8];
1936 1.1 jmcneill } __packed __aligned(4);
1937 1.1 jmcneill
1938 1.1 jmcneill struct mcx_cmd_set_flow_table_root_mb_in {
1939 1.1 jmcneill uint8_t cmd_table_type;
1940 1.1 jmcneill uint8_t cmd_reserved0[3];
1941 1.1 jmcneill uint32_t cmd_table_id;
1942 1.1 jmcneill uint8_t cmd_reserved1[56];
1943 1.1 jmcneill } __packed __aligned(4);
1944 1.1 jmcneill
1945 1.1 jmcneill struct mcx_cmd_set_flow_table_root_out {
1946 1.1 jmcneill uint8_t cmd_status;
1947 1.1 jmcneill uint8_t cmd_reserved0[3];
1948 1.1 jmcneill uint32_t cmd_syndrome;
1949 1.1 jmcneill uint8_t cmd_reserved1[8];
1950 1.1 jmcneill } __packed __aligned(4);
1951 1.1 jmcneill
1952 1.1 jmcneill struct mcx_flow_match {
1953 1.1 jmcneill /* outer headers */
1954 1.1 jmcneill uint8_t mc_src_mac[6];
1955 1.1 jmcneill uint16_t mc_ethertype;
1956 1.1 jmcneill uint8_t mc_dest_mac[6];
1957 1.1 jmcneill uint16_t mc_first_vlan;
1958 1.1 jmcneill uint8_t mc_ip_proto;
1959 1.1 jmcneill uint8_t mc_ip_dscp_ecn;
1960 1.1 jmcneill uint8_t mc_vlan_flags;
1961 1.15 jmcneill #define MCX_FLOW_MATCH_IP_FRAG (1 << 5)
1962 1.1 jmcneill uint8_t mc_tcp_flags;
1963 1.1 jmcneill uint16_t mc_tcp_sport;
1964 1.1 jmcneill uint16_t mc_tcp_dport;
1965 1.1 jmcneill uint32_t mc_reserved0;
1966 1.1 jmcneill uint16_t mc_udp_sport;
1967 1.1 jmcneill uint16_t mc_udp_dport;
1968 1.1 jmcneill uint8_t mc_src_ip[16];
1969 1.1 jmcneill uint8_t mc_dest_ip[16];
1970 1.1 jmcneill
1971 1.1 jmcneill /* misc parameters */
1972 1.1 jmcneill uint8_t mc_reserved1[8];
1973 1.1 jmcneill uint16_t mc_second_vlan;
1974 1.1 jmcneill uint8_t mc_reserved2[2];
1975 1.1 jmcneill uint8_t mc_second_vlan_flags;
1976 1.1 jmcneill uint8_t mc_reserved3[15];
1977 1.1 jmcneill uint32_t mc_outer_ipv6_flow_label;
1978 1.1 jmcneill uint8_t mc_reserved4[32];
1979 1.1 jmcneill
1980 1.1 jmcneill uint8_t mc_reserved[384];
1981 1.1 jmcneill } __packed __aligned(4);
1982 1.1 jmcneill
1983 1.1 jmcneill CTASSERT(sizeof(struct mcx_flow_match) == 512);
1984 1.1 jmcneill
1985 1.1 jmcneill struct mcx_cmd_create_flow_group_in {
1986 1.1 jmcneill uint16_t cmd_opcode;
1987 1.1 jmcneill uint8_t cmd_reserved0[4];
1988 1.1 jmcneill uint16_t cmd_op_mod;
1989 1.1 jmcneill uint8_t cmd_reserved1[8];
1990 1.1 jmcneill } __packed __aligned(4);
1991 1.1 jmcneill
1992 1.1 jmcneill struct mcx_cmd_create_flow_group_mb_in {
1993 1.1 jmcneill uint8_t cmd_table_type;
1994 1.1 jmcneill uint8_t cmd_reserved0[3];
1995 1.1 jmcneill uint32_t cmd_table_id;
1996 1.1 jmcneill uint8_t cmd_reserved1[4];
1997 1.1 jmcneill uint32_t cmd_start_flow_index;
1998 1.1 jmcneill uint8_t cmd_reserved2[4];
1999 1.1 jmcneill uint32_t cmd_end_flow_index;
2000 1.1 jmcneill uint8_t cmd_reserved3[23];
2001 1.1 jmcneill uint8_t cmd_match_criteria_enable;
2002 1.1 jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0)
2003 1.1 jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1)
2004 1.1 jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2)
2005 1.1 jmcneill struct mcx_flow_match cmd_match_criteria;
2006 1.1 jmcneill uint8_t cmd_reserved4[448];
2007 1.1 jmcneill } __packed __aligned(4);
2008 1.1 jmcneill
2009 1.1 jmcneill struct mcx_cmd_create_flow_group_out {
2010 1.1 jmcneill uint8_t cmd_status;
2011 1.1 jmcneill uint8_t cmd_reserved0[3];
2012 1.1 jmcneill uint32_t cmd_syndrome;
2013 1.1 jmcneill uint32_t cmd_group_id;
2014 1.1 jmcneill uint8_t cmd_reserved1[4];
2015 1.1 jmcneill } __packed __aligned(4);
2016 1.1 jmcneill
2017 1.1 jmcneill struct mcx_flow_ctx {
2018 1.1 jmcneill uint8_t fc_reserved0[4];
2019 1.1 jmcneill uint32_t fc_group_id;
2020 1.1 jmcneill uint32_t fc_flow_tag;
2021 1.1 jmcneill uint32_t fc_action;
2022 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0)
2023 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1)
2024 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2)
2025 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3)
2026 1.1 jmcneill uint32_t fc_dest_list_size;
2027 1.1 jmcneill uint32_t fc_counter_list_size;
2028 1.1 jmcneill uint8_t fc_reserved1[40];
2029 1.1 jmcneill struct mcx_flow_match fc_match_value;
2030 1.1 jmcneill uint8_t fc_reserved2[192];
2031 1.1 jmcneill } __packed __aligned(4);
2032 1.1 jmcneill
2033 1.1 jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24)
2034 1.1 jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24)
2035 1.1 jmcneill
2036 1.1 jmcneill struct mcx_cmd_destroy_flow_group_in {
2037 1.1 jmcneill uint16_t cmd_opcode;
2038 1.1 jmcneill uint8_t cmd_reserved0[4];
2039 1.1 jmcneill uint16_t cmd_op_mod;
2040 1.1 jmcneill uint8_t cmd_reserved1[8];
2041 1.1 jmcneill } __packed __aligned(4);
2042 1.1 jmcneill
2043 1.1 jmcneill struct mcx_cmd_destroy_flow_group_mb_in {
2044 1.1 jmcneill uint8_t cmd_table_type;
2045 1.1 jmcneill uint8_t cmd_reserved0[3];
2046 1.1 jmcneill uint32_t cmd_table_id;
2047 1.1 jmcneill uint32_t cmd_group_id;
2048 1.1 jmcneill uint8_t cmd_reserved1[36];
2049 1.1 jmcneill } __packed __aligned(4);
2050 1.1 jmcneill
2051 1.1 jmcneill struct mcx_cmd_destroy_flow_group_out {
2052 1.1 jmcneill uint8_t cmd_status;
2053 1.1 jmcneill uint8_t cmd_reserved0[3];
2054 1.1 jmcneill uint32_t cmd_syndrome;
2055 1.1 jmcneill uint8_t cmd_reserved1[8];
2056 1.1 jmcneill } __packed __aligned(4);
2057 1.1 jmcneill
2058 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_in {
2059 1.1 jmcneill uint16_t cmd_opcode;
2060 1.1 jmcneill uint8_t cmd_reserved0[4];
2061 1.1 jmcneill uint16_t cmd_op_mod;
2062 1.1 jmcneill uint8_t cmd_reserved1[8];
2063 1.1 jmcneill } __packed __aligned(4);
2064 1.1 jmcneill
2065 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_mb_in {
2066 1.1 jmcneill uint8_t cmd_table_type;
2067 1.1 jmcneill uint8_t cmd_reserved0[3];
2068 1.1 jmcneill uint32_t cmd_table_id;
2069 1.1 jmcneill uint32_t cmd_modify_enable_mask;
2070 1.1 jmcneill uint8_t cmd_reserved1[4];
2071 1.1 jmcneill uint32_t cmd_flow_index;
2072 1.1 jmcneill uint8_t cmd_reserved2[28];
2073 1.1 jmcneill struct mcx_flow_ctx cmd_flow_ctx;
2074 1.1 jmcneill } __packed __aligned(4);
2075 1.1 jmcneill
2076 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_out {
2077 1.1 jmcneill uint8_t cmd_status;
2078 1.1 jmcneill uint8_t cmd_reserved0[3];
2079 1.1 jmcneill uint32_t cmd_syndrome;
2080 1.1 jmcneill uint8_t cmd_reserved1[8];
2081 1.1 jmcneill } __packed __aligned(4);
2082 1.1 jmcneill
2083 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_in {
2084 1.1 jmcneill uint16_t cmd_opcode;
2085 1.1 jmcneill uint8_t cmd_reserved0[4];
2086 1.1 jmcneill uint16_t cmd_op_mod;
2087 1.1 jmcneill uint8_t cmd_reserved1[8];
2088 1.1 jmcneill } __packed __aligned(4);
2089 1.1 jmcneill
2090 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_in {
2091 1.1 jmcneill uint8_t cmd_table_type;
2092 1.1 jmcneill uint8_t cmd_reserved0[3];
2093 1.1 jmcneill uint32_t cmd_table_id;
2094 1.1 jmcneill uint8_t cmd_reserved1[8];
2095 1.1 jmcneill uint32_t cmd_flow_index;
2096 1.1 jmcneill uint8_t cmd_reserved2[28];
2097 1.1 jmcneill } __packed __aligned(4);
2098 1.1 jmcneill
2099 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_out {
2100 1.1 jmcneill uint8_t cmd_status;
2101 1.1 jmcneill uint8_t cmd_reserved0[3];
2102 1.1 jmcneill uint32_t cmd_syndrome;
2103 1.1 jmcneill uint8_t cmd_reserved1[8];
2104 1.1 jmcneill } __packed __aligned(4);
2105 1.1 jmcneill
2106 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_out {
2107 1.1 jmcneill uint8_t cmd_reserved0[48];
2108 1.1 jmcneill struct mcx_flow_ctx cmd_flow_ctx;
2109 1.1 jmcneill } __packed __aligned(4);
2110 1.1 jmcneill
2111 1.1 jmcneill struct mcx_cmd_delete_flow_table_entry_in {
2112 1.1 jmcneill uint16_t cmd_opcode;
2113 1.1 jmcneill uint8_t cmd_reserved0[4];
2114 1.1 jmcneill uint16_t cmd_op_mod;
2115 1.1 jmcneill uint8_t cmd_reserved1[8];
2116 1.1 jmcneill } __packed __aligned(4);
2117 1.1 jmcneill
2118 1.1 jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in {
2119 1.1 jmcneill uint8_t cmd_table_type;
2120 1.1 jmcneill uint8_t cmd_reserved0[3];
2121 1.1 jmcneill uint32_t cmd_table_id;
2122 1.1 jmcneill uint8_t cmd_reserved1[8];
2123 1.1 jmcneill uint32_t cmd_flow_index;
2124 1.1 jmcneill uint8_t cmd_reserved2[28];
2125 1.1 jmcneill } __packed __aligned(4);
2126 1.1 jmcneill
2127 1.1 jmcneill struct mcx_cmd_delete_flow_table_entry_out {
2128 1.1 jmcneill uint8_t cmd_status;
2129 1.1 jmcneill uint8_t cmd_reserved0[3];
2130 1.1 jmcneill uint32_t cmd_syndrome;
2131 1.1 jmcneill uint8_t cmd_reserved1[8];
2132 1.1 jmcneill } __packed __aligned(4);
2133 1.1 jmcneill
2134 1.1 jmcneill struct mcx_cmd_query_flow_group_in {
2135 1.1 jmcneill uint16_t cmd_opcode;
2136 1.1 jmcneill uint8_t cmd_reserved0[4];
2137 1.1 jmcneill uint16_t cmd_op_mod;
2138 1.1 jmcneill uint8_t cmd_reserved1[8];
2139 1.1 jmcneill } __packed __aligned(4);
2140 1.1 jmcneill
2141 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_in {
2142 1.1 jmcneill uint8_t cmd_table_type;
2143 1.1 jmcneill uint8_t cmd_reserved0[3];
2144 1.1 jmcneill uint32_t cmd_table_id;
2145 1.1 jmcneill uint32_t cmd_group_id;
2146 1.1 jmcneill uint8_t cmd_reserved1[36];
2147 1.1 jmcneill } __packed __aligned(4);
2148 1.1 jmcneill
2149 1.1 jmcneill struct mcx_cmd_query_flow_group_out {
2150 1.1 jmcneill uint8_t cmd_status;
2151 1.1 jmcneill uint8_t cmd_reserved0[3];
2152 1.1 jmcneill uint32_t cmd_syndrome;
2153 1.1 jmcneill uint8_t cmd_reserved1[8];
2154 1.1 jmcneill } __packed __aligned(4);
2155 1.1 jmcneill
2156 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_out {
2157 1.1 jmcneill uint8_t cmd_reserved0[12];
2158 1.1 jmcneill uint32_t cmd_start_flow_index;
2159 1.1 jmcneill uint8_t cmd_reserved1[4];
2160 1.1 jmcneill uint32_t cmd_end_flow_index;
2161 1.1 jmcneill uint8_t cmd_reserved2[20];
2162 1.1 jmcneill uint32_t cmd_match_criteria_enable;
2163 1.1 jmcneill uint8_t cmd_match_criteria[512];
2164 1.1 jmcneill uint8_t cmd_reserved4[448];
2165 1.1 jmcneill } __packed __aligned(4);
2166 1.1 jmcneill
2167 1.1 jmcneill struct mcx_cmd_query_flow_table_in {
2168 1.1 jmcneill uint16_t cmd_opcode;
2169 1.1 jmcneill uint8_t cmd_reserved0[4];
2170 1.1 jmcneill uint16_t cmd_op_mod;
2171 1.1 jmcneill uint8_t cmd_reserved1[8];
2172 1.1 jmcneill } __packed __aligned(4);
2173 1.1 jmcneill
2174 1.1 jmcneill struct mcx_cmd_query_flow_table_mb_in {
2175 1.1 jmcneill uint8_t cmd_table_type;
2176 1.1 jmcneill uint8_t cmd_reserved0[3];
2177 1.1 jmcneill uint32_t cmd_table_id;
2178 1.1 jmcneill uint8_t cmd_reserved1[40];
2179 1.1 jmcneill } __packed __aligned(4);
2180 1.1 jmcneill
2181 1.1 jmcneill struct mcx_cmd_query_flow_table_out {
2182 1.1 jmcneill uint8_t cmd_status;
2183 1.1 jmcneill uint8_t cmd_reserved0[3];
2184 1.1 jmcneill uint32_t cmd_syndrome;
2185 1.1 jmcneill uint8_t cmd_reserved1[8];
2186 1.1 jmcneill } __packed __aligned(4);
2187 1.1 jmcneill
2188 1.1 jmcneill struct mcx_cmd_query_flow_table_mb_out {
2189 1.1 jmcneill uint8_t cmd_reserved0[4];
2190 1.1 jmcneill struct mcx_flow_table_ctx cmd_ctx;
2191 1.1 jmcneill } __packed __aligned(4);
2192 1.1 jmcneill
2193 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_in {
2194 1.1 jmcneill uint16_t cmd_opcode;
2195 1.1 jmcneill uint8_t cmd_reserved0[4];
2196 1.1 jmcneill uint16_t cmd_op_mod;
2197 1.1 jmcneill uint8_t cmd_reserved1[8];
2198 1.1 jmcneill } __packed __aligned(4);
2199 1.1 jmcneill
2200 1.1 jmcneill struct mcx_cmd_query_rq_in {
2201 1.1 jmcneill uint16_t cmd_opcode;
2202 1.1 jmcneill uint8_t cmd_reserved0[4];
2203 1.1 jmcneill uint16_t cmd_op_mod;
2204 1.1 jmcneill uint32_t cmd_rqn;
2205 1.1 jmcneill uint8_t cmd_reserved1[4];
2206 1.1 jmcneill } __packed __aligned(4);
2207 1.1 jmcneill
2208 1.1 jmcneill struct mcx_cmd_query_rq_out {
2209 1.1 jmcneill uint8_t cmd_status;
2210 1.1 jmcneill uint8_t cmd_reserved0[3];
2211 1.1 jmcneill uint32_t cmd_syndrome;
2212 1.1 jmcneill uint8_t cmd_reserved1[8];
2213 1.1 jmcneill } __packed __aligned(4);
2214 1.1 jmcneill
2215 1.1 jmcneill struct mcx_cmd_query_rq_mb_out {
2216 1.1 jmcneill uint8_t cmd_reserved0[16];
2217 1.1 jmcneill struct mcx_rq_ctx cmd_ctx;
2218 1.1 jmcneill };
2219 1.1 jmcneill
2220 1.1 jmcneill struct mcx_cmd_query_sq_in {
2221 1.1 jmcneill uint16_t cmd_opcode;
2222 1.1 jmcneill uint8_t cmd_reserved0[4];
2223 1.1 jmcneill uint16_t cmd_op_mod;
2224 1.1 jmcneill uint32_t cmd_sqn;
2225 1.1 jmcneill uint8_t cmd_reserved1[4];
2226 1.1 jmcneill } __packed __aligned(4);
2227 1.1 jmcneill
2228 1.1 jmcneill struct mcx_cmd_query_sq_out {
2229 1.1 jmcneill uint8_t cmd_status;
2230 1.1 jmcneill uint8_t cmd_reserved0[3];
2231 1.1 jmcneill uint32_t cmd_syndrome;
2232 1.1 jmcneill uint8_t cmd_reserved1[8];
2233 1.1 jmcneill } __packed __aligned(4);
2234 1.1 jmcneill
2235 1.1 jmcneill struct mcx_cmd_query_sq_mb_out {
2236 1.1 jmcneill uint8_t cmd_reserved0[16];
2237 1.1 jmcneill struct mcx_sq_ctx cmd_ctx;
2238 1.1 jmcneill };
2239 1.1 jmcneill
2240 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_out {
2241 1.1 jmcneill uint8_t cmd_status;
2242 1.1 jmcneill uint8_t cmd_reserved0[3];
2243 1.1 jmcneill uint32_t cmd_syndrome;
2244 1.1 jmcneill uint8_t cmd_reserved1[2];
2245 1.1 jmcneill uint16_t cmd_flow_counter_id;
2246 1.1 jmcneill uint8_t cmd_reserved2[4];
2247 1.1 jmcneill } __packed __aligned(4);
2248 1.1 jmcneill
2249 1.1 jmcneill struct mcx_wq_doorbell {
2250 1.1 jmcneill uint32_t db_recv_counter;
2251 1.1 jmcneill uint32_t db_send_counter;
2252 1.1 jmcneill } __packed __aligned(8);
2253 1.1 jmcneill
2254 1.1 jmcneill struct mcx_dmamem {
2255 1.1 jmcneill bus_dmamap_t mxm_map;
2256 1.1 jmcneill bus_dma_segment_t mxm_seg;
2257 1.1 jmcneill int mxm_nsegs;
2258 1.1 jmcneill size_t mxm_size;
2259 1.1 jmcneill void *mxm_kva;
2260 1.1 jmcneill };
2261 1.1 jmcneill #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map)
2262 1.1 jmcneill #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr)
2263 1.1 jmcneill #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva)
2264 1.15 jmcneill #define MCX_DMA_OFF(_mxm, _off) ((void *)((char *)(_mxm)->mxm_kva + (_off)))
2265 1.1 jmcneill #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size)
2266 1.1 jmcneill
2267 1.1 jmcneill struct mcx_hwmem {
2268 1.1 jmcneill bus_dmamap_t mhm_map;
2269 1.1 jmcneill bus_dma_segment_t *mhm_segs;
2270 1.1 jmcneill unsigned int mhm_seg_count;
2271 1.1 jmcneill unsigned int mhm_npages;
2272 1.1 jmcneill };
2273 1.1 jmcneill
2274 1.1 jmcneill struct mcx_slot {
2275 1.1 jmcneill bus_dmamap_t ms_map;
2276 1.1 jmcneill struct mbuf *ms_m;
2277 1.1 jmcneill };
2278 1.1 jmcneill
2279 1.15 jmcneill struct mcx_eq {
2280 1.15 jmcneill int eq_n;
2281 1.15 jmcneill uint32_t eq_cons;
2282 1.15 jmcneill struct mcx_dmamem eq_mem;
2283 1.15 jmcneill };
2284 1.15 jmcneill
2285 1.1 jmcneill struct mcx_cq {
2286 1.1 jmcneill int cq_n;
2287 1.1 jmcneill struct mcx_dmamem cq_mem;
2288 1.15 jmcneill bus_addr_t cq_doorbell;
2289 1.1 jmcneill uint32_t cq_cons;
2290 1.1 jmcneill uint32_t cq_count;
2291 1.1 jmcneill };
2292 1.1 jmcneill
2293 1.1 jmcneill struct mcx_calibration {
2294 1.1 jmcneill uint64_t c_timestamp; /* previous mcx chip time */
2295 1.1 jmcneill uint64_t c_uptime; /* previous kernel nanouptime */
2296 1.1 jmcneill uint64_t c_tbase; /* mcx chip time */
2297 1.1 jmcneill uint64_t c_ubase; /* kernel nanouptime */
2298 1.15 jmcneill uint64_t c_ratio;
2299 1.1 jmcneill };
2300 1.1 jmcneill
2301 1.1 jmcneill #define MCX_CALIBRATE_FIRST 2
2302 1.15 jmcneill #define MCX_CALIBRATE_NORMAL 32
2303 1.1 jmcneill
2304 1.1 jmcneill struct mcx_rxring {
2305 1.1 jmcneill u_int rxr_total;
2306 1.1 jmcneill u_int rxr_inuse;
2307 1.1 jmcneill };
2308 1.1 jmcneill
2309 1.1 jmcneill MBUFQ_HEAD(mcx_mbufq);
2310 1.1 jmcneill
2311 1.15 jmcneill struct mcx_rx {
2312 1.15 jmcneill struct mcx_softc *rx_softc;
2313 1.15 jmcneill
2314 1.15 jmcneill int rx_rqn;
2315 1.15 jmcneill struct mcx_dmamem rx_rq_mem;
2316 1.15 jmcneill struct mcx_slot *rx_slots;
2317 1.15 jmcneill bus_addr_t rx_doorbell;
2318 1.15 jmcneill
2319 1.15 jmcneill uint32_t rx_prod;
2320 1.15 jmcneill callout_t rx_refill;
2321 1.15 jmcneill struct mcx_rxring rx_rxr;
2322 1.15 jmcneill } __aligned(64);
2323 1.15 jmcneill
2324 1.15 jmcneill struct mcx_tx {
2325 1.15 jmcneill struct mcx_softc *tx_softc;
2326 1.16 jmcneill kmutex_t tx_lock;
2327 1.16 jmcneill pcq_t *tx_pcq;
2328 1.16 jmcneill void *tx_softint;
2329 1.15 jmcneill
2330 1.15 jmcneill int tx_uar;
2331 1.15 jmcneill int tx_sqn;
2332 1.15 jmcneill struct mcx_dmamem tx_sq_mem;
2333 1.15 jmcneill struct mcx_slot *tx_slots;
2334 1.15 jmcneill bus_addr_t tx_doorbell;
2335 1.15 jmcneill int tx_bf_offset;
2336 1.15 jmcneill
2337 1.15 jmcneill uint32_t tx_cons;
2338 1.15 jmcneill uint32_t tx_prod;
2339 1.15 jmcneill } __aligned(64);
2340 1.15 jmcneill
2341 1.15 jmcneill struct mcx_queues {
2342 1.15 jmcneill void *q_ihc;
2343 1.15 jmcneill struct mcx_softc *q_sc;
2344 1.15 jmcneill int q_uar;
2345 1.15 jmcneill int q_index;
2346 1.15 jmcneill struct mcx_rx q_rx;
2347 1.15 jmcneill struct mcx_tx q_tx;
2348 1.15 jmcneill struct mcx_cq q_cq;
2349 1.15 jmcneill struct mcx_eq q_eq;
2350 1.15 jmcneill #if NKSTAT > 0
2351 1.15 jmcneill struct kstat *q_kstat;
2352 1.15 jmcneill #endif
2353 1.15 jmcneill };
2354 1.15 jmcneill
2355 1.15 jmcneill struct mcx_flow_group {
2356 1.15 jmcneill int g_id;
2357 1.15 jmcneill int g_table;
2358 1.15 jmcneill int g_start;
2359 1.15 jmcneill int g_size;
2360 1.15 jmcneill };
2361 1.15 jmcneill
2362 1.15 jmcneill #define MCX_FLOW_GROUP_PROMISC 0
2363 1.15 jmcneill #define MCX_FLOW_GROUP_ALLMULTI 1
2364 1.15 jmcneill #define MCX_FLOW_GROUP_MAC 2
2365 1.15 jmcneill #define MCX_FLOW_GROUP_RSS_L4 3
2366 1.15 jmcneill #define MCX_FLOW_GROUP_RSS_L3 4
2367 1.15 jmcneill #define MCX_FLOW_GROUP_RSS_NONE 5
2368 1.15 jmcneill #define MCX_NUM_FLOW_GROUPS 6
2369 1.15 jmcneill
2370 1.15 jmcneill #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \
2371 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_DST_IP
2372 1.15 jmcneill #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \
2373 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_DPORT
2374 1.15 jmcneill
2375 1.15 jmcneill #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\
2376 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV4
2377 1.15 jmcneill #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \
2378 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV6
2379 1.15 jmcneill #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
2380 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV4
2381 1.15 jmcneill #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
2382 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV6
2383 1.15 jmcneill #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4
2384 1.15 jmcneill #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6
2385 1.15 jmcneill
2386 1.15 jmcneill /*
2387 1.15 jmcneill * There are a few different pieces involved in configuring RSS.
2388 1.15 jmcneill * A Receive Queue Table (RQT) is the indirection table that maps packets to
2389 1.15 jmcneill * different rx queues based on a hash value. We only create one, because
2390 1.15 jmcneill * we want to scatter any traffic we can apply RSS to across all our rx
2391 1.15 jmcneill * queues. Anything else will only be delivered to the first rx queue,
2392 1.15 jmcneill * which doesn't require an RQT.
2393 1.15 jmcneill *
2394 1.15 jmcneill * A Transport Interface Receive (TIR) delivers packets to either a single rx
2395 1.15 jmcneill * queue or an RQT, and in the latter case, specifies the set of fields
2396 1.15 jmcneill * hashed, the hash function, and the hash key. We need one of these for each
2397 1.15 jmcneill * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6,
2398 1.15 jmcneill * and one for non-RSS traffic.
2399 1.15 jmcneill *
2400 1.15 jmcneill * Flow tables hold flow table entries in sequence. The first entry that
2401 1.15 jmcneill * matches a packet is applied, sending the packet to either another flow
2402 1.15 jmcneill * table or a TIR. We use one flow table to select packets based on
2403 1.15 jmcneill * destination MAC address, and a second to apply RSS. The entries in the
2404 1.15 jmcneill * first table send matching packets to the second, and the entries in the
2405 1.15 jmcneill * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR.
2406 1.15 jmcneill *
2407 1.15 jmcneill * The flow table entry that delivers packets to an RSS TIR must include match
2408 1.15 jmcneill * criteria that ensure packets delivered to the TIR include all the fields
2409 1.15 jmcneill * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must
2410 1.15 jmcneill * only accept v4 TCP packets. Accordingly, we need flow table entries for
2411 1.15 jmcneill * each TIR.
2412 1.15 jmcneill *
2413 1.15 jmcneill * All of this is a lot more flexible than we need, and we can describe most
2414 1.15 jmcneill * of the stuff we need with a simple array.
2415 1.15 jmcneill *
2416 1.15 jmcneill * An RSS config creates a TIR with hashing enabled on a set of fields,
2417 1.15 jmcneill * pointing to either the first rx queue or the RQT containing all the rx
2418 1.15 jmcneill * queues, and a flow table entry that matches on an ether type and
2419 1.15 jmcneill * optionally an ip proto, that delivers packets to the TIR.
2420 1.15 jmcneill */
2421 1.15 jmcneill static struct mcx_rss_rule {
2422 1.15 jmcneill int hash_sel;
2423 1.15 jmcneill int flow_group;
2424 1.15 jmcneill int ethertype;
2425 1.15 jmcneill int ip_proto;
2426 1.15 jmcneill } mcx_rss_config[] = {
2427 1.15 jmcneill /* udp and tcp for v4/v6 */
2428 1.15 jmcneill { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4,
2429 1.15 jmcneill ETHERTYPE_IP, IPPROTO_TCP },
2430 1.15 jmcneill { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4,
2431 1.15 jmcneill ETHERTYPE_IPV6, IPPROTO_TCP },
2432 1.15 jmcneill { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4,
2433 1.15 jmcneill ETHERTYPE_IP, IPPROTO_UDP },
2434 1.15 jmcneill { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4,
2435 1.15 jmcneill ETHERTYPE_IPV6, IPPROTO_UDP },
2436 1.15 jmcneill
2437 1.15 jmcneill /* other v4/v6 */
2438 1.15 jmcneill { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3,
2439 1.15 jmcneill ETHERTYPE_IP, 0 },
2440 1.15 jmcneill { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3,
2441 1.15 jmcneill ETHERTYPE_IPV6, 0 },
2442 1.15 jmcneill
2443 1.15 jmcneill /* non v4/v6 */
2444 1.15 jmcneill { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 }
2445 1.15 jmcneill };
2446 1.15 jmcneill
2447 1.1 jmcneill struct mcx_softc {
2448 1.1 jmcneill device_t sc_dev;
2449 1.1 jmcneill struct ethercom sc_ec;
2450 1.1 jmcneill struct ifmedia sc_media;
2451 1.1 jmcneill uint64_t sc_media_status;
2452 1.1 jmcneill uint64_t sc_media_active;
2453 1.12 thorpej kmutex_t sc_media_mutex;
2454 1.1 jmcneill
2455 1.1 jmcneill pci_chipset_tag_t sc_pc;
2456 1.1 jmcneill pci_intr_handle_t *sc_intrs;
2457 1.15 jmcneill void *sc_ihc;
2458 1.1 jmcneill pcitag_t sc_tag;
2459 1.1 jmcneill
2460 1.1 jmcneill bus_dma_tag_t sc_dmat;
2461 1.1 jmcneill bus_space_tag_t sc_memt;
2462 1.1 jmcneill bus_space_handle_t sc_memh;
2463 1.1 jmcneill bus_size_t sc_mems;
2464 1.1 jmcneill
2465 1.1 jmcneill struct mcx_dmamem sc_cmdq_mem;
2466 1.1 jmcneill unsigned int sc_cmdq_mask;
2467 1.1 jmcneill unsigned int sc_cmdq_size;
2468 1.1 jmcneill
2469 1.1 jmcneill unsigned int sc_cmdq_token;
2470 1.1 jmcneill
2471 1.1 jmcneill struct mcx_hwmem sc_boot_pages;
2472 1.1 jmcneill struct mcx_hwmem sc_init_pages;
2473 1.1 jmcneill struct mcx_hwmem sc_regular_pages;
2474 1.1 jmcneill
2475 1.1 jmcneill int sc_uar;
2476 1.1 jmcneill int sc_pd;
2477 1.1 jmcneill int sc_tdomain;
2478 1.1 jmcneill uint32_t sc_lkey;
2479 1.15 jmcneill int sc_tis;
2480 1.15 jmcneill int sc_tir[__arraycount(mcx_rss_config)];
2481 1.15 jmcneill int sc_rqt;
2482 1.1 jmcneill
2483 1.1 jmcneill struct mcx_dmamem sc_doorbell_mem;
2484 1.1 jmcneill
2485 1.15 jmcneill struct mcx_eq sc_admin_eq;
2486 1.15 jmcneill struct mcx_eq sc_queue_eq;
2487 1.15 jmcneill
2488 1.1 jmcneill int sc_hardmtu;
2489 1.15 jmcneill int sc_rxbufsz;
2490 1.15 jmcneill
2491 1.15 jmcneill int sc_bf_size;
2492 1.15 jmcneill int sc_max_rqt_size;
2493 1.1 jmcneill
2494 1.1 jmcneill struct workqueue *sc_workq;
2495 1.1 jmcneill struct work sc_port_change;
2496 1.1 jmcneill
2497 1.15 jmcneill int sc_mac_flow_table_id;
2498 1.15 jmcneill int sc_rss_flow_table_id;
2499 1.15 jmcneill struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS];
2500 1.1 jmcneill int sc_promisc_flow_enabled;
2501 1.1 jmcneill int sc_allmulti_flow_enabled;
2502 1.1 jmcneill int sc_mcast_flow_base;
2503 1.1 jmcneill int sc_extra_mcast;
2504 1.1 jmcneill uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];
2505 1.1 jmcneill
2506 1.1 jmcneill struct mcx_calibration sc_calibration[2];
2507 1.1 jmcneill unsigned int sc_calibration_gen;
2508 1.1 jmcneill callout_t sc_calibrate;
2509 1.15 jmcneill uint32_t sc_mhz;
2510 1.15 jmcneill uint32_t sc_khz;
2511 1.1 jmcneill
2512 1.15 jmcneill struct mcx_queues *sc_queues;
2513 1.15 jmcneill unsigned int sc_nqueues;
2514 1.1 jmcneill
2515 1.15 jmcneill int sc_mcam_reg;
2516 1.1 jmcneill
2517 1.15 jmcneill #if NKSTAT > 0
2518 1.15 jmcneill struct kstat *sc_kstat_ieee8023;
2519 1.15 jmcneill struct kstat *sc_kstat_rfc2863;
2520 1.15 jmcneill struct kstat *sc_kstat_rfc2819;
2521 1.15 jmcneill struct kstat *sc_kstat_rfc3635;
2522 1.15 jmcneill unsigned int sc_kstat_mtmp_count;
2523 1.15 jmcneill struct kstat **sc_kstat_mtmp;
2524 1.15 jmcneill #endif
2525 1.1 jmcneill
2526 1.15 jmcneill struct timecounter sc_timecounter;
2527 1.1 jmcneill };
2528 1.1 jmcneill #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
2529 1.1 jmcneill
2530 1.1 jmcneill static int mcx_match(device_t, cfdata_t, void *);
2531 1.1 jmcneill static void mcx_attach(device_t, device_t, void *);
2532 1.1 jmcneill
2533 1.15 jmcneill static void * mcx_establish_intr(struct mcx_softc *, int, kcpuset_t *,
2534 1.15 jmcneill int (*)(void *), void *, const char *);
2535 1.15 jmcneill
2536 1.1 jmcneill static void mcx_rxr_init(struct mcx_rxring *, u_int, u_int);
2537 1.1 jmcneill static u_int mcx_rxr_get(struct mcx_rxring *, u_int);
2538 1.1 jmcneill static void mcx_rxr_put(struct mcx_rxring *, u_int);
2539 1.1 jmcneill static u_int mcx_rxr_inuse(struct mcx_rxring *);
2540 1.1 jmcneill
2541 1.15 jmcneill #if NKSTAT > 0
2542 1.15 jmcneill static void mcx_kstat_attach(struct mcx_softc *);
2543 1.15 jmcneill #endif
2544 1.15 jmcneill
2545 1.15 jmcneill static void mcx_timecounter_attach(struct mcx_softc *);
2546 1.15 jmcneill
2547 1.1 jmcneill static int mcx_version(struct mcx_softc *);
2548 1.1 jmcneill static int mcx_init_wait(struct mcx_softc *);
2549 1.1 jmcneill static int mcx_enable_hca(struct mcx_softc *);
2550 1.1 jmcneill static int mcx_teardown_hca(struct mcx_softc *, uint16_t);
2551 1.1 jmcneill static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
2552 1.1 jmcneill int);
2553 1.1 jmcneill static int mcx_issi(struct mcx_softc *);
2554 1.1 jmcneill static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
2555 1.1 jmcneill static int mcx_hca_max_caps(struct mcx_softc *);
2556 1.1 jmcneill static int mcx_hca_set_caps(struct mcx_softc *);
2557 1.1 jmcneill static int mcx_init_hca(struct mcx_softc *);
2558 1.1 jmcneill static int mcx_set_driver_version(struct mcx_softc *);
2559 1.1 jmcneill static int mcx_iff(struct mcx_softc *);
2560 1.15 jmcneill static int mcx_alloc_uar(struct mcx_softc *, int *);
2561 1.1 jmcneill static int mcx_alloc_pd(struct mcx_softc *);
2562 1.1 jmcneill static int mcx_alloc_tdomain(struct mcx_softc *);
2563 1.15 jmcneill static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int,
2564 1.15 jmcneill uint64_t, int);
2565 1.1 jmcneill static int mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *);
2566 1.1 jmcneill static int mcx_query_special_contexts(struct mcx_softc *);
2567 1.1 jmcneill static int mcx_set_port_mtu(struct mcx_softc *, int);
2568 1.15 jmcneill static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int,
2569 1.15 jmcneill int);
2570 1.15 jmcneill static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *);
2571 1.15 jmcneill static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int,
2572 1.15 jmcneill int);
2573 1.15 jmcneill static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *);
2574 1.15 jmcneill static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *);
2575 1.15 jmcneill static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int);
2576 1.15 jmcneill static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *);
2577 1.15 jmcneill static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *);
2578 1.15 jmcneill static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *,
2579 1.15 jmcneill int *);
2580 1.15 jmcneill static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t,
2581 1.15 jmcneill int *);
2582 1.15 jmcneill static int mcx_destroy_tir(struct mcx_softc *, int);
2583 1.15 jmcneill static int mcx_create_tis(struct mcx_softc *, int *);
2584 1.15 jmcneill static int mcx_destroy_tis(struct mcx_softc *, int);
2585 1.15 jmcneill static int mcx_create_rqt(struct mcx_softc *, int, int *, int *);
2586 1.15 jmcneill static int mcx_destroy_rqt(struct mcx_softc *, int);
2587 1.15 jmcneill static int mcx_create_flow_table(struct mcx_softc *, int, int, int *);
2588 1.15 jmcneill static int mcx_set_flow_table_root(struct mcx_softc *, int);
2589 1.15 jmcneill static int mcx_destroy_flow_table(struct mcx_softc *, int);
2590 1.15 jmcneill static int mcx_create_flow_group(struct mcx_softc *, int, int, int,
2591 1.1 jmcneill int, int, struct mcx_flow_match *);
2592 1.1 jmcneill static int mcx_destroy_flow_group(struct mcx_softc *, int);
2593 1.15 jmcneill static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int,
2594 1.15 jmcneill const uint8_t *, uint32_t);
2595 1.15 jmcneill static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int,
2596 1.15 jmcneill int, int, uint32_t);
2597 1.1 jmcneill static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int);
2598 1.1 jmcneill
2599 1.15 jmcneill #if NKSTAT > 0
2600 1.15 jmcneill static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *);
2601 1.15 jmcneill static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *);
2602 1.15 jmcneill static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *);
2603 1.15 jmcneill static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *);
2604 1.15 jmcneill #endif
2605 1.15 jmcneill
2606 1.1 jmcneill #if 0
2607 1.15 jmcneill static int mcx_dump_flow_table(struct mcx_softc *, int);
2608 1.15 jmcneill static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int);
2609 1.15 jmcneill static int mcx_dump_flow_group(struct mcx_softc *, int);
2610 1.1 jmcneill #endif
2611 1.1 jmcneill
2612 1.1 jmcneill
2613 1.1 jmcneill /*
2614 1.1 jmcneill static void mcx_cmdq_dump(const struct mcx_cmdq_entry *);
2615 1.1 jmcneill static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
2616 1.1 jmcneill */
2617 1.1 jmcneill static void mcx_refill(void *);
2618 1.15 jmcneill static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *,
2619 1.15 jmcneill struct mcx_cq_entry *, struct mcx_mbufq *,
2620 1.15 jmcneill const struct mcx_calibration *);
2621 1.15 jmcneill static int mcx_process_txeof(struct mcx_softc *, struct mcx_tx *,
2622 1.15 jmcneill struct mcx_cq_entry *);
2623 1.15 jmcneill static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *,
2624 1.15 jmcneill struct mcx_cq *);
2625 1.15 jmcneill
2626 1.15 jmcneill static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int);
2627 1.15 jmcneill static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int);
2628 1.15 jmcneill static int mcx_admin_intr(void *);
2629 1.15 jmcneill static int mcx_cq_intr(void *);
2630 1.1 jmcneill
2631 1.1 jmcneill static int mcx_init(struct ifnet *);
2632 1.1 jmcneill static void mcx_stop(struct ifnet *, int);
2633 1.1 jmcneill static int mcx_ioctl(struct ifnet *, u_long, void *);
2634 1.1 jmcneill static void mcx_start(struct ifnet *);
2635 1.16 jmcneill static int mcx_transmit(struct ifnet *, struct mbuf *);
2636 1.16 jmcneill static void mcx_deferred_transmit(void *);
2637 1.1 jmcneill static void mcx_media_add_types(struct mcx_softc *);
2638 1.1 jmcneill static void mcx_media_status(struct ifnet *, struct ifmediareq *);
2639 1.1 jmcneill static int mcx_media_change(struct ifnet *);
2640 1.1 jmcneill #if 0
2641 1.1 jmcneill static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
2642 1.1 jmcneill #endif
2643 1.1 jmcneill static void mcx_port_change(struct work *, void *);
2644 1.1 jmcneill
2645 1.1 jmcneill static void mcx_calibrate_first(struct mcx_softc *);
2646 1.1 jmcneill static void mcx_calibrate(void *);
2647 1.1 jmcneill
2648 1.1 jmcneill static inline uint32_t
2649 1.1 jmcneill mcx_rd(struct mcx_softc *, bus_size_t);
2650 1.1 jmcneill static inline void
2651 1.1 jmcneill mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
2652 1.1 jmcneill static inline void
2653 1.1 jmcneill mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
2654 1.1 jmcneill
2655 1.1 jmcneill static uint64_t mcx_timer(struct mcx_softc *);
2656 1.1 jmcneill
2657 1.1 jmcneill static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
2658 1.1 jmcneill bus_size_t, u_int align);
2659 1.1 jmcneill static void mcx_dmamem_zero(struct mcx_dmamem *);
2660 1.1 jmcneill static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
2661 1.1 jmcneill
2662 1.1 jmcneill static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
2663 1.1 jmcneill unsigned int);
2664 1.1 jmcneill static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
2665 1.1 jmcneill
2666 1.1 jmcneill CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL);
2667 1.1 jmcneill
2668 1.1 jmcneill static const struct {
2669 1.1 jmcneill pci_vendor_id_t vendor;
2670 1.1 jmcneill pci_product_id_t product;
2671 1.1 jmcneill } mcx_devices[] = {
2672 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 },
2673 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF },
2674 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 },
2675 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF },
2676 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 },
2677 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF },
2678 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 },
2679 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF },
2680 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908 },
2681 1.29 msaitoh { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908VF },
2682 1.26 msaitoh { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 },
2683 1.26 msaitoh { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2894 },
2684 1.1 jmcneill };
2685 1.1 jmcneill
2686 1.13 jmcneill struct mcx_eth_proto_capability {
2687 1.13 jmcneill uint64_t cap_media;
2688 1.13 jmcneill uint64_t cap_baudrate;
2689 1.13 jmcneill };
2690 1.13 jmcneill
2691 1.13 jmcneill static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = {
2692 1.13 jmcneill [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) },
2693 1.13 jmcneill [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) },
2694 1.13 jmcneill [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) },
2695 1.13 jmcneill [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) },
2696 1.13 jmcneill [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) },
2697 1.13 jmcneill [MCX_ETHER_CAP_20G_KR2] = { IFM_20G_KR2, IF_Gbps(20) },
2698 1.13 jmcneill [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) },
2699 1.13 jmcneill [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) },
2700 1.13 jmcneill [MCX_ETHER_CAP_56G_R4] = { IFM_56G_R4, IF_Gbps(56) },
2701 1.13 jmcneill [MCX_ETHER_CAP_10G_CR] = { IFM_10G_CR1, IF_Gbps(10) },
2702 1.13 jmcneill [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) },
2703 1.13 jmcneill [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) },
2704 1.13 jmcneill [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) },
2705 1.13 jmcneill [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) },
2706 1.13 jmcneill [MCX_ETHER_CAP_50G_SR2] = { IFM_50G_SR2, IF_Gbps(50) },
2707 1.13 jmcneill [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) },
2708 1.13 jmcneill [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) },
2709 1.13 jmcneill [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) },
2710 1.13 jmcneill [MCX_ETHER_CAP_100G_LR4] = { IFM_100G_LR4, IF_Gbps(100) },
2711 1.13 jmcneill [MCX_ETHER_CAP_100_TX] = { IFM_100_TX, IF_Mbps(100) },
2712 1.13 jmcneill [MCX_ETHER_CAP_1000_T] = { IFM_1000_T, IF_Gbps(1) },
2713 1.13 jmcneill [MCX_ETHER_CAP_10G_T] = { IFM_10G_T, IF_Gbps(10) },
2714 1.13 jmcneill [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) },
2715 1.13 jmcneill [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) },
2716 1.13 jmcneill [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) },
2717 1.13 jmcneill [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) },
2718 1.13 jmcneill [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) },
2719 1.1 jmcneill };
2720 1.1 jmcneill
2721 1.1 jmcneill static int
2722 1.13 jmcneill mcx_get_id(uint32_t val)
2723 1.13 jmcneill {
2724 1.13 jmcneill return be32toh(val) & 0x00ffffff;
2725 1.13 jmcneill }
2726 1.13 jmcneill
2727 1.13 jmcneill static int
2728 1.1 jmcneill mcx_match(device_t parent, cfdata_t cf, void *aux)
2729 1.1 jmcneill {
2730 1.1 jmcneill struct pci_attach_args *pa = aux;
2731 1.1 jmcneill int n;
2732 1.1 jmcneill
2733 1.1 jmcneill for (n = 0; n < __arraycount(mcx_devices); n++) {
2734 1.1 jmcneill if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor &&
2735 1.1 jmcneill PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product)
2736 1.1 jmcneill return 1;
2737 1.1 jmcneill }
2738 1.1 jmcneill
2739 1.1 jmcneill return 0;
2740 1.1 jmcneill }
2741 1.1 jmcneill
2742 1.1 jmcneill void
2743 1.1 jmcneill mcx_attach(device_t parent, device_t self, void *aux)
2744 1.1 jmcneill {
2745 1.1 jmcneill struct mcx_softc *sc = device_private(self);
2746 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
2747 1.1 jmcneill struct pci_attach_args *pa = aux;
2748 1.22 jmcneill struct ifcapreq ifcr;
2749 1.1 jmcneill uint8_t enaddr[ETHER_ADDR_LEN];
2750 1.1 jmcneill int counts[PCI_INTR_TYPE_SIZE];
2751 1.15 jmcneill char intrxname[32];
2752 1.1 jmcneill pcireg_t memtype;
2753 1.1 jmcneill uint32_t r;
2754 1.1 jmcneill unsigned int cq_stride;
2755 1.1 jmcneill unsigned int cq_size;
2756 1.15 jmcneill int i, msix;
2757 1.15 jmcneill kcpuset_t *affinity;
2758 1.1 jmcneill
2759 1.1 jmcneill sc->sc_dev = self;
2760 1.1 jmcneill sc->sc_pc = pa->pa_pc;
2761 1.1 jmcneill sc->sc_tag = pa->pa_tag;
2762 1.11 thorpej if (pci_dma64_available(pa))
2763 1.11 thorpej sc->sc_dmat = pa->pa_dmat64;
2764 1.11 thorpej else
2765 1.11 thorpej sc->sc_dmat = pa->pa_dmat;
2766 1.1 jmcneill
2767 1.1 jmcneill /* Map the PCI memory space */
2768 1.1 jmcneill memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
2769 1.1 jmcneill if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
2770 1.21 jmcneill #ifdef __NetBSD__
2771 1.21 jmcneill 0,
2772 1.21 jmcneill #else
2773 1.21 jmcneill BUS_SPACE_MAP_PREFETCHABLE,
2774 1.21 jmcneill #endif
2775 1.21 jmcneill &sc->sc_memt, &sc->sc_memh,
2776 1.1 jmcneill NULL, &sc->sc_mems)) {
2777 1.1 jmcneill aprint_error(": unable to map register memory\n");
2778 1.1 jmcneill return;
2779 1.1 jmcneill }
2780 1.1 jmcneill
2781 1.1 jmcneill pci_aprint_devinfo(pa, "Ethernet controller");
2782 1.1 jmcneill
2783 1.12 thorpej mutex_init(&sc->sc_media_mutex, MUTEX_DEFAULT, IPL_SOFTNET);
2784 1.12 thorpej
2785 1.1 jmcneill if (mcx_version(sc) != 0) {
2786 1.1 jmcneill /* error printed by mcx_version */
2787 1.1 jmcneill goto unmap;
2788 1.1 jmcneill }
2789 1.1 jmcneill
2790 1.1 jmcneill r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
2791 1.1 jmcneill cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
2792 1.1 jmcneill cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
2793 1.1 jmcneill if (cq_size > MCX_MAX_CQE) {
2794 1.1 jmcneill aprint_error_dev(self,
2795 1.1 jmcneill "command queue size overflow %u\n", cq_size);
2796 1.1 jmcneill goto unmap;
2797 1.1 jmcneill }
2798 1.1 jmcneill if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
2799 1.1 jmcneill aprint_error_dev(self,
2800 1.1 jmcneill "command queue entry size underflow %u\n", cq_stride);
2801 1.1 jmcneill goto unmap;
2802 1.1 jmcneill }
2803 1.1 jmcneill if (cq_stride * cq_size > MCX_PAGE_SIZE) {
2804 1.1 jmcneill aprint_error_dev(self, "command queue page overflow\n");
2805 1.1 jmcneill goto unmap;
2806 1.1 jmcneill }
2807 1.1 jmcneill
2808 1.15 jmcneill if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE,
2809 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
2810 1.1 jmcneill aprint_error_dev(self, "unable to allocate doorbell memory\n");
2811 1.1 jmcneill goto unmap;
2812 1.1 jmcneill }
2813 1.1 jmcneill
2814 1.1 jmcneill if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
2815 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
2816 1.1 jmcneill aprint_error_dev(self, "unable to allocate command queue\n");
2817 1.1 jmcneill goto dbfree;
2818 1.1 jmcneill }
2819 1.1 jmcneill
2820 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
2821 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t),
2822 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
2823 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
2824 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t),
2825 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
2826 1.1 jmcneill
2827 1.1 jmcneill if (mcx_init_wait(sc) != 0) {
2828 1.1 jmcneill aprint_error_dev(self, "timeout waiting for init\n");
2829 1.1 jmcneill goto cqfree;
2830 1.1 jmcneill }
2831 1.1 jmcneill
2832 1.1 jmcneill sc->sc_cmdq_mask = cq_size - 1;
2833 1.1 jmcneill sc->sc_cmdq_size = cq_stride;
2834 1.1 jmcneill
2835 1.1 jmcneill if (mcx_enable_hca(sc) != 0) {
2836 1.1 jmcneill /* error printed by mcx_enable_hca */
2837 1.1 jmcneill goto cqfree;
2838 1.1 jmcneill }
2839 1.1 jmcneill
2840 1.1 jmcneill if (mcx_issi(sc) != 0) {
2841 1.1 jmcneill /* error printed by mcx_issi */
2842 1.1 jmcneill goto teardown;
2843 1.1 jmcneill }
2844 1.1 jmcneill
2845 1.1 jmcneill if (mcx_pages(sc, &sc->sc_boot_pages,
2846 1.1 jmcneill htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
2847 1.1 jmcneill /* error printed by mcx_pages */
2848 1.1 jmcneill goto teardown;
2849 1.1 jmcneill }
2850 1.1 jmcneill
2851 1.1 jmcneill if (mcx_hca_max_caps(sc) != 0) {
2852 1.1 jmcneill /* error printed by mcx_hca_max_caps */
2853 1.1 jmcneill goto teardown;
2854 1.1 jmcneill }
2855 1.1 jmcneill
2856 1.1 jmcneill if (mcx_hca_set_caps(sc) != 0) {
2857 1.1 jmcneill /* error printed by mcx_hca_set_caps */
2858 1.1 jmcneill goto teardown;
2859 1.1 jmcneill }
2860 1.1 jmcneill
2861 1.1 jmcneill if (mcx_pages(sc, &sc->sc_init_pages,
2862 1.1 jmcneill htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
2863 1.1 jmcneill /* error printed by mcx_pages */
2864 1.1 jmcneill goto teardown;
2865 1.1 jmcneill }
2866 1.1 jmcneill
2867 1.1 jmcneill if (mcx_init_hca(sc) != 0) {
2868 1.1 jmcneill /* error printed by mcx_init_hca */
2869 1.1 jmcneill goto teardown;
2870 1.1 jmcneill }
2871 1.1 jmcneill
2872 1.1 jmcneill if (mcx_pages(sc, &sc->sc_regular_pages,
2873 1.1 jmcneill htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
2874 1.1 jmcneill /* error printed by mcx_pages */
2875 1.1 jmcneill goto teardown;
2876 1.1 jmcneill }
2877 1.1 jmcneill
2878 1.1 jmcneill /* apparently not necessary? */
2879 1.1 jmcneill if (mcx_set_driver_version(sc) != 0) {
2880 1.1 jmcneill /* error printed by mcx_set_driver_version */
2881 1.1 jmcneill goto teardown;
2882 1.1 jmcneill }
2883 1.1 jmcneill
2884 1.1 jmcneill if (mcx_iff(sc) != 0) { /* modify nic vport context */
2885 1.1 jmcneill /* error printed by mcx_iff? */
2886 1.1 jmcneill goto teardown;
2887 1.1 jmcneill }
2888 1.1 jmcneill
2889 1.15 jmcneill if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) {
2890 1.1 jmcneill /* error printed by mcx_alloc_uar */
2891 1.1 jmcneill goto teardown;
2892 1.1 jmcneill }
2893 1.1 jmcneill
2894 1.1 jmcneill if (mcx_alloc_pd(sc) != 0) {
2895 1.1 jmcneill /* error printed by mcx_alloc_pd */
2896 1.1 jmcneill goto teardown;
2897 1.1 jmcneill }
2898 1.1 jmcneill
2899 1.1 jmcneill if (mcx_alloc_tdomain(sc) != 0) {
2900 1.1 jmcneill /* error printed by mcx_alloc_tdomain */
2901 1.1 jmcneill goto teardown;
2902 1.1 jmcneill }
2903 1.1 jmcneill
2904 1.1 jmcneill /*
2905 1.1 jmcneill * PRM makes no mention of msi interrupts, just legacy and msi-x.
2906 1.1 jmcneill * mellanox support tells me legacy interrupts are not supported,
2907 1.1 jmcneill * so we're stuck with just msi-x.
2908 1.1 jmcneill */
2909 1.15 jmcneill counts[PCI_INTR_TYPE_MSIX] = -1;
2910 1.1 jmcneill counts[PCI_INTR_TYPE_MSI] = 0;
2911 1.1 jmcneill counts[PCI_INTR_TYPE_INTX] = 0;
2912 1.1 jmcneill if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) {
2913 1.1 jmcneill aprint_error_dev(self, "unable to allocate interrupt\n");
2914 1.1 jmcneill goto teardown;
2915 1.1 jmcneill }
2916 1.15 jmcneill if (counts[PCI_INTR_TYPE_MSIX] < 2) {
2917 1.15 jmcneill aprint_error_dev(self, "not enough MSI-X vectors\n");
2918 1.15 jmcneill goto teardown;
2919 1.15 jmcneill }
2920 1.1 jmcneill KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX);
2921 1.15 jmcneill snprintf(intrxname, sizeof(intrxname), "%s adminq", DEVNAME(sc));
2922 1.15 jmcneill sc->sc_ihc = mcx_establish_intr(sc, 0, NULL, mcx_admin_intr, sc,
2923 1.15 jmcneill intrxname);
2924 1.15 jmcneill if (sc->sc_ihc == NULL) {
2925 1.15 jmcneill aprint_error_dev(self, "couldn't establish adminq interrupt\n");
2926 1.1 jmcneill goto teardown;
2927 1.1 jmcneill }
2928 1.1 jmcneill
2929 1.15 jmcneill if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar,
2930 1.15 jmcneill (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
2931 1.15 jmcneill (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
2932 1.15 jmcneill (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
2933 1.15 jmcneill (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) {
2934 1.1 jmcneill /* error printed by mcx_create_eq */
2935 1.1 jmcneill goto teardown;
2936 1.1 jmcneill }
2937 1.1 jmcneill
2938 1.1 jmcneill if (mcx_query_nic_vport_context(sc, enaddr) != 0) {
2939 1.1 jmcneill /* error printed by mcx_query_nic_vport_context */
2940 1.1 jmcneill goto teardown;
2941 1.1 jmcneill }
2942 1.1 jmcneill
2943 1.1 jmcneill if (mcx_query_special_contexts(sc) != 0) {
2944 1.1 jmcneill /* error printed by mcx_query_special_contexts */
2945 1.1 jmcneill goto teardown;
2946 1.1 jmcneill }
2947 1.1 jmcneill
2948 1.1 jmcneill if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
2949 1.1 jmcneill /* error printed by mcx_set_port_mtu */
2950 1.1 jmcneill goto teardown;
2951 1.1 jmcneill }
2952 1.1 jmcneill
2953 1.1 jmcneill aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
2954 1.1 jmcneill ether_sprintf(enaddr));
2955 1.1 jmcneill
2956 1.15 jmcneill msix = counts[PCI_INTR_TYPE_MSIX];
2957 1.15 jmcneill msix--; /* admin ops took one */
2958 1.15 jmcneill
2959 1.15 jmcneill sc->sc_nqueues = uimin(MCX_MAX_QUEUES, msix);
2960 1.15 jmcneill sc->sc_nqueues = uimin(sc->sc_nqueues, ncpu);
2961 1.23 andvar /* Round down to a power of two. */
2962 1.23 andvar sc->sc_nqueues = 1U << ilog2(sc->sc_nqueues);
2963 1.15 jmcneill sc->sc_queues = kmem_zalloc(sc->sc_nqueues * sizeof(*sc->sc_queues),
2964 1.15 jmcneill KM_SLEEP);
2965 1.15 jmcneill
2966 1.1 jmcneill strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
2967 1.1 jmcneill ifp->if_softc = sc;
2968 1.1 jmcneill ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
2969 1.1 jmcneill #ifdef MCX_MPSAFE
2970 1.1 jmcneill ifp->if_extflags = IFEF_MPSAFE;
2971 1.1 jmcneill #endif
2972 1.1 jmcneill ifp->if_init = mcx_init;
2973 1.1 jmcneill ifp->if_stop = mcx_stop;
2974 1.1 jmcneill ifp->if_ioctl = mcx_ioctl;
2975 1.1 jmcneill ifp->if_start = mcx_start;
2976 1.16 jmcneill if (sc->sc_nqueues > 1) {
2977 1.16 jmcneill ifp->if_transmit = mcx_transmit;
2978 1.16 jmcneill }
2979 1.1 jmcneill ifp->if_mtu = sc->sc_hardmtu;
2980 1.15 jmcneill ifp->if_capabilities = IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
2981 1.15 jmcneill IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
2982 1.15 jmcneill IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx |
2983 1.15 jmcneill IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
2984 1.15 jmcneill IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx;
2985 1.1 jmcneill IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
2986 1.1 jmcneill IFQ_SET_READY(&ifp->if_snd);
2987 1.1 jmcneill
2988 1.15 jmcneill sc->sc_ec.ec_capabilities = ETHERCAP_JUMBO_MTU |
2989 1.15 jmcneill ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING;
2990 1.15 jmcneill sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;
2991 1.1 jmcneill
2992 1.1 jmcneill sc->sc_ec.ec_ifmedia = &sc->sc_media;
2993 1.12 thorpej ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, mcx_media_change,
2994 1.12 thorpej mcx_media_status, &sc->sc_media_mutex);
2995 1.1 jmcneill mcx_media_add_types(sc);
2996 1.1 jmcneill ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
2997 1.1 jmcneill ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
2998 1.1 jmcneill
2999 1.1 jmcneill if_attach(ifp);
3000 1.22 jmcneill
3001 1.22 jmcneill /* Enable hardware offload by default */
3002 1.22 jmcneill memset(&ifcr, 0, sizeof(ifcr));
3003 1.22 jmcneill ifcr.ifcr_capenable = ifp->if_capabilities;
3004 1.22 jmcneill ifioctl_common(ifp, SIOCSIFCAP, &ifcr);
3005 1.22 jmcneill
3006 1.1 jmcneill if_deferred_start_init(ifp, NULL);
3007 1.1 jmcneill
3008 1.1 jmcneill ether_ifattach(ifp, enaddr);
3009 1.1 jmcneill
3010 1.15 jmcneill kcpuset_create(&affinity, false);
3011 1.15 jmcneill kcpuset_set(affinity, 0);
3012 1.15 jmcneill
3013 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
3014 1.15 jmcneill struct mcx_queues *q = &sc->sc_queues[i];
3015 1.15 jmcneill struct mcx_rx *rx = &q->q_rx;
3016 1.15 jmcneill struct mcx_tx *tx = &q->q_tx;
3017 1.15 jmcneill int vec;
3018 1.15 jmcneill
3019 1.15 jmcneill vec = i + 1;
3020 1.15 jmcneill q->q_sc = sc;
3021 1.15 jmcneill q->q_index = i;
3022 1.15 jmcneill
3023 1.15 jmcneill if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
3024 1.15 jmcneill aprint_error_dev(self, "unable to alloc uar %d\n", i);
3025 1.15 jmcneill goto teardown;
3026 1.15 jmcneill }
3027 1.15 jmcneill
3028 1.15 jmcneill if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
3029 1.15 jmcneill aprint_error_dev(self,
3030 1.15 jmcneill "unable to create event queue %d\n", i);
3031 1.15 jmcneill goto teardown;
3032 1.15 jmcneill }
3033 1.15 jmcneill
3034 1.15 jmcneill rx->rx_softc = sc;
3035 1.15 jmcneill callout_init(&rx->rx_refill, CALLOUT_FLAGS);
3036 1.15 jmcneill callout_setfunc(&rx->rx_refill, mcx_refill, rx);
3037 1.15 jmcneill
3038 1.15 jmcneill tx->tx_softc = sc;
3039 1.16 jmcneill mutex_init(&tx->tx_lock, MUTEX_DEFAULT, IPL_NET);
3040 1.16 jmcneill tx->tx_pcq = pcq_create(MCX_TXQ_NUM, KM_SLEEP);
3041 1.16 jmcneill tx->tx_softint = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,
3042 1.16 jmcneill mcx_deferred_transmit, tx);
3043 1.15 jmcneill
3044 1.15 jmcneill snprintf(intrxname, sizeof(intrxname), "%s queue %d",
3045 1.15 jmcneill DEVNAME(sc), i);
3046 1.15 jmcneill q->q_ihc = mcx_establish_intr(sc, vec, affinity, mcx_cq_intr,
3047 1.15 jmcneill q, intrxname);
3048 1.15 jmcneill }
3049 1.15 jmcneill
3050 1.1 jmcneill callout_init(&sc->sc_calibrate, CALLOUT_FLAGS);
3051 1.1 jmcneill callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc);
3052 1.1 jmcneill
3053 1.1 jmcneill if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc,
3054 1.1 jmcneill PRI_NONE, IPL_NET, 0) != 0) {
3055 1.1 jmcneill aprint_error_dev(self, "couldn't create port change workq\n");
3056 1.1 jmcneill goto teardown;
3057 1.1 jmcneill }
3058 1.1 jmcneill
3059 1.1 jmcneill mcx_port_change(&sc->sc_port_change, sc);
3060 1.1 jmcneill
3061 1.15 jmcneill sc->sc_mac_flow_table_id = -1;
3062 1.15 jmcneill sc->sc_rss_flow_table_id = -1;
3063 1.15 jmcneill sc->sc_rqt = -1;
3064 1.1 jmcneill for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
3065 1.15 jmcneill struct mcx_flow_group *mfg = &sc->sc_flow_group[i];
3066 1.15 jmcneill mfg->g_id = -1;
3067 1.15 jmcneill mfg->g_table = -1;
3068 1.15 jmcneill mfg->g_size = 0;
3069 1.15 jmcneill mfg->g_start = 0;
3070 1.1 jmcneill }
3071 1.1 jmcneill sc->sc_extra_mcast = 0;
3072 1.1 jmcneill memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));
3073 1.15 jmcneill
3074 1.15 jmcneill #if NKSTAT > 0
3075 1.15 jmcneill mcx_kstat_attach(sc);
3076 1.15 jmcneill #endif
3077 1.15 jmcneill mcx_timecounter_attach(sc);
3078 1.1 jmcneill return;
3079 1.1 jmcneill
3080 1.1 jmcneill teardown:
3081 1.1 jmcneill mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
3082 1.1 jmcneill /* error printed by mcx_teardown_hca, and we're already unwinding */
3083 1.1 jmcneill cqfree:
3084 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
3085 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
3086 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3087 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
3088 1.1 jmcneill MCX_CMDQ_INTERFACE_DISABLED);
3089 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t),
3090 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3091 1.1 jmcneill
3092 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
3093 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
3094 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3095 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
3096 1.1 jmcneill
3097 1.1 jmcneill mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
3098 1.1 jmcneill dbfree:
3099 1.1 jmcneill mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
3100 1.1 jmcneill unmap:
3101 1.1 jmcneill bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
3102 1.1 jmcneill sc->sc_mems = 0;
3103 1.1 jmcneill }
3104 1.1 jmcneill
3105 1.15 jmcneill static void *
3106 1.15 jmcneill mcx_establish_intr(struct mcx_softc *sc, int index, kcpuset_t *affinity,
3107 1.15 jmcneill int (*func)(void *), void *arg, const char *xname)
3108 1.15 jmcneill {
3109 1.15 jmcneill char intrbuf[PCI_INTRSTR_LEN];
3110 1.15 jmcneill const char *intrstr;
3111 1.15 jmcneill void *ih;
3112 1.15 jmcneill
3113 1.15 jmcneill pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[index], PCI_INTR_MPSAFE,
3114 1.15 jmcneill true);
3115 1.15 jmcneill
3116 1.15 jmcneill intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[index], intrbuf,
3117 1.15 jmcneill sizeof(intrbuf));
3118 1.15 jmcneill ih = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[index], IPL_NET,
3119 1.15 jmcneill func, arg, xname);
3120 1.15 jmcneill if (ih == NULL) {
3121 1.15 jmcneill aprint_error_dev(sc->sc_dev,
3122 1.15 jmcneill "unable to establish interrupt%s%s\n",
3123 1.15 jmcneill intrstr ? " at " : "",
3124 1.15 jmcneill intrstr ? intrstr : "");
3125 1.15 jmcneill return NULL;
3126 1.15 jmcneill }
3127 1.15 jmcneill
3128 1.15 jmcneill if (affinity != NULL && index > 0) {
3129 1.15 jmcneill /* Round-robin affinity */
3130 1.15 jmcneill kcpuset_zero(affinity);
3131 1.15 jmcneill kcpuset_set(affinity, (index - 1) % ncpu);
3132 1.15 jmcneill interrupt_distribute(ih, affinity, NULL);
3133 1.15 jmcneill }
3134 1.15 jmcneill
3135 1.15 jmcneill return ih;
3136 1.15 jmcneill }
3137 1.15 jmcneill
3138 1.1 jmcneill static void
3139 1.1 jmcneill mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm)
3140 1.1 jmcneill {
3141 1.1 jmcneill rxr->rxr_total = hwm;
3142 1.1 jmcneill rxr->rxr_inuse = 0;
3143 1.1 jmcneill }
3144 1.1 jmcneill
3145 1.1 jmcneill static u_int
3146 1.1 jmcneill mcx_rxr_get(struct mcx_rxring *rxr, u_int max)
3147 1.1 jmcneill {
3148 1.1 jmcneill const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse);
3149 1.1 jmcneill
3150 1.1 jmcneill rxr->rxr_inuse += taken;
3151 1.1 jmcneill
3152 1.1 jmcneill return taken;
3153 1.1 jmcneill }
3154 1.1 jmcneill
3155 1.1 jmcneill static void
3156 1.1 jmcneill mcx_rxr_put(struct mcx_rxring *rxr, u_int n)
3157 1.1 jmcneill {
3158 1.1 jmcneill rxr->rxr_inuse -= n;
3159 1.1 jmcneill }
3160 1.1 jmcneill
3161 1.1 jmcneill static u_int
3162 1.1 jmcneill mcx_rxr_inuse(struct mcx_rxring *rxr)
3163 1.1 jmcneill {
3164 1.1 jmcneill return rxr->rxr_inuse;
3165 1.1 jmcneill }
3166 1.1 jmcneill
3167 1.1 jmcneill static int
3168 1.1 jmcneill mcx_version(struct mcx_softc *sc)
3169 1.1 jmcneill {
3170 1.1 jmcneill uint32_t fw0, fw1;
3171 1.1 jmcneill uint16_t cmdif;
3172 1.1 jmcneill
3173 1.1 jmcneill fw0 = mcx_rd(sc, MCX_FW_VER);
3174 1.1 jmcneill fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
3175 1.1 jmcneill
3176 1.1 jmcneill aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0),
3177 1.1 jmcneill MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
3178 1.1 jmcneill
3179 1.1 jmcneill cmdif = MCX_CMDIF(fw1);
3180 1.1 jmcneill if (cmdif != MCX_CMD_IF_SUPPORTED) {
3181 1.1 jmcneill aprint_error_dev(sc->sc_dev,
3182 1.1 jmcneill "unsupported command interface %u\n", cmdif);
3183 1.1 jmcneill return (-1);
3184 1.1 jmcneill }
3185 1.1 jmcneill
3186 1.1 jmcneill return (0);
3187 1.1 jmcneill }
3188 1.1 jmcneill
3189 1.1 jmcneill static int
3190 1.1 jmcneill mcx_init_wait(struct mcx_softc *sc)
3191 1.1 jmcneill {
3192 1.1 jmcneill unsigned int i;
3193 1.1 jmcneill uint32_t r;
3194 1.1 jmcneill
3195 1.1 jmcneill for (i = 0; i < 2000; i++) {
3196 1.1 jmcneill r = mcx_rd(sc, MCX_STATE);
3197 1.1 jmcneill if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
3198 1.1 jmcneill return (0);
3199 1.1 jmcneill
3200 1.1 jmcneill delay(1000);
3201 1.1 jmcneill mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
3202 1.1 jmcneill BUS_SPACE_BARRIER_READ);
3203 1.1 jmcneill }
3204 1.1 jmcneill
3205 1.1 jmcneill return (-1);
3206 1.1 jmcneill }
3207 1.1 jmcneill
3208 1.1 jmcneill static uint8_t
3209 1.1 jmcneill mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3210 1.1 jmcneill unsigned int msec)
3211 1.1 jmcneill {
3212 1.1 jmcneill unsigned int i;
3213 1.1 jmcneill
3214 1.1 jmcneill for (i = 0; i < msec; i++) {
3215 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
3216 1.1 jmcneill 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
3217 1.1 jmcneill
3218 1.1 jmcneill if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
3219 1.13 jmcneill MCX_CQ_STATUS_OWN_SW)
3220 1.1 jmcneill return (0);
3221 1.1 jmcneill
3222 1.1 jmcneill delay(1000);
3223 1.1 jmcneill }
3224 1.1 jmcneill
3225 1.1 jmcneill return (ETIMEDOUT);
3226 1.1 jmcneill }
3227 1.1 jmcneill
3228 1.1 jmcneill static uint32_t
3229 1.1 jmcneill mcx_mix_u64(uint32_t xor, uint64_t u64)
3230 1.1 jmcneill {
3231 1.1 jmcneill xor ^= u64 >> 32;
3232 1.1 jmcneill xor ^= u64;
3233 1.1 jmcneill
3234 1.1 jmcneill return (xor);
3235 1.1 jmcneill }
3236 1.1 jmcneill
3237 1.1 jmcneill static uint32_t
3238 1.1 jmcneill mcx_mix_u32(uint32_t xor, uint32_t u32)
3239 1.1 jmcneill {
3240 1.1 jmcneill xor ^= u32;
3241 1.1 jmcneill
3242 1.1 jmcneill return (xor);
3243 1.1 jmcneill }
3244 1.1 jmcneill
3245 1.1 jmcneill static uint32_t
3246 1.1 jmcneill mcx_mix_u8(uint32_t xor, uint8_t u8)
3247 1.1 jmcneill {
3248 1.1 jmcneill xor ^= u8;
3249 1.1 jmcneill
3250 1.1 jmcneill return (xor);
3251 1.1 jmcneill }
3252 1.1 jmcneill
3253 1.1 jmcneill static uint8_t
3254 1.1 jmcneill mcx_mix_done(uint32_t xor)
3255 1.1 jmcneill {
3256 1.1 jmcneill xor ^= xor >> 16;
3257 1.1 jmcneill xor ^= xor >> 8;
3258 1.1 jmcneill
3259 1.1 jmcneill return (xor);
3260 1.1 jmcneill }
3261 1.1 jmcneill
3262 1.1 jmcneill static uint8_t
3263 1.1 jmcneill mcx_xor(const void *buf, size_t len)
3264 1.1 jmcneill {
3265 1.1 jmcneill const uint32_t *dwords = buf;
3266 1.1 jmcneill uint32_t xor = 0xff;
3267 1.1 jmcneill size_t i;
3268 1.1 jmcneill
3269 1.1 jmcneill len /= sizeof(*dwords);
3270 1.1 jmcneill
3271 1.1 jmcneill for (i = 0; i < len; i++)
3272 1.1 jmcneill xor ^= dwords[i];
3273 1.1 jmcneill
3274 1.1 jmcneill return (mcx_mix_done(xor));
3275 1.1 jmcneill }
3276 1.1 jmcneill
3277 1.1 jmcneill static uint8_t
3278 1.1 jmcneill mcx_cmdq_token(struct mcx_softc *sc)
3279 1.1 jmcneill {
3280 1.1 jmcneill uint8_t token;
3281 1.1 jmcneill
3282 1.1 jmcneill do {
3283 1.1 jmcneill token = ++sc->sc_cmdq_token;
3284 1.1 jmcneill } while (token == 0);
3285 1.1 jmcneill
3286 1.1 jmcneill return (token);
3287 1.1 jmcneill }
3288 1.1 jmcneill
3289 1.1 jmcneill static void
3290 1.1 jmcneill mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3291 1.1 jmcneill uint32_t ilen, uint32_t olen, uint8_t token)
3292 1.1 jmcneill {
3293 1.1 jmcneill memset(cqe, 0, sc->sc_cmdq_size);
3294 1.1 jmcneill
3295 1.1 jmcneill cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
3296 1.1 jmcneill be32enc(&cqe->cq_input_length, ilen);
3297 1.1 jmcneill be32enc(&cqe->cq_output_length, olen);
3298 1.1 jmcneill cqe->cq_token = token;
3299 1.1 jmcneill cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
3300 1.1 jmcneill }
3301 1.1 jmcneill
3302 1.1 jmcneill static void
3303 1.1 jmcneill mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
3304 1.1 jmcneill {
3305 1.1 jmcneill cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
3306 1.1 jmcneill }
3307 1.1 jmcneill
3308 1.1 jmcneill static int
3309 1.1 jmcneill mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
3310 1.1 jmcneill {
3311 1.1 jmcneill /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */
3312 1.1 jmcneill return (0);
3313 1.1 jmcneill }
3314 1.1 jmcneill
3315 1.1 jmcneill static void *
3316 1.1 jmcneill mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
3317 1.1 jmcneill {
3318 1.1 jmcneill return (&cqe->cq_input_data);
3319 1.1 jmcneill }
3320 1.1 jmcneill
3321 1.1 jmcneill static void *
3322 1.1 jmcneill mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
3323 1.1 jmcneill {
3324 1.1 jmcneill return (&cqe->cq_output_data);
3325 1.1 jmcneill }
3326 1.1 jmcneill
3327 1.1 jmcneill static void
3328 1.1 jmcneill mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3329 1.1 jmcneill unsigned int slot)
3330 1.1 jmcneill {
3331 1.1 jmcneill mcx_cmdq_sign(cqe);
3332 1.1 jmcneill
3333 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
3334 1.1 jmcneill 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
3335 1.1 jmcneill
3336 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
3337 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t),
3338 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3339 1.1 jmcneill }
3340 1.1 jmcneill
3341 1.1 jmcneill static int
3342 1.1 jmcneill mcx_enable_hca(struct mcx_softc *sc)
3343 1.1 jmcneill {
3344 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3345 1.1 jmcneill struct mcx_cmd_enable_hca_in *in;
3346 1.1 jmcneill struct mcx_cmd_enable_hca_out *out;
3347 1.1 jmcneill int error;
3348 1.1 jmcneill uint8_t status;
3349 1.1 jmcneill
3350 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3351 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3352 1.1 jmcneill
3353 1.1 jmcneill in = mcx_cmdq_in(cqe);
3354 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
3355 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3356 1.1 jmcneill in->cmd_function_id = htobe16(0);
3357 1.1 jmcneill
3358 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3359 1.1 jmcneill
3360 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3361 1.1 jmcneill if (error != 0) {
3362 1.1 jmcneill printf(", hca enable timeout\n");
3363 1.1 jmcneill return (-1);
3364 1.1 jmcneill }
3365 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
3366 1.1 jmcneill printf(", hca enable command corrupt\n");
3367 1.1 jmcneill return (-1);
3368 1.1 jmcneill }
3369 1.1 jmcneill
3370 1.1 jmcneill status = cqe->cq_output_data[0];
3371 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
3372 1.1 jmcneill printf(", hca enable failed (%x)\n", status);
3373 1.1 jmcneill return (-1);
3374 1.1 jmcneill }
3375 1.1 jmcneill
3376 1.1 jmcneill return (0);
3377 1.1 jmcneill }
3378 1.1 jmcneill
3379 1.1 jmcneill static int
3380 1.1 jmcneill mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
3381 1.1 jmcneill {
3382 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3383 1.1 jmcneill struct mcx_cmd_teardown_hca_in *in;
3384 1.1 jmcneill struct mcx_cmd_teardown_hca_out *out;
3385 1.1 jmcneill int error;
3386 1.1 jmcneill uint8_t status;
3387 1.1 jmcneill
3388 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3389 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3390 1.1 jmcneill
3391 1.1 jmcneill in = mcx_cmdq_in(cqe);
3392 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
3393 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3394 1.1 jmcneill in->cmd_profile = profile;
3395 1.1 jmcneill
3396 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3397 1.1 jmcneill
3398 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3399 1.1 jmcneill if (error != 0) {
3400 1.1 jmcneill printf(", hca teardown timeout\n");
3401 1.1 jmcneill return (-1);
3402 1.1 jmcneill }
3403 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
3404 1.1 jmcneill printf(", hca teardown command corrupt\n");
3405 1.1 jmcneill return (-1);
3406 1.1 jmcneill }
3407 1.1 jmcneill
3408 1.1 jmcneill status = cqe->cq_output_data[0];
3409 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
3410 1.1 jmcneill printf(", hca teardown failed (%x)\n", status);
3411 1.1 jmcneill return (-1);
3412 1.1 jmcneill }
3413 1.1 jmcneill
3414 1.1 jmcneill return (0);
3415 1.1 jmcneill }
3416 1.1 jmcneill
3417 1.1 jmcneill static int
3418 1.1 jmcneill mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
3419 1.1 jmcneill unsigned int nmb, uint64_t *ptr, uint8_t token)
3420 1.1 jmcneill {
3421 1.1 jmcneill uint8_t *kva;
3422 1.1 jmcneill uint64_t dva;
3423 1.1 jmcneill int i;
3424 1.1 jmcneill int error;
3425 1.1 jmcneill
3426 1.1 jmcneill error = mcx_dmamem_alloc(sc, mxm,
3427 1.1 jmcneill nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
3428 1.1 jmcneill if (error != 0)
3429 1.1 jmcneill return (error);
3430 1.1 jmcneill
3431 1.1 jmcneill mcx_dmamem_zero(mxm);
3432 1.1 jmcneill
3433 1.1 jmcneill dva = MCX_DMA_DVA(mxm);
3434 1.1 jmcneill kva = MCX_DMA_KVA(mxm);
3435 1.1 jmcneill for (i = 0; i < nmb; i++) {
3436 1.1 jmcneill struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
3437 1.1 jmcneill
3438 1.1 jmcneill /* patch the cqe or mbox pointing at this one */
3439 1.1 jmcneill be64enc(ptr, dva);
3440 1.1 jmcneill
3441 1.1 jmcneill /* fill in this mbox */
3442 1.1 jmcneill be32enc(&mbox->mb_block_number, i);
3443 1.1 jmcneill mbox->mb_token = token;
3444 1.1 jmcneill
3445 1.1 jmcneill /* move to the next one */
3446 1.1 jmcneill ptr = &mbox->mb_next_ptr;
3447 1.1 jmcneill
3448 1.1 jmcneill dva += MCX_CMDQ_MAILBOX_SIZE;
3449 1.1 jmcneill kva += MCX_CMDQ_MAILBOX_SIZE;
3450 1.1 jmcneill }
3451 1.1 jmcneill
3452 1.1 jmcneill return (0);
3453 1.1 jmcneill }
3454 1.1 jmcneill
3455 1.1 jmcneill static uint32_t
3456 1.1 jmcneill mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
3457 1.1 jmcneill {
3458 1.1 jmcneill uint32_t xor = 0xff;
3459 1.1 jmcneill
3460 1.1 jmcneill /* only 3 fields get set, so mix them directly */
3461 1.1 jmcneill xor = mcx_mix_u64(xor, mb->mb_next_ptr);
3462 1.1 jmcneill xor = mcx_mix_u32(xor, mb->mb_block_number);
3463 1.1 jmcneill xor = mcx_mix_u8(xor, mb->mb_token);
3464 1.1 jmcneill
3465 1.1 jmcneill return (mcx_mix_done(xor));
3466 1.1 jmcneill }
3467 1.1 jmcneill
3468 1.1 jmcneill static void
3469 1.1 jmcneill mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
3470 1.1 jmcneill {
3471 1.1 jmcneill uint8_t *kva;
3472 1.1 jmcneill int i;
3473 1.1 jmcneill
3474 1.1 jmcneill kva = MCX_DMA_KVA(mxm);
3475 1.1 jmcneill
3476 1.1 jmcneill for (i = 0; i < nmb; i++) {
3477 1.1 jmcneill struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
3478 1.1 jmcneill uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
3479 1.1 jmcneill mb->mb_ctrl_signature = sig;
3480 1.1 jmcneill mb->mb_signature = sig ^
3481 1.1 jmcneill mcx_xor(mb->mb_data, sizeof(mb->mb_data));
3482 1.1 jmcneill
3483 1.1 jmcneill kva += MCX_CMDQ_MAILBOX_SIZE;
3484 1.1 jmcneill }
3485 1.1 jmcneill }
3486 1.1 jmcneill
3487 1.1 jmcneill static void
3488 1.1 jmcneill mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
3489 1.1 jmcneill {
3490 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
3491 1.1 jmcneill 0, MCX_DMA_LEN(mxm), ops);
3492 1.1 jmcneill }
3493 1.1 jmcneill
3494 1.1 jmcneill static struct mcx_cmdq_mailbox *
3495 1.1 jmcneill mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
3496 1.1 jmcneill {
3497 1.1 jmcneill uint8_t *kva;
3498 1.1 jmcneill
3499 1.1 jmcneill kva = MCX_DMA_KVA(mxm);
3500 1.1 jmcneill kva += i * MCX_CMDQ_MAILBOX_SIZE;
3501 1.1 jmcneill
3502 1.1 jmcneill return ((struct mcx_cmdq_mailbox *)kva);
3503 1.1 jmcneill }
3504 1.1 jmcneill
3505 1.1 jmcneill static inline void *
3506 1.1 jmcneill mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
3507 1.1 jmcneill {
3508 1.1 jmcneill return (&mb->mb_data);
3509 1.1 jmcneill }
3510 1.1 jmcneill
3511 1.1 jmcneill static void
3512 1.1 jmcneill mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
3513 1.1 jmcneill void *b, size_t len)
3514 1.1 jmcneill {
3515 1.1 jmcneill uint8_t *buf = b;
3516 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
3517 1.1 jmcneill int i;
3518 1.1 jmcneill
3519 1.1 jmcneill mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
3520 1.1 jmcneill for (i = 0; i < nmb; i++) {
3521 1.1 jmcneill
3522 1.1 jmcneill memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len));
3523 1.1 jmcneill
3524 1.1 jmcneill if (sizeof(mb->mb_data) >= len)
3525 1.1 jmcneill break;
3526 1.1 jmcneill
3527 1.1 jmcneill buf += sizeof(mb->mb_data);
3528 1.1 jmcneill len -= sizeof(mb->mb_data);
3529 1.1 jmcneill mb++;
3530 1.1 jmcneill }
3531 1.1 jmcneill }
3532 1.1 jmcneill
3533 1.1 jmcneill static void
3534 1.13 jmcneill mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages,
3535 1.13 jmcneill struct mcx_dmamem *buf)
3536 1.13 jmcneill {
3537 1.13 jmcneill uint64_t *pas;
3538 1.13 jmcneill int mbox, mbox_pages, i;
3539 1.13 jmcneill
3540 1.13 jmcneill mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE;
3541 1.13 jmcneill offset %= MCX_CMDQ_MAILBOX_DATASIZE;
3542 1.13 jmcneill
3543 1.13 jmcneill pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
3544 1.13 jmcneill pas += (offset / sizeof(*pas));
3545 1.13 jmcneill mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas);
3546 1.13 jmcneill for (i = 0; i < npages; i++) {
3547 1.13 jmcneill if (i == mbox_pages) {
3548 1.13 jmcneill mbox++;
3549 1.13 jmcneill pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
3550 1.13 jmcneill mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas);
3551 1.13 jmcneill }
3552 1.13 jmcneill *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE));
3553 1.13 jmcneill pas++;
3554 1.13 jmcneill }
3555 1.13 jmcneill }
3556 1.13 jmcneill
3557 1.13 jmcneill static void
3558 1.1 jmcneill mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
3559 1.1 jmcneill {
3560 1.1 jmcneill uint8_t *buf = b;
3561 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
3562 1.1 jmcneill int i;
3563 1.1 jmcneill
3564 1.1 jmcneill mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
3565 1.1 jmcneill for (i = 0; i < nmb; i++) {
3566 1.1 jmcneill memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len));
3567 1.1 jmcneill
3568 1.1 jmcneill if (sizeof(mb->mb_data) >= len)
3569 1.1 jmcneill break;
3570 1.1 jmcneill
3571 1.1 jmcneill buf += sizeof(mb->mb_data);
3572 1.1 jmcneill len -= sizeof(mb->mb_data);
3573 1.1 jmcneill mb++;
3574 1.1 jmcneill }
3575 1.1 jmcneill }
3576 1.1 jmcneill
3577 1.1 jmcneill static void
3578 1.1 jmcneill mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
3579 1.1 jmcneill {
3580 1.1 jmcneill mcx_dmamem_free(sc, mxm);
3581 1.1 jmcneill }
3582 1.1 jmcneill
3583 1.1 jmcneill #if 0
3584 1.1 jmcneill static void
3585 1.1 jmcneill mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
3586 1.1 jmcneill {
3587 1.1 jmcneill unsigned int i;
3588 1.1 jmcneill
3589 1.1 jmcneill printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
3590 1.1 jmcneill be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr));
3591 1.1 jmcneill
3592 1.1 jmcneill printf(", idata ");
3593 1.1 jmcneill for (i = 0; i < sizeof(cqe->cq_input_data); i++)
3594 1.1 jmcneill printf("%02x", cqe->cq_input_data[i]);
3595 1.1 jmcneill
3596 1.1 jmcneill printf(", odata ");
3597 1.1 jmcneill for (i = 0; i < sizeof(cqe->cq_output_data); i++)
3598 1.1 jmcneill printf("%02x", cqe->cq_output_data[i]);
3599 1.1 jmcneill
3600 1.1 jmcneill printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
3601 1.1 jmcneill be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length),
3602 1.1 jmcneill cqe->cq_token, cqe->cq_signature, cqe->cq_status);
3603 1.1 jmcneill }
3604 1.1 jmcneill
3605 1.1 jmcneill static void
3606 1.1 jmcneill mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
3607 1.1 jmcneill {
3608 1.1 jmcneill int i, j;
3609 1.1 jmcneill uint8_t *d;
3610 1.1 jmcneill
3611 1.1 jmcneill for (i = 0; i < num; i++) {
3612 1.1 jmcneill struct mcx_cmdq_mailbox *mbox;
3613 1.1 jmcneill mbox = mcx_cq_mbox(mboxes, i);
3614 1.1 jmcneill
3615 1.1 jmcneill d = mcx_cq_mbox_data(mbox);
3616 1.1 jmcneill for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
3617 1.1 jmcneill if (j != 0 && (j % 16 == 0))
3618 1.1 jmcneill printf("\n");
3619 1.1 jmcneill printf("%.2x ", d[j]);
3620 1.1 jmcneill }
3621 1.1 jmcneill }
3622 1.1 jmcneill }
3623 1.1 jmcneill #endif
3624 1.1 jmcneill
3625 1.1 jmcneill static int
3626 1.1 jmcneill mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
3627 1.1 jmcneill int len)
3628 1.1 jmcneill {
3629 1.1 jmcneill struct mcx_dmamem mxm;
3630 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3631 1.1 jmcneill struct mcx_cmd_access_reg_in *in;
3632 1.1 jmcneill struct mcx_cmd_access_reg_out *out;
3633 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3634 1.1 jmcneill int error, nmb;
3635 1.1 jmcneill
3636 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3637 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
3638 1.1 jmcneill token);
3639 1.1 jmcneill
3640 1.1 jmcneill in = mcx_cmdq_in(cqe);
3641 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
3642 1.1 jmcneill in->cmd_op_mod = htobe16(op);
3643 1.1 jmcneill in->cmd_register_id = htobe16(reg);
3644 1.1 jmcneill
3645 1.1 jmcneill nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
3646 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
3647 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
3648 1.1 jmcneill printf(", unable to allocate access reg mailboxen\n");
3649 1.1 jmcneill return (-1);
3650 1.1 jmcneill }
3651 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
3652 1.1 jmcneill mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
3653 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, nmb);
3654 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
3655 1.1 jmcneill
3656 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3657 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3658 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
3659 1.1 jmcneill
3660 1.1 jmcneill if (error != 0) {
3661 1.1 jmcneill printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
3662 1.1 jmcneill (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
3663 1.1 jmcneill goto free;
3664 1.1 jmcneill }
3665 1.1 jmcneill error = mcx_cmdq_verify(cqe);
3666 1.1 jmcneill if (error != 0) {
3667 1.1 jmcneill printf("%s: access reg (%s %x) reply corrupt\n",
3668 1.1 jmcneill (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
3669 1.1 jmcneill reg);
3670 1.1 jmcneill goto free;
3671 1.1 jmcneill }
3672 1.1 jmcneill
3673 1.1 jmcneill out = mcx_cmdq_out(cqe);
3674 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
3675 1.1 jmcneill printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
3676 1.1 jmcneill DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
3677 1.15 jmcneill reg, out->cmd_status, be32toh(out->cmd_syndrome));
3678 1.1 jmcneill error = -1;
3679 1.1 jmcneill goto free;
3680 1.1 jmcneill }
3681 1.1 jmcneill
3682 1.1 jmcneill mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
3683 1.1 jmcneill free:
3684 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
3685 1.1 jmcneill
3686 1.1 jmcneill return (error);
3687 1.1 jmcneill }
3688 1.1 jmcneill
3689 1.1 jmcneill static int
3690 1.15 jmcneill mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3691 1.15 jmcneill unsigned int slot)
3692 1.1 jmcneill {
3693 1.1 jmcneill struct mcx_cmd_set_issi_in *in;
3694 1.1 jmcneill struct mcx_cmd_set_issi_out *out;
3695 1.1 jmcneill uint8_t status;
3696 1.1 jmcneill
3697 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3698 1.1 jmcneill
3699 1.1 jmcneill in = mcx_cmdq_in(cqe);
3700 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
3701 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3702 1.1 jmcneill in->cmd_current_issi = htobe16(MCX_ISSI);
3703 1.1 jmcneill
3704 1.1 jmcneill mcx_cmdq_post(sc, cqe, slot);
3705 1.1 jmcneill if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
3706 1.1 jmcneill return (-1);
3707 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0)
3708 1.1 jmcneill return (-1);
3709 1.1 jmcneill
3710 1.1 jmcneill status = cqe->cq_output_data[0];
3711 1.1 jmcneill if (status != MCX_CQ_STATUS_OK)
3712 1.1 jmcneill return (-1);
3713 1.1 jmcneill
3714 1.1 jmcneill return (0);
3715 1.1 jmcneill }
3716 1.1 jmcneill
3717 1.1 jmcneill static int
3718 1.1 jmcneill mcx_issi(struct mcx_softc *sc)
3719 1.1 jmcneill {
3720 1.1 jmcneill struct mcx_dmamem mxm;
3721 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3722 1.1 jmcneill struct mcx_cmd_query_issi_in *in;
3723 1.1 jmcneill struct mcx_cmd_query_issi_il_out *out;
3724 1.1 jmcneill struct mcx_cmd_query_issi_mb_out *mb;
3725 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3726 1.1 jmcneill uint8_t status;
3727 1.1 jmcneill int error;
3728 1.1 jmcneill
3729 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3730 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
3731 1.1 jmcneill
3732 1.1 jmcneill in = mcx_cmdq_in(cqe);
3733 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
3734 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3735 1.1 jmcneill
3736 1.1 jmcneill CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
3737 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
3738 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
3739 1.1 jmcneill printf(", unable to allocate query issi mailbox\n");
3740 1.1 jmcneill return (-1);
3741 1.1 jmcneill }
3742 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
3743 1.1 jmcneill
3744 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3745 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3746 1.1 jmcneill if (error != 0) {
3747 1.1 jmcneill printf(", query issi timeout\n");
3748 1.1 jmcneill goto free;
3749 1.1 jmcneill }
3750 1.1 jmcneill error = mcx_cmdq_verify(cqe);
3751 1.1 jmcneill if (error != 0) {
3752 1.1 jmcneill printf(", query issi reply corrupt\n");
3753 1.1 jmcneill goto free;
3754 1.1 jmcneill }
3755 1.1 jmcneill
3756 1.1 jmcneill status = cqe->cq_output_data[0];
3757 1.1 jmcneill switch (status) {
3758 1.1 jmcneill case MCX_CQ_STATUS_OK:
3759 1.1 jmcneill break;
3760 1.1 jmcneill case MCX_CQ_STATUS_BAD_OPCODE:
3761 1.1 jmcneill /* use ISSI 0 */
3762 1.1 jmcneill goto free;
3763 1.1 jmcneill default:
3764 1.1 jmcneill printf(", query issi failed (%x)\n", status);
3765 1.1 jmcneill error = -1;
3766 1.1 jmcneill goto free;
3767 1.1 jmcneill }
3768 1.1 jmcneill
3769 1.1 jmcneill out = mcx_cmdq_out(cqe);
3770 1.1 jmcneill if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
3771 1.1 jmcneill /* use ISSI 1 */
3772 1.1 jmcneill goto free;
3773 1.1 jmcneill }
3774 1.1 jmcneill
3775 1.1 jmcneill /* don't need to read cqe anymore, can be used for SET ISSI */
3776 1.1 jmcneill
3777 1.1 jmcneill mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
3778 1.1 jmcneill CTASSERT(MCX_ISSI < NBBY);
3779 1.1 jmcneill /* XXX math is hard */
3780 1.1 jmcneill if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
3781 1.1 jmcneill /* use ISSI 0 */
3782 1.1 jmcneill goto free;
3783 1.1 jmcneill }
3784 1.1 jmcneill
3785 1.1 jmcneill if (mcx_set_issi(sc, cqe, 0) != 0) {
3786 1.1 jmcneill /* ignore the error, just use ISSI 0 */
3787 1.1 jmcneill } else {
3788 1.1 jmcneill /* use ISSI 1 */
3789 1.1 jmcneill }
3790 1.1 jmcneill
3791 1.1 jmcneill free:
3792 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
3793 1.1 jmcneill return (error);
3794 1.1 jmcneill }
3795 1.1 jmcneill
3796 1.1 jmcneill static int
3797 1.1 jmcneill mcx_query_pages(struct mcx_softc *sc, uint16_t type,
3798 1.15 jmcneill int32_t *npages, uint16_t *func_id)
3799 1.1 jmcneill {
3800 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3801 1.1 jmcneill struct mcx_cmd_query_pages_in *in;
3802 1.1 jmcneill struct mcx_cmd_query_pages_out *out;
3803 1.1 jmcneill
3804 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3805 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3806 1.1 jmcneill
3807 1.1 jmcneill in = mcx_cmdq_in(cqe);
3808 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
3809 1.1 jmcneill in->cmd_op_mod = type;
3810 1.1 jmcneill
3811 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3812 1.1 jmcneill if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
3813 1.1 jmcneill printf(", query pages timeout\n");
3814 1.1 jmcneill return (-1);
3815 1.1 jmcneill }
3816 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
3817 1.1 jmcneill printf(", query pages reply corrupt\n");
3818 1.1 jmcneill return (-1);
3819 1.1 jmcneill }
3820 1.1 jmcneill
3821 1.1 jmcneill out = mcx_cmdq_out(cqe);
3822 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
3823 1.1 jmcneill printf(", query pages failed (%x)\n", out->cmd_status);
3824 1.1 jmcneill return (-1);
3825 1.1 jmcneill }
3826 1.1 jmcneill
3827 1.1 jmcneill *func_id = out->cmd_func_id;
3828 1.1 jmcneill *npages = be32dec(&out->cmd_num_pages);
3829 1.1 jmcneill
3830 1.1 jmcneill return (0);
3831 1.1 jmcneill }
3832 1.1 jmcneill
3833 1.1 jmcneill struct bus_dma_iter {
3834 1.1 jmcneill bus_dmamap_t i_map;
3835 1.1 jmcneill bus_size_t i_offset;
3836 1.1 jmcneill unsigned int i_index;
3837 1.1 jmcneill };
3838 1.1 jmcneill
3839 1.1 jmcneill static void
3840 1.1 jmcneill bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
3841 1.1 jmcneill {
3842 1.1 jmcneill i->i_map = map;
3843 1.1 jmcneill i->i_offset = 0;
3844 1.1 jmcneill i->i_index = 0;
3845 1.1 jmcneill }
3846 1.1 jmcneill
3847 1.1 jmcneill static bus_addr_t
3848 1.1 jmcneill bus_dma_iter_addr(struct bus_dma_iter *i)
3849 1.1 jmcneill {
3850 1.1 jmcneill return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
3851 1.1 jmcneill }
3852 1.1 jmcneill
3853 1.1 jmcneill static void
3854 1.1 jmcneill bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
3855 1.1 jmcneill {
3856 1.1 jmcneill bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
3857 1.1 jmcneill bus_size_t diff;
3858 1.1 jmcneill
3859 1.1 jmcneill do {
3860 1.1 jmcneill diff = seg->ds_len - i->i_offset;
3861 1.1 jmcneill if (size < diff)
3862 1.1 jmcneill break;
3863 1.1 jmcneill
3864 1.1 jmcneill size -= diff;
3865 1.1 jmcneill
3866 1.1 jmcneill seg++;
3867 1.1 jmcneill
3868 1.1 jmcneill i->i_offset = 0;
3869 1.1 jmcneill i->i_index++;
3870 1.1 jmcneill } while (size > 0);
3871 1.1 jmcneill
3872 1.1 jmcneill i->i_offset += size;
3873 1.1 jmcneill }
3874 1.1 jmcneill
3875 1.1 jmcneill static int
3876 1.1 jmcneill mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
3877 1.1 jmcneill {
3878 1.1 jmcneill struct mcx_dmamem mxm;
3879 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3880 1.1 jmcneill struct mcx_cmd_manage_pages_in *in;
3881 1.1 jmcneill struct mcx_cmd_manage_pages_out *out;
3882 1.1 jmcneill unsigned int paslen, nmb, i, j, npages;
3883 1.1 jmcneill struct bus_dma_iter iter;
3884 1.1 jmcneill uint64_t *pas;
3885 1.1 jmcneill uint8_t status;
3886 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3887 1.1 jmcneill int error;
3888 1.1 jmcneill
3889 1.1 jmcneill npages = mhm->mhm_npages;
3890 1.1 jmcneill
3891 1.1 jmcneill paslen = sizeof(*pas) * npages;
3892 1.1 jmcneill nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
3893 1.1 jmcneill
3894 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3895 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
3896 1.1 jmcneill
3897 1.1 jmcneill in = mcx_cmdq_in(cqe);
3898 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
3899 1.1 jmcneill in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
3900 1.1 jmcneill in->cmd_func_id = func_id;
3901 1.1 jmcneill be32enc(&in->cmd_input_num_entries, npages);
3902 1.1 jmcneill
3903 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
3904 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
3905 1.1 jmcneill printf(", unable to allocate manage pages mailboxen\n");
3906 1.1 jmcneill return (-1);
3907 1.1 jmcneill }
3908 1.1 jmcneill
3909 1.1 jmcneill bus_dma_iter_init(&iter, mhm->mhm_map);
3910 1.1 jmcneill for (i = 0; i < nmb; i++) {
3911 1.1 jmcneill unsigned int lim;
3912 1.1 jmcneill
3913 1.1 jmcneill pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
3914 1.1 jmcneill lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
3915 1.1 jmcneill
3916 1.1 jmcneill for (j = 0; j < lim; j++) {
3917 1.1 jmcneill be64enc(&pas[j], bus_dma_iter_addr(&iter));
3918 1.1 jmcneill bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
3919 1.1 jmcneill }
3920 1.1 jmcneill
3921 1.1 jmcneill npages -= lim;
3922 1.1 jmcneill }
3923 1.1 jmcneill
3924 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, nmb);
3925 1.1 jmcneill
3926 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3927 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3928 1.1 jmcneill if (error != 0) {
3929 1.1 jmcneill printf(", manage pages timeout\n");
3930 1.1 jmcneill goto free;
3931 1.1 jmcneill }
3932 1.1 jmcneill error = mcx_cmdq_verify(cqe);
3933 1.1 jmcneill if (error != 0) {
3934 1.1 jmcneill printf(", manage pages reply corrupt\n");
3935 1.1 jmcneill goto free;
3936 1.1 jmcneill }
3937 1.1 jmcneill
3938 1.1 jmcneill status = cqe->cq_output_data[0];
3939 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
3940 1.1 jmcneill printf(", manage pages failed (%x)\n", status);
3941 1.1 jmcneill error = -1;
3942 1.1 jmcneill goto free;
3943 1.1 jmcneill }
3944 1.1 jmcneill
3945 1.1 jmcneill free:
3946 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
3947 1.1 jmcneill
3948 1.1 jmcneill return (error);
3949 1.1 jmcneill }
3950 1.1 jmcneill
3951 1.1 jmcneill static int
3952 1.1 jmcneill mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
3953 1.1 jmcneill {
3954 1.15 jmcneill int32_t npages;
3955 1.1 jmcneill uint16_t func_id;
3956 1.1 jmcneill
3957 1.1 jmcneill if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
3958 1.1 jmcneill /* error printed by mcx_query_pages */
3959 1.1 jmcneill return (-1);
3960 1.1 jmcneill }
3961 1.1 jmcneill
3962 1.15 jmcneill if (npages < 1)
3963 1.1 jmcneill return (0);
3964 1.1 jmcneill
3965 1.1 jmcneill if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
3966 1.1 jmcneill printf(", unable to allocate hwmem\n");
3967 1.1 jmcneill return (-1);
3968 1.1 jmcneill }
3969 1.1 jmcneill
3970 1.1 jmcneill if (mcx_add_pages(sc, mhm, func_id) != 0) {
3971 1.1 jmcneill printf(", unable to add hwmem\n");
3972 1.1 jmcneill goto free;
3973 1.1 jmcneill }
3974 1.1 jmcneill
3975 1.1 jmcneill return (0);
3976 1.1 jmcneill
3977 1.1 jmcneill free:
3978 1.1 jmcneill mcx_hwmem_free(sc, mhm);
3979 1.1 jmcneill
3980 1.1 jmcneill return (-1);
3981 1.1 jmcneill }
3982 1.1 jmcneill
3983 1.1 jmcneill static int
3984 1.1 jmcneill mcx_hca_max_caps(struct mcx_softc *sc)
3985 1.1 jmcneill {
3986 1.1 jmcneill struct mcx_dmamem mxm;
3987 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3988 1.1 jmcneill struct mcx_cmd_query_hca_cap_in *in;
3989 1.1 jmcneill struct mcx_cmd_query_hca_cap_out *out;
3990 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
3991 1.1 jmcneill struct mcx_cap_device *hca;
3992 1.1 jmcneill uint8_t status;
3993 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3994 1.1 jmcneill int error;
3995 1.1 jmcneill
3996 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3997 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
3998 1.1 jmcneill token);
3999 1.1 jmcneill
4000 1.1 jmcneill in = mcx_cmdq_in(cqe);
4001 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
4002 1.1 jmcneill in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
4003 1.1 jmcneill MCX_CMD_QUERY_HCA_CAP_DEVICE);
4004 1.1 jmcneill
4005 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
4006 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
4007 1.1 jmcneill printf(", unable to allocate query hca caps mailboxen\n");
4008 1.1 jmcneill return (-1);
4009 1.1 jmcneill }
4010 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
4011 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
4012 1.1 jmcneill
4013 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4014 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4015 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
4016 1.1 jmcneill
4017 1.1 jmcneill if (error != 0) {
4018 1.1 jmcneill printf(", query hca caps timeout\n");
4019 1.1 jmcneill goto free;
4020 1.1 jmcneill }
4021 1.1 jmcneill error = mcx_cmdq_verify(cqe);
4022 1.1 jmcneill if (error != 0) {
4023 1.1 jmcneill printf(", query hca caps reply corrupt\n");
4024 1.1 jmcneill goto free;
4025 1.1 jmcneill }
4026 1.1 jmcneill
4027 1.1 jmcneill status = cqe->cq_output_data[0];
4028 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4029 1.1 jmcneill printf(", query hca caps failed (%x)\n", status);
4030 1.1 jmcneill error = -1;
4031 1.1 jmcneill goto free;
4032 1.1 jmcneill }
4033 1.1 jmcneill
4034 1.1 jmcneill mb = mcx_cq_mbox(&mxm, 0);
4035 1.1 jmcneill hca = mcx_cq_mbox_data(mb);
4036 1.1 jmcneill
4037 1.15 jmcneill if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE)
4038 1.15 jmcneill != MCX_CAP_DEVICE_PORT_TYPE_ETH) {
4039 1.15 jmcneill printf(", not in ethernet mode\n");
4040 1.15 jmcneill error = -1;
4041 1.15 jmcneill goto free;
4042 1.15 jmcneill }
4043 1.1 jmcneill if (hca->log_pg_sz > PAGE_SHIFT) {
4044 1.1 jmcneill printf(", minimum system page shift %u is too large\n",
4045 1.1 jmcneill hca->log_pg_sz);
4046 1.1 jmcneill error = -1;
4047 1.1 jmcneill goto free;
4048 1.1 jmcneill }
4049 1.1 jmcneill /*
4050 1.1 jmcneill * blueflame register is split into two buffers, and we must alternate
4051 1.1 jmcneill * between the two of them.
4052 1.1 jmcneill */
4053 1.1 jmcneill sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
4054 1.15 jmcneill sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size);
4055 1.24 skrll
4056 1.15 jmcneill if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG)
4057 1.15 jmcneill sc->sc_mcam_reg = 1;
4058 1.15 jmcneill
4059 1.15 jmcneill sc->sc_mhz = be32dec(&hca->device_frequency_mhz);
4060 1.15 jmcneill sc->sc_khz = be32dec(&hca->device_frequency_khz);
4061 1.1 jmcneill
4062 1.1 jmcneill free:
4063 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4064 1.1 jmcneill
4065 1.1 jmcneill return (error);
4066 1.1 jmcneill }
4067 1.1 jmcneill
4068 1.1 jmcneill static int
4069 1.1 jmcneill mcx_hca_set_caps(struct mcx_softc *sc)
4070 1.1 jmcneill {
4071 1.1 jmcneill struct mcx_dmamem mxm;
4072 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4073 1.1 jmcneill struct mcx_cmd_query_hca_cap_in *in;
4074 1.1 jmcneill struct mcx_cmd_query_hca_cap_out *out;
4075 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
4076 1.1 jmcneill struct mcx_cap_device *hca;
4077 1.1 jmcneill uint8_t status;
4078 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
4079 1.1 jmcneill int error;
4080 1.1 jmcneill
4081 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4082 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
4083 1.1 jmcneill token);
4084 1.1 jmcneill
4085 1.1 jmcneill in = mcx_cmdq_in(cqe);
4086 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
4087 1.1 jmcneill in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
4088 1.1 jmcneill MCX_CMD_QUERY_HCA_CAP_DEVICE);
4089 1.1 jmcneill
4090 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
4091 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
4092 1.1 jmcneill printf(", unable to allocate manage pages mailboxen\n");
4093 1.1 jmcneill return (-1);
4094 1.1 jmcneill }
4095 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
4096 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
4097 1.1 jmcneill
4098 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4099 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4100 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
4101 1.1 jmcneill
4102 1.1 jmcneill if (error != 0) {
4103 1.1 jmcneill printf(", query hca caps timeout\n");
4104 1.1 jmcneill goto free;
4105 1.1 jmcneill }
4106 1.1 jmcneill error = mcx_cmdq_verify(cqe);
4107 1.1 jmcneill if (error != 0) {
4108 1.1 jmcneill printf(", query hca caps reply corrupt\n");
4109 1.1 jmcneill goto free;
4110 1.1 jmcneill }
4111 1.1 jmcneill
4112 1.1 jmcneill status = cqe->cq_output_data[0];
4113 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4114 1.1 jmcneill printf(", query hca caps failed (%x)\n", status);
4115 1.1 jmcneill error = -1;
4116 1.1 jmcneill goto free;
4117 1.1 jmcneill }
4118 1.1 jmcneill
4119 1.1 jmcneill mb = mcx_cq_mbox(&mxm, 0);
4120 1.1 jmcneill hca = mcx_cq_mbox_data(mb);
4121 1.1 jmcneill
4122 1.1 jmcneill hca->log_pg_sz = PAGE_SHIFT;
4123 1.1 jmcneill
4124 1.1 jmcneill free:
4125 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4126 1.1 jmcneill
4127 1.1 jmcneill return (error);
4128 1.1 jmcneill }
4129 1.1 jmcneill
4130 1.1 jmcneill
4131 1.1 jmcneill static int
4132 1.1 jmcneill mcx_init_hca(struct mcx_softc *sc)
4133 1.1 jmcneill {
4134 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4135 1.1 jmcneill struct mcx_cmd_init_hca_in *in;
4136 1.1 jmcneill struct mcx_cmd_init_hca_out *out;
4137 1.1 jmcneill int error;
4138 1.1 jmcneill uint8_t status;
4139 1.1 jmcneill
4140 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4141 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4142 1.1 jmcneill
4143 1.1 jmcneill in = mcx_cmdq_in(cqe);
4144 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
4145 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4146 1.1 jmcneill
4147 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4148 1.1 jmcneill
4149 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4150 1.1 jmcneill if (error != 0) {
4151 1.1 jmcneill printf(", hca init timeout\n");
4152 1.1 jmcneill return (-1);
4153 1.1 jmcneill }
4154 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4155 1.1 jmcneill printf(", hca init command corrupt\n");
4156 1.1 jmcneill return (-1);
4157 1.1 jmcneill }
4158 1.1 jmcneill
4159 1.1 jmcneill status = cqe->cq_output_data[0];
4160 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4161 1.1 jmcneill printf(", hca init failed (%x)\n", status);
4162 1.1 jmcneill return (-1);
4163 1.1 jmcneill }
4164 1.1 jmcneill
4165 1.1 jmcneill return (0);
4166 1.1 jmcneill }
4167 1.1 jmcneill
4168 1.1 jmcneill static int
4169 1.1 jmcneill mcx_set_driver_version(struct mcx_softc *sc)
4170 1.1 jmcneill {
4171 1.1 jmcneill struct mcx_dmamem mxm;
4172 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4173 1.1 jmcneill struct mcx_cmd_set_driver_version_in *in;
4174 1.1 jmcneill struct mcx_cmd_set_driver_version_out *out;
4175 1.1 jmcneill int error;
4176 1.1 jmcneill int token;
4177 1.1 jmcneill uint8_t status;
4178 1.1 jmcneill
4179 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4180 1.1 jmcneill token = mcx_cmdq_token(sc);
4181 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) +
4182 1.1 jmcneill sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);
4183 1.1 jmcneill
4184 1.1 jmcneill in = mcx_cmdq_in(cqe);
4185 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
4186 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4187 1.1 jmcneill
4188 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4189 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
4190 1.1 jmcneill printf(", unable to allocate set driver version mailboxen\n");
4191 1.1 jmcneill return (-1);
4192 1.1 jmcneill }
4193 1.1 jmcneill strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
4194 1.1 jmcneill "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
4195 1.1 jmcneill
4196 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4197 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4198 1.1 jmcneill
4199 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4200 1.1 jmcneill if (error != 0) {
4201 1.1 jmcneill printf(", set driver version timeout\n");
4202 1.1 jmcneill goto free;
4203 1.1 jmcneill }
4204 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4205 1.1 jmcneill printf(", set driver version command corrupt\n");
4206 1.1 jmcneill goto free;
4207 1.1 jmcneill }
4208 1.1 jmcneill
4209 1.1 jmcneill status = cqe->cq_output_data[0];
4210 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4211 1.1 jmcneill printf(", set driver version failed (%x)\n", status);
4212 1.1 jmcneill error = -1;
4213 1.1 jmcneill goto free;
4214 1.1 jmcneill }
4215 1.1 jmcneill
4216 1.1 jmcneill free:
4217 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4218 1.1 jmcneill
4219 1.1 jmcneill return (error);
4220 1.1 jmcneill }
4221 1.1 jmcneill
4222 1.1 jmcneill static int
4223 1.1 jmcneill mcx_iff(struct mcx_softc *sc)
4224 1.1 jmcneill {
4225 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
4226 1.1 jmcneill struct mcx_dmamem mxm;
4227 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4228 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_in *in;
4229 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_out *out;
4230 1.1 jmcneill struct mcx_nic_vport_ctx *ctx;
4231 1.1 jmcneill int error;
4232 1.1 jmcneill int token;
4233 1.1 jmcneill int insize;
4234 1.15 jmcneill uint32_t dest;
4235 1.15 jmcneill
4236 1.15 jmcneill dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
4237 1.15 jmcneill sc->sc_rss_flow_table_id;
4238 1.1 jmcneill
4239 1.1 jmcneill /* enable or disable the promisc flow */
4240 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_PROMISC)) {
4241 1.1 jmcneill if (sc->sc_promisc_flow_enabled == 0) {
4242 1.15 jmcneill mcx_set_flow_table_entry_mac(sc,
4243 1.15 jmcneill MCX_FLOW_GROUP_PROMISC, 0, NULL, dest);
4244 1.1 jmcneill sc->sc_promisc_flow_enabled = 1;
4245 1.1 jmcneill }
4246 1.1 jmcneill } else if (sc->sc_promisc_flow_enabled != 0) {
4247 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
4248 1.1 jmcneill sc->sc_promisc_flow_enabled = 0;
4249 1.1 jmcneill }
4250 1.1 jmcneill
4251 1.1 jmcneill /* enable or disable the all-multicast flow */
4252 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
4253 1.1 jmcneill if (sc->sc_allmulti_flow_enabled == 0) {
4254 1.1 jmcneill uint8_t mcast[ETHER_ADDR_LEN];
4255 1.1 jmcneill
4256 1.1 jmcneill memset(mcast, 0, sizeof(mcast));
4257 1.1 jmcneill mcast[0] = 0x01;
4258 1.15 jmcneill mcx_set_flow_table_entry_mac(sc,
4259 1.15 jmcneill MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest);
4260 1.1 jmcneill sc->sc_allmulti_flow_enabled = 1;
4261 1.1 jmcneill }
4262 1.1 jmcneill } else if (sc->sc_allmulti_flow_enabled != 0) {
4263 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
4264 1.1 jmcneill sc->sc_allmulti_flow_enabled = 0;
4265 1.1 jmcneill }
4266 1.1 jmcneill
4267 1.1 jmcneill insize = sizeof(struct mcx_nic_vport_ctx) + 240;
4268 1.1 jmcneill
4269 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4270 1.1 jmcneill token = mcx_cmdq_token(sc);
4271 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4272 1.1 jmcneill
4273 1.1 jmcneill in = mcx_cmdq_in(cqe);
4274 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
4275 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4276 1.1 jmcneill in->cmd_field_select = htobe32(
4277 1.1 jmcneill MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
4278 1.1 jmcneill MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
4279 1.1 jmcneill
4280 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
4281 1.15 jmcneill printf(", unable to allocate modify "
4282 1.15 jmcneill "nic vport context mailboxen\n");
4283 1.1 jmcneill return (-1);
4284 1.1 jmcneill }
4285 1.1 jmcneill ctx = (struct mcx_nic_vport_ctx *)
4286 1.1 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
4287 1.1 jmcneill ctx->vp_mtu = htobe32(sc->sc_hardmtu);
4288 1.1 jmcneill /*
4289 1.15 jmcneill * always leave promisc-all enabled on the vport since we
4290 1.15 jmcneill * can't give it a vlan list, and we're already doing multicast
4291 1.15 jmcneill * filtering in the flow table.
4292 1.1 jmcneill */
4293 1.1 jmcneill ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);
4294 1.1 jmcneill
4295 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4296 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4297 1.1 jmcneill
4298 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4299 1.1 jmcneill if (error != 0) {
4300 1.1 jmcneill printf(", modify nic vport context timeout\n");
4301 1.1 jmcneill goto free;
4302 1.1 jmcneill }
4303 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4304 1.1 jmcneill printf(", modify nic vport context command corrupt\n");
4305 1.1 jmcneill goto free;
4306 1.1 jmcneill }
4307 1.1 jmcneill
4308 1.1 jmcneill out = mcx_cmdq_out(cqe);
4309 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4310 1.1 jmcneill printf(", modify nic vport context failed (%x, %x)\n",
4311 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4312 1.1 jmcneill error = -1;
4313 1.1 jmcneill goto free;
4314 1.1 jmcneill }
4315 1.1 jmcneill
4316 1.1 jmcneill free:
4317 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4318 1.1 jmcneill
4319 1.1 jmcneill return (error);
4320 1.1 jmcneill }
4321 1.1 jmcneill
4322 1.1 jmcneill static int
4323 1.15 jmcneill mcx_alloc_uar(struct mcx_softc *sc, int *uar)
4324 1.1 jmcneill {
4325 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4326 1.1 jmcneill struct mcx_cmd_alloc_uar_in *in;
4327 1.1 jmcneill struct mcx_cmd_alloc_uar_out *out;
4328 1.1 jmcneill int error;
4329 1.1 jmcneill
4330 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4331 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4332 1.1 jmcneill
4333 1.1 jmcneill in = mcx_cmdq_in(cqe);
4334 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
4335 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4336 1.1 jmcneill
4337 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4338 1.1 jmcneill
4339 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4340 1.1 jmcneill if (error != 0) {
4341 1.1 jmcneill printf(", alloc uar timeout\n");
4342 1.1 jmcneill return (-1);
4343 1.1 jmcneill }
4344 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4345 1.1 jmcneill printf(", alloc uar command corrupt\n");
4346 1.1 jmcneill return (-1);
4347 1.1 jmcneill }
4348 1.1 jmcneill
4349 1.1 jmcneill out = mcx_cmdq_out(cqe);
4350 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4351 1.1 jmcneill printf(", alloc uar failed (%x)\n", out->cmd_status);
4352 1.1 jmcneill return (-1);
4353 1.1 jmcneill }
4354 1.1 jmcneill
4355 1.15 jmcneill *uar = mcx_get_id(out->cmd_uar);
4356 1.1 jmcneill return (0);
4357 1.1 jmcneill }
4358 1.1 jmcneill
4359 1.1 jmcneill static int
4360 1.15 jmcneill mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar,
4361 1.15 jmcneill uint64_t events, int vector)
4362 1.1 jmcneill {
4363 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4364 1.1 jmcneill struct mcx_dmamem mxm;
4365 1.1 jmcneill struct mcx_cmd_create_eq_in *in;
4366 1.1 jmcneill struct mcx_cmd_create_eq_mb_in *mbin;
4367 1.1 jmcneill struct mcx_cmd_create_eq_out *out;
4368 1.1 jmcneill struct mcx_eq_entry *eqe;
4369 1.1 jmcneill int error;
4370 1.1 jmcneill uint64_t *pas;
4371 1.1 jmcneill int insize, npages, paslen, i, token;
4372 1.1 jmcneill
4373 1.15 jmcneill eq->eq_cons = 0;
4374 1.1 jmcneill
4375 1.1 jmcneill npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
4376 1.1 jmcneill MCX_PAGE_SIZE);
4377 1.1 jmcneill paslen = npages * sizeof(*pas);
4378 1.1 jmcneill insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
4379 1.1 jmcneill
4380 1.15 jmcneill if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE,
4381 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
4382 1.1 jmcneill printf(", unable to allocate event queue memory\n");
4383 1.1 jmcneill return (-1);
4384 1.1 jmcneill }
4385 1.1 jmcneill
4386 1.15 jmcneill eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
4387 1.1 jmcneill for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
4388 1.1 jmcneill eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
4389 1.1 jmcneill }
4390 1.1 jmcneill
4391 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4392 1.1 jmcneill token = mcx_cmdq_token(sc);
4393 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4394 1.1 jmcneill
4395 1.1 jmcneill in = mcx_cmdq_in(cqe);
4396 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
4397 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4398 1.1 jmcneill
4399 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4400 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4401 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
4402 1.1 jmcneill printf(", unable to allocate create eq mailboxen\n");
4403 1.15 jmcneill goto free_eq;
4404 1.1 jmcneill }
4405 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4406 1.1 jmcneill mbin->cmd_eq_ctx.eq_uar_size = htobe32(
4407 1.15 jmcneill (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar);
4408 1.15 jmcneill mbin->cmd_eq_ctx.eq_intr = vector;
4409 1.15 jmcneill mbin->cmd_event_bitmask = htobe64(events);
4410 1.15 jmcneill
4411 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
4412 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
4413 1.1 jmcneill
4414 1.1 jmcneill /* physical addresses follow the mailbox in data */
4415 1.15 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem);
4416 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
4417 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4418 1.1 jmcneill
4419 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4420 1.1 jmcneill if (error != 0) {
4421 1.1 jmcneill printf(", create eq timeout\n");
4422 1.15 jmcneill goto free_mxm;
4423 1.1 jmcneill }
4424 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4425 1.1 jmcneill printf(", create eq command corrupt\n");
4426 1.15 jmcneill goto free_mxm;
4427 1.1 jmcneill }
4428 1.1 jmcneill
4429 1.1 jmcneill out = mcx_cmdq_out(cqe);
4430 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4431 1.1 jmcneill printf(", create eq failed (%x, %x)\n", out->cmd_status,
4432 1.1 jmcneill be32toh(out->cmd_syndrome));
4433 1.15 jmcneill goto free_mxm;
4434 1.1 jmcneill }
4435 1.1 jmcneill
4436 1.15 jmcneill eq->eq_n = mcx_get_id(out->cmd_eqn);
4437 1.15 jmcneill
4438 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
4439 1.15 jmcneill
4440 1.15 jmcneill mcx_arm_eq(sc, eq, uar);
4441 1.15 jmcneill
4442 1.15 jmcneill return (0);
4443 1.15 jmcneill
4444 1.15 jmcneill free_mxm:
4445 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
4446 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
4447 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4448 1.15 jmcneill free_eq:
4449 1.15 jmcneill mcx_dmamem_free(sc, &eq->eq_mem);
4450 1.15 jmcneill return (-1);
4451 1.1 jmcneill }
4452 1.1 jmcneill
4453 1.1 jmcneill static int
4454 1.1 jmcneill mcx_alloc_pd(struct mcx_softc *sc)
4455 1.1 jmcneill {
4456 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4457 1.1 jmcneill struct mcx_cmd_alloc_pd_in *in;
4458 1.1 jmcneill struct mcx_cmd_alloc_pd_out *out;
4459 1.1 jmcneill int error;
4460 1.1 jmcneill
4461 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4462 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4463 1.1 jmcneill
4464 1.1 jmcneill in = mcx_cmdq_in(cqe);
4465 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
4466 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4467 1.1 jmcneill
4468 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4469 1.1 jmcneill
4470 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4471 1.1 jmcneill if (error != 0) {
4472 1.1 jmcneill printf(", alloc pd timeout\n");
4473 1.1 jmcneill return (-1);
4474 1.1 jmcneill }
4475 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4476 1.1 jmcneill printf(", alloc pd command corrupt\n");
4477 1.1 jmcneill return (-1);
4478 1.1 jmcneill }
4479 1.1 jmcneill
4480 1.1 jmcneill out = mcx_cmdq_out(cqe);
4481 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4482 1.1 jmcneill printf(", alloc pd failed (%x)\n", out->cmd_status);
4483 1.1 jmcneill return (-1);
4484 1.1 jmcneill }
4485 1.1 jmcneill
4486 1.13 jmcneill sc->sc_pd = mcx_get_id(out->cmd_pd);
4487 1.1 jmcneill return (0);
4488 1.1 jmcneill }
4489 1.1 jmcneill
4490 1.1 jmcneill static int
4491 1.1 jmcneill mcx_alloc_tdomain(struct mcx_softc *sc)
4492 1.1 jmcneill {
4493 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4494 1.1 jmcneill struct mcx_cmd_alloc_td_in *in;
4495 1.1 jmcneill struct mcx_cmd_alloc_td_out *out;
4496 1.1 jmcneill int error;
4497 1.1 jmcneill
4498 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4499 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4500 1.1 jmcneill
4501 1.1 jmcneill in = mcx_cmdq_in(cqe);
4502 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
4503 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4504 1.1 jmcneill
4505 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4506 1.1 jmcneill
4507 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4508 1.1 jmcneill if (error != 0) {
4509 1.1 jmcneill printf(", alloc transport domain timeout\n");
4510 1.1 jmcneill return (-1);
4511 1.1 jmcneill }
4512 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4513 1.1 jmcneill printf(", alloc transport domain command corrupt\n");
4514 1.1 jmcneill return (-1);
4515 1.1 jmcneill }
4516 1.1 jmcneill
4517 1.1 jmcneill out = mcx_cmdq_out(cqe);
4518 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4519 1.1 jmcneill printf(", alloc transport domain failed (%x)\n",
4520 1.1 jmcneill out->cmd_status);
4521 1.1 jmcneill return (-1);
4522 1.1 jmcneill }
4523 1.1 jmcneill
4524 1.13 jmcneill sc->sc_tdomain = mcx_get_id(out->cmd_tdomain);
4525 1.1 jmcneill return (0);
4526 1.1 jmcneill }
4527 1.1 jmcneill
4528 1.1 jmcneill static int
4529 1.1 jmcneill mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr)
4530 1.1 jmcneill {
4531 1.1 jmcneill struct mcx_dmamem mxm;
4532 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4533 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_in *in;
4534 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_out *out;
4535 1.1 jmcneill struct mcx_nic_vport_ctx *ctx;
4536 1.1 jmcneill uint8_t *addr;
4537 1.1 jmcneill int error, token, i;
4538 1.1 jmcneill
4539 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4540 1.1 jmcneill token = mcx_cmdq_token(sc);
4541 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
4542 1.1 jmcneill
4543 1.1 jmcneill in = mcx_cmdq_in(cqe);
4544 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
4545 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4546 1.1 jmcneill in->cmd_allowed_list_type = 0;
4547 1.1 jmcneill
4548 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4549 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
4550 1.15 jmcneill printf(", unable to allocate "
4551 1.15 jmcneill "query nic vport context mailboxen\n");
4552 1.1 jmcneill return (-1);
4553 1.1 jmcneill }
4554 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4555 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4556 1.1 jmcneill
4557 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4558 1.1 jmcneill if (error != 0) {
4559 1.1 jmcneill printf(", query nic vport context timeout\n");
4560 1.1 jmcneill goto free;
4561 1.1 jmcneill }
4562 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4563 1.1 jmcneill printf(", query nic vport context command corrupt\n");
4564 1.1 jmcneill goto free;
4565 1.1 jmcneill }
4566 1.1 jmcneill
4567 1.1 jmcneill out = mcx_cmdq_out(cqe);
4568 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4569 1.1 jmcneill printf(", query nic vport context failed (%x, %x)\n",
4570 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4571 1.1 jmcneill error = -1;
4572 1.1 jmcneill goto free;
4573 1.1 jmcneill }
4574 1.1 jmcneill
4575 1.15 jmcneill ctx = (struct mcx_nic_vport_ctx *)
4576 1.15 jmcneill mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4577 1.1 jmcneill addr = (uint8_t *)&ctx->vp_perm_addr;
4578 1.1 jmcneill for (i = 0; i < ETHER_ADDR_LEN; i++) {
4579 1.1 jmcneill enaddr[i] = addr[i + 2];
4580 1.1 jmcneill }
4581 1.1 jmcneill free:
4582 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4583 1.1 jmcneill
4584 1.1 jmcneill return (error);
4585 1.1 jmcneill }
4586 1.1 jmcneill
4587 1.1 jmcneill static int
4588 1.1 jmcneill mcx_query_special_contexts(struct mcx_softc *sc)
4589 1.1 jmcneill {
4590 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4591 1.1 jmcneill struct mcx_cmd_query_special_ctx_in *in;
4592 1.1 jmcneill struct mcx_cmd_query_special_ctx_out *out;
4593 1.1 jmcneill int error;
4594 1.1 jmcneill
4595 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4596 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4597 1.1 jmcneill
4598 1.1 jmcneill in = mcx_cmdq_in(cqe);
4599 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
4600 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4601 1.1 jmcneill
4602 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4603 1.1 jmcneill
4604 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4605 1.1 jmcneill if (error != 0) {
4606 1.1 jmcneill printf(", query special contexts timeout\n");
4607 1.1 jmcneill return (-1);
4608 1.1 jmcneill }
4609 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4610 1.1 jmcneill printf(", query special contexts command corrupt\n");
4611 1.1 jmcneill return (-1);
4612 1.1 jmcneill }
4613 1.1 jmcneill
4614 1.1 jmcneill out = mcx_cmdq_out(cqe);
4615 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4616 1.1 jmcneill printf(", query special contexts failed (%x)\n",
4617 1.1 jmcneill out->cmd_status);
4618 1.1 jmcneill return (-1);
4619 1.1 jmcneill }
4620 1.1 jmcneill
4621 1.1 jmcneill sc->sc_lkey = be32toh(out->cmd_resd_lkey);
4622 1.1 jmcneill return (0);
4623 1.1 jmcneill }
4624 1.1 jmcneill
4625 1.1 jmcneill static int
4626 1.1 jmcneill mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
4627 1.1 jmcneill {
4628 1.1 jmcneill struct mcx_reg_pmtu pmtu;
4629 1.1 jmcneill int error;
4630 1.1 jmcneill
4631 1.1 jmcneill /* read max mtu */
4632 1.1 jmcneill memset(&pmtu, 0, sizeof(pmtu));
4633 1.1 jmcneill pmtu.rp_local_port = 1;
4634 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
4635 1.1 jmcneill sizeof(pmtu));
4636 1.1 jmcneill if (error != 0) {
4637 1.1 jmcneill printf(", unable to get port MTU\n");
4638 1.1 jmcneill return error;
4639 1.1 jmcneill }
4640 1.1 jmcneill
4641 1.1 jmcneill mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu));
4642 1.1 jmcneill pmtu.rp_admin_mtu = htobe16(mtu);
4643 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
4644 1.1 jmcneill sizeof(pmtu));
4645 1.1 jmcneill if (error != 0) {
4646 1.1 jmcneill printf(", unable to set port MTU\n");
4647 1.1 jmcneill return error;
4648 1.1 jmcneill }
4649 1.1 jmcneill
4650 1.1 jmcneill sc->sc_hardmtu = mtu;
4651 1.15 jmcneill sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long));
4652 1.1 jmcneill return 0;
4653 1.1 jmcneill }
4654 1.1 jmcneill
4655 1.1 jmcneill static int
4656 1.15 jmcneill mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn)
4657 1.1 jmcneill {
4658 1.1 jmcneill struct mcx_cmdq_entry *cmde;
4659 1.1 jmcneill struct mcx_cq_entry *cqe;
4660 1.1 jmcneill struct mcx_dmamem mxm;
4661 1.1 jmcneill struct mcx_cmd_create_cq_in *in;
4662 1.1 jmcneill struct mcx_cmd_create_cq_mb_in *mbin;
4663 1.1 jmcneill struct mcx_cmd_create_cq_out *out;
4664 1.1 jmcneill int error;
4665 1.1 jmcneill uint64_t *pas;
4666 1.1 jmcneill int insize, npages, paslen, i, token;
4667 1.1 jmcneill
4668 1.15 jmcneill cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db);
4669 1.1 jmcneill
4670 1.1 jmcneill npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
4671 1.1 jmcneill MCX_PAGE_SIZE);
4672 1.1 jmcneill paslen = npages * sizeof(*pas);
4673 1.1 jmcneill insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
4674 1.1 jmcneill
4675 1.1 jmcneill if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
4676 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
4677 1.1 jmcneill printf("%s: unable to allocate completion queue memory\n",
4678 1.1 jmcneill DEVNAME(sc));
4679 1.1 jmcneill return (-1);
4680 1.1 jmcneill }
4681 1.1 jmcneill cqe = MCX_DMA_KVA(&cq->cq_mem);
4682 1.1 jmcneill for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
4683 1.1 jmcneill cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
4684 1.1 jmcneill }
4685 1.1 jmcneill
4686 1.1 jmcneill cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4687 1.1 jmcneill token = mcx_cmdq_token(sc);
4688 1.1 jmcneill mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
4689 1.1 jmcneill
4690 1.1 jmcneill in = mcx_cmdq_in(cmde);
4691 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
4692 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4693 1.1 jmcneill
4694 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4695 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4696 1.1 jmcneill &cmde->cq_input_ptr, token) != 0) {
4697 1.15 jmcneill printf("%s: unable to allocate create cq mailboxen\n",
4698 1.15 jmcneill DEVNAME(sc));
4699 1.15 jmcneill goto free_cq;
4700 1.1 jmcneill }
4701 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4702 1.1 jmcneill mbin->cmd_cq_ctx.cq_uar_size = htobe32(
4703 1.15 jmcneill (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar);
4704 1.1 jmcneill mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
4705 1.1 jmcneill mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
4706 1.1 jmcneill (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
4707 1.1 jmcneill MCX_CQ_MOD_COUNTER);
4708 1.1 jmcneill mbin->cmd_cq_ctx.cq_doorbell = htobe64(
4709 1.15 jmcneill MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell);
4710 1.15 jmcneill
4711 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4712 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
4713 1.1 jmcneill
4714 1.1 jmcneill /* physical addresses follow the mailbox in data */
4715 1.13 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem);
4716 1.1 jmcneill mcx_cmdq_post(sc, cmde, 0);
4717 1.1 jmcneill
4718 1.1 jmcneill error = mcx_cmdq_poll(sc, cmde, 1000);
4719 1.1 jmcneill if (error != 0) {
4720 1.1 jmcneill printf("%s: create cq timeout\n", DEVNAME(sc));
4721 1.15 jmcneill goto free_mxm;
4722 1.1 jmcneill }
4723 1.1 jmcneill if (mcx_cmdq_verify(cmde) != 0) {
4724 1.1 jmcneill printf("%s: create cq command corrupt\n", DEVNAME(sc));
4725 1.15 jmcneill goto free_mxm;
4726 1.1 jmcneill }
4727 1.1 jmcneill
4728 1.1 jmcneill out = mcx_cmdq_out(cmde);
4729 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4730 1.1 jmcneill printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
4731 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4732 1.15 jmcneill goto free_mxm;
4733 1.1 jmcneill }
4734 1.1 jmcneill
4735 1.13 jmcneill cq->cq_n = mcx_get_id(out->cmd_cqn);
4736 1.1 jmcneill cq->cq_cons = 0;
4737 1.1 jmcneill cq->cq_count = 0;
4738 1.1 jmcneill
4739 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4740 1.15 jmcneill
4741 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4742 1.15 jmcneill cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
4743 1.15 jmcneill BUS_DMASYNC_PREWRITE);
4744 1.15 jmcneill
4745 1.15 jmcneill mcx_arm_cq(sc, cq, uar);
4746 1.15 jmcneill
4747 1.15 jmcneill return (0);
4748 1.15 jmcneill
4749 1.15 jmcneill free_mxm:
4750 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4751 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
4752 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
4753 1.15 jmcneill free_cq:
4754 1.15 jmcneill mcx_dmamem_free(sc, &cq->cq_mem);
4755 1.15 jmcneill return (-1);
4756 1.1 jmcneill }
4757 1.1 jmcneill
4758 1.1 jmcneill static int
4759 1.15 jmcneill mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq)
4760 1.1 jmcneill {
4761 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4762 1.1 jmcneill struct mcx_cmd_destroy_cq_in *in;
4763 1.1 jmcneill struct mcx_cmd_destroy_cq_out *out;
4764 1.1 jmcneill int error;
4765 1.1 jmcneill int token;
4766 1.1 jmcneill
4767 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4768 1.1 jmcneill token = mcx_cmdq_token(sc);
4769 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
4770 1.1 jmcneill
4771 1.1 jmcneill in = mcx_cmdq_in(cqe);
4772 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
4773 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4774 1.15 jmcneill in->cmd_cqn = htobe32(cq->cq_n);
4775 1.1 jmcneill
4776 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4777 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4778 1.1 jmcneill if (error != 0) {
4779 1.1 jmcneill printf("%s: destroy cq timeout\n", DEVNAME(sc));
4780 1.1 jmcneill return error;
4781 1.1 jmcneill }
4782 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4783 1.1 jmcneill printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
4784 1.1 jmcneill return error;
4785 1.1 jmcneill }
4786 1.1 jmcneill
4787 1.1 jmcneill out = mcx_cmdq_out(cqe);
4788 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4789 1.1 jmcneill printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
4790 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4791 1.1 jmcneill return -1;
4792 1.1 jmcneill }
4793 1.1 jmcneill
4794 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4795 1.15 jmcneill cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
4796 1.15 jmcneill BUS_DMASYNC_POSTWRITE);
4797 1.15 jmcneill
4798 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4799 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
4800 1.15 jmcneill mcx_dmamem_free(sc, &cq->cq_mem);
4801 1.15 jmcneill
4802 1.15 jmcneill cq->cq_n = 0;
4803 1.15 jmcneill cq->cq_cons = 0;
4804 1.15 jmcneill cq->cq_count = 0;
4805 1.1 jmcneill return 0;
4806 1.1 jmcneill }
4807 1.1 jmcneill
4808 1.1 jmcneill static int
4809 1.15 jmcneill mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn)
4810 1.1 jmcneill {
4811 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4812 1.1 jmcneill struct mcx_dmamem mxm;
4813 1.1 jmcneill struct mcx_cmd_create_rq_in *in;
4814 1.1 jmcneill struct mcx_cmd_create_rq_out *out;
4815 1.1 jmcneill struct mcx_rq_ctx *mbin;
4816 1.1 jmcneill int error;
4817 1.1 jmcneill uint64_t *pas;
4818 1.15 jmcneill uint32_t rq_flags;
4819 1.13 jmcneill int insize, npages, paslen, token;
4820 1.1 jmcneill
4821 1.15 jmcneill rx->rx_doorbell = MCX_WQ_DOORBELL_BASE +
4822 1.15 jmcneill (db * MCX_WQ_DOORBELL_STRIDE);
4823 1.15 jmcneill
4824 1.1 jmcneill npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
4825 1.1 jmcneill MCX_PAGE_SIZE);
4826 1.1 jmcneill paslen = npages * sizeof(*pas);
4827 1.1 jmcneill insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
4828 1.1 jmcneill
4829 1.15 jmcneill if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE,
4830 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
4831 1.1 jmcneill printf("%s: unable to allocate receive queue memory\n",
4832 1.1 jmcneill DEVNAME(sc));
4833 1.1 jmcneill return (-1);
4834 1.1 jmcneill }
4835 1.1 jmcneill
4836 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4837 1.1 jmcneill token = mcx_cmdq_token(sc);
4838 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4839 1.1 jmcneill
4840 1.1 jmcneill in = mcx_cmdq_in(cqe);
4841 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
4842 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4843 1.1 jmcneill
4844 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4845 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4846 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
4847 1.1 jmcneill printf("%s: unable to allocate create rq mailboxen\n",
4848 1.1 jmcneill DEVNAME(sc));
4849 1.15 jmcneill goto free_rq;
4850 1.15 jmcneill }
4851 1.15 jmcneill mbin = (struct mcx_rq_ctx *)
4852 1.15 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
4853 1.15 jmcneill rq_flags = MCX_RQ_CTX_RLKEY;
4854 1.15 jmcneill mbin->rq_flags = htobe32(rq_flags);
4855 1.1 jmcneill mbin->rq_cqn = htobe32(cqn);
4856 1.1 jmcneill mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
4857 1.1 jmcneill mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
4858 1.1 jmcneill mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
4859 1.15 jmcneill rx->rx_doorbell);
4860 1.1 jmcneill mbin->rq_wq.wq_log_stride = htobe16(4);
4861 1.1 jmcneill mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
4862 1.1 jmcneill
4863 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4864 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
4865 1.15 jmcneill
4866 1.1 jmcneill /* physical addresses follow the mailbox in data */
4867 1.15 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem);
4868 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4869 1.1 jmcneill
4870 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4871 1.1 jmcneill if (error != 0) {
4872 1.1 jmcneill printf("%s: create rq timeout\n", DEVNAME(sc));
4873 1.15 jmcneill goto free_mxm;
4874 1.1 jmcneill }
4875 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4876 1.1 jmcneill printf("%s: create rq command corrupt\n", DEVNAME(sc));
4877 1.15 jmcneill goto free_mxm;
4878 1.1 jmcneill }
4879 1.1 jmcneill
4880 1.1 jmcneill out = mcx_cmdq_out(cqe);
4881 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4882 1.1 jmcneill printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
4883 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4884 1.15 jmcneill goto free_mxm;
4885 1.1 jmcneill }
4886 1.1 jmcneill
4887 1.15 jmcneill rx->rx_rqn = mcx_get_id(out->cmd_rqn);
4888 1.1 jmcneill
4889 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
4890 1.15 jmcneill
4891 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4892 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
4893 1.15 jmcneill
4894 1.15 jmcneill return (0);
4895 1.1 jmcneill
4896 1.15 jmcneill free_mxm:
4897 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4898 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
4899 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4900 1.15 jmcneill free_rq:
4901 1.15 jmcneill mcx_dmamem_free(sc, &rx->rx_rq_mem);
4902 1.15 jmcneill return (-1);
4903 1.1 jmcneill }
4904 1.1 jmcneill
4905 1.1 jmcneill static int
4906 1.15 jmcneill mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx)
4907 1.1 jmcneill {
4908 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4909 1.1 jmcneill struct mcx_dmamem mxm;
4910 1.1 jmcneill struct mcx_cmd_modify_rq_in *in;
4911 1.1 jmcneill struct mcx_cmd_modify_rq_mb_in *mbin;
4912 1.1 jmcneill struct mcx_cmd_modify_rq_out *out;
4913 1.1 jmcneill int error;
4914 1.1 jmcneill int token;
4915 1.1 jmcneill
4916 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4917 1.1 jmcneill token = mcx_cmdq_token(sc);
4918 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
4919 1.15 jmcneill sizeof(*out), token);
4920 1.1 jmcneill
4921 1.1 jmcneill in = mcx_cmdq_in(cqe);
4922 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
4923 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4924 1.15 jmcneill in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn);
4925 1.1 jmcneill
4926 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4927 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
4928 1.15 jmcneill printf("%s: unable to allocate modify rq mailbox\n",
4929 1.15 jmcneill DEVNAME(sc));
4930 1.1 jmcneill return (-1);
4931 1.1 jmcneill }
4932 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4933 1.1 jmcneill mbin->cmd_rq_ctx.rq_flags = htobe32(
4934 1.1 jmcneill MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
4935 1.1 jmcneill
4936 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4937 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4938 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4939 1.1 jmcneill if (error != 0) {
4940 1.1 jmcneill printf("%s: modify rq timeout\n", DEVNAME(sc));
4941 1.1 jmcneill goto free;
4942 1.1 jmcneill }
4943 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4944 1.1 jmcneill printf("%s: modify rq command corrupt\n", DEVNAME(sc));
4945 1.1 jmcneill goto free;
4946 1.1 jmcneill }
4947 1.1 jmcneill
4948 1.1 jmcneill out = mcx_cmdq_out(cqe);
4949 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4950 1.1 jmcneill printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
4951 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4952 1.1 jmcneill error = -1;
4953 1.1 jmcneill goto free;
4954 1.1 jmcneill }
4955 1.1 jmcneill
4956 1.1 jmcneill free:
4957 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4958 1.1 jmcneill return (error);
4959 1.1 jmcneill }
4960 1.1 jmcneill
4961 1.1 jmcneill static int
4962 1.15 jmcneill mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx)
4963 1.1 jmcneill {
4964 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4965 1.1 jmcneill struct mcx_cmd_destroy_rq_in *in;
4966 1.1 jmcneill struct mcx_cmd_destroy_rq_out *out;
4967 1.1 jmcneill int error;
4968 1.1 jmcneill int token;
4969 1.1 jmcneill
4970 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4971 1.1 jmcneill token = mcx_cmdq_token(sc);
4972 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
4973 1.1 jmcneill
4974 1.1 jmcneill in = mcx_cmdq_in(cqe);
4975 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
4976 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4977 1.15 jmcneill in->cmd_rqn = htobe32(rx->rx_rqn);
4978 1.1 jmcneill
4979 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4980 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4981 1.1 jmcneill if (error != 0) {
4982 1.1 jmcneill printf("%s: destroy rq timeout\n", DEVNAME(sc));
4983 1.1 jmcneill return error;
4984 1.1 jmcneill }
4985 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4986 1.1 jmcneill printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
4987 1.1 jmcneill return error;
4988 1.1 jmcneill }
4989 1.1 jmcneill
4990 1.1 jmcneill out = mcx_cmdq_out(cqe);
4991 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4992 1.1 jmcneill printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
4993 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4994 1.1 jmcneill return -1;
4995 1.1 jmcneill }
4996 1.1 jmcneill
4997 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4998 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
4999 1.15 jmcneill
5000 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
5001 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
5002 1.15 jmcneill mcx_dmamem_free(sc, &rx->rx_rq_mem);
5003 1.15 jmcneill
5004 1.15 jmcneill rx->rx_rqn = 0;
5005 1.1 jmcneill return 0;
5006 1.1 jmcneill }
5007 1.1 jmcneill
5008 1.1 jmcneill static int
5009 1.15 jmcneill mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn)
5010 1.1 jmcneill {
5011 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5012 1.1 jmcneill struct mcx_dmamem mxm;
5013 1.1 jmcneill struct mcx_cmd_create_tir_in *in;
5014 1.1 jmcneill struct mcx_cmd_create_tir_mb_in *mbin;
5015 1.1 jmcneill struct mcx_cmd_create_tir_out *out;
5016 1.1 jmcneill int error;
5017 1.1 jmcneill int token;
5018 1.1 jmcneill
5019 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5020 1.1 jmcneill token = mcx_cmdq_token(sc);
5021 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5022 1.15 jmcneill sizeof(*out), token);
5023 1.1 jmcneill
5024 1.1 jmcneill in = mcx_cmdq_in(cqe);
5025 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
5026 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5027 1.1 jmcneill
5028 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5029 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5030 1.1 jmcneill printf("%s: unable to allocate create tir mailbox\n",
5031 1.1 jmcneill DEVNAME(sc));
5032 1.1 jmcneill return (-1);
5033 1.1 jmcneill }
5034 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5035 1.1 jmcneill /* leave disp_type = 0, so packets get sent to the inline rqn */
5036 1.15 jmcneill mbin->cmd_inline_rqn = htobe32(rx->rx_rqn);
5037 1.1 jmcneill mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
5038 1.1 jmcneill
5039 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5040 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5041 1.1 jmcneill if (error != 0) {
5042 1.1 jmcneill printf("%s: create tir timeout\n", DEVNAME(sc));
5043 1.1 jmcneill goto free;
5044 1.1 jmcneill }
5045 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5046 1.1 jmcneill printf("%s: create tir command corrupt\n", DEVNAME(sc));
5047 1.1 jmcneill goto free;
5048 1.1 jmcneill }
5049 1.1 jmcneill
5050 1.1 jmcneill out = mcx_cmdq_out(cqe);
5051 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5052 1.1 jmcneill printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
5053 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5054 1.1 jmcneill error = -1;
5055 1.1 jmcneill goto free;
5056 1.1 jmcneill }
5057 1.1 jmcneill
5058 1.15 jmcneill *tirn = mcx_get_id(out->cmd_tirn);
5059 1.15 jmcneill free:
5060 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
5061 1.15 jmcneill return (error);
5062 1.15 jmcneill }
5063 1.15 jmcneill
5064 1.15 jmcneill static int
5065 1.15 jmcneill mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel,
5066 1.15 jmcneill int *tirn)
5067 1.15 jmcneill {
5068 1.15 jmcneill struct mcx_cmdq_entry *cqe;
5069 1.15 jmcneill struct mcx_dmamem mxm;
5070 1.15 jmcneill struct mcx_cmd_create_tir_in *in;
5071 1.15 jmcneill struct mcx_cmd_create_tir_mb_in *mbin;
5072 1.15 jmcneill struct mcx_cmd_create_tir_out *out;
5073 1.15 jmcneill int error;
5074 1.15 jmcneill int token;
5075 1.15 jmcneill
5076 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5077 1.15 jmcneill token = mcx_cmdq_token(sc);
5078 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5079 1.15 jmcneill sizeof(*out), token);
5080 1.15 jmcneill
5081 1.15 jmcneill in = mcx_cmdq_in(cqe);
5082 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
5083 1.15 jmcneill in->cmd_op_mod = htobe16(0);
5084 1.15 jmcneill
5085 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5086 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5087 1.15 jmcneill printf("%s: unable to allocate create tir mailbox\n",
5088 1.15 jmcneill DEVNAME(sc));
5089 1.15 jmcneill return (-1);
5090 1.15 jmcneill }
5091 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5092 1.15 jmcneill mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT
5093 1.15 jmcneill << MCX_TIR_CTX_DISP_TYPE_SHIFT);
5094 1.15 jmcneill mbin->cmd_indir_table = htobe32(rqtn);
5095 1.15 jmcneill mbin->cmd_tdomain = htobe32(sc->sc_tdomain |
5096 1.15 jmcneill MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT);
5097 1.15 jmcneill mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel);
5098 1.15 jmcneill stoeplitz_to_key(&mbin->cmd_rx_hash_key,
5099 1.15 jmcneill sizeof(mbin->cmd_rx_hash_key));
5100 1.15 jmcneill
5101 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
5102 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5103 1.15 jmcneill if (error != 0) {
5104 1.15 jmcneill printf("%s: create tir timeout\n", DEVNAME(sc));
5105 1.15 jmcneill goto free;
5106 1.15 jmcneill }
5107 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5108 1.15 jmcneill printf("%s: create tir command corrupt\n", DEVNAME(sc));
5109 1.15 jmcneill goto free;
5110 1.15 jmcneill }
5111 1.15 jmcneill
5112 1.15 jmcneill out = mcx_cmdq_out(cqe);
5113 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5114 1.15 jmcneill printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
5115 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5116 1.15 jmcneill error = -1;
5117 1.15 jmcneill goto free;
5118 1.15 jmcneill }
5119 1.15 jmcneill
5120 1.15 jmcneill *tirn = mcx_get_id(out->cmd_tirn);
5121 1.1 jmcneill free:
5122 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5123 1.1 jmcneill return (error);
5124 1.1 jmcneill }
5125 1.1 jmcneill
5126 1.1 jmcneill static int
5127 1.15 jmcneill mcx_destroy_tir(struct mcx_softc *sc, int tirn)
5128 1.1 jmcneill {
5129 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5130 1.1 jmcneill struct mcx_cmd_destroy_tir_in *in;
5131 1.1 jmcneill struct mcx_cmd_destroy_tir_out *out;
5132 1.1 jmcneill int error;
5133 1.1 jmcneill int token;
5134 1.1 jmcneill
5135 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5136 1.1 jmcneill token = mcx_cmdq_token(sc);
5137 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5138 1.1 jmcneill
5139 1.1 jmcneill in = mcx_cmdq_in(cqe);
5140 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
5141 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5142 1.15 jmcneill in->cmd_tirn = htobe32(tirn);
5143 1.1 jmcneill
5144 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5145 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5146 1.1 jmcneill if (error != 0) {
5147 1.1 jmcneill printf("%s: destroy tir timeout\n", DEVNAME(sc));
5148 1.1 jmcneill return error;
5149 1.1 jmcneill }
5150 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5151 1.1 jmcneill printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
5152 1.1 jmcneill return error;
5153 1.1 jmcneill }
5154 1.1 jmcneill
5155 1.1 jmcneill out = mcx_cmdq_out(cqe);
5156 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5157 1.1 jmcneill printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
5158 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5159 1.1 jmcneill return -1;
5160 1.1 jmcneill }
5161 1.1 jmcneill
5162 1.15 jmcneill return (0);
5163 1.1 jmcneill }
5164 1.1 jmcneill
5165 1.1 jmcneill static int
5166 1.15 jmcneill mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db,
5167 1.15 jmcneill int cqn)
5168 1.1 jmcneill {
5169 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5170 1.1 jmcneill struct mcx_dmamem mxm;
5171 1.1 jmcneill struct mcx_cmd_create_sq_in *in;
5172 1.1 jmcneill struct mcx_sq_ctx *mbin;
5173 1.1 jmcneill struct mcx_cmd_create_sq_out *out;
5174 1.1 jmcneill int error;
5175 1.1 jmcneill uint64_t *pas;
5176 1.13 jmcneill int insize, npages, paslen, token;
5177 1.1 jmcneill
5178 1.15 jmcneill tx->tx_doorbell = MCX_WQ_DOORBELL_BASE +
5179 1.15 jmcneill (db * MCX_WQ_DOORBELL_STRIDE) + 4;
5180 1.15 jmcneill
5181 1.1 jmcneill npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
5182 1.1 jmcneill MCX_PAGE_SIZE);
5183 1.1 jmcneill paslen = npages * sizeof(*pas);
5184 1.1 jmcneill insize = sizeof(struct mcx_sq_ctx) + paslen;
5185 1.1 jmcneill
5186 1.15 jmcneill if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE,
5187 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
5188 1.15 jmcneill printf("%s: unable to allocate send queue memory\n",
5189 1.15 jmcneill DEVNAME(sc));
5190 1.1 jmcneill return (-1);
5191 1.1 jmcneill }
5192 1.1 jmcneill
5193 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5194 1.1 jmcneill token = mcx_cmdq_token(sc);
5195 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
5196 1.1 jmcneill token);
5197 1.1 jmcneill
5198 1.1 jmcneill in = mcx_cmdq_in(cqe);
5199 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
5200 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5201 1.1 jmcneill
5202 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
5203 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
5204 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
5205 1.15 jmcneill printf("%s: unable to allocate create sq mailboxen\n",
5206 1.15 jmcneill DEVNAME(sc));
5207 1.15 jmcneill goto free_sq;
5208 1.1 jmcneill }
5209 1.15 jmcneill mbin = (struct mcx_sq_ctx *)
5210 1.15 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
5211 1.1 jmcneill mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
5212 1.1 jmcneill (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
5213 1.1 jmcneill mbin->sq_cqn = htobe32(cqn);
5214 1.1 jmcneill mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
5215 1.15 jmcneill mbin->sq_tis_num = htobe32(sc->sc_tis);
5216 1.1 jmcneill mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
5217 1.1 jmcneill mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
5218 1.15 jmcneill mbin->sq_wq.wq_uar_page = htobe32(uar);
5219 1.1 jmcneill mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
5220 1.15 jmcneill tx->tx_doorbell);
5221 1.1 jmcneill mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
5222 1.1 jmcneill mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
5223 1.1 jmcneill
5224 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5225 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
5226 1.15 jmcneill
5227 1.1 jmcneill /* physical addresses follow the mailbox in data */
5228 1.15 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10,
5229 1.15 jmcneill npages, &tx->tx_sq_mem);
5230 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5231 1.1 jmcneill
5232 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5233 1.1 jmcneill if (error != 0) {
5234 1.1 jmcneill printf("%s: create sq timeout\n", DEVNAME(sc));
5235 1.15 jmcneill goto free_mxm;
5236 1.1 jmcneill }
5237 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5238 1.1 jmcneill printf("%s: create sq command corrupt\n", DEVNAME(sc));
5239 1.15 jmcneill goto free_mxm;
5240 1.1 jmcneill }
5241 1.1 jmcneill
5242 1.1 jmcneill out = mcx_cmdq_out(cqe);
5243 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5244 1.1 jmcneill printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
5245 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5246 1.15 jmcneill goto free_mxm;
5247 1.1 jmcneill }
5248 1.1 jmcneill
5249 1.15 jmcneill tx->tx_uar = uar;
5250 1.15 jmcneill tx->tx_sqn = mcx_get_id(out->cmd_sqn);
5251 1.15 jmcneill
5252 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
5253 1.15 jmcneill
5254 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
5255 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
5256 1.15 jmcneill
5257 1.15 jmcneill return (0);
5258 1.1 jmcneill
5259 1.15 jmcneill free_mxm:
5260 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5261 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
5262 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5263 1.15 jmcneill free_sq:
5264 1.15 jmcneill mcx_dmamem_free(sc, &tx->tx_sq_mem);
5265 1.15 jmcneill return (-1);
5266 1.1 jmcneill }
5267 1.1 jmcneill
5268 1.1 jmcneill static int
5269 1.15 jmcneill mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx)
5270 1.1 jmcneill {
5271 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5272 1.1 jmcneill struct mcx_cmd_destroy_sq_in *in;
5273 1.1 jmcneill struct mcx_cmd_destroy_sq_out *out;
5274 1.1 jmcneill int error;
5275 1.1 jmcneill int token;
5276 1.1 jmcneill
5277 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5278 1.1 jmcneill token = mcx_cmdq_token(sc);
5279 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5280 1.1 jmcneill
5281 1.1 jmcneill in = mcx_cmdq_in(cqe);
5282 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
5283 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5284 1.15 jmcneill in->cmd_sqn = htobe32(tx->tx_sqn);
5285 1.1 jmcneill
5286 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5287 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5288 1.1 jmcneill if (error != 0) {
5289 1.1 jmcneill printf("%s: destroy sq timeout\n", DEVNAME(sc));
5290 1.1 jmcneill return error;
5291 1.1 jmcneill }
5292 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5293 1.1 jmcneill printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
5294 1.1 jmcneill return error;
5295 1.1 jmcneill }
5296 1.1 jmcneill
5297 1.1 jmcneill out = mcx_cmdq_out(cqe);
5298 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5299 1.1 jmcneill printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
5300 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5301 1.1 jmcneill return -1;
5302 1.1 jmcneill }
5303 1.1 jmcneill
5304 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
5305 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
5306 1.15 jmcneill
5307 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5308 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
5309 1.15 jmcneill mcx_dmamem_free(sc, &tx->tx_sq_mem);
5310 1.15 jmcneill
5311 1.15 jmcneill tx->tx_sqn = 0;
5312 1.1 jmcneill return 0;
5313 1.1 jmcneill }
5314 1.1 jmcneill
5315 1.1 jmcneill static int
5316 1.15 jmcneill mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx)
5317 1.1 jmcneill {
5318 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5319 1.1 jmcneill struct mcx_dmamem mxm;
5320 1.1 jmcneill struct mcx_cmd_modify_sq_in *in;
5321 1.1 jmcneill struct mcx_cmd_modify_sq_mb_in *mbin;
5322 1.1 jmcneill struct mcx_cmd_modify_sq_out *out;
5323 1.1 jmcneill int error;
5324 1.1 jmcneill int token;
5325 1.1 jmcneill
5326 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5327 1.1 jmcneill token = mcx_cmdq_token(sc);
5328 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5329 1.15 jmcneill sizeof(*out), token);
5330 1.1 jmcneill
5331 1.1 jmcneill in = mcx_cmdq_in(cqe);
5332 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
5333 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5334 1.15 jmcneill in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn);
5335 1.1 jmcneill
5336 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5337 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5338 1.1 jmcneill printf("%s: unable to allocate modify sq mailbox\n",
5339 1.1 jmcneill DEVNAME(sc));
5340 1.1 jmcneill return (-1);
5341 1.1 jmcneill }
5342 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5343 1.1 jmcneill mbin->cmd_sq_ctx.sq_flags = htobe32(
5344 1.1 jmcneill MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
5345 1.1 jmcneill
5346 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5347 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5348 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5349 1.1 jmcneill if (error != 0) {
5350 1.1 jmcneill printf("%s: modify sq timeout\n", DEVNAME(sc));
5351 1.1 jmcneill goto free;
5352 1.1 jmcneill }
5353 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5354 1.1 jmcneill printf("%s: modify sq command corrupt\n", DEVNAME(sc));
5355 1.1 jmcneill goto free;
5356 1.1 jmcneill }
5357 1.1 jmcneill
5358 1.1 jmcneill out = mcx_cmdq_out(cqe);
5359 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5360 1.1 jmcneill printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
5361 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5362 1.1 jmcneill error = -1;
5363 1.1 jmcneill goto free;
5364 1.1 jmcneill }
5365 1.1 jmcneill
5366 1.1 jmcneill free:
5367 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5368 1.1 jmcneill return (error);
5369 1.1 jmcneill }
5370 1.1 jmcneill
5371 1.1 jmcneill static int
5372 1.15 jmcneill mcx_create_tis(struct mcx_softc *sc, int *tis)
5373 1.1 jmcneill {
5374 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5375 1.1 jmcneill struct mcx_dmamem mxm;
5376 1.1 jmcneill struct mcx_cmd_create_tis_in *in;
5377 1.1 jmcneill struct mcx_cmd_create_tis_mb_in *mbin;
5378 1.1 jmcneill struct mcx_cmd_create_tis_out *out;
5379 1.1 jmcneill int error;
5380 1.1 jmcneill int token;
5381 1.1 jmcneill
5382 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5383 1.1 jmcneill token = mcx_cmdq_token(sc);
5384 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5385 1.15 jmcneill sizeof(*out), token);
5386 1.1 jmcneill
5387 1.1 jmcneill in = mcx_cmdq_in(cqe);
5388 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
5389 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5390 1.1 jmcneill
5391 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5392 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5393 1.15 jmcneill printf("%s: unable to allocate create tis mailbox\n",
5394 1.15 jmcneill DEVNAME(sc));
5395 1.1 jmcneill return (-1);
5396 1.1 jmcneill }
5397 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5398 1.1 jmcneill mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
5399 1.1 jmcneill
5400 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5401 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5402 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5403 1.1 jmcneill if (error != 0) {
5404 1.1 jmcneill printf("%s: create tis timeout\n", DEVNAME(sc));
5405 1.1 jmcneill goto free;
5406 1.1 jmcneill }
5407 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5408 1.1 jmcneill printf("%s: create tis command corrupt\n", DEVNAME(sc));
5409 1.1 jmcneill goto free;
5410 1.1 jmcneill }
5411 1.1 jmcneill
5412 1.1 jmcneill out = mcx_cmdq_out(cqe);
5413 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5414 1.1 jmcneill printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
5415 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5416 1.1 jmcneill error = -1;
5417 1.1 jmcneill goto free;
5418 1.1 jmcneill }
5419 1.1 jmcneill
5420 1.15 jmcneill *tis = mcx_get_id(out->cmd_tisn);
5421 1.1 jmcneill free:
5422 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5423 1.1 jmcneill return (error);
5424 1.1 jmcneill }
5425 1.1 jmcneill
5426 1.1 jmcneill static int
5427 1.15 jmcneill mcx_destroy_tis(struct mcx_softc *sc, int tis)
5428 1.1 jmcneill {
5429 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5430 1.1 jmcneill struct mcx_cmd_destroy_tis_in *in;
5431 1.1 jmcneill struct mcx_cmd_destroy_tis_out *out;
5432 1.1 jmcneill int error;
5433 1.1 jmcneill int token;
5434 1.1 jmcneill
5435 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5436 1.1 jmcneill token = mcx_cmdq_token(sc);
5437 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5438 1.1 jmcneill
5439 1.1 jmcneill in = mcx_cmdq_in(cqe);
5440 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
5441 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5442 1.15 jmcneill in->cmd_tisn = htobe32(tis);
5443 1.1 jmcneill
5444 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5445 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5446 1.1 jmcneill if (error != 0) {
5447 1.1 jmcneill printf("%s: destroy tis timeout\n", DEVNAME(sc));
5448 1.1 jmcneill return error;
5449 1.1 jmcneill }
5450 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5451 1.1 jmcneill printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
5452 1.1 jmcneill return error;
5453 1.1 jmcneill }
5454 1.1 jmcneill
5455 1.1 jmcneill out = mcx_cmdq_out(cqe);
5456 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5457 1.1 jmcneill printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
5458 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5459 1.1 jmcneill return -1;
5460 1.1 jmcneill }
5461 1.1 jmcneill
5462 1.15 jmcneill return 0;
5463 1.15 jmcneill }
5464 1.15 jmcneill
5465 1.15 jmcneill static int
5466 1.15 jmcneill mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt)
5467 1.15 jmcneill {
5468 1.15 jmcneill struct mcx_cmdq_entry *cqe;
5469 1.15 jmcneill struct mcx_dmamem mxm;
5470 1.15 jmcneill struct mcx_cmd_create_rqt_in *in;
5471 1.15 jmcneill struct mcx_cmd_create_rqt_mb_in *mbin;
5472 1.15 jmcneill struct mcx_cmd_create_rqt_out *out;
5473 1.15 jmcneill struct mcx_rqt_ctx *rqt_ctx;
5474 1.15 jmcneill int *rqtn;
5475 1.15 jmcneill int error;
5476 1.15 jmcneill int token;
5477 1.15 jmcneill int i;
5478 1.15 jmcneill
5479 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5480 1.15 jmcneill token = mcx_cmdq_token(sc);
5481 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) +
5482 1.15 jmcneill (size * sizeof(int)), sizeof(*out), token);
5483 1.15 jmcneill
5484 1.15 jmcneill in = mcx_cmdq_in(cqe);
5485 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT);
5486 1.15 jmcneill in->cmd_op_mod = htobe16(0);
5487 1.15 jmcneill
5488 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5489 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5490 1.15 jmcneill printf("%s: unable to allocate create rqt mailbox\n",
5491 1.15 jmcneill DEVNAME(sc));
5492 1.15 jmcneill return (-1);
5493 1.15 jmcneill }
5494 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5495 1.15 jmcneill rqt_ctx = &mbin->cmd_rqt;
5496 1.15 jmcneill rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size);
5497 1.15 jmcneill rqt_ctx->cmd_rqt_actual_size = htobe16(size);
5498 1.15 jmcneill
5499 1.15 jmcneill /* rqt list follows the rqt context */
5500 1.15 jmcneill rqtn = (int *)(rqt_ctx + 1);
5501 1.15 jmcneill for (i = 0; i < size; i++) {
5502 1.15 jmcneill rqtn[i] = htobe32(rqns[i]);
5503 1.15 jmcneill }
5504 1.15 jmcneill
5505 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5506 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
5507 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5508 1.15 jmcneill if (error != 0) {
5509 1.15 jmcneill printf("%s: create rqt timeout\n", DEVNAME(sc));
5510 1.15 jmcneill goto free;
5511 1.15 jmcneill }
5512 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5513 1.15 jmcneill printf("%s: create rqt command corrupt\n", DEVNAME(sc));
5514 1.15 jmcneill goto free;
5515 1.15 jmcneill }
5516 1.15 jmcneill
5517 1.15 jmcneill out = mcx_cmdq_out(cqe);
5518 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5519 1.15 jmcneill printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc),
5520 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5521 1.15 jmcneill error = -1;
5522 1.15 jmcneill goto free;
5523 1.15 jmcneill }
5524 1.15 jmcneill
5525 1.15 jmcneill *rqt = mcx_get_id(out->cmd_rqtn);
5526 1.15 jmcneill return (0);
5527 1.15 jmcneill free:
5528 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
5529 1.15 jmcneill return (error);
5530 1.15 jmcneill }
5531 1.15 jmcneill
5532 1.15 jmcneill static int
5533 1.15 jmcneill mcx_destroy_rqt(struct mcx_softc *sc, int rqt)
5534 1.15 jmcneill {
5535 1.15 jmcneill struct mcx_cmdq_entry *cqe;
5536 1.15 jmcneill struct mcx_cmd_destroy_rqt_in *in;
5537 1.15 jmcneill struct mcx_cmd_destroy_rqt_out *out;
5538 1.15 jmcneill int error;
5539 1.15 jmcneill int token;
5540 1.15 jmcneill
5541 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5542 1.15 jmcneill token = mcx_cmdq_token(sc);
5543 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5544 1.15 jmcneill
5545 1.15 jmcneill in = mcx_cmdq_in(cqe);
5546 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT);
5547 1.15 jmcneill in->cmd_op_mod = htobe16(0);
5548 1.15 jmcneill in->cmd_rqtn = htobe32(rqt);
5549 1.15 jmcneill
5550 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
5551 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5552 1.15 jmcneill if (error != 0) {
5553 1.15 jmcneill printf("%s: destroy rqt timeout\n", DEVNAME(sc));
5554 1.15 jmcneill return error;
5555 1.15 jmcneill }
5556 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5557 1.15 jmcneill printf("%s: destroy rqt command corrupt\n", DEVNAME(sc));
5558 1.15 jmcneill return error;
5559 1.15 jmcneill }
5560 1.15 jmcneill
5561 1.15 jmcneill out = mcx_cmdq_out(cqe);
5562 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5563 1.15 jmcneill printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc),
5564 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5565 1.15 jmcneill return -1;
5566 1.15 jmcneill }
5567 1.15 jmcneill
5568 1.1 jmcneill return 0;
5569 1.1 jmcneill }
5570 1.1 jmcneill
5571 1.1 jmcneill #if 0
5572 1.1 jmcneill static int
5573 1.1 jmcneill mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
5574 1.1 jmcneill {
5575 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5576 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_in *in;
5577 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_out *out;
5578 1.1 jmcneill int error;
5579 1.1 jmcneill
5580 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5581 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
5582 1.1 jmcneill
5583 1.1 jmcneill in = mcx_cmdq_in(cqe);
5584 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
5585 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5586 1.1 jmcneill
5587 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5588 1.1 jmcneill
5589 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5590 1.1 jmcneill if (error != 0) {
5591 1.1 jmcneill printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
5592 1.1 jmcneill return (-1);
5593 1.1 jmcneill }
5594 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5595 1.1 jmcneill printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
5596 1.1 jmcneill return (-1);
5597 1.1 jmcneill }
5598 1.1 jmcneill
5599 1.1 jmcneill out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
5600 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5601 1.1 jmcneill printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
5602 1.1 jmcneill out->cmd_status);
5603 1.1 jmcneill return (-1);
5604 1.1 jmcneill }
5605 1.1 jmcneill
5606 1.1 jmcneill sc->sc_flow_counter_id[i] = be16toh(out->cmd_flow_counter_id);
5607 1.1 jmcneill printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
5608 1.1 jmcneill
5609 1.1 jmcneill return (0);
5610 1.1 jmcneill }
5611 1.1 jmcneill #endif
5612 1.1 jmcneill
5613 1.1 jmcneill static int
5614 1.15 jmcneill mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level,
5615 1.15 jmcneill int *flow_table_id)
5616 1.1 jmcneill {
5617 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5618 1.1 jmcneill struct mcx_dmamem mxm;
5619 1.1 jmcneill struct mcx_cmd_create_flow_table_in *in;
5620 1.1 jmcneill struct mcx_cmd_create_flow_table_mb_in *mbin;
5621 1.1 jmcneill struct mcx_cmd_create_flow_table_out *out;
5622 1.1 jmcneill int error;
5623 1.1 jmcneill int token;
5624 1.1 jmcneill
5625 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5626 1.1 jmcneill token = mcx_cmdq_token(sc);
5627 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5628 1.15 jmcneill sizeof(*out), token);
5629 1.1 jmcneill
5630 1.1 jmcneill in = mcx_cmdq_in(cqe);
5631 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
5632 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5633 1.1 jmcneill
5634 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5635 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5636 1.1 jmcneill printf("%s: unable to allocate create flow table mailbox\n",
5637 1.1 jmcneill DEVNAME(sc));
5638 1.1 jmcneill return (-1);
5639 1.1 jmcneill }
5640 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5641 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5642 1.1 jmcneill mbin->cmd_ctx.ft_log_size = log_size;
5643 1.15 jmcneill mbin->cmd_ctx.ft_level = level;
5644 1.1 jmcneill
5645 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5646 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5647 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5648 1.1 jmcneill if (error != 0) {
5649 1.1 jmcneill printf("%s: create flow table timeout\n", DEVNAME(sc));
5650 1.1 jmcneill goto free;
5651 1.1 jmcneill }
5652 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5653 1.1 jmcneill printf("%s: create flow table command corrupt\n", DEVNAME(sc));
5654 1.1 jmcneill goto free;
5655 1.1 jmcneill }
5656 1.1 jmcneill
5657 1.1 jmcneill out = mcx_cmdq_out(cqe);
5658 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5659 1.1 jmcneill printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
5660 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5661 1.1 jmcneill error = -1;
5662 1.1 jmcneill goto free;
5663 1.1 jmcneill }
5664 1.1 jmcneill
5665 1.15 jmcneill *flow_table_id = mcx_get_id(out->cmd_table_id);
5666 1.1 jmcneill free:
5667 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5668 1.1 jmcneill return (error);
5669 1.1 jmcneill }
5670 1.1 jmcneill
5671 1.1 jmcneill static int
5672 1.15 jmcneill mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id)
5673 1.1 jmcneill {
5674 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5675 1.1 jmcneill struct mcx_dmamem mxm;
5676 1.1 jmcneill struct mcx_cmd_set_flow_table_root_in *in;
5677 1.1 jmcneill struct mcx_cmd_set_flow_table_root_mb_in *mbin;
5678 1.1 jmcneill struct mcx_cmd_set_flow_table_root_out *out;
5679 1.1 jmcneill int error;
5680 1.1 jmcneill int token;
5681 1.1 jmcneill
5682 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5683 1.1 jmcneill token = mcx_cmdq_token(sc);
5684 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5685 1.15 jmcneill sizeof(*out), token);
5686 1.1 jmcneill
5687 1.1 jmcneill in = mcx_cmdq_in(cqe);
5688 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
5689 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5690 1.1 jmcneill
5691 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5692 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5693 1.1 jmcneill printf("%s: unable to allocate set flow table root mailbox\n",
5694 1.1 jmcneill DEVNAME(sc));
5695 1.1 jmcneill return (-1);
5696 1.1 jmcneill }
5697 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5698 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5699 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
5700 1.1 jmcneill
5701 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5702 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5703 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5704 1.1 jmcneill if (error != 0) {
5705 1.1 jmcneill printf("%s: set flow table root timeout\n", DEVNAME(sc));
5706 1.1 jmcneill goto free;
5707 1.1 jmcneill }
5708 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5709 1.1 jmcneill printf("%s: set flow table root command corrupt\n",
5710 1.1 jmcneill DEVNAME(sc));
5711 1.1 jmcneill goto free;
5712 1.1 jmcneill }
5713 1.1 jmcneill
5714 1.1 jmcneill out = mcx_cmdq_out(cqe);
5715 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5716 1.1 jmcneill printf("%s: set flow table root failed (%x, %x)\n",
5717 1.1 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
5718 1.1 jmcneill error = -1;
5719 1.1 jmcneill goto free;
5720 1.1 jmcneill }
5721 1.1 jmcneill
5722 1.1 jmcneill free:
5723 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5724 1.1 jmcneill return (error);
5725 1.1 jmcneill }
5726 1.1 jmcneill
5727 1.1 jmcneill static int
5728 1.15 jmcneill mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id)
5729 1.1 jmcneill {
5730 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5731 1.1 jmcneill struct mcx_dmamem mxm;
5732 1.1 jmcneill struct mcx_cmd_destroy_flow_table_in *in;
5733 1.1 jmcneill struct mcx_cmd_destroy_flow_table_mb_in *mb;
5734 1.1 jmcneill struct mcx_cmd_destroy_flow_table_out *out;
5735 1.1 jmcneill int error;
5736 1.1 jmcneill int token;
5737 1.1 jmcneill
5738 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5739 1.1 jmcneill token = mcx_cmdq_token(sc);
5740 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
5741 1.1 jmcneill
5742 1.1 jmcneill in = mcx_cmdq_in(cqe);
5743 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
5744 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5745 1.1 jmcneill
5746 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5747 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5748 1.1 jmcneill printf("%s: unable to allocate destroy flow table mailbox\n",
5749 1.1 jmcneill DEVNAME(sc));
5750 1.1 jmcneill return (-1);
5751 1.1 jmcneill }
5752 1.1 jmcneill mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5753 1.1 jmcneill mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5754 1.15 jmcneill mb->cmd_table_id = htobe32(flow_table_id);
5755 1.1 jmcneill
5756 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5757 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5758 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5759 1.1 jmcneill if (error != 0) {
5760 1.1 jmcneill printf("%s: destroy flow table timeout\n", DEVNAME(sc));
5761 1.1 jmcneill goto free;
5762 1.1 jmcneill }
5763 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5764 1.15 jmcneill printf("%s: destroy flow table command corrupt\n",
5765 1.15 jmcneill DEVNAME(sc));
5766 1.1 jmcneill goto free;
5767 1.1 jmcneill }
5768 1.1 jmcneill
5769 1.1 jmcneill out = mcx_cmdq_out(cqe);
5770 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5771 1.1 jmcneill printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
5772 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5773 1.1 jmcneill error = -1;
5774 1.1 jmcneill goto free;
5775 1.1 jmcneill }
5776 1.1 jmcneill
5777 1.1 jmcneill free:
5778 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5779 1.1 jmcneill return (error);
5780 1.1 jmcneill }
5781 1.1 jmcneill
5782 1.1 jmcneill
5783 1.1 jmcneill static int
5784 1.15 jmcneill mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group,
5785 1.15 jmcneill int start, int size, int match_enable, struct mcx_flow_match *match)
5786 1.1 jmcneill {
5787 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5788 1.1 jmcneill struct mcx_dmamem mxm;
5789 1.1 jmcneill struct mcx_cmd_create_flow_group_in *in;
5790 1.1 jmcneill struct mcx_cmd_create_flow_group_mb_in *mbin;
5791 1.1 jmcneill struct mcx_cmd_create_flow_group_out *out;
5792 1.15 jmcneill struct mcx_flow_group *mfg;
5793 1.1 jmcneill int error;
5794 1.1 jmcneill int token;
5795 1.1 jmcneill
5796 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5797 1.1 jmcneill token = mcx_cmdq_token(sc);
5798 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
5799 1.1 jmcneill token);
5800 1.1 jmcneill
5801 1.1 jmcneill in = mcx_cmdq_in(cqe);
5802 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
5803 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5804 1.1 jmcneill
5805 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5806 1.1 jmcneill != 0) {
5807 1.1 jmcneill printf("%s: unable to allocate create flow group mailbox\n",
5808 1.1 jmcneill DEVNAME(sc));
5809 1.1 jmcneill return (-1);
5810 1.1 jmcneill }
5811 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5812 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5813 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
5814 1.1 jmcneill mbin->cmd_start_flow_index = htobe32(start);
5815 1.1 jmcneill mbin->cmd_end_flow_index = htobe32(start + (size - 1));
5816 1.1 jmcneill
5817 1.1 jmcneill mbin->cmd_match_criteria_enable = match_enable;
5818 1.1 jmcneill memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));
5819 1.1 jmcneill
5820 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
5821 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5822 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5823 1.1 jmcneill if (error != 0) {
5824 1.1 jmcneill printf("%s: create flow group timeout\n", DEVNAME(sc));
5825 1.1 jmcneill goto free;
5826 1.1 jmcneill }
5827 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5828 1.1 jmcneill printf("%s: create flow group command corrupt\n", DEVNAME(sc));
5829 1.1 jmcneill goto free;
5830 1.1 jmcneill }
5831 1.1 jmcneill
5832 1.1 jmcneill out = mcx_cmdq_out(cqe);
5833 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5834 1.1 jmcneill printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
5835 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5836 1.1 jmcneill error = -1;
5837 1.1 jmcneill goto free;
5838 1.1 jmcneill }
5839 1.1 jmcneill
5840 1.15 jmcneill mfg = &sc->sc_flow_group[group];
5841 1.15 jmcneill mfg->g_id = mcx_get_id(out->cmd_group_id);
5842 1.15 jmcneill mfg->g_table = flow_table_id;
5843 1.15 jmcneill mfg->g_start = start;
5844 1.15 jmcneill mfg->g_size = size;
5845 1.1 jmcneill
5846 1.1 jmcneill free:
5847 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5848 1.1 jmcneill return (error);
5849 1.1 jmcneill }
5850 1.1 jmcneill
5851 1.1 jmcneill static int
5852 1.1 jmcneill mcx_destroy_flow_group(struct mcx_softc *sc, int group)
5853 1.1 jmcneill {
5854 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5855 1.1 jmcneill struct mcx_dmamem mxm;
5856 1.1 jmcneill struct mcx_cmd_destroy_flow_group_in *in;
5857 1.1 jmcneill struct mcx_cmd_destroy_flow_group_mb_in *mb;
5858 1.1 jmcneill struct mcx_cmd_destroy_flow_group_out *out;
5859 1.15 jmcneill struct mcx_flow_group *mfg;
5860 1.1 jmcneill int error;
5861 1.1 jmcneill int token;
5862 1.1 jmcneill
5863 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5864 1.1 jmcneill token = mcx_cmdq_token(sc);
5865 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
5866 1.1 jmcneill
5867 1.1 jmcneill in = mcx_cmdq_in(cqe);
5868 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
5869 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5870 1.1 jmcneill
5871 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
5872 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5873 1.1 jmcneill printf("%s: unable to allocate destroy flow group mailbox\n",
5874 1.1 jmcneill DEVNAME(sc));
5875 1.1 jmcneill return (-1);
5876 1.1 jmcneill }
5877 1.1 jmcneill mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5878 1.1 jmcneill mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5879 1.15 jmcneill mfg = &sc->sc_flow_group[group];
5880 1.15 jmcneill mb->cmd_table_id = htobe32(mfg->g_table);
5881 1.15 jmcneill mb->cmd_group_id = htobe32(mfg->g_id);
5882 1.1 jmcneill
5883 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
5884 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5885 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5886 1.1 jmcneill if (error != 0) {
5887 1.1 jmcneill printf("%s: destroy flow group timeout\n", DEVNAME(sc));
5888 1.1 jmcneill goto free;
5889 1.1 jmcneill }
5890 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5891 1.1 jmcneill printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
5892 1.1 jmcneill goto free;
5893 1.1 jmcneill }
5894 1.1 jmcneill
5895 1.1 jmcneill out = mcx_cmdq_out(cqe);
5896 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5897 1.1 jmcneill printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
5898 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5899 1.1 jmcneill error = -1;
5900 1.1 jmcneill goto free;
5901 1.1 jmcneill }
5902 1.1 jmcneill
5903 1.15 jmcneill mfg->g_id = -1;
5904 1.15 jmcneill mfg->g_table = -1;
5905 1.15 jmcneill mfg->g_size = 0;
5906 1.15 jmcneill mfg->g_start = 0;
5907 1.1 jmcneill free:
5908 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5909 1.1 jmcneill return (error);
5910 1.1 jmcneill }
5911 1.1 jmcneill
5912 1.1 jmcneill static int
5913 1.15 jmcneill mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index,
5914 1.15 jmcneill const uint8_t *macaddr, uint32_t dest)
5915 1.1 jmcneill {
5916 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5917 1.1 jmcneill struct mcx_dmamem mxm;
5918 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_in *in;
5919 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
5920 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_out *out;
5921 1.15 jmcneill struct mcx_flow_group *mfg;
5922 1.15 jmcneill uint32_t *pdest;
5923 1.1 jmcneill int error;
5924 1.1 jmcneill int token;
5925 1.1 jmcneill
5926 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5927 1.1 jmcneill token = mcx_cmdq_token(sc);
5928 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
5929 1.1 jmcneill sizeof(*out), token);
5930 1.1 jmcneill
5931 1.1 jmcneill in = mcx_cmdq_in(cqe);
5932 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
5933 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5934 1.1 jmcneill
5935 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5936 1.1 jmcneill != 0) {
5937 1.1 jmcneill printf("%s: unable to allocate set flow table entry mailbox\n",
5938 1.1 jmcneill DEVNAME(sc));
5939 1.1 jmcneill return (-1);
5940 1.1 jmcneill }
5941 1.15 jmcneill
5942 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5943 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5944 1.15 jmcneill
5945 1.15 jmcneill mfg = &sc->sc_flow_group[group];
5946 1.15 jmcneill mbin->cmd_table_id = htobe32(mfg->g_table);
5947 1.15 jmcneill mbin->cmd_flow_index = htobe32(mfg->g_start + index);
5948 1.15 jmcneill mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
5949 1.1 jmcneill
5950 1.1 jmcneill /* flow context ends at offset 0x330, 0x130 into the second mbox */
5951 1.15 jmcneill pdest = (uint32_t *)
5952 1.1 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
5953 1.1 jmcneill mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
5954 1.1 jmcneill mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
5955 1.15 jmcneill *pdest = htobe32(dest);
5956 1.1 jmcneill
5957 1.1 jmcneill /* the only thing we match on at the moment is the dest mac address */
5958 1.1 jmcneill if (macaddr != NULL) {
5959 1.1 jmcneill memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
5960 1.1 jmcneill ETHER_ADDR_LEN);
5961 1.1 jmcneill }
5962 1.1 jmcneill
5963 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
5964 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5965 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5966 1.1 jmcneill if (error != 0) {
5967 1.1 jmcneill printf("%s: set flow table entry timeout\n", DEVNAME(sc));
5968 1.1 jmcneill goto free;
5969 1.1 jmcneill }
5970 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5971 1.1 jmcneill printf("%s: set flow table entry command corrupt\n",
5972 1.1 jmcneill DEVNAME(sc));
5973 1.1 jmcneill goto free;
5974 1.1 jmcneill }
5975 1.1 jmcneill
5976 1.1 jmcneill out = mcx_cmdq_out(cqe);
5977 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5978 1.1 jmcneill printf("%s: set flow table entry failed (%x, %x)\n",
5979 1.1 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
5980 1.1 jmcneill error = -1;
5981 1.1 jmcneill goto free;
5982 1.1 jmcneill }
5983 1.1 jmcneill
5984 1.1 jmcneill free:
5985 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5986 1.1 jmcneill return (error);
5987 1.1 jmcneill }
5988 1.1 jmcneill
5989 1.1 jmcneill static int
5990 1.15 jmcneill mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index,
5991 1.15 jmcneill int ethertype, int ip_proto, uint32_t dest)
5992 1.1 jmcneill {
5993 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5994 1.1 jmcneill struct mcx_dmamem mxm;
5995 1.15 jmcneill struct mcx_cmd_set_flow_table_entry_in *in;
5996 1.15 jmcneill struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
5997 1.15 jmcneill struct mcx_cmd_set_flow_table_entry_out *out;
5998 1.15 jmcneill struct mcx_flow_group *mfg;
5999 1.15 jmcneill uint32_t *pdest;
6000 1.1 jmcneill int error;
6001 1.1 jmcneill int token;
6002 1.1 jmcneill
6003 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6004 1.1 jmcneill token = mcx_cmdq_token(sc);
6005 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
6006 1.15 jmcneill sizeof(*out), token);
6007 1.1 jmcneill
6008 1.1 jmcneill in = mcx_cmdq_in(cqe);
6009 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
6010 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6011 1.1 jmcneill
6012 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
6013 1.15 jmcneill != 0) {
6014 1.15 jmcneill printf("%s: unable to allocate set flow table entry mailbox\n",
6015 1.1 jmcneill DEVNAME(sc));
6016 1.1 jmcneill return (-1);
6017 1.1 jmcneill }
6018 1.15 jmcneill
6019 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6020 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
6021 1.15 jmcneill
6022 1.15 jmcneill mfg = &sc->sc_flow_group[group];
6023 1.15 jmcneill mbin->cmd_table_id = htobe32(mfg->g_table);
6024 1.15 jmcneill mbin->cmd_flow_index = htobe32(mfg->g_start + index);
6025 1.15 jmcneill mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
6026 1.15 jmcneill
6027 1.15 jmcneill /* flow context ends at offset 0x330, 0x130 into the second mbox */
6028 1.15 jmcneill pdest = (uint32_t *)
6029 1.15 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
6030 1.15 jmcneill mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
6031 1.15 jmcneill mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
6032 1.15 jmcneill *pdest = htobe32(dest);
6033 1.15 jmcneill
6034 1.15 jmcneill mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype);
6035 1.15 jmcneill mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto;
6036 1.1 jmcneill
6037 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
6038 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6039 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6040 1.1 jmcneill if (error != 0) {
6041 1.15 jmcneill printf("%s: set flow table entry timeout\n", DEVNAME(sc));
6042 1.1 jmcneill goto free;
6043 1.1 jmcneill }
6044 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6045 1.15 jmcneill printf("%s: set flow table entry command corrupt\n",
6046 1.1 jmcneill DEVNAME(sc));
6047 1.1 jmcneill goto free;
6048 1.1 jmcneill }
6049 1.1 jmcneill
6050 1.1 jmcneill out = mcx_cmdq_out(cqe);
6051 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6052 1.15 jmcneill printf("%s: set flow table entry failed (%x, %x)\n",
6053 1.15 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
6054 1.1 jmcneill error = -1;
6055 1.1 jmcneill goto free;
6056 1.1 jmcneill }
6057 1.1 jmcneill
6058 1.1 jmcneill free:
6059 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
6060 1.1 jmcneill return (error);
6061 1.1 jmcneill }
6062 1.1 jmcneill
6063 1.15 jmcneill static int
6064 1.15 jmcneill mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
6065 1.1 jmcneill {
6066 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6067 1.1 jmcneill struct mcx_dmamem mxm;
6068 1.15 jmcneill struct mcx_cmd_delete_flow_table_entry_in *in;
6069 1.15 jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
6070 1.15 jmcneill struct mcx_cmd_delete_flow_table_entry_out *out;
6071 1.15 jmcneill struct mcx_flow_group *mfg;
6072 1.1 jmcneill int error;
6073 1.15 jmcneill int token;
6074 1.15 jmcneill
6075 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6076 1.15 jmcneill token = mcx_cmdq_token(sc);
6077 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
6078 1.15 jmcneill token);
6079 1.15 jmcneill
6080 1.15 jmcneill in = mcx_cmdq_in(cqe);
6081 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
6082 1.15 jmcneill in->cmd_op_mod = htobe16(0);
6083 1.15 jmcneill
6084 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6085 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
6086 1.15 jmcneill printf("%s: unable to allocate "
6087 1.15 jmcneill "delete flow table entry mailbox\n", DEVNAME(sc));
6088 1.15 jmcneill return (-1);
6089 1.15 jmcneill }
6090 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6091 1.15 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
6092 1.15 jmcneill
6093 1.15 jmcneill mfg = &sc->sc_flow_group[group];
6094 1.15 jmcneill mbin->cmd_table_id = htobe32(mfg->g_table);
6095 1.15 jmcneill mbin->cmd_flow_index = htobe32(mfg->g_start + index);
6096 1.15 jmcneill
6097 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
6098 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
6099 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6100 1.15 jmcneill if (error != 0) {
6101 1.15 jmcneill printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
6102 1.15 jmcneill goto free;
6103 1.15 jmcneill }
6104 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6105 1.15 jmcneill printf("%s: delete flow table entry command corrupt\n",
6106 1.15 jmcneill DEVNAME(sc));
6107 1.15 jmcneill goto free;
6108 1.15 jmcneill }
6109 1.15 jmcneill
6110 1.15 jmcneill out = mcx_cmdq_out(cqe);
6111 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6112 1.15 jmcneill printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
6113 1.15 jmcneill DEVNAME(sc), group, index, out->cmd_status,
6114 1.15 jmcneill be32toh(out->cmd_syndrome));
6115 1.15 jmcneill error = -1;
6116 1.15 jmcneill goto free;
6117 1.15 jmcneill }
6118 1.15 jmcneill
6119 1.15 jmcneill free:
6120 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
6121 1.15 jmcneill return (error);
6122 1.15 jmcneill }
6123 1.15 jmcneill
6124 1.15 jmcneill #if 0
6125 1.15 jmcneill int
6126 1.15 jmcneill mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id)
6127 1.15 jmcneill {
6128 1.15 jmcneill struct mcx_dmamem mxm;
6129 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6130 1.15 jmcneill struct mcx_cmd_query_flow_table_in *in;
6131 1.15 jmcneill struct mcx_cmd_query_flow_table_mb_in *mbin;
6132 1.15 jmcneill struct mcx_cmd_query_flow_table_out *out;
6133 1.15 jmcneill struct mcx_cmd_query_flow_table_mb_out *mbout;
6134 1.15 jmcneill uint8_t token = mcx_cmdq_token(sc);
6135 1.15 jmcneill int error;
6136 1.15 jmcneill int i;
6137 1.15 jmcneill uint8_t *dump;
6138 1.1 jmcneill
6139 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6140 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6141 1.1 jmcneill sizeof(*out) + sizeof(*mbout) + 16, token);
6142 1.1 jmcneill
6143 1.1 jmcneill in = mcx_cmdq_in(cqe);
6144 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
6145 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6146 1.1 jmcneill
6147 1.1 jmcneill CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6148 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
6149 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6150 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6151 1.1 jmcneill printf(", unable to allocate query flow table mailboxes\n");
6152 1.1 jmcneill return (-1);
6153 1.1 jmcneill }
6154 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6155 1.1 jmcneill
6156 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6157 1.1 jmcneill mbin->cmd_table_type = 0;
6158 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
6159 1.1 jmcneill
6160 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6161 1.1 jmcneill
6162 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6163 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6164 1.1 jmcneill if (error != 0) {
6165 1.1 jmcneill printf("%s: query flow table timeout\n", DEVNAME(sc));
6166 1.1 jmcneill goto free;
6167 1.1 jmcneill }
6168 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6169 1.1 jmcneill if (error != 0) {
6170 1.1 jmcneill printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
6171 1.1 jmcneill goto free;
6172 1.1 jmcneill }
6173 1.1 jmcneill
6174 1.1 jmcneill out = mcx_cmdq_out(cqe);
6175 1.1 jmcneill switch (out->cmd_status) {
6176 1.1 jmcneill case MCX_CQ_STATUS_OK:
6177 1.1 jmcneill break;
6178 1.1 jmcneill default:
6179 1.1 jmcneill printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
6180 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6181 1.1 jmcneill error = -1;
6182 1.1 jmcneill goto free;
6183 1.1 jmcneill }
6184 1.1 jmcneill
6185 1.1 jmcneill mbout = (struct mcx_cmd_query_flow_table_mb_out *)
6186 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6187 1.1 jmcneill dump = (uint8_t *)mbout + 8;
6188 1.1 jmcneill for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
6189 1.1 jmcneill printf("%.2x ", dump[i]);
6190 1.1 jmcneill if (i % 16 == 15)
6191 1.1 jmcneill printf("\n");
6192 1.1 jmcneill }
6193 1.1 jmcneill free:
6194 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6195 1.1 jmcneill return (error);
6196 1.1 jmcneill }
6197 1.1 jmcneill int
6198 1.15 jmcneill mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index)
6199 1.1 jmcneill {
6200 1.1 jmcneill struct mcx_dmamem mxm;
6201 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6202 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_in *in;
6203 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
6204 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_out *out;
6205 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
6206 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6207 1.1 jmcneill int error;
6208 1.1 jmcneill int i;
6209 1.1 jmcneill uint8_t *dump;
6210 1.1 jmcneill
6211 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6212 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6213 1.1 jmcneill sizeof(*out) + sizeof(*mbout) + 16, token);
6214 1.1 jmcneill
6215 1.1 jmcneill in = mcx_cmdq_in(cqe);
6216 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
6217 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6218 1.1 jmcneill
6219 1.1 jmcneill CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6220 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6221 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6222 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6223 1.15 jmcneill printf(", unable to allocate "
6224 1.15 jmcneill "query flow table entry mailboxes\n");
6225 1.1 jmcneill return (-1);
6226 1.1 jmcneill }
6227 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6228 1.1 jmcneill
6229 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6230 1.1 jmcneill mbin->cmd_table_type = 0;
6231 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
6232 1.1 jmcneill mbin->cmd_flow_index = htobe32(index);
6233 1.1 jmcneill
6234 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6235 1.1 jmcneill
6236 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6237 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6238 1.1 jmcneill if (error != 0) {
6239 1.1 jmcneill printf("%s: query flow table entry timeout\n", DEVNAME(sc));
6240 1.1 jmcneill goto free;
6241 1.1 jmcneill }
6242 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6243 1.1 jmcneill if (error != 0) {
6244 1.1 jmcneill printf("%s: query flow table entry reply corrupt\n",
6245 1.1 jmcneill DEVNAME(sc));
6246 1.1 jmcneill goto free;
6247 1.1 jmcneill }
6248 1.1 jmcneill
6249 1.1 jmcneill out = mcx_cmdq_out(cqe);
6250 1.1 jmcneill switch (out->cmd_status) {
6251 1.1 jmcneill case MCX_CQ_STATUS_OK:
6252 1.1 jmcneill break;
6253 1.1 jmcneill default:
6254 1.1 jmcneill printf("%s: query flow table entry failed (%x/%x)\n",
6255 1.1 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
6256 1.1 jmcneill error = -1;
6257 1.1 jmcneill goto free;
6258 1.1 jmcneill }
6259 1.1 jmcneill
6260 1.1 jmcneill mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
6261 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6262 1.1 jmcneill dump = (uint8_t *)mbout;
6263 1.1 jmcneill for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6264 1.1 jmcneill printf("%.2x ", dump[i]);
6265 1.1 jmcneill if (i % 16 == 15)
6266 1.1 jmcneill printf("\n");
6267 1.1 jmcneill }
6268 1.1 jmcneill
6269 1.1 jmcneill free:
6270 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6271 1.1 jmcneill return (error);
6272 1.1 jmcneill }
6273 1.1 jmcneill
6274 1.1 jmcneill int
6275 1.15 jmcneill mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id)
6276 1.1 jmcneill {
6277 1.1 jmcneill struct mcx_dmamem mxm;
6278 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6279 1.1 jmcneill struct mcx_cmd_query_flow_group_in *in;
6280 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_in *mbin;
6281 1.1 jmcneill struct mcx_cmd_query_flow_group_out *out;
6282 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_out *mbout;
6283 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6284 1.1 jmcneill int error;
6285 1.1 jmcneill int i;
6286 1.1 jmcneill uint8_t *dump;
6287 1.1 jmcneill
6288 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6289 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6290 1.1 jmcneill sizeof(*out) + sizeof(*mbout) + 16, token);
6291 1.1 jmcneill
6292 1.1 jmcneill in = mcx_cmdq_in(cqe);
6293 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
6294 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6295 1.1 jmcneill
6296 1.1 jmcneill CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6297 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6298 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6299 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6300 1.1 jmcneill printf(", unable to allocate query flow group mailboxes\n");
6301 1.1 jmcneill return (-1);
6302 1.1 jmcneill }
6303 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6304 1.1 jmcneill
6305 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6306 1.1 jmcneill mbin->cmd_table_type = 0;
6307 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
6308 1.1 jmcneill mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
6309 1.1 jmcneill
6310 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6311 1.1 jmcneill
6312 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6313 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6314 1.1 jmcneill if (error != 0) {
6315 1.1 jmcneill printf("%s: query flow group timeout\n", DEVNAME(sc));
6316 1.1 jmcneill goto free;
6317 1.1 jmcneill }
6318 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6319 1.1 jmcneill if (error != 0) {
6320 1.1 jmcneill printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
6321 1.1 jmcneill goto free;
6322 1.1 jmcneill }
6323 1.1 jmcneill
6324 1.1 jmcneill out = mcx_cmdq_out(cqe);
6325 1.1 jmcneill switch (out->cmd_status) {
6326 1.1 jmcneill case MCX_CQ_STATUS_OK:
6327 1.1 jmcneill break;
6328 1.1 jmcneill default:
6329 1.1 jmcneill printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
6330 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6331 1.1 jmcneill error = -1;
6332 1.1 jmcneill goto free;
6333 1.1 jmcneill }
6334 1.1 jmcneill
6335 1.1 jmcneill mbout = (struct mcx_cmd_query_flow_group_mb_out *)
6336 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6337 1.1 jmcneill dump = (uint8_t *)mbout;
6338 1.1 jmcneill for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6339 1.1 jmcneill printf("%.2x ", dump[i]);
6340 1.1 jmcneill if (i % 16 == 15)
6341 1.1 jmcneill printf("\n");
6342 1.1 jmcneill }
6343 1.1 jmcneill dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
6344 1.1 jmcneill for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6345 1.1 jmcneill printf("%.2x ", dump[i]);
6346 1.1 jmcneill if (i % 16 == 15)
6347 1.1 jmcneill printf("\n");
6348 1.1 jmcneill }
6349 1.1 jmcneill
6350 1.1 jmcneill free:
6351 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6352 1.1 jmcneill return (error);
6353 1.1 jmcneill }
6354 1.1 jmcneill
6355 1.15 jmcneill static int
6356 1.15 jmcneill mcx_dump_counters(struct mcx_softc *sc)
6357 1.15 jmcneill {
6358 1.15 jmcneill struct mcx_dmamem mxm;
6359 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6360 1.15 jmcneill struct mcx_cmd_query_vport_counters_in *in;
6361 1.15 jmcneill struct mcx_cmd_query_vport_counters_mb_in *mbin;
6362 1.15 jmcneill struct mcx_cmd_query_vport_counters_out *out;
6363 1.15 jmcneill struct mcx_nic_vport_counters *counters;
6364 1.15 jmcneill int error, token;
6365 1.15 jmcneill
6366 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6367 1.15 jmcneill token = mcx_cmdq_token(sc);
6368 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6369 1.15 jmcneill sizeof(*out) + sizeof(*counters), token);
6370 1.15 jmcneill
6371 1.15 jmcneill in = mcx_cmdq_in(cqe);
6372 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
6373 1.15 jmcneill in->cmd_op_mod = htobe16(0);
6374 1.15 jmcneill
6375 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
6376 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6377 1.15 jmcneill printf(", unable to allocate "
6378 1.15 jmcneill "query nic vport counters mailboxen\n");
6379 1.15 jmcneill return (-1);
6380 1.15 jmcneill }
6381 1.15 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6382 1.15 jmcneill
6383 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6384 1.15 jmcneill mbin->cmd_clear = 0x80;
6385 1.15 jmcneill
6386 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6387 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
6388 1.15 jmcneill
6389 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6390 1.15 jmcneill if (error != 0) {
6391 1.15 jmcneill printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
6392 1.15 jmcneill goto free;
6393 1.15 jmcneill }
6394 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6395 1.15 jmcneill printf("%s: query nic vport counters command corrupt\n",
6396 1.15 jmcneill DEVNAME(sc));
6397 1.15 jmcneill goto free;
6398 1.15 jmcneill }
6399 1.15 jmcneill
6400 1.15 jmcneill out = mcx_cmdq_out(cqe);
6401 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6402 1.15 jmcneill printf("%s: query nic vport counters failed (%x, %x)\n",
6403 1.15 jmcneill DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
6404 1.15 jmcneill error = -1;
6405 1.15 jmcneill goto free;
6406 1.15 jmcneill }
6407 1.15 jmcneill
6408 1.15 jmcneill counters = (struct mcx_nic_vport_counters *)
6409 1.15 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6410 1.15 jmcneill if (counters->rx_bcast.packets + counters->tx_bcast.packets +
6411 1.15 jmcneill counters->rx_ucast.packets + counters->tx_ucast.packets +
6412 1.15 jmcneill counters->rx_err.packets + counters->tx_err.packets)
6413 1.15 jmcneill printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
6414 1.15 jmcneill DEVNAME(sc),
6415 1.15 jmcneill betoh64(counters->tx_err.packets),
6416 1.15 jmcneill betoh64(counters->rx_err.packets),
6417 1.15 jmcneill betoh64(counters->tx_ucast.packets),
6418 1.15 jmcneill betoh64(counters->rx_ucast.packets),
6419 1.15 jmcneill betoh64(counters->tx_bcast.packets),
6420 1.15 jmcneill betoh64(counters->rx_bcast.packets));
6421 1.15 jmcneill free:
6422 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
6423 1.15 jmcneill
6424 1.15 jmcneill return (error);
6425 1.15 jmcneill }
6426 1.15 jmcneill
6427 1.15 jmcneill static int
6428 1.15 jmcneill mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
6429 1.15 jmcneill {
6430 1.15 jmcneill struct mcx_dmamem mxm;
6431 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6432 1.15 jmcneill struct mcx_cmd_query_flow_counter_in *in;
6433 1.15 jmcneill struct mcx_cmd_query_flow_counter_mb_in *mbin;
6434 1.15 jmcneill struct mcx_cmd_query_flow_counter_out *out;
6435 1.15 jmcneill struct mcx_counter *counters;
6436 1.15 jmcneill int error, token;
6437 1.15 jmcneill
6438 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6439 1.15 jmcneill token = mcx_cmdq_token(sc);
6440 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
6441 1.15 jmcneill sizeof(*counters), token);
6442 1.15 jmcneill
6443 1.15 jmcneill in = mcx_cmdq_in(cqe);
6444 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
6445 1.15 jmcneill in->cmd_op_mod = htobe16(0);
6446 1.15 jmcneill
6447 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
6448 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6449 1.15 jmcneill printf(", unable to allocate query flow counter mailboxen\n");
6450 1.15 jmcneill return (-1);
6451 1.15 jmcneill }
6452 1.15 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6453 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6454 1.15 jmcneill mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
6455 1.15 jmcneill mbin->cmd_clear = 0x80;
6456 1.15 jmcneill
6457 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6458 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
6459 1.15 jmcneill
6460 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6461 1.15 jmcneill if (error != 0) {
6462 1.15 jmcneill printf("%s: query flow counter timeout\n", DEVNAME(sc));
6463 1.15 jmcneill goto free;
6464 1.15 jmcneill }
6465 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6466 1.15 jmcneill printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
6467 1.15 jmcneill goto free;
6468 1.15 jmcneill }
6469 1.15 jmcneill
6470 1.15 jmcneill out = mcx_cmdq_out(cqe);
6471 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6472 1.15 jmcneill printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
6473 1.15 jmcneill out->cmd_status, betoh32(out->cmd_syndrome));
6474 1.15 jmcneill error = -1;
6475 1.15 jmcneill goto free;
6476 1.15 jmcneill }
6477 1.15 jmcneill
6478 1.15 jmcneill counters = (struct mcx_counter *)
6479 1.15 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6480 1.15 jmcneill if (counters->packets)
6481 1.15 jmcneill printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
6482 1.15 jmcneill betoh64(counters->packets));
6483 1.15 jmcneill free:
6484 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
6485 1.15 jmcneill
6486 1.15 jmcneill return (error);
6487 1.15 jmcneill }
6488 1.15 jmcneill
6489 1.15 jmcneill #endif
6490 1.15 jmcneill
6491 1.15 jmcneill #if NKSTAT > 0
6492 1.15 jmcneill
6493 1.1 jmcneill int
6494 1.15 jmcneill mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx)
6495 1.1 jmcneill {
6496 1.1 jmcneill struct mcx_dmamem mxm;
6497 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6498 1.1 jmcneill struct mcx_cmd_query_rq_in *in;
6499 1.1 jmcneill struct mcx_cmd_query_rq_out *out;
6500 1.1 jmcneill struct mcx_cmd_query_rq_mb_out *mbout;
6501 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6502 1.1 jmcneill int error;
6503 1.1 jmcneill
6504 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6505 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
6506 1.1 jmcneill token);
6507 1.1 jmcneill
6508 1.1 jmcneill in = mcx_cmdq_in(cqe);
6509 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
6510 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6511 1.15 jmcneill in->cmd_rqn = htobe32(rx->rx_rqn);
6512 1.1 jmcneill
6513 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6514 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6515 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6516 1.15 jmcneill printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc));
6517 1.1 jmcneill return (-1);
6518 1.1 jmcneill }
6519 1.1 jmcneill
6520 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6521 1.1 jmcneill
6522 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6523 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6524 1.1 jmcneill if (error != 0) {
6525 1.1 jmcneill printf("%s: query rq timeout\n", DEVNAME(sc));
6526 1.1 jmcneill goto free;
6527 1.1 jmcneill }
6528 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6529 1.1 jmcneill if (error != 0) {
6530 1.1 jmcneill printf("%s: query rq reply corrupt\n", DEVNAME(sc));
6531 1.1 jmcneill goto free;
6532 1.1 jmcneill }
6533 1.1 jmcneill
6534 1.1 jmcneill out = mcx_cmdq_out(cqe);
6535 1.1 jmcneill switch (out->cmd_status) {
6536 1.1 jmcneill case MCX_CQ_STATUS_OK:
6537 1.1 jmcneill break;
6538 1.1 jmcneill default:
6539 1.1 jmcneill printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
6540 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6541 1.1 jmcneill error = -1;
6542 1.1 jmcneill goto free;
6543 1.1 jmcneill }
6544 1.1 jmcneill
6545 1.1 jmcneill mbout = (struct mcx_cmd_query_rq_mb_out *)
6546 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6547 1.15 jmcneill memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx));
6548 1.1 jmcneill
6549 1.1 jmcneill free:
6550 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6551 1.1 jmcneill return (error);
6552 1.1 jmcneill }
6553 1.1 jmcneill
6554 1.1 jmcneill int
6555 1.15 jmcneill mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx)
6556 1.1 jmcneill {
6557 1.1 jmcneill struct mcx_dmamem mxm;
6558 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6559 1.1 jmcneill struct mcx_cmd_query_sq_in *in;
6560 1.1 jmcneill struct mcx_cmd_query_sq_out *out;
6561 1.1 jmcneill struct mcx_cmd_query_sq_mb_out *mbout;
6562 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6563 1.1 jmcneill int error;
6564 1.1 jmcneill
6565 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6566 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
6567 1.1 jmcneill token);
6568 1.1 jmcneill
6569 1.1 jmcneill in = mcx_cmdq_in(cqe);
6570 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
6571 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6572 1.15 jmcneill in->cmd_sqn = htobe32(tx->tx_sqn);
6573 1.1 jmcneill
6574 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6575 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6576 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6577 1.15 jmcneill printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc));
6578 1.1 jmcneill return (-1);
6579 1.1 jmcneill }
6580 1.1 jmcneill
6581 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6582 1.1 jmcneill
6583 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6584 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6585 1.1 jmcneill if (error != 0) {
6586 1.1 jmcneill printf("%s: query sq timeout\n", DEVNAME(sc));
6587 1.1 jmcneill goto free;
6588 1.1 jmcneill }
6589 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6590 1.1 jmcneill if (error != 0) {
6591 1.1 jmcneill printf("%s: query sq reply corrupt\n", DEVNAME(sc));
6592 1.1 jmcneill goto free;
6593 1.1 jmcneill }
6594 1.1 jmcneill
6595 1.1 jmcneill out = mcx_cmdq_out(cqe);
6596 1.1 jmcneill switch (out->cmd_status) {
6597 1.1 jmcneill case MCX_CQ_STATUS_OK:
6598 1.1 jmcneill break;
6599 1.1 jmcneill default:
6600 1.1 jmcneill printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
6601 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6602 1.1 jmcneill error = -1;
6603 1.1 jmcneill goto free;
6604 1.1 jmcneill }
6605 1.1 jmcneill
6606 1.1 jmcneill mbout = (struct mcx_cmd_query_sq_mb_out *)
6607 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6608 1.15 jmcneill memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx));
6609 1.1 jmcneill
6610 1.1 jmcneill free:
6611 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6612 1.1 jmcneill return (error);
6613 1.1 jmcneill }
6614 1.1 jmcneill
6615 1.15 jmcneill int
6616 1.15 jmcneill mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx)
6617 1.1 jmcneill {
6618 1.1 jmcneill struct mcx_dmamem mxm;
6619 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6620 1.15 jmcneill struct mcx_cmd_query_cq_in *in;
6621 1.15 jmcneill struct mcx_cmd_query_cq_out *out;
6622 1.15 jmcneill struct mcx_cq_ctx *ctx;
6623 1.15 jmcneill uint8_t token = mcx_cmdq_token(sc);
6624 1.15 jmcneill int error;
6625 1.1 jmcneill
6626 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6627 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
6628 1.15 jmcneill token);
6629 1.1 jmcneill
6630 1.1 jmcneill in = mcx_cmdq_in(cqe);
6631 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ);
6632 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6633 1.15 jmcneill in->cmd_cqn = htobe32(cq->cq_n);
6634 1.1 jmcneill
6635 1.15 jmcneill CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6636 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6637 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6638 1.15 jmcneill printf("%s: unable to allocate query cq mailboxes\n",
6639 1.15 jmcneill DEVNAME(sc));
6640 1.1 jmcneill return (-1);
6641 1.1 jmcneill }
6642 1.1 jmcneill
6643 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6644 1.1 jmcneill
6645 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6646 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6647 1.1 jmcneill if (error != 0) {
6648 1.15 jmcneill printf("%s: query cq timeout\n", DEVNAME(sc));
6649 1.1 jmcneill goto free;
6650 1.1 jmcneill }
6651 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6652 1.15 jmcneill printf("%s: query cq reply corrupt\n", DEVNAME(sc));
6653 1.1 jmcneill goto free;
6654 1.1 jmcneill }
6655 1.1 jmcneill
6656 1.1 jmcneill out = mcx_cmdq_out(cqe);
6657 1.15 jmcneill switch (out->cmd_status) {
6658 1.15 jmcneill case MCX_CQ_STATUS_OK:
6659 1.15 jmcneill break;
6660 1.15 jmcneill default:
6661 1.15 jmcneill printf("%s: query qc failed (%x/%x)\n", DEVNAME(sc),
6662 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6663 1.1 jmcneill error = -1;
6664 1.1 jmcneill goto free;
6665 1.1 jmcneill }
6666 1.1 jmcneill
6667 1.15 jmcneill ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6668 1.15 jmcneill memcpy(cq_ctx, ctx, sizeof(*cq_ctx));
6669 1.1 jmcneill free:
6670 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
6671 1.1 jmcneill return (error);
6672 1.1 jmcneill }
6673 1.1 jmcneill
6674 1.15 jmcneill int
6675 1.15 jmcneill mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx)
6676 1.1 jmcneill {
6677 1.1 jmcneill struct mcx_dmamem mxm;
6678 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6679 1.15 jmcneill struct mcx_cmd_query_eq_in *in;
6680 1.15 jmcneill struct mcx_cmd_query_eq_out *out;
6681 1.15 jmcneill struct mcq_eq_ctx *ctx;
6682 1.15 jmcneill uint8_t token = mcx_cmdq_token(sc);
6683 1.15 jmcneill int error;
6684 1.1 jmcneill
6685 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6686 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
6687 1.15 jmcneill token);
6688 1.1 jmcneill
6689 1.1 jmcneill in = mcx_cmdq_in(cqe);
6690 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ);
6691 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6692 1.15 jmcneill in->cmd_eqn = htobe32(eq->eq_n);
6693 1.1 jmcneill
6694 1.15 jmcneill CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6695 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6696 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6697 1.15 jmcneill printf("%s: unable to allocate query eq mailboxes\n",
6698 1.15 jmcneill DEVNAME(sc));
6699 1.1 jmcneill return (-1);
6700 1.1 jmcneill }
6701 1.1 jmcneill
6702 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6703 1.15 jmcneill
6704 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6705 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6706 1.1 jmcneill if (error != 0) {
6707 1.15 jmcneill printf("%s: query eq timeout\n", DEVNAME(sc));
6708 1.1 jmcneill goto free;
6709 1.1 jmcneill }
6710 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6711 1.15 jmcneill printf("%s: query eq reply corrupt\n", DEVNAME(sc));
6712 1.1 jmcneill goto free;
6713 1.1 jmcneill }
6714 1.1 jmcneill
6715 1.1 jmcneill out = mcx_cmdq_out(cqe);
6716 1.15 jmcneill switch (out->cmd_status) {
6717 1.15 jmcneill case MCX_CQ_STATUS_OK:
6718 1.15 jmcneill break;
6719 1.15 jmcneill default:
6720 1.15 jmcneill printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc),
6721 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6722 1.1 jmcneill error = -1;
6723 1.1 jmcneill goto free;
6724 1.1 jmcneill }
6725 1.1 jmcneill
6726 1.15 jmcneill ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6727 1.15 jmcneill memcpy(eq_ctx, ctx, sizeof(*eq_ctx));
6728 1.1 jmcneill free:
6729 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
6730 1.1 jmcneill return (error);
6731 1.1 jmcneill }
6732 1.1 jmcneill
6733 1.15 jmcneill #endif /* NKSTAT > 0 */
6734 1.15 jmcneill
6735 1.1 jmcneill
6736 1.15 jmcneill static inline unsigned int
6737 1.15 jmcneill mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots)
6738 1.1 jmcneill {
6739 1.15 jmcneill struct mcx_rq_entry *ring, *rqe;
6740 1.1 jmcneill struct mcx_slot *ms;
6741 1.1 jmcneill struct mbuf *m;
6742 1.1 jmcneill uint slot, p, fills;
6743 1.1 jmcneill
6744 1.15 jmcneill ring = MCX_DMA_KVA(&rx->rx_rq_mem);
6745 1.15 jmcneill p = rx->rx_prod;
6746 1.15 jmcneill
6747 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
6748 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
6749 1.15 jmcneill
6750 1.1 jmcneill slot = (p % (1 << MCX_LOG_RQ_SIZE));
6751 1.1 jmcneill rqe = ring;
6752 1.1 jmcneill for (fills = 0; fills < nslots; fills++) {
6753 1.15 jmcneill slot = p % (1 << MCX_LOG_RQ_SIZE);
6754 1.15 jmcneill
6755 1.15 jmcneill ms = &rx->rx_slots[slot];
6756 1.15 jmcneill rqe = &ring[slot];
6757 1.15 jmcneill
6758 1.1 jmcneill m = NULL;
6759 1.1 jmcneill MGETHDR(m, M_DONTWAIT, MT_DATA);
6760 1.1 jmcneill if (m == NULL)
6761 1.1 jmcneill break;
6762 1.1 jmcneill
6763 1.1 jmcneill MCLGET(m, M_DONTWAIT);
6764 1.1 jmcneill if ((m->m_flags & M_EXT) == 0) {
6765 1.1 jmcneill m_freem(m);
6766 1.1 jmcneill break;
6767 1.1 jmcneill }
6768 1.1 jmcneill
6769 1.15 jmcneill m->m_len = m->m_pkthdr.len = sc->sc_hardmtu;
6770 1.15 jmcneill m_adj(m, m->m_ext.ext_size - sc->sc_rxbufsz);
6771 1.15 jmcneill m_adj(m, ETHER_ALIGN);
6772 1.15 jmcneill
6773 1.1 jmcneill if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
6774 1.1 jmcneill BUS_DMA_NOWAIT) != 0) {
6775 1.1 jmcneill m_freem(m);
6776 1.1 jmcneill break;
6777 1.1 jmcneill }
6778 1.3 tnn bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, BUS_DMASYNC_PREREAD);
6779 1.1 jmcneill ms->ms_m = m;
6780 1.1 jmcneill
6781 1.15 jmcneill be32enc(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len);
6782 1.15 jmcneill be64enc(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr);
6783 1.15 jmcneill be32enc(&rqe->rqe_lkey, sc->sc_lkey);
6784 1.1 jmcneill
6785 1.1 jmcneill p++;
6786 1.1 jmcneill }
6787 1.1 jmcneill
6788 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
6789 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
6790 1.15 jmcneill
6791 1.15 jmcneill rx->rx_prod = p;
6792 1.1 jmcneill
6793 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
6794 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
6795 1.15 jmcneill be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell),
6796 1.15 jmcneill p & MCX_WQ_DOORBELL_MASK);
6797 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
6798 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
6799 1.1 jmcneill
6800 1.1 jmcneill return (nslots - fills);
6801 1.1 jmcneill }
6802 1.1 jmcneill
6803 1.1 jmcneill static int
6804 1.15 jmcneill mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx)
6805 1.1 jmcneill {
6806 1.1 jmcneill u_int slots;
6807 1.1 jmcneill
6808 1.15 jmcneill slots = mcx_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE));
6809 1.1 jmcneill if (slots == 0)
6810 1.1 jmcneill return (1);
6811 1.1 jmcneill
6812 1.15 jmcneill slots = mcx_rx_fill_slots(sc, rx, slots);
6813 1.15 jmcneill mcx_rxr_put(&rx->rx_rxr, slots);
6814 1.1 jmcneill return (0);
6815 1.1 jmcneill }
6816 1.1 jmcneill
6817 1.1 jmcneill void
6818 1.15 jmcneill mcx_refill(void *xrx)
6819 1.1 jmcneill {
6820 1.15 jmcneill struct mcx_rx *rx = xrx;
6821 1.15 jmcneill struct mcx_softc *sc = rx->rx_softc;
6822 1.1 jmcneill
6823 1.15 jmcneill mcx_rx_fill(sc, rx);
6824 1.1 jmcneill
6825 1.15 jmcneill if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
6826 1.15 jmcneill callout_schedule(&rx->rx_refill, 1);
6827 1.1 jmcneill }
6828 1.1 jmcneill
6829 1.15 jmcneill static int
6830 1.15 jmcneill mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx,
6831 1.15 jmcneill struct mcx_cq_entry *cqe)
6832 1.1 jmcneill {
6833 1.1 jmcneill struct mcx_slot *ms;
6834 1.1 jmcneill bus_dmamap_t map;
6835 1.1 jmcneill int slot, slots;
6836 1.1 jmcneill
6837 1.1 jmcneill slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
6838 1.1 jmcneill
6839 1.15 jmcneill ms = &tx->tx_slots[slot];
6840 1.1 jmcneill map = ms->ms_map;
6841 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
6842 1.1 jmcneill BUS_DMASYNC_POSTWRITE);
6843 1.1 jmcneill
6844 1.1 jmcneill slots = 1;
6845 1.1 jmcneill if (map->dm_nsegs > 1)
6846 1.1 jmcneill slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;
6847 1.1 jmcneill
6848 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, map);
6849 1.1 jmcneill m_freem(ms->ms_m);
6850 1.1 jmcneill ms->ms_m = NULL;
6851 1.15 jmcneill
6852 1.15 jmcneill return (slots);
6853 1.1 jmcneill }
6854 1.1 jmcneill
6855 1.1 jmcneill static uint64_t
6856 1.1 jmcneill mcx_uptime(void)
6857 1.1 jmcneill {
6858 1.1 jmcneill struct timespec ts;
6859 1.1 jmcneill
6860 1.1 jmcneill nanouptime(&ts);
6861 1.1 jmcneill
6862 1.1 jmcneill return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec);
6863 1.1 jmcneill }
6864 1.1 jmcneill
6865 1.1 jmcneill static void
6866 1.1 jmcneill mcx_calibrate_first(struct mcx_softc *sc)
6867 1.1 jmcneill {
6868 1.1 jmcneill struct mcx_calibration *c = &sc->sc_calibration[0];
6869 1.15 jmcneill int s;
6870 1.1 jmcneill
6871 1.1 jmcneill sc->sc_calibration_gen = 0;
6872 1.1 jmcneill
6873 1.15 jmcneill s = splhigh(); /* crit_enter? */
6874 1.1 jmcneill c->c_ubase = mcx_uptime();
6875 1.1 jmcneill c->c_tbase = mcx_timer(sc);
6876 1.15 jmcneill splx(s);
6877 1.15 jmcneill c->c_ratio = 0;
6878 1.1 jmcneill
6879 1.15 jmcneill #if notyet
6880 1.1 jmcneill callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz);
6881 1.15 jmcneill #endif
6882 1.1 jmcneill }
6883 1.1 jmcneill
6884 1.15 jmcneill #define MCX_TIMESTAMP_SHIFT 24
6885 1.1 jmcneill
6886 1.1 jmcneill static void
6887 1.1 jmcneill mcx_calibrate(void *arg)
6888 1.1 jmcneill {
6889 1.1 jmcneill struct mcx_softc *sc = arg;
6890 1.1 jmcneill struct mcx_calibration *nc, *pc;
6891 1.15 jmcneill uint64_t udiff, tdiff;
6892 1.1 jmcneill unsigned int gen;
6893 1.15 jmcneill int s;
6894 1.1 jmcneill
6895 1.1 jmcneill if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING))
6896 1.1 jmcneill return;
6897 1.1 jmcneill
6898 1.1 jmcneill callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz);
6899 1.1 jmcneill
6900 1.1 jmcneill gen = sc->sc_calibration_gen;
6901 1.1 jmcneill pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
6902 1.1 jmcneill gen++;
6903 1.1 jmcneill nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
6904 1.1 jmcneill
6905 1.1 jmcneill nc->c_uptime = pc->c_ubase;
6906 1.1 jmcneill nc->c_timestamp = pc->c_tbase;
6907 1.1 jmcneill
6908 1.15 jmcneill s = splhigh(); /* crit_enter? */
6909 1.1 jmcneill nc->c_ubase = mcx_uptime();
6910 1.1 jmcneill nc->c_tbase = mcx_timer(sc);
6911 1.15 jmcneill splx(s);
6912 1.1 jmcneill
6913 1.15 jmcneill udiff = nc->c_ubase - nc->c_uptime;
6914 1.15 jmcneill tdiff = nc->c_tbase - nc->c_timestamp;
6915 1.15 jmcneill
6916 1.15 jmcneill /*
6917 1.15 jmcneill * udiff is the wall clock time between calibration ticks,
6918 1.15 jmcneill * which should be 32 seconds or 32 billion nanoseconds. if
6919 1.15 jmcneill * we squint, 1 billion nanoseconds is kind of like a 32 bit
6920 1.15 jmcneill * number, so 32 billion should still have a lot of high bits
6921 1.15 jmcneill * spare. we use this space by shifting the nanoseconds up
6922 1.15 jmcneill * 24 bits so we have a nice big number to divide by the
6923 1.15 jmcneill * number of mcx timer ticks.
6924 1.15 jmcneill */
6925 1.15 jmcneill nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff;
6926 1.1 jmcneill
6927 1.1 jmcneill membar_producer();
6928 1.1 jmcneill sc->sc_calibration_gen = gen;
6929 1.1 jmcneill }
6930 1.1 jmcneill
6931 1.1 jmcneill static int
6932 1.15 jmcneill mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx,
6933 1.15 jmcneill struct mcx_cq_entry *cqe, struct mcx_mbufq *mq,
6934 1.15 jmcneill const struct mcx_calibration *c)
6935 1.1 jmcneill {
6936 1.15 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
6937 1.1 jmcneill struct mcx_slot *ms;
6938 1.1 jmcneill struct mbuf *m;
6939 1.20 jmcneill uint32_t flags, len;
6940 1.1 jmcneill int slot;
6941 1.1 jmcneill
6942 1.20 jmcneill len = be32dec(&cqe->cq_byte_cnt);
6943 1.1 jmcneill slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
6944 1.1 jmcneill
6945 1.15 jmcneill ms = &rx->rx_slots[slot];
6946 1.20 jmcneill bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, len, BUS_DMASYNC_POSTREAD);
6947 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
6948 1.1 jmcneill
6949 1.1 jmcneill m = ms->ms_m;
6950 1.1 jmcneill ms->ms_m = NULL;
6951 1.1 jmcneill
6952 1.1 jmcneill m_set_rcvif(m, &sc->sc_ec.ec_if);
6953 1.20 jmcneill m->m_pkthdr.len = m->m_len = len;
6954 1.1 jmcneill
6955 1.1 jmcneill #if 0
6956 1.1 jmcneill if (cqe->cq_rx_hash_type) {
6957 1.15 jmcneill m->m_pkthdr.ph_flowid = be32toh(cqe->cq_rx_hash);
6958 1.15 jmcneill m->m_pkthdr.csum_flags |= M_FLOWID;
6959 1.1 jmcneill }
6960 1.1 jmcneill #endif
6961 1.1 jmcneill
6962 1.15 jmcneill flags = be32dec(&cqe->cq_flags);
6963 1.15 jmcneill if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) {
6964 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx)
6965 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
6966 1.15 jmcneill }
6967 1.15 jmcneill if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) {
6968 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx)
6969 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
6970 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx)
6971 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_TCPv6;
6972 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx)
6973 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
6974 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx)
6975 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_UDPv6;
6976 1.15 jmcneill }
6977 1.15 jmcneill if (flags & MCX_CQ_ENTRY_FLAGS_CV) {
6978 1.15 jmcneill vlan_set_tag(m, flags & MCX_CQ_ENTRY_FLAGS_VLAN_MASK);
6979 1.15 jmcneill }
6980 1.15 jmcneill
6981 1.15 jmcneill #if notyet
6982 1.15 jmcneill if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_LINK0) && c->c_ratio) {
6983 1.15 jmcneill uint64_t t = be64dec(&cqe->cq_timestamp);
6984 1.15 jmcneill t -= c->c_timestamp;
6985 1.15 jmcneill t *= c->c_ratio;
6986 1.15 jmcneill t >>= MCX_TIMESTAMP_SHIFT;
6987 1.15 jmcneill t += c->c_uptime;
6988 1.1 jmcneill
6989 1.15 jmcneill m->m_pkthdr.ph_timestamp = t;
6990 1.1 jmcneill SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
6991 1.1 jmcneill }
6992 1.1 jmcneill #endif
6993 1.1 jmcneill
6994 1.1 jmcneill MBUFQ_ENQUEUE(mq, m);
6995 1.1 jmcneill
6996 1.1 jmcneill return (1);
6997 1.1 jmcneill }
6998 1.1 jmcneill
6999 1.1 jmcneill static struct mcx_cq_entry *
7000 1.1 jmcneill mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
7001 1.1 jmcneill {
7002 1.1 jmcneill struct mcx_cq_entry *cqe;
7003 1.1 jmcneill int next;
7004 1.1 jmcneill
7005 1.1 jmcneill cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
7006 1.1 jmcneill next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
7007 1.1 jmcneill
7008 1.1 jmcneill if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
7009 1.1 jmcneill ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
7010 1.1 jmcneill return (&cqe[next]);
7011 1.1 jmcneill }
7012 1.1 jmcneill
7013 1.1 jmcneill return (NULL);
7014 1.1 jmcneill }
7015 1.1 jmcneill
7016 1.1 jmcneill static void
7017 1.15 jmcneill mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar)
7018 1.1 jmcneill {
7019 1.15 jmcneill struct mcx_cq_doorbell *db;
7020 1.1 jmcneill bus_size_t offset;
7021 1.1 jmcneill uint32_t val;
7022 1.1 jmcneill uint64_t uval;
7023 1.1 jmcneill
7024 1.1 jmcneill val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
7025 1.1 jmcneill val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
7026 1.1 jmcneill
7027 1.15 jmcneill db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell);
7028 1.15 jmcneill
7029 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7030 1.15 jmcneill cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE);
7031 1.1 jmcneill
7032 1.15 jmcneill be32enc(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
7033 1.15 jmcneill be32enc(&db->db_arm_ci, val);
7034 1.15 jmcneill
7035 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7036 1.15 jmcneill cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE);
7037 1.15 jmcneill
7038 1.15 jmcneill offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL;
7039 1.15 jmcneill
7040 1.15 jmcneill uval = (uint64_t)val << 32;
7041 1.1 jmcneill uval |= cq->cq_n;
7042 1.15 jmcneill
7043 1.15 jmcneill bus_space_write_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval));
7044 1.15 jmcneill mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE);
7045 1.1 jmcneill }
7046 1.1 jmcneill
7047 1.1 jmcneill void
7048 1.15 jmcneill mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq)
7049 1.1 jmcneill {
7050 1.15 jmcneill struct mcx_rx *rx = &q->q_rx;
7051 1.15 jmcneill struct mcx_tx *tx = &q->q_tx;
7052 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
7053 1.1 jmcneill const struct mcx_calibration *c;
7054 1.1 jmcneill unsigned int gen;
7055 1.1 jmcneill struct mcx_cq_entry *cqe;
7056 1.1 jmcneill struct mcx_mbufq mq;
7057 1.1 jmcneill struct mbuf *m;
7058 1.1 jmcneill int rxfree, txfree;
7059 1.1 jmcneill
7060 1.1 jmcneill MBUFQ_INIT(&mq);
7061 1.1 jmcneill
7062 1.1 jmcneill gen = sc->sc_calibration_gen;
7063 1.1 jmcneill membar_consumer();
7064 1.1 jmcneill c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
7065 1.1 jmcneill
7066 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
7067 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
7068 1.15 jmcneill
7069 1.1 jmcneill rxfree = 0;
7070 1.1 jmcneill txfree = 0;
7071 1.1 jmcneill while ((cqe = mcx_next_cq_entry(sc, cq))) {
7072 1.1 jmcneill uint8_t opcode;
7073 1.1 jmcneill opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
7074 1.1 jmcneill switch (opcode) {
7075 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_REQ:
7076 1.15 jmcneill txfree += mcx_process_txeof(sc, tx, cqe);
7077 1.1 jmcneill break;
7078 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_SEND:
7079 1.15 jmcneill rxfree += mcx_process_rx(sc, rx, cqe, &mq, c);
7080 1.1 jmcneill break;
7081 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
7082 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
7083 1.1 jmcneill /* uint8_t *cqp = (uint8_t *)cqe; */
7084 1.15 jmcneill /* printf("%s: cq completion error: %x\n",
7085 1.15 jmcneill DEVNAME(sc), cqp[0x37]); */
7086 1.1 jmcneill break;
7087 1.1 jmcneill
7088 1.1 jmcneill default:
7089 1.15 jmcneill /* printf("%s: cq completion opcode %x??\n",
7090 1.15 jmcneill DEVNAME(sc), opcode); */
7091 1.1 jmcneill break;
7092 1.1 jmcneill }
7093 1.1 jmcneill
7094 1.1 jmcneill cq->cq_cons++;
7095 1.1 jmcneill }
7096 1.1 jmcneill
7097 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
7098 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
7099 1.15 jmcneill
7100 1.1 jmcneill if (rxfree > 0) {
7101 1.15 jmcneill mcx_rxr_put(&rx->rx_rxr, rxfree);
7102 1.1 jmcneill while (MBUFQ_FIRST(&mq) != NULL) {
7103 1.1 jmcneill MBUFQ_DEQUEUE(&mq, m);
7104 1.1 jmcneill if_percpuq_enqueue(ifp->if_percpuq, m);
7105 1.1 jmcneill }
7106 1.1 jmcneill
7107 1.15 jmcneill mcx_rx_fill(sc, rx);
7108 1.15 jmcneill if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
7109 1.15 jmcneill callout_schedule(&rx->rx_refill, 1);
7110 1.1 jmcneill }
7111 1.17 jmcneill
7112 1.17 jmcneill cq->cq_count++;
7113 1.17 jmcneill mcx_arm_cq(sc, cq, q->q_uar);
7114 1.17 jmcneill
7115 1.1 jmcneill if (txfree > 0) {
7116 1.15 jmcneill tx->tx_cons += txfree;
7117 1.1 jmcneill if_schedule_deferred_start(ifp);
7118 1.1 jmcneill }
7119 1.1 jmcneill }
7120 1.1 jmcneill
7121 1.15 jmcneill
7122 1.1 jmcneill static void
7123 1.15 jmcneill mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar)
7124 1.1 jmcneill {
7125 1.1 jmcneill bus_size_t offset;
7126 1.1 jmcneill uint32_t val;
7127 1.1 jmcneill
7128 1.15 jmcneill offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM;
7129 1.15 jmcneill val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff);
7130 1.1 jmcneill
7131 1.1 jmcneill mcx_wr(sc, offset, val);
7132 1.15 jmcneill mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE);
7133 1.1 jmcneill }
7134 1.1 jmcneill
7135 1.1 jmcneill static struct mcx_eq_entry *
7136 1.15 jmcneill mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq)
7137 1.1 jmcneill {
7138 1.1 jmcneill struct mcx_eq_entry *eqe;
7139 1.1 jmcneill int next;
7140 1.1 jmcneill
7141 1.15 jmcneill eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
7142 1.15 jmcneill next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE);
7143 1.15 jmcneill if ((eqe[next].eq_owner & 1) ==
7144 1.15 jmcneill ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
7145 1.15 jmcneill eq->eq_cons++;
7146 1.1 jmcneill return (&eqe[next]);
7147 1.1 jmcneill }
7148 1.1 jmcneill return (NULL);
7149 1.1 jmcneill }
7150 1.1 jmcneill
7151 1.1 jmcneill int
7152 1.15 jmcneill mcx_admin_intr(void *xsc)
7153 1.1 jmcneill {
7154 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)xsc;
7155 1.15 jmcneill struct mcx_eq *eq = &sc->sc_admin_eq;
7156 1.1 jmcneill struct mcx_eq_entry *eqe;
7157 1.1 jmcneill
7158 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7159 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
7160 1.15 jmcneill
7161 1.15 jmcneill while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
7162 1.1 jmcneill switch (eqe->eq_event_type) {
7163 1.1 jmcneill case MCX_EVENT_TYPE_LAST_WQE:
7164 1.1 jmcneill /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
7165 1.1 jmcneill break;
7166 1.1 jmcneill
7167 1.1 jmcneill case MCX_EVENT_TYPE_CQ_ERROR:
7168 1.1 jmcneill /* printf("%s: cq error\n", DEVNAME(sc)); */
7169 1.1 jmcneill break;
7170 1.1 jmcneill
7171 1.1 jmcneill case MCX_EVENT_TYPE_CMD_COMPLETION:
7172 1.1 jmcneill /* wakeup probably */
7173 1.1 jmcneill break;
7174 1.1 jmcneill
7175 1.1 jmcneill case MCX_EVENT_TYPE_PORT_CHANGE:
7176 1.1 jmcneill workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL);
7177 1.1 jmcneill break;
7178 1.1 jmcneill
7179 1.1 jmcneill default:
7180 1.1 jmcneill /* printf("%s: something happened\n", DEVNAME(sc)); */
7181 1.1 jmcneill break;
7182 1.1 jmcneill }
7183 1.1 jmcneill }
7184 1.15 jmcneill
7185 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7186 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
7187 1.15 jmcneill
7188 1.15 jmcneill mcx_arm_eq(sc, eq, sc->sc_uar);
7189 1.15 jmcneill
7190 1.15 jmcneill return (1);
7191 1.15 jmcneill }
7192 1.15 jmcneill
7193 1.15 jmcneill int
7194 1.15 jmcneill mcx_cq_intr(void *xq)
7195 1.15 jmcneill {
7196 1.15 jmcneill struct mcx_queues *q = (struct mcx_queues *)xq;
7197 1.15 jmcneill struct mcx_softc *sc = q->q_sc;
7198 1.15 jmcneill struct mcx_eq *eq = &q->q_eq;
7199 1.15 jmcneill struct mcx_eq_entry *eqe;
7200 1.15 jmcneill int cqn;
7201 1.15 jmcneill
7202 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7203 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
7204 1.15 jmcneill
7205 1.15 jmcneill while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
7206 1.15 jmcneill switch (eqe->eq_event_type) {
7207 1.15 jmcneill case MCX_EVENT_TYPE_COMPLETION:
7208 1.15 jmcneill cqn = be32toh(eqe->eq_event_data[6]);
7209 1.15 jmcneill if (cqn == q->q_cq.cq_n)
7210 1.15 jmcneill mcx_process_cq(sc, q, &q->q_cq);
7211 1.15 jmcneill break;
7212 1.15 jmcneill }
7213 1.15 jmcneill }
7214 1.15 jmcneill
7215 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7216 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
7217 1.15 jmcneill
7218 1.15 jmcneill mcx_arm_eq(sc, eq, q->q_uar);
7219 1.15 jmcneill
7220 1.1 jmcneill return (1);
7221 1.1 jmcneill }
7222 1.1 jmcneill
7223 1.1 jmcneill static void
7224 1.1 jmcneill mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
7225 1.1 jmcneill int total)
7226 1.1 jmcneill {
7227 1.1 jmcneill struct mcx_slot *ms;
7228 1.1 jmcneill
7229 1.1 jmcneill int i = allocated;
7230 1.1 jmcneill while (i-- > 0) {
7231 1.1 jmcneill ms = &slots[i];
7232 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
7233 1.28 rin m_freem(ms->ms_m);
7234 1.1 jmcneill }
7235 1.1 jmcneill kmem_free(slots, total * sizeof(*ms));
7236 1.1 jmcneill }
7237 1.1 jmcneill
7238 1.1 jmcneill static int
7239 1.15 jmcneill mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q)
7240 1.1 jmcneill {
7241 1.15 jmcneill struct mcx_rx *rx;
7242 1.15 jmcneill struct mcx_tx *tx;
7243 1.1 jmcneill struct mcx_slot *ms;
7244 1.15 jmcneill int i;
7245 1.1 jmcneill
7246 1.15 jmcneill rx = &q->q_rx;
7247 1.15 jmcneill rx->rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE),
7248 1.1 jmcneill KM_SLEEP);
7249 1.1 jmcneill
7250 1.1 jmcneill for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
7251 1.15 jmcneill ms = &rx->rx_slots[i];
7252 1.1 jmcneill if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
7253 1.1 jmcneill sc->sc_hardmtu, 0,
7254 1.1 jmcneill BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
7255 1.1 jmcneill &ms->ms_map) != 0) {
7256 1.1 jmcneill printf("%s: failed to allocate rx dma maps\n",
7257 1.1 jmcneill DEVNAME(sc));
7258 1.1 jmcneill goto destroy_rx_slots;
7259 1.1 jmcneill }
7260 1.15 jmcneill }
7261 1.15 jmcneill
7262 1.15 jmcneill tx = &q->q_tx;
7263 1.15 jmcneill tx->tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE),
7264 1.15 jmcneill KM_SLEEP);
7265 1.15 jmcneill
7266 1.15 jmcneill for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
7267 1.15 jmcneill ms = &tx->tx_slots[i];
7268 1.15 jmcneill if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
7269 1.15 jmcneill MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
7270 1.15 jmcneill BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
7271 1.15 jmcneill &ms->ms_map) != 0) {
7272 1.15 jmcneill printf("%s: failed to allocate tx dma maps\n",
7273 1.15 jmcneill DEVNAME(sc));
7274 1.15 jmcneill goto destroy_tx_slots;
7275 1.15 jmcneill }
7276 1.15 jmcneill }
7277 1.15 jmcneill
7278 1.15 jmcneill if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index,
7279 1.15 jmcneill q->q_eq.eq_n) != 0)
7280 1.15 jmcneill goto destroy_tx_slots;
7281 1.15 jmcneill
7282 1.15 jmcneill if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n)
7283 1.15 jmcneill != 0)
7284 1.15 jmcneill goto destroy_cq;
7285 1.15 jmcneill
7286 1.15 jmcneill if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0)
7287 1.15 jmcneill goto destroy_sq;
7288 1.15 jmcneill
7289 1.15 jmcneill return 0;
7290 1.15 jmcneill
7291 1.15 jmcneill destroy_sq:
7292 1.15 jmcneill mcx_destroy_sq(sc, tx);
7293 1.15 jmcneill destroy_cq:
7294 1.15 jmcneill mcx_destroy_cq(sc, &q->q_cq);
7295 1.15 jmcneill destroy_tx_slots:
7296 1.15 jmcneill mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
7297 1.15 jmcneill tx->tx_slots = NULL;
7298 1.15 jmcneill
7299 1.15 jmcneill i = (1 << MCX_LOG_RQ_SIZE);
7300 1.15 jmcneill destroy_rx_slots:
7301 1.15 jmcneill mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
7302 1.15 jmcneill rx->rx_slots = NULL;
7303 1.15 jmcneill return ENOMEM;
7304 1.15 jmcneill }
7305 1.15 jmcneill
7306 1.15 jmcneill static int
7307 1.15 jmcneill mcx_rss_group_entry_count(struct mcx_softc *sc, int group)
7308 1.15 jmcneill {
7309 1.15 jmcneill int i;
7310 1.15 jmcneill int count;
7311 1.15 jmcneill
7312 1.15 jmcneill count = 0;
7313 1.15 jmcneill for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7314 1.15 jmcneill if (mcx_rss_config[i].flow_group == group)
7315 1.15 jmcneill count++;
7316 1.15 jmcneill }
7317 1.15 jmcneill
7318 1.15 jmcneill return count;
7319 1.15 jmcneill }
7320 1.15 jmcneill
7321 1.15 jmcneill static int
7322 1.15 jmcneill mcx_init(struct ifnet *ifp)
7323 1.15 jmcneill {
7324 1.15 jmcneill struct mcx_softc *sc = ifp->if_softc;
7325 1.15 jmcneill struct mcx_rx *rx;
7326 1.15 jmcneill struct mcx_tx *tx;
7327 1.15 jmcneill int i, start, count, flow_group, flow_index;
7328 1.15 jmcneill struct mcx_flow_match match_crit;
7329 1.15 jmcneill struct mcx_rss_rule *rss;
7330 1.15 jmcneill uint32_t dest;
7331 1.15 jmcneill int rqns[MCX_MAX_QUEUES] = { 0 };
7332 1.15 jmcneill
7333 1.15 jmcneill if (ISSET(ifp->if_flags, IFF_RUNNING))
7334 1.15 jmcneill mcx_stop(ifp, 0);
7335 1.1 jmcneill
7336 1.15 jmcneill if (mcx_create_tis(sc, &sc->sc_tis) != 0)
7337 1.15 jmcneill goto down;
7338 1.1 jmcneill
7339 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7340 1.15 jmcneill if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
7341 1.15 jmcneill goto down;
7342 1.1 jmcneill }
7343 1.1 jmcneill }
7344 1.1 jmcneill
7345 1.15 jmcneill /* RSS flow table and flow groups */
7346 1.15 jmcneill if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1,
7347 1.15 jmcneill &sc->sc_rss_flow_table_id) != 0)
7348 1.1 jmcneill goto down;
7349 1.1 jmcneill
7350 1.15 jmcneill dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
7351 1.15 jmcneill sc->sc_rss_flow_table_id;
7352 1.1 jmcneill
7353 1.15 jmcneill /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */
7354 1.15 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7355 1.15 jmcneill match_crit.mc_ethertype = 0xffff;
7356 1.15 jmcneill match_crit.mc_ip_proto = 0xff;
7357 1.15 jmcneill match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG;
7358 1.15 jmcneill start = 0;
7359 1.15 jmcneill count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4);
7360 1.15 jmcneill if (count != 0) {
7361 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7362 1.15 jmcneill MCX_FLOW_GROUP_RSS_L4, start, count,
7363 1.15 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7364 1.15 jmcneill goto down;
7365 1.15 jmcneill start += count;
7366 1.15 jmcneill }
7367 1.1 jmcneill
7368 1.15 jmcneill /* L3 RSS flow group (v4/v6, including fragments) */
7369 1.15 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7370 1.15 jmcneill match_crit.mc_ethertype = 0xffff;
7371 1.15 jmcneill count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3);
7372 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7373 1.15 jmcneill MCX_FLOW_GROUP_RSS_L3, start, count,
7374 1.15 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7375 1.1 jmcneill goto down;
7376 1.15 jmcneill start += count;
7377 1.1 jmcneill
7378 1.15 jmcneill /* non-RSS flow group */
7379 1.15 jmcneill count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE);
7380 1.15 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7381 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7382 1.15 jmcneill MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0)
7383 1.1 jmcneill goto down;
7384 1.1 jmcneill
7385 1.15 jmcneill /* Root flow table, matching packets based on mac address */
7386 1.15 jmcneill if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0,
7387 1.15 jmcneill &sc->sc_mac_flow_table_id) != 0)
7388 1.1 jmcneill goto down;
7389 1.1 jmcneill
7390 1.1 jmcneill /* promisc flow group */
7391 1.1 jmcneill start = 0;
7392 1.1 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7393 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7394 1.15 jmcneill MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0)
7395 1.1 jmcneill goto down;
7396 1.1 jmcneill sc->sc_promisc_flow_enabled = 0;
7397 1.1 jmcneill start++;
7398 1.1 jmcneill
7399 1.1 jmcneill /* all multicast flow group */
7400 1.1 jmcneill match_crit.mc_dest_mac[0] = 0x01;
7401 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7402 1.15 jmcneill MCX_FLOW_GROUP_ALLMULTI, start, 1,
7403 1.1 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7404 1.1 jmcneill goto down;
7405 1.1 jmcneill sc->sc_allmulti_flow_enabled = 0;
7406 1.1 jmcneill start++;
7407 1.1 jmcneill
7408 1.1 jmcneill /* mac address matching flow group */
7409 1.1 jmcneill memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
7410 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7411 1.15 jmcneill MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
7412 1.1 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7413 1.1 jmcneill goto down;
7414 1.1 jmcneill
7415 1.1 jmcneill /* flow table entries for unicast and broadcast */
7416 1.1 jmcneill start = 0;
7417 1.15 jmcneill if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
7418 1.15 jmcneill LLADDR(satosdl(ifp->if_dl->ifa_addr)), dest) != 0)
7419 1.1 jmcneill goto down;
7420 1.1 jmcneill start++;
7421 1.1 jmcneill
7422 1.15 jmcneill if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
7423 1.15 jmcneill etherbroadcastaddr, dest) != 0)
7424 1.1 jmcneill goto down;
7425 1.1 jmcneill start++;
7426 1.1 jmcneill
7427 1.1 jmcneill /* multicast entries go after that */
7428 1.1 jmcneill sc->sc_mcast_flow_base = start;
7429 1.1 jmcneill
7430 1.1 jmcneill /* re-add any existing multicast flows */
7431 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7432 1.1 jmcneill if (sc->sc_mcast_flows[i][0] != 0) {
7433 1.15 jmcneill mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC,
7434 1.1 jmcneill sc->sc_mcast_flow_base + i,
7435 1.15 jmcneill sc->sc_mcast_flows[i], dest);
7436 1.1 jmcneill }
7437 1.1 jmcneill }
7438 1.1 jmcneill
7439 1.15 jmcneill if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0)
7440 1.1 jmcneill goto down;
7441 1.1 jmcneill
7442 1.15 jmcneill /*
7443 1.15 jmcneill * the RQT can be any size as long as it's a power of two.
7444 1.15 jmcneill * since we also restrict the number of queues to a power of two,
7445 1.15 jmcneill * we can just put each rx queue in once.
7446 1.15 jmcneill */
7447 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++)
7448 1.15 jmcneill rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;
7449 1.15 jmcneill
7450 1.15 jmcneill if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0)
7451 1.1 jmcneill goto down;
7452 1.1 jmcneill
7453 1.15 jmcneill start = 0;
7454 1.15 jmcneill flow_index = 0;
7455 1.15 jmcneill flow_group = -1;
7456 1.15 jmcneill for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7457 1.15 jmcneill rss = &mcx_rss_config[i];
7458 1.15 jmcneill if (rss->flow_group != flow_group) {
7459 1.15 jmcneill flow_group = rss->flow_group;
7460 1.15 jmcneill flow_index = 0;
7461 1.15 jmcneill }
7462 1.15 jmcneill
7463 1.15 jmcneill if (rss->hash_sel == 0) {
7464 1.15 jmcneill if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx,
7465 1.15 jmcneill &sc->sc_tir[i]) != 0)
7466 1.15 jmcneill goto down;
7467 1.15 jmcneill } else {
7468 1.15 jmcneill if (mcx_create_tir_indirect(sc, sc->sc_rqt,
7469 1.15 jmcneill rss->hash_sel, &sc->sc_tir[i]) != 0)
7470 1.15 jmcneill goto down;
7471 1.15 jmcneill }
7472 1.15 jmcneill
7473 1.15 jmcneill if (mcx_set_flow_table_entry_proto(sc, flow_group,
7474 1.15 jmcneill flow_index, rss->ethertype, rss->ip_proto,
7475 1.15 jmcneill MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0)
7476 1.15 jmcneill goto down;
7477 1.15 jmcneill flow_index++;
7478 1.15 jmcneill }
7479 1.15 jmcneill
7480 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7481 1.15 jmcneill struct mcx_queues *q = &sc->sc_queues[i];
7482 1.15 jmcneill rx = &q->q_rx;
7483 1.15 jmcneill tx = &q->q_tx;
7484 1.15 jmcneill
7485 1.15 jmcneill /* start the queues */
7486 1.15 jmcneill if (mcx_ready_sq(sc, tx) != 0)
7487 1.15 jmcneill goto down;
7488 1.15 jmcneill
7489 1.15 jmcneill if (mcx_ready_rq(sc, rx) != 0)
7490 1.15 jmcneill goto down;
7491 1.15 jmcneill
7492 1.15 jmcneill mcx_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
7493 1.15 jmcneill rx->rx_prod = 0;
7494 1.15 jmcneill mcx_rx_fill(sc, rx);
7495 1.1 jmcneill
7496 1.15 jmcneill tx->tx_cons = 0;
7497 1.15 jmcneill tx->tx_prod = 0;
7498 1.15 jmcneill }
7499 1.1 jmcneill
7500 1.1 jmcneill mcx_calibrate_first(sc);
7501 1.1 jmcneill
7502 1.1 jmcneill SET(ifp->if_flags, IFF_RUNNING);
7503 1.1 jmcneill CLR(ifp->if_flags, IFF_OACTIVE);
7504 1.1 jmcneill if_schedule_deferred_start(ifp);
7505 1.1 jmcneill
7506 1.1 jmcneill return 0;
7507 1.1 jmcneill down:
7508 1.1 jmcneill mcx_stop(ifp, 0);
7509 1.1 jmcneill return EIO;
7510 1.1 jmcneill }
7511 1.1 jmcneill
7512 1.1 jmcneill static void
7513 1.1 jmcneill mcx_stop(struct ifnet *ifp, int disable)
7514 1.1 jmcneill {
7515 1.1 jmcneill struct mcx_softc *sc = ifp->if_softc;
7516 1.15 jmcneill struct mcx_rss_rule *rss;
7517 1.15 jmcneill int group, i, flow_group, flow_index;
7518 1.1 jmcneill
7519 1.1 jmcneill CLR(ifp->if_flags, IFF_RUNNING);
7520 1.1 jmcneill
7521 1.1 jmcneill /*
7522 1.1 jmcneill * delete flow table entries first, so no packets can arrive
7523 1.1 jmcneill * after the barriers
7524 1.1 jmcneill */
7525 1.1 jmcneill if (sc->sc_promisc_flow_enabled)
7526 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
7527 1.1 jmcneill if (sc->sc_allmulti_flow_enabled)
7528 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
7529 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
7530 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
7531 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7532 1.1 jmcneill if (sc->sc_mcast_flows[i][0] != 0) {
7533 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
7534 1.1 jmcneill sc->sc_mcast_flow_base + i);
7535 1.1 jmcneill }
7536 1.1 jmcneill }
7537 1.1 jmcneill
7538 1.15 jmcneill flow_group = -1;
7539 1.15 jmcneill flow_index = 0;
7540 1.15 jmcneill for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7541 1.15 jmcneill rss = &mcx_rss_config[i];
7542 1.15 jmcneill if (rss->flow_group != flow_group) {
7543 1.15 jmcneill flow_group = rss->flow_group;
7544 1.15 jmcneill flow_index = 0;
7545 1.15 jmcneill }
7546 1.15 jmcneill
7547 1.15 jmcneill mcx_delete_flow_table_entry(sc, flow_group, flow_index);
7548 1.15 jmcneill
7549 1.15 jmcneill mcx_destroy_tir(sc, sc->sc_tir[i]);
7550 1.15 jmcneill sc->sc_tir[i] = 0;
7551 1.15 jmcneill
7552 1.15 jmcneill flow_index++;
7553 1.15 jmcneill }
7554 1.15 jmcneill
7555 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7556 1.15 jmcneill callout_halt(&sc->sc_queues[i].q_rx.rx_refill, NULL);
7557 1.15 jmcneill }
7558 1.15 jmcneill
7559 1.1 jmcneill callout_halt(&sc->sc_calibrate, NULL);
7560 1.1 jmcneill
7561 1.1 jmcneill for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
7562 1.15 jmcneill if (sc->sc_flow_group[group].g_id != -1)
7563 1.15 jmcneill mcx_destroy_flow_group(sc, group);
7564 1.15 jmcneill }
7565 1.15 jmcneill
7566 1.15 jmcneill if (sc->sc_mac_flow_table_id != -1) {
7567 1.15 jmcneill mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id);
7568 1.15 jmcneill sc->sc_mac_flow_table_id = -1;
7569 1.15 jmcneill }
7570 1.15 jmcneill if (sc->sc_rss_flow_table_id != -1) {
7571 1.15 jmcneill mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id);
7572 1.15 jmcneill sc->sc_rss_flow_table_id = -1;
7573 1.15 jmcneill }
7574 1.15 jmcneill if (sc->sc_rqt != -1) {
7575 1.15 jmcneill mcx_destroy_rqt(sc, sc->sc_rqt);
7576 1.15 jmcneill sc->sc_rqt = -1;
7577 1.15 jmcneill }
7578 1.15 jmcneill
7579 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7580 1.15 jmcneill struct mcx_queues *q = &sc->sc_queues[i];
7581 1.15 jmcneill struct mcx_rx *rx = &q->q_rx;
7582 1.15 jmcneill struct mcx_tx *tx = &q->q_tx;
7583 1.15 jmcneill struct mcx_cq *cq = &q->q_cq;
7584 1.15 jmcneill
7585 1.15 jmcneill if (rx->rx_rqn != 0)
7586 1.15 jmcneill mcx_destroy_rq(sc, rx);
7587 1.15 jmcneill
7588 1.15 jmcneill if (tx->tx_sqn != 0)
7589 1.15 jmcneill mcx_destroy_sq(sc, tx);
7590 1.15 jmcneill
7591 1.15 jmcneill if (tx->tx_slots != NULL) {
7592 1.15 jmcneill mcx_free_slots(sc, tx->tx_slots,
7593 1.15 jmcneill (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE));
7594 1.15 jmcneill tx->tx_slots = NULL;
7595 1.15 jmcneill }
7596 1.15 jmcneill if (rx->rx_slots != NULL) {
7597 1.15 jmcneill mcx_free_slots(sc, rx->rx_slots,
7598 1.15 jmcneill (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE));
7599 1.15 jmcneill rx->rx_slots = NULL;
7600 1.15 jmcneill }
7601 1.15 jmcneill
7602 1.15 jmcneill if (cq->cq_n != 0)
7603 1.15 jmcneill mcx_destroy_cq(sc, cq);
7604 1.15 jmcneill }
7605 1.15 jmcneill if (sc->sc_tis != 0) {
7606 1.15 jmcneill mcx_destroy_tis(sc, sc->sc_tis);
7607 1.15 jmcneill sc->sc_tis = 0;
7608 1.1 jmcneill }
7609 1.1 jmcneill }
7610 1.1 jmcneill
7611 1.1 jmcneill static int
7612 1.1 jmcneill mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data)
7613 1.1 jmcneill {
7614 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
7615 1.1 jmcneill struct ifreq *ifr = (struct ifreq *)data;
7616 1.7 jmcneill struct ethercom *ec = &sc->sc_ec;
7617 1.1 jmcneill uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
7618 1.7 jmcneill struct ether_multi *enm;
7619 1.7 jmcneill struct ether_multistep step;
7620 1.7 jmcneill int s, i, flags, error = 0;
7621 1.15 jmcneill uint32_t dest;
7622 1.1 jmcneill
7623 1.1 jmcneill s = splnet();
7624 1.1 jmcneill switch (cmd) {
7625 1.1 jmcneill
7626 1.1 jmcneill case SIOCADDMULTI:
7627 1.1 jmcneill if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
7628 1.1 jmcneill error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
7629 1.7 jmcneill if (error != 0) {
7630 1.7 jmcneill splx(s);
7631 1.1 jmcneill return (error);
7632 1.7 jmcneill }
7633 1.1 jmcneill
7634 1.15 jmcneill dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
7635 1.15 jmcneill sc->sc_rss_flow_table_id;
7636 1.15 jmcneill
7637 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7638 1.1 jmcneill if (sc->sc_mcast_flows[i][0] == 0) {
7639 1.1 jmcneill memcpy(sc->sc_mcast_flows[i], addrlo,
7640 1.1 jmcneill ETHER_ADDR_LEN);
7641 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_RUNNING)) {
7642 1.15 jmcneill mcx_set_flow_table_entry_mac(sc,
7643 1.1 jmcneill MCX_FLOW_GROUP_MAC,
7644 1.1 jmcneill sc->sc_mcast_flow_base + i,
7645 1.15 jmcneill sc->sc_mcast_flows[i], dest);
7646 1.1 jmcneill }
7647 1.1 jmcneill break;
7648 1.1 jmcneill }
7649 1.1 jmcneill }
7650 1.1 jmcneill
7651 1.1 jmcneill if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
7652 1.1 jmcneill if (i == MCX_NUM_MCAST_FLOWS) {
7653 1.1 jmcneill SET(ifp->if_flags, IFF_ALLMULTI);
7654 1.1 jmcneill sc->sc_extra_mcast++;
7655 1.1 jmcneill error = ENETRESET;
7656 1.1 jmcneill }
7657 1.1 jmcneill
7658 1.7 jmcneill if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN)) {
7659 1.1 jmcneill SET(ifp->if_flags, IFF_ALLMULTI);
7660 1.1 jmcneill error = ENETRESET;
7661 1.1 jmcneill }
7662 1.1 jmcneill }
7663 1.1 jmcneill }
7664 1.1 jmcneill break;
7665 1.1 jmcneill
7666 1.1 jmcneill case SIOCDELMULTI:
7667 1.1 jmcneill if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
7668 1.1 jmcneill error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
7669 1.7 jmcneill if (error != 0) {
7670 1.7 jmcneill splx(s);
7671 1.1 jmcneill return (error);
7672 1.7 jmcneill }
7673 1.1 jmcneill
7674 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7675 1.1 jmcneill if (memcmp(sc->sc_mcast_flows[i], addrlo,
7676 1.1 jmcneill ETHER_ADDR_LEN) == 0) {
7677 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_RUNNING)) {
7678 1.1 jmcneill mcx_delete_flow_table_entry(sc,
7679 1.1 jmcneill MCX_FLOW_GROUP_MAC,
7680 1.1 jmcneill sc->sc_mcast_flow_base + i);
7681 1.1 jmcneill }
7682 1.1 jmcneill sc->sc_mcast_flows[i][0] = 0;
7683 1.1 jmcneill break;
7684 1.1 jmcneill }
7685 1.1 jmcneill }
7686 1.1 jmcneill
7687 1.1 jmcneill if (i == MCX_NUM_MCAST_FLOWS)
7688 1.1 jmcneill sc->sc_extra_mcast--;
7689 1.1 jmcneill
7690 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
7691 1.7 jmcneill sc->sc_extra_mcast == 0) {
7692 1.7 jmcneill flags = 0;
7693 1.7 jmcneill ETHER_LOCK(ec);
7694 1.7 jmcneill ETHER_FIRST_MULTI(step, ec, enm);
7695 1.7 jmcneill while (enm != NULL) {
7696 1.7 jmcneill if (memcmp(enm->enm_addrlo,
7697 1.7 jmcneill enm->enm_addrhi, ETHER_ADDR_LEN)) {
7698 1.7 jmcneill SET(flags, IFF_ALLMULTI);
7699 1.7 jmcneill break;
7700 1.7 jmcneill }
7701 1.7 jmcneill ETHER_NEXT_MULTI(step, enm);
7702 1.7 jmcneill }
7703 1.7 jmcneill ETHER_UNLOCK(ec);
7704 1.7 jmcneill if (!ISSET(flags, IFF_ALLMULTI)) {
7705 1.7 jmcneill CLR(ifp->if_flags, IFF_ALLMULTI);
7706 1.7 jmcneill error = ENETRESET;
7707 1.7 jmcneill }
7708 1.1 jmcneill }
7709 1.1 jmcneill }
7710 1.1 jmcneill break;
7711 1.1 jmcneill
7712 1.1 jmcneill default:
7713 1.1 jmcneill error = ether_ioctl(ifp, cmd, data);
7714 1.1 jmcneill }
7715 1.1 jmcneill
7716 1.1 jmcneill if (error == ENETRESET) {
7717 1.1 jmcneill if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
7718 1.1 jmcneill (IFF_UP | IFF_RUNNING))
7719 1.1 jmcneill mcx_iff(sc);
7720 1.1 jmcneill error = 0;
7721 1.1 jmcneill }
7722 1.1 jmcneill splx(s);
7723 1.1 jmcneill
7724 1.1 jmcneill return (error);
7725 1.1 jmcneill }
7726 1.1 jmcneill
7727 1.1 jmcneill #if 0
7728 1.1 jmcneill static int
7729 1.1 jmcneill mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
7730 1.1 jmcneill {
7731 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
7732 1.1 jmcneill struct mcx_reg_mcia mcia;
7733 1.1 jmcneill struct mcx_reg_pmlp pmlp;
7734 1.1 jmcneill int offset, error;
7735 1.1 jmcneill
7736 1.1 jmcneill /* get module number */
7737 1.1 jmcneill memset(&pmlp, 0, sizeof(pmlp));
7738 1.1 jmcneill pmlp.rp_local_port = 1;
7739 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
7740 1.1 jmcneill sizeof(pmlp));
7741 1.1 jmcneill if (error != 0) {
7742 1.1 jmcneill printf("%s: unable to get eeprom module number\n",
7743 1.1 jmcneill DEVNAME(sc));
7744 1.1 jmcneill return error;
7745 1.1 jmcneill }
7746 1.1 jmcneill
7747 1.1 jmcneill for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
7748 1.1 jmcneill memset(&mcia, 0, sizeof(mcia));
7749 1.1 jmcneill mcia.rm_l = 0;
7750 1.1 jmcneill mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) &
7751 1.1 jmcneill MCX_PMLP_MODULE_NUM_MASK;
7752 1.1 jmcneill mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */
7753 1.1 jmcneill mcia.rm_page_num = sff->sff_page;
7754 1.1 jmcneill mcia.rm_dev_addr = htobe16(offset);
7755 1.1 jmcneill mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);
7756 1.1 jmcneill
7757 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
7758 1.1 jmcneill &mcia, sizeof(mcia));
7759 1.1 jmcneill if (error != 0) {
7760 1.1 jmcneill printf("%s: unable to read eeprom at %x\n",
7761 1.1 jmcneill DEVNAME(sc), offset);
7762 1.1 jmcneill return error;
7763 1.1 jmcneill }
7764 1.1 jmcneill
7765 1.1 jmcneill memcpy(sff->sff_data + offset, mcia.rm_data,
7766 1.1 jmcneill MCX_MCIA_EEPROM_BYTES);
7767 1.1 jmcneill }
7768 1.1 jmcneill
7769 1.1 jmcneill return 0;
7770 1.1 jmcneill }
7771 1.1 jmcneill #endif
7772 1.1 jmcneill
7773 1.1 jmcneill static int
7774 1.1 jmcneill mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
7775 1.1 jmcneill {
7776 1.1 jmcneill switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
7777 1.1 jmcneill BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
7778 1.1 jmcneill case 0:
7779 1.1 jmcneill break;
7780 1.1 jmcneill
7781 1.1 jmcneill case EFBIG:
7782 1.6 nonaka if (m_defrag(m, M_DONTWAIT) != NULL &&
7783 1.1 jmcneill bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
7784 1.1 jmcneill BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
7785 1.1 jmcneill break;
7786 1.1 jmcneill
7787 1.1 jmcneill /* FALLTHROUGH */
7788 1.1 jmcneill default:
7789 1.1 jmcneill return (1);
7790 1.1 jmcneill }
7791 1.1 jmcneill
7792 1.1 jmcneill ms->ms_m = m;
7793 1.1 jmcneill return (0);
7794 1.1 jmcneill }
7795 1.1 jmcneill
7796 1.1 jmcneill static void
7797 1.16 jmcneill mcx_send_common_locked(struct ifnet *ifp, struct mcx_tx *tx, bool is_transmit)
7798 1.1 jmcneill {
7799 1.1 jmcneill struct mcx_softc *sc = ifp->if_softc;
7800 1.1 jmcneill struct mcx_sq_entry *sq, *sqe;
7801 1.1 jmcneill struct mcx_sq_entry_seg *sqs;
7802 1.1 jmcneill struct mcx_slot *ms;
7803 1.1 jmcneill bus_dmamap_t map;
7804 1.1 jmcneill struct mbuf *m;
7805 1.1 jmcneill u_int idx, free, used;
7806 1.1 jmcneill uint64_t *bf;
7807 1.15 jmcneill uint32_t csum;
7808 1.1 jmcneill size_t bf_base;
7809 1.1 jmcneill int i, seg, nseg;
7810 1.1 jmcneill
7811 1.16 jmcneill KASSERT(mutex_owned(&tx->tx_lock));
7812 1.16 jmcneill
7813 1.16 jmcneill if ((ifp->if_flags & IFF_RUNNING) == 0)
7814 1.16 jmcneill return;
7815 1.16 jmcneill
7816 1.15 jmcneill bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
7817 1.1 jmcneill
7818 1.15 jmcneill idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE);
7819 1.15 jmcneill free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod;
7820 1.1 jmcneill
7821 1.1 jmcneill used = 0;
7822 1.1 jmcneill bf = NULL;
7823 1.15 jmcneill
7824 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
7825 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
7826 1.15 jmcneill
7827 1.15 jmcneill sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem);
7828 1.1 jmcneill
7829 1.1 jmcneill for (;;) {
7830 1.1 jmcneill if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
7831 1.1 jmcneill SET(ifp->if_flags, IFF_OACTIVE);
7832 1.1 jmcneill break;
7833 1.1 jmcneill }
7834 1.1 jmcneill
7835 1.16 jmcneill if (is_transmit) {
7836 1.16 jmcneill m = pcq_get(tx->tx_pcq);
7837 1.16 jmcneill } else {
7838 1.16 jmcneill IFQ_DEQUEUE(&ifp->if_snd, m);
7839 1.16 jmcneill }
7840 1.1 jmcneill if (m == NULL) {
7841 1.1 jmcneill break;
7842 1.1 jmcneill }
7843 1.1 jmcneill
7844 1.1 jmcneill sqe = sq + idx;
7845 1.15 jmcneill ms = &tx->tx_slots[idx];
7846 1.1 jmcneill memset(sqe, 0, sizeof(*sqe));
7847 1.1 jmcneill
7848 1.1 jmcneill /* ctrl segment */
7849 1.1 jmcneill sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
7850 1.15 jmcneill ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
7851 1.1 jmcneill /* always generate a completion event */
7852 1.1 jmcneill sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);
7853 1.1 jmcneill
7854 1.1 jmcneill /* eth segment */
7855 1.15 jmcneill csum = 0;
7856 1.15 jmcneill if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
7857 1.15 jmcneill csum |= MCX_SQE_L3_CSUM;
7858 1.15 jmcneill if (m->m_pkthdr.csum_flags &
7859 1.15 jmcneill (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6))
7860 1.15 jmcneill csum |= MCX_SQE_L4_CSUM;
7861 1.15 jmcneill sqe->sqe_mss_csum = htobe32(csum);
7862 1.1 jmcneill sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
7863 1.15 jmcneill if (vlan_has_tag(m)) {
7864 1.15 jmcneill struct ether_vlan_header *evh;
7865 1.15 jmcneill evh = (struct ether_vlan_header *)
7866 1.15 jmcneill &sqe->sqe_inline_headers;
7867 1.15 jmcneill
7868 1.15 jmcneill m_copydata(m, 0, ETHER_HDR_LEN, evh);
7869 1.15 jmcneill evh->evl_proto = evh->evl_encap_proto;
7870 1.15 jmcneill evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
7871 1.15 jmcneill evh->evl_tag = htons(vlan_get_tag(m));
7872 1.15 jmcneill m_adj(m, ETHER_HDR_LEN);
7873 1.15 jmcneill } else {
7874 1.15 jmcneill m_copydata(m, 0, MCX_SQ_INLINE_SIZE,
7875 1.15 jmcneill sqe->sqe_inline_headers);
7876 1.15 jmcneill m_adj(m, MCX_SQ_INLINE_SIZE);
7877 1.15 jmcneill }
7878 1.1 jmcneill
7879 1.1 jmcneill if (mcx_load_mbuf(sc, ms, m) != 0) {
7880 1.1 jmcneill m_freem(m);
7881 1.10 thorpej if_statinc(ifp, if_oerrors);
7882 1.1 jmcneill continue;
7883 1.1 jmcneill }
7884 1.1 jmcneill bf = (uint64_t *)sqe;
7885 1.1 jmcneill
7886 1.1 jmcneill if (ifp->if_bpf != NULL)
7887 1.1 jmcneill bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers,
7888 1.1 jmcneill MCX_SQ_INLINE_SIZE, m, BPF_D_OUT);
7889 1.1 jmcneill
7890 1.1 jmcneill map = ms->ms_map;
7891 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
7892 1.1 jmcneill BUS_DMASYNC_PREWRITE);
7893 1.1 jmcneill
7894 1.1 jmcneill sqe->sqe_ds_sq_num =
7895 1.15 jmcneill htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) |
7896 1.1 jmcneill (map->dm_nsegs + 3));
7897 1.1 jmcneill
7898 1.1 jmcneill /* data segment - first wqe has one segment */
7899 1.1 jmcneill sqs = sqe->sqe_segs;
7900 1.1 jmcneill seg = 0;
7901 1.1 jmcneill nseg = 1;
7902 1.1 jmcneill for (i = 0; i < map->dm_nsegs; i++) {
7903 1.1 jmcneill if (seg == nseg) {
7904 1.1 jmcneill /* next slot */
7905 1.1 jmcneill idx++;
7906 1.1 jmcneill if (idx == (1 << MCX_LOG_SQ_SIZE))
7907 1.1 jmcneill idx = 0;
7908 1.15 jmcneill tx->tx_prod++;
7909 1.1 jmcneill used++;
7910 1.1 jmcneill
7911 1.1 jmcneill sqs = (struct mcx_sq_entry_seg *)(sq + idx);
7912 1.1 jmcneill seg = 0;
7913 1.1 jmcneill nseg = MCX_SQ_SEGS_PER_SLOT;
7914 1.1 jmcneill }
7915 1.1 jmcneill sqs[seg].sqs_byte_count =
7916 1.1 jmcneill htobe32(map->dm_segs[i].ds_len);
7917 1.1 jmcneill sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
7918 1.1 jmcneill sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
7919 1.1 jmcneill seg++;
7920 1.1 jmcneill }
7921 1.1 jmcneill
7922 1.1 jmcneill idx++;
7923 1.1 jmcneill if (idx == (1 << MCX_LOG_SQ_SIZE))
7924 1.1 jmcneill idx = 0;
7925 1.15 jmcneill tx->tx_prod++;
7926 1.1 jmcneill used++;
7927 1.1 jmcneill }
7928 1.1 jmcneill
7929 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
7930 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
7931 1.15 jmcneill
7932 1.1 jmcneill if (used) {
7933 1.15 jmcneill bus_size_t blueflame;
7934 1.1 jmcneill
7935 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7936 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
7937 1.15 jmcneill be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell),
7938 1.15 jmcneill tx->tx_prod & MCX_WQ_DOORBELL_MASK);
7939 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7940 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
7941 1.1 jmcneill
7942 1.1 jmcneill /*
7943 1.1 jmcneill * write the first 64 bits of the last sqe we produced
7944 1.1 jmcneill * to the blue flame buffer
7945 1.1 jmcneill */
7946 1.15 jmcneill
7947 1.15 jmcneill blueflame = bf_base + tx->tx_bf_offset;
7948 1.1 jmcneill bus_space_write_8(sc->sc_memt, sc->sc_memh,
7949 1.15 jmcneill blueflame, *bf);
7950 1.15 jmcneill mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE);
7951 1.15 jmcneill
7952 1.1 jmcneill /* next write goes to the other buffer */
7953 1.15 jmcneill tx->tx_bf_offset ^= sc->sc_bf_size;
7954 1.1 jmcneill }
7955 1.1 jmcneill }
7956 1.1 jmcneill
7957 1.1 jmcneill static void
7958 1.16 jmcneill mcx_start(struct ifnet *ifp)
7959 1.16 jmcneill {
7960 1.16 jmcneill struct mcx_softc *sc = ifp->if_softc;
7961 1.16 jmcneill /* mcx_start() always uses TX ring[0] */
7962 1.16 jmcneill struct mcx_tx *tx = &sc->sc_queues[0].q_tx;
7963 1.16 jmcneill
7964 1.16 jmcneill mutex_enter(&tx->tx_lock);
7965 1.16 jmcneill if (!ISSET(ifp->if_flags, IFF_OACTIVE)) {
7966 1.16 jmcneill mcx_send_common_locked(ifp, tx, false);
7967 1.16 jmcneill }
7968 1.16 jmcneill mutex_exit(&tx->tx_lock);
7969 1.16 jmcneill }
7970 1.16 jmcneill
7971 1.16 jmcneill static int
7972 1.16 jmcneill mcx_transmit(struct ifnet *ifp, struct mbuf *m)
7973 1.16 jmcneill {
7974 1.16 jmcneill struct mcx_softc *sc = ifp->if_softc;
7975 1.16 jmcneill struct mcx_tx *tx;
7976 1.16 jmcneill
7977 1.16 jmcneill tx = &sc->sc_queues[cpu_index(curcpu()) % sc->sc_nqueues].q_tx;
7978 1.16 jmcneill if (__predict_false(!pcq_put(tx->tx_pcq, m))) {
7979 1.16 jmcneill m_freem(m);
7980 1.16 jmcneill return ENOBUFS;
7981 1.16 jmcneill }
7982 1.16 jmcneill
7983 1.16 jmcneill if (mutex_tryenter(&tx->tx_lock)) {
7984 1.16 jmcneill mcx_send_common_locked(ifp, tx, true);
7985 1.16 jmcneill mutex_exit(&tx->tx_lock);
7986 1.16 jmcneill } else {
7987 1.16 jmcneill softint_schedule(tx->tx_softint);
7988 1.16 jmcneill }
7989 1.16 jmcneill
7990 1.16 jmcneill return 0;
7991 1.16 jmcneill }
7992 1.16 jmcneill
7993 1.16 jmcneill static void
7994 1.16 jmcneill mcx_deferred_transmit(void *arg)
7995 1.16 jmcneill {
7996 1.16 jmcneill struct mcx_tx *tx = arg;
7997 1.16 jmcneill struct mcx_softc *sc = tx->tx_softc;
7998 1.16 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
7999 1.16 jmcneill
8000 1.16 jmcneill mutex_enter(&tx->tx_lock);
8001 1.16 jmcneill if (pcq_peek(tx->tx_pcq) != NULL) {
8002 1.16 jmcneill mcx_send_common_locked(ifp, tx, true);
8003 1.16 jmcneill }
8004 1.16 jmcneill mutex_exit(&tx->tx_lock);
8005 1.16 jmcneill }
8006 1.16 jmcneill
8007 1.1 jmcneill
8008 1.1 jmcneill static void
8009 1.1 jmcneill mcx_media_add_types(struct mcx_softc *sc)
8010 1.1 jmcneill {
8011 1.1 jmcneill struct mcx_reg_ptys ptys;
8012 1.1 jmcneill int i;
8013 1.1 jmcneill uint32_t proto_cap;
8014 1.1 jmcneill
8015 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8016 1.1 jmcneill ptys.rp_local_port = 1;
8017 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8018 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8019 1.1 jmcneill sizeof(ptys)) != 0) {
8020 1.1 jmcneill printf("%s: unable to read port type/speed\n", DEVNAME(sc));
8021 1.1 jmcneill return;
8022 1.1 jmcneill }
8023 1.1 jmcneill
8024 1.1 jmcneill proto_cap = be32toh(ptys.rp_eth_proto_cap);
8025 1.1 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8026 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8027 1.13 jmcneill if (!ISSET(proto_cap, 1U << i))
8028 1.13 jmcneill continue;
8029 1.13 jmcneill
8030 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8031 1.13 jmcneill if (cap->cap_media == 0)
8032 1.13 jmcneill continue;
8033 1.13 jmcneill
8034 1.13 jmcneill ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
8035 1.1 jmcneill }
8036 1.1 jmcneill }
8037 1.1 jmcneill
8038 1.1 jmcneill static void
8039 1.1 jmcneill mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
8040 1.1 jmcneill {
8041 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
8042 1.1 jmcneill struct mcx_reg_ptys ptys;
8043 1.1 jmcneill int i;
8044 1.13 jmcneill uint32_t proto_oper;
8045 1.1 jmcneill uint64_t media_oper;
8046 1.1 jmcneill
8047 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8048 1.1 jmcneill ptys.rp_local_port = 1;
8049 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8050 1.1 jmcneill
8051 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8052 1.1 jmcneill sizeof(ptys)) != 0) {
8053 1.1 jmcneill printf("%s: unable to read port type/speed\n", DEVNAME(sc));
8054 1.1 jmcneill return;
8055 1.1 jmcneill }
8056 1.1 jmcneill
8057 1.1 jmcneill proto_oper = be32toh(ptys.rp_eth_proto_oper);
8058 1.1 jmcneill
8059 1.1 jmcneill media_oper = 0;
8060 1.13 jmcneill
8061 1.1 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8062 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8063 1.13 jmcneill if (!ISSET(proto_oper, 1U << i))
8064 1.13 jmcneill continue;
8065 1.13 jmcneill
8066 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8067 1.13 jmcneill
8068 1.13 jmcneill if (cap->cap_media != 0)
8069 1.13 jmcneill media_oper = cap->cap_media;
8070 1.1 jmcneill }
8071 1.1 jmcneill
8072 1.1 jmcneill ifmr->ifm_status = IFM_AVALID;
8073 1.1 jmcneill if (proto_oper != 0) {
8074 1.1 jmcneill ifmr->ifm_status |= IFM_ACTIVE;
8075 1.27 andvar ifmr->ifm_active = IFM_ETHER | IFM_FDX | IFM_AUTO | media_oper;
8076 1.1 jmcneill /* txpause, rxpause, duplex? */
8077 1.1 jmcneill }
8078 1.1 jmcneill }
8079 1.1 jmcneill
8080 1.1 jmcneill static int
8081 1.1 jmcneill mcx_media_change(struct ifnet *ifp)
8082 1.1 jmcneill {
8083 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
8084 1.1 jmcneill struct mcx_reg_ptys ptys;
8085 1.1 jmcneill struct mcx_reg_paos paos;
8086 1.1 jmcneill uint32_t media;
8087 1.1 jmcneill int i, error;
8088 1.1 jmcneill
8089 1.1 jmcneill if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
8090 1.1 jmcneill return EINVAL;
8091 1.1 jmcneill
8092 1.1 jmcneill error = 0;
8093 1.1 jmcneill
8094 1.1 jmcneill if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
8095 1.1 jmcneill /* read ptys to get supported media */
8096 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8097 1.1 jmcneill ptys.rp_local_port = 1;
8098 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8099 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
8100 1.1 jmcneill &ptys, sizeof(ptys)) != 0) {
8101 1.1 jmcneill printf("%s: unable to read port type/speed\n",
8102 1.1 jmcneill DEVNAME(sc));
8103 1.1 jmcneill return EIO;
8104 1.1 jmcneill }
8105 1.1 jmcneill
8106 1.1 jmcneill media = be32toh(ptys.rp_eth_proto_cap);
8107 1.1 jmcneill } else {
8108 1.1 jmcneill /* map media type */
8109 1.1 jmcneill media = 0;
8110 1.1 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8111 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8112 1.13 jmcneill
8113 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8114 1.13 jmcneill if (cap->cap_media ==
8115 1.1 jmcneill IFM_SUBTYPE(sc->sc_media.ifm_media)) {
8116 1.1 jmcneill media = (1 << i);
8117 1.1 jmcneill break;
8118 1.1 jmcneill }
8119 1.1 jmcneill }
8120 1.1 jmcneill }
8121 1.1 jmcneill
8122 1.1 jmcneill /* disable the port */
8123 1.1 jmcneill memset(&paos, 0, sizeof(paos));
8124 1.1 jmcneill paos.rp_local_port = 1;
8125 1.1 jmcneill paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
8126 1.1 jmcneill paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
8127 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
8128 1.1 jmcneill sizeof(paos)) != 0) {
8129 1.1 jmcneill printf("%s: unable to set port state to down\n", DEVNAME(sc));
8130 1.1 jmcneill return EIO;
8131 1.1 jmcneill }
8132 1.1 jmcneill
8133 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8134 1.1 jmcneill ptys.rp_local_port = 1;
8135 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8136 1.1 jmcneill ptys.rp_eth_proto_admin = htobe32(media);
8137 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
8138 1.1 jmcneill sizeof(ptys)) != 0) {
8139 1.1 jmcneill printf("%s: unable to set port media type/speed\n",
8140 1.1 jmcneill DEVNAME(sc));
8141 1.1 jmcneill error = EIO;
8142 1.1 jmcneill }
8143 1.1 jmcneill
8144 1.1 jmcneill /* re-enable the port to start negotiation */
8145 1.1 jmcneill memset(&paos, 0, sizeof(paos));
8146 1.1 jmcneill paos.rp_local_port = 1;
8147 1.1 jmcneill paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
8148 1.1 jmcneill paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
8149 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
8150 1.1 jmcneill sizeof(paos)) != 0) {
8151 1.1 jmcneill printf("%s: unable to set port state to up\n", DEVNAME(sc));
8152 1.1 jmcneill error = EIO;
8153 1.1 jmcneill }
8154 1.1 jmcneill
8155 1.1 jmcneill return error;
8156 1.1 jmcneill }
8157 1.1 jmcneill
8158 1.1 jmcneill static void
8159 1.1 jmcneill mcx_port_change(struct work *wk, void *xsc)
8160 1.1 jmcneill {
8161 1.1 jmcneill struct mcx_softc *sc = xsc;
8162 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
8163 1.13 jmcneill struct mcx_reg_ptys ptys = {
8164 1.13 jmcneill .rp_local_port = 1,
8165 1.13 jmcneill .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH,
8166 1.13 jmcneill };
8167 1.1 jmcneill int link_state = LINK_STATE_DOWN;
8168 1.1 jmcneill
8169 1.13 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8170 1.13 jmcneill sizeof(ptys)) == 0) {
8171 1.13 jmcneill uint32_t proto_oper = be32toh(ptys.rp_eth_proto_oper);
8172 1.13 jmcneill uint64_t baudrate = 0;
8173 1.13 jmcneill unsigned int i;
8174 1.13 jmcneill
8175 1.15 jmcneill if (proto_oper != 0)
8176 1.15 jmcneill link_state = LINK_STATE_UP;
8177 1.15 jmcneill
8178 1.13 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8179 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8180 1.13 jmcneill if (!ISSET(proto_oper, 1U << i))
8181 1.13 jmcneill continue;
8182 1.13 jmcneill
8183 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8184 1.13 jmcneill if (cap->cap_baudrate == 0)
8185 1.13 jmcneill continue;
8186 1.13 jmcneill
8187 1.13 jmcneill baudrate = cap->cap_baudrate;
8188 1.13 jmcneill break;
8189 1.13 jmcneill }
8190 1.13 jmcneill
8191 1.13 jmcneill ifp->if_baudrate = baudrate;
8192 1.1 jmcneill }
8193 1.1 jmcneill
8194 1.1 jmcneill if (link_state != ifp->if_link_state) {
8195 1.1 jmcneill if_link_state_change(ifp, link_state);
8196 1.1 jmcneill }
8197 1.1 jmcneill }
8198 1.1 jmcneill
8199 1.1 jmcneill
8200 1.1 jmcneill static inline uint32_t
8201 1.1 jmcneill mcx_rd(struct mcx_softc *sc, bus_size_t r)
8202 1.1 jmcneill {
8203 1.1 jmcneill uint32_t word;
8204 1.1 jmcneill
8205 1.1 jmcneill word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r);
8206 1.1 jmcneill
8207 1.1 jmcneill return (be32toh(word));
8208 1.1 jmcneill }
8209 1.1 jmcneill
8210 1.1 jmcneill static inline void
8211 1.1 jmcneill mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
8212 1.1 jmcneill {
8213 1.1 jmcneill bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
8214 1.1 jmcneill }
8215 1.1 jmcneill
8216 1.1 jmcneill static inline void
8217 1.1 jmcneill mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
8218 1.1 jmcneill {
8219 1.21 jmcneill #ifndef __NetBSD__
8220 1.1 jmcneill bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
8221 1.21 jmcneill #endif
8222 1.1 jmcneill }
8223 1.1 jmcneill
8224 1.1 jmcneill static uint64_t
8225 1.1 jmcneill mcx_timer(struct mcx_softc *sc)
8226 1.1 jmcneill {
8227 1.1 jmcneill uint32_t hi, lo, ni;
8228 1.1 jmcneill
8229 1.1 jmcneill hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
8230 1.1 jmcneill for (;;) {
8231 1.1 jmcneill lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
8232 1.1 jmcneill mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
8233 1.1 jmcneill ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
8234 1.1 jmcneill
8235 1.1 jmcneill if (ni == hi)
8236 1.1 jmcneill break;
8237 1.1 jmcneill
8238 1.1 jmcneill hi = ni;
8239 1.1 jmcneill }
8240 1.1 jmcneill
8241 1.1 jmcneill return (((uint64_t)hi << 32) | (uint64_t)lo);
8242 1.1 jmcneill }
8243 1.1 jmcneill
8244 1.1 jmcneill static int
8245 1.1 jmcneill mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
8246 1.1 jmcneill bus_size_t size, u_int align)
8247 1.1 jmcneill {
8248 1.1 jmcneill mxm->mxm_size = size;
8249 1.1 jmcneill
8250 1.1 jmcneill if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
8251 1.1 jmcneill mxm->mxm_size, 0,
8252 1.1 jmcneill BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
8253 1.1 jmcneill &mxm->mxm_map) != 0)
8254 1.1 jmcneill return (1);
8255 1.1 jmcneill if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
8256 1.1 jmcneill align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
8257 1.1 jmcneill BUS_DMA_WAITOK) != 0)
8258 1.1 jmcneill goto destroy;
8259 1.1 jmcneill if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
8260 1.19 jmcneill mxm->mxm_size, &mxm->mxm_kva,
8261 1.19 jmcneill BUS_DMA_WAITOK | BUS_DMA_COHERENT) != 0)
8262 1.1 jmcneill goto free;
8263 1.1 jmcneill if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
8264 1.1 jmcneill mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
8265 1.1 jmcneill goto unmap;
8266 1.1 jmcneill
8267 1.1 jmcneill mcx_dmamem_zero(mxm);
8268 1.1 jmcneill
8269 1.1 jmcneill return (0);
8270 1.1 jmcneill unmap:
8271 1.1 jmcneill bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
8272 1.1 jmcneill free:
8273 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
8274 1.1 jmcneill destroy:
8275 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
8276 1.1 jmcneill return (1);
8277 1.1 jmcneill }
8278 1.1 jmcneill
8279 1.1 jmcneill static void
8280 1.1 jmcneill mcx_dmamem_zero(struct mcx_dmamem *mxm)
8281 1.1 jmcneill {
8282 1.1 jmcneill memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
8283 1.1 jmcneill }
8284 1.1 jmcneill
8285 1.1 jmcneill static void
8286 1.1 jmcneill mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
8287 1.1 jmcneill {
8288 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
8289 1.1 jmcneill bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
8290 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
8291 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
8292 1.1 jmcneill }
8293 1.1 jmcneill
8294 1.1 jmcneill static int
8295 1.1 jmcneill mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
8296 1.1 jmcneill {
8297 1.1 jmcneill bus_dma_segment_t *segs;
8298 1.1 jmcneill bus_size_t len = pages * MCX_PAGE_SIZE;
8299 1.1 jmcneill size_t seglen;
8300 1.1 jmcneill
8301 1.1 jmcneill segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP);
8302 1.1 jmcneill seglen = sizeof(*segs) * pages;
8303 1.1 jmcneill
8304 1.1 jmcneill if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
8305 1.1 jmcneill segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
8306 1.1 jmcneill goto free_segs;
8307 1.1 jmcneill
8308 1.1 jmcneill if (mhm->mhm_seg_count < pages) {
8309 1.1 jmcneill size_t nseglen;
8310 1.1 jmcneill
8311 1.1 jmcneill mhm->mhm_segs = kmem_alloc(
8312 1.1 jmcneill sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP);
8313 1.1 jmcneill
8314 1.1 jmcneill nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
8315 1.1 jmcneill
8316 1.1 jmcneill memcpy(mhm->mhm_segs, segs, nseglen);
8317 1.1 jmcneill
8318 1.1 jmcneill kmem_free(segs, seglen);
8319 1.1 jmcneill
8320 1.1 jmcneill segs = mhm->mhm_segs;
8321 1.1 jmcneill seglen = nseglen;
8322 1.1 jmcneill } else
8323 1.1 jmcneill mhm->mhm_segs = segs;
8324 1.1 jmcneill
8325 1.1 jmcneill if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
8326 1.1 jmcneill MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
8327 1.1 jmcneill &mhm->mhm_map) != 0)
8328 1.1 jmcneill goto free_dmamem;
8329 1.1 jmcneill
8330 1.1 jmcneill if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
8331 1.1 jmcneill mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
8332 1.1 jmcneill goto destroy;
8333 1.1 jmcneill
8334 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
8335 1.1 jmcneill 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
8336 1.1 jmcneill
8337 1.1 jmcneill mhm->mhm_npages = pages;
8338 1.1 jmcneill
8339 1.1 jmcneill return (0);
8340 1.1 jmcneill
8341 1.1 jmcneill destroy:
8342 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
8343 1.1 jmcneill free_dmamem:
8344 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
8345 1.1 jmcneill free_segs:
8346 1.1 jmcneill kmem_free(segs, seglen);
8347 1.1 jmcneill mhm->mhm_segs = NULL;
8348 1.1 jmcneill
8349 1.1 jmcneill return (-1);
8350 1.1 jmcneill }
8351 1.1 jmcneill
8352 1.1 jmcneill static void
8353 1.1 jmcneill mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
8354 1.1 jmcneill {
8355 1.1 jmcneill if (mhm->mhm_npages == 0)
8356 1.1 jmcneill return;
8357 1.1 jmcneill
8358 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
8359 1.1 jmcneill 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
8360 1.1 jmcneill
8361 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
8362 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
8363 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
8364 1.1 jmcneill kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
8365 1.1 jmcneill
8366 1.1 jmcneill mhm->mhm_npages = 0;
8367 1.1 jmcneill }
8368 1.15 jmcneill
8369 1.15 jmcneill #if NKSTAT > 0
8370 1.15 jmcneill struct mcx_ppcnt {
8371 1.15 jmcneill char name[KSTAT_KV_NAMELEN];
8372 1.15 jmcneill enum kstat_kv_unit unit;
8373 1.15 jmcneill };
8374 1.15 jmcneill
8375 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = {
8376 1.15 jmcneill { "Good Tx", KSTAT_KV_U_PACKETS, },
8377 1.15 jmcneill { "Good Rx", KSTAT_KV_U_PACKETS, },
8378 1.15 jmcneill { "FCS errs", KSTAT_KV_U_PACKETS, },
8379 1.15 jmcneill { "Alignment Errs", KSTAT_KV_U_PACKETS, },
8380 1.15 jmcneill { "Good Tx", KSTAT_KV_U_BYTES, },
8381 1.15 jmcneill { "Good Rx", KSTAT_KV_U_BYTES, },
8382 1.15 jmcneill { "Multicast Tx", KSTAT_KV_U_PACKETS, },
8383 1.15 jmcneill { "Broadcast Tx", KSTAT_KV_U_PACKETS, },
8384 1.15 jmcneill { "Multicast Rx", KSTAT_KV_U_PACKETS, },
8385 1.15 jmcneill { "Broadcast Rx", KSTAT_KV_U_PACKETS, },
8386 1.15 jmcneill { "In Range Len", KSTAT_KV_U_PACKETS, },
8387 1.15 jmcneill { "Out Of Range Len", KSTAT_KV_U_PACKETS, },
8388 1.15 jmcneill { "Frame Too Long", KSTAT_KV_U_PACKETS, },
8389 1.15 jmcneill { "Symbol Errs", KSTAT_KV_U_PACKETS, },
8390 1.15 jmcneill { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, },
8391 1.15 jmcneill { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, },
8392 1.15 jmcneill { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, },
8393 1.15 jmcneill { "Pause Rx", KSTAT_KV_U_PACKETS, },
8394 1.15 jmcneill { "Pause Tx", KSTAT_KV_U_PACKETS, },
8395 1.15 jmcneill };
8396 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count);
8397 1.15 jmcneill
8398 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = {
8399 1.15 jmcneill { "Rx Bytes", KSTAT_KV_U_BYTES, },
8400 1.15 jmcneill { "Rx Unicast", KSTAT_KV_U_PACKETS, },
8401 1.15 jmcneill { "Rx Discards", KSTAT_KV_U_PACKETS, },
8402 1.15 jmcneill { "Rx Errors", KSTAT_KV_U_PACKETS, },
8403 1.15 jmcneill { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, },
8404 1.15 jmcneill { "Tx Bytes", KSTAT_KV_U_BYTES, },
8405 1.15 jmcneill { "Tx Unicast", KSTAT_KV_U_PACKETS, },
8406 1.15 jmcneill { "Tx Discards", KSTAT_KV_U_PACKETS, },
8407 1.15 jmcneill { "Tx Errors", KSTAT_KV_U_PACKETS, },
8408 1.15 jmcneill { "Rx Multicast", KSTAT_KV_U_PACKETS, },
8409 1.15 jmcneill { "Rx Broadcast", KSTAT_KV_U_PACKETS, },
8410 1.15 jmcneill { "Tx Multicast", KSTAT_KV_U_PACKETS, },
8411 1.15 jmcneill { "Tx Broadcast", KSTAT_KV_U_PACKETS, },
8412 1.15 jmcneill };
8413 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count);
8414 1.15 jmcneill
8415 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = {
8416 1.15 jmcneill { "Drop Events", KSTAT_KV_U_PACKETS, },
8417 1.15 jmcneill { "Octets", KSTAT_KV_U_BYTES, },
8418 1.15 jmcneill { "Packets", KSTAT_KV_U_PACKETS, },
8419 1.15 jmcneill { "Broadcasts", KSTAT_KV_U_PACKETS, },
8420 1.15 jmcneill { "Multicasts", KSTAT_KV_U_PACKETS, },
8421 1.15 jmcneill { "CRC Align Errs", KSTAT_KV_U_PACKETS, },
8422 1.15 jmcneill { "Undersize", KSTAT_KV_U_PACKETS, },
8423 1.15 jmcneill { "Oversize", KSTAT_KV_U_PACKETS, },
8424 1.15 jmcneill { "Fragments", KSTAT_KV_U_PACKETS, },
8425 1.15 jmcneill { "Jabbers", KSTAT_KV_U_PACKETS, },
8426 1.15 jmcneill { "Collisions", KSTAT_KV_U_NONE, },
8427 1.15 jmcneill { "64B", KSTAT_KV_U_PACKETS, },
8428 1.15 jmcneill { "65-127B", KSTAT_KV_U_PACKETS, },
8429 1.15 jmcneill { "128-255B", KSTAT_KV_U_PACKETS, },
8430 1.15 jmcneill { "256-511B", KSTAT_KV_U_PACKETS, },
8431 1.15 jmcneill { "512-1023B", KSTAT_KV_U_PACKETS, },
8432 1.15 jmcneill { "1024-1518B", KSTAT_KV_U_PACKETS, },
8433 1.15 jmcneill { "1519-2047B", KSTAT_KV_U_PACKETS, },
8434 1.15 jmcneill { "2048-4095B", KSTAT_KV_U_PACKETS, },
8435 1.15 jmcneill { "4096-8191B", KSTAT_KV_U_PACKETS, },
8436 1.15 jmcneill { "8192-10239B", KSTAT_KV_U_PACKETS, },
8437 1.15 jmcneill };
8438 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count);
8439 1.15 jmcneill
8440 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = {
8441 1.15 jmcneill { "Alignment Errs", KSTAT_KV_U_PACKETS, },
8442 1.15 jmcneill { "FCS Errs", KSTAT_KV_U_PACKETS, },
8443 1.15 jmcneill { "Single Colls", KSTAT_KV_U_PACKETS, },
8444 1.15 jmcneill { "Multiple Colls", KSTAT_KV_U_PACKETS, },
8445 1.15 jmcneill { "SQE Test Errs", KSTAT_KV_U_NONE, },
8446 1.15 jmcneill { "Deferred Tx", KSTAT_KV_U_PACKETS, },
8447 1.15 jmcneill { "Late Colls", KSTAT_KV_U_NONE, },
8448 1.15 jmcneill { "Exess Colls", KSTAT_KV_U_NONE, },
8449 1.15 jmcneill { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, },
8450 1.15 jmcneill { "CSM Sense Errs", KSTAT_KV_U_NONE, },
8451 1.15 jmcneill { "Too Long", KSTAT_KV_U_PACKETS, },
8452 1.15 jmcneill { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, },
8453 1.15 jmcneill { "Symbol Errs", KSTAT_KV_U_NONE, },
8454 1.15 jmcneill { "Unknown Control", KSTAT_KV_U_PACKETS, },
8455 1.15 jmcneill { "Pause Rx", KSTAT_KV_U_PACKETS, },
8456 1.15 jmcneill { "Pause Tx", KSTAT_KV_U_PACKETS, },
8457 1.15 jmcneill };
8458 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count);
8459 1.15 jmcneill
8460 1.15 jmcneill struct mcx_kstat_ppcnt {
8461 1.15 jmcneill const char *ksp_name;
8462 1.15 jmcneill const struct mcx_ppcnt *ksp_tpl;
8463 1.15 jmcneill unsigned int ksp_n;
8464 1.15 jmcneill uint8_t ksp_grp;
8465 1.15 jmcneill };
8466 1.15 jmcneill
8467 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = {
8468 1.15 jmcneill .ksp_name = "ieee802.3",
8469 1.15 jmcneill .ksp_tpl = mcx_ppcnt_ieee8023_tpl,
8470 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_ieee8023_tpl),
8471 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023,
8472 1.15 jmcneill };
8473 1.15 jmcneill
8474 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = {
8475 1.15 jmcneill .ksp_name = "rfc2863",
8476 1.15 jmcneill .ksp_tpl = mcx_ppcnt_rfc2863_tpl,
8477 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_rfc2863_tpl),
8478 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863,
8479 1.15 jmcneill };
8480 1.15 jmcneill
8481 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = {
8482 1.15 jmcneill .ksp_name = "rfc2819",
8483 1.15 jmcneill .ksp_tpl = mcx_ppcnt_rfc2819_tpl,
8484 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_rfc2819_tpl),
8485 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819,
8486 1.15 jmcneill };
8487 1.15 jmcneill
8488 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = {
8489 1.15 jmcneill .ksp_name = "rfc3635",
8490 1.15 jmcneill .ksp_tpl = mcx_ppcnt_rfc3635_tpl,
8491 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_rfc3635_tpl),
8492 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635,
8493 1.15 jmcneill };
8494 1.15 jmcneill
8495 1.15 jmcneill static int mcx_kstat_ppcnt_read(struct kstat *);
8496 1.15 jmcneill
8497 1.15 jmcneill static void mcx_kstat_attach_tmps(struct mcx_softc *sc);
8498 1.15 jmcneill static void mcx_kstat_attach_queues(struct mcx_softc *sc);
8499 1.15 jmcneill
8500 1.15 jmcneill static struct kstat *
8501 1.15 jmcneill mcx_kstat_attach_ppcnt(struct mcx_softc *sc,
8502 1.15 jmcneill const struct mcx_kstat_ppcnt *ksp)
8503 1.15 jmcneill {
8504 1.15 jmcneill struct kstat *ks;
8505 1.15 jmcneill struct kstat_kv *kvs;
8506 1.15 jmcneill unsigned int i;
8507 1.15 jmcneill
8508 1.15 jmcneill ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0);
8509 1.15 jmcneill if (ks == NULL)
8510 1.15 jmcneill return (NULL);
8511 1.15 jmcneill
8512 1.15 jmcneill kvs = mallocarray(ksp->ksp_n, sizeof(*kvs),
8513 1.15 jmcneill M_DEVBUF, M_WAITOK);
8514 1.15 jmcneill
8515 1.15 jmcneill for (i = 0; i < ksp->ksp_n; i++) {
8516 1.15 jmcneill const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i];
8517 1.15 jmcneill
8518 1.15 jmcneill kstat_kv_unit_init(&kvs[i], tpl->name,
8519 1.15 jmcneill KSTAT_KV_T_COUNTER64, tpl->unit);
8520 1.15 jmcneill }
8521 1.15 jmcneill
8522 1.15 jmcneill ks->ks_softc = sc;
8523 1.15 jmcneill ks->ks_ptr = (void *)ksp;
8524 1.15 jmcneill ks->ks_data = kvs;
8525 1.15 jmcneill ks->ks_datalen = ksp->ksp_n * sizeof(*kvs);
8526 1.15 jmcneill ks->ks_read = mcx_kstat_ppcnt_read;
8527 1.15 jmcneill
8528 1.15 jmcneill kstat_install(ks);
8529 1.15 jmcneill
8530 1.15 jmcneill return (ks);
8531 1.15 jmcneill }
8532 1.15 jmcneill
8533 1.15 jmcneill static void
8534 1.15 jmcneill mcx_kstat_attach(struct mcx_softc *sc)
8535 1.15 jmcneill {
8536 1.15 jmcneill sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc,
8537 1.15 jmcneill &mcx_kstat_ppcnt_ieee8023);
8538 1.15 jmcneill sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc,
8539 1.15 jmcneill &mcx_kstat_ppcnt_rfc2863);
8540 1.15 jmcneill sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc,
8541 1.15 jmcneill &mcx_kstat_ppcnt_rfc2819);
8542 1.15 jmcneill sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc,
8543 1.15 jmcneill &mcx_kstat_ppcnt_rfc3635);
8544 1.15 jmcneill
8545 1.15 jmcneill mcx_kstat_attach_tmps(sc);
8546 1.15 jmcneill mcx_kstat_attach_queues(sc);
8547 1.15 jmcneill }
8548 1.15 jmcneill
8549 1.15 jmcneill static int
8550 1.15 jmcneill mcx_kstat_ppcnt_read(struct kstat *ks)
8551 1.15 jmcneill {
8552 1.15 jmcneill struct mcx_softc *sc = ks->ks_softc;
8553 1.15 jmcneill struct mcx_kstat_ppcnt *ksp = ks->ks_ptr;
8554 1.15 jmcneill struct mcx_reg_ppcnt ppcnt = {
8555 1.15 jmcneill .ppcnt_grp = ksp->ksp_grp,
8556 1.15 jmcneill .ppcnt_local_port = 1,
8557 1.15 jmcneill };
8558 1.15 jmcneill struct kstat_kv *kvs = ks->ks_data;
8559 1.15 jmcneill uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set;
8560 1.15 jmcneill unsigned int i;
8561 1.15 jmcneill int rv;
8562 1.15 jmcneill
8563 1.15 jmcneill KERNEL_LOCK(); /* XXX */
8564 1.15 jmcneill rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ,
8565 1.15 jmcneill &ppcnt, sizeof(ppcnt));
8566 1.15 jmcneill KERNEL_UNLOCK();
8567 1.15 jmcneill if (rv != 0)
8568 1.15 jmcneill return (EIO);
8569 1.15 jmcneill
8570 1.15 jmcneill nanouptime(&ks->ks_updated);
8571 1.15 jmcneill
8572 1.15 jmcneill for (i = 0; i < ksp->ksp_n; i++)
8573 1.15 jmcneill kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]);
8574 1.15 jmcneill
8575 1.15 jmcneill return (0);
8576 1.15 jmcneill }
8577 1.15 jmcneill
8578 1.15 jmcneill struct mcx_kstat_mtmp {
8579 1.15 jmcneill struct kstat_kv ktmp_name;
8580 1.15 jmcneill struct kstat_kv ktmp_temperature;
8581 1.15 jmcneill struct kstat_kv ktmp_threshold_lo;
8582 1.15 jmcneill struct kstat_kv ktmp_threshold_hi;
8583 1.15 jmcneill };
8584 1.15 jmcneill
8585 1.15 jmcneill static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = {
8586 1.15 jmcneill KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR),
8587 1.15 jmcneill KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP),
8588 1.15 jmcneill KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP),
8589 1.15 jmcneill KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP),
8590 1.15 jmcneill };
8591 1.15 jmcneill
8592 1.15 jmcneill static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 };
8593 1.15 jmcneill
8594 1.15 jmcneill static int mcx_kstat_mtmp_read(struct kstat *);
8595 1.15 jmcneill
8596 1.15 jmcneill static void
8597 1.15 jmcneill mcx_kstat_attach_tmps(struct mcx_softc *sc)
8598 1.15 jmcneill {
8599 1.15 jmcneill struct kstat *ks;
8600 1.15 jmcneill struct mcx_reg_mcam mcam;
8601 1.15 jmcneill struct mcx_reg_mtcap mtcap;
8602 1.15 jmcneill struct mcx_kstat_mtmp *ktmp;
8603 1.15 jmcneill uint64_t map;
8604 1.15 jmcneill unsigned int i, n;
8605 1.15 jmcneill
8606 1.15 jmcneill memset(&mtcap, 0, sizeof(mtcap));
8607 1.15 jmcneill memset(&mcam, 0, sizeof(mcam));
8608 1.15 jmcneill
8609 1.15 jmcneill if (sc->sc_mcam_reg == 0) {
8610 1.15 jmcneill /* no management capabilities */
8611 1.15 jmcneill return;
8612 1.15 jmcneill }
8613 1.15 jmcneill
8614 1.15 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ,
8615 1.15 jmcneill &mcam, sizeof(mcam)) != 0) {
8616 1.15 jmcneill /* unable to check management capabilities? */
8617 1.15 jmcneill return;
8618 1.15 jmcneill }
8619 1.15 jmcneill
8620 1.15 jmcneill if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask,
8621 1.15 jmcneill MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) {
8622 1.15 jmcneill /* no sensor map */
8623 1.15 jmcneill return;
8624 1.15 jmcneill }
8625 1.15 jmcneill
8626 1.15 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ,
8627 1.15 jmcneill &mtcap, sizeof(mtcap)) != 0) {
8628 1.15 jmcneill /* unable to find temperature sensors */
8629 1.15 jmcneill return;
8630 1.15 jmcneill }
8631 1.15 jmcneill
8632 1.15 jmcneill sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count;
8633 1.15 jmcneill sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count,
8634 1.15 jmcneill sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK);
8635 1.15 jmcneill
8636 1.15 jmcneill n = 0;
8637 1.15 jmcneill map = bemtoh64(&mtcap.mtcap_sensor_map);
8638 1.15 jmcneill for (i = 0; i < sizeof(map) * NBBY; i++) {
8639 1.15 jmcneill if (!ISSET(map, (1ULL << i)))
8640 1.15 jmcneill continue;
8641 1.15 jmcneill
8642 1.15 jmcneill ks = kstat_create(DEVNAME(sc), 0, "temperature", i,
8643 1.15 jmcneill KSTAT_T_KV, 0);
8644 1.15 jmcneill if (ks == NULL) {
8645 1.15 jmcneill /* unable to attach temperature sensor %u, i */
8646 1.15 jmcneill continue;
8647 1.15 jmcneill }
8648 1.15 jmcneill
8649 1.15 jmcneill ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO);
8650 1.15 jmcneill *ktmp = mcx_kstat_mtmp_tpl;
8651 1.15 jmcneill
8652 1.15 jmcneill ks->ks_data = ktmp;
8653 1.15 jmcneill ks->ks_datalen = sizeof(*ktmp);
8654 1.15 jmcneill TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval);
8655 1.15 jmcneill ks->ks_read = mcx_kstat_mtmp_read;
8656 1.15 jmcneill
8657 1.15 jmcneill ks->ks_softc = sc;
8658 1.15 jmcneill kstat_install(ks);
8659 1.15 jmcneill
8660 1.15 jmcneill sc->sc_kstat_mtmp[n++] = ks;
8661 1.15 jmcneill if (n >= sc->sc_kstat_mtmp_count)
8662 1.15 jmcneill break;
8663 1.15 jmcneill }
8664 1.15 jmcneill }
8665 1.15 jmcneill
8666 1.15 jmcneill static uint64_t
8667 1.15 jmcneill mcx_tmp_to_uK(uint16_t *t)
8668 1.15 jmcneill {
8669 1.15 jmcneill int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */
8670 1.15 jmcneill mt *= 1000000 / 8; /* convert to uC */
8671 1.15 jmcneill mt += 273150000; /* convert to uK */
8672 1.15 jmcneill
8673 1.15 jmcneill return (mt);
8674 1.15 jmcneill }
8675 1.15 jmcneill
8676 1.15 jmcneill static int
8677 1.15 jmcneill mcx_kstat_mtmp_read(struct kstat *ks)
8678 1.15 jmcneill {
8679 1.15 jmcneill struct mcx_softc *sc = ks->ks_softc;
8680 1.15 jmcneill struct mcx_kstat_mtmp *ktmp = ks->ks_data;
8681 1.15 jmcneill struct mcx_reg_mtmp mtmp;
8682 1.15 jmcneill int rv;
8683 1.15 jmcneill struct timeval updated;
8684 1.15 jmcneill
8685 1.15 jmcneill TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated);
8686 1.15 jmcneill
8687 1.15 jmcneill if (!ratecheck(&updated, &mcx_kstat_mtmp_rate))
8688 1.15 jmcneill return (0);
8689 1.15 jmcneill
8690 1.15 jmcneill memset(&mtmp, 0, sizeof(mtmp));
8691 1.15 jmcneill htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit);
8692 1.15 jmcneill
8693 1.15 jmcneill KERNEL_LOCK(); /* XXX */
8694 1.15 jmcneill rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ,
8695 1.15 jmcneill &mtmp, sizeof(mtmp));
8696 1.15 jmcneill KERNEL_UNLOCK();
8697 1.15 jmcneill if (rv != 0)
8698 1.15 jmcneill return (EIO);
8699 1.15 jmcneill
8700 1.15 jmcneill memset(kstat_kv_istr(&ktmp->ktmp_name), 0,
8701 1.15 jmcneill sizeof(kstat_kv_istr(&ktmp->ktmp_name)));
8702 1.15 jmcneill memcpy(kstat_kv_istr(&ktmp->ktmp_name),
8703 1.15 jmcneill mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name));
8704 1.15 jmcneill kstat_kv_temp(&ktmp->ktmp_temperature) =
8705 1.15 jmcneill mcx_tmp_to_uK(&mtmp.mtmp_temperature);
8706 1.15 jmcneill kstat_kv_temp(&ktmp->ktmp_threshold_lo) =
8707 1.15 jmcneill mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo);
8708 1.15 jmcneill kstat_kv_temp(&ktmp->ktmp_threshold_hi) =
8709 1.15 jmcneill mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi);
8710 1.15 jmcneill
8711 1.15 jmcneill TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated);
8712 1.15 jmcneill
8713 1.15 jmcneill return (0);
8714 1.15 jmcneill }
8715 1.15 jmcneill
8716 1.15 jmcneill struct mcx_queuestat {
8717 1.15 jmcneill char name[KSTAT_KV_NAMELEN];
8718 1.15 jmcneill enum kstat_kv_type type;
8719 1.15 jmcneill };
8720 1.15 jmcneill
8721 1.15 jmcneill static const struct mcx_queuestat mcx_queue_kstat_tpl[] = {
8722 1.15 jmcneill { "RQ SW prod", KSTAT_KV_T_COUNTER64 },
8723 1.15 jmcneill { "RQ HW prod", KSTAT_KV_T_COUNTER64 },
8724 1.15 jmcneill { "RQ HW cons", KSTAT_KV_T_COUNTER64 },
8725 1.15 jmcneill { "RQ HW state", KSTAT_KV_T_ISTR },
8726 1.15 jmcneill
8727 1.15 jmcneill { "SQ SW prod", KSTAT_KV_T_COUNTER64 },
8728 1.15 jmcneill { "SQ SW cons", KSTAT_KV_T_COUNTER64 },
8729 1.15 jmcneill { "SQ HW prod", KSTAT_KV_T_COUNTER64 },
8730 1.15 jmcneill { "SQ HW cons", KSTAT_KV_T_COUNTER64 },
8731 1.15 jmcneill { "SQ HW state", KSTAT_KV_T_ISTR },
8732 1.15 jmcneill
8733 1.15 jmcneill { "CQ SW cons", KSTAT_KV_T_COUNTER64 },
8734 1.15 jmcneill { "CQ HW prod", KSTAT_KV_T_COUNTER64 },
8735 1.15 jmcneill { "CQ HW cons", KSTAT_KV_T_COUNTER64 },
8736 1.15 jmcneill { "CQ HW notify", KSTAT_KV_T_COUNTER64 },
8737 1.15 jmcneill { "CQ HW solicit", KSTAT_KV_T_COUNTER64 },
8738 1.15 jmcneill { "CQ HW status", KSTAT_KV_T_ISTR },
8739 1.15 jmcneill { "CQ HW state", KSTAT_KV_T_ISTR },
8740 1.15 jmcneill
8741 1.15 jmcneill { "EQ SW cons", KSTAT_KV_T_COUNTER64 },
8742 1.15 jmcneill { "EQ HW prod", KSTAT_KV_T_COUNTER64 },
8743 1.15 jmcneill { "EQ HW cons", KSTAT_KV_T_COUNTER64 },
8744 1.15 jmcneill { "EQ HW status", KSTAT_KV_T_ISTR },
8745 1.15 jmcneill { "EQ HW state", KSTAT_KV_T_ISTR },
8746 1.15 jmcneill };
8747 1.15 jmcneill
8748 1.15 jmcneill static int mcx_kstat_queue_read(struct kstat *);
8749 1.15 jmcneill
8750 1.15 jmcneill static void
8751 1.15 jmcneill mcx_kstat_attach_queues(struct mcx_softc *sc)
8752 1.15 jmcneill {
8753 1.15 jmcneill struct kstat *ks;
8754 1.15 jmcneill struct kstat_kv *kvs;
8755 1.15 jmcneill int q, i;
8756 1.15 jmcneill
8757 1.15 jmcneill for (q = 0; q < sc->sc_nqueues; q++) {
8758 1.15 jmcneill ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
8759 1.15 jmcneill KSTAT_T_KV, 0);
8760 1.15 jmcneill if (ks == NULL) {
8761 1.15 jmcneill /* unable to attach queue stats %u, q */
8762 1.15 jmcneill continue;
8763 1.15 jmcneill }
8764 1.15 jmcneill
8765 1.15 jmcneill kvs = mallocarray(nitems(mcx_queue_kstat_tpl),
8766 1.15 jmcneill sizeof(*kvs), M_DEVBUF, M_WAITOK);
8767 1.15 jmcneill
8768 1.15 jmcneill for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) {
8769 1.15 jmcneill const struct mcx_queuestat *tpl =
8770 1.15 jmcneill &mcx_queue_kstat_tpl[i];
8771 1.15 jmcneill
8772 1.15 jmcneill kstat_kv_init(&kvs[i], tpl->name, tpl->type);
8773 1.15 jmcneill }
8774 1.15 jmcneill
8775 1.15 jmcneill ks->ks_softc = &sc->sc_queues[q];
8776 1.15 jmcneill ks->ks_data = kvs;
8777 1.15 jmcneill ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs);
8778 1.15 jmcneill ks->ks_read = mcx_kstat_queue_read;
8779 1.15 jmcneill
8780 1.15 jmcneill sc->sc_queues[q].q_kstat = ks;
8781 1.15 jmcneill kstat_install(ks);
8782 1.15 jmcneill }
8783 1.15 jmcneill }
8784 1.15 jmcneill
8785 1.15 jmcneill static int
8786 1.15 jmcneill mcx_kstat_queue_read(struct kstat *ks)
8787 1.15 jmcneill {
8788 1.15 jmcneill struct mcx_queues *q = ks->ks_softc;
8789 1.15 jmcneill struct mcx_softc *sc = q->q_sc;
8790 1.15 jmcneill struct kstat_kv *kvs = ks->ks_data;
8791 1.15 jmcneill union {
8792 1.15 jmcneill struct mcx_rq_ctx rq;
8793 1.15 jmcneill struct mcx_sq_ctx sq;
8794 1.15 jmcneill struct mcx_cq_ctx cq;
8795 1.15 jmcneill struct mcx_eq_ctx eq;
8796 1.15 jmcneill } u;
8797 1.15 jmcneill const char *text;
8798 1.15 jmcneill int error = 0;
8799 1.15 jmcneill
8800 1.15 jmcneill KERNEL_LOCK();
8801 1.24 skrll
8802 1.15 jmcneill if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) {
8803 1.15 jmcneill error = EIO;
8804 1.15 jmcneill goto out;
8805 1.15 jmcneill }
8806 1.15 jmcneill
8807 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_rx.rx_prod;
8808 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter);
8809 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter);
8810 1.15 jmcneill switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >>
8811 1.15 jmcneill MCX_RQ_CTX_STATE_SHIFT) {
8812 1.15 jmcneill case MCX_RQ_CTX_STATE_RST:
8813 1.15 jmcneill text = "RST";
8814 1.15 jmcneill break;
8815 1.15 jmcneill case MCX_RQ_CTX_STATE_RDY:
8816 1.15 jmcneill text = "RDY";
8817 1.15 jmcneill break;
8818 1.15 jmcneill case MCX_RQ_CTX_STATE_ERR:
8819 1.15 jmcneill text = "ERR";
8820 1.15 jmcneill break;
8821 1.15 jmcneill default:
8822 1.15 jmcneill text = "unknown";
8823 1.15 jmcneill break;
8824 1.15 jmcneill }
8825 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8826 1.15 jmcneill kvs++;
8827 1.15 jmcneill
8828 1.15 jmcneill if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) {
8829 1.15 jmcneill error = EIO;
8830 1.15 jmcneill goto out;
8831 1.15 jmcneill }
8832 1.15 jmcneill
8833 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_tx.tx_prod;
8834 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_tx.tx_cons;
8835 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter);
8836 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter);
8837 1.15 jmcneill switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >>
8838 1.15 jmcneill MCX_SQ_CTX_STATE_SHIFT) {
8839 1.15 jmcneill case MCX_SQ_CTX_STATE_RST:
8840 1.15 jmcneill text = "RST";
8841 1.15 jmcneill break;
8842 1.15 jmcneill case MCX_SQ_CTX_STATE_RDY:
8843 1.15 jmcneill text = "RDY";
8844 1.15 jmcneill break;
8845 1.15 jmcneill case MCX_SQ_CTX_STATE_ERR:
8846 1.15 jmcneill text = "ERR";
8847 1.15 jmcneill break;
8848 1.15 jmcneill default:
8849 1.15 jmcneill text = "unknown";
8850 1.15 jmcneill break;
8851 1.15 jmcneill }
8852 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8853 1.15 jmcneill kvs++;
8854 1.15 jmcneill
8855 1.15 jmcneill if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) {
8856 1.15 jmcneill error = EIO;
8857 1.15 jmcneill goto out;
8858 1.15 jmcneill }
8859 1.15 jmcneill
8860 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_cq.cq_cons;
8861 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter);
8862 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter);
8863 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified);
8864 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit);
8865 1.15 jmcneill
8866 1.15 jmcneill switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >>
8867 1.15 jmcneill MCX_CQ_CTX_STATUS_SHIFT) {
8868 1.15 jmcneill case MCX_CQ_CTX_STATUS_OK:
8869 1.15 jmcneill text = "OK";
8870 1.15 jmcneill break;
8871 1.15 jmcneill case MCX_CQ_CTX_STATUS_OVERFLOW:
8872 1.15 jmcneill text = "overflow";
8873 1.15 jmcneill break;
8874 1.15 jmcneill case MCX_CQ_CTX_STATUS_WRITE_FAIL:
8875 1.15 jmcneill text = "write fail";
8876 1.15 jmcneill break;
8877 1.15 jmcneill default:
8878 1.15 jmcneill text = "unknown";
8879 1.15 jmcneill break;
8880 1.15 jmcneill }
8881 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8882 1.15 jmcneill kvs++;
8883 1.15 jmcneill
8884 1.15 jmcneill switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >>
8885 1.15 jmcneill MCX_CQ_CTX_STATE_SHIFT) {
8886 1.15 jmcneill case MCX_CQ_CTX_STATE_SOLICITED:
8887 1.15 jmcneill text = "solicited";
8888 1.15 jmcneill break;
8889 1.15 jmcneill case MCX_CQ_CTX_STATE_ARMED:
8890 1.15 jmcneill text = "armed";
8891 1.15 jmcneill break;
8892 1.15 jmcneill case MCX_CQ_CTX_STATE_FIRED:
8893 1.15 jmcneill text = "fired";
8894 1.15 jmcneill break;
8895 1.15 jmcneill default:
8896 1.15 jmcneill text = "unknown";
8897 1.15 jmcneill break;
8898 1.15 jmcneill }
8899 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8900 1.15 jmcneill kvs++;
8901 1.15 jmcneill
8902 1.15 jmcneill if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) {
8903 1.15 jmcneill error = EIO;
8904 1.15 jmcneill goto out;
8905 1.15 jmcneill }
8906 1.15 jmcneill
8907 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_eq.eq_cons;
8908 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter);
8909 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter);
8910 1.15 jmcneill
8911 1.15 jmcneill switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >>
8912 1.15 jmcneill MCX_EQ_CTX_STATUS_SHIFT) {
8913 1.15 jmcneill case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE:
8914 1.15 jmcneill text = "write fail";
8915 1.15 jmcneill break;
8916 1.15 jmcneill case MCX_EQ_CTX_STATUS_OK:
8917 1.15 jmcneill text = "OK";
8918 1.15 jmcneill break;
8919 1.15 jmcneill default:
8920 1.15 jmcneill text = "unknown";
8921 1.15 jmcneill break;
8922 1.15 jmcneill }
8923 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8924 1.15 jmcneill kvs++;
8925 1.15 jmcneill
8926 1.15 jmcneill switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >>
8927 1.15 jmcneill MCX_EQ_CTX_STATE_SHIFT) {
8928 1.15 jmcneill case MCX_EQ_CTX_STATE_ARMED:
8929 1.15 jmcneill text = "armed";
8930 1.15 jmcneill break;
8931 1.15 jmcneill case MCX_EQ_CTX_STATE_FIRED:
8932 1.15 jmcneill text = "fired";
8933 1.15 jmcneill break;
8934 1.15 jmcneill default:
8935 1.15 jmcneill text = "unknown";
8936 1.15 jmcneill break;
8937 1.15 jmcneill }
8938 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8939 1.15 jmcneill kvs++;
8940 1.15 jmcneill
8941 1.15 jmcneill nanouptime(&ks->ks_updated);
8942 1.15 jmcneill out:
8943 1.15 jmcneill KERNEL_UNLOCK();
8944 1.15 jmcneill return (error);
8945 1.15 jmcneill }
8946 1.15 jmcneill
8947 1.15 jmcneill #endif /* NKSTAT > 0 */
8948 1.15 jmcneill
8949 1.15 jmcneill static unsigned int
8950 1.15 jmcneill mcx_timecounter_read(struct timecounter *tc)
8951 1.15 jmcneill {
8952 1.15 jmcneill struct mcx_softc *sc = tc->tc_priv;
8953 1.15 jmcneill
8954 1.15 jmcneill return (mcx_rd(sc, MCX_INTERNAL_TIMER_L));
8955 1.15 jmcneill }
8956 1.15 jmcneill
8957 1.15 jmcneill static void
8958 1.15 jmcneill mcx_timecounter_attach(struct mcx_softc *sc)
8959 1.15 jmcneill {
8960 1.15 jmcneill struct timecounter *tc = &sc->sc_timecounter;
8961 1.15 jmcneill
8962 1.15 jmcneill tc->tc_get_timecount = mcx_timecounter_read;
8963 1.15 jmcneill tc->tc_counter_mask = ~0U;
8964 1.15 jmcneill tc->tc_frequency = sc->sc_khz * 1000;
8965 1.15 jmcneill tc->tc_name = device_xname(sc->sc_dev);
8966 1.15 jmcneill tc->tc_quality = -100;
8967 1.15 jmcneill tc->tc_priv = sc;
8968 1.15 jmcneill
8969 1.15 jmcneill tc_init(tc);
8970 1.15 jmcneill }
8971