if_mcx.c revision 1.20 1 /* $NetBSD: if_mcx.c,v 1.20 2021/09/25 15:16:36 jmcneill Exp $ */
2 /* $OpenBSD: if_mcx.c,v 1.101 2021/06/02 19:16:11 patrick Exp $ */
3
4 /*
5 * Copyright (c) 2017 David Gwynne <dlg (at) openbsd.org>
6 * Copyright (c) 2019 Jonathan Matthew <jmatthew (at) openbsd.org>
7 *
8 * Permission to use, copy, modify, and distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 */
20
21 #ifdef _KERNEL_OPT
22 #include "opt_net_mpsafe.h"
23 #endif
24
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(0, "$NetBSD: if_mcx.c,v 1.20 2021/09/25 15:16:36 jmcneill Exp $");
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/sockio.h>
31 #include <sys/mbuf.h>
32 #include <sys/kernel.h>
33 #include <sys/socket.h>
34 #include <sys/device.h>
35 #include <sys/pool.h>
36 #include <sys/queue.h>
37 #include <sys/callout.h>
38 #include <sys/workqueue.h>
39 #include <sys/atomic.h>
40 #include <sys/timetc.h>
41 #include <sys/kmem.h>
42 #include <sys/bus.h>
43 #include <sys/interrupt.h>
44 #include <sys/pcq.h>
45 #include <sys/cpu.h>
46
47 #include <machine/intr.h>
48
49 #include <net/if.h>
50 #include <net/if_dl.h>
51 #include <net/if_ether.h>
52 #include <net/if_media.h>
53 #include <net/if_vlanvar.h>
54 #include <net/toeplitz.h>
55
56 #include <net/bpf.h>
57
58 #include <netinet/in.h>
59
60 #include <dev/pci/pcireg.h>
61 #include <dev/pci/pcivar.h>
62 #include <dev/pci/pcidevs.h>
63
64 /* TODO: Port kstat key/value stuff to evcnt/sysmon */
65 #define NKSTAT 0
66
67 /* XXX This driver is not yet MP-safe; don't claim to be! */
68 /* #ifdef NET_MPSAFE */
69 /* #define MCX_MPSAFE 1 */
70 /* #define CALLOUT_FLAGS CALLOUT_MPSAFE */
71 /* #else */
72 #define CALLOUT_FLAGS 0
73 /* #endif */
74
75 #define MCX_TXQ_NUM 2048
76
77 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
78 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
79
80 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */
81
82 #define MCX_FW_VER 0x0000
83 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff)
84 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16)
85 #define MCX_CMDIF_FW_SUBVER 0x0004
86 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff)
87 #define MCX_CMDIF(_v) ((_v) >> 16)
88
89 #define MCX_ISSI 1 /* as per the PRM */
90 #define MCX_CMD_IF_SUPPORTED 5
91
92 #define MCX_HARDMTU 9500
93
94 #define MCX_PAGE_SHIFT 12
95 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT)
96
97 /* queue sizes */
98 #define MCX_LOG_EQ_SIZE 7
99 #define MCX_LOG_CQ_SIZE 12
100 #define MCX_LOG_RQ_SIZE 10
101 #define MCX_LOG_SQ_SIZE 11
102
103 #define MCX_MAX_QUEUES 16
104
105 /* completion event moderation - about 10khz, or 90% of the cq */
106 #define MCX_CQ_MOD_PERIOD 50
107 #define MCX_CQ_MOD_COUNTER \
108 (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)
109
110 #define MCX_LOG_SQ_ENTRY_SIZE 6
111 #define MCX_SQ_ENTRY_MAX_SLOTS 4
112 #define MCX_SQ_SEGS_PER_SLOT \
113 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
114 #define MCX_SQ_MAX_SEGMENTS \
115 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)
116
117 #define MCX_LOG_FLOW_TABLE_SIZE 5
118 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */
119 #define MCX_NUM_MCAST_FLOWS \
120 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)
121
122 #define MCX_SQ_INLINE_SIZE 18
123 CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE);
124
125 /* doorbell offsets */
126 #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE
127
128 #define MCX_CQ_DOORBELL_BASE 0
129 #define MCX_CQ_DOORBELL_STRIDE 64
130
131 #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2
132 #define MCX_WQ_DOORBELL_STRIDE 64
133 /* make sure the doorbells fit */
134 CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE);
135 CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE <
136 MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE);
137
138 #define MCX_WQ_DOORBELL_MASK 0xffff
139
140 /* uar registers */
141 #define MCX_UAR_CQ_DOORBELL 0x20
142 #define MCX_UAR_EQ_DOORBELL_ARM 0x40
143 #define MCX_UAR_EQ_DOORBELL 0x48
144 #define MCX_UAR_BF 0x800
145
146 #define MCX_CMDQ_ADDR_HI 0x0010
147 #define MCX_CMDQ_ADDR_LO 0x0014
148 #define MCX_CMDQ_ADDR_NMASK 0xfff
149 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf)
150 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf)
151 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8)
152 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8)
153 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8)
154
155 #define MCX_CMDQ_DOORBELL 0x0018
156
157 #define MCX_STATE 0x01fc
158 #define MCX_STATE_MASK (1U << 31)
159 #define MCX_STATE_INITIALIZING (1 << 31)
160 #define MCX_STATE_READY (0 << 31)
161 #define MCX_STATE_INTERFACE_MASK (0x3 << 24)
162 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24)
163 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24)
164
165 #define MCX_INTERNAL_TIMER 0x1000
166 #define MCX_INTERNAL_TIMER_H 0x1000
167 #define MCX_INTERNAL_TIMER_L 0x1004
168
169 #define MCX_CLEAR_INT 0x100c
170
171 #define MCX_REG_OP_WRITE 0
172 #define MCX_REG_OP_READ 1
173
174 #define MCX_REG_PMLP 0x5002
175 #define MCX_REG_PMTU 0x5003
176 #define MCX_REG_PTYS 0x5004
177 #define MCX_REG_PAOS 0x5006
178 #define MCX_REG_PFCC 0x5007
179 #define MCX_REG_PPCNT 0x5008
180 #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */
181 #define MCX_REG_MTMP 0x900a /* mgmt temp */
182 #define MCX_REG_MCIA 0x9014
183 #define MCX_REG_MCAM 0x907f
184
185 #define MCX_ETHER_CAP_SGMII 0
186 #define MCX_ETHER_CAP_1000_KX 1
187 #define MCX_ETHER_CAP_10G_CX4 2
188 #define MCX_ETHER_CAP_10G_KX4 3
189 #define MCX_ETHER_CAP_10G_KR 4
190 #define MCX_ETHER_CAP_20G_KR2 5
191 #define MCX_ETHER_CAP_40G_CR4 6
192 #define MCX_ETHER_CAP_40G_KR4 7
193 #define MCX_ETHER_CAP_56G_R4 8
194 #define MCX_ETHER_CAP_10G_CR 12
195 #define MCX_ETHER_CAP_10G_SR 13
196 #define MCX_ETHER_CAP_10G_LR 14
197 #define MCX_ETHER_CAP_40G_SR4 15
198 #define MCX_ETHER_CAP_40G_LR4 16
199 #define MCX_ETHER_CAP_50G_SR2 18
200 #define MCX_ETHER_CAP_100G_CR4 20
201 #define MCX_ETHER_CAP_100G_SR4 21
202 #define MCX_ETHER_CAP_100G_KR4 22
203 #define MCX_ETHER_CAP_100G_LR4 23
204 #define MCX_ETHER_CAP_100_TX 24
205 #define MCX_ETHER_CAP_1000_T 25
206 #define MCX_ETHER_CAP_10G_T 26
207 #define MCX_ETHER_CAP_25G_CR 27
208 #define MCX_ETHER_CAP_25G_KR 28
209 #define MCX_ETHER_CAP_25G_SR 29
210 #define MCX_ETHER_CAP_50G_CR2 30
211 #define MCX_ETHER_CAP_50G_KR2 31
212
213 #define MCX_MAX_CQE 32
214
215 #define MCX_CMD_QUERY_HCA_CAP 0x100
216 #define MCX_CMD_QUERY_ADAPTER 0x101
217 #define MCX_CMD_INIT_HCA 0x102
218 #define MCX_CMD_TEARDOWN_HCA 0x103
219 #define MCX_CMD_ENABLE_HCA 0x104
220 #define MCX_CMD_DISABLE_HCA 0x105
221 #define MCX_CMD_QUERY_PAGES 0x107
222 #define MCX_CMD_MANAGE_PAGES 0x108
223 #define MCX_CMD_SET_HCA_CAP 0x109
224 #define MCX_CMD_QUERY_ISSI 0x10a
225 #define MCX_CMD_SET_ISSI 0x10b
226 #define MCX_CMD_SET_DRIVER_VERSION 0x10d
227 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203
228 #define MCX_CMD_CREATE_EQ 0x301
229 #define MCX_CMD_DESTROY_EQ 0x302
230 #define MCX_CMD_QUERY_EQ 0x303
231 #define MCX_CMD_CREATE_CQ 0x400
232 #define MCX_CMD_DESTROY_CQ 0x401
233 #define MCX_CMD_QUERY_CQ 0x402
234 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754
235 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
236 0x755
237 #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770
238 #define MCX_CMD_ALLOC_PD 0x800
239 #define MCX_CMD_ALLOC_UAR 0x802
240 #define MCX_CMD_ACCESS_REG 0x805
241 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816
242 #define MCX_CMD_CREATE_TIR 0x900
243 #define MCX_CMD_DESTROY_TIR 0x902
244 #define MCX_CMD_CREATE_SQ 0x904
245 #define MCX_CMD_MODIFY_SQ 0x905
246 #define MCX_CMD_DESTROY_SQ 0x906
247 #define MCX_CMD_QUERY_SQ 0x907
248 #define MCX_CMD_CREATE_RQ 0x908
249 #define MCX_CMD_MODIFY_RQ 0x909
250 #define MCX_CMD_DESTROY_RQ 0x90a
251 #define MCX_CMD_QUERY_RQ 0x90b
252 #define MCX_CMD_CREATE_TIS 0x912
253 #define MCX_CMD_DESTROY_TIS 0x914
254 #define MCX_CMD_CREATE_RQT 0x916
255 #define MCX_CMD_DESTROY_RQT 0x918
256 #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f
257 #define MCX_CMD_CREATE_FLOW_TABLE 0x930
258 #define MCX_CMD_DESTROY_FLOW_TABLE 0x931
259 #define MCX_CMD_QUERY_FLOW_TABLE 0x932
260 #define MCX_CMD_CREATE_FLOW_GROUP 0x933
261 #define MCX_CMD_DESTROY_FLOW_GROUP 0x934
262 #define MCX_CMD_QUERY_FLOW_GROUP 0x935
263 #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936
264 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937
265 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938
266 #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939
267 #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b
268
269 #define MCX_QUEUE_STATE_RST 0
270 #define MCX_QUEUE_STATE_RDY 1
271 #define MCX_QUEUE_STATE_ERR 3
272
273 #define MCX_FLOW_TABLE_TYPE_RX 0
274 #define MCX_FLOW_TABLE_TYPE_TX 1
275
276 #define MCX_CMDQ_INLINE_DATASIZE 16
277
278 struct mcx_cmdq_entry {
279 uint8_t cq_type;
280 #define MCX_CMDQ_TYPE_PCIE 0x7
281 uint8_t cq_reserved0[3];
282
283 uint32_t cq_input_length;
284 uint64_t cq_input_ptr;
285 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
286
287 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
288 uint64_t cq_output_ptr;
289 uint32_t cq_output_length;
290
291 uint8_t cq_token;
292 uint8_t cq_signature;
293 uint8_t cq_reserved1[1];
294 uint8_t cq_status;
295 #define MCX_CQ_STATUS_SHIFT 1
296 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT)
297 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT)
298 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT)
299 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT)
300 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT)
301 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT)
302 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT)
303 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT)
304 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT)
305 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT)
306 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT)
307 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT)
308 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT)
309 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT)
310 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
311 (0x10 << MCX_CQ_STATUS_SHIFT)
312 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT)
313 #define MCX_CQ_STATUS_OWN_MASK 0x1
314 #define MCX_CQ_STATUS_OWN_SW 0x0
315 #define MCX_CQ_STATUS_OWN_HW 0x1
316 } __packed __aligned(8);
317
318 #define MCX_CMDQ_MAILBOX_DATASIZE 512
319
320 struct mcx_cmdq_mailbox {
321 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
322 uint8_t mb_reserved0[48];
323 uint64_t mb_next_ptr;
324 uint32_t mb_block_number;
325 uint8_t mb_reserved1[1];
326 uint8_t mb_token;
327 uint8_t mb_ctrl_signature;
328 uint8_t mb_signature;
329 } __packed __aligned(8);
330
331 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10)
332 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \
333 MCX_CMDQ_MAILBOX_ALIGN)
334 /*
335 * command mailbox structures
336 */
337
338 struct mcx_cmd_enable_hca_in {
339 uint16_t cmd_opcode;
340 uint8_t cmd_reserved0[4];
341 uint16_t cmd_op_mod;
342 uint8_t cmd_reserved1[2];
343 uint16_t cmd_function_id;
344 uint8_t cmd_reserved2[4];
345 } __packed __aligned(4);
346
347 struct mcx_cmd_enable_hca_out {
348 uint8_t cmd_status;
349 uint8_t cmd_reserved0[3];
350 uint32_t cmd_syndrome;
351 uint8_t cmd_reserved1[4];
352 } __packed __aligned(4);
353
354 struct mcx_cmd_init_hca_in {
355 uint16_t cmd_opcode;
356 uint8_t cmd_reserved0[4];
357 uint16_t cmd_op_mod;
358 uint8_t cmd_reserved1[8];
359 } __packed __aligned(4);
360
361 struct mcx_cmd_init_hca_out {
362 uint8_t cmd_status;
363 uint8_t cmd_reserved0[3];
364 uint32_t cmd_syndrome;
365 uint8_t cmd_reserved1[8];
366 } __packed __aligned(4);
367
368 struct mcx_cmd_teardown_hca_in {
369 uint16_t cmd_opcode;
370 uint8_t cmd_reserved0[4];
371 uint16_t cmd_op_mod;
372 uint8_t cmd_reserved1[2];
373 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0
374 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1
375 uint16_t cmd_profile;
376 uint8_t cmd_reserved2[4];
377 } __packed __aligned(4);
378
379 struct mcx_cmd_teardown_hca_out {
380 uint8_t cmd_status;
381 uint8_t cmd_reserved0[3];
382 uint32_t cmd_syndrome;
383 uint8_t cmd_reserved1[8];
384 } __packed __aligned(4);
385
386 struct mcx_cmd_access_reg_in {
387 uint16_t cmd_opcode;
388 uint8_t cmd_reserved0[4];
389 uint16_t cmd_op_mod;
390 uint8_t cmd_reserved1[2];
391 uint16_t cmd_register_id;
392 uint32_t cmd_argument;
393 } __packed __aligned(4);
394
395 struct mcx_cmd_access_reg_out {
396 uint8_t cmd_status;
397 uint8_t cmd_reserved0[3];
398 uint32_t cmd_syndrome;
399 uint8_t cmd_reserved1[8];
400 } __packed __aligned(4);
401
402 struct mcx_reg_pmtu {
403 uint8_t rp_reserved1;
404 uint8_t rp_local_port;
405 uint8_t rp_reserved2[2];
406 uint16_t rp_max_mtu;
407 uint8_t rp_reserved3[2];
408 uint16_t rp_admin_mtu;
409 uint8_t rp_reserved4[2];
410 uint16_t rp_oper_mtu;
411 uint8_t rp_reserved5[2];
412 } __packed __aligned(4);
413
414 struct mcx_reg_ptys {
415 uint8_t rp_reserved1;
416 uint8_t rp_local_port;
417 uint8_t rp_reserved2;
418 uint8_t rp_proto_mask;
419 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2)
420 uint8_t rp_reserved3[8];
421 uint32_t rp_eth_proto_cap;
422 uint8_t rp_reserved4[8];
423 uint32_t rp_eth_proto_admin;
424 uint8_t rp_reserved5[8];
425 uint32_t rp_eth_proto_oper;
426 uint8_t rp_reserved6[24];
427 } __packed __aligned(4);
428
429 struct mcx_reg_paos {
430 uint8_t rp_reserved1;
431 uint8_t rp_local_port;
432 uint8_t rp_admin_status;
433 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1
434 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2
435 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3
436 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4
437 uint8_t rp_oper_status;
438 #define MCX_REG_PAOS_OPER_STATUS_UP 1
439 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2
440 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4
441 uint8_t rp_admin_state_update;
442 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7)
443 uint8_t rp_reserved2[11];
444 } __packed __aligned(4);
445
446 struct mcx_reg_pfcc {
447 uint8_t rp_reserved1;
448 uint8_t rp_local_port;
449 uint8_t rp_reserved2[3];
450 uint8_t rp_prio_mask_tx;
451 uint8_t rp_reserved3;
452 uint8_t rp_prio_mask_rx;
453 uint8_t rp_pptx_aptx;
454 uint8_t rp_pfctx;
455 uint8_t rp_fctx_dis;
456 uint8_t rp_reserved4;
457 uint8_t rp_pprx_aprx;
458 uint8_t rp_pfcrx;
459 uint8_t rp_reserved5[2];
460 uint16_t rp_dev_stall_min;
461 uint16_t rp_dev_stall_crit;
462 uint8_t rp_reserved6[12];
463 } __packed __aligned(4);
464
465 #define MCX_PMLP_MODULE_NUM_MASK 0xff
466 struct mcx_reg_pmlp {
467 uint8_t rp_rxtx;
468 uint8_t rp_local_port;
469 uint8_t rp_reserved0;
470 uint8_t rp_width;
471 uint32_t rp_lane0_mapping;
472 uint32_t rp_lane1_mapping;
473 uint32_t rp_lane2_mapping;
474 uint32_t rp_lane3_mapping;
475 uint8_t rp_reserved1[44];
476 } __packed __aligned(4);
477
478 struct mcx_reg_ppcnt {
479 uint8_t ppcnt_swid;
480 uint8_t ppcnt_local_port;
481 uint8_t ppcnt_pnat;
482 uint8_t ppcnt_grp;
483 #define MCX_REG_PPCNT_GRP_IEEE8023 0x00
484 #define MCX_REG_PPCNT_GRP_RFC2863 0x01
485 #define MCX_REG_PPCNT_GRP_RFC2819 0x02
486 #define MCX_REG_PPCNT_GRP_RFC3635 0x03
487 #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10
488 #define MCX_REG_PPCNT_GRP_PER_TC 0x11
489 #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11
490
491 uint8_t ppcnt_clr;
492 uint8_t ppcnt_reserved1[2];
493 uint8_t ppcnt_prio_tc;
494 #define MCX_REG_PPCNT_CLR (1 << 7)
495
496 uint8_t ppcnt_counter_set[248];
497 } __packed __aligned(8);
498 CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256);
499 CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) %
500 sizeof(uint64_t)) == 0);
501
502 enum mcx_ppcnt_ieee8023 {
503 frames_transmitted_ok,
504 frames_received_ok,
505 frame_check_sequence_errors,
506 alignment_errors,
507 octets_transmitted_ok,
508 octets_received_ok,
509 multicast_frames_xmitted_ok,
510 broadcast_frames_xmitted_ok,
511 multicast_frames_received_ok,
512 broadcast_frames_received_ok,
513 in_range_length_errors,
514 out_of_range_length_field,
515 frame_too_long_errors,
516 symbol_error_during_carrier,
517 mac_control_frames_transmitted,
518 mac_control_frames_received,
519 unsupported_opcodes_received,
520 pause_mac_ctrl_frames_received,
521 pause_mac_ctrl_frames_transmitted,
522
523 mcx_ppcnt_ieee8023_count
524 };
525 CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98);
526
527 enum mcx_ppcnt_rfc2863 {
528 in_octets,
529 in_ucast_pkts,
530 in_discards,
531 in_errors,
532 in_unknown_protos,
533 out_octets,
534 out_ucast_pkts,
535 out_discards,
536 out_errors,
537 in_multicast_pkts,
538 in_broadcast_pkts,
539 out_multicast_pkts,
540 out_broadcast_pkts,
541
542 mcx_ppcnt_rfc2863_count
543 };
544 CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68);
545
546 enum mcx_ppcnt_rfc2819 {
547 drop_events,
548 octets,
549 pkts,
550 broadcast_pkts,
551 multicast_pkts,
552 crc_align_errors,
553 undersize_pkts,
554 oversize_pkts,
555 fragments,
556 jabbers,
557 collisions,
558 pkts64octets,
559 pkts65to127octets,
560 pkts128to255octets,
561 pkts256to511octets,
562 pkts512to1023octets,
563 pkts1024to1518octets,
564 pkts1519to2047octets,
565 pkts2048to4095octets,
566 pkts4096to8191octets,
567 pkts8192to10239octets,
568
569 mcx_ppcnt_rfc2819_count
570 };
571 CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8);
572
573 enum mcx_ppcnt_rfc3635 {
574 dot3stats_alignment_errors,
575 dot3stats_fcs_errors,
576 dot3stats_single_collision_frames,
577 dot3stats_multiple_collision_frames,
578 dot3stats_sqe_test_errors,
579 dot3stats_deferred_transmissions,
580 dot3stats_late_collisions,
581 dot3stats_excessive_collisions,
582 dot3stats_internal_mac_transmit_errors,
583 dot3stats_carrier_sense_errors,
584 dot3stats_frame_too_longs,
585 dot3stats_internal_mac_receive_errors,
586 dot3stats_symbol_errors,
587 dot3control_in_unknown_opcodes,
588 dot3in_pause_frames,
589 dot3out_pause_frames,
590
591 mcx_ppcnt_rfc3635_count
592 };
593 CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80);
594
595 struct mcx_reg_mcam {
596 uint8_t _reserved1[1];
597 uint8_t mcam_feature_group;
598 uint8_t _reserved2[1];
599 uint8_t mcam_access_reg_group;
600 uint8_t _reserved3[4];
601 uint8_t mcam_access_reg_cap_mask[16];
602 uint8_t _reserved4[16];
603 uint8_t mcam_feature_cap_mask[16];
604 uint8_t _reserved5[16];
605 } __packed __aligned(4);
606
607 #define MCX_BITFIELD_BIT(bf, b) (bf[(sizeof bf - 1) - (b / 8)] & (b % 8))
608
609 #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP 6
610
611 struct mcx_reg_mtcap {
612 uint8_t _reserved1[3];
613 uint8_t mtcap_sensor_count;
614 uint8_t _reserved2[4];
615
616 uint64_t mtcap_sensor_map;
617 };
618
619 struct mcx_reg_mtmp {
620 uint8_t _reserved1[2];
621 uint16_t mtmp_sensor_index;
622
623 uint8_t _reserved2[2];
624 uint16_t mtmp_temperature;
625
626 uint16_t mtmp_mte_mtr;
627 #define MCX_REG_MTMP_MTE (1 << 15)
628 #define MCX_REG_MTMP_MTR (1 << 14)
629 uint16_t mtmp_max_temperature;
630
631 uint16_t mtmp_tee;
632 #define MCX_REG_MTMP_TEE_NOPE (0 << 14)
633 #define MCX_REG_MTMP_TEE_GENERATE (1 << 14)
634 #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14)
635 uint16_t mtmp_temperature_threshold_hi;
636
637 uint8_t _reserved3[2];
638 uint16_t mtmp_temperature_threshold_lo;
639
640 uint8_t _reserved4[4];
641
642 uint8_t mtmp_sensor_name[8];
643 };
644 CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20);
645 CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18);
646
647 #define MCX_MCIA_EEPROM_BYTES 32
648 struct mcx_reg_mcia {
649 uint8_t rm_l;
650 uint8_t rm_module;
651 uint8_t rm_reserved0;
652 uint8_t rm_status;
653 uint8_t rm_i2c_addr;
654 uint8_t rm_page_num;
655 uint16_t rm_dev_addr;
656 uint16_t rm_reserved1;
657 uint16_t rm_size;
658 uint32_t rm_reserved2;
659 uint8_t rm_data[48];
660 } __packed __aligned(4);
661
662 struct mcx_cmd_query_issi_in {
663 uint16_t cmd_opcode;
664 uint8_t cmd_reserved0[4];
665 uint16_t cmd_op_mod;
666 uint8_t cmd_reserved1[8];
667 } __packed __aligned(4);
668
669 struct mcx_cmd_query_issi_il_out {
670 uint8_t cmd_status;
671 uint8_t cmd_reserved0[3];
672 uint32_t cmd_syndrome;
673 uint8_t cmd_reserved1[2];
674 uint16_t cmd_current_issi;
675 uint8_t cmd_reserved2[4];
676 } __packed __aligned(4);
677
678 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
679
680 struct mcx_cmd_query_issi_mb_out {
681 uint8_t cmd_reserved2[16];
682 uint8_t cmd_supported_issi[80]; /* very big endian */
683 } __packed __aligned(4);
684
685 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
686
687 struct mcx_cmd_set_issi_in {
688 uint16_t cmd_opcode;
689 uint8_t cmd_reserved0[4];
690 uint16_t cmd_op_mod;
691 uint8_t cmd_reserved1[2];
692 uint16_t cmd_current_issi;
693 uint8_t cmd_reserved2[4];
694 } __packed __aligned(4);
695
696 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
697
698 struct mcx_cmd_set_issi_out {
699 uint8_t cmd_status;
700 uint8_t cmd_reserved0[3];
701 uint32_t cmd_syndrome;
702 uint8_t cmd_reserved1[8];
703 } __packed __aligned(4);
704
705 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
706
707 struct mcx_cmd_query_pages_in {
708 uint16_t cmd_opcode;
709 uint8_t cmd_reserved0[4];
710 uint16_t cmd_op_mod;
711 #define MCX_CMD_QUERY_PAGES_BOOT 0x01
712 #define MCX_CMD_QUERY_PAGES_INIT 0x02
713 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03
714 uint8_t cmd_reserved1[8];
715 } __packed __aligned(4);
716
717 struct mcx_cmd_query_pages_out {
718 uint8_t cmd_status;
719 uint8_t cmd_reserved0[3];
720 uint32_t cmd_syndrome;
721 uint8_t cmd_reserved1[2];
722 uint16_t cmd_func_id;
723 int32_t cmd_num_pages;
724 } __packed __aligned(4);
725
726 struct mcx_cmd_manage_pages_in {
727 uint16_t cmd_opcode;
728 uint8_t cmd_reserved0[4];
729 uint16_t cmd_op_mod;
730 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
731 0x00
732 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
733 0x01
734 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
735 0x02
736 uint8_t cmd_reserved1[2];
737 uint16_t cmd_func_id;
738 uint32_t cmd_input_num_entries;
739 } __packed __aligned(4);
740
741 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
742
743 struct mcx_cmd_manage_pages_out {
744 uint8_t cmd_status;
745 uint8_t cmd_reserved0[3];
746 uint32_t cmd_syndrome;
747 uint32_t cmd_output_num_entries;
748 uint8_t cmd_reserved1[4];
749 } __packed __aligned(4);
750
751 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
752
753 struct mcx_cmd_query_hca_cap_in {
754 uint16_t cmd_opcode;
755 uint8_t cmd_reserved0[4];
756 uint16_t cmd_op_mod;
757 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0)
758 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0)
759 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1)
760 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1)
761 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1)
762 uint8_t cmd_reserved1[8];
763 } __packed __aligned(4);
764
765 struct mcx_cmd_query_hca_cap_out {
766 uint8_t cmd_status;
767 uint8_t cmd_reserved0[3];
768 uint32_t cmd_syndrome;
769 uint8_t cmd_reserved1[8];
770 } __packed __aligned(4);
771
772 #define MCX_HCA_CAP_LEN 0x1000
773 #define MCX_HCA_CAP_NMAILBOXES \
774 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
775
776 #if __GNUC_PREREQ__(4, 3)
777 #define __counter__ __COUNTER__
778 #else
779 #define __counter__ __LINE__
780 #endif
781
782 #define __token(_tok, _num) _tok##_num
783 #define _token(_tok, _num) __token(_tok, _num)
784 #define __reserved__ _token(__reserved, __counter__)
785
786 struct mcx_cap_device {
787 uint8_t reserved0[16];
788
789 uint8_t log_max_srq_sz;
790 uint8_t log_max_qp_sz;
791 uint8_t __reserved__[1];
792 uint8_t log_max_qp; /* 5 bits */
793 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f
794
795 uint8_t __reserved__[1];
796 uint8_t log_max_srq; /* 5 bits */
797 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f
798 uint8_t __reserved__[2];
799
800 uint8_t __reserved__[1];
801 uint8_t log_max_cq_sz;
802 uint8_t __reserved__[1];
803 uint8_t log_max_cq; /* 5 bits */
804 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f
805
806 uint8_t log_max_eq_sz;
807 uint8_t log_max_mkey; /* 6 bits */
808 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f
809 uint8_t __reserved__[1];
810 uint8_t log_max_eq; /* 4 bits */
811 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f
812
813 uint8_t max_indirection;
814 uint8_t log_max_mrw_sz; /* 7 bits */
815 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f
816 uint8_t teardown_log_max_msf_list_size;
817 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80
818 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
819 0x3f
820 uint8_t log_max_klm_list_size; /* 6 bits */
821 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
822 0x3f
823
824 uint8_t __reserved__[1];
825 uint8_t log_max_ra_req_dc; /* 6 bits */
826 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f
827 uint8_t __reserved__[1];
828 uint8_t log_max_ra_res_dc; /* 6 bits */
829 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
830 0x3f
831
832 uint8_t __reserved__[1];
833 uint8_t log_max_ra_req_qp; /* 6 bits */
834 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
835 0x3f
836 uint8_t __reserved__[1];
837 uint8_t log_max_ra_res_qp; /* 6 bits */
838 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
839 0x3f
840
841 uint8_t flags1;
842 #define MCX_CAP_DEVICE_END_PAD 0x80
843 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40
844 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
845 0x20
846 #define MCX_CAP_DEVICE_START_PAD 0x10
847 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \
848 0x08
849 uint8_t __reserved__[1];
850 uint16_t gid_table_size;
851
852 uint16_t flags2;
853 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000
854 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000
855 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
856 0x2000
857 #define MCX_CAP_DEVICE_DEBUG 0x1000
858 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
859 0x8000
860 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000
861 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff
862 uint16_t pkey_table_size;
863
864 uint8_t flags3;
865 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
866 0x80
867 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
868 0x40
869 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20
870 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10
871 #define MCX_CAP_DEVICE_ETS 0x04
872 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02
873 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
874 0x01
875 uint8_t local_ca_ack_delay; /* 5 bits */
876 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
877 0x1f
878 #define MCX_CAP_DEVICE_MCAM_REG 0x40
879 uint8_t port_type;
880 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
881 0x80
882 #define MCX_CAP_DEVICE_PORT_TYPE 0x03
883 #define MCX_CAP_DEVICE_PORT_TYPE_ETH 0x01
884 uint8_t num_ports;
885
886 uint8_t snapshot_log_max_msg;
887 #define MCX_CAP_DEVICE_SNAPSHOT 0x80
888 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f
889 uint8_t max_tc; /* 4 bits */
890 #define MCX_CAP_DEVICE_MAX_TC 0x0f
891 uint8_t flags4;
892 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80
893 #define MCX_CAP_DEVICE_DCBX 0x40
894 #define MCX_CAP_DEVICE_ROL_S 0x02
895 #define MCX_CAP_DEVICE_ROL_G 0x01
896 uint8_t wol;
897 #define MCX_CAP_DEVICE_WOL_S 0x40
898 #define MCX_CAP_DEVICE_WOL_G 0x20
899 #define MCX_CAP_DEVICE_WOL_A 0x10
900 #define MCX_CAP_DEVICE_WOL_B 0x08
901 #define MCX_CAP_DEVICE_WOL_M 0x04
902 #define MCX_CAP_DEVICE_WOL_U 0x02
903 #define MCX_CAP_DEVICE_WOL_P 0x01
904
905 uint16_t stat_rate_support;
906 uint8_t __reserved__[1];
907 uint8_t cqe_version; /* 4 bits */
908 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f
909
910 uint32_t flags5;
911 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
912 0x80000000
913 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000
914 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
915 0x10000000
916 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
917 0x08000000
918 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000
919 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000
920 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000
921 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000
922 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000
923 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000
924 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800
925 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400
926 #define MCX_CAP_DEVICE_SHO 0x00000100
927 #define MCX_CAP_DEVICE_TPH 0x00000080
928 #define MCX_CAP_DEVICE_RF 0x00000040
929 #define MCX_CAP_DEVICE_DCT 0x00000020
930 #define MCX_CAP_DEVICE_QOS 0x00000010
931 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008
932 #define MCX_CAP_DEVICE_ROCE 0x00000004
933 #define MCX_CAP_DEVICE_ATOMIC 0x00000002
934
935 uint32_t flags6;
936 #define MCX_CAP_DEVICE_CQ_OI 0x80000000
937 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000
938 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000
939 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
940 0x10000000
941 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000
942 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000
943 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000
944 #define MCX_CAP_DEVICE_PG 0x01000000
945 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000
946 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
947 0x00400000
948 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
949 0x00200000
950 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
951 0x00100000
952 #define MCX_CAP_DEVICE_CD 0x00080000
953 #define MCX_CAP_DEVICE_ATM 0x00040000
954 #define MCX_CAP_DEVICE_APM 0x00020000
955 #define MCX_CAP_DEVICE_IMAICL 0x00010000
956 #define MCX_CAP_DEVICE_QKV 0x00000200
957 #define MCX_CAP_DEVICE_PKV 0x00000100
958 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080
959 #define MCX_CAP_DEVICE_XRC 0x00000008
960 #define MCX_CAP_DEVICE_UD 0x00000004
961 #define MCX_CAP_DEVICE_UC 0x00000002
962 #define MCX_CAP_DEVICE_RC 0x00000001
963
964 uint8_t uar_flags;
965 #define MCX_CAP_DEVICE_UAR_4K 0x80
966 uint8_t uar_sz; /* 6 bits */
967 #define MCX_CAP_DEVICE_UAR_SZ 0x3f
968 uint8_t __reserved__[1];
969 uint8_t log_pg_sz;
970
971 uint8_t flags7;
972 #define MCX_CAP_DEVICE_BF 0x80
973 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40
974 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
975 0x20
976 uint8_t log_bf_reg_size; /* 5 bits */
977 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f
978 uint8_t __reserved__[2];
979
980 uint16_t num_of_diagnostic_counters;
981 uint16_t max_wqe_sz_sq;
982
983 uint8_t __reserved__[2];
984 uint16_t max_wqe_sz_rq;
985
986 uint8_t __reserved__[2];
987 uint16_t max_wqe_sz_sq_dc;
988
989 uint32_t max_qp_mcg; /* 25 bits */
990 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff
991
992 uint8_t __reserved__[3];
993 uint8_t log_max_mcq;
994
995 uint8_t log_max_transport_domain; /* 5 bits */
996 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
997 0x1f
998 uint8_t log_max_pd; /* 5 bits */
999 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f
1000 uint8_t __reserved__[1];
1001 uint8_t log_max_xrcd; /* 5 bits */
1002 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f
1003
1004 uint8_t __reserved__[2];
1005 uint16_t max_flow_counter;
1006
1007 uint8_t log_max_rq; /* 5 bits */
1008 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f
1009 uint8_t log_max_sq; /* 5 bits */
1010 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f
1011 uint8_t log_max_tir; /* 5 bits */
1012 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f
1013 uint8_t log_max_tis; /* 5 bits */
1014 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f
1015
1016 uint8_t flags8;
1017 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
1018 0x80
1019 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f
1020 uint8_t log_max_rqt; /* 5 bits */
1021 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f
1022 uint8_t log_max_rqt_size; /* 5 bits */
1023 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f
1024 uint8_t log_max_tis_per_sq; /* 5 bits */
1025 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
1026 0x1f
1027
1028 uint8_t flags9;
1029 #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \
1030 0x80
1031 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \
1032 0x1f
1033 uint8_t log_min_stride_sz_rq; /* 5 bits */
1034 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \
1035 0x1f
1036 uint8_t log_max_stride_sz_sq; /* 5 bits */
1037 #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \
1038 0x1f
1039 uint8_t log_min_stride_sz_sq; /* 5 bits */
1040 #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \
1041 0x1f
1042
1043 uint8_t log_max_hairpin_queues;
1044 #define MXC_CAP_DEVICE_HAIRPIN 0x80
1045 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \
1046 0x1f
1047 uint8_t log_min_hairpin_queues;
1048 #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \
1049 0x1f
1050 uint8_t log_max_hairpin_num_packets;
1051 #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \
1052 0x1f
1053 uint8_t log_max_mq_sz;
1054 #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \
1055 0x1f
1056
1057 uint8_t log_min_hairpin_wq_data_sz;
1058 #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \
1059 0x80
1060 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \
1061 0x40
1062 #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \
1063 0x20
1064 #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \
1065 0x1f
1066 uint8_t log_max_vlan_list;
1067 #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \
1068 0x80
1069 #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \
1070 0x1f
1071 uint8_t log_max_current_mc_list;
1072 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \
1073 0x1f
1074 uint8_t log_max_current_uc_list;
1075 #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \
1076 0x1f
1077
1078 uint8_t __reserved__[4];
1079
1080 uint32_t create_qp_start_hint; /* 24 bits */
1081
1082 uint8_t log_max_uctx; /* 5 bits */
1083 #define MXC_CAP_DEVICE_LOG_MAX_UCTX 0x1f
1084 uint8_t log_max_umem; /* 5 bits */
1085 #define MXC_CAP_DEVICE_LOG_MAX_UMEM 0x1f
1086 uint16_t max_num_eqs;
1087
1088 uint8_t log_max_l2_table; /* 5 bits */
1089 #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE 0x1f
1090 uint8_t __reserved__[1];
1091 uint16_t log_uar_page_sz;
1092
1093 uint8_t __reserved__[8];
1094
1095 uint32_t device_frequency_mhz;
1096 uint32_t device_frequency_khz;
1097 } __packed __aligned(8);
1098
1099 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
1100 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
1101 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
1102 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
1103 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
1104 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
1105 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98);
1106 CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c);
1107 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
1108
1109 struct mcx_cmd_set_driver_version_in {
1110 uint16_t cmd_opcode;
1111 uint8_t cmd_reserved0[4];
1112 uint16_t cmd_op_mod;
1113 uint8_t cmd_reserved1[8];
1114 } __packed __aligned(4);
1115
1116 struct mcx_cmd_set_driver_version_out {
1117 uint8_t cmd_status;
1118 uint8_t cmd_reserved0[3];
1119 uint32_t cmd_syndrome;
1120 uint8_t cmd_reserved1[8];
1121 } __packed __aligned(4);
1122
1123 struct mcx_cmd_set_driver_version {
1124 uint8_t cmd_driver_version[64];
1125 } __packed __aligned(8);
1126
1127 struct mcx_cmd_modify_nic_vport_context_in {
1128 uint16_t cmd_opcode;
1129 uint8_t cmd_reserved0[4];
1130 uint16_t cmd_op_mod;
1131 uint8_t cmd_reserved1[4];
1132 uint32_t cmd_field_select;
1133 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04
1134 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10
1135 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40
1136 } __packed __aligned(4);
1137
1138 struct mcx_cmd_modify_nic_vport_context_out {
1139 uint8_t cmd_status;
1140 uint8_t cmd_reserved0[3];
1141 uint32_t cmd_syndrome;
1142 uint8_t cmd_reserved1[8];
1143 } __packed __aligned(4);
1144
1145 struct mcx_cmd_query_nic_vport_context_in {
1146 uint16_t cmd_opcode;
1147 uint8_t cmd_reserved0[4];
1148 uint16_t cmd_op_mod;
1149 uint8_t cmd_reserved1[4];
1150 uint8_t cmd_allowed_list_type;
1151 uint8_t cmd_reserved2[3];
1152 } __packed __aligned(4);
1153
1154 struct mcx_cmd_query_nic_vport_context_out {
1155 uint8_t cmd_status;
1156 uint8_t cmd_reserved0[3];
1157 uint32_t cmd_syndrome;
1158 uint8_t cmd_reserved1[8];
1159 } __packed __aligned(4);
1160
1161 struct mcx_nic_vport_ctx {
1162 uint32_t vp_min_wqe_inline_mode;
1163 uint8_t vp_reserved0[32];
1164 uint32_t vp_mtu;
1165 uint8_t vp_reserved1[200];
1166 uint16_t vp_flags;
1167 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0)
1168 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24)
1169 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24)
1170 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13)
1171 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14)
1172 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15)
1173 uint16_t vp_allowed_list_size;
1174 uint64_t vp_perm_addr;
1175 uint8_t vp_reserved2[4];
1176 /* allowed list follows */
1177 } __packed __aligned(4);
1178
1179 struct mcx_counter {
1180 uint64_t packets;
1181 uint64_t octets;
1182 } __packed __aligned(4);
1183
1184 struct mcx_nic_vport_counters {
1185 struct mcx_counter rx_err;
1186 struct mcx_counter tx_err;
1187 uint8_t reserved0[64]; /* 0x30 */
1188 struct mcx_counter rx_bcast;
1189 struct mcx_counter tx_bcast;
1190 struct mcx_counter rx_ucast;
1191 struct mcx_counter tx_ucast;
1192 struct mcx_counter rx_mcast;
1193 struct mcx_counter tx_mcast;
1194 uint8_t reserved1[0x210 - 0xd0];
1195 } __packed __aligned(4);
1196
1197 struct mcx_cmd_query_vport_counters_in {
1198 uint16_t cmd_opcode;
1199 uint8_t cmd_reserved0[4];
1200 uint16_t cmd_op_mod;
1201 uint8_t cmd_reserved1[8];
1202 } __packed __aligned(4);
1203
1204 struct mcx_cmd_query_vport_counters_mb_in {
1205 uint8_t cmd_reserved0[8];
1206 uint8_t cmd_clear;
1207 uint8_t cmd_reserved1[7];
1208 } __packed __aligned(4);
1209
1210 struct mcx_cmd_query_vport_counters_out {
1211 uint8_t cmd_status;
1212 uint8_t cmd_reserved0[3];
1213 uint32_t cmd_syndrome;
1214 uint8_t cmd_reserved1[8];
1215 } __packed __aligned(4);
1216
1217 struct mcx_cmd_query_flow_counter_in {
1218 uint16_t cmd_opcode;
1219 uint8_t cmd_reserved0[4];
1220 uint16_t cmd_op_mod;
1221 uint8_t cmd_reserved1[8];
1222 } __packed __aligned(4);
1223
1224 struct mcx_cmd_query_flow_counter_mb_in {
1225 uint8_t cmd_reserved0[8];
1226 uint8_t cmd_clear;
1227 uint8_t cmd_reserved1[5];
1228 uint16_t cmd_flow_counter_id;
1229 } __packed __aligned(4);
1230
1231 struct mcx_cmd_query_flow_counter_out {
1232 uint8_t cmd_status;
1233 uint8_t cmd_reserved0[3];
1234 uint32_t cmd_syndrome;
1235 uint8_t cmd_reserved1[8];
1236 } __packed __aligned(4);
1237
1238 struct mcx_cmd_alloc_uar_in {
1239 uint16_t cmd_opcode;
1240 uint8_t cmd_reserved0[4];
1241 uint16_t cmd_op_mod;
1242 uint8_t cmd_reserved1[8];
1243 } __packed __aligned(4);
1244
1245 struct mcx_cmd_alloc_uar_out {
1246 uint8_t cmd_status;
1247 uint8_t cmd_reserved0[3];
1248 uint32_t cmd_syndrome;
1249 uint32_t cmd_uar;
1250 uint8_t cmd_reserved1[4];
1251 } __packed __aligned(4);
1252
1253 struct mcx_cmd_query_special_ctx_in {
1254 uint16_t cmd_opcode;
1255 uint8_t cmd_reserved0[4];
1256 uint16_t cmd_op_mod;
1257 uint8_t cmd_reserved1[8];
1258 } __packed __aligned(4);
1259
1260 struct mcx_cmd_query_special_ctx_out {
1261 uint8_t cmd_status;
1262 uint8_t cmd_reserved0[3];
1263 uint32_t cmd_syndrome;
1264 uint8_t cmd_reserved1[4];
1265 uint32_t cmd_resd_lkey;
1266 } __packed __aligned(4);
1267
1268 struct mcx_eq_ctx {
1269 uint32_t eq_status;
1270 #define MCX_EQ_CTX_STATE_SHIFT 8
1271 #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT)
1272 #define MCX_EQ_CTX_STATE_ARMED 0x9
1273 #define MCX_EQ_CTX_STATE_FIRED 0xa
1274 #define MCX_EQ_CTX_OI_SHIFT 17
1275 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT)
1276 #define MCX_EQ_CTX_EC_SHIFT 18
1277 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT)
1278 #define MCX_EQ_CTX_STATUS_SHIFT 28
1279 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT)
1280 #define MCX_EQ_CTX_STATUS_OK 0x0
1281 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa
1282 uint32_t eq_reserved1;
1283 uint32_t eq_page_offset;
1284 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5
1285 uint32_t eq_uar_size;
1286 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff
1287 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24
1288 uint32_t eq_reserved2;
1289 uint8_t eq_reserved3[3];
1290 uint8_t eq_intr;
1291 uint32_t eq_log_page_size;
1292 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24
1293 uint32_t eq_reserved4[3];
1294 uint32_t eq_consumer_counter;
1295 uint32_t eq_producer_counter;
1296 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff
1297 uint32_t eq_reserved5[4];
1298 } __packed __aligned(4);
1299
1300 CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
1301
1302 struct mcx_cmd_create_eq_in {
1303 uint16_t cmd_opcode;
1304 uint8_t cmd_reserved0[4];
1305 uint16_t cmd_op_mod;
1306 uint8_t cmd_reserved1[8];
1307 } __packed __aligned(4);
1308
1309 struct mcx_cmd_create_eq_mb_in {
1310 struct mcx_eq_ctx cmd_eq_ctx;
1311 uint8_t cmd_reserved0[8];
1312 uint64_t cmd_event_bitmask;
1313 #define MCX_EVENT_TYPE_COMPLETION 0x00
1314 #define MCX_EVENT_TYPE_CQ_ERROR 0x04
1315 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08
1316 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09
1317 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a
1318 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b
1319 #define MCX_EVENT_TYPE_LAST_WQE 0x13
1320 uint8_t cmd_reserved1[176];
1321 } __packed __aligned(4);
1322
1323 struct mcx_cmd_create_eq_out {
1324 uint8_t cmd_status;
1325 uint8_t cmd_reserved0[3];
1326 uint32_t cmd_syndrome;
1327 uint32_t cmd_eqn;
1328 uint8_t cmd_reserved1[4];
1329 } __packed __aligned(4);
1330
1331 struct mcx_cmd_query_eq_in {
1332 uint16_t cmd_opcode;
1333 uint8_t cmd_reserved0[4];
1334 uint16_t cmd_op_mod;
1335 uint32_t cmd_eqn;
1336 uint8_t cmd_reserved1[4];
1337 } __packed __aligned(4);
1338
1339 struct mcx_cmd_query_eq_out {
1340 uint8_t cmd_status;
1341 uint8_t cmd_reserved0[3];
1342 uint32_t cmd_syndrome;
1343 uint8_t cmd_reserved1[8];
1344 } __packed __aligned(4);
1345
1346 struct mcx_eq_entry {
1347 uint8_t eq_reserved1;
1348 uint8_t eq_event_type;
1349 uint8_t eq_reserved2;
1350 uint8_t eq_event_sub_type;
1351
1352 uint8_t eq_reserved3[28];
1353 uint32_t eq_event_data[7];
1354 uint8_t eq_reserved4[2];
1355 uint8_t eq_signature;
1356 uint8_t eq_owner;
1357 #define MCX_EQ_ENTRY_OWNER_INIT 1
1358 } __packed __aligned(4);
1359
1360 CTASSERT(sizeof(struct mcx_eq_entry) == 64);
1361
1362 struct mcx_cmd_alloc_pd_in {
1363 uint16_t cmd_opcode;
1364 uint8_t cmd_reserved0[4];
1365 uint16_t cmd_op_mod;
1366 uint8_t cmd_reserved1[8];
1367 } __packed __aligned(4);
1368
1369 struct mcx_cmd_alloc_pd_out {
1370 uint8_t cmd_status;
1371 uint8_t cmd_reserved0[3];
1372 uint32_t cmd_syndrome;
1373 uint32_t cmd_pd;
1374 uint8_t cmd_reserved1[4];
1375 } __packed __aligned(4);
1376
1377 struct mcx_cmd_alloc_td_in {
1378 uint16_t cmd_opcode;
1379 uint8_t cmd_reserved0[4];
1380 uint16_t cmd_op_mod;
1381 uint8_t cmd_reserved1[8];
1382 } __packed __aligned(4);
1383
1384 struct mcx_cmd_alloc_td_out {
1385 uint8_t cmd_status;
1386 uint8_t cmd_reserved0[3];
1387 uint32_t cmd_syndrome;
1388 uint32_t cmd_tdomain;
1389 uint8_t cmd_reserved1[4];
1390 } __packed __aligned(4);
1391
1392 struct mcx_cmd_create_tir_in {
1393 uint16_t cmd_opcode;
1394 uint8_t cmd_reserved0[4];
1395 uint16_t cmd_op_mod;
1396 uint8_t cmd_reserved1[8];
1397 } __packed __aligned(4);
1398
1399 struct mcx_cmd_create_tir_mb_in {
1400 uint8_t cmd_reserved0[20];
1401 uint32_t cmd_disp_type;
1402 #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0
1403 #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1
1404 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28
1405 uint8_t cmd_reserved1[8];
1406 uint32_t cmd_lro;
1407 uint8_t cmd_reserved2[8];
1408 uint32_t cmd_inline_rqn;
1409 uint32_t cmd_indir_table;
1410 uint32_t cmd_tdomain;
1411 #define MCX_TIR_CTX_HASH_TOEPLITZ 2
1412 #define MCX_TIR_CTX_HASH_SHIFT 28
1413 uint8_t cmd_rx_hash_key[40];
1414 uint32_t cmd_rx_hash_sel_outer;
1415 #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0)
1416 #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1)
1417 #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2)
1418 #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3)
1419 #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31)
1420 #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31)
1421 #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30)
1422 #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30)
1423 uint32_t cmd_rx_hash_sel_inner;
1424 uint8_t cmd_reserved3[152];
1425 } __packed __aligned(4);
1426
1427 struct mcx_cmd_create_tir_out {
1428 uint8_t cmd_status;
1429 uint8_t cmd_reserved0[3];
1430 uint32_t cmd_syndrome;
1431 uint32_t cmd_tirn;
1432 uint8_t cmd_reserved1[4];
1433 } __packed __aligned(4);
1434
1435 struct mcx_cmd_destroy_tir_in {
1436 uint16_t cmd_opcode;
1437 uint8_t cmd_reserved0[4];
1438 uint16_t cmd_op_mod;
1439 uint32_t cmd_tirn;
1440 uint8_t cmd_reserved1[4];
1441 } __packed __aligned(4);
1442
1443 struct mcx_cmd_destroy_tir_out {
1444 uint8_t cmd_status;
1445 uint8_t cmd_reserved0[3];
1446 uint32_t cmd_syndrome;
1447 uint8_t cmd_reserved1[8];
1448 } __packed __aligned(4);
1449
1450 struct mcx_cmd_create_tis_in {
1451 uint16_t cmd_opcode;
1452 uint8_t cmd_reserved0[4];
1453 uint16_t cmd_op_mod;
1454 uint8_t cmd_reserved1[8];
1455 } __packed __aligned(4);
1456
1457 struct mcx_cmd_create_tis_mb_in {
1458 uint8_t cmd_reserved[16];
1459 uint32_t cmd_prio;
1460 uint8_t cmd_reserved1[32];
1461 uint32_t cmd_tdomain;
1462 uint8_t cmd_reserved2[120];
1463 } __packed __aligned(4);
1464
1465 struct mcx_cmd_create_tis_out {
1466 uint8_t cmd_status;
1467 uint8_t cmd_reserved0[3];
1468 uint32_t cmd_syndrome;
1469 uint32_t cmd_tisn;
1470 uint8_t cmd_reserved1[4];
1471 } __packed __aligned(4);
1472
1473 struct mcx_cmd_destroy_tis_in {
1474 uint16_t cmd_opcode;
1475 uint8_t cmd_reserved0[4];
1476 uint16_t cmd_op_mod;
1477 uint32_t cmd_tisn;
1478 uint8_t cmd_reserved1[4];
1479 } __packed __aligned(4);
1480
1481 struct mcx_cmd_destroy_tis_out {
1482 uint8_t cmd_status;
1483 uint8_t cmd_reserved0[3];
1484 uint32_t cmd_syndrome;
1485 uint8_t cmd_reserved1[8];
1486 } __packed __aligned(4);
1487
1488 struct mcx_cmd_create_rqt_in {
1489 uint16_t cmd_opcode;
1490 uint8_t cmd_reserved0[4];
1491 uint16_t cmd_op_mod;
1492 uint8_t cmd_reserved1[8];
1493 } __packed __aligned(4);
1494
1495 struct mcx_rqt_ctx {
1496 uint8_t cmd_reserved0[20];
1497 uint16_t cmd_reserved1;
1498 uint16_t cmd_rqt_max_size;
1499 uint16_t cmd_reserved2;
1500 uint16_t cmd_rqt_actual_size;
1501 uint8_t cmd_reserved3[212];
1502 } __packed __aligned(4);
1503
1504 struct mcx_cmd_create_rqt_mb_in {
1505 uint8_t cmd_reserved0[16];
1506 struct mcx_rqt_ctx cmd_rqt;
1507 } __packed __aligned(4);
1508
1509 struct mcx_cmd_create_rqt_out {
1510 uint8_t cmd_status;
1511 uint8_t cmd_reserved0[3];
1512 uint32_t cmd_syndrome;
1513 uint32_t cmd_rqtn;
1514 uint8_t cmd_reserved1[4];
1515 } __packed __aligned(4);
1516
1517 struct mcx_cmd_destroy_rqt_in {
1518 uint16_t cmd_opcode;
1519 uint8_t cmd_reserved0[4];
1520 uint16_t cmd_op_mod;
1521 uint32_t cmd_rqtn;
1522 uint8_t cmd_reserved1[4];
1523 } __packed __aligned(4);
1524
1525 struct mcx_cmd_destroy_rqt_out {
1526 uint8_t cmd_status;
1527 uint8_t cmd_reserved0[3];
1528 uint32_t cmd_syndrome;
1529 uint8_t cmd_reserved1[8];
1530 } __packed __aligned(4);
1531
1532 struct mcx_cq_ctx {
1533 uint32_t cq_status;
1534 #define MCX_CQ_CTX_STATUS_SHIFT 28
1535 #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT)
1536 #define MCX_CQ_CTX_STATUS_OK 0x0
1537 #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9
1538 #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa
1539 #define MCX_CQ_CTX_STATE_SHIFT 8
1540 #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT)
1541 #define MCX_CQ_CTX_STATE_SOLICITED 0x6
1542 #define MCX_CQ_CTX_STATE_ARMED 0x9
1543 #define MCX_CQ_CTX_STATE_FIRED 0xa
1544 uint32_t cq_reserved1;
1545 uint32_t cq_page_offset;
1546 uint32_t cq_uar_size;
1547 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff
1548 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24
1549 uint32_t cq_period_max_count;
1550 #define MCX_CQ_CTX_PERIOD_SHIFT 16
1551 uint32_t cq_eqn;
1552 uint32_t cq_log_page_size;
1553 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24
1554 uint32_t cq_reserved2;
1555 uint32_t cq_last_notified;
1556 uint32_t cq_last_solicit;
1557 uint32_t cq_consumer_counter;
1558 uint32_t cq_producer_counter;
1559 uint8_t cq_reserved3[8];
1560 uint64_t cq_doorbell;
1561 } __packed __aligned(4);
1562
1563 CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
1564
1565 struct mcx_cmd_create_cq_in {
1566 uint16_t cmd_opcode;
1567 uint8_t cmd_reserved0[4];
1568 uint16_t cmd_op_mod;
1569 uint8_t cmd_reserved1[8];
1570 } __packed __aligned(4);
1571
1572 struct mcx_cmd_create_cq_mb_in {
1573 struct mcx_cq_ctx cmd_cq_ctx;
1574 uint8_t cmd_reserved1[192];
1575 } __packed __aligned(4);
1576
1577 struct mcx_cmd_create_cq_out {
1578 uint8_t cmd_status;
1579 uint8_t cmd_reserved0[3];
1580 uint32_t cmd_syndrome;
1581 uint32_t cmd_cqn;
1582 uint8_t cmd_reserved1[4];
1583 } __packed __aligned(4);
1584
1585 struct mcx_cmd_destroy_cq_in {
1586 uint16_t cmd_opcode;
1587 uint8_t cmd_reserved0[4];
1588 uint16_t cmd_op_mod;
1589 uint32_t cmd_cqn;
1590 uint8_t cmd_reserved1[4];
1591 } __packed __aligned(4);
1592
1593 struct mcx_cmd_destroy_cq_out {
1594 uint8_t cmd_status;
1595 uint8_t cmd_reserved0[3];
1596 uint32_t cmd_syndrome;
1597 uint8_t cmd_reserved1[8];
1598 } __packed __aligned(4);
1599
1600 struct mcx_cmd_query_cq_in {
1601 uint16_t cmd_opcode;
1602 uint8_t cmd_reserved0[4];
1603 uint16_t cmd_op_mod;
1604 uint32_t cmd_cqn;
1605 uint8_t cmd_reserved1[4];
1606 } __packed __aligned(4);
1607
1608 struct mcx_cmd_query_cq_out {
1609 uint8_t cmd_status;
1610 uint8_t cmd_reserved0[3];
1611 uint32_t cmd_syndrome;
1612 uint8_t cmd_reserved1[8];
1613 } __packed __aligned(4);
1614
1615 struct mcx_cq_entry {
1616 uint32_t __reserved__;
1617 uint32_t cq_lro;
1618 uint32_t cq_lro_ack_seq_num;
1619 uint32_t cq_rx_hash;
1620 uint8_t cq_rx_hash_type;
1621 uint8_t cq_ml_path;
1622 uint16_t __reserved__;
1623 uint32_t cq_checksum;
1624 uint32_t __reserved__;
1625 uint32_t cq_flags;
1626 #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26)
1627 #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25)
1628 #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24)
1629 #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16)
1630 #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff)
1631
1632 uint32_t cq_lro_srqn;
1633 uint32_t __reserved__[2];
1634 uint32_t cq_byte_cnt;
1635 uint64_t cq_timestamp;
1636 uint8_t cq_rx_drops;
1637 uint8_t cq_flow_tag[3];
1638 uint16_t cq_wqe_count;
1639 uint8_t cq_signature;
1640 uint8_t cq_opcode_owner;
1641 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0)
1642 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1)
1643 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2
1644 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4
1645
1646 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0
1647 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1
1648 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2
1649 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3
1650
1651 #define MCX_CQ_ENTRY_OPCODE_REQ 0
1652 #define MCX_CQ_ENTRY_OPCODE_SEND 2
1653 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13
1654 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14
1655 #define MCX_CQ_ENTRY_OPCODE_INVALID 15
1656
1657 } __packed __aligned(4);
1658
1659 CTASSERT(sizeof(struct mcx_cq_entry) == 64);
1660
1661 struct mcx_cq_doorbell {
1662 uint32_t db_update_ci;
1663 uint32_t db_arm_ci;
1664 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28
1665 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24)
1666 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff)
1667 } __packed __aligned(8);
1668
1669 struct mcx_wq_ctx {
1670 uint8_t wq_type;
1671 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4)
1672 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3)
1673 uint8_t wq_reserved0[5];
1674 uint16_t wq_lwm;
1675 uint32_t wq_pd;
1676 uint32_t wq_uar_page;
1677 uint64_t wq_doorbell;
1678 uint32_t wq_hw_counter;
1679 uint32_t wq_sw_counter;
1680 uint16_t wq_log_stride;
1681 uint8_t wq_log_page_sz;
1682 uint8_t wq_log_size;
1683 uint8_t wq_reserved1[156];
1684 } __packed __aligned(4);
1685
1686 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
1687
1688 struct mcx_sq_ctx {
1689 uint32_t sq_flags;
1690 #define MCX_SQ_CTX_RLKEY (1U << 31)
1691 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29)
1692 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28)
1693 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24
1694 #define MCX_SQ_CTX_STATE_SHIFT 20
1695 #define MCX_SQ_CTX_STATE_MASK (0xf << 20)
1696 #define MCX_SQ_CTX_STATE_RST 0
1697 #define MCX_SQ_CTX_STATE_RDY 1
1698 #define MCX_SQ_CTX_STATE_ERR 3
1699 uint32_t sq_user_index;
1700 uint32_t sq_cqn;
1701 uint32_t sq_reserved1[5];
1702 uint32_t sq_tis_lst_sz;
1703 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16
1704 uint32_t sq_reserved2[2];
1705 uint32_t sq_tis_num;
1706 struct mcx_wq_ctx sq_wq;
1707 } __packed __aligned(4);
1708
1709 struct mcx_sq_entry_seg {
1710 uint32_t sqs_byte_count;
1711 uint32_t sqs_lkey;
1712 uint64_t sqs_addr;
1713 } __packed __aligned(4);
1714
1715 struct mcx_sq_entry {
1716 /* control segment */
1717 uint32_t sqe_opcode_index;
1718 #define MCX_SQE_WQE_INDEX_SHIFT 8
1719 #define MCX_SQE_WQE_OPCODE_NOP 0x00
1720 #define MCX_SQE_WQE_OPCODE_SEND 0x0a
1721 uint32_t sqe_ds_sq_num;
1722 #define MCX_SQE_SQ_NUM_SHIFT 8
1723 uint32_t sqe_signature;
1724 #define MCX_SQE_SIGNATURE_SHIFT 24
1725 #define MCX_SQE_SOLICITED_EVENT 0x02
1726 #define MCX_SQE_CE_CQE_ON_ERR 0x00
1727 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04
1728 #define MCX_SQE_CE_CQE_ALWAYS 0x08
1729 #define MCX_SQE_CE_CQE_SOLICIT 0x0C
1730 #define MCX_SQE_FM_NO_FENCE 0x00
1731 #define MCX_SQE_FM_SMALL_FENCE 0x40
1732 uint32_t sqe_mkey;
1733
1734 /* ethernet segment */
1735 uint32_t sqe_reserved1;
1736 uint32_t sqe_mss_csum;
1737 #define MCX_SQE_L4_CSUM (1 << 31)
1738 #define MCX_SQE_L3_CSUM (1 << 30)
1739 uint32_t sqe_reserved2;
1740 uint16_t sqe_inline_header_size;
1741 uint16_t sqe_inline_headers[9];
1742
1743 /* data segment */
1744 struct mcx_sq_entry_seg sqe_segs[1];
1745 } __packed __aligned(64);
1746
1747 CTASSERT(sizeof(struct mcx_sq_entry) == 64);
1748
1749 struct mcx_cmd_create_sq_in {
1750 uint16_t cmd_opcode;
1751 uint8_t cmd_reserved0[4];
1752 uint16_t cmd_op_mod;
1753 uint8_t cmd_reserved1[8];
1754 } __packed __aligned(4);
1755
1756 struct mcx_cmd_create_sq_out {
1757 uint8_t cmd_status;
1758 uint8_t cmd_reserved0[3];
1759 uint32_t cmd_syndrome;
1760 uint32_t cmd_sqn;
1761 uint8_t cmd_reserved1[4];
1762 } __packed __aligned(4);
1763
1764 struct mcx_cmd_modify_sq_in {
1765 uint16_t cmd_opcode;
1766 uint8_t cmd_reserved0[4];
1767 uint16_t cmd_op_mod;
1768 uint32_t cmd_sq_state;
1769 uint8_t cmd_reserved1[4];
1770 } __packed __aligned(4);
1771
1772 struct mcx_cmd_modify_sq_mb_in {
1773 uint32_t cmd_modify_hi;
1774 uint32_t cmd_modify_lo;
1775 uint8_t cmd_reserved0[8];
1776 struct mcx_sq_ctx cmd_sq_ctx;
1777 } __packed __aligned(4);
1778
1779 struct mcx_cmd_modify_sq_out {
1780 uint8_t cmd_status;
1781 uint8_t cmd_reserved0[3];
1782 uint32_t cmd_syndrome;
1783 uint8_t cmd_reserved1[8];
1784 } __packed __aligned(4);
1785
1786 struct mcx_cmd_destroy_sq_in {
1787 uint16_t cmd_opcode;
1788 uint8_t cmd_reserved0[4];
1789 uint16_t cmd_op_mod;
1790 uint32_t cmd_sqn;
1791 uint8_t cmd_reserved1[4];
1792 } __packed __aligned(4);
1793
1794 struct mcx_cmd_destroy_sq_out {
1795 uint8_t cmd_status;
1796 uint8_t cmd_reserved0[3];
1797 uint32_t cmd_syndrome;
1798 uint8_t cmd_reserved1[8];
1799 } __packed __aligned(4);
1800
1801
1802 struct mcx_rq_ctx {
1803 uint32_t rq_flags;
1804 #define MCX_RQ_CTX_RLKEY (1U << 31)
1805 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28)
1806 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24
1807 #define MCX_RQ_CTX_STATE_SHIFT 20
1808 #define MCX_RQ_CTX_STATE_MASK (0xf << 20)
1809 #define MCX_RQ_CTX_STATE_RST 0
1810 #define MCX_RQ_CTX_STATE_RDY 1
1811 #define MCX_RQ_CTX_STATE_ERR 3
1812 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18)
1813 uint32_t rq_user_index;
1814 uint32_t rq_cqn;
1815 uint32_t rq_reserved1;
1816 uint32_t rq_rmpn;
1817 uint32_t rq_reserved2[7];
1818 struct mcx_wq_ctx rq_wq;
1819 } __packed __aligned(4);
1820
1821 struct mcx_rq_entry {
1822 uint32_t rqe_byte_count;
1823 uint32_t rqe_lkey;
1824 uint64_t rqe_addr;
1825 } __packed __aligned(16);
1826
1827 struct mcx_cmd_create_rq_in {
1828 uint16_t cmd_opcode;
1829 uint8_t cmd_reserved0[4];
1830 uint16_t cmd_op_mod;
1831 uint8_t cmd_reserved1[8];
1832 } __packed __aligned(4);
1833
1834 struct mcx_cmd_create_rq_out {
1835 uint8_t cmd_status;
1836 uint8_t cmd_reserved0[3];
1837 uint32_t cmd_syndrome;
1838 uint32_t cmd_rqn;
1839 uint8_t cmd_reserved1[4];
1840 } __packed __aligned(4);
1841
1842 struct mcx_cmd_modify_rq_in {
1843 uint16_t cmd_opcode;
1844 uint8_t cmd_reserved0[4];
1845 uint16_t cmd_op_mod;
1846 uint32_t cmd_rq_state;
1847 uint8_t cmd_reserved1[4];
1848 } __packed __aligned(4);
1849
1850 struct mcx_cmd_modify_rq_mb_in {
1851 uint32_t cmd_modify_hi;
1852 uint32_t cmd_modify_lo;
1853 uint8_t cmd_reserved0[8];
1854 struct mcx_rq_ctx cmd_rq_ctx;
1855 } __packed __aligned(4);
1856
1857 struct mcx_cmd_modify_rq_out {
1858 uint8_t cmd_status;
1859 uint8_t cmd_reserved0[3];
1860 uint32_t cmd_syndrome;
1861 uint8_t cmd_reserved1[8];
1862 } __packed __aligned(4);
1863
1864 struct mcx_cmd_destroy_rq_in {
1865 uint16_t cmd_opcode;
1866 uint8_t cmd_reserved0[4];
1867 uint16_t cmd_op_mod;
1868 uint32_t cmd_rqn;
1869 uint8_t cmd_reserved1[4];
1870 } __packed __aligned(4);
1871
1872 struct mcx_cmd_destroy_rq_out {
1873 uint8_t cmd_status;
1874 uint8_t cmd_reserved0[3];
1875 uint32_t cmd_syndrome;
1876 uint8_t cmd_reserved1[8];
1877 } __packed __aligned(4);
1878
1879 struct mcx_cmd_create_flow_table_in {
1880 uint16_t cmd_opcode;
1881 uint8_t cmd_reserved0[4];
1882 uint16_t cmd_op_mod;
1883 uint8_t cmd_reserved1[8];
1884 } __packed __aligned(4);
1885
1886 struct mcx_flow_table_ctx {
1887 uint8_t ft_miss_action;
1888 uint8_t ft_level;
1889 uint8_t ft_reserved0;
1890 uint8_t ft_log_size;
1891 uint32_t ft_table_miss_id;
1892 uint8_t ft_reserved1[28];
1893 } __packed __aligned(4);
1894
1895 struct mcx_cmd_create_flow_table_mb_in {
1896 uint8_t cmd_table_type;
1897 uint8_t cmd_reserved0[7];
1898 struct mcx_flow_table_ctx cmd_ctx;
1899 } __packed __aligned(4);
1900
1901 struct mcx_cmd_create_flow_table_out {
1902 uint8_t cmd_status;
1903 uint8_t cmd_reserved0[3];
1904 uint32_t cmd_syndrome;
1905 uint32_t cmd_table_id;
1906 uint8_t cmd_reserved1[4];
1907 } __packed __aligned(4);
1908
1909 struct mcx_cmd_destroy_flow_table_in {
1910 uint16_t cmd_opcode;
1911 uint8_t cmd_reserved0[4];
1912 uint16_t cmd_op_mod;
1913 uint8_t cmd_reserved1[8];
1914 } __packed __aligned(4);
1915
1916 struct mcx_cmd_destroy_flow_table_mb_in {
1917 uint8_t cmd_table_type;
1918 uint8_t cmd_reserved0[3];
1919 uint32_t cmd_table_id;
1920 uint8_t cmd_reserved1[40];
1921 } __packed __aligned(4);
1922
1923 struct mcx_cmd_destroy_flow_table_out {
1924 uint8_t cmd_status;
1925 uint8_t cmd_reserved0[3];
1926 uint32_t cmd_syndrome;
1927 uint8_t cmd_reserved1[8];
1928 } __packed __aligned(4);
1929
1930 struct mcx_cmd_set_flow_table_root_in {
1931 uint16_t cmd_opcode;
1932 uint8_t cmd_reserved0[4];
1933 uint16_t cmd_op_mod;
1934 uint8_t cmd_reserved1[8];
1935 } __packed __aligned(4);
1936
1937 struct mcx_cmd_set_flow_table_root_mb_in {
1938 uint8_t cmd_table_type;
1939 uint8_t cmd_reserved0[3];
1940 uint32_t cmd_table_id;
1941 uint8_t cmd_reserved1[56];
1942 } __packed __aligned(4);
1943
1944 struct mcx_cmd_set_flow_table_root_out {
1945 uint8_t cmd_status;
1946 uint8_t cmd_reserved0[3];
1947 uint32_t cmd_syndrome;
1948 uint8_t cmd_reserved1[8];
1949 } __packed __aligned(4);
1950
1951 struct mcx_flow_match {
1952 /* outer headers */
1953 uint8_t mc_src_mac[6];
1954 uint16_t mc_ethertype;
1955 uint8_t mc_dest_mac[6];
1956 uint16_t mc_first_vlan;
1957 uint8_t mc_ip_proto;
1958 uint8_t mc_ip_dscp_ecn;
1959 uint8_t mc_vlan_flags;
1960 #define MCX_FLOW_MATCH_IP_FRAG (1 << 5)
1961 uint8_t mc_tcp_flags;
1962 uint16_t mc_tcp_sport;
1963 uint16_t mc_tcp_dport;
1964 uint32_t mc_reserved0;
1965 uint16_t mc_udp_sport;
1966 uint16_t mc_udp_dport;
1967 uint8_t mc_src_ip[16];
1968 uint8_t mc_dest_ip[16];
1969
1970 /* misc parameters */
1971 uint8_t mc_reserved1[8];
1972 uint16_t mc_second_vlan;
1973 uint8_t mc_reserved2[2];
1974 uint8_t mc_second_vlan_flags;
1975 uint8_t mc_reserved3[15];
1976 uint32_t mc_outer_ipv6_flow_label;
1977 uint8_t mc_reserved4[32];
1978
1979 uint8_t mc_reserved[384];
1980 } __packed __aligned(4);
1981
1982 CTASSERT(sizeof(struct mcx_flow_match) == 512);
1983
1984 struct mcx_cmd_create_flow_group_in {
1985 uint16_t cmd_opcode;
1986 uint8_t cmd_reserved0[4];
1987 uint16_t cmd_op_mod;
1988 uint8_t cmd_reserved1[8];
1989 } __packed __aligned(4);
1990
1991 struct mcx_cmd_create_flow_group_mb_in {
1992 uint8_t cmd_table_type;
1993 uint8_t cmd_reserved0[3];
1994 uint32_t cmd_table_id;
1995 uint8_t cmd_reserved1[4];
1996 uint32_t cmd_start_flow_index;
1997 uint8_t cmd_reserved2[4];
1998 uint32_t cmd_end_flow_index;
1999 uint8_t cmd_reserved3[23];
2000 uint8_t cmd_match_criteria_enable;
2001 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0)
2002 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1)
2003 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2)
2004 struct mcx_flow_match cmd_match_criteria;
2005 uint8_t cmd_reserved4[448];
2006 } __packed __aligned(4);
2007
2008 struct mcx_cmd_create_flow_group_out {
2009 uint8_t cmd_status;
2010 uint8_t cmd_reserved0[3];
2011 uint32_t cmd_syndrome;
2012 uint32_t cmd_group_id;
2013 uint8_t cmd_reserved1[4];
2014 } __packed __aligned(4);
2015
2016 struct mcx_flow_ctx {
2017 uint8_t fc_reserved0[4];
2018 uint32_t fc_group_id;
2019 uint32_t fc_flow_tag;
2020 uint32_t fc_action;
2021 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0)
2022 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1)
2023 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2)
2024 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3)
2025 uint32_t fc_dest_list_size;
2026 uint32_t fc_counter_list_size;
2027 uint8_t fc_reserved1[40];
2028 struct mcx_flow_match fc_match_value;
2029 uint8_t fc_reserved2[192];
2030 } __packed __aligned(4);
2031
2032 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24)
2033 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24)
2034
2035 struct mcx_cmd_destroy_flow_group_in {
2036 uint16_t cmd_opcode;
2037 uint8_t cmd_reserved0[4];
2038 uint16_t cmd_op_mod;
2039 uint8_t cmd_reserved1[8];
2040 } __packed __aligned(4);
2041
2042 struct mcx_cmd_destroy_flow_group_mb_in {
2043 uint8_t cmd_table_type;
2044 uint8_t cmd_reserved0[3];
2045 uint32_t cmd_table_id;
2046 uint32_t cmd_group_id;
2047 uint8_t cmd_reserved1[36];
2048 } __packed __aligned(4);
2049
2050 struct mcx_cmd_destroy_flow_group_out {
2051 uint8_t cmd_status;
2052 uint8_t cmd_reserved0[3];
2053 uint32_t cmd_syndrome;
2054 uint8_t cmd_reserved1[8];
2055 } __packed __aligned(4);
2056
2057 struct mcx_cmd_set_flow_table_entry_in {
2058 uint16_t cmd_opcode;
2059 uint8_t cmd_reserved0[4];
2060 uint16_t cmd_op_mod;
2061 uint8_t cmd_reserved1[8];
2062 } __packed __aligned(4);
2063
2064 struct mcx_cmd_set_flow_table_entry_mb_in {
2065 uint8_t cmd_table_type;
2066 uint8_t cmd_reserved0[3];
2067 uint32_t cmd_table_id;
2068 uint32_t cmd_modify_enable_mask;
2069 uint8_t cmd_reserved1[4];
2070 uint32_t cmd_flow_index;
2071 uint8_t cmd_reserved2[28];
2072 struct mcx_flow_ctx cmd_flow_ctx;
2073 } __packed __aligned(4);
2074
2075 struct mcx_cmd_set_flow_table_entry_out {
2076 uint8_t cmd_status;
2077 uint8_t cmd_reserved0[3];
2078 uint32_t cmd_syndrome;
2079 uint8_t cmd_reserved1[8];
2080 } __packed __aligned(4);
2081
2082 struct mcx_cmd_query_flow_table_entry_in {
2083 uint16_t cmd_opcode;
2084 uint8_t cmd_reserved0[4];
2085 uint16_t cmd_op_mod;
2086 uint8_t cmd_reserved1[8];
2087 } __packed __aligned(4);
2088
2089 struct mcx_cmd_query_flow_table_entry_mb_in {
2090 uint8_t cmd_table_type;
2091 uint8_t cmd_reserved0[3];
2092 uint32_t cmd_table_id;
2093 uint8_t cmd_reserved1[8];
2094 uint32_t cmd_flow_index;
2095 uint8_t cmd_reserved2[28];
2096 } __packed __aligned(4);
2097
2098 struct mcx_cmd_query_flow_table_entry_out {
2099 uint8_t cmd_status;
2100 uint8_t cmd_reserved0[3];
2101 uint32_t cmd_syndrome;
2102 uint8_t cmd_reserved1[8];
2103 } __packed __aligned(4);
2104
2105 struct mcx_cmd_query_flow_table_entry_mb_out {
2106 uint8_t cmd_reserved0[48];
2107 struct mcx_flow_ctx cmd_flow_ctx;
2108 } __packed __aligned(4);
2109
2110 struct mcx_cmd_delete_flow_table_entry_in {
2111 uint16_t cmd_opcode;
2112 uint8_t cmd_reserved0[4];
2113 uint16_t cmd_op_mod;
2114 uint8_t cmd_reserved1[8];
2115 } __packed __aligned(4);
2116
2117 struct mcx_cmd_delete_flow_table_entry_mb_in {
2118 uint8_t cmd_table_type;
2119 uint8_t cmd_reserved0[3];
2120 uint32_t cmd_table_id;
2121 uint8_t cmd_reserved1[8];
2122 uint32_t cmd_flow_index;
2123 uint8_t cmd_reserved2[28];
2124 } __packed __aligned(4);
2125
2126 struct mcx_cmd_delete_flow_table_entry_out {
2127 uint8_t cmd_status;
2128 uint8_t cmd_reserved0[3];
2129 uint32_t cmd_syndrome;
2130 uint8_t cmd_reserved1[8];
2131 } __packed __aligned(4);
2132
2133 struct mcx_cmd_query_flow_group_in {
2134 uint16_t cmd_opcode;
2135 uint8_t cmd_reserved0[4];
2136 uint16_t cmd_op_mod;
2137 uint8_t cmd_reserved1[8];
2138 } __packed __aligned(4);
2139
2140 struct mcx_cmd_query_flow_group_mb_in {
2141 uint8_t cmd_table_type;
2142 uint8_t cmd_reserved0[3];
2143 uint32_t cmd_table_id;
2144 uint32_t cmd_group_id;
2145 uint8_t cmd_reserved1[36];
2146 } __packed __aligned(4);
2147
2148 struct mcx_cmd_query_flow_group_out {
2149 uint8_t cmd_status;
2150 uint8_t cmd_reserved0[3];
2151 uint32_t cmd_syndrome;
2152 uint8_t cmd_reserved1[8];
2153 } __packed __aligned(4);
2154
2155 struct mcx_cmd_query_flow_group_mb_out {
2156 uint8_t cmd_reserved0[12];
2157 uint32_t cmd_start_flow_index;
2158 uint8_t cmd_reserved1[4];
2159 uint32_t cmd_end_flow_index;
2160 uint8_t cmd_reserved2[20];
2161 uint32_t cmd_match_criteria_enable;
2162 uint8_t cmd_match_criteria[512];
2163 uint8_t cmd_reserved4[448];
2164 } __packed __aligned(4);
2165
2166 struct mcx_cmd_query_flow_table_in {
2167 uint16_t cmd_opcode;
2168 uint8_t cmd_reserved0[4];
2169 uint16_t cmd_op_mod;
2170 uint8_t cmd_reserved1[8];
2171 } __packed __aligned(4);
2172
2173 struct mcx_cmd_query_flow_table_mb_in {
2174 uint8_t cmd_table_type;
2175 uint8_t cmd_reserved0[3];
2176 uint32_t cmd_table_id;
2177 uint8_t cmd_reserved1[40];
2178 } __packed __aligned(4);
2179
2180 struct mcx_cmd_query_flow_table_out {
2181 uint8_t cmd_status;
2182 uint8_t cmd_reserved0[3];
2183 uint32_t cmd_syndrome;
2184 uint8_t cmd_reserved1[8];
2185 } __packed __aligned(4);
2186
2187 struct mcx_cmd_query_flow_table_mb_out {
2188 uint8_t cmd_reserved0[4];
2189 struct mcx_flow_table_ctx cmd_ctx;
2190 } __packed __aligned(4);
2191
2192 struct mcx_cmd_alloc_flow_counter_in {
2193 uint16_t cmd_opcode;
2194 uint8_t cmd_reserved0[4];
2195 uint16_t cmd_op_mod;
2196 uint8_t cmd_reserved1[8];
2197 } __packed __aligned(4);
2198
2199 struct mcx_cmd_query_rq_in {
2200 uint16_t cmd_opcode;
2201 uint8_t cmd_reserved0[4];
2202 uint16_t cmd_op_mod;
2203 uint32_t cmd_rqn;
2204 uint8_t cmd_reserved1[4];
2205 } __packed __aligned(4);
2206
2207 struct mcx_cmd_query_rq_out {
2208 uint8_t cmd_status;
2209 uint8_t cmd_reserved0[3];
2210 uint32_t cmd_syndrome;
2211 uint8_t cmd_reserved1[8];
2212 } __packed __aligned(4);
2213
2214 struct mcx_cmd_query_rq_mb_out {
2215 uint8_t cmd_reserved0[16];
2216 struct mcx_rq_ctx cmd_ctx;
2217 };
2218
2219 struct mcx_cmd_query_sq_in {
2220 uint16_t cmd_opcode;
2221 uint8_t cmd_reserved0[4];
2222 uint16_t cmd_op_mod;
2223 uint32_t cmd_sqn;
2224 uint8_t cmd_reserved1[4];
2225 } __packed __aligned(4);
2226
2227 struct mcx_cmd_query_sq_out {
2228 uint8_t cmd_status;
2229 uint8_t cmd_reserved0[3];
2230 uint32_t cmd_syndrome;
2231 uint8_t cmd_reserved1[8];
2232 } __packed __aligned(4);
2233
2234 struct mcx_cmd_query_sq_mb_out {
2235 uint8_t cmd_reserved0[16];
2236 struct mcx_sq_ctx cmd_ctx;
2237 };
2238
2239 struct mcx_cmd_alloc_flow_counter_out {
2240 uint8_t cmd_status;
2241 uint8_t cmd_reserved0[3];
2242 uint32_t cmd_syndrome;
2243 uint8_t cmd_reserved1[2];
2244 uint16_t cmd_flow_counter_id;
2245 uint8_t cmd_reserved2[4];
2246 } __packed __aligned(4);
2247
2248 struct mcx_wq_doorbell {
2249 uint32_t db_recv_counter;
2250 uint32_t db_send_counter;
2251 } __packed __aligned(8);
2252
2253 struct mcx_dmamem {
2254 bus_dmamap_t mxm_map;
2255 bus_dma_segment_t mxm_seg;
2256 int mxm_nsegs;
2257 size_t mxm_size;
2258 void *mxm_kva;
2259 };
2260 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map)
2261 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr)
2262 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva)
2263 #define MCX_DMA_OFF(_mxm, _off) ((void *)((char *)(_mxm)->mxm_kva + (_off)))
2264 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size)
2265
2266 struct mcx_hwmem {
2267 bus_dmamap_t mhm_map;
2268 bus_dma_segment_t *mhm_segs;
2269 unsigned int mhm_seg_count;
2270 unsigned int mhm_npages;
2271 };
2272
2273 struct mcx_slot {
2274 bus_dmamap_t ms_map;
2275 struct mbuf *ms_m;
2276 };
2277
2278 struct mcx_eq {
2279 int eq_n;
2280 uint32_t eq_cons;
2281 struct mcx_dmamem eq_mem;
2282 };
2283
2284 struct mcx_cq {
2285 int cq_n;
2286 struct mcx_dmamem cq_mem;
2287 bus_addr_t cq_doorbell;
2288 uint32_t cq_cons;
2289 uint32_t cq_count;
2290 };
2291
2292 struct mcx_calibration {
2293 uint64_t c_timestamp; /* previous mcx chip time */
2294 uint64_t c_uptime; /* previous kernel nanouptime */
2295 uint64_t c_tbase; /* mcx chip time */
2296 uint64_t c_ubase; /* kernel nanouptime */
2297 uint64_t c_ratio;
2298 };
2299
2300 #define MCX_CALIBRATE_FIRST 2
2301 #define MCX_CALIBRATE_NORMAL 32
2302
2303 struct mcx_rxring {
2304 u_int rxr_total;
2305 u_int rxr_inuse;
2306 };
2307
2308 MBUFQ_HEAD(mcx_mbufq);
2309
2310 struct mcx_rx {
2311 struct mcx_softc *rx_softc;
2312
2313 int rx_rqn;
2314 struct mcx_dmamem rx_rq_mem;
2315 struct mcx_slot *rx_slots;
2316 bus_addr_t rx_doorbell;
2317
2318 uint32_t rx_prod;
2319 callout_t rx_refill;
2320 struct mcx_rxring rx_rxr;
2321 } __aligned(64);
2322
2323 struct mcx_tx {
2324 struct mcx_softc *tx_softc;
2325 kmutex_t tx_lock;
2326 pcq_t *tx_pcq;
2327 void *tx_softint;
2328
2329 int tx_uar;
2330 int tx_sqn;
2331 struct mcx_dmamem tx_sq_mem;
2332 struct mcx_slot *tx_slots;
2333 bus_addr_t tx_doorbell;
2334 int tx_bf_offset;
2335
2336 uint32_t tx_cons;
2337 uint32_t tx_prod;
2338 } __aligned(64);
2339
2340 struct mcx_queues {
2341 void *q_ihc;
2342 struct mcx_softc *q_sc;
2343 int q_uar;
2344 int q_index;
2345 struct mcx_rx q_rx;
2346 struct mcx_tx q_tx;
2347 struct mcx_cq q_cq;
2348 struct mcx_eq q_eq;
2349 #if NKSTAT > 0
2350 struct kstat *q_kstat;
2351 #endif
2352 };
2353
2354 struct mcx_flow_group {
2355 int g_id;
2356 int g_table;
2357 int g_start;
2358 int g_size;
2359 };
2360
2361 #define MCX_FLOW_GROUP_PROMISC 0
2362 #define MCX_FLOW_GROUP_ALLMULTI 1
2363 #define MCX_FLOW_GROUP_MAC 2
2364 #define MCX_FLOW_GROUP_RSS_L4 3
2365 #define MCX_FLOW_GROUP_RSS_L3 4
2366 #define MCX_FLOW_GROUP_RSS_NONE 5
2367 #define MCX_NUM_FLOW_GROUPS 6
2368
2369 #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \
2370 MCX_TIR_CTX_HASH_SEL_DST_IP
2371 #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \
2372 MCX_TIR_CTX_HASH_SEL_DPORT
2373
2374 #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\
2375 MCX_TIR_CTX_HASH_SEL_IPV4
2376 #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \
2377 MCX_TIR_CTX_HASH_SEL_IPV6
2378 #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
2379 MCX_TIR_CTX_HASH_SEL_IPV4
2380 #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
2381 MCX_TIR_CTX_HASH_SEL_IPV6
2382 #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4
2383 #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6
2384
2385 /*
2386 * There are a few different pieces involved in configuring RSS.
2387 * A Receive Queue Table (RQT) is the indirection table that maps packets to
2388 * different rx queues based on a hash value. We only create one, because
2389 * we want to scatter any traffic we can apply RSS to across all our rx
2390 * queues. Anything else will only be delivered to the first rx queue,
2391 * which doesn't require an RQT.
2392 *
2393 * A Transport Interface Receive (TIR) delivers packets to either a single rx
2394 * queue or an RQT, and in the latter case, specifies the set of fields
2395 * hashed, the hash function, and the hash key. We need one of these for each
2396 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6,
2397 * and one for non-RSS traffic.
2398 *
2399 * Flow tables hold flow table entries in sequence. The first entry that
2400 * matches a packet is applied, sending the packet to either another flow
2401 * table or a TIR. We use one flow table to select packets based on
2402 * destination MAC address, and a second to apply RSS. The entries in the
2403 * first table send matching packets to the second, and the entries in the
2404 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR.
2405 *
2406 * The flow table entry that delivers packets to an RSS TIR must include match
2407 * criteria that ensure packets delivered to the TIR include all the fields
2408 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must
2409 * only accept v4 TCP packets. Accordingly, we need flow table entries for
2410 * each TIR.
2411 *
2412 * All of this is a lot more flexible than we need, and we can describe most
2413 * of the stuff we need with a simple array.
2414 *
2415 * An RSS config creates a TIR with hashing enabled on a set of fields,
2416 * pointing to either the first rx queue or the RQT containing all the rx
2417 * queues, and a flow table entry that matches on an ether type and
2418 * optionally an ip proto, that delivers packets to the TIR.
2419 */
2420 static struct mcx_rss_rule {
2421 int hash_sel;
2422 int flow_group;
2423 int ethertype;
2424 int ip_proto;
2425 } mcx_rss_config[] = {
2426 /* udp and tcp for v4/v6 */
2427 { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4,
2428 ETHERTYPE_IP, IPPROTO_TCP },
2429 { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4,
2430 ETHERTYPE_IPV6, IPPROTO_TCP },
2431 { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4,
2432 ETHERTYPE_IP, IPPROTO_UDP },
2433 { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4,
2434 ETHERTYPE_IPV6, IPPROTO_UDP },
2435
2436 /* other v4/v6 */
2437 { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3,
2438 ETHERTYPE_IP, 0 },
2439 { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3,
2440 ETHERTYPE_IPV6, 0 },
2441
2442 /* non v4/v6 */
2443 { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 }
2444 };
2445
2446 struct mcx_softc {
2447 device_t sc_dev;
2448 struct ethercom sc_ec;
2449 struct ifmedia sc_media;
2450 uint64_t sc_media_status;
2451 uint64_t sc_media_active;
2452 kmutex_t sc_media_mutex;
2453
2454 pci_chipset_tag_t sc_pc;
2455 pci_intr_handle_t *sc_intrs;
2456 void *sc_ihc;
2457 pcitag_t sc_tag;
2458
2459 bus_dma_tag_t sc_dmat;
2460 bus_space_tag_t sc_memt;
2461 bus_space_handle_t sc_memh;
2462 bus_size_t sc_mems;
2463
2464 struct mcx_dmamem sc_cmdq_mem;
2465 unsigned int sc_cmdq_mask;
2466 unsigned int sc_cmdq_size;
2467
2468 unsigned int sc_cmdq_token;
2469
2470 struct mcx_hwmem sc_boot_pages;
2471 struct mcx_hwmem sc_init_pages;
2472 struct mcx_hwmem sc_regular_pages;
2473
2474 int sc_uar;
2475 int sc_pd;
2476 int sc_tdomain;
2477 uint32_t sc_lkey;
2478 int sc_tis;
2479 int sc_tir[__arraycount(mcx_rss_config)];
2480 int sc_rqt;
2481
2482 struct mcx_dmamem sc_doorbell_mem;
2483
2484 struct mcx_eq sc_admin_eq;
2485 struct mcx_eq sc_queue_eq;
2486
2487 int sc_hardmtu;
2488 int sc_rxbufsz;
2489
2490 int sc_bf_size;
2491 int sc_max_rqt_size;
2492
2493 struct workqueue *sc_workq;
2494 struct work sc_port_change;
2495
2496 int sc_mac_flow_table_id;
2497 int sc_rss_flow_table_id;
2498 struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS];
2499 int sc_promisc_flow_enabled;
2500 int sc_allmulti_flow_enabled;
2501 int sc_mcast_flow_base;
2502 int sc_extra_mcast;
2503 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];
2504
2505 struct mcx_calibration sc_calibration[2];
2506 unsigned int sc_calibration_gen;
2507 callout_t sc_calibrate;
2508 uint32_t sc_mhz;
2509 uint32_t sc_khz;
2510
2511 struct mcx_queues *sc_queues;
2512 unsigned int sc_nqueues;
2513
2514 int sc_mcam_reg;
2515
2516 #if NKSTAT > 0
2517 struct kstat *sc_kstat_ieee8023;
2518 struct kstat *sc_kstat_rfc2863;
2519 struct kstat *sc_kstat_rfc2819;
2520 struct kstat *sc_kstat_rfc3635;
2521 unsigned int sc_kstat_mtmp_count;
2522 struct kstat **sc_kstat_mtmp;
2523 #endif
2524
2525 struct timecounter sc_timecounter;
2526 };
2527 #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
2528
2529 static int mcx_match(device_t, cfdata_t, void *);
2530 static void mcx_attach(device_t, device_t, void *);
2531
2532 static void * mcx_establish_intr(struct mcx_softc *, int, kcpuset_t *,
2533 int (*)(void *), void *, const char *);
2534
2535 static void mcx_rxr_init(struct mcx_rxring *, u_int, u_int);
2536 static u_int mcx_rxr_get(struct mcx_rxring *, u_int);
2537 static void mcx_rxr_put(struct mcx_rxring *, u_int);
2538 static u_int mcx_rxr_inuse(struct mcx_rxring *);
2539
2540 #if NKSTAT > 0
2541 static void mcx_kstat_attach(struct mcx_softc *);
2542 #endif
2543
2544 static void mcx_timecounter_attach(struct mcx_softc *);
2545
2546 static int mcx_version(struct mcx_softc *);
2547 static int mcx_init_wait(struct mcx_softc *);
2548 static int mcx_enable_hca(struct mcx_softc *);
2549 static int mcx_teardown_hca(struct mcx_softc *, uint16_t);
2550 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
2551 int);
2552 static int mcx_issi(struct mcx_softc *);
2553 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
2554 static int mcx_hca_max_caps(struct mcx_softc *);
2555 static int mcx_hca_set_caps(struct mcx_softc *);
2556 static int mcx_init_hca(struct mcx_softc *);
2557 static int mcx_set_driver_version(struct mcx_softc *);
2558 static int mcx_iff(struct mcx_softc *);
2559 static int mcx_alloc_uar(struct mcx_softc *, int *);
2560 static int mcx_alloc_pd(struct mcx_softc *);
2561 static int mcx_alloc_tdomain(struct mcx_softc *);
2562 static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int,
2563 uint64_t, int);
2564 static int mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *);
2565 static int mcx_query_special_contexts(struct mcx_softc *);
2566 static int mcx_set_port_mtu(struct mcx_softc *, int);
2567 static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int,
2568 int);
2569 static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *);
2570 static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int,
2571 int);
2572 static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *);
2573 static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *);
2574 static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int);
2575 static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *);
2576 static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *);
2577 static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *,
2578 int *);
2579 static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t,
2580 int *);
2581 static int mcx_destroy_tir(struct mcx_softc *, int);
2582 static int mcx_create_tis(struct mcx_softc *, int *);
2583 static int mcx_destroy_tis(struct mcx_softc *, int);
2584 static int mcx_create_rqt(struct mcx_softc *, int, int *, int *);
2585 static int mcx_destroy_rqt(struct mcx_softc *, int);
2586 static int mcx_create_flow_table(struct mcx_softc *, int, int, int *);
2587 static int mcx_set_flow_table_root(struct mcx_softc *, int);
2588 static int mcx_destroy_flow_table(struct mcx_softc *, int);
2589 static int mcx_create_flow_group(struct mcx_softc *, int, int, int,
2590 int, int, struct mcx_flow_match *);
2591 static int mcx_destroy_flow_group(struct mcx_softc *, int);
2592 static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int,
2593 const uint8_t *, uint32_t);
2594 static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int,
2595 int, int, uint32_t);
2596 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int);
2597
2598 #if NKSTAT > 0
2599 static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *);
2600 static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *);
2601 static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *);
2602 static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *);
2603 #endif
2604
2605 #if 0
2606 static int mcx_dump_flow_table(struct mcx_softc *, int);
2607 static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int);
2608 static int mcx_dump_flow_group(struct mcx_softc *, int);
2609 #endif
2610
2611
2612 /*
2613 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *);
2614 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
2615 */
2616 static void mcx_refill(void *);
2617 static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *,
2618 struct mcx_cq_entry *, struct mcx_mbufq *,
2619 const struct mcx_calibration *);
2620 static int mcx_process_txeof(struct mcx_softc *, struct mcx_tx *,
2621 struct mcx_cq_entry *);
2622 static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *,
2623 struct mcx_cq *);
2624
2625 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int);
2626 static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int);
2627 static int mcx_admin_intr(void *);
2628 static int mcx_cq_intr(void *);
2629
2630 static int mcx_init(struct ifnet *);
2631 static void mcx_stop(struct ifnet *, int);
2632 static int mcx_ioctl(struct ifnet *, u_long, void *);
2633 static void mcx_start(struct ifnet *);
2634 static int mcx_transmit(struct ifnet *, struct mbuf *);
2635 static void mcx_deferred_transmit(void *);
2636 static void mcx_watchdog(struct ifnet *);
2637 static void mcx_media_add_types(struct mcx_softc *);
2638 static void mcx_media_status(struct ifnet *, struct ifmediareq *);
2639 static int mcx_media_change(struct ifnet *);
2640 #if 0
2641 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
2642 #endif
2643 static void mcx_port_change(struct work *, void *);
2644
2645 static void mcx_calibrate_first(struct mcx_softc *);
2646 static void mcx_calibrate(void *);
2647
2648 static inline uint32_t
2649 mcx_rd(struct mcx_softc *, bus_size_t);
2650 static inline void
2651 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
2652 static inline void
2653 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
2654
2655 static uint64_t mcx_timer(struct mcx_softc *);
2656
2657 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
2658 bus_size_t, u_int align);
2659 static void mcx_dmamem_zero(struct mcx_dmamem *);
2660 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
2661
2662 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
2663 unsigned int);
2664 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
2665
2666 CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL);
2667
2668 static const struct {
2669 pci_vendor_id_t vendor;
2670 pci_product_id_t product;
2671 } mcx_devices[] = {
2672 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 },
2673 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF },
2674 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 },
2675 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF },
2676 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 },
2677 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF },
2678 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 },
2679 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF },
2680 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908 },
2681 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 },
2682 };
2683
2684 struct mcx_eth_proto_capability {
2685 uint64_t cap_media;
2686 uint64_t cap_baudrate;
2687 };
2688
2689 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = {
2690 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) },
2691 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) },
2692 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) },
2693 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) },
2694 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) },
2695 [MCX_ETHER_CAP_20G_KR2] = { IFM_20G_KR2, IF_Gbps(20) },
2696 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) },
2697 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) },
2698 [MCX_ETHER_CAP_56G_R4] = { IFM_56G_R4, IF_Gbps(56) },
2699 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_CR1, IF_Gbps(10) },
2700 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) },
2701 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) },
2702 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) },
2703 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) },
2704 [MCX_ETHER_CAP_50G_SR2] = { IFM_50G_SR2, IF_Gbps(50) },
2705 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) },
2706 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) },
2707 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) },
2708 [MCX_ETHER_CAP_100G_LR4] = { IFM_100G_LR4, IF_Gbps(100) },
2709 [MCX_ETHER_CAP_100_TX] = { IFM_100_TX, IF_Mbps(100) },
2710 [MCX_ETHER_CAP_1000_T] = { IFM_1000_T, IF_Gbps(1) },
2711 [MCX_ETHER_CAP_10G_T] = { IFM_10G_T, IF_Gbps(10) },
2712 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) },
2713 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) },
2714 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) },
2715 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) },
2716 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) },
2717 };
2718
2719 static int
2720 mcx_get_id(uint32_t val)
2721 {
2722 return be32toh(val) & 0x00ffffff;
2723 }
2724
2725 static int
2726 mcx_match(device_t parent, cfdata_t cf, void *aux)
2727 {
2728 struct pci_attach_args *pa = aux;
2729 int n;
2730
2731 for (n = 0; n < __arraycount(mcx_devices); n++) {
2732 if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor &&
2733 PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product)
2734 return 1;
2735 }
2736
2737 return 0;
2738 }
2739
2740 void
2741 mcx_attach(device_t parent, device_t self, void *aux)
2742 {
2743 struct mcx_softc *sc = device_private(self);
2744 struct ifnet *ifp = &sc->sc_ec.ec_if;
2745 struct pci_attach_args *pa = aux;
2746 uint8_t enaddr[ETHER_ADDR_LEN];
2747 int counts[PCI_INTR_TYPE_SIZE];
2748 char intrxname[32];
2749 pcireg_t memtype;
2750 uint32_t r;
2751 unsigned int cq_stride;
2752 unsigned int cq_size;
2753 int i, msix;
2754 kcpuset_t *affinity;
2755
2756 sc->sc_dev = self;
2757 sc->sc_pc = pa->pa_pc;
2758 sc->sc_tag = pa->pa_tag;
2759 if (pci_dma64_available(pa))
2760 sc->sc_dmat = pa->pa_dmat64;
2761 else
2762 sc->sc_dmat = pa->pa_dmat;
2763
2764 /* Map the PCI memory space */
2765 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
2766 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
2767 0 /*BUS_SPACE_MAP_PREFETCHABLE*/, &sc->sc_memt, &sc->sc_memh,
2768 NULL, &sc->sc_mems)) {
2769 aprint_error(": unable to map register memory\n");
2770 return;
2771 }
2772
2773 pci_aprint_devinfo(pa, "Ethernet controller");
2774
2775 mutex_init(&sc->sc_media_mutex, MUTEX_DEFAULT, IPL_SOFTNET);
2776
2777 if (mcx_version(sc) != 0) {
2778 /* error printed by mcx_version */
2779 goto unmap;
2780 }
2781
2782 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
2783 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
2784 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
2785 if (cq_size > MCX_MAX_CQE) {
2786 aprint_error_dev(self,
2787 "command queue size overflow %u\n", cq_size);
2788 goto unmap;
2789 }
2790 if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
2791 aprint_error_dev(self,
2792 "command queue entry size underflow %u\n", cq_stride);
2793 goto unmap;
2794 }
2795 if (cq_stride * cq_size > MCX_PAGE_SIZE) {
2796 aprint_error_dev(self, "command queue page overflow\n");
2797 goto unmap;
2798 }
2799
2800 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE,
2801 MCX_PAGE_SIZE) != 0) {
2802 aprint_error_dev(self, "unable to allocate doorbell memory\n");
2803 goto unmap;
2804 }
2805
2806 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
2807 MCX_PAGE_SIZE) != 0) {
2808 aprint_error_dev(self, "unable to allocate command queue\n");
2809 goto dbfree;
2810 }
2811
2812 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
2813 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t),
2814 BUS_SPACE_BARRIER_WRITE);
2815 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
2816 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t),
2817 BUS_SPACE_BARRIER_WRITE);
2818
2819 if (mcx_init_wait(sc) != 0) {
2820 aprint_error_dev(self, "timeout waiting for init\n");
2821 goto cqfree;
2822 }
2823
2824 sc->sc_cmdq_mask = cq_size - 1;
2825 sc->sc_cmdq_size = cq_stride;
2826
2827 if (mcx_enable_hca(sc) != 0) {
2828 /* error printed by mcx_enable_hca */
2829 goto cqfree;
2830 }
2831
2832 if (mcx_issi(sc) != 0) {
2833 /* error printed by mcx_issi */
2834 goto teardown;
2835 }
2836
2837 if (mcx_pages(sc, &sc->sc_boot_pages,
2838 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
2839 /* error printed by mcx_pages */
2840 goto teardown;
2841 }
2842
2843 if (mcx_hca_max_caps(sc) != 0) {
2844 /* error printed by mcx_hca_max_caps */
2845 goto teardown;
2846 }
2847
2848 if (mcx_hca_set_caps(sc) != 0) {
2849 /* error printed by mcx_hca_set_caps */
2850 goto teardown;
2851 }
2852
2853 if (mcx_pages(sc, &sc->sc_init_pages,
2854 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
2855 /* error printed by mcx_pages */
2856 goto teardown;
2857 }
2858
2859 if (mcx_init_hca(sc) != 0) {
2860 /* error printed by mcx_init_hca */
2861 goto teardown;
2862 }
2863
2864 if (mcx_pages(sc, &sc->sc_regular_pages,
2865 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
2866 /* error printed by mcx_pages */
2867 goto teardown;
2868 }
2869
2870 /* apparently not necessary? */
2871 if (mcx_set_driver_version(sc) != 0) {
2872 /* error printed by mcx_set_driver_version */
2873 goto teardown;
2874 }
2875
2876 if (mcx_iff(sc) != 0) { /* modify nic vport context */
2877 /* error printed by mcx_iff? */
2878 goto teardown;
2879 }
2880
2881 if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) {
2882 /* error printed by mcx_alloc_uar */
2883 goto teardown;
2884 }
2885
2886 if (mcx_alloc_pd(sc) != 0) {
2887 /* error printed by mcx_alloc_pd */
2888 goto teardown;
2889 }
2890
2891 if (mcx_alloc_tdomain(sc) != 0) {
2892 /* error printed by mcx_alloc_tdomain */
2893 goto teardown;
2894 }
2895
2896 /*
2897 * PRM makes no mention of msi interrupts, just legacy and msi-x.
2898 * mellanox support tells me legacy interrupts are not supported,
2899 * so we're stuck with just msi-x.
2900 */
2901 counts[PCI_INTR_TYPE_MSIX] = -1;
2902 counts[PCI_INTR_TYPE_MSI] = 0;
2903 counts[PCI_INTR_TYPE_INTX] = 0;
2904 if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) {
2905 aprint_error_dev(self, "unable to allocate interrupt\n");
2906 goto teardown;
2907 }
2908 if (counts[PCI_INTR_TYPE_MSIX] < 2) {
2909 aprint_error_dev(self, "not enough MSI-X vectors\n");
2910 goto teardown;
2911 }
2912 KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX);
2913 snprintf(intrxname, sizeof(intrxname), "%s adminq", DEVNAME(sc));
2914 sc->sc_ihc = mcx_establish_intr(sc, 0, NULL, mcx_admin_intr, sc,
2915 intrxname);
2916 if (sc->sc_ihc == NULL) {
2917 aprint_error_dev(self, "couldn't establish adminq interrupt\n");
2918 goto teardown;
2919 }
2920
2921 if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar,
2922 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
2923 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
2924 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
2925 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) {
2926 /* error printed by mcx_create_eq */
2927 goto teardown;
2928 }
2929
2930 if (mcx_query_nic_vport_context(sc, enaddr) != 0) {
2931 /* error printed by mcx_query_nic_vport_context */
2932 goto teardown;
2933 }
2934
2935 if (mcx_query_special_contexts(sc) != 0) {
2936 /* error printed by mcx_query_special_contexts */
2937 goto teardown;
2938 }
2939
2940 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
2941 /* error printed by mcx_set_port_mtu */
2942 goto teardown;
2943 }
2944
2945 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
2946 ether_sprintf(enaddr));
2947
2948 msix = counts[PCI_INTR_TYPE_MSIX];
2949 msix--; /* admin ops took one */
2950
2951 sc->sc_nqueues = uimin(MCX_MAX_QUEUES, msix);
2952 sc->sc_nqueues = uimin(sc->sc_nqueues, ncpu);
2953 sc->sc_queues = kmem_zalloc(sc->sc_nqueues * sizeof(*sc->sc_queues),
2954 KM_SLEEP);
2955
2956 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
2957 ifp->if_softc = sc;
2958 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
2959 #ifdef MCX_MPSAFE
2960 ifp->if_extflags = IFEF_MPSAFE;
2961 #endif
2962 ifp->if_init = mcx_init;
2963 ifp->if_stop = mcx_stop;
2964 ifp->if_ioctl = mcx_ioctl;
2965 ifp->if_start = mcx_start;
2966 if (sc->sc_nqueues > 1) {
2967 ifp->if_transmit = mcx_transmit;
2968 }
2969 ifp->if_watchdog = mcx_watchdog;
2970 ifp->if_mtu = sc->sc_hardmtu;
2971 ifp->if_capabilities = IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
2972 IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
2973 IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx |
2974 IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
2975 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx;
2976 IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
2977 IFQ_SET_READY(&ifp->if_snd);
2978
2979 sc->sc_ec.ec_capabilities = ETHERCAP_JUMBO_MTU |
2980 ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING;
2981 sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;
2982
2983 sc->sc_ec.ec_ifmedia = &sc->sc_media;
2984 ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, mcx_media_change,
2985 mcx_media_status, &sc->sc_media_mutex);
2986 mcx_media_add_types(sc);
2987 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
2988 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
2989
2990 if_attach(ifp);
2991 if_deferred_start_init(ifp, NULL);
2992
2993 ether_ifattach(ifp, enaddr);
2994
2995 kcpuset_create(&affinity, false);
2996 kcpuset_set(affinity, 0);
2997
2998 for (i = 0; i < sc->sc_nqueues; i++) {
2999 struct mcx_queues *q = &sc->sc_queues[i];
3000 struct mcx_rx *rx = &q->q_rx;
3001 struct mcx_tx *tx = &q->q_tx;
3002 int vec;
3003
3004 vec = i + 1;
3005 q->q_sc = sc;
3006 q->q_index = i;
3007
3008 if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
3009 aprint_error_dev(self, "unable to alloc uar %d\n", i);
3010 goto teardown;
3011 }
3012
3013 if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
3014 aprint_error_dev(self,
3015 "unable to create event queue %d\n", i);
3016 goto teardown;
3017 }
3018
3019 rx->rx_softc = sc;
3020 callout_init(&rx->rx_refill, CALLOUT_FLAGS);
3021 callout_setfunc(&rx->rx_refill, mcx_refill, rx);
3022
3023 tx->tx_softc = sc;
3024 mutex_init(&tx->tx_lock, MUTEX_DEFAULT, IPL_NET);
3025 tx->tx_pcq = pcq_create(MCX_TXQ_NUM, KM_SLEEP);
3026 tx->tx_softint = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,
3027 mcx_deferred_transmit, tx);
3028
3029 snprintf(intrxname, sizeof(intrxname), "%s queue %d",
3030 DEVNAME(sc), i);
3031 q->q_ihc = mcx_establish_intr(sc, vec, affinity, mcx_cq_intr,
3032 q, intrxname);
3033 }
3034
3035 callout_init(&sc->sc_calibrate, CALLOUT_FLAGS);
3036 callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc);
3037
3038 if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc,
3039 PRI_NONE, IPL_NET, 0) != 0) {
3040 aprint_error_dev(self, "couldn't create port change workq\n");
3041 goto teardown;
3042 }
3043
3044 mcx_port_change(&sc->sc_port_change, sc);
3045
3046 sc->sc_mac_flow_table_id = -1;
3047 sc->sc_rss_flow_table_id = -1;
3048 sc->sc_rqt = -1;
3049 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
3050 struct mcx_flow_group *mfg = &sc->sc_flow_group[i];
3051 mfg->g_id = -1;
3052 mfg->g_table = -1;
3053 mfg->g_size = 0;
3054 mfg->g_start = 0;
3055 }
3056 sc->sc_extra_mcast = 0;
3057 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));
3058
3059 #if NKSTAT > 0
3060 mcx_kstat_attach(sc);
3061 #endif
3062 mcx_timecounter_attach(sc);
3063 return;
3064
3065 teardown:
3066 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
3067 /* error printed by mcx_teardown_hca, and we're already unwinding */
3068 cqfree:
3069 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
3070 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
3071 BUS_SPACE_BARRIER_WRITE);
3072 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
3073 MCX_CMDQ_INTERFACE_DISABLED);
3074 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t),
3075 BUS_SPACE_BARRIER_WRITE);
3076
3077 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
3078 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
3079 BUS_SPACE_BARRIER_WRITE);
3080 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
3081
3082 mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
3083 dbfree:
3084 mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
3085 unmap:
3086 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
3087 sc->sc_mems = 0;
3088 }
3089
3090 static void *
3091 mcx_establish_intr(struct mcx_softc *sc, int index, kcpuset_t *affinity,
3092 int (*func)(void *), void *arg, const char *xname)
3093 {
3094 char intrbuf[PCI_INTRSTR_LEN];
3095 const char *intrstr;
3096 void *ih;
3097
3098 pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[index], PCI_INTR_MPSAFE,
3099 true);
3100
3101 intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[index], intrbuf,
3102 sizeof(intrbuf));
3103 ih = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[index], IPL_NET,
3104 func, arg, xname);
3105 if (ih == NULL) {
3106 aprint_error_dev(sc->sc_dev,
3107 "unable to establish interrupt%s%s\n",
3108 intrstr ? " at " : "",
3109 intrstr ? intrstr : "");
3110 return NULL;
3111 }
3112
3113 if (affinity != NULL && index > 0) {
3114 /* Round-robin affinity */
3115 kcpuset_zero(affinity);
3116 kcpuset_set(affinity, (index - 1) % ncpu);
3117 interrupt_distribute(ih, affinity, NULL);
3118 }
3119
3120 return ih;
3121 }
3122
3123 static void
3124 mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm)
3125 {
3126 rxr->rxr_total = hwm;
3127 rxr->rxr_inuse = 0;
3128 }
3129
3130 static u_int
3131 mcx_rxr_get(struct mcx_rxring *rxr, u_int max)
3132 {
3133 const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse);
3134
3135 rxr->rxr_inuse += taken;
3136
3137 return taken;
3138 }
3139
3140 static void
3141 mcx_rxr_put(struct mcx_rxring *rxr, u_int n)
3142 {
3143 rxr->rxr_inuse -= n;
3144 }
3145
3146 static u_int
3147 mcx_rxr_inuse(struct mcx_rxring *rxr)
3148 {
3149 return rxr->rxr_inuse;
3150 }
3151
3152 static int
3153 mcx_version(struct mcx_softc *sc)
3154 {
3155 uint32_t fw0, fw1;
3156 uint16_t cmdif;
3157
3158 fw0 = mcx_rd(sc, MCX_FW_VER);
3159 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
3160
3161 aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0),
3162 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
3163
3164 cmdif = MCX_CMDIF(fw1);
3165 if (cmdif != MCX_CMD_IF_SUPPORTED) {
3166 aprint_error_dev(sc->sc_dev,
3167 "unsupported command interface %u\n", cmdif);
3168 return (-1);
3169 }
3170
3171 return (0);
3172 }
3173
3174 static int
3175 mcx_init_wait(struct mcx_softc *sc)
3176 {
3177 unsigned int i;
3178 uint32_t r;
3179
3180 for (i = 0; i < 2000; i++) {
3181 r = mcx_rd(sc, MCX_STATE);
3182 if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
3183 return (0);
3184
3185 delay(1000);
3186 mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
3187 BUS_SPACE_BARRIER_READ);
3188 }
3189
3190 return (-1);
3191 }
3192
3193 static uint8_t
3194 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3195 unsigned int msec)
3196 {
3197 unsigned int i;
3198
3199 for (i = 0; i < msec; i++) {
3200 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
3201 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
3202
3203 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
3204 MCX_CQ_STATUS_OWN_SW)
3205 return (0);
3206
3207 delay(1000);
3208 }
3209
3210 return (ETIMEDOUT);
3211 }
3212
3213 static uint32_t
3214 mcx_mix_u64(uint32_t xor, uint64_t u64)
3215 {
3216 xor ^= u64 >> 32;
3217 xor ^= u64;
3218
3219 return (xor);
3220 }
3221
3222 static uint32_t
3223 mcx_mix_u32(uint32_t xor, uint32_t u32)
3224 {
3225 xor ^= u32;
3226
3227 return (xor);
3228 }
3229
3230 static uint32_t
3231 mcx_mix_u8(uint32_t xor, uint8_t u8)
3232 {
3233 xor ^= u8;
3234
3235 return (xor);
3236 }
3237
3238 static uint8_t
3239 mcx_mix_done(uint32_t xor)
3240 {
3241 xor ^= xor >> 16;
3242 xor ^= xor >> 8;
3243
3244 return (xor);
3245 }
3246
3247 static uint8_t
3248 mcx_xor(const void *buf, size_t len)
3249 {
3250 const uint32_t *dwords = buf;
3251 uint32_t xor = 0xff;
3252 size_t i;
3253
3254 len /= sizeof(*dwords);
3255
3256 for (i = 0; i < len; i++)
3257 xor ^= dwords[i];
3258
3259 return (mcx_mix_done(xor));
3260 }
3261
3262 static uint8_t
3263 mcx_cmdq_token(struct mcx_softc *sc)
3264 {
3265 uint8_t token;
3266
3267 do {
3268 token = ++sc->sc_cmdq_token;
3269 } while (token == 0);
3270
3271 return (token);
3272 }
3273
3274 static void
3275 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3276 uint32_t ilen, uint32_t olen, uint8_t token)
3277 {
3278 memset(cqe, 0, sc->sc_cmdq_size);
3279
3280 cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
3281 be32enc(&cqe->cq_input_length, ilen);
3282 be32enc(&cqe->cq_output_length, olen);
3283 cqe->cq_token = token;
3284 cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
3285 }
3286
3287 static void
3288 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
3289 {
3290 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
3291 }
3292
3293 static int
3294 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
3295 {
3296 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */
3297 return (0);
3298 }
3299
3300 static void *
3301 mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
3302 {
3303 return (&cqe->cq_input_data);
3304 }
3305
3306 static void *
3307 mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
3308 {
3309 return (&cqe->cq_output_data);
3310 }
3311
3312 static void
3313 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3314 unsigned int slot)
3315 {
3316 mcx_cmdq_sign(cqe);
3317
3318 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
3319 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
3320
3321 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
3322 mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t),
3323 BUS_SPACE_BARRIER_WRITE);
3324 }
3325
3326 static int
3327 mcx_enable_hca(struct mcx_softc *sc)
3328 {
3329 struct mcx_cmdq_entry *cqe;
3330 struct mcx_cmd_enable_hca_in *in;
3331 struct mcx_cmd_enable_hca_out *out;
3332 int error;
3333 uint8_t status;
3334
3335 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3336 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3337
3338 in = mcx_cmdq_in(cqe);
3339 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
3340 in->cmd_op_mod = htobe16(0);
3341 in->cmd_function_id = htobe16(0);
3342
3343 mcx_cmdq_post(sc, cqe, 0);
3344
3345 error = mcx_cmdq_poll(sc, cqe, 1000);
3346 if (error != 0) {
3347 printf(", hca enable timeout\n");
3348 return (-1);
3349 }
3350 if (mcx_cmdq_verify(cqe) != 0) {
3351 printf(", hca enable command corrupt\n");
3352 return (-1);
3353 }
3354
3355 status = cqe->cq_output_data[0];
3356 if (status != MCX_CQ_STATUS_OK) {
3357 printf(", hca enable failed (%x)\n", status);
3358 return (-1);
3359 }
3360
3361 return (0);
3362 }
3363
3364 static int
3365 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
3366 {
3367 struct mcx_cmdq_entry *cqe;
3368 struct mcx_cmd_teardown_hca_in *in;
3369 struct mcx_cmd_teardown_hca_out *out;
3370 int error;
3371 uint8_t status;
3372
3373 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3374 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3375
3376 in = mcx_cmdq_in(cqe);
3377 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
3378 in->cmd_op_mod = htobe16(0);
3379 in->cmd_profile = profile;
3380
3381 mcx_cmdq_post(sc, cqe, 0);
3382
3383 error = mcx_cmdq_poll(sc, cqe, 1000);
3384 if (error != 0) {
3385 printf(", hca teardown timeout\n");
3386 return (-1);
3387 }
3388 if (mcx_cmdq_verify(cqe) != 0) {
3389 printf(", hca teardown command corrupt\n");
3390 return (-1);
3391 }
3392
3393 status = cqe->cq_output_data[0];
3394 if (status != MCX_CQ_STATUS_OK) {
3395 printf(", hca teardown failed (%x)\n", status);
3396 return (-1);
3397 }
3398
3399 return (0);
3400 }
3401
3402 static int
3403 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
3404 unsigned int nmb, uint64_t *ptr, uint8_t token)
3405 {
3406 uint8_t *kva;
3407 uint64_t dva;
3408 int i;
3409 int error;
3410
3411 error = mcx_dmamem_alloc(sc, mxm,
3412 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
3413 if (error != 0)
3414 return (error);
3415
3416 mcx_dmamem_zero(mxm);
3417
3418 dva = MCX_DMA_DVA(mxm);
3419 kva = MCX_DMA_KVA(mxm);
3420 for (i = 0; i < nmb; i++) {
3421 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
3422
3423 /* patch the cqe or mbox pointing at this one */
3424 be64enc(ptr, dva);
3425
3426 /* fill in this mbox */
3427 be32enc(&mbox->mb_block_number, i);
3428 mbox->mb_token = token;
3429
3430 /* move to the next one */
3431 ptr = &mbox->mb_next_ptr;
3432
3433 dva += MCX_CMDQ_MAILBOX_SIZE;
3434 kva += MCX_CMDQ_MAILBOX_SIZE;
3435 }
3436
3437 return (0);
3438 }
3439
3440 static uint32_t
3441 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
3442 {
3443 uint32_t xor = 0xff;
3444
3445 /* only 3 fields get set, so mix them directly */
3446 xor = mcx_mix_u64(xor, mb->mb_next_ptr);
3447 xor = mcx_mix_u32(xor, mb->mb_block_number);
3448 xor = mcx_mix_u8(xor, mb->mb_token);
3449
3450 return (mcx_mix_done(xor));
3451 }
3452
3453 static void
3454 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
3455 {
3456 uint8_t *kva;
3457 int i;
3458
3459 kva = MCX_DMA_KVA(mxm);
3460
3461 for (i = 0; i < nmb; i++) {
3462 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
3463 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
3464 mb->mb_ctrl_signature = sig;
3465 mb->mb_signature = sig ^
3466 mcx_xor(mb->mb_data, sizeof(mb->mb_data));
3467
3468 kva += MCX_CMDQ_MAILBOX_SIZE;
3469 }
3470 }
3471
3472 static void
3473 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
3474 {
3475 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
3476 0, MCX_DMA_LEN(mxm), ops);
3477 }
3478
3479 static struct mcx_cmdq_mailbox *
3480 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
3481 {
3482 uint8_t *kva;
3483
3484 kva = MCX_DMA_KVA(mxm);
3485 kva += i * MCX_CMDQ_MAILBOX_SIZE;
3486
3487 return ((struct mcx_cmdq_mailbox *)kva);
3488 }
3489
3490 static inline void *
3491 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
3492 {
3493 return (&mb->mb_data);
3494 }
3495
3496 static void
3497 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
3498 void *b, size_t len)
3499 {
3500 uint8_t *buf = b;
3501 struct mcx_cmdq_mailbox *mb;
3502 int i;
3503
3504 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
3505 for (i = 0; i < nmb; i++) {
3506
3507 memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len));
3508
3509 if (sizeof(mb->mb_data) >= len)
3510 break;
3511
3512 buf += sizeof(mb->mb_data);
3513 len -= sizeof(mb->mb_data);
3514 mb++;
3515 }
3516 }
3517
3518 static void
3519 mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages,
3520 struct mcx_dmamem *buf)
3521 {
3522 uint64_t *pas;
3523 int mbox, mbox_pages, i;
3524
3525 mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE;
3526 offset %= MCX_CMDQ_MAILBOX_DATASIZE;
3527
3528 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
3529 pas += (offset / sizeof(*pas));
3530 mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas);
3531 for (i = 0; i < npages; i++) {
3532 if (i == mbox_pages) {
3533 mbox++;
3534 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
3535 mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas);
3536 }
3537 *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE));
3538 pas++;
3539 }
3540 }
3541
3542 static void
3543 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
3544 {
3545 uint8_t *buf = b;
3546 struct mcx_cmdq_mailbox *mb;
3547 int i;
3548
3549 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
3550 for (i = 0; i < nmb; i++) {
3551 memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len));
3552
3553 if (sizeof(mb->mb_data) >= len)
3554 break;
3555
3556 buf += sizeof(mb->mb_data);
3557 len -= sizeof(mb->mb_data);
3558 mb++;
3559 }
3560 }
3561
3562 static void
3563 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
3564 {
3565 mcx_dmamem_free(sc, mxm);
3566 }
3567
3568 #if 0
3569 static void
3570 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
3571 {
3572 unsigned int i;
3573
3574 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
3575 be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr));
3576
3577 printf(", idata ");
3578 for (i = 0; i < sizeof(cqe->cq_input_data); i++)
3579 printf("%02x", cqe->cq_input_data[i]);
3580
3581 printf(", odata ");
3582 for (i = 0; i < sizeof(cqe->cq_output_data); i++)
3583 printf("%02x", cqe->cq_output_data[i]);
3584
3585 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
3586 be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length),
3587 cqe->cq_token, cqe->cq_signature, cqe->cq_status);
3588 }
3589
3590 static void
3591 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
3592 {
3593 int i, j;
3594 uint8_t *d;
3595
3596 for (i = 0; i < num; i++) {
3597 struct mcx_cmdq_mailbox *mbox;
3598 mbox = mcx_cq_mbox(mboxes, i);
3599
3600 d = mcx_cq_mbox_data(mbox);
3601 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
3602 if (j != 0 && (j % 16 == 0))
3603 printf("\n");
3604 printf("%.2x ", d[j]);
3605 }
3606 }
3607 }
3608 #endif
3609
3610 static int
3611 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
3612 int len)
3613 {
3614 struct mcx_dmamem mxm;
3615 struct mcx_cmdq_entry *cqe;
3616 struct mcx_cmd_access_reg_in *in;
3617 struct mcx_cmd_access_reg_out *out;
3618 uint8_t token = mcx_cmdq_token(sc);
3619 int error, nmb;
3620
3621 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3622 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
3623 token);
3624
3625 in = mcx_cmdq_in(cqe);
3626 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
3627 in->cmd_op_mod = htobe16(op);
3628 in->cmd_register_id = htobe16(reg);
3629
3630 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
3631 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
3632 &cqe->cq_output_ptr, token) != 0) {
3633 printf(", unable to allocate access reg mailboxen\n");
3634 return (-1);
3635 }
3636 cqe->cq_input_ptr = cqe->cq_output_ptr;
3637 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
3638 mcx_cmdq_mboxes_sign(&mxm, nmb);
3639 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
3640
3641 mcx_cmdq_post(sc, cqe, 0);
3642 error = mcx_cmdq_poll(sc, cqe, 1000);
3643 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
3644
3645 if (error != 0) {
3646 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
3647 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
3648 goto free;
3649 }
3650 error = mcx_cmdq_verify(cqe);
3651 if (error != 0) {
3652 printf("%s: access reg (%s %x) reply corrupt\n",
3653 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
3654 reg);
3655 goto free;
3656 }
3657
3658 out = mcx_cmdq_out(cqe);
3659 if (out->cmd_status != MCX_CQ_STATUS_OK) {
3660 printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
3661 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
3662 reg, out->cmd_status, be32toh(out->cmd_syndrome));
3663 error = -1;
3664 goto free;
3665 }
3666
3667 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
3668 free:
3669 mcx_dmamem_free(sc, &mxm);
3670
3671 return (error);
3672 }
3673
3674 static int
3675 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3676 unsigned int slot)
3677 {
3678 struct mcx_cmd_set_issi_in *in;
3679 struct mcx_cmd_set_issi_out *out;
3680 uint8_t status;
3681
3682 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3683
3684 in = mcx_cmdq_in(cqe);
3685 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
3686 in->cmd_op_mod = htobe16(0);
3687 in->cmd_current_issi = htobe16(MCX_ISSI);
3688
3689 mcx_cmdq_post(sc, cqe, slot);
3690 if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
3691 return (-1);
3692 if (mcx_cmdq_verify(cqe) != 0)
3693 return (-1);
3694
3695 status = cqe->cq_output_data[0];
3696 if (status != MCX_CQ_STATUS_OK)
3697 return (-1);
3698
3699 return (0);
3700 }
3701
3702 static int
3703 mcx_issi(struct mcx_softc *sc)
3704 {
3705 struct mcx_dmamem mxm;
3706 struct mcx_cmdq_entry *cqe;
3707 struct mcx_cmd_query_issi_in *in;
3708 struct mcx_cmd_query_issi_il_out *out;
3709 struct mcx_cmd_query_issi_mb_out *mb;
3710 uint8_t token = mcx_cmdq_token(sc);
3711 uint8_t status;
3712 int error;
3713
3714 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3715 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
3716
3717 in = mcx_cmdq_in(cqe);
3718 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
3719 in->cmd_op_mod = htobe16(0);
3720
3721 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
3722 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
3723 &cqe->cq_output_ptr, token) != 0) {
3724 printf(", unable to allocate query issi mailbox\n");
3725 return (-1);
3726 }
3727 mcx_cmdq_mboxes_sign(&mxm, 1);
3728
3729 mcx_cmdq_post(sc, cqe, 0);
3730 error = mcx_cmdq_poll(sc, cqe, 1000);
3731 if (error != 0) {
3732 printf(", query issi timeout\n");
3733 goto free;
3734 }
3735 error = mcx_cmdq_verify(cqe);
3736 if (error != 0) {
3737 printf(", query issi reply corrupt\n");
3738 goto free;
3739 }
3740
3741 status = cqe->cq_output_data[0];
3742 switch (status) {
3743 case MCX_CQ_STATUS_OK:
3744 break;
3745 case MCX_CQ_STATUS_BAD_OPCODE:
3746 /* use ISSI 0 */
3747 goto free;
3748 default:
3749 printf(", query issi failed (%x)\n", status);
3750 error = -1;
3751 goto free;
3752 }
3753
3754 out = mcx_cmdq_out(cqe);
3755 if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
3756 /* use ISSI 1 */
3757 goto free;
3758 }
3759
3760 /* don't need to read cqe anymore, can be used for SET ISSI */
3761
3762 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
3763 CTASSERT(MCX_ISSI < NBBY);
3764 /* XXX math is hard */
3765 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
3766 /* use ISSI 0 */
3767 goto free;
3768 }
3769
3770 if (mcx_set_issi(sc, cqe, 0) != 0) {
3771 /* ignore the error, just use ISSI 0 */
3772 } else {
3773 /* use ISSI 1 */
3774 }
3775
3776 free:
3777 mcx_cq_mboxes_free(sc, &mxm);
3778 return (error);
3779 }
3780
3781 static int
3782 mcx_query_pages(struct mcx_softc *sc, uint16_t type,
3783 int32_t *npages, uint16_t *func_id)
3784 {
3785 struct mcx_cmdq_entry *cqe;
3786 struct mcx_cmd_query_pages_in *in;
3787 struct mcx_cmd_query_pages_out *out;
3788
3789 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3790 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3791
3792 in = mcx_cmdq_in(cqe);
3793 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
3794 in->cmd_op_mod = type;
3795
3796 mcx_cmdq_post(sc, cqe, 0);
3797 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
3798 printf(", query pages timeout\n");
3799 return (-1);
3800 }
3801 if (mcx_cmdq_verify(cqe) != 0) {
3802 printf(", query pages reply corrupt\n");
3803 return (-1);
3804 }
3805
3806 out = mcx_cmdq_out(cqe);
3807 if (out->cmd_status != MCX_CQ_STATUS_OK) {
3808 printf(", query pages failed (%x)\n", out->cmd_status);
3809 return (-1);
3810 }
3811
3812 *func_id = out->cmd_func_id;
3813 *npages = be32dec(&out->cmd_num_pages);
3814
3815 return (0);
3816 }
3817
3818 struct bus_dma_iter {
3819 bus_dmamap_t i_map;
3820 bus_size_t i_offset;
3821 unsigned int i_index;
3822 };
3823
3824 static void
3825 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
3826 {
3827 i->i_map = map;
3828 i->i_offset = 0;
3829 i->i_index = 0;
3830 }
3831
3832 static bus_addr_t
3833 bus_dma_iter_addr(struct bus_dma_iter *i)
3834 {
3835 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
3836 }
3837
3838 static void
3839 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
3840 {
3841 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
3842 bus_size_t diff;
3843
3844 do {
3845 diff = seg->ds_len - i->i_offset;
3846 if (size < diff)
3847 break;
3848
3849 size -= diff;
3850
3851 seg++;
3852
3853 i->i_offset = 0;
3854 i->i_index++;
3855 } while (size > 0);
3856
3857 i->i_offset += size;
3858 }
3859
3860 static int
3861 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
3862 {
3863 struct mcx_dmamem mxm;
3864 struct mcx_cmdq_entry *cqe;
3865 struct mcx_cmd_manage_pages_in *in;
3866 struct mcx_cmd_manage_pages_out *out;
3867 unsigned int paslen, nmb, i, j, npages;
3868 struct bus_dma_iter iter;
3869 uint64_t *pas;
3870 uint8_t status;
3871 uint8_t token = mcx_cmdq_token(sc);
3872 int error;
3873
3874 npages = mhm->mhm_npages;
3875
3876 paslen = sizeof(*pas) * npages;
3877 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
3878
3879 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3880 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
3881
3882 in = mcx_cmdq_in(cqe);
3883 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
3884 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
3885 in->cmd_func_id = func_id;
3886 be32enc(&in->cmd_input_num_entries, npages);
3887
3888 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
3889 &cqe->cq_input_ptr, token) != 0) {
3890 printf(", unable to allocate manage pages mailboxen\n");
3891 return (-1);
3892 }
3893
3894 bus_dma_iter_init(&iter, mhm->mhm_map);
3895 for (i = 0; i < nmb; i++) {
3896 unsigned int lim;
3897
3898 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
3899 lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
3900
3901 for (j = 0; j < lim; j++) {
3902 be64enc(&pas[j], bus_dma_iter_addr(&iter));
3903 bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
3904 }
3905
3906 npages -= lim;
3907 }
3908
3909 mcx_cmdq_mboxes_sign(&mxm, nmb);
3910
3911 mcx_cmdq_post(sc, cqe, 0);
3912 error = mcx_cmdq_poll(sc, cqe, 1000);
3913 if (error != 0) {
3914 printf(", manage pages timeout\n");
3915 goto free;
3916 }
3917 error = mcx_cmdq_verify(cqe);
3918 if (error != 0) {
3919 printf(", manage pages reply corrupt\n");
3920 goto free;
3921 }
3922
3923 status = cqe->cq_output_data[0];
3924 if (status != MCX_CQ_STATUS_OK) {
3925 printf(", manage pages failed (%x)\n", status);
3926 error = -1;
3927 goto free;
3928 }
3929
3930 free:
3931 mcx_dmamem_free(sc, &mxm);
3932
3933 return (error);
3934 }
3935
3936 static int
3937 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
3938 {
3939 int32_t npages;
3940 uint16_t func_id;
3941
3942 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
3943 /* error printed by mcx_query_pages */
3944 return (-1);
3945 }
3946
3947 if (npages < 1)
3948 return (0);
3949
3950 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
3951 printf(", unable to allocate hwmem\n");
3952 return (-1);
3953 }
3954
3955 if (mcx_add_pages(sc, mhm, func_id) != 0) {
3956 printf(", unable to add hwmem\n");
3957 goto free;
3958 }
3959
3960 return (0);
3961
3962 free:
3963 mcx_hwmem_free(sc, mhm);
3964
3965 return (-1);
3966 }
3967
3968 static int
3969 mcx_hca_max_caps(struct mcx_softc *sc)
3970 {
3971 struct mcx_dmamem mxm;
3972 struct mcx_cmdq_entry *cqe;
3973 struct mcx_cmd_query_hca_cap_in *in;
3974 struct mcx_cmd_query_hca_cap_out *out;
3975 struct mcx_cmdq_mailbox *mb;
3976 struct mcx_cap_device *hca;
3977 uint8_t status;
3978 uint8_t token = mcx_cmdq_token(sc);
3979 int error;
3980
3981 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3982 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
3983 token);
3984
3985 in = mcx_cmdq_in(cqe);
3986 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
3987 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
3988 MCX_CMD_QUERY_HCA_CAP_DEVICE);
3989
3990 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
3991 &cqe->cq_output_ptr, token) != 0) {
3992 printf(", unable to allocate query hca caps mailboxen\n");
3993 return (-1);
3994 }
3995 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
3996 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
3997
3998 mcx_cmdq_post(sc, cqe, 0);
3999 error = mcx_cmdq_poll(sc, cqe, 1000);
4000 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
4001
4002 if (error != 0) {
4003 printf(", query hca caps timeout\n");
4004 goto free;
4005 }
4006 error = mcx_cmdq_verify(cqe);
4007 if (error != 0) {
4008 printf(", query hca caps reply corrupt\n");
4009 goto free;
4010 }
4011
4012 status = cqe->cq_output_data[0];
4013 if (status != MCX_CQ_STATUS_OK) {
4014 printf(", query hca caps failed (%x)\n", status);
4015 error = -1;
4016 goto free;
4017 }
4018
4019 mb = mcx_cq_mbox(&mxm, 0);
4020 hca = mcx_cq_mbox_data(mb);
4021
4022 if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE)
4023 != MCX_CAP_DEVICE_PORT_TYPE_ETH) {
4024 printf(", not in ethernet mode\n");
4025 error = -1;
4026 goto free;
4027 }
4028 if (hca->log_pg_sz > PAGE_SHIFT) {
4029 printf(", minimum system page shift %u is too large\n",
4030 hca->log_pg_sz);
4031 error = -1;
4032 goto free;
4033 }
4034 /*
4035 * blueflame register is split into two buffers, and we must alternate
4036 * between the two of them.
4037 */
4038 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
4039 sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size);
4040
4041 if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG)
4042 sc->sc_mcam_reg = 1;
4043
4044 sc->sc_mhz = be32dec(&hca->device_frequency_mhz);
4045 sc->sc_khz = be32dec(&hca->device_frequency_khz);
4046
4047 free:
4048 mcx_dmamem_free(sc, &mxm);
4049
4050 return (error);
4051 }
4052
4053 static int
4054 mcx_hca_set_caps(struct mcx_softc *sc)
4055 {
4056 struct mcx_dmamem mxm;
4057 struct mcx_cmdq_entry *cqe;
4058 struct mcx_cmd_query_hca_cap_in *in;
4059 struct mcx_cmd_query_hca_cap_out *out;
4060 struct mcx_cmdq_mailbox *mb;
4061 struct mcx_cap_device *hca;
4062 uint8_t status;
4063 uint8_t token = mcx_cmdq_token(sc);
4064 int error;
4065
4066 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4067 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
4068 token);
4069
4070 in = mcx_cmdq_in(cqe);
4071 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
4072 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
4073 MCX_CMD_QUERY_HCA_CAP_DEVICE);
4074
4075 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
4076 &cqe->cq_output_ptr, token) != 0) {
4077 printf(", unable to allocate manage pages mailboxen\n");
4078 return (-1);
4079 }
4080 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
4081 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
4082
4083 mcx_cmdq_post(sc, cqe, 0);
4084 error = mcx_cmdq_poll(sc, cqe, 1000);
4085 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
4086
4087 if (error != 0) {
4088 printf(", query hca caps timeout\n");
4089 goto free;
4090 }
4091 error = mcx_cmdq_verify(cqe);
4092 if (error != 0) {
4093 printf(", query hca caps reply corrupt\n");
4094 goto free;
4095 }
4096
4097 status = cqe->cq_output_data[0];
4098 if (status != MCX_CQ_STATUS_OK) {
4099 printf(", query hca caps failed (%x)\n", status);
4100 error = -1;
4101 goto free;
4102 }
4103
4104 mb = mcx_cq_mbox(&mxm, 0);
4105 hca = mcx_cq_mbox_data(mb);
4106
4107 hca->log_pg_sz = PAGE_SHIFT;
4108
4109 free:
4110 mcx_dmamem_free(sc, &mxm);
4111
4112 return (error);
4113 }
4114
4115
4116 static int
4117 mcx_init_hca(struct mcx_softc *sc)
4118 {
4119 struct mcx_cmdq_entry *cqe;
4120 struct mcx_cmd_init_hca_in *in;
4121 struct mcx_cmd_init_hca_out *out;
4122 int error;
4123 uint8_t status;
4124
4125 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4126 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4127
4128 in = mcx_cmdq_in(cqe);
4129 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
4130 in->cmd_op_mod = htobe16(0);
4131
4132 mcx_cmdq_post(sc, cqe, 0);
4133
4134 error = mcx_cmdq_poll(sc, cqe, 1000);
4135 if (error != 0) {
4136 printf(", hca init timeout\n");
4137 return (-1);
4138 }
4139 if (mcx_cmdq_verify(cqe) != 0) {
4140 printf(", hca init command corrupt\n");
4141 return (-1);
4142 }
4143
4144 status = cqe->cq_output_data[0];
4145 if (status != MCX_CQ_STATUS_OK) {
4146 printf(", hca init failed (%x)\n", status);
4147 return (-1);
4148 }
4149
4150 return (0);
4151 }
4152
4153 static int
4154 mcx_set_driver_version(struct mcx_softc *sc)
4155 {
4156 struct mcx_dmamem mxm;
4157 struct mcx_cmdq_entry *cqe;
4158 struct mcx_cmd_set_driver_version_in *in;
4159 struct mcx_cmd_set_driver_version_out *out;
4160 int error;
4161 int token;
4162 uint8_t status;
4163
4164 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4165 token = mcx_cmdq_token(sc);
4166 mcx_cmdq_init(sc, cqe, sizeof(*in) +
4167 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);
4168
4169 in = mcx_cmdq_in(cqe);
4170 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
4171 in->cmd_op_mod = htobe16(0);
4172
4173 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4174 &cqe->cq_input_ptr, token) != 0) {
4175 printf(", unable to allocate set driver version mailboxen\n");
4176 return (-1);
4177 }
4178 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
4179 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
4180
4181 mcx_cmdq_mboxes_sign(&mxm, 1);
4182 mcx_cmdq_post(sc, cqe, 0);
4183
4184 error = mcx_cmdq_poll(sc, cqe, 1000);
4185 if (error != 0) {
4186 printf(", set driver version timeout\n");
4187 goto free;
4188 }
4189 if (mcx_cmdq_verify(cqe) != 0) {
4190 printf(", set driver version command corrupt\n");
4191 goto free;
4192 }
4193
4194 status = cqe->cq_output_data[0];
4195 if (status != MCX_CQ_STATUS_OK) {
4196 printf(", set driver version failed (%x)\n", status);
4197 error = -1;
4198 goto free;
4199 }
4200
4201 free:
4202 mcx_dmamem_free(sc, &mxm);
4203
4204 return (error);
4205 }
4206
4207 static int
4208 mcx_iff(struct mcx_softc *sc)
4209 {
4210 struct ifnet *ifp = &sc->sc_ec.ec_if;
4211 struct mcx_dmamem mxm;
4212 struct mcx_cmdq_entry *cqe;
4213 struct mcx_cmd_modify_nic_vport_context_in *in;
4214 struct mcx_cmd_modify_nic_vport_context_out *out;
4215 struct mcx_nic_vport_ctx *ctx;
4216 int error;
4217 int token;
4218 int insize;
4219 uint32_t dest;
4220
4221 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
4222 sc->sc_rss_flow_table_id;
4223
4224 /* enable or disable the promisc flow */
4225 if (ISSET(ifp->if_flags, IFF_PROMISC)) {
4226 if (sc->sc_promisc_flow_enabled == 0) {
4227 mcx_set_flow_table_entry_mac(sc,
4228 MCX_FLOW_GROUP_PROMISC, 0, NULL, dest);
4229 sc->sc_promisc_flow_enabled = 1;
4230 }
4231 } else if (sc->sc_promisc_flow_enabled != 0) {
4232 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
4233 sc->sc_promisc_flow_enabled = 0;
4234 }
4235
4236 /* enable or disable the all-multicast flow */
4237 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
4238 if (sc->sc_allmulti_flow_enabled == 0) {
4239 uint8_t mcast[ETHER_ADDR_LEN];
4240
4241 memset(mcast, 0, sizeof(mcast));
4242 mcast[0] = 0x01;
4243 mcx_set_flow_table_entry_mac(sc,
4244 MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest);
4245 sc->sc_allmulti_flow_enabled = 1;
4246 }
4247 } else if (sc->sc_allmulti_flow_enabled != 0) {
4248 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
4249 sc->sc_allmulti_flow_enabled = 0;
4250 }
4251
4252 insize = sizeof(struct mcx_nic_vport_ctx) + 240;
4253
4254 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4255 token = mcx_cmdq_token(sc);
4256 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4257
4258 in = mcx_cmdq_in(cqe);
4259 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
4260 in->cmd_op_mod = htobe16(0);
4261 in->cmd_field_select = htobe32(
4262 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
4263 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
4264
4265 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
4266 printf(", unable to allocate modify "
4267 "nic vport context mailboxen\n");
4268 return (-1);
4269 }
4270 ctx = (struct mcx_nic_vport_ctx *)
4271 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
4272 ctx->vp_mtu = htobe32(sc->sc_hardmtu);
4273 /*
4274 * always leave promisc-all enabled on the vport since we
4275 * can't give it a vlan list, and we're already doing multicast
4276 * filtering in the flow table.
4277 */
4278 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);
4279
4280 mcx_cmdq_mboxes_sign(&mxm, 1);
4281 mcx_cmdq_post(sc, cqe, 0);
4282
4283 error = mcx_cmdq_poll(sc, cqe, 1000);
4284 if (error != 0) {
4285 printf(", modify nic vport context timeout\n");
4286 goto free;
4287 }
4288 if (mcx_cmdq_verify(cqe) != 0) {
4289 printf(", modify nic vport context command corrupt\n");
4290 goto free;
4291 }
4292
4293 out = mcx_cmdq_out(cqe);
4294 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4295 printf(", modify nic vport context failed (%x, %x)\n",
4296 out->cmd_status, be32toh(out->cmd_syndrome));
4297 error = -1;
4298 goto free;
4299 }
4300
4301 free:
4302 mcx_dmamem_free(sc, &mxm);
4303
4304 return (error);
4305 }
4306
4307 static int
4308 mcx_alloc_uar(struct mcx_softc *sc, int *uar)
4309 {
4310 struct mcx_cmdq_entry *cqe;
4311 struct mcx_cmd_alloc_uar_in *in;
4312 struct mcx_cmd_alloc_uar_out *out;
4313 int error;
4314
4315 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4316 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4317
4318 in = mcx_cmdq_in(cqe);
4319 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
4320 in->cmd_op_mod = htobe16(0);
4321
4322 mcx_cmdq_post(sc, cqe, 0);
4323
4324 error = mcx_cmdq_poll(sc, cqe, 1000);
4325 if (error != 0) {
4326 printf(", alloc uar timeout\n");
4327 return (-1);
4328 }
4329 if (mcx_cmdq_verify(cqe) != 0) {
4330 printf(", alloc uar command corrupt\n");
4331 return (-1);
4332 }
4333
4334 out = mcx_cmdq_out(cqe);
4335 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4336 printf(", alloc uar failed (%x)\n", out->cmd_status);
4337 return (-1);
4338 }
4339
4340 *uar = mcx_get_id(out->cmd_uar);
4341 return (0);
4342 }
4343
4344 static int
4345 mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar,
4346 uint64_t events, int vector)
4347 {
4348 struct mcx_cmdq_entry *cqe;
4349 struct mcx_dmamem mxm;
4350 struct mcx_cmd_create_eq_in *in;
4351 struct mcx_cmd_create_eq_mb_in *mbin;
4352 struct mcx_cmd_create_eq_out *out;
4353 struct mcx_eq_entry *eqe;
4354 int error;
4355 uint64_t *pas;
4356 int insize, npages, paslen, i, token;
4357
4358 eq->eq_cons = 0;
4359
4360 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
4361 MCX_PAGE_SIZE);
4362 paslen = npages * sizeof(*pas);
4363 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
4364
4365 if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE,
4366 MCX_PAGE_SIZE) != 0) {
4367 printf(", unable to allocate event queue memory\n");
4368 return (-1);
4369 }
4370
4371 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
4372 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
4373 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
4374 }
4375
4376 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4377 token = mcx_cmdq_token(sc);
4378 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4379
4380 in = mcx_cmdq_in(cqe);
4381 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
4382 in->cmd_op_mod = htobe16(0);
4383
4384 if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4385 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4386 &cqe->cq_input_ptr, token) != 0) {
4387 printf(", unable to allocate create eq mailboxen\n");
4388 goto free_eq;
4389 }
4390 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4391 mbin->cmd_eq_ctx.eq_uar_size = htobe32(
4392 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar);
4393 mbin->cmd_eq_ctx.eq_intr = vector;
4394 mbin->cmd_event_bitmask = htobe64(events);
4395
4396 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
4397 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
4398
4399 /* physical addresses follow the mailbox in data */
4400 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem);
4401 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
4402 mcx_cmdq_post(sc, cqe, 0);
4403
4404 error = mcx_cmdq_poll(sc, cqe, 1000);
4405 if (error != 0) {
4406 printf(", create eq timeout\n");
4407 goto free_mxm;
4408 }
4409 if (mcx_cmdq_verify(cqe) != 0) {
4410 printf(", create eq command corrupt\n");
4411 goto free_mxm;
4412 }
4413
4414 out = mcx_cmdq_out(cqe);
4415 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4416 printf(", create eq failed (%x, %x)\n", out->cmd_status,
4417 be32toh(out->cmd_syndrome));
4418 goto free_mxm;
4419 }
4420
4421 eq->eq_n = mcx_get_id(out->cmd_eqn);
4422
4423 mcx_dmamem_free(sc, &mxm);
4424
4425 mcx_arm_eq(sc, eq, uar);
4426
4427 return (0);
4428
4429 free_mxm:
4430 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
4431 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
4432 mcx_dmamem_free(sc, &mxm);
4433 free_eq:
4434 mcx_dmamem_free(sc, &eq->eq_mem);
4435 return (-1);
4436 }
4437
4438 static int
4439 mcx_alloc_pd(struct mcx_softc *sc)
4440 {
4441 struct mcx_cmdq_entry *cqe;
4442 struct mcx_cmd_alloc_pd_in *in;
4443 struct mcx_cmd_alloc_pd_out *out;
4444 int error;
4445
4446 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4447 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4448
4449 in = mcx_cmdq_in(cqe);
4450 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
4451 in->cmd_op_mod = htobe16(0);
4452
4453 mcx_cmdq_post(sc, cqe, 0);
4454
4455 error = mcx_cmdq_poll(sc, cqe, 1000);
4456 if (error != 0) {
4457 printf(", alloc pd timeout\n");
4458 return (-1);
4459 }
4460 if (mcx_cmdq_verify(cqe) != 0) {
4461 printf(", alloc pd command corrupt\n");
4462 return (-1);
4463 }
4464
4465 out = mcx_cmdq_out(cqe);
4466 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4467 printf(", alloc pd failed (%x)\n", out->cmd_status);
4468 return (-1);
4469 }
4470
4471 sc->sc_pd = mcx_get_id(out->cmd_pd);
4472 return (0);
4473 }
4474
4475 static int
4476 mcx_alloc_tdomain(struct mcx_softc *sc)
4477 {
4478 struct mcx_cmdq_entry *cqe;
4479 struct mcx_cmd_alloc_td_in *in;
4480 struct mcx_cmd_alloc_td_out *out;
4481 int error;
4482
4483 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4484 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4485
4486 in = mcx_cmdq_in(cqe);
4487 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
4488 in->cmd_op_mod = htobe16(0);
4489
4490 mcx_cmdq_post(sc, cqe, 0);
4491
4492 error = mcx_cmdq_poll(sc, cqe, 1000);
4493 if (error != 0) {
4494 printf(", alloc transport domain timeout\n");
4495 return (-1);
4496 }
4497 if (mcx_cmdq_verify(cqe) != 0) {
4498 printf(", alloc transport domain command corrupt\n");
4499 return (-1);
4500 }
4501
4502 out = mcx_cmdq_out(cqe);
4503 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4504 printf(", alloc transport domain failed (%x)\n",
4505 out->cmd_status);
4506 return (-1);
4507 }
4508
4509 sc->sc_tdomain = mcx_get_id(out->cmd_tdomain);
4510 return (0);
4511 }
4512
4513 static int
4514 mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr)
4515 {
4516 struct mcx_dmamem mxm;
4517 struct mcx_cmdq_entry *cqe;
4518 struct mcx_cmd_query_nic_vport_context_in *in;
4519 struct mcx_cmd_query_nic_vport_context_out *out;
4520 struct mcx_nic_vport_ctx *ctx;
4521 uint8_t *addr;
4522 int error, token, i;
4523
4524 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4525 token = mcx_cmdq_token(sc);
4526 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
4527
4528 in = mcx_cmdq_in(cqe);
4529 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
4530 in->cmd_op_mod = htobe16(0);
4531 in->cmd_allowed_list_type = 0;
4532
4533 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4534 &cqe->cq_output_ptr, token) != 0) {
4535 printf(", unable to allocate "
4536 "query nic vport context mailboxen\n");
4537 return (-1);
4538 }
4539 mcx_cmdq_mboxes_sign(&mxm, 1);
4540 mcx_cmdq_post(sc, cqe, 0);
4541
4542 error = mcx_cmdq_poll(sc, cqe, 1000);
4543 if (error != 0) {
4544 printf(", query nic vport context timeout\n");
4545 goto free;
4546 }
4547 if (mcx_cmdq_verify(cqe) != 0) {
4548 printf(", query nic vport context command corrupt\n");
4549 goto free;
4550 }
4551
4552 out = mcx_cmdq_out(cqe);
4553 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4554 printf(", query nic vport context failed (%x, %x)\n",
4555 out->cmd_status, be32toh(out->cmd_syndrome));
4556 error = -1;
4557 goto free;
4558 }
4559
4560 ctx = (struct mcx_nic_vport_ctx *)
4561 mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4562 addr = (uint8_t *)&ctx->vp_perm_addr;
4563 for (i = 0; i < ETHER_ADDR_LEN; i++) {
4564 enaddr[i] = addr[i + 2];
4565 }
4566 free:
4567 mcx_dmamem_free(sc, &mxm);
4568
4569 return (error);
4570 }
4571
4572 static int
4573 mcx_query_special_contexts(struct mcx_softc *sc)
4574 {
4575 struct mcx_cmdq_entry *cqe;
4576 struct mcx_cmd_query_special_ctx_in *in;
4577 struct mcx_cmd_query_special_ctx_out *out;
4578 int error;
4579
4580 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4581 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4582
4583 in = mcx_cmdq_in(cqe);
4584 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
4585 in->cmd_op_mod = htobe16(0);
4586
4587 mcx_cmdq_post(sc, cqe, 0);
4588
4589 error = mcx_cmdq_poll(sc, cqe, 1000);
4590 if (error != 0) {
4591 printf(", query special contexts timeout\n");
4592 return (-1);
4593 }
4594 if (mcx_cmdq_verify(cqe) != 0) {
4595 printf(", query special contexts command corrupt\n");
4596 return (-1);
4597 }
4598
4599 out = mcx_cmdq_out(cqe);
4600 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4601 printf(", query special contexts failed (%x)\n",
4602 out->cmd_status);
4603 return (-1);
4604 }
4605
4606 sc->sc_lkey = be32toh(out->cmd_resd_lkey);
4607 return (0);
4608 }
4609
4610 static int
4611 mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
4612 {
4613 struct mcx_reg_pmtu pmtu;
4614 int error;
4615
4616 /* read max mtu */
4617 memset(&pmtu, 0, sizeof(pmtu));
4618 pmtu.rp_local_port = 1;
4619 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
4620 sizeof(pmtu));
4621 if (error != 0) {
4622 printf(", unable to get port MTU\n");
4623 return error;
4624 }
4625
4626 mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu));
4627 pmtu.rp_admin_mtu = htobe16(mtu);
4628 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
4629 sizeof(pmtu));
4630 if (error != 0) {
4631 printf(", unable to set port MTU\n");
4632 return error;
4633 }
4634
4635 sc->sc_hardmtu = mtu;
4636 sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long));
4637 return 0;
4638 }
4639
4640 static int
4641 mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn)
4642 {
4643 struct mcx_cmdq_entry *cmde;
4644 struct mcx_cq_entry *cqe;
4645 struct mcx_dmamem mxm;
4646 struct mcx_cmd_create_cq_in *in;
4647 struct mcx_cmd_create_cq_mb_in *mbin;
4648 struct mcx_cmd_create_cq_out *out;
4649 int error;
4650 uint64_t *pas;
4651 int insize, npages, paslen, i, token;
4652
4653 cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db);
4654
4655 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
4656 MCX_PAGE_SIZE);
4657 paslen = npages * sizeof(*pas);
4658 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
4659
4660 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
4661 MCX_PAGE_SIZE) != 0) {
4662 printf("%s: unable to allocate completion queue memory\n",
4663 DEVNAME(sc));
4664 return (-1);
4665 }
4666 cqe = MCX_DMA_KVA(&cq->cq_mem);
4667 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
4668 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
4669 }
4670
4671 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4672 token = mcx_cmdq_token(sc);
4673 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
4674
4675 in = mcx_cmdq_in(cmde);
4676 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
4677 in->cmd_op_mod = htobe16(0);
4678
4679 if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4680 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4681 &cmde->cq_input_ptr, token) != 0) {
4682 printf("%s: unable to allocate create cq mailboxen\n",
4683 DEVNAME(sc));
4684 goto free_cq;
4685 }
4686 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4687 mbin->cmd_cq_ctx.cq_uar_size = htobe32(
4688 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar);
4689 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
4690 mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
4691 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
4692 MCX_CQ_MOD_COUNTER);
4693 mbin->cmd_cq_ctx.cq_doorbell = htobe64(
4694 MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell);
4695
4696 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4697 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
4698
4699 /* physical addresses follow the mailbox in data */
4700 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem);
4701 mcx_cmdq_post(sc, cmde, 0);
4702
4703 error = mcx_cmdq_poll(sc, cmde, 1000);
4704 if (error != 0) {
4705 printf("%s: create cq timeout\n", DEVNAME(sc));
4706 goto free_mxm;
4707 }
4708 if (mcx_cmdq_verify(cmde) != 0) {
4709 printf("%s: create cq command corrupt\n", DEVNAME(sc));
4710 goto free_mxm;
4711 }
4712
4713 out = mcx_cmdq_out(cmde);
4714 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4715 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
4716 out->cmd_status, be32toh(out->cmd_syndrome));
4717 goto free_mxm;
4718 }
4719
4720 cq->cq_n = mcx_get_id(out->cmd_cqn);
4721 cq->cq_cons = 0;
4722 cq->cq_count = 0;
4723
4724 mcx_dmamem_free(sc, &mxm);
4725
4726 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4727 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
4728 BUS_DMASYNC_PREWRITE);
4729
4730 mcx_arm_cq(sc, cq, uar);
4731
4732 return (0);
4733
4734 free_mxm:
4735 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4736 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
4737 mcx_dmamem_free(sc, &mxm);
4738 free_cq:
4739 mcx_dmamem_free(sc, &cq->cq_mem);
4740 return (-1);
4741 }
4742
4743 static int
4744 mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq)
4745 {
4746 struct mcx_cmdq_entry *cqe;
4747 struct mcx_cmd_destroy_cq_in *in;
4748 struct mcx_cmd_destroy_cq_out *out;
4749 int error;
4750 int token;
4751
4752 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4753 token = mcx_cmdq_token(sc);
4754 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
4755
4756 in = mcx_cmdq_in(cqe);
4757 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
4758 in->cmd_op_mod = htobe16(0);
4759 in->cmd_cqn = htobe32(cq->cq_n);
4760
4761 mcx_cmdq_post(sc, cqe, 0);
4762 error = mcx_cmdq_poll(sc, cqe, 1000);
4763 if (error != 0) {
4764 printf("%s: destroy cq timeout\n", DEVNAME(sc));
4765 return error;
4766 }
4767 if (mcx_cmdq_verify(cqe) != 0) {
4768 printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
4769 return error;
4770 }
4771
4772 out = mcx_cmdq_out(cqe);
4773 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4774 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
4775 out->cmd_status, be32toh(out->cmd_syndrome));
4776 return -1;
4777 }
4778
4779 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4780 cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
4781 BUS_DMASYNC_POSTWRITE);
4782
4783 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4784 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
4785 mcx_dmamem_free(sc, &cq->cq_mem);
4786
4787 cq->cq_n = 0;
4788 cq->cq_cons = 0;
4789 cq->cq_count = 0;
4790 return 0;
4791 }
4792
4793 static int
4794 mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn)
4795 {
4796 struct mcx_cmdq_entry *cqe;
4797 struct mcx_dmamem mxm;
4798 struct mcx_cmd_create_rq_in *in;
4799 struct mcx_cmd_create_rq_out *out;
4800 struct mcx_rq_ctx *mbin;
4801 int error;
4802 uint64_t *pas;
4803 uint32_t rq_flags;
4804 int insize, npages, paslen, token;
4805
4806 rx->rx_doorbell = MCX_WQ_DOORBELL_BASE +
4807 (db * MCX_WQ_DOORBELL_STRIDE);
4808
4809 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
4810 MCX_PAGE_SIZE);
4811 paslen = npages * sizeof(*pas);
4812 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
4813
4814 if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE,
4815 MCX_PAGE_SIZE) != 0) {
4816 printf("%s: unable to allocate receive queue memory\n",
4817 DEVNAME(sc));
4818 return (-1);
4819 }
4820
4821 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4822 token = mcx_cmdq_token(sc);
4823 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4824
4825 in = mcx_cmdq_in(cqe);
4826 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
4827 in->cmd_op_mod = htobe16(0);
4828
4829 if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4830 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4831 &cqe->cq_input_ptr, token) != 0) {
4832 printf("%s: unable to allocate create rq mailboxen\n",
4833 DEVNAME(sc));
4834 goto free_rq;
4835 }
4836 mbin = (struct mcx_rq_ctx *)
4837 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
4838 rq_flags = MCX_RQ_CTX_RLKEY;
4839 mbin->rq_flags = htobe32(rq_flags);
4840 mbin->rq_cqn = htobe32(cqn);
4841 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
4842 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
4843 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
4844 rx->rx_doorbell);
4845 mbin->rq_wq.wq_log_stride = htobe16(4);
4846 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
4847
4848 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4849 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
4850
4851 /* physical addresses follow the mailbox in data */
4852 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem);
4853 mcx_cmdq_post(sc, cqe, 0);
4854
4855 error = mcx_cmdq_poll(sc, cqe, 1000);
4856 if (error != 0) {
4857 printf("%s: create rq timeout\n", DEVNAME(sc));
4858 goto free_mxm;
4859 }
4860 if (mcx_cmdq_verify(cqe) != 0) {
4861 printf("%s: create rq command corrupt\n", DEVNAME(sc));
4862 goto free_mxm;
4863 }
4864
4865 out = mcx_cmdq_out(cqe);
4866 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4867 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
4868 out->cmd_status, be32toh(out->cmd_syndrome));
4869 goto free_mxm;
4870 }
4871
4872 rx->rx_rqn = mcx_get_id(out->cmd_rqn);
4873
4874 mcx_dmamem_free(sc, &mxm);
4875
4876 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4877 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
4878
4879 return (0);
4880
4881 free_mxm:
4882 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4883 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
4884 mcx_dmamem_free(sc, &mxm);
4885 free_rq:
4886 mcx_dmamem_free(sc, &rx->rx_rq_mem);
4887 return (-1);
4888 }
4889
4890 static int
4891 mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx)
4892 {
4893 struct mcx_cmdq_entry *cqe;
4894 struct mcx_dmamem mxm;
4895 struct mcx_cmd_modify_rq_in *in;
4896 struct mcx_cmd_modify_rq_mb_in *mbin;
4897 struct mcx_cmd_modify_rq_out *out;
4898 int error;
4899 int token;
4900
4901 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4902 token = mcx_cmdq_token(sc);
4903 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
4904 sizeof(*out), token);
4905
4906 in = mcx_cmdq_in(cqe);
4907 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
4908 in->cmd_op_mod = htobe16(0);
4909 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn);
4910
4911 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4912 &cqe->cq_input_ptr, token) != 0) {
4913 printf("%s: unable to allocate modify rq mailbox\n",
4914 DEVNAME(sc));
4915 return (-1);
4916 }
4917 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4918 mbin->cmd_rq_ctx.rq_flags = htobe32(
4919 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
4920
4921 mcx_cmdq_mboxes_sign(&mxm, 1);
4922 mcx_cmdq_post(sc, cqe, 0);
4923 error = mcx_cmdq_poll(sc, cqe, 1000);
4924 if (error != 0) {
4925 printf("%s: modify rq timeout\n", DEVNAME(sc));
4926 goto free;
4927 }
4928 if (mcx_cmdq_verify(cqe) != 0) {
4929 printf("%s: modify rq command corrupt\n", DEVNAME(sc));
4930 goto free;
4931 }
4932
4933 out = mcx_cmdq_out(cqe);
4934 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4935 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
4936 out->cmd_status, be32toh(out->cmd_syndrome));
4937 error = -1;
4938 goto free;
4939 }
4940
4941 free:
4942 mcx_dmamem_free(sc, &mxm);
4943 return (error);
4944 }
4945
4946 static int
4947 mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx)
4948 {
4949 struct mcx_cmdq_entry *cqe;
4950 struct mcx_cmd_destroy_rq_in *in;
4951 struct mcx_cmd_destroy_rq_out *out;
4952 int error;
4953 int token;
4954
4955 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4956 token = mcx_cmdq_token(sc);
4957 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
4958
4959 in = mcx_cmdq_in(cqe);
4960 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
4961 in->cmd_op_mod = htobe16(0);
4962 in->cmd_rqn = htobe32(rx->rx_rqn);
4963
4964 mcx_cmdq_post(sc, cqe, 0);
4965 error = mcx_cmdq_poll(sc, cqe, 1000);
4966 if (error != 0) {
4967 printf("%s: destroy rq timeout\n", DEVNAME(sc));
4968 return error;
4969 }
4970 if (mcx_cmdq_verify(cqe) != 0) {
4971 printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
4972 return error;
4973 }
4974
4975 out = mcx_cmdq_out(cqe);
4976 if (out->cmd_status != MCX_CQ_STATUS_OK) {
4977 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
4978 out->cmd_status, be32toh(out->cmd_syndrome));
4979 return -1;
4980 }
4981
4982 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4983 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
4984
4985 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4986 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
4987 mcx_dmamem_free(sc, &rx->rx_rq_mem);
4988
4989 rx->rx_rqn = 0;
4990 return 0;
4991 }
4992
4993 static int
4994 mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn)
4995 {
4996 struct mcx_cmdq_entry *cqe;
4997 struct mcx_dmamem mxm;
4998 struct mcx_cmd_create_tir_in *in;
4999 struct mcx_cmd_create_tir_mb_in *mbin;
5000 struct mcx_cmd_create_tir_out *out;
5001 int error;
5002 int token;
5003
5004 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5005 token = mcx_cmdq_token(sc);
5006 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5007 sizeof(*out), token);
5008
5009 in = mcx_cmdq_in(cqe);
5010 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
5011 in->cmd_op_mod = htobe16(0);
5012
5013 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5014 &cqe->cq_input_ptr, token) != 0) {
5015 printf("%s: unable to allocate create tir mailbox\n",
5016 DEVNAME(sc));
5017 return (-1);
5018 }
5019 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5020 /* leave disp_type = 0, so packets get sent to the inline rqn */
5021 mbin->cmd_inline_rqn = htobe32(rx->rx_rqn);
5022 mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
5023
5024 mcx_cmdq_post(sc, cqe, 0);
5025 error = mcx_cmdq_poll(sc, cqe, 1000);
5026 if (error != 0) {
5027 printf("%s: create tir timeout\n", DEVNAME(sc));
5028 goto free;
5029 }
5030 if (mcx_cmdq_verify(cqe) != 0) {
5031 printf("%s: create tir command corrupt\n", DEVNAME(sc));
5032 goto free;
5033 }
5034
5035 out = mcx_cmdq_out(cqe);
5036 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5037 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
5038 out->cmd_status, be32toh(out->cmd_syndrome));
5039 error = -1;
5040 goto free;
5041 }
5042
5043 *tirn = mcx_get_id(out->cmd_tirn);
5044 free:
5045 mcx_dmamem_free(sc, &mxm);
5046 return (error);
5047 }
5048
5049 static int
5050 mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel,
5051 int *tirn)
5052 {
5053 struct mcx_cmdq_entry *cqe;
5054 struct mcx_dmamem mxm;
5055 struct mcx_cmd_create_tir_in *in;
5056 struct mcx_cmd_create_tir_mb_in *mbin;
5057 struct mcx_cmd_create_tir_out *out;
5058 int error;
5059 int token;
5060
5061 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5062 token = mcx_cmdq_token(sc);
5063 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5064 sizeof(*out), token);
5065
5066 in = mcx_cmdq_in(cqe);
5067 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
5068 in->cmd_op_mod = htobe16(0);
5069
5070 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5071 &cqe->cq_input_ptr, token) != 0) {
5072 printf("%s: unable to allocate create tir mailbox\n",
5073 DEVNAME(sc));
5074 return (-1);
5075 }
5076 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5077 mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT
5078 << MCX_TIR_CTX_DISP_TYPE_SHIFT);
5079 mbin->cmd_indir_table = htobe32(rqtn);
5080 mbin->cmd_tdomain = htobe32(sc->sc_tdomain |
5081 MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT);
5082 mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel);
5083 stoeplitz_to_key(&mbin->cmd_rx_hash_key,
5084 sizeof(mbin->cmd_rx_hash_key));
5085
5086 mcx_cmdq_post(sc, cqe, 0);
5087 error = mcx_cmdq_poll(sc, cqe, 1000);
5088 if (error != 0) {
5089 printf("%s: create tir timeout\n", DEVNAME(sc));
5090 goto free;
5091 }
5092 if (mcx_cmdq_verify(cqe) != 0) {
5093 printf("%s: create tir command corrupt\n", DEVNAME(sc));
5094 goto free;
5095 }
5096
5097 out = mcx_cmdq_out(cqe);
5098 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5099 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
5100 out->cmd_status, be32toh(out->cmd_syndrome));
5101 error = -1;
5102 goto free;
5103 }
5104
5105 *tirn = mcx_get_id(out->cmd_tirn);
5106 free:
5107 mcx_dmamem_free(sc, &mxm);
5108 return (error);
5109 }
5110
5111 static int
5112 mcx_destroy_tir(struct mcx_softc *sc, int tirn)
5113 {
5114 struct mcx_cmdq_entry *cqe;
5115 struct mcx_cmd_destroy_tir_in *in;
5116 struct mcx_cmd_destroy_tir_out *out;
5117 int error;
5118 int token;
5119
5120 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5121 token = mcx_cmdq_token(sc);
5122 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5123
5124 in = mcx_cmdq_in(cqe);
5125 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
5126 in->cmd_op_mod = htobe16(0);
5127 in->cmd_tirn = htobe32(tirn);
5128
5129 mcx_cmdq_post(sc, cqe, 0);
5130 error = mcx_cmdq_poll(sc, cqe, 1000);
5131 if (error != 0) {
5132 printf("%s: destroy tir timeout\n", DEVNAME(sc));
5133 return error;
5134 }
5135 if (mcx_cmdq_verify(cqe) != 0) {
5136 printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
5137 return error;
5138 }
5139
5140 out = mcx_cmdq_out(cqe);
5141 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5142 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
5143 out->cmd_status, be32toh(out->cmd_syndrome));
5144 return -1;
5145 }
5146
5147 return (0);
5148 }
5149
5150 static int
5151 mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db,
5152 int cqn)
5153 {
5154 struct mcx_cmdq_entry *cqe;
5155 struct mcx_dmamem mxm;
5156 struct mcx_cmd_create_sq_in *in;
5157 struct mcx_sq_ctx *mbin;
5158 struct mcx_cmd_create_sq_out *out;
5159 int error;
5160 uint64_t *pas;
5161 int insize, npages, paslen, token;
5162
5163 tx->tx_doorbell = MCX_WQ_DOORBELL_BASE +
5164 (db * MCX_WQ_DOORBELL_STRIDE) + 4;
5165
5166 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
5167 MCX_PAGE_SIZE);
5168 paslen = npages * sizeof(*pas);
5169 insize = sizeof(struct mcx_sq_ctx) + paslen;
5170
5171 if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE,
5172 MCX_PAGE_SIZE) != 0) {
5173 printf("%s: unable to allocate send queue memory\n",
5174 DEVNAME(sc));
5175 return (-1);
5176 }
5177
5178 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5179 token = mcx_cmdq_token(sc);
5180 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
5181 token);
5182
5183 in = mcx_cmdq_in(cqe);
5184 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
5185 in->cmd_op_mod = htobe16(0);
5186
5187 if (mcx_cmdq_mboxes_alloc(sc, &mxm,
5188 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
5189 &cqe->cq_input_ptr, token) != 0) {
5190 printf("%s: unable to allocate create sq mailboxen\n",
5191 DEVNAME(sc));
5192 goto free_sq;
5193 }
5194 mbin = (struct mcx_sq_ctx *)
5195 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
5196 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
5197 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
5198 mbin->sq_cqn = htobe32(cqn);
5199 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
5200 mbin->sq_tis_num = htobe32(sc->sc_tis);
5201 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
5202 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
5203 mbin->sq_wq.wq_uar_page = htobe32(uar);
5204 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
5205 tx->tx_doorbell);
5206 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
5207 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
5208
5209 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5210 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
5211
5212 /* physical addresses follow the mailbox in data */
5213 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10,
5214 npages, &tx->tx_sq_mem);
5215 mcx_cmdq_post(sc, cqe, 0);
5216
5217 error = mcx_cmdq_poll(sc, cqe, 1000);
5218 if (error != 0) {
5219 printf("%s: create sq timeout\n", DEVNAME(sc));
5220 goto free_mxm;
5221 }
5222 if (mcx_cmdq_verify(cqe) != 0) {
5223 printf("%s: create sq command corrupt\n", DEVNAME(sc));
5224 goto free_mxm;
5225 }
5226
5227 out = mcx_cmdq_out(cqe);
5228 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5229 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
5230 out->cmd_status, be32toh(out->cmd_syndrome));
5231 goto free_mxm;
5232 }
5233
5234 tx->tx_uar = uar;
5235 tx->tx_sqn = mcx_get_id(out->cmd_sqn);
5236
5237 mcx_dmamem_free(sc, &mxm);
5238
5239 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
5240 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
5241
5242 return (0);
5243
5244 free_mxm:
5245 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5246 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
5247 mcx_dmamem_free(sc, &mxm);
5248 free_sq:
5249 mcx_dmamem_free(sc, &tx->tx_sq_mem);
5250 return (-1);
5251 }
5252
5253 static int
5254 mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx)
5255 {
5256 struct mcx_cmdq_entry *cqe;
5257 struct mcx_cmd_destroy_sq_in *in;
5258 struct mcx_cmd_destroy_sq_out *out;
5259 int error;
5260 int token;
5261
5262 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5263 token = mcx_cmdq_token(sc);
5264 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5265
5266 in = mcx_cmdq_in(cqe);
5267 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
5268 in->cmd_op_mod = htobe16(0);
5269 in->cmd_sqn = htobe32(tx->tx_sqn);
5270
5271 mcx_cmdq_post(sc, cqe, 0);
5272 error = mcx_cmdq_poll(sc, cqe, 1000);
5273 if (error != 0) {
5274 printf("%s: destroy sq timeout\n", DEVNAME(sc));
5275 return error;
5276 }
5277 if (mcx_cmdq_verify(cqe) != 0) {
5278 printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
5279 return error;
5280 }
5281
5282 out = mcx_cmdq_out(cqe);
5283 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5284 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
5285 out->cmd_status, be32toh(out->cmd_syndrome));
5286 return -1;
5287 }
5288
5289 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
5290 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
5291
5292 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5293 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
5294 mcx_dmamem_free(sc, &tx->tx_sq_mem);
5295
5296 tx->tx_sqn = 0;
5297 return 0;
5298 }
5299
5300 static int
5301 mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx)
5302 {
5303 struct mcx_cmdq_entry *cqe;
5304 struct mcx_dmamem mxm;
5305 struct mcx_cmd_modify_sq_in *in;
5306 struct mcx_cmd_modify_sq_mb_in *mbin;
5307 struct mcx_cmd_modify_sq_out *out;
5308 int error;
5309 int token;
5310
5311 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5312 token = mcx_cmdq_token(sc);
5313 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5314 sizeof(*out), token);
5315
5316 in = mcx_cmdq_in(cqe);
5317 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
5318 in->cmd_op_mod = htobe16(0);
5319 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn);
5320
5321 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5322 &cqe->cq_input_ptr, token) != 0) {
5323 printf("%s: unable to allocate modify sq mailbox\n",
5324 DEVNAME(sc));
5325 return (-1);
5326 }
5327 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5328 mbin->cmd_sq_ctx.sq_flags = htobe32(
5329 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
5330
5331 mcx_cmdq_mboxes_sign(&mxm, 1);
5332 mcx_cmdq_post(sc, cqe, 0);
5333 error = mcx_cmdq_poll(sc, cqe, 1000);
5334 if (error != 0) {
5335 printf("%s: modify sq timeout\n", DEVNAME(sc));
5336 goto free;
5337 }
5338 if (mcx_cmdq_verify(cqe) != 0) {
5339 printf("%s: modify sq command corrupt\n", DEVNAME(sc));
5340 goto free;
5341 }
5342
5343 out = mcx_cmdq_out(cqe);
5344 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5345 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
5346 out->cmd_status, be32toh(out->cmd_syndrome));
5347 error = -1;
5348 goto free;
5349 }
5350
5351 free:
5352 mcx_dmamem_free(sc, &mxm);
5353 return (error);
5354 }
5355
5356 static int
5357 mcx_create_tis(struct mcx_softc *sc, int *tis)
5358 {
5359 struct mcx_cmdq_entry *cqe;
5360 struct mcx_dmamem mxm;
5361 struct mcx_cmd_create_tis_in *in;
5362 struct mcx_cmd_create_tis_mb_in *mbin;
5363 struct mcx_cmd_create_tis_out *out;
5364 int error;
5365 int token;
5366
5367 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5368 token = mcx_cmdq_token(sc);
5369 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5370 sizeof(*out), token);
5371
5372 in = mcx_cmdq_in(cqe);
5373 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
5374 in->cmd_op_mod = htobe16(0);
5375
5376 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5377 &cqe->cq_input_ptr, token) != 0) {
5378 printf("%s: unable to allocate create tis mailbox\n",
5379 DEVNAME(sc));
5380 return (-1);
5381 }
5382 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5383 mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
5384
5385 mcx_cmdq_mboxes_sign(&mxm, 1);
5386 mcx_cmdq_post(sc, cqe, 0);
5387 error = mcx_cmdq_poll(sc, cqe, 1000);
5388 if (error != 0) {
5389 printf("%s: create tis timeout\n", DEVNAME(sc));
5390 goto free;
5391 }
5392 if (mcx_cmdq_verify(cqe) != 0) {
5393 printf("%s: create tis command corrupt\n", DEVNAME(sc));
5394 goto free;
5395 }
5396
5397 out = mcx_cmdq_out(cqe);
5398 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5399 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
5400 out->cmd_status, be32toh(out->cmd_syndrome));
5401 error = -1;
5402 goto free;
5403 }
5404
5405 *tis = mcx_get_id(out->cmd_tisn);
5406 free:
5407 mcx_dmamem_free(sc, &mxm);
5408 return (error);
5409 }
5410
5411 static int
5412 mcx_destroy_tis(struct mcx_softc *sc, int tis)
5413 {
5414 struct mcx_cmdq_entry *cqe;
5415 struct mcx_cmd_destroy_tis_in *in;
5416 struct mcx_cmd_destroy_tis_out *out;
5417 int error;
5418 int token;
5419
5420 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5421 token = mcx_cmdq_token(sc);
5422 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5423
5424 in = mcx_cmdq_in(cqe);
5425 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
5426 in->cmd_op_mod = htobe16(0);
5427 in->cmd_tisn = htobe32(tis);
5428
5429 mcx_cmdq_post(sc, cqe, 0);
5430 error = mcx_cmdq_poll(sc, cqe, 1000);
5431 if (error != 0) {
5432 printf("%s: destroy tis timeout\n", DEVNAME(sc));
5433 return error;
5434 }
5435 if (mcx_cmdq_verify(cqe) != 0) {
5436 printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
5437 return error;
5438 }
5439
5440 out = mcx_cmdq_out(cqe);
5441 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5442 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
5443 out->cmd_status, be32toh(out->cmd_syndrome));
5444 return -1;
5445 }
5446
5447 return 0;
5448 }
5449
5450 static int
5451 mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt)
5452 {
5453 struct mcx_cmdq_entry *cqe;
5454 struct mcx_dmamem mxm;
5455 struct mcx_cmd_create_rqt_in *in;
5456 struct mcx_cmd_create_rqt_mb_in *mbin;
5457 struct mcx_cmd_create_rqt_out *out;
5458 struct mcx_rqt_ctx *rqt_ctx;
5459 int *rqtn;
5460 int error;
5461 int token;
5462 int i;
5463
5464 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5465 token = mcx_cmdq_token(sc);
5466 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) +
5467 (size * sizeof(int)), sizeof(*out), token);
5468
5469 in = mcx_cmdq_in(cqe);
5470 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT);
5471 in->cmd_op_mod = htobe16(0);
5472
5473 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5474 &cqe->cq_input_ptr, token) != 0) {
5475 printf("%s: unable to allocate create rqt mailbox\n",
5476 DEVNAME(sc));
5477 return (-1);
5478 }
5479 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5480 rqt_ctx = &mbin->cmd_rqt;
5481 rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size);
5482 rqt_ctx->cmd_rqt_actual_size = htobe16(size);
5483
5484 /* rqt list follows the rqt context */
5485 rqtn = (int *)(rqt_ctx + 1);
5486 for (i = 0; i < size; i++) {
5487 rqtn[i] = htobe32(rqns[i]);
5488 }
5489
5490 mcx_cmdq_mboxes_sign(&mxm, 1);
5491 mcx_cmdq_post(sc, cqe, 0);
5492 error = mcx_cmdq_poll(sc, cqe, 1000);
5493 if (error != 0) {
5494 printf("%s: create rqt timeout\n", DEVNAME(sc));
5495 goto free;
5496 }
5497 if (mcx_cmdq_verify(cqe) != 0) {
5498 printf("%s: create rqt command corrupt\n", DEVNAME(sc));
5499 goto free;
5500 }
5501
5502 out = mcx_cmdq_out(cqe);
5503 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5504 printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc),
5505 out->cmd_status, be32toh(out->cmd_syndrome));
5506 error = -1;
5507 goto free;
5508 }
5509
5510 *rqt = mcx_get_id(out->cmd_rqtn);
5511 return (0);
5512 free:
5513 mcx_dmamem_free(sc, &mxm);
5514 return (error);
5515 }
5516
5517 static int
5518 mcx_destroy_rqt(struct mcx_softc *sc, int rqt)
5519 {
5520 struct mcx_cmdq_entry *cqe;
5521 struct mcx_cmd_destroy_rqt_in *in;
5522 struct mcx_cmd_destroy_rqt_out *out;
5523 int error;
5524 int token;
5525
5526 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5527 token = mcx_cmdq_token(sc);
5528 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5529
5530 in = mcx_cmdq_in(cqe);
5531 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT);
5532 in->cmd_op_mod = htobe16(0);
5533 in->cmd_rqtn = htobe32(rqt);
5534
5535 mcx_cmdq_post(sc, cqe, 0);
5536 error = mcx_cmdq_poll(sc, cqe, 1000);
5537 if (error != 0) {
5538 printf("%s: destroy rqt timeout\n", DEVNAME(sc));
5539 return error;
5540 }
5541 if (mcx_cmdq_verify(cqe) != 0) {
5542 printf("%s: destroy rqt command corrupt\n", DEVNAME(sc));
5543 return error;
5544 }
5545
5546 out = mcx_cmdq_out(cqe);
5547 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5548 printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc),
5549 out->cmd_status, be32toh(out->cmd_syndrome));
5550 return -1;
5551 }
5552
5553 return 0;
5554 }
5555
5556 #if 0
5557 static int
5558 mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
5559 {
5560 struct mcx_cmdq_entry *cqe;
5561 struct mcx_cmd_alloc_flow_counter_in *in;
5562 struct mcx_cmd_alloc_flow_counter_out *out;
5563 int error;
5564
5565 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5566 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
5567
5568 in = mcx_cmdq_in(cqe);
5569 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
5570 in->cmd_op_mod = htobe16(0);
5571
5572 mcx_cmdq_post(sc, cqe, 0);
5573
5574 error = mcx_cmdq_poll(sc, cqe, 1000);
5575 if (error != 0) {
5576 printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
5577 return (-1);
5578 }
5579 if (mcx_cmdq_verify(cqe) != 0) {
5580 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
5581 return (-1);
5582 }
5583
5584 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
5585 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5586 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
5587 out->cmd_status);
5588 return (-1);
5589 }
5590
5591 sc->sc_flow_counter_id[i] = be16toh(out->cmd_flow_counter_id);
5592 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
5593
5594 return (0);
5595 }
5596 #endif
5597
5598 static int
5599 mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level,
5600 int *flow_table_id)
5601 {
5602 struct mcx_cmdq_entry *cqe;
5603 struct mcx_dmamem mxm;
5604 struct mcx_cmd_create_flow_table_in *in;
5605 struct mcx_cmd_create_flow_table_mb_in *mbin;
5606 struct mcx_cmd_create_flow_table_out *out;
5607 int error;
5608 int token;
5609
5610 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5611 token = mcx_cmdq_token(sc);
5612 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5613 sizeof(*out), token);
5614
5615 in = mcx_cmdq_in(cqe);
5616 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
5617 in->cmd_op_mod = htobe16(0);
5618
5619 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5620 &cqe->cq_input_ptr, token) != 0) {
5621 printf("%s: unable to allocate create flow table mailbox\n",
5622 DEVNAME(sc));
5623 return (-1);
5624 }
5625 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5626 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5627 mbin->cmd_ctx.ft_log_size = log_size;
5628 mbin->cmd_ctx.ft_level = level;
5629
5630 mcx_cmdq_mboxes_sign(&mxm, 1);
5631 mcx_cmdq_post(sc, cqe, 0);
5632 error = mcx_cmdq_poll(sc, cqe, 1000);
5633 if (error != 0) {
5634 printf("%s: create flow table timeout\n", DEVNAME(sc));
5635 goto free;
5636 }
5637 if (mcx_cmdq_verify(cqe) != 0) {
5638 printf("%s: create flow table command corrupt\n", DEVNAME(sc));
5639 goto free;
5640 }
5641
5642 out = mcx_cmdq_out(cqe);
5643 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5644 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
5645 out->cmd_status, be32toh(out->cmd_syndrome));
5646 error = -1;
5647 goto free;
5648 }
5649
5650 *flow_table_id = mcx_get_id(out->cmd_table_id);
5651 free:
5652 mcx_dmamem_free(sc, &mxm);
5653 return (error);
5654 }
5655
5656 static int
5657 mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id)
5658 {
5659 struct mcx_cmdq_entry *cqe;
5660 struct mcx_dmamem mxm;
5661 struct mcx_cmd_set_flow_table_root_in *in;
5662 struct mcx_cmd_set_flow_table_root_mb_in *mbin;
5663 struct mcx_cmd_set_flow_table_root_out *out;
5664 int error;
5665 int token;
5666
5667 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5668 token = mcx_cmdq_token(sc);
5669 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5670 sizeof(*out), token);
5671
5672 in = mcx_cmdq_in(cqe);
5673 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
5674 in->cmd_op_mod = htobe16(0);
5675
5676 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5677 &cqe->cq_input_ptr, token) != 0) {
5678 printf("%s: unable to allocate set flow table root mailbox\n",
5679 DEVNAME(sc));
5680 return (-1);
5681 }
5682 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5683 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5684 mbin->cmd_table_id = htobe32(flow_table_id);
5685
5686 mcx_cmdq_mboxes_sign(&mxm, 1);
5687 mcx_cmdq_post(sc, cqe, 0);
5688 error = mcx_cmdq_poll(sc, cqe, 1000);
5689 if (error != 0) {
5690 printf("%s: set flow table root timeout\n", DEVNAME(sc));
5691 goto free;
5692 }
5693 if (mcx_cmdq_verify(cqe) != 0) {
5694 printf("%s: set flow table root command corrupt\n",
5695 DEVNAME(sc));
5696 goto free;
5697 }
5698
5699 out = mcx_cmdq_out(cqe);
5700 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5701 printf("%s: set flow table root failed (%x, %x)\n",
5702 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
5703 error = -1;
5704 goto free;
5705 }
5706
5707 free:
5708 mcx_dmamem_free(sc, &mxm);
5709 return (error);
5710 }
5711
5712 static int
5713 mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id)
5714 {
5715 struct mcx_cmdq_entry *cqe;
5716 struct mcx_dmamem mxm;
5717 struct mcx_cmd_destroy_flow_table_in *in;
5718 struct mcx_cmd_destroy_flow_table_mb_in *mb;
5719 struct mcx_cmd_destroy_flow_table_out *out;
5720 int error;
5721 int token;
5722
5723 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5724 token = mcx_cmdq_token(sc);
5725 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
5726
5727 in = mcx_cmdq_in(cqe);
5728 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
5729 in->cmd_op_mod = htobe16(0);
5730
5731 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5732 &cqe->cq_input_ptr, token) != 0) {
5733 printf("%s: unable to allocate destroy flow table mailbox\n",
5734 DEVNAME(sc));
5735 return (-1);
5736 }
5737 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5738 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5739 mb->cmd_table_id = htobe32(flow_table_id);
5740
5741 mcx_cmdq_mboxes_sign(&mxm, 1);
5742 mcx_cmdq_post(sc, cqe, 0);
5743 error = mcx_cmdq_poll(sc, cqe, 1000);
5744 if (error != 0) {
5745 printf("%s: destroy flow table timeout\n", DEVNAME(sc));
5746 goto free;
5747 }
5748 if (mcx_cmdq_verify(cqe) != 0) {
5749 printf("%s: destroy flow table command corrupt\n",
5750 DEVNAME(sc));
5751 goto free;
5752 }
5753
5754 out = mcx_cmdq_out(cqe);
5755 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5756 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
5757 out->cmd_status, be32toh(out->cmd_syndrome));
5758 error = -1;
5759 goto free;
5760 }
5761
5762 free:
5763 mcx_dmamem_free(sc, &mxm);
5764 return (error);
5765 }
5766
5767
5768 static int
5769 mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group,
5770 int start, int size, int match_enable, struct mcx_flow_match *match)
5771 {
5772 struct mcx_cmdq_entry *cqe;
5773 struct mcx_dmamem mxm;
5774 struct mcx_cmd_create_flow_group_in *in;
5775 struct mcx_cmd_create_flow_group_mb_in *mbin;
5776 struct mcx_cmd_create_flow_group_out *out;
5777 struct mcx_flow_group *mfg;
5778 int error;
5779 int token;
5780
5781 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5782 token = mcx_cmdq_token(sc);
5783 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
5784 token);
5785
5786 in = mcx_cmdq_in(cqe);
5787 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
5788 in->cmd_op_mod = htobe16(0);
5789
5790 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5791 != 0) {
5792 printf("%s: unable to allocate create flow group mailbox\n",
5793 DEVNAME(sc));
5794 return (-1);
5795 }
5796 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5797 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5798 mbin->cmd_table_id = htobe32(flow_table_id);
5799 mbin->cmd_start_flow_index = htobe32(start);
5800 mbin->cmd_end_flow_index = htobe32(start + (size - 1));
5801
5802 mbin->cmd_match_criteria_enable = match_enable;
5803 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));
5804
5805 mcx_cmdq_mboxes_sign(&mxm, 2);
5806 mcx_cmdq_post(sc, cqe, 0);
5807 error = mcx_cmdq_poll(sc, cqe, 1000);
5808 if (error != 0) {
5809 printf("%s: create flow group timeout\n", DEVNAME(sc));
5810 goto free;
5811 }
5812 if (mcx_cmdq_verify(cqe) != 0) {
5813 printf("%s: create flow group command corrupt\n", DEVNAME(sc));
5814 goto free;
5815 }
5816
5817 out = mcx_cmdq_out(cqe);
5818 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5819 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
5820 out->cmd_status, be32toh(out->cmd_syndrome));
5821 error = -1;
5822 goto free;
5823 }
5824
5825 mfg = &sc->sc_flow_group[group];
5826 mfg->g_id = mcx_get_id(out->cmd_group_id);
5827 mfg->g_table = flow_table_id;
5828 mfg->g_start = start;
5829 mfg->g_size = size;
5830
5831 free:
5832 mcx_dmamem_free(sc, &mxm);
5833 return (error);
5834 }
5835
5836 static int
5837 mcx_destroy_flow_group(struct mcx_softc *sc, int group)
5838 {
5839 struct mcx_cmdq_entry *cqe;
5840 struct mcx_dmamem mxm;
5841 struct mcx_cmd_destroy_flow_group_in *in;
5842 struct mcx_cmd_destroy_flow_group_mb_in *mb;
5843 struct mcx_cmd_destroy_flow_group_out *out;
5844 struct mcx_flow_group *mfg;
5845 int error;
5846 int token;
5847
5848 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5849 token = mcx_cmdq_token(sc);
5850 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
5851
5852 in = mcx_cmdq_in(cqe);
5853 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
5854 in->cmd_op_mod = htobe16(0);
5855
5856 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
5857 &cqe->cq_input_ptr, token) != 0) {
5858 printf("%s: unable to allocate destroy flow group mailbox\n",
5859 DEVNAME(sc));
5860 return (-1);
5861 }
5862 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5863 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5864 mfg = &sc->sc_flow_group[group];
5865 mb->cmd_table_id = htobe32(mfg->g_table);
5866 mb->cmd_group_id = htobe32(mfg->g_id);
5867
5868 mcx_cmdq_mboxes_sign(&mxm, 2);
5869 mcx_cmdq_post(sc, cqe, 0);
5870 error = mcx_cmdq_poll(sc, cqe, 1000);
5871 if (error != 0) {
5872 printf("%s: destroy flow group timeout\n", DEVNAME(sc));
5873 goto free;
5874 }
5875 if (mcx_cmdq_verify(cqe) != 0) {
5876 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
5877 goto free;
5878 }
5879
5880 out = mcx_cmdq_out(cqe);
5881 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5882 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
5883 out->cmd_status, be32toh(out->cmd_syndrome));
5884 error = -1;
5885 goto free;
5886 }
5887
5888 mfg->g_id = -1;
5889 mfg->g_table = -1;
5890 mfg->g_size = 0;
5891 mfg->g_start = 0;
5892 free:
5893 mcx_dmamem_free(sc, &mxm);
5894 return (error);
5895 }
5896
5897 static int
5898 mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index,
5899 const uint8_t *macaddr, uint32_t dest)
5900 {
5901 struct mcx_cmdq_entry *cqe;
5902 struct mcx_dmamem mxm;
5903 struct mcx_cmd_set_flow_table_entry_in *in;
5904 struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
5905 struct mcx_cmd_set_flow_table_entry_out *out;
5906 struct mcx_flow_group *mfg;
5907 uint32_t *pdest;
5908 int error;
5909 int token;
5910
5911 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5912 token = mcx_cmdq_token(sc);
5913 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
5914 sizeof(*out), token);
5915
5916 in = mcx_cmdq_in(cqe);
5917 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
5918 in->cmd_op_mod = htobe16(0);
5919
5920 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5921 != 0) {
5922 printf("%s: unable to allocate set flow table entry mailbox\n",
5923 DEVNAME(sc));
5924 return (-1);
5925 }
5926
5927 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5928 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5929
5930 mfg = &sc->sc_flow_group[group];
5931 mbin->cmd_table_id = htobe32(mfg->g_table);
5932 mbin->cmd_flow_index = htobe32(mfg->g_start + index);
5933 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
5934
5935 /* flow context ends at offset 0x330, 0x130 into the second mbox */
5936 pdest = (uint32_t *)
5937 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
5938 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
5939 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
5940 *pdest = htobe32(dest);
5941
5942 /* the only thing we match on at the moment is the dest mac address */
5943 if (macaddr != NULL) {
5944 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
5945 ETHER_ADDR_LEN);
5946 }
5947
5948 mcx_cmdq_mboxes_sign(&mxm, 2);
5949 mcx_cmdq_post(sc, cqe, 0);
5950 error = mcx_cmdq_poll(sc, cqe, 1000);
5951 if (error != 0) {
5952 printf("%s: set flow table entry timeout\n", DEVNAME(sc));
5953 goto free;
5954 }
5955 if (mcx_cmdq_verify(cqe) != 0) {
5956 printf("%s: set flow table entry command corrupt\n",
5957 DEVNAME(sc));
5958 goto free;
5959 }
5960
5961 out = mcx_cmdq_out(cqe);
5962 if (out->cmd_status != MCX_CQ_STATUS_OK) {
5963 printf("%s: set flow table entry failed (%x, %x)\n",
5964 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
5965 error = -1;
5966 goto free;
5967 }
5968
5969 free:
5970 mcx_dmamem_free(sc, &mxm);
5971 return (error);
5972 }
5973
5974 static int
5975 mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index,
5976 int ethertype, int ip_proto, uint32_t dest)
5977 {
5978 struct mcx_cmdq_entry *cqe;
5979 struct mcx_dmamem mxm;
5980 struct mcx_cmd_set_flow_table_entry_in *in;
5981 struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
5982 struct mcx_cmd_set_flow_table_entry_out *out;
5983 struct mcx_flow_group *mfg;
5984 uint32_t *pdest;
5985 int error;
5986 int token;
5987
5988 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5989 token = mcx_cmdq_token(sc);
5990 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
5991 sizeof(*out), token);
5992
5993 in = mcx_cmdq_in(cqe);
5994 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
5995 in->cmd_op_mod = htobe16(0);
5996
5997 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5998 != 0) {
5999 printf("%s: unable to allocate set flow table entry mailbox\n",
6000 DEVNAME(sc));
6001 return (-1);
6002 }
6003
6004 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6005 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
6006
6007 mfg = &sc->sc_flow_group[group];
6008 mbin->cmd_table_id = htobe32(mfg->g_table);
6009 mbin->cmd_flow_index = htobe32(mfg->g_start + index);
6010 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
6011
6012 /* flow context ends at offset 0x330, 0x130 into the second mbox */
6013 pdest = (uint32_t *)
6014 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
6015 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
6016 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
6017 *pdest = htobe32(dest);
6018
6019 mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype);
6020 mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto;
6021
6022 mcx_cmdq_mboxes_sign(&mxm, 2);
6023 mcx_cmdq_post(sc, cqe, 0);
6024 error = mcx_cmdq_poll(sc, cqe, 1000);
6025 if (error != 0) {
6026 printf("%s: set flow table entry timeout\n", DEVNAME(sc));
6027 goto free;
6028 }
6029 if (mcx_cmdq_verify(cqe) != 0) {
6030 printf("%s: set flow table entry command corrupt\n",
6031 DEVNAME(sc));
6032 goto free;
6033 }
6034
6035 out = mcx_cmdq_out(cqe);
6036 if (out->cmd_status != MCX_CQ_STATUS_OK) {
6037 printf("%s: set flow table entry failed (%x, %x)\n",
6038 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
6039 error = -1;
6040 goto free;
6041 }
6042
6043 free:
6044 mcx_dmamem_free(sc, &mxm);
6045 return (error);
6046 }
6047
6048 static int
6049 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
6050 {
6051 struct mcx_cmdq_entry *cqe;
6052 struct mcx_dmamem mxm;
6053 struct mcx_cmd_delete_flow_table_entry_in *in;
6054 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
6055 struct mcx_cmd_delete_flow_table_entry_out *out;
6056 struct mcx_flow_group *mfg;
6057 int error;
6058 int token;
6059
6060 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6061 token = mcx_cmdq_token(sc);
6062 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
6063 token);
6064
6065 in = mcx_cmdq_in(cqe);
6066 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
6067 in->cmd_op_mod = htobe16(0);
6068
6069 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6070 &cqe->cq_input_ptr, token) != 0) {
6071 printf("%s: unable to allocate "
6072 "delete flow table entry mailbox\n", DEVNAME(sc));
6073 return (-1);
6074 }
6075 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6076 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
6077
6078 mfg = &sc->sc_flow_group[group];
6079 mbin->cmd_table_id = htobe32(mfg->g_table);
6080 mbin->cmd_flow_index = htobe32(mfg->g_start + index);
6081
6082 mcx_cmdq_mboxes_sign(&mxm, 2);
6083 mcx_cmdq_post(sc, cqe, 0);
6084 error = mcx_cmdq_poll(sc, cqe, 1000);
6085 if (error != 0) {
6086 printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
6087 goto free;
6088 }
6089 if (mcx_cmdq_verify(cqe) != 0) {
6090 printf("%s: delete flow table entry command corrupt\n",
6091 DEVNAME(sc));
6092 goto free;
6093 }
6094
6095 out = mcx_cmdq_out(cqe);
6096 if (out->cmd_status != MCX_CQ_STATUS_OK) {
6097 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
6098 DEVNAME(sc), group, index, out->cmd_status,
6099 be32toh(out->cmd_syndrome));
6100 error = -1;
6101 goto free;
6102 }
6103
6104 free:
6105 mcx_dmamem_free(sc, &mxm);
6106 return (error);
6107 }
6108
6109 #if 0
6110 int
6111 mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id)
6112 {
6113 struct mcx_dmamem mxm;
6114 struct mcx_cmdq_entry *cqe;
6115 struct mcx_cmd_query_flow_table_in *in;
6116 struct mcx_cmd_query_flow_table_mb_in *mbin;
6117 struct mcx_cmd_query_flow_table_out *out;
6118 struct mcx_cmd_query_flow_table_mb_out *mbout;
6119 uint8_t token = mcx_cmdq_token(sc);
6120 int error;
6121 int i;
6122 uint8_t *dump;
6123
6124 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6125 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6126 sizeof(*out) + sizeof(*mbout) + 16, token);
6127
6128 in = mcx_cmdq_in(cqe);
6129 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
6130 in->cmd_op_mod = htobe16(0);
6131
6132 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6133 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
6134 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6135 &cqe->cq_output_ptr, token) != 0) {
6136 printf(", unable to allocate query flow table mailboxes\n");
6137 return (-1);
6138 }
6139 cqe->cq_input_ptr = cqe->cq_output_ptr;
6140
6141 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6142 mbin->cmd_table_type = 0;
6143 mbin->cmd_table_id = htobe32(flow_table_id);
6144
6145 mcx_cmdq_mboxes_sign(&mxm, 1);
6146
6147 mcx_cmdq_post(sc, cqe, 0);
6148 error = mcx_cmdq_poll(sc, cqe, 1000);
6149 if (error != 0) {
6150 printf("%s: query flow table timeout\n", DEVNAME(sc));
6151 goto free;
6152 }
6153 error = mcx_cmdq_verify(cqe);
6154 if (error != 0) {
6155 printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
6156 goto free;
6157 }
6158
6159 out = mcx_cmdq_out(cqe);
6160 switch (out->cmd_status) {
6161 case MCX_CQ_STATUS_OK:
6162 break;
6163 default:
6164 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
6165 out->cmd_status, be32toh(out->cmd_syndrome));
6166 error = -1;
6167 goto free;
6168 }
6169
6170 mbout = (struct mcx_cmd_query_flow_table_mb_out *)
6171 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6172 dump = (uint8_t *)mbout + 8;
6173 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
6174 printf("%.2x ", dump[i]);
6175 if (i % 16 == 15)
6176 printf("\n");
6177 }
6178 free:
6179 mcx_cq_mboxes_free(sc, &mxm);
6180 return (error);
6181 }
6182 int
6183 mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index)
6184 {
6185 struct mcx_dmamem mxm;
6186 struct mcx_cmdq_entry *cqe;
6187 struct mcx_cmd_query_flow_table_entry_in *in;
6188 struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
6189 struct mcx_cmd_query_flow_table_entry_out *out;
6190 struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
6191 uint8_t token = mcx_cmdq_token(sc);
6192 int error;
6193 int i;
6194 uint8_t *dump;
6195
6196 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6197 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6198 sizeof(*out) + sizeof(*mbout) + 16, token);
6199
6200 in = mcx_cmdq_in(cqe);
6201 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
6202 in->cmd_op_mod = htobe16(0);
6203
6204 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6205 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6206 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6207 &cqe->cq_output_ptr, token) != 0) {
6208 printf(", unable to allocate "
6209 "query flow table entry mailboxes\n");
6210 return (-1);
6211 }
6212 cqe->cq_input_ptr = cqe->cq_output_ptr;
6213
6214 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6215 mbin->cmd_table_type = 0;
6216 mbin->cmd_table_id = htobe32(flow_table_id);
6217 mbin->cmd_flow_index = htobe32(index);
6218
6219 mcx_cmdq_mboxes_sign(&mxm, 1);
6220
6221 mcx_cmdq_post(sc, cqe, 0);
6222 error = mcx_cmdq_poll(sc, cqe, 1000);
6223 if (error != 0) {
6224 printf("%s: query flow table entry timeout\n", DEVNAME(sc));
6225 goto free;
6226 }
6227 error = mcx_cmdq_verify(cqe);
6228 if (error != 0) {
6229 printf("%s: query flow table entry reply corrupt\n",
6230 DEVNAME(sc));
6231 goto free;
6232 }
6233
6234 out = mcx_cmdq_out(cqe);
6235 switch (out->cmd_status) {
6236 case MCX_CQ_STATUS_OK:
6237 break;
6238 default:
6239 printf("%s: query flow table entry failed (%x/%x)\n",
6240 DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
6241 error = -1;
6242 goto free;
6243 }
6244
6245 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
6246 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6247 dump = (uint8_t *)mbout;
6248 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6249 printf("%.2x ", dump[i]);
6250 if (i % 16 == 15)
6251 printf("\n");
6252 }
6253
6254 free:
6255 mcx_cq_mboxes_free(sc, &mxm);
6256 return (error);
6257 }
6258
6259 int
6260 mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id)
6261 {
6262 struct mcx_dmamem mxm;
6263 struct mcx_cmdq_entry *cqe;
6264 struct mcx_cmd_query_flow_group_in *in;
6265 struct mcx_cmd_query_flow_group_mb_in *mbin;
6266 struct mcx_cmd_query_flow_group_out *out;
6267 struct mcx_cmd_query_flow_group_mb_out *mbout;
6268 uint8_t token = mcx_cmdq_token(sc);
6269 int error;
6270 int i;
6271 uint8_t *dump;
6272
6273 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6274 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6275 sizeof(*out) + sizeof(*mbout) + 16, token);
6276
6277 in = mcx_cmdq_in(cqe);
6278 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
6279 in->cmd_op_mod = htobe16(0);
6280
6281 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6282 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6283 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6284 &cqe->cq_output_ptr, token) != 0) {
6285 printf(", unable to allocate query flow group mailboxes\n");
6286 return (-1);
6287 }
6288 cqe->cq_input_ptr = cqe->cq_output_ptr;
6289
6290 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6291 mbin->cmd_table_type = 0;
6292 mbin->cmd_table_id = htobe32(flow_table_id);
6293 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
6294
6295 mcx_cmdq_mboxes_sign(&mxm, 1);
6296
6297 mcx_cmdq_post(sc, cqe, 0);
6298 error = mcx_cmdq_poll(sc, cqe, 1000);
6299 if (error != 0) {
6300 printf("%s: query flow group timeout\n", DEVNAME(sc));
6301 goto free;
6302 }
6303 error = mcx_cmdq_verify(cqe);
6304 if (error != 0) {
6305 printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
6306 goto free;
6307 }
6308
6309 out = mcx_cmdq_out(cqe);
6310 switch (out->cmd_status) {
6311 case MCX_CQ_STATUS_OK:
6312 break;
6313 default:
6314 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
6315 out->cmd_status, be32toh(out->cmd_syndrome));
6316 error = -1;
6317 goto free;
6318 }
6319
6320 mbout = (struct mcx_cmd_query_flow_group_mb_out *)
6321 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6322 dump = (uint8_t *)mbout;
6323 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6324 printf("%.2x ", dump[i]);
6325 if (i % 16 == 15)
6326 printf("\n");
6327 }
6328 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
6329 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6330 printf("%.2x ", dump[i]);
6331 if (i % 16 == 15)
6332 printf("\n");
6333 }
6334
6335 free:
6336 mcx_cq_mboxes_free(sc, &mxm);
6337 return (error);
6338 }
6339
6340 static int
6341 mcx_dump_counters(struct mcx_softc *sc)
6342 {
6343 struct mcx_dmamem mxm;
6344 struct mcx_cmdq_entry *cqe;
6345 struct mcx_cmd_query_vport_counters_in *in;
6346 struct mcx_cmd_query_vport_counters_mb_in *mbin;
6347 struct mcx_cmd_query_vport_counters_out *out;
6348 struct mcx_nic_vport_counters *counters;
6349 int error, token;
6350
6351 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6352 token = mcx_cmdq_token(sc);
6353 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6354 sizeof(*out) + sizeof(*counters), token);
6355
6356 in = mcx_cmdq_in(cqe);
6357 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
6358 in->cmd_op_mod = htobe16(0);
6359
6360 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
6361 &cqe->cq_output_ptr, token) != 0) {
6362 printf(", unable to allocate "
6363 "query nic vport counters mailboxen\n");
6364 return (-1);
6365 }
6366 cqe->cq_input_ptr = cqe->cq_output_ptr;
6367
6368 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6369 mbin->cmd_clear = 0x80;
6370
6371 mcx_cmdq_mboxes_sign(&mxm, 1);
6372 mcx_cmdq_post(sc, cqe, 0);
6373
6374 error = mcx_cmdq_poll(sc, cqe, 1000);
6375 if (error != 0) {
6376 printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
6377 goto free;
6378 }
6379 if (mcx_cmdq_verify(cqe) != 0) {
6380 printf("%s: query nic vport counters command corrupt\n",
6381 DEVNAME(sc));
6382 goto free;
6383 }
6384
6385 out = mcx_cmdq_out(cqe);
6386 if (out->cmd_status != MCX_CQ_STATUS_OK) {
6387 printf("%s: query nic vport counters failed (%x, %x)\n",
6388 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
6389 error = -1;
6390 goto free;
6391 }
6392
6393 counters = (struct mcx_nic_vport_counters *)
6394 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6395 if (counters->rx_bcast.packets + counters->tx_bcast.packets +
6396 counters->rx_ucast.packets + counters->tx_ucast.packets +
6397 counters->rx_err.packets + counters->tx_err.packets)
6398 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
6399 DEVNAME(sc),
6400 betoh64(counters->tx_err.packets),
6401 betoh64(counters->rx_err.packets),
6402 betoh64(counters->tx_ucast.packets),
6403 betoh64(counters->rx_ucast.packets),
6404 betoh64(counters->tx_bcast.packets),
6405 betoh64(counters->rx_bcast.packets));
6406 free:
6407 mcx_dmamem_free(sc, &mxm);
6408
6409 return (error);
6410 }
6411
6412 static int
6413 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
6414 {
6415 struct mcx_dmamem mxm;
6416 struct mcx_cmdq_entry *cqe;
6417 struct mcx_cmd_query_flow_counter_in *in;
6418 struct mcx_cmd_query_flow_counter_mb_in *mbin;
6419 struct mcx_cmd_query_flow_counter_out *out;
6420 struct mcx_counter *counters;
6421 int error, token;
6422
6423 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6424 token = mcx_cmdq_token(sc);
6425 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
6426 sizeof(*counters), token);
6427
6428 in = mcx_cmdq_in(cqe);
6429 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
6430 in->cmd_op_mod = htobe16(0);
6431
6432 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
6433 &cqe->cq_output_ptr, token) != 0) {
6434 printf(", unable to allocate query flow counter mailboxen\n");
6435 return (-1);
6436 }
6437 cqe->cq_input_ptr = cqe->cq_output_ptr;
6438 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6439 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
6440 mbin->cmd_clear = 0x80;
6441
6442 mcx_cmdq_mboxes_sign(&mxm, 1);
6443 mcx_cmdq_post(sc, cqe, 0);
6444
6445 error = mcx_cmdq_poll(sc, cqe, 1000);
6446 if (error != 0) {
6447 printf("%s: query flow counter timeout\n", DEVNAME(sc));
6448 goto free;
6449 }
6450 if (mcx_cmdq_verify(cqe) != 0) {
6451 printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
6452 goto free;
6453 }
6454
6455 out = mcx_cmdq_out(cqe);
6456 if (out->cmd_status != MCX_CQ_STATUS_OK) {
6457 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
6458 out->cmd_status, betoh32(out->cmd_syndrome));
6459 error = -1;
6460 goto free;
6461 }
6462
6463 counters = (struct mcx_counter *)
6464 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6465 if (counters->packets)
6466 printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
6467 betoh64(counters->packets));
6468 free:
6469 mcx_dmamem_free(sc, &mxm);
6470
6471 return (error);
6472 }
6473
6474 #endif
6475
6476 #if NKSTAT > 0
6477
6478 int
6479 mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx)
6480 {
6481 struct mcx_dmamem mxm;
6482 struct mcx_cmdq_entry *cqe;
6483 struct mcx_cmd_query_rq_in *in;
6484 struct mcx_cmd_query_rq_out *out;
6485 struct mcx_cmd_query_rq_mb_out *mbout;
6486 uint8_t token = mcx_cmdq_token(sc);
6487 int error;
6488
6489 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6490 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
6491 token);
6492
6493 in = mcx_cmdq_in(cqe);
6494 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
6495 in->cmd_op_mod = htobe16(0);
6496 in->cmd_rqn = htobe32(rx->rx_rqn);
6497
6498 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6499 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6500 &cqe->cq_output_ptr, token) != 0) {
6501 printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc));
6502 return (-1);
6503 }
6504
6505 mcx_cmdq_mboxes_sign(&mxm, 1);
6506
6507 mcx_cmdq_post(sc, cqe, 0);
6508 error = mcx_cmdq_poll(sc, cqe, 1000);
6509 if (error != 0) {
6510 printf("%s: query rq timeout\n", DEVNAME(sc));
6511 goto free;
6512 }
6513 error = mcx_cmdq_verify(cqe);
6514 if (error != 0) {
6515 printf("%s: query rq reply corrupt\n", DEVNAME(sc));
6516 goto free;
6517 }
6518
6519 out = mcx_cmdq_out(cqe);
6520 switch (out->cmd_status) {
6521 case MCX_CQ_STATUS_OK:
6522 break;
6523 default:
6524 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
6525 out->cmd_status, be32toh(out->cmd_syndrome));
6526 error = -1;
6527 goto free;
6528 }
6529
6530 mbout = (struct mcx_cmd_query_rq_mb_out *)
6531 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6532 memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx));
6533
6534 free:
6535 mcx_cq_mboxes_free(sc, &mxm);
6536 return (error);
6537 }
6538
6539 int
6540 mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx)
6541 {
6542 struct mcx_dmamem mxm;
6543 struct mcx_cmdq_entry *cqe;
6544 struct mcx_cmd_query_sq_in *in;
6545 struct mcx_cmd_query_sq_out *out;
6546 struct mcx_cmd_query_sq_mb_out *mbout;
6547 uint8_t token = mcx_cmdq_token(sc);
6548 int error;
6549
6550 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6551 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
6552 token);
6553
6554 in = mcx_cmdq_in(cqe);
6555 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
6556 in->cmd_op_mod = htobe16(0);
6557 in->cmd_sqn = htobe32(tx->tx_sqn);
6558
6559 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6560 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6561 &cqe->cq_output_ptr, token) != 0) {
6562 printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc));
6563 return (-1);
6564 }
6565
6566 mcx_cmdq_mboxes_sign(&mxm, 1);
6567
6568 mcx_cmdq_post(sc, cqe, 0);
6569 error = mcx_cmdq_poll(sc, cqe, 1000);
6570 if (error != 0) {
6571 printf("%s: query sq timeout\n", DEVNAME(sc));
6572 goto free;
6573 }
6574 error = mcx_cmdq_verify(cqe);
6575 if (error != 0) {
6576 printf("%s: query sq reply corrupt\n", DEVNAME(sc));
6577 goto free;
6578 }
6579
6580 out = mcx_cmdq_out(cqe);
6581 switch (out->cmd_status) {
6582 case MCX_CQ_STATUS_OK:
6583 break;
6584 default:
6585 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
6586 out->cmd_status, be32toh(out->cmd_syndrome));
6587 error = -1;
6588 goto free;
6589 }
6590
6591 mbout = (struct mcx_cmd_query_sq_mb_out *)
6592 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6593 memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx));
6594
6595 free:
6596 mcx_cq_mboxes_free(sc, &mxm);
6597 return (error);
6598 }
6599
6600 int
6601 mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx)
6602 {
6603 struct mcx_dmamem mxm;
6604 struct mcx_cmdq_entry *cqe;
6605 struct mcx_cmd_query_cq_in *in;
6606 struct mcx_cmd_query_cq_out *out;
6607 struct mcx_cq_ctx *ctx;
6608 uint8_t token = mcx_cmdq_token(sc);
6609 int error;
6610
6611 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6612 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
6613 token);
6614
6615 in = mcx_cmdq_in(cqe);
6616 in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ);
6617 in->cmd_op_mod = htobe16(0);
6618 in->cmd_cqn = htobe32(cq->cq_n);
6619
6620 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6621 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6622 &cqe->cq_output_ptr, token) != 0) {
6623 printf("%s: unable to allocate query cq mailboxes\n",
6624 DEVNAME(sc));
6625 return (-1);
6626 }
6627
6628 mcx_cmdq_mboxes_sign(&mxm, 1);
6629
6630 mcx_cmdq_post(sc, cqe, 0);
6631 error = mcx_cmdq_poll(sc, cqe, 1000);
6632 if (error != 0) {
6633 printf("%s: query cq timeout\n", DEVNAME(sc));
6634 goto free;
6635 }
6636 if (mcx_cmdq_verify(cqe) != 0) {
6637 printf("%s: query cq reply corrupt\n", DEVNAME(sc));
6638 goto free;
6639 }
6640
6641 out = mcx_cmdq_out(cqe);
6642 switch (out->cmd_status) {
6643 case MCX_CQ_STATUS_OK:
6644 break;
6645 default:
6646 printf("%s: query qc failed (%x/%x)\n", DEVNAME(sc),
6647 out->cmd_status, be32toh(out->cmd_syndrome));
6648 error = -1;
6649 goto free;
6650 }
6651
6652 ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6653 memcpy(cq_ctx, ctx, sizeof(*cq_ctx));
6654 free:
6655 mcx_dmamem_free(sc, &mxm);
6656 return (error);
6657 }
6658
6659 int
6660 mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx)
6661 {
6662 struct mcx_dmamem mxm;
6663 struct mcx_cmdq_entry *cqe;
6664 struct mcx_cmd_query_eq_in *in;
6665 struct mcx_cmd_query_eq_out *out;
6666 struct mcq_eq_ctx *ctx;
6667 uint8_t token = mcx_cmdq_token(sc);
6668 int error;
6669
6670 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6671 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
6672 token);
6673
6674 in = mcx_cmdq_in(cqe);
6675 in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ);
6676 in->cmd_op_mod = htobe16(0);
6677 in->cmd_eqn = htobe32(eq->eq_n);
6678
6679 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6680 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6681 &cqe->cq_output_ptr, token) != 0) {
6682 printf("%s: unable to allocate query eq mailboxes\n",
6683 DEVNAME(sc));
6684 return (-1);
6685 }
6686
6687 mcx_cmdq_mboxes_sign(&mxm, 1);
6688
6689 mcx_cmdq_post(sc, cqe, 0);
6690 error = mcx_cmdq_poll(sc, cqe, 1000);
6691 if (error != 0) {
6692 printf("%s: query eq timeout\n", DEVNAME(sc));
6693 goto free;
6694 }
6695 if (mcx_cmdq_verify(cqe) != 0) {
6696 printf("%s: query eq reply corrupt\n", DEVNAME(sc));
6697 goto free;
6698 }
6699
6700 out = mcx_cmdq_out(cqe);
6701 switch (out->cmd_status) {
6702 case MCX_CQ_STATUS_OK:
6703 break;
6704 default:
6705 printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc),
6706 out->cmd_status, be32toh(out->cmd_syndrome));
6707 error = -1;
6708 goto free;
6709 }
6710
6711 ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6712 memcpy(eq_ctx, ctx, sizeof(*eq_ctx));
6713 free:
6714 mcx_dmamem_free(sc, &mxm);
6715 return (error);
6716 }
6717
6718 #endif /* NKSTAT > 0 */
6719
6720
6721 static inline unsigned int
6722 mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots)
6723 {
6724 struct mcx_rq_entry *ring, *rqe;
6725 struct mcx_slot *ms;
6726 struct mbuf *m;
6727 uint slot, p, fills;
6728
6729 ring = MCX_DMA_KVA(&rx->rx_rq_mem);
6730 p = rx->rx_prod;
6731
6732 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
6733 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
6734
6735 slot = (p % (1 << MCX_LOG_RQ_SIZE));
6736 rqe = ring;
6737 for (fills = 0; fills < nslots; fills++) {
6738 slot = p % (1 << MCX_LOG_RQ_SIZE);
6739
6740 ms = &rx->rx_slots[slot];
6741 rqe = &ring[slot];
6742
6743 m = NULL;
6744 MGETHDR(m, M_DONTWAIT, MT_DATA);
6745 if (m == NULL)
6746 break;
6747
6748 MCLGET(m, M_DONTWAIT);
6749 if ((m->m_flags & M_EXT) == 0) {
6750 m_freem(m);
6751 break;
6752 }
6753
6754 m->m_len = m->m_pkthdr.len = sc->sc_hardmtu;
6755 m_adj(m, m->m_ext.ext_size - sc->sc_rxbufsz);
6756 m_adj(m, ETHER_ALIGN);
6757
6758 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
6759 BUS_DMA_NOWAIT) != 0) {
6760 m_freem(m);
6761 break;
6762 }
6763 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, BUS_DMASYNC_PREREAD);
6764 ms->ms_m = m;
6765
6766 be32enc(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len);
6767 be64enc(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr);
6768 be32enc(&rqe->rqe_lkey, sc->sc_lkey);
6769
6770 p++;
6771 }
6772
6773 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
6774 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
6775
6776 rx->rx_prod = p;
6777
6778 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
6779 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
6780 be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell),
6781 p & MCX_WQ_DOORBELL_MASK);
6782 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
6783 rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
6784
6785 return (nslots - fills);
6786 }
6787
6788 static int
6789 mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx)
6790 {
6791 u_int slots;
6792
6793 slots = mcx_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE));
6794 if (slots == 0)
6795 return (1);
6796
6797 slots = mcx_rx_fill_slots(sc, rx, slots);
6798 mcx_rxr_put(&rx->rx_rxr, slots);
6799 return (0);
6800 }
6801
6802 void
6803 mcx_refill(void *xrx)
6804 {
6805 struct mcx_rx *rx = xrx;
6806 struct mcx_softc *sc = rx->rx_softc;
6807
6808 mcx_rx_fill(sc, rx);
6809
6810 if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
6811 callout_schedule(&rx->rx_refill, 1);
6812 }
6813
6814 static int
6815 mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx,
6816 struct mcx_cq_entry *cqe)
6817 {
6818 struct mcx_slot *ms;
6819 bus_dmamap_t map;
6820 int slot, slots;
6821
6822 slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
6823
6824 ms = &tx->tx_slots[slot];
6825 map = ms->ms_map;
6826 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
6827 BUS_DMASYNC_POSTWRITE);
6828
6829 slots = 1;
6830 if (map->dm_nsegs > 1)
6831 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;
6832
6833 bus_dmamap_unload(sc->sc_dmat, map);
6834 m_freem(ms->ms_m);
6835 ms->ms_m = NULL;
6836
6837 return (slots);
6838 }
6839
6840 static uint64_t
6841 mcx_uptime(void)
6842 {
6843 struct timespec ts;
6844
6845 nanouptime(&ts);
6846
6847 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec);
6848 }
6849
6850 static void
6851 mcx_calibrate_first(struct mcx_softc *sc)
6852 {
6853 struct mcx_calibration *c = &sc->sc_calibration[0];
6854 int s;
6855
6856 sc->sc_calibration_gen = 0;
6857
6858 s = splhigh(); /* crit_enter? */
6859 c->c_ubase = mcx_uptime();
6860 c->c_tbase = mcx_timer(sc);
6861 splx(s);
6862 c->c_ratio = 0;
6863
6864 #if notyet
6865 callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz);
6866 #endif
6867 }
6868
6869 #define MCX_TIMESTAMP_SHIFT 24
6870
6871 static void
6872 mcx_calibrate(void *arg)
6873 {
6874 struct mcx_softc *sc = arg;
6875 struct mcx_calibration *nc, *pc;
6876 uint64_t udiff, tdiff;
6877 unsigned int gen;
6878 int s;
6879
6880 if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING))
6881 return;
6882
6883 callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz);
6884
6885 gen = sc->sc_calibration_gen;
6886 pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
6887 gen++;
6888 nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
6889
6890 nc->c_uptime = pc->c_ubase;
6891 nc->c_timestamp = pc->c_tbase;
6892
6893 s = splhigh(); /* crit_enter? */
6894 nc->c_ubase = mcx_uptime();
6895 nc->c_tbase = mcx_timer(sc);
6896 splx(s);
6897
6898 udiff = nc->c_ubase - nc->c_uptime;
6899 tdiff = nc->c_tbase - nc->c_timestamp;
6900
6901 /*
6902 * udiff is the wall clock time between calibration ticks,
6903 * which should be 32 seconds or 32 billion nanoseconds. if
6904 * we squint, 1 billion nanoseconds is kind of like a 32 bit
6905 * number, so 32 billion should still have a lot of high bits
6906 * spare. we use this space by shifting the nanoseconds up
6907 * 24 bits so we have a nice big number to divide by the
6908 * number of mcx timer ticks.
6909 */
6910 nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff;
6911
6912 membar_producer();
6913 sc->sc_calibration_gen = gen;
6914 }
6915
6916 static int
6917 mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx,
6918 struct mcx_cq_entry *cqe, struct mcx_mbufq *mq,
6919 const struct mcx_calibration *c)
6920 {
6921 struct ifnet *ifp = &sc->sc_ec.ec_if;
6922 struct mcx_slot *ms;
6923 struct mbuf *m;
6924 uint32_t flags, len;
6925 int slot;
6926
6927 len = be32dec(&cqe->cq_byte_cnt);
6928 slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
6929
6930 ms = &rx->rx_slots[slot];
6931 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, len, BUS_DMASYNC_POSTREAD);
6932 bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
6933
6934 m = ms->ms_m;
6935 ms->ms_m = NULL;
6936
6937 m_set_rcvif(m, &sc->sc_ec.ec_if);
6938 m->m_pkthdr.len = m->m_len = len;
6939
6940 #if 0
6941 if (cqe->cq_rx_hash_type) {
6942 m->m_pkthdr.ph_flowid = be32toh(cqe->cq_rx_hash);
6943 m->m_pkthdr.csum_flags |= M_FLOWID;
6944 }
6945 #endif
6946
6947 flags = be32dec(&cqe->cq_flags);
6948 if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) {
6949 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx)
6950 m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
6951 }
6952 if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) {
6953 if (ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx)
6954 m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
6955 if (ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx)
6956 m->m_pkthdr.csum_flags |= M_CSUM_TCPv6;
6957 if (ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx)
6958 m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
6959 if (ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx)
6960 m->m_pkthdr.csum_flags |= M_CSUM_UDPv6;
6961 }
6962 if (flags & MCX_CQ_ENTRY_FLAGS_CV) {
6963 vlan_set_tag(m, flags & MCX_CQ_ENTRY_FLAGS_VLAN_MASK);
6964 }
6965
6966 #if notyet
6967 if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_LINK0) && c->c_ratio) {
6968 uint64_t t = be64dec(&cqe->cq_timestamp);
6969 t -= c->c_timestamp;
6970 t *= c->c_ratio;
6971 t >>= MCX_TIMESTAMP_SHIFT;
6972 t += c->c_uptime;
6973
6974 m->m_pkthdr.ph_timestamp = t;
6975 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
6976 }
6977 #endif
6978
6979 MBUFQ_ENQUEUE(mq, m);
6980
6981 return (1);
6982 }
6983
6984 static struct mcx_cq_entry *
6985 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
6986 {
6987 struct mcx_cq_entry *cqe;
6988 int next;
6989
6990 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
6991 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
6992
6993 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
6994 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
6995 return (&cqe[next]);
6996 }
6997
6998 return (NULL);
6999 }
7000
7001 static void
7002 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar)
7003 {
7004 struct mcx_cq_doorbell *db;
7005 bus_size_t offset;
7006 uint32_t val;
7007 uint64_t uval;
7008
7009 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
7010 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
7011
7012 db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell);
7013
7014 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7015 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE);
7016
7017 be32enc(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
7018 be32enc(&db->db_arm_ci, val);
7019
7020 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7021 cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE);
7022
7023 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL;
7024
7025 uval = (uint64_t)val << 32;
7026 uval |= cq->cq_n;
7027
7028 bus_space_write_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval));
7029 mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE);
7030 }
7031
7032 void
7033 mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq)
7034 {
7035 struct mcx_rx *rx = &q->q_rx;
7036 struct mcx_tx *tx = &q->q_tx;
7037 struct ifnet *ifp = &sc->sc_ec.ec_if;
7038 const struct mcx_calibration *c;
7039 unsigned int gen;
7040 struct mcx_cq_entry *cqe;
7041 struct mcx_mbufq mq;
7042 struct mbuf *m;
7043 int rxfree, txfree;
7044
7045 MBUFQ_INIT(&mq);
7046
7047 gen = sc->sc_calibration_gen;
7048 membar_consumer();
7049 c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
7050
7051 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
7052 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
7053
7054 rxfree = 0;
7055 txfree = 0;
7056 while ((cqe = mcx_next_cq_entry(sc, cq))) {
7057 uint8_t opcode;
7058 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
7059 switch (opcode) {
7060 case MCX_CQ_ENTRY_OPCODE_REQ:
7061 txfree += mcx_process_txeof(sc, tx, cqe);
7062 break;
7063 case MCX_CQ_ENTRY_OPCODE_SEND:
7064 rxfree += mcx_process_rx(sc, rx, cqe, &mq, c);
7065 break;
7066 case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
7067 case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
7068 /* uint8_t *cqp = (uint8_t *)cqe; */
7069 /* printf("%s: cq completion error: %x\n",
7070 DEVNAME(sc), cqp[0x37]); */
7071 break;
7072
7073 default:
7074 /* printf("%s: cq completion opcode %x??\n",
7075 DEVNAME(sc), opcode); */
7076 break;
7077 }
7078
7079 cq->cq_cons++;
7080 }
7081
7082 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
7083 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
7084
7085 if (rxfree > 0) {
7086 mcx_rxr_put(&rx->rx_rxr, rxfree);
7087 while (MBUFQ_FIRST(&mq) != NULL) {
7088 MBUFQ_DEQUEUE(&mq, m);
7089 if_percpuq_enqueue(ifp->if_percpuq, m);
7090 }
7091
7092 mcx_rx_fill(sc, rx);
7093 if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
7094 callout_schedule(&rx->rx_refill, 1);
7095 }
7096
7097 cq->cq_count++;
7098 mcx_arm_cq(sc, cq, q->q_uar);
7099
7100 if (txfree > 0) {
7101 tx->tx_cons += txfree;
7102 if_schedule_deferred_start(ifp);
7103 }
7104 }
7105
7106
7107 static void
7108 mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar)
7109 {
7110 bus_size_t offset;
7111 uint32_t val;
7112
7113 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM;
7114 val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff);
7115
7116 mcx_wr(sc, offset, val);
7117 mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE);
7118 }
7119
7120 static struct mcx_eq_entry *
7121 mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq)
7122 {
7123 struct mcx_eq_entry *eqe;
7124 int next;
7125
7126 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
7127 next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE);
7128 if ((eqe[next].eq_owner & 1) ==
7129 ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
7130 eq->eq_cons++;
7131 return (&eqe[next]);
7132 }
7133 return (NULL);
7134 }
7135
7136 int
7137 mcx_admin_intr(void *xsc)
7138 {
7139 struct mcx_softc *sc = (struct mcx_softc *)xsc;
7140 struct mcx_eq *eq = &sc->sc_admin_eq;
7141 struct mcx_eq_entry *eqe;
7142
7143 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7144 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
7145
7146 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
7147 switch (eqe->eq_event_type) {
7148 case MCX_EVENT_TYPE_LAST_WQE:
7149 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
7150 break;
7151
7152 case MCX_EVENT_TYPE_CQ_ERROR:
7153 /* printf("%s: cq error\n", DEVNAME(sc)); */
7154 break;
7155
7156 case MCX_EVENT_TYPE_CMD_COMPLETION:
7157 /* wakeup probably */
7158 break;
7159
7160 case MCX_EVENT_TYPE_PORT_CHANGE:
7161 workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL);
7162 break;
7163
7164 default:
7165 /* printf("%s: something happened\n", DEVNAME(sc)); */
7166 break;
7167 }
7168 }
7169
7170 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7171 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
7172
7173 mcx_arm_eq(sc, eq, sc->sc_uar);
7174
7175 return (1);
7176 }
7177
7178 int
7179 mcx_cq_intr(void *xq)
7180 {
7181 struct mcx_queues *q = (struct mcx_queues *)xq;
7182 struct mcx_softc *sc = q->q_sc;
7183 struct mcx_eq *eq = &q->q_eq;
7184 struct mcx_eq_entry *eqe;
7185 int cqn;
7186
7187 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7188 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
7189
7190 while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
7191 switch (eqe->eq_event_type) {
7192 case MCX_EVENT_TYPE_COMPLETION:
7193 cqn = be32toh(eqe->eq_event_data[6]);
7194 if (cqn == q->q_cq.cq_n)
7195 mcx_process_cq(sc, q, &q->q_cq);
7196 break;
7197 }
7198 }
7199
7200 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7201 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
7202
7203 mcx_arm_eq(sc, eq, q->q_uar);
7204
7205 return (1);
7206 }
7207
7208 static void
7209 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
7210 int total)
7211 {
7212 struct mcx_slot *ms;
7213
7214 int i = allocated;
7215 while (i-- > 0) {
7216 ms = &slots[i];
7217 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
7218 if (ms->ms_m != NULL)
7219 m_freem(ms->ms_m);
7220 }
7221 kmem_free(slots, total * sizeof(*ms));
7222 }
7223
7224 static int
7225 mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q)
7226 {
7227 struct mcx_rx *rx;
7228 struct mcx_tx *tx;
7229 struct mcx_slot *ms;
7230 int i;
7231
7232 rx = &q->q_rx;
7233 rx->rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE),
7234 KM_SLEEP);
7235
7236 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
7237 ms = &rx->rx_slots[i];
7238 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
7239 sc->sc_hardmtu, 0,
7240 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
7241 &ms->ms_map) != 0) {
7242 printf("%s: failed to allocate rx dma maps\n",
7243 DEVNAME(sc));
7244 goto destroy_rx_slots;
7245 }
7246 }
7247
7248 tx = &q->q_tx;
7249 tx->tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE),
7250 KM_SLEEP);
7251
7252 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
7253 ms = &tx->tx_slots[i];
7254 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
7255 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
7256 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
7257 &ms->ms_map) != 0) {
7258 printf("%s: failed to allocate tx dma maps\n",
7259 DEVNAME(sc));
7260 goto destroy_tx_slots;
7261 }
7262 }
7263
7264 if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index,
7265 q->q_eq.eq_n) != 0)
7266 goto destroy_tx_slots;
7267
7268 if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n)
7269 != 0)
7270 goto destroy_cq;
7271
7272 if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0)
7273 goto destroy_sq;
7274
7275 return 0;
7276
7277 destroy_sq:
7278 mcx_destroy_sq(sc, tx);
7279 destroy_cq:
7280 mcx_destroy_cq(sc, &q->q_cq);
7281 destroy_tx_slots:
7282 mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
7283 tx->tx_slots = NULL;
7284
7285 i = (1 << MCX_LOG_RQ_SIZE);
7286 destroy_rx_slots:
7287 mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
7288 rx->rx_slots = NULL;
7289 return ENOMEM;
7290 }
7291
7292 static int
7293 mcx_rss_group_entry_count(struct mcx_softc *sc, int group)
7294 {
7295 int i;
7296 int count;
7297
7298 count = 0;
7299 for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7300 if (mcx_rss_config[i].flow_group == group)
7301 count++;
7302 }
7303
7304 return count;
7305 }
7306
7307 static int
7308 mcx_init(struct ifnet *ifp)
7309 {
7310 struct mcx_softc *sc = ifp->if_softc;
7311 struct mcx_rx *rx;
7312 struct mcx_tx *tx;
7313 int i, start, count, flow_group, flow_index;
7314 struct mcx_flow_match match_crit;
7315 struct mcx_rss_rule *rss;
7316 uint32_t dest;
7317 int rqns[MCX_MAX_QUEUES] = { 0 };
7318
7319 if (ISSET(ifp->if_flags, IFF_RUNNING))
7320 mcx_stop(ifp, 0);
7321
7322 if (mcx_create_tis(sc, &sc->sc_tis) != 0)
7323 goto down;
7324
7325 for (i = 0; i < sc->sc_nqueues; i++) {
7326 if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
7327 goto down;
7328 }
7329 }
7330
7331 /* RSS flow table and flow groups */
7332 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1,
7333 &sc->sc_rss_flow_table_id) != 0)
7334 goto down;
7335
7336 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
7337 sc->sc_rss_flow_table_id;
7338
7339 /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */
7340 memset(&match_crit, 0, sizeof(match_crit));
7341 match_crit.mc_ethertype = 0xffff;
7342 match_crit.mc_ip_proto = 0xff;
7343 match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG;
7344 start = 0;
7345 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4);
7346 if (count != 0) {
7347 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7348 MCX_FLOW_GROUP_RSS_L4, start, count,
7349 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7350 goto down;
7351 start += count;
7352 }
7353
7354 /* L3 RSS flow group (v4/v6, including fragments) */
7355 memset(&match_crit, 0, sizeof(match_crit));
7356 match_crit.mc_ethertype = 0xffff;
7357 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3);
7358 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7359 MCX_FLOW_GROUP_RSS_L3, start, count,
7360 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7361 goto down;
7362 start += count;
7363
7364 /* non-RSS flow group */
7365 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE);
7366 memset(&match_crit, 0, sizeof(match_crit));
7367 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7368 MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0)
7369 goto down;
7370
7371 /* Root flow table, matching packets based on mac address */
7372 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0,
7373 &sc->sc_mac_flow_table_id) != 0)
7374 goto down;
7375
7376 /* promisc flow group */
7377 start = 0;
7378 memset(&match_crit, 0, sizeof(match_crit));
7379 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7380 MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0)
7381 goto down;
7382 sc->sc_promisc_flow_enabled = 0;
7383 start++;
7384
7385 /* all multicast flow group */
7386 match_crit.mc_dest_mac[0] = 0x01;
7387 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7388 MCX_FLOW_GROUP_ALLMULTI, start, 1,
7389 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7390 goto down;
7391 sc->sc_allmulti_flow_enabled = 0;
7392 start++;
7393
7394 /* mac address matching flow group */
7395 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
7396 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7397 MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
7398 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7399 goto down;
7400
7401 /* flow table entries for unicast and broadcast */
7402 start = 0;
7403 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
7404 LLADDR(satosdl(ifp->if_dl->ifa_addr)), dest) != 0)
7405 goto down;
7406 start++;
7407
7408 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
7409 etherbroadcastaddr, dest) != 0)
7410 goto down;
7411 start++;
7412
7413 /* multicast entries go after that */
7414 sc->sc_mcast_flow_base = start;
7415
7416 /* re-add any existing multicast flows */
7417 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7418 if (sc->sc_mcast_flows[i][0] != 0) {
7419 mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC,
7420 sc->sc_mcast_flow_base + i,
7421 sc->sc_mcast_flows[i], dest);
7422 }
7423 }
7424
7425 if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0)
7426 goto down;
7427
7428 /*
7429 * the RQT can be any size as long as it's a power of two.
7430 * since we also restrict the number of queues to a power of two,
7431 * we can just put each rx queue in once.
7432 */
7433 for (i = 0; i < sc->sc_nqueues; i++)
7434 rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;
7435
7436 if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0)
7437 goto down;
7438
7439 start = 0;
7440 flow_index = 0;
7441 flow_group = -1;
7442 for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7443 rss = &mcx_rss_config[i];
7444 if (rss->flow_group != flow_group) {
7445 flow_group = rss->flow_group;
7446 flow_index = 0;
7447 }
7448
7449 if (rss->hash_sel == 0) {
7450 if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx,
7451 &sc->sc_tir[i]) != 0)
7452 goto down;
7453 } else {
7454 if (mcx_create_tir_indirect(sc, sc->sc_rqt,
7455 rss->hash_sel, &sc->sc_tir[i]) != 0)
7456 goto down;
7457 }
7458
7459 if (mcx_set_flow_table_entry_proto(sc, flow_group,
7460 flow_index, rss->ethertype, rss->ip_proto,
7461 MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0)
7462 goto down;
7463 flow_index++;
7464 }
7465
7466 for (i = 0; i < sc->sc_nqueues; i++) {
7467 struct mcx_queues *q = &sc->sc_queues[i];
7468 rx = &q->q_rx;
7469 tx = &q->q_tx;
7470
7471 /* start the queues */
7472 if (mcx_ready_sq(sc, tx) != 0)
7473 goto down;
7474
7475 if (mcx_ready_rq(sc, rx) != 0)
7476 goto down;
7477
7478 mcx_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
7479 rx->rx_prod = 0;
7480 mcx_rx_fill(sc, rx);
7481
7482 tx->tx_cons = 0;
7483 tx->tx_prod = 0;
7484 }
7485
7486 mcx_calibrate_first(sc);
7487
7488 SET(ifp->if_flags, IFF_RUNNING);
7489 CLR(ifp->if_flags, IFF_OACTIVE);
7490 if_schedule_deferred_start(ifp);
7491
7492 return 0;
7493 down:
7494 mcx_stop(ifp, 0);
7495 return EIO;
7496 }
7497
7498 static void
7499 mcx_stop(struct ifnet *ifp, int disable)
7500 {
7501 struct mcx_softc *sc = ifp->if_softc;
7502 struct mcx_rss_rule *rss;
7503 int group, i, flow_group, flow_index;
7504
7505 CLR(ifp->if_flags, IFF_RUNNING);
7506
7507 /*
7508 * delete flow table entries first, so no packets can arrive
7509 * after the barriers
7510 */
7511 if (sc->sc_promisc_flow_enabled)
7512 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
7513 if (sc->sc_allmulti_flow_enabled)
7514 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
7515 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
7516 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
7517 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7518 if (sc->sc_mcast_flows[i][0] != 0) {
7519 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
7520 sc->sc_mcast_flow_base + i);
7521 }
7522 }
7523
7524 flow_group = -1;
7525 flow_index = 0;
7526 for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7527 rss = &mcx_rss_config[i];
7528 if (rss->flow_group != flow_group) {
7529 flow_group = rss->flow_group;
7530 flow_index = 0;
7531 }
7532
7533 mcx_delete_flow_table_entry(sc, flow_group, flow_index);
7534
7535 mcx_destroy_tir(sc, sc->sc_tir[i]);
7536 sc->sc_tir[i] = 0;
7537
7538 flow_index++;
7539 }
7540
7541 for (i = 0; i < sc->sc_nqueues; i++) {
7542 callout_halt(&sc->sc_queues[i].q_rx.rx_refill, NULL);
7543 }
7544
7545 callout_halt(&sc->sc_calibrate, NULL);
7546
7547 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
7548 if (sc->sc_flow_group[group].g_id != -1)
7549 mcx_destroy_flow_group(sc, group);
7550 }
7551
7552 if (sc->sc_mac_flow_table_id != -1) {
7553 mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id);
7554 sc->sc_mac_flow_table_id = -1;
7555 }
7556 if (sc->sc_rss_flow_table_id != -1) {
7557 mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id);
7558 sc->sc_rss_flow_table_id = -1;
7559 }
7560 if (sc->sc_rqt != -1) {
7561 mcx_destroy_rqt(sc, sc->sc_rqt);
7562 sc->sc_rqt = -1;
7563 }
7564
7565 for (i = 0; i < sc->sc_nqueues; i++) {
7566 struct mcx_queues *q = &sc->sc_queues[i];
7567 struct mcx_rx *rx = &q->q_rx;
7568 struct mcx_tx *tx = &q->q_tx;
7569 struct mcx_cq *cq = &q->q_cq;
7570
7571 if (rx->rx_rqn != 0)
7572 mcx_destroy_rq(sc, rx);
7573
7574 if (tx->tx_sqn != 0)
7575 mcx_destroy_sq(sc, tx);
7576
7577 if (tx->tx_slots != NULL) {
7578 mcx_free_slots(sc, tx->tx_slots,
7579 (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE));
7580 tx->tx_slots = NULL;
7581 }
7582 if (rx->rx_slots != NULL) {
7583 mcx_free_slots(sc, rx->rx_slots,
7584 (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE));
7585 rx->rx_slots = NULL;
7586 }
7587
7588 if (cq->cq_n != 0)
7589 mcx_destroy_cq(sc, cq);
7590 }
7591 if (sc->sc_tis != 0) {
7592 mcx_destroy_tis(sc, sc->sc_tis);
7593 sc->sc_tis = 0;
7594 }
7595 }
7596
7597 static int
7598 mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data)
7599 {
7600 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
7601 struct ifreq *ifr = (struct ifreq *)data;
7602 struct ethercom *ec = &sc->sc_ec;
7603 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
7604 struct ether_multi *enm;
7605 struct ether_multistep step;
7606 int s, i, flags, error = 0;
7607 uint32_t dest;
7608
7609 s = splnet();
7610 switch (cmd) {
7611
7612 case SIOCADDMULTI:
7613 if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
7614 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
7615 if (error != 0) {
7616 splx(s);
7617 return (error);
7618 }
7619
7620 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
7621 sc->sc_rss_flow_table_id;
7622
7623 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7624 if (sc->sc_mcast_flows[i][0] == 0) {
7625 memcpy(sc->sc_mcast_flows[i], addrlo,
7626 ETHER_ADDR_LEN);
7627 if (ISSET(ifp->if_flags, IFF_RUNNING)) {
7628 mcx_set_flow_table_entry_mac(sc,
7629 MCX_FLOW_GROUP_MAC,
7630 sc->sc_mcast_flow_base + i,
7631 sc->sc_mcast_flows[i], dest);
7632 }
7633 break;
7634 }
7635 }
7636
7637 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
7638 if (i == MCX_NUM_MCAST_FLOWS) {
7639 SET(ifp->if_flags, IFF_ALLMULTI);
7640 sc->sc_extra_mcast++;
7641 error = ENETRESET;
7642 }
7643
7644 if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN)) {
7645 SET(ifp->if_flags, IFF_ALLMULTI);
7646 error = ENETRESET;
7647 }
7648 }
7649 }
7650 break;
7651
7652 case SIOCDELMULTI:
7653 if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
7654 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
7655 if (error != 0) {
7656 splx(s);
7657 return (error);
7658 }
7659
7660 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7661 if (memcmp(sc->sc_mcast_flows[i], addrlo,
7662 ETHER_ADDR_LEN) == 0) {
7663 if (ISSET(ifp->if_flags, IFF_RUNNING)) {
7664 mcx_delete_flow_table_entry(sc,
7665 MCX_FLOW_GROUP_MAC,
7666 sc->sc_mcast_flow_base + i);
7667 }
7668 sc->sc_mcast_flows[i][0] = 0;
7669 break;
7670 }
7671 }
7672
7673 if (i == MCX_NUM_MCAST_FLOWS)
7674 sc->sc_extra_mcast--;
7675
7676 if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
7677 sc->sc_extra_mcast == 0) {
7678 flags = 0;
7679 ETHER_LOCK(ec);
7680 ETHER_FIRST_MULTI(step, ec, enm);
7681 while (enm != NULL) {
7682 if (memcmp(enm->enm_addrlo,
7683 enm->enm_addrhi, ETHER_ADDR_LEN)) {
7684 SET(flags, IFF_ALLMULTI);
7685 break;
7686 }
7687 ETHER_NEXT_MULTI(step, enm);
7688 }
7689 ETHER_UNLOCK(ec);
7690 if (!ISSET(flags, IFF_ALLMULTI)) {
7691 CLR(ifp->if_flags, IFF_ALLMULTI);
7692 error = ENETRESET;
7693 }
7694 }
7695 }
7696 break;
7697
7698 default:
7699 error = ether_ioctl(ifp, cmd, data);
7700 }
7701
7702 if (error == ENETRESET) {
7703 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
7704 (IFF_UP | IFF_RUNNING))
7705 mcx_iff(sc);
7706 error = 0;
7707 }
7708 splx(s);
7709
7710 return (error);
7711 }
7712
7713 #if 0
7714 static int
7715 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
7716 {
7717 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
7718 struct mcx_reg_mcia mcia;
7719 struct mcx_reg_pmlp pmlp;
7720 int offset, error;
7721
7722 /* get module number */
7723 memset(&pmlp, 0, sizeof(pmlp));
7724 pmlp.rp_local_port = 1;
7725 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
7726 sizeof(pmlp));
7727 if (error != 0) {
7728 printf("%s: unable to get eeprom module number\n",
7729 DEVNAME(sc));
7730 return error;
7731 }
7732
7733 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
7734 memset(&mcia, 0, sizeof(mcia));
7735 mcia.rm_l = 0;
7736 mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) &
7737 MCX_PMLP_MODULE_NUM_MASK;
7738 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */
7739 mcia.rm_page_num = sff->sff_page;
7740 mcia.rm_dev_addr = htobe16(offset);
7741 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);
7742
7743 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
7744 &mcia, sizeof(mcia));
7745 if (error != 0) {
7746 printf("%s: unable to read eeprom at %x\n",
7747 DEVNAME(sc), offset);
7748 return error;
7749 }
7750
7751 memcpy(sff->sff_data + offset, mcia.rm_data,
7752 MCX_MCIA_EEPROM_BYTES);
7753 }
7754
7755 return 0;
7756 }
7757 #endif
7758
7759 static int
7760 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
7761 {
7762 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
7763 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
7764 case 0:
7765 break;
7766
7767 case EFBIG:
7768 if (m_defrag(m, M_DONTWAIT) != NULL &&
7769 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
7770 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
7771 break;
7772
7773 /* FALLTHROUGH */
7774 default:
7775 return (1);
7776 }
7777
7778 ms->ms_m = m;
7779 return (0);
7780 }
7781
7782 static void
7783 mcx_send_common_locked(struct ifnet *ifp, struct mcx_tx *tx, bool is_transmit)
7784 {
7785 struct mcx_softc *sc = ifp->if_softc;
7786 struct mcx_sq_entry *sq, *sqe;
7787 struct mcx_sq_entry_seg *sqs;
7788 struct mcx_slot *ms;
7789 bus_dmamap_t map;
7790 struct mbuf *m;
7791 u_int idx, free, used;
7792 uint64_t *bf;
7793 uint32_t csum;
7794 size_t bf_base;
7795 int i, seg, nseg;
7796
7797 KASSERT(mutex_owned(&tx->tx_lock));
7798
7799 if ((ifp->if_flags & IFF_RUNNING) == 0)
7800 return;
7801
7802 bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
7803
7804 idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE);
7805 free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod;
7806
7807 used = 0;
7808 bf = NULL;
7809
7810 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
7811 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
7812
7813 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem);
7814
7815 for (;;) {
7816 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
7817 SET(ifp->if_flags, IFF_OACTIVE);
7818 break;
7819 }
7820
7821 if (is_transmit) {
7822 m = pcq_get(tx->tx_pcq);
7823 } else {
7824 IFQ_DEQUEUE(&ifp->if_snd, m);
7825 }
7826 if (m == NULL) {
7827 break;
7828 }
7829
7830 sqe = sq + idx;
7831 ms = &tx->tx_slots[idx];
7832 memset(sqe, 0, sizeof(*sqe));
7833
7834 /* ctrl segment */
7835 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
7836 ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
7837 /* always generate a completion event */
7838 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);
7839
7840 /* eth segment */
7841 csum = 0;
7842 if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
7843 csum |= MCX_SQE_L3_CSUM;
7844 if (m->m_pkthdr.csum_flags &
7845 (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6))
7846 csum |= MCX_SQE_L4_CSUM;
7847 sqe->sqe_mss_csum = htobe32(csum);
7848 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
7849 if (vlan_has_tag(m)) {
7850 struct ether_vlan_header *evh;
7851 evh = (struct ether_vlan_header *)
7852 &sqe->sqe_inline_headers;
7853
7854 m_copydata(m, 0, ETHER_HDR_LEN, evh);
7855 evh->evl_proto = evh->evl_encap_proto;
7856 evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
7857 evh->evl_tag = htons(vlan_get_tag(m));
7858 m_adj(m, ETHER_HDR_LEN);
7859 } else {
7860 m_copydata(m, 0, MCX_SQ_INLINE_SIZE,
7861 sqe->sqe_inline_headers);
7862 m_adj(m, MCX_SQ_INLINE_SIZE);
7863 }
7864
7865 if (mcx_load_mbuf(sc, ms, m) != 0) {
7866 m_freem(m);
7867 if_statinc(ifp, if_oerrors);
7868 continue;
7869 }
7870 bf = (uint64_t *)sqe;
7871
7872 if (ifp->if_bpf != NULL)
7873 bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers,
7874 MCX_SQ_INLINE_SIZE, m, BPF_D_OUT);
7875
7876 map = ms->ms_map;
7877 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
7878 BUS_DMASYNC_PREWRITE);
7879
7880 sqe->sqe_ds_sq_num =
7881 htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) |
7882 (map->dm_nsegs + 3));
7883
7884 /* data segment - first wqe has one segment */
7885 sqs = sqe->sqe_segs;
7886 seg = 0;
7887 nseg = 1;
7888 for (i = 0; i < map->dm_nsegs; i++) {
7889 if (seg == nseg) {
7890 /* next slot */
7891 idx++;
7892 if (idx == (1 << MCX_LOG_SQ_SIZE))
7893 idx = 0;
7894 tx->tx_prod++;
7895 used++;
7896
7897 sqs = (struct mcx_sq_entry_seg *)(sq + idx);
7898 seg = 0;
7899 nseg = MCX_SQ_SEGS_PER_SLOT;
7900 }
7901 sqs[seg].sqs_byte_count =
7902 htobe32(map->dm_segs[i].ds_len);
7903 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
7904 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
7905 seg++;
7906 }
7907
7908 idx++;
7909 if (idx == (1 << MCX_LOG_SQ_SIZE))
7910 idx = 0;
7911 tx->tx_prod++;
7912 used++;
7913 }
7914
7915 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
7916 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
7917
7918 if (used) {
7919 bus_size_t blueflame;
7920
7921 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7922 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
7923 be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell),
7924 tx->tx_prod & MCX_WQ_DOORBELL_MASK);
7925 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7926 tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
7927
7928 /*
7929 * write the first 64 bits of the last sqe we produced
7930 * to the blue flame buffer
7931 */
7932
7933 blueflame = bf_base + tx->tx_bf_offset;
7934 bus_space_write_8(sc->sc_memt, sc->sc_memh,
7935 blueflame, *bf);
7936 mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE);
7937
7938 /* next write goes to the other buffer */
7939 tx->tx_bf_offset ^= sc->sc_bf_size;
7940 }
7941 }
7942
7943 static void
7944 mcx_start(struct ifnet *ifp)
7945 {
7946 struct mcx_softc *sc = ifp->if_softc;
7947 /* mcx_start() always uses TX ring[0] */
7948 struct mcx_tx *tx = &sc->sc_queues[0].q_tx;
7949
7950 mutex_enter(&tx->tx_lock);
7951 if (!ISSET(ifp->if_flags, IFF_OACTIVE)) {
7952 mcx_send_common_locked(ifp, tx, false);
7953 }
7954 mutex_exit(&tx->tx_lock);
7955 }
7956
7957 static int
7958 mcx_transmit(struct ifnet *ifp, struct mbuf *m)
7959 {
7960 struct mcx_softc *sc = ifp->if_softc;
7961 struct mcx_tx *tx;
7962
7963 tx = &sc->sc_queues[cpu_index(curcpu()) % sc->sc_nqueues].q_tx;
7964 if (__predict_false(!pcq_put(tx->tx_pcq, m))) {
7965 m_freem(m);
7966 return ENOBUFS;
7967 }
7968
7969 if (mutex_tryenter(&tx->tx_lock)) {
7970 mcx_send_common_locked(ifp, tx, true);
7971 mutex_exit(&tx->tx_lock);
7972 } else {
7973 softint_schedule(tx->tx_softint);
7974 }
7975
7976 return 0;
7977 }
7978
7979 static void
7980 mcx_deferred_transmit(void *arg)
7981 {
7982 struct mcx_tx *tx = arg;
7983 struct mcx_softc *sc = tx->tx_softc;
7984 struct ifnet *ifp = &sc->sc_ec.ec_if;
7985
7986 mutex_enter(&tx->tx_lock);
7987 if (pcq_peek(tx->tx_pcq) != NULL) {
7988 mcx_send_common_locked(ifp, tx, true);
7989 }
7990 mutex_exit(&tx->tx_lock);
7991 }
7992
7993 static void
7994 mcx_watchdog(struct ifnet *ifp)
7995 {
7996 }
7997
7998 static void
7999 mcx_media_add_types(struct mcx_softc *sc)
8000 {
8001 struct mcx_reg_ptys ptys;
8002 int i;
8003 uint32_t proto_cap;
8004
8005 memset(&ptys, 0, sizeof(ptys));
8006 ptys.rp_local_port = 1;
8007 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8008 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8009 sizeof(ptys)) != 0) {
8010 printf("%s: unable to read port type/speed\n", DEVNAME(sc));
8011 return;
8012 }
8013
8014 proto_cap = be32toh(ptys.rp_eth_proto_cap);
8015 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8016 const struct mcx_eth_proto_capability *cap;
8017 if (!ISSET(proto_cap, 1U << i))
8018 continue;
8019
8020 cap = &mcx_eth_cap_map[i];
8021 if (cap->cap_media == 0)
8022 continue;
8023
8024 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
8025 }
8026 }
8027
8028 static void
8029 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
8030 {
8031 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
8032 struct mcx_reg_ptys ptys;
8033 int i;
8034 uint32_t proto_oper;
8035 uint64_t media_oper;
8036
8037 memset(&ptys, 0, sizeof(ptys));
8038 ptys.rp_local_port = 1;
8039 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8040
8041 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8042 sizeof(ptys)) != 0) {
8043 printf("%s: unable to read port type/speed\n", DEVNAME(sc));
8044 return;
8045 }
8046
8047 proto_oper = be32toh(ptys.rp_eth_proto_oper);
8048
8049 media_oper = 0;
8050
8051 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8052 const struct mcx_eth_proto_capability *cap;
8053 if (!ISSET(proto_oper, 1U << i))
8054 continue;
8055
8056 cap = &mcx_eth_cap_map[i];
8057
8058 if (cap->cap_media != 0)
8059 media_oper = cap->cap_media;
8060 }
8061
8062 ifmr->ifm_status = IFM_AVALID;
8063 if (proto_oper != 0) {
8064 ifmr->ifm_status |= IFM_ACTIVE;
8065 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper;
8066 /* txpause, rxpause, duplex? */
8067 }
8068 }
8069
8070 static int
8071 mcx_media_change(struct ifnet *ifp)
8072 {
8073 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
8074 struct mcx_reg_ptys ptys;
8075 struct mcx_reg_paos paos;
8076 uint32_t media;
8077 int i, error;
8078
8079 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
8080 return EINVAL;
8081
8082 error = 0;
8083
8084 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
8085 /* read ptys to get supported media */
8086 memset(&ptys, 0, sizeof(ptys));
8087 ptys.rp_local_port = 1;
8088 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8089 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
8090 &ptys, sizeof(ptys)) != 0) {
8091 printf("%s: unable to read port type/speed\n",
8092 DEVNAME(sc));
8093 return EIO;
8094 }
8095
8096 media = be32toh(ptys.rp_eth_proto_cap);
8097 } else {
8098 /* map media type */
8099 media = 0;
8100 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8101 const struct mcx_eth_proto_capability *cap;
8102
8103 cap = &mcx_eth_cap_map[i];
8104 if (cap->cap_media ==
8105 IFM_SUBTYPE(sc->sc_media.ifm_media)) {
8106 media = (1 << i);
8107 break;
8108 }
8109 }
8110 }
8111
8112 /* disable the port */
8113 memset(&paos, 0, sizeof(paos));
8114 paos.rp_local_port = 1;
8115 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
8116 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
8117 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
8118 sizeof(paos)) != 0) {
8119 printf("%s: unable to set port state to down\n", DEVNAME(sc));
8120 return EIO;
8121 }
8122
8123 memset(&ptys, 0, sizeof(ptys));
8124 ptys.rp_local_port = 1;
8125 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8126 ptys.rp_eth_proto_admin = htobe32(media);
8127 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
8128 sizeof(ptys)) != 0) {
8129 printf("%s: unable to set port media type/speed\n",
8130 DEVNAME(sc));
8131 error = EIO;
8132 }
8133
8134 /* re-enable the port to start negotiation */
8135 memset(&paos, 0, sizeof(paos));
8136 paos.rp_local_port = 1;
8137 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
8138 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
8139 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
8140 sizeof(paos)) != 0) {
8141 printf("%s: unable to set port state to up\n", DEVNAME(sc));
8142 error = EIO;
8143 }
8144
8145 return error;
8146 }
8147
8148 static void
8149 mcx_port_change(struct work *wk, void *xsc)
8150 {
8151 struct mcx_softc *sc = xsc;
8152 struct ifnet *ifp = &sc->sc_ec.ec_if;
8153 struct mcx_reg_ptys ptys = {
8154 .rp_local_port = 1,
8155 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH,
8156 };
8157 int link_state = LINK_STATE_DOWN;
8158
8159 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8160 sizeof(ptys)) == 0) {
8161 uint32_t proto_oper = be32toh(ptys.rp_eth_proto_oper);
8162 uint64_t baudrate = 0;
8163 unsigned int i;
8164
8165 if (proto_oper != 0)
8166 link_state = LINK_STATE_UP;
8167
8168 for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8169 const struct mcx_eth_proto_capability *cap;
8170 if (!ISSET(proto_oper, 1U << i))
8171 continue;
8172
8173 cap = &mcx_eth_cap_map[i];
8174 if (cap->cap_baudrate == 0)
8175 continue;
8176
8177 baudrate = cap->cap_baudrate;
8178 break;
8179 }
8180
8181 ifp->if_baudrate = baudrate;
8182 }
8183
8184 if (link_state != ifp->if_link_state) {
8185 if_link_state_change(ifp, link_state);
8186 }
8187 }
8188
8189
8190 static inline uint32_t
8191 mcx_rd(struct mcx_softc *sc, bus_size_t r)
8192 {
8193 uint32_t word;
8194
8195 word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r);
8196
8197 return (be32toh(word));
8198 }
8199
8200 static inline void
8201 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
8202 {
8203 bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
8204 }
8205
8206 static inline void
8207 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
8208 {
8209 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
8210 }
8211
8212 static uint64_t
8213 mcx_timer(struct mcx_softc *sc)
8214 {
8215 uint32_t hi, lo, ni;
8216
8217 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
8218 for (;;) {
8219 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
8220 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
8221 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
8222
8223 if (ni == hi)
8224 break;
8225
8226 hi = ni;
8227 }
8228
8229 return (((uint64_t)hi << 32) | (uint64_t)lo);
8230 }
8231
8232 static int
8233 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
8234 bus_size_t size, u_int align)
8235 {
8236 mxm->mxm_size = size;
8237
8238 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
8239 mxm->mxm_size, 0,
8240 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
8241 &mxm->mxm_map) != 0)
8242 return (1);
8243 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
8244 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
8245 BUS_DMA_WAITOK) != 0)
8246 goto destroy;
8247 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
8248 mxm->mxm_size, &mxm->mxm_kva,
8249 BUS_DMA_WAITOK | BUS_DMA_COHERENT) != 0)
8250 goto free;
8251 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
8252 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
8253 goto unmap;
8254
8255 mcx_dmamem_zero(mxm);
8256
8257 return (0);
8258 unmap:
8259 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
8260 free:
8261 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
8262 destroy:
8263 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
8264 return (1);
8265 }
8266
8267 static void
8268 mcx_dmamem_zero(struct mcx_dmamem *mxm)
8269 {
8270 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
8271 }
8272
8273 static void
8274 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
8275 {
8276 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
8277 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
8278 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
8279 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
8280 }
8281
8282 static int
8283 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
8284 {
8285 bus_dma_segment_t *segs;
8286 bus_size_t len = pages * MCX_PAGE_SIZE;
8287 size_t seglen;
8288
8289 segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP);
8290 seglen = sizeof(*segs) * pages;
8291
8292 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
8293 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
8294 goto free_segs;
8295
8296 if (mhm->mhm_seg_count < pages) {
8297 size_t nseglen;
8298
8299 mhm->mhm_segs = kmem_alloc(
8300 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP);
8301
8302 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
8303
8304 memcpy(mhm->mhm_segs, segs, nseglen);
8305
8306 kmem_free(segs, seglen);
8307
8308 segs = mhm->mhm_segs;
8309 seglen = nseglen;
8310 } else
8311 mhm->mhm_segs = segs;
8312
8313 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
8314 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
8315 &mhm->mhm_map) != 0)
8316 goto free_dmamem;
8317
8318 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
8319 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
8320 goto destroy;
8321
8322 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
8323 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
8324
8325 mhm->mhm_npages = pages;
8326
8327 return (0);
8328
8329 destroy:
8330 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
8331 free_dmamem:
8332 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
8333 free_segs:
8334 kmem_free(segs, seglen);
8335 mhm->mhm_segs = NULL;
8336
8337 return (-1);
8338 }
8339
8340 static void
8341 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
8342 {
8343 if (mhm->mhm_npages == 0)
8344 return;
8345
8346 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
8347 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
8348
8349 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
8350 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
8351 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
8352 kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
8353
8354 mhm->mhm_npages = 0;
8355 }
8356
8357 #if NKSTAT > 0
8358 struct mcx_ppcnt {
8359 char name[KSTAT_KV_NAMELEN];
8360 enum kstat_kv_unit unit;
8361 };
8362
8363 static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = {
8364 { "Good Tx", KSTAT_KV_U_PACKETS, },
8365 { "Good Rx", KSTAT_KV_U_PACKETS, },
8366 { "FCS errs", KSTAT_KV_U_PACKETS, },
8367 { "Alignment Errs", KSTAT_KV_U_PACKETS, },
8368 { "Good Tx", KSTAT_KV_U_BYTES, },
8369 { "Good Rx", KSTAT_KV_U_BYTES, },
8370 { "Multicast Tx", KSTAT_KV_U_PACKETS, },
8371 { "Broadcast Tx", KSTAT_KV_U_PACKETS, },
8372 { "Multicast Rx", KSTAT_KV_U_PACKETS, },
8373 { "Broadcast Rx", KSTAT_KV_U_PACKETS, },
8374 { "In Range Len", KSTAT_KV_U_PACKETS, },
8375 { "Out Of Range Len", KSTAT_KV_U_PACKETS, },
8376 { "Frame Too Long", KSTAT_KV_U_PACKETS, },
8377 { "Symbol Errs", KSTAT_KV_U_PACKETS, },
8378 { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, },
8379 { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, },
8380 { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, },
8381 { "Pause Rx", KSTAT_KV_U_PACKETS, },
8382 { "Pause Tx", KSTAT_KV_U_PACKETS, },
8383 };
8384 CTASSERT(__arraycount(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count);
8385
8386 static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = {
8387 { "Rx Bytes", KSTAT_KV_U_BYTES, },
8388 { "Rx Unicast", KSTAT_KV_U_PACKETS, },
8389 { "Rx Discards", KSTAT_KV_U_PACKETS, },
8390 { "Rx Errors", KSTAT_KV_U_PACKETS, },
8391 { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, },
8392 { "Tx Bytes", KSTAT_KV_U_BYTES, },
8393 { "Tx Unicast", KSTAT_KV_U_PACKETS, },
8394 { "Tx Discards", KSTAT_KV_U_PACKETS, },
8395 { "Tx Errors", KSTAT_KV_U_PACKETS, },
8396 { "Rx Multicast", KSTAT_KV_U_PACKETS, },
8397 { "Rx Broadcast", KSTAT_KV_U_PACKETS, },
8398 { "Tx Multicast", KSTAT_KV_U_PACKETS, },
8399 { "Tx Broadcast", KSTAT_KV_U_PACKETS, },
8400 };
8401 CTASSERT(__arraycount(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count);
8402
8403 static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = {
8404 { "Drop Events", KSTAT_KV_U_PACKETS, },
8405 { "Octets", KSTAT_KV_U_BYTES, },
8406 { "Packets", KSTAT_KV_U_PACKETS, },
8407 { "Broadcasts", KSTAT_KV_U_PACKETS, },
8408 { "Multicasts", KSTAT_KV_U_PACKETS, },
8409 { "CRC Align Errs", KSTAT_KV_U_PACKETS, },
8410 { "Undersize", KSTAT_KV_U_PACKETS, },
8411 { "Oversize", KSTAT_KV_U_PACKETS, },
8412 { "Fragments", KSTAT_KV_U_PACKETS, },
8413 { "Jabbers", KSTAT_KV_U_PACKETS, },
8414 { "Collisions", KSTAT_KV_U_NONE, },
8415 { "64B", KSTAT_KV_U_PACKETS, },
8416 { "65-127B", KSTAT_KV_U_PACKETS, },
8417 { "128-255B", KSTAT_KV_U_PACKETS, },
8418 { "256-511B", KSTAT_KV_U_PACKETS, },
8419 { "512-1023B", KSTAT_KV_U_PACKETS, },
8420 { "1024-1518B", KSTAT_KV_U_PACKETS, },
8421 { "1519-2047B", KSTAT_KV_U_PACKETS, },
8422 { "2048-4095B", KSTAT_KV_U_PACKETS, },
8423 { "4096-8191B", KSTAT_KV_U_PACKETS, },
8424 { "8192-10239B", KSTAT_KV_U_PACKETS, },
8425 };
8426 CTASSERT(__arraycount(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count);
8427
8428 static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = {
8429 { "Alignment Errs", KSTAT_KV_U_PACKETS, },
8430 { "FCS Errs", KSTAT_KV_U_PACKETS, },
8431 { "Single Colls", KSTAT_KV_U_PACKETS, },
8432 { "Multiple Colls", KSTAT_KV_U_PACKETS, },
8433 { "SQE Test Errs", KSTAT_KV_U_NONE, },
8434 { "Deferred Tx", KSTAT_KV_U_PACKETS, },
8435 { "Late Colls", KSTAT_KV_U_NONE, },
8436 { "Exess Colls", KSTAT_KV_U_NONE, },
8437 { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, },
8438 { "CSM Sense Errs", KSTAT_KV_U_NONE, },
8439 { "Too Long", KSTAT_KV_U_PACKETS, },
8440 { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, },
8441 { "Symbol Errs", KSTAT_KV_U_NONE, },
8442 { "Unknown Control", KSTAT_KV_U_PACKETS, },
8443 { "Pause Rx", KSTAT_KV_U_PACKETS, },
8444 { "Pause Tx", KSTAT_KV_U_PACKETS, },
8445 };
8446 CTASSERT(__arraycount(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count);
8447
8448 struct mcx_kstat_ppcnt {
8449 const char *ksp_name;
8450 const struct mcx_ppcnt *ksp_tpl;
8451 unsigned int ksp_n;
8452 uint8_t ksp_grp;
8453 };
8454
8455 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = {
8456 .ksp_name = "ieee802.3",
8457 .ksp_tpl = mcx_ppcnt_ieee8023_tpl,
8458 .ksp_n = __arraycount(mcx_ppcnt_ieee8023_tpl),
8459 .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023,
8460 };
8461
8462 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = {
8463 .ksp_name = "rfc2863",
8464 .ksp_tpl = mcx_ppcnt_rfc2863_tpl,
8465 .ksp_n = __arraycount(mcx_ppcnt_rfc2863_tpl),
8466 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863,
8467 };
8468
8469 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = {
8470 .ksp_name = "rfc2819",
8471 .ksp_tpl = mcx_ppcnt_rfc2819_tpl,
8472 .ksp_n = __arraycount(mcx_ppcnt_rfc2819_tpl),
8473 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819,
8474 };
8475
8476 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = {
8477 .ksp_name = "rfc3635",
8478 .ksp_tpl = mcx_ppcnt_rfc3635_tpl,
8479 .ksp_n = __arraycount(mcx_ppcnt_rfc3635_tpl),
8480 .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635,
8481 };
8482
8483 static int mcx_kstat_ppcnt_read(struct kstat *);
8484
8485 static void mcx_kstat_attach_tmps(struct mcx_softc *sc);
8486 static void mcx_kstat_attach_queues(struct mcx_softc *sc);
8487
8488 static struct kstat *
8489 mcx_kstat_attach_ppcnt(struct mcx_softc *sc,
8490 const struct mcx_kstat_ppcnt *ksp)
8491 {
8492 struct kstat *ks;
8493 struct kstat_kv *kvs;
8494 unsigned int i;
8495
8496 ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0);
8497 if (ks == NULL)
8498 return (NULL);
8499
8500 kvs = mallocarray(ksp->ksp_n, sizeof(*kvs),
8501 M_DEVBUF, M_WAITOK);
8502
8503 for (i = 0; i < ksp->ksp_n; i++) {
8504 const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i];
8505
8506 kstat_kv_unit_init(&kvs[i], tpl->name,
8507 KSTAT_KV_T_COUNTER64, tpl->unit);
8508 }
8509
8510 ks->ks_softc = sc;
8511 ks->ks_ptr = (void *)ksp;
8512 ks->ks_data = kvs;
8513 ks->ks_datalen = ksp->ksp_n * sizeof(*kvs);
8514 ks->ks_read = mcx_kstat_ppcnt_read;
8515
8516 kstat_install(ks);
8517
8518 return (ks);
8519 }
8520
8521 static void
8522 mcx_kstat_attach(struct mcx_softc *sc)
8523 {
8524 sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc,
8525 &mcx_kstat_ppcnt_ieee8023);
8526 sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc,
8527 &mcx_kstat_ppcnt_rfc2863);
8528 sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc,
8529 &mcx_kstat_ppcnt_rfc2819);
8530 sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc,
8531 &mcx_kstat_ppcnt_rfc3635);
8532
8533 mcx_kstat_attach_tmps(sc);
8534 mcx_kstat_attach_queues(sc);
8535 }
8536
8537 static int
8538 mcx_kstat_ppcnt_read(struct kstat *ks)
8539 {
8540 struct mcx_softc *sc = ks->ks_softc;
8541 struct mcx_kstat_ppcnt *ksp = ks->ks_ptr;
8542 struct mcx_reg_ppcnt ppcnt = {
8543 .ppcnt_grp = ksp->ksp_grp,
8544 .ppcnt_local_port = 1,
8545 };
8546 struct kstat_kv *kvs = ks->ks_data;
8547 uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set;
8548 unsigned int i;
8549 int rv;
8550
8551 KERNEL_LOCK(); /* XXX */
8552 rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ,
8553 &ppcnt, sizeof(ppcnt));
8554 KERNEL_UNLOCK();
8555 if (rv != 0)
8556 return (EIO);
8557
8558 nanouptime(&ks->ks_updated);
8559
8560 for (i = 0; i < ksp->ksp_n; i++)
8561 kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]);
8562
8563 return (0);
8564 }
8565
8566 struct mcx_kstat_mtmp {
8567 struct kstat_kv ktmp_name;
8568 struct kstat_kv ktmp_temperature;
8569 struct kstat_kv ktmp_threshold_lo;
8570 struct kstat_kv ktmp_threshold_hi;
8571 };
8572
8573 static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = {
8574 KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR),
8575 KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP),
8576 KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP),
8577 KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP),
8578 };
8579
8580 static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 };
8581
8582 static int mcx_kstat_mtmp_read(struct kstat *);
8583
8584 static void
8585 mcx_kstat_attach_tmps(struct mcx_softc *sc)
8586 {
8587 struct kstat *ks;
8588 struct mcx_reg_mcam mcam;
8589 struct mcx_reg_mtcap mtcap;
8590 struct mcx_kstat_mtmp *ktmp;
8591 uint64_t map;
8592 unsigned int i, n;
8593
8594 memset(&mtcap, 0, sizeof(mtcap));
8595 memset(&mcam, 0, sizeof(mcam));
8596
8597 if (sc->sc_mcam_reg == 0) {
8598 /* no management capabilities */
8599 return;
8600 }
8601
8602 if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ,
8603 &mcam, sizeof(mcam)) != 0) {
8604 /* unable to check management capabilities? */
8605 return;
8606 }
8607
8608 if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask,
8609 MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) {
8610 /* no sensor map */
8611 return;
8612 }
8613
8614 if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ,
8615 &mtcap, sizeof(mtcap)) != 0) {
8616 /* unable to find temperature sensors */
8617 return;
8618 }
8619
8620 sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count;
8621 sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count,
8622 sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK);
8623
8624 n = 0;
8625 map = bemtoh64(&mtcap.mtcap_sensor_map);
8626 for (i = 0; i < sizeof(map) * NBBY; i++) {
8627 if (!ISSET(map, (1ULL << i)))
8628 continue;
8629
8630 ks = kstat_create(DEVNAME(sc), 0, "temperature", i,
8631 KSTAT_T_KV, 0);
8632 if (ks == NULL) {
8633 /* unable to attach temperature sensor %u, i */
8634 continue;
8635 }
8636
8637 ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO);
8638 *ktmp = mcx_kstat_mtmp_tpl;
8639
8640 ks->ks_data = ktmp;
8641 ks->ks_datalen = sizeof(*ktmp);
8642 TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval);
8643 ks->ks_read = mcx_kstat_mtmp_read;
8644
8645 ks->ks_softc = sc;
8646 kstat_install(ks);
8647
8648 sc->sc_kstat_mtmp[n++] = ks;
8649 if (n >= sc->sc_kstat_mtmp_count)
8650 break;
8651 }
8652 }
8653
8654 static uint64_t
8655 mcx_tmp_to_uK(uint16_t *t)
8656 {
8657 int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */
8658 mt *= 1000000 / 8; /* convert to uC */
8659 mt += 273150000; /* convert to uK */
8660
8661 return (mt);
8662 }
8663
8664 static int
8665 mcx_kstat_mtmp_read(struct kstat *ks)
8666 {
8667 struct mcx_softc *sc = ks->ks_softc;
8668 struct mcx_kstat_mtmp *ktmp = ks->ks_data;
8669 struct mcx_reg_mtmp mtmp;
8670 int rv;
8671 struct timeval updated;
8672
8673 TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated);
8674
8675 if (!ratecheck(&updated, &mcx_kstat_mtmp_rate))
8676 return (0);
8677
8678 memset(&mtmp, 0, sizeof(mtmp));
8679 htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit);
8680
8681 KERNEL_LOCK(); /* XXX */
8682 rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ,
8683 &mtmp, sizeof(mtmp));
8684 KERNEL_UNLOCK();
8685 if (rv != 0)
8686 return (EIO);
8687
8688 memset(kstat_kv_istr(&ktmp->ktmp_name), 0,
8689 sizeof(kstat_kv_istr(&ktmp->ktmp_name)));
8690 memcpy(kstat_kv_istr(&ktmp->ktmp_name),
8691 mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name));
8692 kstat_kv_temp(&ktmp->ktmp_temperature) =
8693 mcx_tmp_to_uK(&mtmp.mtmp_temperature);
8694 kstat_kv_temp(&ktmp->ktmp_threshold_lo) =
8695 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo);
8696 kstat_kv_temp(&ktmp->ktmp_threshold_hi) =
8697 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi);
8698
8699 TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated);
8700
8701 return (0);
8702 }
8703
8704 struct mcx_queuestat {
8705 char name[KSTAT_KV_NAMELEN];
8706 enum kstat_kv_type type;
8707 };
8708
8709 static const struct mcx_queuestat mcx_queue_kstat_tpl[] = {
8710 { "RQ SW prod", KSTAT_KV_T_COUNTER64 },
8711 { "RQ HW prod", KSTAT_KV_T_COUNTER64 },
8712 { "RQ HW cons", KSTAT_KV_T_COUNTER64 },
8713 { "RQ HW state", KSTAT_KV_T_ISTR },
8714
8715 { "SQ SW prod", KSTAT_KV_T_COUNTER64 },
8716 { "SQ SW cons", KSTAT_KV_T_COUNTER64 },
8717 { "SQ HW prod", KSTAT_KV_T_COUNTER64 },
8718 { "SQ HW cons", KSTAT_KV_T_COUNTER64 },
8719 { "SQ HW state", KSTAT_KV_T_ISTR },
8720
8721 { "CQ SW cons", KSTAT_KV_T_COUNTER64 },
8722 { "CQ HW prod", KSTAT_KV_T_COUNTER64 },
8723 { "CQ HW cons", KSTAT_KV_T_COUNTER64 },
8724 { "CQ HW notify", KSTAT_KV_T_COUNTER64 },
8725 { "CQ HW solicit", KSTAT_KV_T_COUNTER64 },
8726 { "CQ HW status", KSTAT_KV_T_ISTR },
8727 { "CQ HW state", KSTAT_KV_T_ISTR },
8728
8729 { "EQ SW cons", KSTAT_KV_T_COUNTER64 },
8730 { "EQ HW prod", KSTAT_KV_T_COUNTER64 },
8731 { "EQ HW cons", KSTAT_KV_T_COUNTER64 },
8732 { "EQ HW status", KSTAT_KV_T_ISTR },
8733 { "EQ HW state", KSTAT_KV_T_ISTR },
8734 };
8735
8736 static int mcx_kstat_queue_read(struct kstat *);
8737
8738 static void
8739 mcx_kstat_attach_queues(struct mcx_softc *sc)
8740 {
8741 struct kstat *ks;
8742 struct kstat_kv *kvs;
8743 int q, i;
8744
8745 for (q = 0; q < sc->sc_nqueues; q++) {
8746 ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
8747 KSTAT_T_KV, 0);
8748 if (ks == NULL) {
8749 /* unable to attach queue stats %u, q */
8750 continue;
8751 }
8752
8753 kvs = mallocarray(nitems(mcx_queue_kstat_tpl),
8754 sizeof(*kvs), M_DEVBUF, M_WAITOK);
8755
8756 for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) {
8757 const struct mcx_queuestat *tpl =
8758 &mcx_queue_kstat_tpl[i];
8759
8760 kstat_kv_init(&kvs[i], tpl->name, tpl->type);
8761 }
8762
8763 ks->ks_softc = &sc->sc_queues[q];
8764 ks->ks_data = kvs;
8765 ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs);
8766 ks->ks_read = mcx_kstat_queue_read;
8767
8768 sc->sc_queues[q].q_kstat = ks;
8769 kstat_install(ks);
8770 }
8771 }
8772
8773 static int
8774 mcx_kstat_queue_read(struct kstat *ks)
8775 {
8776 struct mcx_queues *q = ks->ks_softc;
8777 struct mcx_softc *sc = q->q_sc;
8778 struct kstat_kv *kvs = ks->ks_data;
8779 union {
8780 struct mcx_rq_ctx rq;
8781 struct mcx_sq_ctx sq;
8782 struct mcx_cq_ctx cq;
8783 struct mcx_eq_ctx eq;
8784 } u;
8785 const char *text;
8786 int error = 0;
8787
8788 KERNEL_LOCK();
8789
8790 if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) {
8791 error = EIO;
8792 goto out;
8793 }
8794
8795 kstat_kv_u64(kvs++) = q->q_rx.rx_prod;
8796 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter);
8797 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter);
8798 switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >>
8799 MCX_RQ_CTX_STATE_SHIFT) {
8800 case MCX_RQ_CTX_STATE_RST:
8801 text = "RST";
8802 break;
8803 case MCX_RQ_CTX_STATE_RDY:
8804 text = "RDY";
8805 break;
8806 case MCX_RQ_CTX_STATE_ERR:
8807 text = "ERR";
8808 break;
8809 default:
8810 text = "unknown";
8811 break;
8812 }
8813 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8814 kvs++;
8815
8816 if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) {
8817 error = EIO;
8818 goto out;
8819 }
8820
8821 kstat_kv_u64(kvs++) = q->q_tx.tx_prod;
8822 kstat_kv_u64(kvs++) = q->q_tx.tx_cons;
8823 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter);
8824 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter);
8825 switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >>
8826 MCX_SQ_CTX_STATE_SHIFT) {
8827 case MCX_SQ_CTX_STATE_RST:
8828 text = "RST";
8829 break;
8830 case MCX_SQ_CTX_STATE_RDY:
8831 text = "RDY";
8832 break;
8833 case MCX_SQ_CTX_STATE_ERR:
8834 text = "ERR";
8835 break;
8836 default:
8837 text = "unknown";
8838 break;
8839 }
8840 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8841 kvs++;
8842
8843 if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) {
8844 error = EIO;
8845 goto out;
8846 }
8847
8848 kstat_kv_u64(kvs++) = q->q_cq.cq_cons;
8849 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter);
8850 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter);
8851 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified);
8852 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit);
8853
8854 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >>
8855 MCX_CQ_CTX_STATUS_SHIFT) {
8856 case MCX_CQ_CTX_STATUS_OK:
8857 text = "OK";
8858 break;
8859 case MCX_CQ_CTX_STATUS_OVERFLOW:
8860 text = "overflow";
8861 break;
8862 case MCX_CQ_CTX_STATUS_WRITE_FAIL:
8863 text = "write fail";
8864 break;
8865 default:
8866 text = "unknown";
8867 break;
8868 }
8869 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8870 kvs++;
8871
8872 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >>
8873 MCX_CQ_CTX_STATE_SHIFT) {
8874 case MCX_CQ_CTX_STATE_SOLICITED:
8875 text = "solicited";
8876 break;
8877 case MCX_CQ_CTX_STATE_ARMED:
8878 text = "armed";
8879 break;
8880 case MCX_CQ_CTX_STATE_FIRED:
8881 text = "fired";
8882 break;
8883 default:
8884 text = "unknown";
8885 break;
8886 }
8887 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8888 kvs++;
8889
8890 if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) {
8891 error = EIO;
8892 goto out;
8893 }
8894
8895 kstat_kv_u64(kvs++) = q->q_eq.eq_cons;
8896 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter);
8897 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter);
8898
8899 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >>
8900 MCX_EQ_CTX_STATUS_SHIFT) {
8901 case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE:
8902 text = "write fail";
8903 break;
8904 case MCX_EQ_CTX_STATUS_OK:
8905 text = "OK";
8906 break;
8907 default:
8908 text = "unknown";
8909 break;
8910 }
8911 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8912 kvs++;
8913
8914 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >>
8915 MCX_EQ_CTX_STATE_SHIFT) {
8916 case MCX_EQ_CTX_STATE_ARMED:
8917 text = "armed";
8918 break;
8919 case MCX_EQ_CTX_STATE_FIRED:
8920 text = "fired";
8921 break;
8922 default:
8923 text = "unknown";
8924 break;
8925 }
8926 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8927 kvs++;
8928
8929 nanouptime(&ks->ks_updated);
8930 out:
8931 KERNEL_UNLOCK();
8932 return (error);
8933 }
8934
8935 #endif /* NKSTAT > 0 */
8936
8937 static unsigned int
8938 mcx_timecounter_read(struct timecounter *tc)
8939 {
8940 struct mcx_softc *sc = tc->tc_priv;
8941
8942 return (mcx_rd(sc, MCX_INTERNAL_TIMER_L));
8943 }
8944
8945 static void
8946 mcx_timecounter_attach(struct mcx_softc *sc)
8947 {
8948 struct timecounter *tc = &sc->sc_timecounter;
8949
8950 tc->tc_get_timecount = mcx_timecounter_read;
8951 tc->tc_counter_mask = ~0U;
8952 tc->tc_frequency = sc->sc_khz * 1000;
8953 tc->tc_name = device_xname(sc->sc_dev);
8954 tc->tc_quality = -100;
8955 tc->tc_priv = sc;
8956
8957 tc_init(tc);
8958 }
8959