if_mcx.c revision 1.17 1 1.17 jmcneill /* $NetBSD: if_mcx.c,v 1.17 2021/02/20 13:31:35 jmcneill Exp $ */
2 1.17 jmcneill /* $OpenBSD: if_mcx.c,v 1.99 2021/02/15 03:42:00 dlg Exp $ */
3 1.1 jmcneill
4 1.1 jmcneill /*
5 1.1 jmcneill * Copyright (c) 2017 David Gwynne <dlg (at) openbsd.org>
6 1.1 jmcneill * Copyright (c) 2019 Jonathan Matthew <jmatthew (at) openbsd.org>
7 1.1 jmcneill *
8 1.1 jmcneill * Permission to use, copy, modify, and distribute this software for any
9 1.1 jmcneill * purpose with or without fee is hereby granted, provided that the above
10 1.1 jmcneill * copyright notice and this permission notice appear in all copies.
11 1.1 jmcneill *
12 1.1 jmcneill * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 1.1 jmcneill * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 1.1 jmcneill * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 1.1 jmcneill * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 1.1 jmcneill * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 1.1 jmcneill * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 1.1 jmcneill * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 1.1 jmcneill */
20 1.1 jmcneill
21 1.1 jmcneill #ifdef _KERNEL_OPT
22 1.1 jmcneill #include "opt_net_mpsafe.h"
23 1.1 jmcneill #endif
24 1.1 jmcneill
25 1.14 jmcneill #include <sys/cdefs.h>
26 1.17 jmcneill __KERNEL_RCSID(0, "$NetBSD: if_mcx.c,v 1.17 2021/02/20 13:31:35 jmcneill Exp $");
27 1.14 jmcneill
28 1.1 jmcneill #include <sys/param.h>
29 1.1 jmcneill #include <sys/systm.h>
30 1.1 jmcneill #include <sys/sockio.h>
31 1.1 jmcneill #include <sys/mbuf.h>
32 1.1 jmcneill #include <sys/kernel.h>
33 1.1 jmcneill #include <sys/socket.h>
34 1.1 jmcneill #include <sys/device.h>
35 1.1 jmcneill #include <sys/pool.h>
36 1.1 jmcneill #include <sys/queue.h>
37 1.1 jmcneill #include <sys/callout.h>
38 1.1 jmcneill #include <sys/workqueue.h>
39 1.1 jmcneill #include <sys/atomic.h>
40 1.15 jmcneill #include <sys/timetc.h>
41 1.1 jmcneill #include <sys/kmem.h>
42 1.1 jmcneill #include <sys/bus.h>
43 1.15 jmcneill #include <sys/interrupt.h>
44 1.16 jmcneill #include <sys/pcq.h>
45 1.16 jmcneill #include <sys/cpu.h>
46 1.1 jmcneill
47 1.1 jmcneill #include <machine/intr.h>
48 1.1 jmcneill
49 1.1 jmcneill #include <net/if.h>
50 1.1 jmcneill #include <net/if_dl.h>
51 1.1 jmcneill #include <net/if_ether.h>
52 1.1 jmcneill #include <net/if_media.h>
53 1.15 jmcneill #include <net/if_vlanvar.h>
54 1.15 jmcneill #include <net/toeplitz.h>
55 1.1 jmcneill
56 1.1 jmcneill #include <net/bpf.h>
57 1.1 jmcneill
58 1.1 jmcneill #include <netinet/in.h>
59 1.1 jmcneill
60 1.1 jmcneill #include <dev/pci/pcireg.h>
61 1.1 jmcneill #include <dev/pci/pcivar.h>
62 1.1 jmcneill #include <dev/pci/pcidevs.h>
63 1.1 jmcneill
64 1.15 jmcneill /* TODO: Port kstat key/value stuff to evcnt/sysmon */
65 1.15 jmcneill #define NKSTAT 0
66 1.15 jmcneill
67 1.12 thorpej /* XXX This driver is not yet MP-safe; don't claim to be! */
68 1.12 thorpej /* #ifdef NET_MPSAFE */
69 1.12 thorpej /* #define MCX_MPSAFE 1 */
70 1.12 thorpej /* #define CALLOUT_FLAGS CALLOUT_MPSAFE */
71 1.12 thorpej /* #else */
72 1.1 jmcneill #define CALLOUT_FLAGS 0
73 1.12 thorpej /* #endif */
74 1.1 jmcneill
75 1.16 jmcneill #define MCX_TXQ_NUM 2048
76 1.16 jmcneill
77 1.1 jmcneill #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)
78 1.1 jmcneill #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)
79 1.1 jmcneill
80 1.1 jmcneill #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */
81 1.1 jmcneill
82 1.15 jmcneill #define MCX_FW_VER 0x0000
83 1.15 jmcneill #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff)
84 1.15 jmcneill #define MCX_FW_VER_MINOR(_v) ((_v) >> 16)
85 1.15 jmcneill #define MCX_CMDIF_FW_SUBVER 0x0004
86 1.15 jmcneill #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff)
87 1.15 jmcneill #define MCX_CMDIF(_v) ((_v) >> 16)
88 1.1 jmcneill
89 1.15 jmcneill #define MCX_ISSI 1 /* as per the PRM */
90 1.15 jmcneill #define MCX_CMD_IF_SUPPORTED 5
91 1.1 jmcneill
92 1.15 jmcneill #define MCX_HARDMTU 9500
93 1.1 jmcneill
94 1.15 jmcneill #define MCX_PAGE_SHIFT 12
95 1.15 jmcneill #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT)
96 1.1 jmcneill
97 1.1 jmcneill /* queue sizes */
98 1.15 jmcneill #define MCX_LOG_EQ_SIZE 7
99 1.15 jmcneill #define MCX_LOG_CQ_SIZE 12
100 1.15 jmcneill #define MCX_LOG_RQ_SIZE 10
101 1.15 jmcneill #define MCX_LOG_SQ_SIZE 11
102 1.15 jmcneill
103 1.15 jmcneill #define MCX_MAX_QUEUES 16
104 1.1 jmcneill
105 1.1 jmcneill /* completion event moderation - about 10khz, or 90% of the cq */
106 1.15 jmcneill #define MCX_CQ_MOD_PERIOD 50
107 1.15 jmcneill #define MCX_CQ_MOD_COUNTER \
108 1.15 jmcneill (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10)
109 1.15 jmcneill
110 1.15 jmcneill #define MCX_LOG_SQ_ENTRY_SIZE 6
111 1.15 jmcneill #define MCX_SQ_ENTRY_MAX_SLOTS 4
112 1.15 jmcneill #define MCX_SQ_SEGS_PER_SLOT \
113 1.1 jmcneill (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg))
114 1.15 jmcneill #define MCX_SQ_MAX_SEGMENTS \
115 1.1 jmcneill 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT)
116 1.1 jmcneill
117 1.15 jmcneill #define MCX_LOG_FLOW_TABLE_SIZE 5
118 1.15 jmcneill #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */
119 1.15 jmcneill #define MCX_NUM_MCAST_FLOWS \
120 1.1 jmcneill ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS)
121 1.1 jmcneill
122 1.15 jmcneill #define MCX_SQ_INLINE_SIZE 18
123 1.15 jmcneill CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE);
124 1.1 jmcneill
125 1.1 jmcneill /* doorbell offsets */
126 1.15 jmcneill #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE
127 1.1 jmcneill
128 1.15 jmcneill #define MCX_CQ_DOORBELL_BASE 0
129 1.15 jmcneill #define MCX_CQ_DOORBELL_STRIDE 64
130 1.15 jmcneill
131 1.15 jmcneill #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2
132 1.15 jmcneill #define MCX_WQ_DOORBELL_STRIDE 64
133 1.15 jmcneill /* make sure the doorbells fit */
134 1.15 jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE);
135 1.15 jmcneill CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE <
136 1.15 jmcneill MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE);
137 1.15 jmcneill
138 1.15 jmcneill #define MCX_WQ_DOORBELL_MASK 0xffff
139 1.1 jmcneill
140 1.1 jmcneill /* uar registers */
141 1.15 jmcneill #define MCX_UAR_CQ_DOORBELL 0x20
142 1.15 jmcneill #define MCX_UAR_EQ_DOORBELL_ARM 0x40
143 1.15 jmcneill #define MCX_UAR_EQ_DOORBELL 0x48
144 1.15 jmcneill #define MCX_UAR_BF 0x800
145 1.1 jmcneill
146 1.15 jmcneill #define MCX_CMDQ_ADDR_HI 0x0010
147 1.15 jmcneill #define MCX_CMDQ_ADDR_LO 0x0014
148 1.1 jmcneill #define MCX_CMDQ_ADDR_NMASK 0xfff
149 1.1 jmcneill #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf)
150 1.1 jmcneill #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf)
151 1.1 jmcneill #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8)
152 1.1 jmcneill #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8)
153 1.1 jmcneill #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8)
154 1.1 jmcneill
155 1.1 jmcneill #define MCX_CMDQ_DOORBELL 0x0018
156 1.1 jmcneill
157 1.15 jmcneill #define MCX_STATE 0x01fc
158 1.15 jmcneill #define MCX_STATE_MASK (1U << 31)
159 1.15 jmcneill #define MCX_STATE_INITIALIZING (1 << 31)
160 1.15 jmcneill #define MCX_STATE_READY (0 << 31)
161 1.15 jmcneill #define MCX_STATE_INTERFACE_MASK (0x3 << 24)
162 1.15 jmcneill #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24)
163 1.15 jmcneill #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24)
164 1.15 jmcneill
165 1.15 jmcneill #define MCX_INTERNAL_TIMER 0x1000
166 1.15 jmcneill #define MCX_INTERNAL_TIMER_H 0x1000
167 1.15 jmcneill #define MCX_INTERNAL_TIMER_L 0x1004
168 1.15 jmcneill
169 1.15 jmcneill #define MCX_CLEAR_INT 0x100c
170 1.15 jmcneill
171 1.15 jmcneill #define MCX_REG_OP_WRITE 0
172 1.15 jmcneill #define MCX_REG_OP_READ 1
173 1.15 jmcneill
174 1.15 jmcneill #define MCX_REG_PMLP 0x5002
175 1.15 jmcneill #define MCX_REG_PMTU 0x5003
176 1.15 jmcneill #define MCX_REG_PTYS 0x5004
177 1.15 jmcneill #define MCX_REG_PAOS 0x5006
178 1.15 jmcneill #define MCX_REG_PFCC 0x5007
179 1.15 jmcneill #define MCX_REG_PPCNT 0x5008
180 1.15 jmcneill #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */
181 1.15 jmcneill #define MCX_REG_MTMP 0x900a /* mgmt temp */
182 1.15 jmcneill #define MCX_REG_MCIA 0x9014
183 1.15 jmcneill #define MCX_REG_MCAM 0x907f
184 1.15 jmcneill
185 1.15 jmcneill #define MCX_ETHER_CAP_SGMII 0
186 1.15 jmcneill #define MCX_ETHER_CAP_1000_KX 1
187 1.15 jmcneill #define MCX_ETHER_CAP_10G_CX4 2
188 1.15 jmcneill #define MCX_ETHER_CAP_10G_KX4 3
189 1.15 jmcneill #define MCX_ETHER_CAP_10G_KR 4
190 1.15 jmcneill #define MCX_ETHER_CAP_20G_KR2 5
191 1.15 jmcneill #define MCX_ETHER_CAP_40G_CR4 6
192 1.15 jmcneill #define MCX_ETHER_CAP_40G_KR4 7
193 1.15 jmcneill #define MCX_ETHER_CAP_56G_R4 8
194 1.15 jmcneill #define MCX_ETHER_CAP_10G_CR 12
195 1.15 jmcneill #define MCX_ETHER_CAP_10G_SR 13
196 1.15 jmcneill #define MCX_ETHER_CAP_10G_LR 14
197 1.15 jmcneill #define MCX_ETHER_CAP_40G_SR4 15
198 1.15 jmcneill #define MCX_ETHER_CAP_40G_LR4 16
199 1.15 jmcneill #define MCX_ETHER_CAP_50G_SR2 18
200 1.15 jmcneill #define MCX_ETHER_CAP_100G_CR4 20
201 1.15 jmcneill #define MCX_ETHER_CAP_100G_SR4 21
202 1.15 jmcneill #define MCX_ETHER_CAP_100G_KR4 22
203 1.15 jmcneill #define MCX_ETHER_CAP_100G_LR4 23
204 1.15 jmcneill #define MCX_ETHER_CAP_100_TX 24
205 1.15 jmcneill #define MCX_ETHER_CAP_1000_T 25
206 1.15 jmcneill #define MCX_ETHER_CAP_10G_T 26
207 1.15 jmcneill #define MCX_ETHER_CAP_25G_CR 27
208 1.15 jmcneill #define MCX_ETHER_CAP_25G_KR 28
209 1.15 jmcneill #define MCX_ETHER_CAP_25G_SR 29
210 1.15 jmcneill #define MCX_ETHER_CAP_50G_CR2 30
211 1.15 jmcneill #define MCX_ETHER_CAP_50G_KR2 31
212 1.15 jmcneill
213 1.15 jmcneill #define MCX_MAX_CQE 32
214 1.15 jmcneill
215 1.15 jmcneill #define MCX_CMD_QUERY_HCA_CAP 0x100
216 1.15 jmcneill #define MCX_CMD_QUERY_ADAPTER 0x101
217 1.15 jmcneill #define MCX_CMD_INIT_HCA 0x102
218 1.15 jmcneill #define MCX_CMD_TEARDOWN_HCA 0x103
219 1.15 jmcneill #define MCX_CMD_ENABLE_HCA 0x104
220 1.15 jmcneill #define MCX_CMD_DISABLE_HCA 0x105
221 1.15 jmcneill #define MCX_CMD_QUERY_PAGES 0x107
222 1.15 jmcneill #define MCX_CMD_MANAGE_PAGES 0x108
223 1.15 jmcneill #define MCX_CMD_SET_HCA_CAP 0x109
224 1.15 jmcneill #define MCX_CMD_QUERY_ISSI 0x10a
225 1.15 jmcneill #define MCX_CMD_SET_ISSI 0x10b
226 1.15 jmcneill #define MCX_CMD_SET_DRIVER_VERSION 0x10d
227 1.15 jmcneill #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203
228 1.15 jmcneill #define MCX_CMD_CREATE_EQ 0x301
229 1.15 jmcneill #define MCX_CMD_DESTROY_EQ 0x302
230 1.15 jmcneill #define MCX_CMD_QUERY_EQ 0x303
231 1.15 jmcneill #define MCX_CMD_CREATE_CQ 0x400
232 1.15 jmcneill #define MCX_CMD_DESTROY_CQ 0x401
233 1.15 jmcneill #define MCX_CMD_QUERY_CQ 0x402
234 1.15 jmcneill #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754
235 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \
236 1.15 jmcneill 0x755
237 1.15 jmcneill #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770
238 1.15 jmcneill #define MCX_CMD_ALLOC_PD 0x800
239 1.15 jmcneill #define MCX_CMD_ALLOC_UAR 0x802
240 1.15 jmcneill #define MCX_CMD_ACCESS_REG 0x805
241 1.15 jmcneill #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816
242 1.15 jmcneill #define MCX_CMD_CREATE_TIR 0x900
243 1.15 jmcneill #define MCX_CMD_DESTROY_TIR 0x902
244 1.15 jmcneill #define MCX_CMD_CREATE_SQ 0x904
245 1.15 jmcneill #define MCX_CMD_MODIFY_SQ 0x905
246 1.15 jmcneill #define MCX_CMD_DESTROY_SQ 0x906
247 1.15 jmcneill #define MCX_CMD_QUERY_SQ 0x907
248 1.15 jmcneill #define MCX_CMD_CREATE_RQ 0x908
249 1.15 jmcneill #define MCX_CMD_MODIFY_RQ 0x909
250 1.15 jmcneill #define MCX_CMD_DESTROY_RQ 0x90a
251 1.15 jmcneill #define MCX_CMD_QUERY_RQ 0x90b
252 1.15 jmcneill #define MCX_CMD_CREATE_TIS 0x912
253 1.15 jmcneill #define MCX_CMD_DESTROY_TIS 0x914
254 1.15 jmcneill #define MCX_CMD_CREATE_RQT 0x916
255 1.15 jmcneill #define MCX_CMD_DESTROY_RQT 0x918
256 1.15 jmcneill #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f
257 1.15 jmcneill #define MCX_CMD_CREATE_FLOW_TABLE 0x930
258 1.15 jmcneill #define MCX_CMD_DESTROY_FLOW_TABLE 0x931
259 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_TABLE 0x932
260 1.15 jmcneill #define MCX_CMD_CREATE_FLOW_GROUP 0x933
261 1.15 jmcneill #define MCX_CMD_DESTROY_FLOW_GROUP 0x934
262 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_GROUP 0x935
263 1.15 jmcneill #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936
264 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937
265 1.15 jmcneill #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938
266 1.15 jmcneill #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939
267 1.15 jmcneill #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b
268 1.15 jmcneill
269 1.15 jmcneill #define MCX_QUEUE_STATE_RST 0
270 1.15 jmcneill #define MCX_QUEUE_STATE_RDY 1
271 1.15 jmcneill #define MCX_QUEUE_STATE_ERR 3
272 1.1 jmcneill
273 1.15 jmcneill #define MCX_FLOW_TABLE_TYPE_RX 0
274 1.15 jmcneill #define MCX_FLOW_TABLE_TYPE_TX 1
275 1.1 jmcneill
276 1.15 jmcneill #define MCX_CMDQ_INLINE_DATASIZE 16
277 1.1 jmcneill
278 1.1 jmcneill struct mcx_cmdq_entry {
279 1.1 jmcneill uint8_t cq_type;
280 1.1 jmcneill #define MCX_CMDQ_TYPE_PCIE 0x7
281 1.1 jmcneill uint8_t cq_reserved0[3];
282 1.1 jmcneill
283 1.1 jmcneill uint32_t cq_input_length;
284 1.1 jmcneill uint64_t cq_input_ptr;
285 1.1 jmcneill uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE];
286 1.1 jmcneill
287 1.1 jmcneill uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE];
288 1.1 jmcneill uint64_t cq_output_ptr;
289 1.1 jmcneill uint32_t cq_output_length;
290 1.1 jmcneill
291 1.1 jmcneill uint8_t cq_token;
292 1.1 jmcneill uint8_t cq_signature;
293 1.1 jmcneill uint8_t cq_reserved1[1];
294 1.1 jmcneill uint8_t cq_status;
295 1.1 jmcneill #define MCX_CQ_STATUS_SHIFT 1
296 1.1 jmcneill #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT)
297 1.1 jmcneill #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT)
298 1.1 jmcneill #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT)
299 1.1 jmcneill #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT)
300 1.1 jmcneill #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT)
301 1.1 jmcneill #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT)
302 1.1 jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT)
303 1.1 jmcneill #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT)
304 1.1 jmcneill #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT)
305 1.1 jmcneill #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT)
306 1.1 jmcneill #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT)
307 1.1 jmcneill #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT)
308 1.1 jmcneill #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT)
309 1.1 jmcneill #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT)
310 1.1 jmcneill #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \
311 1.1 jmcneill (0x10 << MCX_CQ_STATUS_SHIFT)
312 1.1 jmcneill #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT)
313 1.1 jmcneill #define MCX_CQ_STATUS_OWN_MASK 0x1
314 1.1 jmcneill #define MCX_CQ_STATUS_OWN_SW 0x0
315 1.1 jmcneill #define MCX_CQ_STATUS_OWN_HW 0x1
316 1.1 jmcneill } __packed __aligned(8);
317 1.1 jmcneill
318 1.1 jmcneill #define MCX_CMDQ_MAILBOX_DATASIZE 512
319 1.1 jmcneill
320 1.1 jmcneill struct mcx_cmdq_mailbox {
321 1.1 jmcneill uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE];
322 1.1 jmcneill uint8_t mb_reserved0[48];
323 1.1 jmcneill uint64_t mb_next_ptr;
324 1.1 jmcneill uint32_t mb_block_number;
325 1.1 jmcneill uint8_t mb_reserved1[1];
326 1.1 jmcneill uint8_t mb_token;
327 1.1 jmcneill uint8_t mb_ctrl_signature;
328 1.1 jmcneill uint8_t mb_signature;
329 1.1 jmcneill } __packed __aligned(8);
330 1.1 jmcneill
331 1.1 jmcneill #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10)
332 1.1 jmcneill #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \
333 1.1 jmcneill MCX_CMDQ_MAILBOX_ALIGN)
334 1.1 jmcneill /*
335 1.1 jmcneill * command mailbox structres
336 1.1 jmcneill */
337 1.1 jmcneill
338 1.1 jmcneill struct mcx_cmd_enable_hca_in {
339 1.1 jmcneill uint16_t cmd_opcode;
340 1.1 jmcneill uint8_t cmd_reserved0[4];
341 1.1 jmcneill uint16_t cmd_op_mod;
342 1.1 jmcneill uint8_t cmd_reserved1[2];
343 1.1 jmcneill uint16_t cmd_function_id;
344 1.1 jmcneill uint8_t cmd_reserved2[4];
345 1.1 jmcneill } __packed __aligned(4);
346 1.1 jmcneill
347 1.1 jmcneill struct mcx_cmd_enable_hca_out {
348 1.1 jmcneill uint8_t cmd_status;
349 1.1 jmcneill uint8_t cmd_reserved0[3];
350 1.1 jmcneill uint32_t cmd_syndrome;
351 1.1 jmcneill uint8_t cmd_reserved1[4];
352 1.1 jmcneill } __packed __aligned(4);
353 1.1 jmcneill
354 1.1 jmcneill struct mcx_cmd_init_hca_in {
355 1.1 jmcneill uint16_t cmd_opcode;
356 1.1 jmcneill uint8_t cmd_reserved0[4];
357 1.1 jmcneill uint16_t cmd_op_mod;
358 1.1 jmcneill uint8_t cmd_reserved1[8];
359 1.1 jmcneill } __packed __aligned(4);
360 1.1 jmcneill
361 1.1 jmcneill struct mcx_cmd_init_hca_out {
362 1.1 jmcneill uint8_t cmd_status;
363 1.1 jmcneill uint8_t cmd_reserved0[3];
364 1.1 jmcneill uint32_t cmd_syndrome;
365 1.1 jmcneill uint8_t cmd_reserved1[8];
366 1.1 jmcneill } __packed __aligned(4);
367 1.1 jmcneill
368 1.1 jmcneill struct mcx_cmd_teardown_hca_in {
369 1.1 jmcneill uint16_t cmd_opcode;
370 1.1 jmcneill uint8_t cmd_reserved0[4];
371 1.1 jmcneill uint16_t cmd_op_mod;
372 1.1 jmcneill uint8_t cmd_reserved1[2];
373 1.1 jmcneill #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0
374 1.1 jmcneill #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1
375 1.1 jmcneill uint16_t cmd_profile;
376 1.1 jmcneill uint8_t cmd_reserved2[4];
377 1.1 jmcneill } __packed __aligned(4);
378 1.1 jmcneill
379 1.1 jmcneill struct mcx_cmd_teardown_hca_out {
380 1.1 jmcneill uint8_t cmd_status;
381 1.1 jmcneill uint8_t cmd_reserved0[3];
382 1.1 jmcneill uint32_t cmd_syndrome;
383 1.1 jmcneill uint8_t cmd_reserved1[8];
384 1.1 jmcneill } __packed __aligned(4);
385 1.1 jmcneill
386 1.1 jmcneill struct mcx_cmd_access_reg_in {
387 1.1 jmcneill uint16_t cmd_opcode;
388 1.1 jmcneill uint8_t cmd_reserved0[4];
389 1.1 jmcneill uint16_t cmd_op_mod;
390 1.1 jmcneill uint8_t cmd_reserved1[2];
391 1.1 jmcneill uint16_t cmd_register_id;
392 1.1 jmcneill uint32_t cmd_argument;
393 1.1 jmcneill } __packed __aligned(4);
394 1.1 jmcneill
395 1.1 jmcneill struct mcx_cmd_access_reg_out {
396 1.1 jmcneill uint8_t cmd_status;
397 1.1 jmcneill uint8_t cmd_reserved0[3];
398 1.1 jmcneill uint32_t cmd_syndrome;
399 1.1 jmcneill uint8_t cmd_reserved1[8];
400 1.1 jmcneill } __packed __aligned(4);
401 1.1 jmcneill
402 1.1 jmcneill struct mcx_reg_pmtu {
403 1.1 jmcneill uint8_t rp_reserved1;
404 1.1 jmcneill uint8_t rp_local_port;
405 1.1 jmcneill uint8_t rp_reserved2[2];
406 1.1 jmcneill uint16_t rp_max_mtu;
407 1.1 jmcneill uint8_t rp_reserved3[2];
408 1.1 jmcneill uint16_t rp_admin_mtu;
409 1.1 jmcneill uint8_t rp_reserved4[2];
410 1.1 jmcneill uint16_t rp_oper_mtu;
411 1.1 jmcneill uint8_t rp_reserved5[2];
412 1.1 jmcneill } __packed __aligned(4);
413 1.1 jmcneill
414 1.1 jmcneill struct mcx_reg_ptys {
415 1.1 jmcneill uint8_t rp_reserved1;
416 1.1 jmcneill uint8_t rp_local_port;
417 1.1 jmcneill uint8_t rp_reserved2;
418 1.1 jmcneill uint8_t rp_proto_mask;
419 1.1 jmcneill #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2)
420 1.1 jmcneill uint8_t rp_reserved3[8];
421 1.1 jmcneill uint32_t rp_eth_proto_cap;
422 1.1 jmcneill uint8_t rp_reserved4[8];
423 1.1 jmcneill uint32_t rp_eth_proto_admin;
424 1.1 jmcneill uint8_t rp_reserved5[8];
425 1.1 jmcneill uint32_t rp_eth_proto_oper;
426 1.1 jmcneill uint8_t rp_reserved6[24];
427 1.1 jmcneill } __packed __aligned(4);
428 1.1 jmcneill
429 1.1 jmcneill struct mcx_reg_paos {
430 1.1 jmcneill uint8_t rp_reserved1;
431 1.1 jmcneill uint8_t rp_local_port;
432 1.1 jmcneill uint8_t rp_admin_status;
433 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP 1
434 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2
435 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3
436 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4
437 1.1 jmcneill uint8_t rp_oper_status;
438 1.1 jmcneill #define MCX_REG_PAOS_OPER_STATUS_UP 1
439 1.1 jmcneill #define MCX_REG_PAOS_OPER_STATUS_DOWN 2
440 1.1 jmcneill #define MCX_REG_PAOS_OPER_STATUS_FAILED 4
441 1.1 jmcneill uint8_t rp_admin_state_update;
442 1.1 jmcneill #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7)
443 1.1 jmcneill uint8_t rp_reserved2[11];
444 1.1 jmcneill } __packed __aligned(4);
445 1.1 jmcneill
446 1.1 jmcneill struct mcx_reg_pfcc {
447 1.1 jmcneill uint8_t rp_reserved1;
448 1.1 jmcneill uint8_t rp_local_port;
449 1.1 jmcneill uint8_t rp_reserved2[3];
450 1.1 jmcneill uint8_t rp_prio_mask_tx;
451 1.1 jmcneill uint8_t rp_reserved3;
452 1.1 jmcneill uint8_t rp_prio_mask_rx;
453 1.1 jmcneill uint8_t rp_pptx_aptx;
454 1.1 jmcneill uint8_t rp_pfctx;
455 1.1 jmcneill uint8_t rp_fctx_dis;
456 1.1 jmcneill uint8_t rp_reserved4;
457 1.1 jmcneill uint8_t rp_pprx_aprx;
458 1.1 jmcneill uint8_t rp_pfcrx;
459 1.1 jmcneill uint8_t rp_reserved5[2];
460 1.1 jmcneill uint16_t rp_dev_stall_min;
461 1.1 jmcneill uint16_t rp_dev_stall_crit;
462 1.1 jmcneill uint8_t rp_reserved6[12];
463 1.1 jmcneill } __packed __aligned(4);
464 1.1 jmcneill
465 1.1 jmcneill #define MCX_PMLP_MODULE_NUM_MASK 0xff
466 1.1 jmcneill struct mcx_reg_pmlp {
467 1.1 jmcneill uint8_t rp_rxtx;
468 1.1 jmcneill uint8_t rp_local_port;
469 1.1 jmcneill uint8_t rp_reserved0;
470 1.1 jmcneill uint8_t rp_width;
471 1.1 jmcneill uint32_t rp_lane0_mapping;
472 1.1 jmcneill uint32_t rp_lane1_mapping;
473 1.1 jmcneill uint32_t rp_lane2_mapping;
474 1.1 jmcneill uint32_t rp_lane3_mapping;
475 1.1 jmcneill uint8_t rp_reserved1[44];
476 1.1 jmcneill } __packed __aligned(4);
477 1.1 jmcneill
478 1.15 jmcneill struct mcx_reg_ppcnt {
479 1.15 jmcneill uint8_t ppcnt_swid;
480 1.15 jmcneill uint8_t ppcnt_local_port;
481 1.15 jmcneill uint8_t ppcnt_pnat;
482 1.15 jmcneill uint8_t ppcnt_grp;
483 1.15 jmcneill #define MCX_REG_PPCNT_GRP_IEEE8023 0x00
484 1.15 jmcneill #define MCX_REG_PPCNT_GRP_RFC2863 0x01
485 1.15 jmcneill #define MCX_REG_PPCNT_GRP_RFC2819 0x02
486 1.15 jmcneill #define MCX_REG_PPCNT_GRP_RFC3635 0x03
487 1.15 jmcneill #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10
488 1.15 jmcneill #define MCX_REG_PPCNT_GRP_PER_TC 0x11
489 1.15 jmcneill #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11
490 1.15 jmcneill
491 1.15 jmcneill uint8_t ppcnt_clr;
492 1.15 jmcneill uint8_t ppcnt_reserved1[2];
493 1.15 jmcneill uint8_t ppcnt_prio_tc;
494 1.15 jmcneill #define MCX_REG_PPCNT_CLR (1 << 7)
495 1.15 jmcneill
496 1.15 jmcneill uint8_t ppcnt_counter_set[248];
497 1.15 jmcneill } __packed __aligned(8);
498 1.15 jmcneill CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256);
499 1.15 jmcneill CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) %
500 1.15 jmcneill sizeof(uint64_t)) == 0);
501 1.15 jmcneill
502 1.15 jmcneill enum mcx_ppcnt_ieee8023 {
503 1.15 jmcneill frames_transmitted_ok,
504 1.15 jmcneill frames_received_ok,
505 1.15 jmcneill frame_check_sequence_errors,
506 1.15 jmcneill alignment_errors,
507 1.15 jmcneill octets_transmitted_ok,
508 1.15 jmcneill octets_received_ok,
509 1.15 jmcneill multicast_frames_xmitted_ok,
510 1.15 jmcneill broadcast_frames_xmitted_ok,
511 1.15 jmcneill multicast_frames_received_ok,
512 1.15 jmcneill broadcast_frames_received_ok,
513 1.15 jmcneill in_range_length_errors,
514 1.15 jmcneill out_of_range_length_field,
515 1.15 jmcneill frame_too_long_errors,
516 1.15 jmcneill symbol_error_during_carrier,
517 1.15 jmcneill mac_control_frames_transmitted,
518 1.15 jmcneill mac_control_frames_received,
519 1.15 jmcneill unsupported_opcodes_received,
520 1.15 jmcneill pause_mac_ctrl_frames_received,
521 1.15 jmcneill pause_mac_ctrl_frames_transmitted,
522 1.15 jmcneill
523 1.15 jmcneill mcx_ppcnt_ieee8023_count
524 1.15 jmcneill };
525 1.15 jmcneill CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98);
526 1.15 jmcneill
527 1.15 jmcneill enum mcx_ppcnt_rfc2863 {
528 1.15 jmcneill in_octets,
529 1.15 jmcneill in_ucast_pkts,
530 1.15 jmcneill in_discards,
531 1.15 jmcneill in_errors,
532 1.15 jmcneill in_unknown_protos,
533 1.15 jmcneill out_octets,
534 1.15 jmcneill out_ucast_pkts,
535 1.15 jmcneill out_discards,
536 1.15 jmcneill out_errors,
537 1.15 jmcneill in_multicast_pkts,
538 1.15 jmcneill in_broadcast_pkts,
539 1.15 jmcneill out_multicast_pkts,
540 1.15 jmcneill out_broadcast_pkts,
541 1.15 jmcneill
542 1.15 jmcneill mcx_ppcnt_rfc2863_count
543 1.15 jmcneill };
544 1.15 jmcneill CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68);
545 1.15 jmcneill
546 1.15 jmcneill enum mcx_ppcnt_rfc2819 {
547 1.15 jmcneill drop_events,
548 1.15 jmcneill octets,
549 1.15 jmcneill pkts,
550 1.15 jmcneill broadcast_pkts,
551 1.15 jmcneill multicast_pkts,
552 1.15 jmcneill crc_align_errors,
553 1.15 jmcneill undersize_pkts,
554 1.15 jmcneill oversize_pkts,
555 1.15 jmcneill fragments,
556 1.15 jmcneill jabbers,
557 1.15 jmcneill collisions,
558 1.15 jmcneill pkts64octets,
559 1.15 jmcneill pkts65to127octets,
560 1.15 jmcneill pkts128to255octets,
561 1.15 jmcneill pkts256to511octets,
562 1.15 jmcneill pkts512to1023octets,
563 1.15 jmcneill pkts1024to1518octets,
564 1.15 jmcneill pkts1519to2047octets,
565 1.15 jmcneill pkts2048to4095octets,
566 1.15 jmcneill pkts4096to8191octets,
567 1.15 jmcneill pkts8192to10239octets,
568 1.15 jmcneill
569 1.15 jmcneill mcx_ppcnt_rfc2819_count
570 1.15 jmcneill };
571 1.15 jmcneill CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8);
572 1.15 jmcneill
573 1.15 jmcneill enum mcx_ppcnt_rfc3635 {
574 1.15 jmcneill dot3stats_alignment_errors,
575 1.15 jmcneill dot3stats_fcs_errors,
576 1.15 jmcneill dot3stats_single_collision_frames,
577 1.15 jmcneill dot3stats_multiple_collision_frames,
578 1.15 jmcneill dot3stats_sqe_test_errors,
579 1.15 jmcneill dot3stats_deferred_transmissions,
580 1.15 jmcneill dot3stats_late_collisions,
581 1.15 jmcneill dot3stats_excessive_collisions,
582 1.15 jmcneill dot3stats_internal_mac_transmit_errors,
583 1.15 jmcneill dot3stats_carrier_sense_errors,
584 1.15 jmcneill dot3stats_frame_too_longs,
585 1.15 jmcneill dot3stats_internal_mac_receive_errors,
586 1.15 jmcneill dot3stats_symbol_errors,
587 1.15 jmcneill dot3control_in_unknown_opcodes,
588 1.15 jmcneill dot3in_pause_frames,
589 1.15 jmcneill dot3out_pause_frames,
590 1.15 jmcneill
591 1.15 jmcneill mcx_ppcnt_rfc3635_count
592 1.15 jmcneill };
593 1.15 jmcneill CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80);
594 1.15 jmcneill
595 1.15 jmcneill struct mcx_reg_mcam {
596 1.15 jmcneill uint8_t _reserved1[1];
597 1.15 jmcneill uint8_t mcam_feature_group;
598 1.15 jmcneill uint8_t _reserved2[1];
599 1.15 jmcneill uint8_t mcam_access_reg_group;
600 1.15 jmcneill uint8_t _reserved3[4];
601 1.15 jmcneill uint8_t mcam_access_reg_cap_mask[16];
602 1.15 jmcneill uint8_t _reserved4[16];
603 1.15 jmcneill uint8_t mcam_feature_cap_mask[16];
604 1.15 jmcneill uint8_t _reserved5[16];
605 1.15 jmcneill } __packed __aligned(4);
606 1.15 jmcneill
607 1.15 jmcneill #define MCX_BITFIELD_BIT(bf, b) (bf[(sizeof bf - 1) - (b / 8)] & (b % 8))
608 1.15 jmcneill
609 1.15 jmcneill #define MCX_MCAM_FEATURE_CAP_SENSOR_MAP 6
610 1.15 jmcneill
611 1.15 jmcneill struct mcx_reg_mtcap {
612 1.15 jmcneill uint8_t _reserved1[3];
613 1.15 jmcneill uint8_t mtcap_sensor_count;
614 1.15 jmcneill uint8_t _reserved2[4];
615 1.15 jmcneill
616 1.15 jmcneill uint64_t mtcap_sensor_map;
617 1.15 jmcneill };
618 1.15 jmcneill
619 1.15 jmcneill struct mcx_reg_mtmp {
620 1.15 jmcneill uint8_t _reserved1[2];
621 1.15 jmcneill uint16_t mtmp_sensor_index;
622 1.15 jmcneill
623 1.15 jmcneill uint8_t _reserved2[2];
624 1.15 jmcneill uint16_t mtmp_temperature;
625 1.15 jmcneill
626 1.15 jmcneill uint16_t mtmp_mte_mtr;
627 1.15 jmcneill #define MCX_REG_MTMP_MTE (1 << 15)
628 1.15 jmcneill #define MCX_REG_MTMP_MTR (1 << 14)
629 1.15 jmcneill uint16_t mtmp_max_temperature;
630 1.15 jmcneill
631 1.15 jmcneill uint16_t mtmp_tee;
632 1.15 jmcneill #define MCX_REG_MTMP_TEE_NOPE (0 << 14)
633 1.15 jmcneill #define MCX_REG_MTMP_TEE_GENERATE (1 << 14)
634 1.15 jmcneill #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14)
635 1.15 jmcneill uint16_t mtmp_temperature_threshold_hi;
636 1.15 jmcneill
637 1.15 jmcneill uint8_t _reserved3[2];
638 1.15 jmcneill uint16_t mtmp_temperature_threshold_lo;
639 1.15 jmcneill
640 1.15 jmcneill uint8_t _reserved4[4];
641 1.15 jmcneill
642 1.15 jmcneill uint8_t mtmp_sensor_name[8];
643 1.15 jmcneill };
644 1.15 jmcneill CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20);
645 1.15 jmcneill CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18);
646 1.15 jmcneill
647 1.1 jmcneill #define MCX_MCIA_EEPROM_BYTES 32
648 1.1 jmcneill struct mcx_reg_mcia {
649 1.1 jmcneill uint8_t rm_l;
650 1.1 jmcneill uint8_t rm_module;
651 1.1 jmcneill uint8_t rm_reserved0;
652 1.1 jmcneill uint8_t rm_status;
653 1.1 jmcneill uint8_t rm_i2c_addr;
654 1.1 jmcneill uint8_t rm_page_num;
655 1.1 jmcneill uint16_t rm_dev_addr;
656 1.1 jmcneill uint16_t rm_reserved1;
657 1.1 jmcneill uint16_t rm_size;
658 1.1 jmcneill uint32_t rm_reserved2;
659 1.1 jmcneill uint8_t rm_data[48];
660 1.1 jmcneill } __packed __aligned(4);
661 1.1 jmcneill
662 1.1 jmcneill struct mcx_cmd_query_issi_in {
663 1.1 jmcneill uint16_t cmd_opcode;
664 1.1 jmcneill uint8_t cmd_reserved0[4];
665 1.1 jmcneill uint16_t cmd_op_mod;
666 1.1 jmcneill uint8_t cmd_reserved1[8];
667 1.1 jmcneill } __packed __aligned(4);
668 1.1 jmcneill
669 1.1 jmcneill struct mcx_cmd_query_issi_il_out {
670 1.1 jmcneill uint8_t cmd_status;
671 1.1 jmcneill uint8_t cmd_reserved0[3];
672 1.1 jmcneill uint32_t cmd_syndrome;
673 1.1 jmcneill uint8_t cmd_reserved1[2];
674 1.1 jmcneill uint16_t cmd_current_issi;
675 1.1 jmcneill uint8_t cmd_reserved2[4];
676 1.1 jmcneill } __packed __aligned(4);
677 1.1 jmcneill
678 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE);
679 1.1 jmcneill
680 1.1 jmcneill struct mcx_cmd_query_issi_mb_out {
681 1.1 jmcneill uint8_t cmd_reserved2[16];
682 1.1 jmcneill uint8_t cmd_supported_issi[80]; /* very big endian */
683 1.1 jmcneill } __packed __aligned(4);
684 1.1 jmcneill
685 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE);
686 1.1 jmcneill
687 1.1 jmcneill struct mcx_cmd_set_issi_in {
688 1.1 jmcneill uint16_t cmd_opcode;
689 1.1 jmcneill uint8_t cmd_reserved0[4];
690 1.1 jmcneill uint16_t cmd_op_mod;
691 1.1 jmcneill uint8_t cmd_reserved1[2];
692 1.1 jmcneill uint16_t cmd_current_issi;
693 1.1 jmcneill uint8_t cmd_reserved2[4];
694 1.1 jmcneill } __packed __aligned(4);
695 1.1 jmcneill
696 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE);
697 1.1 jmcneill
698 1.1 jmcneill struct mcx_cmd_set_issi_out {
699 1.1 jmcneill uint8_t cmd_status;
700 1.1 jmcneill uint8_t cmd_reserved0[3];
701 1.1 jmcneill uint32_t cmd_syndrome;
702 1.1 jmcneill uint8_t cmd_reserved1[8];
703 1.1 jmcneill } __packed __aligned(4);
704 1.1 jmcneill
705 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE);
706 1.1 jmcneill
707 1.1 jmcneill struct mcx_cmd_query_pages_in {
708 1.1 jmcneill uint16_t cmd_opcode;
709 1.1 jmcneill uint8_t cmd_reserved0[4];
710 1.1 jmcneill uint16_t cmd_op_mod;
711 1.1 jmcneill #define MCX_CMD_QUERY_PAGES_BOOT 0x01
712 1.1 jmcneill #define MCX_CMD_QUERY_PAGES_INIT 0x02
713 1.1 jmcneill #define MCX_CMD_QUERY_PAGES_REGULAR 0x03
714 1.1 jmcneill uint8_t cmd_reserved1[8];
715 1.1 jmcneill } __packed __aligned(4);
716 1.1 jmcneill
717 1.1 jmcneill struct mcx_cmd_query_pages_out {
718 1.1 jmcneill uint8_t cmd_status;
719 1.1 jmcneill uint8_t cmd_reserved0[3];
720 1.1 jmcneill uint32_t cmd_syndrome;
721 1.1 jmcneill uint8_t cmd_reserved1[2];
722 1.1 jmcneill uint16_t cmd_func_id;
723 1.15 jmcneill int32_t cmd_num_pages;
724 1.1 jmcneill } __packed __aligned(4);
725 1.1 jmcneill
726 1.1 jmcneill struct mcx_cmd_manage_pages_in {
727 1.1 jmcneill uint16_t cmd_opcode;
728 1.1 jmcneill uint8_t cmd_reserved0[4];
729 1.1 jmcneill uint16_t cmd_op_mod;
730 1.1 jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \
731 1.1 jmcneill 0x00
732 1.1 jmcneill #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \
733 1.1 jmcneill 0x01
734 1.1 jmcneill #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \
735 1.1 jmcneill 0x02
736 1.1 jmcneill uint8_t cmd_reserved1[2];
737 1.1 jmcneill uint16_t cmd_func_id;
738 1.1 jmcneill uint32_t cmd_input_num_entries;
739 1.1 jmcneill } __packed __aligned(4);
740 1.1 jmcneill
741 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE);
742 1.1 jmcneill
743 1.1 jmcneill struct mcx_cmd_manage_pages_out {
744 1.1 jmcneill uint8_t cmd_status;
745 1.1 jmcneill uint8_t cmd_reserved0[3];
746 1.1 jmcneill uint32_t cmd_syndrome;
747 1.1 jmcneill uint32_t cmd_output_num_entries;
748 1.1 jmcneill uint8_t cmd_reserved1[4];
749 1.1 jmcneill } __packed __aligned(4);
750 1.1 jmcneill
751 1.1 jmcneill CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE);
752 1.1 jmcneill
753 1.1 jmcneill struct mcx_cmd_query_hca_cap_in {
754 1.1 jmcneill uint16_t cmd_opcode;
755 1.1 jmcneill uint8_t cmd_reserved0[4];
756 1.1 jmcneill uint16_t cmd_op_mod;
757 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0)
758 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0)
759 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1)
760 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1)
761 1.1 jmcneill #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1)
762 1.1 jmcneill uint8_t cmd_reserved1[8];
763 1.1 jmcneill } __packed __aligned(4);
764 1.1 jmcneill
765 1.1 jmcneill struct mcx_cmd_query_hca_cap_out {
766 1.1 jmcneill uint8_t cmd_status;
767 1.1 jmcneill uint8_t cmd_reserved0[3];
768 1.1 jmcneill uint32_t cmd_syndrome;
769 1.1 jmcneill uint8_t cmd_reserved1[8];
770 1.1 jmcneill } __packed __aligned(4);
771 1.1 jmcneill
772 1.1 jmcneill #define MCX_HCA_CAP_LEN 0x1000
773 1.1 jmcneill #define MCX_HCA_CAP_NMAILBOXES \
774 1.1 jmcneill (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE)
775 1.1 jmcneill
776 1.1 jmcneill #if __GNUC_PREREQ__(4, 3)
777 1.1 jmcneill #define __counter__ __COUNTER__
778 1.1 jmcneill #else
779 1.1 jmcneill #define __counter__ __LINE__
780 1.1 jmcneill #endif
781 1.1 jmcneill
782 1.1 jmcneill #define __token(_tok, _num) _tok##_num
783 1.1 jmcneill #define _token(_tok, _num) __token(_tok, _num)
784 1.1 jmcneill #define __reserved__ _token(__reserved, __counter__)
785 1.1 jmcneill
786 1.1 jmcneill struct mcx_cap_device {
787 1.1 jmcneill uint8_t reserved0[16];
788 1.1 jmcneill
789 1.1 jmcneill uint8_t log_max_srq_sz;
790 1.1 jmcneill uint8_t log_max_qp_sz;
791 1.1 jmcneill uint8_t __reserved__[1];
792 1.1 jmcneill uint8_t log_max_qp; /* 5 bits */
793 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f
794 1.1 jmcneill
795 1.1 jmcneill uint8_t __reserved__[1];
796 1.1 jmcneill uint8_t log_max_srq; /* 5 bits */
797 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f
798 1.1 jmcneill uint8_t __reserved__[2];
799 1.1 jmcneill
800 1.1 jmcneill uint8_t __reserved__[1];
801 1.1 jmcneill uint8_t log_max_cq_sz;
802 1.1 jmcneill uint8_t __reserved__[1];
803 1.1 jmcneill uint8_t log_max_cq; /* 5 bits */
804 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f
805 1.1 jmcneill
806 1.1 jmcneill uint8_t log_max_eq_sz;
807 1.1 jmcneill uint8_t log_max_mkey; /* 6 bits */
808 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f
809 1.1 jmcneill uint8_t __reserved__[1];
810 1.1 jmcneill uint8_t log_max_eq; /* 4 bits */
811 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f
812 1.1 jmcneill
813 1.1 jmcneill uint8_t max_indirection;
814 1.1 jmcneill uint8_t log_max_mrw_sz; /* 7 bits */
815 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f
816 1.1 jmcneill uint8_t teardown_log_max_msf_list_size;
817 1.1 jmcneill #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80
818 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \
819 1.1 jmcneill 0x3f
820 1.1 jmcneill uint8_t log_max_klm_list_size; /* 6 bits */
821 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \
822 1.1 jmcneill 0x3f
823 1.1 jmcneill
824 1.1 jmcneill uint8_t __reserved__[1];
825 1.1 jmcneill uint8_t log_max_ra_req_dc; /* 6 bits */
826 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f
827 1.1 jmcneill uint8_t __reserved__[1];
828 1.1 jmcneill uint8_t log_max_ra_res_dc; /* 6 bits */
829 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \
830 1.1 jmcneill 0x3f
831 1.1 jmcneill
832 1.1 jmcneill uint8_t __reserved__[1];
833 1.1 jmcneill uint8_t log_max_ra_req_qp; /* 6 bits */
834 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \
835 1.1 jmcneill 0x3f
836 1.1 jmcneill uint8_t __reserved__[1];
837 1.1 jmcneill uint8_t log_max_ra_res_qp; /* 6 bits */
838 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \
839 1.1 jmcneill 0x3f
840 1.1 jmcneill
841 1.1 jmcneill uint8_t flags1;
842 1.1 jmcneill #define MCX_CAP_DEVICE_END_PAD 0x80
843 1.1 jmcneill #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40
844 1.1 jmcneill #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \
845 1.1 jmcneill 0x20
846 1.1 jmcneill #define MCX_CAP_DEVICE_START_PAD 0x10
847 1.1 jmcneill #define MCX_CAP_DEVICE_128BYTE_CACHELINE \
848 1.1 jmcneill 0x08
849 1.1 jmcneill uint8_t __reserved__[1];
850 1.1 jmcneill uint16_t gid_table_size;
851 1.1 jmcneill
852 1.1 jmcneill uint16_t flags2;
853 1.1 jmcneill #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000
854 1.1 jmcneill #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000
855 1.1 jmcneill #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \
856 1.1 jmcneill 0x2000
857 1.1 jmcneill #define MCX_CAP_DEVICE_DEBUG 0x1000
858 1.1 jmcneill #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \
859 1.1 jmcneill 0x8000
860 1.1 jmcneill #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000
861 1.1 jmcneill #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff
862 1.1 jmcneill uint16_t pkey_table_size;
863 1.1 jmcneill
864 1.1 jmcneill uint8_t flags3;
865 1.1 jmcneill #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \
866 1.1 jmcneill 0x80
867 1.1 jmcneill #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \
868 1.1 jmcneill 0x40
869 1.1 jmcneill #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20
870 1.1 jmcneill #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10
871 1.1 jmcneill #define MCX_CAP_DEVICE_ETS 0x04
872 1.1 jmcneill #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02
873 1.1 jmcneill #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \
874 1.1 jmcneill 0x01
875 1.1 jmcneill uint8_t local_ca_ack_delay; /* 5 bits */
876 1.1 jmcneill #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \
877 1.1 jmcneill 0x1f
878 1.15 jmcneill #define MCX_CAP_DEVICE_MCAM_REG 0x40
879 1.1 jmcneill uint8_t port_type;
880 1.1 jmcneill #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \
881 1.1 jmcneill 0x80
882 1.1 jmcneill #define MCX_CAP_DEVICE_PORT_TYPE 0x03
883 1.15 jmcneill #define MCX_CAP_DEVICE_PORT_TYPE_ETH 0x01
884 1.1 jmcneill uint8_t num_ports;
885 1.1 jmcneill
886 1.1 jmcneill uint8_t snapshot_log_max_msg;
887 1.1 jmcneill #define MCX_CAP_DEVICE_SNAPSHOT 0x80
888 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f
889 1.1 jmcneill uint8_t max_tc; /* 4 bits */
890 1.1 jmcneill #define MCX_CAP_DEVICE_MAX_TC 0x0f
891 1.1 jmcneill uint8_t flags4;
892 1.1 jmcneill #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80
893 1.1 jmcneill #define MCX_CAP_DEVICE_DCBX 0x40
894 1.1 jmcneill #define MCX_CAP_DEVICE_ROL_S 0x02
895 1.1 jmcneill #define MCX_CAP_DEVICE_ROL_G 0x01
896 1.1 jmcneill uint8_t wol;
897 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_S 0x40
898 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_G 0x20
899 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_A 0x10
900 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_B 0x08
901 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_M 0x04
902 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_U 0x02
903 1.1 jmcneill #define MCX_CAP_DEVICE_WOL_P 0x01
904 1.1 jmcneill
905 1.1 jmcneill uint16_t stat_rate_support;
906 1.1 jmcneill uint8_t __reserved__[1];
907 1.1 jmcneill uint8_t cqe_version; /* 4 bits */
908 1.1 jmcneill #define MCX_CAP_DEVICE_CQE_VERSION 0x0f
909 1.1 jmcneill
910 1.1 jmcneill uint32_t flags5;
911 1.1 jmcneill #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \
912 1.1 jmcneill 0x80000000
913 1.1 jmcneill #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000
914 1.1 jmcneill #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \
915 1.1 jmcneill 0x10000000
916 1.1 jmcneill #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \
917 1.1 jmcneill 0x08000000
918 1.1 jmcneill #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000
919 1.1 jmcneill #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000
920 1.1 jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000
921 1.1 jmcneill #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000
922 1.1 jmcneill #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000
923 1.1 jmcneill #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000
924 1.1 jmcneill #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800
925 1.1 jmcneill #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400
926 1.1 jmcneill #define MCX_CAP_DEVICE_SHO 0x00000100
927 1.1 jmcneill #define MCX_CAP_DEVICE_TPH 0x00000080
928 1.1 jmcneill #define MCX_CAP_DEVICE_RF 0x00000040
929 1.1 jmcneill #define MCX_CAP_DEVICE_DCT 0x00000020
930 1.1 jmcneill #define MCX_CAP_DEVICE_QOS 0x00000010
931 1.1 jmcneill #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008
932 1.1 jmcneill #define MCX_CAP_DEVICE_ROCE 0x00000004
933 1.1 jmcneill #define MCX_CAP_DEVICE_ATOMIC 0x00000002
934 1.1 jmcneill
935 1.1 jmcneill uint32_t flags6;
936 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_OI 0x80000000
937 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000
938 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000
939 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \
940 1.1 jmcneill 0x10000000
941 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000
942 1.1 jmcneill #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000
943 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000
944 1.1 jmcneill #define MCX_CAP_DEVICE_PG 0x01000000
945 1.1 jmcneill #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000
946 1.1 jmcneill #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \
947 1.1 jmcneill 0x00400000
948 1.1 jmcneill #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \
949 1.1 jmcneill 0x00200000
950 1.1 jmcneill #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \
951 1.1 jmcneill 0x00100000
952 1.1 jmcneill #define MCX_CAP_DEVICE_CD 0x00080000
953 1.1 jmcneill #define MCX_CAP_DEVICE_ATM 0x00040000
954 1.1 jmcneill #define MCX_CAP_DEVICE_APM 0x00020000
955 1.1 jmcneill #define MCX_CAP_DEVICE_IMAICL 0x00010000
956 1.1 jmcneill #define MCX_CAP_DEVICE_QKV 0x00000200
957 1.1 jmcneill #define MCX_CAP_DEVICE_PKV 0x00000100
958 1.1 jmcneill #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080
959 1.1 jmcneill #define MCX_CAP_DEVICE_XRC 0x00000008
960 1.1 jmcneill #define MCX_CAP_DEVICE_UD 0x00000004
961 1.1 jmcneill #define MCX_CAP_DEVICE_UC 0x00000002
962 1.1 jmcneill #define MCX_CAP_DEVICE_RC 0x00000001
963 1.1 jmcneill
964 1.1 jmcneill uint8_t uar_flags;
965 1.1 jmcneill #define MCX_CAP_DEVICE_UAR_4K 0x80
966 1.1 jmcneill uint8_t uar_sz; /* 6 bits */
967 1.1 jmcneill #define MCX_CAP_DEVICE_UAR_SZ 0x3f
968 1.1 jmcneill uint8_t __reserved__[1];
969 1.1 jmcneill uint8_t log_pg_sz;
970 1.1 jmcneill
971 1.1 jmcneill uint8_t flags7;
972 1.1 jmcneill #define MCX_CAP_DEVICE_BF 0x80
973 1.1 jmcneill #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40
974 1.1 jmcneill #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \
975 1.1 jmcneill 0x20
976 1.1 jmcneill uint8_t log_bf_reg_size; /* 5 bits */
977 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f
978 1.1 jmcneill uint8_t __reserved__[2];
979 1.1 jmcneill
980 1.1 jmcneill uint16_t num_of_diagnostic_counters;
981 1.1 jmcneill uint16_t max_wqe_sz_sq;
982 1.1 jmcneill
983 1.1 jmcneill uint8_t __reserved__[2];
984 1.1 jmcneill uint16_t max_wqe_sz_rq;
985 1.1 jmcneill
986 1.1 jmcneill uint8_t __reserved__[2];
987 1.1 jmcneill uint16_t max_wqe_sz_sq_dc;
988 1.1 jmcneill
989 1.1 jmcneill uint32_t max_qp_mcg; /* 25 bits */
990 1.1 jmcneill #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff
991 1.1 jmcneill
992 1.1 jmcneill uint8_t __reserved__[3];
993 1.1 jmcneill uint8_t log_max_mcq;
994 1.1 jmcneill
995 1.1 jmcneill uint8_t log_max_transport_domain; /* 5 bits */
996 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \
997 1.1 jmcneill 0x1f
998 1.1 jmcneill uint8_t log_max_pd; /* 5 bits */
999 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f
1000 1.1 jmcneill uint8_t __reserved__[1];
1001 1.1 jmcneill uint8_t log_max_xrcd; /* 5 bits */
1002 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f
1003 1.1 jmcneill
1004 1.1 jmcneill uint8_t __reserved__[2];
1005 1.1 jmcneill uint16_t max_flow_counter;
1006 1.1 jmcneill
1007 1.1 jmcneill uint8_t log_max_rq; /* 5 bits */
1008 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f
1009 1.1 jmcneill uint8_t log_max_sq; /* 5 bits */
1010 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f
1011 1.1 jmcneill uint8_t log_max_tir; /* 5 bits */
1012 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f
1013 1.1 jmcneill uint8_t log_max_tis; /* 5 bits */
1014 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f
1015 1.1 jmcneill
1016 1.1 jmcneill uint8_t flags8;
1017 1.1 jmcneill #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \
1018 1.1 jmcneill 0x80
1019 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f
1020 1.1 jmcneill uint8_t log_max_rqt; /* 5 bits */
1021 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f
1022 1.1 jmcneill uint8_t log_max_rqt_size; /* 5 bits */
1023 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f
1024 1.1 jmcneill uint8_t log_max_tis_per_sq; /* 5 bits */
1025 1.1 jmcneill #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \
1026 1.1 jmcneill 0x1f
1027 1.15 jmcneill
1028 1.15 jmcneill uint8_t flags9;
1029 1.15 jmcneill #define MXC_CAP_DEVICE_EXT_STRIDE_NUM_RANGES \
1030 1.15 jmcneill 0x80
1031 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_RQ \
1032 1.15 jmcneill 0x1f
1033 1.15 jmcneill uint8_t log_min_stride_sz_rq; /* 5 bits */
1034 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_RQ \
1035 1.15 jmcneill 0x1f
1036 1.15 jmcneill uint8_t log_max_stride_sz_sq; /* 5 bits */
1037 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_STRIDE_SZ_SQ \
1038 1.15 jmcneill 0x1f
1039 1.15 jmcneill uint8_t log_min_stride_sz_sq; /* 5 bits */
1040 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MIN_STRIDE_SZ_SQ \
1041 1.15 jmcneill 0x1f
1042 1.15 jmcneill
1043 1.15 jmcneill uint8_t log_max_hairpin_queues;
1044 1.15 jmcneill #define MXC_CAP_DEVICE_HAIRPIN 0x80
1045 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_QUEUES \
1046 1.15 jmcneill 0x1f
1047 1.15 jmcneill uint8_t log_min_hairpin_queues;
1048 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MIN_HAIRPIN_QUEUES \
1049 1.15 jmcneill 0x1f
1050 1.15 jmcneill uint8_t log_max_hairpin_num_packets;
1051 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_HAIRPIN_NUM_PACKETS \
1052 1.15 jmcneill 0x1f
1053 1.15 jmcneill uint8_t log_max_mq_sz;
1054 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_WQ_SZ \
1055 1.15 jmcneill 0x1f
1056 1.15 jmcneill
1057 1.15 jmcneill uint8_t log_min_hairpin_wq_data_sz;
1058 1.15 jmcneill #define MXC_CAP_DEVICE_NIC_VPORT_CHANGE_EVENT \
1059 1.15 jmcneill 0x80
1060 1.15 jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_UC \
1061 1.15 jmcneill 0x40
1062 1.15 jmcneill #define MXC_CAP_DEVICE_DISABLE_LOCAL_LB_MC \
1063 1.15 jmcneill 0x20
1064 1.15 jmcneill #define MCX_CAP_DEVICE_LOG_MIN_HAIRPIN_WQ_DATA_SZ \
1065 1.15 jmcneill 0x1f
1066 1.15 jmcneill uint8_t log_max_vlan_list;
1067 1.15 jmcneill #define MXC_CAP_DEVICE_SYSTEM_IMAGE_GUID_MODIFIABLE \
1068 1.15 jmcneill 0x80
1069 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_VLAN_LIST \
1070 1.15 jmcneill 0x1f
1071 1.15 jmcneill uint8_t log_max_current_mc_list;
1072 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_MC_LIST \
1073 1.15 jmcneill 0x1f
1074 1.15 jmcneill uint8_t log_max_current_uc_list;
1075 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_CURRENT_UC_LIST \
1076 1.15 jmcneill 0x1f
1077 1.15 jmcneill
1078 1.15 jmcneill uint8_t __reserved__[4];
1079 1.15 jmcneill
1080 1.15 jmcneill uint32_t create_qp_start_hint; /* 24 bits */
1081 1.15 jmcneill
1082 1.15 jmcneill uint8_t log_max_uctx; /* 5 bits */
1083 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UCTX 0x1f
1084 1.15 jmcneill uint8_t log_max_umem; /* 5 bits */
1085 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_UMEM 0x1f
1086 1.15 jmcneill uint16_t max_num_eqs;
1087 1.15 jmcneill
1088 1.15 jmcneill uint8_t log_max_l2_table; /* 5 bits */
1089 1.15 jmcneill #define MXC_CAP_DEVICE_LOG_MAX_L2_TABLE 0x1f
1090 1.15 jmcneill uint8_t __reserved__[1];
1091 1.15 jmcneill uint16_t log_uar_page_sz;
1092 1.15 jmcneill
1093 1.15 jmcneill uint8_t __reserved__[8];
1094 1.15 jmcneill
1095 1.15 jmcneill uint32_t device_frequency_mhz;
1096 1.15 jmcneill uint32_t device_frequency_khz;
1097 1.1 jmcneill } __packed __aligned(8);
1098 1.1 jmcneill
1099 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20);
1100 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c);
1101 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30);
1102 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38);
1103 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40);
1104 1.1 jmcneill CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c);
1105 1.15 jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_mhz) == 0x98);
1106 1.15 jmcneill CTASSERT(offsetof(struct mcx_cap_device, device_frequency_khz) == 0x9c);
1107 1.1 jmcneill CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE);
1108 1.1 jmcneill
1109 1.1 jmcneill struct mcx_cmd_set_driver_version_in {
1110 1.1 jmcneill uint16_t cmd_opcode;
1111 1.1 jmcneill uint8_t cmd_reserved0[4];
1112 1.1 jmcneill uint16_t cmd_op_mod;
1113 1.1 jmcneill uint8_t cmd_reserved1[8];
1114 1.1 jmcneill } __packed __aligned(4);
1115 1.1 jmcneill
1116 1.1 jmcneill struct mcx_cmd_set_driver_version_out {
1117 1.1 jmcneill uint8_t cmd_status;
1118 1.1 jmcneill uint8_t cmd_reserved0[3];
1119 1.1 jmcneill uint32_t cmd_syndrome;
1120 1.1 jmcneill uint8_t cmd_reserved1[8];
1121 1.1 jmcneill } __packed __aligned(4);
1122 1.1 jmcneill
1123 1.1 jmcneill struct mcx_cmd_set_driver_version {
1124 1.1 jmcneill uint8_t cmd_driver_version[64];
1125 1.1 jmcneill } __packed __aligned(8);
1126 1.1 jmcneill
1127 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_in {
1128 1.1 jmcneill uint16_t cmd_opcode;
1129 1.1 jmcneill uint8_t cmd_reserved0[4];
1130 1.1 jmcneill uint16_t cmd_op_mod;
1131 1.1 jmcneill uint8_t cmd_reserved1[4];
1132 1.1 jmcneill uint32_t cmd_field_select;
1133 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04
1134 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10
1135 1.1 jmcneill #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40
1136 1.1 jmcneill } __packed __aligned(4);
1137 1.1 jmcneill
1138 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_out {
1139 1.1 jmcneill uint8_t cmd_status;
1140 1.1 jmcneill uint8_t cmd_reserved0[3];
1141 1.1 jmcneill uint32_t cmd_syndrome;
1142 1.1 jmcneill uint8_t cmd_reserved1[8];
1143 1.1 jmcneill } __packed __aligned(4);
1144 1.1 jmcneill
1145 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_in {
1146 1.1 jmcneill uint16_t cmd_opcode;
1147 1.1 jmcneill uint8_t cmd_reserved0[4];
1148 1.1 jmcneill uint16_t cmd_op_mod;
1149 1.1 jmcneill uint8_t cmd_reserved1[4];
1150 1.1 jmcneill uint8_t cmd_allowed_list_type;
1151 1.1 jmcneill uint8_t cmd_reserved2[3];
1152 1.1 jmcneill } __packed __aligned(4);
1153 1.1 jmcneill
1154 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_out {
1155 1.1 jmcneill uint8_t cmd_status;
1156 1.1 jmcneill uint8_t cmd_reserved0[3];
1157 1.1 jmcneill uint32_t cmd_syndrome;
1158 1.1 jmcneill uint8_t cmd_reserved1[8];
1159 1.1 jmcneill } __packed __aligned(4);
1160 1.1 jmcneill
1161 1.1 jmcneill struct mcx_nic_vport_ctx {
1162 1.1 jmcneill uint32_t vp_min_wqe_inline_mode;
1163 1.1 jmcneill uint8_t vp_reserved0[32];
1164 1.1 jmcneill uint32_t vp_mtu;
1165 1.1 jmcneill uint8_t vp_reserved1[200];
1166 1.1 jmcneill uint16_t vp_flags;
1167 1.1 jmcneill #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0)
1168 1.1 jmcneill #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24)
1169 1.1 jmcneill #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24)
1170 1.1 jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13)
1171 1.1 jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14)
1172 1.1 jmcneill #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15)
1173 1.1 jmcneill uint16_t vp_allowed_list_size;
1174 1.1 jmcneill uint64_t vp_perm_addr;
1175 1.1 jmcneill uint8_t vp_reserved2[4];
1176 1.1 jmcneill /* allowed list follows */
1177 1.1 jmcneill } __packed __aligned(4);
1178 1.1 jmcneill
1179 1.1 jmcneill struct mcx_counter {
1180 1.1 jmcneill uint64_t packets;
1181 1.1 jmcneill uint64_t octets;
1182 1.1 jmcneill } __packed __aligned(4);
1183 1.1 jmcneill
1184 1.1 jmcneill struct mcx_nic_vport_counters {
1185 1.1 jmcneill struct mcx_counter rx_err;
1186 1.1 jmcneill struct mcx_counter tx_err;
1187 1.1 jmcneill uint8_t reserved0[64]; /* 0x30 */
1188 1.1 jmcneill struct mcx_counter rx_bcast;
1189 1.1 jmcneill struct mcx_counter tx_bcast;
1190 1.1 jmcneill struct mcx_counter rx_ucast;
1191 1.1 jmcneill struct mcx_counter tx_ucast;
1192 1.1 jmcneill struct mcx_counter rx_mcast;
1193 1.1 jmcneill struct mcx_counter tx_mcast;
1194 1.1 jmcneill uint8_t reserved1[0x210 - 0xd0];
1195 1.1 jmcneill } __packed __aligned(4);
1196 1.1 jmcneill
1197 1.1 jmcneill struct mcx_cmd_query_vport_counters_in {
1198 1.1 jmcneill uint16_t cmd_opcode;
1199 1.1 jmcneill uint8_t cmd_reserved0[4];
1200 1.1 jmcneill uint16_t cmd_op_mod;
1201 1.1 jmcneill uint8_t cmd_reserved1[8];
1202 1.1 jmcneill } __packed __aligned(4);
1203 1.1 jmcneill
1204 1.1 jmcneill struct mcx_cmd_query_vport_counters_mb_in {
1205 1.1 jmcneill uint8_t cmd_reserved0[8];
1206 1.1 jmcneill uint8_t cmd_clear;
1207 1.1 jmcneill uint8_t cmd_reserved1[7];
1208 1.1 jmcneill } __packed __aligned(4);
1209 1.1 jmcneill
1210 1.1 jmcneill struct mcx_cmd_query_vport_counters_out {
1211 1.1 jmcneill uint8_t cmd_status;
1212 1.1 jmcneill uint8_t cmd_reserved0[3];
1213 1.1 jmcneill uint32_t cmd_syndrome;
1214 1.1 jmcneill uint8_t cmd_reserved1[8];
1215 1.1 jmcneill } __packed __aligned(4);
1216 1.1 jmcneill
1217 1.1 jmcneill struct mcx_cmd_query_flow_counter_in {
1218 1.1 jmcneill uint16_t cmd_opcode;
1219 1.1 jmcneill uint8_t cmd_reserved0[4];
1220 1.1 jmcneill uint16_t cmd_op_mod;
1221 1.1 jmcneill uint8_t cmd_reserved1[8];
1222 1.1 jmcneill } __packed __aligned(4);
1223 1.1 jmcneill
1224 1.1 jmcneill struct mcx_cmd_query_flow_counter_mb_in {
1225 1.1 jmcneill uint8_t cmd_reserved0[8];
1226 1.1 jmcneill uint8_t cmd_clear;
1227 1.1 jmcneill uint8_t cmd_reserved1[5];
1228 1.1 jmcneill uint16_t cmd_flow_counter_id;
1229 1.1 jmcneill } __packed __aligned(4);
1230 1.1 jmcneill
1231 1.1 jmcneill struct mcx_cmd_query_flow_counter_out {
1232 1.1 jmcneill uint8_t cmd_status;
1233 1.1 jmcneill uint8_t cmd_reserved0[3];
1234 1.1 jmcneill uint32_t cmd_syndrome;
1235 1.1 jmcneill uint8_t cmd_reserved1[8];
1236 1.1 jmcneill } __packed __aligned(4);
1237 1.1 jmcneill
1238 1.1 jmcneill struct mcx_cmd_alloc_uar_in {
1239 1.1 jmcneill uint16_t cmd_opcode;
1240 1.1 jmcneill uint8_t cmd_reserved0[4];
1241 1.1 jmcneill uint16_t cmd_op_mod;
1242 1.1 jmcneill uint8_t cmd_reserved1[8];
1243 1.1 jmcneill } __packed __aligned(4);
1244 1.1 jmcneill
1245 1.1 jmcneill struct mcx_cmd_alloc_uar_out {
1246 1.1 jmcneill uint8_t cmd_status;
1247 1.1 jmcneill uint8_t cmd_reserved0[3];
1248 1.1 jmcneill uint32_t cmd_syndrome;
1249 1.1 jmcneill uint32_t cmd_uar;
1250 1.1 jmcneill uint8_t cmd_reserved1[4];
1251 1.1 jmcneill } __packed __aligned(4);
1252 1.1 jmcneill
1253 1.1 jmcneill struct mcx_cmd_query_special_ctx_in {
1254 1.1 jmcneill uint16_t cmd_opcode;
1255 1.1 jmcneill uint8_t cmd_reserved0[4];
1256 1.1 jmcneill uint16_t cmd_op_mod;
1257 1.1 jmcneill uint8_t cmd_reserved1[8];
1258 1.1 jmcneill } __packed __aligned(4);
1259 1.1 jmcneill
1260 1.1 jmcneill struct mcx_cmd_query_special_ctx_out {
1261 1.1 jmcneill uint8_t cmd_status;
1262 1.1 jmcneill uint8_t cmd_reserved0[3];
1263 1.1 jmcneill uint32_t cmd_syndrome;
1264 1.1 jmcneill uint8_t cmd_reserved1[4];
1265 1.1 jmcneill uint32_t cmd_resd_lkey;
1266 1.1 jmcneill } __packed __aligned(4);
1267 1.1 jmcneill
1268 1.1 jmcneill struct mcx_eq_ctx {
1269 1.1 jmcneill uint32_t eq_status;
1270 1.15 jmcneill #define MCX_EQ_CTX_STATE_SHIFT 8
1271 1.15 jmcneill #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT)
1272 1.15 jmcneill #define MCX_EQ_CTX_STATE_ARMED 0x9
1273 1.15 jmcneill #define MCX_EQ_CTX_STATE_FIRED 0xa
1274 1.1 jmcneill #define MCX_EQ_CTX_OI_SHIFT 17
1275 1.1 jmcneill #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT)
1276 1.1 jmcneill #define MCX_EQ_CTX_EC_SHIFT 18
1277 1.1 jmcneill #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT)
1278 1.1 jmcneill #define MCX_EQ_CTX_STATUS_SHIFT 28
1279 1.1 jmcneill #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT)
1280 1.15 jmcneill #define MCX_EQ_CTX_STATUS_OK 0x0
1281 1.15 jmcneill #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa
1282 1.1 jmcneill uint32_t eq_reserved1;
1283 1.1 jmcneill uint32_t eq_page_offset;
1284 1.1 jmcneill #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5
1285 1.1 jmcneill uint32_t eq_uar_size;
1286 1.1 jmcneill #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff
1287 1.1 jmcneill #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24
1288 1.1 jmcneill uint32_t eq_reserved2;
1289 1.1 jmcneill uint8_t eq_reserved3[3];
1290 1.1 jmcneill uint8_t eq_intr;
1291 1.1 jmcneill uint32_t eq_log_page_size;
1292 1.1 jmcneill #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24
1293 1.1 jmcneill uint32_t eq_reserved4[3];
1294 1.1 jmcneill uint32_t eq_consumer_counter;
1295 1.1 jmcneill uint32_t eq_producer_counter;
1296 1.1 jmcneill #define MCX_EQ_CTX_COUNTER_MASK 0xffffff
1297 1.1 jmcneill uint32_t eq_reserved5[4];
1298 1.1 jmcneill } __packed __aligned(4);
1299 1.1 jmcneill
1300 1.1 jmcneill CTASSERT(sizeof(struct mcx_eq_ctx) == 64);
1301 1.1 jmcneill
1302 1.1 jmcneill struct mcx_cmd_create_eq_in {
1303 1.1 jmcneill uint16_t cmd_opcode;
1304 1.1 jmcneill uint8_t cmd_reserved0[4];
1305 1.1 jmcneill uint16_t cmd_op_mod;
1306 1.1 jmcneill uint8_t cmd_reserved1[8];
1307 1.1 jmcneill } __packed __aligned(4);
1308 1.1 jmcneill
1309 1.1 jmcneill struct mcx_cmd_create_eq_mb_in {
1310 1.1 jmcneill struct mcx_eq_ctx cmd_eq_ctx;
1311 1.1 jmcneill uint8_t cmd_reserved0[8];
1312 1.1 jmcneill uint64_t cmd_event_bitmask;
1313 1.1 jmcneill #define MCX_EVENT_TYPE_COMPLETION 0x00
1314 1.1 jmcneill #define MCX_EVENT_TYPE_CQ_ERROR 0x04
1315 1.1 jmcneill #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08
1316 1.1 jmcneill #define MCX_EVENT_TYPE_PORT_CHANGE 0x09
1317 1.1 jmcneill #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a
1318 1.1 jmcneill #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b
1319 1.1 jmcneill #define MCX_EVENT_TYPE_LAST_WQE 0x13
1320 1.1 jmcneill uint8_t cmd_reserved1[176];
1321 1.1 jmcneill } __packed __aligned(4);
1322 1.1 jmcneill
1323 1.1 jmcneill struct mcx_cmd_create_eq_out {
1324 1.1 jmcneill uint8_t cmd_status;
1325 1.1 jmcneill uint8_t cmd_reserved0[3];
1326 1.1 jmcneill uint32_t cmd_syndrome;
1327 1.1 jmcneill uint32_t cmd_eqn;
1328 1.1 jmcneill uint8_t cmd_reserved1[4];
1329 1.1 jmcneill } __packed __aligned(4);
1330 1.1 jmcneill
1331 1.15 jmcneill struct mcx_cmd_query_eq_in {
1332 1.15 jmcneill uint16_t cmd_opcode;
1333 1.15 jmcneill uint8_t cmd_reserved0[4];
1334 1.15 jmcneill uint16_t cmd_op_mod;
1335 1.15 jmcneill uint32_t cmd_eqn;
1336 1.15 jmcneill uint8_t cmd_reserved1[4];
1337 1.15 jmcneill } __packed __aligned(4);
1338 1.15 jmcneill
1339 1.15 jmcneill struct mcx_cmd_query_eq_out {
1340 1.15 jmcneill uint8_t cmd_status;
1341 1.15 jmcneill uint8_t cmd_reserved0[3];
1342 1.15 jmcneill uint32_t cmd_syndrome;
1343 1.15 jmcneill uint8_t cmd_reserved1[8];
1344 1.15 jmcneill } __packed __aligned(4);
1345 1.15 jmcneill
1346 1.1 jmcneill struct mcx_eq_entry {
1347 1.1 jmcneill uint8_t eq_reserved1;
1348 1.1 jmcneill uint8_t eq_event_type;
1349 1.1 jmcneill uint8_t eq_reserved2;
1350 1.1 jmcneill uint8_t eq_event_sub_type;
1351 1.1 jmcneill
1352 1.1 jmcneill uint8_t eq_reserved3[28];
1353 1.1 jmcneill uint32_t eq_event_data[7];
1354 1.1 jmcneill uint8_t eq_reserved4[2];
1355 1.1 jmcneill uint8_t eq_signature;
1356 1.1 jmcneill uint8_t eq_owner;
1357 1.1 jmcneill #define MCX_EQ_ENTRY_OWNER_INIT 1
1358 1.1 jmcneill } __packed __aligned(4);
1359 1.1 jmcneill
1360 1.1 jmcneill CTASSERT(sizeof(struct mcx_eq_entry) == 64);
1361 1.1 jmcneill
1362 1.1 jmcneill struct mcx_cmd_alloc_pd_in {
1363 1.1 jmcneill uint16_t cmd_opcode;
1364 1.1 jmcneill uint8_t cmd_reserved0[4];
1365 1.1 jmcneill uint16_t cmd_op_mod;
1366 1.1 jmcneill uint8_t cmd_reserved1[8];
1367 1.1 jmcneill } __packed __aligned(4);
1368 1.1 jmcneill
1369 1.1 jmcneill struct mcx_cmd_alloc_pd_out {
1370 1.1 jmcneill uint8_t cmd_status;
1371 1.1 jmcneill uint8_t cmd_reserved0[3];
1372 1.1 jmcneill uint32_t cmd_syndrome;
1373 1.1 jmcneill uint32_t cmd_pd;
1374 1.1 jmcneill uint8_t cmd_reserved1[4];
1375 1.1 jmcneill } __packed __aligned(4);
1376 1.1 jmcneill
1377 1.1 jmcneill struct mcx_cmd_alloc_td_in {
1378 1.1 jmcneill uint16_t cmd_opcode;
1379 1.1 jmcneill uint8_t cmd_reserved0[4];
1380 1.1 jmcneill uint16_t cmd_op_mod;
1381 1.1 jmcneill uint8_t cmd_reserved1[8];
1382 1.1 jmcneill } __packed __aligned(4);
1383 1.1 jmcneill
1384 1.1 jmcneill struct mcx_cmd_alloc_td_out {
1385 1.1 jmcneill uint8_t cmd_status;
1386 1.1 jmcneill uint8_t cmd_reserved0[3];
1387 1.1 jmcneill uint32_t cmd_syndrome;
1388 1.1 jmcneill uint32_t cmd_tdomain;
1389 1.1 jmcneill uint8_t cmd_reserved1[4];
1390 1.1 jmcneill } __packed __aligned(4);
1391 1.1 jmcneill
1392 1.1 jmcneill struct mcx_cmd_create_tir_in {
1393 1.1 jmcneill uint16_t cmd_opcode;
1394 1.1 jmcneill uint8_t cmd_reserved0[4];
1395 1.1 jmcneill uint16_t cmd_op_mod;
1396 1.1 jmcneill uint8_t cmd_reserved1[8];
1397 1.1 jmcneill } __packed __aligned(4);
1398 1.1 jmcneill
1399 1.1 jmcneill struct mcx_cmd_create_tir_mb_in {
1400 1.1 jmcneill uint8_t cmd_reserved0[20];
1401 1.1 jmcneill uint32_t cmd_disp_type;
1402 1.15 jmcneill #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0
1403 1.15 jmcneill #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1
1404 1.1 jmcneill #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28
1405 1.1 jmcneill uint8_t cmd_reserved1[8];
1406 1.1 jmcneill uint32_t cmd_lro;
1407 1.1 jmcneill uint8_t cmd_reserved2[8];
1408 1.1 jmcneill uint32_t cmd_inline_rqn;
1409 1.1 jmcneill uint32_t cmd_indir_table;
1410 1.1 jmcneill uint32_t cmd_tdomain;
1411 1.15 jmcneill #define MCX_TIR_CTX_HASH_TOEPLITZ 2
1412 1.15 jmcneill #define MCX_TIR_CTX_HASH_SHIFT 28
1413 1.1 jmcneill uint8_t cmd_rx_hash_key[40];
1414 1.1 jmcneill uint32_t cmd_rx_hash_sel_outer;
1415 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0)
1416 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1)
1417 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2)
1418 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3)
1419 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31)
1420 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31)
1421 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30)
1422 1.15 jmcneill #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30)
1423 1.1 jmcneill uint32_t cmd_rx_hash_sel_inner;
1424 1.1 jmcneill uint8_t cmd_reserved3[152];
1425 1.1 jmcneill } __packed __aligned(4);
1426 1.1 jmcneill
1427 1.1 jmcneill struct mcx_cmd_create_tir_out {
1428 1.1 jmcneill uint8_t cmd_status;
1429 1.1 jmcneill uint8_t cmd_reserved0[3];
1430 1.1 jmcneill uint32_t cmd_syndrome;
1431 1.1 jmcneill uint32_t cmd_tirn;
1432 1.1 jmcneill uint8_t cmd_reserved1[4];
1433 1.1 jmcneill } __packed __aligned(4);
1434 1.1 jmcneill
1435 1.1 jmcneill struct mcx_cmd_destroy_tir_in {
1436 1.1 jmcneill uint16_t cmd_opcode;
1437 1.1 jmcneill uint8_t cmd_reserved0[4];
1438 1.1 jmcneill uint16_t cmd_op_mod;
1439 1.1 jmcneill uint32_t cmd_tirn;
1440 1.1 jmcneill uint8_t cmd_reserved1[4];
1441 1.1 jmcneill } __packed __aligned(4);
1442 1.1 jmcneill
1443 1.1 jmcneill struct mcx_cmd_destroy_tir_out {
1444 1.1 jmcneill uint8_t cmd_status;
1445 1.1 jmcneill uint8_t cmd_reserved0[3];
1446 1.1 jmcneill uint32_t cmd_syndrome;
1447 1.1 jmcneill uint8_t cmd_reserved1[8];
1448 1.1 jmcneill } __packed __aligned(4);
1449 1.1 jmcneill
1450 1.1 jmcneill struct mcx_cmd_create_tis_in {
1451 1.1 jmcneill uint16_t cmd_opcode;
1452 1.1 jmcneill uint8_t cmd_reserved0[4];
1453 1.1 jmcneill uint16_t cmd_op_mod;
1454 1.1 jmcneill uint8_t cmd_reserved1[8];
1455 1.1 jmcneill } __packed __aligned(4);
1456 1.1 jmcneill
1457 1.1 jmcneill struct mcx_cmd_create_tis_mb_in {
1458 1.1 jmcneill uint8_t cmd_reserved[16];
1459 1.1 jmcneill uint32_t cmd_prio;
1460 1.1 jmcneill uint8_t cmd_reserved1[32];
1461 1.1 jmcneill uint32_t cmd_tdomain;
1462 1.1 jmcneill uint8_t cmd_reserved2[120];
1463 1.1 jmcneill } __packed __aligned(4);
1464 1.1 jmcneill
1465 1.1 jmcneill struct mcx_cmd_create_tis_out {
1466 1.1 jmcneill uint8_t cmd_status;
1467 1.1 jmcneill uint8_t cmd_reserved0[3];
1468 1.1 jmcneill uint32_t cmd_syndrome;
1469 1.1 jmcneill uint32_t cmd_tisn;
1470 1.1 jmcneill uint8_t cmd_reserved1[4];
1471 1.1 jmcneill } __packed __aligned(4);
1472 1.1 jmcneill
1473 1.1 jmcneill struct mcx_cmd_destroy_tis_in {
1474 1.1 jmcneill uint16_t cmd_opcode;
1475 1.1 jmcneill uint8_t cmd_reserved0[4];
1476 1.1 jmcneill uint16_t cmd_op_mod;
1477 1.1 jmcneill uint32_t cmd_tisn;
1478 1.1 jmcneill uint8_t cmd_reserved1[4];
1479 1.1 jmcneill } __packed __aligned(4);
1480 1.1 jmcneill
1481 1.1 jmcneill struct mcx_cmd_destroy_tis_out {
1482 1.1 jmcneill uint8_t cmd_status;
1483 1.1 jmcneill uint8_t cmd_reserved0[3];
1484 1.1 jmcneill uint32_t cmd_syndrome;
1485 1.1 jmcneill uint8_t cmd_reserved1[8];
1486 1.1 jmcneill } __packed __aligned(4);
1487 1.1 jmcneill
1488 1.15 jmcneill struct mcx_cmd_create_rqt_in {
1489 1.15 jmcneill uint16_t cmd_opcode;
1490 1.15 jmcneill uint8_t cmd_reserved0[4];
1491 1.15 jmcneill uint16_t cmd_op_mod;
1492 1.15 jmcneill uint8_t cmd_reserved1[8];
1493 1.15 jmcneill } __packed __aligned(4);
1494 1.15 jmcneill
1495 1.15 jmcneill struct mcx_rqt_ctx {
1496 1.15 jmcneill uint8_t cmd_reserved0[20];
1497 1.15 jmcneill uint16_t cmd_reserved1;
1498 1.15 jmcneill uint16_t cmd_rqt_max_size;
1499 1.15 jmcneill uint16_t cmd_reserved2;
1500 1.15 jmcneill uint16_t cmd_rqt_actual_size;
1501 1.15 jmcneill uint8_t cmd_reserved3[212];
1502 1.15 jmcneill } __packed __aligned(4);
1503 1.15 jmcneill
1504 1.15 jmcneill struct mcx_cmd_create_rqt_mb_in {
1505 1.15 jmcneill uint8_t cmd_reserved0[16];
1506 1.15 jmcneill struct mcx_rqt_ctx cmd_rqt;
1507 1.15 jmcneill } __packed __aligned(4);
1508 1.15 jmcneill
1509 1.15 jmcneill struct mcx_cmd_create_rqt_out {
1510 1.15 jmcneill uint8_t cmd_status;
1511 1.15 jmcneill uint8_t cmd_reserved0[3];
1512 1.15 jmcneill uint32_t cmd_syndrome;
1513 1.15 jmcneill uint32_t cmd_rqtn;
1514 1.15 jmcneill uint8_t cmd_reserved1[4];
1515 1.15 jmcneill } __packed __aligned(4);
1516 1.15 jmcneill
1517 1.15 jmcneill struct mcx_cmd_destroy_rqt_in {
1518 1.15 jmcneill uint16_t cmd_opcode;
1519 1.15 jmcneill uint8_t cmd_reserved0[4];
1520 1.15 jmcneill uint16_t cmd_op_mod;
1521 1.15 jmcneill uint32_t cmd_rqtn;
1522 1.15 jmcneill uint8_t cmd_reserved1[4];
1523 1.15 jmcneill } __packed __aligned(4);
1524 1.15 jmcneill
1525 1.15 jmcneill struct mcx_cmd_destroy_rqt_out {
1526 1.15 jmcneill uint8_t cmd_status;
1527 1.15 jmcneill uint8_t cmd_reserved0[3];
1528 1.15 jmcneill uint32_t cmd_syndrome;
1529 1.15 jmcneill uint8_t cmd_reserved1[8];
1530 1.15 jmcneill } __packed __aligned(4);
1531 1.15 jmcneill
1532 1.1 jmcneill struct mcx_cq_ctx {
1533 1.1 jmcneill uint32_t cq_status;
1534 1.15 jmcneill #define MCX_CQ_CTX_STATUS_SHIFT 28
1535 1.15 jmcneill #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT)
1536 1.15 jmcneill #define MCX_CQ_CTX_STATUS_OK 0x0
1537 1.15 jmcneill #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9
1538 1.15 jmcneill #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa
1539 1.15 jmcneill #define MCX_CQ_CTX_STATE_SHIFT 8
1540 1.15 jmcneill #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT)
1541 1.15 jmcneill #define MCX_CQ_CTX_STATE_SOLICITED 0x6
1542 1.15 jmcneill #define MCX_CQ_CTX_STATE_ARMED 0x9
1543 1.15 jmcneill #define MCX_CQ_CTX_STATE_FIRED 0xa
1544 1.1 jmcneill uint32_t cq_reserved1;
1545 1.1 jmcneill uint32_t cq_page_offset;
1546 1.1 jmcneill uint32_t cq_uar_size;
1547 1.1 jmcneill #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff
1548 1.1 jmcneill #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24
1549 1.1 jmcneill uint32_t cq_period_max_count;
1550 1.1 jmcneill #define MCX_CQ_CTX_PERIOD_SHIFT 16
1551 1.1 jmcneill uint32_t cq_eqn;
1552 1.1 jmcneill uint32_t cq_log_page_size;
1553 1.1 jmcneill #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24
1554 1.1 jmcneill uint32_t cq_reserved2;
1555 1.1 jmcneill uint32_t cq_last_notified;
1556 1.1 jmcneill uint32_t cq_last_solicit;
1557 1.1 jmcneill uint32_t cq_consumer_counter;
1558 1.1 jmcneill uint32_t cq_producer_counter;
1559 1.1 jmcneill uint8_t cq_reserved3[8];
1560 1.1 jmcneill uint64_t cq_doorbell;
1561 1.1 jmcneill } __packed __aligned(4);
1562 1.1 jmcneill
1563 1.1 jmcneill CTASSERT(sizeof(struct mcx_cq_ctx) == 64);
1564 1.1 jmcneill
1565 1.1 jmcneill struct mcx_cmd_create_cq_in {
1566 1.1 jmcneill uint16_t cmd_opcode;
1567 1.1 jmcneill uint8_t cmd_reserved0[4];
1568 1.1 jmcneill uint16_t cmd_op_mod;
1569 1.1 jmcneill uint8_t cmd_reserved1[8];
1570 1.1 jmcneill } __packed __aligned(4);
1571 1.1 jmcneill
1572 1.1 jmcneill struct mcx_cmd_create_cq_mb_in {
1573 1.1 jmcneill struct mcx_cq_ctx cmd_cq_ctx;
1574 1.1 jmcneill uint8_t cmd_reserved1[192];
1575 1.1 jmcneill } __packed __aligned(4);
1576 1.1 jmcneill
1577 1.1 jmcneill struct mcx_cmd_create_cq_out {
1578 1.1 jmcneill uint8_t cmd_status;
1579 1.1 jmcneill uint8_t cmd_reserved0[3];
1580 1.1 jmcneill uint32_t cmd_syndrome;
1581 1.1 jmcneill uint32_t cmd_cqn;
1582 1.1 jmcneill uint8_t cmd_reserved1[4];
1583 1.1 jmcneill } __packed __aligned(4);
1584 1.1 jmcneill
1585 1.1 jmcneill struct mcx_cmd_destroy_cq_in {
1586 1.1 jmcneill uint16_t cmd_opcode;
1587 1.1 jmcneill uint8_t cmd_reserved0[4];
1588 1.1 jmcneill uint16_t cmd_op_mod;
1589 1.1 jmcneill uint32_t cmd_cqn;
1590 1.1 jmcneill uint8_t cmd_reserved1[4];
1591 1.1 jmcneill } __packed __aligned(4);
1592 1.1 jmcneill
1593 1.1 jmcneill struct mcx_cmd_destroy_cq_out {
1594 1.1 jmcneill uint8_t cmd_status;
1595 1.1 jmcneill uint8_t cmd_reserved0[3];
1596 1.1 jmcneill uint32_t cmd_syndrome;
1597 1.1 jmcneill uint8_t cmd_reserved1[8];
1598 1.1 jmcneill } __packed __aligned(4);
1599 1.1 jmcneill
1600 1.15 jmcneill struct mcx_cmd_query_cq_in {
1601 1.15 jmcneill uint16_t cmd_opcode;
1602 1.15 jmcneill uint8_t cmd_reserved0[4];
1603 1.15 jmcneill uint16_t cmd_op_mod;
1604 1.15 jmcneill uint32_t cmd_cqn;
1605 1.15 jmcneill uint8_t cmd_reserved1[4];
1606 1.15 jmcneill } __packed __aligned(4);
1607 1.15 jmcneill
1608 1.15 jmcneill struct mcx_cmd_query_cq_out {
1609 1.15 jmcneill uint8_t cmd_status;
1610 1.15 jmcneill uint8_t cmd_reserved0[3];
1611 1.15 jmcneill uint32_t cmd_syndrome;
1612 1.15 jmcneill uint8_t cmd_reserved1[8];
1613 1.15 jmcneill } __packed __aligned(4);
1614 1.15 jmcneill
1615 1.1 jmcneill struct mcx_cq_entry {
1616 1.1 jmcneill uint32_t __reserved__;
1617 1.1 jmcneill uint32_t cq_lro;
1618 1.1 jmcneill uint32_t cq_lro_ack_seq_num;
1619 1.1 jmcneill uint32_t cq_rx_hash;
1620 1.1 jmcneill uint8_t cq_rx_hash_type;
1621 1.1 jmcneill uint8_t cq_ml_path;
1622 1.1 jmcneill uint16_t __reserved__;
1623 1.1 jmcneill uint32_t cq_checksum;
1624 1.1 jmcneill uint32_t __reserved__;
1625 1.1 jmcneill uint32_t cq_flags;
1626 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26)
1627 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25)
1628 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24)
1629 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16)
1630 1.15 jmcneill #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff)
1631 1.15 jmcneill
1632 1.1 jmcneill uint32_t cq_lro_srqn;
1633 1.1 jmcneill uint32_t __reserved__[2];
1634 1.1 jmcneill uint32_t cq_byte_cnt;
1635 1.1 jmcneill uint64_t cq_timestamp;
1636 1.1 jmcneill uint8_t cq_rx_drops;
1637 1.1 jmcneill uint8_t cq_flow_tag[3];
1638 1.1 jmcneill uint16_t cq_wqe_count;
1639 1.1 jmcneill uint8_t cq_signature;
1640 1.1 jmcneill uint8_t cq_opcode_owner;
1641 1.1 jmcneill #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0)
1642 1.1 jmcneill #define MCX_CQ_ENTRY_FLAG_SE (1 << 1)
1643 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_SHIFT 2
1644 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_SHIFT 4
1645 1.1 jmcneill
1646 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0
1647 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1
1648 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2
1649 1.1 jmcneill #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3
1650 1.1 jmcneill
1651 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ 0
1652 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND 2
1653 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13
1654 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14
1655 1.1 jmcneill #define MCX_CQ_ENTRY_OPCODE_INVALID 15
1656 1.1 jmcneill
1657 1.1 jmcneill } __packed __aligned(4);
1658 1.1 jmcneill
1659 1.1 jmcneill CTASSERT(sizeof(struct mcx_cq_entry) == 64);
1660 1.1 jmcneill
1661 1.1 jmcneill struct mcx_cq_doorbell {
1662 1.1 jmcneill uint32_t db_update_ci;
1663 1.1 jmcneill uint32_t db_arm_ci;
1664 1.1 jmcneill #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28
1665 1.1 jmcneill #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24)
1666 1.1 jmcneill #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff)
1667 1.1 jmcneill } __packed __aligned(8);
1668 1.1 jmcneill
1669 1.1 jmcneill struct mcx_wq_ctx {
1670 1.1 jmcneill uint8_t wq_type;
1671 1.1 jmcneill #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4)
1672 1.1 jmcneill #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3)
1673 1.1 jmcneill uint8_t wq_reserved0[5];
1674 1.1 jmcneill uint16_t wq_lwm;
1675 1.1 jmcneill uint32_t wq_pd;
1676 1.1 jmcneill uint32_t wq_uar_page;
1677 1.1 jmcneill uint64_t wq_doorbell;
1678 1.1 jmcneill uint32_t wq_hw_counter;
1679 1.1 jmcneill uint32_t wq_sw_counter;
1680 1.1 jmcneill uint16_t wq_log_stride;
1681 1.1 jmcneill uint8_t wq_log_page_sz;
1682 1.1 jmcneill uint8_t wq_log_size;
1683 1.1 jmcneill uint8_t wq_reserved1[156];
1684 1.1 jmcneill } __packed __aligned(4);
1685 1.1 jmcneill
1686 1.1 jmcneill CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0);
1687 1.1 jmcneill
1688 1.1 jmcneill struct mcx_sq_ctx {
1689 1.1 jmcneill uint32_t sq_flags;
1690 1.5 msaitoh #define MCX_SQ_CTX_RLKEY (1U << 31)
1691 1.1 jmcneill #define MCX_SQ_CTX_FRE_SHIFT (1 << 29)
1692 1.1 jmcneill #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28)
1693 1.1 jmcneill #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24
1694 1.1 jmcneill #define MCX_SQ_CTX_STATE_SHIFT 20
1695 1.15 jmcneill #define MCX_SQ_CTX_STATE_MASK (0xf << 20)
1696 1.15 jmcneill #define MCX_SQ_CTX_STATE_RST 0
1697 1.15 jmcneill #define MCX_SQ_CTX_STATE_RDY 1
1698 1.15 jmcneill #define MCX_SQ_CTX_STATE_ERR 3
1699 1.1 jmcneill uint32_t sq_user_index;
1700 1.1 jmcneill uint32_t sq_cqn;
1701 1.1 jmcneill uint32_t sq_reserved1[5];
1702 1.1 jmcneill uint32_t sq_tis_lst_sz;
1703 1.1 jmcneill #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16
1704 1.1 jmcneill uint32_t sq_reserved2[2];
1705 1.1 jmcneill uint32_t sq_tis_num;
1706 1.1 jmcneill struct mcx_wq_ctx sq_wq;
1707 1.1 jmcneill } __packed __aligned(4);
1708 1.1 jmcneill
1709 1.1 jmcneill struct mcx_sq_entry_seg {
1710 1.1 jmcneill uint32_t sqs_byte_count;
1711 1.1 jmcneill uint32_t sqs_lkey;
1712 1.1 jmcneill uint64_t sqs_addr;
1713 1.1 jmcneill } __packed __aligned(4);
1714 1.1 jmcneill
1715 1.1 jmcneill struct mcx_sq_entry {
1716 1.1 jmcneill /* control segment */
1717 1.1 jmcneill uint32_t sqe_opcode_index;
1718 1.1 jmcneill #define MCX_SQE_WQE_INDEX_SHIFT 8
1719 1.1 jmcneill #define MCX_SQE_WQE_OPCODE_NOP 0x00
1720 1.1 jmcneill #define MCX_SQE_WQE_OPCODE_SEND 0x0a
1721 1.1 jmcneill uint32_t sqe_ds_sq_num;
1722 1.1 jmcneill #define MCX_SQE_SQ_NUM_SHIFT 8
1723 1.1 jmcneill uint32_t sqe_signature;
1724 1.1 jmcneill #define MCX_SQE_SIGNATURE_SHIFT 24
1725 1.1 jmcneill #define MCX_SQE_SOLICITED_EVENT 0x02
1726 1.1 jmcneill #define MCX_SQE_CE_CQE_ON_ERR 0x00
1727 1.1 jmcneill #define MCX_SQE_CE_CQE_FIRST_ERR 0x04
1728 1.1 jmcneill #define MCX_SQE_CE_CQE_ALWAYS 0x08
1729 1.1 jmcneill #define MCX_SQE_CE_CQE_SOLICIT 0x0C
1730 1.1 jmcneill #define MCX_SQE_FM_NO_FENCE 0x00
1731 1.1 jmcneill #define MCX_SQE_FM_SMALL_FENCE 0x40
1732 1.1 jmcneill uint32_t sqe_mkey;
1733 1.1 jmcneill
1734 1.1 jmcneill /* ethernet segment */
1735 1.1 jmcneill uint32_t sqe_reserved1;
1736 1.1 jmcneill uint32_t sqe_mss_csum;
1737 1.1 jmcneill #define MCX_SQE_L4_CSUM (1 << 31)
1738 1.1 jmcneill #define MCX_SQE_L3_CSUM (1 << 30)
1739 1.1 jmcneill uint32_t sqe_reserved2;
1740 1.1 jmcneill uint16_t sqe_inline_header_size;
1741 1.1 jmcneill uint16_t sqe_inline_headers[9];
1742 1.1 jmcneill
1743 1.1 jmcneill /* data segment */
1744 1.1 jmcneill struct mcx_sq_entry_seg sqe_segs[1];
1745 1.1 jmcneill } __packed __aligned(64);
1746 1.1 jmcneill
1747 1.1 jmcneill CTASSERT(sizeof(struct mcx_sq_entry) == 64);
1748 1.1 jmcneill
1749 1.1 jmcneill struct mcx_cmd_create_sq_in {
1750 1.1 jmcneill uint16_t cmd_opcode;
1751 1.1 jmcneill uint8_t cmd_reserved0[4];
1752 1.1 jmcneill uint16_t cmd_op_mod;
1753 1.1 jmcneill uint8_t cmd_reserved1[8];
1754 1.1 jmcneill } __packed __aligned(4);
1755 1.1 jmcneill
1756 1.1 jmcneill struct mcx_cmd_create_sq_out {
1757 1.1 jmcneill uint8_t cmd_status;
1758 1.1 jmcneill uint8_t cmd_reserved0[3];
1759 1.1 jmcneill uint32_t cmd_syndrome;
1760 1.1 jmcneill uint32_t cmd_sqn;
1761 1.1 jmcneill uint8_t cmd_reserved1[4];
1762 1.1 jmcneill } __packed __aligned(4);
1763 1.1 jmcneill
1764 1.1 jmcneill struct mcx_cmd_modify_sq_in {
1765 1.1 jmcneill uint16_t cmd_opcode;
1766 1.1 jmcneill uint8_t cmd_reserved0[4];
1767 1.1 jmcneill uint16_t cmd_op_mod;
1768 1.1 jmcneill uint32_t cmd_sq_state;
1769 1.1 jmcneill uint8_t cmd_reserved1[4];
1770 1.1 jmcneill } __packed __aligned(4);
1771 1.1 jmcneill
1772 1.1 jmcneill struct mcx_cmd_modify_sq_mb_in {
1773 1.1 jmcneill uint32_t cmd_modify_hi;
1774 1.1 jmcneill uint32_t cmd_modify_lo;
1775 1.1 jmcneill uint8_t cmd_reserved0[8];
1776 1.1 jmcneill struct mcx_sq_ctx cmd_sq_ctx;
1777 1.1 jmcneill } __packed __aligned(4);
1778 1.1 jmcneill
1779 1.1 jmcneill struct mcx_cmd_modify_sq_out {
1780 1.1 jmcneill uint8_t cmd_status;
1781 1.1 jmcneill uint8_t cmd_reserved0[3];
1782 1.1 jmcneill uint32_t cmd_syndrome;
1783 1.1 jmcneill uint8_t cmd_reserved1[8];
1784 1.1 jmcneill } __packed __aligned(4);
1785 1.1 jmcneill
1786 1.1 jmcneill struct mcx_cmd_destroy_sq_in {
1787 1.1 jmcneill uint16_t cmd_opcode;
1788 1.1 jmcneill uint8_t cmd_reserved0[4];
1789 1.1 jmcneill uint16_t cmd_op_mod;
1790 1.1 jmcneill uint32_t cmd_sqn;
1791 1.1 jmcneill uint8_t cmd_reserved1[4];
1792 1.1 jmcneill } __packed __aligned(4);
1793 1.1 jmcneill
1794 1.1 jmcneill struct mcx_cmd_destroy_sq_out {
1795 1.1 jmcneill uint8_t cmd_status;
1796 1.1 jmcneill uint8_t cmd_reserved0[3];
1797 1.1 jmcneill uint32_t cmd_syndrome;
1798 1.1 jmcneill uint8_t cmd_reserved1[8];
1799 1.1 jmcneill } __packed __aligned(4);
1800 1.1 jmcneill
1801 1.1 jmcneill
1802 1.1 jmcneill struct mcx_rq_ctx {
1803 1.1 jmcneill uint32_t rq_flags;
1804 1.5 msaitoh #define MCX_RQ_CTX_RLKEY (1U << 31)
1805 1.1 jmcneill #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28)
1806 1.1 jmcneill #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24
1807 1.1 jmcneill #define MCX_RQ_CTX_STATE_SHIFT 20
1808 1.15 jmcneill #define MCX_RQ_CTX_STATE_MASK (0xf << 20)
1809 1.15 jmcneill #define MCX_RQ_CTX_STATE_RST 0
1810 1.15 jmcneill #define MCX_RQ_CTX_STATE_RDY 1
1811 1.15 jmcneill #define MCX_RQ_CTX_STATE_ERR 3
1812 1.1 jmcneill #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18)
1813 1.1 jmcneill uint32_t rq_user_index;
1814 1.1 jmcneill uint32_t rq_cqn;
1815 1.1 jmcneill uint32_t rq_reserved1;
1816 1.1 jmcneill uint32_t rq_rmpn;
1817 1.1 jmcneill uint32_t rq_reserved2[7];
1818 1.1 jmcneill struct mcx_wq_ctx rq_wq;
1819 1.1 jmcneill } __packed __aligned(4);
1820 1.1 jmcneill
1821 1.1 jmcneill struct mcx_rq_entry {
1822 1.1 jmcneill uint32_t rqe_byte_count;
1823 1.1 jmcneill uint32_t rqe_lkey;
1824 1.1 jmcneill uint64_t rqe_addr;
1825 1.1 jmcneill } __packed __aligned(16);
1826 1.1 jmcneill
1827 1.1 jmcneill struct mcx_cmd_create_rq_in {
1828 1.1 jmcneill uint16_t cmd_opcode;
1829 1.1 jmcneill uint8_t cmd_reserved0[4];
1830 1.1 jmcneill uint16_t cmd_op_mod;
1831 1.1 jmcneill uint8_t cmd_reserved1[8];
1832 1.1 jmcneill } __packed __aligned(4);
1833 1.1 jmcneill
1834 1.1 jmcneill struct mcx_cmd_create_rq_out {
1835 1.1 jmcneill uint8_t cmd_status;
1836 1.1 jmcneill uint8_t cmd_reserved0[3];
1837 1.1 jmcneill uint32_t cmd_syndrome;
1838 1.1 jmcneill uint32_t cmd_rqn;
1839 1.1 jmcneill uint8_t cmd_reserved1[4];
1840 1.1 jmcneill } __packed __aligned(4);
1841 1.1 jmcneill
1842 1.1 jmcneill struct mcx_cmd_modify_rq_in {
1843 1.1 jmcneill uint16_t cmd_opcode;
1844 1.1 jmcneill uint8_t cmd_reserved0[4];
1845 1.1 jmcneill uint16_t cmd_op_mod;
1846 1.1 jmcneill uint32_t cmd_rq_state;
1847 1.1 jmcneill uint8_t cmd_reserved1[4];
1848 1.1 jmcneill } __packed __aligned(4);
1849 1.1 jmcneill
1850 1.1 jmcneill struct mcx_cmd_modify_rq_mb_in {
1851 1.1 jmcneill uint32_t cmd_modify_hi;
1852 1.1 jmcneill uint32_t cmd_modify_lo;
1853 1.1 jmcneill uint8_t cmd_reserved0[8];
1854 1.1 jmcneill struct mcx_rq_ctx cmd_rq_ctx;
1855 1.1 jmcneill } __packed __aligned(4);
1856 1.1 jmcneill
1857 1.1 jmcneill struct mcx_cmd_modify_rq_out {
1858 1.1 jmcneill uint8_t cmd_status;
1859 1.1 jmcneill uint8_t cmd_reserved0[3];
1860 1.1 jmcneill uint32_t cmd_syndrome;
1861 1.1 jmcneill uint8_t cmd_reserved1[8];
1862 1.1 jmcneill } __packed __aligned(4);
1863 1.1 jmcneill
1864 1.1 jmcneill struct mcx_cmd_destroy_rq_in {
1865 1.1 jmcneill uint16_t cmd_opcode;
1866 1.1 jmcneill uint8_t cmd_reserved0[4];
1867 1.1 jmcneill uint16_t cmd_op_mod;
1868 1.1 jmcneill uint32_t cmd_rqn;
1869 1.1 jmcneill uint8_t cmd_reserved1[4];
1870 1.1 jmcneill } __packed __aligned(4);
1871 1.1 jmcneill
1872 1.1 jmcneill struct mcx_cmd_destroy_rq_out {
1873 1.1 jmcneill uint8_t cmd_status;
1874 1.1 jmcneill uint8_t cmd_reserved0[3];
1875 1.1 jmcneill uint32_t cmd_syndrome;
1876 1.1 jmcneill uint8_t cmd_reserved1[8];
1877 1.1 jmcneill } __packed __aligned(4);
1878 1.1 jmcneill
1879 1.1 jmcneill struct mcx_cmd_create_flow_table_in {
1880 1.1 jmcneill uint16_t cmd_opcode;
1881 1.1 jmcneill uint8_t cmd_reserved0[4];
1882 1.1 jmcneill uint16_t cmd_op_mod;
1883 1.1 jmcneill uint8_t cmd_reserved1[8];
1884 1.1 jmcneill } __packed __aligned(4);
1885 1.1 jmcneill
1886 1.1 jmcneill struct mcx_flow_table_ctx {
1887 1.1 jmcneill uint8_t ft_miss_action;
1888 1.1 jmcneill uint8_t ft_level;
1889 1.1 jmcneill uint8_t ft_reserved0;
1890 1.1 jmcneill uint8_t ft_log_size;
1891 1.1 jmcneill uint32_t ft_table_miss_id;
1892 1.1 jmcneill uint8_t ft_reserved1[28];
1893 1.1 jmcneill } __packed __aligned(4);
1894 1.1 jmcneill
1895 1.1 jmcneill struct mcx_cmd_create_flow_table_mb_in {
1896 1.1 jmcneill uint8_t cmd_table_type;
1897 1.1 jmcneill uint8_t cmd_reserved0[7];
1898 1.1 jmcneill struct mcx_flow_table_ctx cmd_ctx;
1899 1.1 jmcneill } __packed __aligned(4);
1900 1.1 jmcneill
1901 1.1 jmcneill struct mcx_cmd_create_flow_table_out {
1902 1.1 jmcneill uint8_t cmd_status;
1903 1.1 jmcneill uint8_t cmd_reserved0[3];
1904 1.1 jmcneill uint32_t cmd_syndrome;
1905 1.1 jmcneill uint32_t cmd_table_id;
1906 1.1 jmcneill uint8_t cmd_reserved1[4];
1907 1.1 jmcneill } __packed __aligned(4);
1908 1.1 jmcneill
1909 1.1 jmcneill struct mcx_cmd_destroy_flow_table_in {
1910 1.1 jmcneill uint16_t cmd_opcode;
1911 1.1 jmcneill uint8_t cmd_reserved0[4];
1912 1.1 jmcneill uint16_t cmd_op_mod;
1913 1.1 jmcneill uint8_t cmd_reserved1[8];
1914 1.1 jmcneill } __packed __aligned(4);
1915 1.1 jmcneill
1916 1.1 jmcneill struct mcx_cmd_destroy_flow_table_mb_in {
1917 1.1 jmcneill uint8_t cmd_table_type;
1918 1.1 jmcneill uint8_t cmd_reserved0[3];
1919 1.1 jmcneill uint32_t cmd_table_id;
1920 1.1 jmcneill uint8_t cmd_reserved1[40];
1921 1.1 jmcneill } __packed __aligned(4);
1922 1.1 jmcneill
1923 1.1 jmcneill struct mcx_cmd_destroy_flow_table_out {
1924 1.1 jmcneill uint8_t cmd_status;
1925 1.1 jmcneill uint8_t cmd_reserved0[3];
1926 1.1 jmcneill uint32_t cmd_syndrome;
1927 1.1 jmcneill uint8_t cmd_reserved1[8];
1928 1.1 jmcneill } __packed __aligned(4);
1929 1.1 jmcneill
1930 1.1 jmcneill struct mcx_cmd_set_flow_table_root_in {
1931 1.1 jmcneill uint16_t cmd_opcode;
1932 1.1 jmcneill uint8_t cmd_reserved0[4];
1933 1.1 jmcneill uint16_t cmd_op_mod;
1934 1.1 jmcneill uint8_t cmd_reserved1[8];
1935 1.1 jmcneill } __packed __aligned(4);
1936 1.1 jmcneill
1937 1.1 jmcneill struct mcx_cmd_set_flow_table_root_mb_in {
1938 1.1 jmcneill uint8_t cmd_table_type;
1939 1.1 jmcneill uint8_t cmd_reserved0[3];
1940 1.1 jmcneill uint32_t cmd_table_id;
1941 1.1 jmcneill uint8_t cmd_reserved1[56];
1942 1.1 jmcneill } __packed __aligned(4);
1943 1.1 jmcneill
1944 1.1 jmcneill struct mcx_cmd_set_flow_table_root_out {
1945 1.1 jmcneill uint8_t cmd_status;
1946 1.1 jmcneill uint8_t cmd_reserved0[3];
1947 1.1 jmcneill uint32_t cmd_syndrome;
1948 1.1 jmcneill uint8_t cmd_reserved1[8];
1949 1.1 jmcneill } __packed __aligned(4);
1950 1.1 jmcneill
1951 1.1 jmcneill struct mcx_flow_match {
1952 1.1 jmcneill /* outer headers */
1953 1.1 jmcneill uint8_t mc_src_mac[6];
1954 1.1 jmcneill uint16_t mc_ethertype;
1955 1.1 jmcneill uint8_t mc_dest_mac[6];
1956 1.1 jmcneill uint16_t mc_first_vlan;
1957 1.1 jmcneill uint8_t mc_ip_proto;
1958 1.1 jmcneill uint8_t mc_ip_dscp_ecn;
1959 1.1 jmcneill uint8_t mc_vlan_flags;
1960 1.15 jmcneill #define MCX_FLOW_MATCH_IP_FRAG (1 << 5)
1961 1.1 jmcneill uint8_t mc_tcp_flags;
1962 1.1 jmcneill uint16_t mc_tcp_sport;
1963 1.1 jmcneill uint16_t mc_tcp_dport;
1964 1.1 jmcneill uint32_t mc_reserved0;
1965 1.1 jmcneill uint16_t mc_udp_sport;
1966 1.1 jmcneill uint16_t mc_udp_dport;
1967 1.1 jmcneill uint8_t mc_src_ip[16];
1968 1.1 jmcneill uint8_t mc_dest_ip[16];
1969 1.1 jmcneill
1970 1.1 jmcneill /* misc parameters */
1971 1.1 jmcneill uint8_t mc_reserved1[8];
1972 1.1 jmcneill uint16_t mc_second_vlan;
1973 1.1 jmcneill uint8_t mc_reserved2[2];
1974 1.1 jmcneill uint8_t mc_second_vlan_flags;
1975 1.1 jmcneill uint8_t mc_reserved3[15];
1976 1.1 jmcneill uint32_t mc_outer_ipv6_flow_label;
1977 1.1 jmcneill uint8_t mc_reserved4[32];
1978 1.1 jmcneill
1979 1.1 jmcneill uint8_t mc_reserved[384];
1980 1.1 jmcneill } __packed __aligned(4);
1981 1.1 jmcneill
1982 1.1 jmcneill CTASSERT(sizeof(struct mcx_flow_match) == 512);
1983 1.1 jmcneill
1984 1.1 jmcneill struct mcx_cmd_create_flow_group_in {
1985 1.1 jmcneill uint16_t cmd_opcode;
1986 1.1 jmcneill uint8_t cmd_reserved0[4];
1987 1.1 jmcneill uint16_t cmd_op_mod;
1988 1.1 jmcneill uint8_t cmd_reserved1[8];
1989 1.1 jmcneill } __packed __aligned(4);
1990 1.1 jmcneill
1991 1.1 jmcneill struct mcx_cmd_create_flow_group_mb_in {
1992 1.1 jmcneill uint8_t cmd_table_type;
1993 1.1 jmcneill uint8_t cmd_reserved0[3];
1994 1.1 jmcneill uint32_t cmd_table_id;
1995 1.1 jmcneill uint8_t cmd_reserved1[4];
1996 1.1 jmcneill uint32_t cmd_start_flow_index;
1997 1.1 jmcneill uint8_t cmd_reserved2[4];
1998 1.1 jmcneill uint32_t cmd_end_flow_index;
1999 1.1 jmcneill uint8_t cmd_reserved3[23];
2000 1.1 jmcneill uint8_t cmd_match_criteria_enable;
2001 1.1 jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0)
2002 1.1 jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1)
2003 1.1 jmcneill #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2)
2004 1.1 jmcneill struct mcx_flow_match cmd_match_criteria;
2005 1.1 jmcneill uint8_t cmd_reserved4[448];
2006 1.1 jmcneill } __packed __aligned(4);
2007 1.1 jmcneill
2008 1.1 jmcneill struct mcx_cmd_create_flow_group_out {
2009 1.1 jmcneill uint8_t cmd_status;
2010 1.1 jmcneill uint8_t cmd_reserved0[3];
2011 1.1 jmcneill uint32_t cmd_syndrome;
2012 1.1 jmcneill uint32_t cmd_group_id;
2013 1.1 jmcneill uint8_t cmd_reserved1[4];
2014 1.1 jmcneill } __packed __aligned(4);
2015 1.1 jmcneill
2016 1.1 jmcneill struct mcx_flow_ctx {
2017 1.1 jmcneill uint8_t fc_reserved0[4];
2018 1.1 jmcneill uint32_t fc_group_id;
2019 1.1 jmcneill uint32_t fc_flow_tag;
2020 1.1 jmcneill uint32_t fc_action;
2021 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0)
2022 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1)
2023 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2)
2024 1.1 jmcneill #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3)
2025 1.1 jmcneill uint32_t fc_dest_list_size;
2026 1.1 jmcneill uint32_t fc_counter_list_size;
2027 1.1 jmcneill uint8_t fc_reserved1[40];
2028 1.1 jmcneill struct mcx_flow_match fc_match_value;
2029 1.1 jmcneill uint8_t fc_reserved2[192];
2030 1.1 jmcneill } __packed __aligned(4);
2031 1.1 jmcneill
2032 1.1 jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24)
2033 1.1 jmcneill #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24)
2034 1.1 jmcneill
2035 1.1 jmcneill struct mcx_cmd_destroy_flow_group_in {
2036 1.1 jmcneill uint16_t cmd_opcode;
2037 1.1 jmcneill uint8_t cmd_reserved0[4];
2038 1.1 jmcneill uint16_t cmd_op_mod;
2039 1.1 jmcneill uint8_t cmd_reserved1[8];
2040 1.1 jmcneill } __packed __aligned(4);
2041 1.1 jmcneill
2042 1.1 jmcneill struct mcx_cmd_destroy_flow_group_mb_in {
2043 1.1 jmcneill uint8_t cmd_table_type;
2044 1.1 jmcneill uint8_t cmd_reserved0[3];
2045 1.1 jmcneill uint32_t cmd_table_id;
2046 1.1 jmcneill uint32_t cmd_group_id;
2047 1.1 jmcneill uint8_t cmd_reserved1[36];
2048 1.1 jmcneill } __packed __aligned(4);
2049 1.1 jmcneill
2050 1.1 jmcneill struct mcx_cmd_destroy_flow_group_out {
2051 1.1 jmcneill uint8_t cmd_status;
2052 1.1 jmcneill uint8_t cmd_reserved0[3];
2053 1.1 jmcneill uint32_t cmd_syndrome;
2054 1.1 jmcneill uint8_t cmd_reserved1[8];
2055 1.1 jmcneill } __packed __aligned(4);
2056 1.1 jmcneill
2057 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_in {
2058 1.1 jmcneill uint16_t cmd_opcode;
2059 1.1 jmcneill uint8_t cmd_reserved0[4];
2060 1.1 jmcneill uint16_t cmd_op_mod;
2061 1.1 jmcneill uint8_t cmd_reserved1[8];
2062 1.1 jmcneill } __packed __aligned(4);
2063 1.1 jmcneill
2064 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_mb_in {
2065 1.1 jmcneill uint8_t cmd_table_type;
2066 1.1 jmcneill uint8_t cmd_reserved0[3];
2067 1.1 jmcneill uint32_t cmd_table_id;
2068 1.1 jmcneill uint32_t cmd_modify_enable_mask;
2069 1.1 jmcneill uint8_t cmd_reserved1[4];
2070 1.1 jmcneill uint32_t cmd_flow_index;
2071 1.1 jmcneill uint8_t cmd_reserved2[28];
2072 1.1 jmcneill struct mcx_flow_ctx cmd_flow_ctx;
2073 1.1 jmcneill } __packed __aligned(4);
2074 1.1 jmcneill
2075 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_out {
2076 1.1 jmcneill uint8_t cmd_status;
2077 1.1 jmcneill uint8_t cmd_reserved0[3];
2078 1.1 jmcneill uint32_t cmd_syndrome;
2079 1.1 jmcneill uint8_t cmd_reserved1[8];
2080 1.1 jmcneill } __packed __aligned(4);
2081 1.1 jmcneill
2082 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_in {
2083 1.1 jmcneill uint16_t cmd_opcode;
2084 1.1 jmcneill uint8_t cmd_reserved0[4];
2085 1.1 jmcneill uint16_t cmd_op_mod;
2086 1.1 jmcneill uint8_t cmd_reserved1[8];
2087 1.1 jmcneill } __packed __aligned(4);
2088 1.1 jmcneill
2089 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_in {
2090 1.1 jmcneill uint8_t cmd_table_type;
2091 1.1 jmcneill uint8_t cmd_reserved0[3];
2092 1.1 jmcneill uint32_t cmd_table_id;
2093 1.1 jmcneill uint8_t cmd_reserved1[8];
2094 1.1 jmcneill uint32_t cmd_flow_index;
2095 1.1 jmcneill uint8_t cmd_reserved2[28];
2096 1.1 jmcneill } __packed __aligned(4);
2097 1.1 jmcneill
2098 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_out {
2099 1.1 jmcneill uint8_t cmd_status;
2100 1.1 jmcneill uint8_t cmd_reserved0[3];
2101 1.1 jmcneill uint32_t cmd_syndrome;
2102 1.1 jmcneill uint8_t cmd_reserved1[8];
2103 1.1 jmcneill } __packed __aligned(4);
2104 1.1 jmcneill
2105 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_out {
2106 1.1 jmcneill uint8_t cmd_reserved0[48];
2107 1.1 jmcneill struct mcx_flow_ctx cmd_flow_ctx;
2108 1.1 jmcneill } __packed __aligned(4);
2109 1.1 jmcneill
2110 1.1 jmcneill struct mcx_cmd_delete_flow_table_entry_in {
2111 1.1 jmcneill uint16_t cmd_opcode;
2112 1.1 jmcneill uint8_t cmd_reserved0[4];
2113 1.1 jmcneill uint16_t cmd_op_mod;
2114 1.1 jmcneill uint8_t cmd_reserved1[8];
2115 1.1 jmcneill } __packed __aligned(4);
2116 1.1 jmcneill
2117 1.1 jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in {
2118 1.1 jmcneill uint8_t cmd_table_type;
2119 1.1 jmcneill uint8_t cmd_reserved0[3];
2120 1.1 jmcneill uint32_t cmd_table_id;
2121 1.1 jmcneill uint8_t cmd_reserved1[8];
2122 1.1 jmcneill uint32_t cmd_flow_index;
2123 1.1 jmcneill uint8_t cmd_reserved2[28];
2124 1.1 jmcneill } __packed __aligned(4);
2125 1.1 jmcneill
2126 1.1 jmcneill struct mcx_cmd_delete_flow_table_entry_out {
2127 1.1 jmcneill uint8_t cmd_status;
2128 1.1 jmcneill uint8_t cmd_reserved0[3];
2129 1.1 jmcneill uint32_t cmd_syndrome;
2130 1.1 jmcneill uint8_t cmd_reserved1[8];
2131 1.1 jmcneill } __packed __aligned(4);
2132 1.1 jmcneill
2133 1.1 jmcneill struct mcx_cmd_query_flow_group_in {
2134 1.1 jmcneill uint16_t cmd_opcode;
2135 1.1 jmcneill uint8_t cmd_reserved0[4];
2136 1.1 jmcneill uint16_t cmd_op_mod;
2137 1.1 jmcneill uint8_t cmd_reserved1[8];
2138 1.1 jmcneill } __packed __aligned(4);
2139 1.1 jmcneill
2140 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_in {
2141 1.1 jmcneill uint8_t cmd_table_type;
2142 1.1 jmcneill uint8_t cmd_reserved0[3];
2143 1.1 jmcneill uint32_t cmd_table_id;
2144 1.1 jmcneill uint32_t cmd_group_id;
2145 1.1 jmcneill uint8_t cmd_reserved1[36];
2146 1.1 jmcneill } __packed __aligned(4);
2147 1.1 jmcneill
2148 1.1 jmcneill struct mcx_cmd_query_flow_group_out {
2149 1.1 jmcneill uint8_t cmd_status;
2150 1.1 jmcneill uint8_t cmd_reserved0[3];
2151 1.1 jmcneill uint32_t cmd_syndrome;
2152 1.1 jmcneill uint8_t cmd_reserved1[8];
2153 1.1 jmcneill } __packed __aligned(4);
2154 1.1 jmcneill
2155 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_out {
2156 1.1 jmcneill uint8_t cmd_reserved0[12];
2157 1.1 jmcneill uint32_t cmd_start_flow_index;
2158 1.1 jmcneill uint8_t cmd_reserved1[4];
2159 1.1 jmcneill uint32_t cmd_end_flow_index;
2160 1.1 jmcneill uint8_t cmd_reserved2[20];
2161 1.1 jmcneill uint32_t cmd_match_criteria_enable;
2162 1.1 jmcneill uint8_t cmd_match_criteria[512];
2163 1.1 jmcneill uint8_t cmd_reserved4[448];
2164 1.1 jmcneill } __packed __aligned(4);
2165 1.1 jmcneill
2166 1.1 jmcneill struct mcx_cmd_query_flow_table_in {
2167 1.1 jmcneill uint16_t cmd_opcode;
2168 1.1 jmcneill uint8_t cmd_reserved0[4];
2169 1.1 jmcneill uint16_t cmd_op_mod;
2170 1.1 jmcneill uint8_t cmd_reserved1[8];
2171 1.1 jmcneill } __packed __aligned(4);
2172 1.1 jmcneill
2173 1.1 jmcneill struct mcx_cmd_query_flow_table_mb_in {
2174 1.1 jmcneill uint8_t cmd_table_type;
2175 1.1 jmcneill uint8_t cmd_reserved0[3];
2176 1.1 jmcneill uint32_t cmd_table_id;
2177 1.1 jmcneill uint8_t cmd_reserved1[40];
2178 1.1 jmcneill } __packed __aligned(4);
2179 1.1 jmcneill
2180 1.1 jmcneill struct mcx_cmd_query_flow_table_out {
2181 1.1 jmcneill uint8_t cmd_status;
2182 1.1 jmcneill uint8_t cmd_reserved0[3];
2183 1.1 jmcneill uint32_t cmd_syndrome;
2184 1.1 jmcneill uint8_t cmd_reserved1[8];
2185 1.1 jmcneill } __packed __aligned(4);
2186 1.1 jmcneill
2187 1.1 jmcneill struct mcx_cmd_query_flow_table_mb_out {
2188 1.1 jmcneill uint8_t cmd_reserved0[4];
2189 1.1 jmcneill struct mcx_flow_table_ctx cmd_ctx;
2190 1.1 jmcneill } __packed __aligned(4);
2191 1.1 jmcneill
2192 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_in {
2193 1.1 jmcneill uint16_t cmd_opcode;
2194 1.1 jmcneill uint8_t cmd_reserved0[4];
2195 1.1 jmcneill uint16_t cmd_op_mod;
2196 1.1 jmcneill uint8_t cmd_reserved1[8];
2197 1.1 jmcneill } __packed __aligned(4);
2198 1.1 jmcneill
2199 1.1 jmcneill struct mcx_cmd_query_rq_in {
2200 1.1 jmcneill uint16_t cmd_opcode;
2201 1.1 jmcneill uint8_t cmd_reserved0[4];
2202 1.1 jmcneill uint16_t cmd_op_mod;
2203 1.1 jmcneill uint32_t cmd_rqn;
2204 1.1 jmcneill uint8_t cmd_reserved1[4];
2205 1.1 jmcneill } __packed __aligned(4);
2206 1.1 jmcneill
2207 1.1 jmcneill struct mcx_cmd_query_rq_out {
2208 1.1 jmcneill uint8_t cmd_status;
2209 1.1 jmcneill uint8_t cmd_reserved0[3];
2210 1.1 jmcneill uint32_t cmd_syndrome;
2211 1.1 jmcneill uint8_t cmd_reserved1[8];
2212 1.1 jmcneill } __packed __aligned(4);
2213 1.1 jmcneill
2214 1.1 jmcneill struct mcx_cmd_query_rq_mb_out {
2215 1.1 jmcneill uint8_t cmd_reserved0[16];
2216 1.1 jmcneill struct mcx_rq_ctx cmd_ctx;
2217 1.1 jmcneill };
2218 1.1 jmcneill
2219 1.1 jmcneill struct mcx_cmd_query_sq_in {
2220 1.1 jmcneill uint16_t cmd_opcode;
2221 1.1 jmcneill uint8_t cmd_reserved0[4];
2222 1.1 jmcneill uint16_t cmd_op_mod;
2223 1.1 jmcneill uint32_t cmd_sqn;
2224 1.1 jmcneill uint8_t cmd_reserved1[4];
2225 1.1 jmcneill } __packed __aligned(4);
2226 1.1 jmcneill
2227 1.1 jmcneill struct mcx_cmd_query_sq_out {
2228 1.1 jmcneill uint8_t cmd_status;
2229 1.1 jmcneill uint8_t cmd_reserved0[3];
2230 1.1 jmcneill uint32_t cmd_syndrome;
2231 1.1 jmcneill uint8_t cmd_reserved1[8];
2232 1.1 jmcneill } __packed __aligned(4);
2233 1.1 jmcneill
2234 1.1 jmcneill struct mcx_cmd_query_sq_mb_out {
2235 1.1 jmcneill uint8_t cmd_reserved0[16];
2236 1.1 jmcneill struct mcx_sq_ctx cmd_ctx;
2237 1.1 jmcneill };
2238 1.1 jmcneill
2239 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_out {
2240 1.1 jmcneill uint8_t cmd_status;
2241 1.1 jmcneill uint8_t cmd_reserved0[3];
2242 1.1 jmcneill uint32_t cmd_syndrome;
2243 1.1 jmcneill uint8_t cmd_reserved1[2];
2244 1.1 jmcneill uint16_t cmd_flow_counter_id;
2245 1.1 jmcneill uint8_t cmd_reserved2[4];
2246 1.1 jmcneill } __packed __aligned(4);
2247 1.1 jmcneill
2248 1.1 jmcneill struct mcx_wq_doorbell {
2249 1.1 jmcneill uint32_t db_recv_counter;
2250 1.1 jmcneill uint32_t db_send_counter;
2251 1.1 jmcneill } __packed __aligned(8);
2252 1.1 jmcneill
2253 1.1 jmcneill struct mcx_dmamem {
2254 1.1 jmcneill bus_dmamap_t mxm_map;
2255 1.1 jmcneill bus_dma_segment_t mxm_seg;
2256 1.1 jmcneill int mxm_nsegs;
2257 1.1 jmcneill size_t mxm_size;
2258 1.1 jmcneill void *mxm_kva;
2259 1.1 jmcneill };
2260 1.1 jmcneill #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map)
2261 1.1 jmcneill #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr)
2262 1.1 jmcneill #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva)
2263 1.15 jmcneill #define MCX_DMA_OFF(_mxm, _off) ((void *)((char *)(_mxm)->mxm_kva + (_off)))
2264 1.1 jmcneill #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size)
2265 1.1 jmcneill
2266 1.1 jmcneill struct mcx_hwmem {
2267 1.1 jmcneill bus_dmamap_t mhm_map;
2268 1.1 jmcneill bus_dma_segment_t *mhm_segs;
2269 1.1 jmcneill unsigned int mhm_seg_count;
2270 1.1 jmcneill unsigned int mhm_npages;
2271 1.1 jmcneill };
2272 1.1 jmcneill
2273 1.1 jmcneill struct mcx_slot {
2274 1.1 jmcneill bus_dmamap_t ms_map;
2275 1.1 jmcneill struct mbuf *ms_m;
2276 1.1 jmcneill };
2277 1.1 jmcneill
2278 1.15 jmcneill struct mcx_eq {
2279 1.15 jmcneill int eq_n;
2280 1.15 jmcneill uint32_t eq_cons;
2281 1.15 jmcneill struct mcx_dmamem eq_mem;
2282 1.15 jmcneill };
2283 1.15 jmcneill
2284 1.1 jmcneill struct mcx_cq {
2285 1.1 jmcneill int cq_n;
2286 1.1 jmcneill struct mcx_dmamem cq_mem;
2287 1.15 jmcneill bus_addr_t cq_doorbell;
2288 1.1 jmcneill uint32_t cq_cons;
2289 1.1 jmcneill uint32_t cq_count;
2290 1.1 jmcneill };
2291 1.1 jmcneill
2292 1.1 jmcneill struct mcx_calibration {
2293 1.1 jmcneill uint64_t c_timestamp; /* previous mcx chip time */
2294 1.1 jmcneill uint64_t c_uptime; /* previous kernel nanouptime */
2295 1.1 jmcneill uint64_t c_tbase; /* mcx chip time */
2296 1.1 jmcneill uint64_t c_ubase; /* kernel nanouptime */
2297 1.15 jmcneill uint64_t c_ratio;
2298 1.1 jmcneill };
2299 1.1 jmcneill
2300 1.1 jmcneill #define MCX_CALIBRATE_FIRST 2
2301 1.15 jmcneill #define MCX_CALIBRATE_NORMAL 32
2302 1.1 jmcneill
2303 1.1 jmcneill struct mcx_rxring {
2304 1.1 jmcneill u_int rxr_total;
2305 1.1 jmcneill u_int rxr_inuse;
2306 1.1 jmcneill };
2307 1.1 jmcneill
2308 1.1 jmcneill MBUFQ_HEAD(mcx_mbufq);
2309 1.1 jmcneill
2310 1.15 jmcneill struct mcx_rx {
2311 1.15 jmcneill struct mcx_softc *rx_softc;
2312 1.15 jmcneill
2313 1.15 jmcneill int rx_rqn;
2314 1.15 jmcneill struct mcx_dmamem rx_rq_mem;
2315 1.15 jmcneill struct mcx_slot *rx_slots;
2316 1.15 jmcneill bus_addr_t rx_doorbell;
2317 1.15 jmcneill
2318 1.15 jmcneill uint32_t rx_prod;
2319 1.15 jmcneill callout_t rx_refill;
2320 1.15 jmcneill struct mcx_rxring rx_rxr;
2321 1.15 jmcneill } __aligned(64);
2322 1.15 jmcneill
2323 1.15 jmcneill struct mcx_tx {
2324 1.15 jmcneill struct mcx_softc *tx_softc;
2325 1.16 jmcneill kmutex_t tx_lock;
2326 1.16 jmcneill pcq_t *tx_pcq;
2327 1.16 jmcneill void *tx_softint;
2328 1.15 jmcneill
2329 1.15 jmcneill int tx_uar;
2330 1.15 jmcneill int tx_sqn;
2331 1.15 jmcneill struct mcx_dmamem tx_sq_mem;
2332 1.15 jmcneill struct mcx_slot *tx_slots;
2333 1.15 jmcneill bus_addr_t tx_doorbell;
2334 1.15 jmcneill int tx_bf_offset;
2335 1.15 jmcneill
2336 1.15 jmcneill uint32_t tx_cons;
2337 1.15 jmcneill uint32_t tx_prod;
2338 1.15 jmcneill } __aligned(64);
2339 1.15 jmcneill
2340 1.15 jmcneill struct mcx_queues {
2341 1.15 jmcneill void *q_ihc;
2342 1.15 jmcneill struct mcx_softc *q_sc;
2343 1.15 jmcneill int q_uar;
2344 1.15 jmcneill int q_index;
2345 1.15 jmcneill struct mcx_rx q_rx;
2346 1.15 jmcneill struct mcx_tx q_tx;
2347 1.15 jmcneill struct mcx_cq q_cq;
2348 1.15 jmcneill struct mcx_eq q_eq;
2349 1.15 jmcneill #if NKSTAT > 0
2350 1.15 jmcneill struct kstat *q_kstat;
2351 1.15 jmcneill #endif
2352 1.15 jmcneill };
2353 1.15 jmcneill
2354 1.15 jmcneill struct mcx_flow_group {
2355 1.15 jmcneill int g_id;
2356 1.15 jmcneill int g_table;
2357 1.15 jmcneill int g_start;
2358 1.15 jmcneill int g_size;
2359 1.15 jmcneill };
2360 1.15 jmcneill
2361 1.15 jmcneill #define MCX_FLOW_GROUP_PROMISC 0
2362 1.15 jmcneill #define MCX_FLOW_GROUP_ALLMULTI 1
2363 1.15 jmcneill #define MCX_FLOW_GROUP_MAC 2
2364 1.15 jmcneill #define MCX_FLOW_GROUP_RSS_L4 3
2365 1.15 jmcneill #define MCX_FLOW_GROUP_RSS_L3 4
2366 1.15 jmcneill #define MCX_FLOW_GROUP_RSS_NONE 5
2367 1.15 jmcneill #define MCX_NUM_FLOW_GROUPS 6
2368 1.15 jmcneill
2369 1.15 jmcneill #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \
2370 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_DST_IP
2371 1.15 jmcneill #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \
2372 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_DPORT
2373 1.15 jmcneill
2374 1.15 jmcneill #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\
2375 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV4
2376 1.15 jmcneill #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \
2377 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV6
2378 1.15 jmcneill #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
2379 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV4
2380 1.15 jmcneill #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \
2381 1.15 jmcneill MCX_TIR_CTX_HASH_SEL_IPV6
2382 1.15 jmcneill #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4
2383 1.15 jmcneill #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6
2384 1.15 jmcneill
2385 1.15 jmcneill /*
2386 1.15 jmcneill * There are a few different pieces involved in configuring RSS.
2387 1.15 jmcneill * A Receive Queue Table (RQT) is the indirection table that maps packets to
2388 1.15 jmcneill * different rx queues based on a hash value. We only create one, because
2389 1.15 jmcneill * we want to scatter any traffic we can apply RSS to across all our rx
2390 1.15 jmcneill * queues. Anything else will only be delivered to the first rx queue,
2391 1.15 jmcneill * which doesn't require an RQT.
2392 1.15 jmcneill *
2393 1.15 jmcneill * A Transport Interface Receive (TIR) delivers packets to either a single rx
2394 1.15 jmcneill * queue or an RQT, and in the latter case, specifies the set of fields
2395 1.15 jmcneill * hashed, the hash function, and the hash key. We need one of these for each
2396 1.15 jmcneill * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6,
2397 1.15 jmcneill * and one for non-RSS traffic.
2398 1.15 jmcneill *
2399 1.15 jmcneill * Flow tables hold flow table entries in sequence. The first entry that
2400 1.15 jmcneill * matches a packet is applied, sending the packet to either another flow
2401 1.15 jmcneill * table or a TIR. We use one flow table to select packets based on
2402 1.15 jmcneill * destination MAC address, and a second to apply RSS. The entries in the
2403 1.15 jmcneill * first table send matching packets to the second, and the entries in the
2404 1.15 jmcneill * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR.
2405 1.15 jmcneill *
2406 1.15 jmcneill * The flow table entry that delivers packets to an RSS TIR must include match
2407 1.15 jmcneill * criteria that ensure packets delivered to the TIR include all the fields
2408 1.15 jmcneill * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must
2409 1.15 jmcneill * only accept v4 TCP packets. Accordingly, we need flow table entries for
2410 1.15 jmcneill * each TIR.
2411 1.15 jmcneill *
2412 1.15 jmcneill * All of this is a lot more flexible than we need, and we can describe most
2413 1.15 jmcneill * of the stuff we need with a simple array.
2414 1.15 jmcneill *
2415 1.15 jmcneill * An RSS config creates a TIR with hashing enabled on a set of fields,
2416 1.15 jmcneill * pointing to either the first rx queue or the RQT containing all the rx
2417 1.15 jmcneill * queues, and a flow table entry that matches on an ether type and
2418 1.15 jmcneill * optionally an ip proto, that delivers packets to the TIR.
2419 1.15 jmcneill */
2420 1.15 jmcneill static struct mcx_rss_rule {
2421 1.15 jmcneill int hash_sel;
2422 1.15 jmcneill int flow_group;
2423 1.15 jmcneill int ethertype;
2424 1.15 jmcneill int ip_proto;
2425 1.15 jmcneill } mcx_rss_config[] = {
2426 1.15 jmcneill /* udp and tcp for v4/v6 */
2427 1.15 jmcneill { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4,
2428 1.15 jmcneill ETHERTYPE_IP, IPPROTO_TCP },
2429 1.15 jmcneill { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4,
2430 1.15 jmcneill ETHERTYPE_IPV6, IPPROTO_TCP },
2431 1.15 jmcneill { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4,
2432 1.15 jmcneill ETHERTYPE_IP, IPPROTO_UDP },
2433 1.15 jmcneill { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4,
2434 1.15 jmcneill ETHERTYPE_IPV6, IPPROTO_UDP },
2435 1.15 jmcneill
2436 1.15 jmcneill /* other v4/v6 */
2437 1.15 jmcneill { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3,
2438 1.15 jmcneill ETHERTYPE_IP, 0 },
2439 1.15 jmcneill { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3,
2440 1.15 jmcneill ETHERTYPE_IPV6, 0 },
2441 1.15 jmcneill
2442 1.15 jmcneill /* non v4/v6 */
2443 1.15 jmcneill { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 }
2444 1.15 jmcneill };
2445 1.15 jmcneill
2446 1.1 jmcneill struct mcx_softc {
2447 1.1 jmcneill device_t sc_dev;
2448 1.1 jmcneill struct ethercom sc_ec;
2449 1.1 jmcneill struct ifmedia sc_media;
2450 1.1 jmcneill uint64_t sc_media_status;
2451 1.1 jmcneill uint64_t sc_media_active;
2452 1.12 thorpej kmutex_t sc_media_mutex;
2453 1.1 jmcneill
2454 1.1 jmcneill pci_chipset_tag_t sc_pc;
2455 1.1 jmcneill pci_intr_handle_t *sc_intrs;
2456 1.15 jmcneill void *sc_ihc;
2457 1.1 jmcneill pcitag_t sc_tag;
2458 1.1 jmcneill
2459 1.1 jmcneill bus_dma_tag_t sc_dmat;
2460 1.1 jmcneill bus_space_tag_t sc_memt;
2461 1.1 jmcneill bus_space_handle_t sc_memh;
2462 1.1 jmcneill bus_size_t sc_mems;
2463 1.1 jmcneill
2464 1.1 jmcneill struct mcx_dmamem sc_cmdq_mem;
2465 1.1 jmcneill unsigned int sc_cmdq_mask;
2466 1.1 jmcneill unsigned int sc_cmdq_size;
2467 1.1 jmcneill
2468 1.1 jmcneill unsigned int sc_cmdq_token;
2469 1.1 jmcneill
2470 1.1 jmcneill struct mcx_hwmem sc_boot_pages;
2471 1.1 jmcneill struct mcx_hwmem sc_init_pages;
2472 1.1 jmcneill struct mcx_hwmem sc_regular_pages;
2473 1.1 jmcneill
2474 1.1 jmcneill int sc_uar;
2475 1.1 jmcneill int sc_pd;
2476 1.1 jmcneill int sc_tdomain;
2477 1.1 jmcneill uint32_t sc_lkey;
2478 1.15 jmcneill int sc_tis;
2479 1.15 jmcneill int sc_tir[__arraycount(mcx_rss_config)];
2480 1.15 jmcneill int sc_rqt;
2481 1.1 jmcneill
2482 1.1 jmcneill struct mcx_dmamem sc_doorbell_mem;
2483 1.1 jmcneill
2484 1.15 jmcneill struct mcx_eq sc_admin_eq;
2485 1.15 jmcneill struct mcx_eq sc_queue_eq;
2486 1.15 jmcneill
2487 1.1 jmcneill int sc_hardmtu;
2488 1.15 jmcneill int sc_rxbufsz;
2489 1.15 jmcneill
2490 1.15 jmcneill int sc_bf_size;
2491 1.15 jmcneill int sc_max_rqt_size;
2492 1.1 jmcneill
2493 1.1 jmcneill struct workqueue *sc_workq;
2494 1.1 jmcneill struct work sc_port_change;
2495 1.1 jmcneill
2496 1.15 jmcneill int sc_mac_flow_table_id;
2497 1.15 jmcneill int sc_rss_flow_table_id;
2498 1.15 jmcneill struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS];
2499 1.1 jmcneill int sc_promisc_flow_enabled;
2500 1.1 jmcneill int sc_allmulti_flow_enabled;
2501 1.1 jmcneill int sc_mcast_flow_base;
2502 1.1 jmcneill int sc_extra_mcast;
2503 1.1 jmcneill uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN];
2504 1.1 jmcneill
2505 1.1 jmcneill struct mcx_calibration sc_calibration[2];
2506 1.1 jmcneill unsigned int sc_calibration_gen;
2507 1.1 jmcneill callout_t sc_calibrate;
2508 1.15 jmcneill uint32_t sc_mhz;
2509 1.15 jmcneill uint32_t sc_khz;
2510 1.1 jmcneill
2511 1.15 jmcneill struct mcx_queues *sc_queues;
2512 1.15 jmcneill unsigned int sc_nqueues;
2513 1.1 jmcneill
2514 1.15 jmcneill int sc_mcam_reg;
2515 1.1 jmcneill
2516 1.15 jmcneill #if NKSTAT > 0
2517 1.15 jmcneill struct kstat *sc_kstat_ieee8023;
2518 1.15 jmcneill struct kstat *sc_kstat_rfc2863;
2519 1.15 jmcneill struct kstat *sc_kstat_rfc2819;
2520 1.15 jmcneill struct kstat *sc_kstat_rfc3635;
2521 1.15 jmcneill unsigned int sc_kstat_mtmp_count;
2522 1.15 jmcneill struct kstat **sc_kstat_mtmp;
2523 1.15 jmcneill #endif
2524 1.1 jmcneill
2525 1.15 jmcneill struct timecounter sc_timecounter;
2526 1.1 jmcneill };
2527 1.1 jmcneill #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
2528 1.1 jmcneill
2529 1.1 jmcneill static int mcx_match(device_t, cfdata_t, void *);
2530 1.1 jmcneill static void mcx_attach(device_t, device_t, void *);
2531 1.1 jmcneill
2532 1.15 jmcneill static void * mcx_establish_intr(struct mcx_softc *, int, kcpuset_t *,
2533 1.15 jmcneill int (*)(void *), void *, const char *);
2534 1.15 jmcneill
2535 1.1 jmcneill static void mcx_rxr_init(struct mcx_rxring *, u_int, u_int);
2536 1.1 jmcneill static u_int mcx_rxr_get(struct mcx_rxring *, u_int);
2537 1.1 jmcneill static void mcx_rxr_put(struct mcx_rxring *, u_int);
2538 1.1 jmcneill static u_int mcx_rxr_inuse(struct mcx_rxring *);
2539 1.1 jmcneill
2540 1.15 jmcneill #if NKSTAT > 0
2541 1.15 jmcneill static void mcx_kstat_attach(struct mcx_softc *);
2542 1.15 jmcneill #endif
2543 1.15 jmcneill
2544 1.15 jmcneill static void mcx_timecounter_attach(struct mcx_softc *);
2545 1.15 jmcneill
2546 1.1 jmcneill static int mcx_version(struct mcx_softc *);
2547 1.1 jmcneill static int mcx_init_wait(struct mcx_softc *);
2548 1.1 jmcneill static int mcx_enable_hca(struct mcx_softc *);
2549 1.1 jmcneill static int mcx_teardown_hca(struct mcx_softc *, uint16_t);
2550 1.1 jmcneill static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *,
2551 1.1 jmcneill int);
2552 1.1 jmcneill static int mcx_issi(struct mcx_softc *);
2553 1.1 jmcneill static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t);
2554 1.1 jmcneill static int mcx_hca_max_caps(struct mcx_softc *);
2555 1.1 jmcneill static int mcx_hca_set_caps(struct mcx_softc *);
2556 1.1 jmcneill static int mcx_init_hca(struct mcx_softc *);
2557 1.1 jmcneill static int mcx_set_driver_version(struct mcx_softc *);
2558 1.1 jmcneill static int mcx_iff(struct mcx_softc *);
2559 1.15 jmcneill static int mcx_alloc_uar(struct mcx_softc *, int *);
2560 1.1 jmcneill static int mcx_alloc_pd(struct mcx_softc *);
2561 1.1 jmcneill static int mcx_alloc_tdomain(struct mcx_softc *);
2562 1.15 jmcneill static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int,
2563 1.15 jmcneill uint64_t, int);
2564 1.1 jmcneill static int mcx_query_nic_vport_context(struct mcx_softc *, uint8_t *);
2565 1.1 jmcneill static int mcx_query_special_contexts(struct mcx_softc *);
2566 1.1 jmcneill static int mcx_set_port_mtu(struct mcx_softc *, int);
2567 1.15 jmcneill static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int,
2568 1.15 jmcneill int);
2569 1.15 jmcneill static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *);
2570 1.15 jmcneill static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int,
2571 1.15 jmcneill int);
2572 1.15 jmcneill static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *);
2573 1.15 jmcneill static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *);
2574 1.15 jmcneill static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int);
2575 1.15 jmcneill static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *);
2576 1.15 jmcneill static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *);
2577 1.15 jmcneill static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *,
2578 1.15 jmcneill int *);
2579 1.15 jmcneill static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t,
2580 1.15 jmcneill int *);
2581 1.15 jmcneill static int mcx_destroy_tir(struct mcx_softc *, int);
2582 1.15 jmcneill static int mcx_create_tis(struct mcx_softc *, int *);
2583 1.15 jmcneill static int mcx_destroy_tis(struct mcx_softc *, int);
2584 1.15 jmcneill static int mcx_create_rqt(struct mcx_softc *, int, int *, int *);
2585 1.15 jmcneill static int mcx_destroy_rqt(struct mcx_softc *, int);
2586 1.15 jmcneill static int mcx_create_flow_table(struct mcx_softc *, int, int, int *);
2587 1.15 jmcneill static int mcx_set_flow_table_root(struct mcx_softc *, int);
2588 1.15 jmcneill static int mcx_destroy_flow_table(struct mcx_softc *, int);
2589 1.15 jmcneill static int mcx_create_flow_group(struct mcx_softc *, int, int, int,
2590 1.1 jmcneill int, int, struct mcx_flow_match *);
2591 1.1 jmcneill static int mcx_destroy_flow_group(struct mcx_softc *, int);
2592 1.15 jmcneill static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int,
2593 1.15 jmcneill const uint8_t *, uint32_t);
2594 1.15 jmcneill static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int,
2595 1.15 jmcneill int, int, uint32_t);
2596 1.1 jmcneill static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int);
2597 1.1 jmcneill
2598 1.15 jmcneill #if NKSTAT > 0
2599 1.15 jmcneill static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *);
2600 1.15 jmcneill static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *);
2601 1.15 jmcneill static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *);
2602 1.15 jmcneill static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *);
2603 1.15 jmcneill #endif
2604 1.15 jmcneill
2605 1.1 jmcneill #if 0
2606 1.15 jmcneill static int mcx_dump_flow_table(struct mcx_softc *, int);
2607 1.15 jmcneill static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int);
2608 1.15 jmcneill static int mcx_dump_flow_group(struct mcx_softc *, int);
2609 1.1 jmcneill #endif
2610 1.1 jmcneill
2611 1.1 jmcneill
2612 1.1 jmcneill /*
2613 1.1 jmcneill static void mcx_cmdq_dump(const struct mcx_cmdq_entry *);
2614 1.1 jmcneill static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int);
2615 1.1 jmcneill */
2616 1.1 jmcneill static void mcx_refill(void *);
2617 1.15 jmcneill static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *,
2618 1.15 jmcneill struct mcx_cq_entry *, struct mcx_mbufq *,
2619 1.15 jmcneill const struct mcx_calibration *);
2620 1.15 jmcneill static int mcx_process_txeof(struct mcx_softc *, struct mcx_tx *,
2621 1.15 jmcneill struct mcx_cq_entry *);
2622 1.15 jmcneill static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *,
2623 1.15 jmcneill struct mcx_cq *);
2624 1.15 jmcneill
2625 1.15 jmcneill static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int);
2626 1.15 jmcneill static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int);
2627 1.15 jmcneill static int mcx_admin_intr(void *);
2628 1.15 jmcneill static int mcx_cq_intr(void *);
2629 1.1 jmcneill
2630 1.1 jmcneill static int mcx_init(struct ifnet *);
2631 1.1 jmcneill static void mcx_stop(struct ifnet *, int);
2632 1.1 jmcneill static int mcx_ioctl(struct ifnet *, u_long, void *);
2633 1.1 jmcneill static void mcx_start(struct ifnet *);
2634 1.16 jmcneill static int mcx_transmit(struct ifnet *, struct mbuf *);
2635 1.16 jmcneill static void mcx_deferred_transmit(void *);
2636 1.1 jmcneill static void mcx_watchdog(struct ifnet *);
2637 1.1 jmcneill static void mcx_media_add_types(struct mcx_softc *);
2638 1.1 jmcneill static void mcx_media_status(struct ifnet *, struct ifmediareq *);
2639 1.1 jmcneill static int mcx_media_change(struct ifnet *);
2640 1.1 jmcneill #if 0
2641 1.1 jmcneill static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *);
2642 1.1 jmcneill #endif
2643 1.1 jmcneill static void mcx_port_change(struct work *, void *);
2644 1.1 jmcneill
2645 1.1 jmcneill static void mcx_calibrate_first(struct mcx_softc *);
2646 1.1 jmcneill static void mcx_calibrate(void *);
2647 1.1 jmcneill
2648 1.1 jmcneill static inline uint32_t
2649 1.1 jmcneill mcx_rd(struct mcx_softc *, bus_size_t);
2650 1.1 jmcneill static inline void
2651 1.1 jmcneill mcx_wr(struct mcx_softc *, bus_size_t, uint32_t);
2652 1.1 jmcneill static inline void
2653 1.1 jmcneill mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int);
2654 1.1 jmcneill
2655 1.1 jmcneill static uint64_t mcx_timer(struct mcx_softc *);
2656 1.1 jmcneill
2657 1.1 jmcneill static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *,
2658 1.1 jmcneill bus_size_t, u_int align);
2659 1.1 jmcneill static void mcx_dmamem_zero(struct mcx_dmamem *);
2660 1.1 jmcneill static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *);
2661 1.1 jmcneill
2662 1.1 jmcneill static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *,
2663 1.1 jmcneill unsigned int);
2664 1.1 jmcneill static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *);
2665 1.1 jmcneill
2666 1.1 jmcneill CFATTACH_DECL_NEW(mcx, sizeof(struct mcx_softc), mcx_match, mcx_attach, NULL, NULL);
2667 1.1 jmcneill
2668 1.1 jmcneill static const struct {
2669 1.1 jmcneill pci_vendor_id_t vendor;
2670 1.1 jmcneill pci_product_id_t product;
2671 1.1 jmcneill } mcx_devices[] = {
2672 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 },
2673 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF },
2674 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 },
2675 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF },
2676 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 },
2677 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF },
2678 1.1 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 },
2679 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF },
2680 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28908 },
2681 1.15 jmcneill { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 },
2682 1.1 jmcneill };
2683 1.1 jmcneill
2684 1.13 jmcneill struct mcx_eth_proto_capability {
2685 1.13 jmcneill uint64_t cap_media;
2686 1.13 jmcneill uint64_t cap_baudrate;
2687 1.13 jmcneill };
2688 1.13 jmcneill
2689 1.13 jmcneill static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = {
2690 1.13 jmcneill [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) },
2691 1.13 jmcneill [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) },
2692 1.13 jmcneill [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) },
2693 1.13 jmcneill [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) },
2694 1.13 jmcneill [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) },
2695 1.13 jmcneill [MCX_ETHER_CAP_20G_KR2] = { IFM_20G_KR2, IF_Gbps(20) },
2696 1.13 jmcneill [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) },
2697 1.13 jmcneill [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) },
2698 1.13 jmcneill [MCX_ETHER_CAP_56G_R4] = { IFM_56G_R4, IF_Gbps(56) },
2699 1.13 jmcneill [MCX_ETHER_CAP_10G_CR] = { IFM_10G_CR1, IF_Gbps(10) },
2700 1.13 jmcneill [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) },
2701 1.13 jmcneill [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) },
2702 1.13 jmcneill [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) },
2703 1.13 jmcneill [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) },
2704 1.13 jmcneill [MCX_ETHER_CAP_50G_SR2] = { IFM_50G_SR2, IF_Gbps(50) },
2705 1.13 jmcneill [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) },
2706 1.13 jmcneill [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) },
2707 1.13 jmcneill [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) },
2708 1.13 jmcneill [MCX_ETHER_CAP_100G_LR4] = { IFM_100G_LR4, IF_Gbps(100) },
2709 1.13 jmcneill [MCX_ETHER_CAP_100_TX] = { IFM_100_TX, IF_Mbps(100) },
2710 1.13 jmcneill [MCX_ETHER_CAP_1000_T] = { IFM_1000_T, IF_Gbps(1) },
2711 1.13 jmcneill [MCX_ETHER_CAP_10G_T] = { IFM_10G_T, IF_Gbps(10) },
2712 1.13 jmcneill [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) },
2713 1.13 jmcneill [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) },
2714 1.13 jmcneill [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) },
2715 1.13 jmcneill [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) },
2716 1.13 jmcneill [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) },
2717 1.1 jmcneill };
2718 1.1 jmcneill
2719 1.1 jmcneill static int
2720 1.13 jmcneill mcx_get_id(uint32_t val)
2721 1.13 jmcneill {
2722 1.13 jmcneill return be32toh(val) & 0x00ffffff;
2723 1.13 jmcneill }
2724 1.13 jmcneill
2725 1.13 jmcneill static int
2726 1.1 jmcneill mcx_match(device_t parent, cfdata_t cf, void *aux)
2727 1.1 jmcneill {
2728 1.1 jmcneill struct pci_attach_args *pa = aux;
2729 1.1 jmcneill int n;
2730 1.1 jmcneill
2731 1.1 jmcneill for (n = 0; n < __arraycount(mcx_devices); n++) {
2732 1.1 jmcneill if (PCI_VENDOR(pa->pa_id) == mcx_devices[n].vendor &&
2733 1.1 jmcneill PCI_PRODUCT(pa->pa_id) == mcx_devices[n].product)
2734 1.1 jmcneill return 1;
2735 1.1 jmcneill }
2736 1.1 jmcneill
2737 1.1 jmcneill return 0;
2738 1.1 jmcneill }
2739 1.1 jmcneill
2740 1.1 jmcneill void
2741 1.1 jmcneill mcx_attach(device_t parent, device_t self, void *aux)
2742 1.1 jmcneill {
2743 1.1 jmcneill struct mcx_softc *sc = device_private(self);
2744 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
2745 1.1 jmcneill struct pci_attach_args *pa = aux;
2746 1.1 jmcneill uint8_t enaddr[ETHER_ADDR_LEN];
2747 1.1 jmcneill int counts[PCI_INTR_TYPE_SIZE];
2748 1.15 jmcneill char intrxname[32];
2749 1.1 jmcneill pcireg_t memtype;
2750 1.1 jmcneill uint32_t r;
2751 1.1 jmcneill unsigned int cq_stride;
2752 1.1 jmcneill unsigned int cq_size;
2753 1.15 jmcneill int i, msix;
2754 1.15 jmcneill kcpuset_t *affinity;
2755 1.1 jmcneill
2756 1.1 jmcneill sc->sc_dev = self;
2757 1.1 jmcneill sc->sc_pc = pa->pa_pc;
2758 1.1 jmcneill sc->sc_tag = pa->pa_tag;
2759 1.11 thorpej if (pci_dma64_available(pa))
2760 1.11 thorpej sc->sc_dmat = pa->pa_dmat64;
2761 1.11 thorpej else
2762 1.11 thorpej sc->sc_dmat = pa->pa_dmat;
2763 1.1 jmcneill
2764 1.1 jmcneill /* Map the PCI memory space */
2765 1.1 jmcneill memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR);
2766 1.1 jmcneill if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype,
2767 1.1 jmcneill 0 /*BUS_SPACE_MAP_PREFETCHABLE*/, &sc->sc_memt, &sc->sc_memh,
2768 1.1 jmcneill NULL, &sc->sc_mems)) {
2769 1.1 jmcneill aprint_error(": unable to map register memory\n");
2770 1.1 jmcneill return;
2771 1.1 jmcneill }
2772 1.1 jmcneill
2773 1.1 jmcneill pci_aprint_devinfo(pa, "Ethernet controller");
2774 1.1 jmcneill
2775 1.12 thorpej mutex_init(&sc->sc_media_mutex, MUTEX_DEFAULT, IPL_SOFTNET);
2776 1.12 thorpej
2777 1.1 jmcneill if (mcx_version(sc) != 0) {
2778 1.1 jmcneill /* error printed by mcx_version */
2779 1.1 jmcneill goto unmap;
2780 1.1 jmcneill }
2781 1.1 jmcneill
2782 1.1 jmcneill r = mcx_rd(sc, MCX_CMDQ_ADDR_LO);
2783 1.1 jmcneill cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */
2784 1.1 jmcneill cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */
2785 1.1 jmcneill if (cq_size > MCX_MAX_CQE) {
2786 1.1 jmcneill aprint_error_dev(self,
2787 1.1 jmcneill "command queue size overflow %u\n", cq_size);
2788 1.1 jmcneill goto unmap;
2789 1.1 jmcneill }
2790 1.1 jmcneill if (cq_stride < sizeof(struct mcx_cmdq_entry)) {
2791 1.1 jmcneill aprint_error_dev(self,
2792 1.1 jmcneill "command queue entry size underflow %u\n", cq_stride);
2793 1.1 jmcneill goto unmap;
2794 1.1 jmcneill }
2795 1.1 jmcneill if (cq_stride * cq_size > MCX_PAGE_SIZE) {
2796 1.1 jmcneill aprint_error_dev(self, "command queue page overflow\n");
2797 1.1 jmcneill goto unmap;
2798 1.1 jmcneill }
2799 1.1 jmcneill
2800 1.15 jmcneill if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE,
2801 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
2802 1.1 jmcneill aprint_error_dev(self, "unable to allocate doorbell memory\n");
2803 1.1 jmcneill goto unmap;
2804 1.1 jmcneill }
2805 1.1 jmcneill
2806 1.1 jmcneill if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE,
2807 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
2808 1.1 jmcneill aprint_error_dev(self, "unable to allocate command queue\n");
2809 1.1 jmcneill goto dbfree;
2810 1.1 jmcneill }
2811 1.1 jmcneill
2812 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
2813 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t),
2814 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
2815 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem));
2816 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t),
2817 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
2818 1.1 jmcneill
2819 1.1 jmcneill if (mcx_init_wait(sc) != 0) {
2820 1.1 jmcneill aprint_error_dev(self, "timeout waiting for init\n");
2821 1.1 jmcneill goto cqfree;
2822 1.1 jmcneill }
2823 1.1 jmcneill
2824 1.1 jmcneill sc->sc_cmdq_mask = cq_size - 1;
2825 1.1 jmcneill sc->sc_cmdq_size = cq_stride;
2826 1.1 jmcneill
2827 1.1 jmcneill if (mcx_enable_hca(sc) != 0) {
2828 1.1 jmcneill /* error printed by mcx_enable_hca */
2829 1.1 jmcneill goto cqfree;
2830 1.1 jmcneill }
2831 1.1 jmcneill
2832 1.1 jmcneill if (mcx_issi(sc) != 0) {
2833 1.1 jmcneill /* error printed by mcx_issi */
2834 1.1 jmcneill goto teardown;
2835 1.1 jmcneill }
2836 1.1 jmcneill
2837 1.1 jmcneill if (mcx_pages(sc, &sc->sc_boot_pages,
2838 1.1 jmcneill htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) {
2839 1.1 jmcneill /* error printed by mcx_pages */
2840 1.1 jmcneill goto teardown;
2841 1.1 jmcneill }
2842 1.1 jmcneill
2843 1.1 jmcneill if (mcx_hca_max_caps(sc) != 0) {
2844 1.1 jmcneill /* error printed by mcx_hca_max_caps */
2845 1.1 jmcneill goto teardown;
2846 1.1 jmcneill }
2847 1.1 jmcneill
2848 1.1 jmcneill if (mcx_hca_set_caps(sc) != 0) {
2849 1.1 jmcneill /* error printed by mcx_hca_set_caps */
2850 1.1 jmcneill goto teardown;
2851 1.1 jmcneill }
2852 1.1 jmcneill
2853 1.1 jmcneill if (mcx_pages(sc, &sc->sc_init_pages,
2854 1.1 jmcneill htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) {
2855 1.1 jmcneill /* error printed by mcx_pages */
2856 1.1 jmcneill goto teardown;
2857 1.1 jmcneill }
2858 1.1 jmcneill
2859 1.1 jmcneill if (mcx_init_hca(sc) != 0) {
2860 1.1 jmcneill /* error printed by mcx_init_hca */
2861 1.1 jmcneill goto teardown;
2862 1.1 jmcneill }
2863 1.1 jmcneill
2864 1.1 jmcneill if (mcx_pages(sc, &sc->sc_regular_pages,
2865 1.1 jmcneill htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) {
2866 1.1 jmcneill /* error printed by mcx_pages */
2867 1.1 jmcneill goto teardown;
2868 1.1 jmcneill }
2869 1.1 jmcneill
2870 1.1 jmcneill /* apparently not necessary? */
2871 1.1 jmcneill if (mcx_set_driver_version(sc) != 0) {
2872 1.1 jmcneill /* error printed by mcx_set_driver_version */
2873 1.1 jmcneill goto teardown;
2874 1.1 jmcneill }
2875 1.1 jmcneill
2876 1.1 jmcneill if (mcx_iff(sc) != 0) { /* modify nic vport context */
2877 1.1 jmcneill /* error printed by mcx_iff? */
2878 1.1 jmcneill goto teardown;
2879 1.1 jmcneill }
2880 1.1 jmcneill
2881 1.15 jmcneill if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) {
2882 1.1 jmcneill /* error printed by mcx_alloc_uar */
2883 1.1 jmcneill goto teardown;
2884 1.1 jmcneill }
2885 1.1 jmcneill
2886 1.1 jmcneill if (mcx_alloc_pd(sc) != 0) {
2887 1.1 jmcneill /* error printed by mcx_alloc_pd */
2888 1.1 jmcneill goto teardown;
2889 1.1 jmcneill }
2890 1.1 jmcneill
2891 1.1 jmcneill if (mcx_alloc_tdomain(sc) != 0) {
2892 1.1 jmcneill /* error printed by mcx_alloc_tdomain */
2893 1.1 jmcneill goto teardown;
2894 1.1 jmcneill }
2895 1.1 jmcneill
2896 1.1 jmcneill /*
2897 1.1 jmcneill * PRM makes no mention of msi interrupts, just legacy and msi-x.
2898 1.1 jmcneill * mellanox support tells me legacy interrupts are not supported,
2899 1.1 jmcneill * so we're stuck with just msi-x.
2900 1.1 jmcneill */
2901 1.15 jmcneill counts[PCI_INTR_TYPE_MSIX] = -1;
2902 1.1 jmcneill counts[PCI_INTR_TYPE_MSI] = 0;
2903 1.1 jmcneill counts[PCI_INTR_TYPE_INTX] = 0;
2904 1.1 jmcneill if (pci_intr_alloc(pa, &sc->sc_intrs, counts, PCI_INTR_TYPE_MSIX) != 0) {
2905 1.1 jmcneill aprint_error_dev(self, "unable to allocate interrupt\n");
2906 1.1 jmcneill goto teardown;
2907 1.1 jmcneill }
2908 1.15 jmcneill if (counts[PCI_INTR_TYPE_MSIX] < 2) {
2909 1.15 jmcneill aprint_error_dev(self, "not enough MSI-X vectors\n");
2910 1.15 jmcneill goto teardown;
2911 1.15 jmcneill }
2912 1.1 jmcneill KASSERT(pci_intr_type(sc->sc_pc, sc->sc_intrs[0]) == PCI_INTR_TYPE_MSIX);
2913 1.15 jmcneill snprintf(intrxname, sizeof(intrxname), "%s adminq", DEVNAME(sc));
2914 1.15 jmcneill sc->sc_ihc = mcx_establish_intr(sc, 0, NULL, mcx_admin_intr, sc,
2915 1.15 jmcneill intrxname);
2916 1.15 jmcneill if (sc->sc_ihc == NULL) {
2917 1.15 jmcneill aprint_error_dev(self, "couldn't establish adminq interrupt\n");
2918 1.1 jmcneill goto teardown;
2919 1.1 jmcneill }
2920 1.1 jmcneill
2921 1.15 jmcneill if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar,
2922 1.15 jmcneill (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) |
2923 1.15 jmcneill (1ull << MCX_EVENT_TYPE_PORT_CHANGE) |
2924 1.15 jmcneill (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) |
2925 1.15 jmcneill (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) {
2926 1.1 jmcneill /* error printed by mcx_create_eq */
2927 1.1 jmcneill goto teardown;
2928 1.1 jmcneill }
2929 1.1 jmcneill
2930 1.1 jmcneill if (mcx_query_nic_vport_context(sc, enaddr) != 0) {
2931 1.1 jmcneill /* error printed by mcx_query_nic_vport_context */
2932 1.1 jmcneill goto teardown;
2933 1.1 jmcneill }
2934 1.1 jmcneill
2935 1.1 jmcneill if (mcx_query_special_contexts(sc) != 0) {
2936 1.1 jmcneill /* error printed by mcx_query_special_contexts */
2937 1.1 jmcneill goto teardown;
2938 1.1 jmcneill }
2939 1.1 jmcneill
2940 1.1 jmcneill if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) {
2941 1.1 jmcneill /* error printed by mcx_set_port_mtu */
2942 1.1 jmcneill goto teardown;
2943 1.1 jmcneill }
2944 1.1 jmcneill
2945 1.1 jmcneill aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
2946 1.1 jmcneill ether_sprintf(enaddr));
2947 1.1 jmcneill
2948 1.15 jmcneill msix = counts[PCI_INTR_TYPE_MSIX];
2949 1.15 jmcneill msix--; /* admin ops took one */
2950 1.15 jmcneill
2951 1.15 jmcneill sc->sc_nqueues = uimin(MCX_MAX_QUEUES, msix);
2952 1.15 jmcneill sc->sc_nqueues = uimin(sc->sc_nqueues, ncpu);
2953 1.15 jmcneill sc->sc_queues = kmem_zalloc(sc->sc_nqueues * sizeof(*sc->sc_queues),
2954 1.15 jmcneill KM_SLEEP);
2955 1.15 jmcneill
2956 1.1 jmcneill strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
2957 1.1 jmcneill ifp->if_softc = sc;
2958 1.1 jmcneill ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
2959 1.1 jmcneill #ifdef MCX_MPSAFE
2960 1.1 jmcneill ifp->if_extflags = IFEF_MPSAFE;
2961 1.1 jmcneill #endif
2962 1.1 jmcneill ifp->if_init = mcx_init;
2963 1.1 jmcneill ifp->if_stop = mcx_stop;
2964 1.1 jmcneill ifp->if_ioctl = mcx_ioctl;
2965 1.1 jmcneill ifp->if_start = mcx_start;
2966 1.16 jmcneill if (sc->sc_nqueues > 1) {
2967 1.16 jmcneill ifp->if_transmit = mcx_transmit;
2968 1.16 jmcneill }
2969 1.1 jmcneill ifp->if_watchdog = mcx_watchdog;
2970 1.1 jmcneill ifp->if_mtu = sc->sc_hardmtu;
2971 1.15 jmcneill ifp->if_capabilities = IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_IPv4_Tx |
2972 1.15 jmcneill IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv4_Tx |
2973 1.15 jmcneill IFCAP_CSUM_UDPv6_Rx | IFCAP_CSUM_UDPv6_Tx |
2974 1.15 jmcneill IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv4_Tx |
2975 1.15 jmcneill IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_TCPv6_Tx;
2976 1.1 jmcneill IFQ_SET_MAXLEN(&ifp->if_snd, 1024);
2977 1.1 jmcneill IFQ_SET_READY(&ifp->if_snd);
2978 1.1 jmcneill
2979 1.15 jmcneill sc->sc_ec.ec_capabilities = ETHERCAP_JUMBO_MTU |
2980 1.15 jmcneill ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING;
2981 1.15 jmcneill sc->sc_ec.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;
2982 1.1 jmcneill
2983 1.1 jmcneill sc->sc_ec.ec_ifmedia = &sc->sc_media;
2984 1.12 thorpej ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, mcx_media_change,
2985 1.12 thorpej mcx_media_status, &sc->sc_media_mutex);
2986 1.1 jmcneill mcx_media_add_types(sc);
2987 1.1 jmcneill ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
2988 1.1 jmcneill ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
2989 1.1 jmcneill
2990 1.1 jmcneill if_attach(ifp);
2991 1.1 jmcneill if_deferred_start_init(ifp, NULL);
2992 1.1 jmcneill
2993 1.1 jmcneill ether_ifattach(ifp, enaddr);
2994 1.1 jmcneill
2995 1.15 jmcneill kcpuset_create(&affinity, false);
2996 1.15 jmcneill kcpuset_set(affinity, 0);
2997 1.15 jmcneill
2998 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
2999 1.15 jmcneill struct mcx_queues *q = &sc->sc_queues[i];
3000 1.15 jmcneill struct mcx_rx *rx = &q->q_rx;
3001 1.15 jmcneill struct mcx_tx *tx = &q->q_tx;
3002 1.15 jmcneill int vec;
3003 1.15 jmcneill
3004 1.15 jmcneill vec = i + 1;
3005 1.15 jmcneill q->q_sc = sc;
3006 1.15 jmcneill q->q_index = i;
3007 1.15 jmcneill
3008 1.15 jmcneill if (mcx_alloc_uar(sc, &q->q_uar) != 0) {
3009 1.15 jmcneill aprint_error_dev(self, "unable to alloc uar %d\n", i);
3010 1.15 jmcneill goto teardown;
3011 1.15 jmcneill }
3012 1.15 jmcneill
3013 1.15 jmcneill if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) {
3014 1.15 jmcneill aprint_error_dev(self,
3015 1.15 jmcneill "unable to create event queue %d\n", i);
3016 1.15 jmcneill goto teardown;
3017 1.15 jmcneill }
3018 1.15 jmcneill
3019 1.15 jmcneill rx->rx_softc = sc;
3020 1.15 jmcneill callout_init(&rx->rx_refill, CALLOUT_FLAGS);
3021 1.15 jmcneill callout_setfunc(&rx->rx_refill, mcx_refill, rx);
3022 1.15 jmcneill
3023 1.15 jmcneill tx->tx_softc = sc;
3024 1.16 jmcneill mutex_init(&tx->tx_lock, MUTEX_DEFAULT, IPL_NET);
3025 1.16 jmcneill tx->tx_pcq = pcq_create(MCX_TXQ_NUM, KM_SLEEP);
3026 1.16 jmcneill tx->tx_softint = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,
3027 1.16 jmcneill mcx_deferred_transmit, tx);
3028 1.15 jmcneill
3029 1.15 jmcneill snprintf(intrxname, sizeof(intrxname), "%s queue %d",
3030 1.15 jmcneill DEVNAME(sc), i);
3031 1.15 jmcneill q->q_ihc = mcx_establish_intr(sc, vec, affinity, mcx_cq_intr,
3032 1.15 jmcneill q, intrxname);
3033 1.15 jmcneill }
3034 1.15 jmcneill
3035 1.1 jmcneill callout_init(&sc->sc_calibrate, CALLOUT_FLAGS);
3036 1.1 jmcneill callout_setfunc(&sc->sc_calibrate, mcx_calibrate, sc);
3037 1.1 jmcneill
3038 1.1 jmcneill if (workqueue_create(&sc->sc_workq, "mcxportchg", mcx_port_change, sc,
3039 1.1 jmcneill PRI_NONE, IPL_NET, 0) != 0) {
3040 1.1 jmcneill aprint_error_dev(self, "couldn't create port change workq\n");
3041 1.1 jmcneill goto teardown;
3042 1.1 jmcneill }
3043 1.1 jmcneill
3044 1.1 jmcneill mcx_port_change(&sc->sc_port_change, sc);
3045 1.1 jmcneill
3046 1.15 jmcneill sc->sc_mac_flow_table_id = -1;
3047 1.15 jmcneill sc->sc_rss_flow_table_id = -1;
3048 1.15 jmcneill sc->sc_rqt = -1;
3049 1.1 jmcneill for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) {
3050 1.15 jmcneill struct mcx_flow_group *mfg = &sc->sc_flow_group[i];
3051 1.15 jmcneill mfg->g_id = -1;
3052 1.15 jmcneill mfg->g_table = -1;
3053 1.15 jmcneill mfg->g_size = 0;
3054 1.15 jmcneill mfg->g_start = 0;
3055 1.1 jmcneill }
3056 1.1 jmcneill sc->sc_extra_mcast = 0;
3057 1.1 jmcneill memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows));
3058 1.15 jmcneill
3059 1.15 jmcneill #if NKSTAT > 0
3060 1.15 jmcneill mcx_kstat_attach(sc);
3061 1.15 jmcneill #endif
3062 1.15 jmcneill mcx_timecounter_attach(sc);
3063 1.1 jmcneill return;
3064 1.1 jmcneill
3065 1.1 jmcneill teardown:
3066 1.1 jmcneill mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL));
3067 1.1 jmcneill /* error printed by mcx_teardown_hca, and we're already unwinding */
3068 1.1 jmcneill cqfree:
3069 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32);
3070 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
3071 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3072 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) |
3073 1.1 jmcneill MCX_CMDQ_INTERFACE_DISABLED);
3074 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t),
3075 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3076 1.1 jmcneill
3077 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0);
3078 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t),
3079 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3080 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED);
3081 1.1 jmcneill
3082 1.1 jmcneill mcx_dmamem_free(sc, &sc->sc_cmdq_mem);
3083 1.1 jmcneill dbfree:
3084 1.1 jmcneill mcx_dmamem_free(sc, &sc->sc_doorbell_mem);
3085 1.1 jmcneill unmap:
3086 1.1 jmcneill bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
3087 1.1 jmcneill sc->sc_mems = 0;
3088 1.1 jmcneill }
3089 1.1 jmcneill
3090 1.15 jmcneill static void *
3091 1.15 jmcneill mcx_establish_intr(struct mcx_softc *sc, int index, kcpuset_t *affinity,
3092 1.15 jmcneill int (*func)(void *), void *arg, const char *xname)
3093 1.15 jmcneill {
3094 1.15 jmcneill char intrbuf[PCI_INTRSTR_LEN];
3095 1.15 jmcneill const char *intrstr;
3096 1.15 jmcneill void *ih;
3097 1.15 jmcneill
3098 1.15 jmcneill pci_intr_setattr(sc->sc_pc, &sc->sc_intrs[index], PCI_INTR_MPSAFE,
3099 1.15 jmcneill true);
3100 1.15 jmcneill
3101 1.15 jmcneill intrstr = pci_intr_string(sc->sc_pc, sc->sc_intrs[index], intrbuf,
3102 1.15 jmcneill sizeof(intrbuf));
3103 1.15 jmcneill ih = pci_intr_establish_xname(sc->sc_pc, sc->sc_intrs[index], IPL_NET,
3104 1.15 jmcneill func, arg, xname);
3105 1.15 jmcneill if (ih == NULL) {
3106 1.15 jmcneill aprint_error_dev(sc->sc_dev,
3107 1.15 jmcneill "unable to establish interrupt%s%s\n",
3108 1.15 jmcneill intrstr ? " at " : "",
3109 1.15 jmcneill intrstr ? intrstr : "");
3110 1.15 jmcneill return NULL;
3111 1.15 jmcneill }
3112 1.15 jmcneill
3113 1.15 jmcneill if (affinity != NULL && index > 0) {
3114 1.15 jmcneill /* Round-robin affinity */
3115 1.15 jmcneill kcpuset_zero(affinity);
3116 1.15 jmcneill kcpuset_set(affinity, (index - 1) % ncpu);
3117 1.15 jmcneill interrupt_distribute(ih, affinity, NULL);
3118 1.15 jmcneill }
3119 1.15 jmcneill
3120 1.15 jmcneill return ih;
3121 1.15 jmcneill }
3122 1.15 jmcneill
3123 1.1 jmcneill static void
3124 1.1 jmcneill mcx_rxr_init(struct mcx_rxring *rxr, u_int lwm __unused, u_int hwm)
3125 1.1 jmcneill {
3126 1.1 jmcneill rxr->rxr_total = hwm;
3127 1.1 jmcneill rxr->rxr_inuse = 0;
3128 1.1 jmcneill }
3129 1.1 jmcneill
3130 1.1 jmcneill static u_int
3131 1.1 jmcneill mcx_rxr_get(struct mcx_rxring *rxr, u_int max)
3132 1.1 jmcneill {
3133 1.1 jmcneill const u_int taken = MIN(max, rxr->rxr_total - rxr->rxr_inuse);
3134 1.1 jmcneill
3135 1.1 jmcneill rxr->rxr_inuse += taken;
3136 1.1 jmcneill
3137 1.1 jmcneill return taken;
3138 1.1 jmcneill }
3139 1.1 jmcneill
3140 1.1 jmcneill static void
3141 1.1 jmcneill mcx_rxr_put(struct mcx_rxring *rxr, u_int n)
3142 1.1 jmcneill {
3143 1.1 jmcneill rxr->rxr_inuse -= n;
3144 1.1 jmcneill }
3145 1.1 jmcneill
3146 1.1 jmcneill static u_int
3147 1.1 jmcneill mcx_rxr_inuse(struct mcx_rxring *rxr)
3148 1.1 jmcneill {
3149 1.1 jmcneill return rxr->rxr_inuse;
3150 1.1 jmcneill }
3151 1.1 jmcneill
3152 1.1 jmcneill static int
3153 1.1 jmcneill mcx_version(struct mcx_softc *sc)
3154 1.1 jmcneill {
3155 1.1 jmcneill uint32_t fw0, fw1;
3156 1.1 jmcneill uint16_t cmdif;
3157 1.1 jmcneill
3158 1.1 jmcneill fw0 = mcx_rd(sc, MCX_FW_VER);
3159 1.1 jmcneill fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER);
3160 1.1 jmcneill
3161 1.1 jmcneill aprint_normal_dev(sc->sc_dev, "FW %u.%u.%04u\n", MCX_FW_VER_MAJOR(fw0),
3162 1.1 jmcneill MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1));
3163 1.1 jmcneill
3164 1.1 jmcneill cmdif = MCX_CMDIF(fw1);
3165 1.1 jmcneill if (cmdif != MCX_CMD_IF_SUPPORTED) {
3166 1.1 jmcneill aprint_error_dev(sc->sc_dev,
3167 1.1 jmcneill "unsupported command interface %u\n", cmdif);
3168 1.1 jmcneill return (-1);
3169 1.1 jmcneill }
3170 1.1 jmcneill
3171 1.1 jmcneill return (0);
3172 1.1 jmcneill }
3173 1.1 jmcneill
3174 1.1 jmcneill static int
3175 1.1 jmcneill mcx_init_wait(struct mcx_softc *sc)
3176 1.1 jmcneill {
3177 1.1 jmcneill unsigned int i;
3178 1.1 jmcneill uint32_t r;
3179 1.1 jmcneill
3180 1.1 jmcneill for (i = 0; i < 2000; i++) {
3181 1.1 jmcneill r = mcx_rd(sc, MCX_STATE);
3182 1.1 jmcneill if ((r & MCX_STATE_MASK) == MCX_STATE_READY)
3183 1.1 jmcneill return (0);
3184 1.1 jmcneill
3185 1.1 jmcneill delay(1000);
3186 1.1 jmcneill mcx_bar(sc, MCX_STATE, sizeof(uint32_t),
3187 1.1 jmcneill BUS_SPACE_BARRIER_READ);
3188 1.1 jmcneill }
3189 1.1 jmcneill
3190 1.1 jmcneill return (-1);
3191 1.1 jmcneill }
3192 1.1 jmcneill
3193 1.1 jmcneill static uint8_t
3194 1.1 jmcneill mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3195 1.1 jmcneill unsigned int msec)
3196 1.1 jmcneill {
3197 1.1 jmcneill unsigned int i;
3198 1.1 jmcneill
3199 1.1 jmcneill for (i = 0; i < msec; i++) {
3200 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
3201 1.1 jmcneill 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW);
3202 1.1 jmcneill
3203 1.1 jmcneill if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) ==
3204 1.13 jmcneill MCX_CQ_STATUS_OWN_SW)
3205 1.1 jmcneill return (0);
3206 1.1 jmcneill
3207 1.1 jmcneill delay(1000);
3208 1.1 jmcneill }
3209 1.1 jmcneill
3210 1.1 jmcneill return (ETIMEDOUT);
3211 1.1 jmcneill }
3212 1.1 jmcneill
3213 1.1 jmcneill static uint32_t
3214 1.1 jmcneill mcx_mix_u64(uint32_t xor, uint64_t u64)
3215 1.1 jmcneill {
3216 1.1 jmcneill xor ^= u64 >> 32;
3217 1.1 jmcneill xor ^= u64;
3218 1.1 jmcneill
3219 1.1 jmcneill return (xor);
3220 1.1 jmcneill }
3221 1.1 jmcneill
3222 1.1 jmcneill static uint32_t
3223 1.1 jmcneill mcx_mix_u32(uint32_t xor, uint32_t u32)
3224 1.1 jmcneill {
3225 1.1 jmcneill xor ^= u32;
3226 1.1 jmcneill
3227 1.1 jmcneill return (xor);
3228 1.1 jmcneill }
3229 1.1 jmcneill
3230 1.1 jmcneill static uint32_t
3231 1.1 jmcneill mcx_mix_u8(uint32_t xor, uint8_t u8)
3232 1.1 jmcneill {
3233 1.1 jmcneill xor ^= u8;
3234 1.1 jmcneill
3235 1.1 jmcneill return (xor);
3236 1.1 jmcneill }
3237 1.1 jmcneill
3238 1.1 jmcneill static uint8_t
3239 1.1 jmcneill mcx_mix_done(uint32_t xor)
3240 1.1 jmcneill {
3241 1.1 jmcneill xor ^= xor >> 16;
3242 1.1 jmcneill xor ^= xor >> 8;
3243 1.1 jmcneill
3244 1.1 jmcneill return (xor);
3245 1.1 jmcneill }
3246 1.1 jmcneill
3247 1.1 jmcneill static uint8_t
3248 1.1 jmcneill mcx_xor(const void *buf, size_t len)
3249 1.1 jmcneill {
3250 1.1 jmcneill const uint32_t *dwords = buf;
3251 1.1 jmcneill uint32_t xor = 0xff;
3252 1.1 jmcneill size_t i;
3253 1.1 jmcneill
3254 1.1 jmcneill len /= sizeof(*dwords);
3255 1.1 jmcneill
3256 1.1 jmcneill for (i = 0; i < len; i++)
3257 1.1 jmcneill xor ^= dwords[i];
3258 1.1 jmcneill
3259 1.1 jmcneill return (mcx_mix_done(xor));
3260 1.1 jmcneill }
3261 1.1 jmcneill
3262 1.1 jmcneill static uint8_t
3263 1.1 jmcneill mcx_cmdq_token(struct mcx_softc *sc)
3264 1.1 jmcneill {
3265 1.1 jmcneill uint8_t token;
3266 1.1 jmcneill
3267 1.1 jmcneill do {
3268 1.1 jmcneill token = ++sc->sc_cmdq_token;
3269 1.1 jmcneill } while (token == 0);
3270 1.1 jmcneill
3271 1.1 jmcneill return (token);
3272 1.1 jmcneill }
3273 1.1 jmcneill
3274 1.1 jmcneill static void
3275 1.1 jmcneill mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3276 1.1 jmcneill uint32_t ilen, uint32_t olen, uint8_t token)
3277 1.1 jmcneill {
3278 1.1 jmcneill memset(cqe, 0, sc->sc_cmdq_size);
3279 1.1 jmcneill
3280 1.1 jmcneill cqe->cq_type = MCX_CMDQ_TYPE_PCIE;
3281 1.1 jmcneill be32enc(&cqe->cq_input_length, ilen);
3282 1.1 jmcneill be32enc(&cqe->cq_output_length, olen);
3283 1.1 jmcneill cqe->cq_token = token;
3284 1.1 jmcneill cqe->cq_status = MCX_CQ_STATUS_OWN_HW;
3285 1.1 jmcneill }
3286 1.1 jmcneill
3287 1.1 jmcneill static void
3288 1.1 jmcneill mcx_cmdq_sign(struct mcx_cmdq_entry *cqe)
3289 1.1 jmcneill {
3290 1.1 jmcneill cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe));
3291 1.1 jmcneill }
3292 1.1 jmcneill
3293 1.1 jmcneill static int
3294 1.1 jmcneill mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe)
3295 1.1 jmcneill {
3296 1.1 jmcneill /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */
3297 1.1 jmcneill return (0);
3298 1.1 jmcneill }
3299 1.1 jmcneill
3300 1.1 jmcneill static void *
3301 1.1 jmcneill mcx_cmdq_in(struct mcx_cmdq_entry *cqe)
3302 1.1 jmcneill {
3303 1.1 jmcneill return (&cqe->cq_input_data);
3304 1.1 jmcneill }
3305 1.1 jmcneill
3306 1.1 jmcneill static void *
3307 1.1 jmcneill mcx_cmdq_out(struct mcx_cmdq_entry *cqe)
3308 1.1 jmcneill {
3309 1.1 jmcneill return (&cqe->cq_output_data);
3310 1.1 jmcneill }
3311 1.1 jmcneill
3312 1.1 jmcneill static void
3313 1.1 jmcneill mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3314 1.1 jmcneill unsigned int slot)
3315 1.1 jmcneill {
3316 1.1 jmcneill mcx_cmdq_sign(cqe);
3317 1.1 jmcneill
3318 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem),
3319 1.1 jmcneill 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW);
3320 1.1 jmcneill
3321 1.1 jmcneill mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot);
3322 1.15 jmcneill mcx_bar(sc, MCX_CMDQ_DOORBELL, sizeof(uint32_t),
3323 1.15 jmcneill BUS_SPACE_BARRIER_WRITE);
3324 1.1 jmcneill }
3325 1.1 jmcneill
3326 1.1 jmcneill static int
3327 1.1 jmcneill mcx_enable_hca(struct mcx_softc *sc)
3328 1.1 jmcneill {
3329 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3330 1.1 jmcneill struct mcx_cmd_enable_hca_in *in;
3331 1.1 jmcneill struct mcx_cmd_enable_hca_out *out;
3332 1.1 jmcneill int error;
3333 1.1 jmcneill uint8_t status;
3334 1.1 jmcneill
3335 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3336 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3337 1.1 jmcneill
3338 1.1 jmcneill in = mcx_cmdq_in(cqe);
3339 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA);
3340 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3341 1.1 jmcneill in->cmd_function_id = htobe16(0);
3342 1.1 jmcneill
3343 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3344 1.1 jmcneill
3345 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3346 1.1 jmcneill if (error != 0) {
3347 1.1 jmcneill printf(", hca enable timeout\n");
3348 1.1 jmcneill return (-1);
3349 1.1 jmcneill }
3350 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
3351 1.1 jmcneill printf(", hca enable command corrupt\n");
3352 1.1 jmcneill return (-1);
3353 1.1 jmcneill }
3354 1.1 jmcneill
3355 1.1 jmcneill status = cqe->cq_output_data[0];
3356 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
3357 1.1 jmcneill printf(", hca enable failed (%x)\n", status);
3358 1.1 jmcneill return (-1);
3359 1.1 jmcneill }
3360 1.1 jmcneill
3361 1.1 jmcneill return (0);
3362 1.1 jmcneill }
3363 1.1 jmcneill
3364 1.1 jmcneill static int
3365 1.1 jmcneill mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile)
3366 1.1 jmcneill {
3367 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3368 1.1 jmcneill struct mcx_cmd_teardown_hca_in *in;
3369 1.1 jmcneill struct mcx_cmd_teardown_hca_out *out;
3370 1.1 jmcneill int error;
3371 1.1 jmcneill uint8_t status;
3372 1.1 jmcneill
3373 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3374 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3375 1.1 jmcneill
3376 1.1 jmcneill in = mcx_cmdq_in(cqe);
3377 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA);
3378 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3379 1.1 jmcneill in->cmd_profile = profile;
3380 1.1 jmcneill
3381 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3382 1.1 jmcneill
3383 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3384 1.1 jmcneill if (error != 0) {
3385 1.1 jmcneill printf(", hca teardown timeout\n");
3386 1.1 jmcneill return (-1);
3387 1.1 jmcneill }
3388 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
3389 1.1 jmcneill printf(", hca teardown command corrupt\n");
3390 1.1 jmcneill return (-1);
3391 1.1 jmcneill }
3392 1.1 jmcneill
3393 1.1 jmcneill status = cqe->cq_output_data[0];
3394 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
3395 1.1 jmcneill printf(", hca teardown failed (%x)\n", status);
3396 1.1 jmcneill return (-1);
3397 1.1 jmcneill }
3398 1.1 jmcneill
3399 1.1 jmcneill return (0);
3400 1.1 jmcneill }
3401 1.1 jmcneill
3402 1.1 jmcneill static int
3403 1.1 jmcneill mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
3404 1.1 jmcneill unsigned int nmb, uint64_t *ptr, uint8_t token)
3405 1.1 jmcneill {
3406 1.1 jmcneill uint8_t *kva;
3407 1.1 jmcneill uint64_t dva;
3408 1.1 jmcneill int i;
3409 1.1 jmcneill int error;
3410 1.1 jmcneill
3411 1.1 jmcneill error = mcx_dmamem_alloc(sc, mxm,
3412 1.1 jmcneill nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN);
3413 1.1 jmcneill if (error != 0)
3414 1.1 jmcneill return (error);
3415 1.1 jmcneill
3416 1.1 jmcneill mcx_dmamem_zero(mxm);
3417 1.1 jmcneill
3418 1.1 jmcneill dva = MCX_DMA_DVA(mxm);
3419 1.1 jmcneill kva = MCX_DMA_KVA(mxm);
3420 1.1 jmcneill for (i = 0; i < nmb; i++) {
3421 1.1 jmcneill struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva;
3422 1.1 jmcneill
3423 1.1 jmcneill /* patch the cqe or mbox pointing at this one */
3424 1.1 jmcneill be64enc(ptr, dva);
3425 1.1 jmcneill
3426 1.1 jmcneill /* fill in this mbox */
3427 1.1 jmcneill be32enc(&mbox->mb_block_number, i);
3428 1.1 jmcneill mbox->mb_token = token;
3429 1.1 jmcneill
3430 1.1 jmcneill /* move to the next one */
3431 1.1 jmcneill ptr = &mbox->mb_next_ptr;
3432 1.1 jmcneill
3433 1.1 jmcneill dva += MCX_CMDQ_MAILBOX_SIZE;
3434 1.1 jmcneill kva += MCX_CMDQ_MAILBOX_SIZE;
3435 1.1 jmcneill }
3436 1.1 jmcneill
3437 1.1 jmcneill return (0);
3438 1.1 jmcneill }
3439 1.1 jmcneill
3440 1.1 jmcneill static uint32_t
3441 1.1 jmcneill mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb)
3442 1.1 jmcneill {
3443 1.1 jmcneill uint32_t xor = 0xff;
3444 1.1 jmcneill
3445 1.1 jmcneill /* only 3 fields get set, so mix them directly */
3446 1.1 jmcneill xor = mcx_mix_u64(xor, mb->mb_next_ptr);
3447 1.1 jmcneill xor = mcx_mix_u32(xor, mb->mb_block_number);
3448 1.1 jmcneill xor = mcx_mix_u8(xor, mb->mb_token);
3449 1.1 jmcneill
3450 1.1 jmcneill return (mcx_mix_done(xor));
3451 1.1 jmcneill }
3452 1.1 jmcneill
3453 1.1 jmcneill static void
3454 1.1 jmcneill mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb)
3455 1.1 jmcneill {
3456 1.1 jmcneill uint8_t *kva;
3457 1.1 jmcneill int i;
3458 1.1 jmcneill
3459 1.1 jmcneill kva = MCX_DMA_KVA(mxm);
3460 1.1 jmcneill
3461 1.1 jmcneill for (i = 0; i < nmb; i++) {
3462 1.1 jmcneill struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva;
3463 1.1 jmcneill uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb);
3464 1.1 jmcneill mb->mb_ctrl_signature = sig;
3465 1.1 jmcneill mb->mb_signature = sig ^
3466 1.1 jmcneill mcx_xor(mb->mb_data, sizeof(mb->mb_data));
3467 1.1 jmcneill
3468 1.1 jmcneill kva += MCX_CMDQ_MAILBOX_SIZE;
3469 1.1 jmcneill }
3470 1.1 jmcneill }
3471 1.1 jmcneill
3472 1.1 jmcneill static void
3473 1.1 jmcneill mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops)
3474 1.1 jmcneill {
3475 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm),
3476 1.1 jmcneill 0, MCX_DMA_LEN(mxm), ops);
3477 1.1 jmcneill }
3478 1.1 jmcneill
3479 1.1 jmcneill static struct mcx_cmdq_mailbox *
3480 1.1 jmcneill mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i)
3481 1.1 jmcneill {
3482 1.1 jmcneill uint8_t *kva;
3483 1.1 jmcneill
3484 1.1 jmcneill kva = MCX_DMA_KVA(mxm);
3485 1.1 jmcneill kva += i * MCX_CMDQ_MAILBOX_SIZE;
3486 1.1 jmcneill
3487 1.1 jmcneill return ((struct mcx_cmdq_mailbox *)kva);
3488 1.1 jmcneill }
3489 1.1 jmcneill
3490 1.1 jmcneill static inline void *
3491 1.1 jmcneill mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb)
3492 1.1 jmcneill {
3493 1.1 jmcneill return (&mb->mb_data);
3494 1.1 jmcneill }
3495 1.1 jmcneill
3496 1.1 jmcneill static void
3497 1.1 jmcneill mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb,
3498 1.1 jmcneill void *b, size_t len)
3499 1.1 jmcneill {
3500 1.1 jmcneill uint8_t *buf = b;
3501 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
3502 1.1 jmcneill int i;
3503 1.1 jmcneill
3504 1.1 jmcneill mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
3505 1.1 jmcneill for (i = 0; i < nmb; i++) {
3506 1.1 jmcneill
3507 1.1 jmcneill memcpy(mb->mb_data, buf, uimin(sizeof(mb->mb_data), len));
3508 1.1 jmcneill
3509 1.1 jmcneill if (sizeof(mb->mb_data) >= len)
3510 1.1 jmcneill break;
3511 1.1 jmcneill
3512 1.1 jmcneill buf += sizeof(mb->mb_data);
3513 1.1 jmcneill len -= sizeof(mb->mb_data);
3514 1.1 jmcneill mb++;
3515 1.1 jmcneill }
3516 1.1 jmcneill }
3517 1.1 jmcneill
3518 1.1 jmcneill static void
3519 1.13 jmcneill mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages,
3520 1.13 jmcneill struct mcx_dmamem *buf)
3521 1.13 jmcneill {
3522 1.13 jmcneill uint64_t *pas;
3523 1.13 jmcneill int mbox, mbox_pages, i;
3524 1.13 jmcneill
3525 1.13 jmcneill mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE;
3526 1.13 jmcneill offset %= MCX_CMDQ_MAILBOX_DATASIZE;
3527 1.13 jmcneill
3528 1.13 jmcneill pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
3529 1.13 jmcneill pas += (offset / sizeof(*pas));
3530 1.13 jmcneill mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas);
3531 1.13 jmcneill for (i = 0; i < npages; i++) {
3532 1.13 jmcneill if (i == mbox_pages) {
3533 1.13 jmcneill mbox++;
3534 1.13 jmcneill pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox));
3535 1.13 jmcneill mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas);
3536 1.13 jmcneill }
3537 1.13 jmcneill *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE));
3538 1.13 jmcneill pas++;
3539 1.13 jmcneill }
3540 1.13 jmcneill }
3541 1.13 jmcneill
3542 1.13 jmcneill static void
3543 1.1 jmcneill mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len)
3544 1.1 jmcneill {
3545 1.1 jmcneill uint8_t *buf = b;
3546 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
3547 1.1 jmcneill int i;
3548 1.1 jmcneill
3549 1.1 jmcneill mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm);
3550 1.1 jmcneill for (i = 0; i < nmb; i++) {
3551 1.1 jmcneill memcpy(buf, mb->mb_data, uimin(sizeof(mb->mb_data), len));
3552 1.1 jmcneill
3553 1.1 jmcneill if (sizeof(mb->mb_data) >= len)
3554 1.1 jmcneill break;
3555 1.1 jmcneill
3556 1.1 jmcneill buf += sizeof(mb->mb_data);
3557 1.1 jmcneill len -= sizeof(mb->mb_data);
3558 1.1 jmcneill mb++;
3559 1.1 jmcneill }
3560 1.1 jmcneill }
3561 1.1 jmcneill
3562 1.1 jmcneill static void
3563 1.1 jmcneill mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
3564 1.1 jmcneill {
3565 1.1 jmcneill mcx_dmamem_free(sc, mxm);
3566 1.1 jmcneill }
3567 1.1 jmcneill
3568 1.1 jmcneill #if 0
3569 1.1 jmcneill static void
3570 1.1 jmcneill mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe)
3571 1.1 jmcneill {
3572 1.1 jmcneill unsigned int i;
3573 1.1 jmcneill
3574 1.1 jmcneill printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type,
3575 1.1 jmcneill be32dec(&cqe->cq_input_length), be64dec(&cqe->cq_input_ptr));
3576 1.1 jmcneill
3577 1.1 jmcneill printf(", idata ");
3578 1.1 jmcneill for (i = 0; i < sizeof(cqe->cq_input_data); i++)
3579 1.1 jmcneill printf("%02x", cqe->cq_input_data[i]);
3580 1.1 jmcneill
3581 1.1 jmcneill printf(", odata ");
3582 1.1 jmcneill for (i = 0; i < sizeof(cqe->cq_output_data); i++)
3583 1.1 jmcneill printf("%02x", cqe->cq_output_data[i]);
3584 1.1 jmcneill
3585 1.1 jmcneill printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x",
3586 1.1 jmcneill be64dec(&cqe->cq_output_ptr), be32dec(&cqe->cq_output_length),
3587 1.1 jmcneill cqe->cq_token, cqe->cq_signature, cqe->cq_status);
3588 1.1 jmcneill }
3589 1.1 jmcneill
3590 1.1 jmcneill static void
3591 1.1 jmcneill mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num)
3592 1.1 jmcneill {
3593 1.1 jmcneill int i, j;
3594 1.1 jmcneill uint8_t *d;
3595 1.1 jmcneill
3596 1.1 jmcneill for (i = 0; i < num; i++) {
3597 1.1 jmcneill struct mcx_cmdq_mailbox *mbox;
3598 1.1 jmcneill mbox = mcx_cq_mbox(mboxes, i);
3599 1.1 jmcneill
3600 1.1 jmcneill d = mcx_cq_mbox_data(mbox);
3601 1.1 jmcneill for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) {
3602 1.1 jmcneill if (j != 0 && (j % 16 == 0))
3603 1.1 jmcneill printf("\n");
3604 1.1 jmcneill printf("%.2x ", d[j]);
3605 1.1 jmcneill }
3606 1.1 jmcneill }
3607 1.1 jmcneill }
3608 1.1 jmcneill #endif
3609 1.1 jmcneill
3610 1.1 jmcneill static int
3611 1.1 jmcneill mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data,
3612 1.1 jmcneill int len)
3613 1.1 jmcneill {
3614 1.1 jmcneill struct mcx_dmamem mxm;
3615 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3616 1.1 jmcneill struct mcx_cmd_access_reg_in *in;
3617 1.1 jmcneill struct mcx_cmd_access_reg_out *out;
3618 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3619 1.1 jmcneill int error, nmb;
3620 1.1 jmcneill
3621 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3622 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len,
3623 1.1 jmcneill token);
3624 1.1 jmcneill
3625 1.1 jmcneill in = mcx_cmdq_in(cqe);
3626 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG);
3627 1.1 jmcneill in->cmd_op_mod = htobe16(op);
3628 1.1 jmcneill in->cmd_register_id = htobe16(reg);
3629 1.1 jmcneill
3630 1.1 jmcneill nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE);
3631 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
3632 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
3633 1.1 jmcneill printf(", unable to allocate access reg mailboxen\n");
3634 1.1 jmcneill return (-1);
3635 1.1 jmcneill }
3636 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
3637 1.1 jmcneill mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len);
3638 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, nmb);
3639 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
3640 1.1 jmcneill
3641 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3642 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3643 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
3644 1.1 jmcneill
3645 1.1 jmcneill if (error != 0) {
3646 1.1 jmcneill printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc),
3647 1.1 jmcneill (op == MCX_REG_OP_WRITE ? "write" : "read"), reg);
3648 1.1 jmcneill goto free;
3649 1.1 jmcneill }
3650 1.1 jmcneill error = mcx_cmdq_verify(cqe);
3651 1.1 jmcneill if (error != 0) {
3652 1.1 jmcneill printf("%s: access reg (%s %x) reply corrupt\n",
3653 1.1 jmcneill (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc),
3654 1.1 jmcneill reg);
3655 1.1 jmcneill goto free;
3656 1.1 jmcneill }
3657 1.1 jmcneill
3658 1.1 jmcneill out = mcx_cmdq_out(cqe);
3659 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
3660 1.1 jmcneill printf("%s: access reg (%s %x) failed (%x, %.6x)\n",
3661 1.1 jmcneill DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"),
3662 1.15 jmcneill reg, out->cmd_status, be32toh(out->cmd_syndrome));
3663 1.1 jmcneill error = -1;
3664 1.1 jmcneill goto free;
3665 1.1 jmcneill }
3666 1.1 jmcneill
3667 1.1 jmcneill mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len);
3668 1.1 jmcneill free:
3669 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
3670 1.1 jmcneill
3671 1.1 jmcneill return (error);
3672 1.1 jmcneill }
3673 1.1 jmcneill
3674 1.1 jmcneill static int
3675 1.15 jmcneill mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe,
3676 1.15 jmcneill unsigned int slot)
3677 1.1 jmcneill {
3678 1.1 jmcneill struct mcx_cmd_set_issi_in *in;
3679 1.1 jmcneill struct mcx_cmd_set_issi_out *out;
3680 1.1 jmcneill uint8_t status;
3681 1.1 jmcneill
3682 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3683 1.1 jmcneill
3684 1.1 jmcneill in = mcx_cmdq_in(cqe);
3685 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI);
3686 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3687 1.1 jmcneill in->cmd_current_issi = htobe16(MCX_ISSI);
3688 1.1 jmcneill
3689 1.1 jmcneill mcx_cmdq_post(sc, cqe, slot);
3690 1.1 jmcneill if (mcx_cmdq_poll(sc, cqe, 1000) != 0)
3691 1.1 jmcneill return (-1);
3692 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0)
3693 1.1 jmcneill return (-1);
3694 1.1 jmcneill
3695 1.1 jmcneill status = cqe->cq_output_data[0];
3696 1.1 jmcneill if (status != MCX_CQ_STATUS_OK)
3697 1.1 jmcneill return (-1);
3698 1.1 jmcneill
3699 1.1 jmcneill return (0);
3700 1.1 jmcneill }
3701 1.1 jmcneill
3702 1.1 jmcneill static int
3703 1.1 jmcneill mcx_issi(struct mcx_softc *sc)
3704 1.1 jmcneill {
3705 1.1 jmcneill struct mcx_dmamem mxm;
3706 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3707 1.1 jmcneill struct mcx_cmd_query_issi_in *in;
3708 1.1 jmcneill struct mcx_cmd_query_issi_il_out *out;
3709 1.1 jmcneill struct mcx_cmd_query_issi_mb_out *mb;
3710 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3711 1.1 jmcneill uint8_t status;
3712 1.1 jmcneill int error;
3713 1.1 jmcneill
3714 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3715 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token);
3716 1.1 jmcneill
3717 1.1 jmcneill in = mcx_cmdq_in(cqe);
3718 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI);
3719 1.1 jmcneill in->cmd_op_mod = htobe16(0);
3720 1.1 jmcneill
3721 1.1 jmcneill CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE);
3722 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
3723 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
3724 1.1 jmcneill printf(", unable to allocate query issi mailbox\n");
3725 1.1 jmcneill return (-1);
3726 1.1 jmcneill }
3727 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
3728 1.1 jmcneill
3729 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3730 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3731 1.1 jmcneill if (error != 0) {
3732 1.1 jmcneill printf(", query issi timeout\n");
3733 1.1 jmcneill goto free;
3734 1.1 jmcneill }
3735 1.1 jmcneill error = mcx_cmdq_verify(cqe);
3736 1.1 jmcneill if (error != 0) {
3737 1.1 jmcneill printf(", query issi reply corrupt\n");
3738 1.1 jmcneill goto free;
3739 1.1 jmcneill }
3740 1.1 jmcneill
3741 1.1 jmcneill status = cqe->cq_output_data[0];
3742 1.1 jmcneill switch (status) {
3743 1.1 jmcneill case MCX_CQ_STATUS_OK:
3744 1.1 jmcneill break;
3745 1.1 jmcneill case MCX_CQ_STATUS_BAD_OPCODE:
3746 1.1 jmcneill /* use ISSI 0 */
3747 1.1 jmcneill goto free;
3748 1.1 jmcneill default:
3749 1.1 jmcneill printf(", query issi failed (%x)\n", status);
3750 1.1 jmcneill error = -1;
3751 1.1 jmcneill goto free;
3752 1.1 jmcneill }
3753 1.1 jmcneill
3754 1.1 jmcneill out = mcx_cmdq_out(cqe);
3755 1.1 jmcneill if (out->cmd_current_issi == htobe16(MCX_ISSI)) {
3756 1.1 jmcneill /* use ISSI 1 */
3757 1.1 jmcneill goto free;
3758 1.1 jmcneill }
3759 1.1 jmcneill
3760 1.1 jmcneill /* don't need to read cqe anymore, can be used for SET ISSI */
3761 1.1 jmcneill
3762 1.1 jmcneill mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
3763 1.1 jmcneill CTASSERT(MCX_ISSI < NBBY);
3764 1.1 jmcneill /* XXX math is hard */
3765 1.1 jmcneill if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) {
3766 1.1 jmcneill /* use ISSI 0 */
3767 1.1 jmcneill goto free;
3768 1.1 jmcneill }
3769 1.1 jmcneill
3770 1.1 jmcneill if (mcx_set_issi(sc, cqe, 0) != 0) {
3771 1.1 jmcneill /* ignore the error, just use ISSI 0 */
3772 1.1 jmcneill } else {
3773 1.1 jmcneill /* use ISSI 1 */
3774 1.1 jmcneill }
3775 1.1 jmcneill
3776 1.1 jmcneill free:
3777 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
3778 1.1 jmcneill return (error);
3779 1.1 jmcneill }
3780 1.1 jmcneill
3781 1.1 jmcneill static int
3782 1.1 jmcneill mcx_query_pages(struct mcx_softc *sc, uint16_t type,
3783 1.15 jmcneill int32_t *npages, uint16_t *func_id)
3784 1.1 jmcneill {
3785 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3786 1.1 jmcneill struct mcx_cmd_query_pages_in *in;
3787 1.1 jmcneill struct mcx_cmd_query_pages_out *out;
3788 1.1 jmcneill
3789 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3790 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
3791 1.1 jmcneill
3792 1.1 jmcneill in = mcx_cmdq_in(cqe);
3793 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES);
3794 1.1 jmcneill in->cmd_op_mod = type;
3795 1.1 jmcneill
3796 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3797 1.1 jmcneill if (mcx_cmdq_poll(sc, cqe, 1000) != 0) {
3798 1.1 jmcneill printf(", query pages timeout\n");
3799 1.1 jmcneill return (-1);
3800 1.1 jmcneill }
3801 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
3802 1.1 jmcneill printf(", query pages reply corrupt\n");
3803 1.1 jmcneill return (-1);
3804 1.1 jmcneill }
3805 1.1 jmcneill
3806 1.1 jmcneill out = mcx_cmdq_out(cqe);
3807 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
3808 1.1 jmcneill printf(", query pages failed (%x)\n", out->cmd_status);
3809 1.1 jmcneill return (-1);
3810 1.1 jmcneill }
3811 1.1 jmcneill
3812 1.1 jmcneill *func_id = out->cmd_func_id;
3813 1.1 jmcneill *npages = be32dec(&out->cmd_num_pages);
3814 1.1 jmcneill
3815 1.1 jmcneill return (0);
3816 1.1 jmcneill }
3817 1.1 jmcneill
3818 1.1 jmcneill struct bus_dma_iter {
3819 1.1 jmcneill bus_dmamap_t i_map;
3820 1.1 jmcneill bus_size_t i_offset;
3821 1.1 jmcneill unsigned int i_index;
3822 1.1 jmcneill };
3823 1.1 jmcneill
3824 1.1 jmcneill static void
3825 1.1 jmcneill bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map)
3826 1.1 jmcneill {
3827 1.1 jmcneill i->i_map = map;
3828 1.1 jmcneill i->i_offset = 0;
3829 1.1 jmcneill i->i_index = 0;
3830 1.1 jmcneill }
3831 1.1 jmcneill
3832 1.1 jmcneill static bus_addr_t
3833 1.1 jmcneill bus_dma_iter_addr(struct bus_dma_iter *i)
3834 1.1 jmcneill {
3835 1.1 jmcneill return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset);
3836 1.1 jmcneill }
3837 1.1 jmcneill
3838 1.1 jmcneill static void
3839 1.1 jmcneill bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size)
3840 1.1 jmcneill {
3841 1.1 jmcneill bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index;
3842 1.1 jmcneill bus_size_t diff;
3843 1.1 jmcneill
3844 1.1 jmcneill do {
3845 1.1 jmcneill diff = seg->ds_len - i->i_offset;
3846 1.1 jmcneill if (size < diff)
3847 1.1 jmcneill break;
3848 1.1 jmcneill
3849 1.1 jmcneill size -= diff;
3850 1.1 jmcneill
3851 1.1 jmcneill seg++;
3852 1.1 jmcneill
3853 1.1 jmcneill i->i_offset = 0;
3854 1.1 jmcneill i->i_index++;
3855 1.1 jmcneill } while (size > 0);
3856 1.1 jmcneill
3857 1.1 jmcneill i->i_offset += size;
3858 1.1 jmcneill }
3859 1.1 jmcneill
3860 1.1 jmcneill static int
3861 1.1 jmcneill mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id)
3862 1.1 jmcneill {
3863 1.1 jmcneill struct mcx_dmamem mxm;
3864 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3865 1.1 jmcneill struct mcx_cmd_manage_pages_in *in;
3866 1.1 jmcneill struct mcx_cmd_manage_pages_out *out;
3867 1.1 jmcneill unsigned int paslen, nmb, i, j, npages;
3868 1.1 jmcneill struct bus_dma_iter iter;
3869 1.1 jmcneill uint64_t *pas;
3870 1.1 jmcneill uint8_t status;
3871 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3872 1.1 jmcneill int error;
3873 1.1 jmcneill
3874 1.1 jmcneill npages = mhm->mhm_npages;
3875 1.1 jmcneill
3876 1.1 jmcneill paslen = sizeof(*pas) * npages;
3877 1.1 jmcneill nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE);
3878 1.1 jmcneill
3879 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3880 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token);
3881 1.1 jmcneill
3882 1.1 jmcneill in = mcx_cmdq_in(cqe);
3883 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES);
3884 1.1 jmcneill in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS);
3885 1.1 jmcneill in->cmd_func_id = func_id;
3886 1.1 jmcneill be32enc(&in->cmd_input_num_entries, npages);
3887 1.1 jmcneill
3888 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb,
3889 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
3890 1.1 jmcneill printf(", unable to allocate manage pages mailboxen\n");
3891 1.1 jmcneill return (-1);
3892 1.1 jmcneill }
3893 1.1 jmcneill
3894 1.1 jmcneill bus_dma_iter_init(&iter, mhm->mhm_map);
3895 1.1 jmcneill for (i = 0; i < nmb; i++) {
3896 1.1 jmcneill unsigned int lim;
3897 1.1 jmcneill
3898 1.1 jmcneill pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i));
3899 1.1 jmcneill lim = uimin(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages);
3900 1.1 jmcneill
3901 1.1 jmcneill for (j = 0; j < lim; j++) {
3902 1.1 jmcneill be64enc(&pas[j], bus_dma_iter_addr(&iter));
3903 1.1 jmcneill bus_dma_iter_add(&iter, MCX_PAGE_SIZE);
3904 1.1 jmcneill }
3905 1.1 jmcneill
3906 1.1 jmcneill npages -= lim;
3907 1.1 jmcneill }
3908 1.1 jmcneill
3909 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, nmb);
3910 1.1 jmcneill
3911 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3912 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
3913 1.1 jmcneill if (error != 0) {
3914 1.1 jmcneill printf(", manage pages timeout\n");
3915 1.1 jmcneill goto free;
3916 1.1 jmcneill }
3917 1.1 jmcneill error = mcx_cmdq_verify(cqe);
3918 1.1 jmcneill if (error != 0) {
3919 1.1 jmcneill printf(", manage pages reply corrupt\n");
3920 1.1 jmcneill goto free;
3921 1.1 jmcneill }
3922 1.1 jmcneill
3923 1.1 jmcneill status = cqe->cq_output_data[0];
3924 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
3925 1.1 jmcneill printf(", manage pages failed (%x)\n", status);
3926 1.1 jmcneill error = -1;
3927 1.1 jmcneill goto free;
3928 1.1 jmcneill }
3929 1.1 jmcneill
3930 1.1 jmcneill free:
3931 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
3932 1.1 jmcneill
3933 1.1 jmcneill return (error);
3934 1.1 jmcneill }
3935 1.1 jmcneill
3936 1.1 jmcneill static int
3937 1.1 jmcneill mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type)
3938 1.1 jmcneill {
3939 1.15 jmcneill int32_t npages;
3940 1.1 jmcneill uint16_t func_id;
3941 1.1 jmcneill
3942 1.1 jmcneill if (mcx_query_pages(sc, type, &npages, &func_id) != 0) {
3943 1.1 jmcneill /* error printed by mcx_query_pages */
3944 1.1 jmcneill return (-1);
3945 1.1 jmcneill }
3946 1.1 jmcneill
3947 1.15 jmcneill if (npages < 1)
3948 1.1 jmcneill return (0);
3949 1.1 jmcneill
3950 1.1 jmcneill if (mcx_hwmem_alloc(sc, mhm, npages) != 0) {
3951 1.1 jmcneill printf(", unable to allocate hwmem\n");
3952 1.1 jmcneill return (-1);
3953 1.1 jmcneill }
3954 1.1 jmcneill
3955 1.1 jmcneill if (mcx_add_pages(sc, mhm, func_id) != 0) {
3956 1.1 jmcneill printf(", unable to add hwmem\n");
3957 1.1 jmcneill goto free;
3958 1.1 jmcneill }
3959 1.1 jmcneill
3960 1.1 jmcneill return (0);
3961 1.1 jmcneill
3962 1.1 jmcneill free:
3963 1.1 jmcneill mcx_hwmem_free(sc, mhm);
3964 1.1 jmcneill
3965 1.1 jmcneill return (-1);
3966 1.1 jmcneill }
3967 1.1 jmcneill
3968 1.1 jmcneill static int
3969 1.1 jmcneill mcx_hca_max_caps(struct mcx_softc *sc)
3970 1.1 jmcneill {
3971 1.1 jmcneill struct mcx_dmamem mxm;
3972 1.1 jmcneill struct mcx_cmdq_entry *cqe;
3973 1.1 jmcneill struct mcx_cmd_query_hca_cap_in *in;
3974 1.1 jmcneill struct mcx_cmd_query_hca_cap_out *out;
3975 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
3976 1.1 jmcneill struct mcx_cap_device *hca;
3977 1.1 jmcneill uint8_t status;
3978 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
3979 1.1 jmcneill int error;
3980 1.1 jmcneill
3981 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
3982 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
3983 1.1 jmcneill token);
3984 1.1 jmcneill
3985 1.1 jmcneill in = mcx_cmdq_in(cqe);
3986 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
3987 1.1 jmcneill in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX |
3988 1.1 jmcneill MCX_CMD_QUERY_HCA_CAP_DEVICE);
3989 1.1 jmcneill
3990 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
3991 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
3992 1.1 jmcneill printf(", unable to allocate query hca caps mailboxen\n");
3993 1.1 jmcneill return (-1);
3994 1.1 jmcneill }
3995 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
3996 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
3997 1.1 jmcneill
3998 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
3999 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4000 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
4001 1.1 jmcneill
4002 1.1 jmcneill if (error != 0) {
4003 1.1 jmcneill printf(", query hca caps timeout\n");
4004 1.1 jmcneill goto free;
4005 1.1 jmcneill }
4006 1.1 jmcneill error = mcx_cmdq_verify(cqe);
4007 1.1 jmcneill if (error != 0) {
4008 1.1 jmcneill printf(", query hca caps reply corrupt\n");
4009 1.1 jmcneill goto free;
4010 1.1 jmcneill }
4011 1.1 jmcneill
4012 1.1 jmcneill status = cqe->cq_output_data[0];
4013 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4014 1.1 jmcneill printf(", query hca caps failed (%x)\n", status);
4015 1.1 jmcneill error = -1;
4016 1.1 jmcneill goto free;
4017 1.1 jmcneill }
4018 1.1 jmcneill
4019 1.1 jmcneill mb = mcx_cq_mbox(&mxm, 0);
4020 1.1 jmcneill hca = mcx_cq_mbox_data(mb);
4021 1.1 jmcneill
4022 1.15 jmcneill if ((hca->port_type & MCX_CAP_DEVICE_PORT_TYPE)
4023 1.15 jmcneill != MCX_CAP_DEVICE_PORT_TYPE_ETH) {
4024 1.15 jmcneill printf(", not in ethernet mode\n");
4025 1.15 jmcneill error = -1;
4026 1.15 jmcneill goto free;
4027 1.15 jmcneill }
4028 1.1 jmcneill if (hca->log_pg_sz > PAGE_SHIFT) {
4029 1.1 jmcneill printf(", minimum system page shift %u is too large\n",
4030 1.1 jmcneill hca->log_pg_sz);
4031 1.1 jmcneill error = -1;
4032 1.1 jmcneill goto free;
4033 1.1 jmcneill }
4034 1.1 jmcneill /*
4035 1.1 jmcneill * blueflame register is split into two buffers, and we must alternate
4036 1.1 jmcneill * between the two of them.
4037 1.1 jmcneill */
4038 1.1 jmcneill sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2;
4039 1.15 jmcneill sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size);
4040 1.15 jmcneill
4041 1.15 jmcneill if (hca->local_ca_ack_delay & MCX_CAP_DEVICE_MCAM_REG)
4042 1.15 jmcneill sc->sc_mcam_reg = 1;
4043 1.15 jmcneill
4044 1.15 jmcneill sc->sc_mhz = be32dec(&hca->device_frequency_mhz);
4045 1.15 jmcneill sc->sc_khz = be32dec(&hca->device_frequency_khz);
4046 1.1 jmcneill
4047 1.1 jmcneill free:
4048 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4049 1.1 jmcneill
4050 1.1 jmcneill return (error);
4051 1.1 jmcneill }
4052 1.1 jmcneill
4053 1.1 jmcneill static int
4054 1.1 jmcneill mcx_hca_set_caps(struct mcx_softc *sc)
4055 1.1 jmcneill {
4056 1.1 jmcneill struct mcx_dmamem mxm;
4057 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4058 1.1 jmcneill struct mcx_cmd_query_hca_cap_in *in;
4059 1.1 jmcneill struct mcx_cmd_query_hca_cap_out *out;
4060 1.1 jmcneill struct mcx_cmdq_mailbox *mb;
4061 1.1 jmcneill struct mcx_cap_device *hca;
4062 1.1 jmcneill uint8_t status;
4063 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
4064 1.1 jmcneill int error;
4065 1.1 jmcneill
4066 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4067 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN,
4068 1.1 jmcneill token);
4069 1.1 jmcneill
4070 1.1 jmcneill in = mcx_cmdq_in(cqe);
4071 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP);
4072 1.1 jmcneill in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT |
4073 1.1 jmcneill MCX_CMD_QUERY_HCA_CAP_DEVICE);
4074 1.1 jmcneill
4075 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES,
4076 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
4077 1.1 jmcneill printf(", unable to allocate manage pages mailboxen\n");
4078 1.1 jmcneill return (-1);
4079 1.1 jmcneill }
4080 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES);
4081 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW);
4082 1.1 jmcneill
4083 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4084 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4085 1.1 jmcneill mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW);
4086 1.1 jmcneill
4087 1.1 jmcneill if (error != 0) {
4088 1.1 jmcneill printf(", query hca caps timeout\n");
4089 1.1 jmcneill goto free;
4090 1.1 jmcneill }
4091 1.1 jmcneill error = mcx_cmdq_verify(cqe);
4092 1.1 jmcneill if (error != 0) {
4093 1.1 jmcneill printf(", query hca caps reply corrupt\n");
4094 1.1 jmcneill goto free;
4095 1.1 jmcneill }
4096 1.1 jmcneill
4097 1.1 jmcneill status = cqe->cq_output_data[0];
4098 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4099 1.1 jmcneill printf(", query hca caps failed (%x)\n", status);
4100 1.1 jmcneill error = -1;
4101 1.1 jmcneill goto free;
4102 1.1 jmcneill }
4103 1.1 jmcneill
4104 1.1 jmcneill mb = mcx_cq_mbox(&mxm, 0);
4105 1.1 jmcneill hca = mcx_cq_mbox_data(mb);
4106 1.1 jmcneill
4107 1.1 jmcneill hca->log_pg_sz = PAGE_SHIFT;
4108 1.1 jmcneill
4109 1.1 jmcneill free:
4110 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4111 1.1 jmcneill
4112 1.1 jmcneill return (error);
4113 1.1 jmcneill }
4114 1.1 jmcneill
4115 1.1 jmcneill
4116 1.1 jmcneill static int
4117 1.1 jmcneill mcx_init_hca(struct mcx_softc *sc)
4118 1.1 jmcneill {
4119 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4120 1.1 jmcneill struct mcx_cmd_init_hca_in *in;
4121 1.1 jmcneill struct mcx_cmd_init_hca_out *out;
4122 1.1 jmcneill int error;
4123 1.1 jmcneill uint8_t status;
4124 1.1 jmcneill
4125 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4126 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4127 1.1 jmcneill
4128 1.1 jmcneill in = mcx_cmdq_in(cqe);
4129 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA);
4130 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4131 1.1 jmcneill
4132 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4133 1.1 jmcneill
4134 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4135 1.1 jmcneill if (error != 0) {
4136 1.1 jmcneill printf(", hca init timeout\n");
4137 1.1 jmcneill return (-1);
4138 1.1 jmcneill }
4139 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4140 1.1 jmcneill printf(", hca init command corrupt\n");
4141 1.1 jmcneill return (-1);
4142 1.1 jmcneill }
4143 1.1 jmcneill
4144 1.1 jmcneill status = cqe->cq_output_data[0];
4145 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4146 1.1 jmcneill printf(", hca init failed (%x)\n", status);
4147 1.1 jmcneill return (-1);
4148 1.1 jmcneill }
4149 1.1 jmcneill
4150 1.1 jmcneill return (0);
4151 1.1 jmcneill }
4152 1.1 jmcneill
4153 1.1 jmcneill static int
4154 1.1 jmcneill mcx_set_driver_version(struct mcx_softc *sc)
4155 1.1 jmcneill {
4156 1.1 jmcneill struct mcx_dmamem mxm;
4157 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4158 1.1 jmcneill struct mcx_cmd_set_driver_version_in *in;
4159 1.1 jmcneill struct mcx_cmd_set_driver_version_out *out;
4160 1.1 jmcneill int error;
4161 1.1 jmcneill int token;
4162 1.1 jmcneill uint8_t status;
4163 1.1 jmcneill
4164 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4165 1.1 jmcneill token = mcx_cmdq_token(sc);
4166 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) +
4167 1.1 jmcneill sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token);
4168 1.1 jmcneill
4169 1.1 jmcneill in = mcx_cmdq_in(cqe);
4170 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION);
4171 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4172 1.1 jmcneill
4173 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4174 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
4175 1.1 jmcneill printf(", unable to allocate set driver version mailboxen\n");
4176 1.1 jmcneill return (-1);
4177 1.1 jmcneill }
4178 1.1 jmcneill strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)),
4179 1.1 jmcneill "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE);
4180 1.1 jmcneill
4181 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4182 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4183 1.1 jmcneill
4184 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4185 1.1 jmcneill if (error != 0) {
4186 1.1 jmcneill printf(", set driver version timeout\n");
4187 1.1 jmcneill goto free;
4188 1.1 jmcneill }
4189 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4190 1.1 jmcneill printf(", set driver version command corrupt\n");
4191 1.1 jmcneill goto free;
4192 1.1 jmcneill }
4193 1.1 jmcneill
4194 1.1 jmcneill status = cqe->cq_output_data[0];
4195 1.1 jmcneill if (status != MCX_CQ_STATUS_OK) {
4196 1.1 jmcneill printf(", set driver version failed (%x)\n", status);
4197 1.1 jmcneill error = -1;
4198 1.1 jmcneill goto free;
4199 1.1 jmcneill }
4200 1.1 jmcneill
4201 1.1 jmcneill free:
4202 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4203 1.1 jmcneill
4204 1.1 jmcneill return (error);
4205 1.1 jmcneill }
4206 1.1 jmcneill
4207 1.1 jmcneill static int
4208 1.1 jmcneill mcx_iff(struct mcx_softc *sc)
4209 1.1 jmcneill {
4210 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
4211 1.1 jmcneill struct mcx_dmamem mxm;
4212 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4213 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_in *in;
4214 1.1 jmcneill struct mcx_cmd_modify_nic_vport_context_out *out;
4215 1.1 jmcneill struct mcx_nic_vport_ctx *ctx;
4216 1.1 jmcneill int error;
4217 1.1 jmcneill int token;
4218 1.1 jmcneill int insize;
4219 1.15 jmcneill uint32_t dest;
4220 1.15 jmcneill
4221 1.15 jmcneill dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
4222 1.15 jmcneill sc->sc_rss_flow_table_id;
4223 1.1 jmcneill
4224 1.1 jmcneill /* enable or disable the promisc flow */
4225 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_PROMISC)) {
4226 1.1 jmcneill if (sc->sc_promisc_flow_enabled == 0) {
4227 1.15 jmcneill mcx_set_flow_table_entry_mac(sc,
4228 1.15 jmcneill MCX_FLOW_GROUP_PROMISC, 0, NULL, dest);
4229 1.1 jmcneill sc->sc_promisc_flow_enabled = 1;
4230 1.1 jmcneill }
4231 1.1 jmcneill } else if (sc->sc_promisc_flow_enabled != 0) {
4232 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
4233 1.1 jmcneill sc->sc_promisc_flow_enabled = 0;
4234 1.1 jmcneill }
4235 1.1 jmcneill
4236 1.1 jmcneill /* enable or disable the all-multicast flow */
4237 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
4238 1.1 jmcneill if (sc->sc_allmulti_flow_enabled == 0) {
4239 1.1 jmcneill uint8_t mcast[ETHER_ADDR_LEN];
4240 1.1 jmcneill
4241 1.1 jmcneill memset(mcast, 0, sizeof(mcast));
4242 1.1 jmcneill mcast[0] = 0x01;
4243 1.15 jmcneill mcx_set_flow_table_entry_mac(sc,
4244 1.15 jmcneill MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest);
4245 1.1 jmcneill sc->sc_allmulti_flow_enabled = 1;
4246 1.1 jmcneill }
4247 1.1 jmcneill } else if (sc->sc_allmulti_flow_enabled != 0) {
4248 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
4249 1.1 jmcneill sc->sc_allmulti_flow_enabled = 0;
4250 1.1 jmcneill }
4251 1.1 jmcneill
4252 1.1 jmcneill insize = sizeof(struct mcx_nic_vport_ctx) + 240;
4253 1.1 jmcneill
4254 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4255 1.1 jmcneill token = mcx_cmdq_token(sc);
4256 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4257 1.1 jmcneill
4258 1.1 jmcneill in = mcx_cmdq_in(cqe);
4259 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT);
4260 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4261 1.1 jmcneill in->cmd_field_select = htobe32(
4262 1.1 jmcneill MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC |
4263 1.1 jmcneill MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU);
4264 1.1 jmcneill
4265 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) {
4266 1.15 jmcneill printf(", unable to allocate modify "
4267 1.15 jmcneill "nic vport context mailboxen\n");
4268 1.1 jmcneill return (-1);
4269 1.1 jmcneill }
4270 1.1 jmcneill ctx = (struct mcx_nic_vport_ctx *)
4271 1.1 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240);
4272 1.1 jmcneill ctx->vp_mtu = htobe32(sc->sc_hardmtu);
4273 1.1 jmcneill /*
4274 1.15 jmcneill * always leave promisc-all enabled on the vport since we
4275 1.15 jmcneill * can't give it a vlan list, and we're already doing multicast
4276 1.15 jmcneill * filtering in the flow table.
4277 1.1 jmcneill */
4278 1.1 jmcneill ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL);
4279 1.1 jmcneill
4280 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4281 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4282 1.1 jmcneill
4283 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4284 1.1 jmcneill if (error != 0) {
4285 1.1 jmcneill printf(", modify nic vport context timeout\n");
4286 1.1 jmcneill goto free;
4287 1.1 jmcneill }
4288 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4289 1.1 jmcneill printf(", modify nic vport context command corrupt\n");
4290 1.1 jmcneill goto free;
4291 1.1 jmcneill }
4292 1.1 jmcneill
4293 1.1 jmcneill out = mcx_cmdq_out(cqe);
4294 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4295 1.1 jmcneill printf(", modify nic vport context failed (%x, %x)\n",
4296 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4297 1.1 jmcneill error = -1;
4298 1.1 jmcneill goto free;
4299 1.1 jmcneill }
4300 1.1 jmcneill
4301 1.1 jmcneill free:
4302 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4303 1.1 jmcneill
4304 1.1 jmcneill return (error);
4305 1.1 jmcneill }
4306 1.1 jmcneill
4307 1.1 jmcneill static int
4308 1.15 jmcneill mcx_alloc_uar(struct mcx_softc *sc, int *uar)
4309 1.1 jmcneill {
4310 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4311 1.1 jmcneill struct mcx_cmd_alloc_uar_in *in;
4312 1.1 jmcneill struct mcx_cmd_alloc_uar_out *out;
4313 1.1 jmcneill int error;
4314 1.1 jmcneill
4315 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4316 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4317 1.1 jmcneill
4318 1.1 jmcneill in = mcx_cmdq_in(cqe);
4319 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR);
4320 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4321 1.1 jmcneill
4322 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4323 1.1 jmcneill
4324 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4325 1.1 jmcneill if (error != 0) {
4326 1.1 jmcneill printf(", alloc uar timeout\n");
4327 1.1 jmcneill return (-1);
4328 1.1 jmcneill }
4329 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4330 1.1 jmcneill printf(", alloc uar command corrupt\n");
4331 1.1 jmcneill return (-1);
4332 1.1 jmcneill }
4333 1.1 jmcneill
4334 1.1 jmcneill out = mcx_cmdq_out(cqe);
4335 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4336 1.1 jmcneill printf(", alloc uar failed (%x)\n", out->cmd_status);
4337 1.1 jmcneill return (-1);
4338 1.1 jmcneill }
4339 1.1 jmcneill
4340 1.15 jmcneill *uar = mcx_get_id(out->cmd_uar);
4341 1.1 jmcneill return (0);
4342 1.1 jmcneill }
4343 1.1 jmcneill
4344 1.1 jmcneill static int
4345 1.15 jmcneill mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar,
4346 1.15 jmcneill uint64_t events, int vector)
4347 1.1 jmcneill {
4348 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4349 1.1 jmcneill struct mcx_dmamem mxm;
4350 1.1 jmcneill struct mcx_cmd_create_eq_in *in;
4351 1.1 jmcneill struct mcx_cmd_create_eq_mb_in *mbin;
4352 1.1 jmcneill struct mcx_cmd_create_eq_out *out;
4353 1.1 jmcneill struct mcx_eq_entry *eqe;
4354 1.1 jmcneill int error;
4355 1.1 jmcneill uint64_t *pas;
4356 1.1 jmcneill int insize, npages, paslen, i, token;
4357 1.1 jmcneill
4358 1.15 jmcneill eq->eq_cons = 0;
4359 1.1 jmcneill
4360 1.1 jmcneill npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry),
4361 1.1 jmcneill MCX_PAGE_SIZE);
4362 1.1 jmcneill paslen = npages * sizeof(*pas);
4363 1.1 jmcneill insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen;
4364 1.1 jmcneill
4365 1.15 jmcneill if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE,
4366 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
4367 1.1 jmcneill printf(", unable to allocate event queue memory\n");
4368 1.1 jmcneill return (-1);
4369 1.1 jmcneill }
4370 1.1 jmcneill
4371 1.15 jmcneill eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
4372 1.1 jmcneill for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) {
4373 1.1 jmcneill eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT;
4374 1.1 jmcneill }
4375 1.1 jmcneill
4376 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4377 1.1 jmcneill token = mcx_cmdq_token(sc);
4378 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4379 1.1 jmcneill
4380 1.1 jmcneill in = mcx_cmdq_in(cqe);
4381 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ);
4382 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4383 1.1 jmcneill
4384 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4385 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4386 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
4387 1.1 jmcneill printf(", unable to allocate create eq mailboxen\n");
4388 1.15 jmcneill goto free_eq;
4389 1.1 jmcneill }
4390 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4391 1.1 jmcneill mbin->cmd_eq_ctx.eq_uar_size = htobe32(
4392 1.15 jmcneill (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar);
4393 1.15 jmcneill mbin->cmd_eq_ctx.eq_intr = vector;
4394 1.15 jmcneill mbin->cmd_event_bitmask = htobe64(events);
4395 1.15 jmcneill
4396 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
4397 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
4398 1.1 jmcneill
4399 1.1 jmcneill /* physical addresses follow the mailbox in data */
4400 1.15 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem);
4401 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE));
4402 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4403 1.1 jmcneill
4404 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4405 1.1 jmcneill if (error != 0) {
4406 1.1 jmcneill printf(", create eq timeout\n");
4407 1.15 jmcneill goto free_mxm;
4408 1.1 jmcneill }
4409 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4410 1.1 jmcneill printf(", create eq command corrupt\n");
4411 1.15 jmcneill goto free_mxm;
4412 1.1 jmcneill }
4413 1.1 jmcneill
4414 1.1 jmcneill out = mcx_cmdq_out(cqe);
4415 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4416 1.1 jmcneill printf(", create eq failed (%x, %x)\n", out->cmd_status,
4417 1.1 jmcneill be32toh(out->cmd_syndrome));
4418 1.15 jmcneill goto free_mxm;
4419 1.1 jmcneill }
4420 1.1 jmcneill
4421 1.15 jmcneill eq->eq_n = mcx_get_id(out->cmd_eqn);
4422 1.15 jmcneill
4423 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
4424 1.15 jmcneill
4425 1.15 jmcneill mcx_arm_eq(sc, eq, uar);
4426 1.15 jmcneill
4427 1.15 jmcneill return (0);
4428 1.15 jmcneill
4429 1.15 jmcneill free_mxm:
4430 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
4431 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
4432 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4433 1.15 jmcneill free_eq:
4434 1.15 jmcneill mcx_dmamem_free(sc, &eq->eq_mem);
4435 1.15 jmcneill return (-1);
4436 1.1 jmcneill }
4437 1.1 jmcneill
4438 1.1 jmcneill static int
4439 1.1 jmcneill mcx_alloc_pd(struct mcx_softc *sc)
4440 1.1 jmcneill {
4441 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4442 1.1 jmcneill struct mcx_cmd_alloc_pd_in *in;
4443 1.1 jmcneill struct mcx_cmd_alloc_pd_out *out;
4444 1.1 jmcneill int error;
4445 1.1 jmcneill
4446 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4447 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4448 1.1 jmcneill
4449 1.1 jmcneill in = mcx_cmdq_in(cqe);
4450 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD);
4451 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4452 1.1 jmcneill
4453 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4454 1.1 jmcneill
4455 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4456 1.1 jmcneill if (error != 0) {
4457 1.1 jmcneill printf(", alloc pd timeout\n");
4458 1.1 jmcneill return (-1);
4459 1.1 jmcneill }
4460 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4461 1.1 jmcneill printf(", alloc pd command corrupt\n");
4462 1.1 jmcneill return (-1);
4463 1.1 jmcneill }
4464 1.1 jmcneill
4465 1.1 jmcneill out = mcx_cmdq_out(cqe);
4466 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4467 1.1 jmcneill printf(", alloc pd failed (%x)\n", out->cmd_status);
4468 1.1 jmcneill return (-1);
4469 1.1 jmcneill }
4470 1.1 jmcneill
4471 1.13 jmcneill sc->sc_pd = mcx_get_id(out->cmd_pd);
4472 1.1 jmcneill return (0);
4473 1.1 jmcneill }
4474 1.1 jmcneill
4475 1.1 jmcneill static int
4476 1.1 jmcneill mcx_alloc_tdomain(struct mcx_softc *sc)
4477 1.1 jmcneill {
4478 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4479 1.1 jmcneill struct mcx_cmd_alloc_td_in *in;
4480 1.1 jmcneill struct mcx_cmd_alloc_td_out *out;
4481 1.1 jmcneill int error;
4482 1.1 jmcneill
4483 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4484 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4485 1.1 jmcneill
4486 1.1 jmcneill in = mcx_cmdq_in(cqe);
4487 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN);
4488 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4489 1.1 jmcneill
4490 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4491 1.1 jmcneill
4492 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4493 1.1 jmcneill if (error != 0) {
4494 1.1 jmcneill printf(", alloc transport domain timeout\n");
4495 1.1 jmcneill return (-1);
4496 1.1 jmcneill }
4497 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4498 1.1 jmcneill printf(", alloc transport domain command corrupt\n");
4499 1.1 jmcneill return (-1);
4500 1.1 jmcneill }
4501 1.1 jmcneill
4502 1.1 jmcneill out = mcx_cmdq_out(cqe);
4503 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4504 1.1 jmcneill printf(", alloc transport domain failed (%x)\n",
4505 1.1 jmcneill out->cmd_status);
4506 1.1 jmcneill return (-1);
4507 1.1 jmcneill }
4508 1.1 jmcneill
4509 1.13 jmcneill sc->sc_tdomain = mcx_get_id(out->cmd_tdomain);
4510 1.1 jmcneill return (0);
4511 1.1 jmcneill }
4512 1.1 jmcneill
4513 1.1 jmcneill static int
4514 1.1 jmcneill mcx_query_nic_vport_context(struct mcx_softc *sc, uint8_t *enaddr)
4515 1.1 jmcneill {
4516 1.1 jmcneill struct mcx_dmamem mxm;
4517 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4518 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_in *in;
4519 1.1 jmcneill struct mcx_cmd_query_nic_vport_context_out *out;
4520 1.1 jmcneill struct mcx_nic_vport_ctx *ctx;
4521 1.1 jmcneill uint8_t *addr;
4522 1.1 jmcneill int error, token, i;
4523 1.1 jmcneill
4524 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4525 1.1 jmcneill token = mcx_cmdq_token(sc);
4526 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token);
4527 1.1 jmcneill
4528 1.1 jmcneill in = mcx_cmdq_in(cqe);
4529 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT);
4530 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4531 1.1 jmcneill in->cmd_allowed_list_type = 0;
4532 1.1 jmcneill
4533 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4534 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
4535 1.15 jmcneill printf(", unable to allocate "
4536 1.15 jmcneill "query nic vport context mailboxen\n");
4537 1.1 jmcneill return (-1);
4538 1.1 jmcneill }
4539 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4540 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4541 1.1 jmcneill
4542 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4543 1.1 jmcneill if (error != 0) {
4544 1.1 jmcneill printf(", query nic vport context timeout\n");
4545 1.1 jmcneill goto free;
4546 1.1 jmcneill }
4547 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4548 1.1 jmcneill printf(", query nic vport context command corrupt\n");
4549 1.1 jmcneill goto free;
4550 1.1 jmcneill }
4551 1.1 jmcneill
4552 1.1 jmcneill out = mcx_cmdq_out(cqe);
4553 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4554 1.1 jmcneill printf(", query nic vport context failed (%x, %x)\n",
4555 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4556 1.1 jmcneill error = -1;
4557 1.1 jmcneill goto free;
4558 1.1 jmcneill }
4559 1.1 jmcneill
4560 1.15 jmcneill ctx = (struct mcx_nic_vport_ctx *)
4561 1.15 jmcneill mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4562 1.1 jmcneill addr = (uint8_t *)&ctx->vp_perm_addr;
4563 1.1 jmcneill for (i = 0; i < ETHER_ADDR_LEN; i++) {
4564 1.1 jmcneill enaddr[i] = addr[i + 2];
4565 1.1 jmcneill }
4566 1.1 jmcneill free:
4567 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4568 1.1 jmcneill
4569 1.1 jmcneill return (error);
4570 1.1 jmcneill }
4571 1.1 jmcneill
4572 1.1 jmcneill static int
4573 1.1 jmcneill mcx_query_special_contexts(struct mcx_softc *sc)
4574 1.1 jmcneill {
4575 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4576 1.1 jmcneill struct mcx_cmd_query_special_ctx_in *in;
4577 1.1 jmcneill struct mcx_cmd_query_special_ctx_out *out;
4578 1.1 jmcneill int error;
4579 1.1 jmcneill
4580 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4581 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
4582 1.1 jmcneill
4583 1.1 jmcneill in = mcx_cmdq_in(cqe);
4584 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS);
4585 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4586 1.1 jmcneill
4587 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4588 1.1 jmcneill
4589 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4590 1.1 jmcneill if (error != 0) {
4591 1.1 jmcneill printf(", query special contexts timeout\n");
4592 1.1 jmcneill return (-1);
4593 1.1 jmcneill }
4594 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4595 1.1 jmcneill printf(", query special contexts command corrupt\n");
4596 1.1 jmcneill return (-1);
4597 1.1 jmcneill }
4598 1.1 jmcneill
4599 1.1 jmcneill out = mcx_cmdq_out(cqe);
4600 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4601 1.1 jmcneill printf(", query special contexts failed (%x)\n",
4602 1.1 jmcneill out->cmd_status);
4603 1.1 jmcneill return (-1);
4604 1.1 jmcneill }
4605 1.1 jmcneill
4606 1.1 jmcneill sc->sc_lkey = be32toh(out->cmd_resd_lkey);
4607 1.1 jmcneill return (0);
4608 1.1 jmcneill }
4609 1.1 jmcneill
4610 1.1 jmcneill static int
4611 1.1 jmcneill mcx_set_port_mtu(struct mcx_softc *sc, int mtu)
4612 1.1 jmcneill {
4613 1.1 jmcneill struct mcx_reg_pmtu pmtu;
4614 1.1 jmcneill int error;
4615 1.1 jmcneill
4616 1.1 jmcneill /* read max mtu */
4617 1.1 jmcneill memset(&pmtu, 0, sizeof(pmtu));
4618 1.1 jmcneill pmtu.rp_local_port = 1;
4619 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu,
4620 1.1 jmcneill sizeof(pmtu));
4621 1.1 jmcneill if (error != 0) {
4622 1.1 jmcneill printf(", unable to get port MTU\n");
4623 1.1 jmcneill return error;
4624 1.1 jmcneill }
4625 1.1 jmcneill
4626 1.1 jmcneill mtu = uimin(mtu, be16toh(pmtu.rp_max_mtu));
4627 1.1 jmcneill pmtu.rp_admin_mtu = htobe16(mtu);
4628 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu,
4629 1.1 jmcneill sizeof(pmtu));
4630 1.1 jmcneill if (error != 0) {
4631 1.1 jmcneill printf(", unable to set port MTU\n");
4632 1.1 jmcneill return error;
4633 1.1 jmcneill }
4634 1.1 jmcneill
4635 1.1 jmcneill sc->sc_hardmtu = mtu;
4636 1.15 jmcneill sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long));
4637 1.1 jmcneill return 0;
4638 1.1 jmcneill }
4639 1.1 jmcneill
4640 1.1 jmcneill static int
4641 1.15 jmcneill mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn)
4642 1.1 jmcneill {
4643 1.1 jmcneill struct mcx_cmdq_entry *cmde;
4644 1.1 jmcneill struct mcx_cq_entry *cqe;
4645 1.1 jmcneill struct mcx_dmamem mxm;
4646 1.1 jmcneill struct mcx_cmd_create_cq_in *in;
4647 1.1 jmcneill struct mcx_cmd_create_cq_mb_in *mbin;
4648 1.1 jmcneill struct mcx_cmd_create_cq_out *out;
4649 1.1 jmcneill int error;
4650 1.1 jmcneill uint64_t *pas;
4651 1.1 jmcneill int insize, npages, paslen, i, token;
4652 1.1 jmcneill
4653 1.15 jmcneill cq->cq_doorbell = MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db);
4654 1.1 jmcneill
4655 1.1 jmcneill npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry),
4656 1.1 jmcneill MCX_PAGE_SIZE);
4657 1.1 jmcneill paslen = npages * sizeof(*pas);
4658 1.1 jmcneill insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen;
4659 1.1 jmcneill
4660 1.1 jmcneill if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE,
4661 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
4662 1.1 jmcneill printf("%s: unable to allocate completion queue memory\n",
4663 1.1 jmcneill DEVNAME(sc));
4664 1.1 jmcneill return (-1);
4665 1.1 jmcneill }
4666 1.1 jmcneill cqe = MCX_DMA_KVA(&cq->cq_mem);
4667 1.1 jmcneill for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) {
4668 1.1 jmcneill cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER;
4669 1.1 jmcneill }
4670 1.1 jmcneill
4671 1.1 jmcneill cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4672 1.1 jmcneill token = mcx_cmdq_token(sc);
4673 1.1 jmcneill mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token);
4674 1.1 jmcneill
4675 1.1 jmcneill in = mcx_cmdq_in(cmde);
4676 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ);
4677 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4678 1.1 jmcneill
4679 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4680 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4681 1.1 jmcneill &cmde->cq_input_ptr, token) != 0) {
4682 1.15 jmcneill printf("%s: unable to allocate create cq mailboxen\n",
4683 1.15 jmcneill DEVNAME(sc));
4684 1.15 jmcneill goto free_cq;
4685 1.1 jmcneill }
4686 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4687 1.1 jmcneill mbin->cmd_cq_ctx.cq_uar_size = htobe32(
4688 1.15 jmcneill (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar);
4689 1.1 jmcneill mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn);
4690 1.1 jmcneill mbin->cmd_cq_ctx.cq_period_max_count = htobe32(
4691 1.1 jmcneill (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) |
4692 1.1 jmcneill MCX_CQ_MOD_COUNTER);
4693 1.1 jmcneill mbin->cmd_cq_ctx.cq_doorbell = htobe64(
4694 1.15 jmcneill MCX_DMA_DVA(&sc->sc_doorbell_mem) + cq->cq_doorbell);
4695 1.15 jmcneill
4696 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4697 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
4698 1.1 jmcneill
4699 1.1 jmcneill /* physical addresses follow the mailbox in data */
4700 1.13 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem);
4701 1.1 jmcneill mcx_cmdq_post(sc, cmde, 0);
4702 1.1 jmcneill
4703 1.1 jmcneill error = mcx_cmdq_poll(sc, cmde, 1000);
4704 1.1 jmcneill if (error != 0) {
4705 1.1 jmcneill printf("%s: create cq timeout\n", DEVNAME(sc));
4706 1.15 jmcneill goto free_mxm;
4707 1.1 jmcneill }
4708 1.1 jmcneill if (mcx_cmdq_verify(cmde) != 0) {
4709 1.1 jmcneill printf("%s: create cq command corrupt\n", DEVNAME(sc));
4710 1.15 jmcneill goto free_mxm;
4711 1.1 jmcneill }
4712 1.1 jmcneill
4713 1.1 jmcneill out = mcx_cmdq_out(cmde);
4714 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4715 1.1 jmcneill printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc),
4716 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4717 1.15 jmcneill goto free_mxm;
4718 1.1 jmcneill }
4719 1.1 jmcneill
4720 1.13 jmcneill cq->cq_n = mcx_get_id(out->cmd_cqn);
4721 1.1 jmcneill cq->cq_cons = 0;
4722 1.1 jmcneill cq->cq_count = 0;
4723 1.1 jmcneill
4724 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4725 1.15 jmcneill
4726 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4727 1.15 jmcneill cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
4728 1.15 jmcneill BUS_DMASYNC_PREWRITE);
4729 1.15 jmcneill
4730 1.15 jmcneill mcx_arm_cq(sc, cq, uar);
4731 1.15 jmcneill
4732 1.15 jmcneill return (0);
4733 1.15 jmcneill
4734 1.15 jmcneill free_mxm:
4735 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4736 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
4737 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
4738 1.15 jmcneill free_cq:
4739 1.15 jmcneill mcx_dmamem_free(sc, &cq->cq_mem);
4740 1.15 jmcneill return (-1);
4741 1.1 jmcneill }
4742 1.1 jmcneill
4743 1.1 jmcneill static int
4744 1.15 jmcneill mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq)
4745 1.1 jmcneill {
4746 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4747 1.1 jmcneill struct mcx_cmd_destroy_cq_in *in;
4748 1.1 jmcneill struct mcx_cmd_destroy_cq_out *out;
4749 1.1 jmcneill int error;
4750 1.1 jmcneill int token;
4751 1.1 jmcneill
4752 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4753 1.1 jmcneill token = mcx_cmdq_token(sc);
4754 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
4755 1.1 jmcneill
4756 1.1 jmcneill in = mcx_cmdq_in(cqe);
4757 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ);
4758 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4759 1.15 jmcneill in->cmd_cqn = htobe32(cq->cq_n);
4760 1.1 jmcneill
4761 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4762 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4763 1.1 jmcneill if (error != 0) {
4764 1.1 jmcneill printf("%s: destroy cq timeout\n", DEVNAME(sc));
4765 1.1 jmcneill return error;
4766 1.1 jmcneill }
4767 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4768 1.1 jmcneill printf("%s: destroy cq command corrupt\n", DEVNAME(sc));
4769 1.1 jmcneill return error;
4770 1.1 jmcneill }
4771 1.1 jmcneill
4772 1.1 jmcneill out = mcx_cmdq_out(cqe);
4773 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4774 1.1 jmcneill printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc),
4775 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4776 1.1 jmcneill return -1;
4777 1.1 jmcneill }
4778 1.1 jmcneill
4779 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4780 1.15 jmcneill cq->cq_doorbell, sizeof(struct mcx_cq_doorbell),
4781 1.15 jmcneill BUS_DMASYNC_POSTWRITE);
4782 1.15 jmcneill
4783 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
4784 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
4785 1.15 jmcneill mcx_dmamem_free(sc, &cq->cq_mem);
4786 1.15 jmcneill
4787 1.15 jmcneill cq->cq_n = 0;
4788 1.15 jmcneill cq->cq_cons = 0;
4789 1.15 jmcneill cq->cq_count = 0;
4790 1.1 jmcneill return 0;
4791 1.1 jmcneill }
4792 1.1 jmcneill
4793 1.1 jmcneill static int
4794 1.15 jmcneill mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn)
4795 1.1 jmcneill {
4796 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4797 1.1 jmcneill struct mcx_dmamem mxm;
4798 1.1 jmcneill struct mcx_cmd_create_rq_in *in;
4799 1.1 jmcneill struct mcx_cmd_create_rq_out *out;
4800 1.1 jmcneill struct mcx_rq_ctx *mbin;
4801 1.1 jmcneill int error;
4802 1.1 jmcneill uint64_t *pas;
4803 1.15 jmcneill uint32_t rq_flags;
4804 1.13 jmcneill int insize, npages, paslen, token;
4805 1.1 jmcneill
4806 1.15 jmcneill rx->rx_doorbell = MCX_WQ_DOORBELL_BASE +
4807 1.15 jmcneill (db * MCX_WQ_DOORBELL_STRIDE);
4808 1.15 jmcneill
4809 1.1 jmcneill npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry),
4810 1.1 jmcneill MCX_PAGE_SIZE);
4811 1.1 jmcneill paslen = npages * sizeof(*pas);
4812 1.1 jmcneill insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen;
4813 1.1 jmcneill
4814 1.15 jmcneill if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE,
4815 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
4816 1.1 jmcneill printf("%s: unable to allocate receive queue memory\n",
4817 1.1 jmcneill DEVNAME(sc));
4818 1.1 jmcneill return (-1);
4819 1.1 jmcneill }
4820 1.1 jmcneill
4821 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4822 1.1 jmcneill token = mcx_cmdq_token(sc);
4823 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token);
4824 1.1 jmcneill
4825 1.1 jmcneill in = mcx_cmdq_in(cqe);
4826 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ);
4827 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4828 1.1 jmcneill
4829 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
4830 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
4831 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
4832 1.1 jmcneill printf("%s: unable to allocate create rq mailboxen\n",
4833 1.1 jmcneill DEVNAME(sc));
4834 1.15 jmcneill goto free_rq;
4835 1.15 jmcneill }
4836 1.15 jmcneill mbin = (struct mcx_rq_ctx *)
4837 1.15 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
4838 1.15 jmcneill rq_flags = MCX_RQ_CTX_RLKEY;
4839 1.15 jmcneill mbin->rq_flags = htobe32(rq_flags);
4840 1.1 jmcneill mbin->rq_cqn = htobe32(cqn);
4841 1.1 jmcneill mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
4842 1.1 jmcneill mbin->rq_wq.wq_pd = htobe32(sc->sc_pd);
4843 1.1 jmcneill mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
4844 1.15 jmcneill rx->rx_doorbell);
4845 1.1 jmcneill mbin->rq_wq.wq_log_stride = htobe16(4);
4846 1.1 jmcneill mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE;
4847 1.1 jmcneill
4848 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4849 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
4850 1.15 jmcneill
4851 1.1 jmcneill /* physical addresses follow the mailbox in data */
4852 1.15 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem);
4853 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4854 1.1 jmcneill
4855 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4856 1.1 jmcneill if (error != 0) {
4857 1.1 jmcneill printf("%s: create rq timeout\n", DEVNAME(sc));
4858 1.15 jmcneill goto free_mxm;
4859 1.1 jmcneill }
4860 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4861 1.1 jmcneill printf("%s: create rq command corrupt\n", DEVNAME(sc));
4862 1.15 jmcneill goto free_mxm;
4863 1.1 jmcneill }
4864 1.1 jmcneill
4865 1.1 jmcneill out = mcx_cmdq_out(cqe);
4866 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4867 1.1 jmcneill printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc),
4868 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4869 1.15 jmcneill goto free_mxm;
4870 1.1 jmcneill }
4871 1.1 jmcneill
4872 1.15 jmcneill rx->rx_rqn = mcx_get_id(out->cmd_rqn);
4873 1.1 jmcneill
4874 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
4875 1.15 jmcneill
4876 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4877 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
4878 1.15 jmcneill
4879 1.15 jmcneill return (0);
4880 1.1 jmcneill
4881 1.15 jmcneill free_mxm:
4882 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4883 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
4884 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4885 1.15 jmcneill free_rq:
4886 1.15 jmcneill mcx_dmamem_free(sc, &rx->rx_rq_mem);
4887 1.15 jmcneill return (-1);
4888 1.1 jmcneill }
4889 1.1 jmcneill
4890 1.1 jmcneill static int
4891 1.15 jmcneill mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx)
4892 1.1 jmcneill {
4893 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4894 1.1 jmcneill struct mcx_dmamem mxm;
4895 1.1 jmcneill struct mcx_cmd_modify_rq_in *in;
4896 1.1 jmcneill struct mcx_cmd_modify_rq_mb_in *mbin;
4897 1.1 jmcneill struct mcx_cmd_modify_rq_out *out;
4898 1.1 jmcneill int error;
4899 1.1 jmcneill int token;
4900 1.1 jmcneill
4901 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4902 1.1 jmcneill token = mcx_cmdq_token(sc);
4903 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
4904 1.15 jmcneill sizeof(*out), token);
4905 1.1 jmcneill
4906 1.1 jmcneill in = mcx_cmdq_in(cqe);
4907 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ);
4908 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4909 1.15 jmcneill in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn);
4910 1.1 jmcneill
4911 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
4912 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
4913 1.15 jmcneill printf("%s: unable to allocate modify rq mailbox\n",
4914 1.15 jmcneill DEVNAME(sc));
4915 1.1 jmcneill return (-1);
4916 1.1 jmcneill }
4917 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
4918 1.1 jmcneill mbin->cmd_rq_ctx.rq_flags = htobe32(
4919 1.1 jmcneill MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT);
4920 1.1 jmcneill
4921 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
4922 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4923 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4924 1.1 jmcneill if (error != 0) {
4925 1.1 jmcneill printf("%s: modify rq timeout\n", DEVNAME(sc));
4926 1.1 jmcneill goto free;
4927 1.1 jmcneill }
4928 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4929 1.1 jmcneill printf("%s: modify rq command corrupt\n", DEVNAME(sc));
4930 1.1 jmcneill goto free;
4931 1.1 jmcneill }
4932 1.1 jmcneill
4933 1.1 jmcneill out = mcx_cmdq_out(cqe);
4934 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4935 1.1 jmcneill printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc),
4936 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4937 1.1 jmcneill error = -1;
4938 1.1 jmcneill goto free;
4939 1.1 jmcneill }
4940 1.1 jmcneill
4941 1.1 jmcneill free:
4942 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
4943 1.1 jmcneill return (error);
4944 1.1 jmcneill }
4945 1.1 jmcneill
4946 1.1 jmcneill static int
4947 1.15 jmcneill mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx)
4948 1.1 jmcneill {
4949 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4950 1.1 jmcneill struct mcx_cmd_destroy_rq_in *in;
4951 1.1 jmcneill struct mcx_cmd_destroy_rq_out *out;
4952 1.1 jmcneill int error;
4953 1.1 jmcneill int token;
4954 1.1 jmcneill
4955 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
4956 1.1 jmcneill token = mcx_cmdq_token(sc);
4957 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
4958 1.1 jmcneill
4959 1.1 jmcneill in = mcx_cmdq_in(cqe);
4960 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ);
4961 1.1 jmcneill in->cmd_op_mod = htobe16(0);
4962 1.15 jmcneill in->cmd_rqn = htobe32(rx->rx_rqn);
4963 1.1 jmcneill
4964 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
4965 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
4966 1.1 jmcneill if (error != 0) {
4967 1.1 jmcneill printf("%s: destroy rq timeout\n", DEVNAME(sc));
4968 1.1 jmcneill return error;
4969 1.1 jmcneill }
4970 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
4971 1.1 jmcneill printf("%s: destroy rq command corrupt\n", DEVNAME(sc));
4972 1.1 jmcneill return error;
4973 1.1 jmcneill }
4974 1.1 jmcneill
4975 1.1 jmcneill out = mcx_cmdq_out(cqe);
4976 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
4977 1.1 jmcneill printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc),
4978 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
4979 1.1 jmcneill return -1;
4980 1.1 jmcneill }
4981 1.1 jmcneill
4982 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
4983 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
4984 1.15 jmcneill
4985 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
4986 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
4987 1.15 jmcneill mcx_dmamem_free(sc, &rx->rx_rq_mem);
4988 1.15 jmcneill
4989 1.15 jmcneill rx->rx_rqn = 0;
4990 1.1 jmcneill return 0;
4991 1.1 jmcneill }
4992 1.1 jmcneill
4993 1.1 jmcneill static int
4994 1.15 jmcneill mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn)
4995 1.1 jmcneill {
4996 1.1 jmcneill struct mcx_cmdq_entry *cqe;
4997 1.1 jmcneill struct mcx_dmamem mxm;
4998 1.1 jmcneill struct mcx_cmd_create_tir_in *in;
4999 1.1 jmcneill struct mcx_cmd_create_tir_mb_in *mbin;
5000 1.1 jmcneill struct mcx_cmd_create_tir_out *out;
5001 1.1 jmcneill int error;
5002 1.1 jmcneill int token;
5003 1.1 jmcneill
5004 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5005 1.1 jmcneill token = mcx_cmdq_token(sc);
5006 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5007 1.15 jmcneill sizeof(*out), token);
5008 1.1 jmcneill
5009 1.1 jmcneill in = mcx_cmdq_in(cqe);
5010 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
5011 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5012 1.1 jmcneill
5013 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5014 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5015 1.1 jmcneill printf("%s: unable to allocate create tir mailbox\n",
5016 1.1 jmcneill DEVNAME(sc));
5017 1.1 jmcneill return (-1);
5018 1.1 jmcneill }
5019 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5020 1.1 jmcneill /* leave disp_type = 0, so packets get sent to the inline rqn */
5021 1.15 jmcneill mbin->cmd_inline_rqn = htobe32(rx->rx_rqn);
5022 1.1 jmcneill mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
5023 1.1 jmcneill
5024 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5025 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5026 1.1 jmcneill if (error != 0) {
5027 1.1 jmcneill printf("%s: create tir timeout\n", DEVNAME(sc));
5028 1.1 jmcneill goto free;
5029 1.1 jmcneill }
5030 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5031 1.1 jmcneill printf("%s: create tir command corrupt\n", DEVNAME(sc));
5032 1.1 jmcneill goto free;
5033 1.1 jmcneill }
5034 1.1 jmcneill
5035 1.1 jmcneill out = mcx_cmdq_out(cqe);
5036 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5037 1.1 jmcneill printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
5038 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5039 1.1 jmcneill error = -1;
5040 1.1 jmcneill goto free;
5041 1.1 jmcneill }
5042 1.1 jmcneill
5043 1.15 jmcneill *tirn = mcx_get_id(out->cmd_tirn);
5044 1.15 jmcneill free:
5045 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
5046 1.15 jmcneill return (error);
5047 1.15 jmcneill }
5048 1.15 jmcneill
5049 1.15 jmcneill static int
5050 1.15 jmcneill mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel,
5051 1.15 jmcneill int *tirn)
5052 1.15 jmcneill {
5053 1.15 jmcneill struct mcx_cmdq_entry *cqe;
5054 1.15 jmcneill struct mcx_dmamem mxm;
5055 1.15 jmcneill struct mcx_cmd_create_tir_in *in;
5056 1.15 jmcneill struct mcx_cmd_create_tir_mb_in *mbin;
5057 1.15 jmcneill struct mcx_cmd_create_tir_out *out;
5058 1.15 jmcneill int error;
5059 1.15 jmcneill int token;
5060 1.15 jmcneill
5061 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5062 1.15 jmcneill token = mcx_cmdq_token(sc);
5063 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5064 1.15 jmcneill sizeof(*out), token);
5065 1.15 jmcneill
5066 1.15 jmcneill in = mcx_cmdq_in(cqe);
5067 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR);
5068 1.15 jmcneill in->cmd_op_mod = htobe16(0);
5069 1.15 jmcneill
5070 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5071 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5072 1.15 jmcneill printf("%s: unable to allocate create tir mailbox\n",
5073 1.15 jmcneill DEVNAME(sc));
5074 1.15 jmcneill return (-1);
5075 1.15 jmcneill }
5076 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5077 1.15 jmcneill mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT
5078 1.15 jmcneill << MCX_TIR_CTX_DISP_TYPE_SHIFT);
5079 1.15 jmcneill mbin->cmd_indir_table = htobe32(rqtn);
5080 1.15 jmcneill mbin->cmd_tdomain = htobe32(sc->sc_tdomain |
5081 1.15 jmcneill MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT);
5082 1.15 jmcneill mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel);
5083 1.15 jmcneill stoeplitz_to_key(&mbin->cmd_rx_hash_key,
5084 1.15 jmcneill sizeof(mbin->cmd_rx_hash_key));
5085 1.15 jmcneill
5086 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
5087 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5088 1.15 jmcneill if (error != 0) {
5089 1.15 jmcneill printf("%s: create tir timeout\n", DEVNAME(sc));
5090 1.15 jmcneill goto free;
5091 1.15 jmcneill }
5092 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5093 1.15 jmcneill printf("%s: create tir command corrupt\n", DEVNAME(sc));
5094 1.15 jmcneill goto free;
5095 1.15 jmcneill }
5096 1.15 jmcneill
5097 1.15 jmcneill out = mcx_cmdq_out(cqe);
5098 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5099 1.15 jmcneill printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc),
5100 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5101 1.15 jmcneill error = -1;
5102 1.15 jmcneill goto free;
5103 1.15 jmcneill }
5104 1.15 jmcneill
5105 1.15 jmcneill *tirn = mcx_get_id(out->cmd_tirn);
5106 1.1 jmcneill free:
5107 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5108 1.1 jmcneill return (error);
5109 1.1 jmcneill }
5110 1.1 jmcneill
5111 1.1 jmcneill static int
5112 1.15 jmcneill mcx_destroy_tir(struct mcx_softc *sc, int tirn)
5113 1.1 jmcneill {
5114 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5115 1.1 jmcneill struct mcx_cmd_destroy_tir_in *in;
5116 1.1 jmcneill struct mcx_cmd_destroy_tir_out *out;
5117 1.1 jmcneill int error;
5118 1.1 jmcneill int token;
5119 1.1 jmcneill
5120 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5121 1.1 jmcneill token = mcx_cmdq_token(sc);
5122 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5123 1.1 jmcneill
5124 1.1 jmcneill in = mcx_cmdq_in(cqe);
5125 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR);
5126 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5127 1.15 jmcneill in->cmd_tirn = htobe32(tirn);
5128 1.1 jmcneill
5129 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5130 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5131 1.1 jmcneill if (error != 0) {
5132 1.1 jmcneill printf("%s: destroy tir timeout\n", DEVNAME(sc));
5133 1.1 jmcneill return error;
5134 1.1 jmcneill }
5135 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5136 1.1 jmcneill printf("%s: destroy tir command corrupt\n", DEVNAME(sc));
5137 1.1 jmcneill return error;
5138 1.1 jmcneill }
5139 1.1 jmcneill
5140 1.1 jmcneill out = mcx_cmdq_out(cqe);
5141 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5142 1.1 jmcneill printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc),
5143 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5144 1.1 jmcneill return -1;
5145 1.1 jmcneill }
5146 1.1 jmcneill
5147 1.15 jmcneill return (0);
5148 1.1 jmcneill }
5149 1.1 jmcneill
5150 1.1 jmcneill static int
5151 1.15 jmcneill mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db,
5152 1.15 jmcneill int cqn)
5153 1.1 jmcneill {
5154 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5155 1.1 jmcneill struct mcx_dmamem mxm;
5156 1.1 jmcneill struct mcx_cmd_create_sq_in *in;
5157 1.1 jmcneill struct mcx_sq_ctx *mbin;
5158 1.1 jmcneill struct mcx_cmd_create_sq_out *out;
5159 1.1 jmcneill int error;
5160 1.1 jmcneill uint64_t *pas;
5161 1.13 jmcneill int insize, npages, paslen, token;
5162 1.1 jmcneill
5163 1.15 jmcneill tx->tx_doorbell = MCX_WQ_DOORBELL_BASE +
5164 1.15 jmcneill (db * MCX_WQ_DOORBELL_STRIDE) + 4;
5165 1.15 jmcneill
5166 1.1 jmcneill npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry),
5167 1.1 jmcneill MCX_PAGE_SIZE);
5168 1.1 jmcneill paslen = npages * sizeof(*pas);
5169 1.1 jmcneill insize = sizeof(struct mcx_sq_ctx) + paslen;
5170 1.1 jmcneill
5171 1.15 jmcneill if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE,
5172 1.1 jmcneill MCX_PAGE_SIZE) != 0) {
5173 1.15 jmcneill printf("%s: unable to allocate send queue memory\n",
5174 1.15 jmcneill DEVNAME(sc));
5175 1.1 jmcneill return (-1);
5176 1.1 jmcneill }
5177 1.1 jmcneill
5178 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5179 1.1 jmcneill token = mcx_cmdq_token(sc);
5180 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out),
5181 1.1 jmcneill token);
5182 1.1 jmcneill
5183 1.1 jmcneill in = mcx_cmdq_in(cqe);
5184 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ);
5185 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5186 1.1 jmcneill
5187 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm,
5188 1.15 jmcneill howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE),
5189 1.1 jmcneill &cqe->cq_input_ptr, token) != 0) {
5190 1.15 jmcneill printf("%s: unable to allocate create sq mailboxen\n",
5191 1.15 jmcneill DEVNAME(sc));
5192 1.15 jmcneill goto free_sq;
5193 1.1 jmcneill }
5194 1.15 jmcneill mbin = (struct mcx_sq_ctx *)
5195 1.15 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10);
5196 1.1 jmcneill mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY |
5197 1.1 jmcneill (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT));
5198 1.1 jmcneill mbin->sq_cqn = htobe32(cqn);
5199 1.1 jmcneill mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT);
5200 1.15 jmcneill mbin->sq_tis_num = htobe32(sc->sc_tis);
5201 1.1 jmcneill mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC;
5202 1.1 jmcneill mbin->sq_wq.wq_pd = htobe32(sc->sc_pd);
5203 1.15 jmcneill mbin->sq_wq.wq_uar_page = htobe32(uar);
5204 1.1 jmcneill mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) +
5205 1.15 jmcneill tx->tx_doorbell);
5206 1.1 jmcneill mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE);
5207 1.1 jmcneill mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE;
5208 1.1 jmcneill
5209 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5210 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
5211 1.15 jmcneill
5212 1.1 jmcneill /* physical addresses follow the mailbox in data */
5213 1.15 jmcneill mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10,
5214 1.15 jmcneill npages, &tx->tx_sq_mem);
5215 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5216 1.1 jmcneill
5217 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5218 1.1 jmcneill if (error != 0) {
5219 1.1 jmcneill printf("%s: create sq timeout\n", DEVNAME(sc));
5220 1.15 jmcneill goto free_mxm;
5221 1.1 jmcneill }
5222 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5223 1.1 jmcneill printf("%s: create sq command corrupt\n", DEVNAME(sc));
5224 1.15 jmcneill goto free_mxm;
5225 1.1 jmcneill }
5226 1.1 jmcneill
5227 1.1 jmcneill out = mcx_cmdq_out(cqe);
5228 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5229 1.1 jmcneill printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc),
5230 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5231 1.15 jmcneill goto free_mxm;
5232 1.1 jmcneill }
5233 1.1 jmcneill
5234 1.15 jmcneill tx->tx_uar = uar;
5235 1.15 jmcneill tx->tx_sqn = mcx_get_id(out->cmd_sqn);
5236 1.15 jmcneill
5237 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
5238 1.15 jmcneill
5239 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
5240 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
5241 1.15 jmcneill
5242 1.15 jmcneill return (0);
5243 1.1 jmcneill
5244 1.15 jmcneill free_mxm:
5245 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5246 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
5247 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5248 1.15 jmcneill free_sq:
5249 1.15 jmcneill mcx_dmamem_free(sc, &tx->tx_sq_mem);
5250 1.15 jmcneill return (-1);
5251 1.1 jmcneill }
5252 1.1 jmcneill
5253 1.1 jmcneill static int
5254 1.15 jmcneill mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx)
5255 1.1 jmcneill {
5256 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5257 1.1 jmcneill struct mcx_cmd_destroy_sq_in *in;
5258 1.1 jmcneill struct mcx_cmd_destroy_sq_out *out;
5259 1.1 jmcneill int error;
5260 1.1 jmcneill int token;
5261 1.1 jmcneill
5262 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5263 1.1 jmcneill token = mcx_cmdq_token(sc);
5264 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5265 1.1 jmcneill
5266 1.1 jmcneill in = mcx_cmdq_in(cqe);
5267 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ);
5268 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5269 1.15 jmcneill in->cmd_sqn = htobe32(tx->tx_sqn);
5270 1.1 jmcneill
5271 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5272 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5273 1.1 jmcneill if (error != 0) {
5274 1.1 jmcneill printf("%s: destroy sq timeout\n", DEVNAME(sc));
5275 1.1 jmcneill return error;
5276 1.1 jmcneill }
5277 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5278 1.1 jmcneill printf("%s: destroy sq command corrupt\n", DEVNAME(sc));
5279 1.1 jmcneill return error;
5280 1.1 jmcneill }
5281 1.1 jmcneill
5282 1.1 jmcneill out = mcx_cmdq_out(cqe);
5283 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5284 1.1 jmcneill printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc),
5285 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5286 1.1 jmcneill return -1;
5287 1.1 jmcneill }
5288 1.1 jmcneill
5289 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
5290 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
5291 1.15 jmcneill
5292 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
5293 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
5294 1.15 jmcneill mcx_dmamem_free(sc, &tx->tx_sq_mem);
5295 1.15 jmcneill
5296 1.15 jmcneill tx->tx_sqn = 0;
5297 1.1 jmcneill return 0;
5298 1.1 jmcneill }
5299 1.1 jmcneill
5300 1.1 jmcneill static int
5301 1.15 jmcneill mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx)
5302 1.1 jmcneill {
5303 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5304 1.1 jmcneill struct mcx_dmamem mxm;
5305 1.1 jmcneill struct mcx_cmd_modify_sq_in *in;
5306 1.1 jmcneill struct mcx_cmd_modify_sq_mb_in *mbin;
5307 1.1 jmcneill struct mcx_cmd_modify_sq_out *out;
5308 1.1 jmcneill int error;
5309 1.1 jmcneill int token;
5310 1.1 jmcneill
5311 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5312 1.1 jmcneill token = mcx_cmdq_token(sc);
5313 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5314 1.15 jmcneill sizeof(*out), token);
5315 1.1 jmcneill
5316 1.1 jmcneill in = mcx_cmdq_in(cqe);
5317 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ);
5318 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5319 1.15 jmcneill in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn);
5320 1.1 jmcneill
5321 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5322 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5323 1.1 jmcneill printf("%s: unable to allocate modify sq mailbox\n",
5324 1.1 jmcneill DEVNAME(sc));
5325 1.1 jmcneill return (-1);
5326 1.1 jmcneill }
5327 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5328 1.1 jmcneill mbin->cmd_sq_ctx.sq_flags = htobe32(
5329 1.1 jmcneill MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT);
5330 1.1 jmcneill
5331 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5332 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5333 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5334 1.1 jmcneill if (error != 0) {
5335 1.1 jmcneill printf("%s: modify sq timeout\n", DEVNAME(sc));
5336 1.1 jmcneill goto free;
5337 1.1 jmcneill }
5338 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5339 1.1 jmcneill printf("%s: modify sq command corrupt\n", DEVNAME(sc));
5340 1.1 jmcneill goto free;
5341 1.1 jmcneill }
5342 1.1 jmcneill
5343 1.1 jmcneill out = mcx_cmdq_out(cqe);
5344 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5345 1.1 jmcneill printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc),
5346 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5347 1.1 jmcneill error = -1;
5348 1.1 jmcneill goto free;
5349 1.1 jmcneill }
5350 1.1 jmcneill
5351 1.1 jmcneill free:
5352 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5353 1.1 jmcneill return (error);
5354 1.1 jmcneill }
5355 1.1 jmcneill
5356 1.1 jmcneill static int
5357 1.15 jmcneill mcx_create_tis(struct mcx_softc *sc, int *tis)
5358 1.1 jmcneill {
5359 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5360 1.1 jmcneill struct mcx_dmamem mxm;
5361 1.1 jmcneill struct mcx_cmd_create_tis_in *in;
5362 1.1 jmcneill struct mcx_cmd_create_tis_mb_in *mbin;
5363 1.1 jmcneill struct mcx_cmd_create_tis_out *out;
5364 1.1 jmcneill int error;
5365 1.1 jmcneill int token;
5366 1.1 jmcneill
5367 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5368 1.1 jmcneill token = mcx_cmdq_token(sc);
5369 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5370 1.15 jmcneill sizeof(*out), token);
5371 1.1 jmcneill
5372 1.1 jmcneill in = mcx_cmdq_in(cqe);
5373 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS);
5374 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5375 1.1 jmcneill
5376 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5377 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5378 1.15 jmcneill printf("%s: unable to allocate create tis mailbox\n",
5379 1.15 jmcneill DEVNAME(sc));
5380 1.1 jmcneill return (-1);
5381 1.1 jmcneill }
5382 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5383 1.1 jmcneill mbin->cmd_tdomain = htobe32(sc->sc_tdomain);
5384 1.1 jmcneill
5385 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5386 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5387 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5388 1.1 jmcneill if (error != 0) {
5389 1.1 jmcneill printf("%s: create tis timeout\n", DEVNAME(sc));
5390 1.1 jmcneill goto free;
5391 1.1 jmcneill }
5392 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5393 1.1 jmcneill printf("%s: create tis command corrupt\n", DEVNAME(sc));
5394 1.1 jmcneill goto free;
5395 1.1 jmcneill }
5396 1.1 jmcneill
5397 1.1 jmcneill out = mcx_cmdq_out(cqe);
5398 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5399 1.1 jmcneill printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc),
5400 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5401 1.1 jmcneill error = -1;
5402 1.1 jmcneill goto free;
5403 1.1 jmcneill }
5404 1.1 jmcneill
5405 1.15 jmcneill *tis = mcx_get_id(out->cmd_tisn);
5406 1.1 jmcneill free:
5407 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5408 1.1 jmcneill return (error);
5409 1.1 jmcneill }
5410 1.1 jmcneill
5411 1.1 jmcneill static int
5412 1.15 jmcneill mcx_destroy_tis(struct mcx_softc *sc, int tis)
5413 1.1 jmcneill {
5414 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5415 1.1 jmcneill struct mcx_cmd_destroy_tis_in *in;
5416 1.1 jmcneill struct mcx_cmd_destroy_tis_out *out;
5417 1.1 jmcneill int error;
5418 1.1 jmcneill int token;
5419 1.1 jmcneill
5420 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5421 1.1 jmcneill token = mcx_cmdq_token(sc);
5422 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5423 1.1 jmcneill
5424 1.1 jmcneill in = mcx_cmdq_in(cqe);
5425 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS);
5426 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5427 1.15 jmcneill in->cmd_tisn = htobe32(tis);
5428 1.1 jmcneill
5429 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5430 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5431 1.1 jmcneill if (error != 0) {
5432 1.1 jmcneill printf("%s: destroy tis timeout\n", DEVNAME(sc));
5433 1.1 jmcneill return error;
5434 1.1 jmcneill }
5435 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5436 1.1 jmcneill printf("%s: destroy tis command corrupt\n", DEVNAME(sc));
5437 1.1 jmcneill return error;
5438 1.1 jmcneill }
5439 1.1 jmcneill
5440 1.1 jmcneill out = mcx_cmdq_out(cqe);
5441 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5442 1.1 jmcneill printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc),
5443 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5444 1.1 jmcneill return -1;
5445 1.1 jmcneill }
5446 1.1 jmcneill
5447 1.15 jmcneill return 0;
5448 1.15 jmcneill }
5449 1.15 jmcneill
5450 1.15 jmcneill static int
5451 1.15 jmcneill mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt)
5452 1.15 jmcneill {
5453 1.15 jmcneill struct mcx_cmdq_entry *cqe;
5454 1.15 jmcneill struct mcx_dmamem mxm;
5455 1.15 jmcneill struct mcx_cmd_create_rqt_in *in;
5456 1.15 jmcneill struct mcx_cmd_create_rqt_mb_in *mbin;
5457 1.15 jmcneill struct mcx_cmd_create_rqt_out *out;
5458 1.15 jmcneill struct mcx_rqt_ctx *rqt_ctx;
5459 1.15 jmcneill int *rqtn;
5460 1.15 jmcneill int error;
5461 1.15 jmcneill int token;
5462 1.15 jmcneill int i;
5463 1.15 jmcneill
5464 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5465 1.15 jmcneill token = mcx_cmdq_token(sc);
5466 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) +
5467 1.15 jmcneill (size * sizeof(int)), sizeof(*out), token);
5468 1.15 jmcneill
5469 1.15 jmcneill in = mcx_cmdq_in(cqe);
5470 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT);
5471 1.15 jmcneill in->cmd_op_mod = htobe16(0);
5472 1.15 jmcneill
5473 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5474 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5475 1.15 jmcneill printf("%s: unable to allocate create rqt mailbox\n",
5476 1.15 jmcneill DEVNAME(sc));
5477 1.15 jmcneill return (-1);
5478 1.15 jmcneill }
5479 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5480 1.15 jmcneill rqt_ctx = &mbin->cmd_rqt;
5481 1.15 jmcneill rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size);
5482 1.15 jmcneill rqt_ctx->cmd_rqt_actual_size = htobe16(size);
5483 1.15 jmcneill
5484 1.15 jmcneill /* rqt list follows the rqt context */
5485 1.15 jmcneill rqtn = (int *)(rqt_ctx + 1);
5486 1.15 jmcneill for (i = 0; i < size; i++) {
5487 1.15 jmcneill rqtn[i] = htobe32(rqns[i]);
5488 1.15 jmcneill }
5489 1.15 jmcneill
5490 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5491 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
5492 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5493 1.15 jmcneill if (error != 0) {
5494 1.15 jmcneill printf("%s: create rqt timeout\n", DEVNAME(sc));
5495 1.15 jmcneill goto free;
5496 1.15 jmcneill }
5497 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5498 1.15 jmcneill printf("%s: create rqt command corrupt\n", DEVNAME(sc));
5499 1.15 jmcneill goto free;
5500 1.15 jmcneill }
5501 1.15 jmcneill
5502 1.15 jmcneill out = mcx_cmdq_out(cqe);
5503 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5504 1.15 jmcneill printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc),
5505 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5506 1.15 jmcneill error = -1;
5507 1.15 jmcneill goto free;
5508 1.15 jmcneill }
5509 1.15 jmcneill
5510 1.15 jmcneill *rqt = mcx_get_id(out->cmd_rqtn);
5511 1.15 jmcneill return (0);
5512 1.15 jmcneill free:
5513 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
5514 1.15 jmcneill return (error);
5515 1.15 jmcneill }
5516 1.15 jmcneill
5517 1.15 jmcneill static int
5518 1.15 jmcneill mcx_destroy_rqt(struct mcx_softc *sc, int rqt)
5519 1.15 jmcneill {
5520 1.15 jmcneill struct mcx_cmdq_entry *cqe;
5521 1.15 jmcneill struct mcx_cmd_destroy_rqt_in *in;
5522 1.15 jmcneill struct mcx_cmd_destroy_rqt_out *out;
5523 1.15 jmcneill int error;
5524 1.15 jmcneill int token;
5525 1.15 jmcneill
5526 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5527 1.15 jmcneill token = mcx_cmdq_token(sc);
5528 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token);
5529 1.15 jmcneill
5530 1.15 jmcneill in = mcx_cmdq_in(cqe);
5531 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT);
5532 1.15 jmcneill in->cmd_op_mod = htobe16(0);
5533 1.15 jmcneill in->cmd_rqtn = htobe32(rqt);
5534 1.15 jmcneill
5535 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
5536 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5537 1.15 jmcneill if (error != 0) {
5538 1.15 jmcneill printf("%s: destroy rqt timeout\n", DEVNAME(sc));
5539 1.15 jmcneill return error;
5540 1.15 jmcneill }
5541 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5542 1.15 jmcneill printf("%s: destroy rqt command corrupt\n", DEVNAME(sc));
5543 1.15 jmcneill return error;
5544 1.15 jmcneill }
5545 1.15 jmcneill
5546 1.15 jmcneill out = mcx_cmdq_out(cqe);
5547 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5548 1.15 jmcneill printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc),
5549 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5550 1.15 jmcneill return -1;
5551 1.15 jmcneill }
5552 1.15 jmcneill
5553 1.1 jmcneill return 0;
5554 1.1 jmcneill }
5555 1.1 jmcneill
5556 1.1 jmcneill #if 0
5557 1.1 jmcneill static int
5558 1.1 jmcneill mcx_alloc_flow_counter(struct mcx_softc *sc, int i)
5559 1.1 jmcneill {
5560 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5561 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_in *in;
5562 1.1 jmcneill struct mcx_cmd_alloc_flow_counter_out *out;
5563 1.1 jmcneill int error;
5564 1.1 jmcneill
5565 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5566 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc));
5567 1.1 jmcneill
5568 1.1 jmcneill in = mcx_cmdq_in(cqe);
5569 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER);
5570 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5571 1.1 jmcneill
5572 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5573 1.1 jmcneill
5574 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5575 1.1 jmcneill if (error != 0) {
5576 1.1 jmcneill printf("%s: alloc flow counter timeout\n", DEVNAME(sc));
5577 1.1 jmcneill return (-1);
5578 1.1 jmcneill }
5579 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5580 1.1 jmcneill printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc));
5581 1.1 jmcneill return (-1);
5582 1.1 jmcneill }
5583 1.1 jmcneill
5584 1.1 jmcneill out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data;
5585 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5586 1.1 jmcneill printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc),
5587 1.1 jmcneill out->cmd_status);
5588 1.1 jmcneill return (-1);
5589 1.1 jmcneill }
5590 1.1 jmcneill
5591 1.1 jmcneill sc->sc_flow_counter_id[i] = be16toh(out->cmd_flow_counter_id);
5592 1.1 jmcneill printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]);
5593 1.1 jmcneill
5594 1.1 jmcneill return (0);
5595 1.1 jmcneill }
5596 1.1 jmcneill #endif
5597 1.1 jmcneill
5598 1.1 jmcneill static int
5599 1.15 jmcneill mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level,
5600 1.15 jmcneill int *flow_table_id)
5601 1.1 jmcneill {
5602 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5603 1.1 jmcneill struct mcx_dmamem mxm;
5604 1.1 jmcneill struct mcx_cmd_create_flow_table_in *in;
5605 1.1 jmcneill struct mcx_cmd_create_flow_table_mb_in *mbin;
5606 1.1 jmcneill struct mcx_cmd_create_flow_table_out *out;
5607 1.1 jmcneill int error;
5608 1.1 jmcneill int token;
5609 1.1 jmcneill
5610 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5611 1.1 jmcneill token = mcx_cmdq_token(sc);
5612 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5613 1.15 jmcneill sizeof(*out), token);
5614 1.1 jmcneill
5615 1.1 jmcneill in = mcx_cmdq_in(cqe);
5616 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE);
5617 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5618 1.1 jmcneill
5619 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5620 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5621 1.1 jmcneill printf("%s: unable to allocate create flow table mailbox\n",
5622 1.1 jmcneill DEVNAME(sc));
5623 1.1 jmcneill return (-1);
5624 1.1 jmcneill }
5625 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5626 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5627 1.1 jmcneill mbin->cmd_ctx.ft_log_size = log_size;
5628 1.15 jmcneill mbin->cmd_ctx.ft_level = level;
5629 1.1 jmcneill
5630 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5631 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5632 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5633 1.1 jmcneill if (error != 0) {
5634 1.1 jmcneill printf("%s: create flow table timeout\n", DEVNAME(sc));
5635 1.1 jmcneill goto free;
5636 1.1 jmcneill }
5637 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5638 1.1 jmcneill printf("%s: create flow table command corrupt\n", DEVNAME(sc));
5639 1.1 jmcneill goto free;
5640 1.1 jmcneill }
5641 1.1 jmcneill
5642 1.1 jmcneill out = mcx_cmdq_out(cqe);
5643 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5644 1.1 jmcneill printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc),
5645 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5646 1.1 jmcneill error = -1;
5647 1.1 jmcneill goto free;
5648 1.1 jmcneill }
5649 1.1 jmcneill
5650 1.15 jmcneill *flow_table_id = mcx_get_id(out->cmd_table_id);
5651 1.1 jmcneill free:
5652 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5653 1.1 jmcneill return (error);
5654 1.1 jmcneill }
5655 1.1 jmcneill
5656 1.1 jmcneill static int
5657 1.15 jmcneill mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id)
5658 1.1 jmcneill {
5659 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5660 1.1 jmcneill struct mcx_dmamem mxm;
5661 1.1 jmcneill struct mcx_cmd_set_flow_table_root_in *in;
5662 1.1 jmcneill struct mcx_cmd_set_flow_table_root_mb_in *mbin;
5663 1.1 jmcneill struct mcx_cmd_set_flow_table_root_out *out;
5664 1.1 jmcneill int error;
5665 1.1 jmcneill int token;
5666 1.1 jmcneill
5667 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5668 1.1 jmcneill token = mcx_cmdq_token(sc);
5669 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
5670 1.15 jmcneill sizeof(*out), token);
5671 1.1 jmcneill
5672 1.1 jmcneill in = mcx_cmdq_in(cqe);
5673 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT);
5674 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5675 1.1 jmcneill
5676 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5677 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5678 1.1 jmcneill printf("%s: unable to allocate set flow table root mailbox\n",
5679 1.1 jmcneill DEVNAME(sc));
5680 1.1 jmcneill return (-1);
5681 1.1 jmcneill }
5682 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5683 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5684 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
5685 1.1 jmcneill
5686 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5687 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5688 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5689 1.1 jmcneill if (error != 0) {
5690 1.1 jmcneill printf("%s: set flow table root timeout\n", DEVNAME(sc));
5691 1.1 jmcneill goto free;
5692 1.1 jmcneill }
5693 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5694 1.1 jmcneill printf("%s: set flow table root command corrupt\n",
5695 1.1 jmcneill DEVNAME(sc));
5696 1.1 jmcneill goto free;
5697 1.1 jmcneill }
5698 1.1 jmcneill
5699 1.1 jmcneill out = mcx_cmdq_out(cqe);
5700 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5701 1.1 jmcneill printf("%s: set flow table root failed (%x, %x)\n",
5702 1.1 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
5703 1.1 jmcneill error = -1;
5704 1.1 jmcneill goto free;
5705 1.1 jmcneill }
5706 1.1 jmcneill
5707 1.1 jmcneill free:
5708 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5709 1.1 jmcneill return (error);
5710 1.1 jmcneill }
5711 1.1 jmcneill
5712 1.1 jmcneill static int
5713 1.15 jmcneill mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id)
5714 1.1 jmcneill {
5715 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5716 1.1 jmcneill struct mcx_dmamem mxm;
5717 1.1 jmcneill struct mcx_cmd_destroy_flow_table_in *in;
5718 1.1 jmcneill struct mcx_cmd_destroy_flow_table_mb_in *mb;
5719 1.1 jmcneill struct mcx_cmd_destroy_flow_table_out *out;
5720 1.1 jmcneill int error;
5721 1.1 jmcneill int token;
5722 1.1 jmcneill
5723 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5724 1.1 jmcneill token = mcx_cmdq_token(sc);
5725 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
5726 1.1 jmcneill
5727 1.1 jmcneill in = mcx_cmdq_in(cqe);
5728 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE);
5729 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5730 1.1 jmcneill
5731 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
5732 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5733 1.1 jmcneill printf("%s: unable to allocate destroy flow table mailbox\n",
5734 1.1 jmcneill DEVNAME(sc));
5735 1.1 jmcneill return (-1);
5736 1.1 jmcneill }
5737 1.1 jmcneill mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5738 1.1 jmcneill mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5739 1.15 jmcneill mb->cmd_table_id = htobe32(flow_table_id);
5740 1.1 jmcneill
5741 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
5742 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5743 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5744 1.1 jmcneill if (error != 0) {
5745 1.1 jmcneill printf("%s: destroy flow table timeout\n", DEVNAME(sc));
5746 1.1 jmcneill goto free;
5747 1.1 jmcneill }
5748 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5749 1.15 jmcneill printf("%s: destroy flow table command corrupt\n",
5750 1.15 jmcneill DEVNAME(sc));
5751 1.1 jmcneill goto free;
5752 1.1 jmcneill }
5753 1.1 jmcneill
5754 1.1 jmcneill out = mcx_cmdq_out(cqe);
5755 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5756 1.1 jmcneill printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc),
5757 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5758 1.1 jmcneill error = -1;
5759 1.1 jmcneill goto free;
5760 1.1 jmcneill }
5761 1.1 jmcneill
5762 1.1 jmcneill free:
5763 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5764 1.1 jmcneill return (error);
5765 1.1 jmcneill }
5766 1.1 jmcneill
5767 1.1 jmcneill
5768 1.1 jmcneill static int
5769 1.15 jmcneill mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group,
5770 1.15 jmcneill int start, int size, int match_enable, struct mcx_flow_match *match)
5771 1.1 jmcneill {
5772 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5773 1.1 jmcneill struct mcx_dmamem mxm;
5774 1.1 jmcneill struct mcx_cmd_create_flow_group_in *in;
5775 1.1 jmcneill struct mcx_cmd_create_flow_group_mb_in *mbin;
5776 1.1 jmcneill struct mcx_cmd_create_flow_group_out *out;
5777 1.15 jmcneill struct mcx_flow_group *mfg;
5778 1.1 jmcneill int error;
5779 1.1 jmcneill int token;
5780 1.1 jmcneill
5781 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5782 1.1 jmcneill token = mcx_cmdq_token(sc);
5783 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
5784 1.1 jmcneill token);
5785 1.1 jmcneill
5786 1.1 jmcneill in = mcx_cmdq_in(cqe);
5787 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP);
5788 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5789 1.1 jmcneill
5790 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5791 1.1 jmcneill != 0) {
5792 1.1 jmcneill printf("%s: unable to allocate create flow group mailbox\n",
5793 1.1 jmcneill DEVNAME(sc));
5794 1.1 jmcneill return (-1);
5795 1.1 jmcneill }
5796 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5797 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5798 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
5799 1.1 jmcneill mbin->cmd_start_flow_index = htobe32(start);
5800 1.1 jmcneill mbin->cmd_end_flow_index = htobe32(start + (size - 1));
5801 1.1 jmcneill
5802 1.1 jmcneill mbin->cmd_match_criteria_enable = match_enable;
5803 1.1 jmcneill memcpy(&mbin->cmd_match_criteria, match, sizeof(*match));
5804 1.1 jmcneill
5805 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
5806 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5807 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5808 1.1 jmcneill if (error != 0) {
5809 1.1 jmcneill printf("%s: create flow group timeout\n", DEVNAME(sc));
5810 1.1 jmcneill goto free;
5811 1.1 jmcneill }
5812 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5813 1.1 jmcneill printf("%s: create flow group command corrupt\n", DEVNAME(sc));
5814 1.1 jmcneill goto free;
5815 1.1 jmcneill }
5816 1.1 jmcneill
5817 1.1 jmcneill out = mcx_cmdq_out(cqe);
5818 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5819 1.1 jmcneill printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc),
5820 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5821 1.1 jmcneill error = -1;
5822 1.1 jmcneill goto free;
5823 1.1 jmcneill }
5824 1.1 jmcneill
5825 1.15 jmcneill mfg = &sc->sc_flow_group[group];
5826 1.15 jmcneill mfg->g_id = mcx_get_id(out->cmd_group_id);
5827 1.15 jmcneill mfg->g_table = flow_table_id;
5828 1.15 jmcneill mfg->g_start = start;
5829 1.15 jmcneill mfg->g_size = size;
5830 1.1 jmcneill
5831 1.1 jmcneill free:
5832 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5833 1.1 jmcneill return (error);
5834 1.1 jmcneill }
5835 1.1 jmcneill
5836 1.1 jmcneill static int
5837 1.1 jmcneill mcx_destroy_flow_group(struct mcx_softc *sc, int group)
5838 1.1 jmcneill {
5839 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5840 1.1 jmcneill struct mcx_dmamem mxm;
5841 1.1 jmcneill struct mcx_cmd_destroy_flow_group_in *in;
5842 1.1 jmcneill struct mcx_cmd_destroy_flow_group_mb_in *mb;
5843 1.1 jmcneill struct mcx_cmd_destroy_flow_group_out *out;
5844 1.15 jmcneill struct mcx_flow_group *mfg;
5845 1.1 jmcneill int error;
5846 1.1 jmcneill int token;
5847 1.1 jmcneill
5848 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5849 1.1 jmcneill token = mcx_cmdq_token(sc);
5850 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token);
5851 1.1 jmcneill
5852 1.1 jmcneill in = mcx_cmdq_in(cqe);
5853 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP);
5854 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5855 1.1 jmcneill
5856 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
5857 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
5858 1.1 jmcneill printf("%s: unable to allocate destroy flow group mailbox\n",
5859 1.1 jmcneill DEVNAME(sc));
5860 1.1 jmcneill return (-1);
5861 1.1 jmcneill }
5862 1.1 jmcneill mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5863 1.1 jmcneill mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5864 1.15 jmcneill mfg = &sc->sc_flow_group[group];
5865 1.15 jmcneill mb->cmd_table_id = htobe32(mfg->g_table);
5866 1.15 jmcneill mb->cmd_group_id = htobe32(mfg->g_id);
5867 1.1 jmcneill
5868 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
5869 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5870 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5871 1.1 jmcneill if (error != 0) {
5872 1.1 jmcneill printf("%s: destroy flow group timeout\n", DEVNAME(sc));
5873 1.1 jmcneill goto free;
5874 1.1 jmcneill }
5875 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5876 1.1 jmcneill printf("%s: destroy flow group command corrupt\n", DEVNAME(sc));
5877 1.1 jmcneill goto free;
5878 1.1 jmcneill }
5879 1.1 jmcneill
5880 1.1 jmcneill out = mcx_cmdq_out(cqe);
5881 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5882 1.1 jmcneill printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc),
5883 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
5884 1.1 jmcneill error = -1;
5885 1.1 jmcneill goto free;
5886 1.1 jmcneill }
5887 1.1 jmcneill
5888 1.15 jmcneill mfg->g_id = -1;
5889 1.15 jmcneill mfg->g_table = -1;
5890 1.15 jmcneill mfg->g_size = 0;
5891 1.15 jmcneill mfg->g_start = 0;
5892 1.1 jmcneill free:
5893 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5894 1.1 jmcneill return (error);
5895 1.1 jmcneill }
5896 1.1 jmcneill
5897 1.1 jmcneill static int
5898 1.15 jmcneill mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index,
5899 1.15 jmcneill const uint8_t *macaddr, uint32_t dest)
5900 1.1 jmcneill {
5901 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5902 1.1 jmcneill struct mcx_dmamem mxm;
5903 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_in *in;
5904 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
5905 1.1 jmcneill struct mcx_cmd_set_flow_table_entry_out *out;
5906 1.15 jmcneill struct mcx_flow_group *mfg;
5907 1.15 jmcneill uint32_t *pdest;
5908 1.1 jmcneill int error;
5909 1.1 jmcneill int token;
5910 1.1 jmcneill
5911 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5912 1.1 jmcneill token = mcx_cmdq_token(sc);
5913 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
5914 1.1 jmcneill sizeof(*out), token);
5915 1.1 jmcneill
5916 1.1 jmcneill in = mcx_cmdq_in(cqe);
5917 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
5918 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5919 1.1 jmcneill
5920 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5921 1.1 jmcneill != 0) {
5922 1.1 jmcneill printf("%s: unable to allocate set flow table entry mailbox\n",
5923 1.1 jmcneill DEVNAME(sc));
5924 1.1 jmcneill return (-1);
5925 1.1 jmcneill }
5926 1.15 jmcneill
5927 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
5928 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
5929 1.15 jmcneill
5930 1.15 jmcneill mfg = &sc->sc_flow_group[group];
5931 1.15 jmcneill mbin->cmd_table_id = htobe32(mfg->g_table);
5932 1.15 jmcneill mbin->cmd_flow_index = htobe32(mfg->g_start + index);
5933 1.15 jmcneill mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
5934 1.1 jmcneill
5935 1.1 jmcneill /* flow context ends at offset 0x330, 0x130 into the second mbox */
5936 1.15 jmcneill pdest = (uint32_t *)
5937 1.1 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
5938 1.1 jmcneill mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
5939 1.1 jmcneill mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
5940 1.15 jmcneill *pdest = htobe32(dest);
5941 1.1 jmcneill
5942 1.1 jmcneill /* the only thing we match on at the moment is the dest mac address */
5943 1.1 jmcneill if (macaddr != NULL) {
5944 1.1 jmcneill memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr,
5945 1.1 jmcneill ETHER_ADDR_LEN);
5946 1.1 jmcneill }
5947 1.1 jmcneill
5948 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
5949 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
5950 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
5951 1.1 jmcneill if (error != 0) {
5952 1.1 jmcneill printf("%s: set flow table entry timeout\n", DEVNAME(sc));
5953 1.1 jmcneill goto free;
5954 1.1 jmcneill }
5955 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
5956 1.1 jmcneill printf("%s: set flow table entry command corrupt\n",
5957 1.1 jmcneill DEVNAME(sc));
5958 1.1 jmcneill goto free;
5959 1.1 jmcneill }
5960 1.1 jmcneill
5961 1.1 jmcneill out = mcx_cmdq_out(cqe);
5962 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
5963 1.1 jmcneill printf("%s: set flow table entry failed (%x, %x)\n",
5964 1.1 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
5965 1.1 jmcneill error = -1;
5966 1.1 jmcneill goto free;
5967 1.1 jmcneill }
5968 1.1 jmcneill
5969 1.1 jmcneill free:
5970 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
5971 1.1 jmcneill return (error);
5972 1.1 jmcneill }
5973 1.1 jmcneill
5974 1.1 jmcneill static int
5975 1.15 jmcneill mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index,
5976 1.15 jmcneill int ethertype, int ip_proto, uint32_t dest)
5977 1.1 jmcneill {
5978 1.1 jmcneill struct mcx_cmdq_entry *cqe;
5979 1.1 jmcneill struct mcx_dmamem mxm;
5980 1.15 jmcneill struct mcx_cmd_set_flow_table_entry_in *in;
5981 1.15 jmcneill struct mcx_cmd_set_flow_table_entry_mb_in *mbin;
5982 1.15 jmcneill struct mcx_cmd_set_flow_table_entry_out *out;
5983 1.15 jmcneill struct mcx_flow_group *mfg;
5984 1.15 jmcneill uint32_t *pdest;
5985 1.1 jmcneill int error;
5986 1.1 jmcneill int token;
5987 1.1 jmcneill
5988 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
5989 1.1 jmcneill token = mcx_cmdq_token(sc);
5990 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest),
5991 1.15 jmcneill sizeof(*out), token);
5992 1.1 jmcneill
5993 1.1 jmcneill in = mcx_cmdq_in(cqe);
5994 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY);
5995 1.1 jmcneill in->cmd_op_mod = htobe16(0);
5996 1.1 jmcneill
5997 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token)
5998 1.15 jmcneill != 0) {
5999 1.15 jmcneill printf("%s: unable to allocate set flow table entry mailbox\n",
6000 1.1 jmcneill DEVNAME(sc));
6001 1.1 jmcneill return (-1);
6002 1.1 jmcneill }
6003 1.15 jmcneill
6004 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6005 1.1 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
6006 1.15 jmcneill
6007 1.15 jmcneill mfg = &sc->sc_flow_group[group];
6008 1.15 jmcneill mbin->cmd_table_id = htobe32(mfg->g_table);
6009 1.15 jmcneill mbin->cmd_flow_index = htobe32(mfg->g_start + index);
6010 1.15 jmcneill mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id);
6011 1.15 jmcneill
6012 1.15 jmcneill /* flow context ends at offset 0x330, 0x130 into the second mbox */
6013 1.15 jmcneill pdest = (uint32_t *)
6014 1.15 jmcneill (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130);
6015 1.15 jmcneill mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD);
6016 1.15 jmcneill mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1);
6017 1.15 jmcneill *pdest = htobe32(dest);
6018 1.15 jmcneill
6019 1.15 jmcneill mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype);
6020 1.15 jmcneill mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto;
6021 1.1 jmcneill
6022 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
6023 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6024 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6025 1.1 jmcneill if (error != 0) {
6026 1.15 jmcneill printf("%s: set flow table entry timeout\n", DEVNAME(sc));
6027 1.1 jmcneill goto free;
6028 1.1 jmcneill }
6029 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6030 1.15 jmcneill printf("%s: set flow table entry command corrupt\n",
6031 1.1 jmcneill DEVNAME(sc));
6032 1.1 jmcneill goto free;
6033 1.1 jmcneill }
6034 1.1 jmcneill
6035 1.1 jmcneill out = mcx_cmdq_out(cqe);
6036 1.1 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6037 1.15 jmcneill printf("%s: set flow table entry failed (%x, %x)\n",
6038 1.15 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
6039 1.1 jmcneill error = -1;
6040 1.1 jmcneill goto free;
6041 1.1 jmcneill }
6042 1.1 jmcneill
6043 1.1 jmcneill free:
6044 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
6045 1.1 jmcneill return (error);
6046 1.1 jmcneill }
6047 1.1 jmcneill
6048 1.15 jmcneill static int
6049 1.15 jmcneill mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index)
6050 1.1 jmcneill {
6051 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6052 1.1 jmcneill struct mcx_dmamem mxm;
6053 1.15 jmcneill struct mcx_cmd_delete_flow_table_entry_in *in;
6054 1.15 jmcneill struct mcx_cmd_delete_flow_table_entry_mb_in *mbin;
6055 1.15 jmcneill struct mcx_cmd_delete_flow_table_entry_out *out;
6056 1.15 jmcneill struct mcx_flow_group *mfg;
6057 1.1 jmcneill int error;
6058 1.15 jmcneill int token;
6059 1.15 jmcneill
6060 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6061 1.15 jmcneill token = mcx_cmdq_token(sc);
6062 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out),
6063 1.15 jmcneill token);
6064 1.15 jmcneill
6065 1.15 jmcneill in = mcx_cmdq_in(cqe);
6066 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY);
6067 1.15 jmcneill in->cmd_op_mod = htobe16(0);
6068 1.15 jmcneill
6069 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6070 1.15 jmcneill &cqe->cq_input_ptr, token) != 0) {
6071 1.15 jmcneill printf("%s: unable to allocate "
6072 1.15 jmcneill "delete flow table entry mailbox\n", DEVNAME(sc));
6073 1.15 jmcneill return (-1);
6074 1.15 jmcneill }
6075 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6076 1.15 jmcneill mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX;
6077 1.15 jmcneill
6078 1.15 jmcneill mfg = &sc->sc_flow_group[group];
6079 1.15 jmcneill mbin->cmd_table_id = htobe32(mfg->g_table);
6080 1.15 jmcneill mbin->cmd_flow_index = htobe32(mfg->g_start + index);
6081 1.15 jmcneill
6082 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 2);
6083 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
6084 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6085 1.15 jmcneill if (error != 0) {
6086 1.15 jmcneill printf("%s: delete flow table entry timeout\n", DEVNAME(sc));
6087 1.15 jmcneill goto free;
6088 1.15 jmcneill }
6089 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6090 1.15 jmcneill printf("%s: delete flow table entry command corrupt\n",
6091 1.15 jmcneill DEVNAME(sc));
6092 1.15 jmcneill goto free;
6093 1.15 jmcneill }
6094 1.15 jmcneill
6095 1.15 jmcneill out = mcx_cmdq_out(cqe);
6096 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6097 1.15 jmcneill printf("%s: delete flow table entry %d:%d failed (%x, %x)\n",
6098 1.15 jmcneill DEVNAME(sc), group, index, out->cmd_status,
6099 1.15 jmcneill be32toh(out->cmd_syndrome));
6100 1.15 jmcneill error = -1;
6101 1.15 jmcneill goto free;
6102 1.15 jmcneill }
6103 1.15 jmcneill
6104 1.15 jmcneill free:
6105 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
6106 1.15 jmcneill return (error);
6107 1.15 jmcneill }
6108 1.15 jmcneill
6109 1.15 jmcneill #if 0
6110 1.15 jmcneill int
6111 1.15 jmcneill mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id)
6112 1.15 jmcneill {
6113 1.15 jmcneill struct mcx_dmamem mxm;
6114 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6115 1.15 jmcneill struct mcx_cmd_query_flow_table_in *in;
6116 1.15 jmcneill struct mcx_cmd_query_flow_table_mb_in *mbin;
6117 1.15 jmcneill struct mcx_cmd_query_flow_table_out *out;
6118 1.15 jmcneill struct mcx_cmd_query_flow_table_mb_out *mbout;
6119 1.15 jmcneill uint8_t token = mcx_cmdq_token(sc);
6120 1.15 jmcneill int error;
6121 1.15 jmcneill int i;
6122 1.15 jmcneill uint8_t *dump;
6123 1.1 jmcneill
6124 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6125 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6126 1.1 jmcneill sizeof(*out) + sizeof(*mbout) + 16, token);
6127 1.1 jmcneill
6128 1.1 jmcneill in = mcx_cmdq_in(cqe);
6129 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE);
6130 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6131 1.1 jmcneill
6132 1.1 jmcneill CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6133 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE);
6134 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6135 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6136 1.1 jmcneill printf(", unable to allocate query flow table mailboxes\n");
6137 1.1 jmcneill return (-1);
6138 1.1 jmcneill }
6139 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6140 1.1 jmcneill
6141 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6142 1.1 jmcneill mbin->cmd_table_type = 0;
6143 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
6144 1.1 jmcneill
6145 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6146 1.1 jmcneill
6147 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6148 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6149 1.1 jmcneill if (error != 0) {
6150 1.1 jmcneill printf("%s: query flow table timeout\n", DEVNAME(sc));
6151 1.1 jmcneill goto free;
6152 1.1 jmcneill }
6153 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6154 1.1 jmcneill if (error != 0) {
6155 1.1 jmcneill printf("%s: query flow table reply corrupt\n", DEVNAME(sc));
6156 1.1 jmcneill goto free;
6157 1.1 jmcneill }
6158 1.1 jmcneill
6159 1.1 jmcneill out = mcx_cmdq_out(cqe);
6160 1.1 jmcneill switch (out->cmd_status) {
6161 1.1 jmcneill case MCX_CQ_STATUS_OK:
6162 1.1 jmcneill break;
6163 1.1 jmcneill default:
6164 1.1 jmcneill printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc),
6165 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6166 1.1 jmcneill error = -1;
6167 1.1 jmcneill goto free;
6168 1.1 jmcneill }
6169 1.1 jmcneill
6170 1.1 jmcneill mbout = (struct mcx_cmd_query_flow_table_mb_out *)
6171 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6172 1.1 jmcneill dump = (uint8_t *)mbout + 8;
6173 1.1 jmcneill for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) {
6174 1.1 jmcneill printf("%.2x ", dump[i]);
6175 1.1 jmcneill if (i % 16 == 15)
6176 1.1 jmcneill printf("\n");
6177 1.1 jmcneill }
6178 1.1 jmcneill free:
6179 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6180 1.1 jmcneill return (error);
6181 1.1 jmcneill }
6182 1.1 jmcneill int
6183 1.15 jmcneill mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index)
6184 1.1 jmcneill {
6185 1.1 jmcneill struct mcx_dmamem mxm;
6186 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6187 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_in *in;
6188 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_in *mbin;
6189 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_out *out;
6190 1.1 jmcneill struct mcx_cmd_query_flow_table_entry_mb_out *mbout;
6191 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6192 1.1 jmcneill int error;
6193 1.1 jmcneill int i;
6194 1.1 jmcneill uint8_t *dump;
6195 1.1 jmcneill
6196 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6197 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6198 1.1 jmcneill sizeof(*out) + sizeof(*mbout) + 16, token);
6199 1.1 jmcneill
6200 1.1 jmcneill in = mcx_cmdq_in(cqe);
6201 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY);
6202 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6203 1.1 jmcneill
6204 1.1 jmcneill CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6205 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6206 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6207 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6208 1.15 jmcneill printf(", unable to allocate "
6209 1.15 jmcneill "query flow table entry mailboxes\n");
6210 1.1 jmcneill return (-1);
6211 1.1 jmcneill }
6212 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6213 1.1 jmcneill
6214 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6215 1.1 jmcneill mbin->cmd_table_type = 0;
6216 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
6217 1.1 jmcneill mbin->cmd_flow_index = htobe32(index);
6218 1.1 jmcneill
6219 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6220 1.1 jmcneill
6221 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6222 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6223 1.1 jmcneill if (error != 0) {
6224 1.1 jmcneill printf("%s: query flow table entry timeout\n", DEVNAME(sc));
6225 1.1 jmcneill goto free;
6226 1.1 jmcneill }
6227 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6228 1.1 jmcneill if (error != 0) {
6229 1.1 jmcneill printf("%s: query flow table entry reply corrupt\n",
6230 1.1 jmcneill DEVNAME(sc));
6231 1.1 jmcneill goto free;
6232 1.1 jmcneill }
6233 1.1 jmcneill
6234 1.1 jmcneill out = mcx_cmdq_out(cqe);
6235 1.1 jmcneill switch (out->cmd_status) {
6236 1.1 jmcneill case MCX_CQ_STATUS_OK:
6237 1.1 jmcneill break;
6238 1.1 jmcneill default:
6239 1.1 jmcneill printf("%s: query flow table entry failed (%x/%x)\n",
6240 1.1 jmcneill DEVNAME(sc), out->cmd_status, be32toh(out->cmd_syndrome));
6241 1.1 jmcneill error = -1;
6242 1.1 jmcneill goto free;
6243 1.1 jmcneill }
6244 1.1 jmcneill
6245 1.1 jmcneill mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *)
6246 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6247 1.1 jmcneill dump = (uint8_t *)mbout;
6248 1.1 jmcneill for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6249 1.1 jmcneill printf("%.2x ", dump[i]);
6250 1.1 jmcneill if (i % 16 == 15)
6251 1.1 jmcneill printf("\n");
6252 1.1 jmcneill }
6253 1.1 jmcneill
6254 1.1 jmcneill free:
6255 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6256 1.1 jmcneill return (error);
6257 1.1 jmcneill }
6258 1.1 jmcneill
6259 1.1 jmcneill int
6260 1.15 jmcneill mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id)
6261 1.1 jmcneill {
6262 1.1 jmcneill struct mcx_dmamem mxm;
6263 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6264 1.1 jmcneill struct mcx_cmd_query_flow_group_in *in;
6265 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_in *mbin;
6266 1.1 jmcneill struct mcx_cmd_query_flow_group_out *out;
6267 1.1 jmcneill struct mcx_cmd_query_flow_group_mb_out *mbout;
6268 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6269 1.1 jmcneill int error;
6270 1.1 jmcneill int i;
6271 1.1 jmcneill uint8_t *dump;
6272 1.1 jmcneill
6273 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6274 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6275 1.1 jmcneill sizeof(*out) + sizeof(*mbout) + 16, token);
6276 1.1 jmcneill
6277 1.1 jmcneill in = mcx_cmdq_in(cqe);
6278 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP);
6279 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6280 1.1 jmcneill
6281 1.1 jmcneill CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE);
6282 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6283 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6284 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6285 1.1 jmcneill printf(", unable to allocate query flow group mailboxes\n");
6286 1.1 jmcneill return (-1);
6287 1.1 jmcneill }
6288 1.1 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6289 1.1 jmcneill
6290 1.1 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6291 1.1 jmcneill mbin->cmd_table_type = 0;
6292 1.15 jmcneill mbin->cmd_table_id = htobe32(flow_table_id);
6293 1.1 jmcneill mbin->cmd_group_id = htobe32(sc->sc_flow_group_id);
6294 1.1 jmcneill
6295 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6296 1.1 jmcneill
6297 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6298 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6299 1.1 jmcneill if (error != 0) {
6300 1.1 jmcneill printf("%s: query flow group timeout\n", DEVNAME(sc));
6301 1.1 jmcneill goto free;
6302 1.1 jmcneill }
6303 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6304 1.1 jmcneill if (error != 0) {
6305 1.1 jmcneill printf("%s: query flow group reply corrupt\n", DEVNAME(sc));
6306 1.1 jmcneill goto free;
6307 1.1 jmcneill }
6308 1.1 jmcneill
6309 1.1 jmcneill out = mcx_cmdq_out(cqe);
6310 1.1 jmcneill switch (out->cmd_status) {
6311 1.1 jmcneill case MCX_CQ_STATUS_OK:
6312 1.1 jmcneill break;
6313 1.1 jmcneill default:
6314 1.1 jmcneill printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc),
6315 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6316 1.1 jmcneill error = -1;
6317 1.1 jmcneill goto free;
6318 1.1 jmcneill }
6319 1.1 jmcneill
6320 1.1 jmcneill mbout = (struct mcx_cmd_query_flow_group_mb_out *)
6321 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6322 1.1 jmcneill dump = (uint8_t *)mbout;
6323 1.1 jmcneill for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6324 1.1 jmcneill printf("%.2x ", dump[i]);
6325 1.1 jmcneill if (i % 16 == 15)
6326 1.1 jmcneill printf("\n");
6327 1.1 jmcneill }
6328 1.1 jmcneill dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1)));
6329 1.1 jmcneill for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) {
6330 1.1 jmcneill printf("%.2x ", dump[i]);
6331 1.1 jmcneill if (i % 16 == 15)
6332 1.1 jmcneill printf("\n");
6333 1.1 jmcneill }
6334 1.1 jmcneill
6335 1.1 jmcneill free:
6336 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6337 1.1 jmcneill return (error);
6338 1.1 jmcneill }
6339 1.1 jmcneill
6340 1.15 jmcneill static int
6341 1.15 jmcneill mcx_dump_counters(struct mcx_softc *sc)
6342 1.15 jmcneill {
6343 1.15 jmcneill struct mcx_dmamem mxm;
6344 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6345 1.15 jmcneill struct mcx_cmd_query_vport_counters_in *in;
6346 1.15 jmcneill struct mcx_cmd_query_vport_counters_mb_in *mbin;
6347 1.15 jmcneill struct mcx_cmd_query_vport_counters_out *out;
6348 1.15 jmcneill struct mcx_nic_vport_counters *counters;
6349 1.15 jmcneill int error, token;
6350 1.15 jmcneill
6351 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6352 1.15 jmcneill token = mcx_cmdq_token(sc);
6353 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin),
6354 1.15 jmcneill sizeof(*out) + sizeof(*counters), token);
6355 1.15 jmcneill
6356 1.15 jmcneill in = mcx_cmdq_in(cqe);
6357 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS);
6358 1.15 jmcneill in->cmd_op_mod = htobe16(0);
6359 1.15 jmcneill
6360 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
6361 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6362 1.15 jmcneill printf(", unable to allocate "
6363 1.15 jmcneill "query nic vport counters mailboxen\n");
6364 1.15 jmcneill return (-1);
6365 1.15 jmcneill }
6366 1.15 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6367 1.15 jmcneill
6368 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6369 1.15 jmcneill mbin->cmd_clear = 0x80;
6370 1.15 jmcneill
6371 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6372 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
6373 1.15 jmcneill
6374 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6375 1.15 jmcneill if (error != 0) {
6376 1.15 jmcneill printf("%s: query nic vport counters timeout\n", DEVNAME(sc));
6377 1.15 jmcneill goto free;
6378 1.15 jmcneill }
6379 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6380 1.15 jmcneill printf("%s: query nic vport counters command corrupt\n",
6381 1.15 jmcneill DEVNAME(sc));
6382 1.15 jmcneill goto free;
6383 1.15 jmcneill }
6384 1.15 jmcneill
6385 1.15 jmcneill out = mcx_cmdq_out(cqe);
6386 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6387 1.15 jmcneill printf("%s: query nic vport counters failed (%x, %x)\n",
6388 1.15 jmcneill DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome));
6389 1.15 jmcneill error = -1;
6390 1.15 jmcneill goto free;
6391 1.15 jmcneill }
6392 1.15 jmcneill
6393 1.15 jmcneill counters = (struct mcx_nic_vport_counters *)
6394 1.15 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6395 1.15 jmcneill if (counters->rx_bcast.packets + counters->tx_bcast.packets +
6396 1.15 jmcneill counters->rx_ucast.packets + counters->tx_ucast.packets +
6397 1.15 jmcneill counters->rx_err.packets + counters->tx_err.packets)
6398 1.15 jmcneill printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n",
6399 1.15 jmcneill DEVNAME(sc),
6400 1.15 jmcneill betoh64(counters->tx_err.packets),
6401 1.15 jmcneill betoh64(counters->rx_err.packets),
6402 1.15 jmcneill betoh64(counters->tx_ucast.packets),
6403 1.15 jmcneill betoh64(counters->rx_ucast.packets),
6404 1.15 jmcneill betoh64(counters->tx_bcast.packets),
6405 1.15 jmcneill betoh64(counters->rx_bcast.packets));
6406 1.15 jmcneill free:
6407 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
6408 1.15 jmcneill
6409 1.15 jmcneill return (error);
6410 1.15 jmcneill }
6411 1.15 jmcneill
6412 1.15 jmcneill static int
6413 1.15 jmcneill mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what)
6414 1.15 jmcneill {
6415 1.15 jmcneill struct mcx_dmamem mxm;
6416 1.15 jmcneill struct mcx_cmdq_entry *cqe;
6417 1.15 jmcneill struct mcx_cmd_query_flow_counter_in *in;
6418 1.15 jmcneill struct mcx_cmd_query_flow_counter_mb_in *mbin;
6419 1.15 jmcneill struct mcx_cmd_query_flow_counter_out *out;
6420 1.15 jmcneill struct mcx_counter *counters;
6421 1.15 jmcneill int error, token;
6422 1.15 jmcneill
6423 1.15 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6424 1.15 jmcneill token = mcx_cmdq_token(sc);
6425 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) +
6426 1.15 jmcneill sizeof(*counters), token);
6427 1.15 jmcneill
6428 1.15 jmcneill in = mcx_cmdq_in(cqe);
6429 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER);
6430 1.15 jmcneill in->cmd_op_mod = htobe16(0);
6431 1.15 jmcneill
6432 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1,
6433 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6434 1.15 jmcneill printf(", unable to allocate query flow counter mailboxen\n");
6435 1.15 jmcneill return (-1);
6436 1.15 jmcneill }
6437 1.15 jmcneill cqe->cq_input_ptr = cqe->cq_output_ptr;
6438 1.15 jmcneill mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0));
6439 1.15 jmcneill mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]);
6440 1.15 jmcneill mbin->cmd_clear = 0x80;
6441 1.15 jmcneill
6442 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6443 1.15 jmcneill mcx_cmdq_post(sc, cqe, 0);
6444 1.15 jmcneill
6445 1.15 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6446 1.15 jmcneill if (error != 0) {
6447 1.15 jmcneill printf("%s: query flow counter timeout\n", DEVNAME(sc));
6448 1.15 jmcneill goto free;
6449 1.15 jmcneill }
6450 1.15 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6451 1.15 jmcneill printf("%s: query flow counter command corrupt\n", DEVNAME(sc));
6452 1.15 jmcneill goto free;
6453 1.15 jmcneill }
6454 1.15 jmcneill
6455 1.15 jmcneill out = mcx_cmdq_out(cqe);
6456 1.15 jmcneill if (out->cmd_status != MCX_CQ_STATUS_OK) {
6457 1.15 jmcneill printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc),
6458 1.15 jmcneill out->cmd_status, betoh32(out->cmd_syndrome));
6459 1.15 jmcneill error = -1;
6460 1.15 jmcneill goto free;
6461 1.15 jmcneill }
6462 1.15 jmcneill
6463 1.15 jmcneill counters = (struct mcx_counter *)
6464 1.15 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6465 1.15 jmcneill if (counters->packets)
6466 1.15 jmcneill printf("%s: %s inflow %llx\n", DEVNAME(sc), what,
6467 1.15 jmcneill betoh64(counters->packets));
6468 1.15 jmcneill free:
6469 1.15 jmcneill mcx_dmamem_free(sc, &mxm);
6470 1.15 jmcneill
6471 1.15 jmcneill return (error);
6472 1.15 jmcneill }
6473 1.15 jmcneill
6474 1.15 jmcneill #endif
6475 1.15 jmcneill
6476 1.15 jmcneill #if NKSTAT > 0
6477 1.15 jmcneill
6478 1.1 jmcneill int
6479 1.15 jmcneill mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx)
6480 1.1 jmcneill {
6481 1.1 jmcneill struct mcx_dmamem mxm;
6482 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6483 1.1 jmcneill struct mcx_cmd_query_rq_in *in;
6484 1.1 jmcneill struct mcx_cmd_query_rq_out *out;
6485 1.1 jmcneill struct mcx_cmd_query_rq_mb_out *mbout;
6486 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6487 1.1 jmcneill int error;
6488 1.1 jmcneill
6489 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6490 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
6491 1.1 jmcneill token);
6492 1.1 jmcneill
6493 1.1 jmcneill in = mcx_cmdq_in(cqe);
6494 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ);
6495 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6496 1.15 jmcneill in->cmd_rqn = htobe32(rx->rx_rqn);
6497 1.1 jmcneill
6498 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6499 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6500 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6501 1.15 jmcneill printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc));
6502 1.1 jmcneill return (-1);
6503 1.1 jmcneill }
6504 1.1 jmcneill
6505 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6506 1.1 jmcneill
6507 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6508 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6509 1.1 jmcneill if (error != 0) {
6510 1.1 jmcneill printf("%s: query rq timeout\n", DEVNAME(sc));
6511 1.1 jmcneill goto free;
6512 1.1 jmcneill }
6513 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6514 1.1 jmcneill if (error != 0) {
6515 1.1 jmcneill printf("%s: query rq reply corrupt\n", DEVNAME(sc));
6516 1.1 jmcneill goto free;
6517 1.1 jmcneill }
6518 1.1 jmcneill
6519 1.1 jmcneill out = mcx_cmdq_out(cqe);
6520 1.1 jmcneill switch (out->cmd_status) {
6521 1.1 jmcneill case MCX_CQ_STATUS_OK:
6522 1.1 jmcneill break;
6523 1.1 jmcneill default:
6524 1.1 jmcneill printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc),
6525 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6526 1.1 jmcneill error = -1;
6527 1.1 jmcneill goto free;
6528 1.1 jmcneill }
6529 1.1 jmcneill
6530 1.1 jmcneill mbout = (struct mcx_cmd_query_rq_mb_out *)
6531 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6532 1.15 jmcneill memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx));
6533 1.1 jmcneill
6534 1.1 jmcneill free:
6535 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6536 1.1 jmcneill return (error);
6537 1.1 jmcneill }
6538 1.1 jmcneill
6539 1.1 jmcneill int
6540 1.15 jmcneill mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx)
6541 1.1 jmcneill {
6542 1.1 jmcneill struct mcx_dmamem mxm;
6543 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6544 1.1 jmcneill struct mcx_cmd_query_sq_in *in;
6545 1.1 jmcneill struct mcx_cmd_query_sq_out *out;
6546 1.1 jmcneill struct mcx_cmd_query_sq_mb_out *mbout;
6547 1.1 jmcneill uint8_t token = mcx_cmdq_token(sc);
6548 1.1 jmcneill int error;
6549 1.1 jmcneill
6550 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6551 1.1 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16,
6552 1.1 jmcneill token);
6553 1.1 jmcneill
6554 1.1 jmcneill in = mcx_cmdq_in(cqe);
6555 1.1 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ);
6556 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6557 1.15 jmcneill in->cmd_sqn = htobe32(tx->tx_sqn);
6558 1.1 jmcneill
6559 1.1 jmcneill CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6560 1.1 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6561 1.1 jmcneill &cqe->cq_output_ptr, token) != 0) {
6562 1.15 jmcneill printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc));
6563 1.1 jmcneill return (-1);
6564 1.1 jmcneill }
6565 1.1 jmcneill
6566 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6567 1.1 jmcneill
6568 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6569 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6570 1.1 jmcneill if (error != 0) {
6571 1.1 jmcneill printf("%s: query sq timeout\n", DEVNAME(sc));
6572 1.1 jmcneill goto free;
6573 1.1 jmcneill }
6574 1.1 jmcneill error = mcx_cmdq_verify(cqe);
6575 1.1 jmcneill if (error != 0) {
6576 1.1 jmcneill printf("%s: query sq reply corrupt\n", DEVNAME(sc));
6577 1.1 jmcneill goto free;
6578 1.1 jmcneill }
6579 1.1 jmcneill
6580 1.1 jmcneill out = mcx_cmdq_out(cqe);
6581 1.1 jmcneill switch (out->cmd_status) {
6582 1.1 jmcneill case MCX_CQ_STATUS_OK:
6583 1.1 jmcneill break;
6584 1.1 jmcneill default:
6585 1.1 jmcneill printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc),
6586 1.1 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6587 1.1 jmcneill error = -1;
6588 1.1 jmcneill goto free;
6589 1.1 jmcneill }
6590 1.1 jmcneill
6591 1.1 jmcneill mbout = (struct mcx_cmd_query_sq_mb_out *)
6592 1.1 jmcneill (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6593 1.15 jmcneill memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx));
6594 1.1 jmcneill
6595 1.1 jmcneill free:
6596 1.1 jmcneill mcx_cq_mboxes_free(sc, &mxm);
6597 1.1 jmcneill return (error);
6598 1.1 jmcneill }
6599 1.1 jmcneill
6600 1.15 jmcneill int
6601 1.15 jmcneill mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx)
6602 1.1 jmcneill {
6603 1.1 jmcneill struct mcx_dmamem mxm;
6604 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6605 1.15 jmcneill struct mcx_cmd_query_cq_in *in;
6606 1.15 jmcneill struct mcx_cmd_query_cq_out *out;
6607 1.15 jmcneill struct mcx_cq_ctx *ctx;
6608 1.15 jmcneill uint8_t token = mcx_cmdq_token(sc);
6609 1.15 jmcneill int error;
6610 1.1 jmcneill
6611 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6612 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
6613 1.15 jmcneill token);
6614 1.1 jmcneill
6615 1.1 jmcneill in = mcx_cmdq_in(cqe);
6616 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ);
6617 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6618 1.15 jmcneill in->cmd_cqn = htobe32(cq->cq_n);
6619 1.1 jmcneill
6620 1.15 jmcneill CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6621 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6622 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6623 1.15 jmcneill printf("%s: unable to allocate query cq mailboxes\n",
6624 1.15 jmcneill DEVNAME(sc));
6625 1.1 jmcneill return (-1);
6626 1.1 jmcneill }
6627 1.1 jmcneill
6628 1.15 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6629 1.1 jmcneill
6630 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6631 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6632 1.1 jmcneill if (error != 0) {
6633 1.15 jmcneill printf("%s: query cq timeout\n", DEVNAME(sc));
6634 1.1 jmcneill goto free;
6635 1.1 jmcneill }
6636 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6637 1.15 jmcneill printf("%s: query cq reply corrupt\n", DEVNAME(sc));
6638 1.1 jmcneill goto free;
6639 1.1 jmcneill }
6640 1.1 jmcneill
6641 1.1 jmcneill out = mcx_cmdq_out(cqe);
6642 1.15 jmcneill switch (out->cmd_status) {
6643 1.15 jmcneill case MCX_CQ_STATUS_OK:
6644 1.15 jmcneill break;
6645 1.15 jmcneill default:
6646 1.15 jmcneill printf("%s: query qc failed (%x/%x)\n", DEVNAME(sc),
6647 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6648 1.1 jmcneill error = -1;
6649 1.1 jmcneill goto free;
6650 1.1 jmcneill }
6651 1.1 jmcneill
6652 1.15 jmcneill ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6653 1.15 jmcneill memcpy(cq_ctx, ctx, sizeof(*cq_ctx));
6654 1.1 jmcneill free:
6655 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
6656 1.1 jmcneill return (error);
6657 1.1 jmcneill }
6658 1.1 jmcneill
6659 1.15 jmcneill int
6660 1.15 jmcneill mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx)
6661 1.1 jmcneill {
6662 1.1 jmcneill struct mcx_dmamem mxm;
6663 1.1 jmcneill struct mcx_cmdq_entry *cqe;
6664 1.15 jmcneill struct mcx_cmd_query_eq_in *in;
6665 1.15 jmcneill struct mcx_cmd_query_eq_out *out;
6666 1.15 jmcneill struct mcq_eq_ctx *ctx;
6667 1.15 jmcneill uint8_t token = mcx_cmdq_token(sc);
6668 1.15 jmcneill int error;
6669 1.1 jmcneill
6670 1.1 jmcneill cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem);
6671 1.15 jmcneill mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16,
6672 1.15 jmcneill token);
6673 1.1 jmcneill
6674 1.1 jmcneill in = mcx_cmdq_in(cqe);
6675 1.15 jmcneill in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ);
6676 1.1 jmcneill in->cmd_op_mod = htobe16(0);
6677 1.15 jmcneill in->cmd_eqn = htobe32(eq->eq_n);
6678 1.1 jmcneill
6679 1.15 jmcneill CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2);
6680 1.15 jmcneill if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2,
6681 1.15 jmcneill &cqe->cq_output_ptr, token) != 0) {
6682 1.15 jmcneill printf("%s: unable to allocate query eq mailboxes\n",
6683 1.15 jmcneill DEVNAME(sc));
6684 1.1 jmcneill return (-1);
6685 1.1 jmcneill }
6686 1.1 jmcneill
6687 1.1 jmcneill mcx_cmdq_mboxes_sign(&mxm, 1);
6688 1.15 jmcneill
6689 1.1 jmcneill mcx_cmdq_post(sc, cqe, 0);
6690 1.1 jmcneill error = mcx_cmdq_poll(sc, cqe, 1000);
6691 1.1 jmcneill if (error != 0) {
6692 1.15 jmcneill printf("%s: query eq timeout\n", DEVNAME(sc));
6693 1.1 jmcneill goto free;
6694 1.1 jmcneill }
6695 1.1 jmcneill if (mcx_cmdq_verify(cqe) != 0) {
6696 1.15 jmcneill printf("%s: query eq reply corrupt\n", DEVNAME(sc));
6697 1.1 jmcneill goto free;
6698 1.1 jmcneill }
6699 1.1 jmcneill
6700 1.1 jmcneill out = mcx_cmdq_out(cqe);
6701 1.15 jmcneill switch (out->cmd_status) {
6702 1.15 jmcneill case MCX_CQ_STATUS_OK:
6703 1.15 jmcneill break;
6704 1.15 jmcneill default:
6705 1.15 jmcneill printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc),
6706 1.15 jmcneill out->cmd_status, be32toh(out->cmd_syndrome));
6707 1.1 jmcneill error = -1;
6708 1.1 jmcneill goto free;
6709 1.1 jmcneill }
6710 1.1 jmcneill
6711 1.15 jmcneill ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)));
6712 1.15 jmcneill memcpy(eq_ctx, ctx, sizeof(*eq_ctx));
6713 1.1 jmcneill free:
6714 1.1 jmcneill mcx_dmamem_free(sc, &mxm);
6715 1.1 jmcneill return (error);
6716 1.1 jmcneill }
6717 1.1 jmcneill
6718 1.15 jmcneill #endif /* NKSTAT > 0 */
6719 1.15 jmcneill
6720 1.1 jmcneill
6721 1.15 jmcneill static inline unsigned int
6722 1.15 jmcneill mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, uint nslots)
6723 1.1 jmcneill {
6724 1.15 jmcneill struct mcx_rq_entry *ring, *rqe;
6725 1.1 jmcneill struct mcx_slot *ms;
6726 1.1 jmcneill struct mbuf *m;
6727 1.1 jmcneill uint slot, p, fills;
6728 1.1 jmcneill
6729 1.15 jmcneill ring = MCX_DMA_KVA(&rx->rx_rq_mem);
6730 1.15 jmcneill p = rx->rx_prod;
6731 1.15 jmcneill
6732 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
6733 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_POSTWRITE);
6734 1.15 jmcneill
6735 1.1 jmcneill slot = (p % (1 << MCX_LOG_RQ_SIZE));
6736 1.1 jmcneill rqe = ring;
6737 1.1 jmcneill for (fills = 0; fills < nslots; fills++) {
6738 1.15 jmcneill slot = p % (1 << MCX_LOG_RQ_SIZE);
6739 1.15 jmcneill
6740 1.15 jmcneill ms = &rx->rx_slots[slot];
6741 1.15 jmcneill rqe = &ring[slot];
6742 1.15 jmcneill
6743 1.1 jmcneill m = NULL;
6744 1.1 jmcneill MGETHDR(m, M_DONTWAIT, MT_DATA);
6745 1.1 jmcneill if (m == NULL)
6746 1.1 jmcneill break;
6747 1.1 jmcneill
6748 1.1 jmcneill MCLGET(m, M_DONTWAIT);
6749 1.1 jmcneill if ((m->m_flags & M_EXT) == 0) {
6750 1.1 jmcneill m_freem(m);
6751 1.1 jmcneill break;
6752 1.1 jmcneill }
6753 1.1 jmcneill
6754 1.15 jmcneill m->m_len = m->m_pkthdr.len = sc->sc_hardmtu;
6755 1.15 jmcneill m_adj(m, m->m_ext.ext_size - sc->sc_rxbufsz);
6756 1.15 jmcneill m_adj(m, ETHER_ALIGN);
6757 1.15 jmcneill
6758 1.1 jmcneill if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
6759 1.1 jmcneill BUS_DMA_NOWAIT) != 0) {
6760 1.1 jmcneill m_freem(m);
6761 1.1 jmcneill break;
6762 1.1 jmcneill }
6763 1.3 tnn bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, BUS_DMASYNC_PREREAD);
6764 1.1 jmcneill ms->ms_m = m;
6765 1.1 jmcneill
6766 1.15 jmcneill be32enc(&rqe->rqe_byte_count, ms->ms_map->dm_segs[0].ds_len);
6767 1.15 jmcneill be64enc(&rqe->rqe_addr, ms->ms_map->dm_segs[0].ds_addr);
6768 1.15 jmcneill be32enc(&rqe->rqe_lkey, sc->sc_lkey);
6769 1.1 jmcneill
6770 1.1 jmcneill p++;
6771 1.1 jmcneill }
6772 1.1 jmcneill
6773 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&rx->rx_rq_mem),
6774 1.15 jmcneill 0, MCX_DMA_LEN(&rx->rx_rq_mem), BUS_DMASYNC_PREWRITE);
6775 1.15 jmcneill
6776 1.15 jmcneill rx->rx_prod = p;
6777 1.1 jmcneill
6778 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
6779 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
6780 1.15 jmcneill be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, rx->rx_doorbell),
6781 1.15 jmcneill p & MCX_WQ_DOORBELL_MASK);
6782 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
6783 1.15 jmcneill rx->rx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
6784 1.1 jmcneill
6785 1.1 jmcneill return (nslots - fills);
6786 1.1 jmcneill }
6787 1.1 jmcneill
6788 1.1 jmcneill static int
6789 1.15 jmcneill mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx)
6790 1.1 jmcneill {
6791 1.1 jmcneill u_int slots;
6792 1.1 jmcneill
6793 1.15 jmcneill slots = mcx_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE));
6794 1.1 jmcneill if (slots == 0)
6795 1.1 jmcneill return (1);
6796 1.1 jmcneill
6797 1.15 jmcneill slots = mcx_rx_fill_slots(sc, rx, slots);
6798 1.15 jmcneill mcx_rxr_put(&rx->rx_rxr, slots);
6799 1.1 jmcneill return (0);
6800 1.1 jmcneill }
6801 1.1 jmcneill
6802 1.1 jmcneill void
6803 1.15 jmcneill mcx_refill(void *xrx)
6804 1.1 jmcneill {
6805 1.15 jmcneill struct mcx_rx *rx = xrx;
6806 1.15 jmcneill struct mcx_softc *sc = rx->rx_softc;
6807 1.1 jmcneill
6808 1.15 jmcneill mcx_rx_fill(sc, rx);
6809 1.1 jmcneill
6810 1.15 jmcneill if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
6811 1.15 jmcneill callout_schedule(&rx->rx_refill, 1);
6812 1.1 jmcneill }
6813 1.1 jmcneill
6814 1.15 jmcneill static int
6815 1.15 jmcneill mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx,
6816 1.15 jmcneill struct mcx_cq_entry *cqe)
6817 1.1 jmcneill {
6818 1.1 jmcneill struct mcx_slot *ms;
6819 1.1 jmcneill bus_dmamap_t map;
6820 1.1 jmcneill int slot, slots;
6821 1.1 jmcneill
6822 1.1 jmcneill slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE);
6823 1.1 jmcneill
6824 1.15 jmcneill ms = &tx->tx_slots[slot];
6825 1.1 jmcneill map = ms->ms_map;
6826 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
6827 1.1 jmcneill BUS_DMASYNC_POSTWRITE);
6828 1.1 jmcneill
6829 1.1 jmcneill slots = 1;
6830 1.1 jmcneill if (map->dm_nsegs > 1)
6831 1.1 jmcneill slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT;
6832 1.1 jmcneill
6833 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, map);
6834 1.1 jmcneill m_freem(ms->ms_m);
6835 1.1 jmcneill ms->ms_m = NULL;
6836 1.15 jmcneill
6837 1.15 jmcneill return (slots);
6838 1.1 jmcneill }
6839 1.1 jmcneill
6840 1.1 jmcneill static uint64_t
6841 1.1 jmcneill mcx_uptime(void)
6842 1.1 jmcneill {
6843 1.1 jmcneill struct timespec ts;
6844 1.1 jmcneill
6845 1.1 jmcneill nanouptime(&ts);
6846 1.1 jmcneill
6847 1.1 jmcneill return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec);
6848 1.1 jmcneill }
6849 1.1 jmcneill
6850 1.1 jmcneill static void
6851 1.1 jmcneill mcx_calibrate_first(struct mcx_softc *sc)
6852 1.1 jmcneill {
6853 1.1 jmcneill struct mcx_calibration *c = &sc->sc_calibration[0];
6854 1.15 jmcneill int s;
6855 1.1 jmcneill
6856 1.1 jmcneill sc->sc_calibration_gen = 0;
6857 1.1 jmcneill
6858 1.15 jmcneill s = splhigh(); /* crit_enter? */
6859 1.1 jmcneill c->c_ubase = mcx_uptime();
6860 1.1 jmcneill c->c_tbase = mcx_timer(sc);
6861 1.15 jmcneill splx(s);
6862 1.15 jmcneill c->c_ratio = 0;
6863 1.1 jmcneill
6864 1.15 jmcneill #if notyet
6865 1.1 jmcneill callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_FIRST * hz);
6866 1.15 jmcneill #endif
6867 1.1 jmcneill }
6868 1.1 jmcneill
6869 1.15 jmcneill #define MCX_TIMESTAMP_SHIFT 24
6870 1.1 jmcneill
6871 1.1 jmcneill static void
6872 1.1 jmcneill mcx_calibrate(void *arg)
6873 1.1 jmcneill {
6874 1.1 jmcneill struct mcx_softc *sc = arg;
6875 1.1 jmcneill struct mcx_calibration *nc, *pc;
6876 1.15 jmcneill uint64_t udiff, tdiff;
6877 1.1 jmcneill unsigned int gen;
6878 1.15 jmcneill int s;
6879 1.1 jmcneill
6880 1.1 jmcneill if (!ISSET(sc->sc_ec.ec_if.if_flags, IFF_RUNNING))
6881 1.1 jmcneill return;
6882 1.1 jmcneill
6883 1.1 jmcneill callout_schedule(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL * hz);
6884 1.1 jmcneill
6885 1.1 jmcneill gen = sc->sc_calibration_gen;
6886 1.1 jmcneill pc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
6887 1.1 jmcneill gen++;
6888 1.1 jmcneill nc = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
6889 1.1 jmcneill
6890 1.1 jmcneill nc->c_uptime = pc->c_ubase;
6891 1.1 jmcneill nc->c_timestamp = pc->c_tbase;
6892 1.1 jmcneill
6893 1.15 jmcneill s = splhigh(); /* crit_enter? */
6894 1.1 jmcneill nc->c_ubase = mcx_uptime();
6895 1.1 jmcneill nc->c_tbase = mcx_timer(sc);
6896 1.15 jmcneill splx(s);
6897 1.1 jmcneill
6898 1.15 jmcneill udiff = nc->c_ubase - nc->c_uptime;
6899 1.15 jmcneill tdiff = nc->c_tbase - nc->c_timestamp;
6900 1.15 jmcneill
6901 1.15 jmcneill /*
6902 1.15 jmcneill * udiff is the wall clock time between calibration ticks,
6903 1.15 jmcneill * which should be 32 seconds or 32 billion nanoseconds. if
6904 1.15 jmcneill * we squint, 1 billion nanoseconds is kind of like a 32 bit
6905 1.15 jmcneill * number, so 32 billion should still have a lot of high bits
6906 1.15 jmcneill * spare. we use this space by shifting the nanoseconds up
6907 1.15 jmcneill * 24 bits so we have a nice big number to divide by the
6908 1.15 jmcneill * number of mcx timer ticks.
6909 1.15 jmcneill */
6910 1.15 jmcneill nc->c_ratio = (udiff << MCX_TIMESTAMP_SHIFT) / tdiff;
6911 1.1 jmcneill
6912 1.1 jmcneill membar_producer();
6913 1.1 jmcneill sc->sc_calibration_gen = gen;
6914 1.1 jmcneill }
6915 1.1 jmcneill
6916 1.1 jmcneill static int
6917 1.15 jmcneill mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx,
6918 1.15 jmcneill struct mcx_cq_entry *cqe, struct mcx_mbufq *mq,
6919 1.15 jmcneill const struct mcx_calibration *c)
6920 1.1 jmcneill {
6921 1.15 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
6922 1.1 jmcneill struct mcx_slot *ms;
6923 1.1 jmcneill struct mbuf *m;
6924 1.15 jmcneill uint32_t flags;
6925 1.1 jmcneill int slot;
6926 1.1 jmcneill
6927 1.1 jmcneill slot = be16toh(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE);
6928 1.1 jmcneill
6929 1.15 jmcneill ms = &rx->rx_slots[slot];
6930 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize,
6931 1.1 jmcneill BUS_DMASYNC_POSTREAD);
6932 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, ms->ms_map);
6933 1.1 jmcneill
6934 1.1 jmcneill m = ms->ms_m;
6935 1.1 jmcneill ms->ms_m = NULL;
6936 1.1 jmcneill
6937 1.1 jmcneill m_set_rcvif(m, &sc->sc_ec.ec_if);
6938 1.1 jmcneill m->m_pkthdr.len = m->m_len = be32dec(&cqe->cq_byte_cnt);
6939 1.1 jmcneill
6940 1.1 jmcneill #if 0
6941 1.1 jmcneill if (cqe->cq_rx_hash_type) {
6942 1.15 jmcneill m->m_pkthdr.ph_flowid = be32toh(cqe->cq_rx_hash);
6943 1.15 jmcneill m->m_pkthdr.csum_flags |= M_FLOWID;
6944 1.1 jmcneill }
6945 1.1 jmcneill #endif
6946 1.1 jmcneill
6947 1.15 jmcneill flags = be32dec(&cqe->cq_flags);
6948 1.15 jmcneill if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) {
6949 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx)
6950 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
6951 1.15 jmcneill }
6952 1.15 jmcneill if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) {
6953 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_TCPv4_Rx)
6954 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
6955 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_TCPv6_Rx)
6956 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_TCPv6;
6957 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_UDPv4_Rx)
6958 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
6959 1.15 jmcneill if (ifp->if_capenable & IFCAP_CSUM_UDPv6_Rx)
6960 1.15 jmcneill m->m_pkthdr.csum_flags |= M_CSUM_UDPv6;
6961 1.15 jmcneill }
6962 1.15 jmcneill if (flags & MCX_CQ_ENTRY_FLAGS_CV) {
6963 1.15 jmcneill vlan_set_tag(m, flags & MCX_CQ_ENTRY_FLAGS_VLAN_MASK);
6964 1.15 jmcneill }
6965 1.15 jmcneill
6966 1.15 jmcneill #if notyet
6967 1.15 jmcneill if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_LINK0) && c->c_ratio) {
6968 1.15 jmcneill uint64_t t = be64dec(&cqe->cq_timestamp);
6969 1.15 jmcneill t -= c->c_timestamp;
6970 1.15 jmcneill t *= c->c_ratio;
6971 1.15 jmcneill t >>= MCX_TIMESTAMP_SHIFT;
6972 1.15 jmcneill t += c->c_uptime;
6973 1.1 jmcneill
6974 1.15 jmcneill m->m_pkthdr.ph_timestamp = t;
6975 1.1 jmcneill SET(m->m_pkthdr.csum_flags, M_TIMESTAMP);
6976 1.1 jmcneill }
6977 1.1 jmcneill #endif
6978 1.1 jmcneill
6979 1.1 jmcneill MBUFQ_ENQUEUE(mq, m);
6980 1.1 jmcneill
6981 1.1 jmcneill return (1);
6982 1.1 jmcneill }
6983 1.1 jmcneill
6984 1.1 jmcneill static struct mcx_cq_entry *
6985 1.1 jmcneill mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq)
6986 1.1 jmcneill {
6987 1.1 jmcneill struct mcx_cq_entry *cqe;
6988 1.1 jmcneill int next;
6989 1.1 jmcneill
6990 1.1 jmcneill cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem);
6991 1.1 jmcneill next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE);
6992 1.1 jmcneill
6993 1.1 jmcneill if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) ==
6994 1.1 jmcneill ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) {
6995 1.1 jmcneill return (&cqe[next]);
6996 1.1 jmcneill }
6997 1.1 jmcneill
6998 1.1 jmcneill return (NULL);
6999 1.1 jmcneill }
7000 1.1 jmcneill
7001 1.1 jmcneill static void
7002 1.15 jmcneill mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar)
7003 1.1 jmcneill {
7004 1.15 jmcneill struct mcx_cq_doorbell *db;
7005 1.1 jmcneill bus_size_t offset;
7006 1.1 jmcneill uint32_t val;
7007 1.1 jmcneill uint64_t uval;
7008 1.1 jmcneill
7009 1.1 jmcneill val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT;
7010 1.1 jmcneill val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
7011 1.1 jmcneill
7012 1.15 jmcneill db = MCX_DMA_OFF(&sc->sc_doorbell_mem, cq->cq_doorbell);
7013 1.15 jmcneill
7014 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7015 1.15 jmcneill cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_POSTWRITE);
7016 1.1 jmcneill
7017 1.15 jmcneill be32enc(&db->db_update_ci, cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK);
7018 1.15 jmcneill be32enc(&db->db_arm_ci, val);
7019 1.15 jmcneill
7020 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7021 1.15 jmcneill cq->cq_doorbell, sizeof(*db), BUS_DMASYNC_PREWRITE);
7022 1.15 jmcneill
7023 1.15 jmcneill offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_CQ_DOORBELL;
7024 1.15 jmcneill
7025 1.15 jmcneill uval = (uint64_t)val << 32;
7026 1.1 jmcneill uval |= cq->cq_n;
7027 1.15 jmcneill
7028 1.15 jmcneill bus_space_write_8(sc->sc_memt, sc->sc_memh, offset, htobe64(uval));
7029 1.15 jmcneill mcx_bar(sc, offset, sizeof(uval), BUS_SPACE_BARRIER_WRITE);
7030 1.1 jmcneill }
7031 1.1 jmcneill
7032 1.1 jmcneill void
7033 1.15 jmcneill mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq)
7034 1.1 jmcneill {
7035 1.15 jmcneill struct mcx_rx *rx = &q->q_rx;
7036 1.15 jmcneill struct mcx_tx *tx = &q->q_tx;
7037 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
7038 1.1 jmcneill const struct mcx_calibration *c;
7039 1.1 jmcneill unsigned int gen;
7040 1.1 jmcneill struct mcx_cq_entry *cqe;
7041 1.1 jmcneill struct mcx_mbufq mq;
7042 1.1 jmcneill struct mbuf *m;
7043 1.1 jmcneill int rxfree, txfree;
7044 1.1 jmcneill
7045 1.1 jmcneill MBUFQ_INIT(&mq);
7046 1.1 jmcneill
7047 1.1 jmcneill gen = sc->sc_calibration_gen;
7048 1.1 jmcneill membar_consumer();
7049 1.1 jmcneill c = &sc->sc_calibration[gen % __arraycount(sc->sc_calibration)];
7050 1.1 jmcneill
7051 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
7052 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_POSTREAD);
7053 1.15 jmcneill
7054 1.1 jmcneill rxfree = 0;
7055 1.1 jmcneill txfree = 0;
7056 1.1 jmcneill while ((cqe = mcx_next_cq_entry(sc, cq))) {
7057 1.1 jmcneill uint8_t opcode;
7058 1.1 jmcneill opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT);
7059 1.1 jmcneill switch (opcode) {
7060 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_REQ:
7061 1.15 jmcneill txfree += mcx_process_txeof(sc, tx, cqe);
7062 1.1 jmcneill break;
7063 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_SEND:
7064 1.15 jmcneill rxfree += mcx_process_rx(sc, rx, cqe, &mq, c);
7065 1.1 jmcneill break;
7066 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_REQ_ERR:
7067 1.1 jmcneill case MCX_CQ_ENTRY_OPCODE_SEND_ERR:
7068 1.1 jmcneill /* uint8_t *cqp = (uint8_t *)cqe; */
7069 1.15 jmcneill /* printf("%s: cq completion error: %x\n",
7070 1.15 jmcneill DEVNAME(sc), cqp[0x37]); */
7071 1.1 jmcneill break;
7072 1.1 jmcneill
7073 1.1 jmcneill default:
7074 1.15 jmcneill /* printf("%s: cq completion opcode %x??\n",
7075 1.15 jmcneill DEVNAME(sc), opcode); */
7076 1.1 jmcneill break;
7077 1.1 jmcneill }
7078 1.1 jmcneill
7079 1.1 jmcneill cq->cq_cons++;
7080 1.1 jmcneill }
7081 1.1 jmcneill
7082 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&cq->cq_mem),
7083 1.15 jmcneill 0, MCX_DMA_LEN(&cq->cq_mem), BUS_DMASYNC_PREREAD);
7084 1.15 jmcneill
7085 1.1 jmcneill if (rxfree > 0) {
7086 1.15 jmcneill mcx_rxr_put(&rx->rx_rxr, rxfree);
7087 1.1 jmcneill while (MBUFQ_FIRST(&mq) != NULL) {
7088 1.1 jmcneill MBUFQ_DEQUEUE(&mq, m);
7089 1.1 jmcneill if_percpuq_enqueue(ifp->if_percpuq, m);
7090 1.1 jmcneill }
7091 1.1 jmcneill
7092 1.15 jmcneill mcx_rx_fill(sc, rx);
7093 1.15 jmcneill if (mcx_rxr_inuse(&rx->rx_rxr) == 0)
7094 1.15 jmcneill callout_schedule(&rx->rx_refill, 1);
7095 1.1 jmcneill }
7096 1.17 jmcneill
7097 1.17 jmcneill cq->cq_count++;
7098 1.17 jmcneill mcx_arm_cq(sc, cq, q->q_uar);
7099 1.17 jmcneill
7100 1.1 jmcneill if (txfree > 0) {
7101 1.15 jmcneill tx->tx_cons += txfree;
7102 1.1 jmcneill if_schedule_deferred_start(ifp);
7103 1.1 jmcneill }
7104 1.1 jmcneill }
7105 1.1 jmcneill
7106 1.15 jmcneill
7107 1.1 jmcneill static void
7108 1.15 jmcneill mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar)
7109 1.1 jmcneill {
7110 1.1 jmcneill bus_size_t offset;
7111 1.1 jmcneill uint32_t val;
7112 1.1 jmcneill
7113 1.15 jmcneill offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM;
7114 1.15 jmcneill val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff);
7115 1.1 jmcneill
7116 1.1 jmcneill mcx_wr(sc, offset, val);
7117 1.15 jmcneill mcx_bar(sc, offset, sizeof(val), BUS_SPACE_BARRIER_WRITE);
7118 1.1 jmcneill }
7119 1.1 jmcneill
7120 1.1 jmcneill static struct mcx_eq_entry *
7121 1.15 jmcneill mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq)
7122 1.1 jmcneill {
7123 1.1 jmcneill struct mcx_eq_entry *eqe;
7124 1.1 jmcneill int next;
7125 1.1 jmcneill
7126 1.15 jmcneill eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem);
7127 1.15 jmcneill next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE);
7128 1.15 jmcneill if ((eqe[next].eq_owner & 1) ==
7129 1.15 jmcneill ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) {
7130 1.15 jmcneill eq->eq_cons++;
7131 1.1 jmcneill return (&eqe[next]);
7132 1.1 jmcneill }
7133 1.1 jmcneill return (NULL);
7134 1.1 jmcneill }
7135 1.1 jmcneill
7136 1.1 jmcneill int
7137 1.15 jmcneill mcx_admin_intr(void *xsc)
7138 1.1 jmcneill {
7139 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)xsc;
7140 1.15 jmcneill struct mcx_eq *eq = &sc->sc_admin_eq;
7141 1.1 jmcneill struct mcx_eq_entry *eqe;
7142 1.1 jmcneill
7143 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7144 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
7145 1.15 jmcneill
7146 1.15 jmcneill while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
7147 1.1 jmcneill switch (eqe->eq_event_type) {
7148 1.1 jmcneill case MCX_EVENT_TYPE_LAST_WQE:
7149 1.1 jmcneill /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */
7150 1.1 jmcneill break;
7151 1.1 jmcneill
7152 1.1 jmcneill case MCX_EVENT_TYPE_CQ_ERROR:
7153 1.1 jmcneill /* printf("%s: cq error\n", DEVNAME(sc)); */
7154 1.1 jmcneill break;
7155 1.1 jmcneill
7156 1.1 jmcneill case MCX_EVENT_TYPE_CMD_COMPLETION:
7157 1.1 jmcneill /* wakeup probably */
7158 1.1 jmcneill break;
7159 1.1 jmcneill
7160 1.1 jmcneill case MCX_EVENT_TYPE_PORT_CHANGE:
7161 1.1 jmcneill workqueue_enqueue(sc->sc_workq, &sc->sc_port_change, NULL);
7162 1.1 jmcneill break;
7163 1.1 jmcneill
7164 1.1 jmcneill default:
7165 1.1 jmcneill /* printf("%s: something happened\n", DEVNAME(sc)); */
7166 1.1 jmcneill break;
7167 1.1 jmcneill }
7168 1.1 jmcneill }
7169 1.15 jmcneill
7170 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7171 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
7172 1.15 jmcneill
7173 1.15 jmcneill mcx_arm_eq(sc, eq, sc->sc_uar);
7174 1.15 jmcneill
7175 1.15 jmcneill return (1);
7176 1.15 jmcneill }
7177 1.15 jmcneill
7178 1.15 jmcneill int
7179 1.15 jmcneill mcx_cq_intr(void *xq)
7180 1.15 jmcneill {
7181 1.15 jmcneill struct mcx_queues *q = (struct mcx_queues *)xq;
7182 1.15 jmcneill struct mcx_softc *sc = q->q_sc;
7183 1.15 jmcneill struct mcx_eq *eq = &q->q_eq;
7184 1.15 jmcneill struct mcx_eq_entry *eqe;
7185 1.15 jmcneill int cqn;
7186 1.15 jmcneill
7187 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7188 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_POSTREAD);
7189 1.15 jmcneill
7190 1.15 jmcneill while ((eqe = mcx_next_eq_entry(sc, eq)) != NULL) {
7191 1.15 jmcneill switch (eqe->eq_event_type) {
7192 1.15 jmcneill case MCX_EVENT_TYPE_COMPLETION:
7193 1.15 jmcneill cqn = be32toh(eqe->eq_event_data[6]);
7194 1.15 jmcneill if (cqn == q->q_cq.cq_n)
7195 1.15 jmcneill mcx_process_cq(sc, q, &q->q_cq);
7196 1.15 jmcneill break;
7197 1.15 jmcneill }
7198 1.15 jmcneill }
7199 1.15 jmcneill
7200 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&eq->eq_mem),
7201 1.15 jmcneill 0, MCX_DMA_LEN(&eq->eq_mem), BUS_DMASYNC_PREREAD);
7202 1.15 jmcneill
7203 1.15 jmcneill mcx_arm_eq(sc, eq, q->q_uar);
7204 1.15 jmcneill
7205 1.1 jmcneill return (1);
7206 1.1 jmcneill }
7207 1.1 jmcneill
7208 1.1 jmcneill static void
7209 1.1 jmcneill mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated,
7210 1.1 jmcneill int total)
7211 1.1 jmcneill {
7212 1.1 jmcneill struct mcx_slot *ms;
7213 1.1 jmcneill
7214 1.1 jmcneill int i = allocated;
7215 1.1 jmcneill while (i-- > 0) {
7216 1.1 jmcneill ms = &slots[i];
7217 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, ms->ms_map);
7218 1.1 jmcneill if (ms->ms_m != NULL)
7219 1.1 jmcneill m_freem(ms->ms_m);
7220 1.1 jmcneill }
7221 1.1 jmcneill kmem_free(slots, total * sizeof(*ms));
7222 1.1 jmcneill }
7223 1.1 jmcneill
7224 1.1 jmcneill static int
7225 1.15 jmcneill mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q)
7226 1.1 jmcneill {
7227 1.15 jmcneill struct mcx_rx *rx;
7228 1.15 jmcneill struct mcx_tx *tx;
7229 1.1 jmcneill struct mcx_slot *ms;
7230 1.15 jmcneill int i;
7231 1.1 jmcneill
7232 1.15 jmcneill rx = &q->q_rx;
7233 1.15 jmcneill rx->rx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_RQ_SIZE),
7234 1.1 jmcneill KM_SLEEP);
7235 1.1 jmcneill
7236 1.1 jmcneill for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) {
7237 1.15 jmcneill ms = &rx->rx_slots[i];
7238 1.1 jmcneill if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1,
7239 1.1 jmcneill sc->sc_hardmtu, 0,
7240 1.1 jmcneill BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
7241 1.1 jmcneill &ms->ms_map) != 0) {
7242 1.1 jmcneill printf("%s: failed to allocate rx dma maps\n",
7243 1.1 jmcneill DEVNAME(sc));
7244 1.1 jmcneill goto destroy_rx_slots;
7245 1.1 jmcneill }
7246 1.15 jmcneill }
7247 1.15 jmcneill
7248 1.15 jmcneill tx = &q->q_tx;
7249 1.15 jmcneill tx->tx_slots = kmem_zalloc(sizeof(*ms) * (1 << MCX_LOG_SQ_SIZE),
7250 1.15 jmcneill KM_SLEEP);
7251 1.15 jmcneill
7252 1.15 jmcneill for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) {
7253 1.15 jmcneill ms = &tx->tx_slots[i];
7254 1.15 jmcneill if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu,
7255 1.15 jmcneill MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0,
7256 1.15 jmcneill BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
7257 1.15 jmcneill &ms->ms_map) != 0) {
7258 1.15 jmcneill printf("%s: failed to allocate tx dma maps\n",
7259 1.15 jmcneill DEVNAME(sc));
7260 1.15 jmcneill goto destroy_tx_slots;
7261 1.15 jmcneill }
7262 1.15 jmcneill }
7263 1.15 jmcneill
7264 1.15 jmcneill if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index,
7265 1.15 jmcneill q->q_eq.eq_n) != 0)
7266 1.15 jmcneill goto destroy_tx_slots;
7267 1.15 jmcneill
7268 1.15 jmcneill if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n)
7269 1.15 jmcneill != 0)
7270 1.15 jmcneill goto destroy_cq;
7271 1.15 jmcneill
7272 1.15 jmcneill if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0)
7273 1.15 jmcneill goto destroy_sq;
7274 1.15 jmcneill
7275 1.15 jmcneill return 0;
7276 1.15 jmcneill
7277 1.15 jmcneill destroy_sq:
7278 1.15 jmcneill mcx_destroy_sq(sc, tx);
7279 1.15 jmcneill destroy_cq:
7280 1.15 jmcneill mcx_destroy_cq(sc, &q->q_cq);
7281 1.15 jmcneill destroy_tx_slots:
7282 1.15 jmcneill mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE));
7283 1.15 jmcneill tx->tx_slots = NULL;
7284 1.15 jmcneill
7285 1.15 jmcneill i = (1 << MCX_LOG_RQ_SIZE);
7286 1.15 jmcneill destroy_rx_slots:
7287 1.15 jmcneill mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE));
7288 1.15 jmcneill rx->rx_slots = NULL;
7289 1.15 jmcneill return ENOMEM;
7290 1.15 jmcneill }
7291 1.15 jmcneill
7292 1.15 jmcneill static int
7293 1.15 jmcneill mcx_rss_group_entry_count(struct mcx_softc *sc, int group)
7294 1.15 jmcneill {
7295 1.15 jmcneill int i;
7296 1.15 jmcneill int count;
7297 1.15 jmcneill
7298 1.15 jmcneill count = 0;
7299 1.15 jmcneill for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7300 1.15 jmcneill if (mcx_rss_config[i].flow_group == group)
7301 1.15 jmcneill count++;
7302 1.15 jmcneill }
7303 1.15 jmcneill
7304 1.15 jmcneill return count;
7305 1.15 jmcneill }
7306 1.15 jmcneill
7307 1.15 jmcneill static int
7308 1.15 jmcneill mcx_init(struct ifnet *ifp)
7309 1.15 jmcneill {
7310 1.15 jmcneill struct mcx_softc *sc = ifp->if_softc;
7311 1.15 jmcneill struct mcx_rx *rx;
7312 1.15 jmcneill struct mcx_tx *tx;
7313 1.15 jmcneill int i, start, count, flow_group, flow_index;
7314 1.15 jmcneill struct mcx_flow_match match_crit;
7315 1.15 jmcneill struct mcx_rss_rule *rss;
7316 1.15 jmcneill uint32_t dest;
7317 1.15 jmcneill int rqns[MCX_MAX_QUEUES] = { 0 };
7318 1.15 jmcneill
7319 1.15 jmcneill if (ISSET(ifp->if_flags, IFF_RUNNING))
7320 1.15 jmcneill mcx_stop(ifp, 0);
7321 1.1 jmcneill
7322 1.15 jmcneill if (mcx_create_tis(sc, &sc->sc_tis) != 0)
7323 1.15 jmcneill goto down;
7324 1.1 jmcneill
7325 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7326 1.15 jmcneill if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) {
7327 1.15 jmcneill goto down;
7328 1.1 jmcneill }
7329 1.1 jmcneill }
7330 1.1 jmcneill
7331 1.15 jmcneill /* RSS flow table and flow groups */
7332 1.15 jmcneill if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1,
7333 1.15 jmcneill &sc->sc_rss_flow_table_id) != 0)
7334 1.1 jmcneill goto down;
7335 1.1 jmcneill
7336 1.15 jmcneill dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
7337 1.15 jmcneill sc->sc_rss_flow_table_id;
7338 1.1 jmcneill
7339 1.15 jmcneill /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */
7340 1.15 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7341 1.15 jmcneill match_crit.mc_ethertype = 0xffff;
7342 1.15 jmcneill match_crit.mc_ip_proto = 0xff;
7343 1.15 jmcneill match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG;
7344 1.15 jmcneill start = 0;
7345 1.15 jmcneill count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4);
7346 1.15 jmcneill if (count != 0) {
7347 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7348 1.15 jmcneill MCX_FLOW_GROUP_RSS_L4, start, count,
7349 1.15 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7350 1.15 jmcneill goto down;
7351 1.15 jmcneill start += count;
7352 1.15 jmcneill }
7353 1.1 jmcneill
7354 1.15 jmcneill /* L3 RSS flow group (v4/v6, including fragments) */
7355 1.15 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7356 1.15 jmcneill match_crit.mc_ethertype = 0xffff;
7357 1.15 jmcneill count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3);
7358 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7359 1.15 jmcneill MCX_FLOW_GROUP_RSS_L3, start, count,
7360 1.15 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7361 1.1 jmcneill goto down;
7362 1.15 jmcneill start += count;
7363 1.1 jmcneill
7364 1.15 jmcneill /* non-RSS flow group */
7365 1.15 jmcneill count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE);
7366 1.15 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7367 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id,
7368 1.15 jmcneill MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0)
7369 1.1 jmcneill goto down;
7370 1.1 jmcneill
7371 1.15 jmcneill /* Root flow table, matching packets based on mac address */
7372 1.15 jmcneill if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0,
7373 1.15 jmcneill &sc->sc_mac_flow_table_id) != 0)
7374 1.1 jmcneill goto down;
7375 1.1 jmcneill
7376 1.1 jmcneill /* promisc flow group */
7377 1.1 jmcneill start = 0;
7378 1.1 jmcneill memset(&match_crit, 0, sizeof(match_crit));
7379 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7380 1.15 jmcneill MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0)
7381 1.1 jmcneill goto down;
7382 1.1 jmcneill sc->sc_promisc_flow_enabled = 0;
7383 1.1 jmcneill start++;
7384 1.1 jmcneill
7385 1.1 jmcneill /* all multicast flow group */
7386 1.1 jmcneill match_crit.mc_dest_mac[0] = 0x01;
7387 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7388 1.15 jmcneill MCX_FLOW_GROUP_ALLMULTI, start, 1,
7389 1.1 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7390 1.1 jmcneill goto down;
7391 1.1 jmcneill sc->sc_allmulti_flow_enabled = 0;
7392 1.1 jmcneill start++;
7393 1.1 jmcneill
7394 1.1 jmcneill /* mac address matching flow group */
7395 1.1 jmcneill memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac));
7396 1.15 jmcneill if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id,
7397 1.15 jmcneill MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start,
7398 1.1 jmcneill MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0)
7399 1.1 jmcneill goto down;
7400 1.1 jmcneill
7401 1.1 jmcneill /* flow table entries for unicast and broadcast */
7402 1.1 jmcneill start = 0;
7403 1.15 jmcneill if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
7404 1.15 jmcneill LLADDR(satosdl(ifp->if_dl->ifa_addr)), dest) != 0)
7405 1.1 jmcneill goto down;
7406 1.1 jmcneill start++;
7407 1.1 jmcneill
7408 1.15 jmcneill if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start,
7409 1.15 jmcneill etherbroadcastaddr, dest) != 0)
7410 1.1 jmcneill goto down;
7411 1.1 jmcneill start++;
7412 1.1 jmcneill
7413 1.1 jmcneill /* multicast entries go after that */
7414 1.1 jmcneill sc->sc_mcast_flow_base = start;
7415 1.1 jmcneill
7416 1.1 jmcneill /* re-add any existing multicast flows */
7417 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7418 1.1 jmcneill if (sc->sc_mcast_flows[i][0] != 0) {
7419 1.15 jmcneill mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC,
7420 1.1 jmcneill sc->sc_mcast_flow_base + i,
7421 1.15 jmcneill sc->sc_mcast_flows[i], dest);
7422 1.1 jmcneill }
7423 1.1 jmcneill }
7424 1.1 jmcneill
7425 1.15 jmcneill if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0)
7426 1.1 jmcneill goto down;
7427 1.1 jmcneill
7428 1.15 jmcneill /*
7429 1.15 jmcneill * the RQT can be any size as long as it's a power of two.
7430 1.15 jmcneill * since we also restrict the number of queues to a power of two,
7431 1.15 jmcneill * we can just put each rx queue in once.
7432 1.15 jmcneill */
7433 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++)
7434 1.15 jmcneill rqns[i] = sc->sc_queues[i].q_rx.rx_rqn;
7435 1.15 jmcneill
7436 1.15 jmcneill if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0)
7437 1.1 jmcneill goto down;
7438 1.1 jmcneill
7439 1.15 jmcneill start = 0;
7440 1.15 jmcneill flow_index = 0;
7441 1.15 jmcneill flow_group = -1;
7442 1.15 jmcneill for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7443 1.15 jmcneill rss = &mcx_rss_config[i];
7444 1.15 jmcneill if (rss->flow_group != flow_group) {
7445 1.15 jmcneill flow_group = rss->flow_group;
7446 1.15 jmcneill flow_index = 0;
7447 1.15 jmcneill }
7448 1.15 jmcneill
7449 1.15 jmcneill if (rss->hash_sel == 0) {
7450 1.15 jmcneill if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx,
7451 1.15 jmcneill &sc->sc_tir[i]) != 0)
7452 1.15 jmcneill goto down;
7453 1.15 jmcneill } else {
7454 1.15 jmcneill if (mcx_create_tir_indirect(sc, sc->sc_rqt,
7455 1.15 jmcneill rss->hash_sel, &sc->sc_tir[i]) != 0)
7456 1.15 jmcneill goto down;
7457 1.15 jmcneill }
7458 1.15 jmcneill
7459 1.15 jmcneill if (mcx_set_flow_table_entry_proto(sc, flow_group,
7460 1.15 jmcneill flow_index, rss->ethertype, rss->ip_proto,
7461 1.15 jmcneill MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0)
7462 1.15 jmcneill goto down;
7463 1.15 jmcneill flow_index++;
7464 1.15 jmcneill }
7465 1.15 jmcneill
7466 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7467 1.15 jmcneill struct mcx_queues *q = &sc->sc_queues[i];
7468 1.15 jmcneill rx = &q->q_rx;
7469 1.15 jmcneill tx = &q->q_tx;
7470 1.15 jmcneill
7471 1.15 jmcneill /* start the queues */
7472 1.15 jmcneill if (mcx_ready_sq(sc, tx) != 0)
7473 1.15 jmcneill goto down;
7474 1.15 jmcneill
7475 1.15 jmcneill if (mcx_ready_rq(sc, rx) != 0)
7476 1.15 jmcneill goto down;
7477 1.15 jmcneill
7478 1.15 jmcneill mcx_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE));
7479 1.15 jmcneill rx->rx_prod = 0;
7480 1.15 jmcneill mcx_rx_fill(sc, rx);
7481 1.1 jmcneill
7482 1.15 jmcneill tx->tx_cons = 0;
7483 1.15 jmcneill tx->tx_prod = 0;
7484 1.15 jmcneill }
7485 1.1 jmcneill
7486 1.1 jmcneill mcx_calibrate_first(sc);
7487 1.1 jmcneill
7488 1.1 jmcneill SET(ifp->if_flags, IFF_RUNNING);
7489 1.1 jmcneill CLR(ifp->if_flags, IFF_OACTIVE);
7490 1.1 jmcneill if_schedule_deferred_start(ifp);
7491 1.1 jmcneill
7492 1.1 jmcneill return 0;
7493 1.1 jmcneill down:
7494 1.1 jmcneill mcx_stop(ifp, 0);
7495 1.1 jmcneill return EIO;
7496 1.1 jmcneill }
7497 1.1 jmcneill
7498 1.1 jmcneill static void
7499 1.1 jmcneill mcx_stop(struct ifnet *ifp, int disable)
7500 1.1 jmcneill {
7501 1.1 jmcneill struct mcx_softc *sc = ifp->if_softc;
7502 1.15 jmcneill struct mcx_rss_rule *rss;
7503 1.15 jmcneill int group, i, flow_group, flow_index;
7504 1.1 jmcneill
7505 1.1 jmcneill CLR(ifp->if_flags, IFF_RUNNING);
7506 1.1 jmcneill
7507 1.1 jmcneill /*
7508 1.1 jmcneill * delete flow table entries first, so no packets can arrive
7509 1.1 jmcneill * after the barriers
7510 1.1 jmcneill */
7511 1.1 jmcneill if (sc->sc_promisc_flow_enabled)
7512 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0);
7513 1.1 jmcneill if (sc->sc_allmulti_flow_enabled)
7514 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0);
7515 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0);
7516 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1);
7517 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7518 1.1 jmcneill if (sc->sc_mcast_flows[i][0] != 0) {
7519 1.1 jmcneill mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC,
7520 1.1 jmcneill sc->sc_mcast_flow_base + i);
7521 1.1 jmcneill }
7522 1.1 jmcneill }
7523 1.1 jmcneill
7524 1.15 jmcneill flow_group = -1;
7525 1.15 jmcneill flow_index = 0;
7526 1.15 jmcneill for (i = 0; i < __arraycount(mcx_rss_config); i++) {
7527 1.15 jmcneill rss = &mcx_rss_config[i];
7528 1.15 jmcneill if (rss->flow_group != flow_group) {
7529 1.15 jmcneill flow_group = rss->flow_group;
7530 1.15 jmcneill flow_index = 0;
7531 1.15 jmcneill }
7532 1.15 jmcneill
7533 1.15 jmcneill mcx_delete_flow_table_entry(sc, flow_group, flow_index);
7534 1.15 jmcneill
7535 1.15 jmcneill mcx_destroy_tir(sc, sc->sc_tir[i]);
7536 1.15 jmcneill sc->sc_tir[i] = 0;
7537 1.15 jmcneill
7538 1.15 jmcneill flow_index++;
7539 1.15 jmcneill }
7540 1.15 jmcneill
7541 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7542 1.15 jmcneill callout_halt(&sc->sc_queues[i].q_rx.rx_refill, NULL);
7543 1.15 jmcneill }
7544 1.15 jmcneill
7545 1.1 jmcneill callout_halt(&sc->sc_calibrate, NULL);
7546 1.1 jmcneill
7547 1.1 jmcneill for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) {
7548 1.15 jmcneill if (sc->sc_flow_group[group].g_id != -1)
7549 1.15 jmcneill mcx_destroy_flow_group(sc, group);
7550 1.15 jmcneill }
7551 1.15 jmcneill
7552 1.15 jmcneill if (sc->sc_mac_flow_table_id != -1) {
7553 1.15 jmcneill mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id);
7554 1.15 jmcneill sc->sc_mac_flow_table_id = -1;
7555 1.15 jmcneill }
7556 1.15 jmcneill if (sc->sc_rss_flow_table_id != -1) {
7557 1.15 jmcneill mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id);
7558 1.15 jmcneill sc->sc_rss_flow_table_id = -1;
7559 1.15 jmcneill }
7560 1.15 jmcneill if (sc->sc_rqt != -1) {
7561 1.15 jmcneill mcx_destroy_rqt(sc, sc->sc_rqt);
7562 1.15 jmcneill sc->sc_rqt = -1;
7563 1.15 jmcneill }
7564 1.15 jmcneill
7565 1.15 jmcneill for (i = 0; i < sc->sc_nqueues; i++) {
7566 1.15 jmcneill struct mcx_queues *q = &sc->sc_queues[i];
7567 1.15 jmcneill struct mcx_rx *rx = &q->q_rx;
7568 1.15 jmcneill struct mcx_tx *tx = &q->q_tx;
7569 1.15 jmcneill struct mcx_cq *cq = &q->q_cq;
7570 1.15 jmcneill
7571 1.15 jmcneill if (rx->rx_rqn != 0)
7572 1.15 jmcneill mcx_destroy_rq(sc, rx);
7573 1.15 jmcneill
7574 1.15 jmcneill if (tx->tx_sqn != 0)
7575 1.15 jmcneill mcx_destroy_sq(sc, tx);
7576 1.15 jmcneill
7577 1.15 jmcneill if (tx->tx_slots != NULL) {
7578 1.15 jmcneill mcx_free_slots(sc, tx->tx_slots,
7579 1.15 jmcneill (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE));
7580 1.15 jmcneill tx->tx_slots = NULL;
7581 1.15 jmcneill }
7582 1.15 jmcneill if (rx->rx_slots != NULL) {
7583 1.15 jmcneill mcx_free_slots(sc, rx->rx_slots,
7584 1.15 jmcneill (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE));
7585 1.15 jmcneill rx->rx_slots = NULL;
7586 1.15 jmcneill }
7587 1.15 jmcneill
7588 1.15 jmcneill if (cq->cq_n != 0)
7589 1.15 jmcneill mcx_destroy_cq(sc, cq);
7590 1.15 jmcneill }
7591 1.15 jmcneill if (sc->sc_tis != 0) {
7592 1.15 jmcneill mcx_destroy_tis(sc, sc->sc_tis);
7593 1.15 jmcneill sc->sc_tis = 0;
7594 1.1 jmcneill }
7595 1.1 jmcneill }
7596 1.1 jmcneill
7597 1.1 jmcneill static int
7598 1.1 jmcneill mcx_ioctl(struct ifnet *ifp, u_long cmd, void *data)
7599 1.1 jmcneill {
7600 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
7601 1.1 jmcneill struct ifreq *ifr = (struct ifreq *)data;
7602 1.7 jmcneill struct ethercom *ec = &sc->sc_ec;
7603 1.1 jmcneill uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
7604 1.7 jmcneill struct ether_multi *enm;
7605 1.7 jmcneill struct ether_multistep step;
7606 1.7 jmcneill int s, i, flags, error = 0;
7607 1.15 jmcneill uint32_t dest;
7608 1.1 jmcneill
7609 1.1 jmcneill s = splnet();
7610 1.1 jmcneill switch (cmd) {
7611 1.1 jmcneill
7612 1.1 jmcneill case SIOCADDMULTI:
7613 1.1 jmcneill if (ether_addmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
7614 1.1 jmcneill error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
7615 1.7 jmcneill if (error != 0) {
7616 1.7 jmcneill splx(s);
7617 1.1 jmcneill return (error);
7618 1.7 jmcneill }
7619 1.1 jmcneill
7620 1.15 jmcneill dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE |
7621 1.15 jmcneill sc->sc_rss_flow_table_id;
7622 1.15 jmcneill
7623 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7624 1.1 jmcneill if (sc->sc_mcast_flows[i][0] == 0) {
7625 1.1 jmcneill memcpy(sc->sc_mcast_flows[i], addrlo,
7626 1.1 jmcneill ETHER_ADDR_LEN);
7627 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_RUNNING)) {
7628 1.15 jmcneill mcx_set_flow_table_entry_mac(sc,
7629 1.1 jmcneill MCX_FLOW_GROUP_MAC,
7630 1.1 jmcneill sc->sc_mcast_flow_base + i,
7631 1.15 jmcneill sc->sc_mcast_flows[i], dest);
7632 1.1 jmcneill }
7633 1.1 jmcneill break;
7634 1.1 jmcneill }
7635 1.1 jmcneill }
7636 1.1 jmcneill
7637 1.1 jmcneill if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
7638 1.1 jmcneill if (i == MCX_NUM_MCAST_FLOWS) {
7639 1.1 jmcneill SET(ifp->if_flags, IFF_ALLMULTI);
7640 1.1 jmcneill sc->sc_extra_mcast++;
7641 1.1 jmcneill error = ENETRESET;
7642 1.1 jmcneill }
7643 1.1 jmcneill
7644 1.7 jmcneill if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN)) {
7645 1.1 jmcneill SET(ifp->if_flags, IFF_ALLMULTI);
7646 1.1 jmcneill error = ENETRESET;
7647 1.1 jmcneill }
7648 1.1 jmcneill }
7649 1.1 jmcneill }
7650 1.1 jmcneill break;
7651 1.1 jmcneill
7652 1.1 jmcneill case SIOCDELMULTI:
7653 1.1 jmcneill if (ether_delmulti(ifreq_getaddr(cmd, ifr), &sc->sc_ec) == ENETRESET) {
7654 1.1 jmcneill error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi);
7655 1.7 jmcneill if (error != 0) {
7656 1.7 jmcneill splx(s);
7657 1.1 jmcneill return (error);
7658 1.7 jmcneill }
7659 1.1 jmcneill
7660 1.1 jmcneill for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) {
7661 1.1 jmcneill if (memcmp(sc->sc_mcast_flows[i], addrlo,
7662 1.1 jmcneill ETHER_ADDR_LEN) == 0) {
7663 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_RUNNING)) {
7664 1.1 jmcneill mcx_delete_flow_table_entry(sc,
7665 1.1 jmcneill MCX_FLOW_GROUP_MAC,
7666 1.1 jmcneill sc->sc_mcast_flow_base + i);
7667 1.1 jmcneill }
7668 1.1 jmcneill sc->sc_mcast_flows[i][0] = 0;
7669 1.1 jmcneill break;
7670 1.1 jmcneill }
7671 1.1 jmcneill }
7672 1.1 jmcneill
7673 1.1 jmcneill if (i == MCX_NUM_MCAST_FLOWS)
7674 1.1 jmcneill sc->sc_extra_mcast--;
7675 1.1 jmcneill
7676 1.1 jmcneill if (ISSET(ifp->if_flags, IFF_ALLMULTI) &&
7677 1.7 jmcneill sc->sc_extra_mcast == 0) {
7678 1.7 jmcneill flags = 0;
7679 1.7 jmcneill ETHER_LOCK(ec);
7680 1.7 jmcneill ETHER_FIRST_MULTI(step, ec, enm);
7681 1.7 jmcneill while (enm != NULL) {
7682 1.7 jmcneill if (memcmp(enm->enm_addrlo,
7683 1.7 jmcneill enm->enm_addrhi, ETHER_ADDR_LEN)) {
7684 1.7 jmcneill SET(flags, IFF_ALLMULTI);
7685 1.7 jmcneill break;
7686 1.7 jmcneill }
7687 1.7 jmcneill ETHER_NEXT_MULTI(step, enm);
7688 1.7 jmcneill }
7689 1.7 jmcneill ETHER_UNLOCK(ec);
7690 1.7 jmcneill if (!ISSET(flags, IFF_ALLMULTI)) {
7691 1.7 jmcneill CLR(ifp->if_flags, IFF_ALLMULTI);
7692 1.7 jmcneill error = ENETRESET;
7693 1.7 jmcneill }
7694 1.1 jmcneill }
7695 1.1 jmcneill }
7696 1.1 jmcneill break;
7697 1.1 jmcneill
7698 1.1 jmcneill default:
7699 1.1 jmcneill error = ether_ioctl(ifp, cmd, data);
7700 1.1 jmcneill }
7701 1.1 jmcneill
7702 1.1 jmcneill if (error == ENETRESET) {
7703 1.1 jmcneill if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) ==
7704 1.1 jmcneill (IFF_UP | IFF_RUNNING))
7705 1.1 jmcneill mcx_iff(sc);
7706 1.1 jmcneill error = 0;
7707 1.1 jmcneill }
7708 1.1 jmcneill splx(s);
7709 1.1 jmcneill
7710 1.1 jmcneill return (error);
7711 1.1 jmcneill }
7712 1.1 jmcneill
7713 1.1 jmcneill #if 0
7714 1.1 jmcneill static int
7715 1.1 jmcneill mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff)
7716 1.1 jmcneill {
7717 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
7718 1.1 jmcneill struct mcx_reg_mcia mcia;
7719 1.1 jmcneill struct mcx_reg_pmlp pmlp;
7720 1.1 jmcneill int offset, error;
7721 1.1 jmcneill
7722 1.1 jmcneill /* get module number */
7723 1.1 jmcneill memset(&pmlp, 0, sizeof(pmlp));
7724 1.1 jmcneill pmlp.rp_local_port = 1;
7725 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp,
7726 1.1 jmcneill sizeof(pmlp));
7727 1.1 jmcneill if (error != 0) {
7728 1.1 jmcneill printf("%s: unable to get eeprom module number\n",
7729 1.1 jmcneill DEVNAME(sc));
7730 1.1 jmcneill return error;
7731 1.1 jmcneill }
7732 1.1 jmcneill
7733 1.1 jmcneill for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) {
7734 1.1 jmcneill memset(&mcia, 0, sizeof(mcia));
7735 1.1 jmcneill mcia.rm_l = 0;
7736 1.1 jmcneill mcia.rm_module = be32toh(pmlp.rp_lane0_mapping) &
7737 1.1 jmcneill MCX_PMLP_MODULE_NUM_MASK;
7738 1.1 jmcneill mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */
7739 1.1 jmcneill mcia.rm_page_num = sff->sff_page;
7740 1.1 jmcneill mcia.rm_dev_addr = htobe16(offset);
7741 1.1 jmcneill mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES);
7742 1.1 jmcneill
7743 1.1 jmcneill error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ,
7744 1.1 jmcneill &mcia, sizeof(mcia));
7745 1.1 jmcneill if (error != 0) {
7746 1.1 jmcneill printf("%s: unable to read eeprom at %x\n",
7747 1.1 jmcneill DEVNAME(sc), offset);
7748 1.1 jmcneill return error;
7749 1.1 jmcneill }
7750 1.1 jmcneill
7751 1.1 jmcneill memcpy(sff->sff_data + offset, mcia.rm_data,
7752 1.1 jmcneill MCX_MCIA_EEPROM_BYTES);
7753 1.1 jmcneill }
7754 1.1 jmcneill
7755 1.1 jmcneill return 0;
7756 1.1 jmcneill }
7757 1.1 jmcneill #endif
7758 1.1 jmcneill
7759 1.1 jmcneill static int
7760 1.1 jmcneill mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m)
7761 1.1 jmcneill {
7762 1.1 jmcneill switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
7763 1.1 jmcneill BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) {
7764 1.1 jmcneill case 0:
7765 1.1 jmcneill break;
7766 1.1 jmcneill
7767 1.1 jmcneill case EFBIG:
7768 1.6 nonaka if (m_defrag(m, M_DONTWAIT) != NULL &&
7769 1.1 jmcneill bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m,
7770 1.1 jmcneill BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0)
7771 1.1 jmcneill break;
7772 1.1 jmcneill
7773 1.1 jmcneill /* FALLTHROUGH */
7774 1.1 jmcneill default:
7775 1.1 jmcneill return (1);
7776 1.1 jmcneill }
7777 1.1 jmcneill
7778 1.1 jmcneill ms->ms_m = m;
7779 1.1 jmcneill return (0);
7780 1.1 jmcneill }
7781 1.1 jmcneill
7782 1.1 jmcneill static void
7783 1.16 jmcneill mcx_send_common_locked(struct ifnet *ifp, struct mcx_tx *tx, bool is_transmit)
7784 1.1 jmcneill {
7785 1.1 jmcneill struct mcx_softc *sc = ifp->if_softc;
7786 1.1 jmcneill struct mcx_sq_entry *sq, *sqe;
7787 1.1 jmcneill struct mcx_sq_entry_seg *sqs;
7788 1.1 jmcneill struct mcx_slot *ms;
7789 1.1 jmcneill bus_dmamap_t map;
7790 1.1 jmcneill struct mbuf *m;
7791 1.1 jmcneill u_int idx, free, used;
7792 1.1 jmcneill uint64_t *bf;
7793 1.15 jmcneill uint32_t csum;
7794 1.1 jmcneill size_t bf_base;
7795 1.1 jmcneill int i, seg, nseg;
7796 1.1 jmcneill
7797 1.16 jmcneill KASSERT(mutex_owned(&tx->tx_lock));
7798 1.16 jmcneill
7799 1.16 jmcneill if ((ifp->if_flags & IFF_RUNNING) == 0)
7800 1.16 jmcneill return;
7801 1.16 jmcneill
7802 1.15 jmcneill bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF;
7803 1.1 jmcneill
7804 1.15 jmcneill idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE);
7805 1.15 jmcneill free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod;
7806 1.1 jmcneill
7807 1.1 jmcneill used = 0;
7808 1.1 jmcneill bf = NULL;
7809 1.15 jmcneill
7810 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
7811 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_POSTWRITE);
7812 1.15 jmcneill
7813 1.15 jmcneill sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem);
7814 1.1 jmcneill
7815 1.1 jmcneill for (;;) {
7816 1.1 jmcneill if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) {
7817 1.1 jmcneill SET(ifp->if_flags, IFF_OACTIVE);
7818 1.1 jmcneill break;
7819 1.1 jmcneill }
7820 1.1 jmcneill
7821 1.16 jmcneill if (is_transmit) {
7822 1.16 jmcneill m = pcq_get(tx->tx_pcq);
7823 1.16 jmcneill } else {
7824 1.16 jmcneill IFQ_DEQUEUE(&ifp->if_snd, m);
7825 1.16 jmcneill }
7826 1.1 jmcneill if (m == NULL) {
7827 1.1 jmcneill break;
7828 1.1 jmcneill }
7829 1.1 jmcneill
7830 1.1 jmcneill sqe = sq + idx;
7831 1.15 jmcneill ms = &tx->tx_slots[idx];
7832 1.1 jmcneill memset(sqe, 0, sizeof(*sqe));
7833 1.1 jmcneill
7834 1.1 jmcneill /* ctrl segment */
7835 1.1 jmcneill sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND |
7836 1.15 jmcneill ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT));
7837 1.1 jmcneill /* always generate a completion event */
7838 1.1 jmcneill sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS);
7839 1.1 jmcneill
7840 1.1 jmcneill /* eth segment */
7841 1.15 jmcneill csum = 0;
7842 1.15 jmcneill if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
7843 1.15 jmcneill csum |= MCX_SQE_L3_CSUM;
7844 1.15 jmcneill if (m->m_pkthdr.csum_flags &
7845 1.15 jmcneill (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6))
7846 1.15 jmcneill csum |= MCX_SQE_L4_CSUM;
7847 1.15 jmcneill sqe->sqe_mss_csum = htobe32(csum);
7848 1.1 jmcneill sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE);
7849 1.15 jmcneill if (vlan_has_tag(m)) {
7850 1.15 jmcneill struct ether_vlan_header *evh;
7851 1.15 jmcneill evh = (struct ether_vlan_header *)
7852 1.15 jmcneill &sqe->sqe_inline_headers;
7853 1.15 jmcneill
7854 1.15 jmcneill m_copydata(m, 0, ETHER_HDR_LEN, evh);
7855 1.15 jmcneill evh->evl_proto = evh->evl_encap_proto;
7856 1.15 jmcneill evh->evl_encap_proto = htons(ETHERTYPE_VLAN);
7857 1.15 jmcneill evh->evl_tag = htons(vlan_get_tag(m));
7858 1.15 jmcneill m_adj(m, ETHER_HDR_LEN);
7859 1.15 jmcneill } else {
7860 1.15 jmcneill m_copydata(m, 0, MCX_SQ_INLINE_SIZE,
7861 1.15 jmcneill sqe->sqe_inline_headers);
7862 1.15 jmcneill m_adj(m, MCX_SQ_INLINE_SIZE);
7863 1.15 jmcneill }
7864 1.1 jmcneill
7865 1.1 jmcneill if (mcx_load_mbuf(sc, ms, m) != 0) {
7866 1.1 jmcneill m_freem(m);
7867 1.10 thorpej if_statinc(ifp, if_oerrors);
7868 1.1 jmcneill continue;
7869 1.1 jmcneill }
7870 1.1 jmcneill bf = (uint64_t *)sqe;
7871 1.1 jmcneill
7872 1.1 jmcneill if (ifp->if_bpf != NULL)
7873 1.1 jmcneill bpf_mtap2(ifp->if_bpf, sqe->sqe_inline_headers,
7874 1.1 jmcneill MCX_SQ_INLINE_SIZE, m, BPF_D_OUT);
7875 1.1 jmcneill
7876 1.1 jmcneill map = ms->ms_map;
7877 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
7878 1.1 jmcneill BUS_DMASYNC_PREWRITE);
7879 1.1 jmcneill
7880 1.1 jmcneill sqe->sqe_ds_sq_num =
7881 1.15 jmcneill htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) |
7882 1.1 jmcneill (map->dm_nsegs + 3));
7883 1.1 jmcneill
7884 1.1 jmcneill /* data segment - first wqe has one segment */
7885 1.1 jmcneill sqs = sqe->sqe_segs;
7886 1.1 jmcneill seg = 0;
7887 1.1 jmcneill nseg = 1;
7888 1.1 jmcneill for (i = 0; i < map->dm_nsegs; i++) {
7889 1.1 jmcneill if (seg == nseg) {
7890 1.1 jmcneill /* next slot */
7891 1.1 jmcneill idx++;
7892 1.1 jmcneill if (idx == (1 << MCX_LOG_SQ_SIZE))
7893 1.1 jmcneill idx = 0;
7894 1.15 jmcneill tx->tx_prod++;
7895 1.1 jmcneill used++;
7896 1.1 jmcneill
7897 1.1 jmcneill sqs = (struct mcx_sq_entry_seg *)(sq + idx);
7898 1.1 jmcneill seg = 0;
7899 1.1 jmcneill nseg = MCX_SQ_SEGS_PER_SLOT;
7900 1.1 jmcneill }
7901 1.1 jmcneill sqs[seg].sqs_byte_count =
7902 1.1 jmcneill htobe32(map->dm_segs[i].ds_len);
7903 1.1 jmcneill sqs[seg].sqs_lkey = htobe32(sc->sc_lkey);
7904 1.1 jmcneill sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr);
7905 1.1 jmcneill seg++;
7906 1.1 jmcneill }
7907 1.1 jmcneill
7908 1.1 jmcneill idx++;
7909 1.1 jmcneill if (idx == (1 << MCX_LOG_SQ_SIZE))
7910 1.1 jmcneill idx = 0;
7911 1.15 jmcneill tx->tx_prod++;
7912 1.1 jmcneill used++;
7913 1.1 jmcneill }
7914 1.1 jmcneill
7915 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&tx->tx_sq_mem),
7916 1.15 jmcneill 0, MCX_DMA_LEN(&tx->tx_sq_mem), BUS_DMASYNC_PREWRITE);
7917 1.15 jmcneill
7918 1.1 jmcneill if (used) {
7919 1.15 jmcneill bus_size_t blueflame;
7920 1.1 jmcneill
7921 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7922 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_POSTWRITE);
7923 1.15 jmcneill be32enc(MCX_DMA_OFF(&sc->sc_doorbell_mem, tx->tx_doorbell),
7924 1.15 jmcneill tx->tx_prod & MCX_WQ_DOORBELL_MASK);
7925 1.15 jmcneill bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_doorbell_mem),
7926 1.15 jmcneill tx->tx_doorbell, sizeof(uint32_t), BUS_DMASYNC_PREWRITE);
7927 1.1 jmcneill
7928 1.1 jmcneill /*
7929 1.1 jmcneill * write the first 64 bits of the last sqe we produced
7930 1.1 jmcneill * to the blue flame buffer
7931 1.1 jmcneill */
7932 1.15 jmcneill
7933 1.15 jmcneill blueflame = bf_base + tx->tx_bf_offset;
7934 1.1 jmcneill bus_space_write_8(sc->sc_memt, sc->sc_memh,
7935 1.15 jmcneill blueflame, *bf);
7936 1.15 jmcneill mcx_bar(sc, blueflame, sizeof(*bf), BUS_SPACE_BARRIER_WRITE);
7937 1.15 jmcneill
7938 1.1 jmcneill /* next write goes to the other buffer */
7939 1.15 jmcneill tx->tx_bf_offset ^= sc->sc_bf_size;
7940 1.1 jmcneill }
7941 1.1 jmcneill }
7942 1.1 jmcneill
7943 1.1 jmcneill static void
7944 1.16 jmcneill mcx_start(struct ifnet *ifp)
7945 1.16 jmcneill {
7946 1.16 jmcneill struct mcx_softc *sc = ifp->if_softc;
7947 1.16 jmcneill /* mcx_start() always uses TX ring[0] */
7948 1.16 jmcneill struct mcx_tx *tx = &sc->sc_queues[0].q_tx;
7949 1.16 jmcneill
7950 1.16 jmcneill mutex_enter(&tx->tx_lock);
7951 1.16 jmcneill if (!ISSET(ifp->if_flags, IFF_OACTIVE)) {
7952 1.16 jmcneill mcx_send_common_locked(ifp, tx, false);
7953 1.16 jmcneill }
7954 1.16 jmcneill mutex_exit(&tx->tx_lock);
7955 1.16 jmcneill }
7956 1.16 jmcneill
7957 1.16 jmcneill static int
7958 1.16 jmcneill mcx_transmit(struct ifnet *ifp, struct mbuf *m)
7959 1.16 jmcneill {
7960 1.16 jmcneill struct mcx_softc *sc = ifp->if_softc;
7961 1.16 jmcneill struct mcx_tx *tx;
7962 1.16 jmcneill
7963 1.16 jmcneill tx = &sc->sc_queues[cpu_index(curcpu()) % sc->sc_nqueues].q_tx;
7964 1.16 jmcneill if (__predict_false(!pcq_put(tx->tx_pcq, m))) {
7965 1.16 jmcneill m_freem(m);
7966 1.16 jmcneill return ENOBUFS;
7967 1.16 jmcneill }
7968 1.16 jmcneill
7969 1.16 jmcneill if (mutex_tryenter(&tx->tx_lock)) {
7970 1.16 jmcneill mcx_send_common_locked(ifp, tx, true);
7971 1.16 jmcneill mutex_exit(&tx->tx_lock);
7972 1.16 jmcneill } else {
7973 1.16 jmcneill softint_schedule(tx->tx_softint);
7974 1.16 jmcneill }
7975 1.16 jmcneill
7976 1.16 jmcneill return 0;
7977 1.16 jmcneill }
7978 1.16 jmcneill
7979 1.16 jmcneill static void
7980 1.16 jmcneill mcx_deferred_transmit(void *arg)
7981 1.16 jmcneill {
7982 1.16 jmcneill struct mcx_tx *tx = arg;
7983 1.16 jmcneill struct mcx_softc *sc = tx->tx_softc;
7984 1.16 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
7985 1.16 jmcneill
7986 1.16 jmcneill mutex_enter(&tx->tx_lock);
7987 1.16 jmcneill if (pcq_peek(tx->tx_pcq) != NULL) {
7988 1.16 jmcneill mcx_send_common_locked(ifp, tx, true);
7989 1.16 jmcneill }
7990 1.16 jmcneill mutex_exit(&tx->tx_lock);
7991 1.16 jmcneill }
7992 1.16 jmcneill
7993 1.16 jmcneill static void
7994 1.1 jmcneill mcx_watchdog(struct ifnet *ifp)
7995 1.1 jmcneill {
7996 1.1 jmcneill }
7997 1.1 jmcneill
7998 1.1 jmcneill static void
7999 1.1 jmcneill mcx_media_add_types(struct mcx_softc *sc)
8000 1.1 jmcneill {
8001 1.1 jmcneill struct mcx_reg_ptys ptys;
8002 1.1 jmcneill int i;
8003 1.1 jmcneill uint32_t proto_cap;
8004 1.1 jmcneill
8005 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8006 1.1 jmcneill ptys.rp_local_port = 1;
8007 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8008 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8009 1.1 jmcneill sizeof(ptys)) != 0) {
8010 1.1 jmcneill printf("%s: unable to read port type/speed\n", DEVNAME(sc));
8011 1.1 jmcneill return;
8012 1.1 jmcneill }
8013 1.1 jmcneill
8014 1.1 jmcneill proto_cap = be32toh(ptys.rp_eth_proto_cap);
8015 1.1 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8016 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8017 1.13 jmcneill if (!ISSET(proto_cap, 1U << i))
8018 1.13 jmcneill continue;
8019 1.13 jmcneill
8020 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8021 1.13 jmcneill if (cap->cap_media == 0)
8022 1.13 jmcneill continue;
8023 1.13 jmcneill
8024 1.13 jmcneill ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL);
8025 1.1 jmcneill }
8026 1.1 jmcneill }
8027 1.1 jmcneill
8028 1.1 jmcneill static void
8029 1.1 jmcneill mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
8030 1.1 jmcneill {
8031 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
8032 1.1 jmcneill struct mcx_reg_ptys ptys;
8033 1.1 jmcneill int i;
8034 1.13 jmcneill uint32_t proto_oper;
8035 1.1 jmcneill uint64_t media_oper;
8036 1.1 jmcneill
8037 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8038 1.1 jmcneill ptys.rp_local_port = 1;
8039 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8040 1.1 jmcneill
8041 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8042 1.1 jmcneill sizeof(ptys)) != 0) {
8043 1.1 jmcneill printf("%s: unable to read port type/speed\n", DEVNAME(sc));
8044 1.1 jmcneill return;
8045 1.1 jmcneill }
8046 1.1 jmcneill
8047 1.1 jmcneill proto_oper = be32toh(ptys.rp_eth_proto_oper);
8048 1.1 jmcneill
8049 1.1 jmcneill media_oper = 0;
8050 1.13 jmcneill
8051 1.1 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8052 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8053 1.13 jmcneill if (!ISSET(proto_oper, 1U << i))
8054 1.13 jmcneill continue;
8055 1.13 jmcneill
8056 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8057 1.13 jmcneill
8058 1.13 jmcneill if (cap->cap_media != 0)
8059 1.13 jmcneill media_oper = cap->cap_media;
8060 1.1 jmcneill }
8061 1.1 jmcneill
8062 1.1 jmcneill ifmr->ifm_status = IFM_AVALID;
8063 1.1 jmcneill if (proto_oper != 0) {
8064 1.1 jmcneill ifmr->ifm_status |= IFM_ACTIVE;
8065 1.1 jmcneill ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper;
8066 1.1 jmcneill /* txpause, rxpause, duplex? */
8067 1.1 jmcneill }
8068 1.1 jmcneill }
8069 1.1 jmcneill
8070 1.1 jmcneill static int
8071 1.1 jmcneill mcx_media_change(struct ifnet *ifp)
8072 1.1 jmcneill {
8073 1.1 jmcneill struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc;
8074 1.1 jmcneill struct mcx_reg_ptys ptys;
8075 1.1 jmcneill struct mcx_reg_paos paos;
8076 1.1 jmcneill uint32_t media;
8077 1.1 jmcneill int i, error;
8078 1.1 jmcneill
8079 1.1 jmcneill if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
8080 1.1 jmcneill return EINVAL;
8081 1.1 jmcneill
8082 1.1 jmcneill error = 0;
8083 1.1 jmcneill
8084 1.1 jmcneill if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) {
8085 1.1 jmcneill /* read ptys to get supported media */
8086 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8087 1.1 jmcneill ptys.rp_local_port = 1;
8088 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8089 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ,
8090 1.1 jmcneill &ptys, sizeof(ptys)) != 0) {
8091 1.1 jmcneill printf("%s: unable to read port type/speed\n",
8092 1.1 jmcneill DEVNAME(sc));
8093 1.1 jmcneill return EIO;
8094 1.1 jmcneill }
8095 1.1 jmcneill
8096 1.1 jmcneill media = be32toh(ptys.rp_eth_proto_cap);
8097 1.1 jmcneill } else {
8098 1.1 jmcneill /* map media type */
8099 1.1 jmcneill media = 0;
8100 1.1 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8101 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8102 1.13 jmcneill
8103 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8104 1.13 jmcneill if (cap->cap_media ==
8105 1.1 jmcneill IFM_SUBTYPE(sc->sc_media.ifm_media)) {
8106 1.1 jmcneill media = (1 << i);
8107 1.1 jmcneill break;
8108 1.1 jmcneill }
8109 1.1 jmcneill }
8110 1.1 jmcneill }
8111 1.1 jmcneill
8112 1.1 jmcneill /* disable the port */
8113 1.1 jmcneill memset(&paos, 0, sizeof(paos));
8114 1.1 jmcneill paos.rp_local_port = 1;
8115 1.1 jmcneill paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN;
8116 1.1 jmcneill paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
8117 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
8118 1.1 jmcneill sizeof(paos)) != 0) {
8119 1.1 jmcneill printf("%s: unable to set port state to down\n", DEVNAME(sc));
8120 1.1 jmcneill return EIO;
8121 1.1 jmcneill }
8122 1.1 jmcneill
8123 1.1 jmcneill memset(&ptys, 0, sizeof(ptys));
8124 1.1 jmcneill ptys.rp_local_port = 1;
8125 1.1 jmcneill ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH;
8126 1.1 jmcneill ptys.rp_eth_proto_admin = htobe32(media);
8127 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys,
8128 1.1 jmcneill sizeof(ptys)) != 0) {
8129 1.1 jmcneill printf("%s: unable to set port media type/speed\n",
8130 1.1 jmcneill DEVNAME(sc));
8131 1.1 jmcneill error = EIO;
8132 1.1 jmcneill }
8133 1.1 jmcneill
8134 1.1 jmcneill /* re-enable the port to start negotiation */
8135 1.1 jmcneill memset(&paos, 0, sizeof(paos));
8136 1.1 jmcneill paos.rp_local_port = 1;
8137 1.1 jmcneill paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP;
8138 1.1 jmcneill paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN;
8139 1.1 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos,
8140 1.1 jmcneill sizeof(paos)) != 0) {
8141 1.1 jmcneill printf("%s: unable to set port state to up\n", DEVNAME(sc));
8142 1.1 jmcneill error = EIO;
8143 1.1 jmcneill }
8144 1.1 jmcneill
8145 1.1 jmcneill return error;
8146 1.1 jmcneill }
8147 1.1 jmcneill
8148 1.1 jmcneill static void
8149 1.1 jmcneill mcx_port_change(struct work *wk, void *xsc)
8150 1.1 jmcneill {
8151 1.1 jmcneill struct mcx_softc *sc = xsc;
8152 1.1 jmcneill struct ifnet *ifp = &sc->sc_ec.ec_if;
8153 1.13 jmcneill struct mcx_reg_ptys ptys = {
8154 1.13 jmcneill .rp_local_port = 1,
8155 1.13 jmcneill .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH,
8156 1.13 jmcneill };
8157 1.1 jmcneill int link_state = LINK_STATE_DOWN;
8158 1.1 jmcneill
8159 1.13 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys,
8160 1.13 jmcneill sizeof(ptys)) == 0) {
8161 1.13 jmcneill uint32_t proto_oper = be32toh(ptys.rp_eth_proto_oper);
8162 1.13 jmcneill uint64_t baudrate = 0;
8163 1.13 jmcneill unsigned int i;
8164 1.13 jmcneill
8165 1.15 jmcneill if (proto_oper != 0)
8166 1.15 jmcneill link_state = LINK_STATE_UP;
8167 1.15 jmcneill
8168 1.13 jmcneill for (i = 0; i < __arraycount(mcx_eth_cap_map); i++) {
8169 1.13 jmcneill const struct mcx_eth_proto_capability *cap;
8170 1.13 jmcneill if (!ISSET(proto_oper, 1U << i))
8171 1.13 jmcneill continue;
8172 1.13 jmcneill
8173 1.13 jmcneill cap = &mcx_eth_cap_map[i];
8174 1.13 jmcneill if (cap->cap_baudrate == 0)
8175 1.13 jmcneill continue;
8176 1.13 jmcneill
8177 1.13 jmcneill baudrate = cap->cap_baudrate;
8178 1.13 jmcneill break;
8179 1.13 jmcneill }
8180 1.13 jmcneill
8181 1.13 jmcneill ifp->if_baudrate = baudrate;
8182 1.1 jmcneill }
8183 1.1 jmcneill
8184 1.1 jmcneill if (link_state != ifp->if_link_state) {
8185 1.1 jmcneill if_link_state_change(ifp, link_state);
8186 1.1 jmcneill }
8187 1.1 jmcneill }
8188 1.1 jmcneill
8189 1.1 jmcneill
8190 1.1 jmcneill static inline uint32_t
8191 1.1 jmcneill mcx_rd(struct mcx_softc *sc, bus_size_t r)
8192 1.1 jmcneill {
8193 1.1 jmcneill uint32_t word;
8194 1.1 jmcneill
8195 1.1 jmcneill word = bus_space_read_4(sc->sc_memt, sc->sc_memh, r);
8196 1.1 jmcneill
8197 1.1 jmcneill return (be32toh(word));
8198 1.1 jmcneill }
8199 1.1 jmcneill
8200 1.1 jmcneill static inline void
8201 1.1 jmcneill mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v)
8202 1.1 jmcneill {
8203 1.1 jmcneill bus_space_write_4(sc->sc_memt, sc->sc_memh, r, htobe32(v));
8204 1.1 jmcneill }
8205 1.1 jmcneill
8206 1.1 jmcneill static inline void
8207 1.1 jmcneill mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f)
8208 1.1 jmcneill {
8209 1.1 jmcneill bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f);
8210 1.1 jmcneill }
8211 1.1 jmcneill
8212 1.1 jmcneill static uint64_t
8213 1.1 jmcneill mcx_timer(struct mcx_softc *sc)
8214 1.1 jmcneill {
8215 1.1 jmcneill uint32_t hi, lo, ni;
8216 1.1 jmcneill
8217 1.1 jmcneill hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
8218 1.1 jmcneill for (;;) {
8219 1.1 jmcneill lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L);
8220 1.1 jmcneill mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ);
8221 1.1 jmcneill ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H);
8222 1.1 jmcneill
8223 1.1 jmcneill if (ni == hi)
8224 1.1 jmcneill break;
8225 1.1 jmcneill
8226 1.1 jmcneill hi = ni;
8227 1.1 jmcneill }
8228 1.1 jmcneill
8229 1.1 jmcneill return (((uint64_t)hi << 32) | (uint64_t)lo);
8230 1.1 jmcneill }
8231 1.1 jmcneill
8232 1.1 jmcneill static int
8233 1.1 jmcneill mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm,
8234 1.1 jmcneill bus_size_t size, u_int align)
8235 1.1 jmcneill {
8236 1.1 jmcneill mxm->mxm_size = size;
8237 1.1 jmcneill
8238 1.1 jmcneill if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1,
8239 1.1 jmcneill mxm->mxm_size, 0,
8240 1.1 jmcneill BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
8241 1.1 jmcneill &mxm->mxm_map) != 0)
8242 1.1 jmcneill return (1);
8243 1.1 jmcneill if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size,
8244 1.1 jmcneill align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs,
8245 1.1 jmcneill BUS_DMA_WAITOK) != 0)
8246 1.1 jmcneill goto destroy;
8247 1.1 jmcneill if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs,
8248 1.1 jmcneill mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0)
8249 1.1 jmcneill goto free;
8250 1.1 jmcneill if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva,
8251 1.1 jmcneill mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0)
8252 1.1 jmcneill goto unmap;
8253 1.1 jmcneill
8254 1.1 jmcneill mcx_dmamem_zero(mxm);
8255 1.1 jmcneill
8256 1.1 jmcneill return (0);
8257 1.1 jmcneill unmap:
8258 1.1 jmcneill bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
8259 1.1 jmcneill free:
8260 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
8261 1.1 jmcneill destroy:
8262 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
8263 1.1 jmcneill return (1);
8264 1.1 jmcneill }
8265 1.1 jmcneill
8266 1.1 jmcneill static void
8267 1.1 jmcneill mcx_dmamem_zero(struct mcx_dmamem *mxm)
8268 1.1 jmcneill {
8269 1.1 jmcneill memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm));
8270 1.1 jmcneill }
8271 1.1 jmcneill
8272 1.1 jmcneill static void
8273 1.1 jmcneill mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm)
8274 1.1 jmcneill {
8275 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map);
8276 1.1 jmcneill bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size);
8277 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1);
8278 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map);
8279 1.1 jmcneill }
8280 1.1 jmcneill
8281 1.1 jmcneill static int
8282 1.1 jmcneill mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages)
8283 1.1 jmcneill {
8284 1.1 jmcneill bus_dma_segment_t *segs;
8285 1.1 jmcneill bus_size_t len = pages * MCX_PAGE_SIZE;
8286 1.1 jmcneill size_t seglen;
8287 1.1 jmcneill
8288 1.1 jmcneill segs = kmem_alloc(sizeof(*segs) * pages, KM_SLEEP);
8289 1.1 jmcneill seglen = sizeof(*segs) * pages;
8290 1.1 jmcneill
8291 1.1 jmcneill if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0,
8292 1.1 jmcneill segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0)
8293 1.1 jmcneill goto free_segs;
8294 1.1 jmcneill
8295 1.1 jmcneill if (mhm->mhm_seg_count < pages) {
8296 1.1 jmcneill size_t nseglen;
8297 1.1 jmcneill
8298 1.1 jmcneill mhm->mhm_segs = kmem_alloc(
8299 1.1 jmcneill sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count, KM_SLEEP);
8300 1.1 jmcneill
8301 1.1 jmcneill nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count;
8302 1.1 jmcneill
8303 1.1 jmcneill memcpy(mhm->mhm_segs, segs, nseglen);
8304 1.1 jmcneill
8305 1.1 jmcneill kmem_free(segs, seglen);
8306 1.1 jmcneill
8307 1.1 jmcneill segs = mhm->mhm_segs;
8308 1.1 jmcneill seglen = nseglen;
8309 1.1 jmcneill } else
8310 1.1 jmcneill mhm->mhm_segs = segs;
8311 1.1 jmcneill
8312 1.1 jmcneill if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE,
8313 1.1 jmcneill MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/,
8314 1.1 jmcneill &mhm->mhm_map) != 0)
8315 1.1 jmcneill goto free_dmamem;
8316 1.1 jmcneill
8317 1.1 jmcneill if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map,
8318 1.1 jmcneill mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0)
8319 1.1 jmcneill goto destroy;
8320 1.1 jmcneill
8321 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
8322 1.1 jmcneill 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW);
8323 1.1 jmcneill
8324 1.1 jmcneill mhm->mhm_npages = pages;
8325 1.1 jmcneill
8326 1.1 jmcneill return (0);
8327 1.1 jmcneill
8328 1.1 jmcneill destroy:
8329 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
8330 1.1 jmcneill free_dmamem:
8331 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
8332 1.1 jmcneill free_segs:
8333 1.1 jmcneill kmem_free(segs, seglen);
8334 1.1 jmcneill mhm->mhm_segs = NULL;
8335 1.1 jmcneill
8336 1.1 jmcneill return (-1);
8337 1.1 jmcneill }
8338 1.1 jmcneill
8339 1.1 jmcneill static void
8340 1.1 jmcneill mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm)
8341 1.1 jmcneill {
8342 1.1 jmcneill if (mhm->mhm_npages == 0)
8343 1.1 jmcneill return;
8344 1.1 jmcneill
8345 1.1 jmcneill bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map,
8346 1.1 jmcneill 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW);
8347 1.1 jmcneill
8348 1.1 jmcneill bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map);
8349 1.1 jmcneill bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map);
8350 1.1 jmcneill bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count);
8351 1.1 jmcneill kmem_free(mhm->mhm_segs, sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count);
8352 1.1 jmcneill
8353 1.1 jmcneill mhm->mhm_npages = 0;
8354 1.1 jmcneill }
8355 1.15 jmcneill
8356 1.15 jmcneill #if NKSTAT > 0
8357 1.15 jmcneill struct mcx_ppcnt {
8358 1.15 jmcneill char name[KSTAT_KV_NAMELEN];
8359 1.15 jmcneill enum kstat_kv_unit unit;
8360 1.15 jmcneill };
8361 1.15 jmcneill
8362 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = {
8363 1.15 jmcneill { "Good Tx", KSTAT_KV_U_PACKETS, },
8364 1.15 jmcneill { "Good Rx", KSTAT_KV_U_PACKETS, },
8365 1.15 jmcneill { "FCS errs", KSTAT_KV_U_PACKETS, },
8366 1.15 jmcneill { "Alignment Errs", KSTAT_KV_U_PACKETS, },
8367 1.15 jmcneill { "Good Tx", KSTAT_KV_U_BYTES, },
8368 1.15 jmcneill { "Good Rx", KSTAT_KV_U_BYTES, },
8369 1.15 jmcneill { "Multicast Tx", KSTAT_KV_U_PACKETS, },
8370 1.15 jmcneill { "Broadcast Tx", KSTAT_KV_U_PACKETS, },
8371 1.15 jmcneill { "Multicast Rx", KSTAT_KV_U_PACKETS, },
8372 1.15 jmcneill { "Broadcast Rx", KSTAT_KV_U_PACKETS, },
8373 1.15 jmcneill { "In Range Len", KSTAT_KV_U_PACKETS, },
8374 1.15 jmcneill { "Out Of Range Len", KSTAT_KV_U_PACKETS, },
8375 1.15 jmcneill { "Frame Too Long", KSTAT_KV_U_PACKETS, },
8376 1.15 jmcneill { "Symbol Errs", KSTAT_KV_U_PACKETS, },
8377 1.15 jmcneill { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, },
8378 1.15 jmcneill { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, },
8379 1.15 jmcneill { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, },
8380 1.15 jmcneill { "Pause Rx", KSTAT_KV_U_PACKETS, },
8381 1.15 jmcneill { "Pause Tx", KSTAT_KV_U_PACKETS, },
8382 1.15 jmcneill };
8383 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count);
8384 1.15 jmcneill
8385 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = {
8386 1.15 jmcneill { "Rx Bytes", KSTAT_KV_U_BYTES, },
8387 1.15 jmcneill { "Rx Unicast", KSTAT_KV_U_PACKETS, },
8388 1.15 jmcneill { "Rx Discards", KSTAT_KV_U_PACKETS, },
8389 1.15 jmcneill { "Rx Errors", KSTAT_KV_U_PACKETS, },
8390 1.15 jmcneill { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, },
8391 1.15 jmcneill { "Tx Bytes", KSTAT_KV_U_BYTES, },
8392 1.15 jmcneill { "Tx Unicast", KSTAT_KV_U_PACKETS, },
8393 1.15 jmcneill { "Tx Discards", KSTAT_KV_U_PACKETS, },
8394 1.15 jmcneill { "Tx Errors", KSTAT_KV_U_PACKETS, },
8395 1.15 jmcneill { "Rx Multicast", KSTAT_KV_U_PACKETS, },
8396 1.15 jmcneill { "Rx Broadcast", KSTAT_KV_U_PACKETS, },
8397 1.15 jmcneill { "Tx Multicast", KSTAT_KV_U_PACKETS, },
8398 1.15 jmcneill { "Tx Broadcast", KSTAT_KV_U_PACKETS, },
8399 1.15 jmcneill };
8400 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count);
8401 1.15 jmcneill
8402 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = {
8403 1.15 jmcneill { "Drop Events", KSTAT_KV_U_PACKETS, },
8404 1.15 jmcneill { "Octets", KSTAT_KV_U_BYTES, },
8405 1.15 jmcneill { "Packets", KSTAT_KV_U_PACKETS, },
8406 1.15 jmcneill { "Broadcasts", KSTAT_KV_U_PACKETS, },
8407 1.15 jmcneill { "Multicasts", KSTAT_KV_U_PACKETS, },
8408 1.15 jmcneill { "CRC Align Errs", KSTAT_KV_U_PACKETS, },
8409 1.15 jmcneill { "Undersize", KSTAT_KV_U_PACKETS, },
8410 1.15 jmcneill { "Oversize", KSTAT_KV_U_PACKETS, },
8411 1.15 jmcneill { "Fragments", KSTAT_KV_U_PACKETS, },
8412 1.15 jmcneill { "Jabbers", KSTAT_KV_U_PACKETS, },
8413 1.15 jmcneill { "Collisions", KSTAT_KV_U_NONE, },
8414 1.15 jmcneill { "64B", KSTAT_KV_U_PACKETS, },
8415 1.15 jmcneill { "65-127B", KSTAT_KV_U_PACKETS, },
8416 1.15 jmcneill { "128-255B", KSTAT_KV_U_PACKETS, },
8417 1.15 jmcneill { "256-511B", KSTAT_KV_U_PACKETS, },
8418 1.15 jmcneill { "512-1023B", KSTAT_KV_U_PACKETS, },
8419 1.15 jmcneill { "1024-1518B", KSTAT_KV_U_PACKETS, },
8420 1.15 jmcneill { "1519-2047B", KSTAT_KV_U_PACKETS, },
8421 1.15 jmcneill { "2048-4095B", KSTAT_KV_U_PACKETS, },
8422 1.15 jmcneill { "4096-8191B", KSTAT_KV_U_PACKETS, },
8423 1.15 jmcneill { "8192-10239B", KSTAT_KV_U_PACKETS, },
8424 1.15 jmcneill };
8425 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count);
8426 1.15 jmcneill
8427 1.15 jmcneill static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = {
8428 1.15 jmcneill { "Alignment Errs", KSTAT_KV_U_PACKETS, },
8429 1.15 jmcneill { "FCS Errs", KSTAT_KV_U_PACKETS, },
8430 1.15 jmcneill { "Single Colls", KSTAT_KV_U_PACKETS, },
8431 1.15 jmcneill { "Multiple Colls", KSTAT_KV_U_PACKETS, },
8432 1.15 jmcneill { "SQE Test Errs", KSTAT_KV_U_NONE, },
8433 1.15 jmcneill { "Deferred Tx", KSTAT_KV_U_PACKETS, },
8434 1.15 jmcneill { "Late Colls", KSTAT_KV_U_NONE, },
8435 1.15 jmcneill { "Exess Colls", KSTAT_KV_U_NONE, },
8436 1.15 jmcneill { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, },
8437 1.15 jmcneill { "CSM Sense Errs", KSTAT_KV_U_NONE, },
8438 1.15 jmcneill { "Too Long", KSTAT_KV_U_PACKETS, },
8439 1.15 jmcneill { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, },
8440 1.15 jmcneill { "Symbol Errs", KSTAT_KV_U_NONE, },
8441 1.15 jmcneill { "Unknown Control", KSTAT_KV_U_PACKETS, },
8442 1.15 jmcneill { "Pause Rx", KSTAT_KV_U_PACKETS, },
8443 1.15 jmcneill { "Pause Tx", KSTAT_KV_U_PACKETS, },
8444 1.15 jmcneill };
8445 1.15 jmcneill CTASSERT(__arraycount(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count);
8446 1.15 jmcneill
8447 1.15 jmcneill struct mcx_kstat_ppcnt {
8448 1.15 jmcneill const char *ksp_name;
8449 1.15 jmcneill const struct mcx_ppcnt *ksp_tpl;
8450 1.15 jmcneill unsigned int ksp_n;
8451 1.15 jmcneill uint8_t ksp_grp;
8452 1.15 jmcneill };
8453 1.15 jmcneill
8454 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = {
8455 1.15 jmcneill .ksp_name = "ieee802.3",
8456 1.15 jmcneill .ksp_tpl = mcx_ppcnt_ieee8023_tpl,
8457 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_ieee8023_tpl),
8458 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023,
8459 1.15 jmcneill };
8460 1.15 jmcneill
8461 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = {
8462 1.15 jmcneill .ksp_name = "rfc2863",
8463 1.15 jmcneill .ksp_tpl = mcx_ppcnt_rfc2863_tpl,
8464 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_rfc2863_tpl),
8465 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863,
8466 1.15 jmcneill };
8467 1.15 jmcneill
8468 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = {
8469 1.15 jmcneill .ksp_name = "rfc2819",
8470 1.15 jmcneill .ksp_tpl = mcx_ppcnt_rfc2819_tpl,
8471 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_rfc2819_tpl),
8472 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819,
8473 1.15 jmcneill };
8474 1.15 jmcneill
8475 1.15 jmcneill static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = {
8476 1.15 jmcneill .ksp_name = "rfc3635",
8477 1.15 jmcneill .ksp_tpl = mcx_ppcnt_rfc3635_tpl,
8478 1.15 jmcneill .ksp_n = __arraycount(mcx_ppcnt_rfc3635_tpl),
8479 1.15 jmcneill .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635,
8480 1.15 jmcneill };
8481 1.15 jmcneill
8482 1.15 jmcneill static int mcx_kstat_ppcnt_read(struct kstat *);
8483 1.15 jmcneill
8484 1.15 jmcneill static void mcx_kstat_attach_tmps(struct mcx_softc *sc);
8485 1.15 jmcneill static void mcx_kstat_attach_queues(struct mcx_softc *sc);
8486 1.15 jmcneill
8487 1.15 jmcneill static struct kstat *
8488 1.15 jmcneill mcx_kstat_attach_ppcnt(struct mcx_softc *sc,
8489 1.15 jmcneill const struct mcx_kstat_ppcnt *ksp)
8490 1.15 jmcneill {
8491 1.15 jmcneill struct kstat *ks;
8492 1.15 jmcneill struct kstat_kv *kvs;
8493 1.15 jmcneill unsigned int i;
8494 1.15 jmcneill
8495 1.15 jmcneill ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0);
8496 1.15 jmcneill if (ks == NULL)
8497 1.15 jmcneill return (NULL);
8498 1.15 jmcneill
8499 1.15 jmcneill kvs = mallocarray(ksp->ksp_n, sizeof(*kvs),
8500 1.15 jmcneill M_DEVBUF, M_WAITOK);
8501 1.15 jmcneill
8502 1.15 jmcneill for (i = 0; i < ksp->ksp_n; i++) {
8503 1.15 jmcneill const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i];
8504 1.15 jmcneill
8505 1.15 jmcneill kstat_kv_unit_init(&kvs[i], tpl->name,
8506 1.15 jmcneill KSTAT_KV_T_COUNTER64, tpl->unit);
8507 1.15 jmcneill }
8508 1.15 jmcneill
8509 1.15 jmcneill ks->ks_softc = sc;
8510 1.15 jmcneill ks->ks_ptr = (void *)ksp;
8511 1.15 jmcneill ks->ks_data = kvs;
8512 1.15 jmcneill ks->ks_datalen = ksp->ksp_n * sizeof(*kvs);
8513 1.15 jmcneill ks->ks_read = mcx_kstat_ppcnt_read;
8514 1.15 jmcneill
8515 1.15 jmcneill kstat_install(ks);
8516 1.15 jmcneill
8517 1.15 jmcneill return (ks);
8518 1.15 jmcneill }
8519 1.15 jmcneill
8520 1.15 jmcneill static void
8521 1.15 jmcneill mcx_kstat_attach(struct mcx_softc *sc)
8522 1.15 jmcneill {
8523 1.15 jmcneill sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc,
8524 1.15 jmcneill &mcx_kstat_ppcnt_ieee8023);
8525 1.15 jmcneill sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc,
8526 1.15 jmcneill &mcx_kstat_ppcnt_rfc2863);
8527 1.15 jmcneill sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc,
8528 1.15 jmcneill &mcx_kstat_ppcnt_rfc2819);
8529 1.15 jmcneill sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc,
8530 1.15 jmcneill &mcx_kstat_ppcnt_rfc3635);
8531 1.15 jmcneill
8532 1.15 jmcneill mcx_kstat_attach_tmps(sc);
8533 1.15 jmcneill mcx_kstat_attach_queues(sc);
8534 1.15 jmcneill }
8535 1.15 jmcneill
8536 1.15 jmcneill static int
8537 1.15 jmcneill mcx_kstat_ppcnt_read(struct kstat *ks)
8538 1.15 jmcneill {
8539 1.15 jmcneill struct mcx_softc *sc = ks->ks_softc;
8540 1.15 jmcneill struct mcx_kstat_ppcnt *ksp = ks->ks_ptr;
8541 1.15 jmcneill struct mcx_reg_ppcnt ppcnt = {
8542 1.15 jmcneill .ppcnt_grp = ksp->ksp_grp,
8543 1.15 jmcneill .ppcnt_local_port = 1,
8544 1.15 jmcneill };
8545 1.15 jmcneill struct kstat_kv *kvs = ks->ks_data;
8546 1.15 jmcneill uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set;
8547 1.15 jmcneill unsigned int i;
8548 1.15 jmcneill int rv;
8549 1.15 jmcneill
8550 1.15 jmcneill KERNEL_LOCK(); /* XXX */
8551 1.15 jmcneill rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ,
8552 1.15 jmcneill &ppcnt, sizeof(ppcnt));
8553 1.15 jmcneill KERNEL_UNLOCK();
8554 1.15 jmcneill if (rv != 0)
8555 1.15 jmcneill return (EIO);
8556 1.15 jmcneill
8557 1.15 jmcneill nanouptime(&ks->ks_updated);
8558 1.15 jmcneill
8559 1.15 jmcneill for (i = 0; i < ksp->ksp_n; i++)
8560 1.15 jmcneill kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]);
8561 1.15 jmcneill
8562 1.15 jmcneill return (0);
8563 1.15 jmcneill }
8564 1.15 jmcneill
8565 1.15 jmcneill struct mcx_kstat_mtmp {
8566 1.15 jmcneill struct kstat_kv ktmp_name;
8567 1.15 jmcneill struct kstat_kv ktmp_temperature;
8568 1.15 jmcneill struct kstat_kv ktmp_threshold_lo;
8569 1.15 jmcneill struct kstat_kv ktmp_threshold_hi;
8570 1.15 jmcneill };
8571 1.15 jmcneill
8572 1.15 jmcneill static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = {
8573 1.15 jmcneill KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR),
8574 1.15 jmcneill KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP),
8575 1.15 jmcneill KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP),
8576 1.15 jmcneill KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP),
8577 1.15 jmcneill };
8578 1.15 jmcneill
8579 1.15 jmcneill static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 };
8580 1.15 jmcneill
8581 1.15 jmcneill static int mcx_kstat_mtmp_read(struct kstat *);
8582 1.15 jmcneill
8583 1.15 jmcneill static void
8584 1.15 jmcneill mcx_kstat_attach_tmps(struct mcx_softc *sc)
8585 1.15 jmcneill {
8586 1.15 jmcneill struct kstat *ks;
8587 1.15 jmcneill struct mcx_reg_mcam mcam;
8588 1.15 jmcneill struct mcx_reg_mtcap mtcap;
8589 1.15 jmcneill struct mcx_kstat_mtmp *ktmp;
8590 1.15 jmcneill uint64_t map;
8591 1.15 jmcneill unsigned int i, n;
8592 1.15 jmcneill
8593 1.15 jmcneill memset(&mtcap, 0, sizeof(mtcap));
8594 1.15 jmcneill memset(&mcam, 0, sizeof(mcam));
8595 1.15 jmcneill
8596 1.15 jmcneill if (sc->sc_mcam_reg == 0) {
8597 1.15 jmcneill /* no management capabilities */
8598 1.15 jmcneill return;
8599 1.15 jmcneill }
8600 1.15 jmcneill
8601 1.15 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_MCAM, MCX_REG_OP_READ,
8602 1.15 jmcneill &mcam, sizeof(mcam)) != 0) {
8603 1.15 jmcneill /* unable to check management capabilities? */
8604 1.15 jmcneill return;
8605 1.15 jmcneill }
8606 1.15 jmcneill
8607 1.15 jmcneill if (MCX_BITFIELD_BIT(mcam.mcam_feature_cap_mask,
8608 1.15 jmcneill MCX_MCAM_FEATURE_CAP_SENSOR_MAP) == 0) {
8609 1.15 jmcneill /* no sensor map */
8610 1.15 jmcneill return;
8611 1.15 jmcneill }
8612 1.15 jmcneill
8613 1.15 jmcneill if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ,
8614 1.15 jmcneill &mtcap, sizeof(mtcap)) != 0) {
8615 1.15 jmcneill /* unable to find temperature sensors */
8616 1.15 jmcneill return;
8617 1.15 jmcneill }
8618 1.15 jmcneill
8619 1.15 jmcneill sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count;
8620 1.15 jmcneill sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count,
8621 1.15 jmcneill sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK);
8622 1.15 jmcneill
8623 1.15 jmcneill n = 0;
8624 1.15 jmcneill map = bemtoh64(&mtcap.mtcap_sensor_map);
8625 1.15 jmcneill for (i = 0; i < sizeof(map) * NBBY; i++) {
8626 1.15 jmcneill if (!ISSET(map, (1ULL << i)))
8627 1.15 jmcneill continue;
8628 1.15 jmcneill
8629 1.15 jmcneill ks = kstat_create(DEVNAME(sc), 0, "temperature", i,
8630 1.15 jmcneill KSTAT_T_KV, 0);
8631 1.15 jmcneill if (ks == NULL) {
8632 1.15 jmcneill /* unable to attach temperature sensor %u, i */
8633 1.15 jmcneill continue;
8634 1.15 jmcneill }
8635 1.15 jmcneill
8636 1.15 jmcneill ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO);
8637 1.15 jmcneill *ktmp = mcx_kstat_mtmp_tpl;
8638 1.15 jmcneill
8639 1.15 jmcneill ks->ks_data = ktmp;
8640 1.15 jmcneill ks->ks_datalen = sizeof(*ktmp);
8641 1.15 jmcneill TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval);
8642 1.15 jmcneill ks->ks_read = mcx_kstat_mtmp_read;
8643 1.15 jmcneill
8644 1.15 jmcneill ks->ks_softc = sc;
8645 1.15 jmcneill kstat_install(ks);
8646 1.15 jmcneill
8647 1.15 jmcneill sc->sc_kstat_mtmp[n++] = ks;
8648 1.15 jmcneill if (n >= sc->sc_kstat_mtmp_count)
8649 1.15 jmcneill break;
8650 1.15 jmcneill }
8651 1.15 jmcneill }
8652 1.15 jmcneill
8653 1.15 jmcneill static uint64_t
8654 1.15 jmcneill mcx_tmp_to_uK(uint16_t *t)
8655 1.15 jmcneill {
8656 1.15 jmcneill int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */
8657 1.15 jmcneill mt *= 1000000 / 8; /* convert to uC */
8658 1.15 jmcneill mt += 273150000; /* convert to uK */
8659 1.15 jmcneill
8660 1.15 jmcneill return (mt);
8661 1.15 jmcneill }
8662 1.15 jmcneill
8663 1.15 jmcneill static int
8664 1.15 jmcneill mcx_kstat_mtmp_read(struct kstat *ks)
8665 1.15 jmcneill {
8666 1.15 jmcneill struct mcx_softc *sc = ks->ks_softc;
8667 1.15 jmcneill struct mcx_kstat_mtmp *ktmp = ks->ks_data;
8668 1.15 jmcneill struct mcx_reg_mtmp mtmp;
8669 1.15 jmcneill int rv;
8670 1.15 jmcneill struct timeval updated;
8671 1.15 jmcneill
8672 1.15 jmcneill TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated);
8673 1.15 jmcneill
8674 1.15 jmcneill if (!ratecheck(&updated, &mcx_kstat_mtmp_rate))
8675 1.15 jmcneill return (0);
8676 1.15 jmcneill
8677 1.15 jmcneill memset(&mtmp, 0, sizeof(mtmp));
8678 1.15 jmcneill htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit);
8679 1.15 jmcneill
8680 1.15 jmcneill KERNEL_LOCK(); /* XXX */
8681 1.15 jmcneill rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ,
8682 1.15 jmcneill &mtmp, sizeof(mtmp));
8683 1.15 jmcneill KERNEL_UNLOCK();
8684 1.15 jmcneill if (rv != 0)
8685 1.15 jmcneill return (EIO);
8686 1.15 jmcneill
8687 1.15 jmcneill memset(kstat_kv_istr(&ktmp->ktmp_name), 0,
8688 1.15 jmcneill sizeof(kstat_kv_istr(&ktmp->ktmp_name)));
8689 1.15 jmcneill memcpy(kstat_kv_istr(&ktmp->ktmp_name),
8690 1.15 jmcneill mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name));
8691 1.15 jmcneill kstat_kv_temp(&ktmp->ktmp_temperature) =
8692 1.15 jmcneill mcx_tmp_to_uK(&mtmp.mtmp_temperature);
8693 1.15 jmcneill kstat_kv_temp(&ktmp->ktmp_threshold_lo) =
8694 1.15 jmcneill mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo);
8695 1.15 jmcneill kstat_kv_temp(&ktmp->ktmp_threshold_hi) =
8696 1.15 jmcneill mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi);
8697 1.15 jmcneill
8698 1.15 jmcneill TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated);
8699 1.15 jmcneill
8700 1.15 jmcneill return (0);
8701 1.15 jmcneill }
8702 1.15 jmcneill
8703 1.15 jmcneill struct mcx_queuestat {
8704 1.15 jmcneill char name[KSTAT_KV_NAMELEN];
8705 1.15 jmcneill enum kstat_kv_type type;
8706 1.15 jmcneill };
8707 1.15 jmcneill
8708 1.15 jmcneill static const struct mcx_queuestat mcx_queue_kstat_tpl[] = {
8709 1.15 jmcneill { "RQ SW prod", KSTAT_KV_T_COUNTER64 },
8710 1.15 jmcneill { "RQ HW prod", KSTAT_KV_T_COUNTER64 },
8711 1.15 jmcneill { "RQ HW cons", KSTAT_KV_T_COUNTER64 },
8712 1.15 jmcneill { "RQ HW state", KSTAT_KV_T_ISTR },
8713 1.15 jmcneill
8714 1.15 jmcneill { "SQ SW prod", KSTAT_KV_T_COUNTER64 },
8715 1.15 jmcneill { "SQ SW cons", KSTAT_KV_T_COUNTER64 },
8716 1.15 jmcneill { "SQ HW prod", KSTAT_KV_T_COUNTER64 },
8717 1.15 jmcneill { "SQ HW cons", KSTAT_KV_T_COUNTER64 },
8718 1.15 jmcneill { "SQ HW state", KSTAT_KV_T_ISTR },
8719 1.15 jmcneill
8720 1.15 jmcneill { "CQ SW cons", KSTAT_KV_T_COUNTER64 },
8721 1.15 jmcneill { "CQ HW prod", KSTAT_KV_T_COUNTER64 },
8722 1.15 jmcneill { "CQ HW cons", KSTAT_KV_T_COUNTER64 },
8723 1.15 jmcneill { "CQ HW notify", KSTAT_KV_T_COUNTER64 },
8724 1.15 jmcneill { "CQ HW solicit", KSTAT_KV_T_COUNTER64 },
8725 1.15 jmcneill { "CQ HW status", KSTAT_KV_T_ISTR },
8726 1.15 jmcneill { "CQ HW state", KSTAT_KV_T_ISTR },
8727 1.15 jmcneill
8728 1.15 jmcneill { "EQ SW cons", KSTAT_KV_T_COUNTER64 },
8729 1.15 jmcneill { "EQ HW prod", KSTAT_KV_T_COUNTER64 },
8730 1.15 jmcneill { "EQ HW cons", KSTAT_KV_T_COUNTER64 },
8731 1.15 jmcneill { "EQ HW status", KSTAT_KV_T_ISTR },
8732 1.15 jmcneill { "EQ HW state", KSTAT_KV_T_ISTR },
8733 1.15 jmcneill };
8734 1.15 jmcneill
8735 1.15 jmcneill static int mcx_kstat_queue_read(struct kstat *);
8736 1.15 jmcneill
8737 1.15 jmcneill static void
8738 1.15 jmcneill mcx_kstat_attach_queues(struct mcx_softc *sc)
8739 1.15 jmcneill {
8740 1.15 jmcneill struct kstat *ks;
8741 1.15 jmcneill struct kstat_kv *kvs;
8742 1.15 jmcneill int q, i;
8743 1.15 jmcneill
8744 1.15 jmcneill for (q = 0; q < sc->sc_nqueues; q++) {
8745 1.15 jmcneill ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q,
8746 1.15 jmcneill KSTAT_T_KV, 0);
8747 1.15 jmcneill if (ks == NULL) {
8748 1.15 jmcneill /* unable to attach queue stats %u, q */
8749 1.15 jmcneill continue;
8750 1.15 jmcneill }
8751 1.15 jmcneill
8752 1.15 jmcneill kvs = mallocarray(nitems(mcx_queue_kstat_tpl),
8753 1.15 jmcneill sizeof(*kvs), M_DEVBUF, M_WAITOK);
8754 1.15 jmcneill
8755 1.15 jmcneill for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) {
8756 1.15 jmcneill const struct mcx_queuestat *tpl =
8757 1.15 jmcneill &mcx_queue_kstat_tpl[i];
8758 1.15 jmcneill
8759 1.15 jmcneill kstat_kv_init(&kvs[i], tpl->name, tpl->type);
8760 1.15 jmcneill }
8761 1.15 jmcneill
8762 1.15 jmcneill ks->ks_softc = &sc->sc_queues[q];
8763 1.15 jmcneill ks->ks_data = kvs;
8764 1.15 jmcneill ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs);
8765 1.15 jmcneill ks->ks_read = mcx_kstat_queue_read;
8766 1.15 jmcneill
8767 1.15 jmcneill sc->sc_queues[q].q_kstat = ks;
8768 1.15 jmcneill kstat_install(ks);
8769 1.15 jmcneill }
8770 1.15 jmcneill }
8771 1.15 jmcneill
8772 1.15 jmcneill static int
8773 1.15 jmcneill mcx_kstat_queue_read(struct kstat *ks)
8774 1.15 jmcneill {
8775 1.15 jmcneill struct mcx_queues *q = ks->ks_softc;
8776 1.15 jmcneill struct mcx_softc *sc = q->q_sc;
8777 1.15 jmcneill struct kstat_kv *kvs = ks->ks_data;
8778 1.15 jmcneill union {
8779 1.15 jmcneill struct mcx_rq_ctx rq;
8780 1.15 jmcneill struct mcx_sq_ctx sq;
8781 1.15 jmcneill struct mcx_cq_ctx cq;
8782 1.15 jmcneill struct mcx_eq_ctx eq;
8783 1.15 jmcneill } u;
8784 1.15 jmcneill const char *text;
8785 1.15 jmcneill int error = 0;
8786 1.15 jmcneill
8787 1.15 jmcneill KERNEL_LOCK();
8788 1.15 jmcneill
8789 1.15 jmcneill if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) {
8790 1.15 jmcneill error = EIO;
8791 1.15 jmcneill goto out;
8792 1.15 jmcneill }
8793 1.15 jmcneill
8794 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_rx.rx_prod;
8795 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter);
8796 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter);
8797 1.15 jmcneill switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >>
8798 1.15 jmcneill MCX_RQ_CTX_STATE_SHIFT) {
8799 1.15 jmcneill case MCX_RQ_CTX_STATE_RST:
8800 1.15 jmcneill text = "RST";
8801 1.15 jmcneill break;
8802 1.15 jmcneill case MCX_RQ_CTX_STATE_RDY:
8803 1.15 jmcneill text = "RDY";
8804 1.15 jmcneill break;
8805 1.15 jmcneill case MCX_RQ_CTX_STATE_ERR:
8806 1.15 jmcneill text = "ERR";
8807 1.15 jmcneill break;
8808 1.15 jmcneill default:
8809 1.15 jmcneill text = "unknown";
8810 1.15 jmcneill break;
8811 1.15 jmcneill }
8812 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8813 1.15 jmcneill kvs++;
8814 1.15 jmcneill
8815 1.15 jmcneill if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) {
8816 1.15 jmcneill error = EIO;
8817 1.15 jmcneill goto out;
8818 1.15 jmcneill }
8819 1.15 jmcneill
8820 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_tx.tx_prod;
8821 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_tx.tx_cons;
8822 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter);
8823 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter);
8824 1.15 jmcneill switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >>
8825 1.15 jmcneill MCX_SQ_CTX_STATE_SHIFT) {
8826 1.15 jmcneill case MCX_SQ_CTX_STATE_RST:
8827 1.15 jmcneill text = "RST";
8828 1.15 jmcneill break;
8829 1.15 jmcneill case MCX_SQ_CTX_STATE_RDY:
8830 1.15 jmcneill text = "RDY";
8831 1.15 jmcneill break;
8832 1.15 jmcneill case MCX_SQ_CTX_STATE_ERR:
8833 1.15 jmcneill text = "ERR";
8834 1.15 jmcneill break;
8835 1.15 jmcneill default:
8836 1.15 jmcneill text = "unknown";
8837 1.15 jmcneill break;
8838 1.15 jmcneill }
8839 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8840 1.15 jmcneill kvs++;
8841 1.15 jmcneill
8842 1.15 jmcneill if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) {
8843 1.15 jmcneill error = EIO;
8844 1.15 jmcneill goto out;
8845 1.15 jmcneill }
8846 1.15 jmcneill
8847 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_cq.cq_cons;
8848 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter);
8849 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter);
8850 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified);
8851 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit);
8852 1.15 jmcneill
8853 1.15 jmcneill switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >>
8854 1.15 jmcneill MCX_CQ_CTX_STATUS_SHIFT) {
8855 1.15 jmcneill case MCX_CQ_CTX_STATUS_OK:
8856 1.15 jmcneill text = "OK";
8857 1.15 jmcneill break;
8858 1.15 jmcneill case MCX_CQ_CTX_STATUS_OVERFLOW:
8859 1.15 jmcneill text = "overflow";
8860 1.15 jmcneill break;
8861 1.15 jmcneill case MCX_CQ_CTX_STATUS_WRITE_FAIL:
8862 1.15 jmcneill text = "write fail";
8863 1.15 jmcneill break;
8864 1.15 jmcneill default:
8865 1.15 jmcneill text = "unknown";
8866 1.15 jmcneill break;
8867 1.15 jmcneill }
8868 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8869 1.15 jmcneill kvs++;
8870 1.15 jmcneill
8871 1.15 jmcneill switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >>
8872 1.15 jmcneill MCX_CQ_CTX_STATE_SHIFT) {
8873 1.15 jmcneill case MCX_CQ_CTX_STATE_SOLICITED:
8874 1.15 jmcneill text = "solicited";
8875 1.15 jmcneill break;
8876 1.15 jmcneill case MCX_CQ_CTX_STATE_ARMED:
8877 1.15 jmcneill text = "armed";
8878 1.15 jmcneill break;
8879 1.15 jmcneill case MCX_CQ_CTX_STATE_FIRED:
8880 1.15 jmcneill text = "fired";
8881 1.15 jmcneill break;
8882 1.15 jmcneill default:
8883 1.15 jmcneill text = "unknown";
8884 1.15 jmcneill break;
8885 1.15 jmcneill }
8886 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8887 1.15 jmcneill kvs++;
8888 1.15 jmcneill
8889 1.15 jmcneill if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) {
8890 1.15 jmcneill error = EIO;
8891 1.15 jmcneill goto out;
8892 1.15 jmcneill }
8893 1.15 jmcneill
8894 1.15 jmcneill kstat_kv_u64(kvs++) = q->q_eq.eq_cons;
8895 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter);
8896 1.15 jmcneill kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter);
8897 1.15 jmcneill
8898 1.15 jmcneill switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >>
8899 1.15 jmcneill MCX_EQ_CTX_STATUS_SHIFT) {
8900 1.15 jmcneill case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE:
8901 1.15 jmcneill text = "write fail";
8902 1.15 jmcneill break;
8903 1.15 jmcneill case MCX_EQ_CTX_STATUS_OK:
8904 1.15 jmcneill text = "OK";
8905 1.15 jmcneill break;
8906 1.15 jmcneill default:
8907 1.15 jmcneill text = "unknown";
8908 1.15 jmcneill break;
8909 1.15 jmcneill }
8910 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8911 1.15 jmcneill kvs++;
8912 1.15 jmcneill
8913 1.15 jmcneill switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >>
8914 1.15 jmcneill MCX_EQ_CTX_STATE_SHIFT) {
8915 1.15 jmcneill case MCX_EQ_CTX_STATE_ARMED:
8916 1.15 jmcneill text = "armed";
8917 1.15 jmcneill break;
8918 1.15 jmcneill case MCX_EQ_CTX_STATE_FIRED:
8919 1.15 jmcneill text = "fired";
8920 1.15 jmcneill break;
8921 1.15 jmcneill default:
8922 1.15 jmcneill text = "unknown";
8923 1.15 jmcneill break;
8924 1.15 jmcneill }
8925 1.15 jmcneill strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs)));
8926 1.15 jmcneill kvs++;
8927 1.15 jmcneill
8928 1.15 jmcneill nanouptime(&ks->ks_updated);
8929 1.15 jmcneill out:
8930 1.15 jmcneill KERNEL_UNLOCK();
8931 1.15 jmcneill return (error);
8932 1.15 jmcneill }
8933 1.15 jmcneill
8934 1.15 jmcneill #endif /* NKSTAT > 0 */
8935 1.15 jmcneill
8936 1.15 jmcneill static unsigned int
8937 1.15 jmcneill mcx_timecounter_read(struct timecounter *tc)
8938 1.15 jmcneill {
8939 1.15 jmcneill struct mcx_softc *sc = tc->tc_priv;
8940 1.15 jmcneill
8941 1.15 jmcneill return (mcx_rd(sc, MCX_INTERNAL_TIMER_L));
8942 1.15 jmcneill }
8943 1.15 jmcneill
8944 1.15 jmcneill static void
8945 1.15 jmcneill mcx_timecounter_attach(struct mcx_softc *sc)
8946 1.15 jmcneill {
8947 1.15 jmcneill struct timecounter *tc = &sc->sc_timecounter;
8948 1.15 jmcneill
8949 1.15 jmcneill tc->tc_get_timecount = mcx_timecounter_read;
8950 1.15 jmcneill tc->tc_counter_mask = ~0U;
8951 1.15 jmcneill tc->tc_frequency = sc->sc_khz * 1000;
8952 1.15 jmcneill tc->tc_name = device_xname(sc->sc_dev);
8953 1.15 jmcneill tc->tc_quality = -100;
8954 1.15 jmcneill tc->tc_priv = sc;
8955 1.15 jmcneill
8956 1.15 jmcneill tc_init(tc);
8957 1.15 jmcneill }
8958