msipic.c revision 1.19 1 1.19 hikaru /* $NetBSD: msipic.c,v 1.19 2019/11/13 02:54:59 hikaru Exp $ */
2 1.1 knakahar
3 1.1 knakahar /*
4 1.1 knakahar * Copyright (c) 2015 Internet Initiative Japan Inc.
5 1.1 knakahar * All rights reserved.
6 1.1 knakahar *
7 1.1 knakahar * Redistribution and use in source and binary forms, with or without
8 1.1 knakahar * modification, are permitted provided that the following conditions
9 1.1 knakahar * are met:
10 1.1 knakahar * 1. Redistributions of source code must retain the above copyright
11 1.1 knakahar * notice, this list of conditions and the following disclaimer.
12 1.1 knakahar * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 knakahar * notice, this list of conditions and the following disclaimer in the
14 1.1 knakahar * documentation and/or other materials provided with the distribution.
15 1.1 knakahar *
16 1.1 knakahar * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
17 1.1 knakahar * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
18 1.1 knakahar * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 1.1 knakahar * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
20 1.1 knakahar * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 1.1 knakahar * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 1.1 knakahar * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 1.1 knakahar * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 1.1 knakahar * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 1.1 knakahar * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 1.1 knakahar * POSSIBILITY OF SUCH DAMAGE.
27 1.1 knakahar */
28 1.1 knakahar
29 1.1 knakahar #include <sys/cdefs.h>
30 1.19 hikaru __KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.19 2019/11/13 02:54:59 hikaru Exp $");
31 1.5 msaitoh
32 1.5 msaitoh #include "opt_intrdebug.h"
33 1.1 knakahar
34 1.1 knakahar #include <sys/types.h>
35 1.1 knakahar #include <sys/param.h>
36 1.1 knakahar #include <sys/systm.h>
37 1.1 knakahar #include <sys/errno.h>
38 1.1 knakahar #include <sys/kmem.h>
39 1.1 knakahar #include <sys/mutex.h>
40 1.17 knakahar #include <sys/bitops.h>
41 1.1 knakahar
42 1.1 knakahar #include <dev/pci/pcivar.h>
43 1.1 knakahar
44 1.1 knakahar #include <machine/i82489reg.h>
45 1.9 nonaka #include <machine/i82489var.h>
46 1.1 knakahar #include <machine/i82093reg.h>
47 1.1 knakahar #include <machine/i82093var.h>
48 1.1 knakahar #include <machine/pic.h>
49 1.1 knakahar #include <machine/lock.h>
50 1.1 knakahar
51 1.1 knakahar #include <x86/pci/msipic.h>
52 1.1 knakahar
53 1.1 knakahar #ifdef INTRDEBUG
54 1.1 knakahar #define MSIPICDEBUG
55 1.1 knakahar #endif
56 1.1 knakahar
57 1.1 knakahar #ifdef MSIPICDEBUG
58 1.1 knakahar #define DPRINTF(msg) printf msg
59 1.1 knakahar #else
60 1.1 knakahar #define DPRINTF(msg)
61 1.1 knakahar #endif
62 1.1 knakahar
63 1.1 knakahar #define BUS_SPACE_WRITE_FLUSH(pc, tag) (void)bus_space_read_4(pc, tag, 0)
64 1.1 knakahar
65 1.1 knakahar #define MSIPICNAMEBUF 16
66 1.1 knakahar
67 1.1 knakahar /*
68 1.1 knakahar * A Pseudo pic for single MSI/MSI-X device.
69 1.1 knakahar * The pic and MSI/MSI-X device are distinbuished by "devid". The "devid"
70 1.1 knakahar * is managed by below "dev_seqs".
71 1.1 knakahar */
72 1.1 knakahar struct msipic {
73 1.1 knakahar int mp_bus;
74 1.1 knakahar int mp_dev;
75 1.1 knakahar int mp_fun;
76 1.1 knakahar
77 1.1 knakahar int mp_devid; /* The device id for the MSI/MSI-X device. */
78 1.1 knakahar int mp_veccnt; /* The number of MSI/MSI-X vectors. */
79 1.1 knakahar
80 1.1 knakahar char mp_pic_name[MSIPICNAMEBUF]; /* The MSI/MSI-X device's name. */
81 1.1 knakahar
82 1.1 knakahar struct pci_attach_args mp_pa;
83 1.1 knakahar bus_space_tag_t mp_bstag;
84 1.1 knakahar bus_space_handle_t mp_bshandle;
85 1.1 knakahar bus_size_t mp_bssize;
86 1.1 knakahar struct pic *mp_pic;
87 1.1 knakahar
88 1.1 knakahar LIST_ENTRY(msipic) mp_list;
89 1.1 knakahar };
90 1.1 knakahar
91 1.1 knakahar static kmutex_t msipic_list_lock;
92 1.1 knakahar
93 1.1 knakahar static LIST_HEAD(, msipic) msipic_list =
94 1.1 knakahar LIST_HEAD_INITIALIZER(msipic_list);
95 1.1 knakahar
96 1.1 knakahar /*
97 1.1 knakahar * This struct managements "devid" to use the same "devid" for the device
98 1.12 msaitoh * re-attached. If the device's bus number and device number and function
99 1.1 knakahar * number are equal, it is assumed re-attached.
100 1.1 knakahar */
101 1.1 knakahar struct dev_last_used_seq {
102 1.1 knakahar bool ds_using;
103 1.1 knakahar int ds_bus;
104 1.1 knakahar int ds_dev;
105 1.1 knakahar int ds_fun;
106 1.1 knakahar };
107 1.1 knakahar /* The number of MSI/MSI-X devices supported by system. */
108 1.1 knakahar #define NUM_MSI_DEVS 256
109 1.1 knakahar /* Record devids to use the same devid when the device is re-attached. */
110 1.1 knakahar static struct dev_last_used_seq dev_seqs[NUM_MSI_DEVS];
111 1.1 knakahar
112 1.4 knakahar static int msipic_allocate_common_msi_devid(const struct pci_attach_args *);
113 1.1 knakahar static void msipic_release_common_msi_devid(int);
114 1.1 knakahar
115 1.1 knakahar static struct pic *msipic_find_msi_pic_locked(int);
116 1.4 knakahar static struct pic *msipic_construct_common_msi_pic(const struct pci_attach_args *,
117 1.1 knakahar struct pic *);
118 1.1 knakahar static void msipic_destruct_common_msi_pic(struct pic *);
119 1.1 knakahar
120 1.1 knakahar static void msi_set_msictl_enablebit(struct pic *, int, int);
121 1.1 knakahar static void msi_hwmask(struct pic *, int);
122 1.1 knakahar static void msi_hwunmask(struct pic *, int);
123 1.1 knakahar static void msi_addroute(struct pic *, struct cpu_info *, int, int, int);
124 1.1 knakahar static void msi_delroute(struct pic *, struct cpu_info *, int, int, int);
125 1.1 knakahar
126 1.1 knakahar static void msix_set_vecctl_mask(struct pic *, int, int);
127 1.1 knakahar static void msix_hwmask(struct pic *, int);
128 1.1 knakahar static void msix_hwunmask(struct pic *, int);
129 1.1 knakahar static void msix_addroute(struct pic *, struct cpu_info *, int, int, int);
130 1.1 knakahar static void msix_delroute(struct pic *, struct cpu_info *, int, int, int);
131 1.1 knakahar
132 1.1 knakahar /*
133 1.1 knakahar * Return new "devid" for the device attached first.
134 1.1 knakahar * Return the same "devid" for the device re-attached after dettached once.
135 1.1 knakahar * Return -1 if the number of attached MSI/MSI-X devices is over NUM_MSI_DEVS.
136 1.1 knakahar */
137 1.1 knakahar static int
138 1.4 knakahar msipic_allocate_common_msi_devid(const struct pci_attach_args *pa)
139 1.1 knakahar {
140 1.1 knakahar pci_chipset_tag_t pc;
141 1.1 knakahar pcitag_t tag;
142 1.1 knakahar int bus, dev, fun, i;
143 1.1 knakahar
144 1.1 knakahar KASSERT(mutex_owned(&msipic_list_lock));
145 1.1 knakahar
146 1.1 knakahar pc = pa->pa_pc;
147 1.1 knakahar tag = pa->pa_tag;
148 1.1 knakahar pci_decompose_tag(pc, tag, &bus, &dev, &fun);
149 1.1 knakahar
150 1.1 knakahar /* if the device was once attached, use same devid */
151 1.1 knakahar for (i = 0; i < NUM_MSI_DEVS; i++) {
152 1.1 knakahar /* skip host bridge */
153 1.1 knakahar if (dev_seqs[i].ds_bus == 0
154 1.1 knakahar && dev_seqs[i].ds_dev == 0
155 1.1 knakahar && dev_seqs[i].ds_fun == 0)
156 1.1 knakahar break;
157 1.1 knakahar
158 1.1 knakahar if (dev_seqs[i].ds_bus == bus
159 1.1 knakahar && dev_seqs[i].ds_dev == dev
160 1.1 knakahar && dev_seqs[i].ds_fun == fun) {
161 1.1 knakahar dev_seqs[i].ds_using = true;
162 1.1 knakahar return i;
163 1.1 knakahar }
164 1.1 knakahar }
165 1.1 knakahar
166 1.1 knakahar for (i = 0; i < NUM_MSI_DEVS; i++) {
167 1.1 knakahar if (dev_seqs[i].ds_using == 0) {
168 1.1 knakahar dev_seqs[i].ds_using = true;
169 1.1 knakahar dev_seqs[i].ds_bus = bus;
170 1.1 knakahar dev_seqs[i].ds_dev = dev;
171 1.1 knakahar dev_seqs[i].ds_fun = fun;
172 1.1 knakahar return i;
173 1.1 knakahar }
174 1.1 knakahar }
175 1.1 knakahar
176 1.1 knakahar DPRINTF(("too many MSI devices.\n"));
177 1.1 knakahar return -1;
178 1.1 knakahar }
179 1.1 knakahar
180 1.1 knakahar /*
181 1.1 knakahar * Set the "devid" unused, but keep reserving the "devid" to reuse when
182 1.1 knakahar * the device is re-attached.
183 1.1 knakahar */
184 1.1 knakahar static void
185 1.1 knakahar msipic_release_common_msi_devid(int devid)
186 1.1 knakahar {
187 1.1 knakahar
188 1.1 knakahar KASSERT(mutex_owned(&msipic_list_lock));
189 1.1 knakahar
190 1.1 knakahar if (devid < 0 || NUM_MSI_DEVS <= devid) {
191 1.1 knakahar DPRINTF(("%s: invalid devid.\n", __func__));
192 1.1 knakahar return;
193 1.1 knakahar }
194 1.1 knakahar
195 1.1 knakahar dev_seqs[devid].ds_using = false;
196 1.1 knakahar /* Keep ds_* to reuse the same devid for the same device. */
197 1.1 knakahar }
198 1.1 knakahar
199 1.1 knakahar static struct pic *
200 1.1 knakahar msipic_find_msi_pic_locked(int devid)
201 1.1 knakahar {
202 1.1 knakahar struct msipic *mpp;
203 1.1 knakahar
204 1.1 knakahar KASSERT(mutex_owned(&msipic_list_lock));
205 1.1 knakahar
206 1.1 knakahar LIST_FOREACH(mpp, &msipic_list, mp_list) {
207 1.15 msaitoh if (mpp->mp_devid == devid)
208 1.1 knakahar return mpp->mp_pic;
209 1.1 knakahar }
210 1.1 knakahar return NULL;
211 1.1 knakahar }
212 1.1 knakahar
213 1.1 knakahar /*
214 1.1 knakahar * Return the msi_pic whose device is already registered.
215 1.1 knakahar * If the device is not registered yet, return NULL.
216 1.1 knakahar */
217 1.1 knakahar struct pic *
218 1.1 knakahar msipic_find_msi_pic(int devid)
219 1.1 knakahar {
220 1.1 knakahar struct pic *msipic;
221 1.1 knakahar
222 1.1 knakahar mutex_enter(&msipic_list_lock);
223 1.1 knakahar msipic = msipic_find_msi_pic_locked(devid);
224 1.1 knakahar mutex_exit(&msipic_list_lock);
225 1.1 knakahar
226 1.1 knakahar return msipic;
227 1.1 knakahar }
228 1.1 knakahar
229 1.1 knakahar /*
230 1.1 knakahar * A common construct process of MSI and MSI-X.
231 1.1 knakahar */
232 1.1 knakahar static struct pic *
233 1.4 knakahar msipic_construct_common_msi_pic(const struct pci_attach_args *pa,
234 1.1 knakahar struct pic *pic_tmpl)
235 1.1 knakahar {
236 1.1 knakahar struct pic *pic;
237 1.1 knakahar struct msipic *msipic;
238 1.1 knakahar int devid;
239 1.1 knakahar
240 1.1 knakahar pic = kmem_alloc(sizeof(*pic), KM_SLEEP);
241 1.1 knakahar msipic = kmem_zalloc(sizeof(*msipic), KM_SLEEP);
242 1.1 knakahar
243 1.1 knakahar mutex_enter(&msipic_list_lock);
244 1.1 knakahar
245 1.1 knakahar devid = msipic_allocate_common_msi_devid(pa);
246 1.1 knakahar if (devid == -1) {
247 1.1 knakahar mutex_exit(&msipic_list_lock);
248 1.1 knakahar kmem_free(pic, sizeof(*pic));
249 1.1 knakahar kmem_free(msipic, sizeof(*msipic));
250 1.1 knakahar return NULL;
251 1.1 knakahar }
252 1.1 knakahar
253 1.1 knakahar memcpy(pic, pic_tmpl, sizeof(*pic));
254 1.14 msaitoh pic->pic_edge_stubs
255 1.14 msaitoh = x2apic_mode ? x2apic_edge_stubs : ioapic_edge_stubs;
256 1.1 knakahar pic->pic_msipic = msipic;
257 1.1 knakahar msipic->mp_pic = pic;
258 1.1 knakahar pci_decompose_tag(pa->pa_pc, pa->pa_tag,
259 1.1 knakahar &msipic->mp_bus, &msipic->mp_dev, &msipic->mp_fun);
260 1.1 knakahar memcpy(&msipic->mp_pa, pa, sizeof(msipic->mp_pa));
261 1.1 knakahar msipic->mp_devid = devid;
262 1.1 knakahar /*
263 1.1 knakahar * pci_msi{,x}_alloc() must be called only once in the device driver.
264 1.1 knakahar */
265 1.1 knakahar KASSERT(msipic_find_msi_pic_locked(msipic->mp_devid) == NULL);
266 1.1 knakahar
267 1.1 knakahar LIST_INSERT_HEAD(&msipic_list, msipic, mp_list);
268 1.1 knakahar
269 1.1 knakahar mutex_exit(&msipic_list_lock);
270 1.1 knakahar
271 1.1 knakahar return pic;
272 1.1 knakahar }
273 1.1 knakahar
274 1.1 knakahar static void
275 1.1 knakahar msipic_destruct_common_msi_pic(struct pic *msi_pic)
276 1.1 knakahar {
277 1.1 knakahar struct msipic *msipic;
278 1.1 knakahar
279 1.1 knakahar if (msi_pic == NULL)
280 1.1 knakahar return;
281 1.1 knakahar
282 1.1 knakahar msipic = msi_pic->pic_msipic;
283 1.1 knakahar mutex_enter(&msipic_list_lock);
284 1.1 knakahar LIST_REMOVE(msipic, mp_list);
285 1.1 knakahar msipic_release_common_msi_devid(msipic->mp_devid);
286 1.1 knakahar mutex_exit(&msipic_list_lock);
287 1.1 knakahar
288 1.1 knakahar kmem_free(msipic, sizeof(*msipic));
289 1.1 knakahar kmem_free(msi_pic, sizeof(*msi_pic));
290 1.1 knakahar }
291 1.1 knakahar
292 1.1 knakahar /*
293 1.1 knakahar * The pic is MSI/MSI-X pic or not.
294 1.1 knakahar */
295 1.1 knakahar bool
296 1.1 knakahar msipic_is_msi_pic(struct pic *pic)
297 1.1 knakahar {
298 1.1 knakahar
299 1.1 knakahar return (pic->pic_msipic != NULL);
300 1.1 knakahar }
301 1.1 knakahar
302 1.1 knakahar /*
303 1.1 knakahar * Return the MSI/MSI-X devid which is unique for each devices.
304 1.1 knakahar */
305 1.1 knakahar int
306 1.1 knakahar msipic_get_devid(struct pic *pic)
307 1.1 knakahar {
308 1.1 knakahar
309 1.1 knakahar KASSERT(msipic_is_msi_pic(pic));
310 1.1 knakahar
311 1.1 knakahar return pic->pic_msipic->mp_devid;
312 1.1 knakahar }
313 1.1 knakahar
314 1.1 knakahar #define MSI_MSICTL_ENABLE 1
315 1.1 knakahar #define MSI_MSICTL_DISABLE 0
316 1.1 knakahar static void
317 1.1 knakahar msi_set_msictl_enablebit(struct pic *pic, int msi_vec, int flag)
318 1.1 knakahar {
319 1.1 knakahar pci_chipset_tag_t pc;
320 1.1 knakahar struct pci_attach_args *pa;
321 1.1 knakahar pcitag_t tag;
322 1.1 knakahar pcireg_t ctl;
323 1.3 martin int off, err __diagused;
324 1.1 knakahar
325 1.1 knakahar pc = NULL;
326 1.1 knakahar pa = &pic->pic_msipic->mp_pa;
327 1.1 knakahar tag = pa->pa_tag;
328 1.3 martin err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
329 1.3 martin KASSERT(err != 0);
330 1.1 knakahar
331 1.1 knakahar /*
332 1.1 knakahar * MSI can establish only one vector at once.
333 1.1 knakahar * So, use whole device mask bit instead of a vector mask bit.
334 1.1 knakahar */
335 1.1 knakahar ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
336 1.1 knakahar if (flag == MSI_MSICTL_ENABLE)
337 1.1 knakahar ctl |= PCI_MSI_CTL_MSI_ENABLE;
338 1.1 knakahar else
339 1.1 knakahar ctl &= ~PCI_MSI_CTL_MSI_ENABLE;
340 1.1 knakahar
341 1.1 knakahar pci_conf_write(pc, tag, off, ctl);
342 1.1 knakahar }
343 1.1 knakahar
344 1.1 knakahar static void
345 1.1 knakahar msi_hwmask(struct pic *pic, int msi_vec)
346 1.1 knakahar {
347 1.1 knakahar
348 1.1 knakahar msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_DISABLE);
349 1.1 knakahar }
350 1.1 knakahar
351 1.1 knakahar /*
352 1.1 knakahar * Do not use pic->hwunmask() immediately after pic->delroute().
353 1.1 knakahar * It is required to use pic->addroute() before pic->hwunmask().
354 1.1 knakahar */
355 1.1 knakahar static void
356 1.1 knakahar msi_hwunmask(struct pic *pic, int msi_vec)
357 1.1 knakahar {
358 1.1 knakahar
359 1.1 knakahar msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_ENABLE);
360 1.1 knakahar }
361 1.1 knakahar
362 1.1 knakahar static void
363 1.1 knakahar msi_addroute(struct pic *pic, struct cpu_info *ci,
364 1.1 knakahar int unused, int idt_vec, int type)
365 1.1 knakahar {
366 1.1 knakahar pci_chipset_tag_t pc;
367 1.1 knakahar struct pci_attach_args *pa;
368 1.1 knakahar pcitag_t tag;
369 1.1 knakahar pcireg_t addr, data, ctl;
370 1.3 martin int off, err __diagused;
371 1.1 knakahar
372 1.1 knakahar pc = NULL;
373 1.1 knakahar pa = &pic->pic_msipic->mp_pa;
374 1.1 knakahar tag = pa->pa_tag;
375 1.3 martin err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
376 1.3 martin KASSERT(err != 0);
377 1.1 knakahar
378 1.1 knakahar /*
379 1.1 knakahar * See Intel 64 and IA-32 Architectures Software Developer's Manual
380 1.1 knakahar * Volume 3 10.11 Message Signalled Interrupts.
381 1.1 knakahar */
382 1.1 knakahar /*
383 1.1 knakahar * "cpuid" for MSI address is local APIC ID. In NetBSD, the ID is
384 1.1 knakahar * the same as ci->ci_cpuid.
385 1.1 knakahar */
386 1.1 knakahar addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
387 1.1 knakahar LAPIC_MSIADDR_DSTID_MASK);
388 1.1 knakahar /* If trigger mode is edge, it don't care level for trigger mode. */
389 1.13 msaitoh data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
390 1.13 msaitoh | LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;
391 1.1 knakahar
392 1.16 msaitoh /*
393 1.16 msaitoh * The size of the message data register is 16bit if the extended
394 1.16 msaitoh * message data is not implemented. If it's 16bit and the per-vector
395 1.16 msaitoh * masking is not capable, the location of the upper 16bit is out of
396 1.16 msaitoh * the MSI capability structure's range. The PCI spec says the upper
397 1.16 msaitoh * 16bit is driven to 0 if the message data register is 16bit. It's the
398 1.16 msaitoh * spec, so it's OK just to write it regardless of the value of the
399 1.16 msaitoh * upper 16bit.
400 1.16 msaitoh */
401 1.1 knakahar ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
402 1.1 knakahar if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
403 1.1 knakahar pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, addr);
404 1.1 knakahar pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 0);
405 1.1 knakahar pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data);
406 1.1 knakahar } else {
407 1.1 knakahar pci_conf_write(pc, tag, off + PCI_MSI_MADDR, addr);
408 1.1 knakahar pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data);
409 1.1 knakahar }
410 1.1 knakahar ctl |= PCI_MSI_CTL_MSI_ENABLE;
411 1.1 knakahar pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
412 1.1 knakahar }
413 1.1 knakahar
414 1.1 knakahar /*
415 1.1 knakahar * Do not use pic->hwunmask() immediately after pic->delroute().
416 1.1 knakahar * It is required to use pic->addroute() before pic->hwunmask().
417 1.1 knakahar */
418 1.1 knakahar static void
419 1.1 knakahar msi_delroute(struct pic *pic, struct cpu_info *ci,
420 1.1 knakahar int msi_vec, int idt_vec, int type)
421 1.1 knakahar {
422 1.1 knakahar
423 1.1 knakahar msi_hwmask(pic, msi_vec);
424 1.1 knakahar }
425 1.1 knakahar
426 1.1 knakahar /*
427 1.1 knakahar * Template for MSI pic.
428 1.1 knakahar * .pic_msipic is set later in construct_msi_pic().
429 1.1 knakahar */
430 1.1 knakahar static struct pic msi_pic_tmpl = {
431 1.1 knakahar .pic_type = PIC_MSI,
432 1.1 knakahar .pic_vecbase = 0,
433 1.1 knakahar .pic_apicid = 0,
434 1.1 knakahar .pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msi_pic */
435 1.1 knakahar .pic_hwmask = msi_hwmask,
436 1.1 knakahar .pic_hwunmask = msi_hwunmask,
437 1.1 knakahar .pic_addroute = msi_addroute,
438 1.1 knakahar .pic_delroute = msi_delroute,
439 1.1 knakahar };
440 1.1 knakahar
441 1.1 knakahar /*
442 1.1 knakahar * Create pseudo pic for a MSI device.
443 1.1 knakahar */
444 1.1 knakahar struct pic *
445 1.4 knakahar msipic_construct_msi_pic(const struct pci_attach_args *pa)
446 1.1 knakahar {
447 1.1 knakahar struct pic *msi_pic;
448 1.1 knakahar char pic_name_buf[MSIPICNAMEBUF];
449 1.1 knakahar
450 1.1 knakahar msi_pic = msipic_construct_common_msi_pic(pa, &msi_pic_tmpl);
451 1.1 knakahar if (msi_pic == NULL) {
452 1.1 knakahar DPRINTF(("cannot allocate MSI pic.\n"));
453 1.1 knakahar return NULL;
454 1.1 knakahar }
455 1.1 knakahar
456 1.1 knakahar memset(pic_name_buf, 0, MSIPICNAMEBUF);
457 1.1 knakahar snprintf(pic_name_buf, MSIPICNAMEBUF, "msi%d",
458 1.1 knakahar msi_pic->pic_msipic->mp_devid);
459 1.1 knakahar strncpy(msi_pic->pic_msipic->mp_pic_name, pic_name_buf,
460 1.1 knakahar MSIPICNAMEBUF - 1);
461 1.1 knakahar msi_pic->pic_name = msi_pic->pic_msipic->mp_pic_name;
462 1.1 knakahar
463 1.1 knakahar return msi_pic;
464 1.1 knakahar }
465 1.1 knakahar
466 1.1 knakahar /*
467 1.1 knakahar * Delete pseudo pic for a MSI device.
468 1.1 knakahar */
469 1.1 knakahar void
470 1.1 knakahar msipic_destruct_msi_pic(struct pic *msi_pic)
471 1.1 knakahar {
472 1.1 knakahar
473 1.1 knakahar msipic_destruct_common_msi_pic(msi_pic);
474 1.1 knakahar }
475 1.1 knakahar
476 1.1 knakahar #define MSIX_VECCTL_HWMASK 1
477 1.1 knakahar #define MSIX_VECCTL_HWUNMASK 0
478 1.1 knakahar static void
479 1.1 knakahar msix_set_vecctl_mask(struct pic *pic, int msix_vec, int flag)
480 1.1 knakahar {
481 1.1 knakahar bus_space_tag_t bstag;
482 1.1 knakahar bus_space_handle_t bshandle;
483 1.1 knakahar uint64_t entry_base;
484 1.1 knakahar uint32_t vecctl;
485 1.1 knakahar
486 1.1 knakahar if (msix_vec < 0) {
487 1.1 knakahar DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
488 1.2 knakahar __func__, msipic_get_devid(pic), msix_vec));
489 1.1 knakahar return;
490 1.1 knakahar }
491 1.1 knakahar
492 1.1 knakahar entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
493 1.1 knakahar
494 1.1 knakahar bstag = pic->pic_msipic->mp_bstag;
495 1.1 knakahar bshandle = pic->pic_msipic->mp_bshandle;
496 1.1 knakahar vecctl = bus_space_read_4(bstag, bshandle,
497 1.1 knakahar entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL);
498 1.1 knakahar if (flag == MSIX_VECCTL_HWMASK)
499 1.8 msaitoh vecctl |= PCI_MSIX_VECTCTL_MASK;
500 1.1 knakahar else
501 1.8 msaitoh vecctl &= ~PCI_MSIX_VECTCTL_MASK;
502 1.1 knakahar
503 1.1 knakahar bus_space_write_4(bstag, bshandle,
504 1.1 knakahar entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, vecctl);
505 1.1 knakahar BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
506 1.1 knakahar }
507 1.1 knakahar
508 1.1 knakahar static void
509 1.1 knakahar msix_hwmask(struct pic *pic, int msix_vec)
510 1.1 knakahar {
511 1.1 knakahar
512 1.1 knakahar msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWMASK);
513 1.1 knakahar }
514 1.1 knakahar
515 1.1 knakahar /*
516 1.1 knakahar * Do not use pic->hwunmask() immediately after pic->delroute().
517 1.1 knakahar * It is required to use pic->addroute() before pic->hwunmask().
518 1.1 knakahar */
519 1.1 knakahar static void
520 1.1 knakahar msix_hwunmask(struct pic *pic, int msix_vec)
521 1.1 knakahar {
522 1.1 knakahar
523 1.1 knakahar msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWUNMASK);
524 1.1 knakahar }
525 1.1 knakahar
526 1.1 knakahar static void
527 1.1 knakahar msix_addroute(struct pic *pic, struct cpu_info *ci,
528 1.1 knakahar int msix_vec, int idt_vec, int type)
529 1.1 knakahar {
530 1.1 knakahar pci_chipset_tag_t pc;
531 1.1 knakahar struct pci_attach_args *pa;
532 1.1 knakahar pcitag_t tag;
533 1.1 knakahar bus_space_tag_t bstag;
534 1.1 knakahar bus_space_handle_t bshandle;
535 1.1 knakahar uint64_t entry_base;
536 1.1 knakahar pcireg_t addr, data, ctl;
537 1.3 martin int off, err __diagused;
538 1.1 knakahar
539 1.1 knakahar if (msix_vec < 0) {
540 1.1 knakahar DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
541 1.2 knakahar __func__, msipic_get_devid(pic), msix_vec));
542 1.1 knakahar return;
543 1.1 knakahar }
544 1.1 knakahar
545 1.1 knakahar pa = &pic->pic_msipic->mp_pa;
546 1.1 knakahar pc = pa->pa_pc;
547 1.1 knakahar tag = pa->pa_tag;
548 1.3 martin err = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
549 1.3 martin KASSERT(err != 0);
550 1.1 knakahar
551 1.19 hikaru /* Disable MSI-X before writing MSI-X table */
552 1.19 hikaru ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
553 1.19 hikaru ctl &= ~PCI_MSIX_CTL_ENABLE;
554 1.19 hikaru pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
555 1.19 hikaru
556 1.1 knakahar entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;
557 1.1 knakahar
558 1.1 knakahar /*
559 1.1 knakahar * See Intel 64 and IA-32 Architectures Software Developer's Manual
560 1.1 knakahar * Volume 3 10.11 Message Signalled Interrupts.
561 1.1 knakahar */
562 1.1 knakahar /*
563 1.1 knakahar * "cpuid" for MSI-X address is local APIC ID. In NetBSD, the ID is
564 1.1 knakahar * the same as ci->ci_cpuid.
565 1.1 knakahar */
566 1.1 knakahar addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
567 1.1 knakahar LAPIC_MSIADDR_DSTID_MASK);
568 1.1 knakahar /* If trigger mode is edge, it don't care level for trigger mode. */
569 1.13 msaitoh data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
570 1.13 msaitoh | LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;
571 1.1 knakahar
572 1.1 knakahar bstag = pic->pic_msipic->mp_bstag;
573 1.1 knakahar bshandle = pic->pic_msipic->mp_bshandle;
574 1.1 knakahar bus_space_write_4(bstag, bshandle,
575 1.1 knakahar entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, addr);
576 1.1 knakahar bus_space_write_4(bstag, bshandle,
577 1.1 knakahar entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 0);
578 1.1 knakahar bus_space_write_4(bstag, bshandle,
579 1.1 knakahar entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data);
580 1.1 knakahar bus_space_write_4(bstag, bshandle,
581 1.1 knakahar entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0);
582 1.1 knakahar BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
583 1.1 knakahar
584 1.1 knakahar ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
585 1.1 knakahar ctl |= PCI_MSIX_CTL_ENABLE;
586 1.1 knakahar pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
587 1.1 knakahar }
588 1.1 knakahar
589 1.1 knakahar /*
590 1.1 knakahar * Do not use pic->hwunmask() immediately after pic->delroute().
591 1.1 knakahar * It is required to use pic->addroute() before pic->hwunmask().
592 1.1 knakahar */
593 1.1 knakahar static void
594 1.1 knakahar msix_delroute(struct pic *pic, struct cpu_info *ci,
595 1.1 knakahar int msix_vec, int vec, int type)
596 1.1 knakahar {
597 1.1 knakahar
598 1.1 knakahar msix_hwmask(pic, msix_vec);
599 1.1 knakahar }
600 1.1 knakahar
601 1.1 knakahar /*
602 1.1 knakahar * Template for MSI-X pic.
603 1.1 knakahar * .pic_msipic is set later in construct_msix_pic().
604 1.1 knakahar */
605 1.1 knakahar static struct pic msix_pic_tmpl = {
606 1.1 knakahar .pic_type = PIC_MSIX,
607 1.1 knakahar .pic_vecbase = 0,
608 1.1 knakahar .pic_apicid = 0,
609 1.1 knakahar .pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msix_pic */
610 1.1 knakahar .pic_hwmask = msix_hwmask,
611 1.1 knakahar .pic_hwunmask = msix_hwunmask,
612 1.1 knakahar .pic_addroute = msix_addroute,
613 1.1 knakahar .pic_delroute = msix_delroute,
614 1.1 knakahar };
615 1.1 knakahar
616 1.1 knakahar struct pic *
617 1.4 knakahar msipic_construct_msix_pic(const struct pci_attach_args *pa)
618 1.1 knakahar {
619 1.1 knakahar struct pic *msix_pic;
620 1.1 knakahar pci_chipset_tag_t pc;
621 1.1 knakahar pcitag_t tag;
622 1.1 knakahar pcireg_t tbl;
623 1.1 knakahar bus_space_tag_t bstag;
624 1.1 knakahar bus_space_handle_t bshandle;
625 1.1 knakahar bus_size_t bssize;
626 1.1 knakahar size_t table_size;
627 1.1 knakahar uint32_t table_offset;
628 1.1 knakahar u_int memtype;
629 1.7 msaitoh bus_addr_t memaddr;
630 1.7 msaitoh int flags;
631 1.1 knakahar int bir, bar, err, off, table_nentry;
632 1.1 knakahar char pic_name_buf[MSIPICNAMEBUF];
633 1.1 knakahar
634 1.6 msaitoh table_nentry = pci_msix_count(pa->pa_pc, pa->pa_tag);
635 1.1 knakahar if (table_nentry == 0) {
636 1.1 knakahar DPRINTF(("MSI-X table entry is 0.\n"));
637 1.1 knakahar return NULL;
638 1.1 knakahar }
639 1.1 knakahar
640 1.1 knakahar pc = pa->pa_pc;
641 1.1 knakahar tag = pa->pa_tag;
642 1.1 knakahar if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL) == 0) {
643 1.1 knakahar DPRINTF(("%s: no msix capability", __func__));
644 1.1 knakahar return NULL;
645 1.1 knakahar }
646 1.1 knakahar
647 1.1 knakahar msix_pic = msipic_construct_common_msi_pic(pa, &msix_pic_tmpl);
648 1.1 knakahar if (msix_pic == NULL) {
649 1.1 knakahar DPRINTF(("cannot allocate MSI-X pic.\n"));
650 1.1 knakahar return NULL;
651 1.1 knakahar }
652 1.1 knakahar
653 1.1 knakahar memset(pic_name_buf, 0, MSIPICNAMEBUF);
654 1.1 knakahar snprintf(pic_name_buf, MSIPICNAMEBUF, "msix%d",
655 1.1 knakahar msix_pic->pic_msipic->mp_devid);
656 1.1 knakahar strncpy(msix_pic->pic_msipic->mp_pic_name, pic_name_buf,
657 1.1 knakahar MSIPICNAMEBUF - 1);
658 1.1 knakahar msix_pic->pic_name = msix_pic->pic_msipic->mp_pic_name;
659 1.1 knakahar
660 1.1 knakahar tbl = pci_conf_read(pc, tag, off + PCI_MSIX_TBLOFFSET);
661 1.1 knakahar table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
662 1.1 knakahar bir = tbl & PCI_MSIX_PBABIR_MASK;
663 1.15 msaitoh switch (bir) {
664 1.1 knakahar case 0:
665 1.1 knakahar bar = PCI_BAR0;
666 1.1 knakahar break;
667 1.1 knakahar case 1:
668 1.1 knakahar bar = PCI_BAR1;
669 1.1 knakahar break;
670 1.1 knakahar case 2:
671 1.1 knakahar bar = PCI_BAR2;
672 1.1 knakahar break;
673 1.1 knakahar case 3:
674 1.1 knakahar bar = PCI_BAR3;
675 1.1 knakahar break;
676 1.1 knakahar case 4:
677 1.1 knakahar bar = PCI_BAR4;
678 1.1 knakahar break;
679 1.1 knakahar case 5:
680 1.1 knakahar bar = PCI_BAR5;
681 1.1 knakahar break;
682 1.1 knakahar default:
683 1.15 msaitoh aprint_error("detect an illegal device! "
684 1.15 msaitoh "The device use reserved BIR values.\n");
685 1.1 knakahar msipic_destruct_common_msi_pic(msix_pic);
686 1.1 knakahar return NULL;
687 1.1 knakahar }
688 1.1 knakahar memtype = pci_mapreg_type(pc, tag, bar);
689 1.15 msaitoh /*
690 1.15 msaitoh * PCI_MSIX_TABLE_ENTRY_SIZE consists below
691 1.15 msaitoh * - Vector Control (32bit)
692 1.15 msaitoh * - Message Data (32bit)
693 1.15 msaitoh * - Message Upper Address (32bit)
694 1.15 msaitoh * - Message Lower Address (32bit)
695 1.15 msaitoh */
696 1.1 knakahar table_size = table_nentry * PCI_MSIX_TABLE_ENTRY_SIZE;
697 1.7 msaitoh #if 0
698 1.1 knakahar err = pci_mapreg_submap(pa, bar, memtype, BUS_SPACE_MAP_LINEAR,
699 1.1 knakahar roundup(table_size, PAGE_SIZE), table_offset,
700 1.1 knakahar &bstag, &bshandle, NULL, &bssize);
701 1.7 msaitoh #else
702 1.7 msaitoh /*
703 1.7 msaitoh * Workaround for PCI prefetchable bit. Some chips (e.g. Intel 82599)
704 1.7 msaitoh * report SERR and MSI-X doesn't work. This problem might not be the
705 1.7 msaitoh * driver's bug but our PCI common part or VMs' bug. Until we find a
706 1.7 msaitoh * real reason, we ignore the prefetchable bit.
707 1.7 msaitoh */
708 1.7 msaitoh if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, bar, memtype,
709 1.7 msaitoh &memaddr, NULL, &flags) != 0) {
710 1.7 msaitoh DPRINTF(("cannot get a map info.\n"));
711 1.7 msaitoh msipic_destruct_common_msi_pic(msix_pic);
712 1.7 msaitoh return NULL;
713 1.7 msaitoh }
714 1.7 msaitoh if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
715 1.7 msaitoh DPRINTF(( "clear prefetchable bit\n"));
716 1.7 msaitoh flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
717 1.7 msaitoh }
718 1.7 msaitoh bssize = roundup(table_size, PAGE_SIZE);
719 1.18 tnn err = _x86_memio_map(pa->pa_memt, memaddr + table_offset, bssize, flags,
720 1.7 msaitoh &bshandle);
721 1.7 msaitoh bstag = pa->pa_memt;
722 1.7 msaitoh #endif
723 1.1 knakahar if (err) {
724 1.1 knakahar DPRINTF(("cannot map msix table.\n"));
725 1.1 knakahar msipic_destruct_common_msi_pic(msix_pic);
726 1.1 knakahar return NULL;
727 1.1 knakahar }
728 1.1 knakahar msix_pic->pic_msipic->mp_bstag = bstag;
729 1.1 knakahar msix_pic->pic_msipic->mp_bshandle = bshandle;
730 1.1 knakahar msix_pic->pic_msipic->mp_bssize = bssize;
731 1.1 knakahar
732 1.1 knakahar return msix_pic;
733 1.1 knakahar }
734 1.1 knakahar
735 1.1 knakahar /*
736 1.1 knakahar * Delete pseudo pic for a MSI-X device.
737 1.1 knakahar */
738 1.1 knakahar void
739 1.1 knakahar msipic_destruct_msix_pic(struct pic *msix_pic)
740 1.1 knakahar {
741 1.1 knakahar struct msipic *msipic;
742 1.1 knakahar
743 1.1 knakahar KASSERT(msipic_is_msi_pic(msix_pic));
744 1.1 knakahar KASSERT(msix_pic->pic_type == PIC_MSIX);
745 1.1 knakahar
746 1.1 knakahar msipic = msix_pic->pic_msipic;
747 1.18 tnn _x86_memio_unmap(msipic->mp_bstag, msipic->mp_bshandle,
748 1.18 tnn msipic->mp_bssize, NULL);
749 1.1 knakahar
750 1.1 knakahar msipic_destruct_common_msi_pic(msix_pic);
751 1.1 knakahar }
752 1.1 knakahar
753 1.1 knakahar /*
754 1.1 knakahar * Set the number of MSI vectors for pseudo MSI pic.
755 1.1 knakahar */
756 1.1 knakahar int
757 1.1 knakahar msipic_set_msi_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs,
758 1.1 knakahar int count)
759 1.1 knakahar {
760 1.1 knakahar
761 1.1 knakahar KASSERT(msipic_is_msi_pic(msi_pic));
762 1.1 knakahar
763 1.17 knakahar if (msi_pic->pic_type == PIC_MSI) {
764 1.17 knakahar pci_chipset_tag_t pc;
765 1.17 knakahar struct pci_attach_args *pa;
766 1.17 knakahar pcitag_t tag;
767 1.17 knakahar int off, err __diagused;
768 1.17 knakahar pcireg_t ctl;
769 1.17 knakahar
770 1.17 knakahar pc = NULL;
771 1.17 knakahar pa = &msi_pic->pic_msipic->mp_pa;
772 1.17 knakahar tag = pa->pa_tag;
773 1.17 knakahar err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
774 1.17 knakahar KASSERT(err != 0);
775 1.17 knakahar
776 1.17 knakahar ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
777 1.17 knakahar ctl &= ~PCI_MSI_CTL_MME_MASK;
778 1.17 knakahar ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK);
779 1.17 knakahar pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
780 1.17 knakahar }
781 1.17 knakahar
782 1.1 knakahar msi_pic->pic_msipic->mp_veccnt = count;
783 1.1 knakahar return 0;
784 1.1 knakahar }
785 1.1 knakahar
786 1.1 knakahar /*
787 1.1 knakahar * Initialize the system to use MSI/MSI-X.
788 1.1 knakahar */
789 1.1 knakahar void
790 1.1 knakahar msipic_init(void)
791 1.1 knakahar {
792 1.1 knakahar
793 1.1 knakahar mutex_init(&msipic_list_lock, MUTEX_DEFAULT, IPL_NONE);
794 1.1 knakahar }
795