virtio_pci.c revision 1.33 1 /* $NetBSD: virtio_pci.c,v 1.33 2021/10/28 01:36:43 yamaguchi Exp $ */
2
3 /*
4 * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 * Copyright (c) 2012 Stefan Fritsch.
6 * Copyright (c) 2010 Minoura Makoto.
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.33 2021/10/28 01:36:43 yamaguchi Exp $");
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kmem.h>
36 #include <sys/module.h>
37 #include <sys/endian.h>
38 #include <sys/interrupt.h>
39 #include <sys/syslog.h>
40
41 #include <sys/device.h>
42
43 #include <dev/pci/pcidevs.h>
44 #include <dev/pci/pcireg.h>
45 #include <dev/pci/pcivar.h>
46
47 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
48 #include <dev/pci/virtio_pcireg.h>
49
50 #define VIRTIO_PRIVATE
51 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
52
53
54 #define VIRTIO_PCI_LOG(_sc, _use_log, _fmt, _args...) \
55 do { \
56 if ((_use_log)) { \
57 log(LOG_DEBUG, "%s: " _fmt, \
58 device_xname((_sc)->sc_dev), \
59 ##_args); \
60 } else { \
61 aprint_error_dev((_sc)->sc_dev, \
62 _fmt, ##_args); \
63 } \
64 } while(0)
65
66 static int virtio_pci_match(device_t, cfdata_t, void *);
67 static void virtio_pci_attach(device_t, device_t, void *);
68 static int virtio_pci_rescan(device_t, const char *, const int *);
69 static int virtio_pci_detach(device_t, int);
70
71
72 #define NMAPREG ((PCI_MAPREG_END - PCI_MAPREG_START) / \
73 sizeof(pcireg_t))
74 struct virtio_pci_softc {
75 struct virtio_softc sc_sc;
76
77 /* IO space */
78 bus_space_tag_t sc_iot;
79 bus_space_handle_t sc_ioh;
80 bus_size_t sc_iosize;
81 bus_size_t sc_mapped_iosize;
82
83 /* BARs */
84 bus_space_tag_t sc_bars_iot[NMAPREG];
85 bus_space_handle_t sc_bars_ioh[NMAPREG];
86 bus_size_t sc_bars_iosize[NMAPREG];
87
88 /* notify space */
89 bus_space_tag_t sc_notify_iot;
90 bus_space_handle_t sc_notify_ioh;
91 bus_size_t sc_notify_iosize;
92 uint32_t sc_notify_off_multiplier;
93
94 /* isr space */
95 bus_space_tag_t sc_isr_iot;
96 bus_space_handle_t sc_isr_ioh;
97 bus_size_t sc_isr_iosize;
98
99 /* generic */
100 struct pci_attach_args sc_pa;
101 pci_intr_handle_t *sc_ihp;
102 void **sc_ihs;
103 int sc_ihs_num;
104 int sc_devcfg_offset; /* for 0.9 */
105 };
106
107 static int virtio_pci_attach_09(device_t, void *);
108 static void virtio_pci_kick_09(struct virtio_softc *, uint16_t);
109 static uint16_t virtio_pci_read_queue_size_09(struct virtio_softc *, uint16_t);
110 static void virtio_pci_setup_queue_09(struct virtio_softc *, uint16_t, uint64_t);
111 static void virtio_pci_set_status_09(struct virtio_softc *, int);
112 static void virtio_pci_negotiate_features_09(struct virtio_softc *, uint64_t);
113
114 static int virtio_pci_attach_10(device_t, void *);
115 static void virtio_pci_kick_10(struct virtio_softc *, uint16_t);
116 static uint16_t virtio_pci_read_queue_size_10(struct virtio_softc *, uint16_t);
117 static void virtio_pci_setup_queue_10(struct virtio_softc *, uint16_t, uint64_t);
118 static void virtio_pci_set_status_10(struct virtio_softc *, int);
119 static void virtio_pci_negotiate_features_10(struct virtio_softc *, uint64_t);
120 static int virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen);
121
122 static int virtio_pci_alloc_interrupts(struct virtio_softc *);
123 static void virtio_pci_free_interrupts(struct virtio_softc *);
124 static int virtio_pci_adjust_config_region(struct virtio_pci_softc *psc);
125 static int virtio_pci_intr(void *arg);
126 static int virtio_pci_msix_queue_intr(void *);
127 static int virtio_pci_msix_config_intr(void *);
128 static int virtio_pci_setup_interrupts_09(struct virtio_softc *, int);
129 static int virtio_pci_setup_interrupts_10(struct virtio_softc *, int);
130 static int virtio_pci_establish_msix_interrupts(struct virtio_softc *,
131 struct pci_attach_args *);
132 static int virtio_pci_establish_intx_interrupt(struct virtio_softc *,
133 struct pci_attach_args *);
134 static bool virtio_pci_msix_enabled(struct virtio_pci_softc *);
135
136 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0
137 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1
138
139 /*
140 * When using PCI attached virtio on aarch64-eb under Qemu, the IO space
141 * suddenly read BIG_ENDIAN where it should stay LITTLE_ENDIAN. The data read
142 * 1 byte at a time seem OK but reading bigger lengths result in swapped
143 * endian. This is most notable on reading 8 byters since we can't use
144 * bus_space_{read,write}_8().
145 */
146
147 #if defined(__aarch64__) && BYTE_ORDER == BIG_ENDIAN
148 # define READ_ENDIAN_09 BIG_ENDIAN /* should be LITTLE_ENDIAN */
149 # define READ_ENDIAN_10 BIG_ENDIAN
150 # define STRUCT_ENDIAN_09 BIG_ENDIAN
151 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN
152 #elif BYTE_ORDER == BIG_ENDIAN
153 # define READ_ENDIAN_09 LITTLE_ENDIAN
154 # define READ_ENDIAN_10 BIG_ENDIAN
155 # define STRUCT_ENDIAN_09 BIG_ENDIAN
156 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN
157 #else /* little endian */
158 # define READ_ENDIAN_09 LITTLE_ENDIAN
159 # define READ_ENDIAN_10 LITTLE_ENDIAN
160 # define STRUCT_ENDIAN_09 LITTLE_ENDIAN
161 # define STRUCT_ENDIAN_10 LITTLE_ENDIAN
162 #endif
163
164
165 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
166 virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
167 virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN);
168
169 static const struct virtio_ops virtio_pci_ops_09 = {
170 .kick = virtio_pci_kick_09,
171 .read_queue_size = virtio_pci_read_queue_size_09,
172 .setup_queue = virtio_pci_setup_queue_09,
173 .set_status = virtio_pci_set_status_09,
174 .neg_features = virtio_pci_negotiate_features_09,
175 .alloc_interrupts = virtio_pci_alloc_interrupts,
176 .free_interrupts = virtio_pci_free_interrupts,
177 .setup_interrupts = virtio_pci_setup_interrupts_09,
178 };
179
180 static const struct virtio_ops virtio_pci_ops_10 = {
181 .kick = virtio_pci_kick_10,
182 .read_queue_size = virtio_pci_read_queue_size_10,
183 .setup_queue = virtio_pci_setup_queue_10,
184 .set_status = virtio_pci_set_status_10,
185 .neg_features = virtio_pci_negotiate_features_10,
186 .alloc_interrupts = virtio_pci_alloc_interrupts,
187 .free_interrupts = virtio_pci_free_interrupts,
188 .setup_interrupts = virtio_pci_setup_interrupts_10,
189 };
190
191 static int
192 virtio_pci_match(device_t parent, cfdata_t match, void *aux)
193 {
194 struct pci_attach_args *pa;
195
196 pa = (struct pci_attach_args *)aux;
197 switch (PCI_VENDOR(pa->pa_id)) {
198 case PCI_VENDOR_QUMRANET:
199 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
200 PCI_PRODUCT(pa->pa_id)) &&
201 (PCI_PRODUCT(pa->pa_id) <=
202 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) &&
203 PCI_REVISION(pa->pa_class) == 0)
204 return 1;
205 if (((PCI_PRODUCT_QUMRANET_VIRTIO_1040 <=
206 PCI_PRODUCT(pa->pa_id)) &&
207 (PCI_PRODUCT(pa->pa_id) <=
208 PCI_PRODUCT_QUMRANET_VIRTIO_107F)) &&
209 PCI_REVISION(pa->pa_class) == 1)
210 return 1;
211 break;
212 }
213
214 return 0;
215 }
216
217 static void
218 virtio_pci_attach(device_t parent, device_t self, void *aux)
219 {
220 struct virtio_pci_softc * const psc = device_private(self);
221 struct virtio_softc * const sc = &psc->sc_sc;
222 struct pci_attach_args *pa = (struct pci_attach_args *)aux;
223 pci_chipset_tag_t pc = pa->pa_pc;
224 pcitag_t tag = pa->pa_tag;
225 int revision;
226 int ret;
227 pcireg_t id;
228 pcireg_t csr;
229
230 revision = PCI_REVISION(pa->pa_class);
231 switch (revision) {
232 case 0:
233 /* subsystem ID shows what I am */
234 id = PCI_SUBSYS_ID(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG));
235 break;
236 case 1:
237 /* pci product number shows what I am */
238 id = PCI_PRODUCT(pa->pa_id) - PCI_PRODUCT_QUMRANET_VIRTIO_1040;
239 break;
240 default:
241 aprint_normal(": unknown revision 0x%02x; giving up\n",
242 revision);
243 return;
244 }
245
246 aprint_normal("\n");
247 aprint_naive("\n");
248 virtio_print_device_type(self, id, revision);
249
250 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
251 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
252 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
253
254 sc->sc_dev = self;
255 psc->sc_pa = *pa;
256 psc->sc_iot = pa->pa_iot;
257
258 sc->sc_dmat = pa->pa_dmat;
259 if (pci_dma64_available(pa))
260 sc->sc_dmat = pa->pa_dmat64;
261
262 /* attach is dependent on revision */
263 ret = 0;
264 if (revision == 1) {
265 /* try to attach 1.0 */
266 ret = virtio_pci_attach_10(self, aux);
267 }
268 if (ret == 0 && revision == 0) {
269 /* revision 0 means 0.9 only or both 0.9 and 1.0 */
270 ret = virtio_pci_attach_09(self, aux);
271 }
272 if (ret) {
273 aprint_error_dev(self, "cannot attach (%d)\n", ret);
274 return;
275 }
276 KASSERT(sc->sc_ops);
277
278 /* preset config region */
279 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
280 if (virtio_pci_adjust_config_region(psc))
281 return;
282
283 /* generic */
284 virtio_device_reset(sc);
285 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
286 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
287
288 sc->sc_childdevid = id;
289 sc->sc_child = NULL;
290 virtio_pci_rescan(self, NULL, NULL);
291 return;
292 }
293
294 /* ARGSUSED */
295 static int
296 virtio_pci_rescan(device_t self, const char *ifattr, const int *locs)
297 {
298 struct virtio_pci_softc * const psc = device_private(self);
299 struct virtio_softc * const sc = &psc->sc_sc;
300 struct virtio_attach_args va;
301
302 if (sc->sc_child) /* Child already attached? */
303 return 0;
304
305 memset(&va, 0, sizeof(va));
306 va.sc_childdevid = sc->sc_childdevid;
307
308 config_found(self, &va, NULL, CFARGS_NONE);
309
310 if (virtio_attach_failed(sc))
311 return 0;
312
313 return 0;
314 }
315
316
317 static int
318 virtio_pci_detach(device_t self, int flags)
319 {
320 struct virtio_pci_softc * const psc = device_private(self);
321 struct virtio_softc * const sc = &psc->sc_sc;
322 int r;
323
324 if (sc->sc_child != NULL) {
325 r = config_detach(sc->sc_child, flags);
326 if (r)
327 return r;
328 }
329
330 /* Check that child detached properly */
331 KASSERT(sc->sc_child == NULL);
332 KASSERT(sc->sc_vqs == NULL);
333 KASSERT(psc->sc_ihs_num == 0);
334
335 if (psc->sc_iosize)
336 bus_space_unmap(psc->sc_iot, psc->sc_ioh,
337 psc->sc_mapped_iosize);
338 psc->sc_iosize = 0;
339
340 return 0;
341 }
342
343
344 static int
345 virtio_pci_attach_09(device_t self, void *aux)
346 //struct virtio_pci_softc *psc, struct pci_attach_args *pa)
347 {
348 struct virtio_pci_softc * const psc = device_private(self);
349 struct pci_attach_args *pa = (struct pci_attach_args *)aux;
350 struct virtio_softc * const sc = &psc->sc_sc;
351 // pci_chipset_tag_t pc = pa->pa_pc;
352 // pcitag_t tag = pa->pa_tag;
353
354 /* complete IO region */
355 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
356 &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
357 aprint_error_dev(self, "can't map i/o space\n");
358 return EIO;
359 }
360 psc->sc_mapped_iosize = psc->sc_iosize;
361
362 /* queue space */
363 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
364 VIRTIO_CONFIG_QUEUE_NOTIFY, 2, &psc->sc_notify_ioh)) {
365 aprint_error_dev(self, "can't map notify i/o space\n");
366 return EIO;
367 }
368 psc->sc_notify_iosize = 2;
369 psc->sc_notify_iot = psc->sc_iot;
370
371 /* ISR space */
372 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
373 VIRTIO_CONFIG_ISR_STATUS, 1, &psc->sc_isr_ioh)) {
374 aprint_error_dev(self, "can't map isr i/o space\n");
375 return EIO;
376 }
377 psc->sc_isr_iosize = 1;
378 psc->sc_isr_iot = psc->sc_iot;
379
380 /* set our version 0.9 ops */
381 sc->sc_ops = &virtio_pci_ops_09;
382 sc->sc_bus_endian = READ_ENDIAN_09;
383 sc->sc_struct_endian = STRUCT_ENDIAN_09;
384 return 0;
385 }
386
387
388 static int
389 virtio_pci_attach_10(device_t self, void *aux)
390 {
391 struct virtio_pci_softc * const psc = device_private(self);
392 struct pci_attach_args *pa = (struct pci_attach_args *)aux;
393 struct virtio_softc * const sc = &psc->sc_sc;
394 pci_chipset_tag_t pc = pa->pa_pc;
395 pcitag_t tag = pa->pa_tag;
396
397 struct virtio_pci_cap common, isr, device;
398 struct virtio_pci_notify_cap notify;
399 int have_device_cfg = 0;
400 bus_size_t bars[NMAPREG] = { 0 };
401 int bars_idx[NMAPREG] = { 0 };
402 struct virtio_pci_cap *caps[] = { &common, &isr, &device, ¬ify.cap };
403 int i, j, ret = 0;
404
405 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_COMMON_CFG,
406 &common, sizeof(common)))
407 return ENODEV;
408 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_NOTIFY_CFG,
409 ¬ify, sizeof(notify)))
410 return ENODEV;
411 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_ISR_CFG,
412 &isr, sizeof(isr)))
413 return ENODEV;
414 if (virtio_pci_find_cap(psc, VIRTIO_PCI_CAP_DEVICE_CFG,
415 &device, sizeof(device)))
416 memset(&device, 0, sizeof(device));
417 else
418 have_device_cfg = 1;
419
420 /* Figure out which bars we need to map */
421 for (i = 0; i < __arraycount(caps); i++) {
422 int bar = caps[i]->bar;
423 bus_size_t len = caps[i]->offset + caps[i]->length;
424 if (caps[i]->length == 0)
425 continue;
426 if (bars[bar] < len)
427 bars[bar] = len;
428 }
429
430 for (i = j = 0; i < __arraycount(bars); i++) {
431 int reg;
432 pcireg_t type;
433 if (bars[i] == 0)
434 continue;
435 reg = PCI_MAPREG_START + i * 4;
436 type = pci_mapreg_type(pc, tag, reg);
437 if (pci_mapreg_map(pa, reg, type, 0,
438 &psc->sc_bars_iot[j], &psc->sc_bars_ioh[j],
439 NULL, &psc->sc_bars_iosize[j])) {
440 aprint_error_dev(self, "can't map bar %u \n", i);
441 ret = EIO;
442 goto err;
443 }
444 aprint_debug_dev(self,
445 "bar[%d]: iot %p, size 0x%" PRIxBUSSIZE "\n",
446 j, psc->sc_bars_iot[j], psc->sc_bars_iosize[j]);
447 bars_idx[i] = j;
448 j++;
449 }
450
451 i = bars_idx[notify.cap.bar];
452 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
453 notify.cap.offset, notify.cap.length,
454 &psc->sc_notify_ioh)) {
455 aprint_error_dev(self, "can't map notify i/o space\n");
456 ret = EIO;
457 goto err;
458 }
459 psc->sc_notify_iosize = notify.cap.length;
460 psc->sc_notify_iot = psc->sc_bars_iot[i];
461 psc->sc_notify_off_multiplier = le32toh(notify.notify_off_multiplier);
462
463 if (have_device_cfg) {
464 i = bars_idx[device.bar];
465 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
466 device.offset, device.length,
467 &sc->sc_devcfg_ioh)) {
468 aprint_error_dev(self, "can't map devcfg i/o space\n");
469 ret = EIO;
470 goto err;
471 }
472 aprint_debug_dev(self,
473 "device.offset = 0x%x, device.length = 0x%x\n",
474 device.offset, device.length);
475 sc->sc_devcfg_iosize = device.length;
476 sc->sc_devcfg_iot = psc->sc_bars_iot[i];
477 }
478
479 i = bars_idx[isr.bar];
480 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
481 isr.offset, isr.length, &psc->sc_isr_ioh)) {
482 aprint_error_dev(self, "can't map isr i/o space\n");
483 ret = EIO;
484 goto err;
485 }
486 psc->sc_isr_iosize = isr.length;
487 psc->sc_isr_iot = psc->sc_bars_iot[i];
488
489 i = bars_idx[common.bar];
490 if (bus_space_subregion(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
491 common.offset, common.length, &psc->sc_ioh)) {
492 aprint_error_dev(self, "can't map common i/o space\n");
493 ret = EIO;
494 goto err;
495 }
496 psc->sc_iosize = common.length;
497 psc->sc_iot = psc->sc_bars_iot[i];
498 psc->sc_mapped_iosize = psc->sc_bars_iosize[i];
499
500 psc->sc_sc.sc_version_1 = 1;
501
502 /* set our version 1.0 ops */
503 sc->sc_ops = &virtio_pci_ops_10;
504 sc->sc_bus_endian = READ_ENDIAN_10;
505 sc->sc_struct_endian = STRUCT_ENDIAN_10;
506 return 0;
507
508 err:
509 /* undo our pci_mapreg_map()s */
510 for (i = 0; i < __arraycount(bars); i++) {
511 if (psc->sc_bars_iosize[i] == 0)
512 continue;
513 bus_space_unmap(psc->sc_bars_iot[i], psc->sc_bars_ioh[i],
514 psc->sc_bars_iosize[i]);
515 }
516 return ret;
517 }
518
519 /* v1.0 attach helper */
520 static int
521 virtio_pci_find_cap(struct virtio_pci_softc *psc, int cfg_type, void *buf, int buflen)
522 {
523 device_t self = psc->sc_sc.sc_dev;
524 pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
525 pcitag_t tag = psc->sc_pa.pa_tag;
526 unsigned int offset, i, len;
527 union {
528 pcireg_t reg[8];
529 struct virtio_pci_cap vcap;
530 } *v = buf;
531
532 if (buflen < sizeof(struct virtio_pci_cap))
533 return ERANGE;
534
535 if (!pci_get_capability(pc, tag, PCI_CAP_VENDSPEC, &offset, &v->reg[0]))
536 return ENOENT;
537
538 do {
539 for (i = 0; i < 4; i++)
540 v->reg[i] =
541 le32toh(pci_conf_read(pc, tag, offset + i * 4));
542 if (v->vcap.cfg_type == cfg_type)
543 break;
544 offset = v->vcap.cap_next;
545 } while (offset != 0);
546
547 if (offset == 0)
548 return ENOENT;
549
550 if (v->vcap.cap_len > sizeof(struct virtio_pci_cap)) {
551 len = roundup(v->vcap.cap_len, sizeof(pcireg_t));
552 if (len > buflen) {
553 aprint_error_dev(self, "%s cap too large\n", __func__);
554 return ERANGE;
555 }
556 for (i = 4; i < len / sizeof(pcireg_t); i++)
557 v->reg[i] =
558 le32toh(pci_conf_read(pc, tag, offset + i * 4));
559 }
560
561 /* endian fixup */
562 v->vcap.offset = le32toh(v->vcap.offset);
563 v->vcap.length = le32toh(v->vcap.length);
564 return 0;
565 }
566
567
568 /* -------------------------------------
569 * Version 0.9 support
570 * -------------------------------------*/
571
572 static void
573 virtio_pci_kick_09(struct virtio_softc *sc, uint16_t idx)
574 {
575 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
576
577 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, 0, idx);
578 }
579
580 /* only applicable for v 0.9 but also called for 1.0 */
581 static int
582 virtio_pci_adjust_config_region(struct virtio_pci_softc *psc)
583 {
584 struct virtio_softc * const sc = (struct virtio_softc *) psc;
585 device_t self = psc->sc_sc.sc_dev;
586
587 if (psc->sc_sc.sc_version_1)
588 return 0;
589
590 sc->sc_devcfg_iosize = psc->sc_iosize - psc->sc_devcfg_offset;
591 sc->sc_devcfg_iot = psc->sc_iot;
592 if (bus_space_subregion(psc->sc_iot, psc->sc_ioh,
593 psc->sc_devcfg_offset, sc->sc_devcfg_iosize,
594 &sc->sc_devcfg_ioh)) {
595 aprint_error_dev(self, "can't map config i/o space\n");
596 return EIO;
597 }
598
599 return 0;
600 }
601
602 static uint16_t
603 virtio_pci_read_queue_size_09(struct virtio_softc *sc, uint16_t idx)
604 {
605 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
606
607 bus_space_write_2(psc->sc_iot, psc->sc_ioh,
608 VIRTIO_CONFIG_QUEUE_SELECT, idx);
609 return bus_space_read_2(psc->sc_iot, psc->sc_ioh,
610 VIRTIO_CONFIG_QUEUE_SIZE);
611 }
612
613 static void
614 virtio_pci_setup_queue_09(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
615 {
616 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
617
618 bus_space_write_2(psc->sc_iot, psc->sc_ioh,
619 VIRTIO_CONFIG_QUEUE_SELECT, idx);
620 bus_space_write_4(psc->sc_iot, psc->sc_ioh,
621 VIRTIO_CONFIG_QUEUE_ADDRESS, addr / VIRTIO_PAGE_SIZE);
622
623 if (psc->sc_ihs_num > 1) {
624 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
625 if (sc->sc_child_mq)
626 vec += idx;
627 bus_space_write_2(psc->sc_iot, psc->sc_ioh,
628 VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
629 }
630 }
631
632 static void
633 virtio_pci_set_status_09(struct virtio_softc *sc, int status)
634 {
635 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
636 int old = 0;
637
638 if (status != 0) {
639 old = bus_space_read_1(psc->sc_iot, psc->sc_ioh,
640 VIRTIO_CONFIG_DEVICE_STATUS);
641 }
642 bus_space_write_1(psc->sc_iot, psc->sc_ioh,
643 VIRTIO_CONFIG_DEVICE_STATUS, status|old);
644 }
645
646 static void
647 virtio_pci_negotiate_features_09(struct virtio_softc *sc, uint64_t guest_features)
648 {
649 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
650 uint32_t r;
651
652 r = bus_space_read_4(psc->sc_iot, psc->sc_ioh,
653 VIRTIO_CONFIG_DEVICE_FEATURES);
654
655 r &= guest_features;
656
657 bus_space_write_4(psc->sc_iot, psc->sc_ioh,
658 VIRTIO_CONFIG_GUEST_FEATURES, r);
659
660 sc->sc_active_features = r;
661 }
662
663 /* -------------------------------------
664 * Version 1.0 support
665 * -------------------------------------*/
666
667 static void
668 virtio_pci_kick_10(struct virtio_softc *sc, uint16_t idx)
669 {
670 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
671 unsigned offset = sc->sc_vqs[idx].vq_notify_off *
672 psc->sc_notify_off_multiplier;
673
674 bus_space_write_2(psc->sc_notify_iot, psc->sc_notify_ioh, offset, idx);
675 }
676
677
678 static uint16_t
679 virtio_pci_read_queue_size_10(struct virtio_softc *sc, uint16_t idx)
680 {
681 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
682 bus_space_tag_t iot = psc->sc_iot;
683 bus_space_handle_t ioh = psc->sc_ioh;
684
685 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, idx);
686 return bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SIZE);
687 }
688
689 /*
690 * By definition little endian only in v1.0 and 8 byters are allowed to be
691 * written as two 4 byters
692 *
693 * This is not a general purpose function that can be used in any
694 * driver. Virtio specifically allows the 8 byte bus transaction
695 * to be split into two 4 byte transactions. Do not copy/use it
696 * in other device drivers unless you know that the device accepts it.
697 */
698 static __inline void
699 virtio_pci_bus_space_write_8(bus_space_tag_t iot, bus_space_handle_t ioh,
700 bus_size_t offset, uint64_t value)
701 {
702 #if defined(__HAVE_BUS_SPACE_8)
703 bus_space_write_8(iot, ioh, offset, value);
704 #elif _QUAD_HIGHWORD
705 bus_space_write_4(iot, ioh, offset, BUS_ADDR_LO32(value));
706 bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_HI32(value));
707 #else
708 bus_space_write_4(iot, ioh, offset, BUS_ADDR_HI32(value));
709 bus_space_write_4(iot, ioh, offset + 4, BUS_ADDR_LO32(value));
710 #endif
711 }
712
713 static void
714 virtio_pci_setup_queue_10(struct virtio_softc *sc, uint16_t idx, uint64_t addr)
715 {
716 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
717 struct virtqueue *vq = &sc->sc_vqs[idx];
718 bus_space_tag_t iot = psc->sc_iot;
719 bus_space_handle_t ioh = psc->sc_ioh;
720 KASSERT(vq->vq_index == idx);
721
722 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, vq->vq_index);
723 if (addr == 0) {
724 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_ENABLE, 0);
725 virtio_pci_bus_space_write_8(iot, ioh,
726 VIRTIO_CONFIG1_QUEUE_DESC, 0);
727 virtio_pci_bus_space_write_8(iot, ioh,
728 VIRTIO_CONFIG1_QUEUE_AVAIL, 0);
729 virtio_pci_bus_space_write_8(iot, ioh,
730 VIRTIO_CONFIG1_QUEUE_USED, 0);
731 } else {
732 virtio_pci_bus_space_write_8(iot, ioh,
733 VIRTIO_CONFIG1_QUEUE_DESC, addr);
734 virtio_pci_bus_space_write_8(iot, ioh,
735 VIRTIO_CONFIG1_QUEUE_AVAIL, addr + vq->vq_availoffset);
736 virtio_pci_bus_space_write_8(iot, ioh,
737 VIRTIO_CONFIG1_QUEUE_USED, addr + vq->vq_usedoffset);
738 bus_space_write_2(iot, ioh,
739 VIRTIO_CONFIG1_QUEUE_ENABLE, 1);
740 vq->vq_notify_off = bus_space_read_2(iot, ioh,
741 VIRTIO_CONFIG1_QUEUE_NOTIFY_OFF);
742 }
743
744 if (psc->sc_ihs_num > 1) {
745 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
746 if (sc->sc_child_mq)
747 vec += idx;
748 bus_space_write_2(iot, ioh,
749 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR, vec);
750 }
751 }
752
753 static void
754 virtio_pci_set_status_10(struct virtio_softc *sc, int status)
755 {
756 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
757 bus_space_tag_t iot = psc->sc_iot;
758 bus_space_handle_t ioh = psc->sc_ioh;
759 int old = 0;
760
761 if (status)
762 old = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS);
763 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS, status | old);
764 }
765
766 void
767 virtio_pci_negotiate_features_10(struct virtio_softc *sc, uint64_t guest_features)
768 {
769 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
770 device_t self = sc->sc_dev;
771 bus_space_tag_t iot = psc->sc_iot;
772 bus_space_handle_t ioh = psc->sc_ioh;
773 uint64_t host, negotiated, device_status;
774
775 guest_features |= VIRTIO_F_VERSION_1;
776 /* notify on empty is 0.9 only */
777 guest_features &= ~VIRTIO_F_NOTIFY_ON_EMPTY;
778 sc->sc_active_features = 0;
779
780 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 0);
781 host = bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE);
782 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE_SELECT, 1);
783 host |= (uint64_t)
784 bus_space_read_4(iot, ioh, VIRTIO_CONFIG1_DEVICE_FEATURE) << 32;
785
786 negotiated = host & guest_features;
787
788 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 0);
789 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
790 negotiated & 0xffffffff);
791 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE_SELECT, 1);
792 bus_space_write_4(iot, ioh, VIRTIO_CONFIG1_DRIVER_FEATURE,
793 negotiated >> 32);
794 virtio_pci_set_status_10(sc, VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK);
795
796 device_status = bus_space_read_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS);
797 if ((device_status & VIRTIO_CONFIG_DEVICE_STATUS_FEATURES_OK) == 0) {
798 aprint_error_dev(self, "feature negotiation failed\n");
799 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
800 VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
801 return;
802 }
803
804 if ((negotiated & VIRTIO_F_VERSION_1) == 0) {
805 aprint_error_dev(self, "host rejected version 1\n");
806 bus_space_write_1(iot, ioh, VIRTIO_CONFIG1_DEVICE_STATUS,
807 VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
808 return;
809 }
810
811 sc->sc_active_features = negotiated;
812 return;
813 }
814
815
816 /* -------------------------------------
817 * Generic PCI interrupt code
818 * -------------------------------------*/
819
820 static int
821 virtio_pci_setup_interrupts_10(struct virtio_softc *sc, int reinit)
822 {
823 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
824 bus_space_tag_t iot = psc->sc_iot;
825 bus_space_handle_t ioh = psc->sc_ioh;
826 int vector, ret, qid;
827
828 if (!virtio_pci_msix_enabled(psc))
829 return 0;
830
831 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
832 bus_space_write_2(iot, ioh,
833 VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR, vector);
834 ret = bus_space_read_2(iot, ioh, VIRTIO_CONFIG1_CONFIG_MSIX_VECTOR);
835 if (ret != vector) {
836 VIRTIO_PCI_LOG(sc, reinit,
837 "can't set config msix vector\n");
838 return -1;
839 }
840
841 for (qid = 0; qid < sc->sc_nvqs; qid++) {
842 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
843
844 if (sc->sc_child_mq)
845 vector += qid;
846 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_SELECT, qid);
847 bus_space_write_2(iot, ioh, VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR,
848 vector);
849 ret = bus_space_read_2(iot, ioh,
850 VIRTIO_CONFIG1_QUEUE_MSIX_VECTOR);
851 if (ret != vector) {
852 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d "
853 "msix vector\n", qid);
854 return -1;
855 }
856 }
857
858 return 0;
859 }
860
861 static int
862 virtio_pci_setup_interrupts_09(struct virtio_softc *sc, int reinit)
863 {
864 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
865 int offset, vector, ret, qid;
866
867 if (!virtio_pci_msix_enabled(psc))
868 return 0;
869
870 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
871 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
872
873 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
874 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
875 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
876 vector, ret);
877 if (ret != vector) {
878 VIRTIO_PCI_LOG(sc, reinit,
879 "can't set config msix vector\n");
880 return -1;
881 }
882
883 for (qid = 0; qid < sc->sc_nvqs; qid++) {
884 offset = VIRTIO_CONFIG_QUEUE_SELECT;
885 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, qid);
886
887 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
888 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
889
890 if (sc->sc_child_mq)
891 vector += qid;
892
893 bus_space_write_2(psc->sc_iot, psc->sc_ioh, offset, vector);
894 ret = bus_space_read_2(psc->sc_iot, psc->sc_ioh, offset);
895 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
896 vector, ret);
897 if (ret != vector) {
898 VIRTIO_PCI_LOG(sc, reinit, "can't set queue %d "
899 "msix vector\n", qid);
900 return -1;
901 }
902 }
903
904 return 0;
905 }
906
907 static int
908 virtio_pci_establish_msix_interrupts(struct virtio_softc *sc,
909 struct pci_attach_args *pa)
910 {
911 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
912 device_t self = sc->sc_dev;
913 pci_chipset_tag_t pc = pa->pa_pc;
914 struct virtqueue *vq;
915 char intrbuf[PCI_INTRSTR_LEN];
916 char intr_xname[INTRDEVNAMEBUF];
917 char const *intrstr;
918 int idx, qid, n;
919
920 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
921 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
922 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
923
924 snprintf(intr_xname, sizeof(intr_xname), "%s config",
925 device_xname(sc->sc_dev));
926
927 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
928 sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
929 if (psc->sc_ihs[idx] == NULL) {
930 aprint_error_dev(self, "couldn't establish MSI-X for config\n");
931 goto error;
932 }
933
934 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
935 if (sc->sc_child_mq) {
936 for (qid = 0; qid < sc->sc_nvqs; qid++) {
937 n = idx + qid;
938 vq = &sc->sc_vqs[qid];
939
940 snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
941 device_xname(sc->sc_dev), qid);
942
943 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE) {
944 pci_intr_setattr(pc, &psc->sc_ihp[n],
945 PCI_INTR_MPSAFE, true);
946 }
947
948 psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n],
949 sc->sc_ipl, vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname);
950 if (psc->sc_ihs[n] == NULL) {
951 aprint_error_dev(self, "couldn't establish MSI-X for a vq\n");
952 goto error;
953 }
954 }
955 } else {
956 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
957 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
958
959 snprintf(intr_xname, sizeof(intr_xname), "%s queues",
960 device_xname(sc->sc_dev));
961 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
962 sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname);
963 if (psc->sc_ihs[idx] == NULL) {
964 aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
965 goto error;
966 }
967 }
968
969 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
970 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
971 aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
972 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
973 if (sc->sc_child_mq) {
974 kcpuset_t *affinity;
975 int affinity_to, r;
976
977 kcpuset_create(&affinity, false);
978
979 for (qid = 0; qid < sc->sc_nvqs; qid++) {
980 n = idx + qid;
981 affinity_to = (qid / 2) % ncpu;
982
983 intrstr = pci_intr_string(pc, psc->sc_ihp[n],
984 intrbuf, sizeof(intrbuf));
985
986 kcpuset_zero(affinity);
987 kcpuset_set(affinity, affinity_to);
988 r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL);
989 if (r == 0) {
990 aprint_normal_dev(self,
991 "for vq #%d interrupting at %s affinity to %u\n",
992 qid, intrstr, affinity_to);
993 } else {
994 aprint_normal_dev(self,
995 "for vq #%d interrupting at %s\n",
996 qid, intrstr);
997 }
998 }
999
1000 kcpuset_destroy(affinity);
1001 } else {
1002 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
1003 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
1004 }
1005
1006 return 0;
1007
1008 error:
1009 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
1010 if (psc->sc_ihs[idx] != NULL)
1011 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
1012 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
1013 if (sc->sc_child_mq) {
1014 for (qid = 0; qid < sc->sc_nvqs; qid++) {
1015 n = idx + qid;
1016 if (psc->sc_ihs[n] == NULL)
1017 continue;
1018 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]);
1019 }
1020
1021 } else {
1022 if (psc->sc_ihs[idx] != NULL)
1023 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
1024 }
1025
1026 return -1;
1027 }
1028
1029 static int
1030 virtio_pci_establish_intx_interrupt(struct virtio_softc *sc,
1031 struct pci_attach_args *pa)
1032 {
1033 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
1034 device_t self = sc->sc_dev;
1035 pci_chipset_tag_t pc = pa->pa_pc;
1036 char intrbuf[PCI_INTRSTR_LEN];
1037 char const *intrstr;
1038
1039 if (sc->sc_flags & VIRTIO_F_INTR_MPSAFE)
1040 pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
1041
1042 psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
1043 sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
1044 if (psc->sc_ihs[0] == NULL) {
1045 aprint_error_dev(self, "couldn't establish INTx\n");
1046 return -1;
1047 }
1048
1049 intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf));
1050 aprint_normal_dev(self, "interrupting at %s\n", intrstr);
1051
1052 return 0;
1053 }
1054
1055 static int
1056 virtio_pci_alloc_interrupts(struct virtio_softc *sc)
1057 {
1058 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
1059 device_t self = sc->sc_dev;
1060 pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
1061 pcitag_t tag = psc->sc_pa.pa_tag;
1062 int error;
1063 int nmsix;
1064 int off;
1065 int counts[PCI_INTR_TYPE_SIZE];
1066 pci_intr_type_t max_type;
1067 pcireg_t ctl;
1068
1069 nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
1070 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
1071
1072 /* We need at least two: one for config and the other for queues */
1073 if ((sc->sc_flags & VIRTIO_F_INTR_MSIX) == 0 || nmsix < 2) {
1074 /* Try INTx only */
1075 max_type = PCI_INTR_TYPE_INTX;
1076 counts[PCI_INTR_TYPE_INTX] = 1;
1077 } else {
1078 /* Try MSI-X first and INTx second */
1079 if (sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) {
1080 nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
1081 } else {
1082 sc->sc_child_mq = false;
1083 }
1084
1085 if (sc->sc_child_mq == false) {
1086 nmsix = 2;
1087 }
1088
1089 max_type = PCI_INTR_TYPE_MSIX;
1090 counts[PCI_INTR_TYPE_MSIX] = nmsix;
1091 counts[PCI_INTR_TYPE_MSI] = 0;
1092 counts[PCI_INTR_TYPE_INTX] = 1;
1093 }
1094
1095 retry:
1096 error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
1097 if (error != 0) {
1098 aprint_error_dev(self, "couldn't map interrupt\n");
1099 return -1;
1100 }
1101
1102 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
1103 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix,
1104 KM_SLEEP);
1105
1106 error = virtio_pci_establish_msix_interrupts(sc, &psc->sc_pa);
1107 if (error != 0) {
1108 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
1109 pci_intr_release(pc, psc->sc_ihp, nmsix);
1110
1111 /* Retry INTx */
1112 max_type = PCI_INTR_TYPE_INTX;
1113 counts[PCI_INTR_TYPE_INTX] = 1;
1114 goto retry;
1115 }
1116
1117 psc->sc_ihs_num = nmsix;
1118 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
1119 virtio_pci_adjust_config_region(psc);
1120 } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
1121 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1,
1122 KM_SLEEP);
1123
1124 error = virtio_pci_establish_intx_interrupt(sc, &psc->sc_pa);
1125 if (error != 0) {
1126 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
1127 pci_intr_release(pc, psc->sc_ihp, 1);
1128 return -1;
1129 }
1130
1131 psc->sc_ihs_num = 1;
1132 psc->sc_devcfg_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
1133 virtio_pci_adjust_config_region(psc);
1134
1135 error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
1136 if (error != 0) {
1137 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
1138 ctl &= ~PCI_MSIX_CTL_ENABLE;
1139 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
1140 }
1141 }
1142
1143 return 0;
1144 }
1145
1146 static void
1147 virtio_pci_free_interrupts(struct virtio_softc *sc)
1148 {
1149 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
1150
1151 for (int i = 0; i < psc->sc_ihs_num; i++) {
1152 if (psc->sc_ihs[i] == NULL)
1153 continue;
1154 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
1155 psc->sc_ihs[i] = NULL;
1156 }
1157
1158 if (psc->sc_ihs_num > 0)
1159 pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num);
1160
1161 if (psc->sc_ihs != NULL) {
1162 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
1163 psc->sc_ihs = NULL;
1164 }
1165 psc->sc_ihs_num = 0;
1166 }
1167
1168 static bool
1169 virtio_pci_msix_enabled(struct virtio_pci_softc *psc)
1170 {
1171 pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
1172
1173 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX)
1174 return true;
1175
1176 return false;
1177 }
1178
1179 /*
1180 * Interrupt handler.
1181 */
1182 static int
1183 virtio_pci_intr(void *arg)
1184 {
1185 struct virtio_softc *sc = arg;
1186 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
1187 int isr, r = 0;
1188
1189 /* check and ack the interrupt */
1190 isr = bus_space_read_1(psc->sc_isr_iot, psc->sc_isr_ioh, 0);
1191 if (isr == 0)
1192 return 0;
1193 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
1194 (sc->sc_config_change != NULL))
1195 r = (sc->sc_config_change)(sc);
1196 if (sc->sc_intrhand != NULL) {
1197 if (sc->sc_soft_ih != NULL)
1198 softint_schedule(sc->sc_soft_ih);
1199 else
1200 r |= (sc->sc_intrhand)(sc);
1201 }
1202
1203 return r;
1204 }
1205
1206 static int
1207 virtio_pci_msix_queue_intr(void *arg)
1208 {
1209 struct virtio_softc *sc = arg;
1210 int r = 0;
1211
1212 if (sc->sc_intrhand != NULL) {
1213 if (sc->sc_soft_ih != NULL)
1214 softint_schedule(sc->sc_soft_ih);
1215 else
1216 r |= (sc->sc_intrhand)(sc);
1217 }
1218
1219 return r;
1220 }
1221
1222 static int
1223 virtio_pci_msix_config_intr(void *arg)
1224 {
1225 struct virtio_softc *sc = arg;
1226 int r = 0;
1227
1228 if (sc->sc_config_change != NULL)
1229 r = (sc->sc_config_change)(sc);
1230 return r;
1231 }
1232
1233 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
1234
1235 #ifdef _MODULE
1236 #include "ioconf.c"
1237 #endif
1238
1239 static int
1240 virtio_pci_modcmd(modcmd_t cmd, void *opaque)
1241 {
1242 int error = 0;
1243
1244 #ifdef _MODULE
1245 switch (cmd) {
1246 case MODULE_CMD_INIT:
1247 error = config_init_component(cfdriver_ioconf_virtio_pci,
1248 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
1249 break;
1250 case MODULE_CMD_FINI:
1251 error = config_fini_component(cfdriver_ioconf_virtio_pci,
1252 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
1253 break;
1254 default:
1255 error = ENOTTY;
1256 break;
1257 }
1258 #endif
1259
1260 return error;
1261 }
1262