virtio_pci.c revision 1.2.2.2 1 1.2.2.2 pgoyette /* $NetBSD: virtio_pci.c,v 1.2.2.2 2019/01/18 08:50:42 pgoyette Exp $ */
2 1.1 cherry
3 1.1 cherry /*
4 1.1 cherry * Copyright (c) 2010 Minoura Makoto.
5 1.1 cherry * All rights reserved.
6 1.1 cherry *
7 1.1 cherry * Redistribution and use in source and binary forms, with or without
8 1.1 cherry * modification, are permitted provided that the following conditions
9 1.1 cherry * are met:
10 1.1 cherry * 1. Redistributions of source code must retain the above copyright
11 1.1 cherry * notice, this list of conditions and the following disclaimer.
12 1.1 cherry * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 cherry * notice, this list of conditions and the following disclaimer in the
14 1.1 cherry * documentation and/or other materials provided with the distribution.
15 1.1 cherry *
16 1.1 cherry * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 1.1 cherry * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 1.1 cherry * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 1.1 cherry * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 1.1 cherry * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 1.1 cherry * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 1.1 cherry * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 1.1 cherry * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 1.1 cherry * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 1.1 cherry * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 1.1 cherry */
27 1.1 cherry
28 1.1 cherry #include <sys/cdefs.h>
29 1.2.2.2 pgoyette __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.2.2.2 2019/01/18 08:50:42 pgoyette Exp $");
30 1.1 cherry
31 1.1 cherry #include <sys/param.h>
32 1.1 cherry #include <sys/systm.h>
33 1.2.2.1 pgoyette #include <sys/kmem.h>
34 1.2.2.1 pgoyette #include <sys/module.h>
35 1.2.2.2 pgoyette #include <sys/interrupt.h>
36 1.1 cherry
37 1.1 cherry #include <sys/device.h>
38 1.1 cherry
39 1.1 cherry #include <dev/pci/pcidevs.h>
40 1.1 cherry #include <dev/pci/pcireg.h>
41 1.1 cherry #include <dev/pci/pcivar.h>
42 1.1 cherry
43 1.1 cherry #define VIRTIO_PRIVATE
44 1.1 cherry
45 1.1 cherry #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */
46 1.1 cherry #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */
47 1.1 cherry
48 1.2.2.1 pgoyette static int virtio_pci_match(device_t, cfdata_t, void *);
49 1.2.2.1 pgoyette static void virtio_pci_attach(device_t, device_t, void *);
50 1.2.2.1 pgoyette static int virtio_pci_rescan(device_t, const char *, const int *);
51 1.2.2.1 pgoyette static int virtio_pci_detach(device_t, int);
52 1.2.2.1 pgoyette
53 1.2.2.1 pgoyette struct virtio_pci_softc {
54 1.2.2.1 pgoyette struct virtio_softc sc_sc;
55 1.2.2.1 pgoyette bus_space_tag_t sc_iot;
56 1.2.2.1 pgoyette bus_space_handle_t sc_ioh;
57 1.2.2.1 pgoyette bus_size_t sc_iosize;
58 1.2.2.1 pgoyette struct pci_attach_args sc_pa;
59 1.2.2.1 pgoyette pci_intr_handle_t *sc_ihp;
60 1.2.2.1 pgoyette void **sc_ihs;
61 1.2.2.1 pgoyette int sc_ihs_num;
62 1.2.2.1 pgoyette int sc_config_offset;
63 1.2.2.1 pgoyette };
64 1.2.2.1 pgoyette
65 1.2.2.1 pgoyette static void virtio_pci_kick(struct virtio_softc *, uint16_t);
66 1.2.2.1 pgoyette static uint8_t virtio_pci_read_device_config_1(struct virtio_softc *, int);
67 1.2.2.1 pgoyette static uint16_t virtio_pci_read_device_config_2(struct virtio_softc *, int);
68 1.2.2.1 pgoyette static uint32_t virtio_pci_read_device_config_4(struct virtio_softc *, int);
69 1.2.2.1 pgoyette static uint64_t virtio_pci_read_device_config_8(struct virtio_softc *, int);
70 1.2.2.1 pgoyette static void virtio_pci_write_device_config_1(struct virtio_softc *, int, uint8_t);
71 1.2.2.1 pgoyette static void virtio_pci_write_device_config_2(struct virtio_softc *, int, uint16_t);
72 1.2.2.1 pgoyette static void virtio_pci_write_device_config_4(struct virtio_softc *, int, uint32_t);
73 1.2.2.1 pgoyette static void virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t);
74 1.2.2.1 pgoyette static uint16_t virtio_pci_read_queue_size(struct virtio_softc *, uint16_t);
75 1.2.2.1 pgoyette static void virtio_pci_setup_queue(struct virtio_softc *, uint16_t, uint32_t);
76 1.2.2.1 pgoyette static void virtio_pci_set_status(struct virtio_softc *, int);
77 1.2.2.1 pgoyette static uint32_t virtio_pci_negotiate_features(struct virtio_softc *, uint32_t);
78 1.2.2.1 pgoyette static int virtio_pci_setup_interrupts(struct virtio_softc *);
79 1.2.2.1 pgoyette static void virtio_pci_free_interrupts(struct virtio_softc *);
80 1.2.2.1 pgoyette
81 1.2.2.1 pgoyette static int virtio_pci_intr(void *arg);
82 1.2.2.1 pgoyette static int virtio_pci_msix_queue_intr(void *);
83 1.2.2.2 pgoyette static int virtio_pci_msix_vq_intr(void *);
84 1.2.2.1 pgoyette static int virtio_pci_msix_config_intr(void *);
85 1.2.2.1 pgoyette static int virtio_pci_setup_msix_vectors(struct virtio_softc *);
86 1.2.2.1 pgoyette static int virtio_pci_setup_msix_interrupts(struct virtio_softc *,
87 1.2.2.1 pgoyette struct pci_attach_args *);
88 1.2.2.1 pgoyette static int virtio_pci_setup_intx_interrupt(struct virtio_softc *,
89 1.2.2.1 pgoyette struct pci_attach_args *);
90 1.2.2.1 pgoyette
91 1.2.2.1 pgoyette #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0
92 1.2.2.1 pgoyette #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1
93 1.2.2.1 pgoyette
94 1.2.2.1 pgoyette /* we use the legacy virtio spec, so the PCI registers are host native
95 1.2.2.1 pgoyette * byte order, not PCI (i.e. LE) byte order */
96 1.2.2.1 pgoyette #if BYTE_ORDER == BIG_ENDIAN
97 1.2.2.1 pgoyette #define REG_HI_OFF 0
98 1.2.2.1 pgoyette #define REG_LO_OFF 4
99 1.2.2.1 pgoyette #ifndef __BUS_SPACE_HAS_STREAM_METHODS
100 1.2.2.1 pgoyette #define bus_space_read_stream_1 bus_space_read_1
101 1.2.2.1 pgoyette #define bus_space_write_stream_1 bus_space_write_1
102 1.2.2.1 pgoyette static inline uint16_t
103 1.2.2.1 pgoyette bus_space_read_stream_2(bus_space_tag_t t, bus_space_handle_t h,
104 1.2.2.1 pgoyette bus_size_t o)
105 1.2.2.1 pgoyette {
106 1.2.2.1 pgoyette return le16toh(bus_space_read_2(t, h, o));
107 1.2.2.1 pgoyette }
108 1.2.2.1 pgoyette static inline void
109 1.2.2.1 pgoyette bus_space_write_stream_2(bus_space_tag_t t, bus_space_handle_t h,
110 1.2.2.1 pgoyette bus_size_t o, uint16_t v)
111 1.2.2.1 pgoyette {
112 1.2.2.1 pgoyette bus_space_write_2(t, h, o, htole16(v));
113 1.2.2.1 pgoyette }
114 1.2.2.1 pgoyette static inline uint32_t
115 1.2.2.1 pgoyette bus_space_read_stream_4(bus_space_tag_t t, bus_space_handle_t h,
116 1.2.2.1 pgoyette bus_size_t o)
117 1.2.2.1 pgoyette {
118 1.2.2.1 pgoyette return le32toh(bus_space_read_4(t, h, o));
119 1.2.2.1 pgoyette }
120 1.2.2.1 pgoyette static inline void
121 1.2.2.1 pgoyette bus_space_write_stream_4(bus_space_tag_t t, bus_space_handle_t h,
122 1.2.2.1 pgoyette bus_size_t o, uint32_t v)
123 1.2.2.1 pgoyette {
124 1.2.2.1 pgoyette bus_space_write_4(t, h, o, htole32(v));
125 1.2.2.1 pgoyette }
126 1.2.2.1 pgoyette #endif
127 1.2.2.1 pgoyette #else
128 1.2.2.1 pgoyette #define REG_HI_OFF 4
129 1.2.2.1 pgoyette #define REG_LO_OFF 0
130 1.2.2.1 pgoyette #ifndef __BUS_SPACE_HAS_STREAM_METHODS
131 1.2.2.1 pgoyette #define bus_space_read_stream_1 bus_space_read_1
132 1.2.2.1 pgoyette #define bus_space_read_stream_2 bus_space_read_2
133 1.2.2.1 pgoyette #define bus_space_read_stream_4 bus_space_read_4
134 1.2.2.1 pgoyette #define bus_space_write_stream_1 bus_space_write_1
135 1.2.2.1 pgoyette #define bus_space_write_stream_2 bus_space_write_2
136 1.2.2.1 pgoyette #define bus_space_write_stream_4 bus_space_write_4
137 1.2.2.1 pgoyette #endif
138 1.2.2.1 pgoyette #endif
139 1.2.2.1 pgoyette
140 1.1 cherry
141 1.1 cherry static const char *virtio_device_name[] = {
142 1.1 cherry "Unknown (0)", /* 0 */
143 1.1 cherry "Network", /* 1 */
144 1.1 cherry "Block", /* 2 */
145 1.1 cherry "Console", /* 3 */
146 1.1 cherry "Entropy", /* 4 */
147 1.1 cherry "Memory Balloon", /* 5 */
148 1.1 cherry "I/O Memory", /* 6 */
149 1.1 cherry "Remote Processor Messaging", /* 7 */
150 1.1 cherry "SCSI", /* 8 */
151 1.1 cherry "9P Transport", /* 9 */
152 1.1 cherry "mac80211 wlan", /* 10 */
153 1.1 cherry };
154 1.1 cherry #define NDEVNAMES __arraycount(virtio_device_name)
155 1.1 cherry
156 1.2.2.1 pgoyette CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc),
157 1.2.2.1 pgoyette virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL,
158 1.2.2.1 pgoyette virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN);
159 1.2.2.1 pgoyette
160 1.2.2.1 pgoyette static const struct virtio_ops virtio_pci_ops = {
161 1.2.2.1 pgoyette .kick = virtio_pci_kick,
162 1.2.2.1 pgoyette .read_dev_cfg_1 = virtio_pci_read_device_config_1,
163 1.2.2.1 pgoyette .read_dev_cfg_2 = virtio_pci_read_device_config_2,
164 1.2.2.1 pgoyette .read_dev_cfg_4 = virtio_pci_read_device_config_4,
165 1.2.2.1 pgoyette .read_dev_cfg_8 = virtio_pci_read_device_config_8,
166 1.2.2.1 pgoyette .write_dev_cfg_1 = virtio_pci_write_device_config_1,
167 1.2.2.1 pgoyette .write_dev_cfg_2 = virtio_pci_write_device_config_2,
168 1.2.2.1 pgoyette .write_dev_cfg_4 = virtio_pci_write_device_config_4,
169 1.2.2.1 pgoyette .write_dev_cfg_8 = virtio_pci_write_device_config_8,
170 1.2.2.1 pgoyette .read_queue_size = virtio_pci_read_queue_size,
171 1.2.2.1 pgoyette .setup_queue = virtio_pci_setup_queue,
172 1.2.2.1 pgoyette .set_status = virtio_pci_set_status,
173 1.2.2.1 pgoyette .neg_features = virtio_pci_negotiate_features,
174 1.2.2.1 pgoyette .setup_interrupts = virtio_pci_setup_interrupts,
175 1.2.2.1 pgoyette .free_interrupts = virtio_pci_free_interrupts,
176 1.2.2.1 pgoyette };
177 1.1 cherry
178 1.1 cherry static int
179 1.2.2.1 pgoyette virtio_pci_match(device_t parent, cfdata_t match, void *aux)
180 1.1 cherry {
181 1.1 cherry struct pci_attach_args *pa;
182 1.1 cherry
183 1.1 cherry pa = (struct pci_attach_args *)aux;
184 1.1 cherry switch (PCI_VENDOR(pa->pa_id)) {
185 1.1 cherry case PCI_VENDOR_QUMRANET:
186 1.1 cherry if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <=
187 1.1 cherry PCI_PRODUCT(pa->pa_id)) &&
188 1.1 cherry (PCI_PRODUCT(pa->pa_id) <=
189 1.1 cherry PCI_PRODUCT_QUMRANET_VIRTIO_103F))
190 1.1 cherry return 1;
191 1.1 cherry break;
192 1.1 cherry }
193 1.1 cherry
194 1.1 cherry return 0;
195 1.1 cherry }
196 1.1 cherry
197 1.1 cherry static void
198 1.2.2.1 pgoyette virtio_pci_attach(device_t parent, device_t self, void *aux)
199 1.1 cherry {
200 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = device_private(self);
201 1.2.2.1 pgoyette struct virtio_softc * const sc = &psc->sc_sc;
202 1.1 cherry struct pci_attach_args *pa = (struct pci_attach_args *)aux;
203 1.1 cherry pci_chipset_tag_t pc = pa->pa_pc;
204 1.1 cherry pcitag_t tag = pa->pa_tag;
205 1.1 cherry int revision;
206 1.1 cherry pcireg_t id;
207 1.2 uwe pcireg_t csr;
208 1.1 cherry
209 1.1 cherry revision = PCI_REVISION(pa->pa_class);
210 1.1 cherry if (revision != 0) {
211 1.1 cherry aprint_normal(": unknown revision 0x%02x; giving up\n",
212 1.1 cherry revision);
213 1.1 cherry return;
214 1.1 cherry }
215 1.1 cherry aprint_normal("\n");
216 1.1 cherry aprint_naive("\n");
217 1.1 cherry
218 1.1 cherry /* subsystem ID shows what I am */
219 1.1 cherry id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG);
220 1.1 cherry aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n",
221 1.1 cherry (PCI_SUBSYS_ID(id) < NDEVNAMES?
222 1.1 cherry virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"),
223 1.1 cherry revision);
224 1.1 cherry
225 1.2 uwe csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
226 1.2 uwe csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE;
227 1.2 uwe pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
228 1.2 uwe
229 1.1 cherry sc->sc_dev = self;
230 1.2.2.1 pgoyette sc->sc_ops = &virtio_pci_ops;
231 1.2.2.1 pgoyette psc->sc_pa = *pa;
232 1.2.2.1 pgoyette psc->sc_iot = pa->pa_iot;
233 1.1 cherry if (pci_dma64_available(pa))
234 1.1 cherry sc->sc_dmat = pa->pa_dmat64;
235 1.1 cherry else
236 1.1 cherry sc->sc_dmat = pa->pa_dmat;
237 1.2.2.1 pgoyette psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
238 1.1 cherry
239 1.1 cherry if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
240 1.2.2.1 pgoyette &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) {
241 1.1 cherry aprint_error_dev(self, "can't map i/o space\n");
242 1.1 cherry return;
243 1.1 cherry }
244 1.1 cherry
245 1.1 cherry virtio_device_reset(sc);
246 1.1 cherry virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
247 1.1 cherry virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
248 1.1 cherry
249 1.1 cherry sc->sc_childdevid = PCI_SUBSYS_ID(id);
250 1.1 cherry sc->sc_child = NULL;
251 1.2.2.1 pgoyette virtio_pci_rescan(self, "virtio", 0);
252 1.1 cherry return;
253 1.1 cherry }
254 1.1 cherry
255 1.1 cherry /* ARGSUSED */
256 1.1 cherry static int
257 1.2.2.1 pgoyette virtio_pci_rescan(device_t self, const char *attr, const int *scan_flags)
258 1.1 cherry {
259 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = device_private(self);
260 1.2.2.1 pgoyette struct virtio_softc * const sc = &psc->sc_sc;
261 1.1 cherry struct virtio_attach_args va;
262 1.1 cherry
263 1.1 cherry if (sc->sc_child) /* Child already attached? */
264 1.1 cherry return 0;
265 1.1 cherry
266 1.1 cherry memset(&va, 0, sizeof(va));
267 1.1 cherry va.sc_childdevid = sc->sc_childdevid;
268 1.1 cherry
269 1.1 cherry config_found_ia(self, attr, &va, NULL);
270 1.1 cherry
271 1.1 cherry if (sc->sc_child == NULL) {
272 1.1 cherry aprint_error_dev(self,
273 1.1 cherry "no matching child driver; not configured\n");
274 1.1 cherry return 0;
275 1.1 cherry }
276 1.2.2.1 pgoyette
277 1.1 cherry if (sc->sc_child == VIRTIO_CHILD_FAILED) {
278 1.1 cherry aprint_error_dev(self,
279 1.1 cherry "virtio configuration failed\n");
280 1.1 cherry return 0;
281 1.1 cherry }
282 1.1 cherry
283 1.1 cherry /*
284 1.1 cherry * Make sure child drivers initialize interrupts via call
285 1.1 cherry * to virtio_child_attach_finish().
286 1.1 cherry */
287 1.2.2.1 pgoyette KASSERT(psc->sc_ihs_num != 0);
288 1.1 cherry
289 1.1 cherry return 0;
290 1.1 cherry }
291 1.1 cherry
292 1.1 cherry
293 1.1 cherry static int
294 1.2.2.1 pgoyette virtio_pci_detach(device_t self, int flags)
295 1.1 cherry {
296 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = device_private(self);
297 1.2.2.1 pgoyette struct virtio_softc * const sc = &psc->sc_sc;
298 1.1 cherry int r;
299 1.1 cherry
300 1.1 cherry if (sc->sc_child != NULL) {
301 1.1 cherry r = config_detach(sc->sc_child, flags);
302 1.1 cherry if (r)
303 1.1 cherry return r;
304 1.1 cherry }
305 1.1 cherry
306 1.1 cherry /* Check that child detached properly */
307 1.1 cherry KASSERT(sc->sc_child == NULL);
308 1.1 cherry KASSERT(sc->sc_vqs == NULL);
309 1.2.2.1 pgoyette KASSERT(psc->sc_ihs_num == 0);
310 1.2.2.1 pgoyette
311 1.2.2.1 pgoyette if (psc->sc_iosize)
312 1.2.2.1 pgoyette bus_space_unmap(psc->sc_iot, psc->sc_ioh, psc->sc_iosize);
313 1.2.2.1 pgoyette psc->sc_iosize = 0;
314 1.2.2.1 pgoyette
315 1.2.2.1 pgoyette return 0;
316 1.2.2.1 pgoyette }
317 1.2.2.1 pgoyette
318 1.2.2.1 pgoyette static void
319 1.2.2.1 pgoyette virtio_pci_kick(struct virtio_softc *sc, uint16_t idx)
320 1.2.2.1 pgoyette {
321 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
322 1.2.2.1 pgoyette
323 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
324 1.2.2.1 pgoyette VIRTIO_CONFIG_QUEUE_NOTIFY, idx);
325 1.2.2.1 pgoyette }
326 1.2.2.1 pgoyette
327 1.2.2.1 pgoyette static uint8_t
328 1.2.2.1 pgoyette virtio_pci_read_device_config_1(struct virtio_softc *sc, int index)
329 1.2.2.1 pgoyette {
330 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
331 1.2.2.1 pgoyette return bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
332 1.2.2.1 pgoyette psc->sc_config_offset + index);
333 1.2.2.1 pgoyette }
334 1.2.2.1 pgoyette
335 1.2.2.1 pgoyette static uint16_t
336 1.2.2.1 pgoyette virtio_pci_read_device_config_2(struct virtio_softc *sc, int index)
337 1.2.2.1 pgoyette {
338 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
339 1.2.2.1 pgoyette return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
340 1.2.2.1 pgoyette psc->sc_config_offset + index);
341 1.2.2.1 pgoyette }
342 1.2.2.1 pgoyette
343 1.2.2.1 pgoyette static uint32_t
344 1.2.2.1 pgoyette virtio_pci_read_device_config_4(struct virtio_softc *sc, int index)
345 1.2.2.1 pgoyette {
346 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
347 1.2.2.1 pgoyette return bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
348 1.2.2.1 pgoyette psc->sc_config_offset + index);
349 1.2.2.1 pgoyette }
350 1.2.2.1 pgoyette
351 1.2.2.1 pgoyette static uint64_t
352 1.2.2.1 pgoyette virtio_pci_read_device_config_8(struct virtio_softc *sc, int index)
353 1.2.2.1 pgoyette {
354 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
355 1.2.2.1 pgoyette uint64_t r;
356 1.2.2.1 pgoyette
357 1.2.2.1 pgoyette r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
358 1.2.2.1 pgoyette psc->sc_config_offset + index + REG_HI_OFF);
359 1.2.2.1 pgoyette r <<= 32;
360 1.2.2.1 pgoyette r |= bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
361 1.2.2.1 pgoyette psc->sc_config_offset + index + REG_LO_OFF);
362 1.2.2.1 pgoyette
363 1.2.2.1 pgoyette return r;
364 1.2.2.1 pgoyette }
365 1.2.2.1 pgoyette
366 1.2.2.1 pgoyette static void
367 1.2.2.1 pgoyette virtio_pci_write_device_config_1(struct virtio_softc *sc, int index,
368 1.2.2.1 pgoyette uint8_t value)
369 1.2.2.1 pgoyette {
370 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
371 1.2.2.1 pgoyette
372 1.2.2.1 pgoyette bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
373 1.2.2.1 pgoyette psc->sc_config_offset + index, value);
374 1.2.2.1 pgoyette }
375 1.2.2.1 pgoyette
376 1.2.2.1 pgoyette static void
377 1.2.2.1 pgoyette virtio_pci_write_device_config_2(struct virtio_softc *sc, int index,
378 1.2.2.1 pgoyette uint16_t value)
379 1.2.2.1 pgoyette {
380 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
381 1.2.2.1 pgoyette
382 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
383 1.2.2.1 pgoyette psc->sc_config_offset + index, value);
384 1.2.2.1 pgoyette }
385 1.2.2.1 pgoyette
386 1.2.2.1 pgoyette static void
387 1.2.2.1 pgoyette virtio_pci_write_device_config_4(struct virtio_softc *sc, int index,
388 1.2.2.1 pgoyette uint32_t value)
389 1.2.2.1 pgoyette {
390 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
391 1.2.2.1 pgoyette
392 1.2.2.1 pgoyette bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
393 1.2.2.1 pgoyette psc->sc_config_offset + index, value);
394 1.2.2.1 pgoyette }
395 1.2.2.1 pgoyette
396 1.2.2.1 pgoyette static void
397 1.2.2.1 pgoyette virtio_pci_write_device_config_8(struct virtio_softc *sc, int index,
398 1.2.2.1 pgoyette uint64_t value)
399 1.2.2.1 pgoyette {
400 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
401 1.2.2.1 pgoyette
402 1.2.2.1 pgoyette bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
403 1.2.2.1 pgoyette psc->sc_config_offset + index + REG_LO_OFF,
404 1.2.2.1 pgoyette value & 0xffffffff);
405 1.2.2.1 pgoyette bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
406 1.2.2.1 pgoyette psc->sc_config_offset + index + REG_HI_OFF,
407 1.2.2.1 pgoyette value >> 32);
408 1.2.2.1 pgoyette }
409 1.2.2.1 pgoyette
410 1.2.2.1 pgoyette static uint16_t
411 1.2.2.1 pgoyette virtio_pci_read_queue_size(struct virtio_softc *sc, uint16_t idx)
412 1.2.2.1 pgoyette {
413 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
414 1.2.2.1 pgoyette
415 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
416 1.2.2.1 pgoyette VIRTIO_CONFIG_QUEUE_SELECT, idx);
417 1.2.2.1 pgoyette return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh,
418 1.2.2.1 pgoyette VIRTIO_CONFIG_QUEUE_SIZE);
419 1.2.2.1 pgoyette }
420 1.2.2.1 pgoyette
421 1.2.2.1 pgoyette static void
422 1.2.2.1 pgoyette virtio_pci_setup_queue(struct virtio_softc *sc, uint16_t idx, uint32_t addr)
423 1.2.2.1 pgoyette {
424 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
425 1.2.2.1 pgoyette
426 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
427 1.2.2.1 pgoyette VIRTIO_CONFIG_QUEUE_SELECT, idx);
428 1.2.2.1 pgoyette bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
429 1.2.2.1 pgoyette VIRTIO_CONFIG_QUEUE_ADDRESS, addr);
430 1.2.2.1 pgoyette
431 1.2.2.1 pgoyette if (psc->sc_ihs_num > 1) {
432 1.2.2.1 pgoyette int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
433 1.2.2.2 pgoyette if (sc->sc_child_mq)
434 1.2.2.1 pgoyette vec += idx;
435 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh,
436 1.2.2.1 pgoyette VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec);
437 1.2.2.1 pgoyette }
438 1.2.2.1 pgoyette }
439 1.2.2.1 pgoyette
440 1.2.2.1 pgoyette static void
441 1.2.2.1 pgoyette virtio_pci_set_status(struct virtio_softc *sc, int status)
442 1.2.2.1 pgoyette {
443 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
444 1.2.2.1 pgoyette int old = 0;
445 1.2.2.1 pgoyette
446 1.2.2.1 pgoyette if (status != 0) {
447 1.2.2.1 pgoyette old = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
448 1.2.2.1 pgoyette VIRTIO_CONFIG_DEVICE_STATUS);
449 1.2.2.1 pgoyette }
450 1.2.2.1 pgoyette bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh,
451 1.2.2.1 pgoyette VIRTIO_CONFIG_DEVICE_STATUS, status|old);
452 1.2.2.1 pgoyette }
453 1.2.2.1 pgoyette
454 1.2.2.1 pgoyette static uint32_t
455 1.2.2.1 pgoyette virtio_pci_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
456 1.2.2.1 pgoyette {
457 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
458 1.2.2.1 pgoyette uint32_t r;
459 1.2.2.1 pgoyette
460 1.2.2.1 pgoyette r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh,
461 1.2.2.1 pgoyette VIRTIO_CONFIG_DEVICE_FEATURES);
462 1.2.2.1 pgoyette r &= guest_features;
463 1.2.2.1 pgoyette bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh,
464 1.2.2.1 pgoyette VIRTIO_CONFIG_GUEST_FEATURES, r);
465 1.2.2.1 pgoyette
466 1.2.2.1 pgoyette return r;
467 1.2.2.1 pgoyette }
468 1.2.2.1 pgoyette
469 1.2.2.1 pgoyette
470 1.2.2.1 pgoyette static int
471 1.2.2.1 pgoyette virtio_pci_setup_msix_vectors(struct virtio_softc *sc)
472 1.2.2.1 pgoyette {
473 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
474 1.2.2.1 pgoyette int offset, vector, ret, qid;
475 1.2.2.1 pgoyette
476 1.2.2.1 pgoyette offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR;
477 1.2.2.1 pgoyette vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
478 1.1 cherry
479 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
480 1.2.2.1 pgoyette ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
481 1.2.2.1 pgoyette aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
482 1.2.2.1 pgoyette vector, ret);
483 1.2.2.1 pgoyette if (ret != vector)
484 1.2.2.1 pgoyette return -1;
485 1.2.2.1 pgoyette
486 1.2.2.1 pgoyette for (qid = 0; qid < sc->sc_nvqs; qid++) {
487 1.2.2.1 pgoyette offset = VIRTIO_CONFIG_QUEUE_SELECT;
488 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, qid);
489 1.2.2.1 pgoyette
490 1.2.2.1 pgoyette offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR;
491 1.2.2.1 pgoyette vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
492 1.2.2.1 pgoyette
493 1.2.2.2 pgoyette if (sc->sc_child_mq)
494 1.2.2.2 pgoyette vector += qid;
495 1.2.2.2 pgoyette
496 1.2.2.1 pgoyette bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector);
497 1.2.2.1 pgoyette ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset);
498 1.2.2.1 pgoyette aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n",
499 1.2.2.1 pgoyette vector, ret);
500 1.2.2.1 pgoyette if (ret != vector)
501 1.2.2.1 pgoyette return -1;
502 1.2.2.1 pgoyette }
503 1.1 cherry
504 1.1 cherry return 0;
505 1.1 cherry }
506 1.2.2.1 pgoyette
507 1.2.2.1 pgoyette static int
508 1.2.2.1 pgoyette virtio_pci_setup_msix_interrupts(struct virtio_softc *sc,
509 1.2.2.1 pgoyette struct pci_attach_args *pa)
510 1.2.2.1 pgoyette {
511 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
512 1.2.2.1 pgoyette device_t self = sc->sc_dev;
513 1.2.2.1 pgoyette pci_chipset_tag_t pc = pa->pa_pc;
514 1.2.2.1 pgoyette char intrbuf[PCI_INTRSTR_LEN];
515 1.2.2.2 pgoyette char intr_xname[INTRDEVNAMEBUF];
516 1.2.2.1 pgoyette char const *intrstr;
517 1.2.2.2 pgoyette int idx, qid, n;
518 1.2.2.1 pgoyette
519 1.2.2.1 pgoyette idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
520 1.2.2.1 pgoyette if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
521 1.2.2.1 pgoyette pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
522 1.2.2.1 pgoyette
523 1.2.2.2 pgoyette snprintf(intr_xname, sizeof(intr_xname), "%s config",
524 1.2.2.2 pgoyette device_xname(sc->sc_dev));
525 1.2.2.2 pgoyette
526 1.2.2.1 pgoyette psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
527 1.2.2.2 pgoyette sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname);
528 1.2.2.1 pgoyette if (psc->sc_ihs[idx] == NULL) {
529 1.2.2.1 pgoyette aprint_error_dev(self, "couldn't establish MSI-X for config\n");
530 1.2.2.1 pgoyette goto error;
531 1.2.2.1 pgoyette }
532 1.2.2.1 pgoyette
533 1.2.2.1 pgoyette idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
534 1.2.2.2 pgoyette if (sc->sc_child_mq) {
535 1.2.2.2 pgoyette for (qid = 0; qid < sc->sc_nvqs; qid++) {
536 1.2.2.2 pgoyette n = idx + qid;
537 1.2.2.2 pgoyette
538 1.2.2.2 pgoyette snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d",
539 1.2.2.2 pgoyette device_xname(sc->sc_dev), qid);
540 1.2.2.2 pgoyette
541 1.2.2.2 pgoyette if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) {
542 1.2.2.2 pgoyette pci_intr_setattr(pc, &psc->sc_ihp[n],
543 1.2.2.2 pgoyette PCI_INTR_MPSAFE, true);
544 1.2.2.2 pgoyette }
545 1.2.2.2 pgoyette
546 1.2.2.2 pgoyette psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n],
547 1.2.2.2 pgoyette sc->sc_ipl, virtio_pci_msix_vq_intr, &sc->sc_vqs[qid],
548 1.2.2.2 pgoyette intr_xname);
549 1.2.2.2 pgoyette if (psc->sc_ihs[n] == NULL) {
550 1.2.2.2 pgoyette aprint_error_dev(self, "couldn't establish MSI-X for a vq\n");
551 1.2.2.2 pgoyette goto error;
552 1.2.2.2 pgoyette }
553 1.2.2.2 pgoyette }
554 1.2.2.2 pgoyette } else {
555 1.2.2.2 pgoyette if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
556 1.2.2.2 pgoyette pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true);
557 1.2.2.1 pgoyette
558 1.2.2.2 pgoyette snprintf(intr_xname, sizeof(intr_xname), "%s queues",
559 1.2.2.2 pgoyette device_xname(sc->sc_dev));
560 1.2.2.2 pgoyette psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx],
561 1.2.2.2 pgoyette sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname);
562 1.2.2.2 pgoyette if (psc->sc_ihs[idx] == NULL) {
563 1.2.2.2 pgoyette aprint_error_dev(self, "couldn't establish MSI-X for queues\n");
564 1.2.2.2 pgoyette goto error;
565 1.2.2.2 pgoyette }
566 1.2.2.1 pgoyette }
567 1.2.2.1 pgoyette
568 1.2.2.1 pgoyette if (virtio_pci_setup_msix_vectors(sc) != 0) {
569 1.2.2.1 pgoyette aprint_error_dev(self, "couldn't setup MSI-X vectors\n");
570 1.2.2.1 pgoyette goto error;
571 1.2.2.1 pgoyette }
572 1.2.2.1 pgoyette
573 1.2.2.1 pgoyette idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
574 1.2.2.1 pgoyette intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
575 1.2.2.1 pgoyette aprint_normal_dev(self, "config interrupting at %s\n", intrstr);
576 1.2.2.1 pgoyette idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
577 1.2.2.2 pgoyette if (sc->sc_child_mq) {
578 1.2.2.2 pgoyette kcpuset_t *affinity;
579 1.2.2.2 pgoyette int affinity_to, r;
580 1.2.2.2 pgoyette
581 1.2.2.2 pgoyette kcpuset_create(&affinity, false);
582 1.2.2.2 pgoyette
583 1.2.2.2 pgoyette for (qid = 0; qid < sc->sc_nvqs; qid++) {
584 1.2.2.2 pgoyette n = idx + qid;
585 1.2.2.2 pgoyette affinity_to = (qid / 2) % ncpu;
586 1.2.2.2 pgoyette
587 1.2.2.2 pgoyette intrstr = pci_intr_string(pc, psc->sc_ihp[n],
588 1.2.2.2 pgoyette intrbuf, sizeof(intrbuf));
589 1.2.2.2 pgoyette
590 1.2.2.2 pgoyette kcpuset_zero(affinity);
591 1.2.2.2 pgoyette kcpuset_set(affinity, affinity_to);
592 1.2.2.2 pgoyette r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL);
593 1.2.2.2 pgoyette if (r == 0) {
594 1.2.2.2 pgoyette aprint_normal_dev(self,
595 1.2.2.2 pgoyette "for vq #%d interrupting at %s affinity to %u\n",
596 1.2.2.2 pgoyette qid, intrstr, affinity_to);
597 1.2.2.2 pgoyette } else {
598 1.2.2.2 pgoyette aprint_normal_dev(self,
599 1.2.2.2 pgoyette "for vq #%d interrupting at %s\n",
600 1.2.2.2 pgoyette qid, intrstr);
601 1.2.2.2 pgoyette }
602 1.2.2.2 pgoyette }
603 1.2.2.2 pgoyette
604 1.2.2.2 pgoyette kcpuset_destroy(affinity);
605 1.2.2.2 pgoyette } else {
606 1.2.2.2 pgoyette intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf));
607 1.2.2.2 pgoyette aprint_normal_dev(self, "queues interrupting at %s\n", intrstr);
608 1.2.2.2 pgoyette }
609 1.2.2.1 pgoyette
610 1.2.2.1 pgoyette return 0;
611 1.2.2.1 pgoyette
612 1.2.2.1 pgoyette error:
613 1.2.2.1 pgoyette idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX;
614 1.2.2.1 pgoyette if (psc->sc_ihs[idx] != NULL)
615 1.2.2.1 pgoyette pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
616 1.2.2.1 pgoyette idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX;
617 1.2.2.2 pgoyette if (sc->sc_child_mq) {
618 1.2.2.2 pgoyette for (qid = 0; qid < sc->sc_nvqs; qid++) {
619 1.2.2.2 pgoyette n = idx + qid;
620 1.2.2.2 pgoyette if (psc->sc_ihs[n] == NULL)
621 1.2.2.2 pgoyette continue;
622 1.2.2.2 pgoyette pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]);
623 1.2.2.2 pgoyette }
624 1.2.2.2 pgoyette
625 1.2.2.2 pgoyette } else {
626 1.2.2.2 pgoyette if (psc->sc_ihs[idx] != NULL)
627 1.2.2.2 pgoyette pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]);
628 1.2.2.2 pgoyette }
629 1.2.2.1 pgoyette
630 1.2.2.1 pgoyette return -1;
631 1.2.2.1 pgoyette }
632 1.2.2.1 pgoyette
633 1.2.2.1 pgoyette static int
634 1.2.2.1 pgoyette virtio_pci_setup_intx_interrupt(struct virtio_softc *sc,
635 1.2.2.1 pgoyette struct pci_attach_args *pa)
636 1.2.2.1 pgoyette {
637 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
638 1.2.2.1 pgoyette device_t self = sc->sc_dev;
639 1.2.2.1 pgoyette pci_chipset_tag_t pc = pa->pa_pc;
640 1.2.2.1 pgoyette char intrbuf[PCI_INTRSTR_LEN];
641 1.2.2.1 pgoyette char const *intrstr;
642 1.2.2.1 pgoyette
643 1.2.2.1 pgoyette if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE)
644 1.2.2.1 pgoyette pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true);
645 1.2.2.1 pgoyette
646 1.2.2.1 pgoyette psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0],
647 1.2.2.1 pgoyette sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev));
648 1.2.2.1 pgoyette if (psc->sc_ihs[0] == NULL) {
649 1.2.2.1 pgoyette aprint_error_dev(self, "couldn't establish INTx\n");
650 1.2.2.1 pgoyette return -1;
651 1.2.2.1 pgoyette }
652 1.2.2.1 pgoyette
653 1.2.2.1 pgoyette intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf));
654 1.2.2.1 pgoyette aprint_normal_dev(self, "interrupting at %s\n", intrstr);
655 1.2.2.1 pgoyette
656 1.2.2.1 pgoyette return 0;
657 1.2.2.1 pgoyette }
658 1.2.2.1 pgoyette
659 1.2.2.1 pgoyette static int
660 1.2.2.1 pgoyette virtio_pci_setup_interrupts(struct virtio_softc *sc)
661 1.2.2.1 pgoyette {
662 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
663 1.2.2.1 pgoyette device_t self = sc->sc_dev;
664 1.2.2.1 pgoyette pci_chipset_tag_t pc = psc->sc_pa.pa_pc;
665 1.2.2.1 pgoyette int error;
666 1.2.2.1 pgoyette int nmsix;
667 1.2.2.1 pgoyette int counts[PCI_INTR_TYPE_SIZE];
668 1.2.2.1 pgoyette pci_intr_type_t max_type;
669 1.2.2.1 pgoyette
670 1.2.2.1 pgoyette nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag);
671 1.2.2.1 pgoyette aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix);
672 1.2.2.1 pgoyette
673 1.2.2.1 pgoyette /* We need at least two: one for config and the other for queues */
674 1.2.2.1 pgoyette if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) {
675 1.2.2.1 pgoyette /* Try INTx only */
676 1.2.2.1 pgoyette max_type = PCI_INTR_TYPE_INTX;
677 1.2.2.1 pgoyette counts[PCI_INTR_TYPE_INTX] = 1;
678 1.2.2.1 pgoyette } else {
679 1.2.2.1 pgoyette /* Try MSI-X first and INTx second */
680 1.2.2.2 pgoyette if (sc->sc_child_mq &&
681 1.2.2.2 pgoyette sc->sc_nvqs > (nmsix - VIRTIO_MSIX_QUEUE_VECTOR_INDEX)) {
682 1.2.2.2 pgoyette nmsix = 2;
683 1.2.2.2 pgoyette sc->sc_child_mq = false;
684 1.2.2.2 pgoyette }
685 1.2.2.2 pgoyette
686 1.2.2.1 pgoyette max_type = PCI_INTR_TYPE_MSIX;
687 1.2.2.2 pgoyette counts[PCI_INTR_TYPE_MSIX] = nmsix;
688 1.2.2.1 pgoyette counts[PCI_INTR_TYPE_MSI] = 0;
689 1.2.2.1 pgoyette counts[PCI_INTR_TYPE_INTX] = 1;
690 1.2.2.1 pgoyette }
691 1.2.2.1 pgoyette
692 1.2.2.1 pgoyette retry:
693 1.2.2.1 pgoyette error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type);
694 1.2.2.1 pgoyette if (error != 0) {
695 1.2.2.1 pgoyette aprint_error_dev(self, "couldn't map interrupt\n");
696 1.2.2.1 pgoyette return -1;
697 1.2.2.1 pgoyette }
698 1.2.2.1 pgoyette
699 1.2.2.1 pgoyette if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) {
700 1.2.2.2 pgoyette psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * nmsix,
701 1.2.2.1 pgoyette KM_SLEEP);
702 1.2.2.1 pgoyette
703 1.2.2.1 pgoyette error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa);
704 1.2.2.1 pgoyette if (error != 0) {
705 1.2.2.2 pgoyette kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix);
706 1.2.2.2 pgoyette pci_intr_release(pc, psc->sc_ihp, nmsix);
707 1.2.2.1 pgoyette
708 1.2.2.1 pgoyette /* Retry INTx */
709 1.2.2.1 pgoyette max_type = PCI_INTR_TYPE_INTX;
710 1.2.2.1 pgoyette counts[PCI_INTR_TYPE_INTX] = 1;
711 1.2.2.1 pgoyette goto retry;
712 1.2.2.1 pgoyette }
713 1.2.2.1 pgoyette
714 1.2.2.2 pgoyette psc->sc_ihs_num = nmsix;
715 1.2.2.1 pgoyette psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
716 1.2.2.1 pgoyette } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) {
717 1.2.2.1 pgoyette psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 1,
718 1.2.2.1 pgoyette KM_SLEEP);
719 1.2.2.1 pgoyette
720 1.2.2.1 pgoyette error = virtio_pci_setup_intx_interrupt(sc, &psc->sc_pa);
721 1.2.2.1 pgoyette if (error != 0) {
722 1.2.2.1 pgoyette kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1);
723 1.2.2.1 pgoyette pci_intr_release(pc, psc->sc_ihp, 1);
724 1.2.2.1 pgoyette return -1;
725 1.2.2.1 pgoyette }
726 1.2.2.1 pgoyette
727 1.2.2.1 pgoyette psc->sc_ihs_num = 1;
728 1.2.2.1 pgoyette psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
729 1.2.2.1 pgoyette }
730 1.2.2.1 pgoyette
731 1.2.2.1 pgoyette return 0;
732 1.2.2.1 pgoyette }
733 1.2.2.1 pgoyette
734 1.2.2.1 pgoyette static void
735 1.2.2.1 pgoyette virtio_pci_free_interrupts(struct virtio_softc *sc)
736 1.2.2.1 pgoyette {
737 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
738 1.2.2.1 pgoyette
739 1.2.2.1 pgoyette for (int i = 0; i < psc->sc_ihs_num; i++) {
740 1.2.2.1 pgoyette if (psc->sc_ihs[i] == NULL)
741 1.2.2.1 pgoyette continue;
742 1.2.2.1 pgoyette pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]);
743 1.2.2.1 pgoyette psc->sc_ihs[i] = NULL;
744 1.2.2.1 pgoyette }
745 1.2.2.1 pgoyette
746 1.2.2.1 pgoyette if (psc->sc_ihs_num > 0)
747 1.2.2.1 pgoyette pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num);
748 1.2.2.1 pgoyette
749 1.2.2.1 pgoyette if (psc->sc_ihs != NULL) {
750 1.2.2.1 pgoyette kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num);
751 1.2.2.1 pgoyette psc->sc_ihs = NULL;
752 1.2.2.1 pgoyette }
753 1.2.2.1 pgoyette psc->sc_ihs_num = 0;
754 1.2.2.1 pgoyette }
755 1.2.2.1 pgoyette
756 1.2.2.1 pgoyette /*
757 1.2.2.1 pgoyette * Interrupt handler.
758 1.2.2.1 pgoyette */
759 1.2.2.1 pgoyette static int
760 1.2.2.1 pgoyette virtio_pci_intr(void *arg)
761 1.2.2.1 pgoyette {
762 1.2.2.1 pgoyette struct virtio_softc *sc = arg;
763 1.2.2.1 pgoyette struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc;
764 1.2.2.1 pgoyette int isr, r = 0;
765 1.2.2.1 pgoyette
766 1.2.2.1 pgoyette /* check and ack the interrupt */
767 1.2.2.1 pgoyette isr = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh,
768 1.2.2.1 pgoyette VIRTIO_CONFIG_ISR_STATUS);
769 1.2.2.1 pgoyette if (isr == 0)
770 1.2.2.1 pgoyette return 0;
771 1.2.2.1 pgoyette if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
772 1.2.2.1 pgoyette (sc->sc_config_change != NULL))
773 1.2.2.1 pgoyette r = (sc->sc_config_change)(sc);
774 1.2.2.1 pgoyette if (sc->sc_intrhand != NULL) {
775 1.2.2.1 pgoyette if (sc->sc_soft_ih != NULL)
776 1.2.2.1 pgoyette softint_schedule(sc->sc_soft_ih);
777 1.2.2.1 pgoyette else
778 1.2.2.1 pgoyette r |= (sc->sc_intrhand)(sc);
779 1.2.2.1 pgoyette }
780 1.2.2.1 pgoyette
781 1.2.2.1 pgoyette return r;
782 1.2.2.1 pgoyette }
783 1.2.2.1 pgoyette
784 1.2.2.1 pgoyette static int
785 1.2.2.1 pgoyette virtio_pci_msix_queue_intr(void *arg)
786 1.2.2.1 pgoyette {
787 1.2.2.1 pgoyette struct virtio_softc *sc = arg;
788 1.2.2.1 pgoyette int r = 0;
789 1.2.2.1 pgoyette
790 1.2.2.1 pgoyette if (sc->sc_intrhand != NULL) {
791 1.2.2.1 pgoyette if (sc->sc_soft_ih != NULL)
792 1.2.2.1 pgoyette softint_schedule(sc->sc_soft_ih);
793 1.2.2.1 pgoyette else
794 1.2.2.1 pgoyette r |= (sc->sc_intrhand)(sc);
795 1.2.2.1 pgoyette }
796 1.2.2.1 pgoyette
797 1.2.2.1 pgoyette return r;
798 1.2.2.1 pgoyette }
799 1.2.2.1 pgoyette
800 1.2.2.1 pgoyette static int
801 1.2.2.2 pgoyette virtio_pci_msix_vq_intr(void *arg)
802 1.2.2.2 pgoyette {
803 1.2.2.2 pgoyette struct virtqueue *vq = arg;
804 1.2.2.2 pgoyette int r = 0;
805 1.2.2.2 pgoyette
806 1.2.2.2 pgoyette if (vq->vq_intrhand != NULL) {
807 1.2.2.2 pgoyette if (vq->vq_soft_ih)
808 1.2.2.2 pgoyette softint_schedule(vq->vq_soft_ih);
809 1.2.2.2 pgoyette else
810 1.2.2.2 pgoyette r |= vq->vq_intrhand(vq);
811 1.2.2.2 pgoyette }
812 1.2.2.2 pgoyette
813 1.2.2.2 pgoyette return r;
814 1.2.2.2 pgoyette }
815 1.2.2.2 pgoyette
816 1.2.2.2 pgoyette static int
817 1.2.2.1 pgoyette virtio_pci_msix_config_intr(void *arg)
818 1.2.2.1 pgoyette {
819 1.2.2.1 pgoyette struct virtio_softc *sc = arg;
820 1.2.2.1 pgoyette int r = 0;
821 1.2.2.1 pgoyette
822 1.2.2.1 pgoyette if (sc->sc_config_change != NULL)
823 1.2.2.1 pgoyette r = (sc->sc_config_change)(sc);
824 1.2.2.1 pgoyette return r;
825 1.2.2.1 pgoyette }
826 1.2.2.1 pgoyette
827 1.2.2.1 pgoyette MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio");
828 1.2.2.1 pgoyette
829 1.2.2.1 pgoyette #ifdef _MODULE
830 1.2.2.1 pgoyette #include "ioconf.c"
831 1.2.2.1 pgoyette #endif
832 1.2.2.1 pgoyette
833 1.2.2.1 pgoyette static int
834 1.2.2.1 pgoyette virtio_pci_modcmd(modcmd_t cmd, void *opaque)
835 1.2.2.1 pgoyette {
836 1.2.2.1 pgoyette int error = 0;
837 1.2.2.1 pgoyette
838 1.2.2.1 pgoyette #ifdef _MODULE
839 1.2.2.1 pgoyette switch (cmd) {
840 1.2.2.1 pgoyette case MODULE_CMD_INIT:
841 1.2.2.1 pgoyette error = config_init_component(cfdriver_ioconf_virtio_pci,
842 1.2.2.1 pgoyette cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
843 1.2.2.1 pgoyette break;
844 1.2.2.1 pgoyette case MODULE_CMD_FINI:
845 1.2.2.1 pgoyette error = config_fini_component(cfdriver_ioconf_virtio_pci,
846 1.2.2.1 pgoyette cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci);
847 1.2.2.1 pgoyette break;
848 1.2.2.1 pgoyette default:
849 1.2.2.1 pgoyette error = ENOTTY;
850 1.2.2.1 pgoyette break;
851 1.2.2.1 pgoyette }
852 1.2.2.1 pgoyette #endif
853 1.2.2.1 pgoyette
854 1.2.2.1 pgoyette return error;
855 1.2.2.1 pgoyette }
856 1.2.2.1 pgoyette
857