if_vioif.c revision 1.42
1 /* $NetBSD: if_vioif.c,v 1.42 2019/01/14 14:32:13 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.42 2019/01/14 14:32:13 yamaguchi Exp $"); 30 31 #ifdef _KERNEL_OPT 32 #include "opt_net_mpsafe.h" 33 #endif 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/condvar.h> 40 #include <sys/device.h> 41 #include <sys/intr.h> 42 #include <sys/kmem.h> 43 #include <sys/mbuf.h> 44 #include <sys/mutex.h> 45 #include <sys/sockio.h> 46 #include <sys/cpu.h> 47 #include <sys/module.h> 48 49 #include <dev/pci/virtioreg.h> 50 #include <dev/pci/virtiovar.h> 51 52 #include <net/if.h> 53 #include <net/if_media.h> 54 #include <net/if_ether.h> 55 56 #include <net/bpf.h> 57 58 #include "ioconf.h" 59 60 #ifdef NET_MPSAFE 61 #define VIOIF_MPSAFE 1 62 #endif 63 64 #ifdef SOFTINT_INTR 65 #define VIOIF_SOFTINT_INTR 1 66 #endif 67 68 /* 69 * if_vioifreg.h: 70 */ 71 /* Configuration registers */ 72 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 73 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 74 75 /* Feature bits */ 76 #define VIRTIO_NET_F_CSUM (1<<0) 77 #define VIRTIO_NET_F_GUEST_CSUM (1<<1) 78 #define VIRTIO_NET_F_MAC (1<<5) 79 #define VIRTIO_NET_F_GSO (1<<6) 80 #define VIRTIO_NET_F_GUEST_TSO4 (1<<7) 81 #define VIRTIO_NET_F_GUEST_TSO6 (1<<8) 82 #define VIRTIO_NET_F_GUEST_ECN (1<<9) 83 #define VIRTIO_NET_F_GUEST_UFO (1<<10) 84 #define VIRTIO_NET_F_HOST_TSO4 (1<<11) 85 #define VIRTIO_NET_F_HOST_TSO6 (1<<12) 86 #define VIRTIO_NET_F_HOST_ECN (1<<13) 87 #define VIRTIO_NET_F_HOST_UFO (1<<14) 88 #define VIRTIO_NET_F_MRG_RXBUF (1<<15) 89 #define VIRTIO_NET_F_STATUS (1<<16) 90 #define VIRTIO_NET_F_CTRL_VQ (1<<17) 91 #define VIRTIO_NET_F_CTRL_RX (1<<18) 92 #define VIRTIO_NET_F_CTRL_VLAN (1<<19) 93 94 #define VIRTIO_NET_FLAG_BITS \ 95 VIRTIO_COMMON_FLAG_BITS \ 96 "\x14""CTRL_VLAN" \ 97 "\x13""CTRL_RX" \ 98 "\x12""CTRL_VQ" \ 99 "\x11""STATUS" \ 100 "\x10""MRG_RXBUF" \ 101 "\x0f""HOST_UFO" \ 102 "\x0e""HOST_ECN" \ 103 "\x0d""HOST_TSO6" \ 104 "\x0c""HOST_TSO4" \ 105 "\x0b""GUEST_UFO" \ 106 "\x0a""GUEST_ECN" \ 107 "\x09""GUEST_TSO6" \ 108 "\x08""GUEST_TSO4" \ 109 "\x07""GSO" \ 110 "\x06""MAC" \ 111 "\x02""GUEST_CSUM" \ 112 "\x01""CSUM" 113 114 /* Status */ 115 #define VIRTIO_NET_S_LINK_UP 1 116 117 /* Packet header structure */ 118 struct virtio_net_hdr { 119 uint8_t flags; 120 uint8_t gso_type; 121 uint16_t hdr_len; 122 uint16_t gso_size; 123 uint16_t csum_start; 124 uint16_t csum_offset; 125 #if 0 126 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */ 127 #endif 128 } __packed; 129 130 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 131 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 132 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 133 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 134 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 135 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 136 137 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 138 139 /* Control virtqueue */ 140 struct virtio_net_ctrl_cmd { 141 uint8_t class; 142 uint8_t command; 143 } __packed; 144 #define VIRTIO_NET_CTRL_RX 0 145 # define VIRTIO_NET_CTRL_RX_PROMISC 0 146 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 147 148 #define VIRTIO_NET_CTRL_MAC 1 149 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 150 151 #define VIRTIO_NET_CTRL_VLAN 2 152 # define VIRTIO_NET_CTRL_VLAN_ADD 0 153 # define VIRTIO_NET_CTRL_VLAN_DEL 1 154 155 struct virtio_net_ctrl_status { 156 uint8_t ack; 157 } __packed; 158 #define VIRTIO_NET_OK 0 159 #define VIRTIO_NET_ERR 1 160 161 struct virtio_net_ctrl_rx { 162 uint8_t onoff; 163 } __packed; 164 165 struct virtio_net_ctrl_mac_tbl { 166 uint32_t nentries; 167 uint8_t macs[][ETHER_ADDR_LEN]; 168 } __packed; 169 170 struct virtio_net_ctrl_vlan { 171 uint16_t id; 172 } __packed; 173 174 175 /* 176 * if_vioifvar.h: 177 */ 178 struct vioif_softc { 179 device_t sc_dev; 180 181 struct virtio_softc *sc_virtio; 182 struct virtqueue sc_vq[3]; 183 #define VQ_RX 0 184 #define VQ_TX 1 185 #define VQ_CTRL 2 186 187 uint8_t sc_mac[ETHER_ADDR_LEN]; 188 struct ethercom sc_ethercom; 189 short sc_deferred_init_done; 190 bool sc_link_active; 191 192 /* bus_dmamem */ 193 bus_dma_segment_t sc_hdr_segs[1]; 194 struct virtio_net_hdr *sc_hdrs; 195 #define sc_rx_hdrs sc_hdrs 196 struct virtio_net_hdr *sc_tx_hdrs; 197 struct virtio_net_ctrl_cmd *sc_ctrl_cmd; 198 struct virtio_net_ctrl_status *sc_ctrl_status; 199 struct virtio_net_ctrl_rx *sc_ctrl_rx; 200 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc; 201 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc; 202 203 /* kmem */ 204 bus_dmamap_t *sc_arrays; 205 #define sc_rxhdr_dmamaps sc_arrays 206 bus_dmamap_t *sc_txhdr_dmamaps; 207 bus_dmamap_t *sc_rx_dmamaps; 208 bus_dmamap_t *sc_tx_dmamaps; 209 struct mbuf **sc_rx_mbufs; 210 struct mbuf **sc_tx_mbufs; 211 212 bus_dmamap_t sc_ctrl_cmd_dmamap; 213 bus_dmamap_t sc_ctrl_status_dmamap; 214 bus_dmamap_t sc_ctrl_rx_dmamap; 215 bus_dmamap_t sc_ctrl_tbl_uc_dmamap; 216 bus_dmamap_t sc_ctrl_tbl_mc_dmamap; 217 218 void *sc_rx_softint; 219 void *sc_ctl_softint; 220 221 enum { 222 FREE, INUSE, DONE 223 } sc_ctrl_inuse; 224 kcondvar_t sc_ctrl_wait; 225 kmutex_t sc_ctrl_wait_lock; 226 kmutex_t sc_tx_lock; 227 kmutex_t sc_rx_lock; 228 bool sc_stopping; 229 230 bool sc_has_ctrl; 231 }; 232 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 233 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 234 235 #define VIOIF_TX_LOCK(_sc) mutex_enter(&(_sc)->sc_tx_lock) 236 #define VIOIF_TX_UNLOCK(_sc) mutex_exit(&(_sc)->sc_tx_lock) 237 #define VIOIF_TX_LOCKED(_sc) mutex_owned(&(_sc)->sc_tx_lock) 238 #define VIOIF_RX_LOCK(_sc) mutex_enter(&(_sc)->sc_rx_lock) 239 #define VIOIF_RX_UNLOCK(_sc) mutex_exit(&(_sc)->sc_rx_lock) 240 #define VIOIF_RX_LOCKED(_sc) mutex_owned(&(_sc)->sc_rx_lock) 241 242 /* cfattach interface functions */ 243 static int vioif_match(device_t, cfdata_t, void *); 244 static void vioif_attach(device_t, device_t, void *); 245 static void vioif_deferred_init(device_t); 246 247 /* ifnet interface functions */ 248 static int vioif_init(struct ifnet *); 249 static void vioif_stop(struct ifnet *, int); 250 static void vioif_start(struct ifnet *); 251 static int vioif_ioctl(struct ifnet *, u_long, void *); 252 static void vioif_watchdog(struct ifnet *); 253 254 /* rx */ 255 static int vioif_add_rx_mbuf(struct vioif_softc *, int); 256 static void vioif_free_rx_mbuf(struct vioif_softc *, int); 257 static void vioif_populate_rx_mbufs(struct vioif_softc *); 258 static void vioif_populate_rx_mbufs_locked(struct vioif_softc *); 259 static int vioif_rx_deq(struct vioif_softc *); 260 static int vioif_rx_deq_locked(struct vioif_softc *); 261 static int vioif_rx_vq_done(struct virtqueue *); 262 static void vioif_rx_softint(void *); 263 static void vioif_rx_drain(struct vioif_softc *); 264 265 /* tx */ 266 static int vioif_tx_vq_done(struct virtqueue *); 267 static int vioif_tx_vq_done_locked(struct virtqueue *); 268 static void vioif_tx_drain(struct vioif_softc *); 269 270 /* other control */ 271 static bool vioif_is_link_up(struct vioif_softc *); 272 static void vioif_update_link_status(struct vioif_softc *); 273 static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 274 static int vioif_set_promisc(struct vioif_softc *, bool); 275 static int vioif_set_allmulti(struct vioif_softc *, bool); 276 static int vioif_set_rx_filter(struct vioif_softc *); 277 static int vioif_rx_filter(struct vioif_softc *); 278 static int vioif_ctrl_vq_done(struct virtqueue *); 279 static int vioif_config_change(struct virtio_softc *); 280 static void vioif_ctl_softint(void *); 281 282 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 283 vioif_match, vioif_attach, NULL, NULL); 284 285 static int 286 vioif_match(device_t parent, cfdata_t match, void *aux) 287 { 288 struct virtio_attach_args *va = aux; 289 290 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK) 291 return 1; 292 293 return 0; 294 } 295 296 /* allocate memory */ 297 /* 298 * dma memory is used for: 299 * sc_rx_hdrs[slot]: metadata array for received frames (READ) 300 * sc_tx_hdrs[slot]: metadata array for frames to be sent (WRITE) 301 * sc_ctrl_cmd: command to be sent via ctrl vq (WRITE) 302 * sc_ctrl_status: return value for a command via ctrl vq (READ) 303 * sc_ctrl_rx: parameter for a VIRTIO_NET_CTRL_RX class command 304 * (WRITE) 305 * sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 306 * class command (WRITE) 307 * sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 308 * class command (WRITE) 309 * sc_ctrl_* structures are allocated only one each; they are protected by 310 * sc_ctrl_inuse variable and sc_ctrl_wait condvar. 311 */ 312 /* 313 * dynamically allocated memory is used for: 314 * sc_rxhdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] 315 * sc_txhdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] 316 * sc_rx_dmamaps[slot]: bus_dmamap_t array for received payload 317 * sc_tx_dmamaps[slot]: bus_dmamap_t array for sent payload 318 * sc_rx_mbufs[slot]: mbuf pointer array for received frames 319 * sc_tx_mbufs[slot]: mbuf pointer array for sent frames 320 */ 321 static int 322 vioif_alloc_mems(struct vioif_softc *sc) 323 { 324 struct virtio_softc *vsc = sc->sc_virtio; 325 int allocsize, allocsize2, r, rsegs, i; 326 void *vaddr; 327 intptr_t p; 328 int rxqsize, txqsize; 329 330 rxqsize = sc->sc_vq[VQ_RX].vq_num; 331 txqsize = sc->sc_vq[VQ_TX].vq_num; 332 333 allocsize = sizeof(struct virtio_net_hdr) * rxqsize; 334 allocsize += sizeof(struct virtio_net_hdr) * txqsize; 335 if (sc->sc_has_ctrl) { 336 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 337 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 338 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 339 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) 340 + sizeof(struct virtio_net_ctrl_mac_tbl) 341 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 342 } 343 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, 344 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 345 if (r != 0) { 346 aprint_error_dev(sc->sc_dev, 347 "DMA memory allocation failed, size %d, " 348 "error code %d\n", allocsize, r); 349 goto err_none; 350 } 351 r = bus_dmamem_map(virtio_dmat(vsc), 352 &sc->sc_hdr_segs[0], 1, allocsize, 353 &vaddr, BUS_DMA_NOWAIT); 354 if (r != 0) { 355 aprint_error_dev(sc->sc_dev, 356 "DMA memory map failed, " 357 "error code %d\n", r); 358 goto err_dmamem_alloc; 359 } 360 361 #define P(p, p0, p0size) do { p0 = (void *) p; \ 362 p += p0size; } while (0) 363 memset(vaddr, 0, allocsize); 364 p = (intptr_t) vaddr; 365 366 P(p, sc->sc_rx_hdrs, sizeof(sc->sc_rx_hdrs[0]) * rxqsize); 367 P(p, sc->sc_tx_hdrs, sizeof(sc->sc_tx_hdrs[0]) * txqsize); 368 if (sc->sc_has_ctrl) { 369 P(p, sc->sc_ctrl_cmd, sizeof(*sc->sc_ctrl_cmd)); 370 P(p, sc->sc_ctrl_status, sizeof(*sc->sc_ctrl_status)); 371 P(p, sc->sc_ctrl_rx, sizeof(*sc->sc_ctrl_rx)); 372 P(p, sc->sc_ctrl_mac_tbl_uc, sizeof(*sc->sc_ctrl_mac_tbl_uc) + 0); 373 P(p, sc->sc_ctrl_mac_tbl_mc, 374 (sizeof(*sc->sc_ctrl_mac_tbl_mc) 375 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES)); 376 } 377 378 allocsize2 = sizeof(bus_dmamap_t) * (rxqsize + txqsize); 379 allocsize2 += sizeof(bus_dmamap_t) * (rxqsize + txqsize); 380 allocsize2 += sizeof(struct mbuf*) * (rxqsize + txqsize); 381 vaddr = kmem_zalloc(allocsize2, KM_SLEEP); 382 p = (intptr_t) vaddr; 383 384 P(p, sc->sc_rxhdr_dmamaps, sizeof(sc->sc_rxhdr_dmamaps[0]) * rxqsize); 385 P(p, sc->sc_txhdr_dmamaps, sizeof(sc->sc_txhdr_dmamaps[0]) * txqsize); 386 P(p, sc->sc_rx_dmamaps, sizeof(sc->sc_rx_dmamaps[0]) * rxqsize); 387 P(p, sc->sc_tx_dmamaps, sizeof(sc->sc_tx_dmamaps[0]) * txqsize); 388 P(p, sc->sc_rx_mbufs, sizeof(sc->sc_rx_mbufs[0]) * rxqsize); 389 P(p, sc->sc_tx_mbufs, sizeof(sc->sc_tx_mbufs[0]) * txqsize); 390 #undef P 391 392 #define C(map, size, nsegs, usage) \ 393 do { \ 394 r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \ 395 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, \ 396 &map); \ 397 if (r != 0) { \ 398 aprint_error_dev(sc->sc_dev, \ 399 "%s dmamap creation failed, " \ 400 "error code %d\n", usage, r); \ 401 goto err_reqs; \ 402 } \ 403 } while (0) 404 #define C_L(map, buf, size, nsegs, rw, usage) \ 405 C(map, size, nsegs, usage); \ 406 do { \ 407 r = bus_dmamap_load(virtio_dmat(vsc), map, \ 408 buf, size, NULL, \ 409 rw | BUS_DMA_NOWAIT); \ 410 if (r != 0) { \ 411 aprint_error_dev(sc->sc_dev, \ 412 usage " dmamap load failed, " \ 413 "error code %d\n", r); \ 414 goto err_reqs; \ 415 } \ 416 } while (0) 417 for (i = 0; i < rxqsize; i++) { 418 C_L(sc->sc_rxhdr_dmamaps[i], &sc->sc_rx_hdrs[i], sizeof(sc->sc_rx_hdrs[0]), 1, 419 BUS_DMA_READ, "rx header"); 420 C(sc->sc_rx_dmamaps[i], MCLBYTES, 1, "rx payload"); 421 } 422 423 for (i = 0; i < txqsize; i++) { 424 C_L(sc->sc_txhdr_dmamaps[i], &sc->sc_tx_hdrs[i], sizeof(sc->sc_tx_hdrs[0]), 1, 425 BUS_DMA_READ, "tx header"); 426 C(sc->sc_tx_dmamaps[i], ETHER_MAX_LEN, VIRTIO_NET_TX_MAXNSEGS, "tx payload"); 427 } 428 429 if (sc->sc_has_ctrl) { 430 /* control vq class & command */ 431 C_L(sc->sc_ctrl_cmd_dmamap, sc->sc_ctrl_cmd, sizeof(*sc->sc_ctrl_cmd), 1, 432 BUS_DMA_WRITE, "control command"); 433 C_L(sc->sc_ctrl_status_dmamap, sc->sc_ctrl_status, sizeof(*sc->sc_ctrl_status), 1, 434 BUS_DMA_READ, "control status"); 435 436 /* control vq rx mode command parameter */ 437 C_L(sc->sc_ctrl_rx_dmamap, sc->sc_ctrl_rx, sizeof(*sc->sc_ctrl_rx), 1, 438 BUS_DMA_WRITE, "rx mode control command"); 439 440 /* control vq MAC filter table for unicast */ 441 /* do not load now since its length is variable */ 442 C(sc->sc_ctrl_tbl_uc_dmamap, sizeof(*sc->sc_ctrl_mac_tbl_uc) + 0, 1, 443 "unicast MAC address filter command"); 444 445 /* control vq MAC filter table for multicast */ 446 C(sc->sc_ctrl_tbl_mc_dmamap, 447 sizeof(*sc->sc_ctrl_mac_tbl_mc) 448 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, 449 "multicast MAC address filter command"); 450 } 451 #undef C_L 452 #undef C 453 454 return 0; 455 456 err_reqs: 457 #define D(map) \ 458 do { \ 459 if (map) { \ 460 bus_dmamap_destroy(virtio_dmat(vsc), map); \ 461 map = NULL; \ 462 } \ 463 } while (0) 464 D(sc->sc_ctrl_tbl_mc_dmamap); 465 D(sc->sc_ctrl_tbl_uc_dmamap); 466 D(sc->sc_ctrl_rx_dmamap); 467 D(sc->sc_ctrl_status_dmamap); 468 D(sc->sc_ctrl_cmd_dmamap); 469 for (i = 0; i < txqsize; i++) { 470 D(sc->sc_tx_dmamaps[i]); 471 D(sc->sc_txhdr_dmamaps[i]); 472 } 473 for (i = 0; i < rxqsize; i++) { 474 D(sc->sc_rx_dmamaps[i]); 475 D(sc->sc_rxhdr_dmamaps[i]); 476 } 477 #undef D 478 if (sc->sc_arrays) { 479 kmem_free(sc->sc_arrays, allocsize2); 480 sc->sc_arrays = 0; 481 } 482 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_hdrs, allocsize); 483 err_dmamem_alloc: 484 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); 485 err_none: 486 return -1; 487 } 488 489 static void 490 vioif_attach(device_t parent, device_t self, void *aux) 491 { 492 struct vioif_softc *sc = device_private(self); 493 struct virtio_softc *vsc = device_private(parent); 494 uint32_t features; 495 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 496 u_int flags; 497 int r, nvqs=0, req_flags; 498 499 if (virtio_child(vsc) != NULL) { 500 aprint_normal(": child already attached for %s; " 501 "something wrong...\n", 502 device_xname(parent)); 503 return; 504 } 505 506 sc->sc_dev = self; 507 sc->sc_virtio = vsc; 508 sc->sc_link_active = false; 509 510 req_flags = 0; 511 512 #ifdef VIOIF_MPSAFE 513 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE; 514 #endif 515 #ifdef VIOIF_SOFTINT_INTR 516 req_flags |= VIRTIO_F_PCI_INTR_SOFTINT; 517 #endif 518 req_flags |= VIRTIO_F_PCI_INTR_MSIX; 519 520 virtio_child_attach_start(vsc, self, IPL_NET, sc->sc_vq, 521 vioif_config_change, virtio_vq_intr, req_flags, 522 (VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 523 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY), 524 VIRTIO_NET_FLAG_BITS); 525 526 features = virtio_features(vsc); 527 528 if (features & VIRTIO_NET_F_MAC) { 529 sc->sc_mac[0] = virtio_read_device_config_1(vsc, 530 VIRTIO_NET_CONFIG_MAC+0); 531 sc->sc_mac[1] = virtio_read_device_config_1(vsc, 532 VIRTIO_NET_CONFIG_MAC+1); 533 sc->sc_mac[2] = virtio_read_device_config_1(vsc, 534 VIRTIO_NET_CONFIG_MAC+2); 535 sc->sc_mac[3] = virtio_read_device_config_1(vsc, 536 VIRTIO_NET_CONFIG_MAC+3); 537 sc->sc_mac[4] = virtio_read_device_config_1(vsc, 538 VIRTIO_NET_CONFIG_MAC+4); 539 sc->sc_mac[5] = virtio_read_device_config_1(vsc, 540 VIRTIO_NET_CONFIG_MAC+5); 541 } else { 542 /* code stolen from sys/net/if_tap.c */ 543 struct timeval tv; 544 uint32_t ui; 545 getmicrouptime(&tv); 546 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 547 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 548 virtio_write_device_config_1(vsc, 549 VIRTIO_NET_CONFIG_MAC+0, 550 sc->sc_mac[0]); 551 virtio_write_device_config_1(vsc, 552 VIRTIO_NET_CONFIG_MAC+1, 553 sc->sc_mac[1]); 554 virtio_write_device_config_1(vsc, 555 VIRTIO_NET_CONFIG_MAC+2, 556 sc->sc_mac[2]); 557 virtio_write_device_config_1(vsc, 558 VIRTIO_NET_CONFIG_MAC+3, 559 sc->sc_mac[3]); 560 virtio_write_device_config_1(vsc, 561 VIRTIO_NET_CONFIG_MAC+4, 562 sc->sc_mac[4]); 563 virtio_write_device_config_1(vsc, 564 VIRTIO_NET_CONFIG_MAC+5, 565 sc->sc_mac[5]); 566 } 567 568 aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(sc->sc_mac)); 569 570 mutex_init(&sc->sc_tx_lock, MUTEX_DEFAULT, IPL_NET); 571 mutex_init(&sc->sc_rx_lock, MUTEX_DEFAULT, IPL_NET); 572 sc->sc_stopping = false; 573 574 /* 575 * Allocating a virtqueue for Rx 576 */ 577 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_RX], VQ_RX, 578 MCLBYTES+sizeof(struct virtio_net_hdr), 2, "rx"); 579 if (r != 0) 580 goto err; 581 nvqs = 1; 582 sc->sc_vq[VQ_RX].vq_done = vioif_rx_vq_done; 583 584 /* 585 * Allocating a virtqueue for Tx 586 */ 587 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_TX], VQ_TX, 588 (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)), 589 VIRTIO_NET_TX_MAXNSEGS + 1, "tx"); 590 if (r != 0) 591 goto err; 592 nvqs = 2; 593 sc->sc_vq[VQ_TX].vq_done = vioif_tx_vq_done; 594 595 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 596 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); /* not urgent; do it later */ 597 598 if ((features & VIRTIO_NET_F_CTRL_VQ) && 599 (features & VIRTIO_NET_F_CTRL_RX)) { 600 /* 601 * Allocating a virtqueue for control channel 602 */ 603 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_CTRL], VQ_CTRL, 604 NBPG, 1, "control"); 605 if (r != 0) { 606 aprint_error_dev(self, "failed to allocate " 607 "a virtqueue for control channel\n"); 608 goto skip; 609 } 610 611 sc->sc_vq[VQ_CTRL].vq_done = vioif_ctrl_vq_done; 612 cv_init(&sc->sc_ctrl_wait, "ctrl_vq"); 613 mutex_init(&sc->sc_ctrl_wait_lock, MUTEX_DEFAULT, IPL_NET); 614 sc->sc_ctrl_inuse = FREE; 615 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 616 sc->sc_has_ctrl = true; 617 nvqs = 3; 618 } 619 skip: 620 621 #ifdef VIOIF_MPSAFE 622 flags = SOFTINT_NET | SOFTINT_MPSAFE; 623 #else 624 flags = SOFTINT_NET; 625 #endif 626 sc->sc_rx_softint = softint_establish(flags, vioif_rx_softint, sc); 627 if (sc->sc_rx_softint == NULL) { 628 aprint_error_dev(self, "cannot establish rx softint\n"); 629 goto err; 630 } 631 632 sc->sc_ctl_softint = softint_establish(flags, vioif_ctl_softint, sc); 633 if (sc->sc_ctl_softint == NULL) { 634 aprint_error_dev(self, "cannot establish ctl softint\n"); 635 goto err; 636 } 637 638 if (vioif_alloc_mems(sc) < 0) 639 goto err; 640 641 if (virtio_child_attach_finish(vsc) != 0) 642 goto err; 643 644 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 645 ifp->if_softc = sc; 646 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 647 ifp->if_start = vioif_start; 648 ifp->if_ioctl = vioif_ioctl; 649 ifp->if_init = vioif_init; 650 ifp->if_stop = vioif_stop; 651 ifp->if_capabilities = 0; 652 ifp->if_watchdog = vioif_watchdog; 653 IFQ_SET_MAXLEN(&ifp->if_snd, MAX(sc->sc_vq[VQ_TX].vq_num, IFQ_MAXLEN)); 654 IFQ_SET_READY(&ifp->if_snd); 655 656 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 657 658 if_attach(ifp); 659 if_deferred_start_init(ifp, NULL); 660 ether_ifattach(ifp, sc->sc_mac); 661 662 return; 663 664 err: 665 mutex_destroy(&sc->sc_tx_lock); 666 mutex_destroy(&sc->sc_rx_lock); 667 668 if (sc->sc_has_ctrl) { 669 cv_destroy(&sc->sc_ctrl_wait); 670 mutex_destroy(&sc->sc_ctrl_wait_lock); 671 } 672 673 while (nvqs > 0) 674 virtio_free_vq(vsc, &sc->sc_vq[--nvqs]); 675 676 virtio_child_attach_failed(vsc); 677 return; 678 } 679 680 /* we need interrupts to make promiscuous mode off */ 681 static void 682 vioif_deferred_init(device_t self) 683 { 684 struct vioif_softc *sc = device_private(self); 685 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 686 int r; 687 688 if (ifp->if_flags & IFF_PROMISC) 689 return; 690 691 r = vioif_set_promisc(sc, false); 692 if (r != 0) 693 aprint_error_dev(self, "resetting promisc mode failed, " 694 "errror code %d\n", r); 695 } 696 697 /* 698 * Interface functions for ifnet 699 */ 700 static int 701 vioif_init(struct ifnet *ifp) 702 { 703 struct vioif_softc *sc = ifp->if_softc; 704 struct virtio_softc *vsc = sc->sc_virtio; 705 706 vioif_stop(ifp, 0); 707 708 virtio_reinit_start(vsc); 709 virtio_negotiate_features(vsc, virtio_features(vsc)); 710 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 711 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 712 if (sc->sc_has_ctrl) 713 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 714 virtio_reinit_end(vsc); 715 716 if (!sc->sc_deferred_init_done) { 717 sc->sc_deferred_init_done = 1; 718 if (sc->sc_has_ctrl) 719 vioif_deferred_init(sc->sc_dev); 720 } 721 722 /* Have to set false before vioif_populate_rx_mbufs */ 723 sc->sc_stopping = false; 724 725 vioif_populate_rx_mbufs(sc); 726 727 vioif_update_link_status(sc); 728 ifp->if_flags |= IFF_RUNNING; 729 ifp->if_flags &= ~IFF_OACTIVE; 730 vioif_rx_filter(sc); 731 732 return 0; 733 } 734 735 static void 736 vioif_stop(struct ifnet *ifp, int disable) 737 { 738 struct vioif_softc *sc = ifp->if_softc; 739 struct virtio_softc *vsc = sc->sc_virtio; 740 741 /* Take the locks to ensure that ongoing TX/RX finish */ 742 VIOIF_TX_LOCK(sc); 743 VIOIF_RX_LOCK(sc); 744 sc->sc_stopping = true; 745 VIOIF_RX_UNLOCK(sc); 746 VIOIF_TX_UNLOCK(sc); 747 748 /* disable interrupts */ 749 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 750 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 751 if (sc->sc_has_ctrl) 752 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 753 754 /* only way to stop I/O and DMA is resetting... */ 755 virtio_reset(vsc); 756 vioif_rx_deq(sc); 757 vioif_tx_drain(sc); 758 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 759 sc->sc_link_active = false; 760 761 if (disable) 762 vioif_rx_drain(sc); 763 } 764 765 static void 766 vioif_start(struct ifnet *ifp) 767 { 768 struct vioif_softc *sc = ifp->if_softc; 769 struct virtio_softc *vsc = sc->sc_virtio; 770 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 771 struct mbuf *m; 772 int queued = 0; 773 774 VIOIF_TX_LOCK(sc); 775 776 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING || 777 !sc->sc_link_active) 778 goto out; 779 780 if (sc->sc_stopping) 781 goto out; 782 783 for (;;) { 784 int slot, r; 785 786 IFQ_DEQUEUE(&ifp->if_snd, m); 787 if (m == NULL) 788 break; 789 790 r = virtio_enqueue_prep(vsc, vq, &slot); 791 if (r == EAGAIN) { 792 ifp->if_flags |= IFF_OACTIVE; 793 m_freem(m); 794 break; 795 } 796 if (r != 0) 797 panic("enqueue_prep for a tx buffer"); 798 799 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 800 sc->sc_tx_dmamaps[slot], 801 m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 802 if (r != 0) { 803 /* maybe just too fragmented */ 804 struct mbuf *newm; 805 806 newm = m_defrag(m, M_NOWAIT); 807 if (newm == NULL) { 808 aprint_error_dev(sc->sc_dev, 809 "m_defrag() failed\n"); 810 goto skip; 811 } 812 813 m = newm; 814 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 815 sc->sc_tx_dmamaps[slot], 816 m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 817 if (r != 0) { 818 aprint_error_dev(sc->sc_dev, 819 "tx dmamap load failed, error code %d\n", 820 r); 821 skip: 822 m_freem(m); 823 virtio_enqueue_abort(vsc, vq, slot); 824 continue; 825 } 826 } 827 828 /* This should actually never fail */ 829 r = virtio_enqueue_reserve(vsc, vq, slot, 830 sc->sc_tx_dmamaps[slot]->dm_nsegs + 1); 831 if (r != 0) { 832 aprint_error_dev(sc->sc_dev, 833 "virtio_enqueue_reserve failed, error code %d\n", 834 r); 835 bus_dmamap_unload(virtio_dmat(vsc), 836 sc->sc_tx_dmamaps[slot]); 837 /* slot already freed by virtio_enqueue_reserve */ 838 m_freem(m); 839 continue; 840 } 841 842 sc->sc_tx_mbufs[slot] = m; 843 844 memset(&sc->sc_tx_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); 845 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 846 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 847 BUS_DMASYNC_PREWRITE); 848 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 849 0, sc->sc_txhdr_dmamaps[slot]->dm_mapsize, 850 BUS_DMASYNC_PREWRITE); 851 virtio_enqueue(vsc, vq, slot, sc->sc_txhdr_dmamaps[slot], true); 852 virtio_enqueue(vsc, vq, slot, sc->sc_tx_dmamaps[slot], true); 853 virtio_enqueue_commit(vsc, vq, slot, false); 854 855 queued++; 856 bpf_mtap(ifp, m, BPF_D_OUT); 857 } 858 859 if (queued > 0) { 860 virtio_enqueue_commit(vsc, vq, -1, true); 861 ifp->if_timer = 5; 862 } 863 864 out: 865 VIOIF_TX_UNLOCK(sc); 866 } 867 868 static int 869 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 870 { 871 int s, r; 872 873 s = splnet(); 874 875 r = ether_ioctl(ifp, cmd, data); 876 if ((r == 0 && cmd == SIOCSIFFLAGS) || 877 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) { 878 if (ifp->if_flags & IFF_RUNNING) 879 r = vioif_rx_filter(ifp->if_softc); 880 else 881 r = 0; 882 } 883 884 splx(s); 885 886 return r; 887 } 888 889 void 890 vioif_watchdog(struct ifnet *ifp) 891 { 892 struct vioif_softc *sc = ifp->if_softc; 893 894 if (ifp->if_flags & IFF_RUNNING) 895 vioif_tx_vq_done(&sc->sc_vq[VQ_TX]); 896 } 897 898 899 /* 900 * Receive implementation 901 */ 902 /* allocate and initialize a mbuf for receive */ 903 static int 904 vioif_add_rx_mbuf(struct vioif_softc *sc, int i) 905 { 906 struct mbuf *m; 907 int r; 908 909 MGETHDR(m, M_DONTWAIT, MT_DATA); 910 if (m == NULL) 911 return ENOBUFS; 912 MCLGET(m, M_DONTWAIT); 913 if ((m->m_flags & M_EXT) == 0) { 914 m_freem(m); 915 return ENOBUFS; 916 } 917 sc->sc_rx_mbufs[i] = m; 918 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 919 r = bus_dmamap_load_mbuf(virtio_dmat(sc->sc_virtio), 920 sc->sc_rx_dmamaps[i], 921 m, BUS_DMA_READ|BUS_DMA_NOWAIT); 922 if (r) { 923 m_freem(m); 924 sc->sc_rx_mbufs[i] = 0; 925 return r; 926 } 927 928 return 0; 929 } 930 931 /* free a mbuf for receive */ 932 static void 933 vioif_free_rx_mbuf(struct vioif_softc *sc, int i) 934 { 935 bus_dmamap_unload(virtio_dmat(sc->sc_virtio), sc->sc_rx_dmamaps[i]); 936 m_freem(sc->sc_rx_mbufs[i]); 937 sc->sc_rx_mbufs[i] = NULL; 938 } 939 940 /* add mbufs for all the empty receive slots */ 941 static void 942 vioif_populate_rx_mbufs(struct vioif_softc *sc) 943 { 944 VIOIF_RX_LOCK(sc); 945 vioif_populate_rx_mbufs_locked(sc); 946 VIOIF_RX_UNLOCK(sc); 947 } 948 949 static void 950 vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) 951 { 952 struct virtio_softc *vsc = sc->sc_virtio; 953 int i, r, ndone = 0; 954 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 955 956 KASSERT(VIOIF_RX_LOCKED(sc)); 957 958 if (sc->sc_stopping) 959 return; 960 961 for (i = 0; i < vq->vq_num; i++) { 962 int slot; 963 r = virtio_enqueue_prep(vsc, vq, &slot); 964 if (r == EAGAIN) 965 break; 966 if (r != 0) 967 panic("enqueue_prep for rx buffers"); 968 if (sc->sc_rx_mbufs[slot] == NULL) { 969 r = vioif_add_rx_mbuf(sc, slot); 970 if (r != 0) { 971 printf("%s: rx mbuf allocation failed, " 972 "error code %d\n", 973 device_xname(sc->sc_dev), r); 974 break; 975 } 976 } 977 r = virtio_enqueue_reserve(vsc, vq, slot, 978 sc->sc_rx_dmamaps[slot]->dm_nsegs + 1); 979 if (r != 0) { 980 vioif_free_rx_mbuf(sc, slot); 981 break; 982 } 983 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 984 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); 985 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 986 0, MCLBYTES, BUS_DMASYNC_PREREAD); 987 virtio_enqueue(vsc, vq, slot, sc->sc_rxhdr_dmamaps[slot], false); 988 virtio_enqueue(vsc, vq, slot, sc->sc_rx_dmamaps[slot], false); 989 virtio_enqueue_commit(vsc, vq, slot, false); 990 ndone++; 991 } 992 if (ndone > 0) 993 virtio_enqueue_commit(vsc, vq, -1, true); 994 } 995 996 /* dequeue received packets */ 997 static int 998 vioif_rx_deq(struct vioif_softc *sc) 999 { 1000 int r; 1001 1002 KASSERT(sc->sc_stopping); 1003 1004 VIOIF_RX_LOCK(sc); 1005 r = vioif_rx_deq_locked(sc); 1006 VIOIF_RX_UNLOCK(sc); 1007 1008 return r; 1009 } 1010 1011 /* dequeue received packets */ 1012 static int 1013 vioif_rx_deq_locked(struct vioif_softc *sc) 1014 { 1015 struct virtio_softc *vsc = sc->sc_virtio; 1016 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1017 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1018 struct mbuf *m; 1019 int r = 0; 1020 int slot, len; 1021 1022 KASSERT(VIOIF_RX_LOCKED(sc)); 1023 1024 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1025 len -= sizeof(struct virtio_net_hdr); 1026 r = 1; 1027 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 1028 0, sizeof(struct virtio_net_hdr), 1029 BUS_DMASYNC_POSTREAD); 1030 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 1031 0, MCLBYTES, 1032 BUS_DMASYNC_POSTREAD); 1033 m = sc->sc_rx_mbufs[slot]; 1034 KASSERT(m != NULL); 1035 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot]); 1036 sc->sc_rx_mbufs[slot] = 0; 1037 virtio_dequeue_commit(vsc, vq, slot); 1038 m_set_rcvif(m, ifp); 1039 m->m_len = m->m_pkthdr.len = len; 1040 1041 VIOIF_RX_UNLOCK(sc); 1042 if_percpuq_enqueue(ifp->if_percpuq, m); 1043 VIOIF_RX_LOCK(sc); 1044 1045 if (sc->sc_stopping) 1046 break; 1047 } 1048 1049 return r; 1050 } 1051 1052 /* rx interrupt; call _dequeue above and schedule a softint */ 1053 static int 1054 vioif_rx_vq_done(struct virtqueue *vq) 1055 { 1056 struct virtio_softc *vsc = vq->vq_owner; 1057 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1058 int r = 0; 1059 1060 #ifdef VIOIF_SOFTINT_INTR 1061 KASSERT(!cpu_intr_p()); 1062 #endif 1063 1064 VIOIF_RX_LOCK(sc); 1065 1066 if (sc->sc_stopping) 1067 goto out; 1068 1069 r = vioif_rx_deq_locked(sc); 1070 if (r) 1071 #ifdef VIOIF_SOFTINT_INTR 1072 vioif_populate_rx_mbufs_locked(sc); 1073 #else 1074 softint_schedule(sc->sc_rx_softint); 1075 #endif 1076 1077 out: 1078 VIOIF_RX_UNLOCK(sc); 1079 return r; 1080 } 1081 1082 /* softint: enqueue receive requests for new incoming packets */ 1083 static void 1084 vioif_rx_softint(void *arg) 1085 { 1086 struct vioif_softc *sc = arg; 1087 1088 vioif_populate_rx_mbufs(sc); 1089 } 1090 1091 /* free all the mbufs; called from if_stop(disable) */ 1092 static void 1093 vioif_rx_drain(struct vioif_softc *sc) 1094 { 1095 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1096 int i; 1097 1098 for (i = 0; i < vq->vq_num; i++) { 1099 if (sc->sc_rx_mbufs[i] == NULL) 1100 continue; 1101 vioif_free_rx_mbuf(sc, i); 1102 } 1103 } 1104 1105 1106 /* 1107 * Transmition implementation 1108 */ 1109 /* actual transmission is done in if_start */ 1110 /* tx interrupt; dequeue and free mbufs */ 1111 /* 1112 * tx interrupt is actually disabled; this should be called upon 1113 * tx vq full and watchdog 1114 */ 1115 static int 1116 vioif_tx_vq_done(struct virtqueue *vq) 1117 { 1118 struct virtio_softc *vsc = vq->vq_owner; 1119 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1120 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1121 int r = 0; 1122 1123 VIOIF_TX_LOCK(sc); 1124 1125 if (sc->sc_stopping) 1126 goto out; 1127 1128 r = vioif_tx_vq_done_locked(vq); 1129 1130 out: 1131 VIOIF_TX_UNLOCK(sc); 1132 if (r) 1133 if_schedule_deferred_start(ifp); 1134 return r; 1135 } 1136 1137 static int 1138 vioif_tx_vq_done_locked(struct virtqueue *vq) 1139 { 1140 struct virtio_softc *vsc = vq->vq_owner; 1141 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1142 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1143 struct mbuf *m; 1144 int r = 0; 1145 int slot, len; 1146 1147 KASSERT(VIOIF_TX_LOCKED(sc)); 1148 1149 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1150 r++; 1151 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 1152 0, sizeof(struct virtio_net_hdr), 1153 BUS_DMASYNC_POSTWRITE); 1154 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 1155 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 1156 BUS_DMASYNC_POSTWRITE); 1157 m = sc->sc_tx_mbufs[slot]; 1158 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot]); 1159 sc->sc_tx_mbufs[slot] = 0; 1160 virtio_dequeue_commit(vsc, vq, slot); 1161 ifp->if_opackets++; 1162 m_freem(m); 1163 } 1164 1165 if (r) 1166 ifp->if_flags &= ~IFF_OACTIVE; 1167 return r; 1168 } 1169 1170 /* free all the mbufs already put on vq; called from if_stop(disable) */ 1171 static void 1172 vioif_tx_drain(struct vioif_softc *sc) 1173 { 1174 struct virtio_softc *vsc = sc->sc_virtio; 1175 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 1176 int i; 1177 1178 KASSERT(sc->sc_stopping); 1179 1180 for (i = 0; i < vq->vq_num; i++) { 1181 if (sc->sc_tx_mbufs[i] == NULL) 1182 continue; 1183 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[i]); 1184 m_freem(sc->sc_tx_mbufs[i]); 1185 sc->sc_tx_mbufs[i] = NULL; 1186 } 1187 } 1188 1189 /* 1190 * Control vq 1191 */ 1192 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1193 static int 1194 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 1195 { 1196 struct virtio_softc *vsc = sc->sc_virtio; 1197 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1198 int r, slot; 1199 1200 if (!sc->sc_has_ctrl) 1201 return ENOTSUP; 1202 1203 mutex_enter(&sc->sc_ctrl_wait_lock); 1204 while (sc->sc_ctrl_inuse != FREE) 1205 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1206 sc->sc_ctrl_inuse = INUSE; 1207 mutex_exit(&sc->sc_ctrl_wait_lock); 1208 1209 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_RX; 1210 sc->sc_ctrl_cmd->command = cmd; 1211 sc->sc_ctrl_rx->onoff = onoff; 1212 1213 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1214 0, sizeof(struct virtio_net_ctrl_cmd), 1215 BUS_DMASYNC_PREWRITE); 1216 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 1217 0, sizeof(struct virtio_net_ctrl_rx), 1218 BUS_DMASYNC_PREWRITE); 1219 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1220 0, sizeof(struct virtio_net_ctrl_status), 1221 BUS_DMASYNC_PREREAD); 1222 1223 r = virtio_enqueue_prep(vsc, vq, &slot); 1224 if (r != 0) 1225 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1226 r = virtio_enqueue_reserve(vsc, vq, slot, 3); 1227 if (r != 0) 1228 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1229 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1230 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_rx_dmamap, true); 1231 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1232 virtio_enqueue_commit(vsc, vq, slot, true); 1233 1234 /* wait for done */ 1235 mutex_enter(&sc->sc_ctrl_wait_lock); 1236 while (sc->sc_ctrl_inuse != DONE) 1237 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1238 mutex_exit(&sc->sc_ctrl_wait_lock); 1239 /* already dequeueued */ 1240 1241 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1242 sizeof(struct virtio_net_ctrl_cmd), 1243 BUS_DMASYNC_POSTWRITE); 1244 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 0, 1245 sizeof(struct virtio_net_ctrl_rx), 1246 BUS_DMASYNC_POSTWRITE); 1247 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1248 sizeof(struct virtio_net_ctrl_status), 1249 BUS_DMASYNC_POSTREAD); 1250 1251 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1252 r = 0; 1253 else { 1254 printf("%s: failed setting rx mode\n", 1255 device_xname(sc->sc_dev)); 1256 r = EIO; 1257 } 1258 1259 mutex_enter(&sc->sc_ctrl_wait_lock); 1260 sc->sc_ctrl_inuse = FREE; 1261 cv_signal(&sc->sc_ctrl_wait); 1262 mutex_exit(&sc->sc_ctrl_wait_lock); 1263 1264 return r; 1265 } 1266 1267 static int 1268 vioif_set_promisc(struct vioif_softc *sc, bool onoff) 1269 { 1270 int r; 1271 1272 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 1273 1274 return r; 1275 } 1276 1277 static int 1278 vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 1279 { 1280 int r; 1281 1282 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 1283 1284 return r; 1285 } 1286 1287 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 1288 static int 1289 vioif_set_rx_filter(struct vioif_softc *sc) 1290 { 1291 /* filter already set in sc_ctrl_mac_tbl */ 1292 struct virtio_softc *vsc = sc->sc_virtio; 1293 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1294 int r, slot; 1295 1296 if (!sc->sc_has_ctrl) 1297 return ENOTSUP; 1298 1299 mutex_enter(&sc->sc_ctrl_wait_lock); 1300 while (sc->sc_ctrl_inuse != FREE) 1301 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1302 sc->sc_ctrl_inuse = INUSE; 1303 mutex_exit(&sc->sc_ctrl_wait_lock); 1304 1305 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_MAC; 1306 sc->sc_ctrl_cmd->command = VIRTIO_NET_CTRL_MAC_TABLE_SET; 1307 1308 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 1309 sc->sc_ctrl_mac_tbl_uc, 1310 (sizeof(struct virtio_net_ctrl_mac_tbl) 1311 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1312 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1313 if (r) { 1314 printf("%s: control command dmamap load failed, " 1315 "error code %d\n", device_xname(sc->sc_dev), r); 1316 goto out; 1317 } 1318 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 1319 sc->sc_ctrl_mac_tbl_mc, 1320 (sizeof(struct virtio_net_ctrl_mac_tbl) 1321 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1322 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1323 if (r) { 1324 printf("%s: control command dmamap load failed, " 1325 "error code %d\n", device_xname(sc->sc_dev), r); 1326 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1327 goto out; 1328 } 1329 1330 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1331 0, sizeof(struct virtio_net_ctrl_cmd), 1332 BUS_DMASYNC_PREWRITE); 1333 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1334 (sizeof(struct virtio_net_ctrl_mac_tbl) 1335 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1336 BUS_DMASYNC_PREWRITE); 1337 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1338 (sizeof(struct virtio_net_ctrl_mac_tbl) 1339 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1340 BUS_DMASYNC_PREWRITE); 1341 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1342 0, sizeof(struct virtio_net_ctrl_status), 1343 BUS_DMASYNC_PREREAD); 1344 1345 r = virtio_enqueue_prep(vsc, vq, &slot); 1346 if (r != 0) 1347 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1348 r = virtio_enqueue_reserve(vsc, vq, slot, 4); 1349 if (r != 0) 1350 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1351 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1352 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_uc_dmamap, true); 1353 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_mc_dmamap, true); 1354 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1355 virtio_enqueue_commit(vsc, vq, slot, true); 1356 1357 /* wait for done */ 1358 mutex_enter(&sc->sc_ctrl_wait_lock); 1359 while (sc->sc_ctrl_inuse != DONE) 1360 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1361 mutex_exit(&sc->sc_ctrl_wait_lock); 1362 /* already dequeueued */ 1363 1364 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1365 sizeof(struct virtio_net_ctrl_cmd), 1366 BUS_DMASYNC_POSTWRITE); 1367 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1368 (sizeof(struct virtio_net_ctrl_mac_tbl) 1369 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1370 BUS_DMASYNC_POSTWRITE); 1371 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1372 (sizeof(struct virtio_net_ctrl_mac_tbl) 1373 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1374 BUS_DMASYNC_POSTWRITE); 1375 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1376 sizeof(struct virtio_net_ctrl_status), 1377 BUS_DMASYNC_POSTREAD); 1378 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1379 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap); 1380 1381 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1382 r = 0; 1383 else { 1384 printf("%s: failed setting rx filter\n", 1385 device_xname(sc->sc_dev)); 1386 r = EIO; 1387 } 1388 1389 out: 1390 mutex_enter(&sc->sc_ctrl_wait_lock); 1391 sc->sc_ctrl_inuse = FREE; 1392 cv_signal(&sc->sc_ctrl_wait); 1393 mutex_exit(&sc->sc_ctrl_wait_lock); 1394 1395 return r; 1396 } 1397 1398 /* ctrl vq interrupt; wake up the command issuer */ 1399 static int 1400 vioif_ctrl_vq_done(struct virtqueue *vq) 1401 { 1402 struct virtio_softc *vsc = vq->vq_owner; 1403 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1404 int r, slot; 1405 1406 r = virtio_dequeue(vsc, vq, &slot, NULL); 1407 if (r == ENOENT) 1408 return 0; 1409 virtio_dequeue_commit(vsc, vq, slot); 1410 1411 mutex_enter(&sc->sc_ctrl_wait_lock); 1412 sc->sc_ctrl_inuse = DONE; 1413 cv_signal(&sc->sc_ctrl_wait); 1414 mutex_exit(&sc->sc_ctrl_wait_lock); 1415 1416 return 1; 1417 } 1418 1419 /* 1420 * If IFF_PROMISC requested, set promiscuous 1421 * If multicast filter small enough (<=MAXENTRIES) set rx filter 1422 * If large multicast filter exist use ALLMULTI 1423 */ 1424 /* 1425 * If setting rx filter fails fall back to ALLMULTI 1426 * If ALLMULTI fails fall back to PROMISC 1427 */ 1428 static int 1429 vioif_rx_filter(struct vioif_softc *sc) 1430 { 1431 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1432 struct ether_multi *enm; 1433 struct ether_multistep step; 1434 int nentries; 1435 int promisc = 0, allmulti = 0, rxfilter = 0; 1436 int r; 1437 1438 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */ 1439 ifp->if_flags |= IFF_PROMISC; 1440 return 0; 1441 } 1442 1443 if (ifp->if_flags & IFF_PROMISC) { 1444 promisc = 1; 1445 goto set; 1446 } 1447 1448 nentries = -1; 1449 ETHER_LOCK(&sc->sc_ethercom); 1450 ETHER_FIRST_MULTI(step, &sc->sc_ethercom, enm); 1451 while (nentries++, enm != NULL) { 1452 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 1453 allmulti = 1; 1454 goto set_unlock; 1455 } 1456 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1457 ETHER_ADDR_LEN)) { 1458 allmulti = 1; 1459 goto set_unlock; 1460 } 1461 memcpy(sc->sc_ctrl_mac_tbl_mc->macs[nentries], 1462 enm->enm_addrlo, ETHER_ADDR_LEN); 1463 ETHER_NEXT_MULTI(step, enm); 1464 } 1465 rxfilter = 1; 1466 1467 set_unlock: 1468 ETHER_UNLOCK(&sc->sc_ethercom); 1469 1470 set: 1471 if (rxfilter) { 1472 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1473 sc->sc_ctrl_mac_tbl_mc->nentries = nentries; 1474 r = vioif_set_rx_filter(sc); 1475 if (r != 0) { 1476 rxfilter = 0; 1477 allmulti = 1; /* fallback */ 1478 } 1479 } else { 1480 /* remove rx filter */ 1481 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1482 sc->sc_ctrl_mac_tbl_mc->nentries = 0; 1483 r = vioif_set_rx_filter(sc); 1484 /* what to do on failure? */ 1485 } 1486 if (allmulti) { 1487 r = vioif_set_allmulti(sc, true); 1488 if (r != 0) { 1489 allmulti = 0; 1490 promisc = 1; /* fallback */ 1491 } 1492 } else { 1493 r = vioif_set_allmulti(sc, false); 1494 /* what to do on failure? */ 1495 } 1496 if (promisc) { 1497 r = vioif_set_promisc(sc, true); 1498 } else { 1499 r = vioif_set_promisc(sc, false); 1500 } 1501 1502 return r; 1503 } 1504 1505 static bool 1506 vioif_is_link_up(struct vioif_softc *sc) 1507 { 1508 struct virtio_softc *vsc = sc->sc_virtio; 1509 uint16_t status; 1510 1511 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 1512 status = virtio_read_device_config_2(vsc, 1513 VIRTIO_NET_CONFIG_STATUS); 1514 else 1515 status = VIRTIO_NET_S_LINK_UP; 1516 1517 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 1518 } 1519 1520 /* change link status */ 1521 static void 1522 vioif_update_link_status(struct vioif_softc *sc) 1523 { 1524 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1525 bool active, changed; 1526 int link; 1527 1528 active = vioif_is_link_up(sc); 1529 changed = false; 1530 1531 VIOIF_TX_LOCK(sc); 1532 if (active) { 1533 if (!sc->sc_link_active) 1534 changed = true; 1535 1536 link = LINK_STATE_UP; 1537 sc->sc_link_active = true; 1538 } else { 1539 if (sc->sc_link_active) 1540 changed = true; 1541 1542 link = LINK_STATE_DOWN; 1543 sc->sc_link_active = false; 1544 } 1545 VIOIF_TX_UNLOCK(sc); 1546 1547 if (changed) 1548 if_link_state_change(ifp, link); 1549 } 1550 1551 static int 1552 vioif_config_change(struct virtio_softc *vsc) 1553 { 1554 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1555 1556 #ifdef VIOIF_SOFTINT_INTR 1557 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1558 #endif 1559 1560 #ifdef VIOIF_SOFTINT_INTR 1561 KASSERT(!cpu_intr_p()); 1562 vioif_update_link_status(sc); 1563 vioif_start(ifp); 1564 #else 1565 softint_schedule(sc->sc_ctl_softint); 1566 #endif 1567 1568 return 0; 1569 } 1570 1571 static void 1572 vioif_ctl_softint(void *arg) 1573 { 1574 struct vioif_softc *sc = arg; 1575 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1576 1577 vioif_update_link_status(sc); 1578 vioif_start(ifp); 1579 } 1580 1581 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 1582 1583 #ifdef _MODULE 1584 #include "ioconf.c" 1585 #endif 1586 1587 static int 1588 if_vioif_modcmd(modcmd_t cmd, void *opaque) 1589 { 1590 int error = 0; 1591 1592 #ifdef _MODULE 1593 switch (cmd) { 1594 case MODULE_CMD_INIT: 1595 error = config_init_component(cfdriver_ioconf_if_vioif, 1596 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1597 break; 1598 case MODULE_CMD_FINI: 1599 error = config_fini_component(cfdriver_ioconf_if_vioif, 1600 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1601 break; 1602 default: 1603 error = ENOTTY; 1604 break; 1605 } 1606 #endif 1607 1608 return error; 1609 } 1610
Indexes created Fri Sep 26 08:10:20 GMT 2025