if_vioif.c revision 1.35
1 /* $NetBSD: if_vioif.c,v 1.35 2017/05/17 18:21:40 jdolecek Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.35 2017/05/17 18:21:40 jdolecek Exp $"); 30 31 #ifdef _KERNEL_OPT 32 #include "opt_net_mpsafe.h" 33 #endif 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/condvar.h> 40 #include <sys/device.h> 41 #include <sys/intr.h> 42 #include <sys/kmem.h> 43 #include <sys/mbuf.h> 44 #include <sys/mutex.h> 45 #include <sys/sockio.h> 46 #include <sys/cpu.h> 47 #include <sys/module.h> 48 49 #include <dev/pci/pcidevs.h> 50 #include <dev/pci/pcireg.h> 51 #include <dev/pci/pcivar.h> 52 #include <dev/pci/virtioreg.h> 53 #include <dev/pci/virtiovar.h> 54 55 #include <net/if.h> 56 #include <net/if_media.h> 57 #include <net/if_ether.h> 58 59 #include <net/bpf.h> 60 61 #include "ioconf.h" 62 63 #ifdef NET_MPSAFE 64 #define VIOIF_MPSAFE 1 65 #endif 66 67 #ifdef SOFTINT_INTR 68 #define VIOIF_SOFTINT_INTR 1 69 #endif 70 71 /* 72 * if_vioifreg.h: 73 */ 74 /* Configuration registers */ 75 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 76 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 77 78 /* Feature bits */ 79 #define VIRTIO_NET_F_CSUM (1<<0) 80 #define VIRTIO_NET_F_GUEST_CSUM (1<<1) 81 #define VIRTIO_NET_F_MAC (1<<5) 82 #define VIRTIO_NET_F_GSO (1<<6) 83 #define VIRTIO_NET_F_GUEST_TSO4 (1<<7) 84 #define VIRTIO_NET_F_GUEST_TSO6 (1<<8) 85 #define VIRTIO_NET_F_GUEST_ECN (1<<9) 86 #define VIRTIO_NET_F_GUEST_UFO (1<<10) 87 #define VIRTIO_NET_F_HOST_TSO4 (1<<11) 88 #define VIRTIO_NET_F_HOST_TSO6 (1<<12) 89 #define VIRTIO_NET_F_HOST_ECN (1<<13) 90 #define VIRTIO_NET_F_HOST_UFO (1<<14) 91 #define VIRTIO_NET_F_MRG_RXBUF (1<<15) 92 #define VIRTIO_NET_F_STATUS (1<<16) 93 #define VIRTIO_NET_F_CTRL_VQ (1<<17) 94 #define VIRTIO_NET_F_CTRL_RX (1<<18) 95 #define VIRTIO_NET_F_CTRL_VLAN (1<<19) 96 97 #define VIRTIO_NET_FLAG_BITS \ 98 VIRTIO_COMMON_FLAG_BITS \ 99 "\x14""CTRL_VLAN" \ 100 "\x13""CTRL_RX" \ 101 "\x12""CTRL_VQ" \ 102 "\x11""STATUS" \ 103 "\x10""MRG_RXBUF" \ 104 "\x0f""HOST_UFO" \ 105 "\x0e""HOST_ECN" \ 106 "\x0d""HOST_TSO6" \ 107 "\x0c""HOST_TSO4" \ 108 "\x0b""GUEST_UFO" \ 109 "\x0a""GUEST_ECN" \ 110 "\x09""GUEST_TSO6" \ 111 "\x08""GUEST_TSO4" \ 112 "\x07""GSO" \ 113 "\x06""MAC" \ 114 "\x02""GUEST_CSUM" \ 115 "\x01""CSUM" 116 117 /* Status */ 118 #define VIRTIO_NET_S_LINK_UP 1 119 120 /* Packet header structure */ 121 struct virtio_net_hdr { 122 uint8_t flags; 123 uint8_t gso_type; 124 uint16_t hdr_len; 125 uint16_t gso_size; 126 uint16_t csum_start; 127 uint16_t csum_offset; 128 #if 0 129 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */ 130 #endif 131 } __packed; 132 133 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 134 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 135 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 136 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 137 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 138 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 139 140 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 141 142 /* Control virtqueue */ 143 struct virtio_net_ctrl_cmd { 144 uint8_t class; 145 uint8_t command; 146 } __packed; 147 #define VIRTIO_NET_CTRL_RX 0 148 # define VIRTIO_NET_CTRL_RX_PROMISC 0 149 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 150 151 #define VIRTIO_NET_CTRL_MAC 1 152 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 153 154 #define VIRTIO_NET_CTRL_VLAN 2 155 # define VIRTIO_NET_CTRL_VLAN_ADD 0 156 # define VIRTIO_NET_CTRL_VLAN_DEL 1 157 158 struct virtio_net_ctrl_status { 159 uint8_t ack; 160 } __packed; 161 #define VIRTIO_NET_OK 0 162 #define VIRTIO_NET_ERR 1 163 164 struct virtio_net_ctrl_rx { 165 uint8_t onoff; 166 } __packed; 167 168 struct virtio_net_ctrl_mac_tbl { 169 uint32_t nentries; 170 uint8_t macs[][ETHER_ADDR_LEN]; 171 } __packed; 172 173 struct virtio_net_ctrl_vlan { 174 uint16_t id; 175 } __packed; 176 177 178 /* 179 * if_vioifvar.h: 180 */ 181 struct vioif_softc { 182 device_t sc_dev; 183 184 struct virtio_softc *sc_virtio; 185 struct virtqueue sc_vq[3]; 186 #define VQ_RX 0 187 #define VQ_TX 1 188 #define VQ_CTRL 2 189 190 uint8_t sc_mac[ETHER_ADDR_LEN]; 191 struct ethercom sc_ethercom; 192 short sc_deferred_init_done; 193 bool sc_link_active; 194 195 /* bus_dmamem */ 196 bus_dma_segment_t sc_hdr_segs[1]; 197 struct virtio_net_hdr *sc_hdrs; 198 #define sc_rx_hdrs sc_hdrs 199 struct virtio_net_hdr *sc_tx_hdrs; 200 struct virtio_net_ctrl_cmd *sc_ctrl_cmd; 201 struct virtio_net_ctrl_status *sc_ctrl_status; 202 struct virtio_net_ctrl_rx *sc_ctrl_rx; 203 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc; 204 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc; 205 206 /* kmem */ 207 bus_dmamap_t *sc_arrays; 208 #define sc_rxhdr_dmamaps sc_arrays 209 bus_dmamap_t *sc_txhdr_dmamaps; 210 bus_dmamap_t *sc_rx_dmamaps; 211 bus_dmamap_t *sc_tx_dmamaps; 212 struct mbuf **sc_rx_mbufs; 213 struct mbuf **sc_tx_mbufs; 214 215 bus_dmamap_t sc_ctrl_cmd_dmamap; 216 bus_dmamap_t sc_ctrl_status_dmamap; 217 bus_dmamap_t sc_ctrl_rx_dmamap; 218 bus_dmamap_t sc_ctrl_tbl_uc_dmamap; 219 bus_dmamap_t sc_ctrl_tbl_mc_dmamap; 220 221 void *sc_rx_softint; 222 void *sc_ctl_softint; 223 224 enum { 225 FREE, INUSE, DONE 226 } sc_ctrl_inuse; 227 kcondvar_t sc_ctrl_wait; 228 kmutex_t sc_ctrl_wait_lock; 229 kmutex_t *sc_tx_lock; 230 kmutex_t *sc_rx_lock; 231 bool sc_stopping; 232 233 bool sc_has_ctrl; 234 }; 235 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 236 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 237 238 #define VIOIF_TX_LOCK(_sc) if ((_sc)->sc_tx_lock) mutex_enter((_sc)->sc_tx_lock) 239 #define VIOIF_TX_UNLOCK(_sc) if ((_sc)->sc_tx_lock) mutex_exit((_sc)->sc_tx_lock) 240 #define VIOIF_TX_LOCKED(_sc) (!(_sc)->sc_tx_lock || mutex_owned((_sc)->sc_tx_lock)) 241 #define VIOIF_RX_LOCK(_sc) if ((_sc)->sc_rx_lock) mutex_enter((_sc)->sc_rx_lock) 242 #define VIOIF_RX_UNLOCK(_sc) if ((_sc)->sc_rx_lock) mutex_exit((_sc)->sc_rx_lock) 243 #define VIOIF_RX_LOCKED(_sc) (!(_sc)->sc_rx_lock || mutex_owned((_sc)->sc_rx_lock)) 244 245 /* cfattach interface functions */ 246 static int vioif_match(device_t, cfdata_t, void *); 247 static void vioif_attach(device_t, device_t, void *); 248 static void vioif_deferred_init(device_t); 249 250 /* ifnet interface functions */ 251 static int vioif_init(struct ifnet *); 252 static void vioif_stop(struct ifnet *, int); 253 static void vioif_start(struct ifnet *); 254 static int vioif_ioctl(struct ifnet *, u_long, void *); 255 static void vioif_watchdog(struct ifnet *); 256 257 /* rx */ 258 static int vioif_add_rx_mbuf(struct vioif_softc *, int); 259 static void vioif_free_rx_mbuf(struct vioif_softc *, int); 260 static void vioif_populate_rx_mbufs(struct vioif_softc *); 261 static void vioif_populate_rx_mbufs_locked(struct vioif_softc *); 262 static int vioif_rx_deq(struct vioif_softc *); 263 static int vioif_rx_deq_locked(struct vioif_softc *); 264 static int vioif_rx_vq_done(struct virtqueue *); 265 static void vioif_rx_softint(void *); 266 static void vioif_rx_drain(struct vioif_softc *); 267 268 /* tx */ 269 static int vioif_tx_vq_done(struct virtqueue *); 270 static int vioif_tx_vq_done_locked(struct virtqueue *); 271 static void vioif_tx_drain(struct vioif_softc *); 272 273 /* other control */ 274 static bool vioif_is_link_up(struct vioif_softc *); 275 static void vioif_update_link_status(struct vioif_softc *); 276 static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 277 static int vioif_set_promisc(struct vioif_softc *, bool); 278 static int vioif_set_allmulti(struct vioif_softc *, bool); 279 static int vioif_set_rx_filter(struct vioif_softc *); 280 static int vioif_rx_filter(struct vioif_softc *); 281 static int vioif_ctrl_vq_done(struct virtqueue *); 282 static int vioif_config_change(struct virtio_softc *); 283 static void vioif_ctl_softint(void *); 284 285 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 286 vioif_match, vioif_attach, NULL, NULL); 287 288 static int 289 vioif_match(device_t parent, cfdata_t match, void *aux) 290 { 291 struct virtio_attach_args *va = aux; 292 293 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK) 294 return 1; 295 296 return 0; 297 } 298 299 /* allocate memory */ 300 /* 301 * dma memory is used for: 302 * sc_rx_hdrs[slot]: metadata array for recieved frames (READ) 303 * sc_tx_hdrs[slot]: metadata array for frames to be sent (WRITE) 304 * sc_ctrl_cmd: command to be sent via ctrl vq (WRITE) 305 * sc_ctrl_status: return value for a command via ctrl vq (READ) 306 * sc_ctrl_rx: parameter for a VIRTIO_NET_CTRL_RX class command 307 * (WRITE) 308 * sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 309 * class command (WRITE) 310 * sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 311 * class command (WRITE) 312 * sc_ctrl_* structures are allocated only one each; they are protected by 313 * sc_ctrl_inuse variable and sc_ctrl_wait condvar. 314 */ 315 /* 316 * dynamically allocated memory is used for: 317 * sc_rxhdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] 318 * sc_txhdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] 319 * sc_rx_dmamaps[slot]: bus_dmamap_t array for recieved payload 320 * sc_tx_dmamaps[slot]: bus_dmamap_t array for sent payload 321 * sc_rx_mbufs[slot]: mbuf pointer array for recieved frames 322 * sc_tx_mbufs[slot]: mbuf pointer array for sent frames 323 */ 324 static int 325 vioif_alloc_mems(struct vioif_softc *sc) 326 { 327 struct virtio_softc *vsc = sc->sc_virtio; 328 int allocsize, allocsize2, r, rsegs, i; 329 void *vaddr; 330 intptr_t p; 331 int rxqsize, txqsize; 332 333 rxqsize = sc->sc_vq[VQ_RX].vq_num; 334 txqsize = sc->sc_vq[VQ_TX].vq_num; 335 336 allocsize = sizeof(struct virtio_net_hdr) * rxqsize; 337 allocsize += sizeof(struct virtio_net_hdr) * txqsize; 338 if (sc->sc_has_ctrl) { 339 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 340 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 341 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 342 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) 343 + sizeof(struct virtio_net_ctrl_mac_tbl) 344 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 345 } 346 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, 347 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 348 if (r != 0) { 349 aprint_error_dev(sc->sc_dev, 350 "DMA memory allocation failed, size %d, " 351 "error code %d\n", allocsize, r); 352 goto err_none; 353 } 354 r = bus_dmamem_map(virtio_dmat(vsc), 355 &sc->sc_hdr_segs[0], 1, allocsize, 356 &vaddr, BUS_DMA_NOWAIT); 357 if (r != 0) { 358 aprint_error_dev(sc->sc_dev, 359 "DMA memory map failed, " 360 "error code %d\n", r); 361 goto err_dmamem_alloc; 362 } 363 sc->sc_hdrs = vaddr; 364 memset(vaddr, 0, allocsize); 365 p = (intptr_t) vaddr; 366 p += sizeof(struct virtio_net_hdr) * rxqsize; 367 #define P(name,size) do { sc->sc_ ##name = (void*) p; \ 368 p += size; } while (0) 369 P(tx_hdrs, sizeof(struct virtio_net_hdr) * txqsize); 370 if (sc->sc_has_ctrl) { 371 P(ctrl_cmd, sizeof(struct virtio_net_ctrl_cmd)); 372 P(ctrl_status, sizeof(struct virtio_net_ctrl_status)); 373 P(ctrl_rx, sizeof(struct virtio_net_ctrl_rx)); 374 P(ctrl_mac_tbl_uc, sizeof(struct virtio_net_ctrl_mac_tbl)); 375 P(ctrl_mac_tbl_mc, 376 (sizeof(struct virtio_net_ctrl_mac_tbl) 377 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES)); 378 } 379 #undef P 380 381 allocsize2 = sizeof(bus_dmamap_t) * (rxqsize + txqsize); 382 allocsize2 += sizeof(bus_dmamap_t) * (rxqsize + txqsize); 383 allocsize2 += sizeof(struct mbuf*) * (rxqsize + txqsize); 384 sc->sc_arrays = kmem_zalloc(allocsize2, KM_SLEEP); 385 if (sc->sc_arrays == NULL) 386 goto err_dmamem_map; 387 sc->sc_txhdr_dmamaps = sc->sc_arrays + rxqsize; 388 sc->sc_rx_dmamaps = sc->sc_txhdr_dmamaps + txqsize; 389 sc->sc_tx_dmamaps = sc->sc_rx_dmamaps + rxqsize; 390 sc->sc_rx_mbufs = (void*) (sc->sc_tx_dmamaps + txqsize); 391 sc->sc_tx_mbufs = sc->sc_rx_mbufs + rxqsize; 392 393 #define C(map, buf, size, nsegs, rw, usage) \ 394 do { \ 395 r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \ 396 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, \ 397 &sc->sc_ ##map); \ 398 if (r != 0) { \ 399 aprint_error_dev(sc->sc_dev, \ 400 usage " dmamap creation failed, " \ 401 "error code %d\n", r); \ 402 goto err_reqs; \ 403 } \ 404 } while (0) 405 #define C_L1(map, buf, size, nsegs, rw, usage) \ 406 C(map, buf, size, nsegs, rw, usage); \ 407 do { \ 408 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ 409 &sc->sc_ ##buf, size, NULL, \ 410 BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ 411 if (r != 0) { \ 412 aprint_error_dev(sc->sc_dev, \ 413 usage " dmamap load failed, " \ 414 "error code %d\n", r); \ 415 goto err_reqs; \ 416 } \ 417 } while (0) 418 #define C_L2(map, buf, size, nsegs, rw, usage) \ 419 C(map, buf, size, nsegs, rw, usage); \ 420 do { \ 421 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ 422 sc->sc_ ##buf, size, NULL, \ 423 BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ 424 if (r != 0) { \ 425 aprint_error_dev(sc->sc_dev, \ 426 usage " dmamap load failed, " \ 427 "error code %d\n", r); \ 428 goto err_reqs; \ 429 } \ 430 } while (0) 431 for (i = 0; i < rxqsize; i++) { 432 C_L1(rxhdr_dmamaps[i], rx_hdrs[i], 433 sizeof(struct virtio_net_hdr), 1, 434 READ, "rx header"); 435 C(rx_dmamaps[i], NULL, MCLBYTES, 1, 0, "rx payload"); 436 } 437 438 for (i = 0; i < txqsize; i++) { 439 C_L1(txhdr_dmamaps[i], tx_hdrs[i], 440 sizeof(struct virtio_net_hdr), 1, 441 WRITE, "tx header"); 442 C(tx_dmamaps[i], NULL, ETHER_MAX_LEN, 16 /* XXX */, 0, 443 "tx payload"); 444 } 445 446 if (sc->sc_has_ctrl) { 447 /* control vq class & command */ 448 C_L2(ctrl_cmd_dmamap, ctrl_cmd, 449 sizeof(struct virtio_net_ctrl_cmd), 1, WRITE, 450 "control command"); 451 452 /* control vq status */ 453 C_L2(ctrl_status_dmamap, ctrl_status, 454 sizeof(struct virtio_net_ctrl_status), 1, READ, 455 "control status"); 456 457 /* control vq rx mode command parameter */ 458 C_L2(ctrl_rx_dmamap, ctrl_rx, 459 sizeof(struct virtio_net_ctrl_rx), 1, WRITE, 460 "rx mode control command"); 461 462 /* control vq MAC filter table for unicast */ 463 /* do not load now since its length is variable */ 464 C(ctrl_tbl_uc_dmamap, NULL, 465 sizeof(struct virtio_net_ctrl_mac_tbl) + 0, 1, WRITE, 466 "unicast MAC address filter command"); 467 468 /* control vq MAC filter table for multicast */ 469 C(ctrl_tbl_mc_dmamap, NULL, 470 (sizeof(struct virtio_net_ctrl_mac_tbl) 471 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES), 472 1, WRITE, "multicast MAC address filter command"); 473 } 474 #undef C_L2 475 #undef C_L1 476 #undef C 477 478 return 0; 479 480 err_reqs: 481 #define D(map) \ 482 do { \ 483 if (sc->sc_ ##map) { \ 484 bus_dmamap_destroy(virtio_dmat(vsc), sc->sc_ ##map); \ 485 sc->sc_ ##map = NULL; \ 486 } \ 487 } while (0) 488 D(ctrl_tbl_mc_dmamap); 489 D(ctrl_tbl_uc_dmamap); 490 D(ctrl_rx_dmamap); 491 D(ctrl_status_dmamap); 492 D(ctrl_cmd_dmamap); 493 for (i = 0; i < txqsize; i++) { 494 D(tx_dmamaps[i]); 495 D(txhdr_dmamaps[i]); 496 } 497 for (i = 0; i < rxqsize; i++) { 498 D(rx_dmamaps[i]); 499 D(rxhdr_dmamaps[i]); 500 } 501 #undef D 502 if (sc->sc_arrays) { 503 kmem_free(sc->sc_arrays, allocsize2); 504 sc->sc_arrays = 0; 505 } 506 err_dmamem_map: 507 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_hdrs, allocsize); 508 err_dmamem_alloc: 509 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); 510 err_none: 511 return -1; 512 } 513 514 static void 515 vioif_attach(device_t parent, device_t self, void *aux) 516 { 517 struct vioif_softc *sc = device_private(self); 518 struct virtio_softc *vsc = device_private(parent); 519 uint32_t features; 520 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 521 u_int flags; 522 int r, nvqs=0, req_flags; 523 524 if (virtio_child(vsc) != NULL) { 525 aprint_normal(": child already attached for %s; " 526 "something wrong...\n", 527 device_xname(parent)); 528 return; 529 } 530 531 sc->sc_dev = self; 532 sc->sc_virtio = vsc; 533 sc->sc_link_active = false; 534 535 req_flags = 0; 536 537 #ifdef VIOIF_MPSAFE 538 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE; 539 #endif 540 #ifdef VIOIF_SOFTINT_INTR 541 req_flags |= VIRTIO_F_PCI_INTR_SOFTINT; 542 #endif 543 req_flags |= VIRTIO_F_PCI_INTR_MSIX; 544 545 virtio_child_attach_start(vsc, self, IPL_NET, sc->sc_vq, 546 vioif_config_change, virtio_vq_intr, req_flags, 547 (VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 548 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY), 549 VIRTIO_NET_FLAG_BITS); 550 551 features = virtio_features(vsc); 552 553 if (features & VIRTIO_NET_F_MAC) { 554 sc->sc_mac[0] = virtio_read_device_config_1(vsc, 555 VIRTIO_NET_CONFIG_MAC+0); 556 sc->sc_mac[1] = virtio_read_device_config_1(vsc, 557 VIRTIO_NET_CONFIG_MAC+1); 558 sc->sc_mac[2] = virtio_read_device_config_1(vsc, 559 VIRTIO_NET_CONFIG_MAC+2); 560 sc->sc_mac[3] = virtio_read_device_config_1(vsc, 561 VIRTIO_NET_CONFIG_MAC+3); 562 sc->sc_mac[4] = virtio_read_device_config_1(vsc, 563 VIRTIO_NET_CONFIG_MAC+4); 564 sc->sc_mac[5] = virtio_read_device_config_1(vsc, 565 VIRTIO_NET_CONFIG_MAC+5); 566 } else { 567 /* code stolen from sys/net/if_tap.c */ 568 struct timeval tv; 569 uint32_t ui; 570 getmicrouptime(&tv); 571 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 572 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 573 virtio_write_device_config_1(vsc, 574 VIRTIO_NET_CONFIG_MAC+0, 575 sc->sc_mac[0]); 576 virtio_write_device_config_1(vsc, 577 VIRTIO_NET_CONFIG_MAC+1, 578 sc->sc_mac[1]); 579 virtio_write_device_config_1(vsc, 580 VIRTIO_NET_CONFIG_MAC+2, 581 sc->sc_mac[2]); 582 virtio_write_device_config_1(vsc, 583 VIRTIO_NET_CONFIG_MAC+3, 584 sc->sc_mac[3]); 585 virtio_write_device_config_1(vsc, 586 VIRTIO_NET_CONFIG_MAC+4, 587 sc->sc_mac[4]); 588 virtio_write_device_config_1(vsc, 589 VIRTIO_NET_CONFIG_MAC+5, 590 sc->sc_mac[5]); 591 } 592 593 aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(sc->sc_mac)); 594 595 #ifdef VIOIF_MPSAFE 596 sc->sc_tx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 597 sc->sc_rx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 598 #else 599 sc->sc_tx_lock = NULL; 600 sc->sc_rx_lock = NULL; 601 #endif 602 sc->sc_stopping = false; 603 604 /* 605 * Allocating a virtqueue for Rx 606 */ 607 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_RX], VQ_RX, 608 MCLBYTES+sizeof(struct virtio_net_hdr), 2, "rx"); 609 if (r != 0) 610 goto err; 611 nvqs = 1; 612 sc->sc_vq[VQ_RX].vq_done = vioif_rx_vq_done; 613 614 /* 615 * Allocating a virtqueue for Tx 616 */ 617 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_TX], VQ_TX, 618 (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)), 619 VIRTIO_NET_TX_MAXNSEGS + 1, "tx"); 620 if (r != 0) 621 goto err; 622 nvqs = 2; 623 sc->sc_vq[VQ_TX].vq_done = vioif_tx_vq_done; 624 625 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 626 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); /* not urgent; do it later */ 627 628 if ((features & VIRTIO_NET_F_CTRL_VQ) && 629 (features & VIRTIO_NET_F_CTRL_RX)) { 630 /* 631 * Allocating a virtqueue for control channel 632 */ 633 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_CTRL], VQ_CTRL, 634 NBPG, 1, "control"); 635 if (r != 0) { 636 aprint_error_dev(self, "failed to allocate " 637 "a virtqueue for control channel\n"); 638 goto skip; 639 } 640 641 sc->sc_vq[VQ_CTRL].vq_done = vioif_ctrl_vq_done; 642 cv_init(&sc->sc_ctrl_wait, "ctrl_vq"); 643 mutex_init(&sc->sc_ctrl_wait_lock, MUTEX_DEFAULT, IPL_NET); 644 sc->sc_ctrl_inuse = FREE; 645 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 646 sc->sc_has_ctrl = true; 647 nvqs = 3; 648 } 649 skip: 650 651 #ifdef VIOIF_MPSAFE 652 flags = SOFTINT_NET | SOFTINT_MPSAFE; 653 #else 654 flags = SOFTINT_NET; 655 #endif 656 sc->sc_rx_softint = softint_establish(flags, vioif_rx_softint, sc); 657 if (sc->sc_rx_softint == NULL) { 658 aprint_error_dev(self, "cannot establish rx softint\n"); 659 goto err; 660 } 661 662 sc->sc_ctl_softint = softint_establish(flags, vioif_ctl_softint, sc); 663 if (sc->sc_ctl_softint == NULL) { 664 aprint_error_dev(self, "cannot establish ctl softint\n"); 665 goto err; 666 } 667 668 if (vioif_alloc_mems(sc) < 0) 669 goto err; 670 671 if (virtio_child_attach_finish(vsc) != 0) 672 goto err; 673 674 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 675 ifp->if_softc = sc; 676 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 677 ifp->if_start = vioif_start; 678 ifp->if_ioctl = vioif_ioctl; 679 ifp->if_init = vioif_init; 680 ifp->if_stop = vioif_stop; 681 ifp->if_capabilities = 0; 682 ifp->if_watchdog = vioif_watchdog; 683 684 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 685 686 if_attach(ifp); 687 if_deferred_start_init(ifp, NULL); 688 ether_ifattach(ifp, sc->sc_mac); 689 690 return; 691 692 err: 693 if (sc->sc_tx_lock) 694 mutex_obj_free(sc->sc_tx_lock); 695 if (sc->sc_rx_lock) 696 mutex_obj_free(sc->sc_rx_lock); 697 698 if (sc->sc_has_ctrl) { 699 cv_destroy(&sc->sc_ctrl_wait); 700 mutex_destroy(&sc->sc_ctrl_wait_lock); 701 } 702 703 while (nvqs > 0) 704 virtio_free_vq(vsc, &sc->sc_vq[--nvqs]); 705 706 virtio_child_attach_failed(vsc); 707 return; 708 } 709 710 /* we need interrupts to make promiscuous mode off */ 711 static void 712 vioif_deferred_init(device_t self) 713 { 714 struct vioif_softc *sc = device_private(self); 715 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 716 int r; 717 718 if (ifp->if_flags & IFF_PROMISC) 719 return; 720 721 r = vioif_set_promisc(sc, false); 722 if (r != 0) 723 aprint_error_dev(self, "resetting promisc mode failed, " 724 "errror code %d\n", r); 725 } 726 727 /* 728 * Interface functions for ifnet 729 */ 730 static int 731 vioif_init(struct ifnet *ifp) 732 { 733 struct vioif_softc *sc = ifp->if_softc; 734 struct virtio_softc *vsc = sc->sc_virtio; 735 736 vioif_stop(ifp, 0); 737 738 virtio_reinit_start(vsc); 739 virtio_negotiate_features(vsc, virtio_features(vsc)); 740 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 741 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 742 if (sc->sc_has_ctrl) 743 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 744 virtio_reinit_end(vsc); 745 746 if (!sc->sc_deferred_init_done) { 747 sc->sc_deferred_init_done = 1; 748 if (sc->sc_has_ctrl) 749 vioif_deferred_init(sc->sc_dev); 750 } 751 752 /* Have to set false before vioif_populate_rx_mbufs */ 753 sc->sc_stopping = false; 754 755 vioif_populate_rx_mbufs(sc); 756 757 vioif_update_link_status(sc); 758 ifp->if_flags |= IFF_RUNNING; 759 ifp->if_flags &= ~IFF_OACTIVE; 760 vioif_rx_filter(sc); 761 762 return 0; 763 } 764 765 static void 766 vioif_stop(struct ifnet *ifp, int disable) 767 { 768 struct vioif_softc *sc = ifp->if_softc; 769 struct virtio_softc *vsc = sc->sc_virtio; 770 771 /* Take the locks to ensure that ongoing TX/RX finish */ 772 VIOIF_TX_LOCK(sc); 773 VIOIF_RX_LOCK(sc); 774 sc->sc_stopping = true; 775 VIOIF_RX_UNLOCK(sc); 776 VIOIF_TX_UNLOCK(sc); 777 778 /* disable interrupts */ 779 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 780 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 781 if (sc->sc_has_ctrl) 782 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 783 784 /* only way to stop I/O and DMA is resetting... */ 785 virtio_reset(vsc); 786 vioif_rx_deq(sc); 787 vioif_tx_drain(sc); 788 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 789 sc->sc_link_active = false; 790 791 if (disable) 792 vioif_rx_drain(sc); 793 } 794 795 static void 796 vioif_start(struct ifnet *ifp) 797 { 798 struct vioif_softc *sc = ifp->if_softc; 799 struct virtio_softc *vsc = sc->sc_virtio; 800 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 801 struct mbuf *m; 802 int queued = 0, retry = 0; 803 804 VIOIF_TX_LOCK(sc); 805 806 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING || 807 !sc->sc_link_active) 808 goto out; 809 810 if (sc->sc_stopping) 811 goto out; 812 813 for (;;) { 814 int slot, r; 815 816 IFQ_DEQUEUE(&ifp->if_snd, m); 817 818 if (m == NULL) 819 break; 820 821 retry: 822 r = virtio_enqueue_prep(vsc, vq, &slot); 823 if (r == EAGAIN) { 824 ifp->if_flags |= IFF_OACTIVE; 825 vioif_tx_vq_done_locked(vq); 826 if (retry++ == 0) 827 goto retry; 828 else 829 break; 830 } 831 if (r != 0) 832 panic("enqueue_prep for a tx buffer"); 833 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 834 sc->sc_tx_dmamaps[slot], 835 m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 836 if (r != 0) { 837 virtio_enqueue_abort(vsc, vq, slot); 838 aprint_error_dev(sc->sc_dev, 839 "tx dmamap load failed, error code %d\n", r); 840 break; 841 } 842 r = virtio_enqueue_reserve(vsc, vq, slot, 843 sc->sc_tx_dmamaps[slot]->dm_nsegs + 1); 844 if (r != 0) { 845 bus_dmamap_unload(virtio_dmat(vsc), 846 sc->sc_tx_dmamaps[slot]); 847 vioif_tx_vq_done_locked(vq); 848 if (retry++ == 0) 849 goto retry; 850 else 851 break; 852 } 853 854 sc->sc_tx_mbufs[slot] = m; 855 856 memset(&sc->sc_tx_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); 857 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 858 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 859 BUS_DMASYNC_PREWRITE); 860 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 861 0, sc->sc_txhdr_dmamaps[slot]->dm_mapsize, 862 BUS_DMASYNC_PREWRITE); 863 virtio_enqueue(vsc, vq, slot, sc->sc_txhdr_dmamaps[slot], true); 864 virtio_enqueue(vsc, vq, slot, sc->sc_tx_dmamaps[slot], true); 865 virtio_enqueue_commit(vsc, vq, slot, false); 866 queued++; 867 bpf_mtap(ifp, m); 868 } 869 870 if (m != NULL) 871 m_freem(m); 872 873 if (queued > 0) { 874 virtio_enqueue_commit(vsc, vq, -1, true); 875 ifp->if_timer = 5; 876 } 877 878 out: 879 VIOIF_TX_UNLOCK(sc); 880 } 881 882 static int 883 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 884 { 885 int s, r; 886 887 s = splnet(); 888 889 r = ether_ioctl(ifp, cmd, data); 890 if ((r == 0 && cmd == SIOCSIFFLAGS) || 891 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) { 892 if (ifp->if_flags & IFF_RUNNING) 893 r = vioif_rx_filter(ifp->if_softc); 894 else 895 r = 0; 896 } 897 898 splx(s); 899 900 return r; 901 } 902 903 void 904 vioif_watchdog(struct ifnet *ifp) 905 { 906 struct vioif_softc *sc = ifp->if_softc; 907 908 if (ifp->if_flags & IFF_RUNNING) 909 vioif_tx_vq_done(&sc->sc_vq[VQ_TX]); 910 } 911 912 913 /* 914 * Recieve implementation 915 */ 916 /* allocate and initialize a mbuf for recieve */ 917 static int 918 vioif_add_rx_mbuf(struct vioif_softc *sc, int i) 919 { 920 struct mbuf *m; 921 int r; 922 923 MGETHDR(m, M_DONTWAIT, MT_DATA); 924 if (m == NULL) 925 return ENOBUFS; 926 MCLGET(m, M_DONTWAIT); 927 if ((m->m_flags & M_EXT) == 0) { 928 m_freem(m); 929 return ENOBUFS; 930 } 931 sc->sc_rx_mbufs[i] = m; 932 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 933 r = bus_dmamap_load_mbuf(virtio_dmat(sc->sc_virtio), 934 sc->sc_rx_dmamaps[i], 935 m, BUS_DMA_READ|BUS_DMA_NOWAIT); 936 if (r) { 937 m_freem(m); 938 sc->sc_rx_mbufs[i] = 0; 939 return r; 940 } 941 942 return 0; 943 } 944 945 /* free a mbuf for recieve */ 946 static void 947 vioif_free_rx_mbuf(struct vioif_softc *sc, int i) 948 { 949 bus_dmamap_unload(virtio_dmat(sc->sc_virtio), sc->sc_rx_dmamaps[i]); 950 m_freem(sc->sc_rx_mbufs[i]); 951 sc->sc_rx_mbufs[i] = NULL; 952 } 953 954 /* add mbufs for all the empty recieve slots */ 955 static void 956 vioif_populate_rx_mbufs(struct vioif_softc *sc) 957 { 958 VIOIF_RX_LOCK(sc); 959 vioif_populate_rx_mbufs_locked(sc); 960 VIOIF_RX_UNLOCK(sc); 961 } 962 963 static void 964 vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) 965 { 966 struct virtio_softc *vsc = sc->sc_virtio; 967 int i, r, ndone = 0; 968 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 969 970 KASSERT(VIOIF_RX_LOCKED(sc)); 971 972 if (sc->sc_stopping) 973 return; 974 975 for (i = 0; i < vq->vq_num; i++) { 976 int slot; 977 r = virtio_enqueue_prep(vsc, vq, &slot); 978 if (r == EAGAIN) 979 break; 980 if (r != 0) 981 panic("enqueue_prep for rx buffers"); 982 if (sc->sc_rx_mbufs[slot] == NULL) { 983 r = vioif_add_rx_mbuf(sc, slot); 984 if (r != 0) { 985 printf("%s: rx mbuf allocation failed, " 986 "error code %d\n", 987 device_xname(sc->sc_dev), r); 988 break; 989 } 990 } 991 r = virtio_enqueue_reserve(vsc, vq, slot, 992 sc->sc_rx_dmamaps[slot]->dm_nsegs + 1); 993 if (r != 0) { 994 vioif_free_rx_mbuf(sc, slot); 995 break; 996 } 997 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 998 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); 999 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 1000 0, MCLBYTES, BUS_DMASYNC_PREREAD); 1001 virtio_enqueue(vsc, vq, slot, sc->sc_rxhdr_dmamaps[slot], false); 1002 virtio_enqueue(vsc, vq, slot, sc->sc_rx_dmamaps[slot], false); 1003 virtio_enqueue_commit(vsc, vq, slot, false); 1004 ndone++; 1005 } 1006 if (ndone > 0) 1007 virtio_enqueue_commit(vsc, vq, -1, true); 1008 } 1009 1010 /* dequeue recieved packets */ 1011 static int 1012 vioif_rx_deq(struct vioif_softc *sc) 1013 { 1014 int r; 1015 1016 KASSERT(sc->sc_stopping); 1017 1018 VIOIF_RX_LOCK(sc); 1019 r = vioif_rx_deq_locked(sc); 1020 VIOIF_RX_UNLOCK(sc); 1021 1022 return r; 1023 } 1024 1025 /* dequeue recieved packets */ 1026 static int 1027 vioif_rx_deq_locked(struct vioif_softc *sc) 1028 { 1029 struct virtio_softc *vsc = sc->sc_virtio; 1030 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1031 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1032 struct mbuf *m; 1033 int r = 0; 1034 int slot, len; 1035 1036 KASSERT(VIOIF_RX_LOCKED(sc)); 1037 1038 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1039 len -= sizeof(struct virtio_net_hdr); 1040 r = 1; 1041 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 1042 0, sizeof(struct virtio_net_hdr), 1043 BUS_DMASYNC_POSTREAD); 1044 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 1045 0, MCLBYTES, 1046 BUS_DMASYNC_POSTREAD); 1047 m = sc->sc_rx_mbufs[slot]; 1048 KASSERT(m != NULL); 1049 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot]); 1050 sc->sc_rx_mbufs[slot] = 0; 1051 virtio_dequeue_commit(vsc, vq, slot); 1052 m_set_rcvif(m, ifp); 1053 m->m_len = m->m_pkthdr.len = len; 1054 1055 VIOIF_RX_UNLOCK(sc); 1056 if_percpuq_enqueue(ifp->if_percpuq, m); 1057 VIOIF_RX_LOCK(sc); 1058 1059 if (sc->sc_stopping) 1060 break; 1061 } 1062 1063 return r; 1064 } 1065 1066 /* rx interrupt; call _dequeue above and schedule a softint */ 1067 static int 1068 vioif_rx_vq_done(struct virtqueue *vq) 1069 { 1070 struct virtio_softc *vsc = vq->vq_owner; 1071 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1072 int r = 0; 1073 1074 #ifdef VIOIF_SOFTINT_INTR 1075 KASSERT(!cpu_intr_p()); 1076 #endif 1077 1078 VIOIF_RX_LOCK(sc); 1079 1080 if (sc->sc_stopping) 1081 goto out; 1082 1083 r = vioif_rx_deq_locked(sc); 1084 if (r) 1085 #ifdef VIOIF_SOFTINT_INTR 1086 vioif_populate_rx_mbufs_locked(sc); 1087 #else 1088 softint_schedule(sc->sc_rx_softint); 1089 #endif 1090 1091 out: 1092 VIOIF_RX_UNLOCK(sc); 1093 return r; 1094 } 1095 1096 /* softint: enqueue recieve requests for new incoming packets */ 1097 static void 1098 vioif_rx_softint(void *arg) 1099 { 1100 struct vioif_softc *sc = arg; 1101 1102 vioif_populate_rx_mbufs(sc); 1103 } 1104 1105 /* free all the mbufs; called from if_stop(disable) */ 1106 static void 1107 vioif_rx_drain(struct vioif_softc *sc) 1108 { 1109 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1110 int i; 1111 1112 for (i = 0; i < vq->vq_num; i++) { 1113 if (sc->sc_rx_mbufs[i] == NULL) 1114 continue; 1115 vioif_free_rx_mbuf(sc, i); 1116 } 1117 } 1118 1119 1120 /* 1121 * Transmition implementation 1122 */ 1123 /* actual transmission is done in if_start */ 1124 /* tx interrupt; dequeue and free mbufs */ 1125 /* 1126 * tx interrupt is actually disabled; this should be called upon 1127 * tx vq full and watchdog 1128 */ 1129 static int 1130 vioif_tx_vq_done(struct virtqueue *vq) 1131 { 1132 struct virtio_softc *vsc = vq->vq_owner; 1133 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1134 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1135 int r = 0; 1136 1137 VIOIF_TX_LOCK(sc); 1138 1139 if (sc->sc_stopping) 1140 goto out; 1141 1142 r = vioif_tx_vq_done_locked(vq); 1143 1144 out: 1145 VIOIF_TX_UNLOCK(sc); 1146 if (r) 1147 if_schedule_deferred_start(ifp); 1148 return r; 1149 } 1150 1151 static int 1152 vioif_tx_vq_done_locked(struct virtqueue *vq) 1153 { 1154 struct virtio_softc *vsc = vq->vq_owner; 1155 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1156 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1157 struct mbuf *m; 1158 int r = 0; 1159 int slot, len; 1160 1161 KASSERT(VIOIF_TX_LOCKED(sc)); 1162 1163 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1164 r++; 1165 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 1166 0, sizeof(struct virtio_net_hdr), 1167 BUS_DMASYNC_POSTWRITE); 1168 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 1169 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 1170 BUS_DMASYNC_POSTWRITE); 1171 m = sc->sc_tx_mbufs[slot]; 1172 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot]); 1173 sc->sc_tx_mbufs[slot] = 0; 1174 virtio_dequeue_commit(vsc, vq, slot); 1175 ifp->if_opackets++; 1176 m_freem(m); 1177 } 1178 1179 if (r) 1180 ifp->if_flags &= ~IFF_OACTIVE; 1181 return r; 1182 } 1183 1184 /* free all the mbufs already put on vq; called from if_stop(disable) */ 1185 static void 1186 vioif_tx_drain(struct vioif_softc *sc) 1187 { 1188 struct virtio_softc *vsc = sc->sc_virtio; 1189 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 1190 int i; 1191 1192 KASSERT(sc->sc_stopping); 1193 1194 for (i = 0; i < vq->vq_num; i++) { 1195 if (sc->sc_tx_mbufs[i] == NULL) 1196 continue; 1197 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[i]); 1198 m_freem(sc->sc_tx_mbufs[i]); 1199 sc->sc_tx_mbufs[i] = NULL; 1200 } 1201 } 1202 1203 /* 1204 * Control vq 1205 */ 1206 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1207 static int 1208 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 1209 { 1210 struct virtio_softc *vsc = sc->sc_virtio; 1211 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1212 int r, slot; 1213 1214 if (!sc->sc_has_ctrl) 1215 return ENOTSUP; 1216 1217 mutex_enter(&sc->sc_ctrl_wait_lock); 1218 while (sc->sc_ctrl_inuse != FREE) 1219 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1220 sc->sc_ctrl_inuse = INUSE; 1221 mutex_exit(&sc->sc_ctrl_wait_lock); 1222 1223 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_RX; 1224 sc->sc_ctrl_cmd->command = cmd; 1225 sc->sc_ctrl_rx->onoff = onoff; 1226 1227 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1228 0, sizeof(struct virtio_net_ctrl_cmd), 1229 BUS_DMASYNC_PREWRITE); 1230 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 1231 0, sizeof(struct virtio_net_ctrl_rx), 1232 BUS_DMASYNC_PREWRITE); 1233 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1234 0, sizeof(struct virtio_net_ctrl_status), 1235 BUS_DMASYNC_PREREAD); 1236 1237 r = virtio_enqueue_prep(vsc, vq, &slot); 1238 if (r != 0) 1239 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1240 r = virtio_enqueue_reserve(vsc, vq, slot, 3); 1241 if (r != 0) 1242 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1243 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1244 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_rx_dmamap, true); 1245 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1246 virtio_enqueue_commit(vsc, vq, slot, true); 1247 1248 /* wait for done */ 1249 mutex_enter(&sc->sc_ctrl_wait_lock); 1250 while (sc->sc_ctrl_inuse != DONE) 1251 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1252 mutex_exit(&sc->sc_ctrl_wait_lock); 1253 /* already dequeueued */ 1254 1255 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1256 sizeof(struct virtio_net_ctrl_cmd), 1257 BUS_DMASYNC_POSTWRITE); 1258 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 0, 1259 sizeof(struct virtio_net_ctrl_rx), 1260 BUS_DMASYNC_POSTWRITE); 1261 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1262 sizeof(struct virtio_net_ctrl_status), 1263 BUS_DMASYNC_POSTREAD); 1264 1265 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1266 r = 0; 1267 else { 1268 printf("%s: failed setting rx mode\n", 1269 device_xname(sc->sc_dev)); 1270 r = EIO; 1271 } 1272 1273 mutex_enter(&sc->sc_ctrl_wait_lock); 1274 sc->sc_ctrl_inuse = FREE; 1275 cv_signal(&sc->sc_ctrl_wait); 1276 mutex_exit(&sc->sc_ctrl_wait_lock); 1277 1278 return r; 1279 } 1280 1281 static int 1282 vioif_set_promisc(struct vioif_softc *sc, bool onoff) 1283 { 1284 int r; 1285 1286 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 1287 1288 return r; 1289 } 1290 1291 static int 1292 vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 1293 { 1294 int r; 1295 1296 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 1297 1298 return r; 1299 } 1300 1301 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 1302 static int 1303 vioif_set_rx_filter(struct vioif_softc *sc) 1304 { 1305 /* filter already set in sc_ctrl_mac_tbl */ 1306 struct virtio_softc *vsc = sc->sc_virtio; 1307 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1308 int r, slot; 1309 1310 if (!sc->sc_has_ctrl) 1311 return ENOTSUP; 1312 1313 mutex_enter(&sc->sc_ctrl_wait_lock); 1314 while (sc->sc_ctrl_inuse != FREE) 1315 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1316 sc->sc_ctrl_inuse = INUSE; 1317 mutex_exit(&sc->sc_ctrl_wait_lock); 1318 1319 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_MAC; 1320 sc->sc_ctrl_cmd->command = VIRTIO_NET_CTRL_MAC_TABLE_SET; 1321 1322 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 1323 sc->sc_ctrl_mac_tbl_uc, 1324 (sizeof(struct virtio_net_ctrl_mac_tbl) 1325 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1326 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1327 if (r) { 1328 printf("%s: control command dmamap load failed, " 1329 "error code %d\n", device_xname(sc->sc_dev), r); 1330 goto out; 1331 } 1332 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 1333 sc->sc_ctrl_mac_tbl_mc, 1334 (sizeof(struct virtio_net_ctrl_mac_tbl) 1335 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1336 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1337 if (r) { 1338 printf("%s: control command dmamap load failed, " 1339 "error code %d\n", device_xname(sc->sc_dev), r); 1340 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1341 goto out; 1342 } 1343 1344 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1345 0, sizeof(struct virtio_net_ctrl_cmd), 1346 BUS_DMASYNC_PREWRITE); 1347 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1348 (sizeof(struct virtio_net_ctrl_mac_tbl) 1349 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1350 BUS_DMASYNC_PREWRITE); 1351 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1352 (sizeof(struct virtio_net_ctrl_mac_tbl) 1353 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1354 BUS_DMASYNC_PREWRITE); 1355 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1356 0, sizeof(struct virtio_net_ctrl_status), 1357 BUS_DMASYNC_PREREAD); 1358 1359 r = virtio_enqueue_prep(vsc, vq, &slot); 1360 if (r != 0) 1361 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1362 r = virtio_enqueue_reserve(vsc, vq, slot, 4); 1363 if (r != 0) 1364 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1365 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1366 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_uc_dmamap, true); 1367 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_mc_dmamap, true); 1368 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1369 virtio_enqueue_commit(vsc, vq, slot, true); 1370 1371 /* wait for done */ 1372 mutex_enter(&sc->sc_ctrl_wait_lock); 1373 while (sc->sc_ctrl_inuse != DONE) 1374 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1375 mutex_exit(&sc->sc_ctrl_wait_lock); 1376 /* already dequeueued */ 1377 1378 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1379 sizeof(struct virtio_net_ctrl_cmd), 1380 BUS_DMASYNC_POSTWRITE); 1381 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1382 (sizeof(struct virtio_net_ctrl_mac_tbl) 1383 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1384 BUS_DMASYNC_POSTWRITE); 1385 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1386 (sizeof(struct virtio_net_ctrl_mac_tbl) 1387 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1388 BUS_DMASYNC_POSTWRITE); 1389 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1390 sizeof(struct virtio_net_ctrl_status), 1391 BUS_DMASYNC_POSTREAD); 1392 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1393 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap); 1394 1395 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1396 r = 0; 1397 else { 1398 printf("%s: failed setting rx filter\n", 1399 device_xname(sc->sc_dev)); 1400 r = EIO; 1401 } 1402 1403 out: 1404 mutex_enter(&sc->sc_ctrl_wait_lock); 1405 sc->sc_ctrl_inuse = FREE; 1406 cv_signal(&sc->sc_ctrl_wait); 1407 mutex_exit(&sc->sc_ctrl_wait_lock); 1408 1409 return r; 1410 } 1411 1412 /* ctrl vq interrupt; wake up the command issuer */ 1413 static int 1414 vioif_ctrl_vq_done(struct virtqueue *vq) 1415 { 1416 struct virtio_softc *vsc = vq->vq_owner; 1417 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1418 int r, slot; 1419 1420 r = virtio_dequeue(vsc, vq, &slot, NULL); 1421 if (r == ENOENT) 1422 return 0; 1423 virtio_dequeue_commit(vsc, vq, slot); 1424 1425 mutex_enter(&sc->sc_ctrl_wait_lock); 1426 sc->sc_ctrl_inuse = DONE; 1427 cv_signal(&sc->sc_ctrl_wait); 1428 mutex_exit(&sc->sc_ctrl_wait_lock); 1429 1430 return 1; 1431 } 1432 1433 /* 1434 * If IFF_PROMISC requested, set promiscuous 1435 * If multicast filter small enough (<=MAXENTRIES) set rx filter 1436 * If large multicast filter exist use ALLMULTI 1437 */ 1438 /* 1439 * If setting rx filter fails fall back to ALLMULTI 1440 * If ALLMULTI fails fall back to PROMISC 1441 */ 1442 static int 1443 vioif_rx_filter(struct vioif_softc *sc) 1444 { 1445 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1446 struct ether_multi *enm; 1447 struct ether_multistep step; 1448 int nentries; 1449 int promisc = 0, allmulti = 0, rxfilter = 0; 1450 int r; 1451 1452 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */ 1453 ifp->if_flags |= IFF_PROMISC; 1454 return 0; 1455 } 1456 1457 if (ifp->if_flags & IFF_PROMISC) { 1458 promisc = 1; 1459 goto set; 1460 } 1461 1462 nentries = -1; 1463 ETHER_LOCK(&sc->sc_ethercom); 1464 ETHER_FIRST_MULTI(step, &sc->sc_ethercom, enm); 1465 while (nentries++, enm != NULL) { 1466 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 1467 allmulti = 1; 1468 goto set_unlock; 1469 } 1470 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1471 ETHER_ADDR_LEN)) { 1472 allmulti = 1; 1473 goto set_unlock; 1474 } 1475 memcpy(sc->sc_ctrl_mac_tbl_mc->macs[nentries], 1476 enm->enm_addrlo, ETHER_ADDR_LEN); 1477 ETHER_NEXT_MULTI(step, enm); 1478 } 1479 rxfilter = 1; 1480 1481 set_unlock: 1482 ETHER_UNLOCK(&sc->sc_ethercom); 1483 1484 set: 1485 if (rxfilter) { 1486 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1487 sc->sc_ctrl_mac_tbl_mc->nentries = nentries; 1488 r = vioif_set_rx_filter(sc); 1489 if (r != 0) { 1490 rxfilter = 0; 1491 allmulti = 1; /* fallback */ 1492 } 1493 } else { 1494 /* remove rx filter */ 1495 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1496 sc->sc_ctrl_mac_tbl_mc->nentries = 0; 1497 r = vioif_set_rx_filter(sc); 1498 /* what to do on failure? */ 1499 } 1500 if (allmulti) { 1501 r = vioif_set_allmulti(sc, true); 1502 if (r != 0) { 1503 allmulti = 0; 1504 promisc = 1; /* fallback */ 1505 } 1506 } else { 1507 r = vioif_set_allmulti(sc, false); 1508 /* what to do on failure? */ 1509 } 1510 if (promisc) { 1511 r = vioif_set_promisc(sc, true); 1512 } else { 1513 r = vioif_set_promisc(sc, false); 1514 } 1515 1516 return r; 1517 } 1518 1519 static bool 1520 vioif_is_link_up(struct vioif_softc *sc) 1521 { 1522 struct virtio_softc *vsc = sc->sc_virtio; 1523 uint16_t status; 1524 1525 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 1526 status = virtio_read_device_config_2(vsc, 1527 VIRTIO_NET_CONFIG_STATUS); 1528 else 1529 status = VIRTIO_NET_S_LINK_UP; 1530 1531 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 1532 } 1533 1534 /* change link status */ 1535 static void 1536 vioif_update_link_status(struct vioif_softc *sc) 1537 { 1538 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1539 bool active, changed; 1540 int link; 1541 1542 active = vioif_is_link_up(sc); 1543 changed = false; 1544 1545 VIOIF_TX_LOCK(sc); 1546 if (active) { 1547 if (!sc->sc_link_active) 1548 changed = true; 1549 1550 link = LINK_STATE_UP; 1551 sc->sc_link_active = true; 1552 } else { 1553 if (sc->sc_link_active) 1554 changed = true; 1555 1556 link = LINK_STATE_DOWN; 1557 sc->sc_link_active = false; 1558 } 1559 VIOIF_TX_UNLOCK(sc); 1560 1561 if (changed) 1562 if_link_state_change(ifp, link); 1563 } 1564 1565 static int 1566 vioif_config_change(struct virtio_softc *vsc) 1567 { 1568 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1569 1570 #ifdef VIOIF_SOFTINT_INTR 1571 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1572 #endif 1573 1574 #ifdef VIOIF_SOFTINT_INTR 1575 KASSERT(!cpu_intr_p()); 1576 vioif_update_link_status(sc); 1577 vioif_start(ifp); 1578 #else 1579 softint_schedule(sc->sc_ctl_softint); 1580 #endif 1581 1582 return 0; 1583 } 1584 1585 static void 1586 vioif_ctl_softint(void *arg) 1587 { 1588 struct vioif_softc *sc = arg; 1589 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1590 1591 vioif_update_link_status(sc); 1592 vioif_start(ifp); 1593 } 1594 1595 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 1596 1597 #ifdef _MODULE 1598 #include "ioconf.c" 1599 #endif 1600 1601 static int 1602 if_vioif_modcmd(modcmd_t cmd, void *opaque) 1603 { 1604 int error = 0; 1605 1606 #ifdef _MODULE 1607 switch (cmd) { 1608 case MODULE_CMD_INIT: 1609 error = config_init_component(cfdriver_ioconf_if_vioif, 1610 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1611 break; 1612 case MODULE_CMD_FINI: 1613 error = config_fini_component(cfdriver_ioconf_if_vioif, 1614 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1615 break; 1616 default: 1617 error = ENOTTY; 1618 break; 1619 } 1620 #endif 1621 1622 return error; 1623 } 1624
Indexes created Tue Oct 14 11:09:46 GMT 2025