if_vioif.c revision 1.34
1 /* $NetBSD: if_vioif.c,v 1.34 2017/03/28 04:10:33 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.34 2017/03/28 04:10:33 ozaki-r Exp $"); 30 31 #ifdef _KERNEL_OPT 32 #include "opt_net_mpsafe.h" 33 #endif 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/condvar.h> 40 #include <sys/device.h> 41 #include <sys/intr.h> 42 #include <sys/kmem.h> 43 #include <sys/mbuf.h> 44 #include <sys/mutex.h> 45 #include <sys/sockio.h> 46 #include <sys/cpu.h> 47 #include <sys/module.h> 48 49 #include <dev/pci/pcidevs.h> 50 #include <dev/pci/pcireg.h> 51 #include <dev/pci/pcivar.h> 52 #include <dev/pci/virtioreg.h> 53 #include <dev/pci/virtiovar.h> 54 55 #include <net/if.h> 56 #include <net/if_media.h> 57 #include <net/if_ether.h> 58 59 #include <net/bpf.h> 60 61 #include "ioconf.h" 62 63 #ifdef NET_MPSAFE 64 #define VIOIF_MPSAFE 1 65 #endif 66 67 #ifdef SOFTINT_INTR 68 #define VIOIF_SOFTINT_INTR 1 69 #endif 70 71 /* 72 * if_vioifreg.h: 73 */ 74 /* Configuration registers */ 75 #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */ 76 #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */ 77 78 /* Feature bits */ 79 #define VIRTIO_NET_F_CSUM (1<<0) 80 #define VIRTIO_NET_F_GUEST_CSUM (1<<1) 81 #define VIRTIO_NET_F_MAC (1<<5) 82 #define VIRTIO_NET_F_GSO (1<<6) 83 #define VIRTIO_NET_F_GUEST_TSO4 (1<<7) 84 #define VIRTIO_NET_F_GUEST_TSO6 (1<<8) 85 #define VIRTIO_NET_F_GUEST_ECN (1<<9) 86 #define VIRTIO_NET_F_GUEST_UFO (1<<10) 87 #define VIRTIO_NET_F_HOST_TSO4 (1<<11) 88 #define VIRTIO_NET_F_HOST_TSO6 (1<<12) 89 #define VIRTIO_NET_F_HOST_ECN (1<<13) 90 #define VIRTIO_NET_F_HOST_UFO (1<<14) 91 #define VIRTIO_NET_F_MRG_RXBUF (1<<15) 92 #define VIRTIO_NET_F_STATUS (1<<16) 93 #define VIRTIO_NET_F_CTRL_VQ (1<<17) 94 #define VIRTIO_NET_F_CTRL_RX (1<<18) 95 #define VIRTIO_NET_F_CTRL_VLAN (1<<19) 96 97 #define VIRTIO_NET_FLAG_BITS \ 98 VIRTIO_COMMON_FLAG_BITS \ 99 "\x14""CTRL_VLAN" \ 100 "\x13""CTRL_RX" \ 101 "\x12""CTRL_VQ" \ 102 "\x11""STATUS" \ 103 "\x10""MRG_RXBUF" \ 104 "\x0f""HOST_UFO" \ 105 "\x0e""HOST_ECN" \ 106 "\x0d""HOST_TSO6" \ 107 "\x0c""HOST_TSO4" \ 108 "\x0b""GUEST_UFO" \ 109 "\x0a""GUEST_ECN" \ 110 "\x09""GUEST_TSO6" \ 111 "\x08""GUEST_TSO4" \ 112 "\x07""GSO" \ 113 "\x06""MAC" \ 114 "\x02""GUEST_CSUM" \ 115 "\x01""CSUM" 116 117 /* Status */ 118 #define VIRTIO_NET_S_LINK_UP 1 119 120 /* Packet header structure */ 121 struct virtio_net_hdr { 122 uint8_t flags; 123 uint8_t gso_type; 124 uint16_t hdr_len; 125 uint16_t gso_size; 126 uint16_t csum_start; 127 uint16_t csum_offset; 128 #if 0 129 uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */ 130 #endif 131 } __packed; 132 133 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */ 134 #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */ 135 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */ 136 #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */ 137 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */ 138 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */ 139 140 #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN) 141 142 /* Control virtqueue */ 143 struct virtio_net_ctrl_cmd { 144 uint8_t class; 145 uint8_t command; 146 } __packed; 147 #define VIRTIO_NET_CTRL_RX 0 148 # define VIRTIO_NET_CTRL_RX_PROMISC 0 149 # define VIRTIO_NET_CTRL_RX_ALLMULTI 1 150 151 #define VIRTIO_NET_CTRL_MAC 1 152 # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 153 154 #define VIRTIO_NET_CTRL_VLAN 2 155 # define VIRTIO_NET_CTRL_VLAN_ADD 0 156 # define VIRTIO_NET_CTRL_VLAN_DEL 1 157 158 struct virtio_net_ctrl_status { 159 uint8_t ack; 160 } __packed; 161 #define VIRTIO_NET_OK 0 162 #define VIRTIO_NET_ERR 1 163 164 struct virtio_net_ctrl_rx { 165 uint8_t onoff; 166 } __packed; 167 168 struct virtio_net_ctrl_mac_tbl { 169 uint32_t nentries; 170 uint8_t macs[][ETHER_ADDR_LEN]; 171 } __packed; 172 173 struct virtio_net_ctrl_vlan { 174 uint16_t id; 175 } __packed; 176 177 178 /* 179 * if_vioifvar.h: 180 */ 181 struct vioif_softc { 182 device_t sc_dev; 183 184 struct virtio_softc *sc_virtio; 185 struct virtqueue sc_vq[3]; 186 #define VQ_RX 0 187 #define VQ_TX 1 188 #define VQ_CTRL 2 189 190 uint8_t sc_mac[ETHER_ADDR_LEN]; 191 struct ethercom sc_ethercom; 192 short sc_deferred_init_done; 193 bool sc_link_active; 194 195 /* bus_dmamem */ 196 bus_dma_segment_t sc_hdr_segs[1]; 197 struct virtio_net_hdr *sc_hdrs; 198 #define sc_rx_hdrs sc_hdrs 199 struct virtio_net_hdr *sc_tx_hdrs; 200 struct virtio_net_ctrl_cmd *sc_ctrl_cmd; 201 struct virtio_net_ctrl_status *sc_ctrl_status; 202 struct virtio_net_ctrl_rx *sc_ctrl_rx; 203 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc; 204 struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc; 205 206 /* kmem */ 207 bus_dmamap_t *sc_arrays; 208 #define sc_rxhdr_dmamaps sc_arrays 209 bus_dmamap_t *sc_txhdr_dmamaps; 210 bus_dmamap_t *sc_rx_dmamaps; 211 bus_dmamap_t *sc_tx_dmamaps; 212 struct mbuf **sc_rx_mbufs; 213 struct mbuf **sc_tx_mbufs; 214 215 bus_dmamap_t sc_ctrl_cmd_dmamap; 216 bus_dmamap_t sc_ctrl_status_dmamap; 217 bus_dmamap_t sc_ctrl_rx_dmamap; 218 bus_dmamap_t sc_ctrl_tbl_uc_dmamap; 219 bus_dmamap_t sc_ctrl_tbl_mc_dmamap; 220 221 void *sc_rx_softint; 222 void *sc_ctl_softint; 223 224 enum { 225 FREE, INUSE, DONE 226 } sc_ctrl_inuse; 227 kcondvar_t sc_ctrl_wait; 228 kmutex_t sc_ctrl_wait_lock; 229 kmutex_t *sc_tx_lock; 230 kmutex_t *sc_rx_lock; 231 bool sc_stopping; 232 233 bool sc_has_ctrl; 234 }; 235 #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */ 236 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */ 237 238 #define VIOIF_TX_LOCK(_sc) if ((_sc)->sc_tx_lock) mutex_enter((_sc)->sc_tx_lock) 239 #define VIOIF_TX_UNLOCK(_sc) if ((_sc)->sc_tx_lock) mutex_exit((_sc)->sc_tx_lock) 240 #define VIOIF_TX_LOCKED(_sc) (!(_sc)->sc_tx_lock || mutex_owned((_sc)->sc_tx_lock)) 241 #define VIOIF_RX_LOCK(_sc) if ((_sc)->sc_rx_lock) mutex_enter((_sc)->sc_rx_lock) 242 #define VIOIF_RX_UNLOCK(_sc) if ((_sc)->sc_rx_lock) mutex_exit((_sc)->sc_rx_lock) 243 #define VIOIF_RX_LOCKED(_sc) (!(_sc)->sc_rx_lock || mutex_owned((_sc)->sc_rx_lock)) 244 245 /* cfattach interface functions */ 246 static int vioif_match(device_t, cfdata_t, void *); 247 static void vioif_attach(device_t, device_t, void *); 248 static void vioif_deferred_init(device_t); 249 250 /* ifnet interface functions */ 251 static int vioif_init(struct ifnet *); 252 static void vioif_stop(struct ifnet *, int); 253 static void vioif_start(struct ifnet *); 254 static int vioif_ioctl(struct ifnet *, u_long, void *); 255 static void vioif_watchdog(struct ifnet *); 256 257 /* rx */ 258 static int vioif_add_rx_mbuf(struct vioif_softc *, int); 259 static void vioif_free_rx_mbuf(struct vioif_softc *, int); 260 static void vioif_populate_rx_mbufs(struct vioif_softc *); 261 static void vioif_populate_rx_mbufs_locked(struct vioif_softc *); 262 static int vioif_rx_deq(struct vioif_softc *); 263 static int vioif_rx_deq_locked(struct vioif_softc *); 264 static int vioif_rx_vq_done(struct virtqueue *); 265 static void vioif_rx_softint(void *); 266 static void vioif_rx_drain(struct vioif_softc *); 267 268 /* tx */ 269 static int vioif_tx_vq_done(struct virtqueue *); 270 static int vioif_tx_vq_done_locked(struct virtqueue *); 271 static void vioif_tx_drain(struct vioif_softc *); 272 273 /* other control */ 274 static bool vioif_is_link_up(struct vioif_softc *); 275 static void vioif_update_link_status(struct vioif_softc *); 276 static int vioif_ctrl_rx(struct vioif_softc *, int, bool); 277 static int vioif_set_promisc(struct vioif_softc *, bool); 278 static int vioif_set_allmulti(struct vioif_softc *, bool); 279 static int vioif_set_rx_filter(struct vioif_softc *); 280 static int vioif_rx_filter(struct vioif_softc *); 281 static int vioif_ctrl_vq_done(struct virtqueue *); 282 static int vioif_config_change(struct virtio_softc *); 283 static void vioif_ctl_softint(void *); 284 285 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), 286 vioif_match, vioif_attach, NULL, NULL); 287 288 static int 289 vioif_match(device_t parent, cfdata_t match, void *aux) 290 { 291 struct virtio_attach_args *va = aux; 292 293 if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK) 294 return 1; 295 296 return 0; 297 } 298 299 /* allocate memory */ 300 /* 301 * dma memory is used for: 302 * sc_rx_hdrs[slot]: metadata array for recieved frames (READ) 303 * sc_tx_hdrs[slot]: metadata array for frames to be sent (WRITE) 304 * sc_ctrl_cmd: command to be sent via ctrl vq (WRITE) 305 * sc_ctrl_status: return value for a command via ctrl vq (READ) 306 * sc_ctrl_rx: parameter for a VIRTIO_NET_CTRL_RX class command 307 * (WRITE) 308 * sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC 309 * class command (WRITE) 310 * sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC 311 * class command (WRITE) 312 * sc_ctrl_* structures are allocated only one each; they are protected by 313 * sc_ctrl_inuse variable and sc_ctrl_wait condvar. 314 */ 315 /* 316 * dynamically allocated memory is used for: 317 * sc_rxhdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot] 318 * sc_txhdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot] 319 * sc_rx_dmamaps[slot]: bus_dmamap_t array for recieved payload 320 * sc_tx_dmamaps[slot]: bus_dmamap_t array for sent payload 321 * sc_rx_mbufs[slot]: mbuf pointer array for recieved frames 322 * sc_tx_mbufs[slot]: mbuf pointer array for sent frames 323 */ 324 static int 325 vioif_alloc_mems(struct vioif_softc *sc) 326 { 327 struct virtio_softc *vsc = sc->sc_virtio; 328 int allocsize, allocsize2, r, rsegs, i; 329 void *vaddr; 330 intptr_t p; 331 int rxqsize, txqsize; 332 333 rxqsize = sc->sc_vq[VQ_RX].vq_num; 334 txqsize = sc->sc_vq[VQ_TX].vq_num; 335 336 allocsize = sizeof(struct virtio_net_hdr) * rxqsize; 337 allocsize += sizeof(struct virtio_net_hdr) * txqsize; 338 if (sc->sc_has_ctrl) { 339 allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1; 340 allocsize += sizeof(struct virtio_net_ctrl_status) * 1; 341 allocsize += sizeof(struct virtio_net_ctrl_rx) * 1; 342 allocsize += sizeof(struct virtio_net_ctrl_mac_tbl) 343 + sizeof(struct virtio_net_ctrl_mac_tbl) 344 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; 345 } 346 r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0, 347 &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 348 if (r != 0) { 349 aprint_error_dev(sc->sc_dev, 350 "DMA memory allocation failed, size %d, " 351 "error code %d\n", allocsize, r); 352 goto err_none; 353 } 354 r = bus_dmamem_map(virtio_dmat(vsc), 355 &sc->sc_hdr_segs[0], 1, allocsize, 356 &vaddr, BUS_DMA_NOWAIT); 357 if (r != 0) { 358 aprint_error_dev(sc->sc_dev, 359 "DMA memory map failed, " 360 "error code %d\n", r); 361 goto err_dmamem_alloc; 362 } 363 sc->sc_hdrs = vaddr; 364 memset(vaddr, 0, allocsize); 365 p = (intptr_t) vaddr; 366 p += sizeof(struct virtio_net_hdr) * rxqsize; 367 #define P(name,size) do { sc->sc_ ##name = (void*) p; \ 368 p += size; } while (0) 369 P(tx_hdrs, sizeof(struct virtio_net_hdr) * txqsize); 370 if (sc->sc_has_ctrl) { 371 P(ctrl_cmd, sizeof(struct virtio_net_ctrl_cmd)); 372 P(ctrl_status, sizeof(struct virtio_net_ctrl_status)); 373 P(ctrl_rx, sizeof(struct virtio_net_ctrl_rx)); 374 P(ctrl_mac_tbl_uc, sizeof(struct virtio_net_ctrl_mac_tbl)); 375 P(ctrl_mac_tbl_mc, 376 (sizeof(struct virtio_net_ctrl_mac_tbl) 377 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES)); 378 } 379 #undef P 380 381 allocsize2 = sizeof(bus_dmamap_t) * (rxqsize + txqsize); 382 allocsize2 += sizeof(bus_dmamap_t) * (rxqsize + txqsize); 383 allocsize2 += sizeof(struct mbuf*) * (rxqsize + txqsize); 384 sc->sc_arrays = kmem_zalloc(allocsize2, KM_SLEEP); 385 if (sc->sc_arrays == NULL) 386 goto err_dmamem_map; 387 sc->sc_txhdr_dmamaps = sc->sc_arrays + rxqsize; 388 sc->sc_rx_dmamaps = sc->sc_txhdr_dmamaps + txqsize; 389 sc->sc_tx_dmamaps = sc->sc_rx_dmamaps + rxqsize; 390 sc->sc_rx_mbufs = (void*) (sc->sc_tx_dmamaps + txqsize); 391 sc->sc_tx_mbufs = sc->sc_rx_mbufs + rxqsize; 392 393 #define C(map, buf, size, nsegs, rw, usage) \ 394 do { \ 395 r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \ 396 BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, \ 397 &sc->sc_ ##map); \ 398 if (r != 0) { \ 399 aprint_error_dev(sc->sc_dev, \ 400 usage " dmamap creation failed, " \ 401 "error code %d\n", r); \ 402 goto err_reqs; \ 403 } \ 404 } while (0) 405 #define C_L1(map, buf, size, nsegs, rw, usage) \ 406 C(map, buf, size, nsegs, rw, usage); \ 407 do { \ 408 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ 409 &sc->sc_ ##buf, size, NULL, \ 410 BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ 411 if (r != 0) { \ 412 aprint_error_dev(sc->sc_dev, \ 413 usage " dmamap load failed, " \ 414 "error code %d\n", r); \ 415 goto err_reqs; \ 416 } \ 417 } while (0) 418 #define C_L2(map, buf, size, nsegs, rw, usage) \ 419 C(map, buf, size, nsegs, rw, usage); \ 420 do { \ 421 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ ##map, \ 422 sc->sc_ ##buf, size, NULL, \ 423 BUS_DMA_ ##rw | BUS_DMA_NOWAIT); \ 424 if (r != 0) { \ 425 aprint_error_dev(sc->sc_dev, \ 426 usage " dmamap load failed, " \ 427 "error code %d\n", r); \ 428 goto err_reqs; \ 429 } \ 430 } while (0) 431 for (i = 0; i < rxqsize; i++) { 432 C_L1(rxhdr_dmamaps[i], rx_hdrs[i], 433 sizeof(struct virtio_net_hdr), 1, 434 READ, "rx header"); 435 C(rx_dmamaps[i], NULL, MCLBYTES, 1, 0, "rx payload"); 436 } 437 438 for (i = 0; i < txqsize; i++) { 439 C_L1(txhdr_dmamaps[i], tx_hdrs[i], 440 sizeof(struct virtio_net_hdr), 1, 441 WRITE, "tx header"); 442 C(tx_dmamaps[i], NULL, ETHER_MAX_LEN, 16 /* XXX */, 0, 443 "tx payload"); 444 } 445 446 if (sc->sc_has_ctrl) { 447 /* control vq class & command */ 448 C_L2(ctrl_cmd_dmamap, ctrl_cmd, 449 sizeof(struct virtio_net_ctrl_cmd), 1, WRITE, 450 "control command"); 451 452 /* control vq status */ 453 C_L2(ctrl_status_dmamap, ctrl_status, 454 sizeof(struct virtio_net_ctrl_status), 1, READ, 455 "control status"); 456 457 /* control vq rx mode command parameter */ 458 C_L2(ctrl_rx_dmamap, ctrl_rx, 459 sizeof(struct virtio_net_ctrl_rx), 1, WRITE, 460 "rx mode control command"); 461 462 /* control vq MAC filter table for unicast */ 463 /* do not load now since its length is variable */ 464 C(ctrl_tbl_uc_dmamap, NULL, 465 sizeof(struct virtio_net_ctrl_mac_tbl) + 0, 1, WRITE, 466 "unicast MAC address filter command"); 467 468 /* control vq MAC filter table for multicast */ 469 C(ctrl_tbl_mc_dmamap, NULL, 470 (sizeof(struct virtio_net_ctrl_mac_tbl) 471 + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES), 472 1, WRITE, "multicast MAC address filter command"); 473 } 474 #undef C_L2 475 #undef C_L1 476 #undef C 477 478 return 0; 479 480 err_reqs: 481 #define D(map) \ 482 do { \ 483 if (sc->sc_ ##map) { \ 484 bus_dmamap_destroy(virtio_dmat(vsc), sc->sc_ ##map); \ 485 sc->sc_ ##map = NULL; \ 486 } \ 487 } while (0) 488 D(ctrl_tbl_mc_dmamap); 489 D(ctrl_tbl_uc_dmamap); 490 D(ctrl_rx_dmamap); 491 D(ctrl_status_dmamap); 492 D(ctrl_cmd_dmamap); 493 for (i = 0; i < txqsize; i++) { 494 D(tx_dmamaps[i]); 495 D(txhdr_dmamaps[i]); 496 } 497 for (i = 0; i < rxqsize; i++) { 498 D(rx_dmamaps[i]); 499 D(rxhdr_dmamaps[i]); 500 } 501 #undef D 502 if (sc->sc_arrays) { 503 kmem_free(sc->sc_arrays, allocsize2); 504 sc->sc_arrays = 0; 505 } 506 err_dmamem_map: 507 bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_hdrs, allocsize); 508 err_dmamem_alloc: 509 bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1); 510 err_none: 511 return -1; 512 } 513 514 static void 515 vioif_attach(device_t parent, device_t self, void *aux) 516 { 517 struct vioif_softc *sc = device_private(self); 518 struct virtio_softc *vsc = device_private(parent); 519 uint32_t features; 520 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 521 u_int flags; 522 int r, nvqs=0, req_flags; 523 524 if (virtio_child(vsc) != NULL) { 525 aprint_normal(": child already attached for %s; " 526 "something wrong...\n", 527 device_xname(parent)); 528 return; 529 } 530 531 sc->sc_dev = self; 532 sc->sc_virtio = vsc; 533 sc->sc_link_active = false; 534 535 req_flags = 0; 536 537 #ifdef VIOIF_MPSAFE 538 req_flags |= VIRTIO_F_PCI_INTR_MPSAFE; 539 #endif 540 #ifdef VIOIF_SOFTINT_INTR 541 req_flags |= VIRTIO_F_PCI_INTR_SOFTINT; 542 #endif 543 req_flags |= VIRTIO_F_PCI_INTR_MSIX; 544 545 virtio_child_attach_start(vsc, self, IPL_NET, sc->sc_vq, 546 vioif_config_change, virtio_vq_intr, req_flags, 547 (VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | 548 VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY), 549 VIRTIO_NET_FLAG_BITS); 550 551 features = virtio_features(vsc); 552 553 if (features & VIRTIO_NET_F_MAC) { 554 sc->sc_mac[0] = virtio_read_device_config_1(vsc, 555 VIRTIO_NET_CONFIG_MAC+0); 556 sc->sc_mac[1] = virtio_read_device_config_1(vsc, 557 VIRTIO_NET_CONFIG_MAC+1); 558 sc->sc_mac[2] = virtio_read_device_config_1(vsc, 559 VIRTIO_NET_CONFIG_MAC+2); 560 sc->sc_mac[3] = virtio_read_device_config_1(vsc, 561 VIRTIO_NET_CONFIG_MAC+3); 562 sc->sc_mac[4] = virtio_read_device_config_1(vsc, 563 VIRTIO_NET_CONFIG_MAC+4); 564 sc->sc_mac[5] = virtio_read_device_config_1(vsc, 565 VIRTIO_NET_CONFIG_MAC+5); 566 } else { 567 /* code stolen from sys/net/if_tap.c */ 568 struct timeval tv; 569 uint32_t ui; 570 getmicrouptime(&tv); 571 ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; 572 memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); 573 virtio_write_device_config_1(vsc, 574 VIRTIO_NET_CONFIG_MAC+0, 575 sc->sc_mac[0]); 576 virtio_write_device_config_1(vsc, 577 VIRTIO_NET_CONFIG_MAC+1, 578 sc->sc_mac[1]); 579 virtio_write_device_config_1(vsc, 580 VIRTIO_NET_CONFIG_MAC+2, 581 sc->sc_mac[2]); 582 virtio_write_device_config_1(vsc, 583 VIRTIO_NET_CONFIG_MAC+3, 584 sc->sc_mac[3]); 585 virtio_write_device_config_1(vsc, 586 VIRTIO_NET_CONFIG_MAC+4, 587 sc->sc_mac[4]); 588 virtio_write_device_config_1(vsc, 589 VIRTIO_NET_CONFIG_MAC+5, 590 sc->sc_mac[5]); 591 } 592 593 aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(sc->sc_mac)); 594 595 #ifdef VIOIF_MPSAFE 596 sc->sc_tx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 597 sc->sc_rx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); 598 #else 599 sc->sc_tx_lock = NULL; 600 sc->sc_rx_lock = NULL; 601 #endif 602 sc->sc_stopping = false; 603 604 /* 605 * Allocating a virtqueue for Rx 606 */ 607 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_RX], VQ_RX, 608 MCLBYTES+sizeof(struct virtio_net_hdr), 2, "rx"); 609 if (r != 0) 610 goto err; 611 nvqs = 1; 612 sc->sc_vq[VQ_RX].vq_done = vioif_rx_vq_done; 613 614 /* 615 * Allocating a virtqueue for Tx 616 */ 617 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_TX], VQ_TX, 618 (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)), 619 VIRTIO_NET_TX_MAXNSEGS + 1, "tx"); 620 if (r != 0) 621 goto err; 622 nvqs = 2; 623 sc->sc_vq[VQ_TX].vq_done = vioif_tx_vq_done; 624 625 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 626 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); /* not urgent; do it later */ 627 628 if ((features & VIRTIO_NET_F_CTRL_VQ) && 629 (features & VIRTIO_NET_F_CTRL_RX)) { 630 /* 631 * Allocating a virtqueue for control channel 632 */ 633 r = virtio_alloc_vq(vsc, &sc->sc_vq[VQ_CTRL], VQ_CTRL, 634 NBPG, 1, "control"); 635 if (r != 0) { 636 aprint_error_dev(self, "failed to allocate " 637 "a virtqueue for control channel\n"); 638 goto skip; 639 } 640 641 sc->sc_vq[VQ_CTRL].vq_done = vioif_ctrl_vq_done; 642 cv_init(&sc->sc_ctrl_wait, "ctrl_vq"); 643 mutex_init(&sc->sc_ctrl_wait_lock, MUTEX_DEFAULT, IPL_NET); 644 sc->sc_ctrl_inuse = FREE; 645 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 646 sc->sc_has_ctrl = true; 647 nvqs = 3; 648 } 649 skip: 650 651 #ifdef VIOIF_MPSAFE 652 flags = SOFTINT_NET | SOFTINT_MPSAFE; 653 #else 654 flags = SOFTINT_NET; 655 #endif 656 sc->sc_rx_softint = softint_establish(flags, vioif_rx_softint, sc); 657 if (sc->sc_rx_softint == NULL) { 658 aprint_error_dev(self, "cannot establish rx softint\n"); 659 goto err; 660 } 661 662 sc->sc_ctl_softint = softint_establish(flags, vioif_ctl_softint, sc); 663 if (sc->sc_ctl_softint == NULL) { 664 aprint_error_dev(self, "cannot establish ctl softint\n"); 665 goto err; 666 } 667 668 if (vioif_alloc_mems(sc) < 0) 669 goto err; 670 671 if (virtio_child_attach_finish(vsc) != 0) 672 goto err; 673 674 strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); 675 ifp->if_softc = sc; 676 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 677 ifp->if_start = vioif_start; 678 ifp->if_ioctl = vioif_ioctl; 679 ifp->if_init = vioif_init; 680 ifp->if_stop = vioif_stop; 681 ifp->if_capabilities = 0; 682 ifp->if_watchdog = vioif_watchdog; 683 684 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 685 686 if_attach(ifp); 687 if_deferred_start_init(ifp, NULL); 688 ether_ifattach(ifp, sc->sc_mac); 689 690 return; 691 692 err: 693 if (sc->sc_tx_lock) 694 mutex_obj_free(sc->sc_tx_lock); 695 if (sc->sc_rx_lock) 696 mutex_obj_free(sc->sc_rx_lock); 697 698 if (sc->sc_has_ctrl) { 699 cv_destroy(&sc->sc_ctrl_wait); 700 mutex_destroy(&sc->sc_ctrl_wait_lock); 701 } 702 703 while (nvqs > 0) 704 virtio_free_vq(vsc, &sc->sc_vq[--nvqs]); 705 706 virtio_child_attach_failed(vsc); 707 return; 708 } 709 710 /* we need interrupts to make promiscuous mode off */ 711 static void 712 vioif_deferred_init(device_t self) 713 { 714 struct vioif_softc *sc = device_private(self); 715 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 716 int r; 717 718 if (ifp->if_flags & IFF_PROMISC) 719 return; 720 721 r = vioif_set_promisc(sc, false); 722 if (r != 0) 723 aprint_error_dev(self, "resetting promisc mode failed, " 724 "errror code %d\n", r); 725 } 726 727 /* 728 * Interface functions for ifnet 729 */ 730 static int 731 vioif_init(struct ifnet *ifp) 732 { 733 struct vioif_softc *sc = ifp->if_softc; 734 struct virtio_softc *vsc = sc->sc_virtio; 735 736 vioif_stop(ifp, 0); 737 738 virtio_reinit_start(vsc); 739 virtio_negotiate_features(vsc, virtio_features(vsc)); 740 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 741 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 742 if (sc->sc_has_ctrl) 743 virtio_start_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 744 virtio_reinit_end(vsc); 745 746 if (!sc->sc_deferred_init_done) { 747 sc->sc_deferred_init_done = 1; 748 if (sc->sc_has_ctrl) 749 vioif_deferred_init(sc->sc_dev); 750 } 751 752 /* Have to set false before vioif_populate_rx_mbufs */ 753 sc->sc_stopping = false; 754 755 vioif_populate_rx_mbufs(sc); 756 757 vioif_update_link_status(sc); 758 ifp->if_flags |= IFF_RUNNING; 759 ifp->if_flags &= ~IFF_OACTIVE; 760 vioif_rx_filter(sc); 761 762 return 0; 763 } 764 765 static void 766 vioif_stop(struct ifnet *ifp, int disable) 767 { 768 struct vioif_softc *sc = ifp->if_softc; 769 struct virtio_softc *vsc = sc->sc_virtio; 770 771 /* Take the locks to ensure that ongoing TX/RX finish */ 772 VIOIF_TX_LOCK(sc); 773 VIOIF_RX_LOCK(sc); 774 sc->sc_stopping = true; 775 VIOIF_RX_UNLOCK(sc); 776 VIOIF_TX_UNLOCK(sc); 777 778 /* disable interrupts */ 779 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_RX]); 780 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_TX]); 781 if (sc->sc_has_ctrl) 782 virtio_stop_vq_intr(vsc, &sc->sc_vq[VQ_CTRL]); 783 784 /* only way to stop I/O and DMA is resetting... */ 785 virtio_reset(vsc); 786 vioif_rx_deq(sc); 787 vioif_tx_drain(sc); 788 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 789 sc->sc_link_active = false; 790 791 if (disable) 792 vioif_rx_drain(sc); 793 } 794 795 static void 796 vioif_start(struct ifnet *ifp) 797 { 798 struct vioif_softc *sc = ifp->if_softc; 799 struct virtio_softc *vsc = sc->sc_virtio; 800 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 801 struct mbuf *m; 802 int queued = 0, retry = 0; 803 804 VIOIF_TX_LOCK(sc); 805 806 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING || 807 !sc->sc_link_active) 808 goto out; 809 810 if (sc->sc_stopping) 811 goto out; 812 813 for (;;) { 814 int slot, r; 815 816 IFQ_DEQUEUE(&ifp->if_snd, m); 817 818 if (m == NULL) 819 break; 820 821 retry: 822 r = virtio_enqueue_prep(vsc, vq, &slot); 823 if (r == EAGAIN) { 824 ifp->if_flags |= IFF_OACTIVE; 825 vioif_tx_vq_done_locked(vq); 826 if (retry++ == 0) 827 goto retry; 828 else 829 break; 830 } 831 if (r != 0) 832 panic("enqueue_prep for a tx buffer"); 833 r = bus_dmamap_load_mbuf(virtio_dmat(vsc), 834 sc->sc_tx_dmamaps[slot], 835 m, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 836 if (r != 0) { 837 virtio_enqueue_abort(vsc, vq, slot); 838 aprint_error_dev(sc->sc_dev, 839 "tx dmamap load failed, error code %d\n", r); 840 break; 841 } 842 r = virtio_enqueue_reserve(vsc, vq, slot, 843 sc->sc_tx_dmamaps[slot]->dm_nsegs + 1); 844 if (r != 0) { 845 bus_dmamap_unload(virtio_dmat(vsc), 846 sc->sc_tx_dmamaps[slot]); 847 ifp->if_flags |= IFF_OACTIVE; 848 vioif_tx_vq_done_locked(vq); 849 if (retry++ == 0) 850 goto retry; 851 else 852 break; 853 } 854 855 sc->sc_tx_mbufs[slot] = m; 856 857 memset(&sc->sc_tx_hdrs[slot], 0, sizeof(struct virtio_net_hdr)); 858 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 859 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 860 BUS_DMASYNC_PREWRITE); 861 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 862 0, sc->sc_txhdr_dmamaps[slot]->dm_mapsize, 863 BUS_DMASYNC_PREWRITE); 864 virtio_enqueue(vsc, vq, slot, sc->sc_txhdr_dmamaps[slot], true); 865 virtio_enqueue(vsc, vq, slot, sc->sc_tx_dmamaps[slot], true); 866 virtio_enqueue_commit(vsc, vq, slot, false); 867 queued++; 868 bpf_mtap(ifp, m); 869 } 870 871 if (m != NULL) { 872 ifp->if_flags |= IFF_OACTIVE; 873 m_freem(m); 874 } 875 876 if (queued > 0) { 877 virtio_enqueue_commit(vsc, vq, -1, true); 878 ifp->if_timer = 5; 879 } 880 881 out: 882 VIOIF_TX_UNLOCK(sc); 883 } 884 885 static int 886 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 887 { 888 int s, r; 889 890 s = splnet(); 891 892 r = ether_ioctl(ifp, cmd, data); 893 if ((r == 0 && cmd == SIOCSIFFLAGS) || 894 (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) { 895 if (ifp->if_flags & IFF_RUNNING) 896 r = vioif_rx_filter(ifp->if_softc); 897 else 898 r = 0; 899 } 900 901 splx(s); 902 903 return r; 904 } 905 906 void 907 vioif_watchdog(struct ifnet *ifp) 908 { 909 struct vioif_softc *sc = ifp->if_softc; 910 911 if (ifp->if_flags & IFF_RUNNING) 912 vioif_tx_vq_done(&sc->sc_vq[VQ_TX]); 913 } 914 915 916 /* 917 * Recieve implementation 918 */ 919 /* allocate and initialize a mbuf for recieve */ 920 static int 921 vioif_add_rx_mbuf(struct vioif_softc *sc, int i) 922 { 923 struct mbuf *m; 924 int r; 925 926 MGETHDR(m, M_DONTWAIT, MT_DATA); 927 if (m == NULL) 928 return ENOBUFS; 929 MCLGET(m, M_DONTWAIT); 930 if ((m->m_flags & M_EXT) == 0) { 931 m_freem(m); 932 return ENOBUFS; 933 } 934 sc->sc_rx_mbufs[i] = m; 935 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 936 r = bus_dmamap_load_mbuf(virtio_dmat(sc->sc_virtio), 937 sc->sc_rx_dmamaps[i], 938 m, BUS_DMA_READ|BUS_DMA_NOWAIT); 939 if (r) { 940 m_freem(m); 941 sc->sc_rx_mbufs[i] = 0; 942 return r; 943 } 944 945 return 0; 946 } 947 948 /* free a mbuf for recieve */ 949 static void 950 vioif_free_rx_mbuf(struct vioif_softc *sc, int i) 951 { 952 bus_dmamap_unload(virtio_dmat(sc->sc_virtio), sc->sc_rx_dmamaps[i]); 953 m_freem(sc->sc_rx_mbufs[i]); 954 sc->sc_rx_mbufs[i] = NULL; 955 } 956 957 /* add mbufs for all the empty recieve slots */ 958 static void 959 vioif_populate_rx_mbufs(struct vioif_softc *sc) 960 { 961 VIOIF_RX_LOCK(sc); 962 vioif_populate_rx_mbufs_locked(sc); 963 VIOIF_RX_UNLOCK(sc); 964 } 965 966 static void 967 vioif_populate_rx_mbufs_locked(struct vioif_softc *sc) 968 { 969 struct virtio_softc *vsc = sc->sc_virtio; 970 int i, r, ndone = 0; 971 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 972 973 KASSERT(VIOIF_RX_LOCKED(sc)); 974 975 if (sc->sc_stopping) 976 return; 977 978 for (i = 0; i < vq->vq_num; i++) { 979 int slot; 980 r = virtio_enqueue_prep(vsc, vq, &slot); 981 if (r == EAGAIN) 982 break; 983 if (r != 0) 984 panic("enqueue_prep for rx buffers"); 985 if (sc->sc_rx_mbufs[slot] == NULL) { 986 r = vioif_add_rx_mbuf(sc, slot); 987 if (r != 0) { 988 printf("%s: rx mbuf allocation failed, " 989 "error code %d\n", 990 device_xname(sc->sc_dev), r); 991 break; 992 } 993 } 994 r = virtio_enqueue_reserve(vsc, vq, slot, 995 sc->sc_rx_dmamaps[slot]->dm_nsegs + 1); 996 if (r != 0) { 997 vioif_free_rx_mbuf(sc, slot); 998 break; 999 } 1000 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 1001 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD); 1002 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 1003 0, MCLBYTES, BUS_DMASYNC_PREREAD); 1004 virtio_enqueue(vsc, vq, slot, sc->sc_rxhdr_dmamaps[slot], false); 1005 virtio_enqueue(vsc, vq, slot, sc->sc_rx_dmamaps[slot], false); 1006 virtio_enqueue_commit(vsc, vq, slot, false); 1007 ndone++; 1008 } 1009 if (ndone > 0) 1010 virtio_enqueue_commit(vsc, vq, -1, true); 1011 } 1012 1013 /* dequeue recieved packets */ 1014 static int 1015 vioif_rx_deq(struct vioif_softc *sc) 1016 { 1017 int r; 1018 1019 KASSERT(sc->sc_stopping); 1020 1021 VIOIF_RX_LOCK(sc); 1022 r = vioif_rx_deq_locked(sc); 1023 VIOIF_RX_UNLOCK(sc); 1024 1025 return r; 1026 } 1027 1028 /* dequeue recieved packets */ 1029 static int 1030 vioif_rx_deq_locked(struct vioif_softc *sc) 1031 { 1032 struct virtio_softc *vsc = sc->sc_virtio; 1033 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1034 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1035 struct mbuf *m; 1036 int r = 0; 1037 int slot, len; 1038 1039 KASSERT(VIOIF_RX_LOCKED(sc)); 1040 1041 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1042 len -= sizeof(struct virtio_net_hdr); 1043 r = 1; 1044 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rxhdr_dmamaps[slot], 1045 0, sizeof(struct virtio_net_hdr), 1046 BUS_DMASYNC_POSTREAD); 1047 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot], 1048 0, MCLBYTES, 1049 BUS_DMASYNC_POSTREAD); 1050 m = sc->sc_rx_mbufs[slot]; 1051 KASSERT(m != NULL); 1052 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_rx_dmamaps[slot]); 1053 sc->sc_rx_mbufs[slot] = 0; 1054 virtio_dequeue_commit(vsc, vq, slot); 1055 m_set_rcvif(m, ifp); 1056 m->m_len = m->m_pkthdr.len = len; 1057 1058 VIOIF_RX_UNLOCK(sc); 1059 if_percpuq_enqueue(ifp->if_percpuq, m); 1060 VIOIF_RX_LOCK(sc); 1061 1062 if (sc->sc_stopping) 1063 break; 1064 } 1065 1066 return r; 1067 } 1068 1069 /* rx interrupt; call _dequeue above and schedule a softint */ 1070 static int 1071 vioif_rx_vq_done(struct virtqueue *vq) 1072 { 1073 struct virtio_softc *vsc = vq->vq_owner; 1074 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1075 int r = 0; 1076 1077 #ifdef VIOIF_SOFTINT_INTR 1078 KASSERT(!cpu_intr_p()); 1079 #endif 1080 1081 VIOIF_RX_LOCK(sc); 1082 1083 if (sc->sc_stopping) 1084 goto out; 1085 1086 r = vioif_rx_deq_locked(sc); 1087 if (r) 1088 #ifdef VIOIF_SOFTINT_INTR 1089 vioif_populate_rx_mbufs_locked(sc); 1090 #else 1091 softint_schedule(sc->sc_rx_softint); 1092 #endif 1093 1094 out: 1095 VIOIF_RX_UNLOCK(sc); 1096 return r; 1097 } 1098 1099 /* softint: enqueue recieve requests for new incoming packets */ 1100 static void 1101 vioif_rx_softint(void *arg) 1102 { 1103 struct vioif_softc *sc = arg; 1104 1105 vioif_populate_rx_mbufs(sc); 1106 } 1107 1108 /* free all the mbufs; called from if_stop(disable) */ 1109 static void 1110 vioif_rx_drain(struct vioif_softc *sc) 1111 { 1112 struct virtqueue *vq = &sc->sc_vq[VQ_RX]; 1113 int i; 1114 1115 for (i = 0; i < vq->vq_num; i++) { 1116 if (sc->sc_rx_mbufs[i] == NULL) 1117 continue; 1118 vioif_free_rx_mbuf(sc, i); 1119 } 1120 } 1121 1122 1123 /* 1124 * Transmition implementation 1125 */ 1126 /* actual transmission is done in if_start */ 1127 /* tx interrupt; dequeue and free mbufs */ 1128 /* 1129 * tx interrupt is actually disabled; this should be called upon 1130 * tx vq full and watchdog 1131 */ 1132 static int 1133 vioif_tx_vq_done(struct virtqueue *vq) 1134 { 1135 struct virtio_softc *vsc = vq->vq_owner; 1136 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1137 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1138 int r = 0; 1139 1140 VIOIF_TX_LOCK(sc); 1141 1142 if (sc->sc_stopping) 1143 goto out; 1144 1145 r = vioif_tx_vq_done_locked(vq); 1146 1147 out: 1148 VIOIF_TX_UNLOCK(sc); 1149 if (r) 1150 if_schedule_deferred_start(ifp); 1151 return r; 1152 } 1153 1154 static int 1155 vioif_tx_vq_done_locked(struct virtqueue *vq) 1156 { 1157 struct virtio_softc *vsc = vq->vq_owner; 1158 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1159 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1160 struct mbuf *m; 1161 int r = 0; 1162 int slot, len; 1163 1164 KASSERT(VIOIF_TX_LOCKED(sc)); 1165 1166 while (virtio_dequeue(vsc, vq, &slot, &len) == 0) { 1167 r++; 1168 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_txhdr_dmamaps[slot], 1169 0, sizeof(struct virtio_net_hdr), 1170 BUS_DMASYNC_POSTWRITE); 1171 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot], 1172 0, sc->sc_tx_dmamaps[slot]->dm_mapsize, 1173 BUS_DMASYNC_POSTWRITE); 1174 m = sc->sc_tx_mbufs[slot]; 1175 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[slot]); 1176 sc->sc_tx_mbufs[slot] = 0; 1177 virtio_dequeue_commit(vsc, vq, slot); 1178 ifp->if_opackets++; 1179 m_freem(m); 1180 } 1181 1182 if (r) 1183 ifp->if_flags &= ~IFF_OACTIVE; 1184 return r; 1185 } 1186 1187 /* free all the mbufs already put on vq; called from if_stop(disable) */ 1188 static void 1189 vioif_tx_drain(struct vioif_softc *sc) 1190 { 1191 struct virtio_softc *vsc = sc->sc_virtio; 1192 struct virtqueue *vq = &sc->sc_vq[VQ_TX]; 1193 int i; 1194 1195 KASSERT(sc->sc_stopping); 1196 1197 for (i = 0; i < vq->vq_num; i++) { 1198 if (sc->sc_tx_mbufs[i] == NULL) 1199 continue; 1200 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_tx_dmamaps[i]); 1201 m_freem(sc->sc_tx_mbufs[i]); 1202 sc->sc_tx_mbufs[i] = NULL; 1203 } 1204 } 1205 1206 /* 1207 * Control vq 1208 */ 1209 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */ 1210 static int 1211 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) 1212 { 1213 struct virtio_softc *vsc = sc->sc_virtio; 1214 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1215 int r, slot; 1216 1217 if (!sc->sc_has_ctrl) 1218 return ENOTSUP; 1219 1220 mutex_enter(&sc->sc_ctrl_wait_lock); 1221 while (sc->sc_ctrl_inuse != FREE) 1222 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1223 sc->sc_ctrl_inuse = INUSE; 1224 mutex_exit(&sc->sc_ctrl_wait_lock); 1225 1226 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_RX; 1227 sc->sc_ctrl_cmd->command = cmd; 1228 sc->sc_ctrl_rx->onoff = onoff; 1229 1230 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1231 0, sizeof(struct virtio_net_ctrl_cmd), 1232 BUS_DMASYNC_PREWRITE); 1233 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 1234 0, sizeof(struct virtio_net_ctrl_rx), 1235 BUS_DMASYNC_PREWRITE); 1236 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1237 0, sizeof(struct virtio_net_ctrl_status), 1238 BUS_DMASYNC_PREREAD); 1239 1240 r = virtio_enqueue_prep(vsc, vq, &slot); 1241 if (r != 0) 1242 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1243 r = virtio_enqueue_reserve(vsc, vq, slot, 3); 1244 if (r != 0) 1245 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1246 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1247 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_rx_dmamap, true); 1248 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1249 virtio_enqueue_commit(vsc, vq, slot, true); 1250 1251 /* wait for done */ 1252 mutex_enter(&sc->sc_ctrl_wait_lock); 1253 while (sc->sc_ctrl_inuse != DONE) 1254 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1255 mutex_exit(&sc->sc_ctrl_wait_lock); 1256 /* already dequeueued */ 1257 1258 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1259 sizeof(struct virtio_net_ctrl_cmd), 1260 BUS_DMASYNC_POSTWRITE); 1261 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_rx_dmamap, 0, 1262 sizeof(struct virtio_net_ctrl_rx), 1263 BUS_DMASYNC_POSTWRITE); 1264 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1265 sizeof(struct virtio_net_ctrl_status), 1266 BUS_DMASYNC_POSTREAD); 1267 1268 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1269 r = 0; 1270 else { 1271 printf("%s: failed setting rx mode\n", 1272 device_xname(sc->sc_dev)); 1273 r = EIO; 1274 } 1275 1276 mutex_enter(&sc->sc_ctrl_wait_lock); 1277 sc->sc_ctrl_inuse = FREE; 1278 cv_signal(&sc->sc_ctrl_wait); 1279 mutex_exit(&sc->sc_ctrl_wait_lock); 1280 1281 return r; 1282 } 1283 1284 static int 1285 vioif_set_promisc(struct vioif_softc *sc, bool onoff) 1286 { 1287 int r; 1288 1289 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff); 1290 1291 return r; 1292 } 1293 1294 static int 1295 vioif_set_allmulti(struct vioif_softc *sc, bool onoff) 1296 { 1297 int r; 1298 1299 r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); 1300 1301 return r; 1302 } 1303 1304 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ 1305 static int 1306 vioif_set_rx_filter(struct vioif_softc *sc) 1307 { 1308 /* filter already set in sc_ctrl_mac_tbl */ 1309 struct virtio_softc *vsc = sc->sc_virtio; 1310 struct virtqueue *vq = &sc->sc_vq[VQ_CTRL]; 1311 int r, slot; 1312 1313 if (!sc->sc_has_ctrl) 1314 return ENOTSUP; 1315 1316 mutex_enter(&sc->sc_ctrl_wait_lock); 1317 while (sc->sc_ctrl_inuse != FREE) 1318 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1319 sc->sc_ctrl_inuse = INUSE; 1320 mutex_exit(&sc->sc_ctrl_wait_lock); 1321 1322 sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_MAC; 1323 sc->sc_ctrl_cmd->command = VIRTIO_NET_CTRL_MAC_TABLE_SET; 1324 1325 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 1326 sc->sc_ctrl_mac_tbl_uc, 1327 (sizeof(struct virtio_net_ctrl_mac_tbl) 1328 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1329 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1330 if (r) { 1331 printf("%s: control command dmamap load failed, " 1332 "error code %d\n", device_xname(sc->sc_dev), r); 1333 goto out; 1334 } 1335 r = bus_dmamap_load(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 1336 sc->sc_ctrl_mac_tbl_mc, 1337 (sizeof(struct virtio_net_ctrl_mac_tbl) 1338 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1339 NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT); 1340 if (r) { 1341 printf("%s: control command dmamap load failed, " 1342 "error code %d\n", device_xname(sc->sc_dev), r); 1343 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1344 goto out; 1345 } 1346 1347 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 1348 0, sizeof(struct virtio_net_ctrl_cmd), 1349 BUS_DMASYNC_PREWRITE); 1350 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1351 (sizeof(struct virtio_net_ctrl_mac_tbl) 1352 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1353 BUS_DMASYNC_PREWRITE); 1354 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1355 (sizeof(struct virtio_net_ctrl_mac_tbl) 1356 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1357 BUS_DMASYNC_PREWRITE); 1358 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 1359 0, sizeof(struct virtio_net_ctrl_status), 1360 BUS_DMASYNC_PREREAD); 1361 1362 r = virtio_enqueue_prep(vsc, vq, &slot); 1363 if (r != 0) 1364 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1365 r = virtio_enqueue_reserve(vsc, vq, slot, 4); 1366 if (r != 0) 1367 panic("%s: control vq busy!?", device_xname(sc->sc_dev)); 1368 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true); 1369 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_uc_dmamap, true); 1370 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_mc_dmamap, true); 1371 virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false); 1372 virtio_enqueue_commit(vsc, vq, slot, true); 1373 1374 /* wait for done */ 1375 mutex_enter(&sc->sc_ctrl_wait_lock); 1376 while (sc->sc_ctrl_inuse != DONE) 1377 cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock); 1378 mutex_exit(&sc->sc_ctrl_wait_lock); 1379 /* already dequeueued */ 1380 1381 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_cmd_dmamap, 0, 1382 sizeof(struct virtio_net_ctrl_cmd), 1383 BUS_DMASYNC_POSTWRITE); 1384 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap, 0, 1385 (sizeof(struct virtio_net_ctrl_mac_tbl) 1386 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries), 1387 BUS_DMASYNC_POSTWRITE); 1388 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap, 0, 1389 (sizeof(struct virtio_net_ctrl_mac_tbl) 1390 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries), 1391 BUS_DMASYNC_POSTWRITE); 1392 bus_dmamap_sync(virtio_dmat(vsc), sc->sc_ctrl_status_dmamap, 0, 1393 sizeof(struct virtio_net_ctrl_status), 1394 BUS_DMASYNC_POSTREAD); 1395 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_uc_dmamap); 1396 bus_dmamap_unload(virtio_dmat(vsc), sc->sc_ctrl_tbl_mc_dmamap); 1397 1398 if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK) 1399 r = 0; 1400 else { 1401 printf("%s: failed setting rx filter\n", 1402 device_xname(sc->sc_dev)); 1403 r = EIO; 1404 } 1405 1406 out: 1407 mutex_enter(&sc->sc_ctrl_wait_lock); 1408 sc->sc_ctrl_inuse = FREE; 1409 cv_signal(&sc->sc_ctrl_wait); 1410 mutex_exit(&sc->sc_ctrl_wait_lock); 1411 1412 return r; 1413 } 1414 1415 /* ctrl vq interrupt; wake up the command issuer */ 1416 static int 1417 vioif_ctrl_vq_done(struct virtqueue *vq) 1418 { 1419 struct virtio_softc *vsc = vq->vq_owner; 1420 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1421 int r, slot; 1422 1423 r = virtio_dequeue(vsc, vq, &slot, NULL); 1424 if (r == ENOENT) 1425 return 0; 1426 virtio_dequeue_commit(vsc, vq, slot); 1427 1428 mutex_enter(&sc->sc_ctrl_wait_lock); 1429 sc->sc_ctrl_inuse = DONE; 1430 cv_signal(&sc->sc_ctrl_wait); 1431 mutex_exit(&sc->sc_ctrl_wait_lock); 1432 1433 return 1; 1434 } 1435 1436 /* 1437 * If IFF_PROMISC requested, set promiscuous 1438 * If multicast filter small enough (<=MAXENTRIES) set rx filter 1439 * If large multicast filter exist use ALLMULTI 1440 */ 1441 /* 1442 * If setting rx filter fails fall back to ALLMULTI 1443 * If ALLMULTI fails fall back to PROMISC 1444 */ 1445 static int 1446 vioif_rx_filter(struct vioif_softc *sc) 1447 { 1448 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1449 struct ether_multi *enm; 1450 struct ether_multistep step; 1451 int nentries; 1452 int promisc = 0, allmulti = 0, rxfilter = 0; 1453 int r; 1454 1455 if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */ 1456 ifp->if_flags |= IFF_PROMISC; 1457 return 0; 1458 } 1459 1460 if (ifp->if_flags & IFF_PROMISC) { 1461 promisc = 1; 1462 goto set; 1463 } 1464 1465 nentries = -1; 1466 ETHER_LOCK(&sc->sc_ethercom); 1467 ETHER_FIRST_MULTI(step, &sc->sc_ethercom, enm); 1468 while (nentries++, enm != NULL) { 1469 if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) { 1470 allmulti = 1; 1471 goto set_unlock; 1472 } 1473 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1474 ETHER_ADDR_LEN)) { 1475 allmulti = 1; 1476 goto set_unlock; 1477 } 1478 memcpy(sc->sc_ctrl_mac_tbl_mc->macs[nentries], 1479 enm->enm_addrlo, ETHER_ADDR_LEN); 1480 ETHER_NEXT_MULTI(step, enm); 1481 } 1482 rxfilter = 1; 1483 1484 set_unlock: 1485 ETHER_UNLOCK(&sc->sc_ethercom); 1486 1487 set: 1488 if (rxfilter) { 1489 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1490 sc->sc_ctrl_mac_tbl_mc->nentries = nentries; 1491 r = vioif_set_rx_filter(sc); 1492 if (r != 0) { 1493 rxfilter = 0; 1494 allmulti = 1; /* fallback */ 1495 } 1496 } else { 1497 /* remove rx filter */ 1498 sc->sc_ctrl_mac_tbl_uc->nentries = 0; 1499 sc->sc_ctrl_mac_tbl_mc->nentries = 0; 1500 r = vioif_set_rx_filter(sc); 1501 /* what to do on failure? */ 1502 } 1503 if (allmulti) { 1504 r = vioif_set_allmulti(sc, true); 1505 if (r != 0) { 1506 allmulti = 0; 1507 promisc = 1; /* fallback */ 1508 } 1509 } else { 1510 r = vioif_set_allmulti(sc, false); 1511 /* what to do on failure? */ 1512 } 1513 if (promisc) { 1514 r = vioif_set_promisc(sc, true); 1515 } else { 1516 r = vioif_set_promisc(sc, false); 1517 } 1518 1519 return r; 1520 } 1521 1522 static bool 1523 vioif_is_link_up(struct vioif_softc *sc) 1524 { 1525 struct virtio_softc *vsc = sc->sc_virtio; 1526 uint16_t status; 1527 1528 if (virtio_features(vsc) & VIRTIO_NET_F_STATUS) 1529 status = virtio_read_device_config_2(vsc, 1530 VIRTIO_NET_CONFIG_STATUS); 1531 else 1532 status = VIRTIO_NET_S_LINK_UP; 1533 1534 return ((status & VIRTIO_NET_S_LINK_UP) != 0); 1535 } 1536 1537 /* change link status */ 1538 static void 1539 vioif_update_link_status(struct vioif_softc *sc) 1540 { 1541 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1542 bool active, changed; 1543 int link; 1544 1545 active = vioif_is_link_up(sc); 1546 changed = false; 1547 1548 VIOIF_TX_LOCK(sc); 1549 if (active) { 1550 if (!sc->sc_link_active) 1551 changed = true; 1552 1553 link = LINK_STATE_UP; 1554 sc->sc_link_active = true; 1555 } else { 1556 if (sc->sc_link_active) 1557 changed = true; 1558 1559 link = LINK_STATE_DOWN; 1560 sc->sc_link_active = false; 1561 } 1562 VIOIF_TX_UNLOCK(sc); 1563 1564 if (changed) 1565 if_link_state_change(ifp, link); 1566 } 1567 1568 static int 1569 vioif_config_change(struct virtio_softc *vsc) 1570 { 1571 struct vioif_softc *sc = device_private(virtio_child(vsc)); 1572 1573 #ifdef VIOIF_SOFTINT_INTR 1574 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1575 #endif 1576 1577 #ifdef VIOIF_SOFTINT_INTR 1578 KASSERT(!cpu_intr_p()); 1579 vioif_update_link_status(sc); 1580 vioif_start(ifp); 1581 #else 1582 softint_schedule(sc->sc_ctl_softint); 1583 #endif 1584 1585 return 0; 1586 } 1587 1588 static void 1589 vioif_ctl_softint(void *arg) 1590 { 1591 struct vioif_softc *sc = arg; 1592 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 1593 1594 vioif_update_link_status(sc); 1595 vioif_start(ifp); 1596 } 1597 1598 MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); 1599 1600 #ifdef _MODULE 1601 #include "ioconf.c" 1602 #endif 1603 1604 static int 1605 if_vioif_modcmd(modcmd_t cmd, void *opaque) 1606 { 1607 int error = 0; 1608 1609 #ifdef _MODULE 1610 switch (cmd) { 1611 case MODULE_CMD_INIT: 1612 error = config_init_component(cfdriver_ioconf_if_vioif, 1613 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1614 break; 1615 case MODULE_CMD_FINI: 1616 error = config_fini_component(cfdriver_ioconf_if_vioif, 1617 cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif); 1618 break; 1619 default: 1620 error = ENOTTY; 1621 break; 1622 } 1623 #endif 1624 1625 return error; 1626 } 1627
Indexes created Fri Sep 26 08:10:20 GMT 2025