pvscsi.c revision 1.3
1 /*- 2 * Copyright (c) 2018 VMware, Inc. 3 * 4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0) 5 */ 6 7 /* 8 9 These files are provided under a dual BSD-2 Clause/GPLv2 license. When 10 using or redistributing this file, you may do so under either license. 11 12 BSD-2 Clause License 13 14 Copyright (c) 2018 VMware, Inc. 15 16 Redistribution and use in source and binary forms, with or without 17 modification, are permitted provided that the following conditions 18 are met: 19 20 * Redistributions of source code must retain the above copyright 21 notice, this list of conditions and the following disclaimer. 22 23 * Redistributions in binary form must reproduce the above copyright 24 notice, this list of conditions and the following disclaimer in 25 the documentation and/or other materials provided with the 26 distribution. 27 28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 29 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 30 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 31 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 32 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 33 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 34 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 35 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 36 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 37 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 38 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 39 40 GPL License Summary 41 42 Copyright (c) 2018 VMware, Inc. 43 44 This program is free software; you can redistribute it and/or modify 45 it under the terms of version 2 of the GNU General Public License as 46 published by the Free Software Foundation. 47 48 This program is distributed in the hope that it will be useful, but 49 WITHOUT ANY WARRANTY; without even the implied warranty of 50 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 51 General Public License for more details. 52 53 You should have received a copy of the GNU General Public License 54 along with this program; if not, write to the Free Software 55 Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 56 The full GNU General Public License is included in this distribution 57 in the file called LICENSE.GPL. 58 59 */ 60 61 #include <sys/cdefs.h> 62 __KERNEL_RCSID(0, "$NetBSD: pvscsi.c,v 1.3 2025/09/06 02:56:30 riastradh Exp $"); 63 64 #include <sys/param.h> 65 66 #include <sys/atomic.h> 67 #include <sys/buf.h> 68 #include <sys/bus.h> 69 #include <sys/cpu.h> 70 #include <sys/device.h> 71 #include <sys/kernel.h> 72 #include <sys/kmem.h> 73 #include <sys/paravirt_membar.h> 74 #include <sys/queue.h> 75 #include <sys/sysctl.h> 76 #include <sys/systm.h> 77 78 #include <dev/pci/pcireg.h> 79 #include <dev/pci/pcivar.h> 80 #include <dev/pci/pcidevs.h> 81 82 #include <dev/scsipi/scsi_all.h> 83 #include <dev/scsipi/scsi_message.h> 84 #include <dev/scsipi/scsiconf.h> 85 #include <dev/scsipi/scsipi_disk.h> 86 #include <dev/scsipi/scsi_disk.h> 87 88 #include "pvscsi.h" 89 90 #define PVSCSI_DEFAULT_NUM_PAGES_REQ_RING 8 91 #define PVSCSI_SENSE_LENGTH 256 92 93 #define PVSCSI_MAXPHYS MAXPHYS 94 #define PVSCSI_MAXPHYS_SEGS ((PVSCSI_MAXPHYS / PAGE_SIZE) + 1) 95 96 #define PVSCSI_CMD_PER_LUN 64 97 #define PVSCSI_MAX_LUN 8 98 #define PVSCSI_MAX_TARGET 16 99 100 //#define PVSCSI_DEBUG_LOGGING 101 102 #ifdef PVSCSI_DEBUG_LOGGING 103 #define DEBUG_PRINTF(level, dev, fmt, ...) \ 104 do { \ 105 if (pvscsi_log_level >= (level)) { \ 106 aprint_normal_dev((dev), (fmt), ##__VA_ARGS__); \ 107 } \ 108 } while(0) 109 #else 110 #define DEBUG_PRINTF(level, dev, fmt, ...) 111 #endif /* PVSCSI_DEBUG_LOGGING */ 112 113 struct pvscsi_softc; 114 struct pvscsi_hcb; 115 struct pvscsi_dma; 116 117 #define VMWARE_PVSCSI_DEVSTR "VMware Paravirtual SCSI Controller" 118 119 static inline uint32_t pvscsi_reg_read(struct pvscsi_softc *sc, 120 uint32_t offset); 121 static inline void pvscsi_reg_write(struct pvscsi_softc *sc, uint32_t offset, 122 uint32_t val); 123 static inline uint32_t pvscsi_read_intr_status(struct pvscsi_softc *sc); 124 static inline void pvscsi_write_intr_status(struct pvscsi_softc *sc, 125 uint32_t val); 126 static inline void pvscsi_intr_enable(struct pvscsi_softc *sc); 127 static inline void pvscsi_intr_disable(struct pvscsi_softc *sc); 128 static void pvscsi_kick_io(struct pvscsi_softc *sc, uint8_t cdb0); 129 static void pvscsi_write_cmd(struct pvscsi_softc *sc, uint32_t cmd, void *data, 130 uint32_t len); 131 static uint32_t pvscsi_get_max_targets(struct pvscsi_softc *sc); 132 static int pvscsi_setup_req_call(struct pvscsi_softc *sc, uint32_t enable); 133 static void pvscsi_setup_rings(struct pvscsi_softc *sc); 134 static void pvscsi_setup_msg_ring(struct pvscsi_softc *sc); 135 static int pvscsi_hw_supports_msg(struct pvscsi_softc *sc); 136 137 static void pvscsi_timeout(void *arg); 138 static void pvscsi_adapter_reset(struct pvscsi_softc *sc); 139 static void pvscsi_bus_reset(struct pvscsi_softc *sc); 140 static void pvscsi_device_reset(struct pvscsi_softc *sc, uint32_t target); 141 static void pvscsi_abort(struct pvscsi_softc *sc, uint32_t target, 142 struct pvscsi_hcb *hcb); 143 144 static void pvscsi_process_completion(struct pvscsi_softc *sc, 145 struct pvscsi_ring_cmp_desc *e); 146 static void pvscsi_process_cmp_ring(struct pvscsi_softc *sc); 147 static void pvscsi_process_msg(struct pvscsi_softc *sc, 148 struct pvscsi_ring_msg_desc *e); 149 static void pvscsi_process_msg_ring(struct pvscsi_softc *sc); 150 151 static void pvscsi_intr_locked(struct pvscsi_softc *sc); 152 static int pvscsi_intr(void *xsc); 153 154 static void pvscsi_scsipi_request(struct scsipi_channel *, 155 scsipi_adapter_req_t, void *); 156 157 static inline uint64_t pvscsi_hcb_to_context(struct pvscsi_softc *sc, 158 struct pvscsi_hcb *hcb); 159 static inline struct pvscsi_hcb *pvscsi_context_to_hcb(struct pvscsi_softc *sc, 160 uint64_t context); 161 static struct pvscsi_hcb * pvscsi_hcb_get(struct pvscsi_softc *sc); 162 static void pvscsi_hcb_put(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb); 163 164 static void pvscsi_dma_free(struct pvscsi_softc *sc, struct pvscsi_dma *dma); 165 static int pvscsi_dma_alloc(struct pvscsi_softc *sc, struct pvscsi_dma *dma, 166 bus_size_t size, bus_size_t alignment); 167 static int pvscsi_dma_alloc_ppns(struct pvscsi_softc *sc, 168 struct pvscsi_dma *dma, uint64_t *ppn_list, uint32_t num_pages); 169 static void pvscsi_dma_free_per_hcb(struct pvscsi_softc *sc, 170 uint32_t hcbs_allocated); 171 static int pvscsi_dma_alloc_per_hcb(struct pvscsi_softc *sc); 172 static void pvscsi_free_rings(struct pvscsi_softc *sc); 173 static int pvscsi_allocate_rings(struct pvscsi_softc *sc); 174 static void pvscsi_free_interrupts(struct pvscsi_softc *sc); 175 static int pvscsi_setup_interrupts(struct pvscsi_softc *sc, const struct pci_attach_args *); 176 static void pvscsi_free_all(struct pvscsi_softc *sc); 177 178 static void pvscsi_attach(device_t, device_t, void *); 179 static int pvscsi_detach(device_t, int); 180 static int pvscsi_probe(device_t, cfdata_t, void *); 181 182 #define pvscsi_get_tunable(_sc, _name, _value) (_value) 183 184 #ifdef PVSCSI_DEBUG_LOGGING 185 static int pvscsi_log_level = 1; 186 #endif 187 188 #define TUNABLE_INT(__x, __d) \ 189 err = sysctl_createv(clog, 0, &rnode, &cnode, \ 190 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, \ 191 #__x, SYSCTL_DESCR(__d), \ 192 NULL, 0, &(pvscsi_ ## __x), sizeof(pvscsi_ ## __x), \ 193 CTL_CREATE, CTL_EOL); \ 194 if (err) \ 195 goto fail; 196 197 static int pvscsi_request_ring_pages = 0; 198 static int pvscsi_use_msg = 1; 199 static int pvscsi_use_msi = 1; 200 static int pvscsi_use_msix = 1; 201 static int pvscsi_use_req_call_threshold = 0; 202 static int pvscsi_max_queue_depth = 0; 203 204 SYSCTL_SETUP(sysctl_hw_pvscsi_setup, "sysctl hw.pvscsi setup") 205 { 206 int err; 207 const struct sysctlnode *rnode; 208 const struct sysctlnode *cnode; 209 210 err = sysctl_createv(clog, 0, NULL, &rnode, 211 CTLFLAG_PERMANENT, CTLTYPE_NODE, "pvscsi", 212 SYSCTL_DESCR("pvscsi global controls"), 213 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); 214 215 if (err) 216 goto fail; 217 218 #ifdef PVSCSI_DEBUG_LOGGING 219 TUNABLE_INT(log_level, "Enable debugging output"); 220 #endif 221 222 TUNABLE_INT(request_ring_pages, "No. of pages for the request ring"); 223 TUNABLE_INT(use_msg, "Use message passing"); 224 TUNABLE_INT(use_msi, "Use MSI interrupt"); 225 TUNABLE_INT(use_msix, "Use MSXI interrupt"); 226 TUNABLE_INT(use_req_call_threshold, "Use request limit"); 227 TUNABLE_INT(max_queue_depth, "Maximum size of request queue"); 228 229 return; 230 fail: 231 aprint_error("%s: sysctl_createv failed (err = %d)\n", __func__, err); 232 } 233 234 struct pvscsi_sg_list { 235 struct pvscsi_sg_element sge[PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT]; 236 }; 237 238 #define PVSCSI_ABORT_TIMEOUT 2 239 #define PVSCSI_RESET_TIMEOUT 10 240 241 #define PVSCSI_HCB_NONE 0 242 #define PVSCSI_HCB_ABORT 1 243 #define PVSCSI_HCB_DEVICE_RESET 2 244 #define PVSCSI_HCB_BUS_RESET 3 245 246 struct pvscsi_hcb { 247 struct scsipi_xfer *xs; 248 struct pvscsi_softc *sc; 249 250 struct pvscsi_ring_req_desc *e; 251 int recovery; 252 SLIST_ENTRY(pvscsi_hcb) links; 253 254 bus_dmamap_t dma_map; 255 bus_addr_t dma_map_offset; 256 bus_size_t dma_map_size; 257 void *sense_buffer; 258 bus_addr_t sense_buffer_paddr; 259 struct pvscsi_sg_list *sg_list; 260 bus_addr_t sg_list_paddr; 261 bus_addr_t sg_list_offset; 262 }; 263 264 struct pvscsi_dma { 265 bus_dmamap_t map; 266 void *vaddr; 267 bus_addr_t paddr; 268 bus_size_t size; 269 bus_dma_segment_t seg[1]; 270 }; 271 272 struct pvscsi_softc { 273 device_t dev; 274 kmutex_t lock; 275 276 device_t sc_scsibus_dv; 277 struct scsipi_adapter sc_adapter; 278 struct scsipi_channel sc_channel; 279 280 struct pvscsi_rings_state *rings_state; 281 struct pvscsi_ring_req_desc *req_ring; 282 struct pvscsi_ring_cmp_desc *cmp_ring; 283 struct pvscsi_ring_msg_desc *msg_ring; 284 uint32_t hcb_cnt; 285 struct pvscsi_hcb *hcbs; 286 SLIST_HEAD(, pvscsi_hcb) free_list; 287 288 bus_dma_tag_t sc_dmat; 289 bus_space_tag_t sc_memt; 290 bus_space_handle_t sc_memh; 291 bus_size_t sc_mems; 292 293 bool use_msg; 294 uint32_t max_targets; 295 int mm_rid; 296 int irq_id; 297 int use_req_call_threshold; 298 299 pci_chipset_tag_t sc_pc; 300 pci_intr_handle_t * sc_pihp; 301 void *sc_ih; 302 303 uint64_t rings_state_ppn; 304 uint32_t req_ring_num_pages; 305 uint64_t req_ring_ppn[PVSCSI_MAX_NUM_PAGES_REQ_RING]; 306 uint32_t cmp_ring_num_pages; 307 uint64_t cmp_ring_ppn[PVSCSI_MAX_NUM_PAGES_CMP_RING]; 308 uint32_t msg_ring_num_pages; 309 uint64_t msg_ring_ppn[PVSCSI_MAX_NUM_PAGES_MSG_RING]; 310 311 struct pvscsi_dma rings_state_dma; 312 struct pvscsi_dma req_ring_dma; 313 struct pvscsi_dma cmp_ring_dma; 314 struct pvscsi_dma msg_ring_dma; 315 316 struct pvscsi_dma sg_list_dma; 317 struct pvscsi_dma sense_buffer_dma; 318 }; 319 320 CFATTACH_DECL3_NEW(pvscsi, sizeof(struct pvscsi_softc), 321 pvscsi_probe, pvscsi_attach, pvscsi_detach, NULL, NULL, NULL, 322 DVF_DETACH_SHUTDOWN); 323 324 static inline uint32_t 325 pvscsi_reg_read(struct pvscsi_softc *sc, uint32_t offset) 326 { 327 328 return (bus_space_read_4(sc->sc_memt, sc->sc_memh, offset)); 329 } 330 331 static inline void 332 pvscsi_reg_write(struct pvscsi_softc *sc, uint32_t offset, uint32_t val) 333 { 334 335 bus_space_write_4(sc->sc_memt, sc->sc_memh, offset, val); 336 } 337 338 static inline uint32_t 339 pvscsi_read_intr_status(struct pvscsi_softc *sc) 340 { 341 342 return (pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_INTR_STATUS)); 343 } 344 345 static inline void 346 pvscsi_write_intr_status(struct pvscsi_softc *sc, uint32_t val) 347 { 348 349 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_STATUS, val); 350 } 351 352 static inline void 353 pvscsi_intr_enable(struct pvscsi_softc *sc) 354 { 355 uint32_t mask; 356 357 mask = PVSCSI_INTR_CMPL_MASK; 358 if (sc->use_msg) { 359 mask |= PVSCSI_INTR_MSG_MASK; 360 } 361 362 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_MASK, mask); 363 } 364 365 static inline void 366 pvscsi_intr_disable(struct pvscsi_softc *sc) 367 { 368 369 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_INTR_MASK, 0); 370 } 371 372 static void 373 pvscsi_kick_io(struct pvscsi_softc *sc, uint8_t cdb0) 374 { 375 struct pvscsi_rings_state *s; 376 377 DEBUG_PRINTF(2, sc->dev, "%s: cdb0 %#x\n", __func__, cdb0); 378 if (cdb0 == SCSI_READ_6_COMMAND || cdb0 == READ_10 || 379 cdb0 == READ_12 || cdb0 == READ_16 || 380 cdb0 == SCSI_WRITE_6_COMMAND || cdb0 == WRITE_10 || 381 cdb0 == WRITE_12 || cdb0 == WRITE_16) { 382 s = sc->rings_state; 383 384 /* 385 * Ensure the command has been published before we test 386 * whether we need to kick the host. 387 */ 388 paravirt_membar_sync(); 389 390 DEBUG_PRINTF(2, sc->dev, "%s req prod %d cons %d\n", __func__, 391 s->req_prod_idx, s->req_cons_idx); 392 if (!sc->use_req_call_threshold || 393 (s->req_prod_idx - s->req_cons_idx) >= 394 s->req_call_threshold) { 395 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_KICK_RW_IO, 0); 396 DEBUG_PRINTF(2, sc->dev, "kicked\n"); 397 } else { 398 DEBUG_PRINTF(2, sc->dev, "wtf\n"); 399 } 400 } else { 401 s = sc->rings_state; 402 DEBUG_PRINTF(1, sc->dev, "%s req prod %d cons %d not checked\n", __func__, 403 s->req_prod_idx, s->req_cons_idx); 404 405 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0); 406 } 407 } 408 409 static void 410 pvscsi_write_cmd(struct pvscsi_softc *sc, uint32_t cmd, void *data, 411 uint32_t len) 412 { 413 uint32_t *data_ptr; 414 int i; 415 416 KASSERTMSG(len % sizeof(uint32_t) == 0, 417 "command size not a multiple of 4"); 418 419 data_ptr = data; 420 len /= sizeof(uint32_t); 421 422 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, cmd); 423 for (i = 0; i < len; ++i) { 424 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND_DATA, 425 data_ptr[i]); 426 } 427 } 428 429 static inline uint64_t pvscsi_hcb_to_context(struct pvscsi_softc *sc, 430 struct pvscsi_hcb *hcb) 431 { 432 433 /* Offset by 1 because context must not be 0 */ 434 return (hcb - sc->hcbs + 1); 435 } 436 437 static inline struct pvscsi_hcb* pvscsi_context_to_hcb(struct pvscsi_softc *sc, 438 uint64_t context) 439 { 440 441 return (sc->hcbs + (context - 1)); 442 } 443 444 static struct pvscsi_hcb * 445 pvscsi_hcb_get(struct pvscsi_softc *sc) 446 { 447 struct pvscsi_hcb *hcb; 448 449 KASSERT(mutex_owned(&sc->lock)); 450 451 hcb = SLIST_FIRST(&sc->free_list); 452 if (hcb) { 453 SLIST_REMOVE_HEAD(&sc->free_list, links); 454 } 455 456 return (hcb); 457 } 458 459 static void 460 pvscsi_hcb_put(struct pvscsi_softc *sc, struct pvscsi_hcb *hcb) 461 { 462 463 KASSERT(mutex_owned(&sc->lock)); 464 hcb->xs = NULL; 465 hcb->e = NULL; 466 hcb->recovery = PVSCSI_HCB_NONE; 467 SLIST_INSERT_HEAD(&sc->free_list, hcb, links); 468 } 469 470 static uint32_t 471 pvscsi_get_max_targets(struct pvscsi_softc *sc) 472 { 473 uint32_t max_targets; 474 475 pvscsi_write_cmd(sc, PVSCSI_CMD_GET_MAX_TARGETS, NULL, 0); 476 477 max_targets = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 478 479 if (max_targets == ~0) { 480 max_targets = 16; 481 } 482 483 return (max_targets); 484 } 485 486 static int pvscsi_setup_req_call(struct pvscsi_softc *sc, uint32_t enable) 487 { 488 uint32_t status; 489 struct pvscsi_cmd_desc_setup_req_call cmd; 490 491 if (!pvscsi_get_tunable(sc, "pvscsi_use_req_call_threshold", 492 pvscsi_use_req_call_threshold)) { 493 return (0); 494 } 495 496 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, 497 PVSCSI_CMD_SETUP_REQCALLTHRESHOLD); 498 status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 499 500 if (status != -1) { 501 memset(&cmd, 0, sizeof(cmd)); 502 cmd.enable = enable; 503 pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_REQCALLTHRESHOLD, 504 &cmd, sizeof(cmd)); 505 status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 506 507 return (status != 0); 508 } else { 509 return (0); 510 } 511 } 512 513 static void 514 pvscsi_dma_free(struct pvscsi_softc *sc, struct pvscsi_dma *dma) 515 { 516 517 bus_dmamap_unload(sc->sc_dmat, dma->map); 518 bus_dmamem_unmap(sc->sc_dmat, dma->vaddr, dma->size); 519 bus_dmamap_destroy(sc->sc_dmat, dma->map); 520 bus_dmamem_free(sc->sc_dmat, dma->seg, __arraycount(dma->seg)); 521 522 memset(dma, 0, sizeof(*dma)); 523 } 524 525 static int 526 pvscsi_dma_alloc(struct pvscsi_softc *sc, struct pvscsi_dma *dma, 527 bus_size_t size, bus_size_t alignment) 528 { 529 int error; 530 int nsegs; 531 532 memset(dma, 0, sizeof(*dma)); 533 534 error = bus_dmamem_alloc(sc->sc_dmat, size, alignment, 0, dma->seg, 535 __arraycount(dma->seg), &nsegs, BUS_DMA_WAITOK); 536 if (error) { 537 aprint_normal_dev(sc->dev, "error allocating dma mem, error %d\n", 538 error); 539 goto fail; 540 } 541 542 error = bus_dmamem_map(sc->sc_dmat, dma->seg, nsegs, size, 543 &dma->vaddr, BUS_DMA_WAITOK); 544 if (error != 0) { 545 device_printf(sc->dev, "Failed to map DMA memory\n"); 546 goto dmamemmap_fail; 547 } 548 549 error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 550 BUS_DMA_WAITOK, &dma->map); 551 if (error != 0) { 552 device_printf(sc->dev, "Failed to create DMA map\n"); 553 goto dmamapcreate_fail; 554 } 555 556 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->vaddr, size, 557 NULL, BUS_DMA_WAITOK); 558 if (error) { 559 aprint_normal_dev(sc->dev, "error mapping dma mam, error %d\n", 560 error); 561 goto dmamapload_fail; 562 } 563 564 dma->paddr = dma->map->dm_segs[0].ds_addr; 565 dma->size = size; 566 567 return 0; 568 569 dmamapload_fail: 570 bus_dmamap_destroy(sc->sc_dmat, dma->map); 571 dmamapcreate_fail: 572 bus_dmamem_unmap(sc->sc_dmat, dma->vaddr, dma->size); 573 dmamemmap_fail: 574 bus_dmamem_free(sc->sc_dmat, dma->seg, __arraycount(dma->seg)); 575 fail: 576 577 return (error); 578 } 579 580 static int 581 pvscsi_dma_alloc_ppns(struct pvscsi_softc *sc, struct pvscsi_dma *dma, 582 uint64_t *ppn_list, uint32_t num_pages) 583 { 584 int error; 585 uint32_t i; 586 uint64_t ppn; 587 588 error = pvscsi_dma_alloc(sc, dma, num_pages * PAGE_SIZE, PAGE_SIZE); 589 if (error) { 590 aprint_normal_dev(sc->dev, "Error allocating pages, error %d\n", 591 error); 592 return (error); 593 } 594 595 ppn = dma->paddr >> PAGE_SHIFT; 596 for (i = 0; i < num_pages; i++) { 597 ppn_list[i] = ppn + i; 598 } 599 600 return (0); 601 } 602 603 static void 604 pvscsi_dma_free_per_hcb(struct pvscsi_softc *sc, uint32_t hcbs_allocated) 605 { 606 int i; 607 struct pvscsi_hcb *hcb; 608 609 for (i = 0; i < hcbs_allocated; ++i) { 610 hcb = sc->hcbs + i; 611 bus_dmamap_destroy(sc->sc_dmat, hcb->dma_map); 612 }; 613 614 pvscsi_dma_free(sc, &sc->sense_buffer_dma); 615 pvscsi_dma_free(sc, &sc->sg_list_dma); 616 } 617 618 static int 619 pvscsi_dma_alloc_per_hcb(struct pvscsi_softc *sc) 620 { 621 int i; 622 int error; 623 struct pvscsi_hcb *hcb; 624 625 i = 0; 626 627 error = pvscsi_dma_alloc(sc, &sc->sg_list_dma, 628 sizeof(struct pvscsi_sg_list) * sc->hcb_cnt, 1); 629 if (error) { 630 aprint_normal_dev(sc->dev, 631 "Error allocation sg list DMA memory, error %d\n", error); 632 goto fail; 633 } 634 635 error = pvscsi_dma_alloc(sc, &sc->sense_buffer_dma, 636 PVSCSI_SENSE_LENGTH * sc->hcb_cnt, 1); 637 if (error) { 638 aprint_normal_dev(sc->dev, 639 "Error allocation buffer DMA memory, error %d\n", error); 640 goto fail; 641 } 642 643 for (i = 0; i < sc->hcb_cnt; ++i) { 644 hcb = sc->hcbs + i; 645 646 error = bus_dmamap_create(sc->sc_dmat, PVSCSI_MAXPHYS, 647 PVSCSI_MAXPHYS_SEGS, PVSCSI_MAXPHYS, 0, 648 BUS_DMA_WAITOK, &hcb->dma_map); 649 if (error) { 650 aprint_normal_dev(sc->dev, 651 "Error creating dma map for hcb %d, error %d\n", 652 i, error); 653 goto fail; 654 } 655 656 hcb->sc = sc; 657 hcb->dma_map_offset = PVSCSI_SENSE_LENGTH * i; 658 hcb->dma_map_size = PVSCSI_SENSE_LENGTH; 659 hcb->sense_buffer = 660 (void *)((char *)sc->sense_buffer_dma.vaddr + 661 PVSCSI_SENSE_LENGTH * i); 662 hcb->sense_buffer_paddr = sc->sense_buffer_dma.paddr + 663 PVSCSI_SENSE_LENGTH * i; 664 665 hcb->sg_list = 666 (struct pvscsi_sg_list *)((char *)sc->sg_list_dma.vaddr + 667 sizeof(struct pvscsi_sg_list) * i); 668 hcb->sg_list_paddr = 669 sc->sg_list_dma.paddr + sizeof(struct pvscsi_sg_list) * i; 670 hcb->sg_list_offset = sizeof(struct pvscsi_sg_list) * i; 671 } 672 673 SLIST_INIT(&sc->free_list); 674 for (i = (sc->hcb_cnt - 1); i >= 0; --i) { 675 hcb = sc->hcbs + i; 676 SLIST_INSERT_HEAD(&sc->free_list, hcb, links); 677 } 678 679 fail: 680 if (error) { 681 pvscsi_dma_free_per_hcb(sc, i); 682 } 683 684 return (error); 685 } 686 687 static void 688 pvscsi_free_rings(struct pvscsi_softc *sc) 689 { 690 691 pvscsi_dma_free(sc, &sc->rings_state_dma); 692 pvscsi_dma_free(sc, &sc->req_ring_dma); 693 pvscsi_dma_free(sc, &sc->cmp_ring_dma); 694 if (sc->use_msg) { 695 pvscsi_dma_free(sc, &sc->msg_ring_dma); 696 } 697 } 698 699 static int 700 pvscsi_allocate_rings(struct pvscsi_softc *sc) 701 { 702 int error; 703 704 error = pvscsi_dma_alloc_ppns(sc, &sc->rings_state_dma, 705 &sc->rings_state_ppn, 1); 706 if (error) { 707 aprint_normal_dev(sc->dev, 708 "Error allocating rings state, error = %d\n", error); 709 goto fail; 710 } 711 sc->rings_state = sc->rings_state_dma.vaddr; 712 713 error = pvscsi_dma_alloc_ppns(sc, &sc->req_ring_dma, sc->req_ring_ppn, 714 sc->req_ring_num_pages); 715 if (error) { 716 aprint_normal_dev(sc->dev, 717 "Error allocating req ring pages, error = %d\n", error); 718 goto fail; 719 } 720 sc->req_ring = sc->req_ring_dma.vaddr; 721 722 error = pvscsi_dma_alloc_ppns(sc, &sc->cmp_ring_dma, sc->cmp_ring_ppn, 723 sc->cmp_ring_num_pages); 724 if (error) { 725 aprint_normal_dev(sc->dev, 726 "Error allocating cmp ring pages, error = %d\n", error); 727 goto fail; 728 } 729 sc->cmp_ring = sc->cmp_ring_dma.vaddr; 730 731 sc->msg_ring = NULL; 732 if (sc->use_msg) { 733 error = pvscsi_dma_alloc_ppns(sc, &sc->msg_ring_dma, 734 sc->msg_ring_ppn, sc->msg_ring_num_pages); 735 if (error) { 736 aprint_normal_dev(sc->dev, 737 "Error allocating cmp ring pages, error = %d\n", 738 error); 739 goto fail; 740 } 741 sc->msg_ring = sc->msg_ring_dma.vaddr; 742 } 743 744 fail: 745 if (error) { 746 pvscsi_free_rings(sc); 747 } 748 return (error); 749 } 750 751 static void 752 pvscsi_setup_rings(struct pvscsi_softc *sc) 753 { 754 struct pvscsi_cmd_desc_setup_rings cmd; 755 uint32_t i; 756 757 memset(&cmd, 0, sizeof(cmd)); 758 759 cmd.rings_state_ppn = sc->rings_state_ppn; 760 761 cmd.req_ring_num_pages = sc->req_ring_num_pages; 762 for (i = 0; i < sc->req_ring_num_pages; ++i) { 763 cmd.req_ring_ppns[i] = sc->req_ring_ppn[i]; 764 } 765 766 cmd.cmp_ring_num_pages = sc->cmp_ring_num_pages; 767 for (i = 0; i < sc->cmp_ring_num_pages; ++i) { 768 cmd.cmp_ring_ppns[i] = sc->cmp_ring_ppn[i]; 769 } 770 771 pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof(cmd)); 772 } 773 774 static int 775 pvscsi_hw_supports_msg(struct pvscsi_softc *sc) 776 { 777 uint32_t status; 778 779 pvscsi_reg_write(sc, PVSCSI_REG_OFFSET_COMMAND, 780 PVSCSI_CMD_SETUP_MSG_RING); 781 status = pvscsi_reg_read(sc, PVSCSI_REG_OFFSET_COMMAND_STATUS); 782 783 return (status != -1); 784 } 785 786 static void 787 pvscsi_setup_msg_ring(struct pvscsi_softc *sc) 788 { 789 struct pvscsi_cmd_desc_setup_msg_ring cmd; 790 uint32_t i; 791 792 KASSERTMSG(sc->use_msg, "msg is not being used"); 793 794 memset(&cmd, 0, sizeof(cmd)); 795 796 cmd.num_pages = sc->msg_ring_num_pages; 797 for (i = 0; i < sc->msg_ring_num_pages; ++i) { 798 cmd.ring_ppns[i] = sc->msg_ring_ppn[i]; 799 } 800 801 pvscsi_write_cmd(sc, PVSCSI_CMD_SETUP_MSG_RING, &cmd, sizeof(cmd)); 802 } 803 804 static void 805 pvscsi_adapter_reset(struct pvscsi_softc *sc) 806 { 807 aprint_normal_dev(sc->dev, "Adapter Reset\n"); 808 809 pvscsi_write_cmd(sc, PVSCSI_CMD_ADAPTER_RESET, NULL, 0); 810 #ifdef PVSCSI_DEBUG_LOGGING 811 uint32_t val = 812 #endif 813 pvscsi_read_intr_status(sc); 814 815 DEBUG_PRINTF(2, sc->dev, "adapter reset done: %u\n", val); 816 } 817 818 static void 819 pvscsi_bus_reset(struct pvscsi_softc *sc) 820 { 821 822 aprint_normal_dev(sc->dev, "Bus Reset\n"); 823 824 pvscsi_write_cmd(sc, PVSCSI_CMD_RESET_BUS, NULL, 0); 825 pvscsi_process_cmp_ring(sc); 826 827 DEBUG_PRINTF(2, sc->dev, "bus reset done\n"); 828 } 829 830 static void 831 pvscsi_device_reset(struct pvscsi_softc *sc, uint32_t target) 832 { 833 struct pvscsi_cmd_desc_reset_device cmd; 834 835 memset(&cmd, 0, sizeof(cmd)); 836 837 cmd.target = target; 838 839 aprint_normal_dev(sc->dev, "Device reset for target %u\n", target); 840 841 pvscsi_write_cmd(sc, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof cmd); 842 pvscsi_process_cmp_ring(sc); 843 844 DEBUG_PRINTF(2, sc->dev, "device reset done\n"); 845 } 846 847 static void 848 pvscsi_abort(struct pvscsi_softc *sc, uint32_t target, struct pvscsi_hcb *hcb) 849 { 850 struct pvscsi_cmd_desc_abort_cmd cmd; 851 uint64_t context; 852 853 pvscsi_process_cmp_ring(sc); 854 855 if (hcb != NULL) { 856 context = pvscsi_hcb_to_context(sc, hcb); 857 858 memset(&cmd, 0, sizeof cmd); 859 cmd.target = target; 860 cmd.context = context; 861 862 aprint_normal_dev(sc->dev, "Abort for target %u context %llx\n", 863 target, (unsigned long long)context); 864 865 pvscsi_write_cmd(sc, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd)); 866 pvscsi_process_cmp_ring(sc); 867 868 DEBUG_PRINTF(2, sc->dev, "abort done\n"); 869 } else { 870 DEBUG_PRINTF(1, sc->dev, 871 "Target %u hcb %p not found for abort\n", target, hcb); 872 } 873 } 874 875 static int 876 pvscsi_probe(device_t dev, cfdata_t cf, void *aux) 877 { 878 const struct pci_attach_args *pa = aux; 879 880 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE && 881 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_PVSCSI) { 882 return 1; 883 } 884 return 0; 885 } 886 887 static void 888 pvscsi_timeout(void *arg) 889 { 890 struct pvscsi_hcb *hcb = arg; 891 struct scsipi_xfer *xs = hcb->xs; 892 893 if (xs == NULL) { 894 /* Already completed */ 895 return; 896 } 897 898 struct pvscsi_softc *sc = hcb->sc; 899 900 mutex_enter(&sc->lock); 901 902 scsipi_printaddr(xs->xs_periph); 903 printf("command timeout, CDB: "); 904 scsipi_print_cdb(xs->cmd); 905 printf("\n"); 906 907 switch (hcb->recovery) { 908 case PVSCSI_HCB_NONE: 909 hcb->recovery = PVSCSI_HCB_ABORT; 910 pvscsi_abort(sc, hcb->e->target, hcb); 911 callout_reset(&xs->xs_callout, 912 mstohz(PVSCSI_ABORT_TIMEOUT * 1000), 913 pvscsi_timeout, hcb); 914 break; 915 case PVSCSI_HCB_ABORT: 916 hcb->recovery = PVSCSI_HCB_DEVICE_RESET; 917 pvscsi_device_reset(sc, hcb->e->target); 918 callout_reset(&xs->xs_callout, 919 mstohz(PVSCSI_RESET_TIMEOUT * 1000), 920 pvscsi_timeout, hcb); 921 break; 922 case PVSCSI_HCB_DEVICE_RESET: 923 hcb->recovery = PVSCSI_HCB_BUS_RESET; 924 pvscsi_bus_reset(sc); 925 callout_reset(&xs->xs_callout, 926 mstohz(PVSCSI_RESET_TIMEOUT * 1000), 927 pvscsi_timeout, hcb); 928 break; 929 case PVSCSI_HCB_BUS_RESET: 930 pvscsi_adapter_reset(sc); 931 break; 932 }; 933 mutex_exit(&sc->lock); 934 } 935 936 static void 937 pvscsi_process_completion(struct pvscsi_softc *sc, 938 struct pvscsi_ring_cmp_desc *e) 939 { 940 struct pvscsi_hcb *hcb; 941 struct scsipi_xfer *xs; 942 uint32_t error = XS_NOERROR; 943 uint32_t btstat; 944 uint32_t sdstat; 945 int op; 946 947 hcb = pvscsi_context_to_hcb(sc, e->context); 948 xs = hcb->xs; 949 950 callout_stop(&xs->xs_callout); 951 952 btstat = e->host_status; 953 sdstat = e->scsi_status; 954 955 xs->status = sdstat; 956 xs->resid = xs->datalen - e->data_len; 957 958 DEBUG_PRINTF(3, sc->dev, 959 "command context %llx btstat %d (%#x) sdstat %d (%#x)\n", 960 (unsigned long long)e->context, btstat, btstat, sdstat, sdstat); 961 962 if ((xs->xs_control & XS_CTL_DATA_IN) == XS_CTL_DATA_IN) { 963 op = BUS_DMASYNC_POSTREAD; 964 } else { 965 op = BUS_DMASYNC_POSTWRITE; 966 } 967 bus_dmamap_sync(sc->sc_dmat, sc->sense_buffer_dma.map, 968 hcb->dma_map_offset, hcb->dma_map_size, op); 969 970 if (btstat == BTSTAT_SUCCESS && sdstat == SCSI_OK) { 971 DEBUG_PRINTF(3, sc->dev, 972 "completing command context %llx success\n", 973 (unsigned long long)e->context); 974 xs->resid = 0; 975 } else { 976 switch (btstat) { 977 case BTSTAT_SUCCESS: 978 case BTSTAT_LINKED_COMMAND_COMPLETED: 979 case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG: 980 switch (sdstat) { 981 case SCSI_OK: 982 xs->resid = 0; 983 error = XS_NOERROR; 984 break; 985 case SCSI_CHECK: 986 error = XS_SENSE; 987 xs->resid = 0; 988 989 memset(&xs->sense, 0, sizeof(xs->sense)); 990 memcpy(&xs->sense, hcb->sense_buffer, 991 MIN(sizeof(xs->sense), e->sense_len)); 992 break; 993 case SCSI_BUSY: 994 case SCSI_QUEUE_FULL: 995 error = XS_NOERROR; 996 break; 997 case SCSI_TERMINATED: 998 // case SCSI_STATUS_TASK_ABORTED: 999 DEBUG_PRINTF(1, sc->dev, 1000 "xs: %p sdstat=0x%x\n", xs, sdstat); 1001 error = XS_DRIVER_STUFFUP; 1002 break; 1003 default: 1004 DEBUG_PRINTF(1, sc->dev, 1005 "xs: %p sdstat=0x%x\n", xs, sdstat); 1006 error = XS_DRIVER_STUFFUP; 1007 break; 1008 } 1009 break; 1010 case BTSTAT_SELTIMEO: 1011 error = XS_SELTIMEOUT; 1012 break; 1013 case BTSTAT_DATARUN: 1014 case BTSTAT_DATA_UNDERRUN: 1015 // xs->resid = xs->datalen - c->data_len; 1016 error = XS_NOERROR; 1017 break; 1018 case BTSTAT_ABORTQUEUE: 1019 case BTSTAT_HATIMEOUT: 1020 error = XS_NOERROR; 1021 break; 1022 case BTSTAT_NORESPONSE: 1023 case BTSTAT_SENTRST: 1024 case BTSTAT_RECVRST: 1025 case BTSTAT_BUSRESET: 1026 error = XS_RESET; 1027 break; 1028 case BTSTAT_SCSIPARITY: 1029 error = XS_DRIVER_STUFFUP; 1030 DEBUG_PRINTF(1, sc->dev, 1031 "xs: %p sdstat=0x%x\n", xs, sdstat); 1032 break; 1033 case BTSTAT_BUSFREE: 1034 error = XS_DRIVER_STUFFUP; 1035 DEBUG_PRINTF(1, sc->dev, 1036 "xs: %p sdstat=0x%x\n", xs, sdstat); 1037 break; 1038 case BTSTAT_INVPHASE: 1039 error = XS_DRIVER_STUFFUP; 1040 DEBUG_PRINTF(1, sc->dev, 1041 "xs: %p sdstat=0x%x\n", xs, sdstat); 1042 break; 1043 case BTSTAT_SENSFAILED: 1044 error = XS_DRIVER_STUFFUP; 1045 DEBUG_PRINTF(1, sc->dev, 1046 "xs: %p sdstat=0x%x\n", xs, sdstat); 1047 break; 1048 case BTSTAT_LUNMISMATCH: 1049 case BTSTAT_TAGREJECT: 1050 case BTSTAT_DISCONNECT: 1051 case BTSTAT_BADMSG: 1052 case BTSTAT_INVPARAM: 1053 error = XS_DRIVER_STUFFUP; 1054 DEBUG_PRINTF(1, sc->dev, 1055 "xs: %p sdstat=0x%x\n", xs, sdstat); 1056 break; 1057 case BTSTAT_HASOFTWARE: 1058 case BTSTAT_HAHARDWARE: 1059 error = XS_DRIVER_STUFFUP; 1060 DEBUG_PRINTF(1, sc->dev, 1061 "xs: %p sdstat=0x%x\n", xs, sdstat); 1062 break; 1063 default: 1064 aprint_normal_dev(sc->dev, "unknown hba status: 0x%x\n", 1065 btstat); 1066 error = XS_DRIVER_STUFFUP; 1067 break; 1068 } 1069 1070 DEBUG_PRINTF(3, sc->dev, 1071 "completing command context %llx btstat %x sdstat %x - error %x\n", 1072 (unsigned long long)e->context, btstat, sdstat, error); 1073 } 1074 1075 xs->error = error; 1076 pvscsi_hcb_put(sc, hcb); 1077 1078 mutex_exit(&sc->lock); 1079 1080 scsipi_done(xs); 1081 1082 mutex_enter(&sc->lock); 1083 } 1084 1085 static void 1086 pvscsi_process_cmp_ring(struct pvscsi_softc *sc) 1087 { 1088 struct pvscsi_ring_cmp_desc *ring; 1089 struct pvscsi_rings_state *s; 1090 struct pvscsi_ring_cmp_desc *e; 1091 uint32_t mask; 1092 1093 KASSERT(mutex_owned(&sc->lock)); 1094 1095 s = sc->rings_state; 1096 ring = sc->cmp_ring; 1097 mask = MASK(s->cmp_num_entries_log2); 1098 1099 while (true) { 1100 size_t crpidx = s->cmp_prod_idx; 1101 membar_acquire(); 1102 1103 if (s->cmp_cons_idx == crpidx) 1104 break; 1105 1106 size_t crcidx = s->cmp_cons_idx & mask; 1107 1108 e = ring + crcidx; 1109 1110 pvscsi_process_completion(sc, e); 1111 1112 /* 1113 * ensure completion processing reads happen before write to 1114 * (increment of) cmp_cons_idx 1115 */ 1116 membar_release(); 1117 s->cmp_cons_idx++; 1118 } 1119 } 1120 1121 static void 1122 pvscsi_process_msg(struct pvscsi_softc *sc, struct pvscsi_ring_msg_desc *e) 1123 { 1124 struct pvscsi_ring_msg_dev_status_changed *desc; 1125 1126 switch (e->type) { 1127 case PVSCSI_MSG_DEV_ADDED: 1128 case PVSCSI_MSG_DEV_REMOVED: { 1129 desc = (struct pvscsi_ring_msg_dev_status_changed *)e; 1130 struct scsibus_softc *ssc = device_private(sc->sc_scsibus_dv); 1131 1132 aprint_normal_dev(sc->dev, "MSG: device %s at scsi%u:%u:%u\n", 1133 desc->type == PVSCSI_MSG_DEV_ADDED ? "addition" : "removal", 1134 desc->bus, desc->target, desc->lun[1]); 1135 1136 if (desc->type == PVSCSI_MSG_DEV_ADDED) { 1137 if (scsi_probe_bus(ssc, 1138 desc->target, desc->lun[1]) != 0) { 1139 aprint_normal_dev(sc->dev, 1140 "Error creating path for dev change.\n"); 1141 break; 1142 } 1143 } else { 1144 if (scsipi_target_detach(ssc->sc_channel, 1145 desc->target, desc->lun[1], 1146 DETACH_FORCE) != 0) { 1147 aprint_normal_dev(sc->dev, 1148 "Error detaching target %d lun %d\n", 1149 desc->target, desc->lun[1]); 1150 }; 1151 1152 } 1153 } break; 1154 default: 1155 aprint_normal_dev(sc->dev, "Unknown msg type 0x%x\n", e->type); 1156 }; 1157 } 1158 1159 static void 1160 pvscsi_process_msg_ring(struct pvscsi_softc *sc) 1161 { 1162 struct pvscsi_ring_msg_desc *ring; 1163 struct pvscsi_rings_state *s; 1164 struct pvscsi_ring_msg_desc *e; 1165 uint32_t mask; 1166 1167 KASSERT(mutex_owned(&sc->lock)); 1168 1169 s = sc->rings_state; 1170 ring = sc->msg_ring; 1171 mask = MASK(s->msg_num_entries_log2); 1172 1173 while (true) { 1174 size_t mpidx = s->msg_prod_idx; // dma read (device -> cpu) 1175 membar_acquire(); 1176 1177 if (s->msg_cons_idx == mpidx) 1178 break; 1179 1180 size_t mcidx = s->msg_cons_idx & mask; 1181 1182 e = ring + mcidx; 1183 1184 pvscsi_process_msg(sc, e); 1185 1186 /* 1187 * ensure message processing reads happen before write to 1188 * (increment of) msg_cons_idx 1189 */ 1190 membar_release(); 1191 s->msg_cons_idx++; 1192 } 1193 } 1194 1195 static void 1196 pvscsi_intr_locked(struct pvscsi_softc *sc) 1197 { 1198 uint32_t val; 1199 1200 KASSERT(mutex_owned(&sc->lock)); 1201 1202 val = pvscsi_read_intr_status(sc); 1203 1204 if ((val & PVSCSI_INTR_ALL_SUPPORTED) != 0) { 1205 pvscsi_write_intr_status(sc, val & PVSCSI_INTR_ALL_SUPPORTED); 1206 pvscsi_process_cmp_ring(sc); 1207 if (sc->use_msg) { 1208 pvscsi_process_msg_ring(sc); 1209 } 1210 } 1211 } 1212 1213 static int 1214 pvscsi_intr(void *xsc) 1215 { 1216 struct pvscsi_softc *sc; 1217 1218 sc = xsc; 1219 1220 mutex_enter(&sc->lock); 1221 pvscsi_intr_locked(xsc); 1222 mutex_exit(&sc->lock); 1223 1224 return 1; 1225 } 1226 1227 static void 1228 pvscsi_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t 1229 request, void *arg) 1230 { 1231 struct pvscsi_softc *sc = device_private(chan->chan_adapter->adapt_dev); 1232 1233 if (request == ADAPTER_REQ_SET_XFER_MODE) { 1234 struct scsipi_xfer_mode *xm = arg; 1235 1236 xm->xm_mode = PERIPH_CAP_TQING; 1237 xm->xm_period = 0; 1238 xm->xm_offset = 0; 1239 scsipi_async_event(chan, ASYNC_EVENT_XFER_MODE, xm); 1240 return; 1241 } else if (request != ADAPTER_REQ_RUN_XFER) { 1242 DEBUG_PRINTF(1, sc->dev, "unhandled %d\n", request); 1243 return; 1244 } 1245 1246 /* request is ADAPTER_REQ_RUN_XFER */ 1247 struct scsipi_xfer *xs = arg; 1248 struct scsipi_periph *periph = xs->xs_periph; 1249 #ifdef SCSIPI_DEBUG 1250 periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS; 1251 #endif 1252 1253 uint32_t req_num_entries_log2; 1254 struct pvscsi_ring_req_desc *ring; 1255 struct pvscsi_ring_req_desc *e; 1256 struct pvscsi_rings_state *s; 1257 struct pvscsi_hcb *hcb; 1258 1259 if (xs->cmdlen < 0 || xs->cmdlen > sizeof(e->cdb)) { 1260 DEBUG_PRINTF(1, sc->dev, "bad cmdlen %zu > %zu\n", 1261 (size_t)xs->cmdlen, sizeof(e->cdb)); 1262 /* not a temporary condition */ 1263 xs->error = XS_DRIVER_STUFFUP; 1264 scsipi_done(xs); 1265 return; 1266 } 1267 1268 ring = sc->req_ring; 1269 s = sc->rings_state; 1270 1271 hcb = NULL; 1272 req_num_entries_log2 = s->req_num_entries_log2; 1273 1274 /* Protect against multiple senders */ 1275 mutex_enter(&sc->lock); 1276 1277 if (s->req_prod_idx - s->cmp_cons_idx >= 1278 (1 << req_num_entries_log2)) { 1279 aprint_normal_dev(sc->dev, 1280 "Not enough room on completion ring.\n"); 1281 xs->error = XS_RESOURCE_SHORTAGE; 1282 goto finish_xs; 1283 } 1284 1285 if (xs->cmdlen > sizeof(e->cdb)) { 1286 DEBUG_PRINTF(1, sc->dev, "cdb length %u too large\n", 1287 xs->cmdlen); 1288 xs->error = XS_DRIVER_STUFFUP; 1289 goto finish_xs; 1290 } 1291 1292 hcb = pvscsi_hcb_get(sc); 1293 if (hcb == NULL) { 1294 aprint_normal_dev(sc->dev, "No free hcbs.\n"); 1295 xs->error = XS_RESOURCE_SHORTAGE; 1296 goto finish_xs; 1297 } 1298 1299 hcb->xs = xs; 1300 1301 const size_t rridx = s->req_prod_idx & MASK(req_num_entries_log2); 1302 e = ring + rridx; 1303 1304 memset(e, 0, sizeof(*e)); 1305 e->bus = 0; 1306 e->target = periph->periph_target; 1307 e->lun[1] = periph->periph_lun; 1308 e->data_addr = 0; 1309 e->data_len = xs->datalen; 1310 e->vcpu_hint = cpu_index(curcpu()); 1311 e->flags = 0; 1312 1313 e->cdb_len = xs->cmdlen; 1314 memcpy(e->cdb, xs->cmd, xs->cmdlen); 1315 1316 e->sense_addr = 0; 1317 e->sense_len = sizeof(xs->sense); 1318 if (e->sense_len > 0) { 1319 e->sense_addr = hcb->sense_buffer_paddr; 1320 } 1321 //e->tag = xs->xs_tag_type; 1322 e->tag = MSG_SIMPLE_Q_TAG; 1323 1324 switch (xs->xs_control & (XS_CTL_DATA_IN | XS_CTL_DATA_OUT)) { 1325 case XS_CTL_DATA_IN: 1326 e->flags |= PVSCSI_FLAG_CMD_DIR_TOHOST; 1327 break; 1328 case XS_CTL_DATA_OUT: 1329 e->flags |= PVSCSI_FLAG_CMD_DIR_TODEVICE; 1330 break; 1331 default: 1332 e->flags |= PVSCSI_FLAG_CMD_DIR_NONE; 1333 break; 1334 } 1335 1336 e->context = pvscsi_hcb_to_context(sc, hcb); 1337 hcb->e = e; 1338 1339 DEBUG_PRINTF(3, sc->dev, 1340 " queuing command %02x context %llx\n", e->cdb[0], 1341 (unsigned long long)e->context); 1342 1343 int flags; 1344 flags = (xs->xs_control & XS_CTL_DATA_IN) ? BUS_DMA_READ : BUS_DMA_WRITE; 1345 flags |= (xs->xs_control & XS_CTL_NOSLEEP) ? BUS_DMA_NOWAIT : BUS_DMA_WAITOK; 1346 1347 int error = bus_dmamap_load(sc->sc_dmat, hcb->dma_map, 1348 xs->data, xs->datalen, NULL, flags); 1349 1350 if (error) { 1351 if (error == ENOMEM || error == EAGAIN) { 1352 xs->error = XS_RESOURCE_SHORTAGE; 1353 } else { 1354 xs->error = XS_DRIVER_STUFFUP; 1355 } 1356 DEBUG_PRINTF(1, sc->dev, 1357 "xs: %p load error %d data %p len %d", 1358 xs, error, xs->data, xs->datalen); 1359 goto error_load; 1360 } 1361 1362 int op = (xs->xs_control & XS_CTL_DATA_IN) ? BUS_DMASYNC_PREREAD : 1363 BUS_DMASYNC_PREWRITE; 1364 int nseg = hcb->dma_map->dm_nsegs; 1365 bus_dma_segment_t *segs = hcb->dma_map->dm_segs; 1366 if (nseg != 0) { 1367 if (nseg > 1) { 1368 struct pvscsi_sg_element *sge; 1369 1370 KASSERTMSG(nseg <= PVSCSI_MAX_SG_ENTRIES_PER_SEGMENT, 1371 "too many sg segments"); 1372 1373 sge = hcb->sg_list->sge; 1374 e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST; 1375 1376 for (size_t i = 0; i < nseg; ++i) { 1377 sge[i].addr = segs[i].ds_addr; 1378 sge[i].length = segs[i].ds_len; 1379 sge[i].flags = 0; 1380 } 1381 1382 e->data_addr = hcb->sg_list_paddr; 1383 1384 bus_dmamap_sync(sc->sc_dmat, 1385 sc->sg_list_dma.map, hcb->sg_list_offset, 1386 sizeof(*sge) * nseg, BUS_DMASYNC_PREWRITE); 1387 } else { 1388 e->data_addr = segs->ds_addr; 1389 } 1390 1391 bus_dmamap_sync(sc->sc_dmat, hcb->dma_map, 0, 1392 xs->datalen, op); 1393 } else { 1394 e->data_addr = 0; 1395 } 1396 1397 /* 1398 * Ensure request record writes happen before write to (increment of) 1399 * req_prod_idx. 1400 */ 1401 membar_producer(); 1402 1403 uint8_t cdb0 = e->cdb[0]; 1404 1405 /* handle timeout */ 1406 if ((xs->xs_control & XS_CTL_POLL) == 0) { 1407 int timeout = mstohz(xs->timeout); 1408 /* start expire timer */ 1409 if (timeout == 0) 1410 timeout = 1; 1411 callout_reset(&xs->xs_callout, timeout, pvscsi_timeout, hcb); 1412 } 1413 1414 s->req_prod_idx++; 1415 1416 /* 1417 * Ensure req_prod_idx write (increment) happens before 1418 * IO is kicked (via a write). 1419 */ 1420 membar_producer(); 1421 1422 pvscsi_kick_io(sc, cdb0); 1423 mutex_exit(&sc->lock); 1424 1425 return; 1426 1427 error_load: 1428 pvscsi_hcb_put(sc, hcb); 1429 1430 finish_xs: 1431 mutex_exit(&sc->lock); 1432 scsipi_done(xs); 1433 } 1434 1435 static void 1436 pvscsi_free_interrupts(struct pvscsi_softc *sc) 1437 { 1438 1439 if (sc->sc_ih != NULL) { 1440 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 1441 sc->sc_ih = NULL; 1442 } 1443 if (sc->sc_pihp != NULL) { 1444 pci_intr_release(sc->sc_pc, sc->sc_pihp, 1); 1445 sc->sc_pihp = NULL; 1446 } 1447 } 1448 1449 static int 1450 pvscsi_setup_interrupts(struct pvscsi_softc *sc, const struct pci_attach_args *pa) 1451 { 1452 int use_msix; 1453 int use_msi; 1454 int counts[PCI_INTR_TYPE_SIZE]; 1455 1456 for (size_t i = 0; i < PCI_INTR_TYPE_SIZE; i++) { 1457 counts[i] = 1; 1458 } 1459 1460 use_msix = pvscsi_get_tunable(sc, "use_msix", pvscsi_use_msix); 1461 use_msi = pvscsi_get_tunable(sc, "use_msi", pvscsi_use_msi); 1462 1463 if (!use_msix) { 1464 counts[PCI_INTR_TYPE_MSIX] = 0; 1465 } 1466 if (!use_msi) { 1467 counts[PCI_INTR_TYPE_MSI] = 0; 1468 } 1469 1470 /* Allocate and establish the interrupt. */ 1471 if (pci_intr_alloc(pa, &sc->sc_pihp, counts, PCI_INTR_TYPE_MSIX)) { 1472 aprint_error_dev(sc->dev, "can't allocate handler\n"); 1473 goto fail; 1474 } 1475 1476 char intrbuf[PCI_INTRSTR_LEN]; 1477 const pci_chipset_tag_t pc = pa->pa_pc; 1478 char const *intrstr = pci_intr_string(pc, sc->sc_pihp[0], intrbuf, 1479 sizeof(intrbuf)); 1480 1481 sc->sc_ih = pci_intr_establish_xname(pc, sc->sc_pihp[0], IPL_BIO, 1482 pvscsi_intr, sc, device_xname(sc->dev)); 1483 if (sc->sc_ih == NULL) { 1484 pci_intr_release(pc, sc->sc_pihp, 1); 1485 sc->sc_pihp = NULL; 1486 aprint_error_dev(sc->dev, "couldn't establish interrupt"); 1487 if (intrstr != NULL) 1488 aprint_error(" at %s", intrstr); 1489 aprint_error("\n"); 1490 goto fail; 1491 } 1492 pci_intr_setattr(pc, sc->sc_pihp, PCI_INTR_MPSAFE, true); 1493 1494 aprint_normal_dev(sc->dev, "interrupting at %s\n", intrstr); 1495 1496 return (0); 1497 1498 fail: 1499 if (sc->sc_ih != NULL) { 1500 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 1501 sc->sc_ih = NULL; 1502 } 1503 if (sc->sc_pihp != NULL) { 1504 pci_intr_release(sc->sc_pc, sc->sc_pihp, 1); 1505 sc->sc_pihp = NULL; 1506 } 1507 if (sc->sc_mems) { 1508 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1509 sc->sc_mems = 0; 1510 } 1511 1512 return 1; 1513 } 1514 1515 static void 1516 pvscsi_free_all(struct pvscsi_softc *sc) 1517 { 1518 1519 pvscsi_dma_free_per_hcb(sc, sc->hcb_cnt); 1520 1521 if (sc->hcbs) { 1522 kmem_free(sc->hcbs, sc->hcb_cnt * sizeof(*sc->hcbs)); 1523 } 1524 1525 pvscsi_free_rings(sc); 1526 1527 pvscsi_free_interrupts(sc); 1528 1529 if (sc->sc_mems) { 1530 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 1531 sc->sc_mems = 0; 1532 } 1533 } 1534 1535 static inline void 1536 pci_enable_busmaster(device_t dev, const pci_chipset_tag_t pc, 1537 const pcitag_t tag) 1538 { 1539 pcireg_t pci_cmd_word; 1540 1541 pci_cmd_word = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1542 if (!(pci_cmd_word & PCI_COMMAND_MASTER_ENABLE)) { 1543 pci_cmd_word |= PCI_COMMAND_MASTER_ENABLE; 1544 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, pci_cmd_word); 1545 } 1546 } 1547 1548 static void 1549 pvscsi_attach(device_t parent, device_t dev, void *aux) 1550 { 1551 const struct pci_attach_args *pa = aux; 1552 struct pvscsi_softc *sc; 1553 int rid; 1554 int error; 1555 int max_queue_depth; 1556 int adapter_queue_size; 1557 1558 sc = device_private(dev); 1559 sc->dev = dev; 1560 1561 struct scsipi_adapter *adapt = &sc->sc_adapter; 1562 struct scsipi_channel *chan = &sc->sc_channel; 1563 1564 mutex_init(&sc->lock, MUTEX_DEFAULT, IPL_BIO); 1565 1566 sc->sc_pc = pa->pa_pc; 1567 pci_enable_busmaster(dev, pa->pa_pc, pa->pa_tag); 1568 1569 pci_aprint_devinfo_fancy(pa, "virtual disk controller", 1570 VMWARE_PVSCSI_DEVSTR, true); 1571 1572 /* 1573 * Map the device. All devices support memory-mapped acccess. 1574 */ 1575 bool memh_valid; 1576 bus_space_tag_t memt; 1577 bus_space_handle_t memh; 1578 bus_size_t mems; 1579 pcireg_t regt; 1580 1581 for (rid = PCI_MAPREG_START; rid < PCI_MAPREG_END; rid += sizeof(regt)) { 1582 regt = pci_mapreg_type(pa->pa_pc, pa->pa_tag, rid); 1583 if (PCI_MAPREG_TYPE(regt) == PCI_MAPREG_TYPE_MEM) 1584 break; 1585 } 1586 1587 if (rid >= PCI_MAPREG_END) { 1588 aprint_error_dev(dev, 1589 "unable to locate device registers\n"); 1590 } 1591 1592 memh_valid = (pci_mapreg_map(pa, rid, regt, 0, &memt, &memh, 1593 NULL, &mems) == 0); 1594 if (!memh_valid) { 1595 aprint_error_dev(dev, 1596 "unable to map device registers\n"); 1597 return; 1598 } 1599 sc->sc_memt = memt; 1600 sc->sc_memh = memh; 1601 sc->sc_mems = mems; 1602 1603 if (pci_dma64_available(pa)) { 1604 sc->sc_dmat = pa->pa_dmat64; 1605 aprint_verbose_dev(sc->dev, "64-bit DMA\n"); 1606 } else { 1607 aprint_verbose_dev(sc->dev, "32-bit DMA\n"); 1608 sc->sc_dmat = pa->pa_dmat; 1609 } 1610 1611 error = pvscsi_setup_interrupts(sc, pa); 1612 if (error) { 1613 aprint_normal_dev(dev, "Interrupt setup failed\n"); 1614 pvscsi_free_all(sc); 1615 return; 1616 } 1617 1618 sc->max_targets = pvscsi_get_max_targets(sc); 1619 1620 sc->use_msg = pvscsi_get_tunable(sc, "use_msg", pvscsi_use_msg) && 1621 pvscsi_hw_supports_msg(sc); 1622 sc->msg_ring_num_pages = sc->use_msg ? 1 : 0; 1623 1624 sc->req_ring_num_pages = pvscsi_get_tunable(sc, "request_ring_pages", 1625 pvscsi_request_ring_pages); 1626 if (sc->req_ring_num_pages <= 0) { 1627 if (sc->max_targets <= 16) { 1628 sc->req_ring_num_pages = 1629 PVSCSI_DEFAULT_NUM_PAGES_REQ_RING; 1630 } else { 1631 sc->req_ring_num_pages = PVSCSI_MAX_NUM_PAGES_REQ_RING; 1632 } 1633 } else if (sc->req_ring_num_pages > PVSCSI_MAX_NUM_PAGES_REQ_RING) { 1634 sc->req_ring_num_pages = PVSCSI_MAX_NUM_PAGES_REQ_RING; 1635 } 1636 sc->cmp_ring_num_pages = sc->req_ring_num_pages; 1637 1638 max_queue_depth = pvscsi_get_tunable(sc, "max_queue_depth", 1639 pvscsi_max_queue_depth); 1640 1641 adapter_queue_size = (sc->req_ring_num_pages * PAGE_SIZE) / 1642 sizeof(struct pvscsi_ring_req_desc); 1643 if (max_queue_depth > 0) { 1644 adapter_queue_size = MIN(adapter_queue_size, max_queue_depth); 1645 } 1646 adapter_queue_size = MIN(adapter_queue_size, 1647 PVSCSI_MAX_REQ_QUEUE_DEPTH); 1648 1649 aprint_normal_dev(sc->dev, "Use Msg: %d\n", sc->use_msg); 1650 aprint_normal_dev(sc->dev, "Max targets: %d\n", sc->max_targets); 1651 aprint_normal_dev(sc->dev, "REQ num pages: %d\n", sc->req_ring_num_pages); 1652 aprint_normal_dev(sc->dev, "CMP num pages: %d\n", sc->cmp_ring_num_pages); 1653 aprint_normal_dev(sc->dev, "MSG num pages: %d\n", sc->msg_ring_num_pages); 1654 aprint_normal_dev(sc->dev, "Queue size: %d\n", adapter_queue_size); 1655 1656 if (pvscsi_allocate_rings(sc)) { 1657 aprint_normal_dev(dev, "ring allocation failed\n"); 1658 pvscsi_free_all(sc); 1659 return; 1660 } 1661 1662 sc->hcb_cnt = adapter_queue_size; 1663 sc->hcbs = kmem_zalloc(sc->hcb_cnt * sizeof(*sc->hcbs), KM_SLEEP); 1664 1665 if (pvscsi_dma_alloc_per_hcb(sc)) { 1666 aprint_normal_dev(dev, "error allocating per hcb dma memory\n"); 1667 pvscsi_free_all(sc); 1668 return; 1669 } 1670 1671 pvscsi_adapter_reset(sc); 1672 1673 /* 1674 * Fill in the scsipi_adapter. 1675 */ 1676 memset(adapt, 0, sizeof(*adapt)); 1677 adapt->adapt_dev = sc->dev; 1678 adapt->adapt_nchannels = 1; 1679 adapt->adapt_openings = MIN(adapter_queue_size, PVSCSI_CMD_PER_LUN); 1680 adapt->adapt_max_periph = adapt->adapt_openings; 1681 adapt->adapt_request = pvscsi_scsipi_request; 1682 adapt->adapt_minphys = minphys; 1683 1684 /* 1685 * Fill in the scsipi_channel. 1686 */ 1687 memset(chan, 0, sizeof(*chan)); 1688 chan->chan_adapter = adapt; 1689 chan->chan_bustype = &scsi_bustype; 1690 chan->chan_channel = 0; 1691 chan->chan_ntargets = MIN(PVSCSI_MAX_TARGET, 16); /* cap reasonably */ 1692 chan->chan_nluns = MIN(PVSCSI_MAX_LUN, 1024); /* cap reasonably */ 1693 chan->chan_id = PVSCSI_MAX_TARGET; 1694 chan->chan_flags = SCSIPI_CHAN_NOSETTLE; 1695 1696 pvscsi_setup_rings(sc); 1697 if (sc->use_msg) { 1698 pvscsi_setup_msg_ring(sc); 1699 } 1700 1701 sc->use_req_call_threshold = pvscsi_setup_req_call(sc, 1); 1702 1703 pvscsi_intr_enable(sc); 1704 1705 sc->sc_scsibus_dv = config_found(sc->dev, &sc->sc_channel, scsiprint, 1706 CFARGS_NONE); 1707 1708 return; 1709 } 1710 1711 static int 1712 pvscsi_detach(device_t dev, int flags) 1713 { 1714 struct pvscsi_softc *sc; 1715 1716 sc = device_private(dev); 1717 1718 pvscsi_intr_disable(sc); 1719 pvscsi_adapter_reset(sc); 1720 1721 pvscsi_free_all(sc); 1722 1723 mutex_destroy(&sc->lock); 1724 1725 return (0); 1726 } 1727
Indexes created Mon Sep 29 21:09:56 GMT 2025