ata.c revision 1.157
1 /* $NetBSD: ata.c,v 1.157 2020/05/02 19:09:56 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.157 2020/05/02 19:09:56 thorpej Exp $"); 29 30 #include "opt_ata.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/device.h> 36 #include <sys/conf.h> 37 #include <sys/fcntl.h> 38 #include <sys/proc.h> 39 #include <sys/kthread.h> 40 #include <sys/errno.h> 41 #include <sys/ataio.h> 42 #include <sys/kmem.h> 43 #include <sys/intr.h> 44 #include <sys/bus.h> 45 #include <sys/once.h> 46 #include <sys/bitops.h> 47 #include <sys/cpu.h> 48 49 #define ATABUS_PRIVATE 50 51 #include <dev/ata/ataconf.h> 52 #include <dev/ata/atareg.h> 53 #include <dev/ata/atavar.h> 54 #include <dev/ic/wdcvar.h> /* for PIOBM */ 55 56 #include "ioconf.h" 57 #include "locators.h" 58 59 #include "atapibus.h" 60 #include "ataraid.h" 61 #include "sata_pmp.h" 62 63 #if NATARAID > 0 64 #include <dev/ata/ata_raidvar.h> 65 #endif 66 #if NSATA_PMP > 0 67 #include <dev/ata/satapmpvar.h> 68 #endif 69 #include <dev/ata/satapmpreg.h> 70 71 #define DEBUG_FUNCS 0x08 72 #define DEBUG_PROBE 0x10 73 #define DEBUG_DETACH 0x20 74 #define DEBUG_XFERS 0x40 75 #ifdef ATADEBUG 76 #ifndef ATADEBUG_MASK 77 #define ATADEBUG_MASK 0 78 #endif 79 int atadebug_mask = ATADEBUG_MASK; 80 #define ATADEBUG_PRINT(args, level) \ 81 if (atadebug_mask & (level)) \ 82 printf args 83 #else 84 #define ATADEBUG_PRINT(args, level) 85 #endif 86 87 static ONCE_DECL(ata_init_ctrl); 88 static struct pool ata_xfer_pool; 89 90 /* 91 * A queue of atabus instances, used to ensure the same bus probe order 92 * for a given hardware configuration at each boot. Kthread probing 93 * devices on a atabus. Only one probing at once. 94 */ 95 static TAILQ_HEAD(, atabus_initq) atabus_initq_head; 96 static kmutex_t atabus_qlock; 97 static kcondvar_t atabus_qcv; 98 static lwp_t * atabus_cfg_lwp; 99 100 /***************************************************************************** 101 * ATA bus layer. 102 * 103 * ATA controllers attach an atabus instance, which handles probing the bus 104 * for drives, etc. 105 *****************************************************************************/ 106 107 dev_type_open(atabusopen); 108 dev_type_close(atabusclose); 109 dev_type_ioctl(atabusioctl); 110 111 const struct cdevsw atabus_cdevsw = { 112 .d_open = atabusopen, 113 .d_close = atabusclose, 114 .d_read = noread, 115 .d_write = nowrite, 116 .d_ioctl = atabusioctl, 117 .d_stop = nostop, 118 .d_tty = notty, 119 .d_poll = nopoll, 120 .d_mmap = nommap, 121 .d_kqfilter = nokqfilter, 122 .d_discard = nodiscard, 123 .d_flag = D_OTHER 124 }; 125 126 static void atabus_childdetached(device_t, device_t); 127 static int atabus_rescan(device_t, const char *, const int *); 128 static bool atabus_resume(device_t, const pmf_qual_t *); 129 static bool atabus_suspend(device_t, const pmf_qual_t *); 130 static void atabusconfig_thread(void *); 131 132 static void ata_channel_idle(struct ata_channel *); 133 static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *); 134 static void ata_channel_freeze_locked(struct ata_channel *); 135 static void ata_thread_wake_locked(struct ata_channel *); 136 137 /* 138 * atabus_init: 139 * 140 * Initialize ATA subsystem structures. 141 */ 142 static int 143 atabus_init(void) 144 { 145 146 pool_init(&ata_xfer_pool, sizeof(struct ata_xfer), 0, 0, 0, 147 "ataspl", NULL, IPL_BIO); 148 TAILQ_INIT(&atabus_initq_head); 149 mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE); 150 cv_init(&atabus_qcv, "atainitq"); 151 return 0; 152 } 153 154 /* 155 * atabusprint: 156 * 157 * Autoconfiguration print routine used by ATA controllers when 158 * attaching an atabus instance. 159 */ 160 int 161 atabusprint(void *aux, const char *pnp) 162 { 163 struct ata_channel *chan = aux; 164 165 if (pnp) 166 aprint_normal("atabus at %s", pnp); 167 aprint_normal(" channel %d", chan->ch_channel); 168 169 return (UNCONF); 170 } 171 172 /* 173 * ataprint: 174 * 175 * Autoconfiguration print routine. 176 */ 177 int 178 ataprint(void *aux, const char *pnp) 179 { 180 struct ata_device *adev = aux; 181 182 if (pnp) 183 aprint_normal("wd at %s", pnp); 184 aprint_normal(" drive %d", adev->adev_drv_data->drive); 185 186 return (UNCONF); 187 } 188 189 /* 190 * ata_channel_attach: 191 * 192 * Common parts of attaching an atabus to an ATA controller channel. 193 */ 194 void 195 ata_channel_attach(struct ata_channel *chp) 196 { 197 if (chp->ch_flags & ATACH_DISABLED) 198 return; 199 200 ata_channel_init(chp); 201 202 KASSERT(chp->ch_queue != NULL); 203 204 chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp, 205 atabusprint); 206 } 207 208 /* 209 * ata_channel_detach: 210 * 211 * Common parts of detaching an atabus to an ATA controller channel. 212 */ 213 void 214 ata_channel_detach(struct ata_channel *chp) 215 { 216 if (chp->ch_flags & ATACH_DISABLED) 217 return; 218 219 ata_channel_destroy(chp); 220 221 chp->ch_flags |= ATACH_DETACHED; 222 } 223 224 static void 225 atabusconfig(struct atabus_softc *atabus_sc) 226 { 227 struct ata_channel *chp = atabus_sc->sc_chan; 228 struct atac_softc *atac = chp->ch_atac; 229 struct atabus_initq *atabus_initq = NULL; 230 int i, error; 231 232 /* we are in the atabus's thread context */ 233 234 /* 235 * Probe for the drives attached to controller, unless a PMP 236 * is already known 237 */ 238 /* XXX for SATA devices we will power up all drives at once */ 239 if (chp->ch_satapmp_nports == 0) 240 (*atac->atac_probe)(chp); 241 242 if (chp->ch_ndrives >= 2) { 243 ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n", 244 chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type), 245 DEBUG_PROBE); 246 } 247 248 /* Make sure the devices probe in atabus order to avoid jitter. */ 249 mutex_enter(&atabus_qlock); 250 for (;;) { 251 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 252 if (atabus_initq->atabus_sc == atabus_sc) 253 break; 254 cv_wait(&atabus_qcv, &atabus_qlock); 255 } 256 mutex_exit(&atabus_qlock); 257 258 ata_channel_lock(chp); 259 260 KASSERT(ata_is_thread_run(chp)); 261 262 /* If no drives, abort here */ 263 if (chp->ch_drive == NULL) 264 goto out; 265 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 266 for (i = 0; i < chp->ch_ndrives; i++) 267 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) 268 break; 269 if (i == chp->ch_ndrives) 270 goto out; 271 272 /* Shortcut in case we've been shutdown */ 273 if (chp->ch_flags & ATACH_SHUTDOWN) 274 goto out; 275 276 ata_channel_unlock(chp); 277 278 if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread, 279 atabus_sc, &atabus_cfg_lwp, 280 "%scnf", device_xname(atac->atac_dev))) != 0) 281 aprint_error_dev(atac->atac_dev, 282 "unable to create config thread: error %d\n", error); 283 return; 284 285 out: 286 ata_channel_unlock(chp); 287 288 mutex_enter(&atabus_qlock); 289 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 290 cv_broadcast(&atabus_qcv); 291 mutex_exit(&atabus_qlock); 292 293 kmem_free(atabus_initq, sizeof(*atabus_initq)); 294 295 ata_delref(chp); 296 297 config_pending_decr(atac->atac_dev); 298 } 299 300 /* 301 * atabus_configthread: finish attach of atabus's childrens, in a separate 302 * kernel thread. 303 */ 304 static void 305 atabusconfig_thread(void *arg) 306 { 307 struct atabus_softc *atabus_sc = arg; 308 struct ata_channel *chp = atabus_sc->sc_chan; 309 struct atac_softc *atac = chp->ch_atac; 310 struct atabus_initq *atabus_initq = NULL; 311 int i, s; 312 313 /* XXX seems wrong */ 314 mutex_enter(&atabus_qlock); 315 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 316 KASSERT(atabus_initq->atabus_sc == atabus_sc); 317 mutex_exit(&atabus_qlock); 318 319 /* 320 * First look for a port multiplier 321 */ 322 if (chp->ch_ndrives == PMP_MAX_DRIVES && 323 chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) { 324 #if NSATA_PMP > 0 325 satapmp_attach(chp); 326 #else 327 aprint_error_dev(atabus_sc->sc_dev, 328 "SATA port multiplier not supported\n"); 329 /* no problems going on, all drives are ATA_DRIVET_NONE */ 330 #endif 331 } 332 333 /* 334 * Attach an ATAPI bus, if needed. 335 */ 336 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 337 for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) { 338 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) { 339 #if NATAPIBUS > 0 340 (*atac->atac_atapibus_attach)(atabus_sc); 341 #else 342 /* 343 * Fake the autoconfig "not configured" message 344 */ 345 aprint_normal("atapibus at %s not configured\n", 346 device_xname(atac->atac_dev)); 347 chp->atapibus = NULL; 348 s = splbio(); 349 for (i = 0; i < chp->ch_ndrives; i++) { 350 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 351 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 352 } 353 splx(s); 354 #endif 355 break; 356 } 357 } 358 359 for (i = 0; i < chp->ch_ndrives; i++) { 360 struct ata_device adev; 361 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA && 362 chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) { 363 continue; 364 } 365 if (chp->ch_drive[i].drv_softc != NULL) 366 continue; 367 memset(&adev, 0, sizeof(struct ata_device)); 368 adev.adev_bustype = atac->atac_bustype_ata; 369 adev.adev_channel = chp->ch_channel; 370 adev.adev_drv_data = &chp->ch_drive[i]; 371 chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev, 372 "ata_hl", &adev, ataprint); 373 if (chp->ch_drive[i].drv_softc != NULL) { 374 ata_probe_caps(&chp->ch_drive[i]); 375 } else { 376 s = splbio(); 377 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 378 splx(s); 379 } 380 } 381 382 /* now that we know the drives, the controller can set its modes */ 383 if (atac->atac_set_modes) { 384 (*atac->atac_set_modes)(chp); 385 ata_print_modes(chp); 386 } 387 #if NATARAID > 0 388 if (atac->atac_cap & ATAC_CAP_RAID) { 389 for (i = 0; i < chp->ch_ndrives; i++) { 390 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) { 391 ata_raid_check_component( 392 chp->ch_drive[i].drv_softc); 393 } 394 } 395 } 396 #endif /* NATARAID > 0 */ 397 398 /* 399 * reset drive_flags for unattached devices, reset state for attached 400 * ones 401 */ 402 s = splbio(); 403 for (i = 0; i < chp->ch_ndrives; i++) { 404 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 405 continue; 406 if (chp->ch_drive[i].drv_softc == NULL) { 407 chp->ch_drive[i].drive_flags = 0; 408 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 409 } else 410 chp->ch_drive[i].state = 0; 411 } 412 splx(s); 413 414 mutex_enter(&atabus_qlock); 415 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 416 cv_broadcast(&atabus_qcv); 417 mutex_exit(&atabus_qlock); 418 419 kmem_free(atabus_initq, sizeof(*atabus_initq)); 420 421 ata_delref(chp); 422 423 config_pending_decr(atac->atac_dev); 424 kthread_exit(0); 425 } 426 427 /* 428 * atabus_thread: 429 * 430 * Worker thread for the ATA bus. 431 */ 432 static void 433 atabus_thread(void *arg) 434 { 435 struct atabus_softc *sc = arg; 436 struct ata_channel *chp = sc->sc_chan; 437 struct ata_queue *chq = chp->ch_queue; 438 struct ata_xfer *xfer; 439 int i, rv; 440 441 ata_channel_lock(chp); 442 KASSERT(ata_is_thread_run(chp)); 443 444 /* 445 * Probe the drives. Reset type to indicate to controllers 446 * that can re-probe that all drives must be probed.. 447 * 448 * Note: ch_ndrives may be changed during the probe. 449 */ 450 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 451 for (i = 0; i < chp->ch_ndrives; i++) { 452 chp->ch_drive[i].drive_flags = 0; 453 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 454 } 455 ata_channel_unlock(chp); 456 457 atabusconfig(sc); 458 459 ata_channel_lock(chp); 460 for (;;) { 461 if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_TH_DRIVE_RESET 462 | ATACH_TH_RECOVERY | ATACH_SHUTDOWN)) == 0 && 463 (chq->queue_active == 0 || chq->queue_freeze == 0)) { 464 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 465 } 466 if (chp->ch_flags & ATACH_SHUTDOWN) { 467 break; 468 } 469 if (chp->ch_flags & ATACH_TH_RESCAN) { 470 chp->ch_flags &= ~ATACH_TH_RESCAN; 471 ata_channel_unlock(chp); 472 atabusconfig(sc); 473 ata_channel_lock(chp); 474 } 475 if (chp->ch_flags & ATACH_TH_RESET) { 476 /* this will unfreeze the channel */ 477 ata_thread_run(chp, AT_WAIT, 478 ATACH_TH_RESET, ATACH_NODRIVE); 479 } else if (chp->ch_flags & ATACH_TH_DRIVE_RESET) { 480 /* this will unfreeze the channel */ 481 for (i = 0; i < chp->ch_ndrives; i++) { 482 struct ata_drive_datas *drvp; 483 484 drvp = &chp->ch_drive[i]; 485 486 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 487 ata_thread_run(chp, 488 AT_WAIT, ATACH_TH_DRIVE_RESET, i); 489 } 490 } 491 chp->ch_flags &= ~ATACH_TH_DRIVE_RESET; 492 } else if (chp->ch_flags & ATACH_TH_RECOVERY) { 493 /* 494 * This will unfreeze the channel; drops locks during 495 * run, so must wrap in splbio()/splx() to avoid 496 * spurious interrupts. XXX MPSAFE 497 */ 498 int s = splbio(); 499 ata_thread_run(chp, AT_WAIT, ATACH_TH_RECOVERY, 500 chp->recovery_tfd); 501 splx(s); 502 } else if (chq->queue_active > 0 && chq->queue_freeze == 1) { 503 /* 504 * Caller has bumped queue_freeze, decrease it. This 505 * flow shalt never be executed for NCQ commands. 506 */ 507 KASSERT((chp->ch_flags & ATACH_NCQ) == 0); 508 KASSERT(chq->queue_active == 1); 509 510 ata_channel_thaw_locked(chp); 511 xfer = ata_queue_get_active_xfer_locked(chp); 512 513 KASSERT(xfer != NULL); 514 KASSERT((xfer->c_flags & C_POLL) == 0); 515 516 switch ((rv = ata_xfer_start(xfer))) { 517 case ATASTART_STARTED: 518 case ATASTART_POLL: 519 case ATASTART_ABORT: 520 break; 521 case ATASTART_TH: 522 default: 523 panic("%s: ata_xfer_start() unexpected rv %d", 524 __func__, rv); 525 /* NOTREACHED */ 526 } 527 } else if (chq->queue_freeze > 1) 528 panic("%s: queue_freeze", __func__); 529 530 /* Try to run down the queue once channel is unfrozen */ 531 if (chq->queue_freeze == 0) { 532 ata_channel_unlock(chp); 533 atastart(chp); 534 ata_channel_lock(chp); 535 } 536 } 537 chp->ch_thread = NULL; 538 cv_signal(&chp->ch_thr_idle); 539 ata_channel_unlock(chp); 540 kthread_exit(0); 541 } 542 543 bool 544 ata_is_thread_run(struct ata_channel *chp) 545 { 546 KASSERT(mutex_owned(&chp->ch_lock)); 547 548 return (chp->ch_thread == curlwp && !cpu_intr_p()); 549 } 550 551 static void 552 ata_thread_wake_locked(struct ata_channel *chp) 553 { 554 KASSERT(mutex_owned(&chp->ch_lock)); 555 ata_channel_freeze_locked(chp); 556 cv_signal(&chp->ch_thr_idle); 557 } 558 559 /* 560 * atabus_match: 561 * 562 * Autoconfiguration match routine. 563 */ 564 static int 565 atabus_match(device_t parent, cfdata_t cf, void *aux) 566 { 567 struct ata_channel *chp = aux; 568 569 if (chp == NULL) 570 return (0); 571 572 if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel && 573 cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT) 574 return (0); 575 576 return (1); 577 } 578 579 /* 580 * atabus_attach: 581 * 582 * Autoconfiguration attach routine. 583 */ 584 static void 585 atabus_attach(device_t parent, device_t self, void *aux) 586 { 587 struct atabus_softc *sc = device_private(self); 588 struct ata_channel *chp = aux; 589 struct atabus_initq *initq; 590 int error; 591 592 sc->sc_chan = chp; 593 594 aprint_normal("\n"); 595 aprint_naive("\n"); 596 597 sc->sc_dev = self; 598 599 if (ata_addref(chp)) 600 return; 601 602 RUN_ONCE(&ata_init_ctrl, atabus_init); 603 604 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 605 initq->atabus_sc = sc; 606 mutex_enter(&atabus_qlock); 607 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 608 mutex_exit(&atabus_qlock); 609 config_pending_incr(sc->sc_dev); 610 611 /* XXX MPSAFE - no KTHREAD_MPSAFE, so protected by KERNEL_LOCK() */ 612 if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc, 613 &chp->ch_thread, "%s", device_xname(self))) != 0) 614 aprint_error_dev(self, 615 "unable to create kernel thread: error %d\n", error); 616 617 if (!pmf_device_register(self, atabus_suspend, atabus_resume)) 618 aprint_error_dev(self, "couldn't establish power handler\n"); 619 } 620 621 /* 622 * atabus_detach: 623 * 624 * Autoconfiguration detach routine. 625 */ 626 static int 627 atabus_detach(device_t self, int flags) 628 { 629 struct atabus_softc *sc = device_private(self); 630 struct ata_channel *chp = sc->sc_chan; 631 device_t dev = NULL; 632 int i, error = 0; 633 634 /* 635 * Detach atapibus and its children. 636 */ 637 if ((dev = chp->atapibus) != NULL) { 638 ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n", 639 device_xname(self), device_xname(dev)), DEBUG_DETACH); 640 641 error = config_detach(dev, flags); 642 if (error) 643 goto out; 644 KASSERT(chp->atapibus == NULL); 645 } 646 647 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 648 649 /* 650 * Detach our other children. 651 */ 652 for (i = 0; i < chp->ch_ndrives; i++) { 653 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 654 continue; 655 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 656 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 657 if ((dev = chp->ch_drive[i].drv_softc) != NULL) { 658 ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__, 659 __LINE__, device_xname(self), device_xname(dev)), 660 DEBUG_DETACH); 661 error = config_detach(dev, flags); 662 if (error) 663 goto out; 664 KASSERT(chp->ch_drive[i].drv_softc == NULL); 665 KASSERT(chp->ch_drive[i].drive_type == 0); 666 } 667 } 668 669 /* Shutdown the channel. */ 670 ata_channel_lock(chp); 671 chp->ch_flags |= ATACH_SHUTDOWN; 672 while (chp->ch_thread != NULL) { 673 cv_signal(&chp->ch_thr_idle); 674 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 675 } 676 ata_channel_unlock(chp); 677 678 atabus_free_drives(chp); 679 680 out: 681 #ifdef ATADEBUG 682 if (dev != NULL && error != 0) 683 ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__, 684 device_xname(self), error, device_xname(dev)), 685 DEBUG_DETACH); 686 #endif /* ATADEBUG */ 687 688 return (error); 689 } 690 691 void 692 atabus_childdetached(device_t self, device_t child) 693 { 694 bool found = false; 695 struct atabus_softc *sc = device_private(self); 696 struct ata_channel *chp = sc->sc_chan; 697 int i; 698 699 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 700 /* 701 * atapibus detached. 702 */ 703 if (child == chp->atapibus) { 704 chp->atapibus = NULL; 705 found = true; 706 for (i = 0; i < chp->ch_ndrives; i++) { 707 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI) 708 continue; 709 KASSERT(chp->ch_drive[i].drv_softc != NULL); 710 chp->ch_drive[i].drv_softc = NULL; 711 chp->ch_drive[i].drive_flags = 0; 712 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 713 } 714 } 715 716 /* 717 * Detach our other children. 718 */ 719 for (i = 0; i < chp->ch_ndrives; i++) { 720 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 721 continue; 722 if (child == chp->ch_drive[i].drv_softc) { 723 chp->ch_drive[i].drv_softc = NULL; 724 chp->ch_drive[i].drive_flags = 0; 725 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 726 chp->ch_satapmp_nports = 0; 727 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 728 found = true; 729 } 730 } 731 732 if (!found) 733 panic("%s: unknown child %p", device_xname(self), 734 (const void *)child); 735 } 736 737 CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc), 738 atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan, 739 atabus_childdetached, DVF_DETACH_SHUTDOWN); 740 741 /***************************************************************************** 742 * Common ATA bus operations. 743 *****************************************************************************/ 744 745 /* allocate/free the channel's ch_drive[] array */ 746 int 747 atabus_alloc_drives(struct ata_channel *chp, int ndrives) 748 { 749 int i; 750 if (chp->ch_ndrives != ndrives) 751 atabus_free_drives(chp); 752 if (chp->ch_drive == NULL) { 753 void *drv; 754 755 ata_channel_unlock(chp); 756 drv = kmem_zalloc(sizeof(*chp->ch_drive) * ndrives, KM_SLEEP); 757 ata_channel_lock(chp); 758 759 if (chp->ch_drive != NULL) { 760 /* lost the race */ 761 kmem_free(drv, sizeof(*chp->ch_drive) * ndrives); 762 return 0; 763 } 764 chp->ch_drive = drv; 765 } 766 for (i = 0; i < ndrives; i++) { 767 chp->ch_drive[i].chnl_softc = chp; 768 chp->ch_drive[i].drive = i; 769 } 770 chp->ch_ndrives = ndrives; 771 return 0; 772 } 773 774 void 775 atabus_free_drives(struct ata_channel *chp) 776 { 777 #ifdef DIAGNOSTIC 778 int i; 779 int dopanic = 0; 780 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 781 for (i = 0; i < chp->ch_ndrives; i++) { 782 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) { 783 printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n", 784 device_xname(chp->atabus), i, 785 chp->ch_drive[i].drive_type); 786 dopanic = 1; 787 } 788 if (chp->ch_drive[i].drv_softc != NULL) { 789 printf("%s: ch_drive[%d] attached to %s\n", 790 device_xname(chp->atabus), i, 791 device_xname(chp->ch_drive[i].drv_softc)); 792 dopanic = 1; 793 } 794 } 795 if (dopanic) 796 panic("atabus_free_drives"); 797 #endif 798 799 if (chp->ch_drive == NULL) 800 return; 801 kmem_free(chp->ch_drive, 802 sizeof(struct ata_drive_datas) * chp->ch_ndrives); 803 chp->ch_ndrives = 0; 804 chp->ch_drive = NULL; 805 } 806 807 /* Get the disk's parameters */ 808 int 809 ata_get_params(struct ata_drive_datas *drvp, uint8_t flags, 810 struct ataparams *prms) 811 { 812 struct ata_xfer *xfer; 813 struct ata_channel *chp = drvp->chnl_softc; 814 struct atac_softc *atac = chp->ch_atac; 815 char *tb; 816 int i, rv; 817 uint16_t *p; 818 819 ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS); 820 821 xfer = ata_get_xfer(chp, false); 822 if (xfer == NULL) { 823 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 824 DEBUG_FUNCS|DEBUG_PROBE); 825 return CMD_AGAIN; 826 } 827 828 tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP); 829 memset(prms, 0, sizeof(struct ataparams)); 830 831 if (drvp->drive_type == ATA_DRIVET_ATA) { 832 xfer->c_ata_c.r_command = WDCC_IDENTIFY; 833 xfer->c_ata_c.r_st_bmask = WDCS_DRDY; 834 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 835 xfer->c_ata_c.timeout = 3000; /* 3s */ 836 } else if (drvp->drive_type == ATA_DRIVET_ATAPI) { 837 xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE; 838 xfer->c_ata_c.r_st_bmask = 0; 839 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 840 xfer->c_ata_c.timeout = 10000; /* 10s */ 841 } else { 842 ATADEBUG_PRINT(("ata_get_parms: no disks\n"), 843 DEBUG_FUNCS|DEBUG_PROBE); 844 rv = CMD_ERR; 845 goto out; 846 } 847 xfer->c_ata_c.flags = AT_READ | flags; 848 xfer->c_ata_c.data = tb; 849 xfer->c_ata_c.bcount = ATA_BSIZE; 850 (*atac->atac_bustype_ata->ata_exec_command)(drvp, xfer); 851 ata_wait_cmd(chp, xfer); 852 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 853 ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n", 854 xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE); 855 rv = CMD_ERR; 856 goto out; 857 } 858 /* if we didn't read any data something is wrong */ 859 if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) { 860 rv = CMD_ERR; 861 goto out; 862 } 863 864 /* Read in parameter block. */ 865 memcpy(prms, tb, sizeof(struct ataparams)); 866 867 /* 868 * Shuffle string byte order. 869 * ATAPI NEC, Mitsumi and Pioneer drives and 870 * old ATA TDK CompactFlash cards 871 * have different byte order. 872 */ 873 #if BYTE_ORDER == BIG_ENDIAN 874 # define M(n) prms->atap_model[(n) ^ 1] 875 #else 876 # define M(n) prms->atap_model[n] 877 #endif 878 if ( 879 #if BYTE_ORDER == BIG_ENDIAN 880 ! 881 #endif 882 ((drvp->drive_type == ATA_DRIVET_ATAPI) ? 883 ((M(0) == 'N' && M(1) == 'E') || 884 (M(0) == 'F' && M(1) == 'X') || 885 (M(0) == 'P' && M(1) == 'i')) : 886 ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) { 887 rv = CMD_OK; 888 goto out; 889 } 890 #undef M 891 for (i = 0; i < sizeof(prms->atap_model); i += 2) { 892 p = (uint16_t *)(prms->atap_model + i); 893 *p = bswap16(*p); 894 } 895 for (i = 0; i < sizeof(prms->atap_serial); i += 2) { 896 p = (uint16_t *)(prms->atap_serial + i); 897 *p = bswap16(*p); 898 } 899 for (i = 0; i < sizeof(prms->atap_revision); i += 2) { 900 p = (uint16_t *)(prms->atap_revision + i); 901 *p = bswap16(*p); 902 } 903 904 rv = CMD_OK; 905 out: 906 kmem_free(tb, ATA_BSIZE); 907 ata_free_xfer(chp, xfer); 908 return rv; 909 } 910 911 int 912 ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags) 913 { 914 struct ata_xfer *xfer; 915 int rv; 916 struct ata_channel *chp = drvp->chnl_softc; 917 struct atac_softc *atac = chp->ch_atac; 918 919 ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS); 920 921 xfer = ata_get_xfer(chp, false); 922 if (xfer == NULL) { 923 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 924 DEBUG_FUNCS|DEBUG_PROBE); 925 return CMD_AGAIN; 926 } 927 928 xfer->c_ata_c.r_command = SET_FEATURES; 929 xfer->c_ata_c.r_st_bmask = 0; 930 xfer->c_ata_c.r_st_pmask = 0; 931 xfer->c_ata_c.r_features = WDSF_SET_MODE; 932 xfer->c_ata_c.r_count = mode; 933 xfer->c_ata_c.flags = flags; 934 xfer->c_ata_c.timeout = 1000; /* 1s */ 935 (*atac->atac_bustype_ata->ata_exec_command)(drvp, xfer); 936 ata_wait_cmd(chp, xfer); 937 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 938 rv = CMD_ERR; 939 goto out; 940 } 941 942 rv = CMD_OK; 943 944 out: 945 ata_free_xfer(chp, xfer); 946 return rv; 947 } 948 949 #if NATA_DMA 950 void 951 ata_dmaerr(struct ata_drive_datas *drvp, int flags) 952 { 953 ata_channel_lock_owned(drvp->chnl_softc); 954 955 /* 956 * Downgrade decision: if we get NERRS_MAX in NXFER. 957 * We start with n_dmaerrs set to NERRS_MAX-1 so that the 958 * first error within the first NXFER ops will immediatly trigger 959 * a downgrade. 960 * If we got an error and n_xfers is bigger than NXFER reset counters. 961 */ 962 drvp->n_dmaerrs++; 963 if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) { 964 ata_downgrade_mode(drvp, flags); 965 drvp->n_dmaerrs = NERRS_MAX-1; 966 drvp->n_xfers = 0; 967 return; 968 } 969 if (drvp->n_xfers > NXFER) { 970 drvp->n_dmaerrs = 1; /* just got an error */ 971 drvp->n_xfers = 1; /* restart counting from this error */ 972 } 973 } 974 #endif /* NATA_DMA */ 975 976 /* 977 * freeze the queue and wait for the controller to be idle. Caller has to 978 * unfreeze/restart the queue 979 */ 980 static void 981 ata_channel_idle(struct ata_channel *chp) 982 { 983 ata_channel_lock(chp); 984 ata_channel_freeze_locked(chp); 985 while (chp->ch_queue->queue_active > 0) { 986 chp->ch_queue->queue_flags |= QF_IDLE_WAIT; 987 cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1); 988 } 989 ata_channel_unlock(chp); 990 } 991 992 /* 993 * Add a command to the queue and start controller. 994 * 995 * MUST BE CALLED AT splbio()! 996 */ 997 void 998 ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 999 { 1000 1001 ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer, 1002 chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1003 1004 /* complete xfer setup */ 1005 xfer->c_chp = chp; 1006 1007 ata_channel_lock(chp); 1008 1009 /* 1010 * Standard commands are added to the end of command list, but 1011 * recovery commands must be run immediatelly. 1012 */ 1013 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1014 SIMPLEQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer, 1015 c_xferchain); 1016 else 1017 SIMPLEQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer, 1018 c_xferchain); 1019 1020 /* 1021 * if polling and can sleep, wait for the xfer to be at head of queue 1022 */ 1023 if ((xfer->c_flags & (C_POLL | C_WAIT)) == (C_POLL | C_WAIT)) { 1024 while (chp->ch_queue->queue_active > 0 || 1025 SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) { 1026 xfer->c_flags |= C_WAITACT; 1027 cv_wait(&chp->ch_queue->c_active, &chp->ch_lock); 1028 xfer->c_flags &= ~C_WAITACT; 1029 } 1030 1031 /* 1032 * Free xfer now if it there was attempt to free it 1033 * while we were waiting. 1034 */ 1035 if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) { 1036 ata_channel_unlock(chp); 1037 1038 ata_free_xfer(chp, xfer); 1039 return; 1040 } 1041 } 1042 1043 ata_channel_unlock(chp); 1044 1045 ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n", 1046 chp->ch_flags), DEBUG_XFERS); 1047 atastart(chp); 1048 } 1049 1050 /* 1051 * Start I/O on a controller, for the given channel. 1052 * The first xfer may be not for our channel if the channel queues 1053 * are shared. 1054 * 1055 * MUST BE CALLED AT splbio()! 1056 * 1057 * XXX FIS-based switching with PMP 1058 * Currently atastart() never schedules concurrent NCQ transfers to more than 1059 * one drive, even when channel has several SATA drives attached via PMP. 1060 * To support concurrent transfers to different drives with PMP, it would be 1061 * necessary to implement FIS-based switching support in controller driver, 1062 * and then adjust error handling and recovery to stop assuming at most 1063 * one active drive. 1064 */ 1065 void 1066 atastart(struct ata_channel *chp) 1067 { 1068 struct atac_softc *atac = chp->ch_atac; 1069 struct ata_queue *chq = chp->ch_queue; 1070 struct ata_xfer *xfer, *axfer; 1071 bool skipq; 1072 1073 #ifdef ATA_DEBUG 1074 int spl1, spl2; 1075 1076 spl1 = splbio(); 1077 spl2 = splbio(); 1078 if (spl2 != spl1) { 1079 printf("atastart: not at splbio()\n"); 1080 panic("atastart"); 1081 } 1082 splx(spl2); 1083 splx(spl1); 1084 #endif /* ATA_DEBUG */ 1085 1086 ata_channel_lock(chp); 1087 1088 again: 1089 /* is there a xfer ? */ 1090 if ((xfer = SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL) { 1091 ATADEBUG_PRINT(("%s(chp=%p): channel %d queue_xfer is empty\n", 1092 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1093 goto out; 1094 } 1095 1096 /* 1097 * if someone is waiting for the command to be active, wake it up 1098 * and let it process the command 1099 */ 1100 if (__predict_false(xfer->c_flags & C_WAITACT)) { 1101 ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d " 1102 "wait active\n", xfer, chp->ch_channel, xfer->c_drive), 1103 DEBUG_XFERS); 1104 cv_broadcast(&chp->ch_queue->c_active); 1105 goto out; 1106 } 1107 1108 skipq = ISSET(xfer->c_flags, C_SKIP_QUEUE); 1109 1110 /* is the queue frozen? */ 1111 if (__predict_false(!skipq && chq->queue_freeze > 0)) { 1112 if (chq->queue_flags & QF_IDLE_WAIT) { 1113 chq->queue_flags &= ~QF_IDLE_WAIT; 1114 cv_signal(&chp->ch_queue->queue_idle); 1115 } 1116 ATADEBUG_PRINT(("%s(chp=%p): channel %d drive %d " 1117 "queue frozen: %d\n", 1118 __func__, chp, chp->ch_channel, xfer->c_drive, 1119 chq->queue_freeze), 1120 DEBUG_XFERS); 1121 goto out; 1122 } 1123 1124 /* all xfers on same queue must belong to the same channel */ 1125 KASSERT(xfer->c_chp == chp); 1126 1127 /* 1128 * Can only take the command if there are no current active 1129 * commands, or if the command is NCQ and the active commands are also 1130 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based 1131 * switching, can only send commands to single drive. 1132 * Need only check first xfer. 1133 * XXX FIS-based switching - revisit 1134 */ 1135 if (!skipq && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) { 1136 if (!ISSET(xfer->c_flags, C_NCQ) || 1137 !ISSET(axfer->c_flags, C_NCQ) || 1138 xfer->c_drive != axfer->c_drive) 1139 goto out; 1140 } 1141 1142 struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive]; 1143 1144 /* 1145 * Are we on limit of active xfers ? If the queue has more 1146 * than 1 openings, we keep one slot reserved for recovery or dump. 1147 */ 1148 KASSERT(chq->queue_active <= chq->queue_openings); 1149 const uint8_t chq_openings = (!skipq && chq->queue_openings > 1) 1150 ? (chq->queue_openings - 1) : chq->queue_openings; 1151 const uint8_t drv_openings = ISSET(xfer->c_flags, C_NCQ) 1152 ? drvp->drv_openings : ATA_MAX_OPENINGS; 1153 if (chq->queue_active >= MIN(chq_openings, drv_openings)) { 1154 if (skipq) { 1155 panic("%s: channel %d busy, xfer not possible", 1156 __func__, chp->ch_channel); 1157 } 1158 1159 ATADEBUG_PRINT(("%s(chp=%p): channel %d completely busy\n", 1160 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1161 goto out; 1162 } 1163 1164 /* Slot allocation can fail if drv_openings < ch_openings */ 1165 if (!ata_queue_alloc_slot(chp, &xfer->c_slot, drv_openings)) 1166 goto out; 1167 1168 if (__predict_false(atac->atac_claim_hw)) { 1169 if (!atac->atac_claim_hw(chp, 0)) { 1170 ata_queue_free_slot(chp, xfer->c_slot); 1171 goto out; 1172 } 1173 } 1174 1175 /* Now committed to start the xfer */ 1176 1177 ATADEBUG_PRINT(("%s(chp=%p): xfer %p channel %d drive %d\n", 1178 __func__, chp, xfer, chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1179 if (drvp->drive_flags & ATA_DRIVE_RESET) { 1180 drvp->drive_flags &= ~ATA_DRIVE_RESET; 1181 drvp->state = 0; 1182 } 1183 1184 if (ISSET(xfer->c_flags, C_NCQ)) 1185 SET(chp->ch_flags, ATACH_NCQ); 1186 else 1187 CLR(chp->ch_flags, ATACH_NCQ); 1188 1189 SIMPLEQ_REMOVE_HEAD(&chq->queue_xfer, c_xferchain); 1190 1191 ata_activate_xfer_locked(chp, xfer); 1192 1193 if (atac->atac_cap & ATAC_CAP_NOIRQ) 1194 KASSERT(xfer->c_flags & C_POLL); 1195 1196 switch (ata_xfer_start(xfer)) { 1197 case ATASTART_TH: 1198 case ATASTART_ABORT: 1199 /* don't start any further commands in this case */ 1200 goto out; 1201 default: 1202 /* nothing to do */ 1203 break; 1204 } 1205 1206 /* Queue more commands if possible, but not during recovery or dump */ 1207 if (!skipq && chq->queue_active < chq->queue_openings) 1208 goto again; 1209 1210 out: 1211 ata_channel_unlock(chp); 1212 } 1213 1214 int 1215 ata_xfer_start(struct ata_xfer *xfer) 1216 { 1217 struct ata_channel *chp = xfer->c_chp; 1218 int rv; 1219 1220 KASSERT(mutex_owned(&chp->ch_lock)); 1221 1222 rv = xfer->ops->c_start(chp, xfer); 1223 switch (rv) { 1224 case ATASTART_STARTED: 1225 /* nothing to do */ 1226 break; 1227 case ATASTART_TH: 1228 /* postpone xfer to thread */ 1229 ata_thread_wake_locked(chp); 1230 break; 1231 case ATASTART_POLL: 1232 /* can happen even in thread context for some ATAPI devices */ 1233 ata_channel_unlock(chp); 1234 KASSERT(xfer->ops != NULL && xfer->ops->c_poll != NULL); 1235 xfer->ops->c_poll(chp, xfer); 1236 ata_channel_lock(chp); 1237 break; 1238 case ATASTART_ABORT: 1239 ata_channel_unlock(chp); 1240 KASSERT(xfer->ops != NULL && xfer->ops->c_abort != NULL); 1241 xfer->ops->c_abort(chp, xfer); 1242 ata_channel_lock(chp); 1243 break; 1244 } 1245 1246 return rv; 1247 } 1248 1249 static void 1250 ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer) 1251 { 1252 struct ata_queue * const chq = chp->ch_queue; 1253 1254 KASSERT(mutex_owned(&chp->ch_lock)); 1255 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0); 1256 1257 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1258 TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain); 1259 else { 1260 /* 1261 * Must go to head, so that ata_queue_get_active_xfer() 1262 * returns the recovery command, and not some other 1263 * random active transfer. 1264 */ 1265 TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain); 1266 } 1267 chq->active_xfers_used |= __BIT(xfer->c_slot); 1268 chq->queue_active++; 1269 } 1270 1271 /* 1272 * Does it's own locking, does not require splbio(). 1273 * flags - whether to block waiting for free xfer 1274 */ 1275 struct ata_xfer * 1276 ata_get_xfer(struct ata_channel *chp, bool waitok) 1277 { 1278 return pool_get(&ata_xfer_pool, 1279 PR_ZERO | (waitok ? PR_WAITOK : PR_NOWAIT)); 1280 } 1281 1282 /* 1283 * ata_deactivate_xfer() must be always called prior to ata_free_xfer() 1284 */ 1285 void 1286 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1287 { 1288 struct ata_queue *chq = chp->ch_queue; 1289 1290 ata_channel_lock(chp); 1291 1292 if (__predict_false(xfer->c_flags & (C_WAITACT|C_WAITTIMO))) { 1293 /* Someone is waiting for this xfer, so we can't free now */ 1294 xfer->c_flags |= C_FREE; 1295 cv_broadcast(&chq->c_active); 1296 ata_channel_unlock(chp); 1297 return; 1298 } 1299 1300 /* XXX move PIOBM and free_gw to deactivate? */ 1301 #if NATA_PIOBM /* XXX wdc dependent code */ 1302 if (__predict_false(xfer->c_flags & C_PIOBM)) { 1303 struct wdc_softc *wdc = CHAN_TO_WDC(chp); 1304 1305 /* finish the busmastering PIO */ 1306 (*wdc->piobm_done)(wdc->dma_arg, 1307 chp->ch_channel, xfer->c_drive); 1308 chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT); 1309 } 1310 #endif 1311 1312 if (__predict_false(chp->ch_atac->atac_free_hw)) 1313 chp->ch_atac->atac_free_hw(chp); 1314 1315 ata_channel_unlock(chp); 1316 1317 if (__predict_true(!ISSET(xfer->c_flags, C_PRIVATE_ALLOC))) 1318 pool_put(&ata_xfer_pool, xfer); 1319 } 1320 1321 void 1322 ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1323 { 1324 struct ata_queue * const chq = chp->ch_queue; 1325 1326 ata_channel_lock(chp); 1327 1328 KASSERT(chq->queue_active > 0); 1329 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0); 1330 1331 /* Stop only when this is last active xfer */ 1332 if (chq->queue_active == 1) 1333 callout_stop(&chp->c_timo_callout); 1334 1335 if (callout_invoking(&chp->c_timo_callout)) 1336 xfer->c_flags |= C_WAITTIMO; 1337 1338 TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain); 1339 chq->active_xfers_used &= ~__BIT(xfer->c_slot); 1340 chq->queue_active--; 1341 1342 ata_queue_free_slot(chp, xfer->c_slot); 1343 1344 if (xfer->c_flags & C_WAIT) 1345 cv_broadcast(&chq->c_cmd_finish); 1346 1347 ata_channel_unlock(chp); 1348 } 1349 1350 /* 1351 * Called in c_intr hook. Must be called before before any deactivations 1352 * are done - if there is drain pending, it calls c_kill_xfer hook which 1353 * deactivates the xfer. 1354 * Calls c_kill_xfer with channel lock free. 1355 * Returns true if caller should just exit without further processing. 1356 * Caller must not further access any part of xfer or any related controller 1357 * structures in that case, it should just return. 1358 */ 1359 bool 1360 ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer) 1361 { 1362 int drive = xfer->c_drive; 1363 bool draining = false; 1364 1365 ata_channel_lock(chp); 1366 1367 if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) { 1368 ata_channel_unlock(chp); 1369 1370 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE); 1371 1372 ata_channel_lock(chp); 1373 chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN; 1374 cv_signal(&chp->ch_queue->queue_drain); 1375 draining = true; 1376 } 1377 1378 ata_channel_unlock(chp); 1379 1380 return draining; 1381 } 1382 1383 /* 1384 * Check for race of normal transfer handling vs. timeout. 1385 */ 1386 bool 1387 ata_timo_xfer_check(struct ata_xfer *xfer) 1388 { 1389 struct ata_channel *chp = xfer->c_chp; 1390 struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive]; 1391 1392 ata_channel_lock(chp); 1393 1394 if (xfer->c_flags & C_WAITTIMO) { 1395 xfer->c_flags &= ~C_WAITTIMO; 1396 1397 /* Handle race vs. ata_free_xfer() */ 1398 if (xfer->c_flags & C_FREE) { 1399 xfer->c_flags &= ~C_FREE; 1400 ata_channel_unlock(chp); 1401 1402 device_printf(drvp->drv_softc, 1403 "xfer %"PRIxPTR" freed while invoking timeout\n", 1404 (intptr_t)xfer & PAGE_MASK); 1405 1406 ata_free_xfer(chp, xfer); 1407 return true; 1408 } 1409 1410 /* Race vs. callout_stop() in ata_deactivate_xfer() */ 1411 ata_channel_unlock(chp); 1412 1413 device_printf(drvp->drv_softc, 1414 "xfer %"PRIxPTR" deactivated while invoking timeout\n", 1415 (intptr_t)xfer & PAGE_MASK); 1416 return true; 1417 } 1418 1419 ata_channel_unlock(chp); 1420 1421 /* No race, proceed with timeout handling */ 1422 return false; 1423 } 1424 1425 /* 1426 * Kill off all active xfers for a ata_channel. 1427 * 1428 * Must be called with channel lock held. 1429 */ 1430 void 1431 ata_kill_active(struct ata_channel *chp, int reason, int flags) 1432 { 1433 struct ata_queue * const chq = chp->ch_queue; 1434 struct ata_xfer *xfer, *xfernext; 1435 1436 KASSERT(mutex_owned(&chp->ch_lock)); 1437 1438 TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) { 1439 ata_channel_unlock(chp); 1440 xfer->ops->c_kill_xfer(xfer->c_chp, xfer, reason); 1441 ata_channel_lock(chp); 1442 } 1443 } 1444 1445 /* 1446 * Kill off all pending xfers for a drive. 1447 */ 1448 void 1449 ata_kill_pending(struct ata_drive_datas *drvp) 1450 { 1451 struct ata_channel * const chp = drvp->chnl_softc; 1452 struct ata_queue * const chq = chp->ch_queue; 1453 struct ata_xfer *xfer; 1454 1455 ata_channel_lock(chp); 1456 1457 /* Kill all pending transfers */ 1458 while ((xfer = SIMPLEQ_FIRST(&chq->queue_xfer))) { 1459 KASSERT(xfer->c_chp == chp); 1460 1461 if (xfer->c_drive != drvp->drive) 1462 continue; 1463 1464 SIMPLEQ_REMOVE_HEAD(&chp->ch_queue->queue_xfer, c_xferchain); 1465 1466 /* 1467 * Keep the lock, so that we get deadlock (and 'locking against 1468 * myself' with LOCKDEBUG), instead of silent 1469 * data corruption, if the hook tries to call back into 1470 * middle layer for inactive xfer. 1471 */ 1472 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE_INACTIVE); 1473 } 1474 1475 /* Wait until all active transfers on the drive finish */ 1476 while (chq->queue_active > 0) { 1477 bool drv_active = false; 1478 1479 TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) { 1480 KASSERT(xfer->c_chp == chp); 1481 1482 if (xfer->c_drive == drvp->drive) { 1483 drv_active = true; 1484 break; 1485 } 1486 } 1487 1488 if (!drv_active) { 1489 /* all finished */ 1490 break; 1491 } 1492 1493 drvp->drive_flags |= ATA_DRIVE_WAITDRAIN; 1494 cv_wait(&chq->queue_drain, &chp->ch_lock); 1495 } 1496 1497 ata_channel_unlock(chp); 1498 } 1499 1500 static void 1501 ata_channel_freeze_locked(struct ata_channel *chp) 1502 { 1503 chp->ch_queue->queue_freeze++; 1504 1505 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1506 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1507 } 1508 1509 void 1510 ata_channel_freeze(struct ata_channel *chp) 1511 { 1512 ata_channel_lock(chp); 1513 ata_channel_freeze_locked(chp); 1514 ata_channel_unlock(chp); 1515 } 1516 1517 void 1518 ata_channel_thaw_locked(struct ata_channel *chp) 1519 { 1520 KASSERT(mutex_owned(&chp->ch_lock)); 1521 KASSERT(chp->ch_queue->queue_freeze > 0); 1522 1523 chp->ch_queue->queue_freeze--; 1524 1525 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1526 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1527 } 1528 1529 /* 1530 * ata_thread_run: 1531 * 1532 * Reset and ATA channel. Channel lock must be held. arg is type-specific. 1533 */ 1534 void 1535 ata_thread_run(struct ata_channel *chp, int flags, int type, int arg) 1536 { 1537 struct atac_softc *atac = chp->ch_atac; 1538 bool threset = false; 1539 struct ata_drive_datas *drvp; 1540 1541 ata_channel_lock_owned(chp); 1542 1543 /* 1544 * If we can poll or wait it's OK, otherwise wake up the 1545 * kernel thread to do it for us. 1546 */ 1547 ATADEBUG_PRINT(("%s flags 0x%x ch_flags 0x%x\n", 1548 __func__, flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS); 1549 if ((flags & (AT_POLL | AT_WAIT)) == 0) { 1550 switch (type) { 1551 case ATACH_TH_RESET: 1552 if (chp->ch_flags & ATACH_TH_RESET) { 1553 /* No need to schedule another reset */ 1554 return; 1555 } 1556 break; 1557 case ATACH_TH_DRIVE_RESET: 1558 { 1559 int drive = arg; 1560 1561 KASSERT(drive <= chp->ch_ndrives); 1562 drvp = &chp->ch_drive[drive]; 1563 1564 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 1565 /* No need to schedule another reset */ 1566 return; 1567 } 1568 drvp->drive_flags |= ATA_DRIVE_TH_RESET; 1569 break; 1570 } 1571 case ATACH_TH_RECOVERY: 1572 { 1573 uint32_t tfd = (uint32_t)arg; 1574 1575 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1576 chp->recovery_tfd = tfd; 1577 break; 1578 } 1579 default: 1580 panic("%s: unknown type: %x", __func__, type); 1581 /* NOTREACHED */ 1582 } 1583 1584 /* 1585 * Block execution of other commands while reset is scheduled 1586 * to a thread. 1587 */ 1588 ata_channel_freeze_locked(chp); 1589 chp->ch_flags |= type; 1590 1591 cv_signal(&chp->ch_thr_idle); 1592 return; 1593 } 1594 1595 /* Block execution of other commands during reset */ 1596 ata_channel_freeze_locked(chp); 1597 1598 /* 1599 * If reset has been scheduled to a thread, then clear 1600 * the flag now so that the thread won't try to execute it if 1601 * we happen to sleep, and thaw one more time after the reset. 1602 */ 1603 if (chp->ch_flags & type) { 1604 chp->ch_flags &= ~type; 1605 threset = true; 1606 } 1607 1608 switch (type) { 1609 case ATACH_TH_RESET: 1610 (*atac->atac_bustype_ata->ata_reset_channel)(chp, flags); 1611 1612 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1613 for (int drive = 0; drive < chp->ch_ndrives; drive++) 1614 chp->ch_drive[drive].state = 0; 1615 break; 1616 1617 case ATACH_TH_DRIVE_RESET: 1618 { 1619 int drive = arg; 1620 1621 KASSERT(drive <= chp->ch_ndrives); 1622 drvp = &chp->ch_drive[drive]; 1623 (*atac->atac_bustype_ata->ata_reset_drive)(drvp, flags, NULL); 1624 drvp->state = 0; 1625 break; 1626 } 1627 1628 case ATACH_TH_RECOVERY: 1629 { 1630 uint32_t tfd = (uint32_t)arg; 1631 1632 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1633 KASSERT(atac->atac_bustype_ata->ata_recovery != NULL); 1634 1635 SET(chp->ch_flags, ATACH_RECOVERING); 1636 (*atac->atac_bustype_ata->ata_recovery)(chp, flags, tfd); 1637 CLR(chp->ch_flags, ATACH_RECOVERING); 1638 break; 1639 } 1640 1641 default: 1642 panic("%s: unknown type: %x", __func__, type); 1643 /* NOTREACHED */ 1644 } 1645 1646 /* 1647 * Thaw one extra time to clear the freeze done when the reset has 1648 * been scheduled to the thread. 1649 */ 1650 if (threset) 1651 ata_channel_thaw_locked(chp); 1652 1653 /* Allow commands to run again */ 1654 ata_channel_thaw_locked(chp); 1655 1656 /* Signal the thread in case there is an xfer to run */ 1657 cv_signal(&chp->ch_thr_idle); 1658 } 1659 1660 int 1661 ata_addref(struct ata_channel *chp) 1662 { 1663 struct atac_softc *atac = chp->ch_atac; 1664 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1665 int s, error = 0; 1666 1667 s = splbio(); 1668 if (adapt->adapt_refcnt++ == 0 && 1669 adapt->adapt_enable != NULL) { 1670 error = (*adapt->adapt_enable)(atac->atac_dev, 1); 1671 if (error) 1672 adapt->adapt_refcnt--; 1673 } 1674 splx(s); 1675 return (error); 1676 } 1677 1678 void 1679 ata_delref(struct ata_channel *chp) 1680 { 1681 struct atac_softc *atac = chp->ch_atac; 1682 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1683 int s; 1684 1685 s = splbio(); 1686 if (adapt->adapt_refcnt-- == 1 && 1687 adapt->adapt_enable != NULL) 1688 (void) (*adapt->adapt_enable)(atac->atac_dev, 0); 1689 splx(s); 1690 } 1691 1692 void 1693 ata_print_modes(struct ata_channel *chp) 1694 { 1695 struct atac_softc *atac = chp->ch_atac; 1696 int drive; 1697 struct ata_drive_datas *drvp; 1698 1699 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1700 for (drive = 0; drive < chp->ch_ndrives; drive++) { 1701 drvp = &chp->ch_drive[drive]; 1702 if (drvp->drive_type == ATA_DRIVET_NONE || 1703 drvp->drv_softc == NULL) 1704 continue; 1705 aprint_verbose("%s(%s:%d:%d): using PIO mode %d", 1706 device_xname(drvp->drv_softc), 1707 device_xname(atac->atac_dev), 1708 chp->ch_channel, drvp->drive, drvp->PIO_mode); 1709 #if NATA_DMA 1710 if (drvp->drive_flags & ATA_DRIVE_DMA) 1711 aprint_verbose(", DMA mode %d", drvp->DMA_mode); 1712 #if NATA_UDMA 1713 if (drvp->drive_flags & ATA_DRIVE_UDMA) { 1714 aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode); 1715 if (drvp->UDMA_mode == 2) 1716 aprint_verbose(" (Ultra/33)"); 1717 else if (drvp->UDMA_mode == 4) 1718 aprint_verbose(" (Ultra/66)"); 1719 else if (drvp->UDMA_mode == 5) 1720 aprint_verbose(" (Ultra/100)"); 1721 else if (drvp->UDMA_mode == 6) 1722 aprint_verbose(" (Ultra/133)"); 1723 } 1724 #endif /* NATA_UDMA */ 1725 #endif /* NATA_DMA */ 1726 #if NATA_DMA || NATA_PIOBM 1727 if (0 1728 #if NATA_DMA 1729 || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) 1730 #endif 1731 #if NATA_PIOBM 1732 /* PIOBM capable controllers use DMA for PIO commands */ 1733 || (atac->atac_cap & ATAC_CAP_PIOBM) 1734 #endif 1735 ) 1736 aprint_verbose(" (using DMA)"); 1737 1738 if (drvp->drive_flags & ATA_DRIVE_NCQ) { 1739 aprint_verbose(", NCQ (%d tags)%s", 1740 ATA_REAL_OPENINGS(chp->ch_queue->queue_openings), 1741 (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO) 1742 ? " w/PRIO" : ""); 1743 } else if (drvp->drive_flags & ATA_DRIVE_WFUA) 1744 aprint_verbose(", WRITE DMA FUA EXT"); 1745 1746 #endif /* NATA_DMA || NATA_PIOBM */ 1747 aprint_verbose("\n"); 1748 } 1749 } 1750 1751 #if NATA_DMA 1752 /* 1753 * downgrade the transfer mode of a drive after an error. return 1 if 1754 * downgrade was possible, 0 otherwise. 1755 * 1756 * MUST BE CALLED AT splbio()! 1757 */ 1758 int 1759 ata_downgrade_mode(struct ata_drive_datas *drvp, int flags) 1760 { 1761 struct ata_channel *chp = drvp->chnl_softc; 1762 struct atac_softc *atac = chp->ch_atac; 1763 device_t drv_dev = drvp->drv_softc; 1764 int cf_flags = device_cfdata(drv_dev)->cf_flags; 1765 1766 ata_channel_lock_owned(drvp->chnl_softc); 1767 1768 /* if drive or controller don't know its mode, we can't do much */ 1769 if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 || 1770 (atac->atac_set_modes == NULL)) 1771 return 0; 1772 /* current drive mode was set by a config flag, let it this way */ 1773 if ((cf_flags & ATA_CONFIG_PIO_SET) || 1774 (cf_flags & ATA_CONFIG_DMA_SET) || 1775 (cf_flags & ATA_CONFIG_UDMA_SET)) 1776 return 0; 1777 1778 #if NATA_UDMA 1779 /* 1780 * If we were using Ultra-DMA mode, downgrade to the next lower mode. 1781 */ 1782 if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) { 1783 drvp->UDMA_mode--; 1784 aprint_error_dev(drv_dev, 1785 "transfer error, downgrading to Ultra-DMA mode %d\n", 1786 drvp->UDMA_mode); 1787 } 1788 #endif 1789 1790 /* 1791 * If we were using ultra-DMA, don't downgrade to multiword DMA. 1792 */ 1793 else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) { 1794 drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA); 1795 drvp->PIO_mode = drvp->PIO_cap; 1796 aprint_error_dev(drv_dev, 1797 "transfer error, downgrading to PIO mode %d\n", 1798 drvp->PIO_mode); 1799 } else /* already using PIO, can't downgrade */ 1800 return 0; 1801 1802 (*atac->atac_set_modes)(chp); 1803 ata_print_modes(chp); 1804 /* reset the channel, which will schedule all drives for setup */ 1805 ata_thread_run(chp, flags, ATACH_TH_RESET, ATACH_NODRIVE); 1806 return 1; 1807 } 1808 #endif /* NATA_DMA */ 1809 1810 /* 1811 * Probe drive's capabilities, for use by the controller later 1812 * Assumes drvp points to an existing drive. 1813 */ 1814 void 1815 ata_probe_caps(struct ata_drive_datas *drvp) 1816 { 1817 struct ataparams params, params2; 1818 struct ata_channel *chp = drvp->chnl_softc; 1819 struct atac_softc *atac = chp->ch_atac; 1820 device_t drv_dev = drvp->drv_softc; 1821 int i, printed = 0; 1822 const char *sep = ""; 1823 int cf_flags; 1824 1825 if (ata_get_params(drvp, AT_WAIT, ¶ms) != CMD_OK) { 1826 /* IDENTIFY failed. Can't tell more about the device */ 1827 return; 1828 } 1829 if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) == 1830 (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) { 1831 /* 1832 * Controller claims 16 and 32 bit transfers. 1833 * Re-do an IDENTIFY with 32-bit transfers, 1834 * and compare results. 1835 */ 1836 ata_channel_lock(chp); 1837 drvp->drive_flags |= ATA_DRIVE_CAP32; 1838 ata_channel_unlock(chp); 1839 ata_get_params(drvp, AT_WAIT, ¶ms2); 1840 if (memcmp(¶ms, ¶ms2, sizeof(struct ataparams)) != 0) { 1841 /* Not good. fall back to 16bits */ 1842 ata_channel_lock(chp); 1843 drvp->drive_flags &= ~ATA_DRIVE_CAP32; 1844 ata_channel_unlock(chp); 1845 } else { 1846 aprint_verbose_dev(drv_dev, "32-bit data port\n"); 1847 } 1848 } 1849 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */ 1850 if (params.atap_ata_major > 0x01 && 1851 params.atap_ata_major != 0xffff) { 1852 for (i = 14; i > 0; i--) { 1853 if (params.atap_ata_major & (1 << i)) { 1854 aprint_verbose_dev(drv_dev, 1855 "ATA version %d\n", i); 1856 drvp->ata_vers = i; 1857 break; 1858 } 1859 } 1860 } 1861 #endif 1862 1863 /* An ATAPI device is at last PIO mode 3 */ 1864 if (drvp->drive_type == ATA_DRIVET_ATAPI) 1865 drvp->PIO_mode = 3; 1866 1867 /* 1868 * It's not in the specs, but it seems that some drive 1869 * returns 0xffff in atap_extensions when this field is invalid 1870 */ 1871 if (params.atap_extensions != 0xffff && 1872 (params.atap_extensions & WDC_EXT_MODES)) { 1873 /* 1874 * XXX some drives report something wrong here (they claim to 1875 * support PIO mode 8 !). As mode is coded on 3 bits in 1876 * SET FEATURE, limit it to 7 (so limit i to 4). 1877 * If higher mode than 7 is found, abort. 1878 */ 1879 for (i = 7; i >= 0; i--) { 1880 if ((params.atap_piomode_supp & (1 << i)) == 0) 1881 continue; 1882 if (i > 4) 1883 return; 1884 /* 1885 * See if mode is accepted. 1886 * If the controller can't set its PIO mode, 1887 * assume the defaults are good, so don't try 1888 * to set it 1889 */ 1890 if (atac->atac_set_modes) 1891 /* 1892 * It's OK to poll here, it's fast enough 1893 * to not bother waiting for interrupt 1894 */ 1895 if (ata_set_mode(drvp, 0x08 | (i + 3), 1896 AT_WAIT) != CMD_OK) 1897 continue; 1898 if (!printed) { 1899 aprint_verbose_dev(drv_dev, 1900 "drive supports PIO mode %d", i + 3); 1901 sep = ","; 1902 printed = 1; 1903 } 1904 /* 1905 * If controller's driver can't set its PIO mode, 1906 * get the highter one for the drive. 1907 */ 1908 if (atac->atac_set_modes == NULL || 1909 atac->atac_pio_cap >= i + 3) { 1910 drvp->PIO_mode = i + 3; 1911 drvp->PIO_cap = i + 3; 1912 break; 1913 } 1914 } 1915 if (!printed) { 1916 /* 1917 * We didn't find a valid PIO mode. 1918 * Assume the values returned for DMA are buggy too 1919 */ 1920 return; 1921 } 1922 ata_channel_lock(chp); 1923 drvp->drive_flags |= ATA_DRIVE_MODE; 1924 ata_channel_unlock(chp); 1925 printed = 0; 1926 for (i = 7; i >= 0; i--) { 1927 if ((params.atap_dmamode_supp & (1 << i)) == 0) 1928 continue; 1929 #if NATA_DMA 1930 if ((atac->atac_cap & ATAC_CAP_DMA) && 1931 atac->atac_set_modes != NULL) 1932 if (ata_set_mode(drvp, 0x20 | i, AT_WAIT) 1933 != CMD_OK) 1934 continue; 1935 #endif 1936 if (!printed) { 1937 aprint_verbose("%s DMA mode %d", sep, i); 1938 sep = ","; 1939 printed = 1; 1940 } 1941 #if NATA_DMA 1942 if (atac->atac_cap & ATAC_CAP_DMA) { 1943 if (atac->atac_set_modes != NULL && 1944 atac->atac_dma_cap < i) 1945 continue; 1946 drvp->DMA_mode = i; 1947 drvp->DMA_cap = i; 1948 ata_channel_lock(chp); 1949 drvp->drive_flags |= ATA_DRIVE_DMA; 1950 ata_channel_unlock(chp); 1951 } 1952 #endif 1953 break; 1954 } 1955 if (params.atap_extensions & WDC_EXT_UDMA_MODES) { 1956 printed = 0; 1957 for (i = 7; i >= 0; i--) { 1958 if ((params.atap_udmamode_supp & (1 << i)) 1959 == 0) 1960 continue; 1961 #if NATA_UDMA 1962 if (atac->atac_set_modes != NULL && 1963 (atac->atac_cap & ATAC_CAP_UDMA)) 1964 if (ata_set_mode(drvp, 0x40 | i, 1965 AT_WAIT) != CMD_OK) 1966 continue; 1967 #endif 1968 if (!printed) { 1969 aprint_verbose("%s Ultra-DMA mode %d", 1970 sep, i); 1971 if (i == 2) 1972 aprint_verbose(" (Ultra/33)"); 1973 else if (i == 4) 1974 aprint_verbose(" (Ultra/66)"); 1975 else if (i == 5) 1976 aprint_verbose(" (Ultra/100)"); 1977 else if (i == 6) 1978 aprint_verbose(" (Ultra/133)"); 1979 sep = ","; 1980 printed = 1; 1981 } 1982 #if NATA_UDMA 1983 if (atac->atac_cap & ATAC_CAP_UDMA) { 1984 if (atac->atac_set_modes != NULL && 1985 atac->atac_udma_cap < i) 1986 continue; 1987 drvp->UDMA_mode = i; 1988 drvp->UDMA_cap = i; 1989 ata_channel_lock(chp); 1990 drvp->drive_flags |= ATA_DRIVE_UDMA; 1991 ata_channel_unlock(chp); 1992 } 1993 #endif 1994 break; 1995 } 1996 } 1997 } 1998 1999 ata_channel_lock(chp); 2000 drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM; 2001 if (drvp->drive_type == ATA_DRIVET_ATAPI) { 2002 if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM) 2003 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2004 } else { 2005 if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM) 2006 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2007 } 2008 ata_channel_unlock(chp); 2009 2010 /* Try to guess ATA version here, if it didn't get reported */ 2011 if (drvp->ata_vers == 0) { 2012 #if NATA_UDMA 2013 if (drvp->drive_flags & ATA_DRIVE_UDMA) 2014 drvp->ata_vers = 4; /* should be at last ATA-4 */ 2015 else 2016 #endif 2017 if (drvp->PIO_cap > 2) 2018 drvp->ata_vers = 2; /* should be at last ATA-2 */ 2019 } 2020 cf_flags = device_cfdata(drv_dev)->cf_flags; 2021 if (cf_flags & ATA_CONFIG_PIO_SET) { 2022 ata_channel_lock(chp); 2023 drvp->PIO_mode = 2024 (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF; 2025 drvp->drive_flags |= ATA_DRIVE_MODE; 2026 ata_channel_unlock(chp); 2027 } 2028 #if NATA_DMA 2029 if ((atac->atac_cap & ATAC_CAP_DMA) == 0) { 2030 /* don't care about DMA modes */ 2031 if (*sep != '\0') 2032 aprint_verbose("\n"); 2033 return; 2034 } 2035 if (cf_flags & ATA_CONFIG_DMA_SET) { 2036 ata_channel_lock(chp); 2037 if ((cf_flags & ATA_CONFIG_DMA_MODES) == 2038 ATA_CONFIG_DMA_DISABLE) { 2039 drvp->drive_flags &= ~ATA_DRIVE_DMA; 2040 } else { 2041 drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >> 2042 ATA_CONFIG_DMA_OFF; 2043 drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE; 2044 } 2045 ata_channel_unlock(chp); 2046 } 2047 2048 /* 2049 * Probe WRITE DMA FUA EXT. Support is mandatory for devices 2050 * supporting LBA48, but nevertheless confirm with the feature flag. 2051 */ 2052 if (drvp->drive_flags & ATA_DRIVE_DMA) { 2053 if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0 2054 && (params.atap_cmd_def & ATA_CMDE_WFE)) { 2055 drvp->drive_flags |= ATA_DRIVE_WFUA; 2056 aprint_verbose("%s WRITE DMA FUA", sep); 2057 sep = ","; 2058 } 2059 } 2060 2061 /* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */ 2062 ata_channel_lock(chp); 2063 drvp->drv_openings = 1; 2064 if (params.atap_sata_caps & SATA_NATIVE_CMDQ) { 2065 if (atac->atac_cap & ATAC_CAP_NCQ) 2066 drvp->drive_flags |= ATA_DRIVE_NCQ; 2067 drvp->drv_openings = 2068 (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1; 2069 aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings); 2070 sep = ","; 2071 2072 if (params.atap_sata_caps & SATA_NCQ_PRIO) { 2073 drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO; 2074 aprint_verbose(" w/PRIO"); 2075 } 2076 } 2077 ata_channel_unlock(chp); 2078 2079 if (*sep != '\0') 2080 aprint_verbose("\n"); 2081 2082 #if NATA_UDMA 2083 if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) { 2084 /* don't care about UDMA modes */ 2085 return; 2086 } 2087 if (cf_flags & ATA_CONFIG_UDMA_SET) { 2088 ata_channel_lock(chp); 2089 if ((cf_flags & ATA_CONFIG_UDMA_MODES) == 2090 ATA_CONFIG_UDMA_DISABLE) { 2091 drvp->drive_flags &= ~ATA_DRIVE_UDMA; 2092 } else { 2093 drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >> 2094 ATA_CONFIG_UDMA_OFF; 2095 drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE; 2096 } 2097 ata_channel_unlock(chp); 2098 } 2099 #endif /* NATA_UDMA */ 2100 #endif /* NATA_DMA */ 2101 } 2102 2103 /* management of the /dev/atabus* devices */ 2104 int 2105 atabusopen(dev_t dev, int flag, int fmt, struct lwp *l) 2106 { 2107 struct atabus_softc *sc; 2108 int error; 2109 2110 sc = device_lookup_private(&atabus_cd, minor(dev)); 2111 if (sc == NULL) 2112 return (ENXIO); 2113 2114 if (sc->sc_flags & ATABUSCF_OPEN) 2115 return (EBUSY); 2116 2117 if ((error = ata_addref(sc->sc_chan)) != 0) 2118 return (error); 2119 2120 sc->sc_flags |= ATABUSCF_OPEN; 2121 2122 return (0); 2123 } 2124 2125 2126 int 2127 atabusclose(dev_t dev, int flag, int fmt, struct lwp *l) 2128 { 2129 struct atabus_softc *sc = 2130 device_lookup_private(&atabus_cd, minor(dev)); 2131 2132 ata_delref(sc->sc_chan); 2133 2134 sc->sc_flags &= ~ATABUSCF_OPEN; 2135 2136 return (0); 2137 } 2138 2139 int 2140 atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l) 2141 { 2142 struct atabus_softc *sc = 2143 device_lookup_private(&atabus_cd, minor(dev)); 2144 struct ata_channel *chp = sc->sc_chan; 2145 int min_drive, max_drive, drive; 2146 int error; 2147 2148 /* 2149 * Enforce write permission for ioctls that change the 2150 * state of the bus. Host adapter specific ioctls must 2151 * be checked by the adapter driver. 2152 */ 2153 switch (cmd) { 2154 case ATABUSIOSCAN: 2155 case ATABUSIODETACH: 2156 case ATABUSIORESET: 2157 if ((flag & FWRITE) == 0) 2158 return (EBADF); 2159 } 2160 2161 switch (cmd) { 2162 case ATABUSIORESET: 2163 ata_channel_lock(chp); 2164 ata_thread_run(sc->sc_chan, AT_WAIT | AT_POLL, 2165 ATACH_TH_RESET, ATACH_NODRIVE); 2166 ata_channel_unlock(chp); 2167 return 0; 2168 case ATABUSIOSCAN: 2169 { 2170 #if 0 2171 struct atabusioscan_args *a= 2172 (struct atabusioscan_args *)addr; 2173 #endif 2174 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2175 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2176 return (EOPNOTSUPP); 2177 return (EOPNOTSUPP); 2178 } 2179 case ATABUSIODETACH: 2180 { 2181 struct atabusiodetach_args *a= 2182 (struct atabusiodetach_args *)addr; 2183 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2184 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2185 return (EOPNOTSUPP); 2186 switch (a->at_dev) { 2187 case -1: 2188 min_drive = 0; 2189 max_drive = 1; 2190 break; 2191 case 0: 2192 case 1: 2193 min_drive = max_drive = a->at_dev; 2194 break; 2195 default: 2196 return (EINVAL); 2197 } 2198 for (drive = min_drive; drive <= max_drive; drive++) { 2199 if (chp->ch_drive[drive].drv_softc != NULL) { 2200 error = config_detach( 2201 chp->ch_drive[drive].drv_softc, 0); 2202 if (error) 2203 return (error); 2204 KASSERT(chp->ch_drive[drive].drv_softc == NULL); 2205 } 2206 } 2207 return 0; 2208 } 2209 default: 2210 return ENOTTY; 2211 } 2212 } 2213 2214 static bool 2215 atabus_suspend(device_t dv, const pmf_qual_t *qual) 2216 { 2217 struct atabus_softc *sc = device_private(dv); 2218 struct ata_channel *chp = sc->sc_chan; 2219 2220 ata_channel_idle(chp); 2221 2222 return true; 2223 } 2224 2225 static bool 2226 atabus_resume(device_t dv, const pmf_qual_t *qual) 2227 { 2228 struct atabus_softc *sc = device_private(dv); 2229 struct ata_channel *chp = sc->sc_chan; 2230 2231 /* 2232 * XXX joerg: with wdc, the first channel unfreezes the controller. 2233 * Move this the reset and queue idling into wdc. 2234 */ 2235 ata_channel_lock(chp); 2236 if (chp->ch_queue->queue_freeze == 0) { 2237 ata_channel_unlock(chp); 2238 goto out; 2239 } 2240 2241 /* unfreeze the queue and reset drives */ 2242 ata_channel_thaw_locked(chp); 2243 2244 /* reset channel only if there are drives attached */ 2245 if (chp->ch_ndrives > 0) 2246 ata_thread_run(chp, AT_WAIT, ATACH_TH_RESET, ATACH_NODRIVE); 2247 2248 ata_channel_unlock(chp); 2249 2250 out: 2251 return true; 2252 } 2253 2254 static int 2255 atabus_rescan(device_t self, const char *ifattr, const int *locators) 2256 { 2257 struct atabus_softc *sc = device_private(self); 2258 struct ata_channel *chp = sc->sc_chan; 2259 struct atabus_initq *initq; 2260 int i; 2261 2262 /* 2263 * we can rescan a port multiplier atabus, even if some devices are 2264 * still attached 2265 */ 2266 if (chp->ch_satapmp_nports == 0) { 2267 if (chp->atapibus != NULL) { 2268 return EBUSY; 2269 } 2270 2271 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 2272 for (i = 0; i < chp->ch_ndrives; i++) { 2273 if (chp->ch_drive[i].drv_softc != NULL) { 2274 return EBUSY; 2275 } 2276 } 2277 } 2278 2279 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 2280 initq->atabus_sc = sc; 2281 mutex_enter(&atabus_qlock); 2282 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 2283 mutex_exit(&atabus_qlock); 2284 config_pending_incr(sc->sc_dev); 2285 2286 ata_channel_lock(chp); 2287 chp->ch_flags |= ATACH_TH_RESCAN; 2288 cv_signal(&chp->ch_thr_idle); 2289 ata_channel_unlock(chp); 2290 2291 return 0; 2292 } 2293 2294 void 2295 ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags) 2296 { 2297 KASSERT(mutex_owned(&chp->ch_lock)); 2298 2299 if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) { 2300 /* 2301 * can't use kpause(), we may be in interrupt context 2302 * or taking a crash dump 2303 */ 2304 delay(ms * 1000); 2305 } else { 2306 int pause = mstohz(ms); 2307 2308 kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock); 2309 } 2310 } 2311 2312 void 2313 atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count, 2314 uint16_t *features, uint8_t *device) 2315 { 2316 if ((xfer->c_flags & C_NCQ) == 0) { 2317 /* FUA handling for non-NCQ drives */ 2318 if (xfer->c_bio.flags & ATA_FUA 2319 && *cmd == WDCC_WRITEDMA_EXT) 2320 *cmd = WDCC_WRITEDMA_FUA_EXT; 2321 2322 return; 2323 } 2324 2325 *cmd = (xfer->c_bio.flags & ATA_READ) ? 2326 WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED; 2327 2328 /* for FPDMA the block count is in features */ 2329 *features = *count; 2330 2331 /* NCQ tag */ 2332 *count = (xfer->c_slot << 3); 2333 2334 if (xfer->c_bio.flags & ATA_PRIO_HIGH) 2335 *count |= WDSC_PRIO_HIGH; 2336 2337 /* other device flags */ 2338 if (xfer->c_bio.flags & ATA_FUA) 2339 *device |= WDSD_FUA; 2340 } 2341 2342 void 2343 ata_wait_cmd(struct ata_channel *chp, struct ata_xfer *xfer) 2344 { 2345 struct ata_queue *chq = chp->ch_queue; 2346 struct ata_command *ata_c = &xfer->c_ata_c; 2347 2348 ata_channel_lock(chp); 2349 2350 while ((ata_c->flags & AT_DONE) == 0) 2351 cv_wait(&chq->c_cmd_finish, &chp->ch_lock); 2352 2353 ata_channel_unlock(chp); 2354 2355 KASSERT((ata_c->flags & AT_DONE) != 0); 2356 } 2357
Indexes created Tue Sep 23 14:10:03 GMT 2025