ata.c revision 1.154
1 /* $NetBSD: ata.c,v 1.154 2020/04/04 21:36:15 jdolecek Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.154 2020/04/04 21:36:15 jdolecek Exp $"); 29 30 #include "opt_ata.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/device.h> 36 #include <sys/conf.h> 37 #include <sys/fcntl.h> 38 #include <sys/proc.h> 39 #include <sys/kthread.h> 40 #include <sys/errno.h> 41 #include <sys/ataio.h> 42 #include <sys/kmem.h> 43 #include <sys/intr.h> 44 #include <sys/bus.h> 45 #include <sys/once.h> 46 #include <sys/bitops.h> 47 #include <sys/cpu.h> 48 49 #define ATABUS_PRIVATE 50 51 #include <dev/ata/ataconf.h> 52 #include <dev/ata/atareg.h> 53 #include <dev/ata/atavar.h> 54 #include <dev/ic/wdcvar.h> /* for PIOBM */ 55 56 #include "ioconf.h" 57 #include "locators.h" 58 59 #include "atapibus.h" 60 #include "ataraid.h" 61 #include "sata_pmp.h" 62 63 #if NATARAID > 0 64 #include <dev/ata/ata_raidvar.h> 65 #endif 66 #if NSATA_PMP > 0 67 #include <dev/ata/satapmpvar.h> 68 #endif 69 #include <dev/ata/satapmpreg.h> 70 71 #define DEBUG_FUNCS 0x08 72 #define DEBUG_PROBE 0x10 73 #define DEBUG_DETACH 0x20 74 #define DEBUG_XFERS 0x40 75 #ifdef ATADEBUG 76 #ifndef ATADEBUG_MASK 77 #define ATADEBUG_MASK 0 78 #endif 79 int atadebug_mask = ATADEBUG_MASK; 80 #define ATADEBUG_PRINT(args, level) \ 81 if (atadebug_mask & (level)) \ 82 printf args 83 #else 84 #define ATADEBUG_PRINT(args, level) 85 #endif 86 87 static ONCE_DECL(ata_init_ctrl); 88 static struct pool ata_xfer_pool; 89 90 /* 91 * A queue of atabus instances, used to ensure the same bus probe order 92 * for a given hardware configuration at each boot. Kthread probing 93 * devices on a atabus. Only one probing at once. 94 */ 95 static TAILQ_HEAD(, atabus_initq) atabus_initq_head; 96 static kmutex_t atabus_qlock; 97 static kcondvar_t atabus_qcv; 98 static lwp_t * atabus_cfg_lwp; 99 100 /***************************************************************************** 101 * ATA bus layer. 102 * 103 * ATA controllers attach an atabus instance, which handles probing the bus 104 * for drives, etc. 105 *****************************************************************************/ 106 107 dev_type_open(atabusopen); 108 dev_type_close(atabusclose); 109 dev_type_ioctl(atabusioctl); 110 111 const struct cdevsw atabus_cdevsw = { 112 .d_open = atabusopen, 113 .d_close = atabusclose, 114 .d_read = noread, 115 .d_write = nowrite, 116 .d_ioctl = atabusioctl, 117 .d_stop = nostop, 118 .d_tty = notty, 119 .d_poll = nopoll, 120 .d_mmap = nommap, 121 .d_kqfilter = nokqfilter, 122 .d_discard = nodiscard, 123 .d_flag = D_OTHER 124 }; 125 126 static void atabus_childdetached(device_t, device_t); 127 static int atabus_rescan(device_t, const char *, const int *); 128 static bool atabus_resume(device_t, const pmf_qual_t *); 129 static bool atabus_suspend(device_t, const pmf_qual_t *); 130 static void atabusconfig_thread(void *); 131 132 static void ata_channel_idle(struct ata_channel *); 133 static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *); 134 static void ata_channel_freeze_locked(struct ata_channel *); 135 static void ata_thread_wake_locked(struct ata_channel *); 136 137 /* 138 * atabus_init: 139 * 140 * Initialize ATA subsystem structures. 141 */ 142 static int 143 atabus_init(void) 144 { 145 146 pool_init(&ata_xfer_pool, sizeof(struct ata_xfer), 0, 0, 0, 147 "ataspl", NULL, IPL_BIO); 148 TAILQ_INIT(&atabus_initq_head); 149 mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE); 150 cv_init(&atabus_qcv, "atainitq"); 151 return 0; 152 } 153 154 /* 155 * atabusprint: 156 * 157 * Autoconfiguration print routine used by ATA controllers when 158 * attaching an atabus instance. 159 */ 160 int 161 atabusprint(void *aux, const char *pnp) 162 { 163 struct ata_channel *chan = aux; 164 165 if (pnp) 166 aprint_normal("atabus at %s", pnp); 167 aprint_normal(" channel %d", chan->ch_channel); 168 169 return (UNCONF); 170 } 171 172 /* 173 * ataprint: 174 * 175 * Autoconfiguration print routine. 176 */ 177 int 178 ataprint(void *aux, const char *pnp) 179 { 180 struct ata_device *adev = aux; 181 182 if (pnp) 183 aprint_normal("wd at %s", pnp); 184 aprint_normal(" drive %d", adev->adev_drv_data->drive); 185 186 return (UNCONF); 187 } 188 189 /* 190 * ata_channel_attach: 191 * 192 * Common parts of attaching an atabus to an ATA controller channel. 193 */ 194 void 195 ata_channel_attach(struct ata_channel *chp) 196 { 197 if (chp->ch_flags & ATACH_DISABLED) 198 return; 199 200 ata_channel_init(chp); 201 202 KASSERT(chp->ch_queue != NULL); 203 204 chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp, 205 atabusprint); 206 } 207 208 /* 209 * ata_channel_detach: 210 * 211 * Common parts of detaching an atabus to an ATA controller channel. 212 */ 213 void 214 ata_channel_detach(struct ata_channel *chp) 215 { 216 if (chp->ch_flags & ATACH_DISABLED) 217 return; 218 219 ata_channel_destroy(chp); 220 221 chp->ch_flags |= ATACH_DETACHED; 222 } 223 224 static void 225 atabusconfig(struct atabus_softc *atabus_sc) 226 { 227 struct ata_channel *chp = atabus_sc->sc_chan; 228 struct atac_softc *atac = chp->ch_atac; 229 struct atabus_initq *atabus_initq = NULL; 230 int i, error; 231 232 /* we are in the atabus's thread context */ 233 234 /* 235 * Probe for the drives attached to controller, unless a PMP 236 * is already known 237 */ 238 /* XXX for SATA devices we will power up all drives at once */ 239 if (chp->ch_satapmp_nports == 0) 240 (*atac->atac_probe)(chp); 241 242 if (chp->ch_ndrives >= 2) { 243 ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n", 244 chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type), 245 DEBUG_PROBE); 246 } 247 248 /* Make sure the devices probe in atabus order to avoid jitter. */ 249 mutex_enter(&atabus_qlock); 250 for (;;) { 251 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 252 if (atabus_initq->atabus_sc == atabus_sc) 253 break; 254 cv_wait(&atabus_qcv, &atabus_qlock); 255 } 256 mutex_exit(&atabus_qlock); 257 258 ata_channel_lock(chp); 259 260 KASSERT(ata_is_thread_run(chp)); 261 262 /* If no drives, abort here */ 263 if (chp->ch_drive == NULL) 264 goto out; 265 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 266 for (i = 0; i < chp->ch_ndrives; i++) 267 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) 268 break; 269 if (i == chp->ch_ndrives) 270 goto out; 271 272 /* Shortcut in case we've been shutdown */ 273 if (chp->ch_flags & ATACH_SHUTDOWN) 274 goto out; 275 276 ata_channel_unlock(chp); 277 278 if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread, 279 atabus_sc, &atabus_cfg_lwp, 280 "%scnf", device_xname(atac->atac_dev))) != 0) 281 aprint_error_dev(atac->atac_dev, 282 "unable to create config thread: error %d\n", error); 283 return; 284 285 out: 286 ata_channel_unlock(chp); 287 288 mutex_enter(&atabus_qlock); 289 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 290 cv_broadcast(&atabus_qcv); 291 mutex_exit(&atabus_qlock); 292 293 kmem_free(atabus_initq, sizeof(*atabus_initq)); 294 295 ata_delref(chp); 296 297 config_pending_decr(atac->atac_dev); 298 } 299 300 /* 301 * atabus_configthread: finish attach of atabus's childrens, in a separate 302 * kernel thread. 303 */ 304 static void 305 atabusconfig_thread(void *arg) 306 { 307 struct atabus_softc *atabus_sc = arg; 308 struct ata_channel *chp = atabus_sc->sc_chan; 309 struct atac_softc *atac = chp->ch_atac; 310 struct atabus_initq *atabus_initq = NULL; 311 int i, s; 312 313 /* XXX seems wrong */ 314 mutex_enter(&atabus_qlock); 315 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 316 KASSERT(atabus_initq->atabus_sc == atabus_sc); 317 mutex_exit(&atabus_qlock); 318 319 /* 320 * First look for a port multiplier 321 */ 322 if (chp->ch_ndrives == PMP_MAX_DRIVES && 323 chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) { 324 #if NSATA_PMP > 0 325 satapmp_attach(chp); 326 #else 327 aprint_error_dev(atabus_sc->sc_dev, 328 "SATA port multiplier not supported\n"); 329 /* no problems going on, all drives are ATA_DRIVET_NONE */ 330 #endif 331 } 332 333 /* 334 * Attach an ATAPI bus, if needed. 335 */ 336 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 337 for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) { 338 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) { 339 #if NATAPIBUS > 0 340 (*atac->atac_atapibus_attach)(atabus_sc); 341 #else 342 /* 343 * Fake the autoconfig "not configured" message 344 */ 345 aprint_normal("atapibus at %s not configured\n", 346 device_xname(atac->atac_dev)); 347 chp->atapibus = NULL; 348 s = splbio(); 349 for (i = 0; i < chp->ch_ndrives; i++) { 350 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 351 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 352 } 353 splx(s); 354 #endif 355 break; 356 } 357 } 358 359 for (i = 0; i < chp->ch_ndrives; i++) { 360 struct ata_device adev; 361 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA && 362 chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) { 363 continue; 364 } 365 if (chp->ch_drive[i].drv_softc != NULL) 366 continue; 367 memset(&adev, 0, sizeof(struct ata_device)); 368 adev.adev_bustype = atac->atac_bustype_ata; 369 adev.adev_channel = chp->ch_channel; 370 adev.adev_drv_data = &chp->ch_drive[i]; 371 chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev, 372 "ata_hl", &adev, ataprint); 373 if (chp->ch_drive[i].drv_softc != NULL) { 374 ata_probe_caps(&chp->ch_drive[i]); 375 } else { 376 s = splbio(); 377 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 378 splx(s); 379 } 380 } 381 382 /* now that we know the drives, the controller can set its modes */ 383 if (atac->atac_set_modes) { 384 (*atac->atac_set_modes)(chp); 385 ata_print_modes(chp); 386 } 387 #if NATARAID > 0 388 if (atac->atac_cap & ATAC_CAP_RAID) { 389 for (i = 0; i < chp->ch_ndrives; i++) { 390 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) { 391 ata_raid_check_component( 392 chp->ch_drive[i].drv_softc); 393 } 394 } 395 } 396 #endif /* NATARAID > 0 */ 397 398 /* 399 * reset drive_flags for unattached devices, reset state for attached 400 * ones 401 */ 402 s = splbio(); 403 for (i = 0; i < chp->ch_ndrives; i++) { 404 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 405 continue; 406 if (chp->ch_drive[i].drv_softc == NULL) { 407 chp->ch_drive[i].drive_flags = 0; 408 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 409 } else 410 chp->ch_drive[i].state = 0; 411 } 412 splx(s); 413 414 mutex_enter(&atabus_qlock); 415 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 416 cv_broadcast(&atabus_qcv); 417 mutex_exit(&atabus_qlock); 418 419 kmem_free(atabus_initq, sizeof(*atabus_initq)); 420 421 ata_delref(chp); 422 423 config_pending_decr(atac->atac_dev); 424 kthread_exit(0); 425 } 426 427 /* 428 * atabus_thread: 429 * 430 * Worker thread for the ATA bus. 431 */ 432 static void 433 atabus_thread(void *arg) 434 { 435 struct atabus_softc *sc = arg; 436 struct ata_channel *chp = sc->sc_chan; 437 struct ata_queue *chq = chp->ch_queue; 438 struct ata_xfer *xfer; 439 int i, rv; 440 441 ata_channel_lock(chp); 442 KASSERT(ata_is_thread_run(chp)); 443 444 /* 445 * Probe the drives. Reset type to indicate to controllers 446 * that can re-probe that all drives must be probed.. 447 * 448 * Note: ch_ndrives may be changed during the probe. 449 */ 450 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 451 for (i = 0; i < chp->ch_ndrives; i++) { 452 chp->ch_drive[i].drive_flags = 0; 453 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 454 } 455 ata_channel_unlock(chp); 456 457 atabusconfig(sc); 458 459 ata_channel_lock(chp); 460 for (;;) { 461 if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_TH_DRIVE_RESET 462 | ATACH_TH_RECOVERY | ATACH_SHUTDOWN)) == 0 && 463 (chq->queue_active == 0 || chq->queue_freeze == 0)) { 464 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 465 } 466 if (chp->ch_flags & ATACH_SHUTDOWN) { 467 break; 468 } 469 if (chp->ch_flags & ATACH_TH_RESCAN) { 470 chp->ch_flags &= ~ATACH_TH_RESCAN; 471 ata_channel_unlock(chp); 472 atabusconfig(sc); 473 ata_channel_lock(chp); 474 } 475 if (chp->ch_flags & ATACH_TH_RESET) { 476 /* this will unfreeze the channel */ 477 ata_thread_run(chp, AT_WAIT, 478 ATACH_TH_RESET, ATACH_NODRIVE); 479 } else if (chp->ch_flags & ATACH_TH_DRIVE_RESET) { 480 /* this will unfreeze the channel */ 481 for (i = 0; i < chp->ch_ndrives; i++) { 482 struct ata_drive_datas *drvp; 483 484 drvp = &chp->ch_drive[i]; 485 486 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 487 ata_thread_run(chp, 488 AT_WAIT, ATACH_TH_DRIVE_RESET, i); 489 } 490 } 491 chp->ch_flags &= ~ATACH_TH_DRIVE_RESET; 492 } else if (chp->ch_flags & ATACH_TH_RECOVERY) { 493 /* 494 * This will unfreeze the channel; drops locks during 495 * run, so must wrap in splbio()/splx() to avoid 496 * spurious interrupts. XXX MPSAFE 497 */ 498 int s = splbio(); 499 ata_thread_run(chp, AT_WAIT, ATACH_TH_RECOVERY, 500 chp->recovery_tfd); 501 splx(s); 502 } else if (chq->queue_active > 0 && chq->queue_freeze == 1) { 503 /* 504 * Caller has bumped queue_freeze, decrease it. This 505 * flow shalt never be executed for NCQ commands. 506 */ 507 KASSERT((chp->ch_flags & ATACH_NCQ) == 0); 508 KASSERT(chq->queue_active == 1); 509 510 ata_channel_thaw_locked(chp); 511 xfer = ata_queue_get_active_xfer_locked(chp); 512 513 KASSERT(xfer != NULL); 514 KASSERT((xfer->c_flags & C_POLL) == 0); 515 516 switch ((rv = ata_xfer_start(xfer))) { 517 case ATASTART_STARTED: 518 case ATASTART_POLL: 519 case ATASTART_ABORT: 520 break; 521 case ATASTART_TH: 522 default: 523 panic("%s: ata_xfer_start() unexpected rv %d", 524 __func__, rv); 525 /* NOTREACHED */ 526 } 527 } else if (chq->queue_freeze > 1) 528 panic("%s: queue_freeze", __func__); 529 530 /* Try to run down the queue once channel is unfrozen */ 531 if (chq->queue_freeze == 0) { 532 ata_channel_unlock(chp); 533 atastart(chp); 534 ata_channel_lock(chp); 535 } 536 } 537 chp->ch_thread = NULL; 538 cv_signal(&chp->ch_thr_idle); 539 ata_channel_unlock(chp); 540 kthread_exit(0); 541 } 542 543 bool 544 ata_is_thread_run(struct ata_channel *chp) 545 { 546 KASSERT(mutex_owned(&chp->ch_lock)); 547 548 return (chp->ch_thread == curlwp && !cpu_intr_p()); 549 } 550 551 static void 552 ata_thread_wake_locked(struct ata_channel *chp) 553 { 554 KASSERT(mutex_owned(&chp->ch_lock)); 555 ata_channel_freeze_locked(chp); 556 cv_signal(&chp->ch_thr_idle); 557 } 558 559 /* 560 * atabus_match: 561 * 562 * Autoconfiguration match routine. 563 */ 564 static int 565 atabus_match(device_t parent, cfdata_t cf, void *aux) 566 { 567 struct ata_channel *chp = aux; 568 569 if (chp == NULL) 570 return (0); 571 572 if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel && 573 cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT) 574 return (0); 575 576 return (1); 577 } 578 579 /* 580 * atabus_attach: 581 * 582 * Autoconfiguration attach routine. 583 */ 584 static void 585 atabus_attach(device_t parent, device_t self, void *aux) 586 { 587 struct atabus_softc *sc = device_private(self); 588 struct ata_channel *chp = aux; 589 struct atabus_initq *initq; 590 int error; 591 592 sc->sc_chan = chp; 593 594 aprint_normal("\n"); 595 aprint_naive("\n"); 596 597 sc->sc_dev = self; 598 599 if (ata_addref(chp)) 600 return; 601 602 RUN_ONCE(&ata_init_ctrl, atabus_init); 603 604 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 605 initq->atabus_sc = sc; 606 mutex_enter(&atabus_qlock); 607 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 608 mutex_exit(&atabus_qlock); 609 config_pending_incr(sc->sc_dev); 610 611 /* XXX MPSAFE - no KTHREAD_MPSAFE, so protected by KERNEL_LOCK() */ 612 if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc, 613 &chp->ch_thread, "%s", device_xname(self))) != 0) 614 aprint_error_dev(self, 615 "unable to create kernel thread: error %d\n", error); 616 617 if (!pmf_device_register(self, atabus_suspend, atabus_resume)) 618 aprint_error_dev(self, "couldn't establish power handler\n"); 619 } 620 621 /* 622 * atabus_detach: 623 * 624 * Autoconfiguration detach routine. 625 */ 626 static int 627 atabus_detach(device_t self, int flags) 628 { 629 struct atabus_softc *sc = device_private(self); 630 struct ata_channel *chp = sc->sc_chan; 631 device_t dev = NULL; 632 int i, error = 0; 633 634 /* 635 * Detach atapibus and its children. 636 */ 637 if ((dev = chp->atapibus) != NULL) { 638 ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n", 639 device_xname(self), device_xname(dev)), DEBUG_DETACH); 640 641 error = config_detach(dev, flags); 642 if (error) 643 goto out; 644 KASSERT(chp->atapibus == NULL); 645 } 646 647 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 648 649 /* 650 * Detach our other children. 651 */ 652 for (i = 0; i < chp->ch_ndrives; i++) { 653 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 654 continue; 655 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 656 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 657 if ((dev = chp->ch_drive[i].drv_softc) != NULL) { 658 ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__, 659 __LINE__, device_xname(self), device_xname(dev)), 660 DEBUG_DETACH); 661 error = config_detach(dev, flags); 662 if (error) 663 goto out; 664 KASSERT(chp->ch_drive[i].drv_softc == NULL); 665 KASSERT(chp->ch_drive[i].drive_type == 0); 666 } 667 } 668 669 /* Shutdown the channel. */ 670 ata_channel_lock(chp); 671 chp->ch_flags |= ATACH_SHUTDOWN; 672 while (chp->ch_thread != NULL) { 673 cv_signal(&chp->ch_thr_idle); 674 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 675 } 676 ata_channel_unlock(chp); 677 678 atabus_free_drives(chp); 679 680 out: 681 #ifdef ATADEBUG 682 if (dev != NULL && error != 0) 683 ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__, 684 device_xname(self), error, device_xname(dev)), 685 DEBUG_DETACH); 686 #endif /* ATADEBUG */ 687 688 return (error); 689 } 690 691 void 692 atabus_childdetached(device_t self, device_t child) 693 { 694 bool found = false; 695 struct atabus_softc *sc = device_private(self); 696 struct ata_channel *chp = sc->sc_chan; 697 int i; 698 699 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 700 /* 701 * atapibus detached. 702 */ 703 if (child == chp->atapibus) { 704 chp->atapibus = NULL; 705 found = true; 706 for (i = 0; i < chp->ch_ndrives; i++) { 707 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI) 708 continue; 709 KASSERT(chp->ch_drive[i].drv_softc != NULL); 710 chp->ch_drive[i].drv_softc = NULL; 711 chp->ch_drive[i].drive_flags = 0; 712 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 713 } 714 } 715 716 /* 717 * Detach our other children. 718 */ 719 for (i = 0; i < chp->ch_ndrives; i++) { 720 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 721 continue; 722 if (child == chp->ch_drive[i].drv_softc) { 723 chp->ch_drive[i].drv_softc = NULL; 724 chp->ch_drive[i].drive_flags = 0; 725 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 726 chp->ch_satapmp_nports = 0; 727 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 728 found = true; 729 } 730 } 731 732 if (!found) 733 panic("%s: unknown child %p", device_xname(self), 734 (const void *)child); 735 } 736 737 CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc), 738 atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan, 739 atabus_childdetached, DVF_DETACH_SHUTDOWN); 740 741 /***************************************************************************** 742 * Common ATA bus operations. 743 *****************************************************************************/ 744 745 /* allocate/free the channel's ch_drive[] array */ 746 int 747 atabus_alloc_drives(struct ata_channel *chp, int ndrives) 748 { 749 int i; 750 if (chp->ch_ndrives != ndrives) 751 atabus_free_drives(chp); 752 if (chp->ch_drive == NULL) { 753 void *drv; 754 755 ata_channel_unlock(chp); 756 drv = kmem_zalloc(sizeof(*chp->ch_drive) * ndrives, KM_SLEEP); 757 ata_channel_lock(chp); 758 759 if (chp->ch_drive != NULL) { 760 /* lost the race */ 761 kmem_free(drv, sizeof(*chp->ch_drive) * ndrives); 762 return 0; 763 } 764 chp->ch_drive = drv; 765 } 766 for (i = 0; i < ndrives; i++) { 767 chp->ch_drive[i].chnl_softc = chp; 768 chp->ch_drive[i].drive = i; 769 } 770 chp->ch_ndrives = ndrives; 771 return 0; 772 } 773 774 void 775 atabus_free_drives(struct ata_channel *chp) 776 { 777 #ifdef DIAGNOSTIC 778 int i; 779 int dopanic = 0; 780 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 781 for (i = 0; i < chp->ch_ndrives; i++) { 782 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) { 783 printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n", 784 device_xname(chp->atabus), i, 785 chp->ch_drive[i].drive_type); 786 dopanic = 1; 787 } 788 if (chp->ch_drive[i].drv_softc != NULL) { 789 printf("%s: ch_drive[%d] attached to %s\n", 790 device_xname(chp->atabus), i, 791 device_xname(chp->ch_drive[i].drv_softc)); 792 dopanic = 1; 793 } 794 } 795 if (dopanic) 796 panic("atabus_free_drives"); 797 #endif 798 799 if (chp->ch_drive == NULL) 800 return; 801 kmem_free(chp->ch_drive, 802 sizeof(struct ata_drive_datas) * chp->ch_ndrives); 803 chp->ch_ndrives = 0; 804 chp->ch_drive = NULL; 805 } 806 807 /* Get the disk's parameters */ 808 int 809 ata_get_params(struct ata_drive_datas *drvp, uint8_t flags, 810 struct ataparams *prms) 811 { 812 struct ata_xfer *xfer; 813 struct ata_channel *chp = drvp->chnl_softc; 814 struct atac_softc *atac = chp->ch_atac; 815 char *tb; 816 int i, rv; 817 uint16_t *p; 818 819 ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS); 820 821 xfer = ata_get_xfer(chp, false); 822 if (xfer == NULL) { 823 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 824 DEBUG_FUNCS|DEBUG_PROBE); 825 return CMD_AGAIN; 826 } 827 828 tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP); 829 memset(prms, 0, sizeof(struct ataparams)); 830 831 if (drvp->drive_type == ATA_DRIVET_ATA) { 832 xfer->c_ata_c.r_command = WDCC_IDENTIFY; 833 xfer->c_ata_c.r_st_bmask = WDCS_DRDY; 834 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 835 xfer->c_ata_c.timeout = 3000; /* 3s */ 836 } else if (drvp->drive_type == ATA_DRIVET_ATAPI) { 837 xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE; 838 xfer->c_ata_c.r_st_bmask = 0; 839 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 840 xfer->c_ata_c.timeout = 10000; /* 10s */ 841 } else { 842 ATADEBUG_PRINT(("ata_get_parms: no disks\n"), 843 DEBUG_FUNCS|DEBUG_PROBE); 844 rv = CMD_ERR; 845 goto out; 846 } 847 xfer->c_ata_c.flags = AT_READ | flags; 848 xfer->c_ata_c.data = tb; 849 xfer->c_ata_c.bcount = ATA_BSIZE; 850 if ((*atac->atac_bustype_ata->ata_exec_command)(drvp, 851 xfer) != ATACMD_COMPLETE) { 852 ATADEBUG_PRINT(("ata_get_parms: wdc_exec_command failed\n"), 853 DEBUG_FUNCS|DEBUG_PROBE); 854 rv = CMD_AGAIN; 855 goto out; 856 } 857 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 858 ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n", 859 xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE); 860 rv = CMD_ERR; 861 goto out; 862 } 863 /* if we didn't read any data something is wrong */ 864 if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) { 865 rv = CMD_ERR; 866 goto out; 867 } 868 869 /* Read in parameter block. */ 870 memcpy(prms, tb, sizeof(struct ataparams)); 871 872 /* 873 * Shuffle string byte order. 874 * ATAPI NEC, Mitsumi and Pioneer drives and 875 * old ATA TDK CompactFlash cards 876 * have different byte order. 877 */ 878 #if BYTE_ORDER == BIG_ENDIAN 879 # define M(n) prms->atap_model[(n) ^ 1] 880 #else 881 # define M(n) prms->atap_model[n] 882 #endif 883 if ( 884 #if BYTE_ORDER == BIG_ENDIAN 885 ! 886 #endif 887 ((drvp->drive_type == ATA_DRIVET_ATAPI) ? 888 ((M(0) == 'N' && M(1) == 'E') || 889 (M(0) == 'F' && M(1) == 'X') || 890 (M(0) == 'P' && M(1) == 'i')) : 891 ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) { 892 rv = CMD_OK; 893 goto out; 894 } 895 #undef M 896 for (i = 0; i < sizeof(prms->atap_model); i += 2) { 897 p = (uint16_t *)(prms->atap_model + i); 898 *p = bswap16(*p); 899 } 900 for (i = 0; i < sizeof(prms->atap_serial); i += 2) { 901 p = (uint16_t *)(prms->atap_serial + i); 902 *p = bswap16(*p); 903 } 904 for (i = 0; i < sizeof(prms->atap_revision); i += 2) { 905 p = (uint16_t *)(prms->atap_revision + i); 906 *p = bswap16(*p); 907 } 908 909 rv = CMD_OK; 910 out: 911 kmem_free(tb, ATA_BSIZE); 912 ata_free_xfer(chp, xfer); 913 return rv; 914 } 915 916 int 917 ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags) 918 { 919 struct ata_xfer *xfer; 920 int rv; 921 struct ata_channel *chp = drvp->chnl_softc; 922 struct atac_softc *atac = chp->ch_atac; 923 924 ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS); 925 926 xfer = ata_get_xfer(chp, false); 927 if (xfer == NULL) { 928 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 929 DEBUG_FUNCS|DEBUG_PROBE); 930 return CMD_AGAIN; 931 } 932 933 xfer->c_ata_c.r_command = SET_FEATURES; 934 xfer->c_ata_c.r_st_bmask = 0; 935 xfer->c_ata_c.r_st_pmask = 0; 936 xfer->c_ata_c.r_features = WDSF_SET_MODE; 937 xfer->c_ata_c.r_count = mode; 938 xfer->c_ata_c.flags = flags; 939 xfer->c_ata_c.timeout = 1000; /* 1s */ 940 if ((*atac->atac_bustype_ata->ata_exec_command)(drvp, 941 xfer) != ATACMD_COMPLETE) { 942 rv = CMD_AGAIN; 943 goto out; 944 } 945 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 946 rv = CMD_ERR; 947 goto out; 948 } 949 950 rv = CMD_OK; 951 952 out: 953 ata_free_xfer(chp, xfer); 954 return rv; 955 } 956 957 #if NATA_DMA 958 void 959 ata_dmaerr(struct ata_drive_datas *drvp, int flags) 960 { 961 ata_channel_lock_owned(drvp->chnl_softc); 962 963 /* 964 * Downgrade decision: if we get NERRS_MAX in NXFER. 965 * We start with n_dmaerrs set to NERRS_MAX-1 so that the 966 * first error within the first NXFER ops will immediatly trigger 967 * a downgrade. 968 * If we got an error and n_xfers is bigger than NXFER reset counters. 969 */ 970 drvp->n_dmaerrs++; 971 if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) { 972 ata_downgrade_mode(drvp, flags); 973 drvp->n_dmaerrs = NERRS_MAX-1; 974 drvp->n_xfers = 0; 975 return; 976 } 977 if (drvp->n_xfers > NXFER) { 978 drvp->n_dmaerrs = 1; /* just got an error */ 979 drvp->n_xfers = 1; /* restart counting from this error */ 980 } 981 } 982 #endif /* NATA_DMA */ 983 984 /* 985 * freeze the queue and wait for the controller to be idle. Caller has to 986 * unfreeze/restart the queue 987 */ 988 static void 989 ata_channel_idle(struct ata_channel *chp) 990 { 991 ata_channel_lock(chp); 992 ata_channel_freeze_locked(chp); 993 while (chp->ch_queue->queue_active > 0) { 994 chp->ch_queue->queue_flags |= QF_IDLE_WAIT; 995 cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1); 996 } 997 ata_channel_unlock(chp); 998 } 999 1000 /* 1001 * Add a command to the queue and start controller. 1002 * 1003 * MUST BE CALLED AT splbio()! 1004 */ 1005 void 1006 ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1007 { 1008 1009 ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer, 1010 chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1011 1012 /* complete xfer setup */ 1013 xfer->c_chp = chp; 1014 1015 ata_channel_lock(chp); 1016 1017 /* 1018 * Standard commands are added to the end of command list, but 1019 * recovery commands must be run immediatelly. 1020 */ 1021 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1022 SIMPLEQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer, 1023 c_xferchain); 1024 else 1025 SIMPLEQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer, 1026 c_xferchain); 1027 1028 /* 1029 * if polling and can sleep, wait for the xfer to be at head of queue 1030 */ 1031 if ((xfer->c_flags & (C_POLL | C_WAIT)) == (C_POLL | C_WAIT)) { 1032 while (chp->ch_queue->queue_active > 0 || 1033 SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) { 1034 xfer->c_flags |= C_WAITACT; 1035 cv_wait(&chp->ch_queue->c_active, &chp->ch_lock); 1036 xfer->c_flags &= ~C_WAITACT; 1037 } 1038 1039 /* 1040 * Free xfer now if it there was attempt to free it 1041 * while we were waiting. 1042 */ 1043 if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) { 1044 ata_channel_unlock(chp); 1045 1046 ata_free_xfer(chp, xfer); 1047 return; 1048 } 1049 } 1050 1051 ata_channel_unlock(chp); 1052 1053 ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n", 1054 chp->ch_flags), DEBUG_XFERS); 1055 atastart(chp); 1056 } 1057 1058 /* 1059 * Start I/O on a controller, for the given channel. 1060 * The first xfer may be not for our channel if the channel queues 1061 * are shared. 1062 * 1063 * MUST BE CALLED AT splbio()! 1064 * 1065 * XXX FIS-based switching with PMP 1066 * Currently atastart() never schedules concurrent NCQ transfers to more than 1067 * one drive, even when channel has several SATA drives attached via PMP. 1068 * To support concurrent transfers to different drives with PMP, it would be 1069 * necessary to implement FIS-based switching support in controller driver, 1070 * and then adjust error handling and recovery to stop assuming at most 1071 * one active drive. 1072 */ 1073 void 1074 atastart(struct ata_channel *chp) 1075 { 1076 struct atac_softc *atac = chp->ch_atac; 1077 struct ata_queue *chq = chp->ch_queue; 1078 struct ata_xfer *xfer, *axfer; 1079 bool skipq; 1080 1081 #ifdef ATA_DEBUG 1082 int spl1, spl2; 1083 1084 spl1 = splbio(); 1085 spl2 = splbio(); 1086 if (spl2 != spl1) { 1087 printf("atastart: not at splbio()\n"); 1088 panic("atastart"); 1089 } 1090 splx(spl2); 1091 splx(spl1); 1092 #endif /* ATA_DEBUG */ 1093 1094 ata_channel_lock(chp); 1095 1096 again: 1097 /* is there a xfer ? */ 1098 if ((xfer = SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL) { 1099 ATADEBUG_PRINT(("%s(chp=%p): channel %d queue_xfer is empty\n", 1100 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1101 goto out; 1102 } 1103 1104 /* 1105 * if someone is waiting for the command to be active, wake it up 1106 * and let it process the command 1107 */ 1108 if (__predict_false(xfer->c_flags & C_WAITACT)) { 1109 ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d " 1110 "wait active\n", xfer, chp->ch_channel, xfer->c_drive), 1111 DEBUG_XFERS); 1112 cv_broadcast(&chp->ch_queue->c_active); 1113 goto out; 1114 } 1115 1116 skipq = ISSET(xfer->c_flags, C_SKIP_QUEUE); 1117 1118 /* is the queue frozen? */ 1119 if (__predict_false(!skipq && chq->queue_freeze > 0)) { 1120 if (chq->queue_flags & QF_IDLE_WAIT) { 1121 chq->queue_flags &= ~QF_IDLE_WAIT; 1122 cv_signal(&chp->ch_queue->queue_idle); 1123 } 1124 ATADEBUG_PRINT(("%s(chp=%p): channel %d drive %d " 1125 "queue frozen: %d\n", 1126 __func__, chp, chp->ch_channel, xfer->c_drive, 1127 chq->queue_freeze), 1128 DEBUG_XFERS); 1129 goto out; 1130 } 1131 1132 /* all xfers on same queue must belong to the same channel */ 1133 KASSERT(xfer->c_chp == chp); 1134 1135 /* 1136 * Can only take the command if there are no current active 1137 * commands, or if the command is NCQ and the active commands are also 1138 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based 1139 * switching, can only send commands to single drive. 1140 * Need only check first xfer. 1141 * XXX FIS-based switching - revisit 1142 */ 1143 if (!skipq && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) { 1144 if (!ISSET(xfer->c_flags, C_NCQ) || 1145 !ISSET(axfer->c_flags, C_NCQ) || 1146 xfer->c_drive != axfer->c_drive) 1147 goto out; 1148 } 1149 1150 struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive]; 1151 1152 /* 1153 * Are we on limit of active xfers ? If the queue has more 1154 * than 1 openings, we keep one slot reserved for recovery or dump. 1155 */ 1156 KASSERT(chq->queue_active <= chq->queue_openings); 1157 const uint8_t chq_openings = (!skipq && chq->queue_openings > 1) 1158 ? (chq->queue_openings - 1) : chq->queue_openings; 1159 const uint8_t drv_openings = ISSET(xfer->c_flags, C_NCQ) 1160 ? drvp->drv_openings : ATA_MAX_OPENINGS; 1161 if (chq->queue_active >= MIN(chq_openings, drv_openings)) { 1162 if (skipq) { 1163 panic("%s: channel %d busy, xfer not possible", 1164 __func__, chp->ch_channel); 1165 } 1166 1167 ATADEBUG_PRINT(("%s(chp=%p): channel %d completely busy\n", 1168 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1169 goto out; 1170 } 1171 1172 /* Slot allocation can fail if drv_openings < ch_openings */ 1173 if (!ata_queue_alloc_slot(chp, &xfer->c_slot, drv_openings)) 1174 goto out; 1175 1176 if (__predict_false(atac->atac_claim_hw)) { 1177 if (!atac->atac_claim_hw(chp, 0)) { 1178 ata_queue_free_slot(chp, xfer->c_slot); 1179 goto out; 1180 } 1181 } 1182 1183 /* Now committed to start the xfer */ 1184 1185 ATADEBUG_PRINT(("%s(chp=%p): xfer %p channel %d drive %d\n", 1186 __func__, chp, xfer, chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1187 if (drvp->drive_flags & ATA_DRIVE_RESET) { 1188 drvp->drive_flags &= ~ATA_DRIVE_RESET; 1189 drvp->state = 0; 1190 } 1191 1192 if (ISSET(xfer->c_flags, C_NCQ)) 1193 SET(chp->ch_flags, ATACH_NCQ); 1194 else 1195 CLR(chp->ch_flags, ATACH_NCQ); 1196 1197 SIMPLEQ_REMOVE_HEAD(&chq->queue_xfer, c_xferchain); 1198 1199 ata_activate_xfer_locked(chp, xfer); 1200 1201 if (atac->atac_cap & ATAC_CAP_NOIRQ) 1202 KASSERT(xfer->c_flags & C_POLL); 1203 1204 switch (ata_xfer_start(xfer)) { 1205 case ATASTART_TH: 1206 case ATASTART_ABORT: 1207 /* don't start any further commands in this case */ 1208 goto out; 1209 default: 1210 /* nothing to do */ 1211 break; 1212 } 1213 1214 /* Queue more commands if possible, but not during recovery or dump */ 1215 if (!skipq && chq->queue_active < chq->queue_openings) 1216 goto again; 1217 1218 out: 1219 ata_channel_unlock(chp); 1220 } 1221 1222 int 1223 ata_xfer_start(struct ata_xfer *xfer) 1224 { 1225 struct ata_channel *chp = xfer->c_chp; 1226 int rv; 1227 1228 KASSERT(mutex_owned(&chp->ch_lock)); 1229 1230 rv = xfer->ops->c_start(chp, xfer); 1231 switch (rv) { 1232 case ATASTART_STARTED: 1233 /* nothing to do */ 1234 break; 1235 case ATASTART_TH: 1236 /* postpone xfer to thread */ 1237 ata_thread_wake_locked(chp); 1238 break; 1239 case ATASTART_POLL: 1240 /* can happen even in thread context for some ATAPI devices */ 1241 ata_channel_unlock(chp); 1242 KASSERT(xfer->ops != NULL && xfer->ops->c_poll != NULL); 1243 xfer->ops->c_poll(chp, xfer); 1244 ata_channel_lock(chp); 1245 break; 1246 case ATASTART_ABORT: 1247 ata_channel_unlock(chp); 1248 KASSERT(xfer->ops != NULL && xfer->ops->c_abort != NULL); 1249 xfer->ops->c_abort(chp, xfer); 1250 ata_channel_lock(chp); 1251 break; 1252 } 1253 1254 return rv; 1255 } 1256 1257 static void 1258 ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer) 1259 { 1260 struct ata_queue * const chq = chp->ch_queue; 1261 1262 KASSERT(mutex_owned(&chp->ch_lock)); 1263 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0); 1264 1265 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1266 TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain); 1267 else { 1268 /* 1269 * Must go to head, so that ata_queue_get_active_xfer() 1270 * returns the recovery command, and not some other 1271 * random active transfer. 1272 */ 1273 TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain); 1274 } 1275 chq->active_xfers_used |= __BIT(xfer->c_slot); 1276 chq->queue_active++; 1277 } 1278 1279 /* 1280 * Does it's own locking, does not require splbio(). 1281 * flags - whether to block waiting for free xfer 1282 */ 1283 struct ata_xfer * 1284 ata_get_xfer(struct ata_channel *chp, bool waitok) 1285 { 1286 return pool_get(&ata_xfer_pool, 1287 PR_ZERO | (waitok ? PR_WAITOK : PR_NOWAIT)); 1288 } 1289 1290 /* 1291 * ata_deactivate_xfer() must be always called prior to ata_free_xfer() 1292 */ 1293 void 1294 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1295 { 1296 struct ata_queue *chq = chp->ch_queue; 1297 1298 ata_channel_lock(chp); 1299 1300 if (__predict_false(xfer->c_flags & (C_WAITACT|C_WAITTIMO))) { 1301 /* Someone is waiting for this xfer, so we can't free now */ 1302 xfer->c_flags |= C_FREE; 1303 cv_broadcast(&chq->c_active); 1304 ata_channel_unlock(chp); 1305 return; 1306 } 1307 1308 /* XXX move PIOBM and free_gw to deactivate? */ 1309 #if NATA_PIOBM /* XXX wdc dependent code */ 1310 if (__predict_false(xfer->c_flags & C_PIOBM)) { 1311 struct wdc_softc *wdc = CHAN_TO_WDC(chp); 1312 1313 /* finish the busmastering PIO */ 1314 (*wdc->piobm_done)(wdc->dma_arg, 1315 chp->ch_channel, xfer->c_drive); 1316 chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT); 1317 } 1318 #endif 1319 1320 if (__predict_false(chp->ch_atac->atac_free_hw)) 1321 chp->ch_atac->atac_free_hw(chp); 1322 1323 ata_channel_unlock(chp); 1324 1325 if (__predict_true(!ISSET(xfer->c_flags, C_PRIVATE_ALLOC))) 1326 pool_put(&ata_xfer_pool, xfer); 1327 } 1328 1329 void 1330 ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1331 { 1332 struct ata_queue * const chq = chp->ch_queue; 1333 1334 ata_channel_lock(chp); 1335 1336 KASSERT(chq->queue_active > 0); 1337 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0); 1338 1339 /* Stop only when this is last active xfer */ 1340 if (chq->queue_active == 1) 1341 callout_stop(&chp->c_timo_callout); 1342 1343 if (callout_invoking(&chp->c_timo_callout)) 1344 xfer->c_flags |= C_WAITTIMO; 1345 1346 TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain); 1347 chq->active_xfers_used &= ~__BIT(xfer->c_slot); 1348 chq->queue_active--; 1349 1350 ata_queue_free_slot(chp, xfer->c_slot); 1351 1352 if (xfer->c_flags & C_WAIT) 1353 cv_broadcast(&chq->c_cmd_finish); 1354 1355 ata_channel_unlock(chp); 1356 } 1357 1358 /* 1359 * Called in c_intr hook. Must be called before before any deactivations 1360 * are done - if there is drain pending, it calls c_kill_xfer hook which 1361 * deactivates the xfer. 1362 * Calls c_kill_xfer with channel lock free. 1363 * Returns true if caller should just exit without further processing. 1364 * Caller must not further access any part of xfer or any related controller 1365 * structures in that case, it should just return. 1366 */ 1367 bool 1368 ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer) 1369 { 1370 int drive = xfer->c_drive; 1371 bool draining = false; 1372 1373 ata_channel_lock(chp); 1374 1375 if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) { 1376 ata_channel_unlock(chp); 1377 1378 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE); 1379 1380 ata_channel_lock(chp); 1381 chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN; 1382 cv_signal(&chp->ch_queue->queue_drain); 1383 draining = true; 1384 } 1385 1386 ata_channel_unlock(chp); 1387 1388 return draining; 1389 } 1390 1391 /* 1392 * Check for race of normal transfer handling vs. timeout. 1393 */ 1394 bool 1395 ata_timo_xfer_check(struct ata_xfer *xfer) 1396 { 1397 struct ata_channel *chp = xfer->c_chp; 1398 struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive]; 1399 1400 ata_channel_lock(chp); 1401 1402 if (xfer->c_flags & C_WAITTIMO) { 1403 xfer->c_flags &= ~C_WAITTIMO; 1404 1405 /* Handle race vs. ata_free_xfer() */ 1406 if (xfer->c_flags & C_FREE) { 1407 xfer->c_flags &= ~C_FREE; 1408 ata_channel_unlock(chp); 1409 1410 device_printf(drvp->drv_softc, 1411 "xfer %"PRIxPTR" freed while invoking timeout\n", 1412 (intptr_t)xfer & PAGE_MASK); 1413 1414 ata_free_xfer(chp, xfer); 1415 return true; 1416 } 1417 1418 /* Race vs. callout_stop() in ata_deactivate_xfer() */ 1419 ata_channel_unlock(chp); 1420 1421 device_printf(drvp->drv_softc, 1422 "xfer %"PRIxPTR" deactivated while invoking timeout\n", 1423 (intptr_t)xfer & PAGE_MASK); 1424 return true; 1425 } 1426 1427 ata_channel_unlock(chp); 1428 1429 /* No race, proceed with timeout handling */ 1430 return false; 1431 } 1432 1433 /* 1434 * Kill off all active xfers for a ata_channel. 1435 * 1436 * Must be called with channel lock held. 1437 */ 1438 void 1439 ata_kill_active(struct ata_channel *chp, int reason, int flags) 1440 { 1441 struct ata_queue * const chq = chp->ch_queue; 1442 struct ata_xfer *xfer, *xfernext; 1443 1444 KASSERT(mutex_owned(&chp->ch_lock)); 1445 1446 TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) { 1447 ata_channel_unlock(chp); 1448 xfer->ops->c_kill_xfer(xfer->c_chp, xfer, reason); 1449 ata_channel_lock(chp); 1450 } 1451 } 1452 1453 /* 1454 * Kill off all pending xfers for a drive. 1455 */ 1456 void 1457 ata_kill_pending(struct ata_drive_datas *drvp) 1458 { 1459 struct ata_channel * const chp = drvp->chnl_softc; 1460 struct ata_queue * const chq = chp->ch_queue; 1461 struct ata_xfer *xfer; 1462 1463 ata_channel_lock(chp); 1464 1465 /* Kill all pending transfers */ 1466 while ((xfer = SIMPLEQ_FIRST(&chq->queue_xfer))) { 1467 KASSERT(xfer->c_chp == chp); 1468 1469 if (xfer->c_drive != drvp->drive) 1470 continue; 1471 1472 SIMPLEQ_REMOVE_HEAD(&chp->ch_queue->queue_xfer, c_xferchain); 1473 1474 /* 1475 * Keep the lock, so that we get deadlock (and 'locking against 1476 * myself' with LOCKDEBUG), instead of silent 1477 * data corruption, if the hook tries to call back into 1478 * middle layer for inactive xfer. 1479 */ 1480 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE_INACTIVE); 1481 } 1482 1483 /* Wait until all active transfers on the drive finish */ 1484 while (chq->queue_active > 0) { 1485 bool drv_active = false; 1486 1487 TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) { 1488 KASSERT(xfer->c_chp == chp); 1489 1490 if (xfer->c_drive == drvp->drive) { 1491 drv_active = true; 1492 break; 1493 } 1494 } 1495 1496 if (!drv_active) { 1497 /* all finished */ 1498 break; 1499 } 1500 1501 drvp->drive_flags |= ATA_DRIVE_WAITDRAIN; 1502 cv_wait(&chq->queue_drain, &chp->ch_lock); 1503 } 1504 1505 ata_channel_unlock(chp); 1506 } 1507 1508 static void 1509 ata_channel_freeze_locked(struct ata_channel *chp) 1510 { 1511 chp->ch_queue->queue_freeze++; 1512 1513 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1514 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1515 } 1516 1517 void 1518 ata_channel_freeze(struct ata_channel *chp) 1519 { 1520 ata_channel_lock(chp); 1521 ata_channel_freeze_locked(chp); 1522 ata_channel_unlock(chp); 1523 } 1524 1525 void 1526 ata_channel_thaw_locked(struct ata_channel *chp) 1527 { 1528 KASSERT(mutex_owned(&chp->ch_lock)); 1529 KASSERT(chp->ch_queue->queue_freeze > 0); 1530 1531 chp->ch_queue->queue_freeze--; 1532 1533 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1534 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1535 } 1536 1537 /* 1538 * ata_thread_run: 1539 * 1540 * Reset and ATA channel. Channel lock must be held. arg is type-specific. 1541 */ 1542 void 1543 ata_thread_run(struct ata_channel *chp, int flags, int type, int arg) 1544 { 1545 struct atac_softc *atac = chp->ch_atac; 1546 bool threset = false; 1547 struct ata_drive_datas *drvp; 1548 1549 ata_channel_lock_owned(chp); 1550 1551 /* 1552 * If we can poll or wait it's OK, otherwise wake up the 1553 * kernel thread to do it for us. 1554 */ 1555 ATADEBUG_PRINT(("%s flags 0x%x ch_flags 0x%x\n", 1556 __func__, flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS); 1557 if ((flags & (AT_POLL | AT_WAIT)) == 0) { 1558 switch (type) { 1559 case ATACH_TH_RESET: 1560 if (chp->ch_flags & ATACH_TH_RESET) { 1561 /* No need to schedule another reset */ 1562 return; 1563 } 1564 break; 1565 case ATACH_TH_DRIVE_RESET: 1566 { 1567 int drive = arg; 1568 1569 KASSERT(drive <= chp->ch_ndrives); 1570 drvp = &chp->ch_drive[drive]; 1571 1572 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 1573 /* No need to schedule another reset */ 1574 return; 1575 } 1576 drvp->drive_flags |= ATA_DRIVE_TH_RESET; 1577 break; 1578 } 1579 case ATACH_TH_RECOVERY: 1580 { 1581 uint32_t tfd = (uint32_t)arg; 1582 1583 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1584 chp->recovery_tfd = tfd; 1585 break; 1586 } 1587 default: 1588 panic("%s: unknown type: %x", __func__, type); 1589 /* NOTREACHED */ 1590 } 1591 1592 /* 1593 * Block execution of other commands while reset is scheduled 1594 * to a thread. 1595 */ 1596 ata_channel_freeze_locked(chp); 1597 chp->ch_flags |= type; 1598 1599 cv_signal(&chp->ch_thr_idle); 1600 return; 1601 } 1602 1603 /* Block execution of other commands during reset */ 1604 ata_channel_freeze_locked(chp); 1605 1606 /* 1607 * If reset has been scheduled to a thread, then clear 1608 * the flag now so that the thread won't try to execute it if 1609 * we happen to sleep, and thaw one more time after the reset. 1610 */ 1611 if (chp->ch_flags & type) { 1612 chp->ch_flags &= ~type; 1613 threset = true; 1614 } 1615 1616 switch (type) { 1617 case ATACH_TH_RESET: 1618 (*atac->atac_bustype_ata->ata_reset_channel)(chp, flags); 1619 1620 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1621 for (int drive = 0; drive < chp->ch_ndrives; drive++) 1622 chp->ch_drive[drive].state = 0; 1623 break; 1624 1625 case ATACH_TH_DRIVE_RESET: 1626 { 1627 int drive = arg; 1628 1629 KASSERT(drive <= chp->ch_ndrives); 1630 drvp = &chp->ch_drive[drive]; 1631 (*atac->atac_bustype_ata->ata_reset_drive)(drvp, flags, NULL); 1632 drvp->state = 0; 1633 break; 1634 } 1635 1636 case ATACH_TH_RECOVERY: 1637 { 1638 uint32_t tfd = (uint32_t)arg; 1639 1640 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1641 KASSERT(atac->atac_bustype_ata->ata_recovery != NULL); 1642 1643 SET(chp->ch_flags, ATACH_RECOVERING); 1644 (*atac->atac_bustype_ata->ata_recovery)(chp, flags, tfd); 1645 CLR(chp->ch_flags, ATACH_RECOVERING); 1646 break; 1647 } 1648 1649 default: 1650 panic("%s: unknown type: %x", __func__, type); 1651 /* NOTREACHED */ 1652 } 1653 1654 /* 1655 * Thaw one extra time to clear the freeze done when the reset has 1656 * been scheduled to the thread. 1657 */ 1658 if (threset) 1659 ata_channel_thaw_locked(chp); 1660 1661 /* Allow commands to run again */ 1662 ata_channel_thaw_locked(chp); 1663 1664 /* Signal the thread in case there is an xfer to run */ 1665 cv_signal(&chp->ch_thr_idle); 1666 } 1667 1668 int 1669 ata_addref(struct ata_channel *chp) 1670 { 1671 struct atac_softc *atac = chp->ch_atac; 1672 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1673 int s, error = 0; 1674 1675 s = splbio(); 1676 if (adapt->adapt_refcnt++ == 0 && 1677 adapt->adapt_enable != NULL) { 1678 error = (*adapt->adapt_enable)(atac->atac_dev, 1); 1679 if (error) 1680 adapt->adapt_refcnt--; 1681 } 1682 splx(s); 1683 return (error); 1684 } 1685 1686 void 1687 ata_delref(struct ata_channel *chp) 1688 { 1689 struct atac_softc *atac = chp->ch_atac; 1690 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1691 int s; 1692 1693 s = splbio(); 1694 if (adapt->adapt_refcnt-- == 1 && 1695 adapt->adapt_enable != NULL) 1696 (void) (*adapt->adapt_enable)(atac->atac_dev, 0); 1697 splx(s); 1698 } 1699 1700 void 1701 ata_print_modes(struct ata_channel *chp) 1702 { 1703 struct atac_softc *atac = chp->ch_atac; 1704 int drive; 1705 struct ata_drive_datas *drvp; 1706 1707 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1708 for (drive = 0; drive < chp->ch_ndrives; drive++) { 1709 drvp = &chp->ch_drive[drive]; 1710 if (drvp->drive_type == ATA_DRIVET_NONE || 1711 drvp->drv_softc == NULL) 1712 continue; 1713 aprint_verbose("%s(%s:%d:%d): using PIO mode %d", 1714 device_xname(drvp->drv_softc), 1715 device_xname(atac->atac_dev), 1716 chp->ch_channel, drvp->drive, drvp->PIO_mode); 1717 #if NATA_DMA 1718 if (drvp->drive_flags & ATA_DRIVE_DMA) 1719 aprint_verbose(", DMA mode %d", drvp->DMA_mode); 1720 #if NATA_UDMA 1721 if (drvp->drive_flags & ATA_DRIVE_UDMA) { 1722 aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode); 1723 if (drvp->UDMA_mode == 2) 1724 aprint_verbose(" (Ultra/33)"); 1725 else if (drvp->UDMA_mode == 4) 1726 aprint_verbose(" (Ultra/66)"); 1727 else if (drvp->UDMA_mode == 5) 1728 aprint_verbose(" (Ultra/100)"); 1729 else if (drvp->UDMA_mode == 6) 1730 aprint_verbose(" (Ultra/133)"); 1731 } 1732 #endif /* NATA_UDMA */ 1733 #endif /* NATA_DMA */ 1734 #if NATA_DMA || NATA_PIOBM 1735 if (0 1736 #if NATA_DMA 1737 || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) 1738 #endif 1739 #if NATA_PIOBM 1740 /* PIOBM capable controllers use DMA for PIO commands */ 1741 || (atac->atac_cap & ATAC_CAP_PIOBM) 1742 #endif 1743 ) 1744 aprint_verbose(" (using DMA)"); 1745 1746 if (drvp->drive_flags & ATA_DRIVE_NCQ) { 1747 aprint_verbose(", NCQ (%d tags)%s", 1748 ATA_REAL_OPENINGS(chp->ch_queue->queue_openings), 1749 (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO) 1750 ? " w/PRIO" : ""); 1751 } else if (drvp->drive_flags & ATA_DRIVE_WFUA) 1752 aprint_verbose(", WRITE DMA FUA EXT"); 1753 1754 #endif /* NATA_DMA || NATA_PIOBM */ 1755 aprint_verbose("\n"); 1756 } 1757 } 1758 1759 #if NATA_DMA 1760 /* 1761 * downgrade the transfer mode of a drive after an error. return 1 if 1762 * downgrade was possible, 0 otherwise. 1763 * 1764 * MUST BE CALLED AT splbio()! 1765 */ 1766 int 1767 ata_downgrade_mode(struct ata_drive_datas *drvp, int flags) 1768 { 1769 struct ata_channel *chp = drvp->chnl_softc; 1770 struct atac_softc *atac = chp->ch_atac; 1771 device_t drv_dev = drvp->drv_softc; 1772 int cf_flags = device_cfdata(drv_dev)->cf_flags; 1773 1774 ata_channel_lock_owned(drvp->chnl_softc); 1775 1776 /* if drive or controller don't know its mode, we can't do much */ 1777 if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 || 1778 (atac->atac_set_modes == NULL)) 1779 return 0; 1780 /* current drive mode was set by a config flag, let it this way */ 1781 if ((cf_flags & ATA_CONFIG_PIO_SET) || 1782 (cf_flags & ATA_CONFIG_DMA_SET) || 1783 (cf_flags & ATA_CONFIG_UDMA_SET)) 1784 return 0; 1785 1786 #if NATA_UDMA 1787 /* 1788 * If we were using Ultra-DMA mode, downgrade to the next lower mode. 1789 */ 1790 if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) { 1791 drvp->UDMA_mode--; 1792 aprint_error_dev(drv_dev, 1793 "transfer error, downgrading to Ultra-DMA mode %d\n", 1794 drvp->UDMA_mode); 1795 } 1796 #endif 1797 1798 /* 1799 * If we were using ultra-DMA, don't downgrade to multiword DMA. 1800 */ 1801 else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) { 1802 drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA); 1803 drvp->PIO_mode = drvp->PIO_cap; 1804 aprint_error_dev(drv_dev, 1805 "transfer error, downgrading to PIO mode %d\n", 1806 drvp->PIO_mode); 1807 } else /* already using PIO, can't downgrade */ 1808 return 0; 1809 1810 (*atac->atac_set_modes)(chp); 1811 ata_print_modes(chp); 1812 /* reset the channel, which will schedule all drives for setup */ 1813 ata_thread_run(chp, flags, ATACH_TH_RESET, ATACH_NODRIVE); 1814 return 1; 1815 } 1816 #endif /* NATA_DMA */ 1817 1818 /* 1819 * Probe drive's capabilities, for use by the controller later 1820 * Assumes drvp points to an existing drive. 1821 */ 1822 void 1823 ata_probe_caps(struct ata_drive_datas *drvp) 1824 { 1825 struct ataparams params, params2; 1826 struct ata_channel *chp = drvp->chnl_softc; 1827 struct atac_softc *atac = chp->ch_atac; 1828 device_t drv_dev = drvp->drv_softc; 1829 int i, printed = 0; 1830 const char *sep = ""; 1831 int cf_flags; 1832 1833 if (ata_get_params(drvp, AT_WAIT, ¶ms) != CMD_OK) { 1834 /* IDENTIFY failed. Can't tell more about the device */ 1835 return; 1836 } 1837 if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) == 1838 (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) { 1839 /* 1840 * Controller claims 16 and 32 bit transfers. 1841 * Re-do an IDENTIFY with 32-bit transfers, 1842 * and compare results. 1843 */ 1844 ata_channel_lock(chp); 1845 drvp->drive_flags |= ATA_DRIVE_CAP32; 1846 ata_channel_unlock(chp); 1847 ata_get_params(drvp, AT_WAIT, ¶ms2); 1848 if (memcmp(¶ms, ¶ms2, sizeof(struct ataparams)) != 0) { 1849 /* Not good. fall back to 16bits */ 1850 ata_channel_lock(chp); 1851 drvp->drive_flags &= ~ATA_DRIVE_CAP32; 1852 ata_channel_unlock(chp); 1853 } else { 1854 aprint_verbose_dev(drv_dev, "32-bit data port\n"); 1855 } 1856 } 1857 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */ 1858 if (params.atap_ata_major > 0x01 && 1859 params.atap_ata_major != 0xffff) { 1860 for (i = 14; i > 0; i--) { 1861 if (params.atap_ata_major & (1 << i)) { 1862 aprint_verbose_dev(drv_dev, 1863 "ATA version %d\n", i); 1864 drvp->ata_vers = i; 1865 break; 1866 } 1867 } 1868 } 1869 #endif 1870 1871 /* An ATAPI device is at last PIO mode 3 */ 1872 if (drvp->drive_type == ATA_DRIVET_ATAPI) 1873 drvp->PIO_mode = 3; 1874 1875 /* 1876 * It's not in the specs, but it seems that some drive 1877 * returns 0xffff in atap_extensions when this field is invalid 1878 */ 1879 if (params.atap_extensions != 0xffff && 1880 (params.atap_extensions & WDC_EXT_MODES)) { 1881 /* 1882 * XXX some drives report something wrong here (they claim to 1883 * support PIO mode 8 !). As mode is coded on 3 bits in 1884 * SET FEATURE, limit it to 7 (so limit i to 4). 1885 * If higher mode than 7 is found, abort. 1886 */ 1887 for (i = 7; i >= 0; i--) { 1888 if ((params.atap_piomode_supp & (1 << i)) == 0) 1889 continue; 1890 if (i > 4) 1891 return; 1892 /* 1893 * See if mode is accepted. 1894 * If the controller can't set its PIO mode, 1895 * assume the defaults are good, so don't try 1896 * to set it 1897 */ 1898 if (atac->atac_set_modes) 1899 /* 1900 * It's OK to poll here, it's fast enough 1901 * to not bother waiting for interrupt 1902 */ 1903 if (ata_set_mode(drvp, 0x08 | (i + 3), 1904 AT_WAIT) != CMD_OK) 1905 continue; 1906 if (!printed) { 1907 aprint_verbose_dev(drv_dev, 1908 "drive supports PIO mode %d", i + 3); 1909 sep = ","; 1910 printed = 1; 1911 } 1912 /* 1913 * If controller's driver can't set its PIO mode, 1914 * get the highter one for the drive. 1915 */ 1916 if (atac->atac_set_modes == NULL || 1917 atac->atac_pio_cap >= i + 3) { 1918 drvp->PIO_mode = i + 3; 1919 drvp->PIO_cap = i + 3; 1920 break; 1921 } 1922 } 1923 if (!printed) { 1924 /* 1925 * We didn't find a valid PIO mode. 1926 * Assume the values returned for DMA are buggy too 1927 */ 1928 return; 1929 } 1930 ata_channel_lock(chp); 1931 drvp->drive_flags |= ATA_DRIVE_MODE; 1932 ata_channel_unlock(chp); 1933 printed = 0; 1934 for (i = 7; i >= 0; i--) { 1935 if ((params.atap_dmamode_supp & (1 << i)) == 0) 1936 continue; 1937 #if NATA_DMA 1938 if ((atac->atac_cap & ATAC_CAP_DMA) && 1939 atac->atac_set_modes != NULL) 1940 if (ata_set_mode(drvp, 0x20 | i, AT_WAIT) 1941 != CMD_OK) 1942 continue; 1943 #endif 1944 if (!printed) { 1945 aprint_verbose("%s DMA mode %d", sep, i); 1946 sep = ","; 1947 printed = 1; 1948 } 1949 #if NATA_DMA 1950 if (atac->atac_cap & ATAC_CAP_DMA) { 1951 if (atac->atac_set_modes != NULL && 1952 atac->atac_dma_cap < i) 1953 continue; 1954 drvp->DMA_mode = i; 1955 drvp->DMA_cap = i; 1956 ata_channel_lock(chp); 1957 drvp->drive_flags |= ATA_DRIVE_DMA; 1958 ata_channel_unlock(chp); 1959 } 1960 #endif 1961 break; 1962 } 1963 if (params.atap_extensions & WDC_EXT_UDMA_MODES) { 1964 printed = 0; 1965 for (i = 7; i >= 0; i--) { 1966 if ((params.atap_udmamode_supp & (1 << i)) 1967 == 0) 1968 continue; 1969 #if NATA_UDMA 1970 if (atac->atac_set_modes != NULL && 1971 (atac->atac_cap & ATAC_CAP_UDMA)) 1972 if (ata_set_mode(drvp, 0x40 | i, 1973 AT_WAIT) != CMD_OK) 1974 continue; 1975 #endif 1976 if (!printed) { 1977 aprint_verbose("%s Ultra-DMA mode %d", 1978 sep, i); 1979 if (i == 2) 1980 aprint_verbose(" (Ultra/33)"); 1981 else if (i == 4) 1982 aprint_verbose(" (Ultra/66)"); 1983 else if (i == 5) 1984 aprint_verbose(" (Ultra/100)"); 1985 else if (i == 6) 1986 aprint_verbose(" (Ultra/133)"); 1987 sep = ","; 1988 printed = 1; 1989 } 1990 #if NATA_UDMA 1991 if (atac->atac_cap & ATAC_CAP_UDMA) { 1992 if (atac->atac_set_modes != NULL && 1993 atac->atac_udma_cap < i) 1994 continue; 1995 drvp->UDMA_mode = i; 1996 drvp->UDMA_cap = i; 1997 ata_channel_lock(chp); 1998 drvp->drive_flags |= ATA_DRIVE_UDMA; 1999 ata_channel_unlock(chp); 2000 } 2001 #endif 2002 break; 2003 } 2004 } 2005 } 2006 2007 ata_channel_lock(chp); 2008 drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM; 2009 if (drvp->drive_type == ATA_DRIVET_ATAPI) { 2010 if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM) 2011 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2012 } else { 2013 if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM) 2014 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2015 } 2016 ata_channel_unlock(chp); 2017 2018 /* Try to guess ATA version here, if it didn't get reported */ 2019 if (drvp->ata_vers == 0) { 2020 #if NATA_UDMA 2021 if (drvp->drive_flags & ATA_DRIVE_UDMA) 2022 drvp->ata_vers = 4; /* should be at last ATA-4 */ 2023 else 2024 #endif 2025 if (drvp->PIO_cap > 2) 2026 drvp->ata_vers = 2; /* should be at last ATA-2 */ 2027 } 2028 cf_flags = device_cfdata(drv_dev)->cf_flags; 2029 if (cf_flags & ATA_CONFIG_PIO_SET) { 2030 ata_channel_lock(chp); 2031 drvp->PIO_mode = 2032 (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF; 2033 drvp->drive_flags |= ATA_DRIVE_MODE; 2034 ata_channel_unlock(chp); 2035 } 2036 #if NATA_DMA 2037 if ((atac->atac_cap & ATAC_CAP_DMA) == 0) { 2038 /* don't care about DMA modes */ 2039 if (*sep != '\0') 2040 aprint_verbose("\n"); 2041 return; 2042 } 2043 if (cf_flags & ATA_CONFIG_DMA_SET) { 2044 ata_channel_lock(chp); 2045 if ((cf_flags & ATA_CONFIG_DMA_MODES) == 2046 ATA_CONFIG_DMA_DISABLE) { 2047 drvp->drive_flags &= ~ATA_DRIVE_DMA; 2048 } else { 2049 drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >> 2050 ATA_CONFIG_DMA_OFF; 2051 drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE; 2052 } 2053 ata_channel_unlock(chp); 2054 } 2055 2056 /* 2057 * Probe WRITE DMA FUA EXT. Support is mandatory for devices 2058 * supporting LBA48, but nevertheless confirm with the feature flag. 2059 */ 2060 if (drvp->drive_flags & ATA_DRIVE_DMA) { 2061 if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0 2062 && (params.atap_cmd_def & ATA_CMDE_WFE)) { 2063 drvp->drive_flags |= ATA_DRIVE_WFUA; 2064 aprint_verbose("%s WRITE DMA FUA", sep); 2065 sep = ","; 2066 } 2067 } 2068 2069 /* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */ 2070 ata_channel_lock(chp); 2071 drvp->drv_openings = 1; 2072 if (params.atap_sata_caps & SATA_NATIVE_CMDQ) { 2073 if (atac->atac_cap & ATAC_CAP_NCQ) 2074 drvp->drive_flags |= ATA_DRIVE_NCQ; 2075 drvp->drv_openings = 2076 (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1; 2077 aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings); 2078 sep = ","; 2079 2080 if (params.atap_sata_caps & SATA_NCQ_PRIO) { 2081 drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO; 2082 aprint_verbose(" w/PRIO"); 2083 } 2084 } 2085 ata_channel_unlock(chp); 2086 2087 if (*sep != '\0') 2088 aprint_verbose("\n"); 2089 2090 #if NATA_UDMA 2091 if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) { 2092 /* don't care about UDMA modes */ 2093 return; 2094 } 2095 if (cf_flags & ATA_CONFIG_UDMA_SET) { 2096 ata_channel_lock(chp); 2097 if ((cf_flags & ATA_CONFIG_UDMA_MODES) == 2098 ATA_CONFIG_UDMA_DISABLE) { 2099 drvp->drive_flags &= ~ATA_DRIVE_UDMA; 2100 } else { 2101 drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >> 2102 ATA_CONFIG_UDMA_OFF; 2103 drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE; 2104 } 2105 ata_channel_unlock(chp); 2106 } 2107 #endif /* NATA_UDMA */ 2108 #endif /* NATA_DMA */ 2109 } 2110 2111 /* management of the /dev/atabus* devices */ 2112 int 2113 atabusopen(dev_t dev, int flag, int fmt, struct lwp *l) 2114 { 2115 struct atabus_softc *sc; 2116 int error; 2117 2118 sc = device_lookup_private(&atabus_cd, minor(dev)); 2119 if (sc == NULL) 2120 return (ENXIO); 2121 2122 if (sc->sc_flags & ATABUSCF_OPEN) 2123 return (EBUSY); 2124 2125 if ((error = ata_addref(sc->sc_chan)) != 0) 2126 return (error); 2127 2128 sc->sc_flags |= ATABUSCF_OPEN; 2129 2130 return (0); 2131 } 2132 2133 2134 int 2135 atabusclose(dev_t dev, int flag, int fmt, struct lwp *l) 2136 { 2137 struct atabus_softc *sc = 2138 device_lookup_private(&atabus_cd, minor(dev)); 2139 2140 ata_delref(sc->sc_chan); 2141 2142 sc->sc_flags &= ~ATABUSCF_OPEN; 2143 2144 return (0); 2145 } 2146 2147 int 2148 atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l) 2149 { 2150 struct atabus_softc *sc = 2151 device_lookup_private(&atabus_cd, minor(dev)); 2152 struct ata_channel *chp = sc->sc_chan; 2153 int min_drive, max_drive, drive; 2154 int error; 2155 2156 /* 2157 * Enforce write permission for ioctls that change the 2158 * state of the bus. Host adapter specific ioctls must 2159 * be checked by the adapter driver. 2160 */ 2161 switch (cmd) { 2162 case ATABUSIOSCAN: 2163 case ATABUSIODETACH: 2164 case ATABUSIORESET: 2165 if ((flag & FWRITE) == 0) 2166 return (EBADF); 2167 } 2168 2169 switch (cmd) { 2170 case ATABUSIORESET: 2171 ata_channel_lock(chp); 2172 ata_thread_run(sc->sc_chan, AT_WAIT | AT_POLL, 2173 ATACH_TH_RESET, ATACH_NODRIVE); 2174 ata_channel_unlock(chp); 2175 return 0; 2176 case ATABUSIOSCAN: 2177 { 2178 #if 0 2179 struct atabusioscan_args *a= 2180 (struct atabusioscan_args *)addr; 2181 #endif 2182 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2183 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2184 return (EOPNOTSUPP); 2185 return (EOPNOTSUPP); 2186 } 2187 case ATABUSIODETACH: 2188 { 2189 struct atabusiodetach_args *a= 2190 (struct atabusiodetach_args *)addr; 2191 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2192 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2193 return (EOPNOTSUPP); 2194 switch (a->at_dev) { 2195 case -1: 2196 min_drive = 0; 2197 max_drive = 1; 2198 break; 2199 case 0: 2200 case 1: 2201 min_drive = max_drive = a->at_dev; 2202 break; 2203 default: 2204 return (EINVAL); 2205 } 2206 for (drive = min_drive; drive <= max_drive; drive++) { 2207 if (chp->ch_drive[drive].drv_softc != NULL) { 2208 error = config_detach( 2209 chp->ch_drive[drive].drv_softc, 0); 2210 if (error) 2211 return (error); 2212 KASSERT(chp->ch_drive[drive].drv_softc == NULL); 2213 } 2214 } 2215 return 0; 2216 } 2217 default: 2218 return ENOTTY; 2219 } 2220 } 2221 2222 static bool 2223 atabus_suspend(device_t dv, const pmf_qual_t *qual) 2224 { 2225 struct atabus_softc *sc = device_private(dv); 2226 struct ata_channel *chp = sc->sc_chan; 2227 2228 ata_channel_idle(chp); 2229 2230 return true; 2231 } 2232 2233 static bool 2234 atabus_resume(device_t dv, const pmf_qual_t *qual) 2235 { 2236 struct atabus_softc *sc = device_private(dv); 2237 struct ata_channel *chp = sc->sc_chan; 2238 2239 /* 2240 * XXX joerg: with wdc, the first channel unfreezes the controller. 2241 * Move this the reset and queue idling into wdc. 2242 */ 2243 ata_channel_lock(chp); 2244 if (chp->ch_queue->queue_freeze == 0) { 2245 ata_channel_unlock(chp); 2246 goto out; 2247 } 2248 2249 /* unfreeze the queue and reset drives */ 2250 ata_channel_thaw_locked(chp); 2251 2252 /* reset channel only if there are drives attached */ 2253 if (chp->ch_ndrives > 0) 2254 ata_thread_run(chp, AT_WAIT, ATACH_TH_RESET, ATACH_NODRIVE); 2255 2256 ata_channel_unlock(chp); 2257 2258 out: 2259 return true; 2260 } 2261 2262 static int 2263 atabus_rescan(device_t self, const char *ifattr, const int *locators) 2264 { 2265 struct atabus_softc *sc = device_private(self); 2266 struct ata_channel *chp = sc->sc_chan; 2267 struct atabus_initq *initq; 2268 int i; 2269 2270 /* 2271 * we can rescan a port multiplier atabus, even if some devices are 2272 * still attached 2273 */ 2274 if (chp->ch_satapmp_nports == 0) { 2275 if (chp->atapibus != NULL) { 2276 return EBUSY; 2277 } 2278 2279 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 2280 for (i = 0; i < chp->ch_ndrives; i++) { 2281 if (chp->ch_drive[i].drv_softc != NULL) { 2282 return EBUSY; 2283 } 2284 } 2285 } 2286 2287 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 2288 initq->atabus_sc = sc; 2289 mutex_enter(&atabus_qlock); 2290 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 2291 mutex_exit(&atabus_qlock); 2292 config_pending_incr(sc->sc_dev); 2293 2294 ata_channel_lock(chp); 2295 chp->ch_flags |= ATACH_TH_RESCAN; 2296 cv_signal(&chp->ch_thr_idle); 2297 ata_channel_unlock(chp); 2298 2299 return 0; 2300 } 2301 2302 void 2303 ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags) 2304 { 2305 KASSERT(mutex_owned(&chp->ch_lock)); 2306 2307 if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) { 2308 /* 2309 * can't use kpause(), we may be in interrupt context 2310 * or taking a crash dump 2311 */ 2312 delay(ms * 1000); 2313 } else { 2314 int pause = mstohz(ms); 2315 2316 kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock); 2317 } 2318 } 2319 2320 void 2321 atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count, 2322 uint16_t *features, uint8_t *device) 2323 { 2324 if ((xfer->c_flags & C_NCQ) == 0) { 2325 /* FUA handling for non-NCQ drives */ 2326 if (xfer->c_bio.flags & ATA_FUA 2327 && *cmd == WDCC_WRITEDMA_EXT) 2328 *cmd = WDCC_WRITEDMA_FUA_EXT; 2329 2330 return; 2331 } 2332 2333 *cmd = (xfer->c_bio.flags & ATA_READ) ? 2334 WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED; 2335 2336 /* for FPDMA the block count is in features */ 2337 *features = *count; 2338 2339 /* NCQ tag */ 2340 *count = (xfer->c_slot << 3); 2341 2342 if (xfer->c_bio.flags & ATA_PRIO_HIGH) 2343 *count |= WDSC_PRIO_HIGH; 2344 2345 /* other device flags */ 2346 if (xfer->c_bio.flags & ATA_FUA) 2347 *device |= WDSD_FUA; 2348 } 2349 2350 void 2351 ata_wait_cmd(struct ata_channel *chp, struct ata_xfer *xfer) 2352 { 2353 struct ata_queue *chq = chp->ch_queue; 2354 struct ata_command *ata_c = &xfer->c_ata_c; 2355 2356 ata_channel_lock(chp); 2357 2358 while ((ata_c->flags & AT_DONE) == 0) 2359 cv_wait(&chq->c_cmd_finish, &chp->ch_lock); 2360 2361 ata_channel_unlock(chp); 2362 2363 KASSERT((ata_c->flags & AT_DONE) != 0); 2364 } 2365
Indexes created Fri Sep 26 08:10:20 GMT 2025