ata.c revision 1.160
1 /* $NetBSD: ata.c,v 1.160 2020/10/03 22:32:50 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.160 2020/10/03 22:32:50 riastradh Exp $"); 29 30 #include "opt_ata.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/device.h> 36 #include <sys/conf.h> 37 #include <sys/fcntl.h> 38 #include <sys/proc.h> 39 #include <sys/kthread.h> 40 #include <sys/errno.h> 41 #include <sys/ataio.h> 42 #include <sys/kmem.h> 43 #include <sys/intr.h> 44 #include <sys/bus.h> 45 #include <sys/once.h> 46 #include <sys/bitops.h> 47 #include <sys/cpu.h> 48 49 #define ATABUS_PRIVATE 50 51 #include <dev/ata/ataconf.h> 52 #include <dev/ata/atareg.h> 53 #include <dev/ata/atavar.h> 54 #include <dev/ic/wdcvar.h> /* for PIOBM */ 55 56 #include "ioconf.h" 57 #include "locators.h" 58 59 #include "atapibus.h" 60 #include "ataraid.h" 61 #include "sata_pmp.h" 62 63 #if NATARAID > 0 64 #include <dev/ata/ata_raidvar.h> 65 #endif 66 #if NSATA_PMP > 0 67 #include <dev/ata/satapmpvar.h> 68 #endif 69 #include <dev/ata/satapmpreg.h> 70 71 #define DEBUG_FUNCS 0x08 72 #define DEBUG_PROBE 0x10 73 #define DEBUG_DETACH 0x20 74 #define DEBUG_XFERS 0x40 75 #ifdef ATADEBUG 76 #ifndef ATADEBUG_MASK 77 #define ATADEBUG_MASK 0 78 #endif 79 int atadebug_mask = ATADEBUG_MASK; 80 #define ATADEBUG_PRINT(args, level) \ 81 if (atadebug_mask & (level)) \ 82 printf args 83 #else 84 #define ATADEBUG_PRINT(args, level) 85 #endif 86 87 #if defined(ATA_DOWNGRADE_MODE) && NATA_DMA 88 static int ata_downgrade_mode(struct ata_drive_datas *, int); 89 #endif 90 91 static ONCE_DECL(ata_init_ctrl); 92 static struct pool ata_xfer_pool; 93 94 /* 95 * A queue of atabus instances, used to ensure the same bus probe order 96 * for a given hardware configuration at each boot. Kthread probing 97 * devices on a atabus. Only one probing at once. 98 */ 99 static TAILQ_HEAD(, atabus_initq) atabus_initq_head; 100 static kmutex_t atabus_qlock; 101 static kcondvar_t atabus_qcv; 102 static lwp_t * atabus_cfg_lwp; 103 104 /***************************************************************************** 105 * ATA bus layer. 106 * 107 * ATA controllers attach an atabus instance, which handles probing the bus 108 * for drives, etc. 109 *****************************************************************************/ 110 111 dev_type_open(atabusopen); 112 dev_type_close(atabusclose); 113 dev_type_ioctl(atabusioctl); 114 115 const struct cdevsw atabus_cdevsw = { 116 .d_open = atabusopen, 117 .d_close = atabusclose, 118 .d_read = noread, 119 .d_write = nowrite, 120 .d_ioctl = atabusioctl, 121 .d_stop = nostop, 122 .d_tty = notty, 123 .d_poll = nopoll, 124 .d_mmap = nommap, 125 .d_kqfilter = nokqfilter, 126 .d_discard = nodiscard, 127 .d_flag = D_OTHER 128 }; 129 130 static void atabus_childdetached(device_t, device_t); 131 static int atabus_rescan(device_t, const char *, const int *); 132 static bool atabus_resume(device_t, const pmf_qual_t *); 133 static bool atabus_suspend(device_t, const pmf_qual_t *); 134 static void atabusconfig_thread(void *); 135 136 static void ata_channel_idle(struct ata_channel *); 137 static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *); 138 static void ata_channel_freeze_locked(struct ata_channel *); 139 static void ata_thread_wake_locked(struct ata_channel *); 140 141 /* 142 * atabus_init: 143 * 144 * Initialize ATA subsystem structures. 145 */ 146 static int 147 atabus_init(void) 148 { 149 150 pool_init(&ata_xfer_pool, sizeof(struct ata_xfer), 0, 0, 0, 151 "ataspl", NULL, IPL_BIO); 152 TAILQ_INIT(&atabus_initq_head); 153 mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE); 154 cv_init(&atabus_qcv, "atainitq"); 155 return 0; 156 } 157 158 /* 159 * atabusprint: 160 * 161 * Autoconfiguration print routine used by ATA controllers when 162 * attaching an atabus instance. 163 */ 164 int 165 atabusprint(void *aux, const char *pnp) 166 { 167 struct ata_channel *chan = aux; 168 169 if (pnp) 170 aprint_normal("atabus at %s", pnp); 171 aprint_normal(" channel %d", chan->ch_channel); 172 173 return (UNCONF); 174 } 175 176 /* 177 * ataprint: 178 * 179 * Autoconfiguration print routine. 180 */ 181 int 182 ataprint(void *aux, const char *pnp) 183 { 184 struct ata_device *adev = aux; 185 186 if (pnp) 187 aprint_normal("wd at %s", pnp); 188 aprint_normal(" drive %d", adev->adev_drv_data->drive); 189 190 return (UNCONF); 191 } 192 193 /* 194 * ata_channel_attach: 195 * 196 * Common parts of attaching an atabus to an ATA controller channel. 197 */ 198 void 199 ata_channel_attach(struct ata_channel *chp) 200 { 201 if (chp->ch_flags & ATACH_DISABLED) 202 return; 203 204 ata_channel_init(chp); 205 206 KASSERT(chp->ch_queue != NULL); 207 208 chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp, 209 atabusprint); 210 } 211 212 /* 213 * ata_channel_detach: 214 * 215 * Common parts of detaching an atabus to an ATA controller channel. 216 */ 217 void 218 ata_channel_detach(struct ata_channel *chp) 219 { 220 if (chp->ch_flags & ATACH_DISABLED) 221 return; 222 223 ata_channel_destroy(chp); 224 225 chp->ch_flags |= ATACH_DETACHED; 226 } 227 228 static void 229 atabusconfig(struct atabus_softc *atabus_sc) 230 { 231 struct ata_channel *chp = atabus_sc->sc_chan; 232 struct atac_softc *atac = chp->ch_atac; 233 struct atabus_initq *atabus_initq = NULL; 234 int i, error; 235 236 /* we are in the atabus's thread context */ 237 238 /* 239 * Probe for the drives attached to controller, unless a PMP 240 * is already known 241 */ 242 /* XXX for SATA devices we will power up all drives at once */ 243 if (chp->ch_satapmp_nports == 0) 244 (*atac->atac_probe)(chp); 245 246 if (chp->ch_ndrives >= 2) { 247 ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n", 248 chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type), 249 DEBUG_PROBE); 250 } 251 252 /* Make sure the devices probe in atabus order to avoid jitter. */ 253 mutex_enter(&atabus_qlock); 254 for (;;) { 255 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 256 if (atabus_initq->atabus_sc == atabus_sc) 257 break; 258 cv_wait(&atabus_qcv, &atabus_qlock); 259 } 260 mutex_exit(&atabus_qlock); 261 262 ata_channel_lock(chp); 263 264 KASSERT(ata_is_thread_run(chp)); 265 266 /* If no drives, abort here */ 267 if (chp->ch_drive == NULL) 268 goto out; 269 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 270 for (i = 0; i < chp->ch_ndrives; i++) 271 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) 272 break; 273 if (i == chp->ch_ndrives) 274 goto out; 275 276 /* Shortcut in case we've been shutdown */ 277 if (chp->ch_flags & ATACH_SHUTDOWN) 278 goto out; 279 280 ata_channel_unlock(chp); 281 282 if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread, 283 atabus_sc, &atabus_cfg_lwp, 284 "%scnf", device_xname(atac->atac_dev))) != 0) 285 aprint_error_dev(atac->atac_dev, 286 "unable to create config thread: error %d\n", error); 287 return; 288 289 out: 290 ata_channel_unlock(chp); 291 292 mutex_enter(&atabus_qlock); 293 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 294 cv_broadcast(&atabus_qcv); 295 mutex_exit(&atabus_qlock); 296 297 kmem_free(atabus_initq, sizeof(*atabus_initq)); 298 299 ata_delref(chp); 300 301 config_pending_decr(atabus_sc->sc_dev); 302 } 303 304 /* 305 * atabus_configthread: finish attach of atabus's childrens, in a separate 306 * kernel thread. 307 */ 308 static void 309 atabusconfig_thread(void *arg) 310 { 311 struct atabus_softc *atabus_sc = arg; 312 struct ata_channel *chp = atabus_sc->sc_chan; 313 struct atac_softc *atac = chp->ch_atac; 314 struct atabus_initq *atabus_initq = NULL; 315 int i, s; 316 317 /* XXX seems wrong */ 318 mutex_enter(&atabus_qlock); 319 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 320 KASSERT(atabus_initq->atabus_sc == atabus_sc); 321 mutex_exit(&atabus_qlock); 322 323 /* 324 * First look for a port multiplier 325 */ 326 if (chp->ch_ndrives == PMP_MAX_DRIVES && 327 chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) { 328 #if NSATA_PMP > 0 329 satapmp_attach(chp); 330 #else 331 aprint_error_dev(atabus_sc->sc_dev, 332 "SATA port multiplier not supported\n"); 333 /* no problems going on, all drives are ATA_DRIVET_NONE */ 334 #endif 335 } 336 337 /* 338 * Attach an ATAPI bus, if needed. 339 */ 340 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 341 for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) { 342 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) { 343 #if NATAPIBUS > 0 344 (*atac->atac_atapibus_attach)(atabus_sc); 345 #else 346 /* 347 * Fake the autoconfig "not configured" message 348 */ 349 aprint_normal("atapibus at %s not configured\n", 350 device_xname(atac->atac_dev)); 351 chp->atapibus = NULL; 352 s = splbio(); 353 for (i = 0; i < chp->ch_ndrives; i++) { 354 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 355 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 356 } 357 splx(s); 358 #endif 359 break; 360 } 361 } 362 363 for (i = 0; i < chp->ch_ndrives; i++) { 364 struct ata_device adev; 365 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA && 366 chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) { 367 continue; 368 } 369 if (chp->ch_drive[i].drv_softc != NULL) 370 continue; 371 memset(&adev, 0, sizeof(struct ata_device)); 372 adev.adev_bustype = atac->atac_bustype_ata; 373 adev.adev_channel = chp->ch_channel; 374 adev.adev_drv_data = &chp->ch_drive[i]; 375 chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev, 376 "ata_hl", &adev, ataprint); 377 if (chp->ch_drive[i].drv_softc != NULL) { 378 ata_probe_caps(&chp->ch_drive[i]); 379 } else { 380 s = splbio(); 381 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 382 splx(s); 383 } 384 } 385 386 /* now that we know the drives, the controller can set its modes */ 387 if (atac->atac_set_modes) { 388 (*atac->atac_set_modes)(chp); 389 ata_print_modes(chp); 390 } 391 #if NATARAID > 0 392 if (atac->atac_cap & ATAC_CAP_RAID) { 393 for (i = 0; i < chp->ch_ndrives; i++) { 394 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) { 395 ata_raid_check_component( 396 chp->ch_drive[i].drv_softc); 397 } 398 } 399 } 400 #endif /* NATARAID > 0 */ 401 402 /* 403 * reset drive_flags for unattached devices, reset state for attached 404 * ones 405 */ 406 s = splbio(); 407 for (i = 0; i < chp->ch_ndrives; i++) { 408 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 409 continue; 410 if (chp->ch_drive[i].drv_softc == NULL) { 411 chp->ch_drive[i].drive_flags = 0; 412 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 413 } else 414 chp->ch_drive[i].state = 0; 415 } 416 splx(s); 417 418 mutex_enter(&atabus_qlock); 419 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 420 cv_broadcast(&atabus_qcv); 421 mutex_exit(&atabus_qlock); 422 423 kmem_free(atabus_initq, sizeof(*atabus_initq)); 424 425 ata_delref(chp); 426 427 config_pending_decr(atabus_sc->sc_dev); 428 kthread_exit(0); 429 } 430 431 /* 432 * atabus_thread: 433 * 434 * Worker thread for the ATA bus. 435 */ 436 static void 437 atabus_thread(void *arg) 438 { 439 struct atabus_softc *sc = arg; 440 struct ata_channel *chp = sc->sc_chan; 441 struct ata_queue *chq = chp->ch_queue; 442 struct ata_xfer *xfer; 443 int i, rv; 444 445 ata_channel_lock(chp); 446 KASSERT(ata_is_thread_run(chp)); 447 448 /* 449 * Probe the drives. Reset type to indicate to controllers 450 * that can re-probe that all drives must be probed.. 451 * 452 * Note: ch_ndrives may be changed during the probe. 453 */ 454 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 455 for (i = 0; i < chp->ch_ndrives; i++) { 456 chp->ch_drive[i].drive_flags = 0; 457 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 458 } 459 ata_channel_unlock(chp); 460 461 atabusconfig(sc); 462 463 ata_channel_lock(chp); 464 for (;;) { 465 if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_TH_DRIVE_RESET 466 | ATACH_TH_RECOVERY | ATACH_SHUTDOWN)) == 0 && 467 (chq->queue_active == 0 || chq->queue_freeze == 0)) { 468 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 469 } 470 if (chp->ch_flags & ATACH_SHUTDOWN) { 471 break; 472 } 473 if (chp->ch_flags & ATACH_TH_RESCAN) { 474 chp->ch_flags &= ~ATACH_TH_RESCAN; 475 ata_channel_unlock(chp); 476 atabusconfig(sc); 477 ata_channel_lock(chp); 478 } 479 if (chp->ch_flags & ATACH_TH_RESET) { 480 /* this will unfreeze the channel */ 481 ata_thread_run(chp, AT_WAIT, 482 ATACH_TH_RESET, ATACH_NODRIVE); 483 } else if (chp->ch_flags & ATACH_TH_DRIVE_RESET) { 484 /* this will unfreeze the channel */ 485 for (i = 0; i < chp->ch_ndrives; i++) { 486 struct ata_drive_datas *drvp; 487 488 drvp = &chp->ch_drive[i]; 489 490 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 491 ata_thread_run(chp, 492 AT_WAIT, ATACH_TH_DRIVE_RESET, i); 493 } 494 } 495 chp->ch_flags &= ~ATACH_TH_DRIVE_RESET; 496 } else if (chp->ch_flags & ATACH_TH_RECOVERY) { 497 /* 498 * This will unfreeze the channel; drops locks during 499 * run, so must wrap in splbio()/splx() to avoid 500 * spurious interrupts. XXX MPSAFE 501 */ 502 int s = splbio(); 503 ata_thread_run(chp, AT_WAIT, ATACH_TH_RECOVERY, 504 chp->recovery_tfd); 505 splx(s); 506 } else if (chq->queue_active > 0 && chq->queue_freeze == 1) { 507 /* 508 * Caller has bumped queue_freeze, decrease it. This 509 * flow shalt never be executed for NCQ commands. 510 */ 511 KASSERT((chp->ch_flags & ATACH_NCQ) == 0); 512 KASSERT(chq->queue_active == 1); 513 514 ata_channel_thaw_locked(chp); 515 xfer = ata_queue_get_active_xfer_locked(chp); 516 517 KASSERT(xfer != NULL); 518 KASSERT((xfer->c_flags & C_POLL) == 0); 519 520 switch ((rv = ata_xfer_start(xfer))) { 521 case ATASTART_STARTED: 522 case ATASTART_POLL: 523 case ATASTART_ABORT: 524 break; 525 case ATASTART_TH: 526 default: 527 panic("%s: ata_xfer_start() unexpected rv %d", 528 __func__, rv); 529 /* NOTREACHED */ 530 } 531 } else if (chq->queue_freeze > 1) 532 panic("%s: queue_freeze", __func__); 533 534 /* Try to run down the queue once channel is unfrozen */ 535 if (chq->queue_freeze == 0) { 536 ata_channel_unlock(chp); 537 atastart(chp); 538 ata_channel_lock(chp); 539 } 540 } 541 chp->ch_thread = NULL; 542 cv_signal(&chp->ch_thr_idle); 543 ata_channel_unlock(chp); 544 kthread_exit(0); 545 } 546 547 bool 548 ata_is_thread_run(struct ata_channel *chp) 549 { 550 KASSERT(mutex_owned(&chp->ch_lock)); 551 552 return (chp->ch_thread == curlwp && !cpu_intr_p()); 553 } 554 555 static void 556 ata_thread_wake_locked(struct ata_channel *chp) 557 { 558 KASSERT(mutex_owned(&chp->ch_lock)); 559 ata_channel_freeze_locked(chp); 560 cv_signal(&chp->ch_thr_idle); 561 } 562 563 /* 564 * atabus_match: 565 * 566 * Autoconfiguration match routine. 567 */ 568 static int 569 atabus_match(device_t parent, cfdata_t cf, void *aux) 570 { 571 struct ata_channel *chp = aux; 572 573 if (chp == NULL) 574 return (0); 575 576 if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel && 577 cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT) 578 return (0); 579 580 return (1); 581 } 582 583 /* 584 * atabus_attach: 585 * 586 * Autoconfiguration attach routine. 587 */ 588 static void 589 atabus_attach(device_t parent, device_t self, void *aux) 590 { 591 struct atabus_softc *sc = device_private(self); 592 struct ata_channel *chp = aux; 593 struct atabus_initq *initq; 594 int error; 595 596 sc->sc_chan = chp; 597 598 aprint_normal("\n"); 599 aprint_naive("\n"); 600 601 sc->sc_dev = self; 602 603 if (ata_addref(chp)) 604 return; 605 606 RUN_ONCE(&ata_init_ctrl, atabus_init); 607 608 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 609 initq->atabus_sc = sc; 610 mutex_enter(&atabus_qlock); 611 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 612 mutex_exit(&atabus_qlock); 613 config_pending_incr(sc->sc_dev); 614 615 /* XXX MPSAFE - no KTHREAD_MPSAFE, so protected by KERNEL_LOCK() */ 616 if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc, 617 &chp->ch_thread, "%s", device_xname(self))) != 0) 618 aprint_error_dev(self, 619 "unable to create kernel thread: error %d\n", error); 620 621 if (!pmf_device_register(self, atabus_suspend, atabus_resume)) 622 aprint_error_dev(self, "couldn't establish power handler\n"); 623 } 624 625 /* 626 * atabus_detach: 627 * 628 * Autoconfiguration detach routine. 629 */ 630 static int 631 atabus_detach(device_t self, int flags) 632 { 633 struct atabus_softc *sc = device_private(self); 634 struct ata_channel *chp = sc->sc_chan; 635 device_t dev = NULL; 636 int i, error = 0; 637 638 /* 639 * Detach atapibus and its children. 640 */ 641 if ((dev = chp->atapibus) != NULL) { 642 ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n", 643 device_xname(self), device_xname(dev)), DEBUG_DETACH); 644 645 error = config_detach(dev, flags); 646 if (error) 647 goto out; 648 KASSERT(chp->atapibus == NULL); 649 } 650 651 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 652 653 /* 654 * Detach our other children. 655 */ 656 for (i = 0; i < chp->ch_ndrives; i++) { 657 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 658 continue; 659 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 660 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 661 if ((dev = chp->ch_drive[i].drv_softc) != NULL) { 662 ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__, 663 __LINE__, device_xname(self), device_xname(dev)), 664 DEBUG_DETACH); 665 error = config_detach(dev, flags); 666 if (error) 667 goto out; 668 KASSERT(chp->ch_drive[i].drv_softc == NULL); 669 KASSERT(chp->ch_drive[i].drive_type == 0); 670 } 671 } 672 673 /* Shutdown the channel. */ 674 ata_channel_lock(chp); 675 chp->ch_flags |= ATACH_SHUTDOWN; 676 while (chp->ch_thread != NULL) { 677 cv_signal(&chp->ch_thr_idle); 678 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 679 } 680 ata_channel_unlock(chp); 681 682 atabus_free_drives(chp); 683 684 out: 685 #ifdef ATADEBUG 686 if (dev != NULL && error != 0) 687 ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__, 688 device_xname(self), error, device_xname(dev)), 689 DEBUG_DETACH); 690 #endif /* ATADEBUG */ 691 692 return (error); 693 } 694 695 void 696 atabus_childdetached(device_t self, device_t child) 697 { 698 bool found = false; 699 struct atabus_softc *sc = device_private(self); 700 struct ata_channel *chp = sc->sc_chan; 701 int i; 702 703 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 704 /* 705 * atapibus detached. 706 */ 707 if (child == chp->atapibus) { 708 chp->atapibus = NULL; 709 found = true; 710 for (i = 0; i < chp->ch_ndrives; i++) { 711 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI) 712 continue; 713 KASSERT(chp->ch_drive[i].drv_softc != NULL); 714 chp->ch_drive[i].drv_softc = NULL; 715 chp->ch_drive[i].drive_flags = 0; 716 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 717 } 718 } 719 720 /* 721 * Detach our other children. 722 */ 723 for (i = 0; i < chp->ch_ndrives; i++) { 724 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 725 continue; 726 if (child == chp->ch_drive[i].drv_softc) { 727 chp->ch_drive[i].drv_softc = NULL; 728 chp->ch_drive[i].drive_flags = 0; 729 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 730 chp->ch_satapmp_nports = 0; 731 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 732 found = true; 733 } 734 } 735 736 if (!found) 737 panic("%s: unknown child %p", device_xname(self), 738 (const void *)child); 739 } 740 741 CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc), 742 atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan, 743 atabus_childdetached, DVF_DETACH_SHUTDOWN); 744 745 /***************************************************************************** 746 * Common ATA bus operations. 747 *****************************************************************************/ 748 749 /* allocate/free the channel's ch_drive[] array */ 750 int 751 atabus_alloc_drives(struct ata_channel *chp, int ndrives) 752 { 753 int i; 754 if (chp->ch_ndrives != ndrives) 755 atabus_free_drives(chp); 756 if (chp->ch_drive == NULL) { 757 void *drv; 758 759 ata_channel_unlock(chp); 760 drv = kmem_zalloc(sizeof(*chp->ch_drive) * ndrives, KM_SLEEP); 761 ata_channel_lock(chp); 762 763 if (chp->ch_drive != NULL) { 764 /* lost the race */ 765 kmem_free(drv, sizeof(*chp->ch_drive) * ndrives); 766 return 0; 767 } 768 chp->ch_drive = drv; 769 } 770 for (i = 0; i < ndrives; i++) { 771 chp->ch_drive[i].chnl_softc = chp; 772 chp->ch_drive[i].drive = i; 773 } 774 chp->ch_ndrives = ndrives; 775 return 0; 776 } 777 778 void 779 atabus_free_drives(struct ata_channel *chp) 780 { 781 #ifdef DIAGNOSTIC 782 int i; 783 int dopanic = 0; 784 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 785 for (i = 0; i < chp->ch_ndrives; i++) { 786 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) { 787 printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n", 788 device_xname(chp->atabus), i, 789 chp->ch_drive[i].drive_type); 790 dopanic = 1; 791 } 792 if (chp->ch_drive[i].drv_softc != NULL) { 793 printf("%s: ch_drive[%d] attached to %s\n", 794 device_xname(chp->atabus), i, 795 device_xname(chp->ch_drive[i].drv_softc)); 796 dopanic = 1; 797 } 798 } 799 if (dopanic) 800 panic("atabus_free_drives"); 801 #endif 802 803 if (chp->ch_drive == NULL) 804 return; 805 kmem_free(chp->ch_drive, 806 sizeof(struct ata_drive_datas) * chp->ch_ndrives); 807 chp->ch_ndrives = 0; 808 chp->ch_drive = NULL; 809 } 810 811 /* Get the disk's parameters */ 812 int 813 ata_get_params(struct ata_drive_datas *drvp, uint8_t flags, 814 struct ataparams *prms) 815 { 816 struct ata_xfer *xfer; 817 struct ata_channel *chp = drvp->chnl_softc; 818 struct atac_softc *atac = chp->ch_atac; 819 char *tb; 820 int i, rv; 821 uint16_t *p; 822 823 ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS); 824 825 xfer = ata_get_xfer(chp, false); 826 if (xfer == NULL) { 827 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 828 DEBUG_FUNCS|DEBUG_PROBE); 829 return CMD_AGAIN; 830 } 831 832 tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP); 833 memset(prms, 0, sizeof(struct ataparams)); 834 835 if (drvp->drive_type == ATA_DRIVET_ATA) { 836 xfer->c_ata_c.r_command = WDCC_IDENTIFY; 837 xfer->c_ata_c.r_st_bmask = WDCS_DRDY; 838 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 839 xfer->c_ata_c.timeout = 3000; /* 3s */ 840 } else if (drvp->drive_type == ATA_DRIVET_ATAPI) { 841 xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE; 842 xfer->c_ata_c.r_st_bmask = 0; 843 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 844 xfer->c_ata_c.timeout = 10000; /* 10s */ 845 } else { 846 ATADEBUG_PRINT(("ata_get_parms: no disks\n"), 847 DEBUG_FUNCS|DEBUG_PROBE); 848 rv = CMD_ERR; 849 goto out; 850 } 851 xfer->c_ata_c.flags = AT_READ | flags; 852 xfer->c_ata_c.data = tb; 853 xfer->c_ata_c.bcount = ATA_BSIZE; 854 (*atac->atac_bustype_ata->ata_exec_command)(drvp, xfer); 855 ata_wait_cmd(chp, xfer); 856 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 857 ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n", 858 xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE); 859 rv = CMD_ERR; 860 goto out; 861 } 862 /* if we didn't read any data something is wrong */ 863 if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) { 864 rv = CMD_ERR; 865 goto out; 866 } 867 868 /* Read in parameter block. */ 869 memcpy(prms, tb, sizeof(struct ataparams)); 870 871 /* 872 * Shuffle string byte order. 873 * ATAPI NEC, Mitsumi and Pioneer drives and 874 * old ATA TDK CompactFlash cards 875 * have different byte order. 876 */ 877 #if BYTE_ORDER == BIG_ENDIAN 878 # define M(n) prms->atap_model[(n) ^ 1] 879 #else 880 # define M(n) prms->atap_model[n] 881 #endif 882 if ( 883 #if BYTE_ORDER == BIG_ENDIAN 884 ! 885 #endif 886 ((drvp->drive_type == ATA_DRIVET_ATAPI) ? 887 ((M(0) == 'N' && M(1) == 'E') || 888 (M(0) == 'F' && M(1) == 'X') || 889 (M(0) == 'P' && M(1) == 'i')) : 890 ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) { 891 rv = CMD_OK; 892 goto out; 893 } 894 #undef M 895 for (i = 0; i < sizeof(prms->atap_model); i += 2) { 896 p = (uint16_t *)(prms->atap_model + i); 897 *p = bswap16(*p); 898 } 899 for (i = 0; i < sizeof(prms->atap_serial); i += 2) { 900 p = (uint16_t *)(prms->atap_serial + i); 901 *p = bswap16(*p); 902 } 903 for (i = 0; i < sizeof(prms->atap_revision); i += 2) { 904 p = (uint16_t *)(prms->atap_revision + i); 905 *p = bswap16(*p); 906 } 907 908 rv = CMD_OK; 909 out: 910 kmem_free(tb, ATA_BSIZE); 911 ata_free_xfer(chp, xfer); 912 return rv; 913 } 914 915 int 916 ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags) 917 { 918 struct ata_xfer *xfer; 919 int rv; 920 struct ata_channel *chp = drvp->chnl_softc; 921 struct atac_softc *atac = chp->ch_atac; 922 923 ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS); 924 925 xfer = ata_get_xfer(chp, false); 926 if (xfer == NULL) { 927 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 928 DEBUG_FUNCS|DEBUG_PROBE); 929 return CMD_AGAIN; 930 } 931 932 xfer->c_ata_c.r_command = SET_FEATURES; 933 xfer->c_ata_c.r_st_bmask = 0; 934 xfer->c_ata_c.r_st_pmask = 0; 935 xfer->c_ata_c.r_features = WDSF_SET_MODE; 936 xfer->c_ata_c.r_count = mode; 937 xfer->c_ata_c.flags = flags; 938 xfer->c_ata_c.timeout = 1000; /* 1s */ 939 (*atac->atac_bustype_ata->ata_exec_command)(drvp, xfer); 940 ata_wait_cmd(chp, xfer); 941 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 942 rv = CMD_ERR; 943 goto out; 944 } 945 946 rv = CMD_OK; 947 948 out: 949 ata_free_xfer(chp, xfer); 950 return rv; 951 } 952 953 #if NATA_DMA 954 void 955 ata_dmaerr(struct ata_drive_datas *drvp, int flags) 956 { 957 ata_channel_lock_owned(drvp->chnl_softc); 958 959 /* 960 * Downgrade decision: if we get NERRS_MAX in NXFER. 961 * We start with n_dmaerrs set to NERRS_MAX-1 so that the 962 * first error within the first NXFER ops will immediatly trigger 963 * a downgrade. 964 * If we got an error and n_xfers is bigger than NXFER reset counters. 965 */ 966 drvp->n_dmaerrs++; 967 if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) { 968 #ifdef ATA_DOWNGRADE_MODE 969 ata_downgrade_mode(drvp, flags); 970 drvp->n_dmaerrs = NERRS_MAX-1; 971 #else 972 static struct timeval last; 973 static const struct timeval serrintvl = { 300, 0 }; 974 975 if (ratecheck(&last, &serrintvl)) { 976 aprint_error_dev(drvp->drv_softc, 977 "excessive DMA errors - %d in last %d transfers\n", 978 drvp->n_dmaerrs, drvp->n_xfers); 979 } 980 #endif 981 drvp->n_xfers = 0; 982 return; 983 } 984 if (drvp->n_xfers > NXFER) { 985 drvp->n_dmaerrs = 1; /* just got an error */ 986 drvp->n_xfers = 1; /* restart counting from this error */ 987 } 988 } 989 #endif /* NATA_DMA */ 990 991 /* 992 * freeze the queue and wait for the controller to be idle. Caller has to 993 * unfreeze/restart the queue 994 */ 995 static void 996 ata_channel_idle(struct ata_channel *chp) 997 { 998 ata_channel_lock(chp); 999 ata_channel_freeze_locked(chp); 1000 while (chp->ch_queue->queue_active > 0) { 1001 chp->ch_queue->queue_flags |= QF_IDLE_WAIT; 1002 cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1); 1003 } 1004 ata_channel_unlock(chp); 1005 } 1006 1007 /* 1008 * Add a command to the queue and start controller. 1009 * 1010 * MUST BE CALLED AT splbio()! 1011 */ 1012 void 1013 ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1014 { 1015 1016 ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer, 1017 chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1018 1019 /* complete xfer setup */ 1020 xfer->c_chp = chp; 1021 1022 ata_channel_lock(chp); 1023 1024 /* 1025 * Standard commands are added to the end of command list, but 1026 * recovery commands must be run immediatelly. 1027 */ 1028 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1029 SIMPLEQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer, 1030 c_xferchain); 1031 else 1032 SIMPLEQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer, 1033 c_xferchain); 1034 1035 /* 1036 * if polling and can sleep, wait for the xfer to be at head of queue 1037 */ 1038 if ((xfer->c_flags & (C_POLL | C_WAIT)) == (C_POLL | C_WAIT)) { 1039 while (chp->ch_queue->queue_active > 0 || 1040 SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) { 1041 xfer->c_flags |= C_WAITACT; 1042 cv_wait(&chp->ch_queue->c_active, &chp->ch_lock); 1043 xfer->c_flags &= ~C_WAITACT; 1044 } 1045 1046 /* 1047 * Free xfer now if it there was attempt to free it 1048 * while we were waiting. 1049 */ 1050 if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) { 1051 ata_channel_unlock(chp); 1052 1053 ata_free_xfer(chp, xfer); 1054 return; 1055 } 1056 } 1057 1058 ata_channel_unlock(chp); 1059 1060 ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n", 1061 chp->ch_flags), DEBUG_XFERS); 1062 atastart(chp); 1063 } 1064 1065 /* 1066 * Start I/O on a controller, for the given channel. 1067 * The first xfer may be not for our channel if the channel queues 1068 * are shared. 1069 * 1070 * MUST BE CALLED AT splbio()! 1071 * 1072 * XXX FIS-based switching with PMP 1073 * Currently atastart() never schedules concurrent NCQ transfers to more than 1074 * one drive, even when channel has several SATA drives attached via PMP. 1075 * To support concurrent transfers to different drives with PMP, it would be 1076 * necessary to implement FIS-based switching support in controller driver, 1077 * and then adjust error handling and recovery to stop assuming at most 1078 * one active drive. 1079 */ 1080 void 1081 atastart(struct ata_channel *chp) 1082 { 1083 struct atac_softc *atac = chp->ch_atac; 1084 struct ata_queue *chq = chp->ch_queue; 1085 struct ata_xfer *xfer, *axfer; 1086 bool skipq; 1087 1088 #ifdef ATA_DEBUG 1089 int spl1, spl2; 1090 1091 spl1 = splbio(); 1092 spl2 = splbio(); 1093 if (spl2 != spl1) { 1094 printf("atastart: not at splbio()\n"); 1095 panic("atastart"); 1096 } 1097 splx(spl2); 1098 splx(spl1); 1099 #endif /* ATA_DEBUG */ 1100 1101 ata_channel_lock(chp); 1102 1103 again: 1104 /* is there a xfer ? */ 1105 if ((xfer = SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL) { 1106 ATADEBUG_PRINT(("%s(chp=%p): channel %d queue_xfer is empty\n", 1107 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1108 goto out; 1109 } 1110 1111 /* 1112 * if someone is waiting for the command to be active, wake it up 1113 * and let it process the command 1114 */ 1115 if (__predict_false(xfer->c_flags & C_WAITACT)) { 1116 ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d " 1117 "wait active\n", xfer, chp->ch_channel, xfer->c_drive), 1118 DEBUG_XFERS); 1119 cv_broadcast(&chp->ch_queue->c_active); 1120 goto out; 1121 } 1122 1123 skipq = ISSET(xfer->c_flags, C_SKIP_QUEUE); 1124 1125 /* is the queue frozen? */ 1126 if (__predict_false(!skipq && chq->queue_freeze > 0)) { 1127 if (chq->queue_flags & QF_IDLE_WAIT) { 1128 chq->queue_flags &= ~QF_IDLE_WAIT; 1129 cv_signal(&chp->ch_queue->queue_idle); 1130 } 1131 ATADEBUG_PRINT(("%s(chp=%p): channel %d drive %d " 1132 "queue frozen: %d\n", 1133 __func__, chp, chp->ch_channel, xfer->c_drive, 1134 chq->queue_freeze), 1135 DEBUG_XFERS); 1136 goto out; 1137 } 1138 1139 /* all xfers on same queue must belong to the same channel */ 1140 KASSERT(xfer->c_chp == chp); 1141 1142 /* 1143 * Can only take the command if there are no current active 1144 * commands, or if the command is NCQ and the active commands are also 1145 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based 1146 * switching, can only send commands to single drive. 1147 * Need only check first xfer. 1148 * XXX FIS-based switching - revisit 1149 */ 1150 if (!skipq && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) { 1151 if (!ISSET(xfer->c_flags, C_NCQ) || 1152 !ISSET(axfer->c_flags, C_NCQ) || 1153 xfer->c_drive != axfer->c_drive) 1154 goto out; 1155 } 1156 1157 struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive]; 1158 1159 /* 1160 * Are we on limit of active xfers ? If the queue has more 1161 * than 1 openings, we keep one slot reserved for recovery or dump. 1162 */ 1163 KASSERT(chq->queue_active <= chq->queue_openings); 1164 const uint8_t chq_openings = (!skipq && chq->queue_openings > 1) 1165 ? (chq->queue_openings - 1) : chq->queue_openings; 1166 const uint8_t drv_openings = ISSET(xfer->c_flags, C_NCQ) 1167 ? drvp->drv_openings : ATA_MAX_OPENINGS; 1168 if (chq->queue_active >= MIN(chq_openings, drv_openings)) { 1169 if (skipq) { 1170 panic("%s: channel %d busy, xfer not possible", 1171 __func__, chp->ch_channel); 1172 } 1173 1174 ATADEBUG_PRINT(("%s(chp=%p): channel %d completely busy\n", 1175 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1176 goto out; 1177 } 1178 1179 /* Slot allocation can fail if drv_openings < ch_openings */ 1180 if (!ata_queue_alloc_slot(chp, &xfer->c_slot, drv_openings)) 1181 goto out; 1182 1183 if (__predict_false(atac->atac_claim_hw)) { 1184 if (!atac->atac_claim_hw(chp, 0)) { 1185 ata_queue_free_slot(chp, xfer->c_slot); 1186 goto out; 1187 } 1188 } 1189 1190 /* Now committed to start the xfer */ 1191 1192 ATADEBUG_PRINT(("%s(chp=%p): xfer %p channel %d drive %d\n", 1193 __func__, chp, xfer, chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1194 if (drvp->drive_flags & ATA_DRIVE_RESET) { 1195 drvp->drive_flags &= ~ATA_DRIVE_RESET; 1196 drvp->state = 0; 1197 } 1198 1199 if (ISSET(xfer->c_flags, C_NCQ)) 1200 SET(chp->ch_flags, ATACH_NCQ); 1201 else 1202 CLR(chp->ch_flags, ATACH_NCQ); 1203 1204 SIMPLEQ_REMOVE_HEAD(&chq->queue_xfer, c_xferchain); 1205 1206 ata_activate_xfer_locked(chp, xfer); 1207 1208 if (atac->atac_cap & ATAC_CAP_NOIRQ) 1209 KASSERT(xfer->c_flags & C_POLL); 1210 1211 switch (ata_xfer_start(xfer)) { 1212 case ATASTART_TH: 1213 case ATASTART_ABORT: 1214 /* don't start any further commands in this case */ 1215 goto out; 1216 default: 1217 /* nothing to do */ 1218 break; 1219 } 1220 1221 /* Queue more commands if possible, but not during recovery or dump */ 1222 if (!skipq && chq->queue_active < chq->queue_openings) 1223 goto again; 1224 1225 out: 1226 ata_channel_unlock(chp); 1227 } 1228 1229 int 1230 ata_xfer_start(struct ata_xfer *xfer) 1231 { 1232 struct ata_channel *chp = xfer->c_chp; 1233 int rv; 1234 1235 KASSERT(mutex_owned(&chp->ch_lock)); 1236 1237 rv = xfer->ops->c_start(chp, xfer); 1238 switch (rv) { 1239 case ATASTART_STARTED: 1240 /* nothing to do */ 1241 break; 1242 case ATASTART_TH: 1243 /* postpone xfer to thread */ 1244 ata_thread_wake_locked(chp); 1245 break; 1246 case ATASTART_POLL: 1247 /* can happen even in thread context for some ATAPI devices */ 1248 ata_channel_unlock(chp); 1249 KASSERT(xfer->ops != NULL && xfer->ops->c_poll != NULL); 1250 xfer->ops->c_poll(chp, xfer); 1251 ata_channel_lock(chp); 1252 break; 1253 case ATASTART_ABORT: 1254 ata_channel_unlock(chp); 1255 KASSERT(xfer->ops != NULL && xfer->ops->c_abort != NULL); 1256 xfer->ops->c_abort(chp, xfer); 1257 ata_channel_lock(chp); 1258 break; 1259 } 1260 1261 return rv; 1262 } 1263 1264 static void 1265 ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer) 1266 { 1267 struct ata_queue * const chq = chp->ch_queue; 1268 1269 KASSERT(mutex_owned(&chp->ch_lock)); 1270 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0); 1271 1272 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1273 TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain); 1274 else { 1275 /* 1276 * Must go to head, so that ata_queue_get_active_xfer() 1277 * returns the recovery command, and not some other 1278 * random active transfer. 1279 */ 1280 TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain); 1281 } 1282 chq->active_xfers_used |= __BIT(xfer->c_slot); 1283 chq->queue_active++; 1284 } 1285 1286 /* 1287 * Does it's own locking, does not require splbio(). 1288 * flags - whether to block waiting for free xfer 1289 */ 1290 struct ata_xfer * 1291 ata_get_xfer(struct ata_channel *chp, bool waitok) 1292 { 1293 return pool_get(&ata_xfer_pool, 1294 PR_ZERO | (waitok ? PR_WAITOK : PR_NOWAIT)); 1295 } 1296 1297 /* 1298 * ata_deactivate_xfer() must be always called prior to ata_free_xfer() 1299 */ 1300 void 1301 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1302 { 1303 struct ata_queue *chq = chp->ch_queue; 1304 1305 ata_channel_lock(chp); 1306 1307 if (__predict_false(xfer->c_flags & (C_WAITACT|C_WAITTIMO))) { 1308 /* Someone is waiting for this xfer, so we can't free now */ 1309 xfer->c_flags |= C_FREE; 1310 cv_broadcast(&chq->c_active); 1311 ata_channel_unlock(chp); 1312 return; 1313 } 1314 1315 /* XXX move PIOBM and free_gw to deactivate? */ 1316 #if NATA_PIOBM /* XXX wdc dependent code */ 1317 if (__predict_false(xfer->c_flags & C_PIOBM)) { 1318 struct wdc_softc *wdc = CHAN_TO_WDC(chp); 1319 1320 /* finish the busmastering PIO */ 1321 (*wdc->piobm_done)(wdc->dma_arg, 1322 chp->ch_channel, xfer->c_drive); 1323 chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT); 1324 } 1325 #endif 1326 1327 if (__predict_false(chp->ch_atac->atac_free_hw)) 1328 chp->ch_atac->atac_free_hw(chp); 1329 1330 ata_channel_unlock(chp); 1331 1332 if (__predict_true(!ISSET(xfer->c_flags, C_PRIVATE_ALLOC))) 1333 pool_put(&ata_xfer_pool, xfer); 1334 } 1335 1336 void 1337 ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1338 { 1339 struct ata_queue * const chq = chp->ch_queue; 1340 1341 ata_channel_lock(chp); 1342 1343 KASSERT(chq->queue_active > 0); 1344 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0); 1345 1346 /* Stop only when this is last active xfer */ 1347 if (chq->queue_active == 1) 1348 callout_stop(&chp->c_timo_callout); 1349 1350 if (callout_invoking(&chp->c_timo_callout)) 1351 xfer->c_flags |= C_WAITTIMO; 1352 1353 TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain); 1354 chq->active_xfers_used &= ~__BIT(xfer->c_slot); 1355 chq->queue_active--; 1356 1357 ata_queue_free_slot(chp, xfer->c_slot); 1358 1359 if (xfer->c_flags & C_WAIT) 1360 cv_broadcast(&chq->c_cmd_finish); 1361 1362 ata_channel_unlock(chp); 1363 } 1364 1365 /* 1366 * Called in c_intr hook. Must be called before before any deactivations 1367 * are done - if there is drain pending, it calls c_kill_xfer hook which 1368 * deactivates the xfer. 1369 * Calls c_kill_xfer with channel lock free. 1370 * Returns true if caller should just exit without further processing. 1371 * Caller must not further access any part of xfer or any related controller 1372 * structures in that case, it should just return. 1373 */ 1374 bool 1375 ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer) 1376 { 1377 int drive = xfer->c_drive; 1378 bool draining = false; 1379 1380 ata_channel_lock(chp); 1381 1382 if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) { 1383 ata_channel_unlock(chp); 1384 1385 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE); 1386 1387 ata_channel_lock(chp); 1388 chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN; 1389 cv_signal(&chp->ch_queue->queue_drain); 1390 draining = true; 1391 } 1392 1393 ata_channel_unlock(chp); 1394 1395 return draining; 1396 } 1397 1398 /* 1399 * Check for race of normal transfer handling vs. timeout. 1400 */ 1401 bool 1402 ata_timo_xfer_check(struct ata_xfer *xfer) 1403 { 1404 struct ata_channel *chp = xfer->c_chp; 1405 struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive]; 1406 1407 ata_channel_lock(chp); 1408 1409 if (xfer->c_flags & C_WAITTIMO) { 1410 xfer->c_flags &= ~C_WAITTIMO; 1411 1412 /* Handle race vs. ata_free_xfer() */ 1413 if (xfer->c_flags & C_FREE) { 1414 xfer->c_flags &= ~C_FREE; 1415 ata_channel_unlock(chp); 1416 1417 device_printf(drvp->drv_softc, 1418 "xfer %"PRIxPTR" freed while invoking timeout\n", 1419 (intptr_t)xfer & PAGE_MASK); 1420 1421 ata_free_xfer(chp, xfer); 1422 return true; 1423 } 1424 1425 /* Race vs. callout_stop() in ata_deactivate_xfer() */ 1426 ata_channel_unlock(chp); 1427 1428 device_printf(drvp->drv_softc, 1429 "xfer %"PRIxPTR" deactivated while invoking timeout\n", 1430 (intptr_t)xfer & PAGE_MASK); 1431 return true; 1432 } 1433 1434 ata_channel_unlock(chp); 1435 1436 /* No race, proceed with timeout handling */ 1437 return false; 1438 } 1439 1440 /* 1441 * Kill off all active xfers for a ata_channel. 1442 * 1443 * Must be called with channel lock held. 1444 */ 1445 void 1446 ata_kill_active(struct ata_channel *chp, int reason, int flags) 1447 { 1448 struct ata_queue * const chq = chp->ch_queue; 1449 struct ata_xfer *xfer, *xfernext; 1450 1451 KASSERT(mutex_owned(&chp->ch_lock)); 1452 1453 TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) { 1454 ata_channel_unlock(chp); 1455 xfer->ops->c_kill_xfer(xfer->c_chp, xfer, reason); 1456 ata_channel_lock(chp); 1457 } 1458 } 1459 1460 /* 1461 * Kill off all pending xfers for a drive. 1462 */ 1463 void 1464 ata_kill_pending(struct ata_drive_datas *drvp) 1465 { 1466 struct ata_channel * const chp = drvp->chnl_softc; 1467 struct ata_queue * const chq = chp->ch_queue; 1468 struct ata_xfer *xfer; 1469 1470 ata_channel_lock(chp); 1471 1472 /* Kill all pending transfers */ 1473 while ((xfer = SIMPLEQ_FIRST(&chq->queue_xfer))) { 1474 KASSERT(xfer->c_chp == chp); 1475 1476 if (xfer->c_drive != drvp->drive) 1477 continue; 1478 1479 SIMPLEQ_REMOVE_HEAD(&chp->ch_queue->queue_xfer, c_xferchain); 1480 1481 /* 1482 * Keep the lock, so that we get deadlock (and 'locking against 1483 * myself' with LOCKDEBUG), instead of silent 1484 * data corruption, if the hook tries to call back into 1485 * middle layer for inactive xfer. 1486 */ 1487 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE_INACTIVE); 1488 } 1489 1490 /* Wait until all active transfers on the drive finish */ 1491 while (chq->queue_active > 0) { 1492 bool drv_active = false; 1493 1494 TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) { 1495 KASSERT(xfer->c_chp == chp); 1496 1497 if (xfer->c_drive == drvp->drive) { 1498 drv_active = true; 1499 break; 1500 } 1501 } 1502 1503 if (!drv_active) { 1504 /* all finished */ 1505 break; 1506 } 1507 1508 drvp->drive_flags |= ATA_DRIVE_WAITDRAIN; 1509 cv_wait(&chq->queue_drain, &chp->ch_lock); 1510 } 1511 1512 ata_channel_unlock(chp); 1513 } 1514 1515 static void 1516 ata_channel_freeze_locked(struct ata_channel *chp) 1517 { 1518 chp->ch_queue->queue_freeze++; 1519 1520 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1521 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1522 } 1523 1524 void 1525 ata_channel_freeze(struct ata_channel *chp) 1526 { 1527 ata_channel_lock(chp); 1528 ata_channel_freeze_locked(chp); 1529 ata_channel_unlock(chp); 1530 } 1531 1532 void 1533 ata_channel_thaw_locked(struct ata_channel *chp) 1534 { 1535 KASSERT(mutex_owned(&chp->ch_lock)); 1536 KASSERT(chp->ch_queue->queue_freeze > 0); 1537 1538 chp->ch_queue->queue_freeze--; 1539 1540 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1541 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1542 } 1543 1544 /* 1545 * ata_thread_run: 1546 * 1547 * Reset and ATA channel. Channel lock must be held. arg is type-specific. 1548 */ 1549 void 1550 ata_thread_run(struct ata_channel *chp, int flags, int type, int arg) 1551 { 1552 struct atac_softc *atac = chp->ch_atac; 1553 bool threset = false; 1554 struct ata_drive_datas *drvp; 1555 1556 ata_channel_lock_owned(chp); 1557 1558 /* 1559 * If we can poll or wait it's OK, otherwise wake up the 1560 * kernel thread to do it for us. 1561 */ 1562 ATADEBUG_PRINT(("%s flags 0x%x ch_flags 0x%x\n", 1563 __func__, flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS); 1564 if ((flags & (AT_POLL | AT_WAIT)) == 0) { 1565 switch (type) { 1566 case ATACH_TH_RESET: 1567 if (chp->ch_flags & ATACH_TH_RESET) { 1568 /* No need to schedule another reset */ 1569 return; 1570 } 1571 break; 1572 case ATACH_TH_DRIVE_RESET: 1573 { 1574 int drive = arg; 1575 1576 KASSERT(drive <= chp->ch_ndrives); 1577 drvp = &chp->ch_drive[drive]; 1578 1579 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 1580 /* No need to schedule another reset */ 1581 return; 1582 } 1583 drvp->drive_flags |= ATA_DRIVE_TH_RESET; 1584 break; 1585 } 1586 case ATACH_TH_RECOVERY: 1587 { 1588 uint32_t tfd = (uint32_t)arg; 1589 1590 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1591 chp->recovery_tfd = tfd; 1592 break; 1593 } 1594 default: 1595 panic("%s: unknown type: %x", __func__, type); 1596 /* NOTREACHED */ 1597 } 1598 1599 /* 1600 * Block execution of other commands while reset is scheduled 1601 * to a thread. 1602 */ 1603 ata_channel_freeze_locked(chp); 1604 chp->ch_flags |= type; 1605 1606 cv_signal(&chp->ch_thr_idle); 1607 return; 1608 } 1609 1610 /* Block execution of other commands during reset */ 1611 ata_channel_freeze_locked(chp); 1612 1613 /* 1614 * If reset has been scheduled to a thread, then clear 1615 * the flag now so that the thread won't try to execute it if 1616 * we happen to sleep, and thaw one more time after the reset. 1617 */ 1618 if (chp->ch_flags & type) { 1619 chp->ch_flags &= ~type; 1620 threset = true; 1621 } 1622 1623 switch (type) { 1624 case ATACH_TH_RESET: 1625 (*atac->atac_bustype_ata->ata_reset_channel)(chp, flags); 1626 1627 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1628 for (int drive = 0; drive < chp->ch_ndrives; drive++) 1629 chp->ch_drive[drive].state = 0; 1630 break; 1631 1632 case ATACH_TH_DRIVE_RESET: 1633 { 1634 int drive = arg; 1635 1636 KASSERT(drive <= chp->ch_ndrives); 1637 drvp = &chp->ch_drive[drive]; 1638 (*atac->atac_bustype_ata->ata_reset_drive)(drvp, flags, NULL); 1639 drvp->state = 0; 1640 break; 1641 } 1642 1643 case ATACH_TH_RECOVERY: 1644 { 1645 uint32_t tfd = (uint32_t)arg; 1646 1647 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1648 KASSERT(atac->atac_bustype_ata->ata_recovery != NULL); 1649 1650 SET(chp->ch_flags, ATACH_RECOVERING); 1651 (*atac->atac_bustype_ata->ata_recovery)(chp, flags, tfd); 1652 CLR(chp->ch_flags, ATACH_RECOVERING); 1653 break; 1654 } 1655 1656 default: 1657 panic("%s: unknown type: %x", __func__, type); 1658 /* NOTREACHED */ 1659 } 1660 1661 /* 1662 * Thaw one extra time to clear the freeze done when the reset has 1663 * been scheduled to the thread. 1664 */ 1665 if (threset) 1666 ata_channel_thaw_locked(chp); 1667 1668 /* Allow commands to run again */ 1669 ata_channel_thaw_locked(chp); 1670 1671 /* Signal the thread in case there is an xfer to run */ 1672 cv_signal(&chp->ch_thr_idle); 1673 } 1674 1675 int 1676 ata_addref(struct ata_channel *chp) 1677 { 1678 struct atac_softc *atac = chp->ch_atac; 1679 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1680 int s, error = 0; 1681 1682 s = splbio(); 1683 if (adapt->adapt_refcnt++ == 0 && 1684 adapt->adapt_enable != NULL) { 1685 error = (*adapt->adapt_enable)(atac->atac_dev, 1); 1686 if (error) 1687 adapt->adapt_refcnt--; 1688 } 1689 splx(s); 1690 return (error); 1691 } 1692 1693 void 1694 ata_delref(struct ata_channel *chp) 1695 { 1696 struct atac_softc *atac = chp->ch_atac; 1697 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1698 int s; 1699 1700 s = splbio(); 1701 if (adapt->adapt_refcnt-- == 1 && 1702 adapt->adapt_enable != NULL) 1703 (void) (*adapt->adapt_enable)(atac->atac_dev, 0); 1704 splx(s); 1705 } 1706 1707 void 1708 ata_print_modes(struct ata_channel *chp) 1709 { 1710 struct atac_softc *atac = chp->ch_atac; 1711 int drive; 1712 struct ata_drive_datas *drvp; 1713 1714 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1715 for (drive = 0; drive < chp->ch_ndrives; drive++) { 1716 drvp = &chp->ch_drive[drive]; 1717 if (drvp->drive_type == ATA_DRIVET_NONE || 1718 drvp->drv_softc == NULL) 1719 continue; 1720 aprint_verbose("%s(%s:%d:%d): using PIO mode %d", 1721 device_xname(drvp->drv_softc), 1722 device_xname(atac->atac_dev), 1723 chp->ch_channel, drvp->drive, drvp->PIO_mode); 1724 #if NATA_DMA 1725 if (drvp->drive_flags & ATA_DRIVE_DMA) 1726 aprint_verbose(", DMA mode %d", drvp->DMA_mode); 1727 #if NATA_UDMA 1728 if (drvp->drive_flags & ATA_DRIVE_UDMA) { 1729 aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode); 1730 if (drvp->UDMA_mode == 2) 1731 aprint_verbose(" (Ultra/33)"); 1732 else if (drvp->UDMA_mode == 4) 1733 aprint_verbose(" (Ultra/66)"); 1734 else if (drvp->UDMA_mode == 5) 1735 aprint_verbose(" (Ultra/100)"); 1736 else if (drvp->UDMA_mode == 6) 1737 aprint_verbose(" (Ultra/133)"); 1738 } 1739 #endif /* NATA_UDMA */ 1740 #endif /* NATA_DMA */ 1741 #if NATA_DMA || NATA_PIOBM 1742 if (0 1743 #if NATA_DMA 1744 || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) 1745 #endif 1746 #if NATA_PIOBM 1747 /* PIOBM capable controllers use DMA for PIO commands */ 1748 || (atac->atac_cap & ATAC_CAP_PIOBM) 1749 #endif 1750 ) 1751 aprint_verbose(" (using DMA)"); 1752 1753 if (drvp->drive_flags & ATA_DRIVE_NCQ) { 1754 aprint_verbose(", NCQ (%d tags)%s", 1755 ATA_REAL_OPENINGS(chp->ch_queue->queue_openings), 1756 (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO) 1757 ? " w/PRIO" : ""); 1758 } else if (drvp->drive_flags & ATA_DRIVE_WFUA) 1759 aprint_verbose(", WRITE DMA FUA EXT"); 1760 1761 #endif /* NATA_DMA || NATA_PIOBM */ 1762 aprint_verbose("\n"); 1763 } 1764 } 1765 1766 #if defined(ATA_DOWNGRADE_MODE) && NATA_DMA 1767 /* 1768 * downgrade the transfer mode of a drive after an error. return 1 if 1769 * downgrade was possible, 0 otherwise. 1770 * 1771 * MUST BE CALLED AT splbio()! 1772 */ 1773 static int 1774 ata_downgrade_mode(struct ata_drive_datas *drvp, int flags) 1775 { 1776 struct ata_channel *chp = drvp->chnl_softc; 1777 struct atac_softc *atac = chp->ch_atac; 1778 device_t drv_dev = drvp->drv_softc; 1779 int cf_flags = device_cfdata(drv_dev)->cf_flags; 1780 1781 ata_channel_lock_owned(drvp->chnl_softc); 1782 1783 /* if drive or controller don't know its mode, we can't do much */ 1784 if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 || 1785 (atac->atac_set_modes == NULL)) 1786 return 0; 1787 /* current drive mode was set by a config flag, let it this way */ 1788 if ((cf_flags & ATA_CONFIG_PIO_SET) || 1789 (cf_flags & ATA_CONFIG_DMA_SET) || 1790 (cf_flags & ATA_CONFIG_UDMA_SET)) 1791 return 0; 1792 1793 #if NATA_UDMA 1794 /* 1795 * If we were using Ultra-DMA mode, downgrade to the next lower mode. 1796 */ 1797 if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) { 1798 drvp->UDMA_mode--; 1799 aprint_error_dev(drv_dev, 1800 "transfer error, downgrading to Ultra-DMA mode %d\n", 1801 drvp->UDMA_mode); 1802 } 1803 #endif 1804 1805 /* 1806 * If we were using ultra-DMA, don't downgrade to multiword DMA. 1807 */ 1808 else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) { 1809 drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA); 1810 drvp->PIO_mode = drvp->PIO_cap; 1811 aprint_error_dev(drv_dev, 1812 "transfer error, downgrading to PIO mode %d\n", 1813 drvp->PIO_mode); 1814 } else /* already using PIO, can't downgrade */ 1815 return 0; 1816 1817 (*atac->atac_set_modes)(chp); 1818 ata_print_modes(chp); 1819 /* reset the channel, which will schedule all drives for setup */ 1820 ata_thread_run(chp, flags, ATACH_TH_RESET, ATACH_NODRIVE); 1821 return 1; 1822 } 1823 #endif /* ATA_DOWNGRADE_MODE && NATA_DMA */ 1824 1825 /* 1826 * Probe drive's capabilities, for use by the controller later 1827 * Assumes drvp points to an existing drive. 1828 */ 1829 void 1830 ata_probe_caps(struct ata_drive_datas *drvp) 1831 { 1832 struct ataparams params, params2; 1833 struct ata_channel *chp = drvp->chnl_softc; 1834 struct atac_softc *atac = chp->ch_atac; 1835 device_t drv_dev = drvp->drv_softc; 1836 int i, printed = 0; 1837 const char *sep = ""; 1838 int cf_flags; 1839 1840 if (ata_get_params(drvp, AT_WAIT, ¶ms) != CMD_OK) { 1841 /* IDENTIFY failed. Can't tell more about the device */ 1842 return; 1843 } 1844 if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) == 1845 (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) { 1846 /* 1847 * Controller claims 16 and 32 bit transfers. 1848 * Re-do an IDENTIFY with 32-bit transfers, 1849 * and compare results. 1850 */ 1851 ata_channel_lock(chp); 1852 drvp->drive_flags |= ATA_DRIVE_CAP32; 1853 ata_channel_unlock(chp); 1854 ata_get_params(drvp, AT_WAIT, ¶ms2); 1855 if (memcmp(¶ms, ¶ms2, sizeof(struct ataparams)) != 0) { 1856 /* Not good. fall back to 16bits */ 1857 ata_channel_lock(chp); 1858 drvp->drive_flags &= ~ATA_DRIVE_CAP32; 1859 ata_channel_unlock(chp); 1860 } else { 1861 aprint_verbose_dev(drv_dev, "32-bit data port\n"); 1862 } 1863 } 1864 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */ 1865 if (params.atap_ata_major > 0x01 && 1866 params.atap_ata_major != 0xffff) { 1867 for (i = 14; i > 0; i--) { 1868 if (params.atap_ata_major & (1 << i)) { 1869 aprint_verbose_dev(drv_dev, 1870 "ATA version %d\n", i); 1871 drvp->ata_vers = i; 1872 break; 1873 } 1874 } 1875 } 1876 #endif 1877 1878 /* An ATAPI device is at last PIO mode 3 */ 1879 if (drvp->drive_type == ATA_DRIVET_ATAPI) 1880 drvp->PIO_mode = 3; 1881 1882 /* 1883 * It's not in the specs, but it seems that some drive 1884 * returns 0xffff in atap_extensions when this field is invalid 1885 */ 1886 if (params.atap_extensions != 0xffff && 1887 (params.atap_extensions & WDC_EXT_MODES)) { 1888 /* 1889 * XXX some drives report something wrong here (they claim to 1890 * support PIO mode 8 !). As mode is coded on 3 bits in 1891 * SET FEATURE, limit it to 7 (so limit i to 4). 1892 * If higher mode than 7 is found, abort. 1893 */ 1894 for (i = 7; i >= 0; i--) { 1895 if ((params.atap_piomode_supp & (1 << i)) == 0) 1896 continue; 1897 if (i > 4) 1898 return; 1899 /* 1900 * See if mode is accepted. 1901 * If the controller can't set its PIO mode, 1902 * assume the defaults are good, so don't try 1903 * to set it 1904 */ 1905 if (atac->atac_set_modes) 1906 /* 1907 * It's OK to poll here, it's fast enough 1908 * to not bother waiting for interrupt 1909 */ 1910 if (ata_set_mode(drvp, 0x08 | (i + 3), 1911 AT_WAIT) != CMD_OK) 1912 continue; 1913 if (!printed) { 1914 aprint_verbose_dev(drv_dev, 1915 "drive supports PIO mode %d", i + 3); 1916 sep = ","; 1917 printed = 1; 1918 } 1919 /* 1920 * If controller's driver can't set its PIO mode, 1921 * get the highter one for the drive. 1922 */ 1923 if (atac->atac_set_modes == NULL || 1924 atac->atac_pio_cap >= i + 3) { 1925 drvp->PIO_mode = i + 3; 1926 drvp->PIO_cap = i + 3; 1927 break; 1928 } 1929 } 1930 if (!printed) { 1931 /* 1932 * We didn't find a valid PIO mode. 1933 * Assume the values returned for DMA are buggy too 1934 */ 1935 return; 1936 } 1937 ata_channel_lock(chp); 1938 drvp->drive_flags |= ATA_DRIVE_MODE; 1939 ata_channel_unlock(chp); 1940 printed = 0; 1941 for (i = 7; i >= 0; i--) { 1942 if ((params.atap_dmamode_supp & (1 << i)) == 0) 1943 continue; 1944 #if NATA_DMA 1945 if ((atac->atac_cap & ATAC_CAP_DMA) && 1946 atac->atac_set_modes != NULL) 1947 if (ata_set_mode(drvp, 0x20 | i, AT_WAIT) 1948 != CMD_OK) 1949 continue; 1950 #endif 1951 if (!printed) { 1952 aprint_verbose("%s DMA mode %d", sep, i); 1953 sep = ","; 1954 printed = 1; 1955 } 1956 #if NATA_DMA 1957 if (atac->atac_cap & ATAC_CAP_DMA) { 1958 if (atac->atac_set_modes != NULL && 1959 atac->atac_dma_cap < i) 1960 continue; 1961 drvp->DMA_mode = i; 1962 drvp->DMA_cap = i; 1963 ata_channel_lock(chp); 1964 drvp->drive_flags |= ATA_DRIVE_DMA; 1965 ata_channel_unlock(chp); 1966 } 1967 #endif 1968 break; 1969 } 1970 if (params.atap_extensions & WDC_EXT_UDMA_MODES) { 1971 printed = 0; 1972 for (i = 7; i >= 0; i--) { 1973 if ((params.atap_udmamode_supp & (1 << i)) 1974 == 0) 1975 continue; 1976 #if NATA_UDMA 1977 if (atac->atac_set_modes != NULL && 1978 (atac->atac_cap & ATAC_CAP_UDMA)) 1979 if (ata_set_mode(drvp, 0x40 | i, 1980 AT_WAIT) != CMD_OK) 1981 continue; 1982 #endif 1983 if (!printed) { 1984 aprint_verbose("%s Ultra-DMA mode %d", 1985 sep, i); 1986 if (i == 2) 1987 aprint_verbose(" (Ultra/33)"); 1988 else if (i == 4) 1989 aprint_verbose(" (Ultra/66)"); 1990 else if (i == 5) 1991 aprint_verbose(" (Ultra/100)"); 1992 else if (i == 6) 1993 aprint_verbose(" (Ultra/133)"); 1994 sep = ","; 1995 printed = 1; 1996 } 1997 #if NATA_UDMA 1998 if (atac->atac_cap & ATAC_CAP_UDMA) { 1999 if (atac->atac_set_modes != NULL && 2000 atac->atac_udma_cap < i) 2001 continue; 2002 drvp->UDMA_mode = i; 2003 drvp->UDMA_cap = i; 2004 ata_channel_lock(chp); 2005 drvp->drive_flags |= ATA_DRIVE_UDMA; 2006 ata_channel_unlock(chp); 2007 } 2008 #endif 2009 break; 2010 } 2011 } 2012 } 2013 2014 ata_channel_lock(chp); 2015 drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM; 2016 if (drvp->drive_type == ATA_DRIVET_ATAPI) { 2017 if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM) 2018 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2019 } else { 2020 if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM) 2021 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2022 } 2023 ata_channel_unlock(chp); 2024 2025 /* Try to guess ATA version here, if it didn't get reported */ 2026 if (drvp->ata_vers == 0) { 2027 #if NATA_UDMA 2028 if (drvp->drive_flags & ATA_DRIVE_UDMA) 2029 drvp->ata_vers = 4; /* should be at last ATA-4 */ 2030 else 2031 #endif 2032 if (drvp->PIO_cap > 2) 2033 drvp->ata_vers = 2; /* should be at last ATA-2 */ 2034 } 2035 cf_flags = device_cfdata(drv_dev)->cf_flags; 2036 if (cf_flags & ATA_CONFIG_PIO_SET) { 2037 ata_channel_lock(chp); 2038 drvp->PIO_mode = 2039 (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF; 2040 drvp->drive_flags |= ATA_DRIVE_MODE; 2041 ata_channel_unlock(chp); 2042 } 2043 #if NATA_DMA 2044 if ((atac->atac_cap & ATAC_CAP_DMA) == 0) { 2045 /* don't care about DMA modes */ 2046 if (*sep != '\0') 2047 aprint_verbose("\n"); 2048 return; 2049 } 2050 if (cf_flags & ATA_CONFIG_DMA_SET) { 2051 ata_channel_lock(chp); 2052 if ((cf_flags & ATA_CONFIG_DMA_MODES) == 2053 ATA_CONFIG_DMA_DISABLE) { 2054 drvp->drive_flags &= ~ATA_DRIVE_DMA; 2055 } else { 2056 drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >> 2057 ATA_CONFIG_DMA_OFF; 2058 drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE; 2059 } 2060 ata_channel_unlock(chp); 2061 } 2062 2063 /* 2064 * Probe WRITE DMA FUA EXT. Support is mandatory for devices 2065 * supporting LBA48, but nevertheless confirm with the feature flag. 2066 */ 2067 if (drvp->drive_flags & ATA_DRIVE_DMA) { 2068 if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0 2069 && (params.atap_cmd_def & ATA_CMDE_WFE)) { 2070 drvp->drive_flags |= ATA_DRIVE_WFUA; 2071 aprint_verbose("%s WRITE DMA FUA", sep); 2072 sep = ","; 2073 } 2074 } 2075 2076 /* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */ 2077 ata_channel_lock(chp); 2078 drvp->drv_openings = 1; 2079 if (params.atap_sata_caps & SATA_NATIVE_CMDQ) { 2080 if (atac->atac_cap & ATAC_CAP_NCQ) 2081 drvp->drive_flags |= ATA_DRIVE_NCQ; 2082 drvp->drv_openings = 2083 (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1; 2084 aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings); 2085 sep = ","; 2086 2087 if (params.atap_sata_caps & SATA_NCQ_PRIO) { 2088 drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO; 2089 aprint_verbose(" w/PRIO"); 2090 } 2091 } 2092 ata_channel_unlock(chp); 2093 2094 if (*sep != '\0') 2095 aprint_verbose("\n"); 2096 2097 #if NATA_UDMA 2098 if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) { 2099 /* don't care about UDMA modes */ 2100 return; 2101 } 2102 if (cf_flags & ATA_CONFIG_UDMA_SET) { 2103 ata_channel_lock(chp); 2104 if ((cf_flags & ATA_CONFIG_UDMA_MODES) == 2105 ATA_CONFIG_UDMA_DISABLE) { 2106 drvp->drive_flags &= ~ATA_DRIVE_UDMA; 2107 } else { 2108 drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >> 2109 ATA_CONFIG_UDMA_OFF; 2110 drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE; 2111 } 2112 ata_channel_unlock(chp); 2113 } 2114 #endif /* NATA_UDMA */ 2115 #endif /* NATA_DMA */ 2116 } 2117 2118 /* management of the /dev/atabus* devices */ 2119 int 2120 atabusopen(dev_t dev, int flag, int fmt, struct lwp *l) 2121 { 2122 struct atabus_softc *sc; 2123 int error; 2124 2125 sc = device_lookup_private(&atabus_cd, minor(dev)); 2126 if (sc == NULL) 2127 return (ENXIO); 2128 2129 if (sc->sc_flags & ATABUSCF_OPEN) 2130 return (EBUSY); 2131 2132 if ((error = ata_addref(sc->sc_chan)) != 0) 2133 return (error); 2134 2135 sc->sc_flags |= ATABUSCF_OPEN; 2136 2137 return (0); 2138 } 2139 2140 2141 int 2142 atabusclose(dev_t dev, int flag, int fmt, struct lwp *l) 2143 { 2144 struct atabus_softc *sc = 2145 device_lookup_private(&atabus_cd, minor(dev)); 2146 2147 ata_delref(sc->sc_chan); 2148 2149 sc->sc_flags &= ~ATABUSCF_OPEN; 2150 2151 return (0); 2152 } 2153 2154 int 2155 atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l) 2156 { 2157 struct atabus_softc *sc = 2158 device_lookup_private(&atabus_cd, minor(dev)); 2159 struct ata_channel *chp = sc->sc_chan; 2160 int min_drive, max_drive, drive; 2161 int error; 2162 2163 /* 2164 * Enforce write permission for ioctls that change the 2165 * state of the bus. Host adapter specific ioctls must 2166 * be checked by the adapter driver. 2167 */ 2168 switch (cmd) { 2169 case ATABUSIOSCAN: 2170 case ATABUSIODETACH: 2171 case ATABUSIORESET: 2172 if ((flag & FWRITE) == 0) 2173 return (EBADF); 2174 } 2175 2176 switch (cmd) { 2177 case ATABUSIORESET: 2178 ata_channel_lock(chp); 2179 ata_thread_run(sc->sc_chan, AT_WAIT | AT_POLL, 2180 ATACH_TH_RESET, ATACH_NODRIVE); 2181 ata_channel_unlock(chp); 2182 return 0; 2183 case ATABUSIOSCAN: 2184 { 2185 #if 0 2186 struct atabusioscan_args *a= 2187 (struct atabusioscan_args *)addr; 2188 #endif 2189 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2190 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2191 return (EOPNOTSUPP); 2192 return (EOPNOTSUPP); 2193 } 2194 case ATABUSIODETACH: 2195 { 2196 struct atabusiodetach_args *a= 2197 (struct atabusiodetach_args *)addr; 2198 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2199 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2200 return (EOPNOTSUPP); 2201 switch (a->at_dev) { 2202 case -1: 2203 min_drive = 0; 2204 max_drive = 1; 2205 break; 2206 case 0: 2207 case 1: 2208 min_drive = max_drive = a->at_dev; 2209 break; 2210 default: 2211 return (EINVAL); 2212 } 2213 for (drive = min_drive; drive <= max_drive; drive++) { 2214 if (chp->ch_drive[drive].drv_softc != NULL) { 2215 error = config_detach( 2216 chp->ch_drive[drive].drv_softc, 0); 2217 if (error) 2218 return (error); 2219 KASSERT(chp->ch_drive[drive].drv_softc == NULL); 2220 } 2221 } 2222 return 0; 2223 } 2224 default: 2225 return ENOTTY; 2226 } 2227 } 2228 2229 static bool 2230 atabus_suspend(device_t dv, const pmf_qual_t *qual) 2231 { 2232 struct atabus_softc *sc = device_private(dv); 2233 struct ata_channel *chp = sc->sc_chan; 2234 2235 ata_channel_idle(chp); 2236 2237 return true; 2238 } 2239 2240 static bool 2241 atabus_resume(device_t dv, const pmf_qual_t *qual) 2242 { 2243 struct atabus_softc *sc = device_private(dv); 2244 struct ata_channel *chp = sc->sc_chan; 2245 2246 /* 2247 * XXX joerg: with wdc, the first channel unfreezes the controller. 2248 * Move this the reset and queue idling into wdc. 2249 */ 2250 ata_channel_lock(chp); 2251 if (chp->ch_queue->queue_freeze == 0) { 2252 ata_channel_unlock(chp); 2253 goto out; 2254 } 2255 2256 /* unfreeze the queue and reset drives */ 2257 ata_channel_thaw_locked(chp); 2258 2259 /* reset channel only if there are drives attached */ 2260 if (chp->ch_ndrives > 0) 2261 ata_thread_run(chp, AT_WAIT, ATACH_TH_RESET, ATACH_NODRIVE); 2262 2263 ata_channel_unlock(chp); 2264 2265 out: 2266 return true; 2267 } 2268 2269 static int 2270 atabus_rescan(device_t self, const char *ifattr, const int *locators) 2271 { 2272 struct atabus_softc *sc = device_private(self); 2273 struct ata_channel *chp = sc->sc_chan; 2274 struct atabus_initq *initq; 2275 int i; 2276 2277 /* 2278 * we can rescan a port multiplier atabus, even if some devices are 2279 * still attached 2280 */ 2281 if (chp->ch_satapmp_nports == 0) { 2282 if (chp->atapibus != NULL) { 2283 return EBUSY; 2284 } 2285 2286 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 2287 for (i = 0; i < chp->ch_ndrives; i++) { 2288 if (chp->ch_drive[i].drv_softc != NULL) { 2289 return EBUSY; 2290 } 2291 } 2292 } 2293 2294 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 2295 initq->atabus_sc = sc; 2296 mutex_enter(&atabus_qlock); 2297 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 2298 mutex_exit(&atabus_qlock); 2299 config_pending_incr(sc->sc_dev); 2300 2301 ata_channel_lock(chp); 2302 chp->ch_flags |= ATACH_TH_RESCAN; 2303 cv_signal(&chp->ch_thr_idle); 2304 ata_channel_unlock(chp); 2305 2306 return 0; 2307 } 2308 2309 void 2310 ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags) 2311 { 2312 KASSERT(mutex_owned(&chp->ch_lock)); 2313 2314 if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) { 2315 /* 2316 * can't use kpause(), we may be in interrupt context 2317 * or taking a crash dump 2318 */ 2319 delay(ms * 1000); 2320 } else { 2321 int pause = mstohz(ms); 2322 2323 kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock); 2324 } 2325 } 2326 2327 void 2328 atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count, 2329 uint16_t *features, uint8_t *device) 2330 { 2331 if ((xfer->c_flags & C_NCQ) == 0) { 2332 /* FUA handling for non-NCQ drives */ 2333 if (xfer->c_bio.flags & ATA_FUA 2334 && *cmd == WDCC_WRITEDMA_EXT) 2335 *cmd = WDCC_WRITEDMA_FUA_EXT; 2336 2337 return; 2338 } 2339 2340 *cmd = (xfer->c_bio.flags & ATA_READ) ? 2341 WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED; 2342 2343 /* for FPDMA the block count is in features */ 2344 *features = *count; 2345 2346 /* NCQ tag */ 2347 *count = (xfer->c_slot << 3); 2348 2349 if (xfer->c_bio.flags & ATA_PRIO_HIGH) 2350 *count |= WDSC_PRIO_HIGH; 2351 2352 /* other device flags */ 2353 if (xfer->c_bio.flags & ATA_FUA) 2354 *device |= WDSD_FUA; 2355 } 2356 2357 void 2358 ata_wait_cmd(struct ata_channel *chp, struct ata_xfer *xfer) 2359 { 2360 struct ata_queue *chq = chp->ch_queue; 2361 struct ata_command *ata_c = &xfer->c_ata_c; 2362 2363 ata_channel_lock(chp); 2364 2365 while ((ata_c->flags & AT_DONE) == 0) 2366 cv_wait(&chq->c_cmd_finish, &chp->ch_lock); 2367 2368 ata_channel_unlock(chp); 2369 2370 KASSERT((ata_c->flags & AT_DONE) != 0); 2371 } 2372
Indexes created Wed Sep 24 14:09:57 GMT 2025