1 /* $NetBSD: ata.c,v 1.173 2026/07/12 20:52:47 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.173 2026/07/12 20:52:47 thorpej Exp $"); 29 30 #include "opt_ata.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/device.h> 36 #include <sys/conf.h> 37 #include <sys/fcntl.h> 38 #include <sys/proc.h> 39 #include <sys/kthread.h> 40 #include <sys/errno.h> 41 #include <sys/ataio.h> 42 #include <sys/kmem.h> 43 #include <sys/intr.h> 44 #include <sys/bus.h> 45 #include <sys/once.h> 46 #include <sys/bitops.h> 47 #include <sys/cpu.h> 48 49 #define ATABUS_PRIVATE 50 51 #include <dev/ata/ataconf.h> 52 #include <dev/ata/atareg.h> 53 #include <dev/ata/atavar.h> 54 #include <dev/ic/wdcvar.h> /* for PIOBM */ 55 56 #include "ioconf.h" 57 #include "locators.h" 58 59 #include "atapibus.h" 60 #include "ataraid.h" 61 #include "sata_pmp.h" 62 63 #if NATARAID > 0 64 #include <dev/ata/ata_raidvar.h> 65 #endif 66 #if NSATA_PMP > 0 67 #include <dev/ata/satapmpvar.h> 68 #endif 69 #include <dev/ata/satapmpreg.h> 70 71 #define DEBUG_FUNCS 0x08 72 #define DEBUG_PROBE 0x10 73 #define DEBUG_DETACH 0x20 74 #define DEBUG_XFERS 0x40 75 #ifdef ATADEBUG 76 #ifndef ATADEBUG_MASK 77 #define ATADEBUG_MASK 0 78 #endif 79 int atadebug_mask = ATADEBUG_MASK; 80 #define ATADEBUG_PRINT(args, level) \ 81 if (atadebug_mask & (level)) \ 82 printf args 83 #else 84 #define ATADEBUG_PRINT(args, level) 85 #endif 86 87 #if !defined(ATA_NO_DOWNGRADE_MODE) && NATA_DMA 88 static int ata_downgrade_mode(struct ata_drive_datas *, int); 89 #endif 90 91 static ONCE_DECL(ata_init_ctrl); 92 static struct pool ata_xfer_pool; 93 94 /* 95 * A queue of atabus instances, used to ensure the same bus probe order 96 * for a given hardware configuration at each boot. Kthread probing 97 * devices on a atabus. Only one probing at once. 98 */ 99 static TAILQ_HEAD(, atabus_initq) atabus_initq_head; 100 static kmutex_t atabus_qlock; 101 static kcondvar_t atabus_qcv; 102 static lwp_t * atabus_cfg_lwp; 103 104 /***************************************************************************** 105 * ATA bus layer. 106 * 107 * ATA controllers attach an atabus instance, which handles probing the bus 108 * for drives, etc. 109 *****************************************************************************/ 110 111 dev_type_open(atabusopen); 112 dev_type_close(atabusclose); 113 dev_type_ioctl(atabusioctl); 114 115 const struct cdevsw atabus_cdevsw = { 116 .d_open = atabusopen, 117 .d_close = atabusclose, 118 .d_read = noread, 119 .d_write = nowrite, 120 .d_ioctl = atabusioctl, 121 .d_stop = nostop, 122 .d_tty = notty, 123 .d_poll = nopoll, 124 .d_mmap = nommap, 125 .d_kqfilter = nokqfilter, 126 .d_discard = nodiscard, 127 .d_flag = D_OTHER 128 }; 129 130 static void atabus_childdetached(device_t, device_t); 131 static int atabus_rescan(device_t, const char *, const int *); 132 static bool atabus_resume(device_t, const pmf_qual_t *); 133 static bool atabus_suspend(device_t, const pmf_qual_t *); 134 static void atabusconfig_thread(void *); 135 136 static void ata_channel_idle(struct ata_channel *); 137 static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *); 138 static void ata_channel_freeze_locked(struct ata_channel *); 139 static void ata_thread_wake_locked(struct ata_channel *); 140 141 /* 142 * atabus_init: 143 * 144 * Initialize ATA subsystem structures. 145 */ 146 static int 147 atabus_init(void) 148 { 149 150 pool_init(&ata_xfer_pool, sizeof(struct ata_xfer), 0, 0, 0, 151 "ataspl", NULL, IPL_BIO); 152 TAILQ_INIT(&atabus_initq_head); 153 mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE); 154 cv_init(&atabus_qcv, "atainitq"); 155 return 0; 156 } 157 158 /* 159 * atabusprint: 160 * 161 * Autoconfiguration print routine used by ATA controllers when 162 * attaching an atabus instance. 163 */ 164 int 165 atabusprint(void *aux, const char *pnp) 166 { 167 struct ata_channel *chan = aux; 168 169 if (pnp) 170 aprint_normal("atabus at %s", pnp); 171 aprint_normal(" channel %d", chan->ch_channel); 172 173 return (UNCONF); 174 } 175 176 /* 177 * ataprint: 178 * 179 * Autoconfiguration print routine. 180 */ 181 int 182 ataprint(void *aux, const char *pnp) 183 { 184 struct ata_device *adev = aux; 185 186 if (pnp) 187 aprint_normal("wd at %s", pnp); 188 aprint_normal(" drive %d", adev->adev_drv_data->drive); 189 190 return (UNCONF); 191 } 192 193 /* 194 * ata_channel_attach: 195 * 196 * Common parts of attaching an atabus to an ATA controller channel. 197 */ 198 void 199 ata_channel_attach(struct ata_channel *chp) 200 { 201 if (chp->ch_flags & ATACH_DISABLED) 202 return; 203 204 ata_channel_init(chp); 205 206 KASSERT(chp->ch_queue != NULL); 207 208 chp->atabus = config_found(chp->ch_atac->atac_dev, chp, atabusprint, 209 CFARGS(.iattr = "ata")); 210 } 211 212 /* 213 * ata_channel_detach: 214 * 215 * Common parts of detaching an atabus to an ATA controller channel. 216 */ 217 void 218 ata_channel_detach(struct ata_channel *chp) 219 { 220 if (chp->ch_flags & ATACH_DISABLED) 221 return; 222 223 ata_channel_destroy(chp); 224 225 chp->ch_flags |= ATACH_DETACHED; 226 } 227 228 static void 229 atabusconfig(struct atabus_softc *atabus_sc) 230 { 231 struct ata_channel *chp = atabus_sc->sc_chan; 232 struct atac_softc *atac = chp->ch_atac; 233 struct atabus_initq *atabus_initq = NULL; 234 int i, error; 235 236 /* we are in the atabus's thread context */ 237 238 /* 239 * Probe for the drives attached to controller, unless a PMP 240 * is already known 241 */ 242 /* XXX for SATA devices we will power up all drives at once */ 243 if (chp->ch_satapmp_nports == 0) 244 (*atac->atac_probe)(chp); 245 246 if (chp->ch_ndrives >= 2) { 247 ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n", 248 chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type), 249 DEBUG_PROBE); 250 } 251 252 /* Make sure the devices probe in atabus order to avoid jitter. */ 253 mutex_enter(&atabus_qlock); 254 for (;;) { 255 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 256 if (atabus_initq->atabus_sc == atabus_sc) 257 break; 258 cv_wait(&atabus_qcv, &atabus_qlock); 259 } 260 mutex_exit(&atabus_qlock); 261 262 ata_channel_lock(chp); 263 264 KASSERT(ata_is_thread_run(chp)); 265 266 /* If no drives, abort here */ 267 if (chp->ch_drive == NULL) 268 goto out; 269 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 270 for (i = 0; i < chp->ch_ndrives; i++) 271 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) 272 break; 273 if (i == chp->ch_ndrives) 274 goto out; 275 276 /* Shortcut in case we've been shutdown */ 277 if (chp->ch_flags & ATACH_SHUTDOWN) 278 goto out; 279 280 ata_channel_unlock(chp); 281 282 if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread, 283 atabus_sc, &atabus_cfg_lwp, 284 "%scnf", device_xname(atac->atac_dev))) != 0) 285 aprint_error_dev(atac->atac_dev, 286 "unable to create config thread: error %d\n", error); 287 return; 288 289 out: 290 ata_channel_unlock(chp); 291 292 mutex_enter(&atabus_qlock); 293 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 294 cv_broadcast(&atabus_qcv); 295 mutex_exit(&atabus_qlock); 296 297 kmem_free(atabus_initq, sizeof(*atabus_initq)); 298 299 ata_delref(chp); 300 301 config_pending_decr(atabus_sc->sc_dev); 302 } 303 304 /* 305 * atabus_configthread: finish attach of atabus's childrens, in a separate 306 * kernel thread. 307 */ 308 static void 309 atabusconfig_thread(void *arg) 310 { 311 struct atabus_softc *atabus_sc = arg; 312 struct ata_channel *chp = atabus_sc->sc_chan; 313 struct atac_softc *atac = chp->ch_atac; 314 struct atabus_initq *atabus_initq = NULL; 315 int i, s; 316 317 /* XXX seems wrong */ 318 mutex_enter(&atabus_qlock); 319 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 320 KASSERT(atabus_initq->atabus_sc == atabus_sc); 321 mutex_exit(&atabus_qlock); 322 323 /* 324 * First look for a port multiplier 325 */ 326 if (chp->ch_ndrives == PMP_MAX_DRIVES && 327 chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) { 328 #if NSATA_PMP > 0 329 satapmp_attach(chp); 330 #else 331 aprint_error_dev(atabus_sc->sc_dev, 332 "SATA port multiplier not supported\n"); 333 /* no problems going on, all drives are ATA_DRIVET_NONE */ 334 #endif 335 } 336 337 /* 338 * Attach an ATAPI bus, if needed. 339 */ 340 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 341 for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) { 342 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) { 343 #if NATAPIBUS > 0 344 (*atac->atac_atapibus_attach)(atabus_sc); 345 #else 346 /* 347 * Fake the autoconfig "not configured" message 348 */ 349 aprint_normal("atapibus at %s not configured\n", 350 device_xname(atac->atac_dev)); 351 chp->atapibus = NULL; 352 s = splbio(); 353 for (i = 0; i < chp->ch_ndrives; i++) { 354 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 355 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 356 } 357 splx(s); 358 #endif 359 break; 360 } 361 } 362 363 for (i = 0; i < chp->ch_ndrives; i++) { 364 struct ata_device adev; 365 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA && 366 chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) { 367 continue; 368 } 369 if (chp->ch_drive[i].drv_softc != NULL) 370 continue; 371 memset(&adev, 0, sizeof(struct ata_device)); 372 adev.adev_bustype = atac->atac_bustype_ata; 373 adev.adev_channel = chp->ch_channel; 374 adev.adev_drv_data = &chp->ch_drive[i]; 375 chp->ch_drive[i].drv_softc = config_found(atabus_sc->sc_dev, 376 &adev, ataprint, 377 CFARGS(.iattr = "ata_hl")); 378 if (chp->ch_drive[i].drv_softc != NULL) { 379 ata_probe_caps(&chp->ch_drive[i]); 380 } else { 381 s = splbio(); 382 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 383 splx(s); 384 } 385 } 386 387 /* now that we know the drives, the controller can set its modes */ 388 if (atac->atac_set_modes) { 389 (*atac->atac_set_modes)(chp); 390 ata_print_modes(chp); 391 } 392 #if NATARAID > 0 393 if (atac->atac_cap & ATAC_CAP_RAID) { 394 for (i = 0; i < chp->ch_ndrives; i++) { 395 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) { 396 ata_raid_check_component( 397 chp->ch_drive[i].drv_softc); 398 } 399 } 400 } 401 #endif /* NATARAID > 0 */ 402 403 /* 404 * reset drive_flags for unattached devices, reset state for attached 405 * ones 406 */ 407 s = splbio(); 408 for (i = 0; i < chp->ch_ndrives; i++) { 409 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 410 continue; 411 if (chp->ch_drive[i].drv_softc == NULL) { 412 chp->ch_drive[i].drive_flags = 0; 413 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 414 } else 415 chp->ch_drive[i].state = 0; 416 } 417 splx(s); 418 419 mutex_enter(&atabus_qlock); 420 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 421 cv_broadcast(&atabus_qcv); 422 mutex_exit(&atabus_qlock); 423 424 kmem_free(atabus_initq, sizeof(*atabus_initq)); 425 426 ata_delref(chp); 427 428 config_pending_decr(atabus_sc->sc_dev); 429 kthread_exit(0); 430 } 431 432 /* 433 * atabus_thread: 434 * 435 * Worker thread for the ATA bus. 436 */ 437 static void 438 atabus_thread(void *arg) 439 { 440 struct atabus_softc *sc = arg; 441 struct ata_channel *chp = sc->sc_chan; 442 struct ata_queue *chq = chp->ch_queue; 443 struct ata_xfer *xfer; 444 int i, rv; 445 446 ata_channel_lock(chp); 447 KASSERT(ata_is_thread_run(chp)); 448 449 /* 450 * Probe the drives. Reset type to indicate to controllers 451 * that can re-probe that all drives must be probed.. 452 * 453 * Note: ch_ndrives may be changed during the probe. 454 */ 455 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 456 for (i = 0; i < chp->ch_ndrives; i++) { 457 chp->ch_drive[i].drive_flags = 0; 458 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 459 } 460 ata_channel_unlock(chp); 461 462 atabusconfig(sc); 463 464 ata_channel_lock(chp); 465 for (;;) { 466 if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_TH_DRIVE_RESET 467 | ATACH_TH_RECOVERY | ATACH_SHUTDOWN)) == 0 && 468 (chq->queue_active == 0 || chq->queue_freeze == 0)) { 469 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 470 } 471 if (chp->ch_flags & ATACH_SHUTDOWN) { 472 break; 473 } 474 if (chp->ch_flags & ATACH_TH_RESCAN) { 475 chp->ch_flags &= ~ATACH_TH_RESCAN; 476 ata_channel_unlock(chp); 477 atabusconfig(sc); 478 ata_channel_lock(chp); 479 } 480 if (chp->ch_flags & ATACH_TH_RESET) { 481 /* this will unfreeze the channel */ 482 ata_thread_run(chp, AT_WAIT, 483 ATACH_TH_RESET, ATACH_NODRIVE); 484 } else if (chp->ch_flags & ATACH_TH_DRIVE_RESET) { 485 /* this will unfreeze the channel */ 486 for (i = 0; i < chp->ch_ndrives; i++) { 487 struct ata_drive_datas *drvp; 488 489 drvp = &chp->ch_drive[i]; 490 491 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 492 ata_thread_run(chp, 493 AT_WAIT, ATACH_TH_DRIVE_RESET, i); 494 } 495 } 496 chp->ch_flags &= ~ATACH_TH_DRIVE_RESET; 497 } else if (chp->ch_flags & ATACH_TH_RECOVERY) { 498 /* 499 * This will unfreeze the channel; drops locks during 500 * run, so must wrap in splbio()/splx() to avoid 501 * spurious interrupts. XXX MPSAFE 502 */ 503 int s = splbio(); 504 ata_thread_run(chp, AT_WAIT, ATACH_TH_RECOVERY, 505 chp->recovery_tfd); 506 splx(s); 507 } else if (chq->queue_active > 0 && chq->queue_freeze == 1) { 508 /* 509 * Caller has bumped queue_freeze, decrease it. This 510 * flow shalt never be executed for NCQ commands. 511 */ 512 KASSERT((chp->ch_flags & ATACH_NCQ) == 0); 513 KASSERT(chq->queue_active == 1); 514 515 ata_channel_thaw_locked(chp); 516 xfer = ata_queue_get_active_xfer_locked(chp); 517 518 KASSERT(xfer != NULL); 519 KASSERT((xfer->c_flags & C_POLL) == 0); 520 521 switch ((rv = ata_xfer_start(xfer))) { 522 case ATASTART_STARTED: 523 case ATASTART_POLL: 524 case ATASTART_ABORT: 525 break; 526 case ATASTART_TH: 527 default: 528 panic("%s: ata_xfer_start() unexpected rv %d", 529 __func__, rv); 530 /* NOTREACHED */ 531 } 532 } else if (chq->queue_freeze > 1) 533 panic("%s: queue_freeze", __func__); 534 535 /* Try to run down the queue once channel is unfrozen */ 536 if (chq->queue_freeze == 0) { 537 ata_channel_unlock(chp); 538 atastart(chp); 539 ata_channel_lock(chp); 540 } 541 } 542 chp->ch_thread = NULL; 543 cv_signal(&chp->ch_thr_idle); 544 ata_channel_unlock(chp); 545 kthread_exit(0); 546 } 547 548 bool 549 ata_is_thread_run(struct ata_channel *chp) 550 { 551 KASSERT(mutex_owned(&chp->ch_lock)); 552 553 return (chp->ch_thread == curlwp && !cpu_intr_p()); 554 } 555 556 static void 557 ata_thread_wake_locked(struct ata_channel *chp) 558 { 559 KASSERT(mutex_owned(&chp->ch_lock)); 560 ata_channel_freeze_locked(chp); 561 cv_signal(&chp->ch_thr_idle); 562 } 563 564 /* 565 * atabus_match: 566 * 567 * Autoconfiguration match routine. 568 */ 569 static int 570 atabus_match(device_t parent, cfdata_t cf, void *aux) 571 { 572 struct ata_channel *chp = aux; 573 574 if (chp == NULL) 575 return (0); 576 577 if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel && 578 cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT) 579 return (0); 580 581 return (1); 582 } 583 584 /* 585 * atabus_attach: 586 * 587 * Autoconfiguration attach routine. 588 */ 589 static void 590 atabus_attach(device_t parent, device_t self, void *aux) 591 { 592 struct atabus_softc *sc = device_private(self); 593 struct ata_channel *chp = aux; 594 struct atabus_initq *initq; 595 int error; 596 597 sc->sc_chan = chp; 598 599 aprint_normal("\n"); 600 aprint_naive("\n"); 601 602 sc->sc_dev = self; 603 604 if (ata_addref(chp)) 605 return; 606 607 RUN_ONCE(&ata_init_ctrl, atabus_init); 608 609 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 610 initq->atabus_sc = sc; 611 mutex_enter(&atabus_qlock); 612 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 613 mutex_exit(&atabus_qlock); 614 config_pending_incr(sc->sc_dev); 615 616 /* XXX MPSAFE - no KTHREAD_MPSAFE, so protected by KERNEL_LOCK() */ 617 if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc, 618 &chp->ch_thread, "%s", device_xname(self))) != 0) 619 aprint_error_dev(self, 620 "unable to create kernel thread: error %d\n", error); 621 622 if (!pmf_device_register(self, atabus_suspend, atabus_resume)) 623 aprint_error_dev(self, "couldn't establish power handler\n"); 624 } 625 626 /* 627 * atabus_detach: 628 * 629 * Autoconfiguration detach routine. 630 */ 631 static int 632 atabus_detach(device_t self, int flags) 633 { 634 struct atabus_softc *sc = device_private(self); 635 struct ata_channel *chp = sc->sc_chan; 636 device_t dev = NULL; 637 int i, error = 0; 638 639 /* 640 * Detach atapibus and its children. 641 */ 642 if ((dev = chp->atapibus) != NULL) { 643 ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n", 644 device_xname(self), device_xname(dev)), DEBUG_DETACH); 645 646 error = config_detach(dev, flags); 647 if (error) 648 goto out; 649 KASSERT(chp->atapibus == NULL); 650 } 651 652 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 653 654 /* 655 * Detach our other children. 656 */ 657 for (i = 0; i < chp->ch_ndrives; i++) { 658 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 659 continue; 660 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 661 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 662 if ((dev = chp->ch_drive[i].drv_softc) != NULL) { 663 ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__, 664 __LINE__, device_xname(self), device_xname(dev)), 665 DEBUG_DETACH); 666 error = config_detach(dev, flags); 667 if (error) 668 goto out; 669 KASSERT(chp->ch_drive[i].drv_softc == NULL); 670 KASSERT(chp->ch_drive[i].drive_type == 0); 671 } 672 } 673 674 /* Shutdown the channel. */ 675 ata_channel_lock(chp); 676 chp->ch_flags |= ATACH_SHUTDOWN; 677 while (chp->ch_thread != NULL) { 678 cv_signal(&chp->ch_thr_idle); 679 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 680 } 681 ata_channel_unlock(chp); 682 683 atabus_free_drives(chp); 684 685 out: 686 #ifdef ATADEBUG 687 if (dev != NULL && error != 0) 688 ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__, 689 device_xname(self), error, device_xname(dev)), 690 DEBUG_DETACH); 691 #endif /* ATADEBUG */ 692 693 return (error); 694 } 695 696 void 697 atabus_childdetached(device_t self, device_t child) 698 { 699 bool found = false; 700 struct atabus_softc *sc = device_private(self); 701 struct ata_channel *chp = sc->sc_chan; 702 int i; 703 704 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 705 /* 706 * atapibus detached. 707 */ 708 if (child == chp->atapibus) { 709 chp->atapibus = NULL; 710 found = true; 711 for (i = 0; i < chp->ch_ndrives; i++) { 712 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI) 713 continue; 714 KASSERT(chp->ch_drive[i].drv_softc != NULL); 715 chp->ch_drive[i].drv_softc = NULL; 716 chp->ch_drive[i].drive_flags = 0; 717 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 718 } 719 } 720 721 /* 722 * Detach our other children. 723 */ 724 for (i = 0; i < chp->ch_ndrives; i++) { 725 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 726 continue; 727 if (child == chp->ch_drive[i].drv_softc) { 728 chp->ch_drive[i].drv_softc = NULL; 729 chp->ch_drive[i].drive_flags = 0; 730 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 731 chp->ch_satapmp_nports = 0; 732 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 733 found = true; 734 } 735 } 736 737 if (!found) 738 panic("%s: unknown child %p", device_xname(self), 739 (const void *)child); 740 } 741 742 CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc), 743 atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan, 744 atabus_childdetached, DVF_DETACH_SHUTDOWN); 745 746 /***************************************************************************** 747 * Common ATA bus operations. 748 *****************************************************************************/ 749 750 /* allocate/free the channel's ch_drive[] array */ 751 int 752 atabus_alloc_drives(struct ata_channel *chp, int ndrives) 753 { 754 int i; 755 if (chp->ch_ndrives != ndrives) 756 atabus_free_drives(chp); 757 if (chp->ch_drive == NULL) { 758 void *drv; 759 760 ata_channel_unlock(chp); 761 drv = kmem_zalloc(sizeof(*chp->ch_drive) * ndrives, KM_SLEEP); 762 ata_channel_lock(chp); 763 764 if (chp->ch_drive != NULL) { 765 /* lost the race */ 766 kmem_free(drv, sizeof(*chp->ch_drive) * ndrives); 767 return 0; 768 } 769 chp->ch_drive = drv; 770 } 771 for (i = 0; i < ndrives; i++) { 772 chp->ch_drive[i].chnl_softc = chp; 773 chp->ch_drive[i].drive = i; 774 } 775 chp->ch_ndrives = ndrives; 776 return 0; 777 } 778 779 void 780 atabus_free_drives(struct ata_channel *chp) 781 { 782 #ifdef DIAGNOSTIC 783 int i; 784 int dopanic = 0; 785 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 786 for (i = 0; i < chp->ch_ndrives; i++) { 787 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) { 788 printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n", 789 device_xname(chp->atabus), i, 790 chp->ch_drive[i].drive_type); 791 dopanic = 1; 792 } 793 if (chp->ch_drive[i].drv_softc != NULL) { 794 printf("%s: ch_drive[%d] attached to %s\n", 795 device_xname(chp->atabus), i, 796 device_xname(chp->ch_drive[i].drv_softc)); 797 dopanic = 1; 798 } 799 } 800 if (dopanic) 801 panic("atabus_free_drives"); 802 #endif 803 804 if (chp->ch_drive == NULL) 805 return; 806 kmem_free(chp->ch_drive, 807 sizeof(struct ata_drive_datas) * chp->ch_ndrives); 808 chp->ch_ndrives = 0; 809 chp->ch_drive = NULL; 810 } 811 812 /* Get the disk's parameters */ 813 int 814 ata_get_params(struct ata_drive_datas *drvp, uint8_t flags, 815 struct ataparams *prms) 816 { 817 struct ata_xfer *xfer; 818 struct ata_channel *chp = drvp->chnl_softc; 819 struct atac_softc *atac = chp->ch_atac; 820 char *tb; 821 int i, rv; 822 uint16_t *p; 823 824 ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS); 825 826 xfer = ata_get_xfer(chp, false); 827 if (xfer == NULL) { 828 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 829 DEBUG_FUNCS|DEBUG_PROBE); 830 return CMD_AGAIN; 831 } 832 833 tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP); 834 memset(prms, 0, sizeof(struct ataparams)); 835 836 if (drvp->drive_type == ATA_DRIVET_ATA) { 837 xfer->c_ata_c.r_command = WDCC_IDENTIFY; 838 xfer->c_ata_c.r_st_bmask = WDCS_DRDY; 839 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 840 xfer->c_ata_c.timeout = 3000; /* 3s */ 841 } else if (drvp->drive_type == ATA_DRIVET_ATAPI) { 842 xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE; 843 xfer->c_ata_c.r_st_bmask = 0; 844 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 845 xfer->c_ata_c.timeout = 10000; /* 10s */ 846 } else { 847 ATADEBUG_PRINT(("ata_get_parms: no disks\n"), 848 DEBUG_FUNCS|DEBUG_PROBE); 849 rv = CMD_ERR; 850 goto out; 851 } 852 xfer->c_ata_c.flags = AT_READ | flags; 853 xfer->c_ata_c.data = tb; 854 xfer->c_ata_c.bcount = ATA_BSIZE; 855 (*atac->atac_bustype_ata->ata_exec_command)(drvp, xfer); 856 ata_wait_cmd(chp, xfer); 857 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 858 ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n", 859 xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE); 860 rv = CMD_ERR; 861 goto out; 862 } 863 /* if we didn't read any data something is wrong */ 864 if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) { 865 rv = CMD_ERR; 866 goto out; 867 } 868 869 /* Read in parameter block. */ 870 memcpy(prms, tb, sizeof(struct ataparams)); 871 872 /* 873 * Shuffle string byte order. 874 * ATAPI NEC, Mitsumi and Pioneer drives and 875 * old ATA TDK CompactFlash cards 876 * have different byte order. 877 */ 878 #if BYTE_ORDER == BIG_ENDIAN 879 # define M(n) prms->atap_model[(n) ^ 1] 880 #else 881 # define M(n) prms->atap_model[n] 882 #endif 883 if ( 884 #if BYTE_ORDER == BIG_ENDIAN 885 ! 886 #endif 887 ((drvp->drive_type == ATA_DRIVET_ATAPI) ? 888 ((M(0) == 'N' && M(1) == 'E') || 889 (M(0) == 'F' && M(1) == 'X') || 890 (M(0) == 'P' && M(1) == 'i')) : 891 ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) { 892 rv = CMD_OK; 893 goto out; 894 } 895 #undef M 896 for (i = 0; i < sizeof(prms->atap_model); i += 2) { 897 p = (uint16_t *)(prms->atap_model + i); 898 *p = bswap16(*p); 899 } 900 for (i = 0; i < sizeof(prms->atap_serial); i += 2) { 901 p = (uint16_t *)(prms->atap_serial + i); 902 *p = bswap16(*p); 903 } 904 for (i = 0; i < sizeof(prms->atap_revision); i += 2) { 905 p = (uint16_t *)(prms->atap_revision + i); 906 *p = bswap16(*p); 907 } 908 909 rv = CMD_OK; 910 out: 911 kmem_free(tb, ATA_BSIZE); 912 ata_free_xfer(chp, xfer); 913 return rv; 914 } 915 916 int 917 ata_set_pio8(struct ata_drive_datas *drvp, uint8_t flags) 918 { 919 struct ata_xfer *xfer; 920 int rv; 921 struct ata_channel *chp = drvp->chnl_softc; 922 struct atac_softc *atac = chp->ch_atac; 923 924 ATADEBUG_PRINT(("ata_set_pio8\n"), DEBUG_FUNCS); 925 926 xfer = ata_get_xfer(chp, false); 927 if (xfer == NULL) { 928 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 929 DEBUG_FUNCS|DEBUG_PROBE); 930 return CMD_AGAIN; 931 } 932 933 xfer->c_ata_c.r_command = SET_FEATURES; 934 xfer->c_ata_c.r_st_bmask = 0; 935 xfer->c_ata_c.r_st_pmask = 0; 936 xfer->c_ata_c.r_features = WDSF_8BIT_PIO_EN; 937 xfer->c_ata_c.r_count = 0; 938 xfer->c_ata_c.flags = flags; 939 xfer->c_ata_c.timeout = 1000; /* 1s */ 940 (*atac->atac_bustype_ata->ata_exec_command)(drvp, xfer); 941 ata_wait_cmd(chp, xfer); 942 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 943 rv = CMD_ERR; 944 goto out; 945 } 946 947 rv = CMD_OK; 948 949 out: 950 ata_free_xfer(chp, xfer); 951 return rv; 952 } 953 954 int 955 ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags) 956 { 957 struct ata_xfer *xfer; 958 int rv; 959 struct ata_channel *chp = drvp->chnl_softc; 960 struct atac_softc *atac = chp->ch_atac; 961 962 ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS); 963 964 xfer = ata_get_xfer(chp, false); 965 if (xfer == NULL) { 966 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 967 DEBUG_FUNCS|DEBUG_PROBE); 968 return CMD_AGAIN; 969 } 970 971 xfer->c_ata_c.r_command = SET_FEATURES; 972 xfer->c_ata_c.r_st_bmask = 0; 973 xfer->c_ata_c.r_st_pmask = 0; 974 xfer->c_ata_c.r_features = WDSF_SET_MODE; 975 xfer->c_ata_c.r_count = mode; 976 xfer->c_ata_c.flags = flags; 977 xfer->c_ata_c.timeout = 1000; /* 1s */ 978 (*atac->atac_bustype_ata->ata_exec_command)(drvp, xfer); 979 ata_wait_cmd(chp, xfer); 980 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 981 rv = CMD_ERR; 982 goto out; 983 } 984 985 rv = CMD_OK; 986 987 out: 988 ata_free_xfer(chp, xfer); 989 return rv; 990 } 991 992 #if NATA_DMA 993 void 994 ata_dmaerr(struct ata_drive_datas *drvp, int flags) 995 { 996 ata_channel_lock_owned(drvp->chnl_softc); 997 998 /* 999 * Downgrade decision: if we get NERRS_MAX in NXFER. 1000 * We start with n_dmaerrs set to NERRS_MAX-1 so that the 1001 * first error within the first NXFER ops will immediately trigger 1002 * a downgrade. 1003 * If we got an error and n_xfers is bigger than NXFER reset counters. 1004 */ 1005 drvp->n_dmaerrs++; 1006 if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) { 1007 #if !defined(ATA_NO_DOWNGRADE_MODE) 1008 ata_downgrade_mode(drvp, flags); 1009 drvp->n_dmaerrs = NERRS_MAX-1; 1010 #else 1011 static struct timeval last; 1012 static const struct timeval serrintvl = { 300, 0 }; 1013 1014 if (ratecheck(&last, &serrintvl)) { 1015 device_printf(drvp->drv_softc, 1016 "excessive DMA errors - %d in last %d transfers\n", 1017 drvp->n_dmaerrs, drvp->n_xfers); 1018 } 1019 #endif 1020 drvp->n_xfers = 0; 1021 return; 1022 } 1023 if (drvp->n_xfers > NXFER) { 1024 drvp->n_dmaerrs = 1; /* just got an error */ 1025 drvp->n_xfers = 1; /* restart counting from this error */ 1026 } 1027 } 1028 #endif /* NATA_DMA */ 1029 1030 /* 1031 * freeze the queue and wait for the controller to be idle. Caller has to 1032 * unfreeze/restart the queue 1033 */ 1034 static void 1035 ata_channel_idle(struct ata_channel *chp) 1036 { 1037 ata_channel_lock(chp); 1038 ata_channel_freeze_locked(chp); 1039 while (chp->ch_queue->queue_active > 0) { 1040 chp->ch_queue->queue_flags |= QF_IDLE_WAIT; 1041 cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1); 1042 } 1043 ata_channel_unlock(chp); 1044 } 1045 1046 /* 1047 * Add a command to the queue and start controller. 1048 * 1049 * MUST BE CALLED AT splbio()! 1050 */ 1051 void 1052 ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1053 { 1054 1055 ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer, 1056 chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1057 1058 /* complete xfer setup */ 1059 xfer->c_chp = chp; 1060 1061 ata_channel_lock(chp); 1062 1063 /* 1064 * Standard commands are added to the end of command list, but 1065 * recovery commands must be run immediately. 1066 */ 1067 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1068 SIMPLEQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer, 1069 c_xferchain); 1070 else 1071 SIMPLEQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer, 1072 c_xferchain); 1073 1074 /* 1075 * if polling and can sleep, wait for the xfer to be at head of queue 1076 */ 1077 if ((xfer->c_flags & (C_POLL | C_WAIT)) == (C_POLL | C_WAIT)) { 1078 while (chp->ch_queue->queue_active > 0 || 1079 SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) { 1080 xfer->c_flags |= C_WAITACT; 1081 cv_wait(&chp->ch_queue->c_active, &chp->ch_lock); 1082 xfer->c_flags &= ~C_WAITACT; 1083 } 1084 1085 /* 1086 * Free xfer now if it there was attempt to free it 1087 * while we were waiting. 1088 */ 1089 if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) { 1090 ata_channel_unlock(chp); 1091 1092 ata_free_xfer(chp, xfer); 1093 return; 1094 } 1095 } 1096 1097 ata_channel_unlock(chp); 1098 1099 ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n", 1100 chp->ch_flags), DEBUG_XFERS); 1101 atastart(chp); 1102 } 1103 1104 /* 1105 * Start I/O on a controller, for the given channel. 1106 * The first xfer may be not for our channel if the channel queues 1107 * are shared. 1108 * 1109 * MUST BE CALLED AT splbio()! 1110 * 1111 * XXX FIS-based switching with PMP 1112 * Currently atastart() never schedules concurrent NCQ transfers to more than 1113 * one drive, even when channel has several SATA drives attached via PMP. 1114 * To support concurrent transfers to different drives with PMP, it would be 1115 * necessary to implement FIS-based switching support in controller driver, 1116 * and then adjust error handling and recovery to stop assuming at most 1117 * one active drive. 1118 */ 1119 void 1120 atastart(struct ata_channel *chp) 1121 { 1122 struct atac_softc *atac = chp->ch_atac; 1123 struct ata_queue *chq = chp->ch_queue; 1124 struct ata_xfer *xfer, *axfer; 1125 bool skipq; 1126 1127 #ifdef ATA_DEBUG 1128 int spl1, spl2; 1129 1130 spl1 = splbio(); 1131 spl2 = splbio(); 1132 if (spl2 != spl1) { 1133 printf("atastart: not at splbio()\n"); 1134 panic("atastart"); 1135 } 1136 splx(spl2); 1137 splx(spl1); 1138 #endif /* ATA_DEBUG */ 1139 1140 ata_channel_lock(chp); 1141 1142 again: 1143 /* is there a xfer ? */ 1144 if ((xfer = SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL) { 1145 ATADEBUG_PRINT(("%s(chp=%p): channel %d queue_xfer is empty\n", 1146 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1147 goto out; 1148 } 1149 1150 /* 1151 * if someone is waiting for the command to be active, wake it up 1152 * and let it process the command 1153 */ 1154 if (__predict_false(xfer->c_flags & C_WAITACT)) { 1155 ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d " 1156 "wait active\n", xfer, chp->ch_channel, xfer->c_drive), 1157 DEBUG_XFERS); 1158 cv_broadcast(&chp->ch_queue->c_active); 1159 goto out; 1160 } 1161 1162 skipq = ISSET(xfer->c_flags, C_SKIP_QUEUE); 1163 1164 /* is the queue frozen? */ 1165 if (__predict_false(!skipq && chq->queue_freeze > 0)) { 1166 if (chq->queue_flags & QF_IDLE_WAIT) { 1167 chq->queue_flags &= ~QF_IDLE_WAIT; 1168 cv_signal(&chp->ch_queue->queue_idle); 1169 } 1170 ATADEBUG_PRINT(("%s(chp=%p): channel %d drive %d " 1171 "queue frozen: %d\n", 1172 __func__, chp, chp->ch_channel, xfer->c_drive, 1173 chq->queue_freeze), 1174 DEBUG_XFERS); 1175 goto out; 1176 } 1177 1178 /* all xfers on same queue must belong to the same channel */ 1179 KASSERT(xfer->c_chp == chp); 1180 1181 /* 1182 * Can only take the command if there are no current active 1183 * commands, or if the command is NCQ and the active commands are also 1184 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based 1185 * switching, can only send commands to single drive. 1186 * Need only check first xfer. 1187 * XXX FIS-based switching - revisit 1188 */ 1189 if (!skipq && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) { 1190 if (!ISSET(xfer->c_flags, C_NCQ) || 1191 !ISSET(axfer->c_flags, C_NCQ) || 1192 xfer->c_drive != axfer->c_drive) 1193 goto out; 1194 } 1195 1196 struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive]; 1197 1198 /* 1199 * Are we on limit of active xfers ? If the queue has more 1200 * than 1 openings, we keep one slot reserved for recovery or dump. 1201 */ 1202 KASSERT(chq->queue_active <= chq->queue_openings); 1203 const uint8_t chq_openings = (!skipq && chq->queue_openings > 1) 1204 ? (chq->queue_openings - 1) : chq->queue_openings; 1205 const uint8_t drv_openings = ISSET(xfer->c_flags, C_NCQ) 1206 ? drvp->drv_openings : ATA_MAX_OPENINGS; 1207 if (chq->queue_active >= MIN(chq_openings, drv_openings)) { 1208 if (skipq) { 1209 panic("%s: channel %d busy, xfer not possible", 1210 __func__, chp->ch_channel); 1211 } 1212 1213 ATADEBUG_PRINT(("%s(chp=%p): channel %d completely busy\n", 1214 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1215 goto out; 1216 } 1217 1218 /* Slot allocation can fail if drv_openings < ch_openings */ 1219 if (!ata_queue_alloc_slot(chp, &xfer->c_slot, drv_openings)) 1220 goto out; 1221 1222 if (__predict_false(atac->atac_claim_hw)) { 1223 if (!atac->atac_claim_hw(chp, 0)) { 1224 ata_queue_free_slot(chp, xfer->c_slot); 1225 goto out; 1226 } 1227 } 1228 1229 /* Now committed to start the xfer */ 1230 1231 ATADEBUG_PRINT(("%s(chp=%p): xfer %p channel %d drive %d\n", 1232 __func__, chp, xfer, chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1233 if (drvp->drive_flags & ATA_DRIVE_RESET) { 1234 drvp->drive_flags &= ~ATA_DRIVE_RESET; 1235 drvp->state = 0; 1236 } 1237 1238 if (ISSET(xfer->c_flags, C_NCQ)) 1239 SET(chp->ch_flags, ATACH_NCQ); 1240 else 1241 CLR(chp->ch_flags, ATACH_NCQ); 1242 1243 SIMPLEQ_REMOVE_HEAD(&chq->queue_xfer, c_xferchain); 1244 1245 ata_activate_xfer_locked(chp, xfer); 1246 1247 if (atac->atac_cap & ATAC_CAP_NOIRQ) 1248 KASSERT(xfer->c_flags & C_POLL); 1249 1250 switch (ata_xfer_start(xfer)) { 1251 case ATASTART_TH: 1252 case ATASTART_ABORT: 1253 /* don't start any further commands in this case */ 1254 goto out; 1255 default: 1256 /* nothing to do */ 1257 break; 1258 } 1259 1260 /* Queue more commands if possible, but not during recovery or dump */ 1261 if (!skipq && chq->queue_active < chq->queue_openings) 1262 goto again; 1263 1264 out: 1265 ata_channel_unlock(chp); 1266 } 1267 1268 int 1269 ata_xfer_start(struct ata_xfer *xfer) 1270 { 1271 struct ata_channel *chp = xfer->c_chp; 1272 int rv, status; 1273 1274 KASSERT(mutex_owned(&chp->ch_lock)); 1275 1276 again: 1277 rv = xfer->ops->c_start(chp, xfer); 1278 switch (rv) { 1279 case ATASTART_STARTED: 1280 /* nothing to do */ 1281 break; 1282 case ATASTART_TH: 1283 /* postpone xfer to thread */ 1284 ata_thread_wake_locked(chp); 1285 break; 1286 case ATASTART_POLL: 1287 /* can happen even in thread context for some ATAPI devices */ 1288 ata_channel_unlock(chp); 1289 KASSERT(xfer->ops != NULL && xfer->ops->c_poll != NULL); 1290 status = xfer->ops->c_poll(chp, xfer); 1291 ata_channel_lock(chp); 1292 if (status == ATAPOLL_AGAIN) 1293 goto again; 1294 break; 1295 case ATASTART_ABORT: 1296 ata_channel_unlock(chp); 1297 KASSERT(xfer->ops != NULL && xfer->ops->c_abort != NULL); 1298 xfer->ops->c_abort(chp, xfer); 1299 ata_channel_lock(chp); 1300 break; 1301 } 1302 1303 return rv; 1304 } 1305 1306 static void 1307 ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer) 1308 { 1309 struct ata_queue * const chq = chp->ch_queue; 1310 1311 KASSERT(mutex_owned(&chp->ch_lock)); 1312 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0); 1313 1314 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1315 TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain); 1316 else { 1317 /* 1318 * Must go to head, so that ata_queue_get_active_xfer() 1319 * returns the recovery command, and not some other 1320 * random active transfer. 1321 */ 1322 TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain); 1323 } 1324 chq->active_xfers_used |= __BIT(xfer->c_slot); 1325 chq->queue_active++; 1326 } 1327 1328 /* 1329 * Does it's own locking, does not require splbio(). 1330 * flags - whether to block waiting for free xfer 1331 */ 1332 struct ata_xfer * 1333 ata_get_xfer(struct ata_channel *chp, bool waitok) 1334 { 1335 return pool_get(&ata_xfer_pool, 1336 PR_ZERO | (waitok ? PR_WAITOK : PR_NOWAIT)); 1337 } 1338 1339 /* 1340 * ata_deactivate_xfer() must be always called prior to ata_free_xfer() 1341 */ 1342 void 1343 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1344 { 1345 struct ata_queue *chq = chp->ch_queue; 1346 1347 ata_channel_lock(chp); 1348 1349 if (__predict_false(xfer->c_flags & (C_WAITACT|C_WAITTIMO))) { 1350 /* Someone is waiting for this xfer, so we can't free now */ 1351 xfer->c_flags |= C_FREE; 1352 cv_broadcast(&chq->c_active); 1353 ata_channel_unlock(chp); 1354 return; 1355 } 1356 1357 /* XXX move PIOBM and free_gw to deactivate? */ 1358 #if NATA_PIOBM /* XXX wdc dependent code */ 1359 if (__predict_false(xfer->c_flags & C_PIOBM)) { 1360 struct wdc_softc *wdc = CHAN_TO_WDC(chp); 1361 1362 /* finish the busmastering PIO */ 1363 (*wdc->piobm_done)(wdc->dma_arg, 1364 chp->ch_channel, xfer->c_drive); 1365 chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT); 1366 } 1367 #endif 1368 1369 if (__predict_false(chp->ch_atac->atac_free_hw)) 1370 chp->ch_atac->atac_free_hw(chp); 1371 1372 ata_channel_unlock(chp); 1373 1374 if (__predict_true(!ISSET(xfer->c_flags, C_PRIVATE_ALLOC))) 1375 pool_put(&ata_xfer_pool, xfer); 1376 } 1377 1378 void 1379 ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1380 { 1381 struct ata_queue * const chq = chp->ch_queue; 1382 1383 ata_channel_lock(chp); 1384 1385 KASSERT(chq->queue_active > 0); 1386 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0); 1387 1388 /* Stop only when this is last active xfer */ 1389 if (chq->queue_active == 1) 1390 callout_stop(&chp->c_timo_callout); 1391 1392 if (callout_invoking(&chp->c_timo_callout)) 1393 xfer->c_flags |= C_WAITTIMO; 1394 1395 TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain); 1396 chq->active_xfers_used &= ~__BIT(xfer->c_slot); 1397 chq->queue_active--; 1398 1399 ata_queue_free_slot(chp, xfer->c_slot); 1400 1401 if (xfer->c_flags & C_WAIT) 1402 cv_broadcast(&chq->c_cmd_finish); 1403 1404 ata_channel_unlock(chp); 1405 } 1406 1407 /* 1408 * Called in c_intr hook. Must be called before before any deactivations 1409 * are done - if there is drain pending, it calls c_kill_xfer hook which 1410 * deactivates the xfer. 1411 * Calls c_kill_xfer with channel lock free. 1412 * Returns true if caller should just exit without further processing. 1413 * Caller must not further access any part of xfer or any related controller 1414 * structures in that case, it should just return. 1415 */ 1416 bool 1417 ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer) 1418 { 1419 int drive = xfer->c_drive; 1420 bool draining = false; 1421 1422 ata_channel_lock(chp); 1423 1424 if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) { 1425 ata_channel_unlock(chp); 1426 1427 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE); 1428 1429 ata_channel_lock(chp); 1430 chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN; 1431 cv_signal(&chp->ch_queue->queue_drain); 1432 draining = true; 1433 } 1434 1435 ata_channel_unlock(chp); 1436 1437 return draining; 1438 } 1439 1440 /* 1441 * Check for race of normal transfer handling vs. timeout. 1442 */ 1443 bool 1444 ata_timo_xfer_check(struct ata_xfer *xfer) 1445 { 1446 struct ata_channel *chp = xfer->c_chp; 1447 struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive]; 1448 1449 ata_channel_lock(chp); 1450 1451 if (xfer->c_flags & C_WAITTIMO) { 1452 xfer->c_flags &= ~C_WAITTIMO; 1453 1454 /* Handle race vs. ata_free_xfer() */ 1455 if (xfer->c_flags & C_FREE) { 1456 xfer->c_flags &= ~C_FREE; 1457 ata_channel_unlock(chp); 1458 1459 device_printf(drvp->drv_softc, 1460 "xfer %"PRIxPTR" freed while invoking timeout\n", 1461 (intptr_t)xfer & PAGE_MASK); 1462 1463 ata_free_xfer(chp, xfer); 1464 return true; 1465 } 1466 1467 /* Race vs. callout_stop() in ata_deactivate_xfer() */ 1468 ata_channel_unlock(chp); 1469 1470 device_printf(drvp->drv_softc, 1471 "xfer %"PRIxPTR" deactivated while invoking timeout\n", 1472 (intptr_t)xfer & PAGE_MASK); 1473 return true; 1474 } 1475 1476 ata_channel_unlock(chp); 1477 1478 /* No race, proceed with timeout handling */ 1479 return false; 1480 } 1481 1482 /* 1483 * Kill off all active xfers for a ata_channel. 1484 * 1485 * Must be called with channel lock held. 1486 */ 1487 void 1488 ata_kill_active(struct ata_channel *chp, int reason, int flags) 1489 { 1490 struct ata_queue * const chq = chp->ch_queue; 1491 struct ata_xfer *xfer, *xfernext; 1492 1493 KASSERT(mutex_owned(&chp->ch_lock)); 1494 1495 TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) { 1496 ata_channel_unlock(chp); 1497 xfer->ops->c_kill_xfer(xfer->c_chp, xfer, reason); 1498 ata_channel_lock(chp); 1499 } 1500 } 1501 1502 /* 1503 * Kill off all pending xfers for a drive. 1504 */ 1505 void 1506 ata_kill_pending(struct ata_drive_datas *drvp) 1507 { 1508 struct ata_channel * const chp = drvp->chnl_softc; 1509 struct ata_queue * const chq = chp->ch_queue; 1510 struct ata_xfer *xfer; 1511 1512 ata_channel_lock(chp); 1513 1514 /* Kill all pending transfers */ 1515 while ((xfer = SIMPLEQ_FIRST(&chq->queue_xfer))) { 1516 KASSERT(xfer->c_chp == chp); 1517 1518 if (xfer->c_drive != drvp->drive) 1519 continue; 1520 1521 SIMPLEQ_REMOVE_HEAD(&chp->ch_queue->queue_xfer, c_xferchain); 1522 1523 /* 1524 * Keep the lock, so that we get deadlock (and 'locking against 1525 * myself' with LOCKDEBUG), instead of silent 1526 * data corruption, if the hook tries to call back into 1527 * middle layer for inactive xfer. 1528 */ 1529 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE_INACTIVE); 1530 } 1531 1532 /* Wait until all active transfers on the drive finish */ 1533 while (chq->queue_active > 0) { 1534 bool drv_active = false; 1535 1536 TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) { 1537 KASSERT(xfer->c_chp == chp); 1538 1539 if (xfer->c_drive == drvp->drive) { 1540 drv_active = true; 1541 break; 1542 } 1543 } 1544 1545 if (!drv_active) { 1546 /* all finished */ 1547 break; 1548 } 1549 1550 drvp->drive_flags |= ATA_DRIVE_WAITDRAIN; 1551 cv_wait(&chq->queue_drain, &chp->ch_lock); 1552 } 1553 1554 ata_channel_unlock(chp); 1555 } 1556 1557 static void 1558 ata_channel_freeze_locked(struct ata_channel *chp) 1559 { 1560 chp->ch_queue->queue_freeze++; 1561 1562 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1563 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1564 } 1565 1566 void 1567 ata_channel_freeze(struct ata_channel *chp) 1568 { 1569 ata_channel_lock(chp); 1570 ata_channel_freeze_locked(chp); 1571 ata_channel_unlock(chp); 1572 } 1573 1574 void 1575 ata_channel_thaw_locked(struct ata_channel *chp) 1576 { 1577 KASSERT(mutex_owned(&chp->ch_lock)); 1578 KASSERT(chp->ch_queue->queue_freeze > 0); 1579 1580 chp->ch_queue->queue_freeze--; 1581 1582 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1583 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1584 } 1585 1586 /* 1587 * ata_thread_run: 1588 * 1589 * Reset and ATA channel. Channel lock must be held. arg is type-specific. 1590 */ 1591 void 1592 ata_thread_run(struct ata_channel *chp, int flags, int type, int arg) 1593 { 1594 struct atac_softc *atac = chp->ch_atac; 1595 bool threset = false; 1596 struct ata_drive_datas *drvp; 1597 1598 ata_channel_lock_owned(chp); 1599 1600 /* 1601 * If we can poll or wait it's OK, otherwise wake up the 1602 * kernel thread to do it for us. 1603 */ 1604 ATADEBUG_PRINT(("%s flags 0x%x ch_flags 0x%x\n", 1605 __func__, flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS); 1606 if ((flags & (AT_POLL | AT_WAIT)) == 0) { 1607 switch (type) { 1608 case ATACH_TH_RESET: 1609 if (chp->ch_flags & ATACH_TH_RESET) { 1610 /* No need to schedule another reset */ 1611 return; 1612 } 1613 break; 1614 case ATACH_TH_DRIVE_RESET: 1615 { 1616 int drive = arg; 1617 1618 KASSERT(drive <= chp->ch_ndrives); 1619 drvp = &chp->ch_drive[drive]; 1620 1621 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 1622 /* No need to schedule another reset */ 1623 return; 1624 } 1625 drvp->drive_flags |= ATA_DRIVE_TH_RESET; 1626 break; 1627 } 1628 case ATACH_TH_RECOVERY: 1629 { 1630 uint32_t tfd = (uint32_t)arg; 1631 1632 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1633 chp->recovery_tfd = tfd; 1634 break; 1635 } 1636 default: 1637 panic("%s: unknown type: %x", __func__, type); 1638 /* NOTREACHED */ 1639 } 1640 1641 if (!(chp->ch_flags & type)) { 1642 /* 1643 * Block execution of other commands while 1644 * reset is scheduled to a thread. 1645 */ 1646 ata_channel_freeze_locked(chp); 1647 chp->ch_flags |= type; 1648 } 1649 1650 cv_signal(&chp->ch_thr_idle); 1651 return; 1652 } 1653 1654 /* Block execution of other commands during reset */ 1655 ata_channel_freeze_locked(chp); 1656 1657 /* 1658 * If reset has been scheduled to a thread, then clear 1659 * the flag now so that the thread won't try to execute it if 1660 * we happen to sleep, and thaw one more time after the reset. 1661 */ 1662 if (chp->ch_flags & type) { 1663 chp->ch_flags &= ~type; 1664 threset = true; 1665 } 1666 1667 switch (type) { 1668 case ATACH_TH_RESET: 1669 (*atac->atac_bustype_ata->ata_reset_channel)(chp, flags); 1670 1671 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1672 for (int drive = 0; drive < chp->ch_ndrives; drive++) 1673 chp->ch_drive[drive].state = 0; 1674 break; 1675 1676 case ATACH_TH_DRIVE_RESET: 1677 { 1678 int drive = arg; 1679 1680 KASSERT(drive <= chp->ch_ndrives); 1681 drvp = &chp->ch_drive[drive]; 1682 (*atac->atac_bustype_ata->ata_reset_drive)(drvp, flags, NULL); 1683 drvp->state = 0; 1684 break; 1685 } 1686 1687 case ATACH_TH_RECOVERY: 1688 { 1689 uint32_t tfd = (uint32_t)arg; 1690 1691 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1692 KASSERT(atac->atac_bustype_ata->ata_recovery != NULL); 1693 1694 SET(chp->ch_flags, ATACH_RECOVERING); 1695 (*atac->atac_bustype_ata->ata_recovery)(chp, flags, tfd); 1696 CLR(chp->ch_flags, ATACH_RECOVERING); 1697 break; 1698 } 1699 1700 default: 1701 panic("%s: unknown type: %x", __func__, type); 1702 /* NOTREACHED */ 1703 } 1704 1705 /* 1706 * Thaw one extra time to clear the freeze done when the reset has 1707 * been scheduled to the thread. 1708 */ 1709 if (threset) 1710 ata_channel_thaw_locked(chp); 1711 1712 /* Allow commands to run again */ 1713 ata_channel_thaw_locked(chp); 1714 1715 /* Signal the thread in case there is an xfer to run */ 1716 cv_signal(&chp->ch_thr_idle); 1717 } 1718 1719 int 1720 ata_addref(struct ata_channel *chp) 1721 { 1722 struct atac_softc *atac = chp->ch_atac; 1723 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1724 int s, error = 0; 1725 1726 s = splbio(); 1727 if (adapt->adapt_refcnt++ == 0 && 1728 adapt->adapt_enable != NULL) { 1729 error = (*adapt->adapt_enable)(atac->atac_dev, 1); 1730 if (error) 1731 adapt->adapt_refcnt--; 1732 } 1733 splx(s); 1734 return (error); 1735 } 1736 1737 void 1738 ata_delref(struct ata_channel *chp) 1739 { 1740 struct atac_softc *atac = chp->ch_atac; 1741 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1742 int s; 1743 1744 s = splbio(); 1745 if (adapt->adapt_refcnt-- == 1 && 1746 adapt->adapt_enable != NULL) 1747 (void) (*adapt->adapt_enable)(atac->atac_dev, 0); 1748 splx(s); 1749 } 1750 1751 void 1752 ata_print_modes(struct ata_channel *chp) 1753 { 1754 struct atac_softc *atac = chp->ch_atac; 1755 int drive; 1756 struct ata_drive_datas *drvp; 1757 1758 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1759 for (drive = 0; drive < chp->ch_ndrives; drive++) { 1760 drvp = &chp->ch_drive[drive]; 1761 if (drvp->drive_type == ATA_DRIVET_NONE || 1762 drvp->drv_softc == NULL) 1763 continue; 1764 aprint_verbose("%s(%s:%d:%d): using PIO mode %d", 1765 device_xname(drvp->drv_softc), 1766 device_xname(atac->atac_dev), 1767 chp->ch_channel, drvp->drive, drvp->PIO_mode); 1768 #if NATA_DMA 1769 if (drvp->drive_flags & ATA_DRIVE_DMA) 1770 aprint_verbose(", DMA mode %d", drvp->DMA_mode); 1771 #if NATA_UDMA 1772 if (drvp->drive_flags & ATA_DRIVE_UDMA) { 1773 aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode); 1774 if (drvp->UDMA_mode == 2) 1775 aprint_verbose(" (Ultra/33)"); 1776 else if (drvp->UDMA_mode == 4) 1777 aprint_verbose(" (Ultra/66)"); 1778 else if (drvp->UDMA_mode == 5) 1779 aprint_verbose(" (Ultra/100)"); 1780 else if (drvp->UDMA_mode == 6) 1781 aprint_verbose(" (Ultra/133)"); 1782 } 1783 #endif /* NATA_UDMA */ 1784 #endif /* NATA_DMA */ 1785 #if NATA_DMA || NATA_PIOBM 1786 if (0 1787 #if NATA_DMA 1788 || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) 1789 #endif 1790 #if NATA_PIOBM 1791 /* PIOBM capable controllers use DMA for PIO commands */ 1792 || (atac->atac_cap & ATAC_CAP_PIOBM) 1793 #endif 1794 ) 1795 aprint_verbose(" (using DMA)"); 1796 1797 if (drvp->drive_flags & ATA_DRIVE_NCQ) { 1798 aprint_verbose(", NCQ (%d tags)%s", 1799 ATA_REAL_OPENINGS(chp->ch_queue->queue_openings), 1800 (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO) 1801 ? " w/PRIO" : ""); 1802 } else if (drvp->drive_flags & ATA_DRIVE_WFUA) 1803 aprint_verbose(", WRITE DMA FUA EXT"); 1804 1805 #endif /* NATA_DMA || NATA_PIOBM */ 1806 aprint_verbose("\n"); 1807 } 1808 } 1809 1810 #if !defined(ATA_NO_DOWNGRADE_MODE) && NATA_DMA 1811 /* 1812 * downgrade the transfer mode of a drive after an error. return 1 if 1813 * downgrade was possible, 0 otherwise. 1814 * 1815 * MUST BE CALLED AT splbio()! 1816 */ 1817 static int 1818 ata_downgrade_mode(struct ata_drive_datas *drvp, int flags) 1819 { 1820 struct ata_channel *chp = drvp->chnl_softc; 1821 struct atac_softc *atac = chp->ch_atac; 1822 device_t drv_dev = drvp->drv_softc; 1823 int cf_flags = device_cfdata(drv_dev)->cf_flags; 1824 1825 ata_channel_lock_owned(drvp->chnl_softc); 1826 1827 /* if drive or controller don't know its mode, we can't do much */ 1828 if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 || 1829 (atac->atac_set_modes == NULL)) 1830 return 0; 1831 /* current drive mode was set by a config flag, let it this way */ 1832 if ((cf_flags & ATA_CONFIG_PIO_SET) || 1833 (cf_flags & ATA_CONFIG_DMA_SET) || 1834 (cf_flags & ATA_CONFIG_UDMA_SET)) 1835 return 0; 1836 1837 #if NATA_UDMA 1838 /* 1839 * If we were using Ultra-DMA mode, downgrade to the next lower mode. 1840 */ 1841 if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) { 1842 drvp->UDMA_mode--; 1843 device_printf(drv_dev, 1844 "transfer error, downgrading to Ultra-DMA mode %d\n", 1845 drvp->UDMA_mode); 1846 } 1847 #endif 1848 1849 /* 1850 * If we were using ultra-DMA, don't downgrade to multiword DMA. 1851 */ 1852 else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) { 1853 drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA); 1854 drvp->PIO_mode = drvp->PIO_cap; 1855 device_printf(drv_dev, 1856 "transfer error, downgrading to PIO mode %d\n", 1857 drvp->PIO_mode); 1858 } else /* already using PIO, can't downgrade */ 1859 return 0; 1860 1861 (*atac->atac_set_modes)(chp); 1862 ata_print_modes(chp); 1863 /* reset the channel, which will schedule all drives for setup */ 1864 ata_thread_run(chp, flags, ATACH_TH_RESET, ATACH_NODRIVE); 1865 return 1; 1866 } 1867 #endif /* ATA_DOWNGRADE_MODE && NATA_DMA */ 1868 1869 /* 1870 * Probe drive's capabilities, for use by the controller later 1871 * Assumes drvp points to an existing drive. 1872 */ 1873 void 1874 ata_probe_caps(struct ata_drive_datas *drvp) 1875 { 1876 struct ataparams params, params2; 1877 struct ata_channel *chp = drvp->chnl_softc; 1878 struct atac_softc *atac = chp->ch_atac; 1879 device_t drv_dev = drvp->drv_softc; 1880 int i, printed = 0; 1881 const char *sep = ""; 1882 int cf_flags; 1883 1884 if (ata_get_params(drvp, AT_WAIT, ¶ms) != CMD_OK) { 1885 /* IDENTIFY failed. Can't tell more about the device */ 1886 return; 1887 } 1888 if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) == 1889 (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) { 1890 /* 1891 * Controller claims 16 and 32 bit transfers. 1892 * Re-do an IDENTIFY with 32-bit transfers, 1893 * and compare results. 1894 */ 1895 ata_channel_lock(chp); 1896 drvp->drive_flags |= ATA_DRIVE_CAP32; 1897 ata_channel_unlock(chp); 1898 ata_get_params(drvp, AT_WAIT, ¶ms2); 1899 if (memcmp(¶ms, ¶ms2, sizeof(struct ataparams)) != 0) { 1900 /* Not good. fall back to 16bits */ 1901 ata_channel_lock(chp); 1902 drvp->drive_flags &= ~ATA_DRIVE_CAP32; 1903 ata_channel_unlock(chp); 1904 } else { 1905 aprint_verbose_dev(drv_dev, "32-bit data port\n"); 1906 } 1907 } 1908 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */ 1909 if (params.atap_ata_major > 0x01 && 1910 params.atap_ata_major != 0xffff) { 1911 for (i = 14; i > 0; i--) { 1912 if (params.atap_ata_major & (1 << i)) { 1913 aprint_verbose_dev(drv_dev, 1914 "ATA version %d\n", i); 1915 drvp->ata_vers = i; 1916 break; 1917 } 1918 } 1919 } 1920 #endif 1921 1922 /* An ATAPI device is at last PIO mode 3 */ 1923 if (drvp->drive_type == ATA_DRIVET_ATAPI) 1924 drvp->PIO_mode = 3; 1925 1926 /* 1927 * It's not in the specs, but it seems that some drive 1928 * returns 0xffff in atap_extensions when this field is invalid 1929 */ 1930 if (params.atap_extensions != 0xffff && 1931 (params.atap_extensions & WDC_EXT_MODES)) { 1932 /* 1933 * XXX some drives report something wrong here (they claim to 1934 * support PIO mode 8 !). As mode is coded on 3 bits in 1935 * SET FEATURE, limit it to 7 (so limit i to 4). 1936 * If higher mode than 7 is found, abort. 1937 */ 1938 for (i = 7; i >= 0; i--) { 1939 if ((params.atap_piomode_supp & (1 << i)) == 0) 1940 continue; 1941 if (i > 4) 1942 return; 1943 /* 1944 * See if mode is accepted. 1945 * If the controller can't set its PIO mode, 1946 * assume the defaults are good, so don't try 1947 * to set it 1948 */ 1949 if (atac->atac_set_modes) 1950 /* 1951 * It's OK to poll here, it's fast enough 1952 * to not bother waiting for interrupt 1953 */ 1954 if (ata_set_mode(drvp, 0x08 | (i + 3), 1955 AT_WAIT) != CMD_OK) 1956 continue; 1957 if (!printed) { 1958 aprint_verbose_dev(drv_dev, 1959 "drive supports PIO mode %d", i + 3); 1960 sep = ","; 1961 printed = 1; 1962 } 1963 /* 1964 * If controller's driver can't set its PIO mode, 1965 * get the higher one for the drive. 1966 */ 1967 if (atac->atac_set_modes == NULL || 1968 atac->atac_pio_cap >= i + 3) { 1969 drvp->PIO_mode = i + 3; 1970 drvp->PIO_cap = i + 3; 1971 break; 1972 } 1973 } 1974 if (!printed) { 1975 /* 1976 * We didn't find a valid PIO mode. 1977 * Assume the values returned for DMA are buggy too 1978 */ 1979 return; 1980 } 1981 ata_channel_lock(chp); 1982 drvp->drive_flags |= ATA_DRIVE_MODE; 1983 ata_channel_unlock(chp); 1984 printed = 0; 1985 for (i = 7; i >= 0; i--) { 1986 if ((params.atap_dmamode_supp & (1 << i)) == 0) 1987 continue; 1988 #if NATA_DMA 1989 if ((atac->atac_cap & ATAC_CAP_DMA) && 1990 atac->atac_set_modes != NULL) 1991 if (ata_set_mode(drvp, 0x20 | i, AT_WAIT) 1992 != CMD_OK) 1993 continue; 1994 #endif 1995 if (!printed) { 1996 aprint_verbose("%s DMA mode %d", sep, i); 1997 sep = ","; 1998 printed = 1; 1999 } 2000 #if NATA_DMA 2001 if (atac->atac_cap & ATAC_CAP_DMA) { 2002 if (atac->atac_set_modes != NULL && 2003 atac->atac_dma_cap < i) 2004 continue; 2005 drvp->DMA_mode = i; 2006 drvp->DMA_cap = i; 2007 ata_channel_lock(chp); 2008 drvp->drive_flags |= ATA_DRIVE_DMA; 2009 ata_channel_unlock(chp); 2010 } 2011 #endif 2012 break; 2013 } 2014 if (params.atap_extensions & WDC_EXT_UDMA_MODES) { 2015 printed = 0; 2016 for (i = 7; i >= 0; i--) { 2017 if ((params.atap_udmamode_supp & (1 << i)) 2018 == 0) 2019 continue; 2020 #if NATA_UDMA 2021 if (atac->atac_set_modes != NULL && 2022 (atac->atac_cap & ATAC_CAP_UDMA)) 2023 if (ata_set_mode(drvp, 0x40 | i, 2024 AT_WAIT) != CMD_OK) 2025 continue; 2026 #endif 2027 if (!printed) { 2028 aprint_verbose("%s Ultra-DMA mode %d", 2029 sep, i); 2030 if (i == 2) 2031 aprint_verbose(" (Ultra/33)"); 2032 else if (i == 4) 2033 aprint_verbose(" (Ultra/66)"); 2034 else if (i == 5) 2035 aprint_verbose(" (Ultra/100)"); 2036 else if (i == 6) 2037 aprint_verbose(" (Ultra/133)"); 2038 sep = ","; 2039 printed = 1; 2040 } 2041 #if NATA_UDMA 2042 if (atac->atac_cap & ATAC_CAP_UDMA) { 2043 if (atac->atac_set_modes != NULL && 2044 atac->atac_udma_cap < i) 2045 continue; 2046 drvp->UDMA_mode = i; 2047 drvp->UDMA_cap = i; 2048 ata_channel_lock(chp); 2049 drvp->drive_flags |= ATA_DRIVE_UDMA; 2050 ata_channel_unlock(chp); 2051 } 2052 #endif 2053 break; 2054 } 2055 } 2056 } 2057 2058 ata_channel_lock(chp); 2059 drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM; 2060 if (drvp->drive_type == ATA_DRIVET_ATAPI) { 2061 if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM) 2062 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2063 } else { 2064 if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM) 2065 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2066 } 2067 ata_channel_unlock(chp); 2068 2069 /* Try to guess ATA version here, if it didn't get reported */ 2070 if (drvp->ata_vers == 0) { 2071 #if NATA_UDMA 2072 if (drvp->drive_flags & ATA_DRIVE_UDMA) 2073 drvp->ata_vers = 4; /* should be at last ATA-4 */ 2074 else 2075 #endif 2076 if (drvp->PIO_cap > 2) 2077 drvp->ata_vers = 2; /* should be at last ATA-2 */ 2078 } 2079 cf_flags = device_cfdata(drv_dev)->cf_flags; 2080 if (cf_flags & ATA_CONFIG_PIO_SET) { 2081 ata_channel_lock(chp); 2082 drvp->PIO_mode = 2083 (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF; 2084 drvp->drive_flags |= ATA_DRIVE_MODE; 2085 ata_channel_unlock(chp); 2086 } 2087 #if NATA_DMA 2088 if ((atac->atac_cap & ATAC_CAP_DMA) == 0) { 2089 /* don't care about DMA modes */ 2090 goto out; 2091 } 2092 if (cf_flags & ATA_CONFIG_DMA_SET) { 2093 ata_channel_lock(chp); 2094 if ((cf_flags & ATA_CONFIG_DMA_MODES) == 2095 ATA_CONFIG_DMA_DISABLE) { 2096 drvp->drive_flags &= ~ATA_DRIVE_DMA; 2097 } else { 2098 drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >> 2099 ATA_CONFIG_DMA_OFF; 2100 drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE; 2101 } 2102 ata_channel_unlock(chp); 2103 } 2104 2105 /* 2106 * Probe WRITE DMA FUA EXT. Support is mandatory for devices 2107 * supporting LBA48, but nevertheless confirm with the feature flag. 2108 */ 2109 if (drvp->drive_flags & ATA_DRIVE_DMA) { 2110 if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0 2111 && (params.atap_cmd_def & ATA_CMDE_WFE)) { 2112 drvp->drive_flags |= ATA_DRIVE_WFUA; 2113 aprint_verbose("%s WRITE DMA FUA", sep); 2114 sep = ","; 2115 } 2116 } 2117 2118 /* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */ 2119 ata_channel_lock(chp); 2120 drvp->drv_openings = 1; 2121 if (params.atap_sata_caps & SATA_NATIVE_CMDQ) { 2122 if (atac->atac_cap & ATAC_CAP_NCQ) 2123 drvp->drive_flags |= ATA_DRIVE_NCQ; 2124 drvp->drv_openings = 2125 (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1; 2126 aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings); 2127 sep = ","; 2128 2129 if (params.atap_sata_caps & SATA_NCQ_PRIO) { 2130 drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO; 2131 aprint_verbose(" w/PRIO"); 2132 } 2133 } 2134 ata_channel_unlock(chp); 2135 2136 #if NATA_UDMA 2137 if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) { 2138 /* don't care about UDMA modes */ 2139 goto out; 2140 } 2141 if (cf_flags & ATA_CONFIG_UDMA_SET) { 2142 ata_channel_lock(chp); 2143 if ((cf_flags & ATA_CONFIG_UDMA_MODES) == 2144 ATA_CONFIG_UDMA_DISABLE) { 2145 drvp->drive_flags &= ~ATA_DRIVE_UDMA; 2146 } else { 2147 drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >> 2148 ATA_CONFIG_UDMA_OFF; 2149 drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE; 2150 } 2151 ata_channel_unlock(chp); 2152 } 2153 #endif /* NATA_UDMA */ 2154 out: 2155 #endif /* NATA_DMA */ 2156 if (*sep != '\0') 2157 aprint_verbose("\n"); 2158 } 2159 2160 /* management of the /dev/atabus* devices */ 2161 int 2162 atabusopen(dev_t dev, int flag, int fmt, struct lwp *l) 2163 { 2164 struct atabus_softc *sc; 2165 int error; 2166 2167 sc = device_lookup_private(&atabus_cd, minor(dev)); 2168 if (sc == NULL) 2169 return (ENXIO); 2170 2171 if (sc->sc_flags & ATABUSCF_OPEN) 2172 return (EBUSY); 2173 2174 if ((error = ata_addref(sc->sc_chan)) != 0) 2175 return (error); 2176 2177 sc->sc_flags |= ATABUSCF_OPEN; 2178 2179 return (0); 2180 } 2181 2182 2183 int 2184 atabusclose(dev_t dev, int flag, int fmt, struct lwp *l) 2185 { 2186 struct atabus_softc *sc = 2187 device_lookup_private(&atabus_cd, minor(dev)); 2188 2189 ata_delref(sc->sc_chan); 2190 2191 sc->sc_flags &= ~ATABUSCF_OPEN; 2192 2193 return (0); 2194 } 2195 2196 int 2197 atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l) 2198 { 2199 struct atabus_softc *sc = 2200 device_lookup_private(&atabus_cd, minor(dev)); 2201 struct ata_channel *chp = sc->sc_chan; 2202 int min_drive, max_drive, drive; 2203 int error; 2204 2205 /* 2206 * Enforce write permission for ioctls that change the 2207 * state of the bus. Host adapter specific ioctls must 2208 * be checked by the adapter driver. 2209 */ 2210 switch (cmd) { 2211 case ATABUSIOSCAN: 2212 case ATABUSIODETACH: 2213 case ATABUSIORESET: 2214 if ((flag & FWRITE) == 0) 2215 return (EBADF); 2216 } 2217 2218 switch (cmd) { 2219 case ATABUSIORESET: 2220 ata_channel_lock(chp); 2221 ata_thread_run(sc->sc_chan, AT_WAIT | AT_POLL, 2222 ATACH_TH_RESET, ATACH_NODRIVE); 2223 ata_channel_unlock(chp); 2224 return 0; 2225 case ATABUSIOSCAN: 2226 { 2227 #if 0 2228 struct atabusioscan_args *a= 2229 (struct atabusioscan_args *)addr; 2230 #endif 2231 if ((chp->ch_ndrives > 0 && 2232 (chp->ch_drive[0].drive_type == ATA_DRIVET_OLD)) || 2233 (chp->ch_ndrives > 1 && 2234 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))) 2235 return (EOPNOTSUPP); 2236 return (EOPNOTSUPP); 2237 } 2238 case ATABUSIODETACH: 2239 { 2240 struct atabusiodetach_args *a= 2241 (struct atabusiodetach_args *)addr; 2242 if ((chp->ch_ndrives > 0 && 2243 (chp->ch_drive[0].drive_type == ATA_DRIVET_OLD)) || 2244 (chp->ch_ndrives > 1 && 2245 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))) 2246 return (EOPNOTSUPP); 2247 switch (a->at_dev) { 2248 case -1: 2249 min_drive = 0; 2250 max_drive = 1; 2251 break; 2252 case 0: 2253 case 1: 2254 min_drive = max_drive = a->at_dev; 2255 break; 2256 default: 2257 return (EINVAL); 2258 } 2259 for (drive = min_drive; drive <= max_drive; drive++) { 2260 if ((drive < chp->ch_ndrives) && 2261 chp->ch_drive[drive].drv_softc != NULL) { 2262 error = config_detach( 2263 chp->ch_drive[drive].drv_softc, 0); 2264 if (error) 2265 return (error); 2266 KASSERT(chp->ch_drive[drive].drv_softc == NULL); 2267 } 2268 } 2269 return 0; 2270 } 2271 default: 2272 return ENOTTY; 2273 } 2274 } 2275 2276 static bool 2277 atabus_suspend(device_t dv, const pmf_qual_t *qual) 2278 { 2279 struct atabus_softc *sc = device_private(dv); 2280 struct ata_channel *chp = sc->sc_chan; 2281 2282 ata_channel_idle(chp); 2283 2284 return true; 2285 } 2286 2287 static bool 2288 atabus_resume(device_t dv, const pmf_qual_t *qual) 2289 { 2290 struct atabus_softc *sc = device_private(dv); 2291 struct ata_channel *chp = sc->sc_chan; 2292 2293 /* 2294 * XXX joerg: with wdc, the first channel unfreezes the controller. 2295 * Move this the reset and queue idling into wdc. 2296 */ 2297 ata_channel_lock(chp); 2298 if (chp->ch_queue->queue_freeze == 0) { 2299 ata_channel_unlock(chp); 2300 goto out; 2301 } 2302 2303 /* unfreeze the queue and reset drives */ 2304 ata_channel_thaw_locked(chp); 2305 2306 /* reset channel only if there are drives attached */ 2307 if (chp->ch_ndrives > 0) 2308 ata_thread_run(chp, AT_WAIT, ATACH_TH_RESET, ATACH_NODRIVE); 2309 2310 ata_channel_unlock(chp); 2311 2312 out: 2313 return true; 2314 } 2315 2316 static int 2317 atabus_rescan(device_t self, const char *ifattr, const int *locators) 2318 { 2319 struct atabus_softc *sc = device_private(self); 2320 struct ata_channel *chp = sc->sc_chan; 2321 struct atabus_initq *initq; 2322 int i; 2323 2324 /* 2325 * we can rescan a port multiplier atabus, even if some devices are 2326 * still attached 2327 */ 2328 if (chp->ch_satapmp_nports == 0) { 2329 if (chp->atapibus != NULL) { 2330 return EBUSY; 2331 } 2332 2333 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 2334 for (i = 0; i < chp->ch_ndrives; i++) { 2335 if (chp->ch_drive[i].drv_softc != NULL) { 2336 return EBUSY; 2337 } 2338 } 2339 } 2340 2341 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 2342 initq->atabus_sc = sc; 2343 mutex_enter(&atabus_qlock); 2344 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 2345 mutex_exit(&atabus_qlock); 2346 config_pending_incr(sc->sc_dev); 2347 2348 ata_channel_lock(chp); 2349 chp->ch_flags |= ATACH_TH_RESCAN; 2350 cv_signal(&chp->ch_thr_idle); 2351 ata_channel_unlock(chp); 2352 2353 return 0; 2354 } 2355 2356 void 2357 ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags) 2358 { 2359 KASSERT(mutex_owned(&chp->ch_lock)); 2360 2361 if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) { 2362 /* 2363 * can't use kpause(), we may be in interrupt context 2364 * or taking a crash dump 2365 */ 2366 delay(ms * 1000); 2367 } else { 2368 int pause = mstohz(ms); 2369 2370 kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock); 2371 } 2372 } 2373 2374 void 2375 atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count, 2376 uint16_t *features, uint8_t *device) 2377 { 2378 if ((xfer->c_flags & C_NCQ) == 0) { 2379 /* FUA handling for non-NCQ drives */ 2380 if (xfer->c_bio.flags & ATA_FUA 2381 && *cmd == WDCC_WRITEDMA_EXT) 2382 *cmd = WDCC_WRITEDMA_FUA_EXT; 2383 2384 return; 2385 } 2386 2387 *cmd = (xfer->c_bio.flags & ATA_READ) ? 2388 WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED; 2389 2390 /* for FPDMA the block count is in features */ 2391 *features = *count; 2392 2393 /* NCQ tag */ 2394 *count = (xfer->c_slot << 3); 2395 2396 if (xfer->c_bio.flags & ATA_PRIO_HIGH) 2397 *count |= WDSC_PRIO_HIGH; 2398 2399 /* other device flags */ 2400 if (xfer->c_bio.flags & ATA_FUA) 2401 *device |= WDSD_FUA; 2402 } 2403 2404 void 2405 ata_wait_cmd(struct ata_channel *chp, struct ata_xfer *xfer) 2406 { 2407 struct ata_queue *chq = chp->ch_queue; 2408 struct ata_command *ata_c = &xfer->c_ata_c; 2409 2410 ata_channel_lock(chp); 2411 2412 while ((ata_c->flags & AT_DONE) == 0) 2413 cv_wait(&chq->c_cmd_finish, &chp->ch_lock); 2414 2415 ata_channel_unlock(chp); 2416 2417 KASSERT((ata_c->flags & AT_DONE) != 0); 2418 } 2419
Indexes created Wed Aug 26 00:26:28 UTC 2026