ata.c revision 1.142
1 /* $NetBSD: ata.c,v 1.142 2018/10/22 20:13:47 jdolecek Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.142 2018/10/22 20:13:47 jdolecek Exp $"); 29 30 #include "opt_ata.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/device.h> 36 #include <sys/conf.h> 37 #include <sys/fcntl.h> 38 #include <sys/proc.h> 39 #include <sys/kthread.h> 40 #include <sys/errno.h> 41 #include <sys/ataio.h> 42 #include <sys/kmem.h> 43 #include <sys/intr.h> 44 #include <sys/bus.h> 45 #include <sys/once.h> 46 #include <sys/bitops.h> 47 48 #define ATABUS_PRIVATE 49 50 #include <dev/ata/ataconf.h> 51 #include <dev/ata/atareg.h> 52 #include <dev/ata/atavar.h> 53 #include <dev/ic/wdcvar.h> /* for PIOBM */ 54 55 #include "ioconf.h" 56 #include "locators.h" 57 58 #include "atapibus.h" 59 #include "ataraid.h" 60 #include "sata_pmp.h" 61 62 #if NATARAID > 0 63 #include <dev/ata/ata_raidvar.h> 64 #endif 65 #if NSATA_PMP > 0 66 #include <dev/ata/satapmpvar.h> 67 #endif 68 #include <dev/ata/satapmpreg.h> 69 70 #define DEBUG_FUNCS 0x08 71 #define DEBUG_PROBE 0x10 72 #define DEBUG_DETACH 0x20 73 #define DEBUG_XFERS 0x40 74 #ifdef ATADEBUG 75 #ifndef ATADEBUG_MASK 76 #define ATADEBUG_MASK 0 77 #endif 78 int atadebug_mask = ATADEBUG_MASK; 79 #define ATADEBUG_PRINT(args, level) \ 80 if (atadebug_mask & (level)) \ 81 printf args 82 #else 83 #define ATADEBUG_PRINT(args, level) 84 #endif 85 86 static ONCE_DECL(ata_init_ctrl); 87 static struct pool ata_xfer_pool; 88 89 /* 90 * A queue of atabus instances, used to ensure the same bus probe order 91 * for a given hardware configuration at each boot. Kthread probing 92 * devices on a atabus. Only one probing at once. 93 */ 94 static TAILQ_HEAD(, atabus_initq) atabus_initq_head; 95 static kmutex_t atabus_qlock; 96 static kcondvar_t atabus_qcv; 97 static lwp_t * atabus_cfg_lwp; 98 99 /***************************************************************************** 100 * ATA bus layer. 101 * 102 * ATA controllers attach an atabus instance, which handles probing the bus 103 * for drives, etc. 104 *****************************************************************************/ 105 106 dev_type_open(atabusopen); 107 dev_type_close(atabusclose); 108 dev_type_ioctl(atabusioctl); 109 110 const struct cdevsw atabus_cdevsw = { 111 .d_open = atabusopen, 112 .d_close = atabusclose, 113 .d_read = noread, 114 .d_write = nowrite, 115 .d_ioctl = atabusioctl, 116 .d_stop = nostop, 117 .d_tty = notty, 118 .d_poll = nopoll, 119 .d_mmap = nommap, 120 .d_kqfilter = nokqfilter, 121 .d_discard = nodiscard, 122 .d_flag = D_OTHER 123 }; 124 125 static void atabus_childdetached(device_t, device_t); 126 static int atabus_rescan(device_t, const char *, const int *); 127 static bool atabus_resume(device_t, const pmf_qual_t *); 128 static bool atabus_suspend(device_t, const pmf_qual_t *); 129 static void atabusconfig_thread(void *); 130 131 static void ata_channel_idle(struct ata_channel *); 132 static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *); 133 static void ata_channel_freeze_locked(struct ata_channel *); 134 static void ata_thread_wake_locked(struct ata_channel *); 135 136 /* 137 * atabus_init: 138 * 139 * Initialize ATA subsystem structures. 140 */ 141 static int 142 atabus_init(void) 143 { 144 145 pool_init(&ata_xfer_pool, sizeof(struct ata_xfer), 0, 0, 0, 146 "ataspl", NULL, IPL_BIO); 147 TAILQ_INIT(&atabus_initq_head); 148 mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE); 149 cv_init(&atabus_qcv, "atainitq"); 150 return 0; 151 } 152 153 /* 154 * atabusprint: 155 * 156 * Autoconfiguration print routine used by ATA controllers when 157 * attaching an atabus instance. 158 */ 159 int 160 atabusprint(void *aux, const char *pnp) 161 { 162 struct ata_channel *chan = aux; 163 164 if (pnp) 165 aprint_normal("atabus at %s", pnp); 166 aprint_normal(" channel %d", chan->ch_channel); 167 168 return (UNCONF); 169 } 170 171 /* 172 * ataprint: 173 * 174 * Autoconfiguration print routine. 175 */ 176 int 177 ataprint(void *aux, const char *pnp) 178 { 179 struct ata_device *adev = aux; 180 181 if (pnp) 182 aprint_normal("wd at %s", pnp); 183 aprint_normal(" drive %d", adev->adev_drv_data->drive); 184 185 return (UNCONF); 186 } 187 188 /* 189 * ata_channel_attach: 190 * 191 * Common parts of attaching an atabus to an ATA controller channel. 192 */ 193 void 194 ata_channel_attach(struct ata_channel *chp) 195 { 196 if (chp->ch_flags & ATACH_DISABLED) 197 return; 198 199 ata_channel_init(chp); 200 201 KASSERT(chp->ch_queue != NULL); 202 203 chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp, 204 atabusprint); 205 } 206 207 /* 208 * ata_channel_detach: 209 * 210 * Common parts of detaching an atabus to an ATA controller channel. 211 */ 212 void 213 ata_channel_detach(struct ata_channel *chp) 214 { 215 if (chp->ch_flags & ATACH_DISABLED) 216 return; 217 218 ata_channel_destroy(chp); 219 } 220 221 static void 222 atabusconfig(struct atabus_softc *atabus_sc) 223 { 224 struct ata_channel *chp = atabus_sc->sc_chan; 225 struct atac_softc *atac = chp->ch_atac; 226 struct atabus_initq *atabus_initq = NULL; 227 int i, error; 228 229 /* we are in the atabus's thread context */ 230 ata_channel_lock(chp); 231 chp->ch_flags |= ATACH_TH_RUN; 232 ata_channel_unlock(chp); 233 234 /* 235 * Probe for the drives attached to controller, unless a PMP 236 * is already known 237 */ 238 /* XXX for SATA devices we will power up all drives at once */ 239 if (chp->ch_satapmp_nports == 0) 240 (*atac->atac_probe)(chp); 241 242 if (chp->ch_ndrives >= 2) { 243 ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n", 244 chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type), 245 DEBUG_PROBE); 246 } 247 248 /* next operations will occurs in a separate thread */ 249 ata_channel_lock(chp); 250 chp->ch_flags &= ~ATACH_TH_RUN; 251 ata_channel_unlock(chp); 252 253 /* Make sure the devices probe in atabus order to avoid jitter. */ 254 mutex_enter(&atabus_qlock); 255 for (;;) { 256 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 257 if (atabus_initq->atabus_sc == atabus_sc) 258 break; 259 cv_wait(&atabus_qcv, &atabus_qlock); 260 } 261 mutex_exit(&atabus_qlock); 262 263 ata_channel_lock(chp); 264 265 /* If no drives, abort here */ 266 if (chp->ch_drive == NULL) 267 goto out; 268 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 269 for (i = 0; i < chp->ch_ndrives; i++) 270 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) 271 break; 272 if (i == chp->ch_ndrives) 273 goto out; 274 275 /* Shortcut in case we've been shutdown */ 276 if (chp->ch_flags & ATACH_SHUTDOWN) 277 goto out; 278 279 ata_channel_unlock(chp); 280 281 if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread, 282 atabus_sc, &atabus_cfg_lwp, 283 "%scnf", device_xname(atac->atac_dev))) != 0) 284 aprint_error_dev(atac->atac_dev, 285 "unable to create config thread: error %d\n", error); 286 return; 287 288 out: 289 ata_channel_unlock(chp); 290 291 mutex_enter(&atabus_qlock); 292 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 293 cv_broadcast(&atabus_qcv); 294 mutex_exit(&atabus_qlock); 295 296 kmem_free(atabus_initq, sizeof(*atabus_initq)); 297 298 ata_delref(chp); 299 300 config_pending_decr(atac->atac_dev); 301 } 302 303 /* 304 * atabus_configthread: finish attach of atabus's childrens, in a separate 305 * kernel thread. 306 */ 307 static void 308 atabusconfig_thread(void *arg) 309 { 310 struct atabus_softc *atabus_sc = arg; 311 struct ata_channel *chp = atabus_sc->sc_chan; 312 struct atac_softc *atac = chp->ch_atac; 313 struct atabus_initq *atabus_initq = NULL; 314 int i, s; 315 316 /* XXX seems wrong */ 317 mutex_enter(&atabus_qlock); 318 atabus_initq = TAILQ_FIRST(&atabus_initq_head); 319 KASSERT(atabus_initq->atabus_sc == atabus_sc); 320 mutex_exit(&atabus_qlock); 321 322 /* 323 * First look for a port multiplier 324 */ 325 if (chp->ch_ndrives == PMP_MAX_DRIVES && 326 chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) { 327 #if NSATA_PMP > 0 328 satapmp_attach(chp); 329 #else 330 aprint_error_dev(atabus_sc->sc_dev, 331 "SATA port multiplier not supported\n"); 332 /* no problems going on, all drives are ATA_DRIVET_NONE */ 333 #endif 334 } 335 336 /* 337 * Attach an ATAPI bus, if needed. 338 */ 339 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 340 for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) { 341 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) { 342 #if NATAPIBUS > 0 343 (*atac->atac_atapibus_attach)(atabus_sc); 344 #else 345 /* 346 * Fake the autoconfig "not configured" message 347 */ 348 aprint_normal("atapibus at %s not configured\n", 349 device_xname(atac->atac_dev)); 350 chp->atapibus = NULL; 351 s = splbio(); 352 for (i = 0; i < chp->ch_ndrives; i++) { 353 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 354 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 355 } 356 splx(s); 357 #endif 358 break; 359 } 360 } 361 362 for (i = 0; i < chp->ch_ndrives; i++) { 363 struct ata_device adev; 364 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA && 365 chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) { 366 continue; 367 } 368 if (chp->ch_drive[i].drv_softc != NULL) 369 continue; 370 memset(&adev, 0, sizeof(struct ata_device)); 371 adev.adev_bustype = atac->atac_bustype_ata; 372 adev.adev_channel = chp->ch_channel; 373 adev.adev_drv_data = &chp->ch_drive[i]; 374 chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev, 375 "ata_hl", &adev, ataprint); 376 if (chp->ch_drive[i].drv_softc != NULL) { 377 ata_probe_caps(&chp->ch_drive[i]); 378 } else { 379 s = splbio(); 380 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 381 splx(s); 382 } 383 } 384 385 /* now that we know the drives, the controller can set its modes */ 386 if (atac->atac_set_modes) { 387 (*atac->atac_set_modes)(chp); 388 ata_print_modes(chp); 389 } 390 #if NATARAID > 0 391 if (atac->atac_cap & ATAC_CAP_RAID) { 392 for (i = 0; i < chp->ch_ndrives; i++) { 393 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) { 394 ata_raid_check_component( 395 chp->ch_drive[i].drv_softc); 396 } 397 } 398 } 399 #endif /* NATARAID > 0 */ 400 401 /* 402 * reset drive_flags for unattached devices, reset state for attached 403 * ones 404 */ 405 s = splbio(); 406 for (i = 0; i < chp->ch_ndrives; i++) { 407 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 408 continue; 409 if (chp->ch_drive[i].drv_softc == NULL) { 410 chp->ch_drive[i].drive_flags = 0; 411 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 412 } else 413 chp->ch_drive[i].state = 0; 414 } 415 splx(s); 416 417 mutex_enter(&atabus_qlock); 418 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq); 419 cv_broadcast(&atabus_qcv); 420 mutex_exit(&atabus_qlock); 421 422 kmem_free(atabus_initq, sizeof(*atabus_initq)); 423 424 ata_delref(chp); 425 426 config_pending_decr(atac->atac_dev); 427 kthread_exit(0); 428 } 429 430 /* 431 * atabus_thread: 432 * 433 * Worker thread for the ATA bus. 434 */ 435 static void 436 atabus_thread(void *arg) 437 { 438 struct atabus_softc *sc = arg; 439 struct ata_channel *chp = sc->sc_chan; 440 struct ata_queue *chq = chp->ch_queue; 441 struct ata_xfer *xfer; 442 int i, rv; 443 444 ata_channel_lock(chp); 445 chp->ch_flags |= ATACH_TH_RUN; 446 447 /* 448 * Probe the drives. Reset type to indicate to controllers 449 * that can re-probe that all drives must be probed.. 450 * 451 * Note: ch_ndrives may be changed during the probe. 452 */ 453 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 454 for (i = 0; i < chp->ch_ndrives; i++) { 455 chp->ch_drive[i].drive_flags = 0; 456 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 457 } 458 ata_channel_unlock(chp); 459 460 atabusconfig(sc); 461 462 ata_channel_lock(chp); 463 for (;;) { 464 if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_TH_DRIVE_RESET 465 | ATACH_TH_RECOVERY | ATACH_SHUTDOWN)) == 0 && 466 (chq->queue_active == 0 || chq->queue_freeze == 0)) { 467 chp->ch_flags &= ~ATACH_TH_RUN; 468 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 469 chp->ch_flags |= ATACH_TH_RUN; 470 } 471 if (chp->ch_flags & ATACH_SHUTDOWN) { 472 break; 473 } 474 if (chp->ch_flags & ATACH_TH_RESCAN) { 475 chp->ch_flags &= ~ATACH_TH_RESCAN; 476 ata_channel_unlock(chp); 477 atabusconfig(sc); 478 ata_channel_lock(chp); 479 } 480 if (chp->ch_flags & ATACH_TH_RESET) { 481 /* this will unfreeze the channel */ 482 ata_thread_run(chp, AT_WAIT, 483 ATACH_TH_RESET, ATACH_NODRIVE); 484 } else if (chp->ch_flags & ATACH_TH_DRIVE_RESET) { 485 /* this will unfreeze the channel */ 486 for (i = 0; i < chp->ch_ndrives; i++) { 487 struct ata_drive_datas *drvp; 488 489 drvp = &chp->ch_drive[i]; 490 491 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 492 ata_thread_run(chp, 493 AT_WAIT, ATACH_TH_DRIVE_RESET, i); 494 } 495 } 496 chp->ch_flags &= ~ATACH_TH_DRIVE_RESET; 497 } else if (chp->ch_flags & ATACH_TH_RECOVERY) { 498 /* 499 * This will unfreeze the channel; drops locks during 500 * run, so must wrap in splbio()/splx() to avoid 501 * spurious interrupts. XXX MPSAFE 502 */ 503 int s = splbio(); 504 ata_thread_run(chp, AT_WAIT, ATACH_TH_RECOVERY, 505 chp->recovery_tfd); 506 splx(s); 507 } else if (chq->queue_active > 0 && chq->queue_freeze == 1) { 508 /* 509 * Caller has bumped queue_freeze, decrease it. This 510 * flow shalt never be executed for NCQ commands. 511 */ 512 KASSERT((chp->ch_flags & ATACH_NCQ) == 0); 513 KASSERT(chq->queue_active == 1); 514 515 ata_channel_thaw_locked(chp); 516 xfer = ata_queue_get_active_xfer_locked(chp); 517 518 KASSERT(xfer != NULL); 519 KASSERT((xfer->c_flags & C_POLL) == 0); 520 521 switch ((rv = ata_xfer_start(xfer))) { 522 case ATASTART_STARTED: 523 case ATASTART_POLL: 524 case ATASTART_ABORT: 525 break; 526 case ATASTART_TH: 527 default: 528 panic("%s: ata_xfer_start() unexpected rv %d", 529 __func__, rv); 530 /* NOTREACHED */ 531 } 532 } else if (chq->queue_freeze > 1) 533 panic("%s: queue_freeze", __func__); 534 535 /* Try to run down the queue once channel is unfrozen */ 536 if (chq->queue_freeze == 0) { 537 ata_channel_unlock(chp); 538 atastart(chp); 539 ata_channel_lock(chp); 540 } 541 } 542 chp->ch_thread = NULL; 543 cv_signal(&chp->ch_thr_idle); 544 ata_channel_unlock(chp); 545 kthread_exit(0); 546 } 547 548 static void 549 ata_thread_wake_locked(struct ata_channel *chp) 550 { 551 KASSERT(mutex_owned(&chp->ch_lock)); 552 ata_channel_freeze_locked(chp); 553 cv_signal(&chp->ch_thr_idle); 554 } 555 556 /* 557 * atabus_match: 558 * 559 * Autoconfiguration match routine. 560 */ 561 static int 562 atabus_match(device_t parent, cfdata_t cf, void *aux) 563 { 564 struct ata_channel *chp = aux; 565 566 if (chp == NULL) 567 return (0); 568 569 if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel && 570 cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT) 571 return (0); 572 573 return (1); 574 } 575 576 /* 577 * atabus_attach: 578 * 579 * Autoconfiguration attach routine. 580 */ 581 static void 582 atabus_attach(device_t parent, device_t self, void *aux) 583 { 584 struct atabus_softc *sc = device_private(self); 585 struct ata_channel *chp = aux; 586 struct atabus_initq *initq; 587 int error; 588 589 sc->sc_chan = chp; 590 591 aprint_normal("\n"); 592 aprint_naive("\n"); 593 594 sc->sc_dev = self; 595 596 if (ata_addref(chp)) 597 return; 598 599 RUN_ONCE(&ata_init_ctrl, atabus_init); 600 601 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 602 initq->atabus_sc = sc; 603 mutex_enter(&atabus_qlock); 604 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 605 mutex_exit(&atabus_qlock); 606 config_pending_incr(sc->sc_dev); 607 608 /* XXX MPSAFE - no KTHREAD_MPSAFE, so protected by KERNEL_LOCK() */ 609 if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc, 610 &chp->ch_thread, "%s", device_xname(self))) != 0) 611 aprint_error_dev(self, 612 "unable to create kernel thread: error %d\n", error); 613 614 if (!pmf_device_register(self, atabus_suspend, atabus_resume)) 615 aprint_error_dev(self, "couldn't establish power handler\n"); 616 } 617 618 /* 619 * atabus_detach: 620 * 621 * Autoconfiguration detach routine. 622 */ 623 static int 624 atabus_detach(device_t self, int flags) 625 { 626 struct atabus_softc *sc = device_private(self); 627 struct ata_channel *chp = sc->sc_chan; 628 device_t dev = NULL; 629 int i, error = 0; 630 631 /* Shutdown the channel. */ 632 ata_channel_lock(chp); 633 chp->ch_flags |= ATACH_SHUTDOWN; 634 while (chp->ch_thread != NULL) { 635 cv_signal(&chp->ch_thr_idle); 636 cv_wait(&chp->ch_thr_idle, &chp->ch_lock); 637 } 638 ata_channel_unlock(chp); 639 640 /* 641 * Detach atapibus and its children. 642 */ 643 if ((dev = chp->atapibus) != NULL) { 644 ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n", 645 device_xname(self), device_xname(dev)), DEBUG_DETACH); 646 647 error = config_detach(dev, flags); 648 if (error) 649 goto out; 650 KASSERT(chp->atapibus == NULL); 651 } 652 653 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 654 655 /* 656 * Detach our other children. 657 */ 658 for (i = 0; i < chp->ch_ndrives; i++) { 659 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 660 continue; 661 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 662 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 663 if ((dev = chp->ch_drive[i].drv_softc) != NULL) { 664 ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__, 665 __LINE__, device_xname(self), device_xname(dev)), 666 DEBUG_DETACH); 667 error = config_detach(dev, flags); 668 if (error) 669 goto out; 670 KASSERT(chp->ch_drive[i].drv_softc == NULL); 671 KASSERT(chp->ch_drive[i].drive_type == 0); 672 } 673 } 674 atabus_free_drives(chp); 675 676 out: 677 #ifdef ATADEBUG 678 if (dev != NULL && error != 0) 679 ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__, 680 device_xname(self), error, device_xname(dev)), 681 DEBUG_DETACH); 682 #endif /* ATADEBUG */ 683 684 return (error); 685 } 686 687 void 688 atabus_childdetached(device_t self, device_t child) 689 { 690 bool found = false; 691 struct atabus_softc *sc = device_private(self); 692 struct ata_channel *chp = sc->sc_chan; 693 int i; 694 695 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 696 /* 697 * atapibus detached. 698 */ 699 if (child == chp->atapibus) { 700 chp->atapibus = NULL; 701 found = true; 702 for (i = 0; i < chp->ch_ndrives; i++) { 703 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI) 704 continue; 705 KASSERT(chp->ch_drive[i].drv_softc != NULL); 706 chp->ch_drive[i].drv_softc = NULL; 707 chp->ch_drive[i].drive_flags = 0; 708 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 709 } 710 } 711 712 /* 713 * Detach our other children. 714 */ 715 for (i = 0; i < chp->ch_ndrives; i++) { 716 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) 717 continue; 718 if (child == chp->ch_drive[i].drv_softc) { 719 chp->ch_drive[i].drv_softc = NULL; 720 chp->ch_drive[i].drive_flags = 0; 721 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM) 722 chp->ch_satapmp_nports = 0; 723 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE; 724 found = true; 725 } 726 } 727 728 if (!found) 729 panic("%s: unknown child %p", device_xname(self), 730 (const void *)child); 731 } 732 733 CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc), 734 atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan, 735 atabus_childdetached, DVF_DETACH_SHUTDOWN); 736 737 /***************************************************************************** 738 * Common ATA bus operations. 739 *****************************************************************************/ 740 741 /* allocate/free the channel's ch_drive[] array */ 742 int 743 atabus_alloc_drives(struct ata_channel *chp, int ndrives) 744 { 745 int i; 746 if (chp->ch_ndrives != ndrives) 747 atabus_free_drives(chp); 748 if (chp->ch_drive == NULL) { 749 chp->ch_drive = kmem_zalloc( 750 sizeof(struct ata_drive_datas) * ndrives, KM_NOSLEEP); 751 } 752 if (chp->ch_drive == NULL) { 753 aprint_error_dev(chp->ch_atac->atac_dev, 754 "can't alloc drive array\n"); 755 chp->ch_ndrives = 0; 756 return ENOMEM; 757 }; 758 for (i = 0; i < ndrives; i++) { 759 chp->ch_drive[i].chnl_softc = chp; 760 chp->ch_drive[i].drive = i; 761 } 762 chp->ch_ndrives = ndrives; 763 return 0; 764 } 765 766 void 767 atabus_free_drives(struct ata_channel *chp) 768 { 769 #ifdef DIAGNOSTIC 770 int i; 771 int dopanic = 0; 772 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 773 for (i = 0; i < chp->ch_ndrives; i++) { 774 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) { 775 printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n", 776 device_xname(chp->atabus), i, 777 chp->ch_drive[i].drive_type); 778 dopanic = 1; 779 } 780 if (chp->ch_drive[i].drv_softc != NULL) { 781 printf("%s: ch_drive[%d] attached to %s\n", 782 device_xname(chp->atabus), i, 783 device_xname(chp->ch_drive[i].drv_softc)); 784 dopanic = 1; 785 } 786 } 787 if (dopanic) 788 panic("atabus_free_drives"); 789 #endif 790 791 if (chp->ch_drive == NULL) 792 return; 793 kmem_free(chp->ch_drive, 794 sizeof(struct ata_drive_datas) * chp->ch_ndrives); 795 chp->ch_ndrives = 0; 796 chp->ch_drive = NULL; 797 } 798 799 /* Get the disk's parameters */ 800 int 801 ata_get_params(struct ata_drive_datas *drvp, uint8_t flags, 802 struct ataparams *prms) 803 { 804 struct ata_xfer *xfer; 805 struct ata_channel *chp = drvp->chnl_softc; 806 struct atac_softc *atac = chp->ch_atac; 807 char *tb; 808 int i, rv; 809 uint16_t *p; 810 811 ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS); 812 813 xfer = ata_get_xfer(chp, false); 814 if (xfer == NULL) { 815 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 816 DEBUG_FUNCS|DEBUG_PROBE); 817 return CMD_AGAIN; 818 } 819 820 tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP); 821 memset(prms, 0, sizeof(struct ataparams)); 822 823 if (drvp->drive_type == ATA_DRIVET_ATA) { 824 xfer->c_ata_c.r_command = WDCC_IDENTIFY; 825 xfer->c_ata_c.r_st_bmask = WDCS_DRDY; 826 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 827 xfer->c_ata_c.timeout = 3000; /* 3s */ 828 } else if (drvp->drive_type == ATA_DRIVET_ATAPI) { 829 xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE; 830 xfer->c_ata_c.r_st_bmask = 0; 831 xfer->c_ata_c.r_st_pmask = WDCS_DRQ; 832 xfer->c_ata_c.timeout = 10000; /* 10s */ 833 } else { 834 ATADEBUG_PRINT(("ata_get_parms: no disks\n"), 835 DEBUG_FUNCS|DEBUG_PROBE); 836 rv = CMD_ERR; 837 goto out; 838 } 839 xfer->c_ata_c.flags = AT_READ | flags; 840 xfer->c_ata_c.data = tb; 841 xfer->c_ata_c.bcount = ATA_BSIZE; 842 if ((*atac->atac_bustype_ata->ata_exec_command)(drvp, 843 xfer) != ATACMD_COMPLETE) { 844 ATADEBUG_PRINT(("ata_get_parms: wdc_exec_command failed\n"), 845 DEBUG_FUNCS|DEBUG_PROBE); 846 rv = CMD_AGAIN; 847 goto out; 848 } 849 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 850 ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n", 851 xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE); 852 rv = CMD_ERR; 853 goto out; 854 } 855 /* if we didn't read any data something is wrong */ 856 if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) { 857 rv = CMD_ERR; 858 goto out; 859 } 860 861 /* Read in parameter block. */ 862 memcpy(prms, tb, sizeof(struct ataparams)); 863 864 /* 865 * Shuffle string byte order. 866 * ATAPI NEC, Mitsumi and Pioneer drives and 867 * old ATA TDK CompactFlash cards 868 * have different byte order. 869 */ 870 #if BYTE_ORDER == BIG_ENDIAN 871 # define M(n) prms->atap_model[(n) ^ 1] 872 #else 873 # define M(n) prms->atap_model[n] 874 #endif 875 if ( 876 #if BYTE_ORDER == BIG_ENDIAN 877 ! 878 #endif 879 ((drvp->drive_type == ATA_DRIVET_ATAPI) ? 880 ((M(0) == 'N' && M(1) == 'E') || 881 (M(0) == 'F' && M(1) == 'X') || 882 (M(0) == 'P' && M(1) == 'i')) : 883 ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) { 884 rv = CMD_OK; 885 goto out; 886 } 887 #undef M 888 for (i = 0; i < sizeof(prms->atap_model); i += 2) { 889 p = (uint16_t *)(prms->atap_model + i); 890 *p = bswap16(*p); 891 } 892 for (i = 0; i < sizeof(prms->atap_serial); i += 2) { 893 p = (uint16_t *)(prms->atap_serial + i); 894 *p = bswap16(*p); 895 } 896 for (i = 0; i < sizeof(prms->atap_revision); i += 2) { 897 p = (uint16_t *)(prms->atap_revision + i); 898 *p = bswap16(*p); 899 } 900 901 rv = CMD_OK; 902 out: 903 kmem_free(tb, ATA_BSIZE); 904 ata_free_xfer(chp, xfer); 905 return rv; 906 } 907 908 int 909 ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags) 910 { 911 struct ata_xfer *xfer; 912 int rv; 913 struct ata_channel *chp = drvp->chnl_softc; 914 struct atac_softc *atac = chp->ch_atac; 915 916 ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS); 917 918 xfer = ata_get_xfer(chp, false); 919 if (xfer == NULL) { 920 ATADEBUG_PRINT(("%s: no xfer\n", __func__), 921 DEBUG_FUNCS|DEBUG_PROBE); 922 return CMD_AGAIN; 923 } 924 925 xfer->c_ata_c.r_command = SET_FEATURES; 926 xfer->c_ata_c.r_st_bmask = 0; 927 xfer->c_ata_c.r_st_pmask = 0; 928 xfer->c_ata_c.r_features = WDSF_SET_MODE; 929 xfer->c_ata_c.r_count = mode; 930 xfer->c_ata_c.flags = flags; 931 xfer->c_ata_c.timeout = 1000; /* 1s */ 932 if ((*atac->atac_bustype_ata->ata_exec_command)(drvp, 933 xfer) != ATACMD_COMPLETE) { 934 rv = CMD_AGAIN; 935 goto out; 936 } 937 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) { 938 rv = CMD_ERR; 939 goto out; 940 } 941 942 rv = CMD_OK; 943 944 out: 945 ata_free_xfer(chp, xfer); 946 return rv; 947 } 948 949 #if NATA_DMA 950 void 951 ata_dmaerr(struct ata_drive_datas *drvp, int flags) 952 { 953 /* 954 * Downgrade decision: if we get NERRS_MAX in NXFER. 955 * We start with n_dmaerrs set to NERRS_MAX-1 so that the 956 * first error within the first NXFER ops will immediatly trigger 957 * a downgrade. 958 * If we got an error and n_xfers is bigger than NXFER reset counters. 959 */ 960 drvp->n_dmaerrs++; 961 if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) { 962 ata_downgrade_mode(drvp, flags); 963 drvp->n_dmaerrs = NERRS_MAX-1; 964 drvp->n_xfers = 0; 965 return; 966 } 967 if (drvp->n_xfers > NXFER) { 968 drvp->n_dmaerrs = 1; /* just got an error */ 969 drvp->n_xfers = 1; /* restart counting from this error */ 970 } 971 } 972 #endif /* NATA_DMA */ 973 974 /* 975 * freeze the queue and wait for the controller to be idle. Caller has to 976 * unfreeze/restart the queue 977 */ 978 static void 979 ata_channel_idle(struct ata_channel *chp) 980 { 981 ata_channel_lock(chp); 982 ata_channel_freeze_locked(chp); 983 while (chp->ch_queue->queue_active > 0) { 984 chp->ch_queue->queue_flags |= QF_IDLE_WAIT; 985 cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1); 986 } 987 ata_channel_unlock(chp); 988 } 989 990 /* 991 * Add a command to the queue and start controller. 992 * 993 * MUST BE CALLED AT splbio()! 994 */ 995 void 996 ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 997 { 998 999 ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer, 1000 chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1001 1002 /* complete xfer setup */ 1003 xfer->c_chp = chp; 1004 1005 ata_channel_lock(chp); 1006 1007 /* 1008 * Standard commands are added to the end of command list, but 1009 * recovery commands must be run immediatelly. 1010 */ 1011 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1012 SIMPLEQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer, 1013 c_xferchain); 1014 else 1015 SIMPLEQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer, 1016 c_xferchain); 1017 1018 /* 1019 * if polling and can sleep, wait for the xfer to be at head of queue 1020 */ 1021 if ((xfer->c_flags & (C_POLL | C_WAIT)) == (C_POLL | C_WAIT)) { 1022 while (chp->ch_queue->queue_active > 0 || 1023 SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) { 1024 xfer->c_flags |= C_WAITACT; 1025 cv_wait(&chp->ch_queue->c_active, &chp->ch_lock); 1026 xfer->c_flags &= ~C_WAITACT; 1027 } 1028 1029 /* 1030 * Free xfer now if it there was attempt to free it 1031 * while we were waiting. 1032 */ 1033 if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) { 1034 ata_channel_unlock(chp); 1035 1036 ata_free_xfer(chp, xfer); 1037 return; 1038 } 1039 } 1040 1041 ata_channel_unlock(chp); 1042 1043 ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n", 1044 chp->ch_flags), DEBUG_XFERS); 1045 atastart(chp); 1046 } 1047 1048 /* 1049 * Start I/O on a controller, for the given channel. 1050 * The first xfer may be not for our channel if the channel queues 1051 * are shared. 1052 * 1053 * MUST BE CALLED AT splbio()! 1054 */ 1055 void 1056 atastart(struct ata_channel *chp) 1057 { 1058 struct atac_softc *atac = chp->ch_atac; 1059 struct ata_queue *chq = chp->ch_queue; 1060 struct ata_xfer *xfer, *axfer; 1061 bool skipq; 1062 1063 #ifdef ATA_DEBUG 1064 int spl1, spl2; 1065 1066 spl1 = splbio(); 1067 spl2 = splbio(); 1068 if (spl2 != spl1) { 1069 printf("atastart: not at splbio()\n"); 1070 panic("atastart"); 1071 } 1072 splx(spl2); 1073 splx(spl1); 1074 #endif /* ATA_DEBUG */ 1075 1076 ata_channel_lock(chp); 1077 1078 again: 1079 /* is there a xfer ? */ 1080 if ((xfer = SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL) { 1081 ATADEBUG_PRINT(("%s(chp=%p): channel %d queue_xfer is empty\n", 1082 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1083 goto out; 1084 } 1085 1086 /* 1087 * if someone is waiting for the command to be active, wake it up 1088 * and let it process the command 1089 */ 1090 if (__predict_false(xfer->c_flags & C_WAITACT)) { 1091 ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d " 1092 "wait active\n", xfer, chp->ch_channel, xfer->c_drive), 1093 DEBUG_XFERS); 1094 cv_broadcast(&chp->ch_queue->c_active); 1095 goto out; 1096 } 1097 1098 skipq = ISSET(xfer->c_flags, C_SKIP_QUEUE); 1099 1100 /* is the queue frozen? */ 1101 if (__predict_false(!skipq && chq->queue_freeze > 0)) { 1102 if (chq->queue_flags & QF_IDLE_WAIT) { 1103 chq->queue_flags &= ~QF_IDLE_WAIT; 1104 cv_signal(&chp->ch_queue->queue_idle); 1105 } 1106 ATADEBUG_PRINT(("%s(chp=%p): channel %d drive %d " 1107 "queue frozen: %d\n", 1108 __func__, chp, chp->ch_channel, xfer->c_drive, 1109 chq->queue_freeze), 1110 DEBUG_XFERS); 1111 goto out; 1112 } 1113 1114 /* all xfers on same queue must belong to the same channel */ 1115 KASSERT(xfer->c_chp == chp); 1116 1117 /* 1118 * Can only take the command if there are no current active 1119 * commands, or if the command is NCQ and the active commands are also 1120 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based 1121 * switching, can only send commands to single drive. 1122 * Need only check first xfer. 1123 * XXX FIS-based switching - revisit 1124 */ 1125 if (!skipq && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) { 1126 if (!ISSET(xfer->c_flags, C_NCQ) || 1127 !ISSET(axfer->c_flags, C_NCQ) || 1128 xfer->c_drive != axfer->c_drive) 1129 goto out; 1130 } 1131 1132 struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive]; 1133 1134 /* 1135 * Are we on limit of active xfers ? If the queue has more 1136 * than 1 openings, we keep one slot reserved for recovery or dump. 1137 */ 1138 KASSERT(chq->queue_active <= chq->queue_openings); 1139 const uint8_t chq_openings = (!skipq && chq->queue_openings > 1) 1140 ? (chq->queue_openings - 1) : chq->queue_openings; 1141 const uint8_t drv_openings = ISSET(xfer->c_flags, C_NCQ) 1142 ? drvp->drv_openings : ATA_MAX_OPENINGS; 1143 if (chq->queue_active >= MIN(chq_openings, drv_openings)) { 1144 if (skipq) { 1145 panic("%s: channel %d busy, xfer not possible", 1146 __func__, chp->ch_channel); 1147 } 1148 1149 ATADEBUG_PRINT(("%s(chp=%p): channel %d completely busy\n", 1150 __func__, chp, chp->ch_channel), DEBUG_XFERS); 1151 goto out; 1152 } 1153 1154 /* Slot allocation can fail if drv_openings < ch_openings */ 1155 if (!ata_queue_alloc_slot(chp, &xfer->c_slot, drv_openings)) 1156 goto out; 1157 1158 if (__predict_false(atac->atac_claim_hw)) { 1159 if (!atac->atac_claim_hw(chp, 0)) { 1160 ata_queue_free_slot(chp, xfer->c_slot); 1161 goto out; 1162 } 1163 } 1164 1165 /* Now committed to start the xfer */ 1166 1167 ATADEBUG_PRINT(("%s(chp=%p): xfer %p channel %d drive %d\n", 1168 __func__, chp, xfer, chp->ch_channel, xfer->c_drive), DEBUG_XFERS); 1169 if (drvp->drive_flags & ATA_DRIVE_RESET) { 1170 drvp->drive_flags &= ~ATA_DRIVE_RESET; 1171 drvp->state = 0; 1172 } 1173 1174 if (ISSET(xfer->c_flags, C_NCQ)) 1175 SET(chp->ch_flags, ATACH_NCQ); 1176 else 1177 CLR(chp->ch_flags, ATACH_NCQ); 1178 1179 SIMPLEQ_REMOVE_HEAD(&chq->queue_xfer, c_xferchain); 1180 1181 ata_activate_xfer_locked(chp, xfer); 1182 1183 if (atac->atac_cap & ATAC_CAP_NOIRQ) 1184 KASSERT(xfer->c_flags & C_POLL); 1185 1186 switch (ata_xfer_start(xfer)) { 1187 case ATASTART_TH: 1188 case ATASTART_ABORT: 1189 /* don't start any further commands in this case */ 1190 goto out; 1191 default: 1192 /* nothing to do */ 1193 break; 1194 } 1195 1196 /* Queue more commands if possible, but not during recovery or dump */ 1197 if (!skipq && chq->queue_active < chq->queue_openings) 1198 goto again; 1199 1200 out: 1201 ata_channel_unlock(chp); 1202 } 1203 1204 int 1205 ata_xfer_start(struct ata_xfer *xfer) 1206 { 1207 struct ata_channel *chp = xfer->c_chp; 1208 int rv; 1209 1210 KASSERT(mutex_owned(&chp->ch_lock)); 1211 1212 rv = xfer->ops->c_start(chp, xfer); 1213 switch (rv) { 1214 case ATASTART_STARTED: 1215 /* nothing to do */ 1216 break; 1217 case ATASTART_TH: 1218 /* postpone xfer to thread */ 1219 ata_thread_wake_locked(chp); 1220 break; 1221 case ATASTART_POLL: 1222 /* can happen even in thread context for some ATAPI devices */ 1223 ata_channel_unlock(chp); 1224 KASSERT(xfer->ops != NULL && xfer->ops->c_poll != NULL); 1225 xfer->ops->c_poll(chp, xfer); 1226 ata_channel_lock(chp); 1227 break; 1228 case ATASTART_ABORT: 1229 ata_channel_unlock(chp); 1230 KASSERT(xfer->ops != NULL && xfer->ops->c_abort != NULL); 1231 xfer->ops->c_abort(chp, xfer); 1232 ata_channel_lock(chp); 1233 break; 1234 } 1235 1236 return rv; 1237 } 1238 1239 static void 1240 ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer) 1241 { 1242 struct ata_queue * const chq = chp->ch_queue; 1243 1244 KASSERT(mutex_owned(&chp->ch_lock)); 1245 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0); 1246 1247 if ((xfer->c_flags & C_SKIP_QUEUE) == 0) 1248 TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain); 1249 else { 1250 /* 1251 * Must go to head, so that ata_queue_get_active_xfer() 1252 * returns the recovery command, and not some other 1253 * random active transfer. 1254 */ 1255 TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain); 1256 } 1257 chq->active_xfers_used |= __BIT(xfer->c_slot); 1258 chq->queue_active++; 1259 } 1260 1261 /* 1262 * Does it's own locking, does not require splbio(). 1263 * flags - whether to block waiting for free xfer 1264 */ 1265 struct ata_xfer * 1266 ata_get_xfer(struct ata_channel *chp, bool waitok) 1267 { 1268 struct ata_xfer *xfer; 1269 1270 xfer = pool_get(&ata_xfer_pool, waitok ? PR_WAITOK : PR_NOWAIT); 1271 KASSERT(!waitok || xfer != NULL); 1272 1273 if (xfer != NULL) { 1274 /* zero everything */ 1275 memset(xfer, 0, sizeof(*xfer)); 1276 } 1277 1278 return xfer; 1279 } 1280 1281 /* 1282 * ata_deactivate_xfer() must be always called prior to ata_free_xfer() 1283 */ 1284 void 1285 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1286 { 1287 struct ata_queue *chq = chp->ch_queue; 1288 1289 ata_channel_lock(chp); 1290 1291 if (__predict_false(xfer->c_flags & (C_WAITACT|C_WAITTIMO))) { 1292 /* Someone is waiting for this xfer, so we can't free now */ 1293 xfer->c_flags |= C_FREE; 1294 cv_broadcast(&chq->c_active); 1295 ata_channel_unlock(chp); 1296 return; 1297 } 1298 1299 /* XXX move PIOBM and free_gw to deactivate? */ 1300 #if NATA_PIOBM /* XXX wdc dependent code */ 1301 if (__predict_false(xfer->c_flags & C_PIOBM)) { 1302 struct wdc_softc *wdc = CHAN_TO_WDC(chp); 1303 1304 /* finish the busmastering PIO */ 1305 (*wdc->piobm_done)(wdc->dma_arg, 1306 chp->ch_channel, xfer->c_drive); 1307 chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT); 1308 } 1309 #endif 1310 1311 if (__predict_false(chp->ch_atac->atac_free_hw)) 1312 chp->ch_atac->atac_free_hw(chp); 1313 1314 ata_channel_unlock(chp); 1315 1316 if (__predict_true(!ISSET(xfer->c_flags, C_PRIVATE_ALLOC))) 1317 pool_put(&ata_xfer_pool, xfer); 1318 } 1319 1320 void 1321 ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 1322 { 1323 struct ata_queue * const chq = chp->ch_queue; 1324 1325 ata_channel_lock(chp); 1326 1327 KASSERT(chq->queue_active > 0); 1328 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0); 1329 1330 /* Stop only when this is last active xfer */ 1331 if (chq->queue_active == 1) 1332 callout_stop(&chp->c_timo_callout); 1333 1334 if (callout_invoking(&chp->c_timo_callout)) 1335 xfer->c_flags |= C_WAITTIMO; 1336 1337 TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain); 1338 chq->active_xfers_used &= ~__BIT(xfer->c_slot); 1339 chq->queue_active--; 1340 1341 ata_queue_free_slot(chp, xfer->c_slot); 1342 1343 if (xfer->c_flags & C_WAIT) 1344 cv_broadcast(&chq->c_cmd_finish); 1345 1346 ata_channel_unlock(chp); 1347 } 1348 1349 /* 1350 * Called in c_intr hook. Must be called before before any deactivations 1351 * are done - if there is drain pending, it calls c_kill_xfer hook which 1352 * deactivates the xfer. 1353 * Calls c_kill_xfer with channel lock free. 1354 * Returns true if caller should just exit without further processing. 1355 * Caller must not further access any part of xfer or any related controller 1356 * structures in that case, it should just return. 1357 */ 1358 bool 1359 ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer) 1360 { 1361 int drive = xfer->c_drive; 1362 bool draining = false; 1363 1364 ata_channel_lock(chp); 1365 1366 if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) { 1367 ata_channel_unlock(chp); 1368 1369 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE); 1370 1371 ata_channel_lock(chp); 1372 chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN; 1373 cv_signal(&chp->ch_queue->queue_drain); 1374 draining = true; 1375 } 1376 1377 ata_channel_unlock(chp); 1378 1379 return draining; 1380 } 1381 1382 /* 1383 * Check for race of normal transfer handling vs. timeout. 1384 */ 1385 bool 1386 ata_timo_xfer_check(struct ata_xfer *xfer) 1387 { 1388 struct ata_channel *chp = xfer->c_chp; 1389 struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive]; 1390 1391 ata_channel_lock(chp); 1392 1393 if (xfer->c_flags & C_WAITTIMO) { 1394 xfer->c_flags &= ~C_WAITTIMO; 1395 1396 /* Handle race vs. ata_free_xfer() */ 1397 if (xfer->c_flags & C_FREE) { 1398 xfer->c_flags &= ~C_FREE; 1399 ata_channel_unlock(chp); 1400 1401 device_printf(drvp->drv_softc, 1402 "xfer %"PRIxPTR" freed while invoking timeout\n", 1403 (intptr_t)xfer & PAGE_MASK); 1404 1405 ata_free_xfer(chp, xfer); 1406 return true; 1407 } 1408 1409 /* Race vs. callout_stop() in ata_deactivate_xfer() */ 1410 ata_channel_unlock(chp); 1411 1412 device_printf(drvp->drv_softc, 1413 "xfer %"PRIxPTR" deactivated while invoking timeout\n", 1414 (intptr_t)xfer & PAGE_MASK); 1415 return true; 1416 } 1417 1418 ata_channel_unlock(chp); 1419 1420 /* No race, proceed with timeout handling */ 1421 return false; 1422 } 1423 1424 /* 1425 * Kill off all active xfers for a ata_channel. 1426 * 1427 * Must be called with channel lock held. 1428 */ 1429 void 1430 ata_kill_active(struct ata_channel *chp, int reason, int flags) 1431 { 1432 struct ata_queue * const chq = chp->ch_queue; 1433 struct ata_xfer *xfer, *xfernext; 1434 1435 KASSERT(mutex_owned(&chp->ch_lock)); 1436 1437 TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) { 1438 xfer->ops->c_kill_xfer(xfer->c_chp, xfer, reason); 1439 } 1440 } 1441 1442 /* 1443 * Kill off all pending xfers for a drive. 1444 */ 1445 void 1446 ata_kill_pending(struct ata_drive_datas *drvp) 1447 { 1448 struct ata_channel * const chp = drvp->chnl_softc; 1449 struct ata_queue * const chq = chp->ch_queue; 1450 struct ata_xfer *xfer; 1451 1452 ata_channel_lock(chp); 1453 1454 /* Kill all pending transfers */ 1455 while ((xfer = SIMPLEQ_FIRST(&chq->queue_xfer))) { 1456 KASSERT(xfer->c_chp == chp); 1457 1458 if (xfer->c_drive != drvp->drive) 1459 continue; 1460 1461 SIMPLEQ_REMOVE_HEAD(&chp->ch_queue->queue_xfer, c_xferchain); 1462 1463 /* 1464 * Keep the lock, so that we get deadlock (and 'locking against 1465 * myself' with LOCKDEBUG), instead of silent 1466 * data corruption, if the hook tries to call back into 1467 * middle layer for inactive xfer. 1468 */ 1469 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE_INACTIVE); 1470 } 1471 1472 /* Wait until all active transfers on the drive finish */ 1473 while (chq->queue_active > 0) { 1474 bool drv_active = false; 1475 1476 TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) { 1477 KASSERT(xfer->c_chp == chp); 1478 1479 if (xfer->c_drive == drvp->drive) { 1480 drv_active = true; 1481 break; 1482 } 1483 } 1484 1485 if (!drv_active) { 1486 /* all finished */ 1487 break; 1488 } 1489 1490 drvp->drive_flags |= ATA_DRIVE_WAITDRAIN; 1491 cv_wait(&chq->queue_drain, &chp->ch_lock); 1492 } 1493 1494 ata_channel_unlock(chp); 1495 } 1496 1497 static void 1498 ata_channel_freeze_locked(struct ata_channel *chp) 1499 { 1500 chp->ch_queue->queue_freeze++; 1501 1502 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1503 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1504 } 1505 1506 void 1507 ata_channel_freeze(struct ata_channel *chp) 1508 { 1509 ata_channel_lock(chp); 1510 ata_channel_freeze_locked(chp); 1511 ata_channel_unlock(chp); 1512 } 1513 1514 void 1515 ata_channel_thaw_locked(struct ata_channel *chp) 1516 { 1517 KASSERT(mutex_owned(&chp->ch_lock)); 1518 KASSERT(chp->ch_queue->queue_freeze > 0); 1519 1520 chp->ch_queue->queue_freeze--; 1521 1522 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp, 1523 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS); 1524 } 1525 1526 /* 1527 * ata_thread_run: 1528 * 1529 * Reset and ATA channel. Channel lock must be held. arg is type-specific. 1530 */ 1531 void 1532 ata_thread_run(struct ata_channel *chp, int flags, int type, int arg) 1533 { 1534 struct atac_softc *atac = chp->ch_atac; 1535 bool threset = false; 1536 struct ata_drive_datas *drvp; 1537 1538 ata_channel_lock_owned(chp); 1539 1540 /* 1541 * If we can poll or wait it's OK, otherwise wake up the 1542 * kernel thread to do it for us. 1543 */ 1544 ATADEBUG_PRINT(("%s flags 0x%x ch_flags 0x%x\n", 1545 __func__, flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS); 1546 if ((flags & (AT_POLL | AT_WAIT)) == 0) { 1547 switch (type) { 1548 case ATACH_TH_RESET: 1549 if (chp->ch_flags & ATACH_TH_RESET) { 1550 /* No need to schedule another reset */ 1551 return; 1552 } 1553 break; 1554 case ATACH_TH_DRIVE_RESET: 1555 { 1556 int drive = arg; 1557 1558 KASSERT(drive <= chp->ch_ndrives); 1559 drvp = &chp->ch_drive[drive]; 1560 1561 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) { 1562 /* No need to schedule another reset */ 1563 return; 1564 } 1565 drvp->drive_flags |= ATA_DRIVE_TH_RESET; 1566 break; 1567 } 1568 case ATACH_TH_RECOVERY: 1569 { 1570 uint32_t tfd = (uint32_t)arg; 1571 1572 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1573 chp->recovery_tfd = tfd; 1574 break; 1575 } 1576 default: 1577 panic("%s: unknown type: %x", __func__, type); 1578 /* NOTREACHED */ 1579 } 1580 1581 /* 1582 * Block execution of other commands while reset is scheduled 1583 * to a thread. 1584 */ 1585 ata_channel_freeze_locked(chp); 1586 chp->ch_flags |= type; 1587 1588 cv_signal(&chp->ch_thr_idle); 1589 return; 1590 } 1591 1592 /* Block execution of other commands during reset */ 1593 ata_channel_freeze_locked(chp); 1594 1595 /* 1596 * If reset has been scheduled to a thread, then clear 1597 * the flag now so that the thread won't try to execute it if 1598 * we happen to sleep, and thaw one more time after the reset. 1599 */ 1600 if (chp->ch_flags & type) { 1601 chp->ch_flags &= ~type; 1602 threset = true; 1603 } 1604 1605 switch (type) { 1606 case ATACH_TH_RESET: 1607 (*atac->atac_bustype_ata->ata_reset_channel)(chp, flags); 1608 1609 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1610 for (int drive = 0; drive < chp->ch_ndrives; drive++) 1611 chp->ch_drive[drive].state = 0; 1612 break; 1613 1614 case ATACH_TH_DRIVE_RESET: 1615 { 1616 int drive = arg; 1617 1618 KASSERT(drive <= chp->ch_ndrives); 1619 drvp = &chp->ch_drive[drive]; 1620 (*atac->atac_bustype_ata->ata_reset_drive)(drvp, flags, NULL); 1621 drvp->state = 0; 1622 break; 1623 } 1624 1625 case ATACH_TH_RECOVERY: 1626 { 1627 uint32_t tfd = (uint32_t)arg; 1628 1629 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0); 1630 KASSERT(atac->atac_bustype_ata->ata_recovery != NULL); 1631 1632 SET(chp->ch_flags, ATACH_RECOVERING); 1633 (*atac->atac_bustype_ata->ata_recovery)(chp, flags, tfd); 1634 CLR(chp->ch_flags, ATACH_RECOVERING); 1635 break; 1636 } 1637 1638 default: 1639 panic("%s: unknown type: %x", __func__, type); 1640 /* NOTREACHED */ 1641 } 1642 1643 /* 1644 * Thaw one extra time to clear the freeze done when the reset has 1645 * been scheduled to the thread. 1646 */ 1647 if (threset) 1648 ata_channel_thaw_locked(chp); 1649 1650 /* Allow commands to run again */ 1651 ata_channel_thaw_locked(chp); 1652 1653 /* Signal the thread in case there is an xfer to run */ 1654 cv_signal(&chp->ch_thr_idle); 1655 } 1656 1657 int 1658 ata_addref(struct ata_channel *chp) 1659 { 1660 struct atac_softc *atac = chp->ch_atac; 1661 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1662 int s, error = 0; 1663 1664 s = splbio(); 1665 if (adapt->adapt_refcnt++ == 0 && 1666 adapt->adapt_enable != NULL) { 1667 error = (*adapt->adapt_enable)(atac->atac_dev, 1); 1668 if (error) 1669 adapt->adapt_refcnt--; 1670 } 1671 splx(s); 1672 return (error); 1673 } 1674 1675 void 1676 ata_delref(struct ata_channel *chp) 1677 { 1678 struct atac_softc *atac = chp->ch_atac; 1679 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic; 1680 int s; 1681 1682 s = splbio(); 1683 if (adapt->adapt_refcnt-- == 1 && 1684 adapt->adapt_enable != NULL) 1685 (void) (*adapt->adapt_enable)(atac->atac_dev, 0); 1686 splx(s); 1687 } 1688 1689 void 1690 ata_print_modes(struct ata_channel *chp) 1691 { 1692 struct atac_softc *atac = chp->ch_atac; 1693 int drive; 1694 struct ata_drive_datas *drvp; 1695 1696 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 1697 for (drive = 0; drive < chp->ch_ndrives; drive++) { 1698 drvp = &chp->ch_drive[drive]; 1699 if (drvp->drive_type == ATA_DRIVET_NONE || 1700 drvp->drv_softc == NULL) 1701 continue; 1702 aprint_verbose("%s(%s:%d:%d): using PIO mode %d", 1703 device_xname(drvp->drv_softc), 1704 device_xname(atac->atac_dev), 1705 chp->ch_channel, drvp->drive, drvp->PIO_mode); 1706 #if NATA_DMA 1707 if (drvp->drive_flags & ATA_DRIVE_DMA) 1708 aprint_verbose(", DMA mode %d", drvp->DMA_mode); 1709 #if NATA_UDMA 1710 if (drvp->drive_flags & ATA_DRIVE_UDMA) { 1711 aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode); 1712 if (drvp->UDMA_mode == 2) 1713 aprint_verbose(" (Ultra/33)"); 1714 else if (drvp->UDMA_mode == 4) 1715 aprint_verbose(" (Ultra/66)"); 1716 else if (drvp->UDMA_mode == 5) 1717 aprint_verbose(" (Ultra/100)"); 1718 else if (drvp->UDMA_mode == 6) 1719 aprint_verbose(" (Ultra/133)"); 1720 } 1721 #endif /* NATA_UDMA */ 1722 #endif /* NATA_DMA */ 1723 #if NATA_DMA || NATA_PIOBM 1724 if (0 1725 #if NATA_DMA 1726 || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) 1727 #endif 1728 #if NATA_PIOBM 1729 /* PIOBM capable controllers use DMA for PIO commands */ 1730 || (atac->atac_cap & ATAC_CAP_PIOBM) 1731 #endif 1732 ) 1733 aprint_verbose(" (using DMA)"); 1734 1735 if (drvp->drive_flags & ATA_DRIVE_NCQ) { 1736 aprint_verbose(", NCQ (%d tags)%s", 1737 ATA_REAL_OPENINGS(chp->ch_queue->queue_openings), 1738 (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO) 1739 ? " w/PRIO" : ""); 1740 } else if (drvp->drive_flags & ATA_DRIVE_WFUA) 1741 aprint_verbose(", WRITE DMA FUA EXT"); 1742 1743 #endif /* NATA_DMA || NATA_PIOBM */ 1744 aprint_verbose("\n"); 1745 } 1746 } 1747 1748 #if NATA_DMA 1749 /* 1750 * downgrade the transfer mode of a drive after an error. return 1 if 1751 * downgrade was possible, 0 otherwise. 1752 * 1753 * MUST BE CALLED AT splbio()! 1754 */ 1755 int 1756 ata_downgrade_mode(struct ata_drive_datas *drvp, int flags) 1757 { 1758 struct ata_channel *chp = drvp->chnl_softc; 1759 struct atac_softc *atac = chp->ch_atac; 1760 device_t drv_dev = drvp->drv_softc; 1761 int cf_flags = device_cfdata(drv_dev)->cf_flags; 1762 1763 /* if drive or controller don't know its mode, we can't do much */ 1764 if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 || 1765 (atac->atac_set_modes == NULL)) 1766 return 0; 1767 /* current drive mode was set by a config flag, let it this way */ 1768 if ((cf_flags & ATA_CONFIG_PIO_SET) || 1769 (cf_flags & ATA_CONFIG_DMA_SET) || 1770 (cf_flags & ATA_CONFIG_UDMA_SET)) 1771 return 0; 1772 1773 #if NATA_UDMA 1774 /* 1775 * If we were using Ultra-DMA mode, downgrade to the next lower mode. 1776 */ 1777 if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) { 1778 drvp->UDMA_mode--; 1779 aprint_error_dev(drv_dev, 1780 "transfer error, downgrading to Ultra-DMA mode %d\n", 1781 drvp->UDMA_mode); 1782 } 1783 #endif 1784 1785 /* 1786 * If we were using ultra-DMA, don't downgrade to multiword DMA. 1787 */ 1788 else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) { 1789 drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA); 1790 drvp->PIO_mode = drvp->PIO_cap; 1791 aprint_error_dev(drv_dev, 1792 "transfer error, downgrading to PIO mode %d\n", 1793 drvp->PIO_mode); 1794 } else /* already using PIO, can't downgrade */ 1795 return 0; 1796 1797 (*atac->atac_set_modes)(chp); 1798 ata_print_modes(chp); 1799 /* reset the channel, which will schedule all drives for setup */ 1800 ata_thread_run(chp, flags, ATACH_TH_RESET, ATACH_NODRIVE); 1801 return 1; 1802 } 1803 #endif /* NATA_DMA */ 1804 1805 /* 1806 * Probe drive's capabilities, for use by the controller later 1807 * Assumes drvp points to an existing drive. 1808 */ 1809 void 1810 ata_probe_caps(struct ata_drive_datas *drvp) 1811 { 1812 struct ataparams params, params2; 1813 struct ata_channel *chp = drvp->chnl_softc; 1814 struct atac_softc *atac = chp->ch_atac; 1815 device_t drv_dev = drvp->drv_softc; 1816 int i, printed = 0; 1817 const char *sep = ""; 1818 int cf_flags; 1819 1820 if (ata_get_params(drvp, AT_WAIT, ¶ms) != CMD_OK) { 1821 /* IDENTIFY failed. Can't tell more about the device */ 1822 return; 1823 } 1824 if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) == 1825 (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) { 1826 /* 1827 * Controller claims 16 and 32 bit transfers. 1828 * Re-do an IDENTIFY with 32-bit transfers, 1829 * and compare results. 1830 */ 1831 ata_channel_lock(chp); 1832 drvp->drive_flags |= ATA_DRIVE_CAP32; 1833 ata_channel_unlock(chp); 1834 ata_get_params(drvp, AT_WAIT, ¶ms2); 1835 if (memcmp(¶ms, ¶ms2, sizeof(struct ataparams)) != 0) { 1836 /* Not good. fall back to 16bits */ 1837 ata_channel_lock(chp); 1838 drvp->drive_flags &= ~ATA_DRIVE_CAP32; 1839 ata_channel_unlock(chp); 1840 } else { 1841 aprint_verbose_dev(drv_dev, "32-bit data port\n"); 1842 } 1843 } 1844 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */ 1845 if (params.atap_ata_major > 0x01 && 1846 params.atap_ata_major != 0xffff) { 1847 for (i = 14; i > 0; i--) { 1848 if (params.atap_ata_major & (1 << i)) { 1849 aprint_verbose_dev(drv_dev, 1850 "ATA version %d\n", i); 1851 drvp->ata_vers = i; 1852 break; 1853 } 1854 } 1855 } 1856 #endif 1857 1858 /* An ATAPI device is at last PIO mode 3 */ 1859 if (drvp->drive_type == ATA_DRIVET_ATAPI) 1860 drvp->PIO_mode = 3; 1861 1862 /* 1863 * It's not in the specs, but it seems that some drive 1864 * returns 0xffff in atap_extensions when this field is invalid 1865 */ 1866 if (params.atap_extensions != 0xffff && 1867 (params.atap_extensions & WDC_EXT_MODES)) { 1868 /* 1869 * XXX some drives report something wrong here (they claim to 1870 * support PIO mode 8 !). As mode is coded on 3 bits in 1871 * SET FEATURE, limit it to 7 (so limit i to 4). 1872 * If higher mode than 7 is found, abort. 1873 */ 1874 for (i = 7; i >= 0; i--) { 1875 if ((params.atap_piomode_supp & (1 << i)) == 0) 1876 continue; 1877 if (i > 4) 1878 return; 1879 /* 1880 * See if mode is accepted. 1881 * If the controller can't set its PIO mode, 1882 * assume the defaults are good, so don't try 1883 * to set it 1884 */ 1885 if (atac->atac_set_modes) 1886 /* 1887 * It's OK to pool here, it's fast enough 1888 * to not bother waiting for interrupt 1889 */ 1890 if (ata_set_mode(drvp, 0x08 | (i + 3), 1891 AT_WAIT) != CMD_OK) 1892 continue; 1893 if (!printed) { 1894 aprint_verbose_dev(drv_dev, 1895 "drive supports PIO mode %d", i + 3); 1896 sep = ","; 1897 printed = 1; 1898 } 1899 /* 1900 * If controller's driver can't set its PIO mode, 1901 * get the highter one for the drive. 1902 */ 1903 if (atac->atac_set_modes == NULL || 1904 atac->atac_pio_cap >= i + 3) { 1905 drvp->PIO_mode = i + 3; 1906 drvp->PIO_cap = i + 3; 1907 break; 1908 } 1909 } 1910 if (!printed) { 1911 /* 1912 * We didn't find a valid PIO mode. 1913 * Assume the values returned for DMA are buggy too 1914 */ 1915 return; 1916 } 1917 ata_channel_lock(chp); 1918 drvp->drive_flags |= ATA_DRIVE_MODE; 1919 ata_channel_unlock(chp); 1920 printed = 0; 1921 for (i = 7; i >= 0; i--) { 1922 if ((params.atap_dmamode_supp & (1 << i)) == 0) 1923 continue; 1924 #if NATA_DMA 1925 if ((atac->atac_cap & ATAC_CAP_DMA) && 1926 atac->atac_set_modes != NULL) 1927 if (ata_set_mode(drvp, 0x20 | i, AT_WAIT) 1928 != CMD_OK) 1929 continue; 1930 #endif 1931 if (!printed) { 1932 aprint_verbose("%s DMA mode %d", sep, i); 1933 sep = ","; 1934 printed = 1; 1935 } 1936 #if NATA_DMA 1937 if (atac->atac_cap & ATAC_CAP_DMA) { 1938 if (atac->atac_set_modes != NULL && 1939 atac->atac_dma_cap < i) 1940 continue; 1941 drvp->DMA_mode = i; 1942 drvp->DMA_cap = i; 1943 ata_channel_lock(chp); 1944 drvp->drive_flags |= ATA_DRIVE_DMA; 1945 ata_channel_unlock(chp); 1946 } 1947 #endif 1948 break; 1949 } 1950 if (params.atap_extensions & WDC_EXT_UDMA_MODES) { 1951 printed = 0; 1952 for (i = 7; i >= 0; i--) { 1953 if ((params.atap_udmamode_supp & (1 << i)) 1954 == 0) 1955 continue; 1956 #if NATA_UDMA 1957 if (atac->atac_set_modes != NULL && 1958 (atac->atac_cap & ATAC_CAP_UDMA)) 1959 if (ata_set_mode(drvp, 0x40 | i, 1960 AT_WAIT) != CMD_OK) 1961 continue; 1962 #endif 1963 if (!printed) { 1964 aprint_verbose("%s Ultra-DMA mode %d", 1965 sep, i); 1966 if (i == 2) 1967 aprint_verbose(" (Ultra/33)"); 1968 else if (i == 4) 1969 aprint_verbose(" (Ultra/66)"); 1970 else if (i == 5) 1971 aprint_verbose(" (Ultra/100)"); 1972 else if (i == 6) 1973 aprint_verbose(" (Ultra/133)"); 1974 sep = ","; 1975 printed = 1; 1976 } 1977 #if NATA_UDMA 1978 if (atac->atac_cap & ATAC_CAP_UDMA) { 1979 if (atac->atac_set_modes != NULL && 1980 atac->atac_udma_cap < i) 1981 continue; 1982 drvp->UDMA_mode = i; 1983 drvp->UDMA_cap = i; 1984 ata_channel_lock(chp); 1985 drvp->drive_flags |= ATA_DRIVE_UDMA; 1986 ata_channel_unlock(chp); 1987 } 1988 #endif 1989 break; 1990 } 1991 } 1992 } 1993 1994 ata_channel_lock(chp); 1995 drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM; 1996 if (drvp->drive_type == ATA_DRIVET_ATAPI) { 1997 if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM) 1998 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 1999 } else { 2000 if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM) 2001 drvp->drive_flags |= ATA_DRIVE_NOSTREAM; 2002 } 2003 ata_channel_unlock(chp); 2004 2005 /* Try to guess ATA version here, if it didn't get reported */ 2006 if (drvp->ata_vers == 0) { 2007 #if NATA_UDMA 2008 if (drvp->drive_flags & ATA_DRIVE_UDMA) 2009 drvp->ata_vers = 4; /* should be at last ATA-4 */ 2010 else 2011 #endif 2012 if (drvp->PIO_cap > 2) 2013 drvp->ata_vers = 2; /* should be at last ATA-2 */ 2014 } 2015 cf_flags = device_cfdata(drv_dev)->cf_flags; 2016 if (cf_flags & ATA_CONFIG_PIO_SET) { 2017 ata_channel_lock(chp); 2018 drvp->PIO_mode = 2019 (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF; 2020 drvp->drive_flags |= ATA_DRIVE_MODE; 2021 ata_channel_unlock(chp); 2022 } 2023 #if NATA_DMA 2024 if ((atac->atac_cap & ATAC_CAP_DMA) == 0) { 2025 /* don't care about DMA modes */ 2026 return; 2027 } 2028 if (cf_flags & ATA_CONFIG_DMA_SET) { 2029 ata_channel_lock(chp); 2030 if ((cf_flags & ATA_CONFIG_DMA_MODES) == 2031 ATA_CONFIG_DMA_DISABLE) { 2032 drvp->drive_flags &= ~ATA_DRIVE_DMA; 2033 } else { 2034 drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >> 2035 ATA_CONFIG_DMA_OFF; 2036 drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE; 2037 } 2038 ata_channel_unlock(chp); 2039 } 2040 2041 /* 2042 * Probe WRITE DMA FUA EXT. Support is mandatory for devices 2043 * supporting LBA48, but nevertheless confirm with the feature flag. 2044 */ 2045 if (drvp->drive_flags & ATA_DRIVE_DMA) { 2046 if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0 2047 && (params.atap_cmd_def & ATA_CMDE_WFE)) { 2048 drvp->drive_flags |= ATA_DRIVE_WFUA; 2049 aprint_verbose("%s WRITE DMA FUA", sep); 2050 sep = ","; 2051 } 2052 } 2053 2054 /* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */ 2055 ata_channel_lock(chp); 2056 drvp->drv_openings = 1; 2057 if (params.atap_sata_caps & SATA_NATIVE_CMDQ) { 2058 if (atac->atac_cap & ATAC_CAP_NCQ) 2059 drvp->drive_flags |= ATA_DRIVE_NCQ; 2060 drvp->drv_openings = 2061 (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1; 2062 aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings); 2063 sep = ","; 2064 2065 if (params.atap_sata_caps & SATA_NCQ_PRIO) { 2066 drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO; 2067 aprint_verbose(" w/PRIO"); 2068 } 2069 } 2070 ata_channel_unlock(chp); 2071 2072 if (printed) 2073 aprint_verbose("\n"); 2074 2075 #if NATA_UDMA 2076 if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) { 2077 /* don't care about UDMA modes */ 2078 return; 2079 } 2080 if (cf_flags & ATA_CONFIG_UDMA_SET) { 2081 ata_channel_lock(chp); 2082 if ((cf_flags & ATA_CONFIG_UDMA_MODES) == 2083 ATA_CONFIG_UDMA_DISABLE) { 2084 drvp->drive_flags &= ~ATA_DRIVE_UDMA; 2085 } else { 2086 drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >> 2087 ATA_CONFIG_UDMA_OFF; 2088 drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE; 2089 } 2090 ata_channel_unlock(chp); 2091 } 2092 #endif /* NATA_UDMA */ 2093 #endif /* NATA_DMA */ 2094 } 2095 2096 /* management of the /dev/atabus* devices */ 2097 int 2098 atabusopen(dev_t dev, int flag, int fmt, struct lwp *l) 2099 { 2100 struct atabus_softc *sc; 2101 int error; 2102 2103 sc = device_lookup_private(&atabus_cd, minor(dev)); 2104 if (sc == NULL) 2105 return (ENXIO); 2106 2107 if (sc->sc_flags & ATABUSCF_OPEN) 2108 return (EBUSY); 2109 2110 if ((error = ata_addref(sc->sc_chan)) != 0) 2111 return (error); 2112 2113 sc->sc_flags |= ATABUSCF_OPEN; 2114 2115 return (0); 2116 } 2117 2118 2119 int 2120 atabusclose(dev_t dev, int flag, int fmt, struct lwp *l) 2121 { 2122 struct atabus_softc *sc = 2123 device_lookup_private(&atabus_cd, minor(dev)); 2124 2125 ata_delref(sc->sc_chan); 2126 2127 sc->sc_flags &= ~ATABUSCF_OPEN; 2128 2129 return (0); 2130 } 2131 2132 int 2133 atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l) 2134 { 2135 struct atabus_softc *sc = 2136 device_lookup_private(&atabus_cd, minor(dev)); 2137 struct ata_channel *chp = sc->sc_chan; 2138 int min_drive, max_drive, drive; 2139 int error; 2140 2141 /* 2142 * Enforce write permission for ioctls that change the 2143 * state of the bus. Host adapter specific ioctls must 2144 * be checked by the adapter driver. 2145 */ 2146 switch (cmd) { 2147 case ATABUSIOSCAN: 2148 case ATABUSIODETACH: 2149 case ATABUSIORESET: 2150 if ((flag & FWRITE) == 0) 2151 return (EBADF); 2152 } 2153 2154 switch (cmd) { 2155 case ATABUSIORESET: 2156 ata_channel_lock(chp); 2157 ata_thread_run(sc->sc_chan, AT_WAIT | AT_POLL, 2158 ATACH_TH_RESET, ATACH_NODRIVE); 2159 ata_channel_unlock(chp); 2160 return 0; 2161 case ATABUSIOSCAN: 2162 { 2163 #if 0 2164 struct atabusioscan_args *a= 2165 (struct atabusioscan_args *)addr; 2166 #endif 2167 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2168 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2169 return (EOPNOTSUPP); 2170 return (EOPNOTSUPP); 2171 } 2172 case ATABUSIODETACH: 2173 { 2174 struct atabusiodetach_args *a= 2175 (struct atabusiodetach_args *)addr; 2176 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) || 2177 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD)) 2178 return (EOPNOTSUPP); 2179 switch (a->at_dev) { 2180 case -1: 2181 min_drive = 0; 2182 max_drive = 1; 2183 break; 2184 case 0: 2185 case 1: 2186 min_drive = max_drive = a->at_dev; 2187 break; 2188 default: 2189 return (EINVAL); 2190 } 2191 for (drive = min_drive; drive <= max_drive; drive++) { 2192 if (chp->ch_drive[drive].drv_softc != NULL) { 2193 error = config_detach( 2194 chp->ch_drive[drive].drv_softc, 0); 2195 if (error) 2196 return (error); 2197 KASSERT(chp->ch_drive[drive].drv_softc == NULL); 2198 } 2199 } 2200 return 0; 2201 } 2202 default: 2203 return ENOTTY; 2204 } 2205 } 2206 2207 static bool 2208 atabus_suspend(device_t dv, const pmf_qual_t *qual) 2209 { 2210 struct atabus_softc *sc = device_private(dv); 2211 struct ata_channel *chp = sc->sc_chan; 2212 2213 ata_channel_idle(chp); 2214 2215 return true; 2216 } 2217 2218 static bool 2219 atabus_resume(device_t dv, const pmf_qual_t *qual) 2220 { 2221 struct atabus_softc *sc = device_private(dv); 2222 struct ata_channel *chp = sc->sc_chan; 2223 2224 /* 2225 * XXX joerg: with wdc, the first channel unfreezes the controler. 2226 * Move this the reset and queue idling into wdc. 2227 */ 2228 ata_channel_lock(chp); 2229 if (chp->ch_queue->queue_freeze == 0) { 2230 ata_channel_unlock(chp); 2231 goto out; 2232 } 2233 2234 /* unfreeze the queue and reset drives */ 2235 ata_channel_thaw_locked(chp); 2236 2237 /* reset channel only if there are drives attached */ 2238 if (chp->ch_ndrives > 0) 2239 ata_thread_run(chp, AT_WAIT, ATACH_TH_RESET, ATACH_NODRIVE); 2240 2241 ata_channel_unlock(chp); 2242 2243 out: 2244 return true; 2245 } 2246 2247 static int 2248 atabus_rescan(device_t self, const char *ifattr, const int *locators) 2249 { 2250 struct atabus_softc *sc = device_private(self); 2251 struct ata_channel *chp = sc->sc_chan; 2252 struct atabus_initq *initq; 2253 int i; 2254 2255 /* 2256 * we can rescan a port multiplier atabus, even if some devices are 2257 * still attached 2258 */ 2259 if (chp->ch_satapmp_nports == 0) { 2260 if (chp->atapibus != NULL) { 2261 return EBUSY; 2262 } 2263 2264 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL); 2265 for (i = 0; i < chp->ch_ndrives; i++) { 2266 if (chp->ch_drive[i].drv_softc != NULL) { 2267 return EBUSY; 2268 } 2269 } 2270 } 2271 2272 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP); 2273 initq->atabus_sc = sc; 2274 mutex_enter(&atabus_qlock); 2275 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq); 2276 mutex_exit(&atabus_qlock); 2277 config_pending_incr(sc->sc_dev); 2278 2279 ata_channel_lock(chp); 2280 chp->ch_flags |= ATACH_TH_RESCAN; 2281 cv_signal(&chp->ch_thr_idle); 2282 ata_channel_unlock(chp); 2283 2284 return 0; 2285 } 2286 2287 void 2288 ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags) 2289 { 2290 KASSERT(mutex_owned(&chp->ch_lock)); 2291 2292 if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) { 2293 /* 2294 * can't use kpause(), we may be in interrupt context 2295 * or taking a crash dump 2296 */ 2297 delay(ms * 1000); 2298 } else { 2299 int pause = mstohz(ms); 2300 2301 kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock); 2302 } 2303 } 2304 2305 void 2306 atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count, 2307 uint16_t *features, uint8_t *device) 2308 { 2309 if ((xfer->c_flags & C_NCQ) == 0) { 2310 /* FUA handling for non-NCQ drives */ 2311 if (xfer->c_bio.flags & ATA_FUA 2312 && *cmd == WDCC_WRITEDMA_EXT) 2313 *cmd = WDCC_WRITEDMA_FUA_EXT; 2314 2315 return; 2316 } 2317 2318 *cmd = (xfer->c_bio.flags & ATA_READ) ? 2319 WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED; 2320 2321 /* for FPDMA the block count is in features */ 2322 *features = *count; 2323 2324 /* NCQ tag */ 2325 *count = (xfer->c_slot << 3); 2326 2327 if (xfer->c_bio.flags & ATA_PRIO_HIGH) 2328 *count |= WDSC_PRIO_HIGH; 2329 2330 /* other device flags */ 2331 if (xfer->c_bio.flags & ATA_FUA) 2332 *device |= WDSD_FUA; 2333 } 2334 2335 void 2336 ata_wait_cmd(struct ata_channel *chp, struct ata_xfer *xfer) 2337 { 2338 struct ata_queue *chq = chp->ch_queue; 2339 struct ata_command *ata_c = &xfer->c_ata_c; 2340 2341 ata_channel_lock(chp); 2342 2343 while ((ata_c->flags & AT_DONE) == 0) 2344 cv_wait(&chq->c_cmd_finish, &chp->ch_lock); 2345 2346 ata_channel_unlock(chp); 2347 2348 KASSERT((ata_c->flags & AT_DONE) != 0); 2349 } 2350
Indexes created Thu Sep 25 16:09:42 GMT 2025