ata.c revision 1.144 1 /* $NetBSD: ata.c,v 1.144 2018/10/24 20:01:13 jdolecek Exp $ */
2
3 /*
4 * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.144 2018/10/24 20:01:13 jdolecek Exp $");
29
30 #include "opt_ata.h"
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/device.h>
36 #include <sys/conf.h>
37 #include <sys/fcntl.h>
38 #include <sys/proc.h>
39 #include <sys/kthread.h>
40 #include <sys/errno.h>
41 #include <sys/ataio.h>
42 #include <sys/kmem.h>
43 #include <sys/intr.h>
44 #include <sys/bus.h>
45 #include <sys/once.h>
46 #include <sys/bitops.h>
47
48 #define ATABUS_PRIVATE
49
50 #include <dev/ata/ataconf.h>
51 #include <dev/ata/atareg.h>
52 #include <dev/ata/atavar.h>
53 #include <dev/ic/wdcvar.h> /* for PIOBM */
54
55 #include "ioconf.h"
56 #include "locators.h"
57
58 #include "atapibus.h"
59 #include "ataraid.h"
60 #include "sata_pmp.h"
61
62 #if NATARAID > 0
63 #include <dev/ata/ata_raidvar.h>
64 #endif
65 #if NSATA_PMP > 0
66 #include <dev/ata/satapmpvar.h>
67 #endif
68 #include <dev/ata/satapmpreg.h>
69
70 #define DEBUG_FUNCS 0x08
71 #define DEBUG_PROBE 0x10
72 #define DEBUG_DETACH 0x20
73 #define DEBUG_XFERS 0x40
74 #ifdef ATADEBUG
75 #ifndef ATADEBUG_MASK
76 #define ATADEBUG_MASK 0
77 #endif
78 int atadebug_mask = ATADEBUG_MASK;
79 #define ATADEBUG_PRINT(args, level) \
80 if (atadebug_mask & (level)) \
81 printf args
82 #else
83 #define ATADEBUG_PRINT(args, level)
84 #endif
85
86 static ONCE_DECL(ata_init_ctrl);
87 static struct pool ata_xfer_pool;
88
89 /*
90 * A queue of atabus instances, used to ensure the same bus probe order
91 * for a given hardware configuration at each boot. Kthread probing
92 * devices on a atabus. Only one probing at once.
93 */
94 static TAILQ_HEAD(, atabus_initq) atabus_initq_head;
95 static kmutex_t atabus_qlock;
96 static kcondvar_t atabus_qcv;
97 static lwp_t * atabus_cfg_lwp;
98
99 /*****************************************************************************
100 * ATA bus layer.
101 *
102 * ATA controllers attach an atabus instance, which handles probing the bus
103 * for drives, etc.
104 *****************************************************************************/
105
106 dev_type_open(atabusopen);
107 dev_type_close(atabusclose);
108 dev_type_ioctl(atabusioctl);
109
110 const struct cdevsw atabus_cdevsw = {
111 .d_open = atabusopen,
112 .d_close = atabusclose,
113 .d_read = noread,
114 .d_write = nowrite,
115 .d_ioctl = atabusioctl,
116 .d_stop = nostop,
117 .d_tty = notty,
118 .d_poll = nopoll,
119 .d_mmap = nommap,
120 .d_kqfilter = nokqfilter,
121 .d_discard = nodiscard,
122 .d_flag = D_OTHER
123 };
124
125 static void atabus_childdetached(device_t, device_t);
126 static int atabus_rescan(device_t, const char *, const int *);
127 static bool atabus_resume(device_t, const pmf_qual_t *);
128 static bool atabus_suspend(device_t, const pmf_qual_t *);
129 static void atabusconfig_thread(void *);
130
131 static void ata_channel_idle(struct ata_channel *);
132 static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *);
133 static void ata_channel_freeze_locked(struct ata_channel *);
134 static void ata_thread_wake_locked(struct ata_channel *);
135
136 /*
137 * atabus_init:
138 *
139 * Initialize ATA subsystem structures.
140 */
141 static int
142 atabus_init(void)
143 {
144
145 pool_init(&ata_xfer_pool, sizeof(struct ata_xfer), 0, 0, 0,
146 "ataspl", NULL, IPL_BIO);
147 TAILQ_INIT(&atabus_initq_head);
148 mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE);
149 cv_init(&atabus_qcv, "atainitq");
150 return 0;
151 }
152
153 /*
154 * atabusprint:
155 *
156 * Autoconfiguration print routine used by ATA controllers when
157 * attaching an atabus instance.
158 */
159 int
160 atabusprint(void *aux, const char *pnp)
161 {
162 struct ata_channel *chan = aux;
163
164 if (pnp)
165 aprint_normal("atabus at %s", pnp);
166 aprint_normal(" channel %d", chan->ch_channel);
167
168 return (UNCONF);
169 }
170
171 /*
172 * ataprint:
173 *
174 * Autoconfiguration print routine.
175 */
176 int
177 ataprint(void *aux, const char *pnp)
178 {
179 struct ata_device *adev = aux;
180
181 if (pnp)
182 aprint_normal("wd at %s", pnp);
183 aprint_normal(" drive %d", adev->adev_drv_data->drive);
184
185 return (UNCONF);
186 }
187
188 /*
189 * ata_channel_attach:
190 *
191 * Common parts of attaching an atabus to an ATA controller channel.
192 */
193 void
194 ata_channel_attach(struct ata_channel *chp)
195 {
196 if (chp->ch_flags & ATACH_DISABLED)
197 return;
198
199 ata_channel_init(chp);
200
201 KASSERT(chp->ch_queue != NULL);
202
203 chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp,
204 atabusprint);
205 }
206
207 /*
208 * ata_channel_detach:
209 *
210 * Common parts of detaching an atabus to an ATA controller channel.
211 */
212 void
213 ata_channel_detach(struct ata_channel *chp)
214 {
215 if (chp->ch_flags & ATACH_DISABLED)
216 return;
217
218 ata_channel_destroy(chp);
219
220 chp->ch_flags |= ATACH_DETACHED;
221 }
222
223 static void
224 atabusconfig(struct atabus_softc *atabus_sc)
225 {
226 struct ata_channel *chp = atabus_sc->sc_chan;
227 struct atac_softc *atac = chp->ch_atac;
228 struct atabus_initq *atabus_initq = NULL;
229 int i, error;
230
231 /* we are in the atabus's thread context */
232 ata_channel_lock(chp);
233 chp->ch_flags |= ATACH_TH_RUN;
234 ata_channel_unlock(chp);
235
236 /*
237 * Probe for the drives attached to controller, unless a PMP
238 * is already known
239 */
240 /* XXX for SATA devices we will power up all drives at once */
241 if (chp->ch_satapmp_nports == 0)
242 (*atac->atac_probe)(chp);
243
244 if (chp->ch_ndrives >= 2) {
245 ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n",
246 chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type),
247 DEBUG_PROBE);
248 }
249
250 /* next operations will occurs in a separate thread */
251 ata_channel_lock(chp);
252 chp->ch_flags &= ~ATACH_TH_RUN;
253 ata_channel_unlock(chp);
254
255 /* Make sure the devices probe in atabus order to avoid jitter. */
256 mutex_enter(&atabus_qlock);
257 for (;;) {
258 atabus_initq = TAILQ_FIRST(&atabus_initq_head);
259 if (atabus_initq->atabus_sc == atabus_sc)
260 break;
261 cv_wait(&atabus_qcv, &atabus_qlock);
262 }
263 mutex_exit(&atabus_qlock);
264
265 ata_channel_lock(chp);
266
267 /* If no drives, abort here */
268 if (chp->ch_drive == NULL)
269 goto out;
270 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
271 for (i = 0; i < chp->ch_ndrives; i++)
272 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE)
273 break;
274 if (i == chp->ch_ndrives)
275 goto out;
276
277 /* Shortcut in case we've been shutdown */
278 if (chp->ch_flags & ATACH_SHUTDOWN)
279 goto out;
280
281 ata_channel_unlock(chp);
282
283 if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread,
284 atabus_sc, &atabus_cfg_lwp,
285 "%scnf", device_xname(atac->atac_dev))) != 0)
286 aprint_error_dev(atac->atac_dev,
287 "unable to create config thread: error %d\n", error);
288 return;
289
290 out:
291 ata_channel_unlock(chp);
292
293 mutex_enter(&atabus_qlock);
294 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
295 cv_broadcast(&atabus_qcv);
296 mutex_exit(&atabus_qlock);
297
298 kmem_free(atabus_initq, sizeof(*atabus_initq));
299
300 ata_delref(chp);
301
302 config_pending_decr(atac->atac_dev);
303 }
304
305 /*
306 * atabus_configthread: finish attach of atabus's childrens, in a separate
307 * kernel thread.
308 */
309 static void
310 atabusconfig_thread(void *arg)
311 {
312 struct atabus_softc *atabus_sc = arg;
313 struct ata_channel *chp = atabus_sc->sc_chan;
314 struct atac_softc *atac = chp->ch_atac;
315 struct atabus_initq *atabus_initq = NULL;
316 int i, s;
317
318 /* XXX seems wrong */
319 mutex_enter(&atabus_qlock);
320 atabus_initq = TAILQ_FIRST(&atabus_initq_head);
321 KASSERT(atabus_initq->atabus_sc == atabus_sc);
322 mutex_exit(&atabus_qlock);
323
324 /*
325 * First look for a port multiplier
326 */
327 if (chp->ch_ndrives == PMP_MAX_DRIVES &&
328 chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) {
329 #if NSATA_PMP > 0
330 satapmp_attach(chp);
331 #else
332 aprint_error_dev(atabus_sc->sc_dev,
333 "SATA port multiplier not supported\n");
334 /* no problems going on, all drives are ATA_DRIVET_NONE */
335 #endif
336 }
337
338 /*
339 * Attach an ATAPI bus, if needed.
340 */
341 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
342 for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) {
343 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) {
344 #if NATAPIBUS > 0
345 (*atac->atac_atapibus_attach)(atabus_sc);
346 #else
347 /*
348 * Fake the autoconfig "not configured" message
349 */
350 aprint_normal("atapibus at %s not configured\n",
351 device_xname(atac->atac_dev));
352 chp->atapibus = NULL;
353 s = splbio();
354 for (i = 0; i < chp->ch_ndrives; i++) {
355 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
356 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
357 }
358 splx(s);
359 #endif
360 break;
361 }
362 }
363
364 for (i = 0; i < chp->ch_ndrives; i++) {
365 struct ata_device adev;
366 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA &&
367 chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) {
368 continue;
369 }
370 if (chp->ch_drive[i].drv_softc != NULL)
371 continue;
372 memset(&adev, 0, sizeof(struct ata_device));
373 adev.adev_bustype = atac->atac_bustype_ata;
374 adev.adev_channel = chp->ch_channel;
375 adev.adev_drv_data = &chp->ch_drive[i];
376 chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev,
377 "ata_hl", &adev, ataprint);
378 if (chp->ch_drive[i].drv_softc != NULL) {
379 ata_probe_caps(&chp->ch_drive[i]);
380 } else {
381 s = splbio();
382 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
383 splx(s);
384 }
385 }
386
387 /* now that we know the drives, the controller can set its modes */
388 if (atac->atac_set_modes) {
389 (*atac->atac_set_modes)(chp);
390 ata_print_modes(chp);
391 }
392 #if NATARAID > 0
393 if (atac->atac_cap & ATAC_CAP_RAID) {
394 for (i = 0; i < chp->ch_ndrives; i++) {
395 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) {
396 ata_raid_check_component(
397 chp->ch_drive[i].drv_softc);
398 }
399 }
400 }
401 #endif /* NATARAID > 0 */
402
403 /*
404 * reset drive_flags for unattached devices, reset state for attached
405 * ones
406 */
407 s = splbio();
408 for (i = 0; i < chp->ch_ndrives; i++) {
409 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
410 continue;
411 if (chp->ch_drive[i].drv_softc == NULL) {
412 chp->ch_drive[i].drive_flags = 0;
413 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
414 } else
415 chp->ch_drive[i].state = 0;
416 }
417 splx(s);
418
419 mutex_enter(&atabus_qlock);
420 TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
421 cv_broadcast(&atabus_qcv);
422 mutex_exit(&atabus_qlock);
423
424 kmem_free(atabus_initq, sizeof(*atabus_initq));
425
426 ata_delref(chp);
427
428 config_pending_decr(atac->atac_dev);
429 kthread_exit(0);
430 }
431
432 /*
433 * atabus_thread:
434 *
435 * Worker thread for the ATA bus.
436 */
437 static void
438 atabus_thread(void *arg)
439 {
440 struct atabus_softc *sc = arg;
441 struct ata_channel *chp = sc->sc_chan;
442 struct ata_queue *chq = chp->ch_queue;
443 struct ata_xfer *xfer;
444 int i, rv;
445
446 ata_channel_lock(chp);
447 chp->ch_flags |= ATACH_TH_RUN;
448
449 /*
450 * Probe the drives. Reset type to indicate to controllers
451 * that can re-probe that all drives must be probed..
452 *
453 * Note: ch_ndrives may be changed during the probe.
454 */
455 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
456 for (i = 0; i < chp->ch_ndrives; i++) {
457 chp->ch_drive[i].drive_flags = 0;
458 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
459 }
460 ata_channel_unlock(chp);
461
462 atabusconfig(sc);
463
464 ata_channel_lock(chp);
465 for (;;) {
466 if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_TH_DRIVE_RESET
467 | ATACH_TH_RECOVERY | ATACH_SHUTDOWN)) == 0 &&
468 (chq->queue_active == 0 || chq->queue_freeze == 0)) {
469 chp->ch_flags &= ~ATACH_TH_RUN;
470 cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
471 chp->ch_flags |= ATACH_TH_RUN;
472 }
473 if (chp->ch_flags & ATACH_SHUTDOWN) {
474 break;
475 }
476 if (chp->ch_flags & ATACH_TH_RESCAN) {
477 chp->ch_flags &= ~ATACH_TH_RESCAN;
478 ata_channel_unlock(chp);
479 atabusconfig(sc);
480 ata_channel_lock(chp);
481 }
482 if (chp->ch_flags & ATACH_TH_RESET) {
483 /* this will unfreeze the channel */
484 ata_thread_run(chp, AT_WAIT,
485 ATACH_TH_RESET, ATACH_NODRIVE);
486 } else if (chp->ch_flags & ATACH_TH_DRIVE_RESET) {
487 /* this will unfreeze the channel */
488 for (i = 0; i < chp->ch_ndrives; i++) {
489 struct ata_drive_datas *drvp;
490
491 drvp = &chp->ch_drive[i];
492
493 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) {
494 ata_thread_run(chp,
495 AT_WAIT, ATACH_TH_DRIVE_RESET, i);
496 }
497 }
498 chp->ch_flags &= ~ATACH_TH_DRIVE_RESET;
499 } else if (chp->ch_flags & ATACH_TH_RECOVERY) {
500 /*
501 * This will unfreeze the channel; drops locks during
502 * run, so must wrap in splbio()/splx() to avoid
503 * spurious interrupts. XXX MPSAFE
504 */
505 int s = splbio();
506 ata_thread_run(chp, AT_WAIT, ATACH_TH_RECOVERY,
507 chp->recovery_tfd);
508 splx(s);
509 } else if (chq->queue_active > 0 && chq->queue_freeze == 1) {
510 /*
511 * Caller has bumped queue_freeze, decrease it. This
512 * flow shalt never be executed for NCQ commands.
513 */
514 KASSERT((chp->ch_flags & ATACH_NCQ) == 0);
515 KASSERT(chq->queue_active == 1);
516
517 ata_channel_thaw_locked(chp);
518 xfer = ata_queue_get_active_xfer_locked(chp);
519
520 KASSERT(xfer != NULL);
521 KASSERT((xfer->c_flags & C_POLL) == 0);
522
523 switch ((rv = ata_xfer_start(xfer))) {
524 case ATASTART_STARTED:
525 case ATASTART_POLL:
526 case ATASTART_ABORT:
527 break;
528 case ATASTART_TH:
529 default:
530 panic("%s: ata_xfer_start() unexpected rv %d",
531 __func__, rv);
532 /* NOTREACHED */
533 }
534 } else if (chq->queue_freeze > 1)
535 panic("%s: queue_freeze", __func__);
536
537 /* Try to run down the queue once channel is unfrozen */
538 if (chq->queue_freeze == 0) {
539 ata_channel_unlock(chp);
540 atastart(chp);
541 ata_channel_lock(chp);
542 }
543 }
544 chp->ch_thread = NULL;
545 cv_signal(&chp->ch_thr_idle);
546 ata_channel_unlock(chp);
547 kthread_exit(0);
548 }
549
550 static void
551 ata_thread_wake_locked(struct ata_channel *chp)
552 {
553 KASSERT(mutex_owned(&chp->ch_lock));
554 ata_channel_freeze_locked(chp);
555 cv_signal(&chp->ch_thr_idle);
556 }
557
558 /*
559 * atabus_match:
560 *
561 * Autoconfiguration match routine.
562 */
563 static int
564 atabus_match(device_t parent, cfdata_t cf, void *aux)
565 {
566 struct ata_channel *chp = aux;
567
568 if (chp == NULL)
569 return (0);
570
571 if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel &&
572 cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT)
573 return (0);
574
575 return (1);
576 }
577
578 /*
579 * atabus_attach:
580 *
581 * Autoconfiguration attach routine.
582 */
583 static void
584 atabus_attach(device_t parent, device_t self, void *aux)
585 {
586 struct atabus_softc *sc = device_private(self);
587 struct ata_channel *chp = aux;
588 struct atabus_initq *initq;
589 int error;
590
591 sc->sc_chan = chp;
592
593 aprint_normal("\n");
594 aprint_naive("\n");
595
596 sc->sc_dev = self;
597
598 if (ata_addref(chp))
599 return;
600
601 RUN_ONCE(&ata_init_ctrl, atabus_init);
602
603 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP);
604 initq->atabus_sc = sc;
605 mutex_enter(&atabus_qlock);
606 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
607 mutex_exit(&atabus_qlock);
608 config_pending_incr(sc->sc_dev);
609
610 /* XXX MPSAFE - no KTHREAD_MPSAFE, so protected by KERNEL_LOCK() */
611 if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc,
612 &chp->ch_thread, "%s", device_xname(self))) != 0)
613 aprint_error_dev(self,
614 "unable to create kernel thread: error %d\n", error);
615
616 if (!pmf_device_register(self, atabus_suspend, atabus_resume))
617 aprint_error_dev(self, "couldn't establish power handler\n");
618 }
619
620 /*
621 * atabus_detach:
622 *
623 * Autoconfiguration detach routine.
624 */
625 static int
626 atabus_detach(device_t self, int flags)
627 {
628 struct atabus_softc *sc = device_private(self);
629 struct ata_channel *chp = sc->sc_chan;
630 device_t dev = NULL;
631 int i, error = 0;
632
633 /* Shutdown the channel. */
634 ata_channel_lock(chp);
635 chp->ch_flags |= ATACH_SHUTDOWN;
636 while (chp->ch_thread != NULL) {
637 cv_signal(&chp->ch_thr_idle);
638 cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
639 }
640 ata_channel_unlock(chp);
641
642 /*
643 * Detach atapibus and its children.
644 */
645 if ((dev = chp->atapibus) != NULL) {
646 ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n",
647 device_xname(self), device_xname(dev)), DEBUG_DETACH);
648
649 error = config_detach(dev, flags);
650 if (error)
651 goto out;
652 KASSERT(chp->atapibus == NULL);
653 }
654
655 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
656
657 /*
658 * Detach our other children.
659 */
660 for (i = 0; i < chp->ch_ndrives; i++) {
661 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
662 continue;
663 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
664 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
665 if ((dev = chp->ch_drive[i].drv_softc) != NULL) {
666 ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__,
667 __LINE__, device_xname(self), device_xname(dev)),
668 DEBUG_DETACH);
669 error = config_detach(dev, flags);
670 if (error)
671 goto out;
672 KASSERT(chp->ch_drive[i].drv_softc == NULL);
673 KASSERT(chp->ch_drive[i].drive_type == 0);
674 }
675 }
676 atabus_free_drives(chp);
677
678 out:
679 #ifdef ATADEBUG
680 if (dev != NULL && error != 0)
681 ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__,
682 device_xname(self), error, device_xname(dev)),
683 DEBUG_DETACH);
684 #endif /* ATADEBUG */
685
686 return (error);
687 }
688
689 void
690 atabus_childdetached(device_t self, device_t child)
691 {
692 bool found = false;
693 struct atabus_softc *sc = device_private(self);
694 struct ata_channel *chp = sc->sc_chan;
695 int i;
696
697 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
698 /*
699 * atapibus detached.
700 */
701 if (child == chp->atapibus) {
702 chp->atapibus = NULL;
703 found = true;
704 for (i = 0; i < chp->ch_ndrives; i++) {
705 if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI)
706 continue;
707 KASSERT(chp->ch_drive[i].drv_softc != NULL);
708 chp->ch_drive[i].drv_softc = NULL;
709 chp->ch_drive[i].drive_flags = 0;
710 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
711 }
712 }
713
714 /*
715 * Detach our other children.
716 */
717 for (i = 0; i < chp->ch_ndrives; i++) {
718 if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
719 continue;
720 if (child == chp->ch_drive[i].drv_softc) {
721 chp->ch_drive[i].drv_softc = NULL;
722 chp->ch_drive[i].drive_flags = 0;
723 if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
724 chp->ch_satapmp_nports = 0;
725 chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
726 found = true;
727 }
728 }
729
730 if (!found)
731 panic("%s: unknown child %p", device_xname(self),
732 (const void *)child);
733 }
734
735 CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc),
736 atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan,
737 atabus_childdetached, DVF_DETACH_SHUTDOWN);
738
739 /*****************************************************************************
740 * Common ATA bus operations.
741 *****************************************************************************/
742
743 /* allocate/free the channel's ch_drive[] array */
744 int
745 atabus_alloc_drives(struct ata_channel *chp, int ndrives)
746 {
747 int i;
748 if (chp->ch_ndrives != ndrives)
749 atabus_free_drives(chp);
750 if (chp->ch_drive == NULL) {
751 chp->ch_drive = kmem_zalloc(
752 sizeof(struct ata_drive_datas) * ndrives, KM_NOSLEEP);
753 }
754 if (chp->ch_drive == NULL) {
755 aprint_error_dev(chp->ch_atac->atac_dev,
756 "can't alloc drive array\n");
757 chp->ch_ndrives = 0;
758 return ENOMEM;
759 };
760 for (i = 0; i < ndrives; i++) {
761 chp->ch_drive[i].chnl_softc = chp;
762 chp->ch_drive[i].drive = i;
763 }
764 chp->ch_ndrives = ndrives;
765 return 0;
766 }
767
768 void
769 atabus_free_drives(struct ata_channel *chp)
770 {
771 #ifdef DIAGNOSTIC
772 int i;
773 int dopanic = 0;
774 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
775 for (i = 0; i < chp->ch_ndrives; i++) {
776 if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) {
777 printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n",
778 device_xname(chp->atabus), i,
779 chp->ch_drive[i].drive_type);
780 dopanic = 1;
781 }
782 if (chp->ch_drive[i].drv_softc != NULL) {
783 printf("%s: ch_drive[%d] attached to %s\n",
784 device_xname(chp->atabus), i,
785 device_xname(chp->ch_drive[i].drv_softc));
786 dopanic = 1;
787 }
788 }
789 if (dopanic)
790 panic("atabus_free_drives");
791 #endif
792
793 if (chp->ch_drive == NULL)
794 return;
795 kmem_free(chp->ch_drive,
796 sizeof(struct ata_drive_datas) * chp->ch_ndrives);
797 chp->ch_ndrives = 0;
798 chp->ch_drive = NULL;
799 }
800
801 /* Get the disk's parameters */
802 int
803 ata_get_params(struct ata_drive_datas *drvp, uint8_t flags,
804 struct ataparams *prms)
805 {
806 struct ata_xfer *xfer;
807 struct ata_channel *chp = drvp->chnl_softc;
808 struct atac_softc *atac = chp->ch_atac;
809 char *tb;
810 int i, rv;
811 uint16_t *p;
812
813 ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS);
814
815 xfer = ata_get_xfer(chp, false);
816 if (xfer == NULL) {
817 ATADEBUG_PRINT(("%s: no xfer\n", __func__),
818 DEBUG_FUNCS|DEBUG_PROBE);
819 return CMD_AGAIN;
820 }
821
822 tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP);
823 memset(prms, 0, sizeof(struct ataparams));
824
825 if (drvp->drive_type == ATA_DRIVET_ATA) {
826 xfer->c_ata_c.r_command = WDCC_IDENTIFY;
827 xfer->c_ata_c.r_st_bmask = WDCS_DRDY;
828 xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
829 xfer->c_ata_c.timeout = 3000; /* 3s */
830 } else if (drvp->drive_type == ATA_DRIVET_ATAPI) {
831 xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE;
832 xfer->c_ata_c.r_st_bmask = 0;
833 xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
834 xfer->c_ata_c.timeout = 10000; /* 10s */
835 } else {
836 ATADEBUG_PRINT(("ata_get_parms: no disks\n"),
837 DEBUG_FUNCS|DEBUG_PROBE);
838 rv = CMD_ERR;
839 goto out;
840 }
841 xfer->c_ata_c.flags = AT_READ | flags;
842 xfer->c_ata_c.data = tb;
843 xfer->c_ata_c.bcount = ATA_BSIZE;
844 if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
845 xfer) != ATACMD_COMPLETE) {
846 ATADEBUG_PRINT(("ata_get_parms: wdc_exec_command failed\n"),
847 DEBUG_FUNCS|DEBUG_PROBE);
848 rv = CMD_AGAIN;
849 goto out;
850 }
851 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
852 ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n",
853 xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE);
854 rv = CMD_ERR;
855 goto out;
856 }
857 /* if we didn't read any data something is wrong */
858 if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) {
859 rv = CMD_ERR;
860 goto out;
861 }
862
863 /* Read in parameter block. */
864 memcpy(prms, tb, sizeof(struct ataparams));
865
866 /*
867 * Shuffle string byte order.
868 * ATAPI NEC, Mitsumi and Pioneer drives and
869 * old ATA TDK CompactFlash cards
870 * have different byte order.
871 */
872 #if BYTE_ORDER == BIG_ENDIAN
873 # define M(n) prms->atap_model[(n) ^ 1]
874 #else
875 # define M(n) prms->atap_model[n]
876 #endif
877 if (
878 #if BYTE_ORDER == BIG_ENDIAN
879 !
880 #endif
881 ((drvp->drive_type == ATA_DRIVET_ATAPI) ?
882 ((M(0) == 'N' && M(1) == 'E') ||
883 (M(0) == 'F' && M(1) == 'X') ||
884 (M(0) == 'P' && M(1) == 'i')) :
885 ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) {
886 rv = CMD_OK;
887 goto out;
888 }
889 #undef M
890 for (i = 0; i < sizeof(prms->atap_model); i += 2) {
891 p = (uint16_t *)(prms->atap_model + i);
892 *p = bswap16(*p);
893 }
894 for (i = 0; i < sizeof(prms->atap_serial); i += 2) {
895 p = (uint16_t *)(prms->atap_serial + i);
896 *p = bswap16(*p);
897 }
898 for (i = 0; i < sizeof(prms->atap_revision); i += 2) {
899 p = (uint16_t *)(prms->atap_revision + i);
900 *p = bswap16(*p);
901 }
902
903 rv = CMD_OK;
904 out:
905 kmem_free(tb, ATA_BSIZE);
906 ata_free_xfer(chp, xfer);
907 return rv;
908 }
909
910 int
911 ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags)
912 {
913 struct ata_xfer *xfer;
914 int rv;
915 struct ata_channel *chp = drvp->chnl_softc;
916 struct atac_softc *atac = chp->ch_atac;
917
918 ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS);
919
920 xfer = ata_get_xfer(chp, false);
921 if (xfer == NULL) {
922 ATADEBUG_PRINT(("%s: no xfer\n", __func__),
923 DEBUG_FUNCS|DEBUG_PROBE);
924 return CMD_AGAIN;
925 }
926
927 xfer->c_ata_c.r_command = SET_FEATURES;
928 xfer->c_ata_c.r_st_bmask = 0;
929 xfer->c_ata_c.r_st_pmask = 0;
930 xfer->c_ata_c.r_features = WDSF_SET_MODE;
931 xfer->c_ata_c.r_count = mode;
932 xfer->c_ata_c.flags = flags;
933 xfer->c_ata_c.timeout = 1000; /* 1s */
934 if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
935 xfer) != ATACMD_COMPLETE) {
936 rv = CMD_AGAIN;
937 goto out;
938 }
939 if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
940 rv = CMD_ERR;
941 goto out;
942 }
943
944 rv = CMD_OK;
945
946 out:
947 ata_free_xfer(chp, xfer);
948 return rv;
949 }
950
951 #if NATA_DMA
952 void
953 ata_dmaerr(struct ata_drive_datas *drvp, int flags)
954 {
955 /*
956 * Downgrade decision: if we get NERRS_MAX in NXFER.
957 * We start with n_dmaerrs set to NERRS_MAX-1 so that the
958 * first error within the first NXFER ops will immediatly trigger
959 * a downgrade.
960 * If we got an error and n_xfers is bigger than NXFER reset counters.
961 */
962 drvp->n_dmaerrs++;
963 if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) {
964 ata_downgrade_mode(drvp, flags);
965 drvp->n_dmaerrs = NERRS_MAX-1;
966 drvp->n_xfers = 0;
967 return;
968 }
969 if (drvp->n_xfers > NXFER) {
970 drvp->n_dmaerrs = 1; /* just got an error */
971 drvp->n_xfers = 1; /* restart counting from this error */
972 }
973 }
974 #endif /* NATA_DMA */
975
976 /*
977 * freeze the queue and wait for the controller to be idle. Caller has to
978 * unfreeze/restart the queue
979 */
980 static void
981 ata_channel_idle(struct ata_channel *chp)
982 {
983 ata_channel_lock(chp);
984 ata_channel_freeze_locked(chp);
985 while (chp->ch_queue->queue_active > 0) {
986 chp->ch_queue->queue_flags |= QF_IDLE_WAIT;
987 cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1);
988 }
989 ata_channel_unlock(chp);
990 }
991
992 /*
993 * Add a command to the queue and start controller.
994 *
995 * MUST BE CALLED AT splbio()!
996 */
997 void
998 ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
999 {
1000
1001 ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer,
1002 chp->ch_channel, xfer->c_drive), DEBUG_XFERS);
1003
1004 /* complete xfer setup */
1005 xfer->c_chp = chp;
1006
1007 ata_channel_lock(chp);
1008
1009 /*
1010 * Standard commands are added to the end of command list, but
1011 * recovery commands must be run immediatelly.
1012 */
1013 if ((xfer->c_flags & C_SKIP_QUEUE) == 0)
1014 SIMPLEQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer,
1015 c_xferchain);
1016 else
1017 SIMPLEQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer,
1018 c_xferchain);
1019
1020 /*
1021 * if polling and can sleep, wait for the xfer to be at head of queue
1022 */
1023 if ((xfer->c_flags & (C_POLL | C_WAIT)) == (C_POLL | C_WAIT)) {
1024 while (chp->ch_queue->queue_active > 0 ||
1025 SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) {
1026 xfer->c_flags |= C_WAITACT;
1027 cv_wait(&chp->ch_queue->c_active, &chp->ch_lock);
1028 xfer->c_flags &= ~C_WAITACT;
1029 }
1030
1031 /*
1032 * Free xfer now if it there was attempt to free it
1033 * while we were waiting.
1034 */
1035 if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) {
1036 ata_channel_unlock(chp);
1037
1038 ata_free_xfer(chp, xfer);
1039 return;
1040 }
1041 }
1042
1043 ata_channel_unlock(chp);
1044
1045 ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n",
1046 chp->ch_flags), DEBUG_XFERS);
1047 atastart(chp);
1048 }
1049
1050 /*
1051 * Start I/O on a controller, for the given channel.
1052 * The first xfer may be not for our channel if the channel queues
1053 * are shared.
1054 *
1055 * MUST BE CALLED AT splbio()!
1056 *
1057 * XXX FIS-based switching with PMP
1058 * Currently atastart() never schedules concurrent NCQ transfers to more than
1059 * one drive, even when channel has several SATA drives attached via PMP.
1060 * To support concurrent transfers to different drives with PMP, it would be
1061 * necessary to implement FIS-based switching support in controller driver,
1062 * and then adjust error handling and recovery to stop assuming at most
1063 * one active drive.
1064 */
1065 void
1066 atastart(struct ata_channel *chp)
1067 {
1068 struct atac_softc *atac = chp->ch_atac;
1069 struct ata_queue *chq = chp->ch_queue;
1070 struct ata_xfer *xfer, *axfer;
1071 bool skipq;
1072
1073 #ifdef ATA_DEBUG
1074 int spl1, spl2;
1075
1076 spl1 = splbio();
1077 spl2 = splbio();
1078 if (spl2 != spl1) {
1079 printf("atastart: not at splbio()\n");
1080 panic("atastart");
1081 }
1082 splx(spl2);
1083 splx(spl1);
1084 #endif /* ATA_DEBUG */
1085
1086 ata_channel_lock(chp);
1087
1088 again:
1089 /* is there a xfer ? */
1090 if ((xfer = SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL) {
1091 ATADEBUG_PRINT(("%s(chp=%p): channel %d queue_xfer is empty\n",
1092 __func__, chp, chp->ch_channel), DEBUG_XFERS);
1093 goto out;
1094 }
1095
1096 /*
1097 * if someone is waiting for the command to be active, wake it up
1098 * and let it process the command
1099 */
1100 if (__predict_false(xfer->c_flags & C_WAITACT)) {
1101 ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d "
1102 "wait active\n", xfer, chp->ch_channel, xfer->c_drive),
1103 DEBUG_XFERS);
1104 cv_broadcast(&chp->ch_queue->c_active);
1105 goto out;
1106 }
1107
1108 skipq = ISSET(xfer->c_flags, C_SKIP_QUEUE);
1109
1110 /* is the queue frozen? */
1111 if (__predict_false(!skipq && chq->queue_freeze > 0)) {
1112 if (chq->queue_flags & QF_IDLE_WAIT) {
1113 chq->queue_flags &= ~QF_IDLE_WAIT;
1114 cv_signal(&chp->ch_queue->queue_idle);
1115 }
1116 ATADEBUG_PRINT(("%s(chp=%p): channel %d drive %d "
1117 "queue frozen: %d\n",
1118 __func__, chp, chp->ch_channel, xfer->c_drive,
1119 chq->queue_freeze),
1120 DEBUG_XFERS);
1121 goto out;
1122 }
1123
1124 /* all xfers on same queue must belong to the same channel */
1125 KASSERT(xfer->c_chp == chp);
1126
1127 /*
1128 * Can only take the command if there are no current active
1129 * commands, or if the command is NCQ and the active commands are also
1130 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based
1131 * switching, can only send commands to single drive.
1132 * Need only check first xfer.
1133 * XXX FIS-based switching - revisit
1134 */
1135 if (!skipq && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) {
1136 if (!ISSET(xfer->c_flags, C_NCQ) ||
1137 !ISSET(axfer->c_flags, C_NCQ) ||
1138 xfer->c_drive != axfer->c_drive)
1139 goto out;
1140 }
1141
1142 struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive];
1143
1144 /*
1145 * Are we on limit of active xfers ? If the queue has more
1146 * than 1 openings, we keep one slot reserved for recovery or dump.
1147 */
1148 KASSERT(chq->queue_active <= chq->queue_openings);
1149 const uint8_t chq_openings = (!skipq && chq->queue_openings > 1)
1150 ? (chq->queue_openings - 1) : chq->queue_openings;
1151 const uint8_t drv_openings = ISSET(xfer->c_flags, C_NCQ)
1152 ? drvp->drv_openings : ATA_MAX_OPENINGS;
1153 if (chq->queue_active >= MIN(chq_openings, drv_openings)) {
1154 if (skipq) {
1155 panic("%s: channel %d busy, xfer not possible",
1156 __func__, chp->ch_channel);
1157 }
1158
1159 ATADEBUG_PRINT(("%s(chp=%p): channel %d completely busy\n",
1160 __func__, chp, chp->ch_channel), DEBUG_XFERS);
1161 goto out;
1162 }
1163
1164 /* Slot allocation can fail if drv_openings < ch_openings */
1165 if (!ata_queue_alloc_slot(chp, &xfer->c_slot, drv_openings))
1166 goto out;
1167
1168 if (__predict_false(atac->atac_claim_hw)) {
1169 if (!atac->atac_claim_hw(chp, 0)) {
1170 ata_queue_free_slot(chp, xfer->c_slot);
1171 goto out;
1172 }
1173 }
1174
1175 /* Now committed to start the xfer */
1176
1177 ATADEBUG_PRINT(("%s(chp=%p): xfer %p channel %d drive %d\n",
1178 __func__, chp, xfer, chp->ch_channel, xfer->c_drive), DEBUG_XFERS);
1179 if (drvp->drive_flags & ATA_DRIVE_RESET) {
1180 drvp->drive_flags &= ~ATA_DRIVE_RESET;
1181 drvp->state = 0;
1182 }
1183
1184 if (ISSET(xfer->c_flags, C_NCQ))
1185 SET(chp->ch_flags, ATACH_NCQ);
1186 else
1187 CLR(chp->ch_flags, ATACH_NCQ);
1188
1189 SIMPLEQ_REMOVE_HEAD(&chq->queue_xfer, c_xferchain);
1190
1191 ata_activate_xfer_locked(chp, xfer);
1192
1193 if (atac->atac_cap & ATAC_CAP_NOIRQ)
1194 KASSERT(xfer->c_flags & C_POLL);
1195
1196 switch (ata_xfer_start(xfer)) {
1197 case ATASTART_TH:
1198 case ATASTART_ABORT:
1199 /* don't start any further commands in this case */
1200 goto out;
1201 default:
1202 /* nothing to do */
1203 break;
1204 }
1205
1206 /* Queue more commands if possible, but not during recovery or dump */
1207 if (!skipq && chq->queue_active < chq->queue_openings)
1208 goto again;
1209
1210 out:
1211 ata_channel_unlock(chp);
1212 }
1213
1214 int
1215 ata_xfer_start(struct ata_xfer *xfer)
1216 {
1217 struct ata_channel *chp = xfer->c_chp;
1218 int rv;
1219
1220 KASSERT(mutex_owned(&chp->ch_lock));
1221
1222 rv = xfer->ops->c_start(chp, xfer);
1223 switch (rv) {
1224 case ATASTART_STARTED:
1225 /* nothing to do */
1226 break;
1227 case ATASTART_TH:
1228 /* postpone xfer to thread */
1229 ata_thread_wake_locked(chp);
1230 break;
1231 case ATASTART_POLL:
1232 /* can happen even in thread context for some ATAPI devices */
1233 ata_channel_unlock(chp);
1234 KASSERT(xfer->ops != NULL && xfer->ops->c_poll != NULL);
1235 xfer->ops->c_poll(chp, xfer);
1236 ata_channel_lock(chp);
1237 break;
1238 case ATASTART_ABORT:
1239 ata_channel_unlock(chp);
1240 KASSERT(xfer->ops != NULL && xfer->ops->c_abort != NULL);
1241 xfer->ops->c_abort(chp, xfer);
1242 ata_channel_lock(chp);
1243 break;
1244 }
1245
1246 return rv;
1247 }
1248
1249 static void
1250 ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer)
1251 {
1252 struct ata_queue * const chq = chp->ch_queue;
1253
1254 KASSERT(mutex_owned(&chp->ch_lock));
1255 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);
1256
1257 if ((xfer->c_flags & C_SKIP_QUEUE) == 0)
1258 TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain);
1259 else {
1260 /*
1261 * Must go to head, so that ata_queue_get_active_xfer()
1262 * returns the recovery command, and not some other
1263 * random active transfer.
1264 */
1265 TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain);
1266 }
1267 chq->active_xfers_used |= __BIT(xfer->c_slot);
1268 chq->queue_active++;
1269 }
1270
1271 /*
1272 * Does it's own locking, does not require splbio().
1273 * flags - whether to block waiting for free xfer
1274 */
1275 struct ata_xfer *
1276 ata_get_xfer(struct ata_channel *chp, bool waitok)
1277 {
1278 struct ata_xfer *xfer;
1279
1280 xfer = pool_get(&ata_xfer_pool, waitok ? PR_WAITOK : PR_NOWAIT);
1281 KASSERT(!waitok || xfer != NULL);
1282
1283 if (xfer != NULL) {
1284 /* zero everything */
1285 memset(xfer, 0, sizeof(*xfer));
1286 }
1287
1288 return xfer;
1289 }
1290
1291 /*
1292 * ata_deactivate_xfer() must be always called prior to ata_free_xfer()
1293 */
1294 void
1295 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
1296 {
1297 struct ata_queue *chq = chp->ch_queue;
1298
1299 ata_channel_lock(chp);
1300
1301 if (__predict_false(xfer->c_flags & (C_WAITACT|C_WAITTIMO))) {
1302 /* Someone is waiting for this xfer, so we can't free now */
1303 xfer->c_flags |= C_FREE;
1304 cv_broadcast(&chq->c_active);
1305 ata_channel_unlock(chp);
1306 return;
1307 }
1308
1309 /* XXX move PIOBM and free_gw to deactivate? */
1310 #if NATA_PIOBM /* XXX wdc dependent code */
1311 if (__predict_false(xfer->c_flags & C_PIOBM)) {
1312 struct wdc_softc *wdc = CHAN_TO_WDC(chp);
1313
1314 /* finish the busmastering PIO */
1315 (*wdc->piobm_done)(wdc->dma_arg,
1316 chp->ch_channel, xfer->c_drive);
1317 chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT);
1318 }
1319 #endif
1320
1321 if (__predict_false(chp->ch_atac->atac_free_hw))
1322 chp->ch_atac->atac_free_hw(chp);
1323
1324 ata_channel_unlock(chp);
1325
1326 if (__predict_true(!ISSET(xfer->c_flags, C_PRIVATE_ALLOC)))
1327 pool_put(&ata_xfer_pool, xfer);
1328 }
1329
1330 void
1331 ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
1332 {
1333 struct ata_queue * const chq = chp->ch_queue;
1334
1335 ata_channel_lock(chp);
1336
1337 KASSERT(chq->queue_active > 0);
1338 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0);
1339
1340 /* Stop only when this is last active xfer */
1341 if (chq->queue_active == 1)
1342 callout_stop(&chp->c_timo_callout);
1343
1344 if (callout_invoking(&chp->c_timo_callout))
1345 xfer->c_flags |= C_WAITTIMO;
1346
1347 TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain);
1348 chq->active_xfers_used &= ~__BIT(xfer->c_slot);
1349 chq->queue_active--;
1350
1351 ata_queue_free_slot(chp, xfer->c_slot);
1352
1353 if (xfer->c_flags & C_WAIT)
1354 cv_broadcast(&chq->c_cmd_finish);
1355
1356 ata_channel_unlock(chp);
1357 }
1358
1359 /*
1360 * Called in c_intr hook. Must be called before before any deactivations
1361 * are done - if there is drain pending, it calls c_kill_xfer hook which
1362 * deactivates the xfer.
1363 * Calls c_kill_xfer with channel lock free.
1364 * Returns true if caller should just exit without further processing.
1365 * Caller must not further access any part of xfer or any related controller
1366 * structures in that case, it should just return.
1367 */
1368 bool
1369 ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer)
1370 {
1371 int drive = xfer->c_drive;
1372 bool draining = false;
1373
1374 ata_channel_lock(chp);
1375
1376 if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) {
1377 ata_channel_unlock(chp);
1378
1379 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE);
1380
1381 ata_channel_lock(chp);
1382 chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN;
1383 cv_signal(&chp->ch_queue->queue_drain);
1384 draining = true;
1385 }
1386
1387 ata_channel_unlock(chp);
1388
1389 return draining;
1390 }
1391
1392 /*
1393 * Check for race of normal transfer handling vs. timeout.
1394 */
1395 bool
1396 ata_timo_xfer_check(struct ata_xfer *xfer)
1397 {
1398 struct ata_channel *chp = xfer->c_chp;
1399 struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
1400
1401 ata_channel_lock(chp);
1402
1403 if (xfer->c_flags & C_WAITTIMO) {
1404 xfer->c_flags &= ~C_WAITTIMO;
1405
1406 /* Handle race vs. ata_free_xfer() */
1407 if (xfer->c_flags & C_FREE) {
1408 xfer->c_flags &= ~C_FREE;
1409 ata_channel_unlock(chp);
1410
1411 device_printf(drvp->drv_softc,
1412 "xfer %"PRIxPTR" freed while invoking timeout\n",
1413 (intptr_t)xfer & PAGE_MASK);
1414
1415 ata_free_xfer(chp, xfer);
1416 return true;
1417 }
1418
1419 /* Race vs. callout_stop() in ata_deactivate_xfer() */
1420 ata_channel_unlock(chp);
1421
1422 device_printf(drvp->drv_softc,
1423 "xfer %"PRIxPTR" deactivated while invoking timeout\n",
1424 (intptr_t)xfer & PAGE_MASK);
1425 return true;
1426 }
1427
1428 ata_channel_unlock(chp);
1429
1430 /* No race, proceed with timeout handling */
1431 return false;
1432 }
1433
1434 /*
1435 * Kill off all active xfers for a ata_channel.
1436 *
1437 * Must be called with channel lock held.
1438 */
1439 void
1440 ata_kill_active(struct ata_channel *chp, int reason, int flags)
1441 {
1442 struct ata_queue * const chq = chp->ch_queue;
1443 struct ata_xfer *xfer, *xfernext;
1444
1445 KASSERT(mutex_owned(&chp->ch_lock));
1446
1447 TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) {
1448 xfer->ops->c_kill_xfer(xfer->c_chp, xfer, reason);
1449 }
1450 }
1451
1452 /*
1453 * Kill off all pending xfers for a drive.
1454 */
1455 void
1456 ata_kill_pending(struct ata_drive_datas *drvp)
1457 {
1458 struct ata_channel * const chp = drvp->chnl_softc;
1459 struct ata_queue * const chq = chp->ch_queue;
1460 struct ata_xfer *xfer;
1461
1462 ata_channel_lock(chp);
1463
1464 /* Kill all pending transfers */
1465 while ((xfer = SIMPLEQ_FIRST(&chq->queue_xfer))) {
1466 KASSERT(xfer->c_chp == chp);
1467
1468 if (xfer->c_drive != drvp->drive)
1469 continue;
1470
1471 SIMPLEQ_REMOVE_HEAD(&chp->ch_queue->queue_xfer, c_xferchain);
1472
1473 /*
1474 * Keep the lock, so that we get deadlock (and 'locking against
1475 * myself' with LOCKDEBUG), instead of silent
1476 * data corruption, if the hook tries to call back into
1477 * middle layer for inactive xfer.
1478 */
1479 xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE_INACTIVE);
1480 }
1481
1482 /* Wait until all active transfers on the drive finish */
1483 while (chq->queue_active > 0) {
1484 bool drv_active = false;
1485
1486 TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
1487 KASSERT(xfer->c_chp == chp);
1488
1489 if (xfer->c_drive == drvp->drive) {
1490 drv_active = true;
1491 break;
1492 }
1493 }
1494
1495 if (!drv_active) {
1496 /* all finished */
1497 break;
1498 }
1499
1500 drvp->drive_flags |= ATA_DRIVE_WAITDRAIN;
1501 cv_wait(&chq->queue_drain, &chp->ch_lock);
1502 }
1503
1504 ata_channel_unlock(chp);
1505 }
1506
1507 static void
1508 ata_channel_freeze_locked(struct ata_channel *chp)
1509 {
1510 chp->ch_queue->queue_freeze++;
1511
1512 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp,
1513 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS);
1514 }
1515
1516 void
1517 ata_channel_freeze(struct ata_channel *chp)
1518 {
1519 ata_channel_lock(chp);
1520 ata_channel_freeze_locked(chp);
1521 ata_channel_unlock(chp);
1522 }
1523
1524 void
1525 ata_channel_thaw_locked(struct ata_channel *chp)
1526 {
1527 KASSERT(mutex_owned(&chp->ch_lock));
1528 KASSERT(chp->ch_queue->queue_freeze > 0);
1529
1530 chp->ch_queue->queue_freeze--;
1531
1532 ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp,
1533 chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS);
1534 }
1535
1536 /*
1537 * ata_thread_run:
1538 *
1539 * Reset and ATA channel. Channel lock must be held. arg is type-specific.
1540 */
1541 void
1542 ata_thread_run(struct ata_channel *chp, int flags, int type, int arg)
1543 {
1544 struct atac_softc *atac = chp->ch_atac;
1545 bool threset = false;
1546 struct ata_drive_datas *drvp;
1547
1548 ata_channel_lock_owned(chp);
1549
1550 /*
1551 * If we can poll or wait it's OK, otherwise wake up the
1552 * kernel thread to do it for us.
1553 */
1554 ATADEBUG_PRINT(("%s flags 0x%x ch_flags 0x%x\n",
1555 __func__, flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS);
1556 if ((flags & (AT_POLL | AT_WAIT)) == 0) {
1557 switch (type) {
1558 case ATACH_TH_RESET:
1559 if (chp->ch_flags & ATACH_TH_RESET) {
1560 /* No need to schedule another reset */
1561 return;
1562 }
1563 break;
1564 case ATACH_TH_DRIVE_RESET:
1565 {
1566 int drive = arg;
1567
1568 KASSERT(drive <= chp->ch_ndrives);
1569 drvp = &chp->ch_drive[drive];
1570
1571 if (drvp->drive_flags & ATA_DRIVE_TH_RESET) {
1572 /* No need to schedule another reset */
1573 return;
1574 }
1575 drvp->drive_flags |= ATA_DRIVE_TH_RESET;
1576 break;
1577 }
1578 case ATACH_TH_RECOVERY:
1579 {
1580 uint32_t tfd = (uint32_t)arg;
1581
1582 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0);
1583 chp->recovery_tfd = tfd;
1584 break;
1585 }
1586 default:
1587 panic("%s: unknown type: %x", __func__, type);
1588 /* NOTREACHED */
1589 }
1590
1591 /*
1592 * Block execution of other commands while reset is scheduled
1593 * to a thread.
1594 */
1595 ata_channel_freeze_locked(chp);
1596 chp->ch_flags |= type;
1597
1598 cv_signal(&chp->ch_thr_idle);
1599 return;
1600 }
1601
1602 /* Block execution of other commands during reset */
1603 ata_channel_freeze_locked(chp);
1604
1605 /*
1606 * If reset has been scheduled to a thread, then clear
1607 * the flag now so that the thread won't try to execute it if
1608 * we happen to sleep, and thaw one more time after the reset.
1609 */
1610 if (chp->ch_flags & type) {
1611 chp->ch_flags &= ~type;
1612 threset = true;
1613 }
1614
1615 switch (type) {
1616 case ATACH_TH_RESET:
1617 (*atac->atac_bustype_ata->ata_reset_channel)(chp, flags);
1618
1619 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
1620 for (int drive = 0; drive < chp->ch_ndrives; drive++)
1621 chp->ch_drive[drive].state = 0;
1622 break;
1623
1624 case ATACH_TH_DRIVE_RESET:
1625 {
1626 int drive = arg;
1627
1628 KASSERT(drive <= chp->ch_ndrives);
1629 drvp = &chp->ch_drive[drive];
1630 (*atac->atac_bustype_ata->ata_reset_drive)(drvp, flags, NULL);
1631 drvp->state = 0;
1632 break;
1633 }
1634
1635 case ATACH_TH_RECOVERY:
1636 {
1637 uint32_t tfd = (uint32_t)arg;
1638
1639 KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0);
1640 KASSERT(atac->atac_bustype_ata->ata_recovery != NULL);
1641
1642 SET(chp->ch_flags, ATACH_RECOVERING);
1643 (*atac->atac_bustype_ata->ata_recovery)(chp, flags, tfd);
1644 CLR(chp->ch_flags, ATACH_RECOVERING);
1645 break;
1646 }
1647
1648 default:
1649 panic("%s: unknown type: %x", __func__, type);
1650 /* NOTREACHED */
1651 }
1652
1653 /*
1654 * Thaw one extra time to clear the freeze done when the reset has
1655 * been scheduled to the thread.
1656 */
1657 if (threset)
1658 ata_channel_thaw_locked(chp);
1659
1660 /* Allow commands to run again */
1661 ata_channel_thaw_locked(chp);
1662
1663 /* Signal the thread in case there is an xfer to run */
1664 cv_signal(&chp->ch_thr_idle);
1665 }
1666
1667 int
1668 ata_addref(struct ata_channel *chp)
1669 {
1670 struct atac_softc *atac = chp->ch_atac;
1671 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
1672 int s, error = 0;
1673
1674 s = splbio();
1675 if (adapt->adapt_refcnt++ == 0 &&
1676 adapt->adapt_enable != NULL) {
1677 error = (*adapt->adapt_enable)(atac->atac_dev, 1);
1678 if (error)
1679 adapt->adapt_refcnt--;
1680 }
1681 splx(s);
1682 return (error);
1683 }
1684
1685 void
1686 ata_delref(struct ata_channel *chp)
1687 {
1688 struct atac_softc *atac = chp->ch_atac;
1689 struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
1690 int s;
1691
1692 s = splbio();
1693 if (adapt->adapt_refcnt-- == 1 &&
1694 adapt->adapt_enable != NULL)
1695 (void) (*adapt->adapt_enable)(atac->atac_dev, 0);
1696 splx(s);
1697 }
1698
1699 void
1700 ata_print_modes(struct ata_channel *chp)
1701 {
1702 struct atac_softc *atac = chp->ch_atac;
1703 int drive;
1704 struct ata_drive_datas *drvp;
1705
1706 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
1707 for (drive = 0; drive < chp->ch_ndrives; drive++) {
1708 drvp = &chp->ch_drive[drive];
1709 if (drvp->drive_type == ATA_DRIVET_NONE ||
1710 drvp->drv_softc == NULL)
1711 continue;
1712 aprint_verbose("%s(%s:%d:%d): using PIO mode %d",
1713 device_xname(drvp->drv_softc),
1714 device_xname(atac->atac_dev),
1715 chp->ch_channel, drvp->drive, drvp->PIO_mode);
1716 #if NATA_DMA
1717 if (drvp->drive_flags & ATA_DRIVE_DMA)
1718 aprint_verbose(", DMA mode %d", drvp->DMA_mode);
1719 #if NATA_UDMA
1720 if (drvp->drive_flags & ATA_DRIVE_UDMA) {
1721 aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode);
1722 if (drvp->UDMA_mode == 2)
1723 aprint_verbose(" (Ultra/33)");
1724 else if (drvp->UDMA_mode == 4)
1725 aprint_verbose(" (Ultra/66)");
1726 else if (drvp->UDMA_mode == 5)
1727 aprint_verbose(" (Ultra/100)");
1728 else if (drvp->UDMA_mode == 6)
1729 aprint_verbose(" (Ultra/133)");
1730 }
1731 #endif /* NATA_UDMA */
1732 #endif /* NATA_DMA */
1733 #if NATA_DMA || NATA_PIOBM
1734 if (0
1735 #if NATA_DMA
1736 || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA))
1737 #endif
1738 #if NATA_PIOBM
1739 /* PIOBM capable controllers use DMA for PIO commands */
1740 || (atac->atac_cap & ATAC_CAP_PIOBM)
1741 #endif
1742 )
1743 aprint_verbose(" (using DMA)");
1744
1745 if (drvp->drive_flags & ATA_DRIVE_NCQ) {
1746 aprint_verbose(", NCQ (%d tags)%s",
1747 ATA_REAL_OPENINGS(chp->ch_queue->queue_openings),
1748 (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO)
1749 ? " w/PRIO" : "");
1750 } else if (drvp->drive_flags & ATA_DRIVE_WFUA)
1751 aprint_verbose(", WRITE DMA FUA EXT");
1752
1753 #endif /* NATA_DMA || NATA_PIOBM */
1754 aprint_verbose("\n");
1755 }
1756 }
1757
1758 #if NATA_DMA
1759 /*
1760 * downgrade the transfer mode of a drive after an error. return 1 if
1761 * downgrade was possible, 0 otherwise.
1762 *
1763 * MUST BE CALLED AT splbio()!
1764 */
1765 int
1766 ata_downgrade_mode(struct ata_drive_datas *drvp, int flags)
1767 {
1768 struct ata_channel *chp = drvp->chnl_softc;
1769 struct atac_softc *atac = chp->ch_atac;
1770 device_t drv_dev = drvp->drv_softc;
1771 int cf_flags = device_cfdata(drv_dev)->cf_flags;
1772
1773 /* if drive or controller don't know its mode, we can't do much */
1774 if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 ||
1775 (atac->atac_set_modes == NULL))
1776 return 0;
1777 /* current drive mode was set by a config flag, let it this way */
1778 if ((cf_flags & ATA_CONFIG_PIO_SET) ||
1779 (cf_flags & ATA_CONFIG_DMA_SET) ||
1780 (cf_flags & ATA_CONFIG_UDMA_SET))
1781 return 0;
1782
1783 #if NATA_UDMA
1784 /*
1785 * If we were using Ultra-DMA mode, downgrade to the next lower mode.
1786 */
1787 if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) {
1788 drvp->UDMA_mode--;
1789 aprint_error_dev(drv_dev,
1790 "transfer error, downgrading to Ultra-DMA mode %d\n",
1791 drvp->UDMA_mode);
1792 }
1793 #endif
1794
1795 /*
1796 * If we were using ultra-DMA, don't downgrade to multiword DMA.
1797 */
1798 else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) {
1799 drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA);
1800 drvp->PIO_mode = drvp->PIO_cap;
1801 aprint_error_dev(drv_dev,
1802 "transfer error, downgrading to PIO mode %d\n",
1803 drvp->PIO_mode);
1804 } else /* already using PIO, can't downgrade */
1805 return 0;
1806
1807 (*atac->atac_set_modes)(chp);
1808 ata_print_modes(chp);
1809 /* reset the channel, which will schedule all drives for setup */
1810 ata_thread_run(chp, flags, ATACH_TH_RESET, ATACH_NODRIVE);
1811 return 1;
1812 }
1813 #endif /* NATA_DMA */
1814
1815 /*
1816 * Probe drive's capabilities, for use by the controller later
1817 * Assumes drvp points to an existing drive.
1818 */
1819 void
1820 ata_probe_caps(struct ata_drive_datas *drvp)
1821 {
1822 struct ataparams params, params2;
1823 struct ata_channel *chp = drvp->chnl_softc;
1824 struct atac_softc *atac = chp->ch_atac;
1825 device_t drv_dev = drvp->drv_softc;
1826 int i, printed = 0;
1827 const char *sep = "";
1828 int cf_flags;
1829
1830 if (ata_get_params(drvp, AT_WAIT, ¶ms) != CMD_OK) {
1831 /* IDENTIFY failed. Can't tell more about the device */
1832 return;
1833 }
1834 if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) ==
1835 (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) {
1836 /*
1837 * Controller claims 16 and 32 bit transfers.
1838 * Re-do an IDENTIFY with 32-bit transfers,
1839 * and compare results.
1840 */
1841 ata_channel_lock(chp);
1842 drvp->drive_flags |= ATA_DRIVE_CAP32;
1843 ata_channel_unlock(chp);
1844 ata_get_params(drvp, AT_WAIT, ¶ms2);
1845 if (memcmp(¶ms, ¶ms2, sizeof(struct ataparams)) != 0) {
1846 /* Not good. fall back to 16bits */
1847 ata_channel_lock(chp);
1848 drvp->drive_flags &= ~ATA_DRIVE_CAP32;
1849 ata_channel_unlock(chp);
1850 } else {
1851 aprint_verbose_dev(drv_dev, "32-bit data port\n");
1852 }
1853 }
1854 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */
1855 if (params.atap_ata_major > 0x01 &&
1856 params.atap_ata_major != 0xffff) {
1857 for (i = 14; i > 0; i--) {
1858 if (params.atap_ata_major & (1 << i)) {
1859 aprint_verbose_dev(drv_dev,
1860 "ATA version %d\n", i);
1861 drvp->ata_vers = i;
1862 break;
1863 }
1864 }
1865 }
1866 #endif
1867
1868 /* An ATAPI device is at last PIO mode 3 */
1869 if (drvp->drive_type == ATA_DRIVET_ATAPI)
1870 drvp->PIO_mode = 3;
1871
1872 /*
1873 * It's not in the specs, but it seems that some drive
1874 * returns 0xffff in atap_extensions when this field is invalid
1875 */
1876 if (params.atap_extensions != 0xffff &&
1877 (params.atap_extensions & WDC_EXT_MODES)) {
1878 /*
1879 * XXX some drives report something wrong here (they claim to
1880 * support PIO mode 8 !). As mode is coded on 3 bits in
1881 * SET FEATURE, limit it to 7 (so limit i to 4).
1882 * If higher mode than 7 is found, abort.
1883 */
1884 for (i = 7; i >= 0; i--) {
1885 if ((params.atap_piomode_supp & (1 << i)) == 0)
1886 continue;
1887 if (i > 4)
1888 return;
1889 /*
1890 * See if mode is accepted.
1891 * If the controller can't set its PIO mode,
1892 * assume the defaults are good, so don't try
1893 * to set it
1894 */
1895 if (atac->atac_set_modes)
1896 /*
1897 * It's OK to pool here, it's fast enough
1898 * to not bother waiting for interrupt
1899 */
1900 if (ata_set_mode(drvp, 0x08 | (i + 3),
1901 AT_WAIT) != CMD_OK)
1902 continue;
1903 if (!printed) {
1904 aprint_verbose_dev(drv_dev,
1905 "drive supports PIO mode %d", i + 3);
1906 sep = ",";
1907 printed = 1;
1908 }
1909 /*
1910 * If controller's driver can't set its PIO mode,
1911 * get the highter one for the drive.
1912 */
1913 if (atac->atac_set_modes == NULL ||
1914 atac->atac_pio_cap >= i + 3) {
1915 drvp->PIO_mode = i + 3;
1916 drvp->PIO_cap = i + 3;
1917 break;
1918 }
1919 }
1920 if (!printed) {
1921 /*
1922 * We didn't find a valid PIO mode.
1923 * Assume the values returned for DMA are buggy too
1924 */
1925 return;
1926 }
1927 ata_channel_lock(chp);
1928 drvp->drive_flags |= ATA_DRIVE_MODE;
1929 ata_channel_unlock(chp);
1930 printed = 0;
1931 for (i = 7; i >= 0; i--) {
1932 if ((params.atap_dmamode_supp & (1 << i)) == 0)
1933 continue;
1934 #if NATA_DMA
1935 if ((atac->atac_cap & ATAC_CAP_DMA) &&
1936 atac->atac_set_modes != NULL)
1937 if (ata_set_mode(drvp, 0x20 | i, AT_WAIT)
1938 != CMD_OK)
1939 continue;
1940 #endif
1941 if (!printed) {
1942 aprint_verbose("%s DMA mode %d", sep, i);
1943 sep = ",";
1944 printed = 1;
1945 }
1946 #if NATA_DMA
1947 if (atac->atac_cap & ATAC_CAP_DMA) {
1948 if (atac->atac_set_modes != NULL &&
1949 atac->atac_dma_cap < i)
1950 continue;
1951 drvp->DMA_mode = i;
1952 drvp->DMA_cap = i;
1953 ata_channel_lock(chp);
1954 drvp->drive_flags |= ATA_DRIVE_DMA;
1955 ata_channel_unlock(chp);
1956 }
1957 #endif
1958 break;
1959 }
1960 if (params.atap_extensions & WDC_EXT_UDMA_MODES) {
1961 printed = 0;
1962 for (i = 7; i >= 0; i--) {
1963 if ((params.atap_udmamode_supp & (1 << i))
1964 == 0)
1965 continue;
1966 #if NATA_UDMA
1967 if (atac->atac_set_modes != NULL &&
1968 (atac->atac_cap & ATAC_CAP_UDMA))
1969 if (ata_set_mode(drvp, 0x40 | i,
1970 AT_WAIT) != CMD_OK)
1971 continue;
1972 #endif
1973 if (!printed) {
1974 aprint_verbose("%s Ultra-DMA mode %d",
1975 sep, i);
1976 if (i == 2)
1977 aprint_verbose(" (Ultra/33)");
1978 else if (i == 4)
1979 aprint_verbose(" (Ultra/66)");
1980 else if (i == 5)
1981 aprint_verbose(" (Ultra/100)");
1982 else if (i == 6)
1983 aprint_verbose(" (Ultra/133)");
1984 sep = ",";
1985 printed = 1;
1986 }
1987 #if NATA_UDMA
1988 if (atac->atac_cap & ATAC_CAP_UDMA) {
1989 if (atac->atac_set_modes != NULL &&
1990 atac->atac_udma_cap < i)
1991 continue;
1992 drvp->UDMA_mode = i;
1993 drvp->UDMA_cap = i;
1994 ata_channel_lock(chp);
1995 drvp->drive_flags |= ATA_DRIVE_UDMA;
1996 ata_channel_unlock(chp);
1997 }
1998 #endif
1999 break;
2000 }
2001 }
2002 }
2003
2004 ata_channel_lock(chp);
2005 drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM;
2006 if (drvp->drive_type == ATA_DRIVET_ATAPI) {
2007 if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM)
2008 drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
2009 } else {
2010 if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM)
2011 drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
2012 }
2013 ata_channel_unlock(chp);
2014
2015 /* Try to guess ATA version here, if it didn't get reported */
2016 if (drvp->ata_vers == 0) {
2017 #if NATA_UDMA
2018 if (drvp->drive_flags & ATA_DRIVE_UDMA)
2019 drvp->ata_vers = 4; /* should be at last ATA-4 */
2020 else
2021 #endif
2022 if (drvp->PIO_cap > 2)
2023 drvp->ata_vers = 2; /* should be at last ATA-2 */
2024 }
2025 cf_flags = device_cfdata(drv_dev)->cf_flags;
2026 if (cf_flags & ATA_CONFIG_PIO_SET) {
2027 ata_channel_lock(chp);
2028 drvp->PIO_mode =
2029 (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF;
2030 drvp->drive_flags |= ATA_DRIVE_MODE;
2031 ata_channel_unlock(chp);
2032 }
2033 #if NATA_DMA
2034 if ((atac->atac_cap & ATAC_CAP_DMA) == 0) {
2035 /* don't care about DMA modes */
2036 return;
2037 }
2038 if (cf_flags & ATA_CONFIG_DMA_SET) {
2039 ata_channel_lock(chp);
2040 if ((cf_flags & ATA_CONFIG_DMA_MODES) ==
2041 ATA_CONFIG_DMA_DISABLE) {
2042 drvp->drive_flags &= ~ATA_DRIVE_DMA;
2043 } else {
2044 drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >>
2045 ATA_CONFIG_DMA_OFF;
2046 drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE;
2047 }
2048 ata_channel_unlock(chp);
2049 }
2050
2051 /*
2052 * Probe WRITE DMA FUA EXT. Support is mandatory for devices
2053 * supporting LBA48, but nevertheless confirm with the feature flag.
2054 */
2055 if (drvp->drive_flags & ATA_DRIVE_DMA) {
2056 if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0
2057 && (params.atap_cmd_def & ATA_CMDE_WFE)) {
2058 drvp->drive_flags |= ATA_DRIVE_WFUA;
2059 aprint_verbose("%s WRITE DMA FUA", sep);
2060 sep = ",";
2061 }
2062 }
2063
2064 /* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */
2065 ata_channel_lock(chp);
2066 drvp->drv_openings = 1;
2067 if (params.atap_sata_caps & SATA_NATIVE_CMDQ) {
2068 if (atac->atac_cap & ATAC_CAP_NCQ)
2069 drvp->drive_flags |= ATA_DRIVE_NCQ;
2070 drvp->drv_openings =
2071 (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1;
2072 aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings);
2073 sep = ",";
2074
2075 if (params.atap_sata_caps & SATA_NCQ_PRIO) {
2076 drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO;
2077 aprint_verbose(" w/PRIO");
2078 }
2079 }
2080 ata_channel_unlock(chp);
2081
2082 if (printed)
2083 aprint_verbose("\n");
2084
2085 #if NATA_UDMA
2086 if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) {
2087 /* don't care about UDMA modes */
2088 return;
2089 }
2090 if (cf_flags & ATA_CONFIG_UDMA_SET) {
2091 ata_channel_lock(chp);
2092 if ((cf_flags & ATA_CONFIG_UDMA_MODES) ==
2093 ATA_CONFIG_UDMA_DISABLE) {
2094 drvp->drive_flags &= ~ATA_DRIVE_UDMA;
2095 } else {
2096 drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >>
2097 ATA_CONFIG_UDMA_OFF;
2098 drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE;
2099 }
2100 ata_channel_unlock(chp);
2101 }
2102 #endif /* NATA_UDMA */
2103 #endif /* NATA_DMA */
2104 }
2105
2106 /* management of the /dev/atabus* devices */
2107 int
2108 atabusopen(dev_t dev, int flag, int fmt, struct lwp *l)
2109 {
2110 struct atabus_softc *sc;
2111 int error;
2112
2113 sc = device_lookup_private(&atabus_cd, minor(dev));
2114 if (sc == NULL)
2115 return (ENXIO);
2116
2117 if (sc->sc_flags & ATABUSCF_OPEN)
2118 return (EBUSY);
2119
2120 if ((error = ata_addref(sc->sc_chan)) != 0)
2121 return (error);
2122
2123 sc->sc_flags |= ATABUSCF_OPEN;
2124
2125 return (0);
2126 }
2127
2128
2129 int
2130 atabusclose(dev_t dev, int flag, int fmt, struct lwp *l)
2131 {
2132 struct atabus_softc *sc =
2133 device_lookup_private(&atabus_cd, minor(dev));
2134
2135 ata_delref(sc->sc_chan);
2136
2137 sc->sc_flags &= ~ATABUSCF_OPEN;
2138
2139 return (0);
2140 }
2141
2142 int
2143 atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l)
2144 {
2145 struct atabus_softc *sc =
2146 device_lookup_private(&atabus_cd, minor(dev));
2147 struct ata_channel *chp = sc->sc_chan;
2148 int min_drive, max_drive, drive;
2149 int error;
2150
2151 /*
2152 * Enforce write permission for ioctls that change the
2153 * state of the bus. Host adapter specific ioctls must
2154 * be checked by the adapter driver.
2155 */
2156 switch (cmd) {
2157 case ATABUSIOSCAN:
2158 case ATABUSIODETACH:
2159 case ATABUSIORESET:
2160 if ((flag & FWRITE) == 0)
2161 return (EBADF);
2162 }
2163
2164 switch (cmd) {
2165 case ATABUSIORESET:
2166 ata_channel_lock(chp);
2167 ata_thread_run(sc->sc_chan, AT_WAIT | AT_POLL,
2168 ATACH_TH_RESET, ATACH_NODRIVE);
2169 ata_channel_unlock(chp);
2170 return 0;
2171 case ATABUSIOSCAN:
2172 {
2173 #if 0
2174 struct atabusioscan_args *a=
2175 (struct atabusioscan_args *)addr;
2176 #endif
2177 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
2178 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
2179 return (EOPNOTSUPP);
2180 return (EOPNOTSUPP);
2181 }
2182 case ATABUSIODETACH:
2183 {
2184 struct atabusiodetach_args *a=
2185 (struct atabusiodetach_args *)addr;
2186 if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
2187 (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
2188 return (EOPNOTSUPP);
2189 switch (a->at_dev) {
2190 case -1:
2191 min_drive = 0;
2192 max_drive = 1;
2193 break;
2194 case 0:
2195 case 1:
2196 min_drive = max_drive = a->at_dev;
2197 break;
2198 default:
2199 return (EINVAL);
2200 }
2201 for (drive = min_drive; drive <= max_drive; drive++) {
2202 if (chp->ch_drive[drive].drv_softc != NULL) {
2203 error = config_detach(
2204 chp->ch_drive[drive].drv_softc, 0);
2205 if (error)
2206 return (error);
2207 KASSERT(chp->ch_drive[drive].drv_softc == NULL);
2208 }
2209 }
2210 return 0;
2211 }
2212 default:
2213 return ENOTTY;
2214 }
2215 }
2216
2217 static bool
2218 atabus_suspend(device_t dv, const pmf_qual_t *qual)
2219 {
2220 struct atabus_softc *sc = device_private(dv);
2221 struct ata_channel *chp = sc->sc_chan;
2222
2223 ata_channel_idle(chp);
2224
2225 return true;
2226 }
2227
2228 static bool
2229 atabus_resume(device_t dv, const pmf_qual_t *qual)
2230 {
2231 struct atabus_softc *sc = device_private(dv);
2232 struct ata_channel *chp = sc->sc_chan;
2233
2234 /*
2235 * XXX joerg: with wdc, the first channel unfreezes the controler.
2236 * Move this the reset and queue idling into wdc.
2237 */
2238 ata_channel_lock(chp);
2239 if (chp->ch_queue->queue_freeze == 0) {
2240 ata_channel_unlock(chp);
2241 goto out;
2242 }
2243
2244 /* unfreeze the queue and reset drives */
2245 ata_channel_thaw_locked(chp);
2246
2247 /* reset channel only if there are drives attached */
2248 if (chp->ch_ndrives > 0)
2249 ata_thread_run(chp, AT_WAIT, ATACH_TH_RESET, ATACH_NODRIVE);
2250
2251 ata_channel_unlock(chp);
2252
2253 out:
2254 return true;
2255 }
2256
2257 static int
2258 atabus_rescan(device_t self, const char *ifattr, const int *locators)
2259 {
2260 struct atabus_softc *sc = device_private(self);
2261 struct ata_channel *chp = sc->sc_chan;
2262 struct atabus_initq *initq;
2263 int i;
2264
2265 /*
2266 * we can rescan a port multiplier atabus, even if some devices are
2267 * still attached
2268 */
2269 if (chp->ch_satapmp_nports == 0) {
2270 if (chp->atapibus != NULL) {
2271 return EBUSY;
2272 }
2273
2274 KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
2275 for (i = 0; i < chp->ch_ndrives; i++) {
2276 if (chp->ch_drive[i].drv_softc != NULL) {
2277 return EBUSY;
2278 }
2279 }
2280 }
2281
2282 initq = kmem_zalloc(sizeof(*initq), KM_SLEEP);
2283 initq->atabus_sc = sc;
2284 mutex_enter(&atabus_qlock);
2285 TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
2286 mutex_exit(&atabus_qlock);
2287 config_pending_incr(sc->sc_dev);
2288
2289 ata_channel_lock(chp);
2290 chp->ch_flags |= ATACH_TH_RESCAN;
2291 cv_signal(&chp->ch_thr_idle);
2292 ata_channel_unlock(chp);
2293
2294 return 0;
2295 }
2296
2297 void
2298 ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags)
2299 {
2300 KASSERT(mutex_owned(&chp->ch_lock));
2301
2302 if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) {
2303 /*
2304 * can't use kpause(), we may be in interrupt context
2305 * or taking a crash dump
2306 */
2307 delay(ms * 1000);
2308 } else {
2309 int pause = mstohz(ms);
2310
2311 kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock);
2312 }
2313 }
2314
2315 void
2316 atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count,
2317 uint16_t *features, uint8_t *device)
2318 {
2319 if ((xfer->c_flags & C_NCQ) == 0) {
2320 /* FUA handling for non-NCQ drives */
2321 if (xfer->c_bio.flags & ATA_FUA
2322 && *cmd == WDCC_WRITEDMA_EXT)
2323 *cmd = WDCC_WRITEDMA_FUA_EXT;
2324
2325 return;
2326 }
2327
2328 *cmd = (xfer->c_bio.flags & ATA_READ) ?
2329 WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED;
2330
2331 /* for FPDMA the block count is in features */
2332 *features = *count;
2333
2334 /* NCQ tag */
2335 *count = (xfer->c_slot << 3);
2336
2337 if (xfer->c_bio.flags & ATA_PRIO_HIGH)
2338 *count |= WDSC_PRIO_HIGH;
2339
2340 /* other device flags */
2341 if (xfer->c_bio.flags & ATA_FUA)
2342 *device |= WDSD_FUA;
2343 }
2344
2345 void
2346 ata_wait_cmd(struct ata_channel *chp, struct ata_xfer *xfer)
2347 {
2348 struct ata_queue *chq = chp->ch_queue;
2349 struct ata_command *ata_c = &xfer->c_ata_c;
2350
2351 ata_channel_lock(chp);
2352
2353 while ((ata_c->flags & AT_DONE) == 0)
2354 cv_wait(&chq->c_cmd_finish, &chp->ch_lock);
2355
2356 ata_channel_unlock(chp);
2357
2358 KASSERT((ata_c->flags & AT_DONE) != 0);
2359 }
2360