ata_subr.c revision 1.6.2.3
1 /* $NetBSD: ata_subr.c,v 1.6.2.3 2018/09/17 19:00:43 jdolecek Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2001 Manuel Bouyer. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __KERNEL_RCSID(0, "$NetBSD: ata_subr.c,v 1.6.2.3 2018/09/17 19:00:43 jdolecek Exp $"); 29 30 #include "opt_ata.h" 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/kernel.h> 35 #include <sys/malloc.h> 36 #include <sys/device.h> 37 #include <sys/conf.h> 38 #include <sys/fcntl.h> 39 #include <sys/proc.h> 40 #include <sys/kthread.h> 41 #include <sys/errno.h> 42 #include <sys/ataio.h> 43 #include <sys/kmem.h> 44 #include <sys/intr.h> 45 #include <sys/bus.h> 46 #include <sys/once.h> 47 #include <sys/bitops.h> 48 49 #define ATABUS_PRIVATE 50 51 #include <dev/ata/ataconf.h> 52 #include <dev/ata/atareg.h> 53 #include <dev/ata/atavar.h> 54 #include <dev/ic/wdcvar.h> /* for PIOBM */ 55 56 #define DEBUG_FUNCS 0x08 57 #define DEBUG_PROBE 0x10 58 #define DEBUG_DETACH 0x20 59 #define DEBUG_XFERS 0x40 60 #ifdef ATADEBUG 61 extern int atadebug_mask; 62 #define ATADEBUG_PRINT(args, level) \ 63 if (atadebug_mask & (level)) \ 64 printf args 65 #else 66 #define ATADEBUG_PRINT(args, level) 67 #endif 68 69 void 70 ata_queue_reset(struct ata_queue *chq) 71 { 72 /* make sure that we can use polled commands */ 73 SIMPLEQ_INIT(&chq->queue_xfer); 74 TAILQ_INIT(&chq->active_xfers); 75 chq->queue_freeze = 0; 76 chq->queue_active = 0; 77 chq->active_xfers_used = 0; 78 chq->queue_xfers_avail = __BIT(chq->queue_openings) - 1; 79 } 80 81 struct ata_xfer * 82 ata_queue_hwslot_to_xfer(struct ata_channel *chp, int hwslot) 83 { 84 struct ata_queue *chq = chp->ch_queue; 85 struct ata_xfer *xfer = NULL; 86 87 ata_channel_lock(chp); 88 89 KASSERTMSG(hwslot < chq->queue_openings, "hwslot %d > openings %d", 90 hwslot, chq->queue_openings); 91 KASSERTMSG((chq->active_xfers_used & __BIT(hwslot)) != 0, 92 "hwslot %d not active", hwslot); 93 94 /* Usually the first entry will be the one */ 95 TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) { 96 if (xfer->c_slot == hwslot) 97 break; 98 } 99 100 ata_channel_unlock(chp); 101 102 KASSERTMSG((xfer != NULL), 103 "%s: xfer with slot %d not found (active %x)", __func__, 104 hwslot, chq->active_xfers_used); 105 106 return xfer; 107 } 108 109 struct ata_xfer * 110 ata_queue_get_active_xfer_locked(struct ata_channel *chp) 111 { 112 struct ata_xfer *xfer; 113 114 KASSERT(mutex_owned(&chp->ch_lock)); 115 xfer = TAILQ_FIRST(&chp->ch_queue->active_xfers); 116 117 if (xfer && ISSET(xfer->c_flags, C_NCQ)) { 118 /* Spurious call, never return NCQ xfer from this interface */ 119 xfer = NULL; 120 } 121 122 return xfer; 123 } 124 125 /* 126 * This interface is supposed only to be used when there is exactly 127 * one outstanding command, when there is no information about the slot, 128 * which triggered the command. ata_queue_hwslot_to_xfer() interface 129 * is preferred in all NCQ cases. 130 */ 131 struct ata_xfer * 132 ata_queue_get_active_xfer(struct ata_channel *chp) 133 { 134 struct ata_xfer *xfer = NULL; 135 136 ata_channel_lock(chp); 137 xfer = ata_queue_get_active_xfer_locked(chp); 138 ata_channel_unlock(chp); 139 140 return xfer; 141 } 142 143 struct ata_xfer * 144 ata_queue_drive_active_xfer(struct ata_channel *chp, int drive) 145 { 146 struct ata_xfer *xfer = NULL; 147 148 ata_channel_lock(chp); 149 150 TAILQ_FOREACH(xfer, &chp->ch_queue->active_xfers, c_activechain) { 151 if (xfer->c_drive == drive) 152 break; 153 } 154 KASSERT(xfer != NULL); 155 156 ata_channel_unlock(chp); 157 158 return xfer; 159 } 160 161 struct ata_queue * 162 ata_queue_alloc(uint8_t openings) 163 { 164 if (openings == 0) 165 openings = 1; 166 167 if (openings > ATA_MAX_OPENINGS) 168 openings = ATA_MAX_OPENINGS; 169 170 struct ata_queue *chq = malloc(offsetof(struct ata_queue, queue_xfers[openings]), 171 M_DEVBUF, M_WAITOK | M_ZERO); 172 173 chq->queue_openings = openings; 174 ata_queue_reset(chq); 175 176 cv_init(&chq->queue_busy, "ataqbusy"); 177 cv_init(&chq->queue_drain, "atdrn"); 178 cv_init(&chq->queue_idle, "qidl"); 179 180 cv_init(&chq->c_active, "ataact"); 181 cv_init(&chq->c_cmd_finish, "atafin"); 182 183 for (uint8_t i = 0; i < openings; i++) 184 chq->queue_xfers[i].c_slot = i; 185 186 return chq; 187 } 188 189 void 190 ata_queue_free(struct ata_queue *chq) 191 { 192 cv_destroy(&chq->queue_busy); 193 cv_destroy(&chq->queue_drain); 194 cv_destroy(&chq->queue_idle); 195 196 cv_destroy(&chq->c_active); 197 cv_destroy(&chq->c_cmd_finish); 198 199 free(chq, M_DEVBUF); 200 } 201 202 void 203 ata_channel_init(struct ata_channel *chp) 204 { 205 mutex_init(&chp->ch_lock, MUTEX_DEFAULT, IPL_BIO); 206 cv_init(&chp->ch_thr_idle, "atath"); 207 208 callout_init(&chp->c_timo_callout, 0); /* XXX MPSAFE */ 209 210 /* Optionally setup the queue, too */ 211 if (chp->ch_queue == NULL) { 212 chp->ch_queue = ata_queue_alloc(1); 213 } 214 } 215 216 void 217 ata_channel_destroy(struct ata_channel *chp) 218 { 219 if (chp->ch_queue != NULL) { 220 ata_queue_free(chp->ch_queue); 221 chp->ch_queue = NULL; 222 } 223 224 mutex_enter(&chp->ch_lock); 225 callout_halt(&chp->c_timo_callout, &chp->ch_lock); 226 callout_destroy(&chp->c_timo_callout); 227 mutex_exit(&chp->ch_lock); 228 229 mutex_destroy(&chp->ch_lock); 230 cv_destroy(&chp->ch_thr_idle); 231 } 232 233 /* 234 * Does it's own locking, does not require splbio(). 235 * flags - whether to block waiting for free xfer 236 * openings - limit of openings supported by device, <= 0 means tag not 237 * relevant, and any available xfer can be returned 238 */ 239 struct ata_xfer * 240 ata_get_xfer_ext(struct ata_channel *chp, int flags, uint8_t openings) 241 { 242 struct ata_queue *chq = chp->ch_queue; 243 struct ata_xfer *xfer = NULL; 244 uint32_t avail, slot, mask; 245 int error; 246 247 ATADEBUG_PRINT(("%s: channel %d fl 0x%x op %d qavail 0x%x qact %d", 248 __func__, chp->ch_channel, flags, openings, 249 chq->queue_xfers_avail, chq->queue_active), 250 DEBUG_XFERS); 251 252 ata_channel_lock(chp); 253 254 /* 255 * When openings is just 1, can't reserve anything for 256 * recovery. KASSERT() here is to catch code which naively 257 * relies on C_RECOVERY to work under this condition. 258 */ 259 KASSERT((flags & C_RECOVERY) == 0 || chq->queue_openings > 1); 260 261 if (flags & C_RECOVERY) { 262 mask = UINT32_MAX; 263 } else { 264 if (openings <= 0 || openings > chq->queue_openings) 265 openings = chq->queue_openings; 266 267 if (openings > 1) { 268 mask = __BIT(openings - 1) - 1; 269 } else { 270 mask = UINT32_MAX; 271 } 272 } 273 274 retry: 275 avail = ffs32(chq->queue_xfers_avail & mask); 276 if (avail == 0) { 277 /* 278 * Catch code which tries to get another recovery xfer while 279 * already holding one (wrong recursion). 280 */ 281 KASSERTMSG((flags & C_RECOVERY) == 0, 282 "recovery xfer busy openings %d mask %x avail %x", 283 openings, mask, chq->queue_xfers_avail); 284 285 if (flags & C_WAIT) { 286 chq->queue_flags |= QF_NEED_XFER; 287 error = cv_wait_sig(&chq->queue_busy, &chp->ch_lock); 288 if (error == 0) 289 goto retry; 290 } 291 292 goto out; 293 } 294 295 slot = avail - 1; 296 xfer = &chq->queue_xfers[slot]; 297 chq->queue_xfers_avail &= ~__BIT(slot); 298 299 KASSERT((chq->active_xfers_used & __BIT(slot)) == 0); 300 301 /* zero everything after the callout member */ 302 memset(&xfer->c_startzero, 0, 303 sizeof(struct ata_xfer) - offsetof(struct ata_xfer, c_startzero)); 304 305 out: 306 ata_channel_unlock(chp); 307 308 ATADEBUG_PRINT((" xfer %p\n", xfer), DEBUG_XFERS); 309 return xfer; 310 } 311 312 /* 313 * ata_deactivate_xfer() must be always called prior to ata_free_xfer() 314 */ 315 void 316 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer) 317 { 318 struct ata_queue *chq = chp->ch_queue; 319 320 ata_channel_lock(chp); 321 322 if (xfer->c_flags & (C_WAITACT|C_WAITTIMO)) { 323 /* Someone is waiting for this xfer, so we can't free now */ 324 xfer->c_flags |= C_FREE; 325 cv_broadcast(&chq->c_active); 326 goto out; 327 } 328 329 /* XXX move PIOBM and free_gw to deactivate? */ 330 #if NATA_PIOBM /* XXX wdc dependent code */ 331 if (xfer->c_flags & C_PIOBM) { 332 struct wdc_softc *wdc = CHAN_TO_WDC(chp); 333 334 /* finish the busmastering PIO */ 335 (*wdc->piobm_done)(wdc->dma_arg, 336 chp->ch_channel, xfer->c_drive); 337 chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT); 338 } 339 #endif 340 341 if (chp->ch_atac->atac_free_hw) 342 chp->ch_atac->atac_free_hw(chp); 343 344 KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0); 345 KASSERT((chq->queue_xfers_avail & __BIT(xfer->c_slot)) == 0); 346 chq->queue_xfers_avail |= __BIT(xfer->c_slot); 347 348 out: 349 if (chq->queue_flags & QF_NEED_XFER) { 350 chq->queue_flags &= ~QF_NEED_XFER; 351 cv_broadcast(&chq->queue_busy); 352 } 353 354 ata_channel_unlock(chp); 355 356 ATADEBUG_PRINT(("%s: channel %d xfer %p qavail 0x%x qact %d\n", 357 __func__, chp->ch_channel, xfer, chq->queue_xfers_avail, 358 chq->queue_active), 359 DEBUG_XFERS); 360 } 361 362 void 363 ata_timeout(void *v) 364 { 365 struct ata_channel *chp = v; 366 struct ata_queue *chq = chp->ch_queue; 367 struct ata_xfer *xfer, *nxfer; 368 int s; 369 370 s = splbio(); /* XXX MPSAFE */ 371 372 callout_ack(&chp->c_timo_callout); 373 374 /* 375 * If there is a timeout, means the last enqueued command 376 * timed out, and thus all commands timed out. 377 * XXX locking 378 */ 379 TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, nxfer) { 380 ATADEBUG_PRINT(("%s: slot %d\n", __func__, xfer->c_slot), 381 DEBUG_FUNCS|DEBUG_XFERS); 382 383 if (ata_timo_xfer_check(xfer)) { 384 /* Already logged */ 385 continue; 386 } 387 388 /* Mark as timed out. Do not print anything, wd(4) will. */ 389 xfer->c_flags |= C_TIMEOU; 390 xfer->ops->c_intr(xfer->c_chp, xfer, 0); 391 } 392 393 splx(s); 394 } 395 396 /* 397 * Must be called without any locks, i.e. with both drive and channel locks 398 * released. 399 */ 400 void 401 ata_channel_start(struct ata_channel *chp, int drive, bool start_self) 402 { 403 int i, s; 404 struct ata_drive_datas *drvp; 405 406 s = splbio(); 407 408 KASSERT(chp->ch_ndrives > 0); 409 410 #define ATA_DRIVE_START(chp, drive) \ 411 do { \ 412 KASSERT(drive < chp->ch_ndrives); \ 413 drvp = &chp->ch_drive[drive]; \ 414 \ 415 if (drvp->drive_type != ATA_DRIVET_ATA && \ 416 drvp->drive_type != ATA_DRIVET_ATAPI && \ 417 drvp->drive_type != ATA_DRIVET_OLD) \ 418 continue; \ 419 \ 420 if (drvp->drv_start != NULL) \ 421 (*drvp->drv_start)(drvp->drv_softc); \ 422 } while (0) 423 424 /* 425 * Process drives in round robin fashion starting with next one after 426 * the one which finished transfer. Thus no single drive would 427 * completely starve other drives on same channel. 428 * This loop processes all but the current drive, so won't do anything 429 * if there is only one drive in channel. 430 */ 431 for (i = (drive + 1) % chp->ch_ndrives; i != drive; 432 i = (i + 1) % chp->ch_ndrives) { 433 ATA_DRIVE_START(chp, i); 434 } 435 436 /* Now try to kick off xfers on the current drive */ 437 if (start_self) 438 ATA_DRIVE_START(chp, drive); 439 440 splx(s); 441 #undef ATA_DRIVE_START 442 } 443 444 void 445 ata_channel_lock(struct ata_channel *chp) 446 { 447 mutex_enter(&chp->ch_lock); 448 } 449 450 void 451 ata_channel_unlock(struct ata_channel *chp) 452 { 453 mutex_exit(&chp->ch_lock); 454 } 455 456 void 457 ata_channel_lock_owned(struct ata_channel *chp) 458 { 459 KASSERT(mutex_owned(&chp->ch_lock)); 460 } 461 462 #ifdef ATADEBUG 463 void 464 atachannel_debug(struct ata_channel *chp) 465 { 466 struct ata_queue *chq = chp->ch_queue; 467 468 printf(" ch %s flags 0x%x ndrives %d\n", 469 device_xname(chp->atabus), chp->ch_flags, chp->ch_ndrives); 470 printf(" que: flags 0x%x avail 0x%x used 0x%x\n", 471 chq->queue_flags, chq->queue_xfers_avail, chq->active_xfers_used); 472 printf(" act %d freez %d open %u\n", 473 chq->queue_active, chq->queue_freeze, chq->queue_openings); 474 475 #if 0 476 printf(" xfers:\n"); 477 for(int i=0; i < chq->queue_openings; i++) { 478 struct ata_xfer *xfer = &chq->queue_xfers[i]; 479 480 printf(" #%d sl %d drv %d retr %d fl %x", 481 i, xfer->c_slot, xfer->c_drive, xfer->c_retries, 482 xfer->c_flags); 483 printf(" data %p bcount %d skip %d\n", 484 xfer->c_databuf, xfer->c_bcount, xfer->c_skip); 485 } 486 #endif 487 } 488 #endif /* ATADEBUG */ 489
Indexes created Tue Oct 14 15:09:51 GMT 2025