1 /* $NetBSD: rf_diskqueue.c,v 1.67 2026/10/24 00:20:24 tls Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /**************************************************************************** 30 * 31 * rf_diskqueue.c -- higher-level disk queue code 32 * 33 * the routines here are a generic wrapper around the actual queueing 34 * routines. The code here implements thread scheduling, synchronization, 35 * and locking ops (see below) on top of the lower-level queueing code. 36 * 37 * to support atomic RMW, we implement "locking operations". When a 38 * locking op is dispatched to the lower levels of the driver, the 39 * queue is locked, and no further I/Os are dispatched until the queue 40 * receives & completes a corresponding "unlocking operation". This 41 * code relies on the higher layers to guarantee that a locking op 42 * will always be eventually followed by an unlocking op. The model 43 * is that the higher layers are structured so locking and unlocking 44 * ops occur in pairs, i.e. an unlocking op cannot be generated until 45 * after a locking op reports completion. There is no good way to 46 * check to see that an unlocking op "corresponds" to the op that 47 * currently has the queue locked, so we make no such attempt. Since 48 * by definition there can be only one locking op outstanding on a 49 * disk, this should not be a problem. 50 * 51 * In the kernel, we allow multiple I/Os to be concurrently dispatched 52 * to the disk driver. In order to support locking ops in this 53 * environment, when we decide to do a locking op, we stop dispatching 54 * new I/Os and wait until all dispatched I/Os have completed before 55 * dispatching the locking op. 56 * 57 * Unfortunately, the code is different in the 3 different operating 58 * states (user level, kernel, simulator). In the kernel, I/O is 59 * non-blocking, and we have no disk threads to dispatch for us. 60 * Therefore, we have to dispatch new I/Os to the scsi driver at the 61 * time of enqueue, and also at the time of completion. At user 62 * level, I/O is blocking, and so only the disk threads may dispatch 63 * I/Os. Thus at user level, all we can do at enqueue time is enqueue 64 * and wake up the disk thread to do the dispatch. 65 * 66 ****************************************************************************/ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: rf_diskqueue.c,v 1.67 2026/10/24 00:20:24 tls Exp $"); 70 71 #include <dev/raidframe/raidframevar.h> 72 73 #include "rf_threadstuff.h" 74 #include "rf_raid.h" 75 #include "rf_diskqueue.h" 76 #include "rf_alloclist.h" 77 #include "rf_acctrace.h" 78 #include "rf_etimer.h" 79 #include "rf_general.h" 80 #include "rf_debugprint.h" 81 #include "rf_shutdown.h" 82 #include "rf_cvscan.h" 83 #include "rf_sstf.h" 84 #include "rf_fifo.h" 85 #include "rf_kintf.h" 86 87 #include <sys/buf.h> 88 89 static void rf_ShutdownDiskQueueSystem(void *); 90 91 #ifndef RF_DEBUG_DISKQUEUE 92 #define RF_DEBUG_DISKQUEUE 0 93 #endif 94 95 #if RF_DEBUG_DISKQUEUE 96 #define Dprintf1(s,a) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),NULL,NULL,NULL,NULL,NULL,NULL,NULL) 97 #define Dprintf2(s,a,b) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),(void *)((unsigned long)b),NULL,NULL,NULL,NULL,NULL,NULL) 98 #define Dprintf3(s,a,b,c) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),(void *)((unsigned long)b),(void *)((unsigned long)c),NULL,NULL,NULL,NULL,NULL) 99 #else 100 #define Dprintf1(s,a) 101 #define Dprintf2(s,a,b) 102 #define Dprintf3(s,a,b,c) 103 #endif 104 105 /***************************************************************************** 106 * 107 * the disk queue switch defines all the functions used in the 108 * different queueing disciplines queue ID, init routine, enqueue 109 * routine, dequeue routine 110 * 111 ****************************************************************************/ 112 113 static const RF_DiskQueueSW_t diskqueuesw[] = { 114 {"fifo", /* FIFO */ 115 rf_FifoCreate, 116 rf_FifoEnqueue, 117 rf_FifoDequeue, 118 rf_FifoPromote}, 119 120 {"cvscan", /* cvscan */ 121 rf_CvscanCreate, 122 rf_CvscanEnqueue, 123 rf_CvscanDequeue, 124 rf_CvscanPromote}, 125 126 {"sstf", /* shortest seek time first */ 127 rf_SstfCreate, 128 rf_SstfEnqueue, 129 rf_SstfDequeue, 130 rf_SstfPromote}, 131 132 {"scan", /* SCAN (two-way elevator) */ 133 rf_ScanCreate, 134 rf_SstfEnqueue, 135 rf_ScanDequeue, 136 rf_SstfPromote}, 137 138 {"cscan", /* CSCAN (one-way elevator) */ 139 rf_CscanCreate, 140 rf_SstfEnqueue, 141 rf_CscanDequeue, 142 rf_SstfPromote}, 143 144 }; 145 #define NUM_DISK_QUEUE_TYPES (sizeof(diskqueuesw)/sizeof(RF_DiskQueueSW_t)) 146 147 /* configures a single disk queue */ 148 149 static void 150 rf_ShutdownDiskQueue(void *arg) 151 { 152 RF_DiskQueue_t *diskqueue = arg; 153 154 rf_destroy_mutex2(diskqueue->mutex); 155 } 156 157 int 158 rf_ConfigureDiskQueue(RF_Raid_t *raidPtr, RF_DiskQueue_t *diskqueue, 159 RF_RowCol_t c, const RF_DiskQueueSW_t *p, 160 RF_SectorCount_t sectPerDisk, dev_t dev, 161 int maxOutstanding, RF_ShutdownList_t **listp, 162 RF_AllocListElem_t *clList) 163 { 164 diskqueue->col = c; 165 diskqueue->qPtr = p; 166 diskqueue->qHdr = (p->Create) (sectPerDisk, clList, listp); 167 diskqueue->dev = dev; 168 diskqueue->numOutstanding = 0; 169 diskqueue->queueLength = 0; 170 diskqueue->maxOutstanding = maxOutstanding; 171 diskqueue->curPriority = RF_IO_NORMAL_PRIORITY; 172 diskqueue->flags = 0; 173 diskqueue->raidPtr = raidPtr; 174 diskqueue->rf_cinfo = &raidPtr->raid_cinfo[c]; 175 rf_init_mutex2(diskqueue->mutex, IPL_VM); 176 rf_ShutdownCreate(listp, rf_ShutdownDiskQueue, diskqueue); 177 return (0); 178 } 179 180 int 181 rf_UpdateDiskQueue(RF_DiskQueue_t *diskqueue, RF_RaidDisk_t *disk) 182 { 183 diskqueue->dev = disk->dev; 184 return(0); 185 } 186 187 static void 188 rf_ShutdownDiskQueueSystem(void *arg) 189 { 190 RF_Raid_t *raidPtr; 191 192 raidPtr = (RF_Raid_t *) arg; 193 194 pool_destroy(&raidPtr->pools.dqd); 195 pool_destroy(&raidPtr->pools.bufio); 196 } 197 198 int 199 rf_ConfigureDiskQueueSystem(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr, 200 RF_Config_t *cfgPtr) 201 202 { 203 size_t stripe_data_bytes, min_dqd, max_dqd, min_bufio, max_bufio; 204 u_int max_stripes_per_bp, c, o, stripes, recon_io; 205 206 c = raidPtr->numCol; 207 o = raidPtr->openings; 208 stripe_data_bytes = (size_t)raidPtr->Layout.dataSectorsPerStripe << 209 raidPtr->logBytesPerSector; 210 /* Total active stripes for 'openings' MAXPHYS I/Os + 1 parity op. */ 211 max_stripes_per_bp = howmany(MAXPHYS, stripe_data_bytes) + 1; 212 stripes = o * max_stripes_per_bp + 1; 213 /* Active recon reads/writes (c-1 exclusive + c-1 forced + floating). */ 214 recon_io = (raidPtr->Layout.map->SubmitReconBuffer == NULL) ? 215 0 : 2 * (c - 1) + raidPtr->numFloatingReconBufs; 216 217 /* 218 * Prime (xmin) for up to 2*c disk ops per active stripe + recon I/Os 219 * so rf_EngineThread / rf_RaidIOThread never block in pool_get(PR_WAITOK). 220 */ 221 min_dqd = 2 * c * stripes + recon_io; 222 max_dqd = uimax(2 * min_dqd, 223 (raidPtr->numCol + raidPtr->numSpare) * raidPtr->maxQueueDepth); 224 225 /* 1 buf_t per RF_DiskQueueData_t. */ 226 min_bufio = min_dqd; 227 max_bufio = max_dqd; 228 229 rf_pool_init(raidPtr, raidPtr->poolNames.dqd, &raidPtr->pools.dqd, sizeof(RF_DiskQueueData_t), 230 "dqd", min_dqd, max_dqd); 231 rf_pool_init(raidPtr, raidPtr->poolNames.bufio, &raidPtr->pools.bufio, sizeof(buf_t), 232 "bufio", min_bufio, max_bufio); 233 rf_ShutdownCreate(listp, rf_ShutdownDiskQueueSystem, raidPtr); 234 235 return (0); 236 } 237 238 int 239 rf_ConfigureDiskQueues(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr, 240 RF_Config_t *cfgPtr) 241 { 242 RF_DiskQueue_t *diskQueues, *spareQueues; 243 const RF_DiskQueueSW_t *p; 244 RF_RowCol_t r,c; 245 int rc, i; 246 247 raidPtr->maxQueueDepth = (uint8_t)cfgPtr->maxOutstandingDiskReqs; 248 if (raidPtr->maxQueueDepth == 0) 249 raidPtr->maxQueueDepth = RF_DEFAULT_QUEUE_DEPTH; 250 251 for (p = NULL, i = 0; i < NUM_DISK_QUEUE_TYPES; i++) { 252 if (!strcmp(diskqueuesw[i].queueType, cfgPtr->diskQueueType)) { 253 p = &diskqueuesw[i]; 254 break; 255 } 256 } 257 if (p == NULL) { 258 RF_ERRORMSG2("Unknown queue type \"%s\". Using %s\n", cfgPtr->diskQueueType, diskqueuesw[0].queueType); 259 p = &diskqueuesw[0]; 260 } 261 raidPtr->qType = p; 262 263 diskQueues = RF_MallocAndAdd( 264 (raidPtr->numCol + RF_MAXSPARE) * sizeof(*diskQueues), 265 raidPtr->cleanupList); 266 if (diskQueues == NULL) 267 return (ENOMEM); 268 raidPtr->Queues = diskQueues; 269 270 for (c = 0; c < raidPtr->numCol; c++) { 271 rc = rf_ConfigureDiskQueue(raidPtr, &diskQueues[c], 272 c, p, 273 raidPtr->sectorsPerDisk, 274 raidPtr->Disks[c].dev, 275 raidPtr->maxQueueDepth, 276 listp, raidPtr->cleanupList); 277 if (rc) 278 return (rc); 279 } 280 281 spareQueues = &raidPtr->Queues[raidPtr->numCol]; 282 for (r = 0; r < raidPtr->maxQueue; r++) { 283 rc = rf_ConfigureDiskQueue(raidPtr, &spareQueues[r], 284 raidPtr->numCol + r, p, 285 raidPtr->sectorsPerDisk, 286 raidPtr->Disks[raidPtr->numCol + r].dev, 287 raidPtr->maxQueueDepth, listp, 288 raidPtr->cleanupList); 289 if (rc) 290 return (rc); 291 } 292 return (0); 293 } 294 /* Enqueue a disk I/O 295 * 296 * In the kernel, I/O is non-blocking and so we'd like to have multiple 297 * I/Os outstanding on the physical disks when possible. 298 * 299 * when any request arrives at a queue, we have two choices: 300 * dispatch it to the lower levels 301 * queue it up 302 * 303 * kernel rules for when to do what: 304 * unlocking req : always dispatch it 305 * normal req : queue empty => dispatch it & set priority 306 * queue not full & priority is ok => dispatch it 307 * else queue it 308 */ 309 void 310 rf_DiskIOEnqueue(RF_DiskQueue_t *queue, RF_DiskQueueData_t *req, int pri) 311 { 312 RF_ETIMER_START(req->qtime); 313 RF_ASSERT(req->type == RF_IO_TYPE_NOP || req->numSector); 314 req->priority = pri; 315 316 #if RF_DEBUG_DISKQUEUE 317 if (rf_queueDebug && (req->numSector == 0)) { 318 printf("Warning: Enqueueing zero-sector access\n"); 319 } 320 #endif 321 RF_LOCK_QUEUE_MUTEX(queue, "DiskIOEnqueue"); 322 if (RF_OK_TO_DISPATCH(queue, req)) { 323 Dprintf2("Dispatching pri %d regular op to c %d (ok to dispatch)\n", pri, queue->col); 324 rf_DispatchKernelIO(queue, req); 325 } else { 326 queue->queueLength++; /* increment count of number of requests waiting in this queue */ 327 Dprintf2("Enqueueing pri %d regular op to c %d (not ok to dispatch)\n", pri, queue->col); 328 req->queue = (void *) queue; 329 (queue->qPtr->Enqueue) (queue->qHdr, req, pri); 330 } 331 RF_UNLOCK_QUEUE_MUTEX(queue, "DiskIOEnqueue"); 332 } 333 334 335 /* get the next set of I/Os started */ 336 void 337 rf_DiskIOComplete(RF_DiskQueue_t *queue, RF_DiskQueueData_t *req, int status) 338 { 339 int done = 0; 340 341 RF_LOCK_QUEUE_MUTEX(queue, "DiskIOComplete"); 342 queue->numOutstanding--; 343 RF_ASSERT(queue->numOutstanding >= 0); 344 345 /* dispatch requests to the disk until we find one that we can't. */ 346 /* no reason to continue once we've filled up the queue */ 347 /* no reason to even start if the queue is locked */ 348 349 while (!done && !RF_QUEUE_FULL(queue)) { 350 req = (queue->qPtr->Dequeue) (queue->qHdr); 351 if (req) { 352 Dprintf2("DiskIOComplete: extracting pri %d req from queue at c %d\n", req->priority, queue->col); 353 queue->queueLength--; /* decrement count of number of requests waiting in this queue */ 354 RF_ASSERT(queue->queueLength >= 0); 355 if (RF_OK_TO_DISPATCH(queue, req)) { 356 Dprintf2("DiskIOComplete: dispatching pri %d regular req to c %d (ok to dispatch)\n", req->priority, queue->col); 357 rf_DispatchKernelIO(queue, req); 358 } else { 359 /* we can't dispatch it, so just re-enqueue it. 360 potential trouble here if disk queues batch reqs */ 361 Dprintf2("DiskIOComplete: re-enqueueing pri %d regular req to c %d\n", req->priority, queue->col); 362 queue->queueLength++; 363 (queue->qPtr->Enqueue) (queue->qHdr, req, req->priority); 364 done = 1; 365 } 366 } else { 367 Dprintf1("DiskIOComplete: no more requests to extract.\n", ""); 368 done = 1; 369 } 370 } 371 372 RF_UNLOCK_QUEUE_MUTEX(queue, "DiskIOComplete"); 373 } 374 /* promotes accesses tagged with the given parityStripeID from low priority 375 * to normal priority. This promotion is optional, meaning that a queue 376 * need not implement it. If there is no promotion routine associated with 377 * a queue, this routine does nothing and returns -1. 378 */ 379 int 380 rf_DiskIOPromote(RF_DiskQueue_t *queue, RF_StripeNum_t parityStripeID, 381 RF_ReconUnitNum_t which_ru) 382 { 383 int retval; 384 385 if (!queue->qPtr->Promote) 386 return (-1); 387 RF_LOCK_QUEUE_MUTEX(queue, "DiskIOPromote"); 388 retval = (queue->qPtr->Promote) (queue->qHdr, parityStripeID, which_ru); 389 RF_UNLOCK_QUEUE_MUTEX(queue, "DiskIOPromote"); 390 return (retval); 391 } 392 393 RF_DiskQueueData_t * 394 rf_CreateDiskQueueData(RF_IoType_t typ, RF_SectorNum_t ssect, 395 RF_SectorCount_t nsect, void *bf, 396 RF_StripeNum_t parityStripeID, 397 RF_ReconUnitNum_t which_ru, 398 void (*wakeF) (void *, int), void *arg, 399 RF_AccTraceEntry_t *tracerec, RF_Raid_t *raidPtr, 400 RF_DiskQueueDataFlags_t flags, const struct buf *mbp) 401 { 402 RF_DiskQueueData_t *p; 403 404 p = pool_get(&raidPtr->pools.dqd, PR_WAITOK | PR_ZERO); 405 KASSERT(p != NULL); 406 407 /* Obtain a buffer from our own pool. It is possible for the 408 regular getiobuf() to run out of memory and return NULL. 409 We need to guarantee that never happens, as RAIDframe 410 doesn't have a good way to recover if memory allocation 411 fails here. 412 */ 413 p->bp = pool_get(&raidPtr->pools.bufio, PR_WAITOK | PR_ZERO); 414 KASSERT(p->bp != NULL); 415 416 buf_init(p->bp); 417 418 SET(p->bp->b_cflags, BC_BUSY); /* mark buffer busy */ 419 if (mbp) { 420 SET(p->bp->b_flags, mbp->b_flags & rf_b_pass); 421 p->bp->b_proc = mbp->b_proc; 422 } 423 424 p->sectorOffset = ssect + rf_protectedSectors; 425 p->numSector = nsect; 426 p->type = typ; 427 p->buf = bf; 428 p->parityStripeID = parityStripeID; 429 p->which_ru = which_ru; 430 p->CompleteFunc = wakeF; 431 p->argument = arg; 432 p->next = NULL; 433 p->tracerec = tracerec; 434 p->priority = RF_IO_NORMAL_PRIORITY; 435 p->raidPtr = raidPtr; 436 p->flags = flags; 437 return (p); 438 } 439 440 void 441 rf_FreeDiskQueueData(RF_DiskQueueData_t *p) 442 { 443 444 buf_destroy(p->bp); 445 446 pool_put(&p->raidPtr->pools.bufio, p->bp); 447 pool_put(&p->raidPtr->pools.dqd, p); 448 } 449
Indexes created Sun Oct 25 00:26:45 UTC 2026