rf_paritylogging.c revision 1.17
1 /* $NetBSD: rf_paritylogging.c,v 1.17 2003/12/29 03:33:48 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: William V. Courtright II 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 30 /* 31 parity logging configuration, dag selection, and mapping is implemented here 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: rf_paritylogging.c,v 1.17 2003/12/29 03:33:48 oster Exp $"); 36 37 #include "rf_archs.h" 38 39 #if RF_INCLUDE_PARITYLOGGING > 0 40 41 #include <dev/raidframe/raidframevar.h> 42 43 #include "rf_raid.h" 44 #include "rf_dag.h" 45 #include "rf_dagutils.h" 46 #include "rf_dagfuncs.h" 47 #include "rf_dagffrd.h" 48 #include "rf_dagffwr.h" 49 #include "rf_dagdegrd.h" 50 #include "rf_dagdegwr.h" 51 #include "rf_paritylog.h" 52 #include "rf_paritylogDiskMgr.h" 53 #include "rf_paritylogging.h" 54 #include "rf_parityloggingdags.h" 55 #include "rf_general.h" 56 #include "rf_map.h" 57 #include "rf_utils.h" 58 #include "rf_shutdown.h" 59 60 typedef struct RF_ParityLoggingConfigInfo_s { 61 RF_RowCol_t **stripeIdentifier; /* filled in at config time & used by 62 * IdentifyStripe */ 63 } RF_ParityLoggingConfigInfo_t; 64 65 static void FreeRegionInfo(RF_Raid_t * raidPtr, RF_RegionId_t regionID); 66 static void rf_ShutdownParityLogging(RF_ThreadArg_t arg); 67 static void rf_ShutdownParityLoggingRegionInfo(RF_ThreadArg_t arg); 68 static void rf_ShutdownParityLoggingPool(RF_ThreadArg_t arg); 69 static void rf_ShutdownParityLoggingRegionBufferPool(RF_ThreadArg_t arg); 70 static void rf_ShutdownParityLoggingParityBufferPool(RF_ThreadArg_t arg); 71 static void rf_ShutdownParityLoggingDiskQueue(RF_ThreadArg_t arg); 72 73 int 74 rf_ConfigureParityLogging( 75 RF_ShutdownList_t ** listp, 76 RF_Raid_t * raidPtr, 77 RF_Config_t * cfgPtr) 78 { 79 int i, j, startdisk, rc; 80 RF_SectorCount_t totalLogCapacity, fragmentation, lastRegionCapacity; 81 RF_SectorCount_t parityBufferCapacity, maxRegionParityRange; 82 RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; 83 RF_ParityLoggingConfigInfo_t *info; 84 RF_ParityLog_t *l = NULL, *next; 85 caddr_t lHeapPtr; 86 87 if (rf_numParityRegions <= 0) 88 return(EINVAL); 89 90 /* 91 * We create multiple entries on the shutdown list here, since 92 * this configuration routine is fairly complicated in and of 93 * itself, and this makes backing out of a failed configuration 94 * much simpler. 95 */ 96 97 raidPtr->numSectorsPerLog = RF_DEFAULT_NUM_SECTORS_PER_LOG; 98 99 /* create a parity logging configuration structure */ 100 RF_MallocAndAdd(info, sizeof(RF_ParityLoggingConfigInfo_t), 101 (RF_ParityLoggingConfigInfo_t *), 102 raidPtr->cleanupList); 103 if (info == NULL) 104 return (ENOMEM); 105 layoutPtr->layoutSpecificInfo = (void *) info; 106 107 /* the stripe identifier must identify the disks in each stripe, IN 108 * THE ORDER THAT THEY APPEAR IN THE STRIPE. */ 109 info->stripeIdentifier = rf_make_2d_array((raidPtr->numCol), 110 (raidPtr->numCol), 111 raidPtr->cleanupList); 112 if (info->stripeIdentifier == NULL) 113 return (ENOMEM); 114 115 startdisk = 0; 116 for (i = 0; i < (raidPtr->numCol); i++) { 117 for (j = 0; j < (raidPtr->numCol); j++) { 118 info->stripeIdentifier[i][j] = (startdisk + j) % 119 (raidPtr->numCol - 1); 120 } 121 if ((--startdisk) < 0) 122 startdisk = raidPtr->numCol - 1 - 1; 123 } 124 125 /* fill in the remaining layout parameters */ 126 layoutPtr->numStripe = layoutPtr->stripeUnitsPerDisk; 127 layoutPtr->numParityCol = 1; 128 layoutPtr->numParityLogCol = 1; 129 layoutPtr->numDataCol = raidPtr->numCol - layoutPtr->numParityCol - 130 layoutPtr->numParityLogCol; 131 layoutPtr->dataSectorsPerStripe = layoutPtr->numDataCol * 132 layoutPtr->sectorsPerStripeUnit; 133 layoutPtr->dataStripeUnitsPerDisk = layoutPtr->stripeUnitsPerDisk; 134 raidPtr->sectorsPerDisk = layoutPtr->stripeUnitsPerDisk * 135 layoutPtr->sectorsPerStripeUnit; 136 137 raidPtr->totalSectors = layoutPtr->stripeUnitsPerDisk * 138 layoutPtr->numDataCol * layoutPtr->sectorsPerStripeUnit; 139 140 /* configure parity log parameters 141 * 142 * parameter comment/constraints 143 * ------------------------------------------- 144 * numParityRegions* all regions (except possibly last) 145 * of equal size 146 * totalInCoreLogCapacity* amount of memory in bytes available 147 * for in-core logs (default 1 MB) 148 * numSectorsPerLog# capacity of an in-core log in sectors 149 * (1 * disk track) 150 * numParityLogs total number of in-core logs, 151 * should be at least numParityRegions 152 * regionLogCapacity size of a region log (except possibly 153 * last one) in sectors 154 * totalLogCapacity total amount of log space in sectors 155 * 156 * where '*' denotes a user settable parameter. 157 * Note that logs are fixed to be the size of a disk track, 158 * value #defined in rf_paritylog.h 159 * 160 */ 161 162 totalLogCapacity = layoutPtr->stripeUnitsPerDisk * layoutPtr->sectorsPerStripeUnit * layoutPtr->numParityLogCol; 163 raidPtr->regionLogCapacity = totalLogCapacity / rf_numParityRegions; 164 if (rf_parityLogDebug) 165 printf("bytes per sector %d\n", raidPtr->bytesPerSector); 166 167 /* reduce fragmentation within a disk region by adjusting the number 168 * of regions in an attempt to allow an integral number of logs to fit 169 * into a disk region */ 170 fragmentation = raidPtr->regionLogCapacity % raidPtr->numSectorsPerLog; 171 if (fragmentation > 0) 172 for (i = 1; i < (raidPtr->numSectorsPerLog / 2); i++) { 173 if (((totalLogCapacity / (rf_numParityRegions + i)) % 174 raidPtr->numSectorsPerLog) < fragmentation) { 175 rf_numParityRegions++; 176 raidPtr->regionLogCapacity = totalLogCapacity / 177 rf_numParityRegions; 178 fragmentation = raidPtr->regionLogCapacity % 179 raidPtr->numSectorsPerLog; 180 } 181 if (((totalLogCapacity / (rf_numParityRegions - i)) % 182 raidPtr->numSectorsPerLog) < fragmentation) { 183 rf_numParityRegions--; 184 raidPtr->regionLogCapacity = totalLogCapacity / 185 rf_numParityRegions; 186 fragmentation = raidPtr->regionLogCapacity % 187 raidPtr->numSectorsPerLog; 188 } 189 } 190 /* ensure integral number of regions per log */ 191 raidPtr->regionLogCapacity = (raidPtr->regionLogCapacity / 192 raidPtr->numSectorsPerLog) * 193 raidPtr->numSectorsPerLog; 194 195 raidPtr->numParityLogs = rf_totalInCoreLogCapacity / 196 (raidPtr->bytesPerSector * raidPtr->numSectorsPerLog); 197 /* to avoid deadlock, must ensure that enough logs exist for each 198 * region to have one simultaneously */ 199 if (raidPtr->numParityLogs < rf_numParityRegions) 200 raidPtr->numParityLogs = rf_numParityRegions; 201 202 /* create region information structs */ 203 printf("Allocating %d bytes for in-core parity region info\n", 204 (int) (rf_numParityRegions * sizeof(RF_RegionInfo_t))); 205 RF_Malloc(raidPtr->regionInfo, 206 (rf_numParityRegions * sizeof(RF_RegionInfo_t)), 207 (RF_RegionInfo_t *)); 208 if (raidPtr->regionInfo == NULL) 209 return (ENOMEM); 210 211 /* last region may not be full capacity */ 212 lastRegionCapacity = raidPtr->regionLogCapacity; 213 while ((rf_numParityRegions - 1) * raidPtr->regionLogCapacity + 214 lastRegionCapacity > totalLogCapacity) 215 lastRegionCapacity = lastRegionCapacity - 216 raidPtr->numSectorsPerLog; 217 218 raidPtr->regionParityRange = raidPtr->sectorsPerDisk / 219 rf_numParityRegions; 220 maxRegionParityRange = raidPtr->regionParityRange; 221 222 /* i can't remember why this line is in the code -wvcii 6/30/95 */ 223 /* if (raidPtr->sectorsPerDisk % rf_numParityRegions > 0) 224 regionParityRange++; */ 225 226 /* build pool of unused parity logs */ 227 printf("Allocating %d bytes for %d parity logs\n", 228 raidPtr->numParityLogs * raidPtr->numSectorsPerLog * 229 raidPtr->bytesPerSector, 230 raidPtr->numParityLogs); 231 RF_Malloc(raidPtr->parityLogBufferHeap, raidPtr->numParityLogs * 232 raidPtr->numSectorsPerLog * raidPtr->bytesPerSector, 233 (caddr_t)); 234 if (raidPtr->parityLogBufferHeap == NULL) 235 return (ENOMEM); 236 lHeapPtr = raidPtr->parityLogBufferHeap; 237 rc = rf_mutex_init(&raidPtr->parityLogPool.mutex); 238 if (rc) { 239 rf_print_unable_to_init_mutex(__FILE__, __LINE__, rc); 240 RF_Free(raidPtr->parityLogBufferHeap, raidPtr->numParityLogs * 241 raidPtr->numSectorsPerLog * raidPtr->bytesPerSector); 242 return (ENOMEM); 243 } 244 for (i = 0; i < raidPtr->numParityLogs; i++) { 245 if (i == 0) { 246 RF_Malloc(raidPtr->parityLogPool.parityLogs, 247 sizeof(RF_ParityLog_t), (RF_ParityLog_t *)); 248 if (raidPtr->parityLogPool.parityLogs == NULL) { 249 RF_Free(raidPtr->parityLogBufferHeap, 250 raidPtr->numParityLogs * 251 raidPtr->numSectorsPerLog * 252 raidPtr->bytesPerSector); 253 return (ENOMEM); 254 } 255 l = raidPtr->parityLogPool.parityLogs; 256 } else { 257 RF_Malloc(l->next, sizeof(RF_ParityLog_t), 258 (RF_ParityLog_t *)); 259 if (l->next == NULL) { 260 RF_Free(raidPtr->parityLogBufferHeap, 261 raidPtr->numParityLogs * 262 raidPtr->numSectorsPerLog * 263 raidPtr->bytesPerSector); 264 for (l = raidPtr->parityLogPool.parityLogs; 265 l; 266 l = next) { 267 next = l->next; 268 if (l->records) 269 RF_Free(l->records, (raidPtr->numSectorsPerLog * sizeof(RF_ParityLogRecord_t))); 270 RF_Free(l, sizeof(RF_ParityLog_t)); 271 } 272 return (ENOMEM); 273 } 274 l = l->next; 275 } 276 l->bufPtr = lHeapPtr; 277 lHeapPtr += raidPtr->numSectorsPerLog * 278 raidPtr->bytesPerSector; 279 RF_Malloc(l->records, (raidPtr->numSectorsPerLog * 280 sizeof(RF_ParityLogRecord_t)), 281 (RF_ParityLogRecord_t *)); 282 if (l->records == NULL) { 283 RF_Free(raidPtr->parityLogBufferHeap, 284 raidPtr->numParityLogs * 285 raidPtr->numSectorsPerLog * 286 raidPtr->bytesPerSector); 287 for (l = raidPtr->parityLogPool.parityLogs; 288 l; 289 l = next) { 290 next = l->next; 291 if (l->records) 292 RF_Free(l->records, 293 (raidPtr->numSectorsPerLog * 294 sizeof(RF_ParityLogRecord_t))); 295 RF_Free(l, sizeof(RF_ParityLog_t)); 296 } 297 return (ENOMEM); 298 } 299 } 300 rc = rf_ShutdownCreate(listp, rf_ShutdownParityLoggingPool, raidPtr); 301 if (rc) { 302 RF_ERRORMSG3("Unable to create shutdown entry file %s line %d rc=%d\n", __FILE__, 303 __LINE__, rc); 304 rf_ShutdownParityLoggingPool(raidPtr); 305 return (rc); 306 } 307 /* build pool of region buffers */ 308 rc = rf_mutex_init(&raidPtr->regionBufferPool.mutex); 309 if (rc) { 310 rf_print_unable_to_init_mutex(__FILE__, __LINE__, rc); 311 return (ENOMEM); 312 } 313 rc = rf_cond_init(&raidPtr->regionBufferPool.cond); 314 if (rc) { 315 rf_print_unable_to_init_cond(__FILE__, __LINE__, rc); 316 rf_mutex_destroy(&raidPtr->regionBufferPool.mutex); 317 return (ENOMEM); 318 } 319 raidPtr->regionBufferPool.bufferSize = raidPtr->regionLogCapacity * 320 raidPtr->bytesPerSector; 321 printf("regionBufferPool.bufferSize %d\n", 322 raidPtr->regionBufferPool.bufferSize); 323 324 /* for now, only one region at a time may be reintegrated */ 325 raidPtr->regionBufferPool.totalBuffers = 1; 326 327 raidPtr->regionBufferPool.availableBuffers = 328 raidPtr->regionBufferPool.totalBuffers; 329 raidPtr->regionBufferPool.availBuffersIndex = 0; 330 raidPtr->regionBufferPool.emptyBuffersIndex = 0; 331 printf("Allocating %d bytes for regionBufferPool\n", 332 (int) (raidPtr->regionBufferPool.totalBuffers * 333 sizeof(caddr_t))); 334 RF_Malloc(raidPtr->regionBufferPool.buffers, 335 raidPtr->regionBufferPool.totalBuffers * sizeof(caddr_t), 336 (caddr_t *)); 337 if (raidPtr->regionBufferPool.buffers == NULL) { 338 rf_mutex_destroy(&raidPtr->regionBufferPool.mutex); 339 rf_cond_destroy(&raidPtr->regionBufferPool.cond); 340 return (ENOMEM); 341 } 342 for (i = 0; i < raidPtr->regionBufferPool.totalBuffers; i++) { 343 printf("Allocating %d bytes for regionBufferPool#%d\n", 344 (int) (raidPtr->regionBufferPool.bufferSize * 345 sizeof(char)), i); 346 RF_Malloc(raidPtr->regionBufferPool.buffers[i], 347 raidPtr->regionBufferPool.bufferSize * sizeof(char), 348 (caddr_t)); 349 if (raidPtr->regionBufferPool.buffers[i] == NULL) { 350 rf_mutex_destroy(&raidPtr->regionBufferPool.mutex); 351 rf_cond_destroy(&raidPtr->regionBufferPool.cond); 352 for (j = 0; j < i; j++) { 353 RF_Free(raidPtr->regionBufferPool.buffers[i], 354 raidPtr->regionBufferPool.bufferSize * 355 sizeof(char)); 356 } 357 RF_Free(raidPtr->regionBufferPool.buffers, 358 raidPtr->regionBufferPool.totalBuffers * 359 sizeof(caddr_t)); 360 return (ENOMEM); 361 } 362 printf("raidPtr->regionBufferPool.buffers[%d] = %lx\n", i, 363 (long) raidPtr->regionBufferPool.buffers[i]); 364 } 365 rc = rf_ShutdownCreate(listp, 366 rf_ShutdownParityLoggingRegionBufferPool, 367 raidPtr); 368 if (rc) { 369 RF_ERRORMSG3("Unable to create shutdown entry file %s line %d rc=%d\n", __FILE__, 370 __LINE__, rc); 371 rf_ShutdownParityLoggingRegionBufferPool(raidPtr); 372 return (rc); 373 } 374 /* build pool of parity buffers */ 375 parityBufferCapacity = maxRegionParityRange; 376 rc = rf_mutex_init(&raidPtr->parityBufferPool.mutex); 377 if (rc) { 378 rf_print_unable_to_init_mutex(__FILE__, __LINE__, rc); 379 return (rc); 380 } 381 rc = rf_cond_init(&raidPtr->parityBufferPool.cond); 382 if (rc) { 383 rf_print_unable_to_init_cond(__FILE__, __LINE__, rc); 384 rf_mutex_destroy(&raidPtr->parityBufferPool.mutex); 385 return (ENOMEM); 386 } 387 raidPtr->parityBufferPool.bufferSize = parityBufferCapacity * 388 raidPtr->bytesPerSector; 389 printf("parityBufferPool.bufferSize %d\n", 390 raidPtr->parityBufferPool.bufferSize); 391 392 /* for now, only one region at a time may be reintegrated */ 393 raidPtr->parityBufferPool.totalBuffers = 1; 394 395 raidPtr->parityBufferPool.availableBuffers = 396 raidPtr->parityBufferPool.totalBuffers; 397 raidPtr->parityBufferPool.availBuffersIndex = 0; 398 raidPtr->parityBufferPool.emptyBuffersIndex = 0; 399 printf("Allocating %d bytes for parityBufferPool of %d units\n", 400 (int) (raidPtr->parityBufferPool.totalBuffers * 401 sizeof(caddr_t)), 402 raidPtr->parityBufferPool.totalBuffers ); 403 RF_Malloc(raidPtr->parityBufferPool.buffers, 404 raidPtr->parityBufferPool.totalBuffers * sizeof(caddr_t), 405 (caddr_t *)); 406 if (raidPtr->parityBufferPool.buffers == NULL) { 407 rf_mutex_destroy(&raidPtr->parityBufferPool.mutex); 408 rf_cond_destroy(&raidPtr->parityBufferPool.cond); 409 return (ENOMEM); 410 } 411 for (i = 0; i < raidPtr->parityBufferPool.totalBuffers; i++) { 412 printf("Allocating %d bytes for parityBufferPool#%d\n", 413 (int) (raidPtr->parityBufferPool.bufferSize * 414 sizeof(char)),i); 415 RF_Malloc(raidPtr->parityBufferPool.buffers[i], 416 raidPtr->parityBufferPool.bufferSize * sizeof(char), 417 (caddr_t)); 418 if (raidPtr->parityBufferPool.buffers == NULL) { 419 rf_mutex_destroy(&raidPtr->parityBufferPool.mutex); 420 rf_cond_destroy(&raidPtr->parityBufferPool.cond); 421 for (j = 0; j < i; j++) { 422 RF_Free(raidPtr->parityBufferPool.buffers[i], 423 raidPtr->regionBufferPool.bufferSize * 424 sizeof(char)); 425 } 426 RF_Free(raidPtr->parityBufferPool.buffers, 427 raidPtr->regionBufferPool.totalBuffers * 428 sizeof(caddr_t)); 429 return (ENOMEM); 430 } 431 printf("parityBufferPool.buffers[%d] = %lx\n", i, 432 (long) raidPtr->parityBufferPool.buffers[i]); 433 } 434 rc = rf_ShutdownCreate(listp, 435 rf_ShutdownParityLoggingParityBufferPool, 436 raidPtr); 437 if (rc) { 438 RF_ERRORMSG3("Unable to create shutdown entry file %s line %d rc=%d\n", __FILE__, 439 __LINE__, rc); 440 rf_ShutdownParityLoggingParityBufferPool(raidPtr); 441 return (rc); 442 } 443 /* initialize parityLogDiskQueue */ 444 rc = rf_create_managed_mutex(listp, 445 &raidPtr->parityLogDiskQueue.mutex); 446 if (rc) { 447 rf_print_unable_to_init_mutex(__FILE__, __LINE__, rc); 448 return (rc); 449 } 450 rc = rf_create_managed_cond(listp, &raidPtr->parityLogDiskQueue.cond); 451 if (rc) { 452 rf_print_unable_to_init_cond(__FILE__, __LINE__, rc); 453 return (rc); 454 } 455 raidPtr->parityLogDiskQueue.flushQueue = NULL; 456 raidPtr->parityLogDiskQueue.reintQueue = NULL; 457 raidPtr->parityLogDiskQueue.bufHead = NULL; 458 raidPtr->parityLogDiskQueue.bufTail = NULL; 459 raidPtr->parityLogDiskQueue.reintHead = NULL; 460 raidPtr->parityLogDiskQueue.reintTail = NULL; 461 raidPtr->parityLogDiskQueue.logBlockHead = NULL; 462 raidPtr->parityLogDiskQueue.logBlockTail = NULL; 463 raidPtr->parityLogDiskQueue.reintBlockHead = NULL; 464 raidPtr->parityLogDiskQueue.reintBlockTail = NULL; 465 raidPtr->parityLogDiskQueue.freeDataList = NULL; 466 raidPtr->parityLogDiskQueue.freeCommonList = NULL; 467 468 rc = rf_ShutdownCreate(listp, 469 rf_ShutdownParityLoggingDiskQueue, 470 raidPtr); 471 if (rc) { 472 RF_ERRORMSG3("Unable to create shutdown entry file %s line %d rc=%d\n", __FILE__, 473 __LINE__, rc); 474 return (rc); 475 } 476 for (i = 0; i < rf_numParityRegions; i++) { 477 rc = rf_mutex_init(&raidPtr->regionInfo[i].mutex); 478 if (rc) { 479 rf_print_unable_to_init_mutex(__FILE__, __LINE__, rc); 480 for (j = 0; j < i; j++) 481 FreeRegionInfo(raidPtr, j); 482 RF_Free(raidPtr->regionInfo, 483 (rf_numParityRegions * 484 sizeof(RF_RegionInfo_t))); 485 return (ENOMEM); 486 } 487 rc = rf_mutex_init(&raidPtr->regionInfo[i].reintMutex); 488 if (rc) { 489 rf_print_unable_to_init_mutex(__FILE__, __LINE__, rc); 490 rf_mutex_destroy(&raidPtr->regionInfo[i].mutex); 491 for (j = 0; j < i; j++) 492 FreeRegionInfo(raidPtr, j); 493 RF_Free(raidPtr->regionInfo, 494 (rf_numParityRegions * 495 sizeof(RF_RegionInfo_t))); 496 return (ENOMEM); 497 } 498 raidPtr->regionInfo[i].reintInProgress = RF_FALSE; 499 raidPtr->regionInfo[i].regionStartAddr = 500 raidPtr->regionLogCapacity * i; 501 raidPtr->regionInfo[i].parityStartAddr = 502 raidPtr->regionParityRange * i; 503 if (i < rf_numParityRegions - 1) { 504 raidPtr->regionInfo[i].capacity = 505 raidPtr->regionLogCapacity; 506 raidPtr->regionInfo[i].numSectorsParity = 507 raidPtr->regionParityRange; 508 } else { 509 raidPtr->regionInfo[i].capacity = 510 lastRegionCapacity; 511 raidPtr->regionInfo[i].numSectorsParity = 512 raidPtr->sectorsPerDisk - 513 raidPtr->regionParityRange * i; 514 if (raidPtr->regionInfo[i].numSectorsParity > 515 maxRegionParityRange) 516 maxRegionParityRange = 517 raidPtr->regionInfo[i].numSectorsParity; 518 } 519 raidPtr->regionInfo[i].diskCount = 0; 520 RF_ASSERT(raidPtr->regionInfo[i].capacity + 521 raidPtr->regionInfo[i].regionStartAddr <= 522 totalLogCapacity); 523 RF_ASSERT(raidPtr->regionInfo[i].parityStartAddr + 524 raidPtr->regionInfo[i].numSectorsParity <= 525 raidPtr->sectorsPerDisk); 526 printf("Allocating %d bytes for region %d\n", 527 (int) (raidPtr->regionInfo[i].capacity * 528 sizeof(RF_DiskMap_t)), i); 529 RF_Malloc(raidPtr->regionInfo[i].diskMap, 530 (raidPtr->regionInfo[i].capacity * 531 sizeof(RF_DiskMap_t)), 532 (RF_DiskMap_t *)); 533 if (raidPtr->regionInfo[i].diskMap == NULL) { 534 rf_mutex_destroy(&raidPtr->regionInfo[i].mutex); 535 rf_mutex_destroy(&raidPtr->regionInfo[i].reintMutex); 536 for (j = 0; j < i; j++) 537 FreeRegionInfo(raidPtr, j); 538 RF_Free(raidPtr->regionInfo, 539 (rf_numParityRegions * 540 sizeof(RF_RegionInfo_t))); 541 return (ENOMEM); 542 } 543 raidPtr->regionInfo[i].loggingEnabled = RF_FALSE; 544 raidPtr->regionInfo[i].coreLog = NULL; 545 } 546 rc = rf_ShutdownCreate(listp, 547 rf_ShutdownParityLoggingRegionInfo, 548 raidPtr); 549 if (rc) { 550 RF_ERRORMSG3("Unable to create shutdown entry file %s line %d rc=%d\n", __FILE__, 551 __LINE__, rc); 552 rf_ShutdownParityLoggingRegionInfo(raidPtr); 553 return (rc); 554 } 555 RF_ASSERT(raidPtr->parityLogDiskQueue.threadState == 0); 556 raidPtr->parityLogDiskQueue.threadState = RF_PLOG_CREATED; 557 rc = RF_CREATE_THREAD(raidPtr->pLogDiskThreadHandle, 558 rf_ParityLoggingDiskManager, raidPtr,"rf_log"); 559 if (rc) { 560 raidPtr->parityLogDiskQueue.threadState = 0; 561 RF_ERRORMSG3("Unable to create parity logging disk thread file %s line %d rc=%d\n", 562 __FILE__, __LINE__, rc); 563 return (ENOMEM); 564 } 565 /* wait for thread to start */ 566 RF_LOCK_MUTEX(raidPtr->parityLogDiskQueue.mutex); 567 while (!(raidPtr->parityLogDiskQueue.threadState & RF_PLOG_RUNNING)) { 568 RF_WAIT_COND(raidPtr->parityLogDiskQueue.cond, 569 raidPtr->parityLogDiskQueue.mutex); 570 } 571 RF_UNLOCK_MUTEX(raidPtr->parityLogDiskQueue.mutex); 572 573 rc = rf_ShutdownCreate(listp, rf_ShutdownParityLogging, raidPtr); 574 if (rc) { 575 RF_ERRORMSG1("Got rc=%d adding parity logging shutdown event\n", rc); 576 rf_ShutdownParityLogging(raidPtr); 577 return (rc); 578 } 579 if (rf_parityLogDebug) { 580 printf(" size of disk log in sectors: %d\n", 581 (int) totalLogCapacity); 582 printf(" total number of parity regions is %d\n", (int) rf_numParityRegions); 583 printf(" nominal sectors of log per parity region is %d\n", (int) raidPtr->regionLogCapacity); 584 printf(" nominal region fragmentation is %d sectors\n", (int) fragmentation); 585 printf(" total number of parity logs is %d\n", raidPtr->numParityLogs); 586 printf(" parity log size is %d sectors\n", raidPtr->numSectorsPerLog); 587 printf(" total in-core log space is %d bytes\n", (int) rf_totalInCoreLogCapacity); 588 } 589 rf_EnableParityLogging(raidPtr); 590 591 return (0); 592 } 593 594 static void 595 FreeRegionInfo( 596 RF_Raid_t * raidPtr, 597 RF_RegionId_t regionID) 598 { 599 RF_LOCK_MUTEX(raidPtr->regionInfo[regionID].mutex); 600 RF_Free(raidPtr->regionInfo[regionID].diskMap, 601 (raidPtr->regionInfo[regionID].capacity * 602 sizeof(RF_DiskMap_t))); 603 if (!rf_forceParityLogReint && raidPtr->regionInfo[regionID].coreLog) { 604 rf_ReleaseParityLogs(raidPtr, 605 raidPtr->regionInfo[regionID].coreLog); 606 raidPtr->regionInfo[regionID].coreLog = NULL; 607 } else { 608 RF_ASSERT(raidPtr->regionInfo[regionID].coreLog == NULL); 609 RF_ASSERT(raidPtr->regionInfo[regionID].diskCount == 0); 610 } 611 RF_UNLOCK_MUTEX(raidPtr->regionInfo[regionID].mutex); 612 rf_mutex_destroy(&raidPtr->regionInfo[regionID].mutex); 613 rf_mutex_destroy(&raidPtr->regionInfo[regionID].reintMutex); 614 } 615 616 617 static void 618 FreeParityLogQueue( 619 RF_Raid_t * raidPtr, 620 RF_ParityLogQueue_t * queue) 621 { 622 RF_ParityLog_t *l1, *l2; 623 624 RF_LOCK_MUTEX(queue->mutex); 625 l1 = queue->parityLogs; 626 while (l1) { 627 l2 = l1; 628 l1 = l2->next; 629 RF_Free(l2->records, (raidPtr->numSectorsPerLog * 630 sizeof(RF_ParityLogRecord_t))); 631 RF_Free(l2, sizeof(RF_ParityLog_t)); 632 } 633 RF_UNLOCK_MUTEX(queue->mutex); 634 rf_mutex_destroy(&queue->mutex); 635 } 636 637 638 static void 639 FreeRegionBufferQueue(RF_RegionBufferQueue_t * queue) 640 { 641 int i; 642 643 RF_LOCK_MUTEX(queue->mutex); 644 if (queue->availableBuffers != queue->totalBuffers) { 645 printf("Attempt to free region queue which is still in use!\n"); 646 RF_ASSERT(0); 647 } 648 for (i = 0; i < queue->totalBuffers; i++) 649 RF_Free(queue->buffers[i], queue->bufferSize); 650 RF_Free(queue->buffers, queue->totalBuffers * sizeof(caddr_t)); 651 RF_UNLOCK_MUTEX(queue->mutex); 652 rf_mutex_destroy(&queue->mutex); 653 } 654 655 static void 656 rf_ShutdownParityLoggingRegionInfo(RF_ThreadArg_t arg) 657 { 658 RF_Raid_t *raidPtr; 659 RF_RegionId_t i; 660 661 raidPtr = (RF_Raid_t *) arg; 662 if (rf_parityLogDebug) { 663 printf("raid%d: ShutdownParityLoggingRegionInfo\n", 664 raidPtr->raidid); 665 } 666 /* free region information structs */ 667 for (i = 0; i < rf_numParityRegions; i++) 668 FreeRegionInfo(raidPtr, i); 669 RF_Free(raidPtr->regionInfo, (rf_numParityRegions * 670 sizeof(raidPtr->regionInfo))); 671 raidPtr->regionInfo = NULL; 672 } 673 674 static void 675 rf_ShutdownParityLoggingPool(RF_ThreadArg_t arg) 676 { 677 RF_Raid_t *raidPtr; 678 679 raidPtr = (RF_Raid_t *) arg; 680 if (rf_parityLogDebug) { 681 printf("raid%d: ShutdownParityLoggingPool\n", raidPtr->raidid); 682 } 683 /* free contents of parityLogPool */ 684 FreeParityLogQueue(raidPtr, &raidPtr->parityLogPool); 685 RF_Free(raidPtr->parityLogBufferHeap, raidPtr->numParityLogs * 686 raidPtr->numSectorsPerLog * raidPtr->bytesPerSector); 687 } 688 689 static void 690 rf_ShutdownParityLoggingRegionBufferPool(RF_ThreadArg_t arg) 691 { 692 RF_Raid_t *raidPtr; 693 694 raidPtr = (RF_Raid_t *) arg; 695 if (rf_parityLogDebug) { 696 printf("raid%d: ShutdownParityLoggingRegionBufferPool\n", 697 raidPtr->raidid); 698 } 699 FreeRegionBufferQueue(&raidPtr->regionBufferPool); 700 } 701 702 static void 703 rf_ShutdownParityLoggingParityBufferPool(RF_ThreadArg_t arg) 704 { 705 RF_Raid_t *raidPtr; 706 707 raidPtr = (RF_Raid_t *) arg; 708 if (rf_parityLogDebug) { 709 printf("raid%d: ShutdownParityLoggingParityBufferPool\n", 710 raidPtr->raidid); 711 } 712 FreeRegionBufferQueue(&raidPtr->parityBufferPool); 713 } 714 715 static void 716 rf_ShutdownParityLoggingDiskQueue(RF_ThreadArg_t arg) 717 { 718 RF_ParityLogData_t *d; 719 RF_CommonLogData_t *c; 720 RF_Raid_t *raidPtr; 721 722 raidPtr = (RF_Raid_t *) arg; 723 if (rf_parityLogDebug) { 724 printf("raid%d: ShutdownParityLoggingDiskQueue\n", 725 raidPtr->raidid); 726 } 727 /* free disk manager stuff */ 728 RF_ASSERT(raidPtr->parityLogDiskQueue.bufHead == NULL); 729 RF_ASSERT(raidPtr->parityLogDiskQueue.bufTail == NULL); 730 RF_ASSERT(raidPtr->parityLogDiskQueue.reintHead == NULL); 731 RF_ASSERT(raidPtr->parityLogDiskQueue.reintTail == NULL); 732 while (raidPtr->parityLogDiskQueue.freeDataList) { 733 d = raidPtr->parityLogDiskQueue.freeDataList; 734 raidPtr->parityLogDiskQueue.freeDataList = 735 raidPtr->parityLogDiskQueue.freeDataList->next; 736 RF_Free(d, sizeof(RF_ParityLogData_t)); 737 } 738 while (raidPtr->parityLogDiskQueue.freeCommonList) { 739 c = raidPtr->parityLogDiskQueue.freeCommonList; 740 rf_mutex_destroy(&c->mutex); 741 raidPtr->parityLogDiskQueue.freeCommonList = 742 raidPtr->parityLogDiskQueue.freeCommonList->next; 743 RF_Free(c, sizeof(RF_CommonLogData_t)); 744 } 745 } 746 747 static void 748 rf_ShutdownParityLogging(RF_ThreadArg_t arg) 749 { 750 RF_Raid_t *raidPtr; 751 752 raidPtr = (RF_Raid_t *) arg; 753 if (rf_parityLogDebug) { 754 printf("raid%d: ShutdownParityLogging\n", raidPtr->raidid); 755 } 756 /* shutdown disk thread */ 757 /* This has the desirable side-effect of forcing all regions to be 758 * reintegrated. This is necessary since all parity log maps are 759 * currently held in volatile memory. */ 760 761 RF_LOCK_MUTEX(raidPtr->parityLogDiskQueue.mutex); 762 raidPtr->parityLogDiskQueue.threadState |= RF_PLOG_TERMINATE; 763 RF_UNLOCK_MUTEX(raidPtr->parityLogDiskQueue.mutex); 764 RF_SIGNAL_COND(raidPtr->parityLogDiskQueue.cond); 765 /* 766 * pLogDiskThread will now terminate when queues are cleared 767 * now wait for it to be done 768 */ 769 RF_LOCK_MUTEX(raidPtr->parityLogDiskQueue.mutex); 770 while (!(raidPtr->parityLogDiskQueue.threadState & RF_PLOG_SHUTDOWN)) { 771 RF_WAIT_COND(raidPtr->parityLogDiskQueue.cond, 772 raidPtr->parityLogDiskQueue.mutex); 773 } 774 RF_UNLOCK_MUTEX(raidPtr->parityLogDiskQueue.mutex); 775 if (rf_parityLogDebug) { 776 printf("raid%d: ShutdownParityLogging done (thread completed)\n", raidPtr->raidid); 777 } 778 } 779 780 int 781 rf_GetDefaultNumFloatingReconBuffersParityLogging(RF_Raid_t * raidPtr) 782 { 783 return (20); 784 } 785 786 RF_HeadSepLimit_t 787 rf_GetDefaultHeadSepLimitParityLogging(RF_Raid_t * raidPtr) 788 { 789 return (10); 790 } 791 /* return the region ID for a given RAID address */ 792 RF_RegionId_t 793 rf_MapRegionIDParityLogging( 794 RF_Raid_t * raidPtr, 795 RF_SectorNum_t address) 796 { 797 RF_RegionId_t regionID; 798 799 /* regionID = address / (raidPtr->regionParityRange * raidPtr->Layout.numDataCol); */ 800 regionID = address / raidPtr->regionParityRange; 801 if (regionID == rf_numParityRegions) { 802 /* last region may be larger than other regions */ 803 regionID--; 804 } 805 RF_ASSERT(address >= raidPtr->regionInfo[regionID].parityStartAddr); 806 RF_ASSERT(address < raidPtr->regionInfo[regionID].parityStartAddr + 807 raidPtr->regionInfo[regionID].numSectorsParity); 808 RF_ASSERT(regionID < rf_numParityRegions); 809 return (regionID); 810 } 811 812 813 /* given a logical RAID sector, determine physical disk address of data */ 814 void 815 rf_MapSectorParityLogging( 816 RF_Raid_t * raidPtr, 817 RF_RaidAddr_t raidSector, 818 RF_RowCol_t * col, 819 RF_SectorNum_t * diskSector, 820 int remap) 821 { 822 RF_StripeNum_t SUID = raidSector / 823 raidPtr->Layout.sectorsPerStripeUnit; 824 /* *col = (SUID % (raidPtr->numCol - 825 * raidPtr->Layout.numParityLogCol)); */ 826 *col = SUID % raidPtr->Layout.numDataCol; 827 *diskSector = (SUID / (raidPtr->Layout.numDataCol)) * 828 raidPtr->Layout.sectorsPerStripeUnit + 829 (raidSector % raidPtr->Layout.sectorsPerStripeUnit); 830 } 831 832 833 /* given a logical RAID sector, determine physical disk address of parity */ 834 void 835 rf_MapParityParityLogging( 836 RF_Raid_t * raidPtr, 837 RF_RaidAddr_t raidSector, 838 RF_RowCol_t * col, 839 RF_SectorNum_t * diskSector, 840 int remap) 841 { 842 RF_StripeNum_t SUID = raidSector / 843 raidPtr->Layout.sectorsPerStripeUnit; 844 845 /* *col = 846 * raidPtr->Layout.numDataCol-(SUID/raidPtr->Layout.numDataCol)%(raidPt 847 * r->numCol - raidPtr->Layout.numParityLogCol); */ 848 *col = raidPtr->Layout.numDataCol; 849 *diskSector = (SUID / (raidPtr->Layout.numDataCol)) * 850 raidPtr->Layout.sectorsPerStripeUnit + 851 (raidSector % raidPtr->Layout.sectorsPerStripeUnit); 852 } 853 854 855 /* given a regionID and sector offset, determine the physical disk address of the parity log */ 856 void 857 rf_MapLogParityLogging( 858 RF_Raid_t * raidPtr, 859 RF_RegionId_t regionID, 860 RF_SectorNum_t regionOffset, 861 RF_RowCol_t * col, 862 RF_SectorNum_t * startSector) 863 { 864 *col = raidPtr->numCol - 1; 865 *startSector = raidPtr->regionInfo[regionID].regionStartAddr + regionOffset; 866 } 867 868 869 /* given a regionID, determine the physical disk address of the logged 870 parity for that region */ 871 void 872 rf_MapRegionParity( 873 RF_Raid_t * raidPtr, 874 RF_RegionId_t regionID, 875 RF_RowCol_t * col, 876 RF_SectorNum_t * startSector, 877 RF_SectorCount_t * numSector) 878 { 879 *col = raidPtr->numCol - 2; 880 *startSector = raidPtr->regionInfo[regionID].parityStartAddr; 881 *numSector = raidPtr->regionInfo[regionID].numSectorsParity; 882 } 883 884 885 /* given a logical RAID address, determine the participating disks in 886 the stripe */ 887 void 888 rf_IdentifyStripeParityLogging( 889 RF_Raid_t * raidPtr, 890 RF_RaidAddr_t addr, 891 RF_RowCol_t ** diskids) 892 { 893 RF_StripeNum_t stripeID = rf_RaidAddressToStripeID(&raidPtr->Layout, 894 addr); 895 RF_ParityLoggingConfigInfo_t *info = (RF_ParityLoggingConfigInfo_t *) 896 raidPtr->Layout.layoutSpecificInfo; 897 *diskids = info->stripeIdentifier[stripeID % raidPtr->numCol]; 898 } 899 900 901 void 902 rf_MapSIDToPSIDParityLogging( 903 RF_RaidLayout_t * layoutPtr, 904 RF_StripeNum_t stripeID, 905 RF_StripeNum_t * psID, 906 RF_ReconUnitNum_t * which_ru) 907 { 908 *which_ru = 0; 909 *psID = stripeID; 910 } 911 912 913 /* select an algorithm for performing an access. Returns two pointers, 914 * one to a function that will return information about the DAG, and 915 * another to a function that will create the dag. 916 */ 917 void 918 rf_ParityLoggingDagSelect( 919 RF_Raid_t * raidPtr, 920 RF_IoType_t type, 921 RF_AccessStripeMap_t * asmp, 922 RF_VoidFuncPtr * createFunc) 923 { 924 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 925 RF_PhysDiskAddr_t *failedPDA = NULL; 926 RF_RowCol_t fcol; 927 RF_RowStatus_t rstat; 928 int prior_recon; 929 930 RF_ASSERT(RF_IO_IS_R_OR_W(type)); 931 932 if (asmp->numDataFailed + asmp->numParityFailed > 1) { 933 RF_ERRORMSG("Multiple disks failed in a single group! Aborting I/O operation.\n"); 934 *createFunc = NULL; 935 return; 936 } else 937 if (asmp->numDataFailed + asmp->numParityFailed == 1) { 938 939 /* if under recon & already reconstructed, redirect 940 * the access to the spare drive and eliminate the 941 * failure indication */ 942 failedPDA = asmp->failedPDAs[0]; 943 fcol = failedPDA->col; 944 rstat = raidPtr->status; 945 prior_recon = (rstat == rf_rs_reconfigured) || ( 946 (rstat == rf_rs_reconstructing) ? 947 rf_CheckRUReconstructed(raidPtr->reconControl->reconMap, failedPDA->startSector) : 0 948 ); 949 if (prior_recon) { 950 RF_RowCol_t oc = failedPDA->col; 951 RF_SectorNum_t oo = failedPDA->startSector; 952 if (layoutPtr->map->flags & 953 RF_DISTRIBUTE_SPARE) { 954 /* redirect to dist spare space */ 955 956 if (failedPDA == asmp->parityInfo) { 957 958 /* parity has failed */ 959 (layoutPtr->map->MapParity) (raidPtr, failedPDA->raidAddress, 960 &failedPDA->col, &failedPDA->startSector, RF_REMAP); 961 962 if (asmp->parityInfo->next) { /* redir 2nd component, 963 * if any */ 964 RF_PhysDiskAddr_t *p = asmp->parityInfo->next; 965 RF_SectorNum_t SUoffs = p->startSector % layoutPtr->sectorsPerStripeUnit; 966 p->col = failedPDA->col; 967 p->startSector = rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, failedPDA->startSector) + 968 SUoffs; /* cheating: 969 * startSector is not 970 * really a RAID address */ 971 } 972 } else 973 if (asmp->parityInfo->next && failedPDA == asmp->parityInfo->next) { 974 RF_ASSERT(0); /* should not ever 975 * happen */ 976 } else { 977 978 /* data has failed */ 979 (layoutPtr->map->MapSector) (raidPtr, failedPDA->raidAddress, 980 &failedPDA->col, &failedPDA->startSector, RF_REMAP); 981 982 } 983 984 } else { 985 /* redirect to dedicated spare space */ 986 987 failedPDA->col = raidPtr->Disks[fcol].spareCol; 988 989 /* the parity may have two distinct 990 * components, both of which may need 991 * to be redirected */ 992 if (asmp->parityInfo->next) { 993 if (failedPDA == asmp->parityInfo) { 994 failedPDA->next->col = failedPDA->col; 995 } else 996 if (failedPDA == asmp->parityInfo->next) { /* paranoid: should never occur */ 997 asmp->parityInfo->col = failedPDA->col; 998 } 999 } 1000 } 1001 1002 RF_ASSERT(failedPDA->col != -1); 1003 1004 if (rf_dagDebug || rf_mapDebug) { 1005 printf("raid%d: Redirected type '%c' c %d o %ld -> c %d o %ld\n", 1006 raidPtr->raidid, type, oc, (long) oo, failedPDA->col, (long) failedPDA->startSector); 1007 } 1008 asmp->numDataFailed = asmp->numParityFailed = 0; 1009 } 1010 } 1011 if (type == RF_IO_TYPE_READ) { 1012 1013 if (asmp->numDataFailed == 0) 1014 *createFunc = (RF_VoidFuncPtr) rf_CreateFaultFreeReadDAG; 1015 else 1016 *createFunc = (RF_VoidFuncPtr) rf_CreateRaidFiveDegradedReadDAG; 1017 1018 } else { 1019 1020 1021 /* if mirroring, always use large writes. If the access 1022 * requires two distinct parity updates, always do a small 1023 * write. If the stripe contains a failure but the access 1024 * does not, do a small write. The first conditional 1025 * (numStripeUnitsAccessed <= numDataCol/2) uses a 1026 * less-than-or-equal rather than just a less-than because 1027 * when G is 3 or 4, numDataCol/2 is 1, and I want 1028 * single-stripe-unit updates to use just one disk. */ 1029 if ((asmp->numDataFailed + asmp->numParityFailed) == 0) { 1030 if (((asmp->numStripeUnitsAccessed <= 1031 (layoutPtr->numDataCol / 2)) && 1032 (layoutPtr->numDataCol != 1)) || 1033 (asmp->parityInfo->next != NULL) || 1034 rf_CheckStripeForFailures(raidPtr, asmp)) { 1035 *createFunc = (RF_VoidFuncPtr) rf_CreateParityLoggingSmallWriteDAG; 1036 } else 1037 *createFunc = (RF_VoidFuncPtr) rf_CreateParityLoggingLargeWriteDAG; 1038 } else 1039 if (asmp->numParityFailed == 1) 1040 *createFunc = (RF_VoidFuncPtr) rf_CreateNonRedundantWriteDAG; 1041 else 1042 if (asmp->numStripeUnitsAccessed != 1 && failedPDA->numSector != layoutPtr->sectorsPerStripeUnit) 1043 *createFunc = NULL; 1044 else 1045 *createFunc = (RF_VoidFuncPtr) rf_CreateDegradedWriteDAG; 1046 } 1047 } 1048 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */ 1049
Indexes created Tue Sep 23 14:10:03 GMT 2025