Home | History | Annotate | Line # | Download | only in raidframe
      1 /*	$NetBSD: rf_disks.c,v 1.99 2026/10/28 21:33:13 tls Exp $	*/
      2 /*-
      3  * Copyright (c) 1999 The NetBSD Foundation, Inc.
      4  * All rights reserved.
      5  *
      6  * This code is derived from software contributed to The NetBSD Foundation
      7  * by Greg Oster
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  *
     18  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     19  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     20  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     21  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     22  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     28  * POSSIBILITY OF SUCH DAMAGE.
     29  */
     30 
     31 /*
     32  * Copyright (c) 1995 Carnegie-Mellon University.
     33  * All rights reserved.
     34  *
     35  * Author: Mark Holland
     36  *
     37  * Permission to use, copy, modify and distribute this software and
     38  * its documentation is hereby granted, provided that both the copyright
     39  * notice and this permission notice appear in all copies of the
     40  * software, derivative works or modified versions, and any portions
     41  * thereof, and that both notices appear in supporting documentation.
     42  *
     43  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
     44  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
     45  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
     46  *
     47  * Carnegie Mellon requests users of this software to return to
     48  *
     49  *  Software Distribution Coordinator  or  Software.Distribution (at) CS.CMU.EDU
     50  *  School of Computer Science
     51  *  Carnegie Mellon University
     52  *  Pittsburgh PA 15213-3890
     53  *
     54  * any improvements or extensions that they make and grant Carnegie the
     55  * rights to redistribute these changes.
     56  */
     57 
     58 /***************************************************************
     59  * rf_disks.c -- code to perform operations on the actual disks
     60  ***************************************************************/
     61 
     62 #include <sys/cdefs.h>
     63 __KERNEL_RCSID(0, "$NetBSD: rf_disks.c,v 1.99 2026/10/28 21:33:13 tls Exp $");
     64 
     65 #include <dev/raidframe/raidframevar.h>
     66 
     67 #include "rf_raid.h"
     68 #include "rf_alloclist.h"
     69 #include "rf_driver.h"
     70 #include "rf_utils.h"
     71 #include "rf_general.h"
     72 #include "rf_options.h"
     73 #include "rf_kintf.h"
     74 #include "rf_netbsd.h"
     75 
     76 #include <sys/param.h>
     77 #include <sys/systm.h>
     78 #include <sys/proc.h>
     79 #include <sys/ioctl.h>
     80 #include <sys/fcntl.h>
     81 #include <sys/vnode.h>
     82 #include <sys/namei.h> /* for pathbuf */
     83 #include <sys/kauth.h>
     84 #include <miscfs/specfs/specdev.h> /* for v_rdev */
     85 
     86 static int rf_AllocDiskStructures(RF_Raid_t *, RF_Config_t *);
     87 static void rf_print_label_status( RF_Raid_t *, int, char *,
     88 				  RF_ComponentLabel_t *);
     89 static int rf_check_label_vitals( RF_Raid_t *, int, int, char *,
     90 				  RF_ComponentLabel_t *, int, int );
     91 
     92 #define DPRINTF6(a,b,c,d,e,f) if (rf_diskDebug) printf(a,b,c,d,e,f)
     93 #define DPRINTF7(a,b,c,d,e,f,g) if (rf_diskDebug) printf(a,b,c,d,e,f,g)
     94 
     95 /**************************************************************************
     96  *
     97  * initialize the disks comprising the array
     98  *
     99  * We want the spare disks to have regular row,col numbers so that we can
    100  * easily substitue a spare for a failed disk.  But, the driver code assumes
    101  * throughout that the array contains numRow by numCol _non-spare_ disks, so
    102  * it's not clear how to fit in the spares.  This is an unfortunate holdover
    103  * from raidSim.  The quick and dirty fix is to make row zero bigger than the
    104  * rest, and put all the spares in it.  This probably needs to get changed
    105  * eventually.
    106  *
    107  **************************************************************************/
    108 
    109 int
    110 rf_ConfigureDisks(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr,
    111 		  RF_Config_t *cfgPtr)
    112 {
    113 	RF_RaidDisk_t *disks;
    114 	RF_SectorCount_t min_numblks = (RF_SectorCount_t) 0x7FFFFFFFFFFFLL;
    115 	RF_RowCol_t c;
    116 	int bs, ret;
    117 	unsigned i, count, foundone = 0, numFailuresThisRow;
    118 	int force;
    119 
    120 	force = cfgPtr->force;
    121 
    122 	ret = rf_AllocDiskStructures(raidPtr, cfgPtr);
    123 	if (ret)
    124 		goto fail;
    125 
    126 	disks = raidPtr->Disks;
    127 
    128 	numFailuresThisRow = 0;
    129 	for (c = 0; c < raidPtr->numCol; c++) {
    130 		ret = rf_ConfigureDisk(raidPtr,
    131 				       &cfgPtr->devnames[0][c][0],
    132 				       &disks[c], c);
    133 
    134 		if (ret)
    135 			goto fail;
    136 
    137 		if (disks[c].status == rf_ds_optimal) {
    138 			ret = raidfetch_component_label(raidPtr, c);
    139 			if (ret)
    140 				goto fail;
    141 
    142 			/* mark it as failed if the label looks bogus... */
    143 			if (!rf_reasonable_label(&raidPtr->raid_cinfo[c].ci_label,0) && !force) {
    144 				disks[c].status = rf_ds_failed;
    145 			}
    146 		}
    147 
    148 		if (disks[c].status != rf_ds_optimal) {
    149 			numFailuresThisRow++;
    150 		} else {
    151 			if (disks[c].numBlocks < min_numblks)
    152 				min_numblks = disks[c].numBlocks;
    153 			DPRINTF6("Disk at col %d: dev %s numBlocks %" PRIu64 " blockSize %d (%ld MB)\n",
    154 				 c, disks[c].devname,
    155 				 disks[c].numBlocks,
    156 				 disks[c].blockSize,
    157 				 (long int) disks[c].numBlocks *
    158 				 disks[c].blockSize / 1024 / 1024);
    159 		}
    160 	}
    161 	/* XXX fix for n-fault tolerant */
    162 	/* XXX this should probably check to see how many failures
    163 	   we can handle for this configuration! */
    164 	if (numFailuresThisRow > 0)
    165 		raidPtr->status = rf_rs_degraded;
    166 
    167 	/* all disks must be the same size & have the same block size, bs must
    168 	 * be a power of 2 */
    169 	bs = 0;
    170 	foundone = 0;
    171 	for (c = 0; c < raidPtr->numCol; c++) {
    172 		if (disks[c].status == rf_ds_optimal) {
    173 			bs = disks[c].blockSize;
    174 			foundone = 1;
    175 			break;
    176 		}
    177 	}
    178 	if (!foundone) {
    179 		RF_ERRORMSG("RAIDFRAME: Did not find any live disks in the array.\n");
    180 		ret = EINVAL;
    181 		goto fail;
    182 	}
    183 	for (count = 0, i = 1; i; i <<= 1)
    184 		if (bs & i)
    185 			count++;
    186 	if (count != 1) {
    187 		RF_ERRORMSG1("Error: block size on disks (%d) must be a power of 2\n", bs);
    188 		ret = EINVAL;
    189 		goto fail;
    190 	}
    191 
    192 	if (rf_CheckLabels( raidPtr, cfgPtr )) {
    193 		printf("raid%d: There were fatal errors\n", raidPtr->raidid);
    194 		if (force != 0) {
    195 			printf("raid%d: Fatal errors being ignored.\n",
    196 			       raidPtr->raidid);
    197 		} else {
    198 			ret = EINVAL;
    199 			goto fail;
    200 		}
    201 	}
    202 
    203 	for (c = 0; c < raidPtr->numCol; c++) {
    204 		if (disks[c].status == rf_ds_optimal) {
    205 			if (disks[c].blockSize != bs) {
    206 				RF_ERRORMSG1("Error: block size of disk at c %d different from disk at c 0\n", c);
    207 				ret = EINVAL;
    208 				goto fail;
    209 			}
    210 			if (disks[c].numBlocks != min_numblks) {
    211 				RF_ERRORMSG2("WARNING: truncating disk at c %d to %d blocks\n",
    212 					     c, (int) min_numblks);
    213 				disks[c].numBlocks = min_numblks;
    214 			}
    215 		}
    216 	}
    217 
    218 	raidPtr->sectorsPerDisk = min_numblks;
    219 	raidPtr->logBytesPerSector = ffs(bs) - 1;
    220 	raidPtr->bytesPerSector = bs;
    221 	raidPtr->sectorMask = bs - 1;
    222 	return (0);
    223 
    224 fail:
    225 
    226 	rf_UnconfigureVnodes( raidPtr );
    227 
    228 	return (ret);
    229 }
    230 
    231 
    232 /****************************************************************************
    233  * set up the data structures describing the spare disks in the array
    234  * recall from the above comment that the spare disk descriptors are stored
    235  * in row zero, which is specially expanded to hold them.
    236  ****************************************************************************/
    237 int
    238 rf_ConfigureSpareDisks(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr,
    239 		       RF_Config_t *cfgPtr)
    240 {
    241 	int     i, ret;
    242 	unsigned int bs;
    243 	RF_RaidDisk_t *disks;
    244 	int     num_spares_done;
    245 
    246 	num_spares_done = 0;
    247 
    248 	/* The space for the spares should have already been allocated by
    249 	 * ConfigureDisks() */
    250 
    251 	disks = &raidPtr->Disks[raidPtr->numCol];
    252 	for (i = 0; i < raidPtr->numSpare; i++) {
    253 		ret = rf_ConfigureDisk(raidPtr, &cfgPtr->spare_names[i][0],
    254 				       &disks[i], raidPtr->numCol + i);
    255 		if (ret)
    256 			goto fail;
    257 		if (disks[i].status != rf_ds_optimal) {
    258 			RF_ERRORMSG1("Warning: spare disk %s failed TUR\n",
    259 				     &cfgPtr->spare_names[i][0]);
    260 		} else {
    261 			disks[i].status = rf_ds_spare;	/* change status to
    262 							 * spare */
    263 			DPRINTF6("Spare Disk %d: dev %s numBlocks %" PRIu64 " blockSize %d (%ld MB)\n", i,
    264 			    disks[i].devname,
    265 			    disks[i].numBlocks, disks[i].blockSize,
    266 			    (long int) disks[i].numBlocks *
    267 				 disks[i].blockSize / 1024 / 1024);
    268 		}
    269 		num_spares_done++;
    270 	}
    271 
    272 	/* check sizes and block sizes on spare disks */
    273 	bs = 1 << raidPtr->logBytesPerSector;
    274 	for (i = 0; i < raidPtr->numSpare; i++) {
    275 		if (disks[i].blockSize != bs) {
    276 			RF_ERRORMSG3("Block size of %d on spare disk %s is not the same as on other disks (%d)\n", disks[i].blockSize, disks[i].devname, bs);
    277 			ret = EINVAL;
    278 			goto fail;
    279 		}
    280 		if (disks[i].numBlocks < raidPtr->sectorsPerDisk) {
    281 			RF_ERRORMSG3("Spare disk %s (%d blocks) is too small to serve as a spare (need %" PRIu64 " blocks)\n",
    282 				     disks[i].devname, disks[i].blockSize,
    283 				     raidPtr->sectorsPerDisk);
    284 			ret = EINVAL;
    285 			goto fail;
    286 		} else
    287 			if (disks[i].numBlocks > raidPtr->sectorsPerDisk) {
    288 				RF_ERRORMSG3("Warning: truncating spare disk %s to %" PRIu64 " blocks (from %" PRIu64 ")\n",
    289 				    disks[i].devname,
    290 				    raidPtr->sectorsPerDisk,
    291 				    disks[i].numBlocks);
    292 
    293 				disks[i].numBlocks = raidPtr->sectorsPerDisk;
    294 			}
    295 	}
    296 
    297 	return (0);
    298 
    299 fail:
    300 
    301 	/* Release the hold on the main components.  We've failed to allocate
    302 	 * a spare, and since we're failing, we need to free things..
    303 
    304 	 XXX failing to allocate a spare is *not* that big of a deal...
    305 	 We *can* survive without it, if need be, esp. if we get hot
    306 	 adding working.
    307 
    308 	 If we don't fail out here, then we need a way to remove this spare...
    309 	 that should be easier to do here than if we are "live"...
    310 
    311 	 */
    312 
    313 	rf_UnconfigureVnodes( raidPtr );
    314 
    315 	return (ret);
    316 }
    317 
    318 static int
    319 rf_AllocDiskStructures(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr)
    320 {
    321 	int ret;
    322 	size_t entries = raidPtr->numCol + RF_MAXSPARE;
    323 
    324 	/* We allocate RF_MAXSPARE on the first row so that we
    325 	   have room to do hot-swapping of spares */
    326 	raidPtr->Disks = RF_MallocAndAdd(
    327 	    entries * sizeof(*raidPtr->Disks), raidPtr->cleanupList);
    328 	if (raidPtr->Disks == NULL) {
    329 		ret = ENOMEM;
    330 		goto fail;
    331 	}
    332 
    333 	/* get space for device specific stuff.. */
    334 	raidPtr->raid_cinfo = RF_MallocAndAdd(
    335 	    entries * sizeof(*raidPtr->raid_cinfo), raidPtr->cleanupList);
    336 	if (raidPtr->raid_cinfo == NULL) {
    337 		ret = ENOMEM;
    338 		goto fail;
    339 	}
    340 
    341 	raidPtr->abortRecon = RF_MallocAndAdd(
    342 	    entries * sizeof(int), raidPtr->cleanupList);
    343 	if (raidPtr->abortRecon == NULL) {
    344 		ret = ENOMEM;
    345 		goto fail;
    346 	}
    347 
    348 
    349 	return(0);
    350 fail:
    351 	rf_UnconfigureVnodes( raidPtr );
    352 
    353 	return(ret);
    354 }
    355 
    356 
    357 /* configure a single disk during auto-configuration at boot */
    358 int
    359 rf_AutoConfigureDisks(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr,
    360 		      RF_AutoConfig_t *auto_config)
    361 {
    362 	RF_RaidDisk_t *disks;
    363 	RF_RaidDisk_t *diskPtr;
    364 	RF_RowCol_t c;
    365 	RF_SectorCount_t min_numblks = (RF_SectorCount_t) 0x7FFFFFFFFFFFLL;
    366 	int bs, ret;
    367 	int numFailuresThisRow;
    368 	RF_AutoConfig_t *ac;
    369 	int parity_good;
    370 	int mod_counter;
    371 	int mod_counter_found;
    372 
    373 #if DEBUG
    374 	printf("Starting autoconfiguration of RAID set...\n");
    375 #endif
    376 
    377 	ret = rf_AllocDiskStructures(raidPtr, cfgPtr);
    378 	if (ret)
    379 		goto fail;
    380 
    381 	disks = raidPtr->Disks;
    382 
    383 	/* assume the parity will be fine.. */
    384 	parity_good = RF_RAID_CLEAN;
    385 
    386 	/* Check for mod_counters that are too low */
    387 	mod_counter_found = 0;
    388 	mod_counter = 0;
    389 	ac = auto_config;
    390 	while(ac!=NULL) {
    391 		if (mod_counter_found==0) {
    392 			mod_counter = ac->clabel->mod_counter;
    393 			mod_counter_found = 1;
    394 		} else {
    395 			if (ac->clabel->mod_counter > mod_counter) {
    396 				mod_counter = ac->clabel->mod_counter;
    397 			}
    398 		}
    399 		ac->flag = 0; /* clear the general purpose flag */
    400 		ac = ac->next;
    401 	}
    402 
    403 	bs = 0;
    404 
    405 	numFailuresThisRow = 0;
    406 	for (c = 0; c < raidPtr->numCol; c++) {
    407 		diskPtr = &disks[c];
    408 
    409 		/* find this row/col in the autoconfig */
    410 #if DEBUG
    411 		printf("Looking for %d in autoconfig\n",c);
    412 #endif
    413 		ac = auto_config;
    414 		while(ac!=NULL) {
    415 			if (ac->clabel==NULL) {
    416 				/* big-time bad news. */
    417 				goto fail;
    418 			}
    419 			if ((ac->clabel->column == c) &&
    420 			    (ac->clabel->mod_counter == mod_counter)) {
    421 				/* it's this one... */
    422 				/* flag it as 'used', so we don't
    423 				   free it later. */
    424 				ac->flag = 1;
    425 #if DEBUG
    426 				printf("Found: %s at %d\n",
    427 				       ac->devname,c);
    428 #endif
    429 
    430 				break;
    431 			}
    432 			ac=ac->next;
    433 		}
    434 
    435 		if (ac==NULL) {
    436 			/* we didn't find an exact match with a
    437 			   correct mod_counter above... can we find
    438 			   one with an incorrect mod_counter to use
    439 			   instead?  (this one, if we find it, will be
    440 			   marked as failed once the set configures)
    441 			*/
    442 			RF_AutoConfig_t *best_ac = NULL;
    443 
    444 			ac = auto_config;
    445 			while(ac!=NULL) {
    446 				if (ac->clabel==NULL) {
    447 					/* big-time bad news. */
    448 					goto fail;
    449 				}
    450 				if (ac->clabel->column == c) {
    451 					if (best_ac == NULL ||
    452 					    ac->clabel->mod_counter >
    453 					    best_ac->clabel->mod_counter) {
    454 						best_ac = ac;
    455 					}
    456 				}
    457 				ac=ac->next;
    458 			}
    459 			if (best_ac != NULL) {
    460 				ac = best_ac;
    461 				/* it's this one...
    462 				   flag it as 'used', so we
    463 				   don't free it later. */
    464 				ac->flag = 1;
    465 #if DEBUG
    466 				printf("Found(low mod_counter): %s at %d\n",
    467 				       ac->devname,c);
    468 #endif
    469 			}
    470 		}
    471 
    472 
    473 
    474 		if (ac!=NULL) {
    475 			/* Found it.  Configure it.. */
    476 			diskPtr->blockSize = ac->clabel->blockSize;
    477 			diskPtr->numBlocks =
    478 			    rf_component_label_numblocks(ac->clabel);
    479 			diskPtr->partitionSize =
    480 			    rf_component_label_partitionsize(ac->clabel);
    481 			/* Note: rf_protectedSectors is already
    482 			   factored into numBlocks here */
    483 			raidPtr->raid_cinfo[c].ci_vp = ac->vp;
    484 			raidPtr->raid_cinfo[c].ci_dev = ac->dev;
    485 
    486 			memcpy(raidget_component_label(raidPtr, c),
    487 			    ac->clabel, sizeof(*ac->clabel));
    488 			snprintf(diskPtr->devname, sizeof(diskPtr->devname),
    489 			    "/dev/%s", ac->devname);
    490 
    491 			/* note the fact that this component was
    492 			   autoconfigured.  You'll need this info
    493 			   later.  Trust me :) */
    494 			diskPtr->auto_configured = 1;
    495 			diskPtr->dev = ac->dev;
    496 
    497 			/*
    498 			 * we allow the user to specify that
    499 			 * only a fraction of the disks should
    500 			 * be used this is just for debug: it
    501 			 * speeds up the parity scan
    502 			 */
    503 
    504 			diskPtr->numBlocks = diskPtr->numBlocks *
    505 				rf_sizePercentage / 100;
    506 
    507 			/* XXX these will get set multiple times,
    508 			   but since we're autoconfiguring, they'd
    509 			   better be always the same each time!
    510 			   If not, this is the least of your worries */
    511 
    512 			bs = diskPtr->blockSize;
    513 			min_numblks = diskPtr->numBlocks;
    514 
    515 			/* this gets done multiple times, but that's
    516 			   fine -- the serial number will be the same
    517 			   for all components, guaranteed */
    518 			raidPtr->serial_number = ac->clabel->serial_number;
    519 			/* check the last time the label was modified */
    520 
    521 			if (ac->clabel->mod_counter != mod_counter) {
    522 				/* Even though we've filled in all of
    523 				   the above, we don't trust this
    524 				   component since its modification
    525 				   counter is not in sync with the
    526 				   rest, and we really consider it to
    527 				   be failed.  */
    528 				disks[c].status = rf_ds_failed;
    529 				numFailuresThisRow++;
    530 			} else {
    531 				if (ac->clabel->clean != RF_RAID_CLEAN) {
    532 					parity_good = RF_RAID_DIRTY;
    533 				}
    534 			}
    535 		} else {
    536 			/* Didn't find it at all!!  Component must
    537 			   really be dead */
    538 			disks[c].status = rf_ds_failed;
    539 			snprintf(disks[c].devname, sizeof(disks[c].devname),
    540 			    "component%d", c);
    541 			numFailuresThisRow++;
    542 		}
    543 	}
    544 	/* XXX fix for n-fault tolerant */
    545 	/* XXX this should probably check to see how many failures
    546 	   we can handle for this configuration! */
    547 	if (numFailuresThisRow > 0) {
    548 		raidPtr->status = rf_rs_degraded;
    549 		raidPtr->numFailures = numFailuresThisRow;
    550 	}
    551 
    552 	/*
    553 	 * Attach any remaining components that were not used in the active
    554 	 * array (i.e. components that were superseded in their column by a
    555 	 * spare with a higher mod_counter) as rf_ds_spared so their vnodes
    556 	 * remain open and they cannot form a rogue RAID set during
    557 	 * recursive-RAID rescans.
    558 	 */
    559 	ac = auto_config;
    560 	while(ac!=NULL) {
    561 		if (ac->flag == 0) {
    562 			if (raidPtr->numSpare < RF_MAXSPARE) {
    563 				RF_RowCol_t sparecol =
    564 				    raidPtr->numCol + raidPtr->numSpare;
    565 
    566 				diskPtr = &disks[sparecol];
    567 				diskPtr->blockSize = ac->clabel->blockSize;
    568 				diskPtr->numBlocks =
    569 				    rf_component_label_numblocks(ac->clabel);
    570 				diskPtr->partitionSize =
    571 				    rf_component_label_partitionsize(
    572 					ac->clabel);
    573 				raidPtr->raid_cinfo[sparecol].ci_vp = ac->vp;
    574 				raidPtr->raid_cinfo[sparecol].ci_dev = ac->dev;
    575 
    576 				memcpy(raidget_component_label(raidPtr,
    577 				    sparecol), ac->clabel,
    578 				    sizeof(*ac->clabel));
    579 				snprintf(diskPtr->devname,
    580 				    sizeof(diskPtr->devname),
    581 				    "/dev/%s", ac->devname);
    582 				diskPtr->auto_configured = 1;
    583 				diskPtr->dev = ac->dev;
    584 				diskPtr->numBlocks = diskPtr->numBlocks *
    585 				    rf_sizePercentage / 100;
    586 				diskPtr->status = rf_ds_spared;
    587 
    588 				printf("raid%d: attaching spared component %s "
    589 				    "(column %d, mod_counter %d < %d)\n",
    590 				    raidPtr->raidid, diskPtr->devname,
    591 				    ac->clabel->column,
    592 				    ac->clabel->mod_counter, mod_counter);
    593 
    594 				ac->flag = 1;
    595 				raidPtr->numSpare++;
    596 				raidPtr->maxQueue = raidPtr->numSpare;
    597 			} else {
    598 				vn_lock(ac->vp, LK_EXCLUSIVE | LK_RETRY);
    599 				VOP_CLOSE(ac->vp, FREAD | FWRITE, NOCRED);
    600 				vput(ac->vp);
    601 				ac->vp = NULL;
    602 #if DEBUG
    603 				printf("Released %s from auto-config set.\n",
    604 				       ac->devname);
    605 #endif
    606 			}
    607 		}
    608 		ac = ac->next;
    609 	}
    610 
    611 	raidPtr->mod_counter = mod_counter;
    612 
    613 	/* note the state of the parity, if any */
    614 	raidPtr->parity_good = parity_good;
    615 	raidPtr->sectorsPerDisk = min_numblks;
    616 	raidPtr->logBytesPerSector = ffs(bs) - 1;
    617 	raidPtr->bytesPerSector = bs;
    618 	raidPtr->sectorMask = bs - 1;
    619 	return (0);
    620 
    621 fail:
    622 
    623 	rf_UnconfigureVnodes( raidPtr );
    624 
    625 	return (ret);
    626 
    627 }
    628 
    629 /* configure a single disk in the array */
    630 int
    631 rf_ConfigureDisk(RF_Raid_t *raidPtr, char *bf, RF_RaidDisk_t *diskPtr,
    632 		 RF_RowCol_t col)
    633 {
    634 	char   *p;
    635 	struct pathbuf *pb;
    636 	struct vnode *vp;
    637 	int     error;
    638 
    639 	p = rf_find_non_white(bf);
    640 	if (p[strlen(p) - 1] == '\n') {
    641 		/* strip off the newline */
    642 		p[strlen(p) - 1] = '\0';
    643 	}
    644 	(void) strcpy(diskPtr->devname, p);
    645 
    646 	/* Let's start by claiming the component is fine and well... */
    647 	diskPtr->status = rf_ds_optimal;
    648 
    649 	raidPtr->raid_cinfo[col].ci_vp = NULL;
    650 	raidPtr->raid_cinfo[col].ci_dev = 0;
    651 
    652 	if (!strcmp("absent", diskPtr->devname)) {
    653 		printf("Ignoring missing component at column %d\n", col);
    654 		snprintf(diskPtr->devname, sizeof(diskPtr->devname),
    655 		    "component%d", col);
    656 		diskPtr->status = rf_ds_failed;
    657 		return (0);
    658 	}
    659 
    660 	pb = pathbuf_create(diskPtr->devname);
    661 	if (pb == NULL) {
    662 		printf("pathbuf_create for device: %s failed!\n",
    663 		       diskPtr->devname);
    664 		return ENOMEM;
    665 	}
    666 	error = vn_bdev_openpath(pb, &vp, curlwp);
    667 	pathbuf_destroy(pb);
    668 	if (error) {
    669 		printf("open device: '%s' failed: %d\n", diskPtr->devname, error);
    670 		if (error == ENXIO) {
    671 			/* the component isn't there... must be dead :-( */
    672 			diskPtr->status = rf_ds_failed;
    673 			return 0;
    674 		} else {
    675 			return (error);
    676 		}
    677 	}
    678 
    679 	if ((error = rf_getdisksize(vp, diskPtr)) != 0)
    680 		return (error);
    681 
    682 	/*
    683 	 * If this raidPtr's bytesPerSector is zero, fill it in with this
    684 	 * components blockSize.  This will give us something to work with
    685 	 * initially, and if it is wrong, we'll get errors later.
    686 	 */
    687 	if (raidPtr->bytesPerSector == 0)
    688 		raidPtr->bytesPerSector = diskPtr->blockSize;
    689 
    690 	if (diskPtr->status == rf_ds_optimal) {
    691 		raidPtr->raid_cinfo[col].ci_vp = vp;
    692 		raidPtr->raid_cinfo[col].ci_dev = vp->v_rdev;
    693 
    694 		/* This component was not automatically configured */
    695 		diskPtr->auto_configured = 0;
    696 		diskPtr->dev = vp->v_rdev;
    697 
    698 		/* we allow the user to specify that only a fraction of the
    699 		 * disks should be used this is just for debug:  it speeds up
    700 		 * the parity scan */
    701 		diskPtr->numBlocks = diskPtr->numBlocks *
    702 			rf_sizePercentage / 100;
    703 	}
    704 	return (0);
    705 }
    706 
    707 static void
    708 rf_print_label_status(RF_Raid_t *raidPtr, int column, char *dev_name,
    709 		      RF_ComponentLabel_t *ci_label)
    710 {
    711 
    712 	printf("raid%d: Component %s being configured at col: %d\n",
    713 	       raidPtr->raidid, dev_name, column );
    714 	printf("         Column: %d Num Columns: %d\n",
    715 	       ci_label->column,
    716 	       ci_label->num_columns);
    717 	printf("         Version: %d Serial Number: %d Mod Counter: %d\n",
    718 	       ci_label->version, ci_label->serial_number,
    719 	       ci_label->mod_counter);
    720 	printf("         Clean: %s Status: %d\n",
    721 	       ci_label->clean ? "Yes" : "No", ci_label->status );
    722 }
    723 
    724 static int rf_check_label_vitals(RF_Raid_t *raidPtr, int row, int column,
    725 				 char *dev_name, RF_ComponentLabel_t *ci_label,
    726 				 int serial_number, int mod_counter)
    727 {
    728 	int fatal_error = 0;
    729 
    730 	if (serial_number != ci_label->serial_number) {
    731 		printf("%s has a different serial number: %d %d\n",
    732 		       dev_name, serial_number, ci_label->serial_number);
    733 		fatal_error = 1;
    734 	}
    735 	if (mod_counter != ci_label->mod_counter) {
    736 		printf("%s has a different modification count: %d %d\n",
    737 		       dev_name, mod_counter, ci_label->mod_counter);
    738 	}
    739 
    740 	if (row != ci_label->row) {
    741 		printf("Row out of alignment for: %s\n", dev_name);
    742 		fatal_error = 1;
    743 	}
    744 	if (column != ci_label->column) {
    745 		printf("Column out of alignment for: %s\n", dev_name);
    746 		fatal_error = 1;
    747 	}
    748 	if (raidPtr->numCol != ci_label->num_columns) {
    749 		printf("Number of columns do not match for: %s\n", dev_name);
    750 		fatal_error = 1;
    751 	}
    752 	if (ci_label->clean == 0) {
    753 		/* it's not clean, but that's not fatal */
    754 		printf("%s is not clean!\n", dev_name);
    755 	}
    756 	return(fatal_error);
    757 }
    758 
    759 
    760 static void
    761 rf_handle_hosed(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr, int hosed_column,
    762     int again)
    763 {
    764 	printf("Hosed component: %s\n", &cfgPtr->devnames[0][hosed_column][0]);
    765 	if (cfgPtr->force)
    766 		return;
    767 
    768 	/* we'll fail this component, as if there are
    769 	   other major errors, we aren't forcing things
    770 	   and we'll abort the config anyways */
    771 	if (again && raidPtr->Disks[hosed_column].status == rf_ds_failed)
    772 		return;
    773 
    774 	raidPtr->Disks[hosed_column].status = rf_ds_failed;
    775 	raidPtr->numFailures++;
    776 	raidPtr->status = rf_rs_degraded;
    777 }
    778 
    779 /*
    780 
    781    rf_CheckLabels() - check all the component labels for consistency.
    782    Return an error if there is anything major amiss.
    783 
    784  */
    785 
    786 int
    787 rf_CheckLabels(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr)
    788 {
    789 	int c;
    790 	char *dev_name;
    791 	RF_ComponentLabel_t *ci_label;
    792 	int serial_number = 0;
    793 	int mod_number = 0;
    794 	int fatal_error = 0;
    795 	int mod_values[4];
    796 	int mod_count[4];
    797 	int ser_values[4];
    798 	int ser_count[4];
    799 	int num_ser;
    800 	int num_mod;
    801 	int i;
    802 	int found;
    803 	int hosed_column;
    804 	int too_fatal;
    805 	int parity_good;
    806 
    807 	hosed_column = -1;
    808 	too_fatal = 0;
    809 
    810 	/*
    811 	   We're going to try to be a little intelligent here.  If one
    812 	   component's label is bogus, and we can identify that it's the
    813 	   *only* one that's gone, we'll mark it as "failed" and allow
    814 	   the configuration to proceed.  This will be the *only* case
    815 	   that we'll proceed if there would be (otherwise) fatal errors.
    816 
    817 	   Basically we simply keep a count of how many components had
    818 	   what serial number.  If all but one agree, we simply mark
    819 	   the disagreeing component as being failed, and allow
    820 	   things to come up "normally".
    821 
    822 	   We do this first for serial numbers, and then for "mod_counter".
    823 
    824 	 */
    825 
    826 	num_ser = 0;
    827 	num_mod = 0;
    828 
    829 	ser_values[0] = ser_values[1] = ser_values[2] = ser_values[3] = 0;
    830 	ser_count[0] = ser_count[1] = ser_count[2] = ser_count[3] = 0;
    831 	mod_values[0] = mod_values[1] = mod_values[2] = mod_values[3] = 0;
    832 	mod_count[0] = mod_count[1] = mod_count[2] = mod_count[3] = 0;
    833 
    834 	for (c = 0; c < raidPtr->numCol; c++) {
    835 		if (raidPtr->Disks[c].status != rf_ds_optimal)
    836 			continue;
    837 		ci_label = raidget_component_label(raidPtr, c);
    838 		found=0;
    839 		for(i=0;i<num_ser;i++) {
    840 			if (ser_values[i] == ci_label->serial_number) {
    841 				ser_count[i]++;
    842 				found=1;
    843 				break;
    844 			}
    845 		}
    846 		if (!found) {
    847 			ser_values[num_ser] = ci_label->serial_number;
    848 			ser_count[num_ser] = 1;
    849 			num_ser++;
    850 			if (num_ser>2) {
    851 				fatal_error = 1;
    852 				break;
    853 			}
    854 		}
    855 		found=0;
    856 		for(i=0;i<num_mod;i++) {
    857 			if (mod_values[i] == ci_label->mod_counter) {
    858 				mod_count[i]++;
    859 				found=1;
    860 				break;
    861 			}
    862 		}
    863 		if (!found) {
    864 			mod_values[num_mod] = ci_label->mod_counter;
    865 			mod_count[num_mod] = 1;
    866 			num_mod++;
    867 			if (num_mod>2) {
    868 				fatal_error = 1;
    869 				break;
    870 			}
    871 		}
    872 	}
    873 #if DEBUG
    874 	printf("raid%d: Summary of serial numbers:\n", raidPtr->raidid);
    875 	for(i=0;i<num_ser;i++) {
    876 		printf("%d %d\n", ser_values[i], ser_count[i]);
    877 	}
    878 	printf("raid%d: Summary of mod counters:\n", raidPtr->raidid);
    879 	for(i=0;i<num_mod;i++) {
    880 		printf("%d %d\n", mod_values[i], mod_count[i]);
    881 	}
    882 #endif
    883 	serial_number = ser_values[0];
    884 	if (num_ser == 2) {
    885 		if ((ser_count[0] == 1) || (ser_count[1] == 1)) {
    886 			/* Locate the maverick component */
    887 			if (ser_count[1] > ser_count[0]) {
    888 				serial_number = ser_values[1];
    889 			}
    890 
    891 			for (c = 0; c < raidPtr->numCol; c++) {
    892 				if (raidPtr->Disks[c].status != rf_ds_optimal)
    893 					continue;
    894 				ci_label = raidget_component_label(raidPtr, c);
    895 				if (serial_number != ci_label->serial_number) {
    896 					hosed_column = c;
    897 					break;
    898 				}
    899 			}
    900 			if (hosed_column != -1)
    901 				rf_handle_hosed(raidPtr, cfgPtr, hosed_column,
    902 				    0);
    903 		} else {
    904 			too_fatal = 1;
    905 		}
    906 		if (cfgPtr->parityConfig == '0') {
    907 			/* We've identified two different serial numbers.
    908 			   RAID 0 can't cope with that, so we'll punt */
    909 			too_fatal = 1;
    910 		}
    911 
    912 	}
    913 
    914 	/* record the serial number for later.  If we bail later, setting
    915 	   this doesn't matter, otherwise we've got the best guess at the
    916 	   correct serial number */
    917 	raidPtr->serial_number = serial_number;
    918 
    919 	mod_number = mod_values[0];
    920 	if (num_mod == 2) {
    921 		if ((mod_count[0] == 1) || (mod_count[1] == 1)) {
    922 			/* Locate the maverick component */
    923 			if (mod_count[1] > mod_count[0]) {
    924 				mod_number = mod_values[1];
    925 			} else if (mod_count[1] < mod_count[0]) {
    926 				mod_number = mod_values[0];
    927 			} else {
    928 				/* counts of different modification values
    929 				   are the same.   Assume greater value is
    930 				   the correct one, all other things
    931 				   considered */
    932 				if (mod_values[0] > mod_values[1]) {
    933 					mod_number = mod_values[0];
    934 				} else {
    935 					mod_number = mod_values[1];
    936 				}
    937 
    938 			}
    939 
    940 			for (c = 0; c < raidPtr->numCol; c++) {
    941 				if (raidPtr->Disks[c].status != rf_ds_optimal)
    942 					continue;
    943 
    944 				ci_label = raidget_component_label(raidPtr, c);
    945 				if (mod_number != ci_label->mod_counter) {
    946 					if (hosed_column == c) {
    947 						/* same one.  Can
    948 						   deal with it.  */
    949 					} else {
    950 						hosed_column = c;
    951 						if (num_ser != 1) {
    952 							too_fatal = 1;
    953 							break;
    954 						}
    955 					}
    956 				}
    957 			}
    958 			if (hosed_column != -1)
    959 				rf_handle_hosed(raidPtr, cfgPtr, hosed_column,
    960 				    1);
    961 		} else {
    962 			too_fatal = 1;
    963 		}
    964 		if (cfgPtr->parityConfig == '0') {
    965 			/* We've identified two different mod counters.
    966 			   RAID 0 can't cope with that, so we'll punt */
    967 			too_fatal = 1;
    968 		}
    969 	}
    970 
    971 	raidPtr->mod_counter = mod_number;
    972 
    973 	if (too_fatal) {
    974 		/* we've had both a serial number mismatch, and a mod_counter
    975 		   mismatch -- and they involved two different components!!
    976 		   Bail -- make things fail so that the user must force
    977 		   the issue... */
    978 		hosed_column = -1;
    979 		fatal_error = 1;
    980 	}
    981 
    982 	if (num_ser > 2) {
    983 		printf("raid%d: Too many different serial numbers!\n",
    984 		       raidPtr->raidid);
    985 		fatal_error = 1;
    986 	}
    987 
    988 	if (num_mod > 2) {
    989 		printf("raid%d: Too many different mod counters!\n",
    990 		       raidPtr->raidid);
    991 		fatal_error = 1;
    992 	}
    993 
    994         for (c = 0; c < raidPtr->numCol; c++) {
    995 		if (raidPtr->Disks[c].status != rf_ds_optimal) {
    996 			hosed_column = c;
    997 			break;
    998 		}
    999 	}
   1000 
   1001 	/* we start by assuming the parity will be good, and flee from
   1002 	   that notion at the slightest sign of trouble */
   1003 
   1004 	parity_good = RF_RAID_CLEAN;
   1005 
   1006 	for (c = 0; c < raidPtr->numCol; c++) {
   1007 		dev_name = &cfgPtr->devnames[0][c][0];
   1008 		ci_label = raidget_component_label(raidPtr, c);
   1009 
   1010 		if (c == hosed_column) {
   1011 			printf("raid%d: Ignoring %s\n",
   1012 			       raidPtr->raidid, dev_name);
   1013 		} else {
   1014 			rf_print_label_status( raidPtr, c, dev_name, ci_label);
   1015 			if (rf_check_label_vitals( raidPtr, 0, c,
   1016 						   dev_name, ci_label,
   1017 						   serial_number,
   1018 						   mod_number )) {
   1019 				fatal_error = 1;
   1020 			}
   1021 			if (ci_label->clean != RF_RAID_CLEAN) {
   1022 				parity_good = RF_RAID_DIRTY;
   1023 			}
   1024 		}
   1025 	}
   1026 
   1027 	if (fatal_error) {
   1028 		parity_good = RF_RAID_DIRTY;
   1029 	}
   1030 
   1031 	/* we note the state of the parity */
   1032 	raidPtr->parity_good = parity_good;
   1033 
   1034 	return(fatal_error);
   1035 }
   1036 
   1037 int
   1038 rf_add_hot_spare(RF_Raid_t *raidPtr, RF_SingleComponent_t *sparePtr)
   1039 {
   1040 	RF_DiskQueue_t *spareQueues;
   1041 	RF_RaidDisk_t *disks;
   1042 	RF_ComponentLabel_t *clabel;
   1043 	char *p;
   1044 	int ret;
   1045 	unsigned int bs;
   1046 	int spare_number;
   1047 	bool reusing = false;
   1048 
   1049 	ret=0;
   1050 
   1051 	rf_lock_mutex2(raidPtr->mutex);
   1052 	while (raidPtr->changing_components == 1) {
   1053 		rf_wait_cond2(raidPtr->changing_components_cv, raidPtr->mutex);
   1054 	}
   1055 	raidPtr->changing_components = 1;
   1056 	rf_unlock_mutex2(raidPtr->mutex);
   1057 
   1058 	/* the beginning of the spares... */
   1059 	disks = &raidPtr->Disks[raidPtr->numCol];
   1060 
   1061 	p = rf_find_non_white(sparePtr->component_name);
   1062 	if (p[strlen(p) - 1] == '\n') {
   1063 		p[strlen(p) - 1] = '\0';
   1064 	}
   1065 
   1066 	spare_number = -1;
   1067 	for (int i = 0; i < raidPtr->numSpare; i++) {
   1068 		if (strcmp(disks[i].devname, p) == 0) {
   1069 			spare_number = i;
   1070 			reusing = true;
   1071 			break;
   1072 		}
   1073 	}
   1074 
   1075 	if (reusing) {
   1076 		struct vnode *vp;
   1077 		int acd;
   1078 
   1079 		if (disks[spare_number].status != rf_ds_spared &&
   1080 		    disks[spare_number].status != rf_ds_failed) {
   1081 			ret = EBUSY;
   1082 			goto out;
   1083 		}
   1084 		rf_SuspendNewRequestsAndWait(raidPtr);
   1085 		vp = raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp;
   1086 		acd = disks[spare_number].auto_configured;
   1087 		raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp =
   1088 		    NULL;
   1089 		raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_dev = 0;
   1090 		disks[spare_number].auto_configured = 0;
   1091 		rf_close_component(raidPtr, vp, acd);
   1092 		rf_ResumeNewRequests(raidPtr);
   1093 	} else {
   1094 		if (raidPtr->numSpare >= RF_MAXSPARE) {
   1095 			RF_ERRORMSG1("Too many spares: %d\n",
   1096 			    raidPtr->numSpare);
   1097 			ret = EINVAL;
   1098 			goto out;
   1099 		}
   1100 		spare_number = raidPtr->numSpare;
   1101 	}
   1102 
   1103 	ret = rf_ConfigureDisk(raidPtr, sparePtr->component_name,
   1104 			       &disks[spare_number],
   1105 			       raidPtr->numCol + spare_number);
   1106 
   1107 	if (ret)
   1108 		goto fail;
   1109 	if (disks[spare_number].status != rf_ds_optimal) {
   1110 		RF_ERRORMSG1("Warning: spare disk %s failed TUR\n",
   1111 			     sparePtr->component_name);
   1112 		rf_close_component(raidPtr, raidPtr->raid_cinfo[raidPtr->numCol+spare_number].ci_vp, 0);
   1113 		raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp =
   1114 		    NULL;
   1115 		ret=EINVAL;
   1116 		goto fail;
   1117 	} else {
   1118 		disks[spare_number].status = rf_ds_spare;
   1119 		DPRINTF6("Spare Disk %d: dev %s numBlocks %" PRIu64 " blockSize %d (%ld MB)\n",
   1120 			 spare_number,
   1121 			 disks[spare_number].devname,
   1122 			 disks[spare_number].numBlocks,
   1123 			 disks[spare_number].blockSize,
   1124 			 (long int) disks[spare_number].numBlocks *
   1125 			 disks[spare_number].blockSize / 1024 / 1024);
   1126 	}
   1127 
   1128 
   1129 	/* check sizes and block sizes on the spare disk */
   1130 	bs = 1 << raidPtr->logBytesPerSector;
   1131 	if (disks[spare_number].blockSize != bs) {
   1132 		RF_ERRORMSG3("Block size of %d on spare disk %s is not the same as on other disks (%d)\n", disks[spare_number].blockSize, disks[spare_number].devname, bs);
   1133 		rf_close_component(raidPtr, raidPtr->raid_cinfo[raidPtr->numCol+spare_number].ci_vp, 0);
   1134 		raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp =
   1135 		    NULL;
   1136 		ret = EINVAL;
   1137 		goto fail;
   1138 	}
   1139 	if (disks[spare_number].numBlocks < raidPtr->sectorsPerDisk) {
   1140 		RF_ERRORMSG3("Spare disk %s (%d blocks) is too small to serve as a spare (need %" PRIu64 " blocks)\n",
   1141 			     disks[spare_number].devname,
   1142 			     disks[spare_number].blockSize,
   1143 			     raidPtr->sectorsPerDisk);
   1144 		rf_close_component(raidPtr, raidPtr->raid_cinfo[raidPtr->numCol+spare_number].ci_vp, 0);
   1145 		raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp =
   1146 		    NULL;
   1147 		ret = EINVAL;
   1148 		goto fail;
   1149 	} else {
   1150 		if (disks[spare_number].numBlocks >
   1151 		    raidPtr->sectorsPerDisk) {
   1152 			RF_ERRORMSG3("Warning: truncating spare disk %s to %" PRIu64 " blocks (from %" PRIu64 ")\n",
   1153 			    disks[spare_number].devname,
   1154 			    raidPtr->sectorsPerDisk,
   1155 			    disks[spare_number].numBlocks);
   1156 
   1157 			disks[spare_number].numBlocks = raidPtr->sectorsPerDisk;
   1158 		}
   1159 	}
   1160 
   1161 	/*
   1162 	 * Clear any stale component label on the spare and verify that
   1163 	 * the disk is writable before making it available.
   1164 	 */
   1165 	clabel = raidget_component_label(raidPtr,
   1166 	    raidPtr->numCol + spare_number);
   1167 	memset(clabel, 0, sizeof(*clabel));
   1168 	ret = raidflush_component_label(raidPtr,
   1169 	    raidPtr->numCol + spare_number);
   1170 	if (ret) {
   1171 		rf_close_component(raidPtr,
   1172 		    raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp,
   1173 		    0);
   1174 		raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp =
   1175 		    NULL;
   1176 		goto fail;
   1177 	}
   1178 
   1179 	/*
   1180 	 * We only grow one initialized diskQueue at a time
   1181 	 * spare_number can be lower than raidPtr->maxQueue (update)
   1182 	 * or they can be equal (initialize new queue)
   1183 	 */
   1184 	RF_ASSERT(spare_number <= raidPtr->maxQueue);
   1185 
   1186 	spareQueues = &raidPtr->Queues[raidPtr->numCol];
   1187 	if (spare_number == raidPtr->maxQueue) {
   1188 		ret = rf_ConfigureDiskQueue(raidPtr, &spareQueues[spare_number],
   1189 					    raidPtr->numCol + spare_number,
   1190 					    raidPtr->qType,
   1191 					    raidPtr->sectorsPerDisk,
   1192 					    raidPtr->Disks[raidPtr->numCol +
   1193 							  spare_number].dev,
   1194 					    raidPtr->maxOutstanding,
   1195 					    &raidPtr->shutdownList,
   1196 					    raidPtr->cleanupList);
   1197 		if (ret)
   1198 			goto fail;
   1199 		rf_lock_mutex2(raidPtr->mutex);
   1200 		raidPtr->maxQueue++;
   1201 		rf_unlock_mutex2(raidPtr->mutex);
   1202 	} else {
   1203 		(void)rf_UpdateDiskQueue(&spareQueues[spare_number],
   1204 			                 &disks[spare_number]);
   1205 	}
   1206 
   1207 fail:
   1208 	if (ret != 0 && reusing) {
   1209 		disks[spare_number].status = rf_ds_failed;
   1210 	}
   1211 out:
   1212 	rf_lock_mutex2(raidPtr->mutex);
   1213 
   1214 	if (ret == 0 && !reusing) {
   1215 		raidPtr->numSpare++;
   1216 	}
   1217 
   1218 	raidPtr->changing_components = 0;
   1219 	rf_signal_cond2(raidPtr->changing_components_cv);
   1220 	rf_unlock_mutex2(raidPtr->mutex);
   1221 
   1222 	return(ret);
   1223 }
   1224 
   1225 int
   1226 rf_remove_hot_spare(RF_Raid_t *raidPtr, RF_SingleComponent_t *sparePtr)
   1227 {
   1228 	int spare_number;
   1229 	int i, acd;
   1230 	RF_RaidDisk_t *disk;
   1231 	struct vnode *vp;
   1232 	int ret = EINVAL;
   1233 
   1234 	spare_number = sparePtr->column - raidPtr->numCol;
   1235 	if (spare_number < 0 || spare_number >= raidPtr->numSpare)
   1236 		return(ret);
   1237 
   1238 	rf_lock_mutex2(raidPtr->mutex);
   1239 	while (raidPtr->changing_components == 1) {
   1240 		rf_wait_cond2(raidPtr->changing_components_cv, raidPtr->mutex);
   1241 	}
   1242 	raidPtr->changing_components = 1;
   1243 	rf_unlock_mutex2(raidPtr->mutex);
   1244 
   1245 	rf_SuspendNewRequestsAndWait(raidPtr);
   1246 
   1247 	disk = &raidPtr->Disks[raidPtr->numCol + spare_number];
   1248 	if (disk->status != rf_ds_spare &&
   1249 	    disk->status != rf_ds_failed &&
   1250 	    disk->status != rf_ds_spared) {
   1251 		printf("Spare is in use %d\n", disk->status);
   1252 		ret = EBUSY;
   1253 		goto out;
   1254 	}
   1255 
   1256 	vp = raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp;
   1257 	acd = disk->auto_configured;
   1258 	if (vp != NULL) {
   1259 		RF_ComponentLabel_t *clabel;
   1260 
   1261 		clabel = raidget_component_label(raidPtr,
   1262 		    raidPtr->numCol + spare_number);
   1263 		memset(clabel, 0, sizeof(*clabel));
   1264 		(void)raidflush_component_label(raidPtr,
   1265 		    raidPtr->numCol + spare_number);
   1266 	}
   1267 	raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_vp = NULL;
   1268 	raidPtr->raid_cinfo[raidPtr->numCol + spare_number].ci_dev = 0;
   1269 
   1270 	/* This component was not automatically configured */
   1271 	disk->auto_configured = 0;
   1272 	disk->dev = 0;
   1273 	disk->numBlocks = 0;
   1274 	disk->partitionSize = 0;
   1275 	disk->status = rf_ds_failed;
   1276 	snprintf(disk->devname, sizeof(disk->devname),
   1277 		 "absent_spare%d", spare_number);
   1278 	rf_close_component(raidPtr, vp, acd);
   1279 
   1280 	rf_lock_mutex2(raidPtr->mutex);
   1281 
   1282 	/* at this point we know spare_number is to be pushed all the way to the end of the array... */
   1283 
   1284 	for (i = raidPtr->numCol + spare_number; i < raidPtr->numCol+raidPtr->numSpare-1; i++) {
   1285 		/* now we work our way up the spare array, swaping the current one for the next one */
   1286 		rf_swap_components(raidPtr, i, i+1);
   1287 	}
   1288 
   1289 	raidPtr->numSpare--;
   1290 	rf_unlock_mutex2(raidPtr->mutex);
   1291 
   1292 	rf_ResumeNewRequests(raidPtr);
   1293 
   1294 	ret = 0;
   1295 
   1296 out:
   1297 
   1298 	rf_lock_mutex2(raidPtr->mutex);
   1299 	raidPtr->changing_components = 0;
   1300 	rf_signal_cond2(raidPtr->changing_components_cv);
   1301 	rf_unlock_mutex2(raidPtr->mutex);
   1302 
   1303 	return(ret);
   1304 }
   1305 
   1306 /*
   1307  * Delete a non hot spare component
   1308  */
   1309 int
   1310 rf_delete_component(RF_Raid_t *raidPtr, RF_SingleComponent_t *component)
   1311 {
   1312 	RF_RaidDisk_t *disk;
   1313 	RF_RowCol_t col = component->column;
   1314 	struct vnode *vp;
   1315 	int acd;
   1316 	int ret = EINVAL;
   1317 
   1318 	if (col < 0 || col >= raidPtr->numCol)
   1319 		return(ret);
   1320 
   1321 	rf_lock_mutex2(raidPtr->mutex);
   1322 	while (raidPtr->changing_components == 1) {
   1323 		rf_wait_cond2(raidPtr->changing_components_cv, raidPtr->mutex);
   1324 	}
   1325 	raidPtr->changing_components = 1;
   1326 	rf_unlock_mutex2(raidPtr->mutex);
   1327 
   1328 	disk = &raidPtr->Disks[col];
   1329 
   1330 	/* 1. This component must be marked as failed or spared  */
   1331 	switch (disk->status) {
   1332 	case rf_ds_failed:
   1333 	case rf_ds_dist_spared:
   1334 	case rf_ds_spared:
   1335 		break;
   1336 	default:
   1337 		ret = EBUSY;
   1338 		goto out;
   1339 	}
   1340 
   1341 	vp = raidPtr->raid_cinfo[col].ci_vp;
   1342 	acd = disk->auto_configured;
   1343 	if (vp != NULL) {
   1344 		RF_ComponentLabel_t *clabel;
   1345 
   1346 		clabel = raidget_component_label(raidPtr, col);
   1347 		memset(clabel, 0, sizeof(*clabel));
   1348 		(void)raidflush_component_label(raidPtr, col);
   1349 	}
   1350 	raidPtr->raid_cinfo[col].ci_vp = NULL;
   1351 	raidPtr->raid_cinfo[col].ci_dev = 0;
   1352 
   1353 	/* This component was not automatically configured */
   1354 	disk->auto_configured = 0;
   1355 	disk->dev = 0;
   1356 	disk->numBlocks = 0;
   1357 	disk->partitionSize = 0;
   1358 	snprintf(disk->devname, sizeof(disk->devname), "component%d", col);
   1359 
   1360 	rf_close_component(raidPtr, vp, acd);
   1361 
   1362 	ret = 0;
   1363 out:
   1364 	rf_lock_mutex2(raidPtr->mutex);
   1365 	raidPtr->changing_components = 0;
   1366 	rf_signal_cond2(raidPtr->changing_components_cv);
   1367 	rf_unlock_mutex2(raidPtr->mutex);
   1368 
   1369 	return(ret);
   1370 }
   1371 
   1372 int
   1373 rf_remove_component(RF_Raid_t *raidPtr, RF_SingleComponent_t *component)
   1374 {
   1375 	RF_RowCol_t col = component->column;
   1376 
   1377 	if (col < 0 || col >= raidPtr->numCol + raidPtr->numSpare)
   1378 		return(EINVAL);
   1379 
   1380 	if (col >= raidPtr->numCol)
   1381 		return rf_remove_hot_spare(raidPtr, component);
   1382 	else
   1383 		return rf_delete_component(raidPtr, component);
   1384 }
   1385 
   1386 int
   1387 rf_incorporate_hot_spare(RF_Raid_t *raidPtr,
   1388     RF_SingleComponent_t *component)
   1389 {
   1390 
   1391 	/* Issues here include how to 'move' this in if there is IO
   1392 	   taking place (e.g. component queues and such) */
   1393 
   1394 	return(EINVAL); /* Not implemented yet. */
   1395 }
   1396 
   1397 void
   1398 rf_swap_components(RF_Raid_t *raidPtr, int a, int b)
   1399 {
   1400 	RF_RaidDisk_t tmp_disk;
   1401 	struct raidcinfo tmp_cinfo;
   1402 	dev_t tmp_qdev;
   1403 
   1404 	/* This function *MUST* be called with all IO suspended. */
   1405 	RF_ASSERT(raidPtr->accesses_suspended > 0);
   1406 
   1407 	/* Swap the disk structures */
   1408 	tmp_disk = raidPtr->Disks[a];
   1409 	raidPtr->Disks[a] = raidPtr->Disks[b];
   1410 	raidPtr->Disks[b] = tmp_disk;
   1411 
   1412 	/* Swap the raid_cinfo structures (vp, ci_dev, ci_label) */
   1413 	tmp_cinfo = raidPtr->raid_cinfo[a];
   1414 	raidPtr->raid_cinfo[a] = raidPtr->raid_cinfo[b];
   1415 	raidPtr->raid_cinfo[b] = tmp_cinfo;
   1416 
   1417 	/* Keep the disk queue dev_t in sync */
   1418 	if (raidPtr->Queues != NULL) {
   1419 		tmp_qdev = raidPtr->Queues[a].dev;
   1420 		raidPtr->Queues[a].dev = raidPtr->Queues[b].dev;
   1421 		raidPtr->Queues[b].dev = tmp_qdev;
   1422 	}
   1423 }
   1424 
   1425