xref: /src/sys/dev/raidframe/rf_decluster.c

/*	$NetBSD: rf_decluster.c,v 1.1 1998/11/13 04:20:28 oster Exp $	*/
/*
 * Copyright (c) 1995 Carnegie-Mellon University.
 * All rights reserved.
 *
 * Author: Mark Holland
 *
 * Permission to use, copy, modify and distribute this software and
 * its documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 *
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 *
 * Carnegie Mellon requests users of this software to return to
 *
 *  Software Distribution Coordinator  or  Software.Distribution (at) CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 *
 * any improvements or extensions that they make and grant Carnegie the
 * rights to redistribute these changes.
 */

/*----------------------------------------------------------------------
 *
 * rf_decluster.c -- code related to the declustered layout
 *
 * Created 10-21-92 (MCH)
 *
 * Nov 93:  adding support for distributed sparing.  This code is a little
 *          complex:  the basic layout used is as follows:
 *          let F = (v-1)/GCD(r,v-1).  The spare space for each set of
 *          F consecutive fulltables is grouped together and placed after
 *          that set of tables.
 *                   +------------------------------+
 *                   |        F fulltables          |
 *                   |        Spare Space           |
 *                   |        F fulltables          |
 *                   |        Spare Space           |
 *                   |            ...               |
 *                   +------------------------------+
 *
 *--------------------------------------------------------------------*/

/*
 * :
 * Log: rf_decluster.c,v
 * Revision 1.51  1996/08/21 19:47:10  jimz
 * fix bogus return values from config
 *
 * Revision 1.50  1996/08/20  22:41:42  jimz
 * better diagnostics for bad blockdesigns
 *
 * Revision 1.49  1996/07/31  16:56:18  jimz
 * dataBytesPerStripe, sectorsPerDisk init arch-indep.
 *
 * Revision 1.48  1996/07/29  14:05:12  jimz
 * fix numPUs/numRUs confusion (everything is now numRUs)
 * clean up some commenting, return values
 *
 * Revision 1.47  1996/07/27  23:36:08  jimz
 * Solaris port of simulator
 *
 * Revision 1.46  1996/07/27  18:40:11  jimz
 * cleanup sweep
 *
 * Revision 1.45  1996/07/18  22:57:14  jimz
 * port simulator to AIX
 *
 * Revision 1.44  1996/07/13  00:00:59  jimz
 * sanitized generalized reconstruction architecture
 * cleaned up head sep, rbuf problems
 *
 * Revision 1.43  1996/06/19  17:53:48  jimz
 * move GetNumSparePUs, InstallSpareTable ops into layout switch
 *
 * Revision 1.42  1996/06/17  03:23:48  jimz
 * switch DeclusteredDS typing
 *
 * Revision 1.41  1996/06/11  08:55:15  jimz
 * improved error-checking at configuration time
 *
 * Revision 1.40  1996/06/10  11:55:47  jimz
 * Straightened out some per-array/not-per-array distinctions, fixed
 * a couple bugs related to confusion. Added shutdown lists. Removed
 * layout shutdown function (now subsumed by shutdown lists).
 *
 * Revision 1.39  1996/06/09  02:36:46  jimz
 * lots of little crufty cleanup- fixup whitespace
 * issues, comment #ifdefs, improve typing in some
 * places (esp size-related)
 *
 * Revision 1.38  1996/06/07  22:26:27  jimz
 * type-ify which_ru (RF_ReconUnitNum_t)
 *
 * Revision 1.37  1996/06/07  21:33:04  jimz
 * begin using consistent types for sector numbers,
 * stripe numbers, row+col numbers, recon unit numbers
 *
 * Revision 1.36  1996/06/03  23:28:26  jimz
 * more bugfixes
 * check in tree to sync for IPDS runs with current bugfixes
 * there still may be a problem with threads in the script test
 * getting I/Os stuck- not trivially reproducible (runs ~50 times
 * in a row without getting stuck)
 *
 * Revision 1.35  1996/06/02  17:31:48  jimz
 * Moved a lot of global stuff into array structure, where it belongs.
 * Fixed up paritylogging, pss modules in this manner. Some general
 * code cleanup. Removed lots of dead code, some dead files.
 *
 * Revision 1.34  1996/05/30  23:22:16  jimz
 * bugfixes of serialization, timing problems
 * more cleanup
 *
 * Revision 1.33  1996/05/30  11:29:41  jimz
 * Numerous bug fixes. Stripe lock release code disagreed with the taking code
 * about when stripes should be locked (I made it consistent: no parity, no lock)
 * There was a lot of extra serialization of I/Os which I've removed- a lot of
 * it was to calculate values for the cache code, which is no longer with us.
 * More types, function, macro cleanup. Added code to properly quiesce the array
 * on shutdown. Made a lot of stuff array-specific which was (bogusly) general
 * before. Fixed memory allocation, freeing bugs.
 *
 * Revision 1.32  1996/05/27  18:56:37  jimz
 * more code cleanup
 * better typing
 * compiles in all 3 environments
 *
 * Revision 1.31  1996/05/24  01:59:45  jimz
 * another checkpoint in code cleanup for release
 * time to sync kernel tree
 *
 * Revision 1.30  1996/05/23  00:33:23  jimz
 * code cleanup: move all debug decls to rf_options.c, all extern
 * debug decls to rf_options.h, all debug vars preceded by rf_
 *
 * Revision 1.29  1996/05/18  19:51:34  jimz
 * major code cleanup- fix syntax, make some types consistent,
 * add prototypes, clean out dead code, et cetera
 *
 * Revision 1.28  1995/12/12  18:10:06  jimz
 * MIN -> RF_MIN, MAX -> RF_MAX, ASSERT -> RF_ASSERT
 * fix 80-column brain damage in comments
 *
 * Revision 1.27  1995/12/01  16:00:08  root
 * added copyright info
 *
 * Revision 1.26  1995/11/28  21:35:12  amiri
 * set the RF_BD_DECLUSTERED flag
 *
 * Revision 1.25  1995/11/17  18:56:00  wvcii
 * added prototyping to MapParity
 *
 * Revision 1.24  1995/07/04  22:25:33  holland
 * increased default num bufs
 *
 * Revision 1.23  1995/07/03  20:23:51  holland
 * changed floating recon bufs & head sep yet again
 *
 * Revision 1.22  1995/07/03  18:12:14  holland
 * changed the way the number of floating recon bufs & the head sep
 * limit are set
 *
 * Revision 1.21  1995/07/02  15:07:42  holland
 * bug fixes related to getting distributed sparing numbers
 *
 * Revision 1.20  1995/06/23  13:41:28  robby
 * updeated to prototypes in rf_layout.h
 *
 */

#ifdef _KERNEL
#define KERNEL
#endif


#include "rf_types.h"
#include "rf_raid.h"
#include "rf_raidframe.h"
#include "rf_configure.h"
#include "rf_decluster.h"
#include "rf_debugMem.h"
#include "rf_utils.h"
#include "rf_alloclist.h"
#include "rf_general.h"
#include "rf_shutdown.h"
#include "rf_sys.h"

extern int rf_copyback_in_progress;                /* debug only */

/* found in rf_kintf.c */
int rf_GetSpareTableFromDaemon(RF_SparetWait_t  *req);

/* configuration code */

int rf_ConfigureDeclustered(
  RF_ShutdownList_t  **listp,
  RF_Raid_t           *raidPtr,
  RF_Config_t         *cfgPtr)
{
    RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout);
    int b, v, k, r, lambda;				/* block design params */
    int i, j;
    RF_RowCol_t *first_avail_slot;
    RF_StripeCount_t complete_FT_count, numCompleteFullTablesPerDisk;
    RF_DeclusteredConfigInfo_t *info;
    RF_StripeCount_t PUsPerDisk, spareRegionDepthInPUs, numCompleteSpareRegionsPerDisk, extraPUsPerDisk;
    RF_StripeCount_t totSparePUsPerDisk;
    RF_SectorNum_t diskOffsetOfLastFullTableInSUs;
    RF_SectorCount_t SpareSpaceInSUs;
    char *cfgBuf = (char *) (cfgPtr->layoutSpecific);
    RF_StripeNum_t l, SUID;

    SUID = l = 0;
    numCompleteSpareRegionsPerDisk = 0;

    /* 1. create layout specific structure */
    RF_MallocAndAdd(info, sizeof(RF_DeclusteredConfigInfo_t), (RF_DeclusteredConfigInfo_t *), raidPtr->cleanupList);
    if (info == NULL)
      return(ENOMEM);
    layoutPtr->layoutSpecificInfo = (void *) info;
    info->SpareTable = NULL;

    /* 2. extract parameters from the config structure */
    if (layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) {
      (void) bcopy(cfgBuf, info->sparemap_fname, RF_SPAREMAP_NAME_LEN);
    }
    cfgBuf += RF_SPAREMAP_NAME_LEN;

    b        = *( (int *) cfgBuf);   cfgBuf += sizeof(int);
    v        = *( (int *) cfgBuf);   cfgBuf += sizeof(int);
    k        = *( (int *) cfgBuf);   cfgBuf += sizeof(int);
    r        = *( (int *) cfgBuf);   cfgBuf += sizeof(int);
    lambda   = *( (int *) cfgBuf);   cfgBuf += sizeof(int);
    raidPtr->noRotate = *( (int *) cfgBuf);   cfgBuf += sizeof(int);

    /* the sparemaps are generated assuming that parity is rotated, so we issue
     * a warning if both distributed sparing and no-rotate are on at the same time
     */
    if ((layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) && raidPtr->noRotate) {
	RF_ERRORMSG("Warning:  distributed sparing specified without parity rotation.\n");
    }

    if (raidPtr->numCol != v) {
        RF_ERRORMSG2("RAID: config error: table element count (%d) not equal to no. of cols (%d)\n", v, raidPtr->numCol);
        return(EINVAL);
    }

    /* 3.  set up the values used in the mapping code */
    info->BlocksPerTable = b;
    info->Lambda = lambda;
    info->NumParityReps = info->groupSize = k;
    info->SUsPerTable = b * (k-1) * layoutPtr->SUsPerPU;/* b blks, k-1 SUs each */
    info->SUsPerFullTable = k * info->SUsPerTable;	/* rot k times */
    info->PUsPerBlock = k-1;
    info->SUsPerBlock = info->PUsPerBlock * layoutPtr->SUsPerPU;
    info->TableDepthInPUs = (b*k) / v;
    info->FullTableDepthInPUs = info->TableDepthInPUs * k;		/* k repetitions */

    /* used only in distributed sparing case */
    info->FullTablesPerSpareRegion = (v-1) / rf_gcd(r, v-1);		/* (v-1)/gcd fulltables */
    info->TablesPerSpareRegion = k * info->FullTablesPerSpareRegion;
    info->SpareSpaceDepthPerRegionInSUs = (r * info->TablesPerSpareRegion / (v-1)) * layoutPtr->SUsPerPU;

    /* check to make sure the block design is sufficiently small */
    if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) {
        if (info->FullTableDepthInPUs * layoutPtr->SUsPerPU + info->SpareSpaceDepthPerRegionInSUs > layoutPtr->stripeUnitsPerDisk) {
	    RF_ERRORMSG3("RAID: config error: Full Table depth (%d) + Spare Space (%d) larger than disk size (%d) (BD too big)\n",
			 (int)info->FullTableDepthInPUs,
			 (int)info->SpareSpaceDepthPerRegionInSUs,
			 (int)layoutPtr->stripeUnitsPerDisk);
	    return(EINVAL);
	}
    } else {
	if (info->TableDepthInPUs * layoutPtr->SUsPerPU > layoutPtr->stripeUnitsPerDisk) {
	    RF_ERRORMSG2("RAID: config error: Table depth (%d) larger than disk size (%d) (BD too big)\n",
			 (int)(info->TableDepthInPUs * layoutPtr->SUsPerPU), \
			 (int)layoutPtr->stripeUnitsPerDisk);
	    return(EINVAL);
	}
    }


    /* compute the size of each disk, and the number of tables in the last fulltable (which
     * need not be complete)
     */
    if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) {

	PUsPerDisk = layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU;
	spareRegionDepthInPUs = (info->TablesPerSpareRegion * info->TableDepthInPUs +
				 (info->TablesPerSpareRegion * info->TableDepthInPUs) / (v-1));
	info->SpareRegionDepthInSUs = spareRegionDepthInPUs * layoutPtr->SUsPerPU;

	numCompleteSpareRegionsPerDisk = PUsPerDisk / spareRegionDepthInPUs;
	info->NumCompleteSRs = numCompleteSpareRegionsPerDisk;
	extraPUsPerDisk = PUsPerDisk % spareRegionDepthInPUs;

	/* assume conservatively that we need the full amount of spare space in one region in order
	 * to provide spares for the partial spare region at the end of the array.  We set "i" to
	 * the number of tables in the partial spare region.  This may actually include some fulltables.
	 */
	extraPUsPerDisk -= (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU);
	if (extraPUsPerDisk <= 0) i = 0;
	else i = extraPUsPerDisk/info->TableDepthInPUs;

	complete_FT_count = raidPtr->numRow * (numCompleteSpareRegionsPerDisk * (info->TablesPerSpareRegion/k) + i/k);
        info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable;
	info->ExtraTablesPerDisk = i % k;

	/* note that in the last spare region, the spare space is complete even though data/parity space is not */
	totSparePUsPerDisk = (numCompleteSpareRegionsPerDisk+1) * (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU);
	info->TotSparePUsPerDisk = totSparePUsPerDisk;

	layoutPtr->stripeUnitsPerDisk =
	    ((complete_FT_count/raidPtr->numRow) * info->FullTableDepthInPUs +	 	/* data & parity space */
	     info->ExtraTablesPerDisk * info->TableDepthInPUs +
	     totSparePUsPerDisk								/* spare space */
	    ) * layoutPtr->SUsPerPU;
	layoutPtr->dataStripeUnitsPerDisk =
	    (complete_FT_count * info->FullTableDepthInPUs + info->ExtraTablesPerDisk * info->TableDepthInPUs)
	    * layoutPtr->SUsPerPU * (k-1) / k;

    } else {
        /* non-dist spare case:  force each disk to contain an integral number of tables */
        layoutPtr->stripeUnitsPerDisk /= (info->TableDepthInPUs * layoutPtr->SUsPerPU);
        layoutPtr->stripeUnitsPerDisk *= (info->TableDepthInPUs * layoutPtr->SUsPerPU);

	/* compute the number of tables in the last fulltable, which need not be complete */
        complete_FT_count =
            ((layoutPtr->stripeUnitsPerDisk/layoutPtr->SUsPerPU) / info->FullTableDepthInPUs) * raidPtr->numRow;

        info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable;
        info->ExtraTablesPerDisk =
		((layoutPtr->stripeUnitsPerDisk/layoutPtr->SUsPerPU) / info->TableDepthInPUs) % k;
    }

    raidPtr->sectorsPerDisk = layoutPtr->stripeUnitsPerDisk * layoutPtr->sectorsPerStripeUnit;

    /* find the disk offset of the stripe unit where the last fulltable starts */
    numCompleteFullTablesPerDisk = complete_FT_count / raidPtr->numRow;
    diskOffsetOfLastFullTableInSUs = numCompleteFullTablesPerDisk * info->FullTableDepthInPUs * layoutPtr->SUsPerPU;
    if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) {
        SpareSpaceInSUs  = numCompleteSpareRegionsPerDisk * info->SpareSpaceDepthPerRegionInSUs;
        diskOffsetOfLastFullTableInSUs += SpareSpaceInSUs;
        info->DiskOffsetOfLastSpareSpaceChunkInSUs =
	    diskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU;
    }
    info->DiskOffsetOfLastFullTableInSUs = diskOffsetOfLastFullTableInSUs;
    info->numCompleteFullTablesPerDisk = numCompleteFullTablesPerDisk;

    /* 4.  create and initialize the lookup tables */
    info->LayoutTable = rf_make_2d_array(b, k, raidPtr->cleanupList);
    if (info->LayoutTable == NULL)
      return(ENOMEM);
    info->OffsetTable = rf_make_2d_array(b, k, raidPtr->cleanupList);
    if (info->OffsetTable == NULL)
      return(ENOMEM);
    info->BlockTable  =	rf_make_2d_array(info->TableDepthInPUs*layoutPtr->SUsPerPU, raidPtr->numCol, raidPtr->cleanupList);
    if (info->BlockTable == NULL)
      return(ENOMEM);

    first_avail_slot = rf_make_1d_array(v, NULL);
    if (first_avail_slot == NULL)
      return(ENOMEM);

    for (i=0; i<b; i++)
      for (j=0; j<k; j++)
        info->LayoutTable[i][j] = *cfgBuf++;

    /* initialize offset table */
    for (i=0; i<b; i++) for (j=0; j<k; j++) {
        info->OffsetTable[i][j] = first_avail_slot[ info->LayoutTable[i][j] ];
        first_avail_slot[ info->LayoutTable[i][j] ]++;
    }

    /* initialize block table */
    for (SUID=l=0; l<layoutPtr->SUsPerPU; l++) {
        for (i=0; i<b; i++) {
            for (j=0; j<k; j++) {
                info->BlockTable[ (info->OffsetTable[i][j] * layoutPtr->SUsPerPU) + l ]
		                [ info->LayoutTable[i][j] ] = SUID;
            }
            SUID++;
        }
    }

    rf_free_1d_array(first_avail_slot, v);

    /* 5.  set up the remaining redundant-but-useful parameters */

    raidPtr->totalSectors = (k*complete_FT_count + raidPtr->numRow*info->ExtraTablesPerDisk) *
    			  info->SUsPerTable * layoutPtr->sectorsPerStripeUnit;
    layoutPtr->numStripe = (raidPtr->totalSectors / layoutPtr->sectorsPerStripeUnit) / (k-1);

    /* strange evaluation order below to try and minimize overflow problems */

    layoutPtr->dataSectorsPerStripe = (k-1) * layoutPtr->sectorsPerStripeUnit;
    layoutPtr->bytesPerStripeUnit = layoutPtr->sectorsPerStripeUnit << raidPtr->logBytesPerSector;
    layoutPtr->numDataCol = k-1;
    layoutPtr->numParityCol = 1;

    return(0);
}

/* declustering with distributed sparing */
static void rf_ShutdownDeclusteredDS(RF_ThreadArg_t);
static void rf_ShutdownDeclusteredDS(arg)
  RF_ThreadArg_t  arg;
{
  RF_DeclusteredConfigInfo_t *info;
  RF_Raid_t *raidPtr;

  raidPtr = (RF_Raid_t *)arg;
  info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo;
  if (info->SpareTable)
    rf_FreeSpareTable(raidPtr);
}

int rf_ConfigureDeclusteredDS(
  RF_ShutdownList_t  **listp,
  RF_Raid_t           *raidPtr,
  RF_Config_t         *cfgPtr)
{
  int rc;

  rc = rf_ConfigureDeclustered(listp, raidPtr, cfgPtr);
  if (rc)
    return(rc);
  rc = rf_ShutdownCreate(listp, rf_ShutdownDeclusteredDS, raidPtr);
  if (rc) {
    RF_ERRORMSG1("Got %d adding shutdown event for DeclusteredDS\n", rc);
    rf_ShutdownDeclusteredDS(raidPtr);
    return(rc);
  }
  return(0);
}

void rf_MapSectorDeclustered(raidPtr, raidSector, row, col, diskSector, remap)
  RF_Raid_t       *raidPtr;
  RF_RaidAddr_t    raidSector;
  RF_RowCol_t     *row;
  RF_RowCol_t     *col;
  RF_SectorNum_t  *diskSector;
  int              remap;
{
    RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout);
    RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo;
    RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit;
    RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset;
    RF_StripeNum_t BlockID, BlockOffset, RepIndex;
    RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable;
    RF_StripeCount_t fulltable_depth  = info->FullTableDepthInPUs * layoutPtr->SUsPerPU;
    RF_StripeNum_t base_suid = 0, outSU, SpareRegion=0, SpareSpace=0;

    rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid);

    FullTableID     = SUID / sus_per_fulltable;		/* fulltable ID within array (across rows) */
    if (raidPtr->numRow == 1) *row = 0;                 /* avoid a mod and a div in the common case */
    else {
      *row            = FullTableID % raidPtr->numRow;
      FullTableID    /= raidPtr->numRow;			/* convert to fulltable ID on this disk */
    }
    if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) {
	SpareRegion = FullTableID / info->FullTablesPerSpareRegion;
        SpareSpace  = SpareRegion * info->SpareSpaceDepthPerRegionInSUs;
    }
    FullTableOffset = SUID % sus_per_fulltable;
    TableID         = FullTableOffset / info->SUsPerTable;
    TableOffset     = FullTableOffset - TableID * info->SUsPerTable;
    BlockID         = TableOffset / info->PUsPerBlock;
    BlockOffset     = TableOffset - BlockID * info->PUsPerBlock;
    BlockID        %= info->BlocksPerTable;
    RepIndex        = info->PUsPerBlock - TableID;
    if (!raidPtr->noRotate) BlockOffset    += ((BlockOffset >= RepIndex) ? 1 : 0);
    *col            = info->LayoutTable[BlockID][BlockOffset];

    /* remap to distributed spare space if indicated */
    if (remap) {
      RF_ASSERT( raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared ||
	     (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal));
      rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU);
    } else {

        outSU	    = base_suid;
        outSU      += FullTableID * fulltable_depth;  				        /* offs to strt of FT */
        outSU	   += SpareSpace;						        /* skip rsvd spare space */
        outSU      += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU;   	        /* offs to strt of tble */
        outSU      += info->OffsetTable[BlockID][BlockOffset] * layoutPtr->SUsPerPU;	/* offs to the PU */
    }
    outSU          += TableOffset / (info->BlocksPerTable * info->PUsPerBlock);	        /* offs to the SU within a PU */

    /* convert SUs to sectors, and, if not aligned to SU boundary, add in offset to sector.  */
    *diskSector     = outSU*layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit);

    RF_ASSERT( *col != -1 );
}


/* prototyping this inexplicably causes the compile of the layout table (rf_layout.c) to fail */
void rf_MapParityDeclustered(
  RF_Raid_t       *raidPtr,
  RF_RaidAddr_t    raidSector,
  RF_RowCol_t     *row,
  RF_RowCol_t     *col,
  RF_SectorNum_t  *diskSector,
  int              remap)
{
    RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout);
    RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo;
    RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit;
    RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset;
    RF_StripeNum_t BlockID, BlockOffset, RepIndex;
    RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable;
    RF_StripeCount_t fulltable_depth  = info->FullTableDepthInPUs * layoutPtr->SUsPerPU;
    RF_StripeNum_t base_suid = 0, outSU, SpareRegion=0, SpareSpace=0;

    rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid);

    /* compute row & (possibly) spare space exactly as before */
    FullTableID     = SUID / sus_per_fulltable;
    if (raidPtr->numRow == 1) *row = 0;                         /* avoid a mod and a div in the common case */
    else {
      *row            = FullTableID % raidPtr->numRow;
      FullTableID    /= raidPtr->numRow;			/* convert to fulltable ID on this disk */
    }
    if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) {
	SpareRegion = FullTableID / info->FullTablesPerSpareRegion;
        SpareSpace  = SpareRegion * info->SpareSpaceDepthPerRegionInSUs;
    }

    /* compute BlockID and RepIndex exactly as before */
    FullTableOffset = SUID % sus_per_fulltable;
    TableID         = FullTableOffset / info->SUsPerTable;
    TableOffset     = FullTableOffset - TableID * info->SUsPerTable;
    /*TableOffset     = FullTableOffset % info->SUsPerTable;*/
    /*BlockID         = (TableOffset / info->PUsPerBlock) % info->BlocksPerTable;*/
    BlockID         = TableOffset / info->PUsPerBlock;
    /*BlockOffset     = TableOffset % info->PUsPerBlock;*/
    BlockOffset     = TableOffset - BlockID * info->PUsPerBlock;
    BlockID        %= info->BlocksPerTable;

    /* the parity block is in the position indicated by RepIndex */
    RepIndex        = (raidPtr->noRotate) ? info->PUsPerBlock : info->PUsPerBlock - TableID;
    *col	    = info->LayoutTable[BlockID][RepIndex];

    if (remap) {
      RF_ASSERT( raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared ||
	     (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal));
      rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU);
    } else {

        /* compute sector as before, except use RepIndex instead of BlockOffset */
        outSU        = base_suid;
        outSU       += FullTableID * fulltable_depth;
        outSU	    += SpareSpace;						/* skip rsvd spare space */
        outSU       += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU;
        outSU       += info->OffsetTable[BlockID][RepIndex] * layoutPtr->SUsPerPU;
    }

    outSU       += TableOffset / (info->BlocksPerTable * info->PUsPerBlock);
    *diskSector  = outSU*layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit);

    RF_ASSERT( *col != -1 );
}

/* returns an array of ints identifying the disks that comprise the stripe containing the indicated address.
 * the caller must _never_ attempt to modify this array.
 */
void rf_IdentifyStripeDeclustered(
  RF_Raid_t        *raidPtr,
  RF_RaidAddr_t     addr,
  RF_RowCol_t     **diskids,
  RF_RowCol_t      *outRow)
{
  RF_RaidLayout_t *layoutPtr           = &(raidPtr->Layout);
  RF_DeclusteredConfigInfo_t *info     = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo;
  RF_StripeCount_t sus_per_fulltable   = info->SUsPerFullTable;
  RF_StripeCount_t fulltable_depth     = info->FullTableDepthInPUs * layoutPtr->SUsPerPU;
  RF_StripeNum_t  base_suid            = 0;
  RF_StripeNum_t SUID                  = rf_RaidAddressToStripeUnitID(layoutPtr, addr);
  RF_StripeNum_t stripeID, FullTableID;
  int tableOffset;

  rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid);
  FullTableID     = SUID / sus_per_fulltable;		/* fulltable ID within array (across rows) */
  *outRow         = FullTableID % raidPtr->numRow;
  stripeID        = rf_StripeUnitIDToStripeID(layoutPtr, SUID);                     /* find stripe offset into array */
  tableOffset     = (stripeID % info->BlocksPerTable);                        /* find offset into block design table */
  *diskids        = info->LayoutTable[tableOffset];
}

/* This returns the default head-separation limit, which is measured
 * in "required units for reconstruction".  Each time a disk fetches
 * a unit, it bumps a counter.  The head-sep code prohibits any disk
 * from getting more than headSepLimit counter values ahead of any
 * other.
 *
 * We assume here that the number of floating recon buffers is already
 * set.  There are r stripes to be reconstructed in each table, and so
 * if we have a total of B buffers, we can have at most B/r tables
 * under recon at any one time.  In each table, lambda units are required
 * from each disk, so given B buffers, the head sep limit has to be
 * (lambda*B)/r units.  We subtract one to avoid weird boundary cases.
 *
 * for example, suppose were given 50 buffers, r=19, and lambda=4 as in
 * the 20.5 design.  There are 19 stripes/table to be reconstructed, so
 * we can have 50/19 tables concurrently under reconstruction, which means
 * we can allow the fastest disk to get 50/19 tables ahead of the slower
 * disk.  There are lambda "required units" for each disk, so the fastest
 * disk can get 4*50/19 = 10 counter values ahead of the slowest.
 *
 * If numBufsToAccumulate is not 1, we need to limit the head sep further
 * because multiple bufs will be required for each stripe under recon.
 */
RF_HeadSepLimit_t rf_GetDefaultHeadSepLimitDeclustered(
  RF_Raid_t  *raidPtr)
{
  RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo;

  return(info->Lambda * raidPtr->numFloatingReconBufs / info->TableDepthInPUs / rf_numBufsToAccumulate);
}

/* returns the default number of recon buffers to use.  The value
 * is somewhat arbitrary...it's intended to be large enough to allow
 * for a reasonably large head-sep limit, but small enough that you
 * don't use up all your system memory with buffers.
 */
int rf_GetDefaultNumFloatingReconBuffersDeclustered(RF_Raid_t *raidPtr)
{
  return(100 * rf_numBufsToAccumulate);
}

/* sectors in the last fulltable of the array need to be handled
 * specially since this fulltable can be incomplete.  this function
 * changes the values of certain params to handle this.
 *
 * the idea here is that MapSector et. al. figure out which disk the
 * addressed unit lives on by computing the modulos of the unit number
 * with the number of units per fulltable, table, etc.  In the last
 * fulltable, there are fewer units per fulltable, so we need to adjust
 * the number of user data units per fulltable to reflect this.
 *
 * so, we (1) convert the fulltable size and depth parameters to
 * the size of the partial fulltable at the end, (2) compute the
 * disk sector offset where this fulltable starts, and (3) convert
 * the users stripe unit number from an offset into the array to
 * an offset into the last fulltable.
 */
void rf_decluster_adjust_params(
  RF_RaidLayout_t   *layoutPtr,
  RF_StripeNum_t    *SUID,
  RF_StripeCount_t  *sus_per_fulltable,
  RF_StripeCount_t  *fulltable_depth,
  RF_StripeNum_t    *base_suid)
{
    RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo;
#if defined(__NetBSD__) && defined(_KERNEL)
    /* Nothing! */
#else
    char pc = layoutPtr->map->parityConfig;
#endif

    if (*SUID >= info->FullTableLimitSUID) {
	/* new full table size is size of last full table on disk */
	*sus_per_fulltable = info->ExtraTablesPerDisk * info->SUsPerTable;

	/* new full table depth is corresponding depth */
	*fulltable_depth = info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU;

	/* set up the new base offset */
	*base_suid = info->DiskOffsetOfLastFullTableInSUs;

	/* convert users array address to an offset into the last fulltable */
	*SUID -= info->FullTableLimitSUID;
    }
}

/*
 * map a stripe ID to a parity stripe ID.
 * See comment above RaidAddressToParityStripeID in layout.c.
 */
void rf_MapSIDToPSIDDeclustered(
  RF_RaidLayout_t    *layoutPtr,
  RF_StripeNum_t      stripeID,
  RF_StripeNum_t     *psID,
  RF_ReconUnitNum_t  *which_ru)
{
    RF_DeclusteredConfigInfo_t *info;

    info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo;

    *psID = (stripeID / (layoutPtr->SUsPerPU * info->BlocksPerTable))
        * info->BlocksPerTable + (stripeID % info->BlocksPerTable);
    *which_ru = (stripeID % (info->BlocksPerTable * layoutPtr->SUsPerPU))
        / info->BlocksPerTable;
    RF_ASSERT( (*which_ru) < layoutPtr->SUsPerPU/layoutPtr->SUsPerRU);
}

/*
 * Called from MapSector and MapParity to retarget an access at the spare unit.
 * Modifies the "col" and "outSU" parameters only.
 */
void rf_remap_to_spare_space(
  RF_RaidLayout_t             *layoutPtr,
  RF_DeclusteredConfigInfo_t  *info,
  RF_RowCol_t                  row,
  RF_StripeNum_t               FullTableID,
  RF_StripeNum_t               TableID,
  RF_SectorNum_t               BlockID,
  RF_StripeNum_t               base_suid,
  RF_StripeNum_t               SpareRegion,
  RF_RowCol_t                 *outCol,
  RF_StripeNum_t              *outSU)
{
    RF_StripeNum_t ftID, spareTableStartSU, TableInSpareRegion, lastSROffset, which_ft;

    /*
     * note that FullTableID and hence SpareRegion may have gotten
     * tweaked by rf_decluster_adjust_params. We detect this by
     * noticing that base_suid is not 0.
     */
    if (base_suid == 0) {
      ftID = FullTableID;
    }
    else {
      /*
       * There may be > 1.0 full tables in the last (i.e. partial)
       * spare region.  find out which of these we're in.
       */
      lastSROffset = info->NumCompleteSRs * info->SpareRegionDepthInSUs;
      which_ft = (info->DiskOffsetOfLastFullTableInSUs - lastSROffset) / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU);

      /* compute the actual full table ID */
      ftID = info->DiskOffsetOfLastFullTableInSUs / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU) + which_ft;
      SpareRegion = info->NumCompleteSRs;
    }
    TableInSpareRegion = (ftID * info->NumParityReps + TableID) % info->TablesPerSpareRegion;

    *outCol = info->SpareTable[TableInSpareRegion][BlockID].spareDisk;
    RF_ASSERT( *outCol != -1);

    spareTableStartSU = (SpareRegion == info->NumCompleteSRs) ?
	    info->DiskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU :
	    (SpareRegion+1) * info->SpareRegionDepthInSUs - info->SpareSpaceDepthPerRegionInSUs;
    *outSU = spareTableStartSU + info->SpareTable[TableInSpareRegion][BlockID].spareBlockOffsetInSUs;
    if (*outSU >= layoutPtr->stripeUnitsPerDisk) {
	printf("rf_remap_to_spare_space: invalid remapped disk SU offset %ld\n",(long)*outSU);
    }
}

int rf_InstallSpareTable(
  RF_Raid_t    *raidPtr,
  RF_RowCol_t   frow,
  RF_RowCol_t   fcol)
{
  RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo;
  RF_SparetWait_t *req;
  int retcode;

  RF_Malloc(req, sizeof(*req), (RF_SparetWait_t *));
  req->C                             = raidPtr->numCol;
  req->G                             = raidPtr->Layout.numDataCol + raidPtr->Layout.numParityCol;
  req->fcol                          = fcol;
  req->SUsPerPU                      = raidPtr->Layout.SUsPerPU;
  req->TablesPerSpareRegion          = info->TablesPerSpareRegion;
  req->BlocksPerTable                = info->BlocksPerTable;
  req->TableDepthInPUs               = info->TableDepthInPUs;
  req->SpareSpaceDepthPerRegionInSUs = info->SpareSpaceDepthPerRegionInSUs;

#ifndef KERNEL
  info->SpareTable = rf_ReadSpareTable(req, info->sparemap_fname);
  RF_Free(req, sizeof(*req));
  retcode = (info->SpareTable) ? 0 : 1;
#else /* !KERNEL */
  retcode = rf_GetSpareTableFromDaemon(req);
  RF_ASSERT(!retcode);                                     /* XXX -- fix this to recover gracefully -- XXX */
#endif /* !KERNEL */

  return(retcode);
}

#ifdef KERNEL
/*
 * Invoked via ioctl to install a spare table in the kernel.
 */
int rf_SetSpareTable(raidPtr, data)
  RF_Raid_t  *raidPtr;
  void       *data;
{
  RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo;
  RF_SpareTableEntry_t **ptrs;
  int i, retcode;

  /* what we need to copyin is a 2-d array, so first copyin the user pointers to the rows in the table */
  RF_Malloc(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **));
  retcode = copyin((caddr_t) data, (caddr_t) ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *));

  if (retcode) return(retcode);

  /* now allocate kernel space for the row pointers */
  RF_Malloc(info->SpareTable, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **));

  /* now allocate kernel space for each row in the table, and copy it in from user space */
  for (i=0; i<info->TablesPerSpareRegion; i++) {
    RF_Malloc(info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t), (RF_SpareTableEntry_t *));
    retcode = copyin(ptrs[i], info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t));
    if (retcode) {
      info->SpareTable = NULL;             /* blow off the memory we've allocated */
      return(retcode);
    }
  }

  /* free up the temporary array we used */
  RF_Free(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *));

  return(0);
}
#endif /* KERNEL */

RF_ReconUnitCount_t rf_GetNumSpareRUsDeclustered(raidPtr)
  RF_Raid_t *raidPtr;
{
  RF_RaidLayout_t *layoutPtr = &raidPtr->Layout;

  return( ((RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo)->TotSparePUsPerDisk );
}


void rf_FreeSpareTable(raidPtr)
  RF_Raid_t  *raidPtr;
{
  long i;
  RF_RaidLayout_t *layoutPtr = &raidPtr->Layout;
  RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo;
  RF_SpareTableEntry_t **table = info->SpareTable;

  for (i=0; i<info->TablesPerSpareRegion; i++) {RF_Free(table[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t));}
  RF_Free(table, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *));
  info->SpareTable = (RF_SpareTableEntry_t **) NULL;
}