rf_raid.h revision 1.52 1 /* $NetBSD: rf_raid.h,v 1.52 2023/09/17 20:07:39 oster Exp $ */
2 /*
3 * Copyright (c) 1995 Carnegie-Mellon University.
4 * All rights reserved.
5 *
6 * Author: Mark Holland
7 *
8 * Permission to use, copy, modify and distribute this software and
9 * its documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
17 *
18 * Carnegie Mellon requests users of this software to return to
19 *
20 * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU
21 * School of Computer Science
22 * Carnegie Mellon University
23 * Pittsburgh PA 15213-3890
24 *
25 * any improvements or extensions that they make and grant Carnegie the
26 * rights to redistribute these changes.
27 */
28
29 /**********************************************
30 * rf_raid.h -- main header file for RAID driver
31 **********************************************/
32
33
34 #ifndef _RF__RF_RAID_H_
35 #define _RF__RF_RAID_H_
36
37 #include <dev/raidframe/raidframevar.h>
38 #include "rf_archs.h"
39 #include "rf_threadstuff.h"
40
41 #include "rf_netbsd.h"
42
43 #include <sys/disklabel.h>
44 #include <sys/types.h>
45 #include <sys/queue.h>
46
47 #include "rf_alloclist.h"
48 #include "rf_stripelocks.h"
49 #include "rf_layout.h"
50 #include "rf_disks.h"
51 #include "rf_debugMem.h"
52 #include "rf_diskqueue.h"
53 #include "rf_reconstruct.h"
54 #include "rf_acctrace.h"
55 #include "rf_fifo.h"
56
57 #if RF_INCLUDE_PARITYLOGGING > 0
58 #include "rf_paritylog.h"
59 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
60
61 #define RF_COMPONENT_LABEL_VERSION_1 1
62 #define RF_COMPONENT_LABEL_VERSION 2
63 #define RF_RAID_DIRTY 0
64 #define RF_RAID_CLEAN 1
65
66
67 /*
68 * Each row in the array is a distinct parity group, so
69 * each has its own status, which is one of the following.
70 */
71 typedef enum RF_RowStatus_e {
72 rf_rs_optimal,
73 rf_rs_degraded,
74 rf_rs_reconstructing,
75 rf_rs_reconfigured
76 } RF_RowStatus_t;
77
78 struct RF_CumulativeStats_s {
79 struct timeval start; /* the time when the stats were last started */
80 struct timeval stop; /* the time when the stats were last stopped */
81 long sum_io_us; /* sum of all user response times (us) */
82 long num_ios; /* total number of I/Os serviced */
83 long num_sect_moved; /* total number of sectors read or written */
84 };
85
86 struct RF_ThroughputStats_s {
87 rf_declare_mutex2(mutex);/* a mutex used to lock the configuration
88 * stuff */
89 struct timeval start; /* timer started when numOutstandingRequests
90 * moves from 0 to 1 */
91 struct timeval stop; /* timer stopped when numOutstandingRequests
92 * moves from 1 to 0 */
93 RF_uint64 sum_io_us; /* total time timer is enabled */
94 RF_uint64 num_ios; /* total number of ios processed by RAIDframe */
95 long num_out_ios; /* number of outstanding ios */
96 };
97
98 struct RF_Raid_s {
99 /* This portion never changes, and can be accessed without locking */
100 /* an exception is Disks[][].status, which requires locking when it is
101 * changed. XXX this is no longer true. numSpare and friends can
102 * change now.
103 */
104 u_int numCol; /* number of columns of disks, typically == #
105 * of disks/rank */
106 u_int numSpare; /* number of spare disks */
107 int maxQueueDepth; /* max disk queue depth */
108 RF_SectorCount_t totalSectors; /* total number of sectors in the
109 * array */
110 RF_SectorCount_t sectorsPerDisk; /* number of sectors on each
111 * disk */
112 u_int logBytesPerSector; /* base-2 log of the number of bytes
113 * in a sector */
114 u_int bytesPerSector; /* bytes in a sector */
115 RF_int32 sectorMask; /* mask of bytes-per-sector */
116
117 RF_RaidLayout_t Layout; /* all information related to layout */
118 RF_RaidDisk_t *Disks; /* all information related to physical disks */
119 RF_DiskQueue_t *Queues; /* all information related to disk queues */
120 u_int maxQueue; /* initialized queues in Queues array */
121 const RF_DiskQueueSW_t *qType;/* pointer to the DiskQueueSW used for the
122 component queues. */
123 /* NOTE: This is an anchor point via which the queues can be
124 * accessed, but the enqueue/dequeue routines in diskqueue.c use a
125 * local copy of this pointer for the actual accesses. */
126 /* The remainder of the structure can change, and therefore requires
127 * locking on reads and updates */
128 rf_declare_mutex2(mutex);/* mutex used to serialize access to
129 * the fields below */
130 RF_RowStatus_t status; /* the status of each row in the array */
131 int valid; /* indicates successful configuration */
132 RF_LockTableEntry_t *lockTable; /* stripe-lock table */
133 RF_LockTableEntry_t *quiesceLock; /* quiesnce table */
134 int numFailures; /* total number of failures in the array */
135 int numNewFailures; /* number of *new* failures (that havn't
136 caused a mod_counter update */
137
138 int parity_good; /* !0 if parity is known to be correct */
139 int serial_number; /* a "serial number" for this set */
140 int mod_counter; /* modification counter for component labels */
141 int clean; /* completely unused and should be removed */
142
143 int openings; /* Number of IO's which can be scheduled
144 simultaneously (high-level - not a
145 per-component limit)*/
146
147 int maxOutstanding; /* maxOutstanding requests (per-component) */
148 int autoconfigure; /* automatically configure this RAID set.
149 0 == no, 1 == yes */
150 int root_partition; /* Use this set as /
151 0 == no, 1 == yes*/
152 int last_unit; /* last unit number (e.g. 0 for /dev/raid0)
153 of this component. Used for autoconfigure
154 only. */
155 int config_order; /* 0 .. n. The order in which the component
156 should be auto-configured. E.g. 0 is will
157 done first, (and would become raid0).
158 This may be in conflict with last_unit!!?! */
159 /* Not currently used. */
160
161 /* queue to gather up requests from KernelWakeupFunc() and let
162 a kernel thread deal with calling rf_DiskIOComplete and any
163 callback functions. */
164 TAILQ_HEAD(iodone_q,RF_DiskQueueData_s) iodone;
165 /* and a lock/cv to protect it */
166 rf_declare_mutex2(iodone_lock);
167 rf_declare_cond2(iodone_cv);
168
169
170 RF_VoidPointerListElem_t *iobuf; /* I/O buffer free list */
171 int iobuf_count; /* count of I/O buffers on the freelist */
172 int numEmergencyBuffers; /* number of these buffers to pre-allocate */
173
174 RF_VoidPointerListElem_t *stripebuf; /* Full-stripe buffer free list */
175 int stripebuf_count; /* count of full-stripe buffers on the freelist */
176 int numEmergencyStripeBuffers; /* number of these buffers to pre-allocate */
177
178 /*
179 * Cleanup stuff
180 */
181 RF_ShutdownList_t *shutdownList; /* shutdown activities */
182 RF_AllocListElem_t *cleanupList; /* memory to be freed at
183 * shutdown time */
184
185 /*
186 * Recon stuff
187 */
188 RF_HeadSepLimit_t headSepLimit;
189 int numFloatingReconBufs;
190 int reconInProgress;
191 int forceRecon;
192 rf_declare_cond2(waitForReconCond); /* goes with raidPtr->mutex */
193 RF_RaidReconDesc_t *reconDesc; /* reconstruction descriptor */
194 RF_ReconCtrl_t *reconControl; /* reconstruction control structure
195 * pointers for each row in the array */
196
197 /*
198 * Array-quiescence stuff
199 */
200 rf_declare_mutex2(access_suspend_mutex);
201 rf_declare_cond2(access_suspend_cv);
202 RF_IoCount_t accesses_suspended;
203 RF_IoCount_t accs_in_flight;
204 int access_suspend_release;
205 int waiting_for_quiescence;
206 RF_CallbackFuncDesc_t *quiesce_wait_list;
207
208 /*
209 * Statistics
210 */
211 RF_StripeCount_t parity_rewrite_stripes_done;
212 RF_StripeCount_t copyback_stripes_done;
213
214 int recon_in_progress;
215 int parity_rewrite_in_progress;
216 int copyback_in_progress;
217 int changing_components;
218
219 rf_declare_cond2(parity_rewrite_cv);
220 rf_declare_cond2(changing_components_cv);
221
222 /*
223 * Engine thread control
224 */
225 rf_declare_mutex2(node_queue_mutex);
226 rf_declare_cond2(node_queue_cv);
227 RF_DagNode_t *node_queue;
228 RF_Thread_t parity_rewrite_thread;
229 RF_Thread_t copyback_thread;
230 RF_Thread_t engine_thread;
231 RF_Thread_t engine_helper_thread;
232 RF_Thread_t recon_thread;
233 int shutdown_engine;
234 int shutdown_raidio;
235 int dags_in_flight; /* debug */
236
237 /*
238 * PSS (Parity Stripe Status) stuff
239 */
240 long pssTableSize;
241
242 /*
243 * Reconstruction stuff
244 */
245 int procsInBufWait;
246 int numFullReconBuffers;
247 #if RF_ACC_TRACE > 0
248 RF_AccTraceEntry_t *recon_tracerecs;
249 #endif
250 unsigned long accumXorTimeUs;
251
252 /*
253 * nAccOutstanding, waitShutdown protected by desc freelist lock
254 * (This may seem strange, since that's a central serialization point
255 * for a per-array piece of data, but otherwise, it'd be an extra
256 * per-array lock, and that'd only be less efficient...)
257 */
258 rf_declare_mutex2(rad_lock);
259 rf_declare_cond2(outstandingCond);
260 int waitShutdown;
261 int nAccOutstanding;
262
263 int *abortRecon; /* Abort background operations requested */
264
265 RF_DiskId_t **diskids;
266
267 int raidid;
268 void *softc;
269 RF_AccTotals_t acc_totals;
270 int keep_acc_totals;
271
272 struct raidcinfo *raid_cinfo; /* array of component info */
273
274 int terminate_disk_queues;
275
276 /*
277 * XXX
278 *
279 * config-specific information should be moved
280 * somewhere else, or at least hung off this
281 * in some generic way
282 */
283 #if RF_INCLUDE_CHAINDECLUSTER > 0
284
285 /* used by rf_compute_workload_shift */
286 RF_RowCol_t hist_diskreq[RF_MAXCOL];
287 #endif
288 /* used by declustering */
289 int noRotate;
290
291 #if RF_INCLUDE_PARITYLOGGING > 0
292 /* used by parity logging */
293 RF_SectorCount_t regionLogCapacity;
294 RF_ParityLogQueue_t parityLogPool; /* pool of unused parity logs */
295 RF_RegionInfo_t *regionInfo; /* array of region state */
296 int numParityLogs;
297 int numSectorsPerLog;
298 int regionParityRange;
299 int logsInUse; /* debugging */
300 RF_ParityLogDiskQueue_t parityLogDiskQueue; /* state of parity
301 * logging disk work */
302 RF_RegionBufferQueue_t regionBufferPool; /* buffers for holding
303 * region log */
304 RF_RegionBufferQueue_t parityBufferPool; /* buffers for holding
305 * parity */
306 void *parityLogBufferHeap; /* pool of unused parity logs */
307 RF_Thread_t pLogDiskThreadHandle;
308
309 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
310 struct rf_paritymap *parity_map;
311 struct RF_Pools_s pools;
312 struct RF_PoolNames_s poolNames;
313 };
314
315 struct raid_softc {
316 struct dk_softc sc_dksc;
317 int sc_unit;
318 int sc_flags; /* flags */
319 int sc_cflags; /* configuration flags */
320 kmutex_t sc_mutex; /* interlock mutex */
321 kcondvar_t sc_cv; /* and the condvar */
322 uint64_t sc_size; /* size of the raid device */
323 char sc_xname[20]; /* XXX external name */
324 RF_Raid_t sc_r;
325 LIST_ENTRY(raid_softc) sc_link;
326 };
327 /* sc_flags */
328 #define RAIDF_INITED 0x01 /* unit has been initialized */
329 #define RAIDF_SHUTDOWN 0x02 /* unit is being shutdown */
330 #define RAIDF_DETACH 0x04 /* detach after final close */
331 #define RAIDF_WANTED 0x08 /* someone waiting to obtain a lock */
332 #define RAIDF_LOCKED 0x10 /* unit is locked */
333 #define RAIDF_UNIT_CHANGED 0x20 /* unit is being changed */
334
335
336 int rf_fail_disk(RF_Raid_t *, struct rf_recon_req *);
337
338 int rf_inited(const struct raid_softc *);
339 int rf_get_unit(const struct raid_softc *);
340 RF_Raid_t *rf_get_raid(struct raid_softc *);
341 int rf_construct(struct raid_softc *, RF_Config_t *);
342
343 #endif /* !_RF__RF_RAID_H_ */
344