rf_pqdegdags.c revision 1.1 1 /* $NetBSD: rf_pqdegdags.c,v 1.1 1998/11/13 04:20:32 oster Exp $ */
2 /*
3 * Copyright (c) 1995 Carnegie-Mellon University.
4 * All rights reserved.
5 *
6 * Author: Daniel Stodolsky
7 *
8 * Permission to use, copy, modify and distribute this software and
9 * its documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
17 *
18 * Carnegie Mellon requests users of this software to return to
19 *
20 * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU
21 * School of Computer Science
22 * Carnegie Mellon University
23 * Pittsburgh PA 15213-3890
24 *
25 * any improvements or extensions that they make and grant Carnegie the
26 * rights to redistribute these changes.
27 */
28
29 /*
30 * rf_pqdegdags.c
31 * Degraded mode dags for double fault cases.
32 */
33
34 /*
35 * :
36 * Log: rf_pqdegdags.c,v
37 * Revision 1.31 1996/11/05 21:10:40 jimz
38 * failed pda generalization
39 *
40 * Revision 1.30 1996/07/31 16:30:05 jimz
41 * asm/asmap fix
42 *
43 * Revision 1.29 1996/07/31 15:35:15 jimz
44 * evenodd changes; bugfixes for double-degraded archs, generalize
45 * some formerly PQ-only functions
46 *
47 * Revision 1.28 1996/07/28 20:31:39 jimz
48 * i386netbsd port
49 * true/false fixup
50 *
51 * Revision 1.27 1996/07/27 23:36:08 jimz
52 * Solaris port of simulator
53 *
54 * Revision 1.26 1996/07/22 19:52:16 jimz
55 * switched node params to RF_DagParam_t, a union of
56 * a 64-bit int and a void *, for better portability
57 * attempted hpux port, but failed partway through for
58 * lack of a single C compiler capable of compiling all
59 * source files
60 *
61 * Revision 1.25 1996/06/09 02:36:46 jimz
62 * lots of little crufty cleanup- fixup whitespace
63 * issues, comment #ifdefs, improve typing in some
64 * places (esp size-related)
65 *
66 * Revision 1.24 1996/06/07 22:26:27 jimz
67 * type-ify which_ru (RF_ReconUnitNum_t)
68 *
69 * Revision 1.23 1996/06/07 21:33:04 jimz
70 * begin using consistent types for sector numbers,
71 * stripe numbers, row+col numbers, recon unit numbers
72 *
73 * Revision 1.22 1996/06/02 17:31:48 jimz
74 * Moved a lot of global stuff into array structure, where it belongs.
75 * Fixed up paritylogging, pss modules in this manner. Some general
76 * code cleanup. Removed lots of dead code, some dead files.
77 *
78 * Revision 1.21 1996/05/31 22:26:54 jimz
79 * fix a lot of mapping problems, memory allocation problems
80 * found some weird lock issues, fixed 'em
81 * more code cleanup
82 *
83 * Revision 1.20 1996/05/30 12:59:18 jimz
84 * make etimer happier, more portable
85 *
86 * Revision 1.19 1996/05/30 11:29:41 jimz
87 * Numerous bug fixes. Stripe lock release code disagreed with the taking code
88 * about when stripes should be locked (I made it consistent: no parity, no lock)
89 * There was a lot of extra serialization of I/Os which I've removed- a lot of
90 * it was to calculate values for the cache code, which is no longer with us.
91 * More types, function, macro cleanup. Added code to properly quiesce the array
92 * on shutdown. Made a lot of stuff array-specific which was (bogusly) general
93 * before. Fixed memory allocation, freeing bugs.
94 *
95 * Revision 1.18 1996/05/27 18:56:37 jimz
96 * more code cleanup
97 * better typing
98 * compiles in all 3 environments
99 *
100 * Revision 1.17 1996/05/24 22:17:04 jimz
101 * continue code + namespace cleanup
102 * typed a bunch of flags
103 *
104 * Revision 1.16 1996/05/24 04:28:55 jimz
105 * release cleanup ckpt
106 *
107 * Revision 1.15 1996/05/23 21:46:35 jimz
108 * checkpoint in code cleanup (release prep)
109 * lots of types, function names have been fixed
110 *
111 * Revision 1.14 1996/05/23 00:33:23 jimz
112 * code cleanup: move all debug decls to rf_options.c, all extern
113 * debug decls to rf_options.h, all debug vars preceded by rf_
114 *
115 * Revision 1.13 1996/05/18 19:51:34 jimz
116 * major code cleanup- fix syntax, make some types consistent,
117 * add prototypes, clean out dead code, et cetera
118 *
119 * Revision 1.12 1996/05/08 21:01:24 jimz
120 * fixed up enum type names that were conflicting with other
121 * enums and function names (ie, "panic")
122 * future naming trends will be towards RF_ and rf_ for
123 * everything raidframe-related
124 *
125 * Revision 1.11 1996/05/03 19:47:50 wvcii
126 * removed include of rf_redstripe.h
127 *
128 * Revision 1.10 1995/12/12 18:10:06 jimz
129 * MIN -> RF_MIN, MAX -> RF_MAX, ASSERT -> RF_ASSERT
130 * fix 80-column brain damage in comments
131 *
132 * Revision 1.9 1995/11/30 16:17:57 wvcii
133 * added copyright info
134 *
135 * Revision 1.8 1995/11/07 15:33:25 wvcii
136 * dag creation routines now generate term node
137 * added asserts
138 * encoded commit point nodes, antecedence types into dags
139 * didn't add commit barrier - the code is a mess and needs to
140 * be cleand up first
141 *
142 */
143
144 #include "rf_archs.h"
145
146 #if (RF_INCLUDE_DECL_PQ > 0) || (RF_INCLUDE_RAID6 > 0)
147
148 #include "rf_types.h"
149 #include "rf_raid.h"
150 #include "rf_dag.h"
151 #include "rf_dagfuncs.h"
152 #include "rf_dagutils.h"
153 #include "rf_etimer.h"
154 #include "rf_acctrace.h"
155 #include "rf_general.h"
156 #include "rf_pqdegdags.h"
157 #include "rf_pq.h"
158 #include "rf_sys.h"
159
160 static void applyPDA(RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda, RF_PhysDiskAddr_t *ppda,
161 RF_PhysDiskAddr_t *qpda, void *bp);
162
163 /*
164 Two data drives have failed, and we are doing a read that covers one of them.
165 We may also be reading some of the surviving drives.
166
167
168 *****************************************************************************************
169 *
170 * creates a DAG to perform a degraded-mode read of data within one stripe.
171 * This DAG is as follows:
172 *
173 * Hdr
174 * |
175 * Block
176 * / / \ \ \ \
177 * Rud ... Rud Rrd ... Rrd Rp Rq
178 * | \ | \ | \ | \ | \ | \
179 *
180 * | |
181 * Unblock X
182 * \ /
183 * ------ T ------
184 *
185 * Each R node is a successor of the L node
186 * One successor arc from each R node goes to U, and the other to X
187 * There is one Rud for each chunk of surviving user data requested by the user,
188 * and one Rrd for each chunk of surviving user data _not_ being read by the user
189 * R = read, ud = user data, rd = recovery (surviving) data, p = P data, q = Qdata
190 * X = pq recovery node, T = terminate
191 *
192 * The block & unblock nodes are leftovers from a previous version. They
193 * do nothing, but I haven't deleted them because it would be a tremendous
194 * effort to put them back in.
195 *
196 * Note: The target buffer for the XOR node is set to the actual user buffer where the
197 * failed data is supposed to end up. This buffer is zero'd by the code here. Thus,
198 * if you create a degraded read dag, use it, and then re-use, you have to be sure to
199 * zero the target buffer prior to the re-use.
200 *
201 * Every buffer read is passed to the pq recovery node, whose job it is to sort out whats
202 * needs and what's not.
203 ****************************************************************************************/
204 /* init a disk node with 2 successors and one predecessor */
205 #define INIT_DISK_NODE(node,name) \
206 rf_InitNode(node, rf_wait, RF_FALSE, rf_DiskReadFunc, rf_DiskReadUndoFunc, rf_GenericWakeupFunc, 2,1,4,0, dag_h, name, allocList); \
207 (node)->succedents[0] = unblockNode; \
208 (node)->succedents[1] = recoveryNode; \
209 (node)->antecedents[0] = blockNode; \
210 (node)->antType[0] = rf_control
211
212 #define DISK_NODE_PARAMS(_node_,_p_) \
213 (_node_).params[0].p = _p_ ; \
214 (_node_).params[1].p = (_p_)->bufPtr; \
215 (_node_).params[2].v = parityStripeID; \
216 (_node_).params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0, 0, which_ru)
217
218 #define DISK_NODE_PDA(node) ((node)->params[0].p)
219
220 RF_CREATE_DAG_FUNC_DECL(rf_PQ_DoubleDegRead)
221 {
222 rf_DoubleDegRead(raidPtr, asmap, dag_h, bp, flags, allocList,
223 "Rq", "PQ Recovery", rf_PQDoubleRecoveryFunc);
224 }
225
226 static void applyPDA(raidPtr,pda,ppda,qpda, bp)
227 RF_Raid_t *raidPtr;
228 RF_PhysDiskAddr_t *pda;
229 RF_PhysDiskAddr_t *ppda;
230 RF_PhysDiskAddr_t *qpda;
231 void *bp;
232 {
233 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout);
234 RF_RaidAddr_t s0off = rf_StripeUnitOffset(layoutPtr, ppda->startSector);
235 RF_SectorCount_t s0len = ppda->numSector, len;
236 RF_SectorNum_t suoffset;
237 unsigned coeff;
238 char *pbuf = ppda->bufPtr;
239 char *qbuf = qpda->bufPtr;
240 char *buf;
241 int delta;
242
243 suoffset = rf_StripeUnitOffset(layoutPtr, pda->startSector);
244 len = pda->numSector;
245 /* see if pda intersects a recovery pda */
246 if ((suoffset < s0off+s0len) && ( suoffset+len > s0off))
247 {
248 buf = pda->bufPtr;
249 coeff = rf_RaidAddressToStripeUnitID(&(raidPtr->Layout),pda->raidAddress);
250 coeff = (coeff % raidPtr->Layout.numDataCol);
251
252 if (suoffset < s0off)
253 {
254 delta = s0off - suoffset;
255 buf += rf_RaidAddressToStripeUnitID(&(raidPtr->Layout),delta);
256 suoffset = s0off;
257 len -= delta;
258 }
259 if (suoffset > s0off)
260 {
261 delta = suoffset - s0off;
262 pbuf += rf_RaidAddressToStripeUnitID(&(raidPtr->Layout),delta);
263 qbuf += rf_RaidAddressToStripeUnitID(&(raidPtr->Layout),delta);
264 }
265 if ((suoffset + len) > (s0len + s0off))
266 len = s0len + s0off - suoffset;
267
268 /* src, dest, len */
269 rf_bxor(buf,pbuf,rf_RaidAddressToByte(raidPtr,len), bp);
270
271 /* dest, src, len, coeff */
272 rf_IncQ((unsigned long *)qbuf,(unsigned long *)buf,rf_RaidAddressToByte(raidPtr,len),coeff);
273 }
274 }
275 /*
276 Recover data in the case of a double failure. There can be two
277 result buffers, one for each chunk of data trying to be recovered.
278 The params are pda's that have not been range restricted or otherwise
279 politely massaged - this should be done here. The last params are the
280 pdas of P and Q, followed by the raidPtr. The list can look like
281
282 pda, pda, ... , p pda, q pda, raidptr, asm
283
284 or
285
286 pda, pda, ... , p_1 pda, p_2 pda, q_1 pda, q_2 pda, raidptr, asm
287
288 depending on wether two chunks of recovery data were required.
289
290 The second condition only arises if there are two failed buffers
291 whose lengths do not add up a stripe unit.
292 */
293
294
295 int rf_PQDoubleRecoveryFunc(node)
296 RF_DagNode_t *node;
297 {
298 int np = node->numParams;
299 RF_AccessStripeMap_t *asmap = (RF_AccessStripeMap_t *) node->params[np-1].p;
300 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[np-2].p;
301 RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) &(raidPtr->Layout);
302 int d, i;
303 unsigned coeff;
304 RF_RaidAddr_t sosAddr, suoffset;
305 RF_SectorCount_t len, secPerSU = layoutPtr->sectorsPerStripeUnit;
306 int two = 0;
307 RF_PhysDiskAddr_t *ppda,*ppda2,*qpda,*qpda2,*pda,npda;
308 char *buf;
309 int numDataCol = layoutPtr->numDataCol;
310 RF_Etimer_t timer;
311 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
312
313 RF_ETIMER_START(timer);
314
315 if (asmap->failedPDAs[1] &&
316 (asmap->failedPDAs[1]->numSector + asmap->failedPDAs[0]->numSector < secPerSU))
317 {
318 RF_ASSERT(0);
319 ppda = node->params[np-6].p;
320 ppda2 = node->params[np-5].p;
321 qpda = node->params[np-4].p;
322 qpda2 = node->params[np-3].p;
323 d = (np-6);
324 two = 1;
325 }
326 else
327 {
328 ppda = node->params[np-4].p;
329 qpda = node->params[np-3].p;
330 d = (np-4);
331 }
332
333 for (i=0; i < d; i++)
334 {
335 pda = node->params[i].p;
336 buf = pda->bufPtr;
337 suoffset = rf_StripeUnitOffset(layoutPtr, pda->startSector);
338 len = pda->numSector;
339 coeff = rf_RaidAddressToStripeUnitID(layoutPtr,pda->raidAddress);
340 /* compute the data unit offset within the column */
341 coeff = (coeff % raidPtr->Layout.numDataCol);
342 /* see if pda intersects a recovery pda */
343 applyPDA(raidPtr,pda,ppda,qpda,node->dagHdr->bp);
344 if (two)
345 applyPDA(raidPtr,pda,ppda,qpda,node->dagHdr->bp);
346 }
347
348 /* ok, we got the parity back to the point where we can recover.
349 We now need to determine the coeff of the columns that need to be
350 recovered. We can also only need to recover a single stripe unit.
351 */
352
353 if (asmap->failedPDAs[1] == NULL)
354 { /* only a single stripe unit to recover. */
355 pda = asmap->failedPDAs[0];
356 sosAddr = rf_RaidAddressOfPrevStripeBoundary(layoutPtr, asmap->raidAddress);
357 /* need to determine the column of the other failed disk */
358 coeff = rf_RaidAddressToStripeUnitID(layoutPtr,pda->raidAddress);
359 /* compute the data unit offset within the column */
360 coeff = (coeff % raidPtr->Layout.numDataCol);
361 for (i=0; i < numDataCol; i++)
362 {
363 npda.raidAddress = sosAddr + (i * secPerSU);
364 (raidPtr->Layout.map->MapSector)(raidPtr,npda.raidAddress, &(npda.row), &(npda.col), &(npda.startSector), 0);
365 /* skip over dead disks */
366 if (RF_DEAD_DISK(raidPtr->Disks[npda.row][npda.col].status))
367 if (i != coeff) break;
368 }
369 RF_ASSERT (i < numDataCol);
370 RF_ASSERT (two==0);
371 /* recover the data. Since we need only want to recover one column, we overwrite the
372 parity with the other one. */
373 if (coeff < i) /* recovering 'a' */
374 rf_PQ_recover((unsigned long *)ppda->bufPtr,(unsigned long *)qpda->bufPtr,(unsigned long *)pda->bufPtr,(unsigned long *)ppda->bufPtr,rf_RaidAddressToByte(raidPtr,pda->numSector), coeff, i);
375 else /* recovering 'b' */
376 rf_PQ_recover((unsigned long *)ppda->bufPtr,(unsigned long *)qpda->bufPtr,(unsigned long *)ppda->bufPtr,(unsigned long *)pda->bufPtr,rf_RaidAddressToByte(raidPtr,pda->numSector), i, coeff);
377 }
378 else
379 RF_PANIC();
380
381 RF_ETIMER_STOP(timer);
382 RF_ETIMER_EVAL(timer);
383 if (tracerec)
384 tracerec->q_us += RF_ETIMER_VAL_US(timer);
385 rf_GenericWakeupFunc(node,0);
386 return(0);
387 }
388
389 int rf_PQWriteDoubleRecoveryFunc(node)
390 RF_DagNode_t *node;
391 {
392 /* The situation:
393
394 We are doing a write that hits only one
395 failed data unit.
396 The other failed data unit is not being overwritten, so
397 we need to generate it.
398
399 For the moment, we assume all the nonfailed data being
400 written is in the shadow of the failed data unit.
401 (i.e,, either a single data unit write or the entire
402 failed stripe unit is being overwritten. )
403
404 Recovery strategy:
405 apply the recovery data to the parity and q.
406 Use P & Q to recover the second failed data unit in P.
407 Zero fill Q, then apply the recovered data to p.
408 Then apply the data being written to the failed drive.
409 Then walk through the surviving drives, applying new data
410 when it exists, othewise the recovery data. Quite a mess.
411
412
413 The params
414
415 read pda0, read pda1, ... read pda (numDataCol-3),
416 write pda0, ... , write pda (numStripeUnitAccess - numDataFailed),
417 failed pda, raidPtr, asmap
418 */
419
420 int np = node->numParams;
421 RF_AccessStripeMap_t *asmap = (RF_AccessStripeMap_t *) node->params[np-1].p;
422 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[np-2].p;
423 RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) &(raidPtr->Layout);
424 int i;
425 RF_RaidAddr_t sosAddr;
426 unsigned coeff;
427 RF_StripeCount_t secPerSU = layoutPtr->sectorsPerStripeUnit;
428 RF_PhysDiskAddr_t *ppda,*qpda,*pda,npda;
429 int numDataCol = layoutPtr->numDataCol;
430 RF_Etimer_t timer;
431 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
432
433 RF_ASSERT(node->numResults == 2);
434 RF_ASSERT(asmap->failedPDAs[1] == NULL);
435 RF_ETIMER_START(timer);
436 ppda = node->results[0];
437 qpda = node->results[1];
438 /* apply the recovery data */
439 for (i=0; i < numDataCol-2; i++)
440 applyPDA(raidPtr,node->params[i].p,ppda,qpda, node->dagHdr->bp);
441
442 /* determine the other failed data unit */
443 pda = asmap->failedPDAs[0];
444 sosAddr = rf_RaidAddressOfPrevStripeBoundary(layoutPtr, asmap->raidAddress);
445 /* need to determine the column of the other failed disk */
446 coeff = rf_RaidAddressToStripeUnitID(layoutPtr,pda->raidAddress);
447 /* compute the data unit offset within the column */
448 coeff = (coeff % raidPtr->Layout.numDataCol);
449 for (i=0; i < numDataCol; i++)
450 {
451 npda.raidAddress = sosAddr + (i * secPerSU);
452 (raidPtr->Layout.map->MapSector)(raidPtr,npda.raidAddress, &(npda.row), &(npda.col), &(npda.startSector), 0);
453 /* skip over dead disks */
454 if (RF_DEAD_DISK(raidPtr->Disks[npda.row][npda.col].status))
455 if (i != coeff) break;
456 }
457 RF_ASSERT (i < numDataCol);
458 /* recover the data. The column we want to recover we write over the parity.
459 The column we don't care about we dump in q. */
460 if (coeff < i) /* recovering 'a' */
461 rf_PQ_recover((unsigned long *)ppda->bufPtr,(unsigned long *)qpda->bufPtr,(unsigned long *)ppda->bufPtr,(unsigned long *)qpda->bufPtr,rf_RaidAddressToByte(raidPtr,pda->numSector), coeff, i);
462 else /* recovering 'b' */
463 rf_PQ_recover((unsigned long *)ppda->bufPtr,(unsigned long *)qpda->bufPtr,(unsigned long *)qpda->bufPtr,(unsigned long *)ppda->bufPtr,rf_RaidAddressToByte(raidPtr,pda->numSector), i, coeff);
464
465 /* OK. The valid data is in P. Zero fill Q, then inc it into it. */
466 bzero(qpda->bufPtr,rf_RaidAddressToByte(raidPtr,qpda->numSector));
467 rf_IncQ((unsigned long *)qpda->bufPtr,(unsigned long *)ppda->bufPtr,rf_RaidAddressToByte(raidPtr,qpda->numSector),i);
468
469 /* now apply all the write data to the buffer */
470 /* single stripe unit write case: the failed data is only thing we are writing. */
471 RF_ASSERT(asmap->numStripeUnitsAccessed == 1);
472 /* dest, src, len, coeff */
473 rf_IncQ((unsigned long *)qpda->bufPtr,(unsigned long *)asmap->failedPDAs[0]->bufPtr,rf_RaidAddressToByte(raidPtr,qpda->numSector),coeff);
474 rf_bxor(asmap->failedPDAs[0]->bufPtr,ppda->bufPtr,rf_RaidAddressToByte(raidPtr,ppda->numSector),node->dagHdr->bp);
475
476 /* now apply all the recovery data */
477 for (i=0; i < numDataCol-2; i++)
478 applyPDA(raidPtr,node->params[i].p,ppda,qpda, node->dagHdr->bp);
479
480 RF_ETIMER_STOP(timer);
481 RF_ETIMER_EVAL(timer);
482 if (tracerec)
483 tracerec->q_us += RF_ETIMER_VAL_US(timer);
484
485 rf_GenericWakeupFunc(node,0);
486 return(0);
487 }
488
489 RF_CREATE_DAG_FUNC_DECL(rf_PQ_DDLargeWrite)
490 {
491 RF_PANIC();
492 }
493
494 /*
495 Two lost data unit write case.
496
497 There are really two cases here:
498
499 (1) The write completely covers the two lost data units.
500 In that case, a reconstruct write that doesn't write the
501 failed data units will do the correct thing. So in this case,
502 the dag looks like
503
504 full stripe read of surviving data units (not being overwriten)
505 write new data (ignoring failed units) compute P&Q
506 write P&Q
507
508
509 (2) The write does not completely cover both failed data units
510 (but touches at least one of them). Then we need to do the
511 equivalent of a reconstruct read to recover the missing data
512 unit from the other stripe.
513
514 For any data we are writing that is not in the "shadow"
515 of the failed units, we need to do a four cycle update.
516 PANIC on this case. for now
517
518 */
519
520 RF_CREATE_DAG_FUNC_DECL(rf_PQ_200_CreateWriteDAG)
521 {
522 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout);
523 RF_SectorCount_t sectorsPerSU = layoutPtr->sectorsPerStripeUnit;
524 int sum;
525 int nf = asmap->numDataFailed;
526
527 sum = asmap->failedPDAs[0]->numSector;
528 if (nf == 2)
529 sum += asmap->failedPDAs[1]->numSector;
530
531 if ((nf == 2) && ( sum == (2*sectorsPerSU)))
532 {
533 /* large write case */
534 rf_PQ_DDLargeWrite(raidPtr, asmap, dag_h, bp, flags, allocList);
535 return;
536 }
537
538
539 if ((nf == asmap->numStripeUnitsAccessed) || (sum >= sectorsPerSU))
540 {
541 /* small write case, no user data not in shadow */
542 rf_PQ_DDSimpleSmallWrite(raidPtr, asmap, dag_h, bp, flags, allocList);
543 return;
544 }
545 RF_PANIC();
546 }
547
548 RF_CREATE_DAG_FUNC_DECL(rf_PQ_DDSimpleSmallWrite)
549 {
550 rf_DoubleDegSmallWrite(raidPtr, asmap, dag_h, bp, flags, allocList, "Rq", "Wq", "PQ Recovery", rf_PQWriteDoubleRecoveryFunc);
551 }
552
553 #endif /* (RF_INCLUDE_DECL_PQ > 0) || (RF_INCLUDE_RAID6 > 0) */
554