rf_dagfuncs.c revision 1.29 1 1.29 christos /* $NetBSD: rf_dagfuncs.c,v 1.29 2007/03/04 06:02:36 christos Exp $ */
2 1.1 oster /*
3 1.1 oster * Copyright (c) 1995 Carnegie-Mellon University.
4 1.1 oster * All rights reserved.
5 1.1 oster *
6 1.1 oster * Author: Mark Holland, William V. Courtright II
7 1.1 oster *
8 1.1 oster * Permission to use, copy, modify and distribute this software and
9 1.1 oster * its documentation is hereby granted, provided that both the copyright
10 1.1 oster * notice and this permission notice appear in all copies of the
11 1.1 oster * software, derivative works or modified versions, and any portions
12 1.1 oster * thereof, and that both notices appear in supporting documentation.
13 1.1 oster *
14 1.1 oster * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
15 1.1 oster * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
16 1.1 oster * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
17 1.1 oster *
18 1.1 oster * Carnegie Mellon requests users of this software to return to
19 1.1 oster *
20 1.1 oster * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU
21 1.1 oster * School of Computer Science
22 1.1 oster * Carnegie Mellon University
23 1.1 oster * Pittsburgh PA 15213-3890
24 1.1 oster *
25 1.1 oster * any improvements or extensions that they make and grant Carnegie the
26 1.1 oster * rights to redistribute these changes.
27 1.1 oster */
28 1.1 oster
29 1.1 oster /*
30 1.1 oster * dagfuncs.c -- DAG node execution routines
31 1.1 oster *
32 1.1 oster * Rules:
33 1.1 oster * 1. Every DAG execution function must eventually cause node->status to
34 1.1 oster * get set to "good" or "bad", and "FinishNode" to be called. In the
35 1.1 oster * case of nodes that complete immediately (xor, NullNodeFunc, etc),
36 1.1 oster * the node execution function can do these two things directly. In
37 1.1 oster * the case of nodes that have to wait for some event (a disk read to
38 1.1 oster * complete, a lock to be released, etc) to occur before they can
39 1.1 oster * complete, this is typically achieved by having whatever module
40 1.1 oster * is doing the operation call GenericWakeupFunc upon completion.
41 1.1 oster * 2. DAG execution functions should check the status in the DAG header
42 1.1 oster * and NOP out their operations if the status is not "enable". However,
43 1.1 oster * execution functions that release resources must be sure to release
44 1.1 oster * them even when they NOP out the function that would use them.
45 1.1 oster * Functions that acquire resources should go ahead and acquire them
46 1.1 oster * even when they NOP, so that a downstream release node will not have
47 1.1 oster * to check to find out whether or not the acquire was suppressed.
48 1.1 oster */
49 1.8 lukem
50 1.8 lukem #include <sys/cdefs.h>
51 1.29 christos __KERNEL_RCSID(0, "$NetBSD: rf_dagfuncs.c,v 1.29 2007/03/04 06:02:36 christos Exp $");
52 1.1 oster
53 1.7 mrg #include <sys/param.h>
54 1.1 oster #include <sys/ioctl.h>
55 1.1 oster
56 1.1 oster #include "rf_archs.h"
57 1.1 oster #include "rf_raid.h"
58 1.1 oster #include "rf_dag.h"
59 1.1 oster #include "rf_layout.h"
60 1.1 oster #include "rf_etimer.h"
61 1.1 oster #include "rf_acctrace.h"
62 1.1 oster #include "rf_diskqueue.h"
63 1.1 oster #include "rf_dagfuncs.h"
64 1.1 oster #include "rf_general.h"
65 1.1 oster #include "rf_engine.h"
66 1.1 oster #include "rf_dagutils.h"
67 1.1 oster
68 1.1 oster #include "rf_kintf.h"
69 1.1 oster
70 1.1 oster #if RF_INCLUDE_PARITYLOGGING > 0
71 1.1 oster #include "rf_paritylog.h"
72 1.3 oster #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
73 1.1 oster
74 1.3 oster int (*rf_DiskReadFunc) (RF_DagNode_t *);
75 1.3 oster int (*rf_DiskWriteFunc) (RF_DagNode_t *);
76 1.3 oster int (*rf_DiskReadUndoFunc) (RF_DagNode_t *);
77 1.3 oster int (*rf_DiskWriteUndoFunc) (RF_DagNode_t *);
78 1.3 oster int (*rf_RegularXorUndoFunc) (RF_DagNode_t *);
79 1.3 oster int (*rf_SimpleXorUndoFunc) (RF_DagNode_t *);
80 1.3 oster int (*rf_RecoveryXorUndoFunc) (RF_DagNode_t *);
81 1.1 oster
82 1.14 oster /*****************************************************************************
83 1.1 oster * main (only) configuration routine for this module
84 1.14 oster ****************************************************************************/
85 1.23 perry int
86 1.28 christos rf_ConfigureDAGFuncs(RF_ShutdownList_t **listp)
87 1.3 oster {
88 1.23 perry RF_ASSERT(((sizeof(long) == 8) && RF_LONGSHIFT == 3) ||
89 1.14 oster ((sizeof(long) == 4) && RF_LONGSHIFT == 2));
90 1.3 oster rf_DiskReadFunc = rf_DiskReadFuncForThreads;
91 1.3 oster rf_DiskReadUndoFunc = rf_DiskUndoFunc;
92 1.3 oster rf_DiskWriteFunc = rf_DiskWriteFuncForThreads;
93 1.3 oster rf_DiskWriteUndoFunc = rf_DiskUndoFunc;
94 1.3 oster rf_RegularXorUndoFunc = rf_NullNodeUndoFunc;
95 1.3 oster rf_SimpleXorUndoFunc = rf_NullNodeUndoFunc;
96 1.3 oster rf_RecoveryXorUndoFunc = rf_NullNodeUndoFunc;
97 1.3 oster return (0);
98 1.1 oster }
99 1.1 oster
100 1.1 oster
101 1.1 oster
102 1.14 oster /*****************************************************************************
103 1.1 oster * the execution function associated with a terminate node
104 1.14 oster ****************************************************************************/
105 1.23 perry int
106 1.15 oster rf_TerminateFunc(RF_DagNode_t *node)
107 1.1 oster {
108 1.3 oster RF_ASSERT(node->dagHdr->numCommits == node->dagHdr->numCommitNodes);
109 1.3 oster node->status = rf_good;
110 1.3 oster return (rf_FinishNode(node, RF_THREAD_CONTEXT));
111 1.1 oster }
112 1.1 oster
113 1.23 perry int
114 1.28 christos rf_TerminateUndoFunc(RF_DagNode_t *node)
115 1.1 oster {
116 1.3 oster return (0);
117 1.1 oster }
118 1.1 oster
119 1.1 oster
120 1.15 oster /*****************************************************************************
121 1.1 oster * execution functions associated with a mirror node
122 1.1 oster *
123 1.1 oster * parameters:
124 1.1 oster *
125 1.1 oster * 0 - physical disk addres of data
126 1.1 oster * 1 - buffer for holding read data
127 1.1 oster * 2 - parity stripe ID
128 1.1 oster * 3 - flags
129 1.1 oster * 4 - physical disk address of mirror (parity)
130 1.1 oster *
131 1.15 oster ****************************************************************************/
132 1.1 oster
133 1.23 perry int
134 1.15 oster rf_DiskReadMirrorIdleFunc(RF_DagNode_t *node)
135 1.1 oster {
136 1.3 oster /* select the mirror copy with the shortest queue and fill in node
137 1.3 oster * parameters with physical disk address */
138 1.1 oster
139 1.3 oster rf_SelectMirrorDiskIdle(node);
140 1.3 oster return (rf_DiskReadFunc(node));
141 1.1 oster }
142 1.1 oster
143 1.11 oster #if (RF_INCLUDE_CHAINDECLUSTER > 0) || (RF_INCLUDE_INTERDECLUSTER > 0) || (RF_DEBUG_VALIDATE_DAG > 0)
144 1.23 perry int
145 1.15 oster rf_DiskReadMirrorPartitionFunc(RF_DagNode_t *node)
146 1.1 oster {
147 1.3 oster /* select the mirror copy with the shortest queue and fill in node
148 1.3 oster * parameters with physical disk address */
149 1.1 oster
150 1.3 oster rf_SelectMirrorDiskPartition(node);
151 1.3 oster return (rf_DiskReadFunc(node));
152 1.1 oster }
153 1.11 oster #endif
154 1.1 oster
155 1.23 perry int
156 1.28 christos rf_DiskReadMirrorUndoFunc(RF_DagNode_t *node)
157 1.1 oster {
158 1.3 oster return (0);
159 1.1 oster }
160 1.1 oster
161 1.1 oster
162 1.1 oster
163 1.1 oster #if RF_INCLUDE_PARITYLOGGING > 0
164 1.14 oster /*****************************************************************************
165 1.1 oster * the execution function associated with a parity log update node
166 1.14 oster ****************************************************************************/
167 1.23 perry int
168 1.15 oster rf_ParityLogUpdateFunc(RF_DagNode_t *node)
169 1.3 oster {
170 1.3 oster RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
171 1.29 christos void *bf = (void *) node->params[1].p;
172 1.3 oster RF_ParityLogData_t *logData;
173 1.19 oster #if RF_ACC_TRACE > 0
174 1.3 oster RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
175 1.3 oster RF_Etimer_t timer;
176 1.19 oster #endif
177 1.3 oster
178 1.3 oster if (node->dagHdr->status == rf_enable) {
179 1.19 oster #if RF_ACC_TRACE > 0
180 1.3 oster RF_ETIMER_START(timer);
181 1.19 oster #endif
182 1.24 christos logData = rf_CreateParityLogData(RF_UPDATE, pda, bf,
183 1.3 oster (RF_Raid_t *) (node->dagHdr->raidPtr),
184 1.3 oster node->wakeFunc, (void *) node,
185 1.3 oster node->dagHdr->tracerec, timer);
186 1.3 oster if (logData)
187 1.3 oster rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
188 1.3 oster else {
189 1.19 oster #if RF_ACC_TRACE > 0
190 1.3 oster RF_ETIMER_STOP(timer);
191 1.3 oster RF_ETIMER_EVAL(timer);
192 1.3 oster tracerec->plog_us += RF_ETIMER_VAL_US(timer);
193 1.19 oster #endif
194 1.3 oster (node->wakeFunc) (node, ENOMEM);
195 1.3 oster }
196 1.1 oster }
197 1.3 oster return (0);
198 1.1 oster }
199 1.1 oster
200 1.1 oster
201 1.15 oster /*****************************************************************************
202 1.1 oster * the execution function associated with a parity log overwrite node
203 1.15 oster ****************************************************************************/
204 1.23 perry int
205 1.15 oster rf_ParityLogOverwriteFunc(RF_DagNode_t *node)
206 1.3 oster {
207 1.3 oster RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
208 1.29 christos void *bf = (void *) node->params[1].p;
209 1.3 oster RF_ParityLogData_t *logData;
210 1.19 oster #if RF_ACC_TRACE > 0
211 1.3 oster RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
212 1.3 oster RF_Etimer_t timer;
213 1.19 oster #endif
214 1.3 oster
215 1.3 oster if (node->dagHdr->status == rf_enable) {
216 1.19 oster #if RF_ACC_TRACE > 0
217 1.3 oster RF_ETIMER_START(timer);
218 1.19 oster #endif
219 1.24 christos logData = rf_CreateParityLogData(RF_OVERWRITE, pda, bf,
220 1.14 oster (RF_Raid_t *) (node->dagHdr->raidPtr),
221 1.3 oster node->wakeFunc, (void *) node, node->dagHdr->tracerec, timer);
222 1.3 oster if (logData)
223 1.3 oster rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE);
224 1.3 oster else {
225 1.19 oster #if RF_ACC_TRACE > 0
226 1.3 oster RF_ETIMER_STOP(timer);
227 1.3 oster RF_ETIMER_EVAL(timer);
228 1.3 oster tracerec->plog_us += RF_ETIMER_VAL_US(timer);
229 1.19 oster #endif
230 1.3 oster (node->wakeFunc) (node, ENOMEM);
231 1.3 oster }
232 1.1 oster }
233 1.3 oster return (0);
234 1.1 oster }
235 1.1 oster
236 1.23 perry int
237 1.28 christos rf_ParityLogUpdateUndoFunc(RF_DagNode_t *node)
238 1.1 oster {
239 1.3 oster return (0);
240 1.1 oster }
241 1.1 oster
242 1.23 perry int
243 1.28 christos rf_ParityLogOverwriteUndoFunc(RF_DagNode_t *node)
244 1.1 oster {
245 1.3 oster return (0);
246 1.1 oster }
247 1.10 oster #endif /* RF_INCLUDE_PARITYLOGGING > 0 */
248 1.10 oster
249 1.14 oster /*****************************************************************************
250 1.1 oster * the execution function associated with a NOP node
251 1.14 oster ****************************************************************************/
252 1.23 perry int
253 1.15 oster rf_NullNodeFunc(RF_DagNode_t *node)
254 1.1 oster {
255 1.3 oster node->status = rf_good;
256 1.3 oster return (rf_FinishNode(node, RF_THREAD_CONTEXT));
257 1.1 oster }
258 1.1 oster
259 1.23 perry int
260 1.15 oster rf_NullNodeUndoFunc(RF_DagNode_t *node)
261 1.1 oster {
262 1.3 oster node->status = rf_undone;
263 1.3 oster return (rf_FinishNode(node, RF_THREAD_CONTEXT));
264 1.1 oster }
265 1.1 oster
266 1.1 oster
267 1.14 oster /*****************************************************************************
268 1.1 oster * the execution function associated with a disk-read node
269 1.14 oster ****************************************************************************/
270 1.23 perry int
271 1.15 oster rf_DiskReadFuncForThreads(RF_DagNode_t *node)
272 1.3 oster {
273 1.3 oster RF_DiskQueueData_t *req;
274 1.3 oster RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
275 1.29 christos void *bf = (void *) node->params[1].p;
276 1.3 oster RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
277 1.3 oster unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
278 1.3 oster unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
279 1.3 oster RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_READ : RF_IO_TYPE_NOP;
280 1.13 oster RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
281 1.3 oster void *b_proc = NULL;
282 1.1 oster
283 1.3 oster if (node->dagHdr->bp)
284 1.3 oster b_proc = (void *) ((struct buf *) node->dagHdr->bp)->b_proc;
285 1.1 oster
286 1.3 oster req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
287 1.24 christos bf, parityStripeID, which_ru,
288 1.3 oster (int (*) (void *, int)) node->wakeFunc,
289 1.22 oster node,
290 1.19 oster #if RF_ACC_TRACE > 0
291 1.19 oster node->dagHdr->tracerec,
292 1.19 oster #else
293 1.19 oster NULL,
294 1.19 oster #endif
295 1.21 oster (void *) (node->dagHdr->raidPtr), 0, b_proc, PR_NOWAIT);
296 1.3 oster if (!req) {
297 1.3 oster (node->wakeFunc) (node, ENOMEM);
298 1.3 oster } else {
299 1.3 oster node->dagFuncData = (void *) req;
300 1.13 oster rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority);
301 1.3 oster }
302 1.3 oster return (0);
303 1.1 oster }
304 1.1 oster
305 1.1 oster
306 1.14 oster /*****************************************************************************
307 1.1 oster * the execution function associated with a disk-write node
308 1.14 oster ****************************************************************************/
309 1.23 perry int
310 1.15 oster rf_DiskWriteFuncForThreads(RF_DagNode_t *node)
311 1.3 oster {
312 1.3 oster RF_DiskQueueData_t *req;
313 1.3 oster RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
314 1.29 christos void *bf = (void *) node->params[1].p;
315 1.3 oster RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v;
316 1.3 oster unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v);
317 1.3 oster unsigned which_ru = RF_EXTRACT_RU(node->params[3].v);
318 1.3 oster RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_WRITE : RF_IO_TYPE_NOP;
319 1.13 oster RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
320 1.3 oster void *b_proc = NULL;
321 1.1 oster
322 1.3 oster if (node->dagHdr->bp)
323 1.3 oster b_proc = (void *) ((struct buf *) node->dagHdr->bp)->b_proc;
324 1.1 oster
325 1.3 oster /* normal processing (rollaway or forward recovery) begins here */
326 1.3 oster req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector,
327 1.24 christos bf, parityStripeID, which_ru,
328 1.3 oster (int (*) (void *, int)) node->wakeFunc,
329 1.22 oster (void *) node,
330 1.19 oster #if RF_ACC_TRACE > 0
331 1.3 oster node->dagHdr->tracerec,
332 1.19 oster #else
333 1.19 oster NULL,
334 1.19 oster #endif
335 1.3 oster (void *) (node->dagHdr->raidPtr),
336 1.21 oster 0, b_proc, PR_NOWAIT);
337 1.3 oster
338 1.3 oster if (!req) {
339 1.3 oster (node->wakeFunc) (node, ENOMEM);
340 1.3 oster } else {
341 1.3 oster node->dagFuncData = (void *) req;
342 1.13 oster rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority);
343 1.3 oster }
344 1.3 oster
345 1.3 oster return (0);
346 1.1 oster }
347 1.14 oster /*****************************************************************************
348 1.1 oster * the undo function for disk nodes
349 1.1 oster * Note: this is not a proper undo of a write node, only locks are released.
350 1.1 oster * old data is not restored to disk!
351 1.14 oster ****************************************************************************/
352 1.23 perry int
353 1.15 oster rf_DiskUndoFunc(RF_DagNode_t *node)
354 1.3 oster {
355 1.3 oster RF_DiskQueueData_t *req;
356 1.3 oster RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p;
357 1.13 oster RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues;
358 1.3 oster
359 1.3 oster req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP,
360 1.3 oster 0L, 0, NULL, 0L, 0,
361 1.3 oster (int (*) (void *, int)) node->wakeFunc,
362 1.3 oster (void *) node,
363 1.19 oster #if RF_ACC_TRACE > 0
364 1.19 oster node->dagHdr->tracerec,
365 1.19 oster #else
366 1.19 oster NULL,
367 1.19 oster #endif
368 1.3 oster (void *) (node->dagHdr->raidPtr),
369 1.21 oster RF_UNLOCK_DISK_QUEUE, NULL, PR_NOWAIT);
370 1.3 oster if (!req)
371 1.3 oster (node->wakeFunc) (node, ENOMEM);
372 1.3 oster else {
373 1.3 oster node->dagFuncData = (void *) req;
374 1.13 oster rf_DiskIOEnqueue(&(dqs[pda->col]), req, RF_IO_NORMAL_PRIORITY);
375 1.3 oster }
376 1.1 oster
377 1.3 oster return (0);
378 1.1 oster }
379 1.3 oster
380 1.14 oster /*****************************************************************************
381 1.14 oster * Callback routine for DiskRead and DiskWrite nodes. When the disk
382 1.14 oster * op completes, the routine is called to set the node status and
383 1.14 oster * inform the execution engine that the node has fired.
384 1.14 oster ****************************************************************************/
385 1.23 perry int
386 1.15 oster rf_GenericWakeupFunc(RF_DagNode_t *node, int status)
387 1.3 oster {
388 1.15 oster
389 1.3 oster switch (node->status) {
390 1.3 oster case rf_fired:
391 1.3 oster if (status)
392 1.3 oster node->status = rf_bad;
393 1.3 oster else
394 1.3 oster node->status = rf_good;
395 1.3 oster break;
396 1.3 oster case rf_recover:
397 1.3 oster /* probably should never reach this case */
398 1.3 oster if (status)
399 1.3 oster node->status = rf_panic;
400 1.3 oster else
401 1.3 oster node->status = rf_undone;
402 1.3 oster break;
403 1.3 oster default:
404 1.4 oster printf("rf_GenericWakeupFunc:");
405 1.4 oster printf("node->status is %d,", node->status);
406 1.4 oster printf("status is %d \n", status);
407 1.3 oster RF_PANIC();
408 1.3 oster break;
409 1.3 oster }
410 1.3 oster if (node->dagFuncData)
411 1.3 oster rf_FreeDiskQueueData((RF_DiskQueueData_t *) node->dagFuncData);
412 1.3 oster return (rf_FinishNode(node, RF_INTR_CONTEXT));
413 1.1 oster }
414 1.1 oster
415 1.1 oster
416 1.14 oster /*****************************************************************************
417 1.14 oster * there are three distinct types of xor nodes:
418 1.14 oster
419 1.14 oster * A "regular xor" is used in the fault-free case where the access
420 1.14 oster * spans a complete stripe unit. It assumes that the result buffer is
421 1.14 oster * one full stripe unit in size, and uses the stripe-unit-offset
422 1.14 oster * values that it computes from the PDAs to determine where within the
423 1.14 oster * stripe unit to XOR each argument buffer.
424 1.14 oster *
425 1.14 oster * A "simple xor" is used in the fault-free case where the access
426 1.14 oster * touches only a portion of one (or two, in some cases) stripe
427 1.14 oster * unit(s). It assumes that all the argument buffers are of the same
428 1.14 oster * size and have the same stripe unit offset.
429 1.14 oster *
430 1.14 oster * A "recovery xor" is used in the degraded-mode case. It's similar
431 1.14 oster * to the regular xor function except that it takes the failed PDA as
432 1.14 oster * an additional parameter, and uses it to determine what portions of
433 1.14 oster * the argument buffers need to be xor'd into the result buffer, and
434 1.14 oster * where in the result buffer they should go.
435 1.14 oster ****************************************************************************/
436 1.1 oster
437 1.1 oster /* xor the params together and store the result in the result field.
438 1.14 oster * assume the result field points to a buffer that is the size of one
439 1.14 oster * SU, and use the pda params to determine where within the buffer to
440 1.14 oster * XOR the input buffers. */
441 1.23 perry int
442 1.15 oster rf_RegularXorFunc(RF_DagNode_t *node)
443 1.3 oster {
444 1.3 oster RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
445 1.19 oster #if RF_ACC_TRACE > 0
446 1.3 oster RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
447 1.3 oster RF_Etimer_t timer;
448 1.19 oster #endif
449 1.3 oster int i, retcode;
450 1.1 oster
451 1.3 oster retcode = 0;
452 1.3 oster if (node->dagHdr->status == rf_enable) {
453 1.3 oster /* don't do the XOR if the input is the same as the output */
454 1.19 oster #if RF_ACC_TRACE > 0
455 1.3 oster RF_ETIMER_START(timer);
456 1.19 oster #endif
457 1.3 oster for (i = 0; i < node->numParams - 1; i += 2)
458 1.3 oster if (node->params[i + 1].p != node->results[0]) {
459 1.3 oster retcode = rf_XorIntoBuffer(raidPtr, (RF_PhysDiskAddr_t *) node->params[i].p,
460 1.17 oster (char *) node->params[i + 1].p, (char *) node->results[0]);
461 1.3 oster }
462 1.19 oster #if RF_ACC_TRACE > 0
463 1.3 oster RF_ETIMER_STOP(timer);
464 1.3 oster RF_ETIMER_EVAL(timer);
465 1.3 oster tracerec->xor_us += RF_ETIMER_VAL_US(timer);
466 1.19 oster #endif
467 1.3 oster }
468 1.3 oster return (rf_GenericWakeupFunc(node, retcode)); /* call wake func
469 1.3 oster * explicitly since no
470 1.3 oster * I/O in this node */
471 1.1 oster }
472 1.1 oster /* xor the inputs into the result buffer, ignoring placement issues */
473 1.23 perry int
474 1.15 oster rf_SimpleXorFunc(RF_DagNode_t *node)
475 1.3 oster {
476 1.3 oster RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
477 1.3 oster int i, retcode = 0;
478 1.19 oster #if RF_ACC_TRACE > 0
479 1.3 oster RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
480 1.3 oster RF_Etimer_t timer;
481 1.19 oster #endif
482 1.1 oster
483 1.3 oster if (node->dagHdr->status == rf_enable) {
484 1.19 oster #if RF_ACC_TRACE > 0
485 1.3 oster RF_ETIMER_START(timer);
486 1.19 oster #endif
487 1.3 oster /* don't do the XOR if the input is the same as the output */
488 1.3 oster for (i = 0; i < node->numParams - 1; i += 2)
489 1.3 oster if (node->params[i + 1].p != node->results[0]) {
490 1.3 oster retcode = rf_bxor((char *) node->params[i + 1].p, (char *) node->results[0],
491 1.17 oster rf_RaidAddressToByte(raidPtr, ((RF_PhysDiskAddr_t *) node->params[i].p)->numSector));
492 1.3 oster }
493 1.19 oster #if RF_ACC_TRACE > 0
494 1.3 oster RF_ETIMER_STOP(timer);
495 1.3 oster RF_ETIMER_EVAL(timer);
496 1.3 oster tracerec->xor_us += RF_ETIMER_VAL_US(timer);
497 1.19 oster #endif
498 1.3 oster }
499 1.3 oster return (rf_GenericWakeupFunc(node, retcode)); /* call wake func
500 1.3 oster * explicitly since no
501 1.3 oster * I/O in this node */
502 1.1 oster }
503 1.14 oster /* this xor is used by the degraded-mode dag functions to recover lost
504 1.14 oster * data. the second-to-last parameter is the PDA for the failed
505 1.14 oster * portion of the access. the code here looks at this PDA and assumes
506 1.14 oster * that the xor target buffer is equal in size to the number of
507 1.14 oster * sectors in the failed PDA. It then uses the other PDAs in the
508 1.14 oster * parameter list to determine where within the target buffer the
509 1.14 oster * corresponding data should be xored. */
510 1.23 perry int
511 1.15 oster rf_RecoveryXorFunc(RF_DagNode_t *node)
512 1.3 oster {
513 1.3 oster RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p;
514 1.3 oster RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) & raidPtr->Layout;
515 1.3 oster RF_PhysDiskAddr_t *failedPDA = (RF_PhysDiskAddr_t *) node->params[node->numParams - 2].p;
516 1.3 oster int i, retcode = 0;
517 1.3 oster RF_PhysDiskAddr_t *pda;
518 1.3 oster int suoffset, failedSUOffset = rf_StripeUnitOffset(layoutPtr, failedPDA->startSector);
519 1.3 oster char *srcbuf, *destbuf;
520 1.19 oster #if RF_ACC_TRACE > 0
521 1.3 oster RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec;
522 1.3 oster RF_Etimer_t timer;
523 1.19 oster #endif
524 1.1 oster
525 1.3 oster if (node->dagHdr->status == rf_enable) {
526 1.19 oster #if RF_ACC_TRACE > 0
527 1.3 oster RF_ETIMER_START(timer);
528 1.19 oster #endif
529 1.3 oster for (i = 0; i < node->numParams - 2; i += 2)
530 1.3 oster if (node->params[i + 1].p != node->results[0]) {
531 1.3 oster pda = (RF_PhysDiskAddr_t *) node->params[i].p;
532 1.3 oster srcbuf = (char *) node->params[i + 1].p;
533 1.3 oster suoffset = rf_StripeUnitOffset(layoutPtr, pda->startSector);
534 1.3 oster destbuf = ((char *) node->results[0]) + rf_RaidAddressToByte(raidPtr, suoffset - failedSUOffset);
535 1.17 oster retcode = rf_bxor(srcbuf, destbuf, rf_RaidAddressToByte(raidPtr, pda->numSector));
536 1.3 oster }
537 1.19 oster #if RF_ACC_TRACE > 0
538 1.3 oster RF_ETIMER_STOP(timer);
539 1.3 oster RF_ETIMER_EVAL(timer);
540 1.3 oster tracerec->xor_us += RF_ETIMER_VAL_US(timer);
541 1.19 oster #endif
542 1.3 oster }
543 1.3 oster return (rf_GenericWakeupFunc(node, retcode));
544 1.1 oster }
545 1.14 oster /*****************************************************************************
546 1.14 oster * The next three functions are utilities used by the above
547 1.14 oster * xor-execution functions.
548 1.14 oster ****************************************************************************/
549 1.1 oster
550 1.1 oster
551 1.1 oster /*
552 1.14 oster * this is just a glorified buffer xor. targbuf points to a buffer
553 1.14 oster * that is one full stripe unit in size. srcbuf points to a buffer
554 1.14 oster * that may be less than 1 SU, but never more. When the access
555 1.14 oster * described by pda is one SU in size (which by implication means it's
556 1.14 oster * SU-aligned), all that happens is (targbuf) <- (srcbuf ^ targbuf).
557 1.14 oster * When the access is less than one SU in size the XOR occurs on only
558 1.14 oster * the portion of targbuf identified in the pda. */
559 1.1 oster
560 1.23 perry int
561 1.15 oster rf_XorIntoBuffer(RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda,
562 1.17 oster char *srcbuf, char *targbuf)
563 1.3 oster {
564 1.3 oster char *targptr;
565 1.3 oster int sectPerSU = raidPtr->Layout.sectorsPerStripeUnit;
566 1.3 oster int SUOffset = pda->startSector % sectPerSU;
567 1.3 oster int length, retcode = 0;
568 1.3 oster
569 1.3 oster RF_ASSERT(pda->numSector <= sectPerSU);
570 1.3 oster
571 1.3 oster targptr = targbuf + rf_RaidAddressToByte(raidPtr, SUOffset);
572 1.3 oster length = rf_RaidAddressToByte(raidPtr, pda->numSector);
573 1.17 oster retcode = rf_bxor(srcbuf, targptr, length);
574 1.3 oster return (retcode);
575 1.1 oster }
576 1.14 oster /* it really should be the case that the buffer pointers (returned by
577 1.14 oster * malloc) are aligned to the natural word size of the machine, so
578 1.14 oster * this is the only case we optimize for. The length should always be
579 1.14 oster * a multiple of the sector size, so there should be no problem with
580 1.14 oster * leftover bytes at the end. */
581 1.23 perry int
582 1.17 oster rf_bxor(char *src, char *dest, int len)
583 1.3 oster {
584 1.3 oster unsigned mask = sizeof(long) - 1, retcode = 0;
585 1.3 oster
586 1.23 perry if (!(((unsigned long) src) & mask) &&
587 1.14 oster !(((unsigned long) dest) & mask) && !(len & mask)) {
588 1.23 perry retcode = rf_longword_bxor((unsigned long *) src,
589 1.23 perry (unsigned long *) dest,
590 1.17 oster len >> RF_LONGSHIFT);
591 1.3 oster } else {
592 1.3 oster RF_ASSERT(0);
593 1.3 oster }
594 1.3 oster return (retcode);
595 1.1 oster }
596 1.1 oster
597 1.14 oster /* When XORing in kernel mode, we need to map each user page to kernel
598 1.14 oster * space before we can access it. We don't want to assume anything
599 1.14 oster * about which input buffers are in kernel/user space, nor about their
600 1.14 oster * alignment, so in each loop we compute the maximum number of bytes
601 1.14 oster * that we can xor without crossing any page boundaries, and do only
602 1.23 perry * this many bytes before the next remap.
603 1.23 perry *
604 1.23 perry * len - is in longwords
605 1.15 oster */
606 1.23 perry int
607 1.17 oster rf_longword_bxor(unsigned long *src, unsigned long *dest, int len)
608 1.3 oster {
609 1.6 augustss unsigned long *end = src + len;
610 1.6 augustss unsigned long d0, d1, d2, d3, s0, s1, s2, s3; /* temps */
611 1.14 oster unsigned long *pg_src, *pg_dest; /* per-page source/dest pointers */
612 1.3 oster int longs_this_time;/* # longwords to xor in the current iteration */
613 1.3 oster
614 1.16 oster pg_src = src;
615 1.16 oster pg_dest = dest;
616 1.3 oster if (!pg_src || !pg_dest)
617 1.3 oster return (EFAULT);
618 1.3 oster
619 1.3 oster while (len >= 4) {
620 1.3 oster longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(pg_src), RF_BLIP(pg_dest)) >> RF_LONGSHIFT); /* note len in longwords */
621 1.3 oster src += longs_this_time;
622 1.3 oster dest += longs_this_time;
623 1.3 oster len -= longs_this_time;
624 1.3 oster while (longs_this_time >= 4) {
625 1.3 oster d0 = pg_dest[0];
626 1.3 oster d1 = pg_dest[1];
627 1.3 oster d2 = pg_dest[2];
628 1.3 oster d3 = pg_dest[3];
629 1.3 oster s0 = pg_src[0];
630 1.3 oster s1 = pg_src[1];
631 1.3 oster s2 = pg_src[2];
632 1.3 oster s3 = pg_src[3];
633 1.3 oster pg_dest[0] = d0 ^ s0;
634 1.3 oster pg_dest[1] = d1 ^ s1;
635 1.3 oster pg_dest[2] = d2 ^ s2;
636 1.3 oster pg_dest[3] = d3 ^ s3;
637 1.3 oster pg_src += 4;
638 1.3 oster pg_dest += 4;
639 1.3 oster longs_this_time -= 4;
640 1.3 oster }
641 1.3 oster while (longs_this_time > 0) { /* cannot cross any page
642 1.3 oster * boundaries here */
643 1.3 oster *pg_dest++ ^= *pg_src++;
644 1.3 oster longs_this_time--;
645 1.3 oster }
646 1.3 oster
647 1.3 oster /* either we're done, or we've reached a page boundary on one
648 1.3 oster * (or possibly both) of the pointers */
649 1.3 oster if (len) {
650 1.3 oster if (RF_PAGE_ALIGNED(src))
651 1.16 oster pg_src = src;
652 1.3 oster if (RF_PAGE_ALIGNED(dest))
653 1.16 oster pg_dest = dest;
654 1.3 oster if (!pg_src || !pg_dest)
655 1.3 oster return (EFAULT);
656 1.3 oster }
657 1.3 oster }
658 1.3 oster while (src < end) {
659 1.3 oster *pg_dest++ ^= *pg_src++;
660 1.3 oster src++;
661 1.3 oster dest++;
662 1.3 oster len--;
663 1.3 oster if (RF_PAGE_ALIGNED(src))
664 1.16 oster pg_src = src;
665 1.3 oster if (RF_PAGE_ALIGNED(dest))
666 1.16 oster pg_dest = dest;
667 1.3 oster }
668 1.3 oster RF_ASSERT(len == 0);
669 1.3 oster return (0);
670 1.1 oster }
671 1.1 oster
672 1.9 oster #if 0
673 1.1 oster /*
674 1.1 oster dst = a ^ b ^ c;
675 1.1 oster a may equal dst
676 1.1 oster see comment above longword_bxor
677 1.15 oster len is length in longwords
678 1.1 oster */
679 1.23 perry int
680 1.15 oster rf_longword_bxor3(unsigned long *dst, unsigned long *a, unsigned long *b,
681 1.15 oster unsigned long *c, int len, void *bp)
682 1.3 oster {
683 1.3 oster unsigned long a0, a1, a2, a3, b0, b1, b2, b3;
684 1.6 augustss unsigned long *pg_a, *pg_b, *pg_c, *pg_dst; /* per-page source/dest
685 1.3 oster * pointers */
686 1.3 oster int longs_this_time;/* # longs to xor in the current iteration */
687 1.3 oster char dst_is_a = 0;
688 1.3 oster
689 1.16 oster pg_a = a;
690 1.16 oster pg_b = b;
691 1.16 oster pg_c = c;
692 1.3 oster if (a == dst) {
693 1.3 oster pg_dst = pg_a;
694 1.3 oster dst_is_a = 1;
695 1.3 oster } else {
696 1.16 oster pg_dst = dst;
697 1.3 oster }
698 1.3 oster
699 1.3 oster /* align dest to cache line. Can't cross a pg boundary on dst here. */
700 1.3 oster while ((((unsigned long) pg_dst) & 0x1f)) {
701 1.3 oster *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
702 1.3 oster dst++;
703 1.3 oster a++;
704 1.3 oster b++;
705 1.3 oster c++;
706 1.3 oster if (RF_PAGE_ALIGNED(a)) {
707 1.16 oster pg_a = a;
708 1.3 oster if (!pg_a)
709 1.3 oster return (EFAULT);
710 1.3 oster }
711 1.3 oster if (RF_PAGE_ALIGNED(b)) {
712 1.16 oster pg_b = a;
713 1.3 oster if (!pg_b)
714 1.3 oster return (EFAULT);
715 1.3 oster }
716 1.3 oster if (RF_PAGE_ALIGNED(c)) {
717 1.16 oster pg_c = a;
718 1.3 oster if (!pg_c)
719 1.3 oster return (EFAULT);
720 1.3 oster }
721 1.3 oster len--;
722 1.3 oster }
723 1.3 oster
724 1.3 oster while (len > 4) {
725 1.3 oster longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(a), RF_MIN(RF_BLIP(b), RF_MIN(RF_BLIP(c), RF_BLIP(dst)))) >> RF_LONGSHIFT);
726 1.3 oster a += longs_this_time;
727 1.3 oster b += longs_this_time;
728 1.3 oster c += longs_this_time;
729 1.3 oster dst += longs_this_time;
730 1.3 oster len -= longs_this_time;
731 1.3 oster while (longs_this_time >= 4) {
732 1.3 oster a0 = pg_a[0];
733 1.3 oster longs_this_time -= 4;
734 1.3 oster
735 1.3 oster a1 = pg_a[1];
736 1.3 oster a2 = pg_a[2];
737 1.3 oster
738 1.3 oster a3 = pg_a[3];
739 1.3 oster pg_a += 4;
740 1.3 oster
741 1.3 oster b0 = pg_b[0];
742 1.3 oster b1 = pg_b[1];
743 1.3 oster
744 1.3 oster b2 = pg_b[2];
745 1.3 oster b3 = pg_b[3];
746 1.3 oster /* start dual issue */
747 1.3 oster a0 ^= b0;
748 1.3 oster b0 = pg_c[0];
749 1.3 oster
750 1.3 oster pg_b += 4;
751 1.3 oster a1 ^= b1;
752 1.3 oster
753 1.3 oster a2 ^= b2;
754 1.3 oster a3 ^= b3;
755 1.3 oster
756 1.3 oster b1 = pg_c[1];
757 1.3 oster a0 ^= b0;
758 1.3 oster
759 1.3 oster b2 = pg_c[2];
760 1.3 oster a1 ^= b1;
761 1.3 oster
762 1.3 oster b3 = pg_c[3];
763 1.3 oster a2 ^= b2;
764 1.3 oster
765 1.3 oster pg_dst[0] = a0;
766 1.3 oster a3 ^= b3;
767 1.3 oster pg_dst[1] = a1;
768 1.3 oster pg_c += 4;
769 1.3 oster pg_dst[2] = a2;
770 1.3 oster pg_dst[3] = a3;
771 1.3 oster pg_dst += 4;
772 1.3 oster }
773 1.3 oster while (longs_this_time > 0) { /* cannot cross any page
774 1.3 oster * boundaries here */
775 1.3 oster *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
776 1.3 oster longs_this_time--;
777 1.3 oster }
778 1.3 oster
779 1.3 oster if (len) {
780 1.3 oster if (RF_PAGE_ALIGNED(a)) {
781 1.16 oster pg_a = a;
782 1.3 oster if (!pg_a)
783 1.3 oster return (EFAULT);
784 1.3 oster if (dst_is_a)
785 1.3 oster pg_dst = pg_a;
786 1.3 oster }
787 1.3 oster if (RF_PAGE_ALIGNED(b)) {
788 1.16 oster pg_b = b;
789 1.3 oster if (!pg_b)
790 1.3 oster return (EFAULT);
791 1.3 oster }
792 1.3 oster if (RF_PAGE_ALIGNED(c)) {
793 1.16 oster pg_c = c;
794 1.3 oster if (!pg_c)
795 1.3 oster return (EFAULT);
796 1.3 oster }
797 1.3 oster if (!dst_is_a)
798 1.3 oster if (RF_PAGE_ALIGNED(dst)) {
799 1.16 oster pg_dst = dst;
800 1.3 oster if (!pg_dst)
801 1.3 oster return (EFAULT);
802 1.3 oster }
803 1.3 oster }
804 1.3 oster }
805 1.3 oster while (len) {
806 1.3 oster *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++;
807 1.3 oster dst++;
808 1.3 oster a++;
809 1.3 oster b++;
810 1.3 oster c++;
811 1.3 oster if (RF_PAGE_ALIGNED(a)) {
812 1.16 oster pg_a = a;
813 1.3 oster if (!pg_a)
814 1.3 oster return (EFAULT);
815 1.3 oster if (dst_is_a)
816 1.3 oster pg_dst = pg_a;
817 1.3 oster }
818 1.3 oster if (RF_PAGE_ALIGNED(b)) {
819 1.16 oster pg_b = b;
820 1.3 oster if (!pg_b)
821 1.3 oster return (EFAULT);
822 1.3 oster }
823 1.3 oster if (RF_PAGE_ALIGNED(c)) {
824 1.16 oster pg_c = c;
825 1.3 oster if (!pg_c)
826 1.3 oster return (EFAULT);
827 1.3 oster }
828 1.3 oster if (!dst_is_a)
829 1.3 oster if (RF_PAGE_ALIGNED(dst)) {
830 1.16 oster pg_dst = dst;
831 1.3 oster if (!pg_dst)
832 1.3 oster return (EFAULT);
833 1.3 oster }
834 1.3 oster len--;
835 1.3 oster }
836 1.3 oster return (0);
837 1.3 oster }
838 1.3 oster
839 1.23 perry int
840 1.23 perry rf_bxor3(unsigned char *dst, unsigned char *a, unsigned char *b,
841 1.15 oster unsigned char *c, unsigned long len, void *bp)
842 1.1 oster {
843 1.3 oster RF_ASSERT(((RF_UL(dst) | RF_UL(a) | RF_UL(b) | RF_UL(c) | len) & 0x7) == 0);
844 1.1 oster
845 1.3 oster return (rf_longword_bxor3((unsigned long *) dst, (unsigned long *) a,
846 1.3 oster (unsigned long *) b, (unsigned long *) c, len >> RF_LONGSHIFT, bp));
847 1.1 oster }
848 1.9 oster #endif
849