Home | History | Annotate | Line # | Download | only in udf
udf_strat_rmw.c revision 1.24
      1 /* $NetBSD: udf_strat_rmw.c,v 1.23 2013/10/18 19:56:55 christos Exp $ */
      2 
      3 /*
      4  * Copyright (c) 2006, 2008 Reinoud Zandijk
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  *
     27  */
     28 
     29 #include <sys/cdefs.h>
     30 #ifndef lint
     31 __KERNEL_RCSID(0, "$NetBSD: udf_strat_rmw.c,v 1.23 2013/10/18 19:56:55 christos Exp $");
     32 #endif /* not lint */
     33 
     34 
     35 #if defined(_KERNEL_OPT)
     36 #include "opt_compat_netbsd.h"
     37 #endif
     38 
     39 #include <sys/param.h>
     40 #include <sys/systm.h>
     41 #include <sys/sysctl.h>
     42 #include <sys/namei.h>
     43 #include <sys/proc.h>
     44 #include <sys/kernel.h>
     45 #include <sys/vnode.h>
     46 #include <miscfs/genfs/genfs_node.h>
     47 #include <sys/mount.h>
     48 #include <sys/buf.h>
     49 #include <sys/file.h>
     50 #include <sys/device.h>
     51 #include <sys/disklabel.h>
     52 #include <sys/ioctl.h>
     53 #include <sys/malloc.h>
     54 #include <sys/dirent.h>
     55 #include <sys/stat.h>
     56 #include <sys/conf.h>
     57 #include <sys/kauth.h>
     58 #include <sys/kthread.h>
     59 #include <dev/clock_subr.h>
     60 
     61 #include <fs/udf/ecma167-udf.h>
     62 #include <fs/udf/udf_mount.h>
     63 
     64 #include "udf.h"
     65 #include "udf_subr.h"
     66 #include "udf_bswap.h"
     67 
     68 
     69 #define VTOI(vnode) ((struct udf_node *) (vnode)->v_data)
     70 #define PRIV(ump) ((struct strat_private *) (ump)->strategy_private)
     71 #define BTOE(buf) ((struct udf_eccline *) ((buf)->b_private))
     72 
     73 /* --------------------------------------------------------------------- */
     74 
     75 #define UDF_MAX_PACKET_SIZE	64			/* DONT change this */
     76 
     77 /* sheduler states */
     78 #define UDF_SHED_WAITING	1			/* waiting on timeout */
     79 #define UDF_SHED_READING	2
     80 #define UDF_SHED_WRITING	3
     81 #define UDF_SHED_SEQWRITING	4
     82 #define UDF_SHED_IDLE		5			/* refcnt'd */
     83 #define UDF_SHED_FREE		6			/* recycleable */
     84 #define UDF_SHED_MAX		6+1
     85 
     86 /* flags */
     87 #define ECC_LOCKED		0x01			/* prevent access   */
     88 #define ECC_WANTED		0x02			/* trying access    */
     89 #define ECC_SEQWRITING		0x04			/* sequential queue */
     90 #define ECC_FLOATING		0x08			/* not queued yet   */
     91 
     92 #define ECC_WAITTIME		10
     93 
     94 
     95 TAILQ_HEAD(ecclineq, udf_eccline);
     96 struct udf_eccline {
     97 	struct udf_mount	 *ump;
     98 	uint64_t		  present;		/* preserve these */
     99 	uint64_t		  readin;		/* bitmap */
    100 	uint64_t		  dirty;		/* bitmap */
    101 	uint64_t		  error;		/* bitmap */
    102 	uint32_t		  refcnt;
    103 
    104 	struct timespec		  wait_time;
    105 	uint32_t		  flags;
    106 	uint32_t		  start_sector;		/* physical */
    107 
    108 	const char		 *fname;
    109 	int			  sline;
    110 
    111 	struct buf		 *buf;
    112 	void			 *blob;
    113 
    114 	struct buf		 *bufs[UDF_MAX_PACKET_SIZE];
    115 	uint32_t		  bufs_bpos[UDF_MAX_PACKET_SIZE];
    116 	int			  bufs_len[UDF_MAX_PACKET_SIZE];
    117 
    118 	int			  queued_on;		/* on which BUFQ list */
    119 	LIST_ENTRY(udf_eccline)   hashchain;		/* on sector lookup  */
    120 };
    121 
    122 
    123 struct strat_private {
    124 	lwp_t			 *queue_lwp;
    125 	kcondvar_t		  discstrat_cv;		/* to wait on       */
    126 	kmutex_t		  discstrat_mutex;	/* disc strategy    */
    127 	kmutex_t		  seqwrite_mutex;	/* protect mappings */
    128 
    129 	int			  thread_running;	/* thread control */
    130 	int			  run_thread;		/* thread control */
    131 	int			  thread_finished;	/* thread control */
    132 	int			  cur_queue;
    133 
    134 	int			  num_floating;
    135 	int			  num_queued[UDF_SHED_MAX];
    136 	struct bufq_state	 *queues[UDF_SHED_MAX];
    137 	struct timespec		  last_queued[UDF_SHED_MAX];
    138 	struct disk_strategy	  old_strategy_setting;
    139 
    140 	struct pool		  eccline_pool;
    141 	struct pool		  ecclineblob_pool;
    142 	LIST_HEAD(, udf_eccline)  eccline_hash[UDF_ECCBUF_HASHSIZE];
    143 };
    144 
    145 /* --------------------------------------------------------------------- */
    146 
    147 #define UDF_LOCK_ECCLINE(eccline) udf_lock_eccline(eccline, __FILE__, __LINE__)
    148 #define UDF_UNLOCK_ECCLINE(eccline) udf_unlock_eccline(eccline, __FILE__, __LINE__)
    149 
    150 /* can be called with or without discstrat lock */
    151 static void
    152 udf_lock_eccline(struct udf_eccline *eccline, const char *fname, int sline)
    153 {
    154 	struct strat_private *priv = PRIV(eccline->ump);
    155 	int waslocked, ret;
    156 
    157 	KASSERT(mutex_owned(&priv->discstrat_mutex));
    158 
    159 	waslocked = mutex_owned(&priv->discstrat_mutex);
    160 	if (!waslocked)
    161 		mutex_enter(&priv->discstrat_mutex);
    162 
    163 	/* wait until its unlocked first */
    164 	eccline->refcnt++;
    165 	while (eccline->flags & ECC_LOCKED) {
    166 		DPRINTF(ECCLINE, ("waiting for lock at %s:%d\n",
    167 					fname, sline));
    168 		DPRINTF(ECCLINE, ("was locked at %s:%d\n",
    169 					eccline->fname, eccline->sline));
    170 		eccline->flags |= ECC_WANTED;
    171 		ret = cv_timedwait(&priv->discstrat_cv, &priv->discstrat_mutex,
    172 			hz/8);
    173 		if (ret == EWOULDBLOCK)
    174 			DPRINTF(LOCKING, ("eccline lock helt, waiting for "
    175 				"release"));
    176 	}
    177 	eccline->flags |= ECC_LOCKED;
    178 	eccline->flags &= ~ECC_WANTED;
    179 	eccline->refcnt--;
    180 
    181 	eccline->fname = fname;
    182 	eccline->sline = sline;
    183 
    184 	if (!waslocked)
    185 		mutex_exit(&priv->discstrat_mutex);
    186 }
    187 
    188 
    189 /* can be called with or without discstrat lock */
    190 static void
    191 udf_unlock_eccline(struct udf_eccline *eccline, const char *fname, int sline)
    192 {
    193 	struct strat_private *priv = PRIV(eccline->ump);
    194 	int waslocked;
    195 
    196 	KASSERT(mutex_owned(&priv->discstrat_mutex));
    197 
    198 	waslocked = mutex_owned(&priv->discstrat_mutex);
    199 	if (!waslocked)
    200 		mutex_enter(&priv->discstrat_mutex);
    201 
    202 	eccline->flags &= ~ECC_LOCKED;
    203 	cv_broadcast(&priv->discstrat_cv);
    204 
    205 	if (!waslocked)
    206 		mutex_exit(&priv->discstrat_mutex);
    207 }
    208 
    209 
    210 /* NOTE discstrat_mutex should be held! */
    211 static void
    212 udf_dispose_eccline(struct udf_eccline *eccline)
    213 {
    214 	struct strat_private *priv = PRIV(eccline->ump);
    215 
    216 	KASSERT(mutex_owned(&priv->discstrat_mutex));
    217 
    218 	DPRINTF(ECCLINE, ("dispose eccline with start sector %d, "
    219 		"present %0"PRIx64"\n", eccline->start_sector,
    220 		eccline->present));
    221 
    222 	KASSERT(eccline->refcnt == 0);
    223 	KASSERT(eccline->dirty  == 0);
    224 	KASSERT(eccline->queued_on == 0);
    225 	KASSERT(eccline->flags & ECC_FLOATING);
    226 	KASSERT(eccline->flags & ECC_LOCKED);
    227 
    228 	LIST_REMOVE(eccline, hashchain);
    229 	priv->num_floating--;
    230 
    231 	putiobuf(eccline->buf);
    232 	pool_put(&priv->ecclineblob_pool, eccline->blob);
    233 	pool_put(&priv->eccline_pool, eccline);
    234 }
    235 
    236 
    237 /* NOTE discstrat_mutex should be held! */
    238 static void
    239 udf_push_eccline(struct udf_eccline *eccline, int newqueue)
    240 {
    241 	struct strat_private *priv = PRIV(eccline->ump);
    242 
    243 	KASSERT(mutex_owned(&priv->discstrat_mutex));
    244 
    245 	DPRINTF(PARANOIA, ("DEBUG: buf %p pushed on queue %d\n", eccline->buf, newqueue));
    246 
    247 	KASSERT(eccline->queued_on == 0);
    248 	KASSERT(eccline->flags & ECC_FLOATING);
    249 
    250 	/* set buffer block numbers to make sure its queued correctly */
    251 	eccline->buf->b_lblkno   = eccline->start_sector;
    252 	eccline->buf->b_blkno    = eccline->start_sector;
    253 	eccline->buf->b_rawblkno = eccline->start_sector;
    254 
    255 	vfs_timestamp(&priv->last_queued[newqueue]);
    256 	eccline->flags &= ~ECC_FLOATING;
    257 	priv->num_floating--;
    258 	eccline->queued_on = newqueue;
    259 	priv->num_queued[newqueue]++;
    260 	bufq_put(priv->queues[newqueue], eccline->buf);
    261 
    262 	UDF_UNLOCK_ECCLINE(eccline);
    263 
    264 	/* XXX tickle disc strategy statemachine */
    265 	if (newqueue != UDF_SHED_IDLE)
    266 		cv_signal(&priv->discstrat_cv);
    267 }
    268 
    269 
    270 static struct udf_eccline *
    271 udf_peek_eccline(struct strat_private *priv, int queued_on)
    272 {
    273 	struct udf_eccline *eccline;
    274 	struct buf *buf;
    275 
    276 	KASSERT(mutex_owned(&priv->discstrat_mutex));
    277 
    278 	for(;;) {
    279 		buf = bufq_peek(priv->queues[queued_on]);
    280 		/* could have been a race, but we'll revisit later */
    281 		if (buf == NULL)
    282 			return NULL;
    283 
    284 		eccline = BTOE(buf);
    285 		UDF_LOCK_ECCLINE(eccline);
    286 
    287 		/* might have changed before we obtained the lock */
    288 		if (eccline->queued_on == queued_on)
    289 			break;
    290 
    291 		UDF_UNLOCK_ECCLINE(eccline);
    292 	}
    293 
    294 	KASSERT(eccline->queued_on == queued_on);
    295 	KASSERT((eccline->flags & ECC_FLOATING) == 0);
    296 
    297 	DPRINTF(PARANOIA, ("DEBUG: buf %p peeked at queue %d\n",
    298 		eccline->buf, queued_on));
    299 
    300 	return eccline;
    301 }
    302 
    303 
    304 static struct udf_eccline *
    305 udf_pop_eccline(struct strat_private *priv, int queued_on)
    306 {
    307 	struct udf_eccline *eccline;
    308 	struct buf *buf;
    309 
    310 	KASSERT(mutex_owned(&priv->discstrat_mutex));
    311 
    312 	for(;;) {
    313 		buf = bufq_get(priv->queues[queued_on]);
    314 		if (buf == NULL) {
    315 			// KASSERT(priv->num_queued[queued_on] == 0);
    316 			return NULL;
    317 		}
    318 
    319 		eccline = BTOE(buf);
    320 		UDF_LOCK_ECCLINE(eccline);
    321 
    322 		/* might have changed before we obtained the lock */
    323 		if (eccline->queued_on == queued_on)
    324 			break;
    325 
    326 		UDF_UNLOCK_ECCLINE(eccline);
    327 	}
    328 
    329 	KASSERT(eccline->queued_on == queued_on);
    330 	KASSERT((eccline->flags & ECC_FLOATING) == 0);
    331 
    332 	priv->num_queued[queued_on]--;
    333 	eccline->queued_on = 0;
    334 
    335 	eccline->flags |= ECC_FLOATING;
    336 	priv->num_floating++;
    337 
    338 	DPRINTF(PARANOIA, ("DEBUG: buf %p popped from queue %d\n",
    339 		eccline->buf, queued_on));
    340 
    341 	return eccline;
    342 }
    343 
    344 
    345 static void
    346 udf_unqueue_eccline(struct strat_private *priv, struct udf_eccline *eccline)
    347 {
    348 	struct buf *ret __diagused;
    349 
    350 	UDF_LOCK_ECCLINE(eccline);
    351 	if (eccline->queued_on == 0) {
    352 		KASSERT(eccline->flags & ECC_FLOATING);
    353 		return;
    354 	}
    355 
    356 	ret = bufq_cancel(priv->queues[eccline->queued_on], eccline->buf);
    357 	KASSERT(ret == eccline->buf);
    358 
    359 	priv->num_queued[eccline->queued_on]--;
    360 	eccline->queued_on = 0;
    361 
    362 	eccline->flags |= ECC_FLOATING;
    363 	priv->num_floating++;
    364 }
    365 
    366 
    367 static struct udf_eccline *
    368 udf_geteccline(struct udf_mount *ump, uint32_t sector, int flags)
    369 {
    370 	struct strat_private *priv = PRIV(ump);
    371 	struct udf_eccline *eccline;
    372 	uint32_t start_sector, lb_size, blobsize;
    373 	uint8_t *eccline_blob;
    374 	int line, line_offset;
    375 	int num_busy;
    376 
    377 	mutex_enter(&priv->discstrat_mutex);
    378 
    379 	/* lookup in our line cache hashtable */
    380 	line_offset  = sector % ump->packet_size;
    381 	start_sector = sector - line_offset;
    382 	line = (start_sector/ump->packet_size) & UDF_ECCBUF_HASHMASK;
    383 
    384 	KASSERT(priv->thread_running);
    385 
    386 retry:
    387 	DPRINTF(ECCLINE, ("get line sector %d, line %d\n", sector, line));
    388 	LIST_FOREACH(eccline, &priv->eccline_hash[line], hashchain) {
    389 		if (eccline->start_sector == start_sector) {
    390 			DPRINTF(ECCLINE, ("\tfound eccline, start_sector %d\n",
    391 				eccline->start_sector));
    392 			udf_unqueue_eccline(priv, eccline);
    393 
    394 			mutex_exit(&priv->discstrat_mutex);
    395 			return eccline;
    396 		}
    397 	}
    398 
    399 	/* not found in eccline cache */
    400 	DPRINTF(ECCLINE, ("\tnot found in eccline cache\n"));
    401 
    402 	lb_size  = udf_rw32(ump->logical_vol->lb_size);
    403 	blobsize = ump->packet_size * lb_size;
    404 
    405 	/* dont allow too many pending requests */
    406 	DPRINTF(ECCLINE, ("\tallocating new eccline\n"));
    407 	num_busy = (priv->num_queued[UDF_SHED_SEQWRITING] + priv->num_floating);
    408 	if ((flags & ECC_SEQWRITING) && (num_busy > UDF_ECCLINE_MAXBUSY)) {
    409 		cv_timedwait(&priv->discstrat_cv,
    410 			&priv->discstrat_mutex, hz/8);
    411 		goto retry;
    412 	}
    413 
    414 	eccline_blob = pool_get(&priv->ecclineblob_pool, PR_NOWAIT);
    415 	eccline = pool_get(&priv->eccline_pool, PR_NOWAIT);
    416 	if ((eccline_blob == NULL) || (eccline == NULL)) {
    417 		if (eccline_blob)
    418 			pool_put(&priv->ecclineblob_pool, eccline_blob);
    419 		if (eccline)
    420 			pool_put(&priv->eccline_pool, eccline);
    421 
    422 		/* out of memory for now; canibalise freelist */
    423 		eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
    424 		if (eccline == NULL) {
    425 			/* serious trouble; wait and retry */
    426 			cv_timedwait(&priv->discstrat_cv,
    427 				&priv->discstrat_mutex, hz/8);
    428 			goto retry;
    429 		}
    430 
    431 		/* push back line if we're waiting for it or its locked */
    432 		if (eccline->flags & ECC_WANTED) {
    433 			/* we won a race, but someone else needed it */
    434 			udf_push_eccline(eccline, UDF_SHED_FREE);
    435 			goto retry;
    436 		}
    437 
    438 		/* unlink this entry */
    439 		LIST_REMOVE(eccline, hashchain);
    440 		KASSERT(eccline->flags & ECC_FLOATING);
    441 		KASSERT(eccline->queued_on == 0);
    442 
    443 		eccline_blob = eccline->blob;
    444 		eccline->flags = ECC_FLOATING | ECC_LOCKED;
    445 	} else {
    446 		eccline->flags = ECC_FLOATING | ECC_LOCKED;
    447 		priv->num_floating++;
    448 	}
    449 
    450 	eccline->queued_on = 0;
    451 	eccline->blob = eccline_blob;
    452 	eccline->buf  = getiobuf(NULL, true);
    453 	eccline->buf->b_private = eccline;	/* IMPORTANT */
    454 
    455 	/* initialise eccline blob */
    456 	/* XXX memset expensive and strictly not needed XXX */
    457 	memset(eccline->blob, 0, blobsize);
    458 
    459 	eccline->ump = ump;
    460 	eccline->present = eccline->readin = eccline->dirty = 0;
    461 	eccline->error = 0;
    462 	eccline->refcnt = 0;
    463 	memset(eccline->bufs, 0, UDF_MAX_PACKET_SIZE * sizeof(struct buf *));
    464 
    465 	eccline->start_sector    = start_sector;
    466 	eccline->buf->b_lblkno   = start_sector;
    467 	eccline->buf->b_blkno    = start_sector;
    468 	eccline->buf->b_rawblkno = start_sector;
    469 
    470 	LIST_INSERT_HEAD(&priv->eccline_hash[line], eccline, hashchain);
    471 
    472 	/*
    473 	 * TODO possible optimalisation for checking overlap with partitions
    474 	 * to get a clue on future eccline usage
    475 	 */
    476 
    477 	KASSERT(eccline->refcnt == 0);
    478 	KASSERT(eccline->flags & ECC_FLOATING);
    479 	KASSERT(eccline->flags & ECC_LOCKED);
    480 	mutex_exit(&priv->discstrat_mutex);
    481 
    482 	return eccline;
    483 }
    484 
    485 
    486 static void
    487 udf_puteccline(struct udf_eccline *eccline)
    488 {
    489 	struct strat_private *priv = PRIV(eccline->ump);
    490 	struct udf_mount *ump = eccline->ump;
    491 	uint64_t allbits = ((uint64_t) 1 << ump->packet_size)-1;
    492 	int new_queue;
    493 
    494 	mutex_enter(&priv->discstrat_mutex);
    495 
    496 	DPRINTF(ECCLINE, ("put eccline start sector %d, refcnt %d\n",
    497 		eccline->start_sector, eccline->refcnt));
    498 
    499 	KASSERT(eccline->flags & ECC_LOCKED);
    500 	KASSERT(eccline->flags & ECC_FLOATING);
    501 
    502 	/* clear all read bits that are already read in */
    503 	if (eccline->readin & eccline->present)
    504 		eccline->readin &= (~eccline->present) & allbits;
    505 
    506 	/* if we have active nodes we dont set it on seqwriting */
    507 	if (eccline->refcnt > 1)
    508 		eccline->flags &= ~ECC_SEQWRITING;
    509 
    510 	/* select state */
    511 	new_queue = UDF_SHED_FREE;
    512 	if (eccline->refcnt > 0)
    513 		new_queue = UDF_SHED_IDLE;
    514 	if (eccline->flags & ECC_WANTED)
    515 		new_queue = UDF_SHED_IDLE;
    516 	if (eccline->readin)
    517 		new_queue = UDF_SHED_READING;
    518 	if (eccline->dirty) {
    519 		new_queue = UDF_SHED_WAITING;
    520 		vfs_timestamp(&eccline->wait_time);
    521 		eccline->wait_time.tv_sec += ECC_WAITTIME;
    522 
    523 		if (eccline->present == allbits) {
    524 			new_queue = UDF_SHED_WRITING;
    525 			if (eccline->flags & ECC_SEQWRITING)
    526 				new_queue = UDF_SHED_SEQWRITING;
    527 		}
    528 	}
    529 	udf_push_eccline(eccline, new_queue);
    530 
    531 	mutex_exit(&priv->discstrat_mutex);
    532 }
    533 
    534 /* --------------------------------------------------------------------- */
    535 
    536 static int
    537 udf_create_nodedscr_rmw(struct udf_strat_args *args)
    538 {
    539 	union dscrptr   **dscrptr  = &args->dscr;
    540 	struct udf_mount *ump      = args->ump;
    541 	struct long_ad   *icb      = args->icb;
    542 	struct udf_eccline *eccline;
    543 	uint64_t bit;
    544 	uint32_t sectornr, lb_size, dummy;
    545 	uint8_t *mem;
    546 	int error, eccsect;
    547 
    548 	error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
    549 	if (error)
    550 		return error;
    551 
    552 	lb_size  = udf_rw32(ump->logical_vol->lb_size);
    553 
    554 	/* get our eccline */
    555 	eccline = udf_geteccline(ump, sectornr, 0);
    556 	eccsect = sectornr - eccline->start_sector;
    557 
    558 	bit = (uint64_t) 1 << eccsect;
    559 	eccline->readin  &= ~bit;	/* just in case */
    560 	eccline->present |=  bit;
    561 	eccline->dirty   &= ~bit;	/* Err... euhm... clean? */
    562 
    563 	eccline->refcnt++;
    564 
    565 	/* clear space */
    566 	mem = ((uint8_t *) eccline->blob) + eccsect * lb_size;
    567 	memset(mem, 0, lb_size);
    568 
    569 	udf_puteccline(eccline);
    570 
    571 	*dscrptr = (union dscrptr *) mem;
    572 	return 0;
    573 }
    574 
    575 
    576 static void
    577 udf_free_nodedscr_rmw(struct udf_strat_args *args)
    578 {
    579 	struct udf_mount *ump  = args->ump;
    580 	struct long_ad   *icb  = args->icb;
    581 	struct udf_eccline *eccline;
    582 	uint64_t bit;
    583 	uint32_t sectornr, dummy;
    584 	int error, eccsect;
    585 
    586 	error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
    587 	if (error)
    588 		return;
    589 
    590 	/* get our eccline */
    591 	eccline = udf_geteccline(ump, sectornr, 0);
    592 	eccsect = sectornr - eccline->start_sector;
    593 
    594 	bit = (uint64_t) 1 << eccsect;
    595 	KASSERT(eccline->present & bit);
    596 
    597 	eccline->readin &= ~bit;	/* just in case */
    598 	/* XXX eccline->dirty? */
    599 
    600 	KASSERT(eccline->refcnt >= 1);
    601 	eccline->refcnt--;
    602 
    603 	udf_puteccline(eccline);
    604 }
    605 
    606 
    607 static int
    608 udf_read_nodedscr_rmw(struct udf_strat_args *args)
    609 {
    610 	union dscrptr   **dscrptr = &args->dscr;
    611 	struct udf_mount *ump = args->ump;
    612 	struct long_ad   *icb = args->icb;
    613 	struct strat_private *priv;
    614 	struct udf_eccline *eccline;
    615 	uint64_t bit;
    616 	uint32_t sectornr, dummy;
    617 	uint8_t *pos;
    618 	int sector_size = ump->discinfo.sector_size;
    619 	int lb_size = udf_rw32(ump->logical_vol->lb_size);
    620 	int i, error, dscrlen, eccsect;
    621 
    622 	lb_size = lb_size;
    623 	KASSERT(sector_size == lb_size);
    624 	error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
    625 	if (error)
    626 		return error;
    627 
    628 	/* get our eccline */
    629 	eccline = udf_geteccline(ump, sectornr, 0);
    630 	eccsect = sectornr - eccline->start_sector;
    631 
    632 	bit = (uint64_t) 1 << eccsect;
    633 	if ((eccline->present & bit) == 0) {
    634 		/* mark bit for readin */
    635 		eccline->readin |= bit;
    636 		eccline->refcnt++;	/* prevent recycling */
    637 		KASSERT(eccline->bufs[eccsect] == NULL);
    638 		udf_puteccline(eccline);
    639 
    640 		/* wait for completion */
    641 		priv = PRIV(eccline->ump);
    642 		mutex_enter(&priv->discstrat_mutex);
    643 		while (((eccline->present | eccline->error) & bit) == 0) {
    644 			error = cv_timedwait(&priv->discstrat_cv,
    645 				&priv->discstrat_mutex,
    646 				hz/8);
    647 			if (error == EWOULDBLOCK)
    648 				DPRINTF(LOCKING, ("eccline waiting for read\n"));
    649 		}
    650 		mutex_exit(&priv->discstrat_mutex);
    651 
    652 		/* reget our line */
    653 		eccline = udf_geteccline(ump, sectornr, 0);
    654 		KASSERT(eccline->refcnt >= 1);
    655 		eccline->refcnt--;	/* undo refcnt */
    656 
    657 		if (eccline->error & bit) {
    658 			*dscrptr = NULL;
    659 			udf_puteccline(eccline);
    660 			return EIO;		/* XXX error code */
    661 		}
    662 	}
    663 
    664 	*dscrptr = (union dscrptr *)
    665 		(((uint8_t *) eccline->blob) + eccsect * sector_size);
    666 
    667 	/* code from read_phys_descr */
    668 	/* check if its a valid tag */
    669 	error = udf_check_tag(*dscrptr);
    670 	if (error) {
    671 		/* check if its an empty block */
    672 		pos = (uint8_t *) *dscrptr;
    673 		for (i = 0; i < sector_size; i++, pos++) {
    674 			if (*pos) break;
    675 		}
    676 		if (i == sector_size) {
    677 			/* return no error but with no dscrptr */
    678 			error = 0;
    679 		}
    680 		*dscrptr = NULL;
    681 		udf_puteccline(eccline);
    682 		return error;
    683 	}
    684 
    685 	/* calculate descriptor size */
    686 	dscrlen = udf_tagsize(*dscrptr, sector_size);
    687 	error = udf_check_tag_payload(*dscrptr, dscrlen);
    688 	if (error) {
    689 		*dscrptr = NULL;
    690 		udf_puteccline(eccline);
    691 		return error;
    692 	}
    693 
    694 	/* we have a hold since it has a node descriptor */
    695 	eccline->refcnt++;
    696 	udf_puteccline(eccline);
    697 
    698 	return 0;
    699 }
    700 
    701 
    702 static int
    703 udf_write_nodedscr_rmw(struct udf_strat_args *args)
    704 {
    705 	union dscrptr    *dscrptr = args->dscr;
    706 	struct udf_mount *ump = args->ump;
    707 	struct long_ad   *icb = args->icb;
    708 	struct udf_node *udf_node = args->udf_node;
    709 	struct udf_eccline *eccline;
    710 	uint64_t bit;
    711 	uint32_t sectornr, logsectornr, dummy;
    712 	// int waitfor  = args->waitfor;
    713 	int sector_size = ump->discinfo.sector_size;
    714 	int lb_size = udf_rw32(ump->logical_vol->lb_size);
    715 	int error, eccsect;
    716 
    717 	lb_size = lb_size;
    718 	KASSERT(sector_size == lb_size);
    719 	sectornr    = 0;
    720 	error = udf_translate_vtop(ump, icb, &sectornr, &dummy);
    721 	if (error)
    722 		return error;
    723 
    724 	/* paranoia: add reference to the vnode to prevent recycling */
    725 	vhold(udf_node->vnode);
    726 
    727 	/* get our eccline */
    728 	eccline = udf_geteccline(ump, sectornr, 0);
    729 	eccsect = sectornr - eccline->start_sector;
    730 
    731 	bit = (uint64_t) 1 << eccsect;
    732 
    733 	/* old callback still pending? */
    734 	if (eccline->bufs[eccsect]) {
    735 		DPRINTF(WRITE, ("udf_write_nodedscr_rmw: writing descriptor"
    736 					" over buffer?\n"));
    737 		nestiobuf_done(eccline->bufs[eccsect],
    738 				eccline->bufs_len[eccsect],
    739 				0);
    740 		eccline->bufs[eccsect] = NULL;
    741 	}
    742 
    743 	/* set sector number in the descriptor and validate */
    744 	dscrptr = (union dscrptr *)
    745 		(((uint8_t *) eccline->blob) + eccsect * sector_size);
    746 	KASSERT(dscrptr == args->dscr);
    747 
    748 	logsectornr = udf_rw32(icb->loc.lb_num);
    749 	dscrptr->tag.tag_loc = udf_rw32(logsectornr);
    750 	udf_validate_tag_and_crc_sums(dscrptr);
    751 
    752 	udf_fixup_node_internals(ump, (uint8_t *) dscrptr, UDF_C_NODE);
    753 
    754 	/* set our flags */
    755 	KASSERT(eccline->present & bit);
    756 	eccline->dirty |= bit;
    757 
    758 	KASSERT(udf_tagsize(dscrptr, sector_size) <= sector_size);
    759 
    760 	udf_node->outstanding_nodedscr--;
    761 	if (udf_node->outstanding_nodedscr == 0) {
    762 		/* XXX still using wakeup! */
    763 		UDF_UNLOCK_NODE(udf_node, 0);
    764 		wakeup(&udf_node->outstanding_nodedscr);
    765 	}
    766 	holdrele(udf_node->vnode);
    767 	udf_puteccline(eccline);
    768 
    769 	/* XXX waitfor not used */
    770 	return 0;
    771 }
    772 
    773 
    774 static void
    775 udf_queuebuf_rmw(struct udf_strat_args *args)
    776 {
    777 	struct udf_mount *ump = args->ump;
    778 	struct buf *buf = args->nestbuf;
    779 	struct desc_tag *tag;
    780 	struct strat_private *priv = PRIV(ump);
    781 	struct udf_eccline *eccline;
    782 	struct long_ad *node_ad_cpy;
    783 	uint64_t bit, *lmapping, *pmapping, *lmappos, *pmappos, blknr;
    784 	uint32_t buf_len, len, sectors, sectornr, our_sectornr;
    785 	uint32_t bpos;
    786 	uint16_t vpart_num;
    787 	uint8_t *fidblk, *src, *dst;
    788 	int sector_size = ump->discinfo.sector_size;
    789 	int blks = sector_size / DEV_BSIZE;
    790 	int eccsect, what, queue, error;
    791 
    792 	KASSERT(ump);
    793 	KASSERT(buf);
    794 	KASSERT(buf->b_iodone == nestiobuf_iodone);
    795 
    796 	blknr        = buf->b_blkno;
    797 	our_sectornr = blknr / blks;
    798 
    799 	what = buf->b_udf_c_type;
    800 	queue = UDF_SHED_READING;
    801 	if ((buf->b_flags & B_READ) == 0) {
    802 		/* writing */
    803 		queue = UDF_SHED_SEQWRITING;
    804 		if (what == UDF_C_ABSOLUTE)
    805 			queue = UDF_SHED_WRITING;
    806 		if (what == UDF_C_DSCR)
    807 			queue = UDF_SHED_WRITING;
    808 		if (what == UDF_C_NODE)
    809 			queue = UDF_SHED_WRITING;
    810 	}
    811 
    812 	if (queue == UDF_SHED_READING) {
    813 		DPRINTF(SHEDULE, ("\nudf_queuebuf_rmw READ %p : sector %d type %d,"
    814 			"b_resid %d, b_bcount %d, b_bufsize %d\n",
    815 			buf, (uint32_t) buf->b_blkno / blks, buf->b_udf_c_type,
    816 			buf->b_resid, buf->b_bcount, buf->b_bufsize));
    817 
    818 		/* mark bits for reading */
    819 		buf_len = buf->b_bcount;
    820 		sectornr = our_sectornr;
    821 		eccline = udf_geteccline(ump, sectornr, 0);
    822 		eccsect = sectornr - eccline->start_sector;
    823 		bpos = 0;
    824 		while (buf_len) {
    825 			len = MIN(buf_len, sector_size);
    826 			if ((eccsect < 0) || (eccsect >= ump->packet_size)) {
    827 				udf_puteccline(eccline);
    828 				eccline = udf_geteccline(ump, sectornr, 0);
    829 				eccsect = sectornr - eccline->start_sector;
    830 			}
    831 			bit = (uint64_t) 1 << eccsect;
    832 			error = eccline->error & bit ? EIO : 0;
    833 			if (eccline->present & bit) {
    834 				src = (uint8_t *) eccline->blob +
    835 					eccsect * sector_size;
    836 				dst = (uint8_t *) buf->b_data + bpos;
    837 				if (!error)
    838 					memcpy(dst, src, len);
    839 				nestiobuf_done(buf, len, error);
    840 			} else {
    841 				eccline->readin |= bit;
    842 				KASSERT(eccline->bufs[eccsect] == NULL);
    843 				eccline->bufs[eccsect] = buf;
    844 				eccline->bufs_bpos[eccsect] = bpos;
    845 				eccline->bufs_len[eccsect] = len;
    846 			}
    847 			bpos += sector_size;
    848 			eccsect++;
    849 			sectornr++;
    850 			buf_len -= len;
    851 		}
    852 		udf_puteccline(eccline);
    853 		return;
    854 	}
    855 
    856 	if (queue == UDF_SHED_WRITING) {
    857 		DPRINTF(SHEDULE, ("\nudf_queuebuf_rmw WRITE %p : sector %d "
    858 			"type %d, b_resid %d, b_bcount %d, b_bufsize %d\n",
    859 			buf, (uint32_t) buf->b_blkno / blks, buf->b_udf_c_type,
    860 			buf->b_resid, buf->b_bcount, buf->b_bufsize));
    861 
    862 		/* if we have FIDs fixup using buffer's sector number(s) */
    863 		if (buf->b_udf_c_type == UDF_C_FIDS)
    864 			panic("UDF_C_FIDS in SHED_WRITING!\n");
    865 
    866 		udf_fixup_node_internals(ump, buf->b_data, buf->b_udf_c_type);
    867 
    868 		/* copy parts into the bufs and set for writing */
    869 		buf_len = buf->b_bcount;
    870 		sectornr = our_sectornr;
    871 		eccline = udf_geteccline(ump, sectornr, 0);
    872 		eccsect = sectornr - eccline->start_sector;
    873 		bpos = 0;
    874 		while (buf_len) {
    875 			len = MIN(buf_len, sector_size);
    876 			if ((eccsect < 0) || (eccsect >= ump->packet_size)) {
    877 				udf_puteccline(eccline);
    878 				eccline = udf_geteccline(ump, sectornr, 0);
    879 				eccsect = sectornr - eccline->start_sector;
    880 			}
    881 			bit = (uint64_t) 1 << eccsect;
    882 			KASSERT((eccline->readin & bit) == 0);
    883 			eccline->present |= bit;
    884 			eccline->dirty   |= bit;
    885 			if (eccline->bufs[eccsect]) {
    886 				/* old callback still pending */
    887 				nestiobuf_done(eccline->bufs[eccsect],
    888 						eccline->bufs_len[eccsect],
    889 						0);
    890 				eccline->bufs[eccsect] = NULL;
    891 			}
    892 
    893 			src = (uint8_t *) buf->b_data + bpos;
    894 			dst = (uint8_t *) eccline->blob + eccsect * sector_size;
    895 			if (len != sector_size)
    896 				memset(dst, 0, sector_size);
    897 			memcpy(dst, src, len);
    898 
    899 			/* note that its finished for this extent */
    900 			eccline->bufs[eccsect] = NULL;
    901 			nestiobuf_done(buf, len, 0);
    902 
    903 			bpos += sector_size;
    904 			eccsect++;
    905 			sectornr++;
    906 			buf_len -= len;
    907 		}
    908 		udf_puteccline(eccline);
    909 		return;
    910 
    911 	}
    912 
    913 	/* sequential writing */
    914 	KASSERT(queue == UDF_SHED_SEQWRITING);
    915 	DPRINTF(SHEDULE, ("\nudf_queuebuf_rmw SEQWRITE %p : sector XXXX "
    916 		"type %d, b_resid %d, b_bcount %d, b_bufsize %d\n",
    917 		buf, buf->b_udf_c_type, buf->b_resid, buf->b_bcount,
    918 		buf->b_bufsize));
    919 	/*
    920 	 * Buffers should not have been allocated to disc addresses yet on
    921 	 * this queue. Note that a buffer can get multiple extents allocated.
    922 	 * Note that it *looks* like the normal writing but its different in
    923 	 * the details.
    924 	 *
    925 	 * lmapping contains lb_num relative to base partition.
    926 	 *
    927 	 * XXX should we try to claim/organize the allocated memory to
    928 	 * block-aligned pieces?
    929 	 */
    930 	mutex_enter(&priv->seqwrite_mutex);
    931 
    932 	lmapping    = ump->la_lmapping;
    933 	node_ad_cpy = ump->la_node_ad_cpy;
    934 
    935 	/* logically allocate buf and map it in the file */
    936 	udf_late_allocate_buf(ump, buf, lmapping, node_ad_cpy, &vpart_num);
    937 
    938 	/* if we have FIDs, fixup using the new allocation table */
    939 	if (buf->b_udf_c_type == UDF_C_FIDS) {
    940 		buf_len = buf->b_bcount;
    941 		bpos = 0;
    942 		lmappos = lmapping;
    943 		while (buf_len) {
    944 			sectornr = *lmappos++;
    945 			len = MIN(buf_len, sector_size);
    946 			fidblk = (uint8_t *) buf->b_data + bpos;
    947 			udf_fixup_fid_block(fidblk, sector_size,
    948 				0, len, sectornr);
    949 			bpos += len;
    950 			buf_len -= len;
    951 		}
    952 	}
    953 	if (buf->b_udf_c_type == UDF_C_METADATA_SBM) {
    954 		if (buf->b_lblkno == 0) {
    955 			/* update the tag location inside */
    956 			tag = (struct desc_tag *) buf->b_data;
    957 			tag->tag_loc = udf_rw32(*lmapping);
    958 			udf_validate_tag_and_crc_sums(buf->b_data);
    959 		}
    960 	}
    961 	udf_fixup_node_internals(ump, buf->b_data, buf->b_udf_c_type);
    962 
    963 	/*
    964 	 * Translate new mappings in lmapping to pmappings.
    965 	 * pmapping to contain lb_nums as used for disc adressing.
    966 	 */
    967 	pmapping = ump->la_pmapping;
    968 	sectors  = (buf->b_bcount + sector_size -1) / sector_size;
    969 	udf_translate_vtop_list(ump, sectors, vpart_num, lmapping, pmapping);
    970 
    971 	/* copy parts into the bufs and set for writing */
    972 	pmappos = pmapping;
    973 	buf_len = buf->b_bcount;
    974 	sectornr = *pmappos++;
    975 	eccline = udf_geteccline(ump, sectornr, ECC_SEQWRITING);
    976 	eccsect = sectornr - eccline->start_sector;
    977 	bpos = 0;
    978 	while (buf_len) {
    979 		len = MIN(buf_len, sector_size);
    980 		eccsect = sectornr - eccline->start_sector;
    981 		if ((eccsect < 0) || (eccsect >= ump->packet_size)) {
    982 			eccline->flags |= ECC_SEQWRITING;
    983 			udf_puteccline(eccline);
    984 			eccline = udf_geteccline(ump, sectornr, ECC_SEQWRITING);
    985 			eccsect = sectornr - eccline->start_sector;
    986 		}
    987 		bit = (uint64_t) 1 << eccsect;
    988 		KASSERT((eccline->readin & bit) == 0);
    989 		eccline->present |= bit;
    990 		eccline->dirty   |= bit;
    991 		eccline->bufs[eccsect] = NULL;
    992 
    993 		src = (uint8_t *) buf->b_data + bpos;
    994 		dst = (uint8_t *)
    995 			eccline->blob + eccsect * sector_size;
    996 		if (len != sector_size)
    997 			memset(dst, 0, sector_size);
    998 		memcpy(dst, src, len);
    999 
   1000 		/* note that its finished for this extent */
   1001 		nestiobuf_done(buf, len, 0);
   1002 
   1003 		bpos += sector_size;
   1004 		sectornr = *pmappos++;
   1005 		buf_len -= len;
   1006 	}
   1007 	eccline->flags |= ECC_SEQWRITING;
   1008 	udf_puteccline(eccline);
   1009 	mutex_exit(&priv->seqwrite_mutex);
   1010 }
   1011 
   1012 /* --------------------------------------------------------------------- */
   1013 
   1014 static void
   1015 udf_shedule_read_callback(struct buf *buf)
   1016 {
   1017 	struct udf_eccline *eccline = BTOE(buf);
   1018 	struct udf_mount *ump = eccline->ump;
   1019 	uint64_t bit;
   1020 	uint8_t *src, *dst;
   1021 	int sector_size = ump->discinfo.sector_size;
   1022 	int error, i, len;
   1023 
   1024 	DPRINTF(ECCLINE, ("read callback called on buf %p\n", buf));
   1025 
   1026 	/* post process read action */
   1027 	KASSERT(eccline->flags & ECC_LOCKED);
   1028 	error = buf->b_error;
   1029 	for (i = 0; i < ump->packet_size; i++) {
   1030 		bit = (uint64_t) 1 << i;
   1031 		src = (uint8_t *) buf->b_data +   i * sector_size;
   1032 		dst = (uint8_t *) eccline->blob + i * sector_size;
   1033 		if (eccline->present & bit)
   1034 			continue;
   1035 		eccline->present |= bit;
   1036 		if (error)
   1037 			eccline->error |= bit;
   1038 		if (eccline->bufs[i]) {
   1039 			dst = (uint8_t *) eccline->bufs[i]->b_data +
   1040 				eccline->bufs_bpos[i];
   1041 			len = eccline->bufs_len[i];
   1042 			if (!error)
   1043 				memcpy(dst, src, len);
   1044 			nestiobuf_done(eccline->bufs[i], len, error);
   1045 			eccline->bufs[i] = NULL;
   1046 		}
   1047 
   1048 	}
   1049 	KASSERT(buf->b_data == eccline->blob);
   1050 	KASSERT(eccline->present == ((uint64_t) 1 << ump->packet_size)-1);
   1051 
   1052 	/*
   1053 	 * XXX TODO what to do on read errors? read in all sectors
   1054 	 * synchronously and allocate a sparable entry?
   1055 	 */
   1056 
   1057 	udf_puteccline(eccline);
   1058 	DPRINTF(ECCLINE, ("read callback finished\n"));
   1059 }
   1060 
   1061 
   1062 static void
   1063 udf_shedule_write_callback(struct buf *buf)
   1064 {
   1065 	struct udf_eccline *eccline = BTOE(buf);
   1066 	struct udf_mount *ump = eccline->ump;
   1067 	uint64_t bit;
   1068 	int error, i;
   1069 
   1070 	DPRINTF(ECCLINE, ("write callback called on buf %p\n", buf));
   1071 
   1072 	/* post process write action */
   1073 	KASSERT(eccline->flags & ECC_LOCKED);
   1074 	error = buf->b_error;
   1075 	for (i = 0; i < ump->packet_size; i++) {
   1076 		bit = (uint64_t) 1 << i;
   1077 		if ((eccline->dirty & bit) == 0)
   1078 			continue;
   1079 		if (error) {
   1080 			eccline->error |= bit;
   1081 		} else {
   1082 			eccline->dirty &= ~bit;
   1083 		}
   1084 
   1085 		KASSERT(eccline->bufs[i] == 0);
   1086 	}
   1087 	KASSERT(eccline->dirty == 0);
   1088 	KASSERT(error == 0);
   1089 
   1090 	/*
   1091 	 * XXX TODO on write errors allocate a sparable entry and reissue
   1092 	 */
   1093 
   1094 	udf_puteccline(eccline);
   1095 	DPRINTF(ECCLINE, ("write callback finished\n"));
   1096 }
   1097 
   1098 
   1099 static void
   1100 udf_issue_eccline(struct udf_eccline *eccline, int queued_on)
   1101 {
   1102 	struct udf_mount *ump = eccline->ump;
   1103 	struct strat_private *priv = PRIV(ump);
   1104 	struct buf *buf, *nestbuf;
   1105 	uint64_t bit, allbits = ((uint64_t) 1 << ump->packet_size)-1;
   1106 	uint32_t start;
   1107 	int sector_size = ump->discinfo.sector_size;
   1108 	int blks = sector_size / DEV_BSIZE;
   1109 	int i;
   1110 
   1111 	KASSERT(eccline->flags & ECC_LOCKED);
   1112 
   1113 	if (queued_on == UDF_SHED_READING) {
   1114 		DPRINTF(SHEDULE, ("udf_issue_eccline reading : "));
   1115 		/* read all bits that are not yet present */
   1116 		eccline->readin = (~eccline->present) & allbits;
   1117 		KASSERT(eccline->readin);
   1118 		start = eccline->start_sector;
   1119 		buf = eccline->buf;
   1120 		buf->b_flags    = B_READ | B_ASYNC;
   1121 		SET(buf->b_cflags, BC_BUSY);	/* mark buffer busy */
   1122 		buf->b_oflags   = 0;
   1123 		buf->b_iodone   = udf_shedule_read_callback;
   1124 		buf->b_data     = eccline->blob;
   1125 		buf->b_bcount   = ump->packet_size * sector_size;
   1126 		buf->b_resid    = buf->b_bcount;
   1127 		buf->b_bufsize  = buf->b_bcount;
   1128 		buf->b_private  = eccline;
   1129 		BIO_SETPRIO(buf, BPRIO_DEFAULT);
   1130 		buf->b_lblkno   = buf->b_blkno = buf->b_rawblkno = start * blks;
   1131 		buf->b_proc     = NULL;
   1132 
   1133 		if (eccline->present != 0) {
   1134 			for (i = 0; i < ump->packet_size; i++) {
   1135 				bit = (uint64_t) 1 << i;
   1136 				if (eccline->present & bit) {
   1137 					nestiobuf_done(buf, sector_size, 0);
   1138 					continue;
   1139 				}
   1140 				nestbuf = getiobuf(NULL, true);
   1141 				nestiobuf_setup(buf, nestbuf, i * sector_size,
   1142 					sector_size);
   1143 				/* adjust blocknumber to read */
   1144 				nestbuf->b_blkno = buf->b_blkno + i*blks;
   1145 				nestbuf->b_rawblkno = buf->b_rawblkno + i*blks;
   1146 
   1147 				DPRINTF(SHEDULE, ("sector %d ", start + i));
   1148 
   1149 				/* mutex dance since it could lock */
   1150 				mutex_exit(&priv->discstrat_mutex);
   1151 					/* call asynchronous */
   1152 					VOP_STRATEGY(ump->devvp, nestbuf);
   1153 				mutex_enter(&priv->discstrat_mutex);
   1154 			}
   1155 			DPRINTF(SHEDULE, ("\n"));
   1156 			return;
   1157 		}
   1158 	} else {
   1159 		/* write or seqwrite */
   1160 		DPRINTF(SHEDULE, ("udf_issue_eccline writing or seqwriting : "));
   1161 		DPRINTF(SHEDULE, ("\n\tpresent %"PRIx64", readin %"PRIx64", "
   1162 			"dirty %"PRIx64"\n\t", eccline->present, eccline->readin,
   1163 			eccline->dirty));
   1164 		KASSERT(eccline->present == allbits);
   1165 
   1166 		start = eccline->start_sector;
   1167 		buf = eccline->buf;
   1168 		buf->b_flags    = B_WRITE | B_ASYNC;
   1169 		SET(buf->b_cflags, BC_BUSY);	/* mark buffer busy */
   1170 		buf->b_oflags   = 0;
   1171 		buf->b_iodone   = udf_shedule_write_callback;
   1172 		buf->b_data     = eccline->blob;
   1173 		buf->b_bcount   = ump->packet_size * sector_size;
   1174 		buf->b_resid    = buf->b_bcount;
   1175 		buf->b_bufsize  = buf->b_bcount;
   1176 		buf->b_private  = eccline;
   1177 		BIO_SETPRIO(buf, BPRIO_DEFAULT);
   1178 		buf->b_lblkno   = buf->b_blkno = buf->b_rawblkno = start * blks;
   1179 		buf->b_proc     = NULL;
   1180 	}
   1181 
   1182 	/* mutex dance since it could lock */
   1183 	mutex_exit(&priv->discstrat_mutex);
   1184 		/* call asynchronous */
   1185 		DPRINTF(SHEDULE, ("sector %d for %d\n",
   1186 			start, ump->packet_size));
   1187 		VOP_STRATEGY(ump->devvp, buf);
   1188 	mutex_enter(&priv->discstrat_mutex);
   1189 }
   1190 
   1191 
   1192 static void
   1193 udf_discstrat_thread(void *arg)
   1194 {
   1195 	struct udf_mount *ump = (struct udf_mount *) arg;
   1196 	struct strat_private *priv = PRIV(ump);
   1197 	struct udf_eccline *eccline;
   1198 	struct timespec now, *last;
   1199 	uint64_t allbits = ((uint64_t) 1 << ump->packet_size)-1;
   1200 	int new_queue, wait, work;
   1201 
   1202 	work = 1;
   1203 	priv->thread_running = 1;
   1204 	mutex_enter(&priv->discstrat_mutex);
   1205 	priv->num_floating = 0;
   1206 	while (priv->run_thread || work || priv->num_floating) {
   1207 		/* get our time */
   1208 		vfs_timestamp(&now);
   1209 
   1210 		/* maintenance: handle eccline state machine */
   1211 		for(;;) {
   1212 			/* only peek at it */
   1213 			eccline = udf_peek_eccline(priv, UDF_SHED_WAITING);
   1214 			if (eccline == NULL)
   1215 				break;
   1216 
   1217 			/* if not reading, wait until the time has come */
   1218 			if ((priv->cur_queue != UDF_SHED_READING) &&
   1219 				(eccline->wait_time.tv_sec - now.tv_sec > 0)) {
   1220 					UDF_UNLOCK_ECCLINE(eccline);
   1221 					/* all others are later, so break off */
   1222 					break;
   1223 			}
   1224 
   1225 			/* release */
   1226 			UDF_UNLOCK_ECCLINE(eccline);
   1227 
   1228 			/* do get it */
   1229 			eccline = udf_pop_eccline(priv, UDF_SHED_WAITING);
   1230 
   1231 			/* requeue according to state */
   1232 			new_queue = UDF_SHED_FREE;	/* unlikely */
   1233 			if (eccline->refcnt > 0)
   1234 				new_queue = UDF_SHED_IDLE;
   1235 			if (eccline->flags & ECC_WANTED)
   1236 				new_queue = UDF_SHED_IDLE;
   1237 			if (eccline->readin)
   1238 				new_queue = UDF_SHED_READING;
   1239 			if (eccline->dirty) {
   1240 				new_queue = UDF_SHED_READING;
   1241 				if (eccline->present == allbits) {
   1242 					new_queue = UDF_SHED_WRITING;
   1243 					if (eccline->flags & ECC_SEQWRITING)
   1244 						new_queue = UDF_SHED_SEQWRITING;
   1245 				}
   1246 			}
   1247 			udf_push_eccline(eccline, new_queue);
   1248 		}
   1249 
   1250 		/* maintenance: free excess ecclines */
   1251 		while (priv->num_queued[UDF_SHED_FREE] > UDF_ECCLINE_MAXFREE) {
   1252 			eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
   1253 			KASSERT(eccline);
   1254 			KASSERT(eccline->refcnt == 0);
   1255 			if (eccline->flags & ECC_WANTED) {
   1256 				/* we won the race, but we dont want to win */
   1257 				DPRINTF(ECCLINE, ("Tried removing, pushed back to free list\n"));
   1258 				udf_push_eccline(eccline, UDF_SHED_IDLE);
   1259 			} else {
   1260 				DPRINTF(ECCLINE, ("Removing entry from free list\n"));
   1261 				udf_dispose_eccline(eccline);
   1262 			}
   1263 		}
   1264 
   1265 		/* process the current selected queue */
   1266 		/* get our time */
   1267 		vfs_timestamp(&now);
   1268 		last = &priv->last_queued[priv->cur_queue];
   1269 
   1270 		/* get our line */
   1271 		eccline = udf_pop_eccline(priv, priv->cur_queue);
   1272 		if (eccline) {
   1273 			wait = 0;
   1274 			new_queue = priv->cur_queue;
   1275 			DPRINTF(ECCLINE, ("UDF_ISSUE_ECCLINE\n"));
   1276 
   1277 			udf_issue_eccline(eccline, priv->cur_queue);
   1278 		} else {
   1279 			/* don't switch too quickly */
   1280 			if (now.tv_sec - last->tv_sec < 2) {
   1281 				/* wait some time */
   1282 				cv_timedwait(&priv->discstrat_cv,
   1283 					&priv->discstrat_mutex, hz);
   1284 				/* we assume there is work to be done */
   1285 				work = 1;
   1286 				continue;
   1287 			}
   1288 
   1289 			/* XXX select on queue lengths ? */
   1290 			wait = 1;
   1291 			/* check if we can/should switch */
   1292 			new_queue = priv->cur_queue;
   1293 			if (bufq_peek(priv->queues[UDF_SHED_READING]))
   1294 				new_queue = UDF_SHED_READING;
   1295 			if (bufq_peek(priv->queues[UDF_SHED_WRITING]))
   1296 				new_queue = UDF_SHED_WRITING;
   1297 			if (bufq_peek(priv->queues[UDF_SHED_SEQWRITING]))
   1298 				new_queue = UDF_SHED_SEQWRITING;
   1299 		}
   1300 
   1301 		/* give room */
   1302 		mutex_exit(&priv->discstrat_mutex);
   1303 
   1304 		if (new_queue != priv->cur_queue) {
   1305 			wait = 0;
   1306 			DPRINTF(SHEDULE, ("switching from %d to %d\n",
   1307 				priv->cur_queue, new_queue));
   1308 			priv->cur_queue = new_queue;
   1309 		}
   1310 		mutex_enter(&priv->discstrat_mutex);
   1311 
   1312 		/* wait for more if needed */
   1313 		if (wait)
   1314 			cv_timedwait(&priv->discstrat_cv,
   1315 				&priv->discstrat_mutex, hz/4);	/* /8 */
   1316 
   1317 		work  = (bufq_peek(priv->queues[UDF_SHED_WAITING]) != NULL);
   1318 		work |= (bufq_peek(priv->queues[UDF_SHED_READING]) != NULL);
   1319 		work |= (bufq_peek(priv->queues[UDF_SHED_WRITING]) != NULL);
   1320 		work |= (bufq_peek(priv->queues[UDF_SHED_SEQWRITING]) != NULL);
   1321 
   1322 		DPRINTF(PARANOIA, ("work : (%d, %d, %d) -> work %d, float %d\n",
   1323 			(bufq_peek(priv->queues[UDF_SHED_READING]) != NULL),
   1324 			(bufq_peek(priv->queues[UDF_SHED_WRITING]) != NULL),
   1325 			(bufq_peek(priv->queues[UDF_SHED_SEQWRITING]) != NULL),
   1326 			work, priv->num_floating));
   1327 	}
   1328 
   1329 	mutex_exit(&priv->discstrat_mutex);
   1330 
   1331 	/* tear down remaining ecclines */
   1332 	mutex_enter(&priv->discstrat_mutex);
   1333 	KASSERT(bufq_peek(priv->queues[UDF_SHED_WAITING]) == NULL);
   1334 	KASSERT(bufq_peek(priv->queues[UDF_SHED_IDLE]) == NULL);
   1335 	KASSERT(bufq_peek(priv->queues[UDF_SHED_READING]) == NULL);
   1336 	KASSERT(bufq_peek(priv->queues[UDF_SHED_WRITING]) == NULL);
   1337 	KASSERT(bufq_peek(priv->queues[UDF_SHED_SEQWRITING]) == NULL);
   1338 
   1339 	KASSERT(priv->num_queued[UDF_SHED_WAITING] == 0);
   1340 	KASSERT(priv->num_queued[UDF_SHED_IDLE] == 0);
   1341 	KASSERT(priv->num_queued[UDF_SHED_READING] == 0);
   1342 	KASSERT(priv->num_queued[UDF_SHED_WRITING] == 0);
   1343 	KASSERT(priv->num_queued[UDF_SHED_SEQWRITING] == 0);
   1344 
   1345 	eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
   1346 	while (eccline) {
   1347 		udf_dispose_eccline(eccline);
   1348 		eccline = udf_pop_eccline(priv, UDF_SHED_FREE);
   1349 	}
   1350 	KASSERT(priv->num_queued[UDF_SHED_FREE] == 0);
   1351 	mutex_exit(&priv->discstrat_mutex);
   1352 
   1353 	priv->thread_running  = 0;
   1354 	priv->thread_finished = 1;
   1355 	wakeup(&priv->run_thread);
   1356 	kthread_exit(0);
   1357 	/* not reached */
   1358 }
   1359 
   1360 /* --------------------------------------------------------------------- */
   1361 
   1362 /*
   1363  * Buffer memory pool allocator.
   1364  */
   1365 
   1366 static void *
   1367 ecclinepool_page_alloc(struct pool *pp, int flags)
   1368 {
   1369         return (void *)uvm_km_alloc(kernel_map,
   1370             MAXBSIZE, MAXBSIZE,
   1371             ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK)
   1372 	    	| UVM_KMF_WIRED /* UVM_KMF_PAGABLE? */);
   1373 }
   1374 
   1375 static void
   1376 ecclinepool_page_free(struct pool *pp, void *v)
   1377 {
   1378         uvm_km_free(kernel_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED);
   1379 }
   1380 
   1381 static struct pool_allocator ecclinepool_allocator = {
   1382         .pa_alloc = ecclinepool_page_alloc,
   1383         .pa_free  = ecclinepool_page_free,
   1384         .pa_pagesz = MAXBSIZE,
   1385 };
   1386 
   1387 
   1388 static void
   1389 udf_discstrat_init_rmw(struct udf_strat_args *args)
   1390 {
   1391 	struct udf_mount *ump = args->ump;
   1392 	struct strat_private *priv = PRIV(ump);
   1393 	uint32_t lb_size, blobsize, hashline;
   1394 	int i;
   1395 
   1396 	KASSERT(ump);
   1397 	KASSERT(ump->logical_vol);
   1398 	KASSERT(priv == NULL);
   1399 
   1400 	lb_size = udf_rw32(ump->logical_vol->lb_size);
   1401 	blobsize = ump->packet_size * lb_size;
   1402 	KASSERT(lb_size > 0);
   1403 	KASSERT(ump->packet_size <= 64);
   1404 
   1405 	/* initialise our memory space */
   1406 	ump->strategy_private = malloc(sizeof(struct strat_private),
   1407 		M_UDFTEMP, M_WAITOK);
   1408 	priv = ump->strategy_private;
   1409 	memset(priv, 0 , sizeof(struct strat_private));
   1410 
   1411 	/* initialise locks */
   1412 	cv_init(&priv->discstrat_cv, "udfstrat");
   1413 	mutex_init(&priv->discstrat_mutex, MUTEX_DEFAULT, IPL_NONE);
   1414 	mutex_init(&priv->seqwrite_mutex, MUTEX_DEFAULT, IPL_NONE);
   1415 
   1416 	/* initialise struct eccline pool */
   1417 	pool_init(&priv->eccline_pool, sizeof(struct udf_eccline),
   1418 		0, 0, 0, "udf_eccline_pool", NULL, IPL_NONE);
   1419 
   1420 	/* initialise eccline blob pool */
   1421         ecclinepool_allocator.pa_pagesz = blobsize;
   1422 	pool_init(&priv->ecclineblob_pool, blobsize,
   1423 		0, 0, 0, "udf_eccline_blob", &ecclinepool_allocator, IPL_NONE);
   1424 
   1425 	/* initialise main queues */
   1426 	for (i = 0; i < UDF_SHED_MAX; i++) {
   1427 		priv->num_queued[i] = 0;
   1428 		vfs_timestamp(&priv->last_queued[i]);
   1429 	}
   1430 	bufq_alloc(&priv->queues[UDF_SHED_WAITING], "fcfs",
   1431 		BUFQ_SORT_RAWBLOCK);
   1432 	bufq_alloc(&priv->queues[UDF_SHED_READING], "disksort",
   1433 		BUFQ_SORT_RAWBLOCK);
   1434 	bufq_alloc(&priv->queues[UDF_SHED_WRITING], "disksort",
   1435 		BUFQ_SORT_RAWBLOCK);
   1436 	bufq_alloc(&priv->queues[UDF_SHED_SEQWRITING], "disksort", 0);
   1437 
   1438 	/* initialise administrative queues */
   1439 	bufq_alloc(&priv->queues[UDF_SHED_IDLE], "fcfs", 0);
   1440 	bufq_alloc(&priv->queues[UDF_SHED_FREE], "fcfs", 0);
   1441 
   1442 	for (hashline = 0; hashline < UDF_ECCBUF_HASHSIZE; hashline++) {
   1443 		LIST_INIT(&priv->eccline_hash[hashline]);
   1444 	}
   1445 
   1446 	/* create our disk strategy thread */
   1447 	priv->cur_queue = UDF_SHED_READING;
   1448 	priv->thread_finished = 0;
   1449 	priv->thread_running  = 0;
   1450 	priv->run_thread      = 1;
   1451 	if (kthread_create(PRI_NONE, 0 /* KTHREAD_MPSAFE*/, NULL /* cpu_info*/,
   1452 		udf_discstrat_thread, ump, &priv->queue_lwp,
   1453 		"%s", "udf_rw")) {
   1454 		panic("fork udf_rw");
   1455 	}
   1456 
   1457 	/* wait for thread to spin up */
   1458 	while (!priv->thread_running) {
   1459 		tsleep(&priv->thread_running, PRIBIO+1, "udfshedstart", hz);
   1460 	}
   1461 }
   1462 
   1463 
   1464 static void
   1465 udf_discstrat_finish_rmw(struct udf_strat_args *args)
   1466 {
   1467 	struct udf_mount *ump = args->ump;
   1468 	struct strat_private *priv = PRIV(ump);
   1469 
   1470 	if (ump == NULL)
   1471 		return;
   1472 
   1473 	/* stop our sheduling thread */
   1474 	KASSERT(priv->run_thread == 1);
   1475 	priv->run_thread = 0;
   1476 	wakeup(priv->queue_lwp);
   1477 	while (!priv->thread_finished) {
   1478 		tsleep(&priv->run_thread, PRIBIO + 1, "udfshedfin", hz);
   1479 	}
   1480 	/* kthread should be finished now */
   1481 
   1482 	/* cleanup our pools */
   1483 	pool_destroy(&priv->eccline_pool);
   1484 	pool_destroy(&priv->ecclineblob_pool);
   1485 
   1486 	cv_destroy(&priv->discstrat_cv);
   1487 	mutex_destroy(&priv->discstrat_mutex);
   1488 	mutex_destroy(&priv->seqwrite_mutex);
   1489 
   1490 	/* free our private space */
   1491 	free(ump->strategy_private, M_UDFTEMP);
   1492 	ump->strategy_private = NULL;
   1493 }
   1494 
   1495 /* --------------------------------------------------------------------- */
   1496 
   1497 struct udf_strategy udf_strat_rmw =
   1498 {
   1499 	udf_create_nodedscr_rmw,
   1500 	udf_free_nodedscr_rmw,
   1501 	udf_read_nodedscr_rmw,
   1502 	udf_write_nodedscr_rmw,
   1503 	udf_queuebuf_rmw,
   1504 	udf_discstrat_init_rmw,
   1505 	udf_discstrat_finish_rmw
   1506 };
   1507 
   1508