vfs_wapbl.c revision 1.6
1 /* $NetBSD: vfs_wapbl.c,v 1.6 2008/11/11 08:29:58 joerg Exp $ */ 2 3 /*- 4 * Copyright (c) 2003,2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * This implements file system independent write ahead filesystem logging. 34 */ 35 36 #define WAPBL_INTERNAL 37 38 #include <sys/cdefs.h> 39 __KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.6 2008/11/11 08:29:58 joerg Exp $"); 40 41 #include <sys/param.h> 42 43 #ifdef _KERNEL 44 #include <sys/param.h> 45 #include <sys/namei.h> 46 #include <sys/proc.h> 47 #include <sys/uio.h> 48 #include <sys/vnode.h> 49 #include <sys/file.h> 50 #include <sys/malloc.h> 51 #include <sys/resourcevar.h> 52 #include <sys/conf.h> 53 #include <sys/mount.h> 54 #include <sys/kernel.h> 55 #include <sys/kauth.h> 56 #include <sys/mutex.h> 57 #include <sys/atomic.h> 58 #include <sys/wapbl.h> 59 60 #if WAPBL_UVM_ALLOC 61 #include <uvm/uvm.h> 62 #endif 63 64 #include <miscfs/specfs/specdev.h> 65 66 MALLOC_JUSTDEFINE(M_WAPBL, "wapbl", "write-ahead physical block logging"); 67 #define wapbl_malloc(s) malloc((s), M_WAPBL, M_WAITOK) 68 #define wapbl_free(a) free((a), M_WAPBL) 69 #define wapbl_calloc(n, s) malloc((n)*(s), M_WAPBL, M_WAITOK | M_ZERO) 70 71 #else /* !_KERNEL */ 72 #include <assert.h> 73 #include <errno.h> 74 #include <stdio.h> 75 #include <stdbool.h> 76 #include <stdlib.h> 77 #include <string.h> 78 79 #include <sys/time.h> 80 #include <sys/wapbl.h> 81 82 #define KDASSERT(x) assert(x) 83 #define KASSERT(x) assert(x) 84 #define wapbl_malloc(s) malloc(s) 85 #define wapbl_free(a) free(a) 86 #define wapbl_calloc(n, s) calloc((n), (s)) 87 88 #endif /* !_KERNEL */ 89 90 /* 91 * INTERNAL DATA STRUCTURES 92 */ 93 94 /* 95 * This structure holds per-mount log information. 96 * 97 * Legend: a = atomic access only 98 * r = read-only after init 99 * l = rwlock held 100 * m = mutex held 101 * u = unlocked access ok 102 * b = bufcache_lock held 103 */ 104 struct wapbl { 105 struct vnode *wl_logvp; /* r: log here */ 106 struct vnode *wl_devvp; /* r: log on this device */ 107 struct mount *wl_mount; /* r: mountpoint wl is associated with */ 108 daddr_t wl_logpbn; /* r: Physical block number of start of log */ 109 int wl_log_dev_bshift; /* r: logarithm of device block size of log 110 device */ 111 int wl_fs_dev_bshift; /* r: logarithm of device block size of 112 filesystem device */ 113 114 unsigned wl_lock_count; /* m: Count of transactions in progress */ 115 116 size_t wl_circ_size; /* r: Number of bytes in buffer of log */ 117 size_t wl_circ_off; /* r: Number of bytes reserved at start */ 118 119 size_t wl_bufcount_max; /* r: Number of buffers reserved for log */ 120 size_t wl_bufbytes_max; /* r: Number of buf bytes reserved for log */ 121 122 off_t wl_head; /* l: Byte offset of log head */ 123 off_t wl_tail; /* l: Byte offset of log tail */ 124 /* 125 * head == tail == 0 means log is empty 126 * head == tail != 0 means log is full 127 * see assertions in wapbl_advance() for other boundary conditions. 128 * only truncate moves the tail, except when flush sets it to 129 * wl_header_size only flush moves the head, except when truncate 130 * sets it to 0. 131 */ 132 133 struct wapbl_wc_header *wl_wc_header; /* l */ 134 void *wl_wc_scratch; /* l: scratch space (XXX: por que?!?) */ 135 136 kmutex_t wl_mtx; /* u: short-term lock */ 137 krwlock_t wl_rwlock; /* u: File system transaction lock */ 138 139 /* 140 * Must be held while accessing 141 * wl_count or wl_bufs or head or tail 142 */ 143 144 /* 145 * Callback called from within the flush routine to flush any extra 146 * bits. Note that flush may be skipped without calling this if 147 * there are no outstanding buffers in the transaction. 148 */ 149 #if _KERNEL 150 wapbl_flush_fn_t wl_flush; /* r */ 151 wapbl_flush_fn_t wl_flush_abort;/* r */ 152 #endif 153 154 size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ 155 size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ 156 size_t wl_bcount; /* m: Total bcount of wl_bufs */ 157 158 LIST_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ 159 160 kcondvar_t wl_reclaimable_cv; /* m (obviously) */ 161 size_t wl_reclaimable_bytes; /* m: Amount of space available for 162 reclamation by truncate */ 163 int wl_error_count; /* m: # of wl_entries with errors */ 164 size_t wl_reserved_bytes; /* never truncate log smaller than this */ 165 166 #ifdef WAPBL_DEBUG_BUFBYTES 167 size_t wl_unsynced_bufbytes; /* Byte count of unsynced buffers */ 168 #endif 169 170 daddr_t *wl_deallocblks;/* l: address of block */ 171 int *wl_dealloclens; /* l: size of block (fragments, kom ihg) */ 172 int wl_dealloccnt; /* l: total count */ 173 int wl_dealloclim; /* l: max count */ 174 175 /* hashtable of inode numbers for allocated but unlinked inodes */ 176 /* synch ??? */ 177 LIST_HEAD(wapbl_ino_head, wapbl_ino) *wl_inohash; 178 u_long wl_inohashmask; 179 int wl_inohashcnt; 180 181 SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* On disk transaction 182 accounting */ 183 }; 184 185 #ifdef WAPBL_DEBUG_PRINT 186 int wapbl_debug_print = WAPBL_DEBUG_PRINT; 187 #endif 188 189 /****************************************************************/ 190 #ifdef _KERNEL 191 192 #ifdef WAPBL_DEBUG 193 struct wapbl *wapbl_debug_wl; 194 #endif 195 196 static int wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail); 197 static int wapbl_write_blocks(struct wapbl *wl, off_t *offp); 198 static int wapbl_write_revocations(struct wapbl *wl, off_t *offp); 199 static int wapbl_write_inodes(struct wapbl *wl, off_t *offp); 200 #endif /* _KERNEL */ 201 202 static int wapbl_replay_prescan(struct wapbl_replay *wr); 203 static int wapbl_replay_get_inodes(struct wapbl_replay *wr); 204 205 static __inline size_t wapbl_space_free(size_t avail, off_t head, 206 off_t tail); 207 static __inline size_t wapbl_space_used(size_t avail, off_t head, 208 off_t tail); 209 210 #ifdef _KERNEL 211 212 #define WAPBL_INODETRK_SIZE 83 213 static int wapbl_ino_pool_refcount; 214 static struct pool wapbl_ino_pool; 215 struct wapbl_ino { 216 LIST_ENTRY(wapbl_ino) wi_hash; 217 ino_t wi_ino; 218 mode_t wi_mode; 219 }; 220 221 static void wapbl_inodetrk_init(struct wapbl *wl, u_int size); 222 static void wapbl_inodetrk_free(struct wapbl *wl); 223 static struct wapbl_ino *wapbl_inodetrk_get(struct wapbl *wl, ino_t ino); 224 225 static size_t wapbl_transaction_len(struct wapbl *wl); 226 static __inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); 227 228 #ifdef DEBUG 229 int wapbl_replay_verify(struct wapbl_replay *, struct vnode *); 230 #endif 231 232 static int wapbl_replay_isopen1(struct wapbl_replay *); 233 234 /* 235 * This is useful for debugging. If set, the log will 236 * only be truncated when necessary. 237 */ 238 int wapbl_lazy_truncate = 0; 239 240 struct wapbl_ops wapbl_ops = { 241 .wo_wapbl_discard = wapbl_discard, 242 .wo_wapbl_replay_isopen = wapbl_replay_isopen1, 243 .wo_wapbl_replay_can_read = wapbl_replay_can_read, 244 .wo_wapbl_replay_read = wapbl_replay_read, 245 .wo_wapbl_add_buf = wapbl_add_buf, 246 .wo_wapbl_remove_buf = wapbl_remove_buf, 247 .wo_wapbl_resize_buf = wapbl_resize_buf, 248 .wo_wapbl_begin = wapbl_begin, 249 .wo_wapbl_end = wapbl_end, 250 .wo_wapbl_junlock_assert= wapbl_junlock_assert, 251 252 /* XXX: the following is only used to say "this is a wapbl buf" */ 253 .wo_wapbl_biodone = wapbl_biodone, 254 }; 255 256 void 257 wapbl_init() 258 { 259 260 malloc_type_attach(M_WAPBL); 261 } 262 263 int 264 wapbl_start(struct wapbl ** wlp, struct mount *mp, struct vnode *vp, 265 daddr_t off, size_t count, size_t blksize, struct wapbl_replay *wr, 266 wapbl_flush_fn_t flushfn, wapbl_flush_fn_t flushabortfn) 267 { 268 struct wapbl *wl; 269 struct vnode *devvp; 270 daddr_t logpbn; 271 int error; 272 int log_dev_bshift = DEV_BSHIFT; 273 int fs_dev_bshift = DEV_BSHIFT; 274 int run; 275 276 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_start: vp=%p off=%" PRId64 277 " count=%zu blksize=%zu\n", vp, off, count, blksize)); 278 279 if (log_dev_bshift > fs_dev_bshift) { 280 WAPBL_PRINTF(WAPBL_PRINT_OPEN, 281 ("wapbl: log device's block size cannot be larger " 282 "than filesystem's\n")); 283 /* 284 * Not currently implemented, although it could be if 285 * needed someday. 286 */ 287 return ENOSYS; 288 } 289 290 if (off < 0) 291 return EINVAL; 292 293 if (blksize < DEV_BSIZE) 294 return EINVAL; 295 if (blksize % DEV_BSIZE) 296 return EINVAL; 297 298 /* XXXTODO: verify that the full load is writable */ 299 300 /* 301 * XXX check for minimum log size 302 * minimum is governed by minimum amount of space 303 * to complete a transaction. (probably truncate) 304 */ 305 /* XXX for now pick something minimal */ 306 if ((count * blksize) < MAXPHYS) { 307 return ENOSPC; 308 } 309 310 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, &run)) != 0) { 311 return error; 312 } 313 314 wl = wapbl_calloc(1, sizeof(*wl)); 315 rw_init(&wl->wl_rwlock); 316 mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); 317 cv_init(&wl->wl_reclaimable_cv, "wapblrec"); 318 LIST_INIT(&wl->wl_bufs); 319 SIMPLEQ_INIT(&wl->wl_entries); 320 321 wl->wl_logvp = vp; 322 wl->wl_devvp = devvp; 323 wl->wl_mount = mp; 324 wl->wl_logpbn = logpbn; 325 wl->wl_log_dev_bshift = log_dev_bshift; 326 wl->wl_fs_dev_bshift = fs_dev_bshift; 327 328 wl->wl_flush = flushfn; 329 wl->wl_flush_abort = flushabortfn; 330 331 /* Reserve two log device blocks for the commit headers */ 332 wl->wl_circ_off = 2<<wl->wl_log_dev_bshift; 333 wl->wl_circ_size = ((count * blksize) - wl->wl_circ_off); 334 /* truncate the log usage to a multiple of log_dev_bshift */ 335 wl->wl_circ_size >>= wl->wl_log_dev_bshift; 336 wl->wl_circ_size <<= wl->wl_log_dev_bshift; 337 338 /* 339 * wl_bufbytes_max limits the size of the in memory transaction space. 340 * - Since buffers are allocated and accounted for in units of 341 * PAGE_SIZE it is required to be a multiple of PAGE_SIZE 342 * (i.e. 1<<PAGE_SHIFT) 343 * - Since the log device has to be written in units of 344 * 1<<wl_log_dev_bshift it is required to be a mulitple of 345 * 1<<wl_log_dev_bshift. 346 * - Since filesystem will provide data in units of 1<<wl_fs_dev_bshift, 347 * it is convenient to be a multiple of 1<<wl_fs_dev_bshift. 348 * Therefore it must be multiple of the least common multiple of those 349 * three quantities. Fortunately, all of those quantities are 350 * guaranteed to be a power of two, and the least common multiple of 351 * a set of numbers which are all powers of two is simply the maximum 352 * of those numbers. Finally, the maximum logarithm of a power of two 353 * is the same as the log of the maximum power of two. So we can do 354 * the following operations to size wl_bufbytes_max: 355 */ 356 357 /* XXX fix actual number of pages reserved per filesystem. */ 358 wl->wl_bufbytes_max = MIN(wl->wl_circ_size, buf_memcalc() / 2); 359 360 /* Round wl_bufbytes_max to the largest power of two constraint */ 361 wl->wl_bufbytes_max >>= PAGE_SHIFT; 362 wl->wl_bufbytes_max <<= PAGE_SHIFT; 363 wl->wl_bufbytes_max >>= wl->wl_log_dev_bshift; 364 wl->wl_bufbytes_max <<= wl->wl_log_dev_bshift; 365 wl->wl_bufbytes_max >>= wl->wl_fs_dev_bshift; 366 wl->wl_bufbytes_max <<= wl->wl_fs_dev_bshift; 367 368 /* XXX maybe use filesystem fragment size instead of 1024 */ 369 /* XXX fix actual number of buffers reserved per filesystem. */ 370 wl->wl_bufcount_max = (nbuf / 2) * 1024; 371 372 /* XXX tie this into resource estimation */ 373 wl->wl_dealloclim = 2 * btodb(wl->wl_bufbytes_max); 374 375 #if WAPBL_UVM_ALLOC 376 wl->wl_deallocblks = (void *) uvm_km_zalloc(kernel_map, 377 round_page(sizeof(*wl->wl_deallocblks) * wl->wl_dealloclim)); 378 KASSERT(wl->wl_deallocblks != NULL); 379 wl->wl_dealloclens = (void *) uvm_km_zalloc(kernel_map, 380 round_page(sizeof(*wl->wl_dealloclens) * wl->wl_dealloclim)); 381 KASSERT(wl->wl_dealloclens != NULL); 382 #else 383 wl->wl_deallocblks = wapbl_malloc(sizeof(*wl->wl_deallocblks) * 384 wl->wl_dealloclim); 385 wl->wl_dealloclens = wapbl_malloc(sizeof(*wl->wl_dealloclens) * 386 wl->wl_dealloclim); 387 #endif 388 389 wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); 390 391 /* Initialize the commit header */ 392 { 393 struct wapbl_wc_header *wc; 394 size_t len = 1<<wl->wl_log_dev_bshift; 395 wc = wapbl_calloc(1, len); 396 wc->wc_type = WAPBL_WC_HEADER; 397 wc->wc_len = len; 398 wc->wc_circ_off = wl->wl_circ_off; 399 wc->wc_circ_size = wl->wl_circ_size; 400 /* XXX wc->wc_fsid */ 401 wc->wc_log_dev_bshift = wl->wl_log_dev_bshift; 402 wc->wc_fs_dev_bshift = wl->wl_fs_dev_bshift; 403 wl->wl_wc_header = wc; 404 wl->wl_wc_scratch = wapbl_malloc(len); 405 } 406 407 /* 408 * if there was an existing set of unlinked but 409 * allocated inodes, preserve it in the new 410 * log. 411 */ 412 if (wr && wr->wr_inodescnt) { 413 int i; 414 415 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 416 ("wapbl_start: reusing log with %d inodes\n", 417 wr->wr_inodescnt)); 418 419 /* 420 * Its only valid to reuse the replay log if its 421 * the same as the new log we just opened. 422 */ 423 KDASSERT(!wapbl_replay_isopen(wr)); 424 KASSERT(devvp->v_rdev == wr->wr_devvp->v_rdev); 425 KASSERT(logpbn == wr->wr_logpbn); 426 KASSERT(wl->wl_circ_size == wr->wr_wc_header.wc_circ_size); 427 KASSERT(wl->wl_circ_off == wr->wr_wc_header.wc_circ_off); 428 KASSERT(wl->wl_log_dev_bshift == 429 wr->wr_wc_header.wc_log_dev_bshift); 430 KASSERT(wl->wl_fs_dev_bshift == 431 wr->wr_wc_header.wc_fs_dev_bshift); 432 433 wl->wl_wc_header->wc_generation = 434 wr->wr_wc_header.wc_generation + 1; 435 436 for (i = 0; i < wr->wr_inodescnt; i++) 437 wapbl_register_inode(wl, wr->wr_inodes[i].wr_inumber, 438 wr->wr_inodes[i].wr_imode); 439 440 /* Make sure new transaction won't overwrite old inodes list */ 441 KDASSERT(wapbl_transaction_len(wl) <= 442 wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, 443 wr->wr_inodestail)); 444 445 wl->wl_head = wl->wl_tail = wr->wr_inodeshead; 446 wl->wl_reclaimable_bytes = wl->wl_reserved_bytes = 447 wapbl_transaction_len(wl); 448 449 error = wapbl_write_inodes(wl, &wl->wl_head); 450 if (error) 451 goto errout; 452 453 KASSERT(wl->wl_head != wl->wl_tail); 454 KASSERT(wl->wl_head != 0); 455 } 456 457 error = wapbl_write_commit(wl, wl->wl_head, wl->wl_tail); 458 if (error) { 459 goto errout; 460 } 461 462 *wlp = wl; 463 #if defined(WAPBL_DEBUG) 464 wapbl_debug_wl = wl; 465 #endif 466 467 return 0; 468 errout: 469 wapbl_discard(wl); 470 wapbl_free(wl->wl_wc_scratch); 471 wapbl_free(wl->wl_wc_header); 472 #if WAPBL_UVM_ALLOC 473 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_deallocblks, 474 round_page(sizeof(*wl->wl_deallocblks * 475 wl->wl_dealloclim))); 476 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_dealloclens, 477 round_page(sizeof(*wl->wl_dealloclens * 478 wl->wl_dealloclim))); 479 #else 480 wapbl_free(wl->wl_deallocblks); 481 wapbl_free(wl->wl_dealloclens); 482 #endif 483 wapbl_inodetrk_free(wl); 484 wapbl_free(wl); 485 486 return error; 487 } 488 489 /* 490 * Like wapbl_flush, only discards the transaction 491 * completely 492 */ 493 494 void 495 wapbl_discard(struct wapbl *wl) 496 { 497 struct wapbl_entry *we; 498 struct buf *bp; 499 int i; 500 501 /* 502 * XXX we may consider using upgrade here 503 * if we want to call flush from inside a transaction 504 */ 505 rw_enter(&wl->wl_rwlock, RW_WRITER); 506 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 507 wl->wl_dealloccnt); 508 509 #ifdef WAPBL_DEBUG_PRINT 510 { 511 struct wapbl_entry *we; 512 pid_t pid = -1; 513 lwpid_t lid = -1; 514 if (curproc) 515 pid = curproc->p_pid; 516 if (curlwp) 517 lid = curlwp->l_lid; 518 #ifdef WAPBL_DEBUG_BUFBYTES 519 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 520 ("wapbl_discard: thread %d.%d discarding " 521 "transaction\n" 522 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 523 "deallocs=%d inodes=%d\n" 524 "\terrcnt = %u, reclaimable=%zu reserved=%zu " 525 "unsynced=%zu\n", 526 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 527 wl->wl_bcount, wl->wl_dealloccnt, 528 wl->wl_inohashcnt, wl->wl_error_count, 529 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 530 wl->wl_unsynced_bufbytes)); 531 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 532 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 533 ("\tentry: bufcount = %zu, reclaimable = %zu, " 534 "error = %d, unsynced = %zu\n", 535 we->we_bufcount, we->we_reclaimable_bytes, 536 we->we_error, we->we_unsynced_bufbytes)); 537 } 538 #else /* !WAPBL_DEBUG_BUFBYTES */ 539 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 540 ("wapbl_discard: thread %d.%d discarding transaction\n" 541 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 542 "deallocs=%d inodes=%d\n" 543 "\terrcnt = %u, reclaimable=%zu reserved=%zu\n", 544 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 545 wl->wl_bcount, wl->wl_dealloccnt, 546 wl->wl_inohashcnt, wl->wl_error_count, 547 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes)); 548 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 549 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 550 ("\tentry: bufcount = %zu, reclaimable = %zu, " 551 "error = %d\n", 552 we->we_bufcount, we->we_reclaimable_bytes, 553 we->we_error)); 554 } 555 #endif /* !WAPBL_DEBUG_BUFBYTES */ 556 } 557 #endif /* WAPBL_DEBUG_PRINT */ 558 559 for (i = 0; i <= wl->wl_inohashmask; i++) { 560 struct wapbl_ino_head *wih; 561 struct wapbl_ino *wi; 562 563 wih = &wl->wl_inohash[i]; 564 while ((wi = LIST_FIRST(wih)) != NULL) { 565 LIST_REMOVE(wi, wi_hash); 566 pool_put(&wapbl_ino_pool, wi); 567 KASSERT(wl->wl_inohashcnt > 0); 568 wl->wl_inohashcnt--; 569 } 570 } 571 572 /* 573 * clean buffer list 574 */ 575 mutex_enter(&bufcache_lock); 576 mutex_enter(&wl->wl_mtx); 577 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 578 if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { 579 /* 580 * The buffer will be unlocked and 581 * removed from the transaction in brelse 582 */ 583 mutex_exit(&wl->wl_mtx); 584 brelsel(bp, 0); 585 mutex_enter(&wl->wl_mtx); 586 } 587 } 588 mutex_exit(&wl->wl_mtx); 589 mutex_exit(&bufcache_lock); 590 591 /* 592 * Remove references to this wl from wl_entries, free any which 593 * no longer have buffers, others will be freed in wapbl_biodone 594 * when they no longer have any buffers. 595 */ 596 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) != NULL) { 597 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 598 /* XXX should we be accumulating wl_error_count 599 * and increasing reclaimable bytes ? */ 600 we->we_wapbl = NULL; 601 if (we->we_bufcount == 0) { 602 #ifdef WAPBL_DEBUG_BUFBYTES 603 KASSERT(we->we_unsynced_bufbytes == 0); 604 #endif 605 wapbl_free(we); 606 } 607 } 608 609 /* Discard list of deallocs */ 610 wl->wl_dealloccnt = 0; 611 /* XXX should we clear wl_reserved_bytes? */ 612 613 KASSERT(wl->wl_bufbytes == 0); 614 KASSERT(wl->wl_bcount == 0); 615 KASSERT(wl->wl_bufcount == 0); 616 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 617 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 618 KASSERT(wl->wl_inohashcnt == 0); 619 620 rw_exit(&wl->wl_rwlock); 621 } 622 623 int 624 wapbl_stop(struct wapbl *wl, int force) 625 { 626 struct vnode *vp; 627 int error; 628 629 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_stop called\n")); 630 error = wapbl_flush(wl, 1); 631 if (error) { 632 if (force) 633 wapbl_discard(wl); 634 else 635 return error; 636 } 637 638 /* Unlinked inodes persist after a flush */ 639 if (wl->wl_inohashcnt) { 640 if (force) { 641 wapbl_discard(wl); 642 } else { 643 return EBUSY; 644 } 645 } 646 647 KASSERT(wl->wl_bufbytes == 0); 648 KASSERT(wl->wl_bcount == 0); 649 KASSERT(wl->wl_bufcount == 0); 650 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 651 KASSERT(wl->wl_dealloccnt == 0); 652 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 653 KASSERT(wl->wl_inohashcnt == 0); 654 655 vp = wl->wl_logvp; 656 657 wapbl_free(wl->wl_wc_scratch); 658 wapbl_free(wl->wl_wc_header); 659 #if WAPBL_UVM_ALLOC 660 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_deallocblks, 661 round_page(sizeof(*wl->wl_deallocblks * 662 wl->wl_dealloclim))); 663 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_dealloclens, 664 round_page(sizeof(*wl->wl_dealloclens * 665 wl->wl_dealloclim))); 666 #else 667 wapbl_free(wl->wl_deallocblks); 668 wapbl_free(wl->wl_dealloclens); 669 #endif 670 wapbl_inodetrk_free(wl); 671 672 cv_destroy(&wl->wl_reclaimable_cv); 673 mutex_destroy(&wl->wl_mtx); 674 rw_destroy(&wl->wl_rwlock); 675 wapbl_free(wl); 676 677 return 0; 678 } 679 680 static int 681 wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) 682 { 683 struct pstats *pstats = curlwp->l_proc->p_stats; 684 struct buf *bp; 685 int error; 686 687 KASSERT((flags & ~(B_WRITE | B_READ)) == 0); 688 KASSERT(devvp->v_type == VBLK); 689 690 if ((flags & (B_WRITE | B_READ)) == B_WRITE) { 691 mutex_enter(&devvp->v_interlock); 692 devvp->v_numoutput++; 693 mutex_exit(&devvp->v_interlock); 694 pstats->p_ru.ru_oublock++; 695 } else { 696 pstats->p_ru.ru_inblock++; 697 } 698 699 bp = getiobuf(devvp, true); 700 bp->b_flags = flags; 701 bp->b_cflags = BC_BUSY; /* silly & dubious */ 702 bp->b_dev = devvp->v_rdev; 703 bp->b_data = data; 704 bp->b_bufsize = bp->b_resid = bp->b_bcount = len; 705 bp->b_blkno = pbn; 706 707 WAPBL_PRINTF(WAPBL_PRINT_IO, 708 ("wapbl_doio: %s %d bytes at block %"PRId64" on dev 0x%x\n", 709 BUF_ISWRITE(bp) ? "write" : "read", bp->b_bcount, 710 bp->b_blkno, bp->b_dev)); 711 712 VOP_STRATEGY(devvp, bp); 713 714 error = biowait(bp); 715 putiobuf(bp); 716 717 if (error) { 718 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 719 ("wapbl_doio: %s %zu bytes at block %" PRId64 720 " on dev 0x%x failed with error %d\n", 721 (((flags & (B_WRITE | B_READ)) == B_WRITE) ? 722 "write" : "read"), 723 len, pbn, devvp->v_rdev, error)); 724 } 725 726 return error; 727 } 728 729 int 730 wapbl_write(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 731 { 732 733 return wapbl_doio(data, len, devvp, pbn, B_WRITE); 734 } 735 736 int 737 wapbl_read(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 738 { 739 740 return wapbl_doio(data, len, devvp, pbn, B_READ); 741 } 742 743 /* 744 * Off is byte offset returns new offset for next write 745 * handles log wraparound 746 */ 747 static int 748 wapbl_circ_write(struct wapbl *wl, void *data, size_t len, off_t *offp) 749 { 750 size_t slen; 751 off_t off = *offp; 752 int error; 753 754 KDASSERT(((len >> wl->wl_log_dev_bshift) << 755 wl->wl_log_dev_bshift) == len); 756 757 if (off < wl->wl_circ_off) 758 off = wl->wl_circ_off; 759 slen = wl->wl_circ_off + wl->wl_circ_size - off; 760 if (slen < len) { 761 error = wapbl_write(data, slen, wl->wl_devvp, 762 wl->wl_logpbn + (off >> wl->wl_log_dev_bshift)); 763 if (error) 764 return error; 765 data = (uint8_t *)data + slen; 766 len -= slen; 767 off = wl->wl_circ_off; 768 } 769 error = wapbl_write(data, len, wl->wl_devvp, 770 wl->wl_logpbn + (off >> wl->wl_log_dev_bshift)); 771 if (error) 772 return error; 773 off += len; 774 if (off >= wl->wl_circ_off + wl->wl_circ_size) 775 off = wl->wl_circ_off; 776 *offp = off; 777 return 0; 778 } 779 780 /****************************************************************/ 781 782 int 783 wapbl_begin(struct wapbl *wl, const char *file, int line) 784 { 785 int doflush; 786 unsigned lockcount; 787 krw_t op; 788 789 KDASSERT(wl); 790 791 /* 792 * XXX: The original code calls for the use of a RW_READER lock 793 * here, but it turns out there are performance issues with high 794 * metadata-rate workloads (e.g. multiple simultaneous tar 795 * extractions). For now, we force the lock to be RW_WRITER, 796 * since that currently has the best performance characteristics 797 * (even for a single tar-file extraction). 798 * 799 */ 800 #define WAPBL_DEBUG_SERIALIZE 1 801 802 #ifdef WAPBL_DEBUG_SERIALIZE 803 op = RW_WRITER; 804 #else 805 op = RW_READER; 806 #endif 807 808 /* 809 * XXX this needs to be made much more sophisticated. 810 * perhaps each wapbl_begin could reserve a specified 811 * number of buffers and bytes. 812 */ 813 mutex_enter(&wl->wl_mtx); 814 lockcount = wl->wl_lock_count; 815 doflush = ((wl->wl_bufbytes + (lockcount * MAXPHYS)) > 816 wl->wl_bufbytes_max / 2) || 817 ((wl->wl_bufcount + (lockcount * 10)) > 818 wl->wl_bufcount_max / 2) || 819 (wapbl_transaction_len(wl) > wl->wl_circ_size / 2); 820 mutex_exit(&wl->wl_mtx); 821 822 if (doflush) { 823 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 824 ("force flush lockcnt=%d bufbytes=%zu " 825 "(max=%zu) bufcount=%zu (max=%zu)\n", 826 lockcount, wl->wl_bufbytes, 827 wl->wl_bufbytes_max, wl->wl_bufcount, 828 wl->wl_bufcount_max)); 829 } 830 831 if (doflush) { 832 int error = wapbl_flush(wl, 0); 833 if (error) 834 return error; 835 } 836 837 rw_enter(&wl->wl_rwlock, op); 838 mutex_enter(&wl->wl_mtx); 839 wl->wl_lock_count++; 840 mutex_exit(&wl->wl_mtx); 841 842 #if defined(WAPBL_DEBUG_PRINT) && defined(WAPBL_DEBUG_SERIALIZE) 843 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 844 ("wapbl_begin thread %d.%d with bufcount=%zu " 845 "bufbytes=%zu bcount=%zu at %s:%d\n", 846 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 847 wl->wl_bufbytes, wl->wl_bcount, file, line)); 848 #endif 849 850 return 0; 851 } 852 853 void 854 wapbl_end(struct wapbl *wl) 855 { 856 857 #if defined(WAPBL_DEBUG_PRINT) && defined(WAPBL_DEBUG_SERIALIZE) 858 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 859 ("wapbl_end thread %d.%d with bufcount=%zu " 860 "bufbytes=%zu bcount=%zu\n", 861 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 862 wl->wl_bufbytes, wl->wl_bcount)); 863 #endif 864 865 mutex_enter(&wl->wl_mtx); 866 KASSERT(wl->wl_lock_count > 0); 867 wl->wl_lock_count--; 868 mutex_exit(&wl->wl_mtx); 869 870 rw_exit(&wl->wl_rwlock); 871 } 872 873 void 874 wapbl_add_buf(struct wapbl *wl, struct buf * bp) 875 { 876 877 KASSERT(bp->b_cflags & BC_BUSY); 878 KASSERT(bp->b_vp); 879 880 wapbl_jlock_assert(wl); 881 882 #if 0 883 /* 884 * XXX this might be an issue for swapfiles. 885 * see uvm_swap.c:1702 886 * 887 * XXX2 why require it then? leap of semantics? 888 */ 889 KASSERT((bp->b_cflags & BC_NOCACHE) == 0); 890 #endif 891 892 mutex_enter(&wl->wl_mtx); 893 if (bp->b_flags & B_LOCKED) { 894 LIST_REMOVE(bp, b_wapbllist); 895 WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, 896 ("wapbl_add_buf thread %d.%d re-adding buf %p " 897 "with %d bytes %d bcount\n", 898 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 899 bp->b_bcount)); 900 } else { 901 /* unlocked by dirty buffers shouldn't exist */ 902 KASSERT(!(bp->b_oflags & BO_DELWRI)); 903 wl->wl_bufbytes += bp->b_bufsize; 904 wl->wl_bcount += bp->b_bcount; 905 wl->wl_bufcount++; 906 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 907 ("wapbl_add_buf thread %d.%d adding buf %p " 908 "with %d bytes %d bcount\n", 909 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 910 bp->b_bcount)); 911 } 912 LIST_INSERT_HEAD(&wl->wl_bufs, bp, b_wapbllist); 913 mutex_exit(&wl->wl_mtx); 914 915 bp->b_flags |= B_LOCKED; 916 } 917 918 static void 919 wapbl_remove_buf_locked(struct wapbl * wl, struct buf *bp) 920 { 921 922 KASSERT(mutex_owned(&wl->wl_mtx)); 923 KASSERT(bp->b_cflags & BC_BUSY); 924 wapbl_jlock_assert(wl); 925 926 #if 0 927 /* 928 * XXX this might be an issue for swapfiles. 929 * see uvm_swap.c:1725 930 * 931 * XXXdeux: see above 932 */ 933 KASSERT((bp->b_flags & BC_NOCACHE) == 0); 934 #endif 935 KASSERT(bp->b_flags & B_LOCKED); 936 937 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 938 ("wapbl_remove_buf thread %d.%d removing buf %p with " 939 "%d bytes %d bcount\n", 940 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, bp->b_bcount)); 941 942 KASSERT(wl->wl_bufbytes >= bp->b_bufsize); 943 wl->wl_bufbytes -= bp->b_bufsize; 944 KASSERT(wl->wl_bcount >= bp->b_bcount); 945 wl->wl_bcount -= bp->b_bcount; 946 KASSERT(wl->wl_bufcount > 0); 947 wl->wl_bufcount--; 948 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 949 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 950 LIST_REMOVE(bp, b_wapbllist); 951 952 bp->b_flags &= ~B_LOCKED; 953 } 954 955 /* called from brelsel() in vfs_bio among other places */ 956 void 957 wapbl_remove_buf(struct wapbl * wl, struct buf *bp) 958 { 959 960 mutex_enter(&wl->wl_mtx); 961 wapbl_remove_buf_locked(wl, bp); 962 mutex_exit(&wl->wl_mtx); 963 } 964 965 void 966 wapbl_resize_buf(struct wapbl *wl, struct buf *bp, long oldsz, long oldcnt) 967 { 968 969 KASSERT(bp->b_cflags & BC_BUSY); 970 971 /* 972 * XXX: why does this depend on B_LOCKED? otherwise the buf 973 * is not for a transaction? if so, why is this called in the 974 * first place? 975 */ 976 if (bp->b_flags & B_LOCKED) { 977 mutex_enter(&wl->wl_mtx); 978 wl->wl_bufbytes += bp->b_bufsize - oldsz; 979 wl->wl_bcount += bp->b_bcount - oldcnt; 980 mutex_exit(&wl->wl_mtx); 981 } 982 } 983 984 #endif /* _KERNEL */ 985 986 /****************************************************************/ 987 /* Some utility inlines */ 988 989 /* This is used to advance the pointer at old to new value at old+delta */ 990 static __inline off_t 991 wapbl_advance(size_t size, size_t off, off_t old, size_t delta) 992 { 993 off_t new; 994 995 /* Define acceptable ranges for inputs. */ 996 KASSERT(delta <= size); 997 KASSERT((old == 0) || (old >= off)); 998 KASSERT(old < (size + off)); 999 1000 if ((old == 0) && (delta != 0)) 1001 new = off + delta; 1002 else if ((old + delta) < (size + off)) 1003 new = old + delta; 1004 else 1005 new = (old + delta) - size; 1006 1007 /* Note some interesting axioms */ 1008 KASSERT((delta != 0) || (new == old)); 1009 KASSERT((delta == 0) || (new != 0)); 1010 KASSERT((delta != (size)) || (new == old)); 1011 1012 /* Define acceptable ranges for output. */ 1013 KASSERT((new == 0) || (new >= off)); 1014 KASSERT(new < (size + off)); 1015 return new; 1016 } 1017 1018 static __inline size_t 1019 wapbl_space_used(size_t avail, off_t head, off_t tail) 1020 { 1021 1022 if (tail == 0) { 1023 KASSERT(head == 0); 1024 return 0; 1025 } 1026 return ((head + (avail - 1) - tail) % avail) + 1; 1027 } 1028 1029 static __inline size_t 1030 wapbl_space_free(size_t avail, off_t head, off_t tail) 1031 { 1032 1033 return avail - wapbl_space_used(avail, head, tail); 1034 } 1035 1036 static __inline void 1037 wapbl_advance_head(size_t size, size_t off, size_t delta, off_t *headp, 1038 off_t *tailp) 1039 { 1040 off_t head = *headp; 1041 off_t tail = *tailp; 1042 1043 KASSERT(delta <= wapbl_space_free(size, head, tail)); 1044 head = wapbl_advance(size, off, head, delta); 1045 if ((tail == 0) && (head != 0)) 1046 tail = off; 1047 *headp = head; 1048 *tailp = tail; 1049 } 1050 1051 static __inline void 1052 wapbl_advance_tail(size_t size, size_t off, size_t delta, off_t *headp, 1053 off_t *tailp) 1054 { 1055 off_t head = *headp; 1056 off_t tail = *tailp; 1057 1058 KASSERT(delta <= wapbl_space_used(size, head, tail)); 1059 tail = wapbl_advance(size, off, tail, delta); 1060 if (head == tail) { 1061 head = tail = 0; 1062 } 1063 *headp = head; 1064 *tailp = tail; 1065 } 1066 1067 #ifdef _KERNEL 1068 1069 /****************************************************************/ 1070 1071 /* 1072 * Remove transactions whose buffers are completely flushed to disk. 1073 * Will block until at least minfree space is available. 1074 * only intended to be called from inside wapbl_flush and therefore 1075 * does not protect against commit races with itself or with flush. 1076 */ 1077 static int 1078 wapbl_truncate(struct wapbl *wl, size_t minfree, int waitonly) 1079 { 1080 size_t delta; 1081 size_t avail; 1082 off_t head; 1083 off_t tail; 1084 int error = 0; 1085 1086 KASSERT(minfree <= (wl->wl_circ_size - wl->wl_reserved_bytes)); 1087 KASSERT(rw_write_held(&wl->wl_rwlock)); 1088 1089 mutex_enter(&wl->wl_mtx); 1090 1091 /* 1092 * First check to see if we have to do a commit 1093 * at all. 1094 */ 1095 avail = wapbl_space_free(wl->wl_circ_size, wl->wl_head, wl->wl_tail); 1096 if (minfree < avail) { 1097 mutex_exit(&wl->wl_mtx); 1098 return 0; 1099 } 1100 minfree -= avail; 1101 while ((wl->wl_error_count == 0) && 1102 (wl->wl_reclaimable_bytes < minfree)) { 1103 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1104 ("wapbl_truncate: sleeping on %p wl=%p bytes=%zd " 1105 "minfree=%zd\n", 1106 &wl->wl_reclaimable_bytes, wl, wl->wl_reclaimable_bytes, 1107 minfree)); 1108 1109 cv_wait(&wl->wl_reclaimable_cv, &wl->wl_mtx); 1110 } 1111 if (wl->wl_reclaimable_bytes < minfree) { 1112 KASSERT(wl->wl_error_count); 1113 /* XXX maybe get actual error from buffer instead someday? */ 1114 error = EIO; 1115 } 1116 head = wl->wl_head; 1117 tail = wl->wl_tail; 1118 delta = wl->wl_reclaimable_bytes; 1119 1120 /* If all of of the entries are flushed, then be sure to keep 1121 * the reserved bytes reserved. Watch out for discarded transactions, 1122 * which could leave more bytes reserved than are reclaimable. 1123 */ 1124 if (SIMPLEQ_EMPTY(&wl->wl_entries) && 1125 (delta >= wl->wl_reserved_bytes)) { 1126 delta -= wl->wl_reserved_bytes; 1127 } 1128 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, &head, 1129 &tail); 1130 KDASSERT(wl->wl_reserved_bytes <= 1131 wapbl_space_used(wl->wl_circ_size, head, tail)); 1132 mutex_exit(&wl->wl_mtx); 1133 1134 if (error) 1135 return error; 1136 1137 if (waitonly) 1138 return 0; 1139 1140 /* 1141 * This is where head, tail and delta are unprotected 1142 * from races against itself or flush. This is ok since 1143 * we only call this routine from inside flush itself. 1144 * 1145 * XXX: how can it race against itself when accessed only 1146 * from behind the write-locked rwlock? 1147 */ 1148 error = wapbl_write_commit(wl, head, tail); 1149 if (error) 1150 return error; 1151 1152 wl->wl_head = head; 1153 wl->wl_tail = tail; 1154 1155 mutex_enter(&wl->wl_mtx); 1156 KASSERT(wl->wl_reclaimable_bytes >= delta); 1157 wl->wl_reclaimable_bytes -= delta; 1158 mutex_exit(&wl->wl_mtx); 1159 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1160 ("wapbl_truncate thread %d.%d truncating %zu bytes\n", 1161 curproc->p_pid, curlwp->l_lid, delta)); 1162 1163 return 0; 1164 } 1165 1166 /****************************************************************/ 1167 1168 void 1169 wapbl_biodone(struct buf *bp) 1170 { 1171 struct wapbl_entry *we = bp->b_private; 1172 struct wapbl *wl = we->we_wapbl; 1173 1174 /* 1175 * Handle possible flushing of buffers after log has been 1176 * decomissioned. 1177 */ 1178 if (!wl) { 1179 KASSERT(we->we_bufcount > 0); 1180 we->we_bufcount--; 1181 #ifdef WAPBL_DEBUG_BUFBYTES 1182 KASSERT(we->we_unsynced_bufbytes >= bp->b_bufsize); 1183 we->we_unsynced_bufbytes -= bp->b_bufsize; 1184 #endif 1185 1186 if (we->we_bufcount == 0) { 1187 #ifdef WAPBL_DEBUG_BUFBYTES 1188 KASSERT(we->we_unsynced_bufbytes == 0); 1189 #endif 1190 wapbl_free(we); 1191 } 1192 1193 brelse(bp, 0); 1194 return; 1195 } 1196 1197 #ifdef ohbother 1198 KDASSERT(bp->b_flags & B_DONE); 1199 KDASSERT(!(bp->b_flags & B_DELWRI)); 1200 KDASSERT(bp->b_flags & B_ASYNC); 1201 KDASSERT(bp->b_flags & B_BUSY); 1202 KDASSERT(!(bp->b_flags & B_LOCKED)); 1203 KDASSERT(!(bp->b_flags & B_READ)); 1204 KDASSERT(!(bp->b_flags & B_INVAL)); 1205 KDASSERT(!(bp->b_flags & B_NOCACHE)); 1206 #endif 1207 1208 if (bp->b_error) { 1209 #ifdef notyet /* Can't currently handle possible dirty buffer reuse */ 1210 XXXpooka: interfaces not fully updated 1211 Note: this was not enabled in the original patch 1212 against netbsd4 either. I don't know if comment 1213 above is true or not. 1214 1215 /* 1216 * If an error occurs, report the error and leave the 1217 * buffer as a delayed write on the LRU queue. 1218 * restarting the write would likely result in 1219 * an error spinloop, so let it be done harmlessly 1220 * by the syncer. 1221 */ 1222 bp->b_flags &= ~(B_DONE); 1223 simple_unlock(&bp->b_interlock); 1224 1225 if (we->we_error == 0) { 1226 mutex_enter(&wl->wl_mtx); 1227 wl->wl_error_count++; 1228 mutex_exit(&wl->wl_mtx); 1229 cv_broadcast(&wl->wl_reclaimable_cv); 1230 } 1231 we->we_error = bp->b_error; 1232 bp->b_error = 0; 1233 brelse(bp); 1234 return; 1235 #else 1236 /* For now, just mark the log permanently errored out */ 1237 1238 mutex_enter(&wl->wl_mtx); 1239 if (wl->wl_error_count == 0) { 1240 wl->wl_error_count++; 1241 cv_broadcast(&wl->wl_reclaimable_cv); 1242 } 1243 mutex_exit(&wl->wl_mtx); 1244 #endif 1245 } 1246 1247 mutex_enter(&wl->wl_mtx); 1248 1249 KASSERT(we->we_bufcount > 0); 1250 we->we_bufcount--; 1251 #ifdef WAPBL_DEBUG_BUFBYTES 1252 KASSERT(we->we_unsynced_bufbytes >= bp->b_bufsize); 1253 we->we_unsynced_bufbytes -= bp->b_bufsize; 1254 KASSERT(wl->wl_unsynced_bufbytes >= bp->b_bufsize); 1255 wl->wl_unsynced_bufbytes -= bp->b_bufsize; 1256 #endif 1257 1258 /* 1259 * If the current transaction can be reclaimed, start 1260 * at the beginning and reclaim any consecutive reclaimable 1261 * transactions. If we successfully reclaim anything, 1262 * then wakeup anyone waiting for the reclaim. 1263 */ 1264 if (we->we_bufcount == 0) { 1265 size_t delta = 0; 1266 int errcnt = 0; 1267 #ifdef WAPBL_DEBUG_BUFBYTES 1268 KDASSERT(we->we_unsynced_bufbytes == 0); 1269 #endif 1270 /* 1271 * clear any posted error, since the buffer it came from 1272 * has successfully flushed by now 1273 */ 1274 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) && 1275 (we->we_bufcount == 0)) { 1276 delta += we->we_reclaimable_bytes; 1277 if (we->we_error) 1278 errcnt++; 1279 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 1280 wapbl_free(we); 1281 } 1282 1283 if (delta) { 1284 wl->wl_reclaimable_bytes += delta; 1285 KASSERT(wl->wl_error_count >= errcnt); 1286 wl->wl_error_count -= errcnt; 1287 cv_broadcast(&wl->wl_reclaimable_cv); 1288 } 1289 } 1290 1291 mutex_exit(&wl->wl_mtx); 1292 brelse(bp, 0); 1293 } 1294 1295 /* 1296 * Write transactions to disk + start I/O for contents 1297 */ 1298 int 1299 wapbl_flush(struct wapbl *wl, int waitfor) 1300 { 1301 struct buf *bp; 1302 struct wapbl_entry *we; 1303 off_t off; 1304 off_t head; 1305 off_t tail; 1306 size_t delta = 0; 1307 size_t flushsize; 1308 size_t reserved; 1309 int error = 0; 1310 1311 /* 1312 * Do a quick check to see if a full flush can be skipped 1313 * This assumes that the flush callback does not need to be called 1314 * unless there are other outstanding bufs. 1315 */ 1316 if (!waitfor) { 1317 size_t nbufs; 1318 mutex_enter(&wl->wl_mtx); /* XXX need mutex here to 1319 protect the KASSERTS */ 1320 nbufs = wl->wl_bufcount; 1321 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1322 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1323 mutex_exit(&wl->wl_mtx); 1324 if (nbufs == 0) 1325 return 0; 1326 } 1327 1328 /* 1329 * XXX we may consider using LK_UPGRADE here 1330 * if we want to call flush from inside a transaction 1331 */ 1332 rw_enter(&wl->wl_rwlock, RW_WRITER); 1333 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 1334 wl->wl_dealloccnt); 1335 1336 /* 1337 * Now that we are fully locked and flushed, 1338 * do another check for nothing to do. 1339 */ 1340 if (wl->wl_bufcount == 0) { 1341 goto out; 1342 } 1343 1344 #if 0 1345 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1346 ("wapbl_flush thread %d.%d flushing entries with " 1347 "bufcount=%zu bufbytes=%zu\n", 1348 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1349 wl->wl_bufbytes)); 1350 #endif 1351 1352 /* Calculate amount of space needed to flush */ 1353 flushsize = wapbl_transaction_len(wl); 1354 1355 if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) { 1356 /* 1357 * XXX this could be handled more gracefully, perhaps place 1358 * only a partial transaction in the log and allow the 1359 * remaining to flush without the protection of the journal. 1360 */ 1361 panic("wapbl_flush: current transaction too big to flush\n"); 1362 } 1363 1364 error = wapbl_truncate(wl, flushsize, 0); 1365 if (error) 1366 goto out2; 1367 1368 off = wl->wl_head; 1369 KASSERT((off == 0) || ((off >= wl->wl_circ_off) && 1370 (off < wl->wl_circ_off + wl->wl_circ_size))); 1371 error = wapbl_write_blocks(wl, &off); 1372 if (error) 1373 goto out2; 1374 error = wapbl_write_revocations(wl, &off); 1375 if (error) 1376 goto out2; 1377 error = wapbl_write_inodes(wl, &off); 1378 if (error) 1379 goto out2; 1380 1381 reserved = 0; 1382 if (wl->wl_inohashcnt) 1383 reserved = wapbl_transaction_inodes_len(wl); 1384 1385 head = wl->wl_head; 1386 tail = wl->wl_tail; 1387 1388 wapbl_advance_head(wl->wl_circ_size, wl->wl_circ_off, flushsize, 1389 &head, &tail); 1390 #ifdef WAPBL_DEBUG 1391 if (head != off) { 1392 panic("lost head! head=%"PRIdMAX" tail=%" PRIdMAX 1393 " off=%"PRIdMAX" flush=%zu\n", 1394 (intmax_t)head, (intmax_t)tail, (intmax_t)off, 1395 flushsize); 1396 } 1397 #else 1398 KASSERT(head == off); 1399 #endif 1400 1401 /* Opportunistically move the tail forward if we can */ 1402 if (!wapbl_lazy_truncate) { 1403 mutex_enter(&wl->wl_mtx); 1404 delta = wl->wl_reclaimable_bytes; 1405 mutex_exit(&wl->wl_mtx); 1406 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, 1407 &head, &tail); 1408 } 1409 1410 error = wapbl_write_commit(wl, head, tail); 1411 if (error) 1412 goto out2; 1413 1414 /* poolme? or kmemme? */ 1415 we = wapbl_calloc(1, sizeof(*we)); 1416 1417 #ifdef WAPBL_DEBUG_BUFBYTES 1418 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1419 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1420 " unsynced=%zu" 1421 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1422 "inodes=%d\n", 1423 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1424 wapbl_space_used(wl->wl_circ_size, head, tail), 1425 wl->wl_unsynced_bufbytes, wl->wl_bufcount, 1426 wl->wl_bufbytes, wl->wl_bcount, wl->wl_dealloccnt, 1427 wl->wl_inohashcnt)); 1428 #else 1429 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1430 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1431 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1432 "inodes=%d\n", 1433 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1434 wapbl_space_used(wl->wl_circ_size, head, tail), 1435 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1436 wl->wl_dealloccnt, wl->wl_inohashcnt)); 1437 #endif 1438 1439 1440 mutex_enter(&bufcache_lock); 1441 mutex_enter(&wl->wl_mtx); 1442 1443 wl->wl_reserved_bytes = reserved; 1444 wl->wl_head = head; 1445 wl->wl_tail = tail; 1446 KASSERT(wl->wl_reclaimable_bytes >= delta); 1447 wl->wl_reclaimable_bytes -= delta; 1448 wl->wl_dealloccnt = 0; 1449 #ifdef WAPBL_DEBUG_BUFBYTES 1450 wl->wl_unsynced_bufbytes += wl->wl_bufbytes; 1451 #endif 1452 1453 we->we_wapbl = wl; 1454 we->we_bufcount = wl->wl_bufcount; 1455 #ifdef WAPBL_DEBUG_BUFBYTES 1456 we->we_unsynced_bufbytes = wl->wl_bufbytes; 1457 #endif 1458 we->we_reclaimable_bytes = flushsize; 1459 we->we_error = 0; 1460 SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); 1461 1462 /* 1463 * this flushes bufs in reverse order than they were queued 1464 * it shouldn't matter, but if we care we could use TAILQ instead. 1465 * XXX Note they will get put on the lru queue when they flush 1466 * so we might actually want to change this to preserve order. 1467 */ 1468 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 1469 if (bbusy(bp, 0, 0, &wl->wl_mtx)) { 1470 continue; 1471 } 1472 bp->b_iodone = wapbl_biodone; 1473 bp->b_private = we; 1474 bremfree(bp); 1475 wapbl_remove_buf_locked(wl, bp); 1476 mutex_exit(&wl->wl_mtx); 1477 mutex_exit(&bufcache_lock); 1478 bawrite(bp); 1479 mutex_enter(&bufcache_lock); 1480 mutex_enter(&wl->wl_mtx); 1481 } 1482 mutex_exit(&wl->wl_mtx); 1483 mutex_exit(&bufcache_lock); 1484 1485 #if 0 1486 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1487 ("wapbl_flush thread %d.%d done flushing entries...\n", 1488 curproc->p_pid, curlwp->l_lid)); 1489 #endif 1490 1491 out: 1492 1493 /* 1494 * If the waitfor flag is set, don't return until everything is 1495 * fully flushed and the on disk log is empty. 1496 */ 1497 if (waitfor) { 1498 error = wapbl_truncate(wl, wl->wl_circ_size - 1499 wl->wl_reserved_bytes, wapbl_lazy_truncate); 1500 } 1501 1502 out2: 1503 if (error) { 1504 wl->wl_flush_abort(wl->wl_mount, wl->wl_deallocblks, 1505 wl->wl_dealloclens, wl->wl_dealloccnt); 1506 } 1507 1508 #ifdef WAPBL_DEBUG_PRINT 1509 if (error) { 1510 pid_t pid = -1; 1511 lwpid_t lid = -1; 1512 if (curproc) 1513 pid = curproc->p_pid; 1514 if (curlwp) 1515 lid = curlwp->l_lid; 1516 mutex_enter(&wl->wl_mtx); 1517 #ifdef WAPBL_DEBUG_BUFBYTES 1518 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1519 ("wapbl_flush: thread %d.%d aborted flush: " 1520 "error = %d\n" 1521 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1522 "deallocs=%d inodes=%d\n" 1523 "\terrcnt = %d, reclaimable=%zu reserved=%zu " 1524 "unsynced=%zu\n", 1525 pid, lid, error, wl->wl_bufcount, 1526 wl->wl_bufbytes, wl->wl_bcount, 1527 wl->wl_dealloccnt, wl->wl_inohashcnt, 1528 wl->wl_error_count, wl->wl_reclaimable_bytes, 1529 wl->wl_reserved_bytes, wl->wl_unsynced_bufbytes)); 1530 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1531 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1532 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1533 "error = %d, unsynced = %zu\n", 1534 we->we_bufcount, we->we_reclaimable_bytes, 1535 we->we_error, we->we_unsynced_bufbytes)); 1536 } 1537 #else 1538 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1539 ("wapbl_flush: thread %d.%d aborted flush: " 1540 "error = %d\n" 1541 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1542 "deallocs=%d inodes=%d\n" 1543 "\terrcnt = %d, reclaimable=%zu reserved=%zu\n", 1544 pid, lid, error, wl->wl_bufcount, 1545 wl->wl_bufbytes, wl->wl_bcount, 1546 wl->wl_dealloccnt, wl->wl_inohashcnt, 1547 wl->wl_error_count, wl->wl_reclaimable_bytes, 1548 wl->wl_reserved_bytes)); 1549 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1550 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1551 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1552 "error = %d\n", we->we_bufcount, 1553 we->we_reclaimable_bytes, we->we_error)); 1554 } 1555 #endif 1556 mutex_exit(&wl->wl_mtx); 1557 } 1558 #endif 1559 1560 rw_exit(&wl->wl_rwlock); 1561 return error; 1562 } 1563 1564 /****************************************************************/ 1565 1566 void 1567 wapbl_jlock_assert(struct wapbl *wl) 1568 { 1569 1570 #ifdef WAPBL_DEBUG_SERIALIZE 1571 KASSERT(rw_write_held(&wl->wl_rwlock)); 1572 #else 1573 KASSERT(rw_read_held(&wl->wl_rwlock) || rw_write_held(&wl->wl_rwlock)); 1574 #endif 1575 } 1576 1577 void 1578 wapbl_junlock_assert(struct wapbl *wl) 1579 { 1580 1581 #ifdef WAPBL_DEBUG_SERIALIZE 1582 KASSERT(!rw_write_held(&wl->wl_rwlock)); 1583 #endif 1584 } 1585 1586 /****************************************************************/ 1587 1588 /* locks missing */ 1589 void 1590 wapbl_print(struct wapbl *wl, 1591 int full, 1592 void (*pr)(const char *, ...)) 1593 { 1594 struct buf *bp; 1595 struct wapbl_entry *we; 1596 (*pr)("wapbl %p", wl); 1597 (*pr)("\nlogvp = %p, devvp = %p, logpbn = %"PRId64"\n", 1598 wl->wl_logvp, wl->wl_devvp, wl->wl_logpbn); 1599 (*pr)("circ = %zu, header = %zu, head = %"PRIdMAX" tail = %"PRIdMAX"\n", 1600 wl->wl_circ_size, wl->wl_circ_off, 1601 (intmax_t)wl->wl_head, (intmax_t)wl->wl_tail); 1602 (*pr)("fs_dev_bshift = %d, log_dev_bshift = %d\n", 1603 wl->wl_log_dev_bshift, wl->wl_fs_dev_bshift); 1604 #ifdef WAPBL_DEBUG_BUFBYTES 1605 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1606 "reserved = %zu errcnt = %d unsynced = %zu\n", 1607 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1608 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1609 wl->wl_error_count, wl->wl_unsynced_bufbytes); 1610 #else 1611 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1612 "reserved = %zu errcnt = %d\n", wl->wl_bufcount, wl->wl_bufbytes, 1613 wl->wl_bcount, wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1614 wl->wl_error_count); 1615 #endif 1616 (*pr)("\tdealloccnt = %d, dealloclim = %d\n", 1617 wl->wl_dealloccnt, wl->wl_dealloclim); 1618 (*pr)("\tinohashcnt = %d, inohashmask = 0x%08x\n", 1619 wl->wl_inohashcnt, wl->wl_inohashmask); 1620 (*pr)("entries:\n"); 1621 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1622 #ifdef WAPBL_DEBUG_BUFBYTES 1623 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d, " 1624 "unsynced = %zu\n", 1625 we->we_bufcount, we->we_reclaimable_bytes, 1626 we->we_error, we->we_unsynced_bufbytes); 1627 #else 1628 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d\n", 1629 we->we_bufcount, we->we_reclaimable_bytes, we->we_error); 1630 #endif 1631 } 1632 if (full) { 1633 int cnt = 0; 1634 (*pr)("bufs ="); 1635 LIST_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { 1636 if (!LIST_NEXT(bp, b_wapbllist)) { 1637 (*pr)(" %p", bp); 1638 } else if ((++cnt % 6) == 0) { 1639 (*pr)(" %p,\n\t", bp); 1640 } else { 1641 (*pr)(" %p,", bp); 1642 } 1643 } 1644 (*pr)("\n"); 1645 1646 (*pr)("dealloced blks = "); 1647 { 1648 int i; 1649 cnt = 0; 1650 for (i = 0; i < wl->wl_dealloccnt; i++) { 1651 (*pr)(" %"PRId64":%d,", 1652 wl->wl_deallocblks[i], 1653 wl->wl_dealloclens[i]); 1654 if ((++cnt % 4) == 0) { 1655 (*pr)("\n\t"); 1656 } 1657 } 1658 } 1659 (*pr)("\n"); 1660 1661 (*pr)("registered inodes = "); 1662 { 1663 int i; 1664 cnt = 0; 1665 for (i = 0; i <= wl->wl_inohashmask; i++) { 1666 struct wapbl_ino_head *wih; 1667 struct wapbl_ino *wi; 1668 1669 wih = &wl->wl_inohash[i]; 1670 LIST_FOREACH(wi, wih, wi_hash) { 1671 if (wi->wi_ino == 0) 1672 continue; 1673 (*pr)(" %"PRId32"/0%06"PRIo32",", 1674 wi->wi_ino, wi->wi_mode); 1675 if ((++cnt % 4) == 0) { 1676 (*pr)("\n\t"); 1677 } 1678 } 1679 } 1680 (*pr)("\n"); 1681 } 1682 } 1683 } 1684 1685 #if defined(WAPBL_DEBUG) || defined(DDB) 1686 void 1687 wapbl_dump(struct wapbl *wl) 1688 { 1689 #if defined(WAPBL_DEBUG) 1690 if (!wl) 1691 wl = wapbl_debug_wl; 1692 #endif 1693 if (!wl) 1694 return; 1695 wapbl_print(wl, 1, printf); 1696 } 1697 #endif 1698 1699 /****************************************************************/ 1700 1701 void 1702 wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len) 1703 { 1704 1705 wapbl_jlock_assert(wl); 1706 1707 /* XXX should eventually instead tie this into resource estimation */ 1708 /* XXX this KASSERT needs locking/mutex analysis */ 1709 KASSERT(wl->wl_dealloccnt < wl->wl_dealloclim); 1710 wl->wl_deallocblks[wl->wl_dealloccnt] = blk; 1711 wl->wl_dealloclens[wl->wl_dealloccnt] = len; 1712 wl->wl_dealloccnt++; 1713 WAPBL_PRINTF(WAPBL_PRINT_ALLOC, 1714 ("wapbl_register_deallocation: blk=%"PRId64" len=%d\n", blk, len)); 1715 } 1716 1717 /****************************************************************/ 1718 1719 static void 1720 wapbl_inodetrk_init(struct wapbl *wl, u_int size) 1721 { 1722 1723 wl->wl_inohash = hashinit(size, HASH_LIST, true, &wl->wl_inohashmask); 1724 if (atomic_inc_uint_nv(&wapbl_ino_pool_refcount) == 1) { 1725 pool_init(&wapbl_ino_pool, sizeof(struct wapbl_ino), 0, 0, 0, 1726 "wapblinopl", &pool_allocator_nointr, IPL_NONE); 1727 } 1728 } 1729 1730 static void 1731 wapbl_inodetrk_free(struct wapbl *wl) 1732 { 1733 1734 /* XXX this KASSERT needs locking/mutex analysis */ 1735 KASSERT(wl->wl_inohashcnt == 0); 1736 hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask); 1737 if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) { 1738 pool_destroy(&wapbl_ino_pool); 1739 } 1740 } 1741 1742 static struct wapbl_ino * 1743 wapbl_inodetrk_get(struct wapbl *wl, ino_t ino) 1744 { 1745 struct wapbl_ino_head *wih; 1746 struct wapbl_ino *wi; 1747 1748 KASSERT(mutex_owned(&wl->wl_mtx)); 1749 1750 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1751 LIST_FOREACH(wi, wih, wi_hash) { 1752 if (ino == wi->wi_ino) 1753 return wi; 1754 } 1755 return 0; 1756 } 1757 1758 void 1759 wapbl_register_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1760 { 1761 struct wapbl_ino_head *wih; 1762 struct wapbl_ino *wi; 1763 1764 wi = pool_get(&wapbl_ino_pool, PR_WAITOK); 1765 1766 mutex_enter(&wl->wl_mtx); 1767 if (wapbl_inodetrk_get(wl, ino) == NULL) { 1768 wi->wi_ino = ino; 1769 wi->wi_mode = mode; 1770 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1771 LIST_INSERT_HEAD(wih, wi, wi_hash); 1772 wl->wl_inohashcnt++; 1773 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1774 ("wapbl_register_inode: ino=%"PRId64"\n", ino)); 1775 mutex_exit(&wl->wl_mtx); 1776 } else { 1777 mutex_exit(&wl->wl_mtx); 1778 pool_put(&wapbl_ino_pool, wi); 1779 } 1780 } 1781 1782 void 1783 wapbl_unregister_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1784 { 1785 struct wapbl_ino *wi; 1786 1787 mutex_enter(&wl->wl_mtx); 1788 wi = wapbl_inodetrk_get(wl, ino); 1789 if (wi) { 1790 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1791 ("wapbl_unregister_inode: ino=%"PRId64"\n", ino)); 1792 KASSERT(wl->wl_inohashcnt > 0); 1793 wl->wl_inohashcnt--; 1794 LIST_REMOVE(wi, wi_hash); 1795 mutex_exit(&wl->wl_mtx); 1796 1797 pool_put(&wapbl_ino_pool, wi); 1798 } else { 1799 mutex_exit(&wl->wl_mtx); 1800 } 1801 } 1802 1803 /****************************************************************/ 1804 1805 static __inline size_t 1806 wapbl_transaction_inodes_len(struct wapbl *wl) 1807 { 1808 int blocklen = 1<<wl->wl_log_dev_bshift; 1809 int iph; 1810 1811 /* Calculate number of inodes described in a inodelist header */ 1812 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 1813 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 1814 1815 KASSERT(iph > 0); 1816 1817 return MAX(1, howmany(wl->wl_inohashcnt, iph))*blocklen; 1818 } 1819 1820 1821 /* Calculate amount of space a transaction will take on disk */ 1822 static size_t 1823 wapbl_transaction_len(struct wapbl *wl) 1824 { 1825 int blocklen = 1<<wl->wl_log_dev_bshift; 1826 size_t len; 1827 int bph; 1828 1829 /* Calculate number of blocks described in a blocklist header */ 1830 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1831 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1832 1833 KASSERT(bph > 0); 1834 1835 len = wl->wl_bcount; 1836 len += howmany(wl->wl_bufcount, bph)*blocklen; 1837 len += howmany(wl->wl_dealloccnt, bph)*blocklen; 1838 len += wapbl_transaction_inodes_len(wl); 1839 1840 return len; 1841 } 1842 1843 /* 1844 * Perform commit operation 1845 * 1846 * Note that generation number incrementation needs to 1847 * be protected against racing with other invocations 1848 * of wapbl_commit. This is ok since this routine 1849 * is only invoked from wapbl_flush 1850 */ 1851 static int 1852 wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail) 1853 { 1854 struct wapbl_wc_header *wc = wl->wl_wc_header; 1855 struct timespec ts; 1856 int error; 1857 int force = 1; 1858 1859 /* XXX Calc checksum here, instead we do this for now */ 1860 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, FWRITE, FSCRED); 1861 if (error) { 1862 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1863 ("wapbl_write_commit: DIOCCACHESYNC on dev 0x%x " 1864 "returned %d\n", wl->wl_devvp->v_rdev, error)); 1865 } 1866 1867 wc->wc_head = head; 1868 wc->wc_tail = tail; 1869 wc->wc_checksum = 0; 1870 wc->wc_version = 1; 1871 getnanotime(&ts); 1872 wc->wc_time = ts.tv_sec;; 1873 wc->wc_timensec = ts.tv_nsec; 1874 1875 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 1876 ("wapbl_write_commit: head = %"PRIdMAX "tail = %"PRIdMAX"\n", 1877 (intmax_t)head, (intmax_t)tail)); 1878 1879 /* 1880 * XXX if generation will rollover, then first zero 1881 * over second commit header before trying to write both headers. 1882 */ 1883 1884 error = wapbl_write(wc, wc->wc_len, wl->wl_devvp, 1885 wl->wl_logpbn + wc->wc_generation % 2); 1886 if (error) 1887 return error; 1888 1889 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, FWRITE, FSCRED); 1890 if (error) { 1891 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1892 ("wapbl_write_commit: DIOCCACHESYNC on dev 0x%x " 1893 "returned %d\n", wl->wl_devvp->v_rdev, error)); 1894 } 1895 1896 /* 1897 * If the generation number was zero, write it out a second time. 1898 * This handles initialization and generation number rollover 1899 */ 1900 if (wc->wc_generation++ == 0) { 1901 error = wapbl_write_commit(wl, head, tail); 1902 /* 1903 * This panic should be able to be removed if we do the 1904 * zero'ing mentioned above, and we are certain to roll 1905 * back generation number on failure. 1906 */ 1907 if (error) 1908 panic("wapbl_write_commit: error writing duplicate " 1909 "log header: %d\n", error); 1910 } 1911 return 0; 1912 } 1913 1914 /* Returns new offset value */ 1915 static int 1916 wapbl_write_blocks(struct wapbl *wl, off_t *offp) 1917 { 1918 struct wapbl_wc_blocklist *wc = 1919 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 1920 int blocklen = 1<<wl->wl_log_dev_bshift; 1921 int bph; 1922 struct buf *bp; 1923 off_t off = *offp; 1924 int error; 1925 1926 KASSERT(rw_write_held(&wl->wl_rwlock)); 1927 1928 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1929 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1930 1931 bp = LIST_FIRST(&wl->wl_bufs); 1932 1933 while (bp) { 1934 int cnt; 1935 struct buf *obp = bp; 1936 1937 KASSERT(bp->b_flags & B_LOCKED); 1938 1939 wc->wc_type = WAPBL_WC_BLOCKS; 1940 wc->wc_len = blocklen; 1941 wc->wc_blkcount = 0; 1942 while (bp && (wc->wc_blkcount < bph)) { 1943 /* 1944 * Make sure all the physical block numbers are up to 1945 * date. If this is not always true on a given 1946 * filesystem, then VOP_BMAP must be called. We 1947 * could call VOP_BMAP here, or else in the filesystem 1948 * specific flush callback, although neither of those 1949 * solutions allow us to take the vnode lock. If a 1950 * filesystem requires that we must take the vnode lock 1951 * to call VOP_BMAP, then we can probably do it in 1952 * bwrite when the vnode lock should already be held 1953 * by the invoking code. 1954 */ 1955 KASSERT((bp->b_vp->v_type == VBLK) || 1956 (bp->b_blkno != bp->b_lblkno)); 1957 KASSERT(bp->b_blkno > 0); 1958 1959 wc->wc_blocks[wc->wc_blkcount].wc_daddr = bp->b_blkno; 1960 wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; 1961 wc->wc_len += bp->b_bcount; 1962 wc->wc_blkcount++; 1963 bp = LIST_NEXT(bp, b_wapbllist); 1964 } 1965 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 1966 ("wapbl_write_blocks: len = %u off = %"PRIdMAX"\n", 1967 wc->wc_len, (intmax_t)off)); 1968 1969 error = wapbl_circ_write(wl, wc, blocklen, &off); 1970 if (error) 1971 return error; 1972 bp = obp; 1973 cnt = 0; 1974 while (bp && (cnt++ < bph)) { 1975 error = wapbl_circ_write(wl, bp->b_data, 1976 bp->b_bcount, &off); 1977 if (error) 1978 return error; 1979 bp = LIST_NEXT(bp, b_wapbllist); 1980 } 1981 } 1982 *offp = off; 1983 return 0; 1984 } 1985 1986 static int 1987 wapbl_write_revocations(struct wapbl *wl, off_t *offp) 1988 { 1989 struct wapbl_wc_blocklist *wc = 1990 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 1991 int i; 1992 int blocklen = 1<<wl->wl_log_dev_bshift; 1993 int bph; 1994 off_t off = *offp; 1995 int error; 1996 1997 if (wl->wl_dealloccnt == 0) 1998 return 0; 1999 2000 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 2001 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 2002 2003 i = 0; 2004 while (i < wl->wl_dealloccnt) { 2005 wc->wc_type = WAPBL_WC_REVOCATIONS; 2006 wc->wc_len = blocklen; 2007 wc->wc_blkcount = 0; 2008 while ((i < wl->wl_dealloccnt) && (wc->wc_blkcount < bph)) { 2009 wc->wc_blocks[wc->wc_blkcount].wc_daddr = 2010 wl->wl_deallocblks[i]; 2011 wc->wc_blocks[wc->wc_blkcount].wc_dlen = 2012 wl->wl_dealloclens[i]; 2013 wc->wc_blkcount++; 2014 i++; 2015 } 2016 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2017 ("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", 2018 wc->wc_len, (intmax_t)off)); 2019 error = wapbl_circ_write(wl, wc, blocklen, &off); 2020 if (error) 2021 return error; 2022 } 2023 *offp = off; 2024 return 0; 2025 } 2026 2027 static int 2028 wapbl_write_inodes(struct wapbl *wl, off_t *offp) 2029 { 2030 struct wapbl_wc_inodelist *wc = 2031 (struct wapbl_wc_inodelist *)wl->wl_wc_scratch; 2032 int i; 2033 int blocklen = 1<<wl->wl_log_dev_bshift; 2034 off_t off = *offp; 2035 int error; 2036 2037 struct wapbl_ino_head *wih; 2038 struct wapbl_ino *wi; 2039 int iph; 2040 2041 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2042 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2043 2044 i = 0; 2045 wih = &wl->wl_inohash[0]; 2046 wi = 0; 2047 do { 2048 wc->wc_type = WAPBL_WC_INODES; 2049 wc->wc_len = blocklen; 2050 wc->wc_inocnt = 0; 2051 wc->wc_clear = (i == 0); 2052 while ((i < wl->wl_inohashcnt) && (wc->wc_inocnt < iph)) { 2053 while (!wi) { 2054 KASSERT((wih - &wl->wl_inohash[0]) 2055 <= wl->wl_inohashmask); 2056 wi = LIST_FIRST(wih++); 2057 } 2058 wc->wc_inodes[wc->wc_inocnt].wc_inumber = wi->wi_ino; 2059 wc->wc_inodes[wc->wc_inocnt].wc_imode = wi->wi_mode; 2060 wc->wc_inocnt++; 2061 i++; 2062 wi = LIST_NEXT(wi, wi_hash); 2063 } 2064 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2065 ("wapbl_write_inodes: len = %u off = %"PRIdMAX"\n", 2066 wc->wc_len, (intmax_t)off)); 2067 error = wapbl_circ_write(wl, wc, blocklen, &off); 2068 if (error) 2069 return error; 2070 } while (i < wl->wl_inohashcnt); 2071 2072 *offp = off; 2073 return 0; 2074 } 2075 2076 #endif /* _KERNEL */ 2077 2078 /****************************************************************/ 2079 2080 #ifdef _KERNEL 2081 static struct pool wapbl_blk_pool; 2082 static int wapbl_blk_pool_refcount; 2083 #endif 2084 struct wapbl_blk { 2085 LIST_ENTRY(wapbl_blk) wb_hash; 2086 daddr_t wb_blk; 2087 off_t wb_off; /* Offset of this block in the log */ 2088 }; 2089 #define WAPBL_BLKPOOL_MIN 83 2090 2091 static void 2092 wapbl_blkhash_init(struct wapbl_replay *wr, u_int size) 2093 { 2094 if (size < WAPBL_BLKPOOL_MIN) 2095 size = WAPBL_BLKPOOL_MIN; 2096 KASSERT(wr->wr_blkhash == 0); 2097 #ifdef _KERNEL 2098 wr->wr_blkhash = hashinit(size, HASH_LIST, true, &wr->wr_blkhashmask); 2099 if (atomic_inc_uint_nv(&wapbl_blk_pool_refcount) == 1) { 2100 pool_init(&wapbl_blk_pool, sizeof(struct wapbl_blk), 0, 0, 0, 2101 "wapblblkpl", &pool_allocator_nointr, IPL_NONE); 2102 } 2103 #else /* ! _KERNEL */ 2104 /* Manually implement hashinit */ 2105 { 2106 int i; 2107 unsigned long hashsize; 2108 for (hashsize = 1; hashsize < size; hashsize <<= 1) 2109 continue; 2110 wr->wr_blkhash = wapbl_malloc(hashsize * sizeof(*wr->wr_blkhash)); 2111 for (i = 0; i < wr->wr_blkhashmask; i++) 2112 LIST_INIT(&wr->wr_blkhash[i]); 2113 wr->wr_blkhashmask = hashsize - 1; 2114 } 2115 #endif /* ! _KERNEL */ 2116 } 2117 2118 static void 2119 wapbl_blkhash_free(struct wapbl_replay *wr) 2120 { 2121 KASSERT(wr->wr_blkhashcnt == 0); 2122 #ifdef _KERNEL 2123 hashdone(wr->wr_blkhash, HASH_LIST, wr->wr_blkhashmask); 2124 if (atomic_dec_uint_nv(&wapbl_blk_pool_refcount) == 0) { 2125 pool_destroy(&wapbl_blk_pool); 2126 } 2127 #else /* ! _KERNEL */ 2128 wapbl_free(wr->wr_blkhash); 2129 #endif /* ! _KERNEL */ 2130 } 2131 2132 static struct wapbl_blk * 2133 wapbl_blkhash_get(struct wapbl_replay *wr, daddr_t blk) 2134 { 2135 struct wapbl_blk_head *wbh; 2136 struct wapbl_blk *wb; 2137 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2138 LIST_FOREACH(wb, wbh, wb_hash) { 2139 if (blk == wb->wb_blk) 2140 return wb; 2141 } 2142 return 0; 2143 } 2144 2145 static void 2146 wapbl_blkhash_ins(struct wapbl_replay *wr, daddr_t blk, off_t off) 2147 { 2148 struct wapbl_blk_head *wbh; 2149 struct wapbl_blk *wb; 2150 wb = wapbl_blkhash_get(wr, blk); 2151 if (wb) { 2152 KASSERT(wb->wb_blk == blk); 2153 wb->wb_off = off; 2154 } else { 2155 #ifdef _KERNEL 2156 wb = pool_get(&wapbl_blk_pool, PR_WAITOK); 2157 #else /* ! _KERNEL */ 2158 wb = wapbl_malloc(sizeof(*wb)); 2159 #endif /* ! _KERNEL */ 2160 wb->wb_blk = blk; 2161 wb->wb_off = off; 2162 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2163 LIST_INSERT_HEAD(wbh, wb, wb_hash); 2164 wr->wr_blkhashcnt++; 2165 } 2166 } 2167 2168 static void 2169 wapbl_blkhash_rem(struct wapbl_replay *wr, daddr_t blk) 2170 { 2171 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2172 if (wb) { 2173 KASSERT(wr->wr_blkhashcnt > 0); 2174 wr->wr_blkhashcnt--; 2175 LIST_REMOVE(wb, wb_hash); 2176 #ifdef _KERNEL 2177 pool_put(&wapbl_blk_pool, wb); 2178 #else /* ! _KERNEL */ 2179 wapbl_free(wb); 2180 #endif /* ! _KERNEL */ 2181 } 2182 } 2183 2184 static void 2185 wapbl_blkhash_clear(struct wapbl_replay *wr) 2186 { 2187 int i; 2188 for (i = 0; i <= wr->wr_blkhashmask; i++) { 2189 struct wapbl_blk *wb; 2190 2191 while ((wb = LIST_FIRST(&wr->wr_blkhash[i]))) { 2192 KASSERT(wr->wr_blkhashcnt > 0); 2193 wr->wr_blkhashcnt--; 2194 LIST_REMOVE(wb, wb_hash); 2195 #ifdef _KERNEL 2196 pool_put(&wapbl_blk_pool, wb); 2197 #else /* ! _KERNEL */ 2198 wapbl_free(wb); 2199 #endif /* ! _KERNEL */ 2200 } 2201 } 2202 KASSERT(wr->wr_blkhashcnt == 0); 2203 } 2204 2205 /****************************************************************/ 2206 2207 static int 2208 wapbl_circ_read(struct wapbl_replay *wr, void *data, size_t len, off_t *offp) 2209 { 2210 size_t slen; 2211 struct wapbl_wc_header *wc = &wr->wr_wc_header; 2212 off_t off = *offp; 2213 int error; 2214 2215 KASSERT(((len >> wc->wc_log_dev_bshift) << 2216 wc->wc_log_dev_bshift) == len); 2217 if (off < wc->wc_circ_off) 2218 off = wc->wc_circ_off; 2219 slen = wc->wc_circ_off + wc->wc_circ_size - off; 2220 if (slen < len) { 2221 error = wapbl_read(data, slen, wr->wr_devvp, 2222 wr->wr_logpbn + (off >> wc->wc_log_dev_bshift)); 2223 if (error) 2224 return error; 2225 data = (uint8_t *)data + slen; 2226 len -= slen; 2227 off = wc->wc_circ_off; 2228 } 2229 error = wapbl_read(data, len, wr->wr_devvp, 2230 wr->wr_logpbn + (off >> wc->wc_log_dev_bshift)); 2231 if (error) 2232 return error; 2233 off += len; 2234 if (off >= wc->wc_circ_off + wc->wc_circ_size) 2235 off = wc->wc_circ_off; 2236 *offp = off; 2237 return 0; 2238 } 2239 2240 static void 2241 wapbl_circ_advance(struct wapbl_replay *wr, size_t len, off_t *offp) 2242 { 2243 size_t slen; 2244 struct wapbl_wc_header *wc = &wr->wr_wc_header; 2245 off_t off = *offp; 2246 2247 KASSERT(((len >> wc->wc_log_dev_bshift) << 2248 wc->wc_log_dev_bshift) == len); 2249 2250 if (off < wc->wc_circ_off) 2251 off = wc->wc_circ_off; 2252 slen = wc->wc_circ_off + wc->wc_circ_size - off; 2253 if (slen < len) { 2254 len -= slen; 2255 off = wc->wc_circ_off; 2256 } 2257 off += len; 2258 if (off >= wc->wc_circ_off + wc->wc_circ_size) 2259 off = wc->wc_circ_off; 2260 *offp = off; 2261 } 2262 2263 /****************************************************************/ 2264 2265 int 2266 wapbl_replay_start(struct wapbl_replay **wrp, struct vnode *vp, 2267 daddr_t off, size_t count, size_t blksize) 2268 { 2269 struct wapbl_replay *wr; 2270 int error; 2271 struct vnode *devvp; 2272 daddr_t logpbn; 2273 uint8_t *scratch; 2274 struct wapbl_wc_header *wch; 2275 struct wapbl_wc_header *wch2; 2276 /* Use this until we read the actual log header */ 2277 int log_dev_bshift = DEV_BSHIFT; 2278 size_t used; 2279 2280 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2281 ("wapbl_replay_start: vp=%p off=%"PRId64 " count=%zu blksize=%zu\n", 2282 vp, off, count, blksize)); 2283 2284 if (off < 0) 2285 return EINVAL; 2286 2287 if (blksize < DEV_BSIZE) 2288 return EINVAL; 2289 if (blksize % DEV_BSIZE) 2290 return EINVAL; 2291 2292 #ifdef _KERNEL 2293 #if 0 2294 /* XXX vp->v_size isn't reliably set for VBLK devices, 2295 * especially root. However, we might still want to verify 2296 * that the full load is readable */ 2297 if ((off + count) * blksize > vp->v_size) 2298 return EINVAL; 2299 #endif 2300 2301 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, 0)) != 0) { 2302 return error; 2303 } 2304 #else /* ! _KERNEL */ 2305 devvp = vp; 2306 logpbn = off; 2307 #endif /* ! _KERNEL */ 2308 2309 scratch = wapbl_malloc(MAXBSIZE); 2310 2311 error = wapbl_read(scratch, 2<<log_dev_bshift, devvp, logpbn); 2312 if (error) 2313 goto errout; 2314 2315 wch = (struct wapbl_wc_header *)scratch; 2316 wch2 = 2317 (struct wapbl_wc_header *)(scratch + (1<<log_dev_bshift)); 2318 /* XXX verify checksums and magic numbers */ 2319 if (wch->wc_type != WAPBL_WC_HEADER) { 2320 printf("Unrecognized wapbl magic: 0x%08x\n", wch->wc_type); 2321 error = EFTYPE; 2322 goto errout; 2323 } 2324 2325 if (wch2->wc_generation > wch->wc_generation) 2326 wch = wch2; 2327 2328 wr = wapbl_calloc(1, sizeof(*wr)); 2329 2330 wr->wr_logvp = vp; 2331 wr->wr_devvp = devvp; 2332 wr->wr_logpbn = logpbn; 2333 2334 wr->wr_scratch = scratch; 2335 2336 memcpy(&wr->wr_wc_header, wch, sizeof(wr->wr_wc_header)); 2337 2338 used = wapbl_space_used(wch->wc_circ_size, wch->wc_head, wch->wc_tail); 2339 2340 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2341 ("wapbl_replay: head=%"PRId64" tail=%"PRId64" off=%"PRId64 2342 " len=%"PRId64" used=%zu\n", 2343 wch->wc_head, wch->wc_tail, wch->wc_circ_off, 2344 wch->wc_circ_size, used)); 2345 2346 wapbl_blkhash_init(wr, (used >> wch->wc_fs_dev_bshift)); 2347 error = wapbl_replay_prescan(wr); 2348 if (error) { 2349 wapbl_replay_stop(wr); 2350 wapbl_replay_free(wr); 2351 return error; 2352 } 2353 2354 error = wapbl_replay_get_inodes(wr); 2355 if (error) { 2356 wapbl_replay_stop(wr); 2357 wapbl_replay_free(wr); 2358 return error; 2359 } 2360 2361 *wrp = wr; 2362 return 0; 2363 2364 errout: 2365 wapbl_free(scratch); 2366 return error; 2367 } 2368 2369 void 2370 wapbl_replay_stop(struct wapbl_replay *wr) 2371 { 2372 2373 if (!wapbl_replay_isopen(wr)) 2374 return; 2375 2376 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, ("wapbl_replay_stop called\n")); 2377 2378 wapbl_free(wr->wr_scratch); 2379 wr->wr_scratch = 0; 2380 2381 wr->wr_logvp = 0; 2382 2383 wapbl_blkhash_clear(wr); 2384 wapbl_blkhash_free(wr); 2385 } 2386 2387 void 2388 wapbl_replay_free(struct wapbl_replay *wr) 2389 { 2390 2391 KDASSERT(!wapbl_replay_isopen(wr)); 2392 2393 if (wr->wr_inodes) 2394 wapbl_free(wr->wr_inodes); 2395 wapbl_free(wr); 2396 } 2397 2398 #ifdef _KERNEL 2399 int 2400 wapbl_replay_isopen1(struct wapbl_replay *wr) 2401 { 2402 2403 return wapbl_replay_isopen(wr); 2404 } 2405 #endif 2406 2407 static int 2408 wapbl_replay_prescan(struct wapbl_replay *wr) 2409 { 2410 off_t off; 2411 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2412 int error; 2413 2414 int logblklen = 1<<wch->wc_log_dev_bshift; 2415 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2416 2417 wapbl_blkhash_clear(wr); 2418 2419 off = wch->wc_tail; 2420 while (off != wch->wc_head) { 2421 struct wapbl_wc_null *wcn; 2422 off_t saveoff = off; 2423 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2424 if (error) 2425 goto errout; 2426 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2427 switch (wcn->wc_type) { 2428 case WAPBL_WC_BLOCKS: 2429 { 2430 struct wapbl_wc_blocklist *wc = 2431 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2432 int i; 2433 for (i = 0; i < wc->wc_blkcount; i++) { 2434 int j, n; 2435 /* 2436 * Enter each physical block into the 2437 * hashtable independently 2438 */ 2439 n = wc->wc_blocks[i].wc_dlen >> 2440 wch->wc_fs_dev_bshift; 2441 for (j = 0; j < n; j++) { 2442 wapbl_blkhash_ins(wr, 2443 wc->wc_blocks[i].wc_daddr + j, 2444 off); 2445 wapbl_circ_advance(wr, 2446 fsblklen, &off); 2447 } 2448 } 2449 } 2450 break; 2451 2452 case WAPBL_WC_REVOCATIONS: 2453 { 2454 struct wapbl_wc_blocklist *wc = 2455 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2456 int i; 2457 for (i = 0; i < wc->wc_blkcount; i++) { 2458 int j, n; 2459 /* 2460 * Remove any blocks found from the 2461 * hashtable 2462 */ 2463 n = wc->wc_blocks[i].wc_dlen >> 2464 wch->wc_fs_dev_bshift; 2465 for (j = 0; j < n; j++) { 2466 wapbl_blkhash_rem(wr, 2467 wc->wc_blocks[i].wc_daddr + j); 2468 } 2469 } 2470 } 2471 break; 2472 2473 case WAPBL_WC_INODES: 2474 { 2475 struct wapbl_wc_inodelist *wc = 2476 (struct wapbl_wc_inodelist *)wr->wr_scratch; 2477 /* 2478 * Keep track of where we found this so we 2479 * can use it later 2480 */ 2481 if (wc->wc_clear) { 2482 wr->wr_inodestail = saveoff; 2483 wr->wr_inodescnt = 0; 2484 } 2485 if (wr->wr_inodestail) 2486 wr->wr_inodeshead = off; 2487 wr->wr_inodescnt += wc->wc_inocnt; 2488 } 2489 break; 2490 default: 2491 printf("Unrecognized wapbl type: 0x%08x\n", 2492 wcn->wc_type); 2493 error = EFTYPE; 2494 goto errout; 2495 } 2496 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2497 if (off != saveoff) { 2498 printf("wapbl_replay: corrupted records\n"); 2499 error = EFTYPE; 2500 goto errout; 2501 } 2502 } 2503 return 0; 2504 2505 errout: 2506 wapbl_blkhash_clear(wr); 2507 return error; 2508 } 2509 2510 static int 2511 wapbl_replay_get_inodes(struct wapbl_replay *wr) 2512 { 2513 off_t off; 2514 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2515 int logblklen = 1<<wch->wc_log_dev_bshift; 2516 int cnt= 0; 2517 2518 KDASSERT(wapbl_replay_isopen(wr)); 2519 2520 if (wr->wr_inodescnt == 0) 2521 return 0; 2522 2523 KASSERT(!wr->wr_inodes); 2524 2525 wr->wr_inodes = wapbl_malloc(wr->wr_inodescnt*sizeof(wr->wr_inodes[0])); 2526 2527 off = wr->wr_inodestail; 2528 2529 while (off != wr->wr_inodeshead) { 2530 struct wapbl_wc_null *wcn; 2531 int error; 2532 off_t saveoff = off; 2533 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2534 if (error) { 2535 wapbl_free(wr->wr_inodes); 2536 wr->wr_inodes = 0; 2537 return error; 2538 } 2539 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2540 switch (wcn->wc_type) { 2541 case WAPBL_WC_BLOCKS: 2542 case WAPBL_WC_REVOCATIONS: 2543 break; 2544 case WAPBL_WC_INODES: 2545 { 2546 struct wapbl_wc_inodelist *wc = 2547 (struct wapbl_wc_inodelist *)wr->wr_scratch; 2548 /* 2549 * Keep track of where we found this so we 2550 * can use it later 2551 */ 2552 if (wc->wc_clear) { 2553 cnt = 0; 2554 } 2555 /* This memcpy assumes that wr_inodes is 2556 * laid out the same as wc_inodes. */ 2557 memcpy(&wr->wr_inodes[cnt], wc->wc_inodes, 2558 wc->wc_inocnt*sizeof(wc->wc_inodes[0])); 2559 cnt += wc->wc_inocnt; 2560 } 2561 break; 2562 default: 2563 KASSERT(0); 2564 } 2565 off = saveoff; 2566 wapbl_circ_advance(wr, wcn->wc_len, &off); 2567 } 2568 KASSERT(cnt == wr->wr_inodescnt); 2569 return 0; 2570 } 2571 2572 #ifdef DEBUG 2573 int 2574 wapbl_replay_verify(struct wapbl_replay *wr, struct vnode *fsdevvp) 2575 { 2576 off_t off; 2577 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2578 int mismatchcnt = 0; 2579 int logblklen = 1<<wch->wc_log_dev_bshift; 2580 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2581 void *scratch1 = wapbl_malloc(MAXBSIZE); 2582 void *scratch2 = wapbl_malloc(MAXBSIZE); 2583 int error = 0; 2584 2585 KDASSERT(wapbl_replay_isopen(wr)); 2586 2587 off = wch->wc_tail; 2588 while (off != wch->wc_head) { 2589 struct wapbl_wc_null *wcn; 2590 #ifdef DEBUG 2591 off_t saveoff = off; 2592 #endif 2593 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2594 if (error) 2595 goto out; 2596 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2597 switch (wcn->wc_type) { 2598 case WAPBL_WC_BLOCKS: 2599 { 2600 struct wapbl_wc_blocklist *wc = 2601 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2602 int i; 2603 for (i = 0; i < wc->wc_blkcount; i++) { 2604 int foundcnt = 0; 2605 int dirtycnt = 0; 2606 int j, n; 2607 /* 2608 * Check each physical block into the 2609 * hashtable independently 2610 */ 2611 n = wc->wc_blocks[i].wc_dlen >> 2612 wch->wc_fs_dev_bshift; 2613 for (j = 0; j < n; j++) { 2614 struct wapbl_blk *wb = 2615 wapbl_blkhash_get(wr, 2616 wc->wc_blocks[i].wc_daddr + j); 2617 if (wb && (wb->wb_off == off)) { 2618 foundcnt++; 2619 error = 2620 wapbl_circ_read(wr, 2621 scratch1, fsblklen, 2622 &off); 2623 if (error) 2624 goto out; 2625 error = 2626 wapbl_read(scratch2, 2627 fsblklen, fsdevvp, 2628 wb->wb_blk); 2629 if (error) 2630 goto out; 2631 if (memcmp(scratch1, 2632 scratch2, 2633 fsblklen)) { 2634 printf( 2635 "wapbl_verify: mismatch block %"PRId64" at off %"PRIdMAX"\n", 2636 wb->wb_blk, (intmax_t)off); 2637 dirtycnt++; 2638 mismatchcnt++; 2639 } 2640 } else { 2641 wapbl_circ_advance(wr, 2642 fsblklen, &off); 2643 } 2644 } 2645 #if 0 2646 /* 2647 * If all of the blocks in an entry 2648 * are clean, then remove all of its 2649 * blocks from the hashtable since they 2650 * never will need replay. 2651 */ 2652 if ((foundcnt != 0) && 2653 (dirtycnt == 0)) { 2654 off = saveoff; 2655 wapbl_circ_advance(wr, 2656 logblklen, &off); 2657 for (j = 0; j < n; j++) { 2658 struct wapbl_blk *wb = 2659 wapbl_blkhash_get(wr, 2660 wc->wc_blocks[i].wc_daddr + j); 2661 if (wb && 2662 (wb->wb_off == off)) { 2663 wapbl_blkhash_rem(wr, wb->wb_blk); 2664 } 2665 wapbl_circ_advance(wr, 2666 fsblklen, &off); 2667 } 2668 } 2669 #endif 2670 } 2671 } 2672 break; 2673 case WAPBL_WC_REVOCATIONS: 2674 case WAPBL_WC_INODES: 2675 break; 2676 default: 2677 KASSERT(0); 2678 } 2679 #ifdef DEBUG 2680 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2681 KASSERT(off == saveoff); 2682 #endif 2683 } 2684 out: 2685 wapbl_free(scratch1); 2686 wapbl_free(scratch2); 2687 if (!error && mismatchcnt) 2688 error = EFTYPE; 2689 return error; 2690 } 2691 #endif 2692 2693 int 2694 wapbl_replay_write(struct wapbl_replay *wr, struct vnode *fsdevvp) 2695 { 2696 off_t off; 2697 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2698 int logblklen = 1<<wch->wc_log_dev_bshift; 2699 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2700 void *scratch1 = wapbl_malloc(MAXBSIZE); 2701 int error = 0; 2702 2703 KDASSERT(wapbl_replay_isopen(wr)); 2704 2705 /* 2706 * This parses the journal for replay, although it could 2707 * just as easily walk the hashtable instead. 2708 */ 2709 2710 off = wch->wc_tail; 2711 while (off != wch->wc_head) { 2712 struct wapbl_wc_null *wcn; 2713 #ifdef DEBUG 2714 off_t saveoff = off; 2715 #endif 2716 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2717 if (error) 2718 goto out; 2719 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2720 switch (wcn->wc_type) { 2721 case WAPBL_WC_BLOCKS: 2722 { 2723 struct wapbl_wc_blocklist *wc = 2724 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2725 int i; 2726 for (i = 0; i < wc->wc_blkcount; i++) { 2727 int j, n; 2728 /* 2729 * Check each physical block against 2730 * the hashtable independently 2731 */ 2732 n = wc->wc_blocks[i].wc_dlen >> 2733 wch->wc_fs_dev_bshift; 2734 for (j = 0; j < n; j++) { 2735 struct wapbl_blk *wb = 2736 wapbl_blkhash_get(wr, 2737 wc->wc_blocks[i].wc_daddr + j); 2738 if (wb && (wb->wb_off == off)) { 2739 error = wapbl_circ_read( 2740 wr, scratch1, 2741 fsblklen, &off); 2742 if (error) 2743 goto out; 2744 error = 2745 wapbl_write(scratch1, 2746 fsblklen, fsdevvp, 2747 wb->wb_blk); 2748 if (error) 2749 goto out; 2750 } else { 2751 wapbl_circ_advance(wr, 2752 fsblklen, &off); 2753 } 2754 } 2755 } 2756 } 2757 break; 2758 case WAPBL_WC_REVOCATIONS: 2759 case WAPBL_WC_INODES: 2760 break; 2761 default: 2762 KASSERT(0); 2763 } 2764 #ifdef DEBUG 2765 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2766 KASSERT(off == saveoff); 2767 #endif 2768 } 2769 out: 2770 wapbl_free(scratch1); 2771 return error; 2772 } 2773 2774 int 2775 wapbl_replay_can_read(struct wapbl_replay *wr, daddr_t blk, long len) 2776 { 2777 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2778 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2779 2780 KDASSERT(wapbl_replay_isopen(wr)); 2781 KASSERT((len % fsblklen) == 0); 2782 2783 while (len != 0) { 2784 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2785 if (wb) 2786 return 1; 2787 len -= fsblklen; 2788 } 2789 return 0; 2790 } 2791 2792 int 2793 wapbl_replay_read(struct wapbl_replay *wr, void *data, daddr_t blk, long len) 2794 { 2795 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2796 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2797 2798 KDASSERT(wapbl_replay_isopen(wr)); 2799 2800 KASSERT((len % fsblklen) == 0); 2801 2802 while (len != 0) { 2803 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2804 if (wb) { 2805 off_t off = wb->wb_off; 2806 int error; 2807 error = wapbl_circ_read(wr, data, fsblklen, &off); 2808 if (error) 2809 return error; 2810 } 2811 data = (uint8_t *)data + fsblklen; 2812 len -= fsblklen; 2813 blk++; 2814 } 2815 return 0; 2816 } 2817
Indexes created Fri Sep 26 20:09:58 GMT 2025