vfs_wapbl.c revision 1.1.2.9
1 /* $NetBSD: vfs_wapbl.c,v 1.1.2.9 2008/07/01 00:05:56 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2003,2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * This implements file system independent write ahead filesystem logging. 34 */ 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.1.2.9 2008/07/01 00:05:56 matt Exp $"); 37 38 #include <sys/param.h> 39 40 #ifdef _KERNEL 41 #include <sys/param.h> 42 #include <sys/namei.h> 43 #include <sys/proc.h> 44 #include <sys/uio.h> 45 #include <sys/vnode.h> 46 #include <sys/file.h> 47 #include <sys/malloc.h> 48 #include <sys/resourcevar.h> 49 #include <sys/conf.h> 50 #include <sys/mount.h> 51 #include <sys/kernel.h> 52 #include <sys/kauth.h> 53 #include <sys/mutex.h> 54 #include <sys/atomic.h> 55 #include <sys/wapbl.h> 56 57 #if WAPBL_UVM_ALLOC 58 #include <uvm/uvm.h> 59 #endif 60 61 #include <miscfs/specfs/specdev.h> 62 63 MALLOC_JUSTDEFINE(M_WAPBL, "wapbl", "write-ahead physical block logging"); 64 #define wapbl_malloc(s) malloc((s), M_WAPBL, M_WAITOK) 65 #define wapbl_free(a) free((a), M_WAPBL) 66 #define wapbl_calloc(n, s) malloc((n)*(s), M_WAPBL, M_WAITOK | M_ZERO) 67 68 #else /* !_KERNEL */ 69 #include <assert.h> 70 #include <errno.h> 71 #include <stdio.h> 72 #include <stdbool.h> 73 #include <stdlib.h> 74 #include <string.h> 75 76 #include <sys/time.h> 77 #include <sys/wapbl.h> 78 79 #define KDASSERT(x) assert(x) 80 #define KASSERT(x) assert(x) 81 #define wapbl_malloc(s) malloc(s) 82 #define wapbl_free(a) free(a) 83 #define wapbl_calloc(n, s) calloc((n), (s)) 84 85 #endif /* !_KERNEL */ 86 87 /* 88 * INTERNAL DATA STRUCTURES 89 */ 90 91 /* 92 * This structure holds per-mount log information. 93 * 94 * Legend: a = atomic access only 95 * r = read-only after init 96 * l = rwlock held 97 * m = mutex held 98 * u = unlocked access ok 99 * b = bufcache_lock held 100 */ 101 struct wapbl { 102 struct vnode *wl_logvp; /* r: log here */ 103 struct vnode *wl_devvp; /* r: log on this device */ 104 struct mount *wl_mount; /* r: mountpoint wl is associated with */ 105 daddr_t wl_logpbn; /* r: Physical block number of start of log */ 106 int wl_log_dev_bshift; /* r: logarithm of device block size of log 107 device */ 108 int wl_fs_dev_bshift; /* r: logarithm of device block size of 109 filesystem device */ 110 111 unsigned wl_lock_count; /* a: Count of transactions in progress */ 112 113 size_t wl_circ_size; /* r: Number of bytes in buffer of log */ 114 size_t wl_circ_off; /* r: Number of bytes reserved at start */ 115 116 size_t wl_bufcount_max; /* r: Number of buffers reserved for log */ 117 size_t wl_bufbytes_max; /* r: Number of buf bytes reserved for log */ 118 119 off_t wl_head; /* l: Byte offset of log head */ 120 off_t wl_tail; /* l: Byte offset of log tail */ 121 /* 122 * head == tail == 0 means log is empty 123 * head == tail != 0 means log is full 124 * see assertions in wapbl_advance() for other boundary conditions. 125 * only truncate moves the tail, except when flush sets it to 126 * wl_header_size only flush moves the head, except when truncate 127 * sets it to 0. 128 */ 129 130 struct wapbl_wc_header *wl_wc_header; /* l */ 131 void *wl_wc_scratch; /* l: scratch space (XXX: por que?!?) */ 132 133 kmutex_t wl_mtx; /* u: short-term lock */ 134 krwlock_t wl_rwlock; /* u: File system transaction lock */ 135 136 /* 137 * Must be held while accessing 138 * wl_count or wl_bufs or head or tail 139 */ 140 141 /* 142 * Callback called from within the flush routine to flush any extra 143 * bits. Note that flush may be skipped without calling this if 144 * there are no outstanding buffers in the transaction. 145 */ 146 wapbl_flush_fn_t wl_flush; /* r */ 147 wapbl_flush_fn_t wl_flush_abort;/* r */ 148 149 size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ 150 size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ 151 size_t wl_bcount; /* m: Total bcount of wl_bufs */ 152 153 LIST_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ 154 155 kcondvar_t wl_reclaimable_cv; /* m (obviously) */ 156 size_t wl_reclaimable_bytes; /* m: Amount of space available for 157 reclamation by truncate */ 158 int wl_error_count; /* m: # of wl_entries with errors */ 159 size_t wl_reserved_bytes; /* never truncate log smaller than this */ 160 161 #ifdef WAPBL_DEBUG_BUFBYTES 162 size_t wl_unsynced_bufbytes; /* Byte count of unsynced buffers */ 163 #endif 164 165 daddr_t *wl_deallocblks;/* l: address of block */ 166 int *wl_dealloclens; /* l: size of block (fragments, kom ihg) */ 167 int wl_dealloccnt; /* l: total count */ 168 int wl_dealloclim; /* l: max count */ 169 170 /* hashtable of inode numbers for allocated but unlinked inodes */ 171 /* synch ??? */ 172 LIST_HEAD(wapbl_ino_head, wapbl_ino) *wl_inohash; 173 u_long wl_inohashmask; 174 int wl_inohashcnt; 175 176 SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* On disk transaction 177 accounting */ 178 }; 179 180 #ifdef WAPBL_DEBUG_PRINT 181 int wapbl_debug_print = WAPBL_DEBUG_PRINT; 182 #endif 183 184 /****************************************************************/ 185 #ifdef _KERNEL 186 187 #ifdef WAPBL_DEBUG 188 struct wapbl *wapbl_debug_wl; 189 #endif 190 191 static int wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail); 192 static int wapbl_write_blocks(struct wapbl *wl, off_t *offp); 193 static int wapbl_write_revocations(struct wapbl *wl, off_t *offp); 194 static int wapbl_write_inodes(struct wapbl *wl, off_t *offp); 195 #endif /* _KERNEL */ 196 197 static int wapbl_replay_prescan(struct wapbl_replay *wr); 198 static int wapbl_replay_get_inodes(struct wapbl_replay *wr); 199 200 static __inline size_t wapbl_space_free(size_t avail, off_t head, 201 off_t tail); 202 static __inline size_t wapbl_space_used(size_t avail, off_t head, 203 off_t tail); 204 205 #ifdef _KERNEL 206 207 #define WAPBL_INODETRK_SIZE 83 208 static int wapbl_ino_pool_refcount; 209 static struct pool wapbl_ino_pool; 210 struct wapbl_ino { 211 LIST_ENTRY(wapbl_ino) wi_hash; 212 ino_t wi_ino; 213 mode_t wi_mode; 214 }; 215 216 static void wapbl_inodetrk_init(struct wapbl *wl, u_int size); 217 static void wapbl_inodetrk_free(struct wapbl *wl); 218 static struct wapbl_ino *wapbl_inodetrk_get(struct wapbl *wl, ino_t ino); 219 220 static size_t wapbl_transaction_len(struct wapbl *wl); 221 static __inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); 222 223 /* 224 * This is useful for debugging. If set, the log will 225 * only be truncated when necessary. 226 */ 227 int wapbl_lazy_truncate = 0; 228 229 struct wapbl_ops wapbl_ops = { 230 .wo_wapbl_discard = wapbl_discard, 231 .wo_wapbl_replay_isopen = wapbl_replay_isopen1, 232 .wo_wapbl_replay_read = wapbl_replay_read, 233 .wo_wapbl_add_buf = wapbl_add_buf, 234 .wo_wapbl_remove_buf = wapbl_remove_buf, 235 .wo_wapbl_resize_buf = wapbl_resize_buf, 236 .wo_wapbl_begin = wapbl_begin, 237 .wo_wapbl_end = wapbl_end, 238 .wo_wapbl_junlock_assert= wapbl_junlock_assert, 239 240 /* XXX: the following is only used to say "this is a wapbl buf" */ 241 .wo_wapbl_biodone = wapbl_biodone, 242 }; 243 244 void 245 wapbl_init() 246 { 247 248 malloc_type_attach(M_WAPBL); 249 } 250 251 int 252 wapbl_start(struct wapbl ** wlp, struct mount *mp, struct vnode *vp, 253 daddr_t off, size_t count, size_t blksize, struct wapbl_replay *wr, 254 wapbl_flush_fn_t flushfn, wapbl_flush_fn_t flushabortfn) 255 { 256 struct wapbl *wl; 257 struct vnode *devvp; 258 daddr_t logpbn; 259 int error; 260 int log_dev_bshift = DEV_BSHIFT; 261 int fs_dev_bshift = DEV_BSHIFT; 262 int run; 263 264 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_start: vp=%p off=%" PRId64 265 " count=%zu blksize=%zu\n", vp, off, count, blksize)); 266 267 if (log_dev_bshift > fs_dev_bshift) { 268 WAPBL_PRINTF(WAPBL_PRINT_OPEN, 269 ("wapbl: log device's block size cannot be larger " 270 "than filesystem's\n")); 271 /* 272 * Not currently implemented, although it could be if 273 * needed someday. 274 */ 275 return ENOSYS; 276 } 277 278 if (off < 0) 279 return EINVAL; 280 281 if (blksize < DEV_BSIZE) 282 return EINVAL; 283 if (blksize % DEV_BSIZE) 284 return EINVAL; 285 286 /* XXXTODO: verify that the full load is writable */ 287 288 /* 289 * XXX check for minimum log size 290 * minimum is governed by minimum amount of space 291 * to complete a transaction. (probably truncate) 292 */ 293 /* XXX for now pick something minimal */ 294 if ((count * blksize) < MAXPHYS) { 295 return ENOSPC; 296 } 297 298 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, &run)) != 0) { 299 return error; 300 } 301 302 wl = wapbl_calloc(1, sizeof(*wl)); 303 rw_init(&wl->wl_rwlock); 304 mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); 305 cv_init(&wl->wl_reclaimable_cv, "wapblrec"); 306 LIST_INIT(&wl->wl_bufs); 307 SIMPLEQ_INIT(&wl->wl_entries); 308 309 wl->wl_logvp = vp; 310 wl->wl_devvp = devvp; 311 wl->wl_mount = mp; 312 wl->wl_logpbn = logpbn; 313 wl->wl_log_dev_bshift = log_dev_bshift; 314 wl->wl_fs_dev_bshift = fs_dev_bshift; 315 316 wl->wl_flush = flushfn; 317 wl->wl_flush_abort = flushabortfn; 318 319 /* Reserve two log device blocks for the commit headers */ 320 wl->wl_circ_off = 2<<wl->wl_log_dev_bshift; 321 wl->wl_circ_size = ((count * blksize) - wl->wl_circ_off); 322 /* truncate the log usage to a multiple of log_dev_bshift */ 323 wl->wl_circ_size >>= wl->wl_log_dev_bshift; 324 wl->wl_circ_size <<= wl->wl_log_dev_bshift; 325 326 /* 327 * wl_bufbytes_max limits the size of the in memory transaction space. 328 * - Since buffers are allocated and accounted for in units of 329 * PAGE_SIZE it is required to be a multiple of PAGE_SIZE 330 * (i.e. 1<<PAGE_SHIFT) 331 * - Since the log device has to be written in units of 332 * 1<<wl_log_dev_bshift it is required to be a mulitple of 333 * 1<<wl_log_dev_bshift. 334 * - Since filesystem will provide data in units of 1<<wl_fs_dev_bshift, 335 * it is convenient to be a multiple of 1<<wl_fs_dev_bshift. 336 * Therefore it must be multiple of the least common multiple of those 337 * three quantities. Fortunately, all of those quantities are 338 * guaranteed to be a power of two, and the least common multiple of 339 * a set of numbers which are all powers of two is simply the maximum 340 * of those numbers. Finally, the maximum logarithm of a power of two 341 * is the same as the log of the maximum power of two. So we can do 342 * the following operations to size wl_bufbytes_max: 343 */ 344 345 /* XXX fix actual number of pages reserved per filesystem. */ 346 wl->wl_bufbytes_max = MIN(wl->wl_circ_size, buf_memcalc() / 2); 347 348 /* Round wl_bufbytes_max to the largest power of two constraint */ 349 wl->wl_bufbytes_max >>= PAGE_SHIFT; 350 wl->wl_bufbytes_max <<= PAGE_SHIFT; 351 wl->wl_bufbytes_max >>= wl->wl_log_dev_bshift; 352 wl->wl_bufbytes_max <<= wl->wl_log_dev_bshift; 353 wl->wl_bufbytes_max >>= wl->wl_fs_dev_bshift; 354 wl->wl_bufbytes_max <<= wl->wl_fs_dev_bshift; 355 356 /* XXX maybe use filesystem fragment size instead of 1024 */ 357 /* XXX fix actual number of buffers reserved per filesystem. */ 358 wl->wl_bufcount_max = (nbuf / 2) * 1024; 359 360 /* XXX tie this into resource estimation */ 361 wl->wl_dealloclim = 2 * btodb(wl->wl_bufbytes_max); 362 363 #if WAPBL_UVM_ALLOC 364 wl->wl_deallocblks = (void *) uvm_km_zalloc(kernel_map, 365 round_page(sizeof(*wl->wl_deallocblks) * wl->wl_dealloclim)); 366 KASSERT(wl->wl_deallocblks != NULL); 367 wl->wl_dealloclens = (void *) uvm_km_zalloc(kernel_map, 368 round_page(sizeof(*wl->wl_dealloclens) * wl->wl_dealloclim)); 369 KASSERT(wl->wl_dealloclens != NULL); 370 #else 371 wl->wl_deallocblks = wapbl_malloc(sizeof(*wl->wl_deallocblks) * 372 wl->wl_dealloclim); 373 wl->wl_dealloclens = wapbl_malloc(sizeof(*wl->wl_dealloclens) * 374 wl->wl_dealloclim); 375 #endif 376 377 wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); 378 379 /* Initialize the commit header */ 380 { 381 struct wapbl_wc_header *wc; 382 size_t len = 1<<wl->wl_log_dev_bshift; 383 wc = wapbl_calloc(1, len); 384 wc->wc_type = WAPBL_WC_HEADER; 385 wc->wc_len = len; 386 wc->wc_circ_off = wl->wl_circ_off; 387 wc->wc_circ_size = wl->wl_circ_size; 388 /* XXX wc->wc_fsid */ 389 wc->wc_log_dev_bshift = wl->wl_log_dev_bshift; 390 wc->wc_fs_dev_bshift = wl->wl_fs_dev_bshift; 391 wl->wl_wc_header = wc; 392 wl->wl_wc_scratch = wapbl_malloc(len); 393 } 394 395 /* 396 * if there was an existing set of unlinked but 397 * allocated inodes, preserve it in the new 398 * log. 399 */ 400 if (wr && wr->wr_inodescnt) { 401 int i; 402 403 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 404 ("wapbl_start: reusing log with %d inodes\n", 405 wr->wr_inodescnt)); 406 407 /* 408 * Its only valid to reuse the replay log if its 409 * the same as the new log we just opened. 410 */ 411 KDASSERT(!wapbl_replay_isopen(wr)); 412 KASSERT(devvp->v_rdev == wr->wr_devvp->v_rdev); 413 KASSERT(logpbn == wr->wr_logpbn); 414 KASSERT(wl->wl_circ_size == wr->wr_wc_header.wc_circ_size); 415 KASSERT(wl->wl_circ_off == wr->wr_wc_header.wc_circ_off); 416 KASSERT(wl->wl_log_dev_bshift == 417 wr->wr_wc_header.wc_log_dev_bshift); 418 KASSERT(wl->wl_fs_dev_bshift == 419 wr->wr_wc_header.wc_fs_dev_bshift); 420 421 wl->wl_wc_header->wc_generation = 422 wr->wr_wc_header.wc_generation + 1; 423 424 for (i = 0; i < wr->wr_inodescnt; i++) 425 wapbl_register_inode(wl, wr->wr_inodes[i].wr_inumber, 426 wr->wr_inodes[i].wr_imode); 427 428 /* Make sure new transaction won't overwrite old inodes list */ 429 KDASSERT(wapbl_transaction_len(wl) <= 430 wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, 431 wr->wr_inodestail)); 432 433 wl->wl_head = wl->wl_tail = wr->wr_inodeshead; 434 wl->wl_reclaimable_bytes = wl->wl_reserved_bytes = 435 wapbl_transaction_len(wl); 436 437 error = wapbl_write_inodes(wl, &wl->wl_head); 438 if (error) 439 goto errout; 440 441 KASSERT(wl->wl_head != wl->wl_tail); 442 KASSERT(wl->wl_head != 0); 443 } 444 445 error = wapbl_write_commit(wl, wl->wl_head, wl->wl_tail); 446 if (error) { 447 goto errout; 448 } 449 450 *wlp = wl; 451 #if defined(WAPBL_DEBUG) 452 wapbl_debug_wl = wl; 453 #endif 454 455 return 0; 456 errout: 457 wapbl_discard(wl); 458 wapbl_free(wl->wl_wc_scratch); 459 wapbl_free(wl->wl_wc_header); 460 #if WAPBL_UVM_ALLOC 461 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_deallocblks, 462 round_page(sizeof(*wl->wl_deallocblks * 463 wl->wl_dealloclim))); 464 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_dealloclens, 465 round_page(sizeof(*wl->wl_dealloclens * 466 wl->wl_dealloclim))); 467 #else 468 wapbl_free(wl->wl_deallocblks); 469 wapbl_free(wl->wl_dealloclens); 470 #endif 471 wapbl_inodetrk_free(wl); 472 wapbl_free(wl); 473 474 return error; 475 } 476 477 /* 478 * Like wapbl_flush, only discards the transaction 479 * completely 480 */ 481 482 void 483 wapbl_discard(struct wapbl *wl) 484 { 485 struct wapbl_entry *we; 486 struct buf *bp; 487 int i; 488 489 /* 490 * XXX we may consider using upgrade here 491 * if we want to call flush from inside a transaction 492 */ 493 rw_enter(&wl->wl_rwlock, RW_WRITER); 494 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 495 wl->wl_dealloccnt); 496 497 #ifdef WAPBL_DEBUG_PRINT 498 { 499 struct wapbl_entry *we; 500 pid_t pid = -1; 501 lwpid_t lid = -1; 502 if (curproc) 503 pid = curproc->p_pid; 504 if (curlwp) 505 lid = curlwp->l_lid; 506 #ifdef WAPBL_DEBUG_BUFBYTES 507 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 508 ("wapbl_discard: thread %d.%d discarding " 509 "transaction\n" 510 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 511 "deallocs=%d inodes=%d\n" 512 "\terrcnt = %u, reclaimable=%zu reserved=%zu " 513 "unsynced=%zu\n", 514 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 515 wl->wl_bcount, wl->wl_dealloccnt, 516 wl->wl_inohashcnt, wl->wl_error_count, 517 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 518 wl->wl_unsynced_bufbytes)); 519 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 520 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 521 ("\tentry: bufcount = %zu, reclaimable = %zu, " 522 "error = %d, unsynced = %zu\n", 523 we->we_bufcount, we->we_reclaimable_bytes, 524 we->we_error, we->we_unsynced_bufbytes)); 525 } 526 #else /* !WAPBL_DEBUG_BUFBYTES */ 527 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 528 ("wapbl_discard: thread %d.%d discarding transaction\n" 529 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 530 "deallocs=%d inodes=%d\n" 531 "\terrcnt = %u, reclaimable=%zu reserved=%zu\n", 532 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 533 wl->wl_bcount, wl->wl_dealloccnt, 534 wl->wl_inohashcnt, wl->wl_error_count, 535 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes)); 536 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 537 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 538 ("\tentry: bufcount = %zu, reclaimable = %zu, " 539 "error = %d\n", 540 we->we_bufcount, we->we_reclaimable_bytes, 541 we->we_error)); 542 } 543 #endif /* !WAPBL_DEBUG_BUFBYTES */ 544 } 545 #endif /* WAPBL_DEBUG_PRINT */ 546 547 for (i = 0; i <= wl->wl_inohashmask; i++) { 548 struct wapbl_ino_head *wih; 549 struct wapbl_ino *wi; 550 551 wih = &wl->wl_inohash[i]; 552 while ((wi = LIST_FIRST(wih)) != NULL) { 553 LIST_REMOVE(wi, wi_hash); 554 pool_put(&wapbl_ino_pool, wi); 555 KASSERT(wl->wl_inohashcnt > 0); 556 wl->wl_inohashcnt--; 557 } 558 } 559 560 /* 561 * clean buffer list 562 */ 563 mutex_enter(&bufcache_lock); 564 mutex_enter(&wl->wl_mtx); 565 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 566 if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { 567 /* 568 * The buffer will be unlocked and 569 * removed from the transaction in brelse 570 */ 571 mutex_exit(&wl->wl_mtx); 572 brelsel(bp, 0); 573 mutex_enter(&wl->wl_mtx); 574 } 575 } 576 mutex_exit(&wl->wl_mtx); 577 mutex_exit(&bufcache_lock); 578 579 /* 580 * Remove references to this wl from wl_entries, free any which 581 * no longer have buffers, others will be freed in wapbl_biodone 582 * when they no longer have any buffers. 583 */ 584 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) != NULL) { 585 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 586 /* XXX should we be accumulating wl_error_count 587 * and increasing reclaimable bytes ? */ 588 we->we_wapbl = NULL; 589 if (we->we_bufcount == 0) { 590 #ifdef WAPBL_DEBUG_BUFBYTES 591 KASSERT(we->we_unsynced_bufbytes == 0); 592 #endif 593 wapbl_free(we); 594 } 595 } 596 597 /* Discard list of deallocs */ 598 wl->wl_dealloccnt = 0; 599 /* XXX should we clear wl_reserved_bytes? */ 600 601 KASSERT(wl->wl_bufbytes == 0); 602 KASSERT(wl->wl_bcount == 0); 603 KASSERT(wl->wl_bufcount == 0); 604 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 605 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 606 KASSERT(wl->wl_inohashcnt == 0); 607 608 rw_exit(&wl->wl_rwlock); 609 } 610 611 int 612 wapbl_stop(struct wapbl *wl, int force) 613 { 614 struct vnode *vp; 615 int error; 616 617 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_stop called\n")); 618 error = wapbl_flush(wl, 1); 619 if (error) { 620 if (force) 621 wapbl_discard(wl); 622 else 623 return error; 624 } 625 626 /* Unlinked inodes persist after a flush */ 627 if (wl->wl_inohashcnt) { 628 if (force) { 629 wapbl_discard(wl); 630 } else { 631 return EBUSY; 632 } 633 } 634 635 KASSERT(wl->wl_bufbytes == 0); 636 KASSERT(wl->wl_bcount == 0); 637 KASSERT(wl->wl_bufcount == 0); 638 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 639 KASSERT(wl->wl_dealloccnt == 0); 640 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 641 KASSERT(wl->wl_inohashcnt == 0); 642 643 vp = wl->wl_logvp; 644 645 wapbl_free(wl->wl_wc_scratch); 646 wapbl_free(wl->wl_wc_header); 647 #if WAPBL_UVM_ALLOC 648 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_deallocblks, 649 round_page(sizeof(*wl->wl_deallocblks * 650 wl->wl_dealloclim))); 651 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_dealloclens, 652 round_page(sizeof(*wl->wl_dealloclens * 653 wl->wl_dealloclim))); 654 #else 655 wapbl_free(wl->wl_deallocblks); 656 wapbl_free(wl->wl_dealloclens); 657 #endif 658 wapbl_inodetrk_free(wl); 659 660 cv_destroy(&wl->wl_reclaimable_cv); 661 mutex_destroy(&wl->wl_mtx); 662 rw_destroy(&wl->wl_rwlock); 663 wapbl_free(wl); 664 665 return 0; 666 } 667 668 static int 669 wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) 670 { 671 struct pstats *pstats = curlwp->l_proc->p_stats; 672 struct buf *bp; 673 int error; 674 675 KASSERT((flags & ~(B_WRITE | B_READ)) == 0); 676 KASSERT(devvp->v_type == VBLK); 677 678 if ((flags & (B_WRITE | B_READ)) == B_WRITE) { 679 mutex_enter(&devvp->v_interlock); 680 devvp->v_numoutput++; 681 mutex_exit(&devvp->v_interlock); 682 pstats->p_ru.ru_oublock++; 683 } else { 684 pstats->p_ru.ru_inblock++; 685 } 686 687 bp = getiobuf(devvp, true); 688 bp->b_flags = flags; 689 bp->b_cflags = BC_BUSY; /* silly & dubious */ 690 bp->b_dev = devvp->v_rdev; 691 bp->b_data = data; 692 bp->b_bufsize = bp->b_resid = bp->b_bcount = len; 693 bp->b_blkno = pbn; 694 695 WAPBL_PRINTF(WAPBL_PRINT_IO, 696 ("wapbl_doio: %s %d bytes at block %"PRId64" on dev 0x%x\n", 697 BUF_ISWRITE(bp) ? "write" : "read", bp->b_bcount, 698 bp->b_blkno, bp->b_dev)); 699 700 VOP_STRATEGY(devvp, bp); 701 702 error = biowait(bp); 703 putiobuf(bp); 704 705 if (error) { 706 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 707 ("wapbl_doio: %s %zu bytes at block %" PRId64 708 " on dev 0x%x failed with error %d\n", 709 (((flags & (B_WRITE | B_READ)) == B_WRITE) ? 710 "write" : "read"), 711 len, pbn, devvp->v_rdev, error)); 712 } 713 714 return error; 715 } 716 717 int 718 wapbl_write(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 719 { 720 721 return wapbl_doio(data, len, devvp, pbn, B_WRITE); 722 } 723 724 int 725 wapbl_read(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 726 { 727 728 return wapbl_doio(data, len, devvp, pbn, B_READ); 729 } 730 731 /* 732 * Off is byte offset returns new offset for next write 733 * handles log wraparound 734 */ 735 static int 736 wapbl_circ_write(struct wapbl *wl, void *data, size_t len, off_t *offp) 737 { 738 size_t slen; 739 off_t off = *offp; 740 int error; 741 742 KDASSERT(((len >> wl->wl_log_dev_bshift) << 743 wl->wl_log_dev_bshift) == len); 744 745 if (off < wl->wl_circ_off) 746 off = wl->wl_circ_off; 747 slen = wl->wl_circ_off + wl->wl_circ_size - off; 748 if (slen < len) { 749 error = wapbl_write(data, slen, wl->wl_devvp, 750 wl->wl_logpbn + (off >> wl->wl_log_dev_bshift)); 751 if (error) 752 return error; 753 data = (uint8_t *)data + slen; 754 len -= slen; 755 off = wl->wl_circ_off; 756 } 757 error = wapbl_write(data, len, wl->wl_devvp, 758 wl->wl_logpbn + (off >> wl->wl_log_dev_bshift)); 759 if (error) 760 return error; 761 off += len; 762 if (off >= wl->wl_circ_off + wl->wl_circ_size) 763 off = wl->wl_circ_off; 764 *offp = off; 765 return 0; 766 } 767 768 /****************************************************************/ 769 770 int 771 wapbl_begin(struct wapbl *wl, const char *file, int line) 772 { 773 int doflush; 774 unsigned lockcount; 775 krw_t op; 776 777 KDASSERT(wl); 778 779 #ifdef WAPBL_DEBUG_SERIALIZE 780 op = RW_WRITER; 781 #else 782 op = RW_READER; 783 #endif 784 785 /* 786 * XXX this needs to be made much more sophisticated. 787 * perhaps each wapbl_begin could reserve a specified 788 * number of buffers and bytes. 789 */ 790 mutex_enter(&wl->wl_mtx); 791 lockcount = wl->wl_lock_count; 792 doflush = ((wl->wl_bufbytes + (lockcount * MAXPHYS)) > 793 wl->wl_bufbytes_max / 2) || 794 ((wl->wl_bufcount + (lockcount * 10)) > 795 wl->wl_bufcount_max / 2) || 796 (wapbl_transaction_len(wl) > wl->wl_circ_size / 2); 797 mutex_exit(&wl->wl_mtx); 798 799 if (doflush) { 800 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 801 ("force flush lockcnt=%d bufbytes=%zu " 802 "(max=%zu) bufcount=%zu (max=%zu)\n", 803 lockcount, wl->wl_bufbytes, 804 wl->wl_bufbytes_max, wl->wl_bufcount, 805 wl->wl_bufcount_max)); 806 } 807 808 if (doflush) { 809 int error = wapbl_flush(wl, 0); 810 if (error) 811 return error; 812 } 813 814 rw_enter(&wl->wl_rwlock, op); 815 mutex_enter(&wl->wl_mtx); 816 wl->wl_lock_count++; 817 mutex_exit(&wl->wl_mtx); 818 819 #if defined(WAPBL_DEBUG_PRINT) && defined(WAPBL_DEBUG_SERIALIZE) 820 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 821 ("wapbl_begin thread %d.%d with bufcount=%zu " 822 "bufbytes=%zu bcount=%zu at %s:%d\n", 823 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 824 wl->wl_bufbytes, wl->wl_bcount, file, line)); 825 #endif 826 827 return 0; 828 } 829 830 void 831 wapbl_end(struct wapbl *wl) 832 { 833 834 #if defined(WAPBL_DEBUG_PRINT) && defined(WAPBL_DEBUG_SERIALIZE) 835 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 836 ("wapbl_end thread %d.%d with bufcount=%zu " 837 "bufbytes=%zu bcount=%zu\n", 838 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 839 wl->wl_bufbytes, wl->wl_bcount)); 840 #endif 841 842 mutex_enter(&wl->wl_mtx); 843 KASSERT(wl->wl_lock_count > 0); 844 wl->wl_lock_count--; 845 mutex_exit(&wl->wl_mtx); 846 847 rw_exit(&wl->wl_rwlock); 848 } 849 850 void 851 wapbl_add_buf(struct wapbl *wl, struct buf * bp) 852 { 853 854 KASSERT(bp->b_cflags & BC_BUSY); 855 KASSERT(bp->b_vp); 856 857 wapbl_jlock_assert(wl); 858 859 #if 0 860 /* 861 * XXX this might be an issue for swapfiles. 862 * see uvm_swap.c:1702 863 * 864 * XXX2 why require it then? leap of semantics? 865 */ 866 KASSERT((bp->b_cflags & BC_NOCACHE) == 0); 867 #endif 868 869 mutex_enter(&wl->wl_mtx); 870 if (bp->b_flags & B_LOCKED) { 871 LIST_REMOVE(bp, b_wapbllist); 872 WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, 873 ("wapbl_add_buf thread %d.%d re-adding buf %p " 874 "with %d bytes %d bcount\n", 875 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 876 bp->b_bcount)); 877 } else { 878 /* unlocked by dirty buffers shouldn't exist */ 879 KASSERT(!(bp->b_oflags & BO_DELWRI)); 880 wl->wl_bufbytes += bp->b_bufsize; 881 wl->wl_bcount += bp->b_bcount; 882 wl->wl_bufcount++; 883 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 884 ("wapbl_add_buf thread %d.%d adding buf %p " 885 "with %d bytes %d bcount\n", 886 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 887 bp->b_bcount)); 888 } 889 LIST_INSERT_HEAD(&wl->wl_bufs, bp, b_wapbllist); 890 mutex_exit(&wl->wl_mtx); 891 892 bp->b_flags |= B_LOCKED; 893 } 894 895 static void 896 wapbl_remove_buf_locked(struct wapbl * wl, struct buf *bp) 897 { 898 899 KASSERT(mutex_owned(&wl->wl_mtx)); 900 KASSERT(bp->b_cflags & BC_BUSY); 901 wapbl_jlock_assert(wl); 902 903 #if 0 904 /* 905 * XXX this might be an issue for swapfiles. 906 * see uvm_swap.c:1725 907 * 908 * XXXdeux: see above 909 */ 910 KASSERT((bp->b_flags & BC_NOCACHE) == 0); 911 #endif 912 KASSERT(bp->b_flags & B_LOCKED); 913 914 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 915 ("wapbl_remove_buf thread %d.%d removing buf %p with " 916 "%d bytes %d bcount\n", 917 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, bp->b_bcount)); 918 919 KASSERT(wl->wl_bufbytes >= bp->b_bufsize); 920 wl->wl_bufbytes -= bp->b_bufsize; 921 KASSERT(wl->wl_bcount >= bp->b_bcount); 922 wl->wl_bcount -= bp->b_bcount; 923 KASSERT(wl->wl_bufcount > 0); 924 wl->wl_bufcount--; 925 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 926 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 927 LIST_REMOVE(bp, b_wapbllist); 928 929 bp->b_flags &= ~B_LOCKED; 930 } 931 932 /* called from brelsel() in vfs_bio among other places */ 933 void 934 wapbl_remove_buf(struct wapbl * wl, struct buf *bp) 935 { 936 937 mutex_enter(&wl->wl_mtx); 938 wapbl_remove_buf_locked(wl, bp); 939 mutex_exit(&wl->wl_mtx); 940 } 941 942 void 943 wapbl_resize_buf(struct wapbl *wl, struct buf *bp, long oldsz, long oldcnt) 944 { 945 946 KASSERT(bp->b_cflags & BC_BUSY); 947 948 /* 949 * XXX: why does this depend on B_LOCKED? otherwise the buf 950 * is not for a transaction? if so, why is this called in the 951 * first place? 952 */ 953 if (bp->b_flags & B_LOCKED) { 954 mutex_enter(&wl->wl_mtx); 955 wl->wl_bufbytes += bp->b_bufsize - oldsz; 956 wl->wl_bcount += bp->b_bcount - oldcnt; 957 mutex_exit(&wl->wl_mtx); 958 } 959 } 960 961 #endif /* _KERNEL */ 962 963 /****************************************************************/ 964 /* Some utility inlines */ 965 966 /* This is used to advance the pointer at old to new value at old+delta */ 967 static __inline off_t 968 wapbl_advance(size_t size, size_t off, off_t old, size_t delta) 969 { 970 off_t new; 971 972 /* Define acceptable ranges for inputs. */ 973 KASSERT(delta <= size); 974 KASSERT((old == 0) || (old >= off)); 975 KASSERT(old < (size + off)); 976 977 if ((old == 0) && (delta != 0)) 978 new = off + delta; 979 else if ((old + delta) < (size + off)) 980 new = old + delta; 981 else 982 new = (old + delta) - size; 983 984 /* Note some interesting axioms */ 985 KASSERT((delta != 0) || (new == old)); 986 KASSERT((delta == 0) || (new != 0)); 987 KASSERT((delta != (size)) || (new == old)); 988 989 /* Define acceptable ranges for output. */ 990 KASSERT((new == 0) || (new >= off)); 991 KASSERT(new < (size + off)); 992 return new; 993 } 994 995 static __inline size_t 996 wapbl_space_used(size_t avail, off_t head, off_t tail) 997 { 998 999 if (tail == 0) { 1000 KASSERT(head == 0); 1001 return 0; 1002 } 1003 return ((head + (avail - 1) - tail) % avail) + 1; 1004 } 1005 1006 static __inline size_t 1007 wapbl_space_free(size_t avail, off_t head, off_t tail) 1008 { 1009 1010 return avail - wapbl_space_used(avail, head, tail); 1011 } 1012 1013 static __inline void 1014 wapbl_advance_head(size_t size, size_t off, size_t delta, off_t *headp, 1015 off_t *tailp) 1016 { 1017 off_t head = *headp; 1018 off_t tail = *tailp; 1019 1020 KASSERT(delta <= wapbl_space_free(size, head, tail)); 1021 head = wapbl_advance(size, off, head, delta); 1022 if ((tail == 0) && (head != 0)) 1023 tail = off; 1024 *headp = head; 1025 *tailp = tail; 1026 } 1027 1028 static __inline void 1029 wapbl_advance_tail(size_t size, size_t off, size_t delta, off_t *headp, 1030 off_t *tailp) 1031 { 1032 off_t head = *headp; 1033 off_t tail = *tailp; 1034 1035 KASSERT(delta <= wapbl_space_used(size, head, tail)); 1036 tail = wapbl_advance(size, off, tail, delta); 1037 if (head == tail) { 1038 head = tail = 0; 1039 } 1040 *headp = head; 1041 *tailp = tail; 1042 } 1043 1044 #ifdef _KERNEL 1045 1046 /****************************************************************/ 1047 1048 /* 1049 * Remove transactions whose buffers are completely flushed to disk. 1050 * Will block until at least minfree space is available. 1051 * only intended to be called from inside wapbl_flush and therefore 1052 * does not protect against commit races with itself or with flush. 1053 */ 1054 static int 1055 wapbl_truncate(struct wapbl *wl, size_t minfree, int waitonly) 1056 { 1057 size_t delta; 1058 size_t avail; 1059 off_t head; 1060 off_t tail; 1061 int error = 0; 1062 1063 KASSERT(minfree <= (wl->wl_circ_size - wl->wl_reserved_bytes)); 1064 KASSERT(rw_write_held(&wl->wl_rwlock)); 1065 1066 mutex_enter(&wl->wl_mtx); 1067 1068 /* 1069 * First check to see if we have to do a commit 1070 * at all. 1071 */ 1072 avail = wapbl_space_free(wl->wl_circ_size, wl->wl_head, wl->wl_tail); 1073 if (minfree < avail) { 1074 mutex_exit(&wl->wl_mtx); 1075 return 0; 1076 } 1077 minfree -= avail; 1078 while ((wl->wl_error_count == 0) && 1079 (wl->wl_reclaimable_bytes < minfree)) { 1080 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1081 ("wapbl_truncate: sleeping on %p wl=%p bytes=%zd " 1082 "minfree=%zd\n", 1083 &wl->wl_reclaimable_bytes, wl, wl->wl_reclaimable_bytes, 1084 minfree)); 1085 1086 cv_wait(&wl->wl_reclaimable_cv, &wl->wl_mtx); 1087 } 1088 if (wl->wl_reclaimable_bytes < minfree) { 1089 KASSERT(wl->wl_error_count); 1090 /* XXX maybe get actual error from buffer instead someday? */ 1091 error = EIO; 1092 } 1093 head = wl->wl_head; 1094 tail = wl->wl_tail; 1095 delta = wl->wl_reclaimable_bytes; 1096 1097 /* If all of of the entries are flushed, then be sure to keep 1098 * the reserved bytes reserved. Watch out for discarded transactions, 1099 * which could leave more bytes reserved than are reclaimable. 1100 */ 1101 if (SIMPLEQ_EMPTY(&wl->wl_entries) && 1102 (delta >= wl->wl_reserved_bytes)) { 1103 delta -= wl->wl_reserved_bytes; 1104 } 1105 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, &head, 1106 &tail); 1107 KDASSERT(wl->wl_reserved_bytes <= 1108 wapbl_space_used(wl->wl_circ_size, head, tail)); 1109 mutex_exit(&wl->wl_mtx); 1110 1111 if (error) 1112 return error; 1113 1114 if (waitonly) 1115 return 0; 1116 1117 /* 1118 * This is where head, tail and delta are unprotected 1119 * from races against itself or flush. This is ok since 1120 * we only call this routine from inside flush itself. 1121 * 1122 * XXX: how can it race against itself when accessed only 1123 * from behind the write-locked rwlock? 1124 */ 1125 error = wapbl_write_commit(wl, head, tail); 1126 if (error) 1127 return error; 1128 1129 wl->wl_head = head; 1130 wl->wl_tail = tail; 1131 1132 mutex_enter(&wl->wl_mtx); 1133 KASSERT(wl->wl_reclaimable_bytes >= delta); 1134 wl->wl_reclaimable_bytes -= delta; 1135 mutex_exit(&wl->wl_mtx); 1136 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1137 ("wapbl_truncate thread %d.%d truncating %zu bytes\n", 1138 curproc->p_pid, curlwp->l_lid, delta)); 1139 1140 return 0; 1141 } 1142 1143 /****************************************************************/ 1144 1145 void 1146 wapbl_biodone(struct buf *bp) 1147 { 1148 struct wapbl_entry *we = bp->b_private; 1149 struct wapbl *wl = we->we_wapbl; 1150 1151 /* 1152 * Handle possible flushing of buffers after log has been 1153 * decomissioned. 1154 */ 1155 if (!wl) { 1156 KASSERT(we->we_bufcount > 0); 1157 we->we_bufcount--; 1158 #ifdef WAPBL_DEBUG_BUFBYTES 1159 KASSERT(we->we_unsynced_bufbytes >= bp->b_bufsize); 1160 we->we_unsynced_bufbytes -= bp->b_bufsize; 1161 #endif 1162 1163 if (we->we_bufcount == 0) { 1164 #ifdef WAPBL_DEBUG_BUFBYTES 1165 KASSERT(we->we_unsynced_bufbytes == 0); 1166 #endif 1167 wapbl_free(we); 1168 } 1169 1170 brelse(bp, 0); 1171 return; 1172 } 1173 1174 #ifdef ohbother 1175 KDASSERT(bp->b_flags & B_DONE); 1176 KDASSERT(!(bp->b_flags & B_DELWRI)); 1177 KDASSERT(bp->b_flags & B_ASYNC); 1178 KDASSERT(bp->b_flags & B_BUSY); 1179 KDASSERT(!(bp->b_flags & B_LOCKED)); 1180 KDASSERT(!(bp->b_flags & B_READ)); 1181 KDASSERT(!(bp->b_flags & B_INVAL)); 1182 KDASSERT(!(bp->b_flags & B_NOCACHE)); 1183 #endif 1184 1185 if (bp->b_error) { 1186 #ifdef notyet /* Can't currently handle possible dirty buffer reuse */ 1187 XXXpooka: interfaces not fully updated 1188 Note: this was not enabled in the original patch 1189 against netbsd4 either. I don't know if comment 1190 above is true or not. 1191 1192 /* 1193 * If an error occurs, report the error and leave the 1194 * buffer as a delayed write on the LRU queue. 1195 * restarting the write would likely result in 1196 * an error spinloop, so let it be done harmlessly 1197 * by the syncer. 1198 */ 1199 bp->b_flags &= ~(B_DONE); 1200 simple_unlock(&bp->b_interlock); 1201 1202 if (we->we_error == 0) { 1203 mutex_enter(&wl->wl_mtx); 1204 wl->wl_error_count++; 1205 mutex_exit(&wl->wl_mtx); 1206 cv_broadcast(&wl->wl_reclaimable_cv); 1207 } 1208 we->we_error = bp->b_error; 1209 bp->b_error = 0; 1210 brelse(bp); 1211 return; 1212 #else 1213 /* For now, just mark the log permanently errored out */ 1214 1215 mutex_enter(&wl->wl_mtx); 1216 if (wl->wl_error_count == 0) { 1217 wl->wl_error_count++; 1218 cv_broadcast(&wl->wl_reclaimable_cv); 1219 } 1220 mutex_exit(&wl->wl_mtx); 1221 #endif 1222 } 1223 1224 mutex_enter(&wl->wl_mtx); 1225 1226 KASSERT(we->we_bufcount > 0); 1227 we->we_bufcount--; 1228 #ifdef WAPBL_DEBUG_BUFBYTES 1229 KASSERT(we->we_unsynced_bufbytes >= bp->b_bufsize); 1230 we->we_unsynced_bufbytes -= bp->b_bufsize; 1231 KASSERT(wl->wl_unsynced_bufbytes >= bp->b_bufsize); 1232 wl->wl_unsynced_bufbytes -= bp->b_bufsize; 1233 #endif 1234 1235 /* 1236 * If the current transaction can be reclaimed, start 1237 * at the beginning and reclaim any consecutive reclaimable 1238 * transactions. If we successfully reclaim anything, 1239 * then wakeup anyone waiting for the reclaim. 1240 */ 1241 if (we->we_bufcount == 0) { 1242 size_t delta = 0; 1243 int errcnt = 0; 1244 #ifdef WAPBL_DEBUG_BUFBYTES 1245 KDASSERT(we->we_unsynced_bufbytes == 0); 1246 #endif 1247 /* 1248 * clear any posted error, since the buffer it came from 1249 * has successfully flushed by now 1250 */ 1251 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) && 1252 (we->we_bufcount == 0)) { 1253 delta += we->we_reclaimable_bytes; 1254 if (we->we_error) 1255 errcnt++; 1256 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 1257 wapbl_free(we); 1258 } 1259 1260 if (delta) { 1261 wl->wl_reclaimable_bytes += delta; 1262 KASSERT(wl->wl_error_count >= errcnt); 1263 wl->wl_error_count -= errcnt; 1264 cv_broadcast(&wl->wl_reclaimable_cv); 1265 } 1266 } 1267 1268 mutex_exit(&wl->wl_mtx); 1269 brelse(bp, 0); 1270 } 1271 1272 /* 1273 * Write transactions to disk + start I/O for contents 1274 */ 1275 int 1276 wapbl_flush(struct wapbl *wl, int waitfor) 1277 { 1278 struct buf *bp; 1279 struct wapbl_entry *we; 1280 off_t off; 1281 off_t head; 1282 off_t tail; 1283 size_t delta = 0; 1284 size_t flushsize; 1285 size_t reserved; 1286 int error = 0; 1287 1288 /* 1289 * Do a quick check to see if a full flush can be skipped 1290 * This assumes that the flush callback does not need to be called 1291 * unless there are other outstanding bufs. 1292 */ 1293 if (!waitfor) { 1294 size_t nbufs; 1295 mutex_enter(&wl->wl_mtx); /* XXX need mutex here to 1296 protect the KASSERTS */ 1297 nbufs = wl->wl_bufcount; 1298 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1299 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1300 mutex_exit(&wl->wl_mtx); 1301 if (nbufs == 0) 1302 return 0; 1303 } 1304 1305 /* 1306 * XXX we may consider using LK_UPGRADE here 1307 * if we want to call flush from inside a transaction 1308 */ 1309 rw_enter(&wl->wl_rwlock, RW_WRITER); 1310 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 1311 wl->wl_dealloccnt); 1312 1313 /* 1314 * Now that we are fully locked and flushed, 1315 * do another check for nothing to do. 1316 */ 1317 if (wl->wl_bufcount == 0) { 1318 goto out; 1319 } 1320 1321 #if 0 1322 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1323 ("wapbl_flush thread %d.%d flushing entries with " 1324 "bufcount=%zu bufbytes=%zu\n", 1325 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1326 wl->wl_bufbytes)); 1327 #endif 1328 1329 /* Calculate amount of space needed to flush */ 1330 flushsize = wapbl_transaction_len(wl); 1331 1332 if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) { 1333 /* 1334 * XXX this could be handled more gracefully, perhaps place 1335 * only a partial transaction in the log and allow the 1336 * remaining to flush without the protection of the journal. 1337 */ 1338 panic("wapbl_flush: current transaction too big to flush\n"); 1339 } 1340 1341 error = wapbl_truncate(wl, flushsize, 0); 1342 if (error) 1343 goto out2; 1344 1345 off = wl->wl_head; 1346 KASSERT((off == 0) || ((off >= wl->wl_circ_off) && 1347 (off < wl->wl_circ_off + wl->wl_circ_size))); 1348 error = wapbl_write_blocks(wl, &off); 1349 if (error) 1350 goto out2; 1351 error = wapbl_write_revocations(wl, &off); 1352 if (error) 1353 goto out2; 1354 error = wapbl_write_inodes(wl, &off); 1355 if (error) 1356 goto out2; 1357 1358 reserved = 0; 1359 if (wl->wl_inohashcnt) 1360 reserved = wapbl_transaction_inodes_len(wl); 1361 1362 head = wl->wl_head; 1363 tail = wl->wl_tail; 1364 1365 wapbl_advance_head(wl->wl_circ_size, wl->wl_circ_off, flushsize, 1366 &head, &tail); 1367 #ifdef WAPBL_DEBUG 1368 if (head != off) { 1369 panic("lost head! head=%"PRIdMAX" tail=%" PRIdMAX 1370 " off=%"PRIdMAX" flush=%zu\n", 1371 (intmax_t)head, (intmax_t)tail, (intmax_t)off, 1372 flushsize); 1373 } 1374 #else 1375 KASSERT(head == off); 1376 #endif 1377 1378 /* Opportunistically move the tail forward if we can */ 1379 if (!wapbl_lazy_truncate) { 1380 mutex_enter(&wl->wl_mtx); 1381 delta = wl->wl_reclaimable_bytes; 1382 mutex_exit(&wl->wl_mtx); 1383 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, 1384 &head, &tail); 1385 } 1386 1387 error = wapbl_write_commit(wl, head, tail); 1388 if (error) 1389 goto out2; 1390 1391 /* poolme? or kmemme? */ 1392 we = wapbl_calloc(1, sizeof(*we)); 1393 1394 #ifdef WAPBL_DEBUG_BUFBYTES 1395 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1396 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1397 " unsynced=%zu" 1398 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1399 "inodes=%d\n", 1400 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1401 wapbl_space_used(wl->wl_circ_size, head, tail), 1402 wl->wl_unsynced_bufbytes, wl->wl_bufcount, 1403 wl->wl_bufbytes, wl->wl_bcount, wl->wl_dealloccnt, 1404 wl->wl_inohashcnt)); 1405 #else 1406 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1407 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1408 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1409 "inodes=%d\n", 1410 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1411 wapbl_space_used(wl->wl_circ_size, head, tail), 1412 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1413 wl->wl_dealloccnt, wl->wl_inohashcnt)); 1414 #endif 1415 1416 1417 mutex_enter(&bufcache_lock); 1418 mutex_enter(&wl->wl_mtx); 1419 1420 wl->wl_reserved_bytes = reserved; 1421 wl->wl_head = head; 1422 wl->wl_tail = tail; 1423 KASSERT(wl->wl_reclaimable_bytes >= delta); 1424 wl->wl_reclaimable_bytes -= delta; 1425 wl->wl_dealloccnt = 0; 1426 #ifdef WAPBL_DEBUG_BUFBYTES 1427 wl->wl_unsynced_bufbytes += wl->wl_bufbytes; 1428 #endif 1429 1430 we->we_wapbl = wl; 1431 we->we_bufcount = wl->wl_bufcount; 1432 #ifdef WAPBL_DEBUG_BUFBYTES 1433 we->we_unsynced_bufbytes = wl->wl_bufbytes; 1434 #endif 1435 we->we_reclaimable_bytes = flushsize; 1436 we->we_error = 0; 1437 SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); 1438 1439 /* 1440 * this flushes bufs in reverse order than they were queued 1441 * it shouldn't matter, but if we care we could use TAILQ instead. 1442 * XXX Note they will get put on the lru queue when they flush 1443 * so we might actually want to change this to preserve order. 1444 */ 1445 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 1446 if (bbusy(bp, 0, 0, &wl->wl_mtx)) { 1447 continue; 1448 } 1449 bp->b_iodone = wapbl_biodone; 1450 bp->b_private = we; 1451 bremfree(bp); 1452 wapbl_remove_buf_locked(wl, bp); 1453 mutex_exit(&wl->wl_mtx); 1454 mutex_exit(&bufcache_lock); 1455 bawrite(bp); 1456 mutex_enter(&bufcache_lock); 1457 mutex_enter(&wl->wl_mtx); 1458 } 1459 mutex_exit(&wl->wl_mtx); 1460 mutex_exit(&bufcache_lock); 1461 1462 #if 0 1463 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1464 ("wapbl_flush thread %d.%d done flushing entries...\n", 1465 curproc->p_pid, curlwp->l_lid)); 1466 #endif 1467 1468 out: 1469 1470 /* 1471 * If the waitfor flag is set, don't return until everything is 1472 * fully flushed and the on disk log is empty. 1473 */ 1474 if (waitfor) { 1475 error = wapbl_truncate(wl, wl->wl_circ_size - 1476 wl->wl_reserved_bytes, wapbl_lazy_truncate); 1477 } 1478 1479 out2: 1480 if (error) { 1481 wl->wl_flush_abort(wl->wl_mount, wl->wl_deallocblks, 1482 wl->wl_dealloclens, wl->wl_dealloccnt); 1483 } 1484 1485 #ifdef WAPBL_DEBUG_PRINT 1486 if (error) { 1487 pid_t pid = -1; 1488 lwpid_t lid = -1; 1489 if (curproc) 1490 pid = curproc->p_pid; 1491 if (curlwp) 1492 lid = curlwp->l_lid; 1493 mutex_enter(&wl->wl_mtx); 1494 #ifdef WAPBL_DEBUG_BUFBYTES 1495 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1496 ("wapbl_flush: thread %d.%d aborted flush: " 1497 "error = %d\n" 1498 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1499 "deallocs=%d inodes=%d\n" 1500 "\terrcnt = %d, reclaimable=%zu reserved=%zu " 1501 "unsynced=%zu\n", 1502 pid, lid, error, wl->wl_bufcount, 1503 wl->wl_bufbytes, wl->wl_bcount, 1504 wl->wl_dealloccnt, wl->wl_inohashcnt, 1505 wl->wl_error_count, wl->wl_reclaimable_bytes, 1506 wl->wl_reserved_bytes, wl->wl_unsynced_bufbytes)); 1507 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1508 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1509 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1510 "error = %d, unsynced = %zu\n", 1511 we->we_bufcount, we->we_reclaimable_bytes, 1512 we->we_error, we->we_unsynced_bufbytes)); 1513 } 1514 #else 1515 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1516 ("wapbl_flush: thread %d.%d aborted flush: " 1517 "error = %d\n" 1518 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1519 "deallocs=%d inodes=%d\n" 1520 "\terrcnt = %d, reclaimable=%zu reserved=%zu\n", 1521 pid, lid, error, wl->wl_bufcount, 1522 wl->wl_bufbytes, wl->wl_bcount, 1523 wl->wl_dealloccnt, wl->wl_inohashcnt, 1524 wl->wl_error_count, wl->wl_reclaimable_bytes, 1525 wl->wl_reserved_bytes)); 1526 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1527 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1528 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1529 "error = %d\n", we->we_bufcount, 1530 we->we_reclaimable_bytes, we->we_error)); 1531 } 1532 #endif 1533 mutex_exit(&wl->wl_mtx); 1534 } 1535 #endif 1536 1537 rw_exit(&wl->wl_rwlock); 1538 return error; 1539 } 1540 1541 /****************************************************************/ 1542 1543 void 1544 wapbl_jlock_assert(struct wapbl *wl) 1545 { 1546 1547 #ifdef WAPBL_DEBUG_SERIALIZE 1548 KASSERT(rw_write_held(&wl->wl_rwlock)); 1549 #else 1550 KASSERT(rw_read_held(&wl->wl_rwlock) || rw_write_held(&wl->wl_rwlock)); 1551 #endif 1552 } 1553 1554 void 1555 wapbl_junlock_assert(struct wapbl *wl) 1556 { 1557 1558 #ifdef WAPBL_DEBUG_SERIALIZE 1559 KASSERT(!rw_write_held(&wl->wl_rwlock)); 1560 #endif 1561 } 1562 1563 /****************************************************************/ 1564 1565 /* locks missing */ 1566 void 1567 wapbl_print(struct wapbl *wl, 1568 int full, 1569 void (*pr)(const char *, ...)) 1570 { 1571 struct buf *bp; 1572 struct wapbl_entry *we; 1573 (*pr)("wapbl %p", wl); 1574 (*pr)("\nlogvp = %p, devvp = %p, logpbn = %"PRId64"\n", 1575 wl->wl_logvp, wl->wl_devvp, wl->wl_logpbn); 1576 (*pr)("circ = %zu, header = %zu, head = %"PRIdMAX" tail = %"PRIdMAX"\n", 1577 wl->wl_circ_size, wl->wl_circ_off, 1578 (intmax_t)wl->wl_head, (intmax_t)wl->wl_tail); 1579 (*pr)("fs_dev_bshift = %d, log_dev_bshift = %d\n", 1580 wl->wl_log_dev_bshift, wl->wl_fs_dev_bshift); 1581 #ifdef WAPBL_DEBUG_BUFBYTES 1582 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1583 "reserved = %zu errcnt = %d unsynced = %zu\n", 1584 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1585 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1586 wl->wl_error_count, wl->wl_unsynced_bufbytes); 1587 #else 1588 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1589 "reserved = %zu errcnt = %d\n", wl->wl_bufcount, wl->wl_bufbytes, 1590 wl->wl_bcount, wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1591 wl->wl_error_count); 1592 #endif 1593 (*pr)("\tdealloccnt = %d, dealloclim = %d\n", 1594 wl->wl_dealloccnt, wl->wl_dealloclim); 1595 (*pr)("\tinohashcnt = %d, inohashmask = 0x%08x\n", 1596 wl->wl_inohashcnt, wl->wl_inohashmask); 1597 (*pr)("entries:\n"); 1598 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1599 #ifdef WAPBL_DEBUG_BUFBYTES 1600 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d, " 1601 "unsynced = %zu\n", 1602 we->we_bufcount, we->we_reclaimable_bytes, 1603 we->we_error, we->we_unsynced_bufbytes); 1604 #else 1605 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d\n", 1606 we->we_bufcount, we->we_reclaimable_bytes, we->we_error); 1607 #endif 1608 } 1609 if (full) { 1610 int cnt = 0; 1611 (*pr)("bufs ="); 1612 LIST_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { 1613 if (!LIST_NEXT(bp, b_wapbllist)) { 1614 (*pr)(" %p", bp); 1615 } else if ((++cnt % 6) == 0) { 1616 (*pr)(" %p,\n\t", bp); 1617 } else { 1618 (*pr)(" %p,", bp); 1619 } 1620 } 1621 (*pr)("\n"); 1622 1623 (*pr)("dealloced blks = "); 1624 { 1625 int i; 1626 cnt = 0; 1627 for (i = 0; i < wl->wl_dealloccnt; i++) { 1628 (*pr)(" %"PRId64":%d,", 1629 wl->wl_deallocblks[i], 1630 wl->wl_dealloclens[i]); 1631 if ((++cnt % 4) == 0) { 1632 (*pr)("\n\t"); 1633 } 1634 } 1635 } 1636 (*pr)("\n"); 1637 1638 (*pr)("registered inodes = "); 1639 { 1640 int i; 1641 cnt = 0; 1642 for (i = 0; i <= wl->wl_inohashmask; i++) { 1643 struct wapbl_ino_head *wih; 1644 struct wapbl_ino *wi; 1645 1646 wih = &wl->wl_inohash[i]; 1647 LIST_FOREACH(wi, wih, wi_hash) { 1648 if (wi->wi_ino == 0) 1649 continue; 1650 (*pr)(" %"PRId32"/0%06"PRIo32",", 1651 wi->wi_ino, wi->wi_mode); 1652 if ((++cnt % 4) == 0) { 1653 (*pr)("\n\t"); 1654 } 1655 } 1656 } 1657 (*pr)("\n"); 1658 } 1659 } 1660 } 1661 1662 #if defined(WAPBL_DEBUG) || defined(DDB) 1663 void 1664 wapbl_dump(struct wapbl *wl) 1665 { 1666 #if defined(WAPBL_DEBUG) 1667 if (!wl) 1668 wl = wapbl_debug_wl; 1669 #endif 1670 if (!wl) 1671 return; 1672 wapbl_print(wl, 1, printf); 1673 } 1674 #endif 1675 1676 /****************************************************************/ 1677 1678 void 1679 wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len) 1680 { 1681 1682 wapbl_jlock_assert(wl); 1683 1684 /* XXX should eventually instead tie this into resource estimation */ 1685 /* XXX this KASSERT needs locking/mutex analysis */ 1686 KASSERT(wl->wl_dealloccnt < wl->wl_dealloclim); 1687 wl->wl_deallocblks[wl->wl_dealloccnt] = blk; 1688 wl->wl_dealloclens[wl->wl_dealloccnt] = len; 1689 wl->wl_dealloccnt++; 1690 WAPBL_PRINTF(WAPBL_PRINT_ALLOC, 1691 ("wapbl_register_deallocation: blk=%"PRId64" len=%d\n", blk, len)); 1692 } 1693 1694 /****************************************************************/ 1695 1696 static void 1697 wapbl_inodetrk_init(struct wapbl *wl, u_int size) 1698 { 1699 1700 wl->wl_inohash = hashinit(size, HASH_LIST, true, &wl->wl_inohashmask); 1701 if (atomic_inc_uint_nv(&wapbl_ino_pool_refcount) == 1) { 1702 pool_init(&wapbl_ino_pool, sizeof(struct wapbl_ino), 0, 0, 0, 1703 "wapblinopl", &pool_allocator_nointr, IPL_NONE); 1704 } 1705 } 1706 1707 static void 1708 wapbl_inodetrk_free(struct wapbl *wl) 1709 { 1710 1711 /* XXX this KASSERT needs locking/mutex analysis */ 1712 KASSERT(wl->wl_inohashcnt == 0); 1713 hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask); 1714 if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) { 1715 pool_destroy(&wapbl_ino_pool); 1716 } 1717 } 1718 1719 static struct wapbl_ino * 1720 wapbl_inodetrk_get(struct wapbl *wl, ino_t ino) 1721 { 1722 struct wapbl_ino_head *wih; 1723 struct wapbl_ino *wi; 1724 1725 KASSERT(mutex_owned(&wl->wl_mtx)); 1726 1727 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1728 LIST_FOREACH(wi, wih, wi_hash) { 1729 if (ino == wi->wi_ino) 1730 return wi; 1731 } 1732 return 0; 1733 } 1734 1735 void 1736 wapbl_register_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1737 { 1738 struct wapbl_ino_head *wih; 1739 struct wapbl_ino *wi; 1740 1741 wi = pool_get(&wapbl_ino_pool, PR_WAITOK); 1742 1743 mutex_enter(&wl->wl_mtx); 1744 if (wapbl_inodetrk_get(wl, ino) == NULL) { 1745 wi->wi_ino = ino; 1746 wi->wi_mode = mode; 1747 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1748 LIST_INSERT_HEAD(wih, wi, wi_hash); 1749 wl->wl_inohashcnt++; 1750 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1751 ("wapbl_register_inode: ino=%"PRId64"\n", ino)); 1752 mutex_exit(&wl->wl_mtx); 1753 } else { 1754 mutex_exit(&wl->wl_mtx); 1755 pool_put(&wapbl_ino_pool, wi); 1756 } 1757 } 1758 1759 void 1760 wapbl_unregister_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1761 { 1762 struct wapbl_ino *wi; 1763 1764 mutex_enter(&wl->wl_mtx); 1765 wi = wapbl_inodetrk_get(wl, ino); 1766 if (wi) { 1767 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1768 ("wapbl_unregister_inode: ino=%"PRId64"\n", ino)); 1769 KASSERT(wl->wl_inohashcnt > 0); 1770 wl->wl_inohashcnt--; 1771 LIST_REMOVE(wi, wi_hash); 1772 mutex_exit(&wl->wl_mtx); 1773 1774 pool_put(&wapbl_ino_pool, wi); 1775 } else { 1776 mutex_exit(&wl->wl_mtx); 1777 } 1778 } 1779 1780 /****************************************************************/ 1781 1782 static __inline size_t 1783 wapbl_transaction_inodes_len(struct wapbl *wl) 1784 { 1785 int blocklen = 1<<wl->wl_log_dev_bshift; 1786 int iph; 1787 1788 /* Calculate number of inodes described in a inodelist header */ 1789 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 1790 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 1791 1792 KASSERT(iph > 0); 1793 1794 return MAX(1, howmany(wl->wl_inohashcnt, iph))*blocklen; 1795 } 1796 1797 1798 /* Calculate amount of space a transaction will take on disk */ 1799 static size_t 1800 wapbl_transaction_len(struct wapbl *wl) 1801 { 1802 int blocklen = 1<<wl->wl_log_dev_bshift; 1803 size_t len; 1804 int bph; 1805 1806 /* Calculate number of blocks described in a blocklist header */ 1807 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1808 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1809 1810 KASSERT(bph > 0); 1811 1812 len = wl->wl_bcount; 1813 len += howmany(wl->wl_bufcount, bph)*blocklen; 1814 len += howmany(wl->wl_dealloccnt, bph)*blocklen; 1815 len += wapbl_transaction_inodes_len(wl); 1816 1817 return len; 1818 } 1819 1820 /* 1821 * Perform commit operation 1822 * 1823 * Note that generation number incrementation needs to 1824 * be protected against racing with other invocations 1825 * of wapbl_commit. This is ok since this routine 1826 * is only invoked from wapbl_flush 1827 */ 1828 static int 1829 wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail) 1830 { 1831 struct wapbl_wc_header *wc = wl->wl_wc_header; 1832 struct timespec ts; 1833 int error; 1834 int force = 1; 1835 1836 /* XXX Calc checksum here, instead we do this for now */ 1837 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, FWRITE, FSCRED); 1838 if (error) { 1839 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1840 ("wapbl_write_commit: DIOCCACHESYNC on dev 0x%x " 1841 "returned %d\n", wl->wl_devvp->v_rdev, error)); 1842 } 1843 1844 wc->wc_head = head; 1845 wc->wc_tail = tail; 1846 wc->wc_checksum = 0; 1847 wc->wc_version = 1; 1848 getnanotime(&ts); /* XXX need higher resolution time here? */ 1849 wc->wc_time = ts.tv_sec;; 1850 wc->wc_timensec = ts.tv_nsec; 1851 1852 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 1853 ("wapbl_write_commit: head = %"PRIdMAX "tail = %"PRIdMAX"\n", 1854 (intmax_t)head, (intmax_t)tail)); 1855 1856 /* 1857 * XXX if generation will rollover, then first zero 1858 * over second commit header before trying to write both headers. 1859 */ 1860 1861 error = wapbl_write(wc, wc->wc_len, wl->wl_devvp, 1862 wl->wl_logpbn + wc->wc_generation % 2); 1863 if (error) 1864 return error; 1865 1866 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, FWRITE, FSCRED); 1867 if (error) { 1868 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1869 ("wapbl_write_commit: DIOCCACHESYNC on dev 0x%x " 1870 "returned %d\n", wl->wl_devvp->v_rdev, error)); 1871 } 1872 1873 /* 1874 * If the generation number was zero, write it out a second time. 1875 * This handles initialization and generation number rollover 1876 */ 1877 if (wc->wc_generation++ == 0) { 1878 error = wapbl_write_commit(wl, head, tail); 1879 /* 1880 * This panic should be able to be removed if we do the 1881 * zero'ing mentioned above, and we are certain to roll 1882 * back generation number on failure. 1883 */ 1884 if (error) 1885 panic("wapbl_write_commit: error writing duplicate " 1886 "log header: %d\n", error); 1887 } 1888 return 0; 1889 } 1890 1891 /* Returns new offset value */ 1892 static int 1893 wapbl_write_blocks(struct wapbl *wl, off_t *offp) 1894 { 1895 struct wapbl_wc_blocklist *wc = 1896 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 1897 int blocklen = 1<<wl->wl_log_dev_bshift; 1898 int bph; 1899 struct buf *bp; 1900 off_t off = *offp; 1901 int error; 1902 1903 KASSERT(rw_write_held(&wl->wl_rwlock)); 1904 1905 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1906 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1907 1908 bp = LIST_FIRST(&wl->wl_bufs); 1909 1910 while (bp) { 1911 int cnt; 1912 struct buf *obp = bp; 1913 1914 KASSERT(bp->b_flags & B_LOCKED); 1915 1916 wc->wc_type = WAPBL_WC_BLOCKS; 1917 wc->wc_len = blocklen; 1918 wc->wc_blkcount = 0; 1919 while (bp && (wc->wc_blkcount < bph)) { 1920 /* 1921 * Make sure all the physical block numbers are up to 1922 * date. If this is not always true on a given 1923 * filesystem, then VOP_BMAP must be called. We 1924 * could call VOP_BMAP here, or else in the filesystem 1925 * specific flush callback, although neither of those 1926 * solutions allow us to take the vnode lock. If a 1927 * filesystem requires that we must take the vnode lock 1928 * to call VOP_BMAP, then we can probably do it in 1929 * bwrite when the vnode lock should already be held 1930 * by the invoking code. 1931 */ 1932 KASSERT((bp->b_vp->v_type == VBLK) || 1933 (bp->b_blkno != bp->b_lblkno)); 1934 KASSERT(bp->b_blkno > 0); 1935 1936 wc->wc_blocks[wc->wc_blkcount].wc_daddr = bp->b_blkno; 1937 wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; 1938 wc->wc_len += bp->b_bcount; 1939 wc->wc_blkcount++; 1940 bp = LIST_NEXT(bp, b_wapbllist); 1941 } 1942 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 1943 ("wapbl_write_blocks: len = %u off = %"PRIdMAX"\n", 1944 wc->wc_len, (intmax_t)off)); 1945 1946 error = wapbl_circ_write(wl, wc, blocklen, &off); 1947 if (error) 1948 return error; 1949 bp = obp; 1950 cnt = 0; 1951 while (bp && (cnt++ < bph)) { 1952 error = wapbl_circ_write(wl, bp->b_data, 1953 bp->b_bcount, &off); 1954 if (error) 1955 return error; 1956 bp = LIST_NEXT(bp, b_wapbllist); 1957 } 1958 } 1959 *offp = off; 1960 return 0; 1961 } 1962 1963 static int 1964 wapbl_write_revocations(struct wapbl *wl, off_t *offp) 1965 { 1966 struct wapbl_wc_blocklist *wc = 1967 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 1968 int i; 1969 int blocklen = 1<<wl->wl_log_dev_bshift; 1970 int bph; 1971 off_t off = *offp; 1972 int error; 1973 1974 if (wl->wl_dealloccnt == 0) 1975 return 0; 1976 1977 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1978 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1979 1980 i = 0; 1981 while (i < wl->wl_dealloccnt) { 1982 wc->wc_type = WAPBL_WC_REVOCATIONS; 1983 wc->wc_len = blocklen; 1984 wc->wc_blkcount = 0; 1985 while ((i < wl->wl_dealloccnt) && (wc->wc_blkcount < bph)) { 1986 wc->wc_blocks[wc->wc_blkcount].wc_daddr = 1987 wl->wl_deallocblks[i]; 1988 wc->wc_blocks[wc->wc_blkcount].wc_dlen = 1989 wl->wl_dealloclens[i]; 1990 wc->wc_blkcount++; 1991 i++; 1992 } 1993 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 1994 ("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", 1995 wc->wc_len, (intmax_t)off)); 1996 error = wapbl_circ_write(wl, wc, blocklen, &off); 1997 if (error) 1998 return error; 1999 } 2000 *offp = off; 2001 return 0; 2002 } 2003 2004 static int 2005 wapbl_write_inodes(struct wapbl *wl, off_t *offp) 2006 { 2007 struct wapbl_wc_inodelist *wc = 2008 (struct wapbl_wc_inodelist *)wl->wl_wc_scratch; 2009 int i; 2010 int blocklen = 1<<wl->wl_log_dev_bshift; 2011 off_t off = *offp; 2012 int error; 2013 2014 struct wapbl_ino_head *wih; 2015 struct wapbl_ino *wi; 2016 int iph; 2017 2018 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2019 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2020 2021 i = 0; 2022 wih = &wl->wl_inohash[0]; 2023 wi = 0; 2024 do { 2025 wc->wc_type = WAPBL_WC_INODES; 2026 wc->wc_len = blocklen; 2027 wc->wc_inocnt = 0; 2028 wc->wc_clear = (i == 0); 2029 while ((i < wl->wl_inohashcnt) && (wc->wc_inocnt < iph)) { 2030 while (!wi) { 2031 KASSERT((wih - &wl->wl_inohash[0]) 2032 <= wl->wl_inohashmask); 2033 wi = LIST_FIRST(wih++); 2034 } 2035 wc->wc_inodes[wc->wc_inocnt].wc_inumber = wi->wi_ino; 2036 wc->wc_inodes[wc->wc_inocnt].wc_imode = wi->wi_mode; 2037 wc->wc_inocnt++; 2038 i++; 2039 wi = LIST_NEXT(wi, wi_hash); 2040 } 2041 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2042 ("wapbl_write_inodes: len = %u off = %"PRIdMAX"\n", 2043 wc->wc_len, (intmax_t)off)); 2044 error = wapbl_circ_write(wl, wc, blocklen, &off); 2045 if (error) 2046 return error; 2047 } while (i < wl->wl_inohashcnt); 2048 2049 *offp = off; 2050 return 0; 2051 } 2052 2053 #endif /* _KERNEL */ 2054 2055 /****************************************************************/ 2056 2057 #ifdef _KERNEL 2058 static struct pool wapbl_blk_pool; 2059 static int wapbl_blk_pool_refcount; 2060 #endif 2061 struct wapbl_blk { 2062 LIST_ENTRY(wapbl_blk) wb_hash; 2063 daddr_t wb_blk; 2064 off_t wb_off; /* Offset of this block in the log */ 2065 }; 2066 #define WAPBL_BLKPOOL_MIN 83 2067 2068 static void 2069 wapbl_blkhash_init(struct wapbl_replay *wr, u_int size) 2070 { 2071 if (size < WAPBL_BLKPOOL_MIN) 2072 size = WAPBL_BLKPOOL_MIN; 2073 KASSERT(wr->wr_blkhash == 0); 2074 #ifdef _KERNEL 2075 wr->wr_blkhash = hashinit(size, HASH_LIST, true, &wr->wr_blkhashmask); 2076 if (atomic_inc_uint_nv(&wapbl_blk_pool_refcount) == 1) { 2077 pool_init(&wapbl_blk_pool, sizeof(struct wapbl_blk), 0, 0, 0, 2078 "wapblblkpl", &pool_allocator_nointr, IPL_NONE); 2079 } 2080 #else /* ! _KERNEL */ 2081 /* Manually implement hashinit */ 2082 { 2083 int i; 2084 unsigned long hashsize; 2085 for (hashsize = 1; hashsize < size; hashsize <<= 1) 2086 continue; 2087 wr->wr_blkhash = wapbl_malloc(hashsize * sizeof(*wr->wr_blkhash)); 2088 for (i = 0; i < wr->wr_blkhashmask; i++) 2089 LIST_INIT(&wr->wr_blkhash[i]); 2090 wr->wr_blkhashmask = hashsize - 1; 2091 } 2092 #endif /* ! _KERNEL */ 2093 } 2094 2095 static void 2096 wapbl_blkhash_free(struct wapbl_replay *wr) 2097 { 2098 KASSERT(wr->wr_blkhashcnt == 0); 2099 #ifdef _KERNEL 2100 hashdone(wr->wr_blkhash, HASH_LIST, wr->wr_blkhashmask); 2101 if (atomic_dec_uint_nv(&wapbl_blk_pool_refcount) == 0) { 2102 pool_destroy(&wapbl_blk_pool); 2103 } 2104 #else /* ! _KERNEL */ 2105 wapbl_free(wr->wr_blkhash); 2106 #endif /* ! _KERNEL */ 2107 } 2108 2109 static struct wapbl_blk * 2110 wapbl_blkhash_get(struct wapbl_replay *wr, daddr_t blk) 2111 { 2112 struct wapbl_blk_head *wbh; 2113 struct wapbl_blk *wb; 2114 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2115 LIST_FOREACH(wb, wbh, wb_hash) { 2116 if (blk == wb->wb_blk) 2117 return wb; 2118 } 2119 return 0; 2120 } 2121 2122 static void 2123 wapbl_blkhash_ins(struct wapbl_replay *wr, daddr_t blk, off_t off) 2124 { 2125 struct wapbl_blk_head *wbh; 2126 struct wapbl_blk *wb; 2127 wb = wapbl_blkhash_get(wr, blk); 2128 if (wb) { 2129 KASSERT(wb->wb_blk == blk); 2130 wb->wb_off = off; 2131 } else { 2132 #ifdef _KERNEL 2133 wb = pool_get(&wapbl_blk_pool, PR_WAITOK); 2134 #else /* ! _KERNEL */ 2135 wb = wapbl_malloc(sizeof(*wb)); 2136 #endif /* ! _KERNEL */ 2137 wb->wb_blk = blk; 2138 wb->wb_off = off; 2139 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2140 LIST_INSERT_HEAD(wbh, wb, wb_hash); 2141 wr->wr_blkhashcnt++; 2142 } 2143 } 2144 2145 static void 2146 wapbl_blkhash_rem(struct wapbl_replay *wr, daddr_t blk) 2147 { 2148 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2149 if (wb) { 2150 KASSERT(wr->wr_blkhashcnt > 0); 2151 wr->wr_blkhashcnt--; 2152 LIST_REMOVE(wb, wb_hash); 2153 #ifdef _KERNEL 2154 pool_put(&wapbl_blk_pool, wb); 2155 #else /* ! _KERNEL */ 2156 wapbl_free(wb); 2157 #endif /* ! _KERNEL */ 2158 } 2159 } 2160 2161 static void 2162 wapbl_blkhash_clear(struct wapbl_replay *wr) 2163 { 2164 int i; 2165 for (i = 0; i <= wr->wr_blkhashmask; i++) { 2166 struct wapbl_blk *wb; 2167 2168 while ((wb = LIST_FIRST(&wr->wr_blkhash[i]))) { 2169 KASSERT(wr->wr_blkhashcnt > 0); 2170 wr->wr_blkhashcnt--; 2171 LIST_REMOVE(wb, wb_hash); 2172 #ifdef _KERNEL 2173 pool_put(&wapbl_blk_pool, wb); 2174 #else /* ! _KERNEL */ 2175 wapbl_free(wb); 2176 #endif /* ! _KERNEL */ 2177 } 2178 } 2179 KASSERT(wr->wr_blkhashcnt == 0); 2180 } 2181 2182 /****************************************************************/ 2183 2184 static int 2185 wapbl_circ_read(struct wapbl_replay *wr, void *data, size_t len, off_t *offp) 2186 { 2187 size_t slen; 2188 struct wapbl_wc_header *wc = &wr->wr_wc_header; 2189 off_t off = *offp; 2190 int error; 2191 2192 KASSERT(((len >> wc->wc_log_dev_bshift) << 2193 wc->wc_log_dev_bshift) == len); 2194 if (off < wc->wc_circ_off) 2195 off = wc->wc_circ_off; 2196 slen = wc->wc_circ_off + wc->wc_circ_size - off; 2197 if (slen < len) { 2198 error = wapbl_read(data, slen, wr->wr_devvp, 2199 wr->wr_logpbn + (off >> wc->wc_log_dev_bshift)); 2200 if (error) 2201 return error; 2202 data = (uint8_t *)data + slen; 2203 len -= slen; 2204 off = wc->wc_circ_off; 2205 } 2206 error = wapbl_read(data, len, wr->wr_devvp, 2207 wr->wr_logpbn + (off >> wc->wc_log_dev_bshift)); 2208 if (error) 2209 return error; 2210 off += len; 2211 if (off >= wc->wc_circ_off + wc->wc_circ_size) 2212 off = wc->wc_circ_off; 2213 *offp = off; 2214 return 0; 2215 } 2216 2217 static void 2218 wapbl_circ_advance(struct wapbl_replay *wr, size_t len, off_t *offp) 2219 { 2220 size_t slen; 2221 struct wapbl_wc_header *wc = &wr->wr_wc_header; 2222 off_t off = *offp; 2223 2224 KASSERT(((len >> wc->wc_log_dev_bshift) << 2225 wc->wc_log_dev_bshift) == len); 2226 2227 if (off < wc->wc_circ_off) 2228 off = wc->wc_circ_off; 2229 slen = wc->wc_circ_off + wc->wc_circ_size - off; 2230 if (slen < len) { 2231 len -= slen; 2232 off = wc->wc_circ_off; 2233 } 2234 off += len; 2235 if (off >= wc->wc_circ_off + wc->wc_circ_size) 2236 off = wc->wc_circ_off; 2237 *offp = off; 2238 } 2239 2240 /****************************************************************/ 2241 2242 int 2243 wapbl_replay_start(struct wapbl_replay **wrp, struct vnode *vp, 2244 daddr_t off, size_t count, size_t blksize) 2245 { 2246 struct wapbl_replay *wr; 2247 int error; 2248 struct vnode *devvp; 2249 daddr_t logpbn; 2250 uint8_t *scratch; 2251 struct wapbl_wc_header *wch; 2252 struct wapbl_wc_header *wch2; 2253 /* Use this until we read the actual log header */ 2254 int log_dev_bshift = DEV_BSHIFT; 2255 size_t used; 2256 2257 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2258 ("wapbl_replay_start: vp=%p off=%"PRId64 " count=%zu blksize=%zu\n", 2259 vp, off, count, blksize)); 2260 2261 if (off < 0) 2262 return EINVAL; 2263 2264 if (blksize < DEV_BSIZE) 2265 return EINVAL; 2266 if (blksize % DEV_BSIZE) 2267 return EINVAL; 2268 2269 #ifdef _KERNEL 2270 #if 0 2271 /* XXX vp->v_size isn't reliably set for VBLK devices, 2272 * especially root. However, we might still want to verify 2273 * that the full load is readable */ 2274 if ((off + count) * blksize > vp->v_size) 2275 return EINVAL; 2276 #endif 2277 2278 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, 0)) != 0) { 2279 return error; 2280 } 2281 #else /* ! _KERNEL */ 2282 devvp = vp; 2283 logpbn = off; 2284 #endif /* ! _KERNEL */ 2285 2286 scratch = wapbl_malloc(MAXBSIZE); 2287 2288 error = wapbl_read(scratch, 2<<log_dev_bshift, devvp, logpbn); 2289 if (error) 2290 goto errout; 2291 2292 wch = (struct wapbl_wc_header *)scratch; 2293 wch2 = 2294 (struct wapbl_wc_header *)(scratch + (1<<log_dev_bshift)); 2295 /* XXX verify checksums and magic numbers */ 2296 if (wch->wc_type != WAPBL_WC_HEADER) { 2297 printf("Unrecognized wapbl magic: 0x%08x\n", wch->wc_type); 2298 error = EFTYPE; 2299 goto errout; 2300 } 2301 2302 if (wch2->wc_generation > wch->wc_generation) 2303 wch = wch2; 2304 2305 wr = wapbl_calloc(1, sizeof(*wr)); 2306 2307 wr->wr_logvp = vp; 2308 wr->wr_devvp = devvp; 2309 wr->wr_logpbn = logpbn; 2310 2311 wr->wr_scratch = scratch; 2312 2313 memcpy(&wr->wr_wc_header, wch, sizeof(wr->wr_wc_header)); 2314 2315 used = wapbl_space_used(wch->wc_circ_size, wch->wc_head, wch->wc_tail); 2316 2317 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2318 ("wapbl_replay: head=%"PRId64" tail=%"PRId64" off=%"PRId64 2319 " len=%"PRId64" used=%zu\n", 2320 wch->wc_head, wch->wc_tail, wch->wc_circ_off, 2321 wch->wc_circ_size, used)); 2322 2323 wapbl_blkhash_init(wr, (used >> wch->wc_fs_dev_bshift)); 2324 error = wapbl_replay_prescan(wr); 2325 if (error) { 2326 wapbl_replay_stop(wr); 2327 wapbl_replay_free(wr); 2328 return error; 2329 } 2330 2331 error = wapbl_replay_get_inodes(wr); 2332 if (error) { 2333 wapbl_replay_stop(wr); 2334 wapbl_replay_free(wr); 2335 return error; 2336 } 2337 2338 *wrp = wr; 2339 return 0; 2340 2341 errout: 2342 wapbl_free(scratch); 2343 return error; 2344 } 2345 2346 void 2347 wapbl_replay_stop(struct wapbl_replay *wr) 2348 { 2349 2350 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, ("wapbl_replay_stop called\n")); 2351 2352 KDASSERT(wapbl_replay_isopen(wr)); 2353 2354 wapbl_free(wr->wr_scratch); 2355 wr->wr_scratch = 0; 2356 2357 wr->wr_logvp = 0; 2358 2359 wapbl_blkhash_clear(wr); 2360 wapbl_blkhash_free(wr); 2361 } 2362 2363 void 2364 wapbl_replay_free(struct wapbl_replay *wr) 2365 { 2366 2367 KDASSERT(!wapbl_replay_isopen(wr)); 2368 2369 if (wr->wr_inodes) 2370 wapbl_free(wr->wr_inodes); 2371 wapbl_free(wr); 2372 } 2373 2374 int 2375 wapbl_replay_isopen1(struct wapbl_replay *wr) 2376 { 2377 2378 return wapbl_replay_isopen(wr); 2379 } 2380 2381 static int 2382 wapbl_replay_prescan(struct wapbl_replay *wr) 2383 { 2384 off_t off; 2385 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2386 int error; 2387 2388 int logblklen = 1<<wch->wc_log_dev_bshift; 2389 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2390 2391 wapbl_blkhash_clear(wr); 2392 2393 off = wch->wc_tail; 2394 while (off != wch->wc_head) { 2395 struct wapbl_wc_null *wcn; 2396 off_t saveoff = off; 2397 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2398 if (error) 2399 goto errout; 2400 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2401 switch (wcn->wc_type) { 2402 case WAPBL_WC_BLOCKS: 2403 { 2404 struct wapbl_wc_blocklist *wc = 2405 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2406 int i; 2407 for (i = 0; i < wc->wc_blkcount; i++) { 2408 int j, n; 2409 /* 2410 * Enter each physical block into the 2411 * hashtable independently 2412 */ 2413 n = wc->wc_blocks[i].wc_dlen >> 2414 wch->wc_fs_dev_bshift; 2415 for (j = 0; j < n; j++) { 2416 wapbl_blkhash_ins(wr, 2417 wc->wc_blocks[i].wc_daddr + j, 2418 off); 2419 wapbl_circ_advance(wr, 2420 fsblklen, &off); 2421 } 2422 } 2423 } 2424 break; 2425 2426 case WAPBL_WC_REVOCATIONS: 2427 { 2428 struct wapbl_wc_blocklist *wc = 2429 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2430 int i; 2431 for (i = 0; i < wc->wc_blkcount; i++) { 2432 int j, n; 2433 /* 2434 * Remove any blocks found from the 2435 * hashtable 2436 */ 2437 n = wc->wc_blocks[i].wc_dlen >> 2438 wch->wc_fs_dev_bshift; 2439 for (j = 0; j < n; j++) { 2440 wapbl_blkhash_rem(wr, 2441 wc->wc_blocks[i].wc_daddr + j); 2442 } 2443 } 2444 } 2445 break; 2446 2447 case WAPBL_WC_INODES: 2448 { 2449 struct wapbl_wc_inodelist *wc = 2450 (struct wapbl_wc_inodelist *)wr->wr_scratch; 2451 /* 2452 * Keep track of where we found this so we 2453 * can use it later 2454 */ 2455 if (wc->wc_clear) { 2456 wr->wr_inodestail = saveoff; 2457 wr->wr_inodescnt = 0; 2458 } 2459 if (wr->wr_inodestail) 2460 wr->wr_inodeshead = off; 2461 wr->wr_inodescnt += wc->wc_inocnt; 2462 } 2463 break; 2464 default: 2465 printf("Unrecognized wapbl type: 0x%08x\n", 2466 wcn->wc_type); 2467 error = EFTYPE; 2468 goto errout; 2469 } 2470 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2471 if (off != saveoff) { 2472 printf("wapbl_replay: corrupted records\n"); 2473 error = EFTYPE; 2474 goto errout; 2475 } 2476 } 2477 return 0; 2478 2479 errout: 2480 wapbl_blkhash_clear(wr); 2481 return error; 2482 } 2483 2484 static int 2485 wapbl_replay_get_inodes(struct wapbl_replay *wr) 2486 { 2487 off_t off; 2488 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2489 int logblklen = 1<<wch->wc_log_dev_bshift; 2490 int cnt= 0; 2491 2492 KDASSERT(wapbl_replay_isopen(wr)); 2493 2494 if (wr->wr_inodescnt == 0) 2495 return 0; 2496 2497 KASSERT(!wr->wr_inodes); 2498 2499 wr->wr_inodes = wapbl_malloc(wr->wr_inodescnt*sizeof(wr->wr_inodes[0])); 2500 2501 off = wr->wr_inodestail; 2502 2503 while (off != wr->wr_inodeshead) { 2504 struct wapbl_wc_null *wcn; 2505 int error; 2506 off_t saveoff = off; 2507 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2508 if (error) { 2509 wapbl_free(wr->wr_inodes); 2510 wr->wr_inodes = 0; 2511 return error; 2512 } 2513 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2514 switch (wcn->wc_type) { 2515 case WAPBL_WC_BLOCKS: 2516 case WAPBL_WC_REVOCATIONS: 2517 break; 2518 case WAPBL_WC_INODES: 2519 { 2520 struct wapbl_wc_inodelist *wc = 2521 (struct wapbl_wc_inodelist *)wr->wr_scratch; 2522 /* 2523 * Keep track of where we found this so we 2524 * can use it later 2525 */ 2526 if (wc->wc_clear) { 2527 cnt = 0; 2528 } 2529 /* This memcpy assumes that wr_inodes is 2530 * laid out the same as wc_inodes. */ 2531 memcpy(&wr->wr_inodes[cnt], wc->wc_inodes, 2532 wc->wc_inocnt*sizeof(wc->wc_inodes[0])); 2533 cnt += wc->wc_inocnt; 2534 } 2535 break; 2536 default: 2537 KASSERT(0); 2538 } 2539 off = saveoff; 2540 wapbl_circ_advance(wr, wcn->wc_len, &off); 2541 } 2542 KASSERT(cnt == wr->wr_inodescnt); 2543 return 0; 2544 } 2545 2546 #ifdef DEBUG 2547 int 2548 wapbl_replay_verify(struct wapbl_replay *wr, struct vnode *fsdevvp) 2549 { 2550 off_t off; 2551 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2552 int mismatchcnt = 0; 2553 int logblklen = 1<<wch->wc_log_dev_bshift; 2554 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2555 void *scratch1 = wapbl_malloc(MAXBSIZE); 2556 void *scratch2 = wapbl_malloc(MAXBSIZE); 2557 int error = 0; 2558 2559 KDASSERT(wapbl_replay_isopen(wr)); 2560 2561 off = wch->wc_tail; 2562 while (off != wch->wc_head) { 2563 struct wapbl_wc_null *wcn; 2564 #ifdef DEBUG 2565 off_t saveoff = off; 2566 #endif 2567 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2568 if (error) 2569 goto out; 2570 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2571 switch (wcn->wc_type) { 2572 case WAPBL_WC_BLOCKS: 2573 { 2574 struct wapbl_wc_blocklist *wc = 2575 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2576 int i; 2577 for (i = 0; i < wc->wc_blkcount; i++) { 2578 int foundcnt = 0; 2579 int dirtycnt = 0; 2580 int j, n; 2581 /* 2582 * Check each physical block into the 2583 * hashtable independently 2584 */ 2585 n = wc->wc_blocks[i].wc_dlen >> 2586 wch->wc_fs_dev_bshift; 2587 for (j = 0; j < n; j++) { 2588 struct wapbl_blk *wb = 2589 wapbl_blkhash_get(wr, 2590 wc->wc_blocks[i].wc_daddr + j); 2591 if (wb && (wb->wb_off == off)) { 2592 foundcnt++; 2593 error = 2594 wapbl_circ_read(wr, 2595 scratch1, fsblklen, 2596 &off); 2597 if (error) 2598 goto out; 2599 error = 2600 wapbl_read(scratch2, 2601 fsblklen, fsdevvp, 2602 wb->wb_blk); 2603 if (error) 2604 goto out; 2605 if (memcmp(scratch1, 2606 scratch2, 2607 fsblklen)) { 2608 printf( 2609 "wapbl_verify: mismatch block %"PRId64" at off %"PRIdMAX"\n", 2610 wb->wb_blk, (intmax_t)off); 2611 dirtycnt++; 2612 mismatchcnt++; 2613 } 2614 } else { 2615 wapbl_circ_advance(wr, 2616 fsblklen, &off); 2617 } 2618 } 2619 #if 0 2620 /* 2621 * If all of the blocks in an entry 2622 * are clean, then remove all of its 2623 * blocks from the hashtable since they 2624 * never will need replay. 2625 */ 2626 if ((foundcnt != 0) && 2627 (dirtycnt == 0)) { 2628 off = saveoff; 2629 wapbl_circ_advance(wr, 2630 logblklen, &off); 2631 for (j = 0; j < n; j++) { 2632 struct wapbl_blk *wb = 2633 wapbl_blkhash_get(wr, 2634 wc->wc_blocks[i].wc_daddr + j); 2635 if (wb && 2636 (wb->wb_off == off)) { 2637 wapbl_blkhash_rem(wr, wb->wb_blk); 2638 } 2639 wapbl_circ_advance(wr, 2640 fsblklen, &off); 2641 } 2642 } 2643 #endif 2644 } 2645 } 2646 break; 2647 case WAPBL_WC_REVOCATIONS: 2648 case WAPBL_WC_INODES: 2649 break; 2650 default: 2651 KASSERT(0); 2652 } 2653 #ifdef DEBUG 2654 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2655 KASSERT(off == saveoff); 2656 #endif 2657 } 2658 out: 2659 wapbl_free(scratch1); 2660 wapbl_free(scratch2); 2661 if (!error && mismatchcnt) 2662 error = EFTYPE; 2663 return error; 2664 } 2665 #endif 2666 2667 int 2668 wapbl_replay_write(struct wapbl_replay *wr, struct vnode *fsdevvp) 2669 { 2670 off_t off; 2671 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2672 int logblklen = 1<<wch->wc_log_dev_bshift; 2673 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2674 void *scratch1 = wapbl_malloc(MAXBSIZE); 2675 int error = 0; 2676 2677 KDASSERT(wapbl_replay_isopen(wr)); 2678 2679 /* 2680 * This parses the journal for replay, although it could 2681 * just as easily walk the hashtable instead. 2682 */ 2683 2684 off = wch->wc_tail; 2685 while (off != wch->wc_head) { 2686 struct wapbl_wc_null *wcn; 2687 #ifdef DEBUG 2688 off_t saveoff = off; 2689 #endif 2690 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2691 if (error) 2692 goto out; 2693 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2694 switch (wcn->wc_type) { 2695 case WAPBL_WC_BLOCKS: 2696 { 2697 struct wapbl_wc_blocklist *wc = 2698 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2699 int i; 2700 for (i = 0; i < wc->wc_blkcount; i++) { 2701 int j, n; 2702 /* 2703 * Check each physical block against 2704 * the hashtable independently 2705 */ 2706 n = wc->wc_blocks[i].wc_dlen >> 2707 wch->wc_fs_dev_bshift; 2708 for (j = 0; j < n; j++) { 2709 struct wapbl_blk *wb = 2710 wapbl_blkhash_get(wr, 2711 wc->wc_blocks[i].wc_daddr + j); 2712 if (wb && (wb->wb_off == off)) { 2713 error = wapbl_circ_read( 2714 wr, scratch1, 2715 fsblklen, &off); 2716 if (error) 2717 goto out; 2718 error = 2719 wapbl_write(scratch1, 2720 fsblklen, fsdevvp, 2721 wb->wb_blk); 2722 if (error) 2723 goto out; 2724 } else { 2725 wapbl_circ_advance(wr, 2726 fsblklen, &off); 2727 } 2728 } 2729 } 2730 } 2731 break; 2732 case WAPBL_WC_REVOCATIONS: 2733 case WAPBL_WC_INODES: 2734 break; 2735 default: 2736 KASSERT(0); 2737 } 2738 #ifdef DEBUG 2739 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2740 KASSERT(off == saveoff); 2741 #endif 2742 } 2743 out: 2744 wapbl_free(scratch1); 2745 return error; 2746 } 2747 2748 int 2749 wapbl_replay_read(struct wapbl_replay *wr, void *data, daddr_t blk, long len) 2750 { 2751 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2752 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2753 2754 KDASSERT(wapbl_replay_isopen(wr)); 2755 2756 KASSERT((len % fsblklen) == 0); 2757 2758 while (len != 0) { 2759 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2760 if (wb) { 2761 off_t off = wb->wb_off; 2762 int error; 2763 error = wapbl_circ_read(wr, data, fsblklen, &off); 2764 if (error) 2765 return error; 2766 } 2767 data = (uint8_t *)data + fsblklen; 2768 len -= fsblklen; 2769 blk++; 2770 } 2771 return 0; 2772 } 2773
Indexes created Fri Oct 03 02:09:57 GMT 2025