vfs_wapbl.c revision 1.3.8.6
1 /* $NetBSD: vfs_wapbl.c,v 1.3.8.6 2011/06/18 17:00:25 bouyer Exp $ */ 2 3 /*- 4 * Copyright (c) 2003, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Wasabi Systems, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * This implements file system independent write ahead filesystem logging. 34 */ 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.3.8.6 2011/06/18 17:00:25 bouyer Exp $"); 37 38 #include <sys/param.h> 39 40 #ifdef _KERNEL 41 #include <sys/param.h> 42 #include <sys/namei.h> 43 #include <sys/proc.h> 44 #include <sys/sysctl.h> 45 #include <sys/uio.h> 46 #include <sys/vnode.h> 47 #include <sys/file.h> 48 #include <sys/malloc.h> 49 #include <sys/resourcevar.h> 50 #include <sys/conf.h> 51 #include <sys/mount.h> 52 #include <sys/kernel.h> 53 #include <sys/kauth.h> 54 #include <sys/mutex.h> 55 #include <sys/atomic.h> 56 #include <sys/wapbl.h> 57 58 #if WAPBL_UVM_ALLOC 59 #include <uvm/uvm.h> 60 #endif 61 62 #include <miscfs/specfs/specdev.h> 63 64 MALLOC_JUSTDEFINE(M_WAPBL, "wapbl", "write-ahead physical block logging"); 65 #define wapbl_malloc(s) malloc((s), M_WAPBL, M_WAITOK) 66 #define wapbl_free(a) free((a), M_WAPBL) 67 #define wapbl_calloc(n, s) malloc((n)*(s), M_WAPBL, M_WAITOK | M_ZERO) 68 69 static struct sysctllog *wapbl_sysctl; 70 static int wapbl_flush_disk_cache = 1; 71 static int wapbl_verbose_commit = 0; 72 73 #else /* !_KERNEL */ 74 #include <assert.h> 75 #include <errno.h> 76 #include <stdio.h> 77 #include <stdbool.h> 78 #include <stdlib.h> 79 #include <string.h> 80 81 #include <sys/time.h> 82 #include <sys/wapbl.h> 83 84 #define KDASSERT(x) assert(x) 85 #define KASSERT(x) assert(x) 86 #define wapbl_malloc(s) malloc(s) 87 #define wapbl_free(a) free(a) 88 #define wapbl_calloc(n, s) calloc((n), (s)) 89 90 #endif /* !_KERNEL */ 91 92 /* 93 * INTERNAL DATA STRUCTURES 94 */ 95 96 /* 97 * This structure holds per-mount log information. 98 * 99 * Legend: a = atomic access only 100 * r = read-only after init 101 * l = rwlock held 102 * m = mutex held 103 * lm = rwlock held writing or mutex held 104 * u = unlocked access ok 105 * b = bufcache_lock held 106 */ 107 struct wapbl { 108 struct vnode *wl_logvp; /* r: log here */ 109 struct vnode *wl_devvp; /* r: log on this device */ 110 struct mount *wl_mount; /* r: mountpoint wl is associated with */ 111 daddr_t wl_logpbn; /* r: Physical block number of start of log */ 112 int wl_log_dev_bshift; /* r: logarithm of device block size of log 113 device */ 114 int wl_fs_dev_bshift; /* r: logarithm of device block size of 115 filesystem device */ 116 117 unsigned wl_lock_count; /* m: Count of transactions in progress */ 118 119 size_t wl_circ_size; /* r: Number of bytes in buffer of log */ 120 size_t wl_circ_off; /* r: Number of bytes reserved at start */ 121 122 size_t wl_bufcount_max; /* r: Number of buffers reserved for log */ 123 size_t wl_bufbytes_max; /* r: Number of buf bytes reserved for log */ 124 125 off_t wl_head; /* l: Byte offset of log head */ 126 off_t wl_tail; /* l: Byte offset of log tail */ 127 /* 128 * head == tail == 0 means log is empty 129 * head == tail != 0 means log is full 130 * see assertions in wapbl_advance() for other boundary conditions. 131 * only truncate moves the tail, except when flush sets it to 132 * wl_header_size only flush moves the head, except when truncate 133 * sets it to 0. 134 */ 135 136 struct wapbl_wc_header *wl_wc_header; /* l */ 137 void *wl_wc_scratch; /* l: scratch space (XXX: por que?!?) */ 138 139 kmutex_t wl_mtx; /* u: short-term lock */ 140 krwlock_t wl_rwlock; /* u: File system transaction lock */ 141 142 /* 143 * Must be held while accessing 144 * wl_count or wl_bufs or head or tail 145 */ 146 147 /* 148 * Callback called from within the flush routine to flush any extra 149 * bits. Note that flush may be skipped without calling this if 150 * there are no outstanding buffers in the transaction. 151 */ 152 wapbl_flush_fn_t wl_flush; /* r */ 153 wapbl_flush_fn_t wl_flush_abort;/* r */ 154 155 size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ 156 size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ 157 size_t wl_bcount; /* m: Total bcount of wl_bufs */ 158 159 LIST_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ 160 161 kcondvar_t wl_reclaimable_cv; /* m (obviously) */ 162 size_t wl_reclaimable_bytes; /* m: Amount of space available for 163 reclamation by truncate */ 164 int wl_error_count; /* m: # of wl_entries with errors */ 165 size_t wl_reserved_bytes; /* never truncate log smaller than this */ 166 167 #ifdef WAPBL_DEBUG_BUFBYTES 168 size_t wl_unsynced_bufbytes; /* Byte count of unsynced buffers */ 169 #endif 170 171 daddr_t *wl_deallocblks;/* lm: address of block */ 172 int *wl_dealloclens; /* lm: size of block */ 173 int wl_dealloccnt; /* lm: total count */ 174 int wl_dealloclim; /* l: max count */ 175 176 /* hashtable of inode numbers for allocated but unlinked inodes */ 177 /* synch ??? */ 178 LIST_HEAD(wapbl_ino_head, wapbl_ino) *wl_inohash; 179 u_long wl_inohashmask; 180 int wl_inohashcnt; 181 182 SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* On disk transaction 183 accounting */ 184 }; 185 186 #ifdef WAPBL_DEBUG_PRINT 187 int wapbl_debug_print = WAPBL_DEBUG_PRINT; 188 #endif 189 190 /****************************************************************/ 191 #ifdef _KERNEL 192 193 #ifdef WAPBL_DEBUG 194 struct wapbl *wapbl_debug_wl; 195 #endif 196 197 static int wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail); 198 static int wapbl_write_blocks(struct wapbl *wl, off_t *offp); 199 static int wapbl_write_revocations(struct wapbl *wl, off_t *offp); 200 static int wapbl_write_inodes(struct wapbl *wl, off_t *offp); 201 #endif /* _KERNEL */ 202 203 static int wapbl_replay_prescan(struct wapbl_replay *wr); 204 static int wapbl_replay_get_inodes(struct wapbl_replay *wr); 205 206 static __inline size_t wapbl_space_free(size_t avail, off_t head, 207 off_t tail); 208 static __inline size_t wapbl_space_used(size_t avail, off_t head, 209 off_t tail); 210 211 #ifdef _KERNEL 212 213 #define WAPBL_INODETRK_SIZE 83 214 static int wapbl_ino_pool_refcount; 215 static struct pool wapbl_ino_pool; 216 struct wapbl_ino { 217 LIST_ENTRY(wapbl_ino) wi_hash; 218 ino_t wi_ino; 219 mode_t wi_mode; 220 }; 221 222 static void wapbl_inodetrk_init(struct wapbl *wl, u_int size); 223 static void wapbl_inodetrk_free(struct wapbl *wl); 224 static struct wapbl_ino *wapbl_inodetrk_get(struct wapbl *wl, ino_t ino); 225 226 static size_t wapbl_transaction_len(struct wapbl *wl); 227 static __inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); 228 229 /* 230 * This is useful for debugging. If set, the log will 231 * only be truncated when necessary. 232 */ 233 int wapbl_lazy_truncate = 0; 234 235 struct wapbl_ops wapbl_ops = { 236 .wo_wapbl_discard = wapbl_discard, 237 .wo_wapbl_replay_isopen = wapbl_replay_isopen1, 238 .wo_wapbl_replay_read = wapbl_replay_read, 239 .wo_wapbl_add_buf = wapbl_add_buf, 240 .wo_wapbl_remove_buf = wapbl_remove_buf, 241 .wo_wapbl_resize_buf = wapbl_resize_buf, 242 .wo_wapbl_begin = wapbl_begin, 243 .wo_wapbl_end = wapbl_end, 244 .wo_wapbl_junlock_assert= wapbl_junlock_assert, 245 246 /* XXX: the following is only used to say "this is a wapbl buf" */ 247 .wo_wapbl_biodone = wapbl_biodone, 248 }; 249 250 void 251 wapbl_init() 252 { 253 int rv; 254 const struct sysctlnode *rnode, *cnode; 255 256 malloc_type_attach(M_WAPBL); 257 258 wapbl_sysctl = NULL; 259 260 rv = sysctl_createv(&wapbl_sysctl, 0, NULL, &rnode, 261 CTLFLAG_PERMANENT, 262 CTLTYPE_NODE, "vfs", NULL, 263 NULL, 0, NULL, 0, 264 CTL_VFS, CTL_EOL); 265 if (rv) 266 return; 267 268 rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &rnode, 269 CTLFLAG_PERMANENT, 270 CTLTYPE_NODE, "wapbl", 271 SYSCTL_DESCR("WAPBL journaling options"), 272 NULL, 0, NULL, 0, 273 CTL_CREATE, CTL_EOL); 274 if (rv) 275 return; 276 277 rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 278 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 279 CTLTYPE_INT, "flush_disk_cache", 280 SYSCTL_DESCR("flush disk cache"), 281 NULL, 0, &wapbl_flush_disk_cache, 0, 282 CTL_CREATE, CTL_EOL); 283 if (rv) 284 return; 285 286 rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 287 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 288 CTLTYPE_INT, "verbose_commit", 289 SYSCTL_DESCR("show time and size of wapbl log commits"), 290 NULL, 0, &wapbl_verbose_commit, 0, 291 CTL_CREATE, CTL_EOL); 292 } 293 294 int 295 wapbl_start(struct wapbl ** wlp, struct mount *mp, struct vnode *vp, 296 daddr_t off, size_t count, size_t blksize, struct wapbl_replay *wr, 297 wapbl_flush_fn_t flushfn, wapbl_flush_fn_t flushabortfn) 298 { 299 struct wapbl *wl; 300 struct vnode *devvp; 301 daddr_t logpbn; 302 int error; 303 int log_dev_bshift = DEV_BSHIFT; 304 int fs_dev_bshift = DEV_BSHIFT; 305 int run; 306 307 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_start: vp=%p off=%" PRId64 308 " count=%zu blksize=%zu\n", vp, off, count, blksize)); 309 310 if (log_dev_bshift > fs_dev_bshift) { 311 WAPBL_PRINTF(WAPBL_PRINT_OPEN, 312 ("wapbl: log device's block size cannot be larger " 313 "than filesystem's\n")); 314 /* 315 * Not currently implemented, although it could be if 316 * needed someday. 317 */ 318 return ENOSYS; 319 } 320 321 if (off < 0) 322 return EINVAL; 323 324 if (blksize < DEV_BSIZE) 325 return EINVAL; 326 if (blksize % DEV_BSIZE) 327 return EINVAL; 328 329 /* XXXTODO: verify that the full load is writable */ 330 331 /* 332 * XXX check for minimum log size 333 * minimum is governed by minimum amount of space 334 * to complete a transaction. (probably truncate) 335 */ 336 /* XXX for now pick something minimal */ 337 if ((count * blksize) < MAXPHYS) { 338 return ENOSPC; 339 } 340 341 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, &run)) != 0) { 342 return error; 343 } 344 345 wl = wapbl_calloc(1, sizeof(*wl)); 346 rw_init(&wl->wl_rwlock); 347 mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); 348 cv_init(&wl->wl_reclaimable_cv, "wapblrec"); 349 LIST_INIT(&wl->wl_bufs); 350 SIMPLEQ_INIT(&wl->wl_entries); 351 352 wl->wl_logvp = vp; 353 wl->wl_devvp = devvp; 354 wl->wl_mount = mp; 355 wl->wl_logpbn = logpbn; 356 wl->wl_log_dev_bshift = log_dev_bshift; 357 wl->wl_fs_dev_bshift = fs_dev_bshift; 358 359 wl->wl_flush = flushfn; 360 wl->wl_flush_abort = flushabortfn; 361 362 /* Reserve two log device blocks for the commit headers */ 363 wl->wl_circ_off = 2<<wl->wl_log_dev_bshift; 364 wl->wl_circ_size = ((count * blksize) - wl->wl_circ_off); 365 /* truncate the log usage to a multiple of log_dev_bshift */ 366 wl->wl_circ_size >>= wl->wl_log_dev_bshift; 367 wl->wl_circ_size <<= wl->wl_log_dev_bshift; 368 369 /* 370 * wl_bufbytes_max limits the size of the in memory transaction space. 371 * - Since buffers are allocated and accounted for in units of 372 * PAGE_SIZE it is required to be a multiple of PAGE_SIZE 373 * (i.e. 1<<PAGE_SHIFT) 374 * - Since the log device has to be written in units of 375 * 1<<wl_log_dev_bshift it is required to be a mulitple of 376 * 1<<wl_log_dev_bshift. 377 * - Since filesystem will provide data in units of 1<<wl_fs_dev_bshift, 378 * it is convenient to be a multiple of 1<<wl_fs_dev_bshift. 379 * Therefore it must be multiple of the least common multiple of those 380 * three quantities. Fortunately, all of those quantities are 381 * guaranteed to be a power of two, and the least common multiple of 382 * a set of numbers which are all powers of two is simply the maximum 383 * of those numbers. Finally, the maximum logarithm of a power of two 384 * is the same as the log of the maximum power of two. So we can do 385 * the following operations to size wl_bufbytes_max: 386 */ 387 388 /* XXX fix actual number of pages reserved per filesystem. */ 389 wl->wl_bufbytes_max = MIN(wl->wl_circ_size, buf_memcalc() / 2); 390 391 /* Round wl_bufbytes_max to the largest power of two constraint */ 392 wl->wl_bufbytes_max >>= PAGE_SHIFT; 393 wl->wl_bufbytes_max <<= PAGE_SHIFT; 394 wl->wl_bufbytes_max >>= wl->wl_log_dev_bshift; 395 wl->wl_bufbytes_max <<= wl->wl_log_dev_bshift; 396 wl->wl_bufbytes_max >>= wl->wl_fs_dev_bshift; 397 wl->wl_bufbytes_max <<= wl->wl_fs_dev_bshift; 398 399 /* XXX maybe use filesystem fragment size instead of 1024 */ 400 /* XXX fix actual number of buffers reserved per filesystem. */ 401 wl->wl_bufcount_max = (nbuf / 2) * 1024; 402 403 /* XXX tie this into resource estimation */ 404 wl->wl_dealloclim = wl->wl_bufbytes_max / mp->mnt_stat.f_bsize / 2; 405 406 #if WAPBL_UVM_ALLOC 407 wl->wl_deallocblks = (void *) uvm_km_zalloc(kernel_map, 408 round_page(sizeof(*wl->wl_deallocblks) * wl->wl_dealloclim)); 409 KASSERT(wl->wl_deallocblks != NULL); 410 wl->wl_dealloclens = (void *) uvm_km_zalloc(kernel_map, 411 round_page(sizeof(*wl->wl_dealloclens) * wl->wl_dealloclim)); 412 KASSERT(wl->wl_dealloclens != NULL); 413 #else 414 wl->wl_deallocblks = wapbl_malloc(sizeof(*wl->wl_deallocblks) * 415 wl->wl_dealloclim); 416 wl->wl_dealloclens = wapbl_malloc(sizeof(*wl->wl_dealloclens) * 417 wl->wl_dealloclim); 418 #endif 419 420 wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); 421 422 /* Initialize the commit header */ 423 { 424 struct wapbl_wc_header *wc; 425 size_t len = 1<<wl->wl_log_dev_bshift; 426 wc = wapbl_calloc(1, len); 427 wc->wc_type = WAPBL_WC_HEADER; 428 wc->wc_len = len; 429 wc->wc_circ_off = wl->wl_circ_off; 430 wc->wc_circ_size = wl->wl_circ_size; 431 /* XXX wc->wc_fsid */ 432 wc->wc_log_dev_bshift = wl->wl_log_dev_bshift; 433 wc->wc_fs_dev_bshift = wl->wl_fs_dev_bshift; 434 wl->wl_wc_header = wc; 435 wl->wl_wc_scratch = wapbl_malloc(len); 436 } 437 438 /* 439 * if there was an existing set of unlinked but 440 * allocated inodes, preserve it in the new 441 * log. 442 */ 443 if (wr && wr->wr_inodescnt) { 444 int i; 445 446 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 447 ("wapbl_start: reusing log with %d inodes\n", 448 wr->wr_inodescnt)); 449 450 /* 451 * Its only valid to reuse the replay log if its 452 * the same as the new log we just opened. 453 */ 454 KDASSERT(!wapbl_replay_isopen(wr)); 455 KASSERT(devvp->v_rdev == wr->wr_devvp->v_rdev); 456 KASSERT(logpbn == wr->wr_logpbn); 457 KASSERT(wl->wl_circ_size == wr->wr_wc_header.wc_circ_size); 458 KASSERT(wl->wl_circ_off == wr->wr_wc_header.wc_circ_off); 459 KASSERT(wl->wl_log_dev_bshift == 460 wr->wr_wc_header.wc_log_dev_bshift); 461 KASSERT(wl->wl_fs_dev_bshift == 462 wr->wr_wc_header.wc_fs_dev_bshift); 463 464 wl->wl_wc_header->wc_generation = 465 wr->wr_wc_header.wc_generation + 1; 466 467 for (i = 0; i < wr->wr_inodescnt; i++) 468 wapbl_register_inode(wl, wr->wr_inodes[i].wr_inumber, 469 wr->wr_inodes[i].wr_imode); 470 471 /* Make sure new transaction won't overwrite old inodes list */ 472 KDASSERT(wapbl_transaction_len(wl) <= 473 wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, 474 wr->wr_inodestail)); 475 476 wl->wl_head = wl->wl_tail = wr->wr_inodeshead; 477 wl->wl_reclaimable_bytes = wl->wl_reserved_bytes = 478 wapbl_transaction_len(wl); 479 480 error = wapbl_write_inodes(wl, &wl->wl_head); 481 if (error) 482 goto errout; 483 484 KASSERT(wl->wl_head != wl->wl_tail); 485 KASSERT(wl->wl_head != 0); 486 } 487 488 error = wapbl_write_commit(wl, wl->wl_head, wl->wl_tail); 489 if (error) { 490 goto errout; 491 } 492 493 *wlp = wl; 494 #if defined(WAPBL_DEBUG) 495 wapbl_debug_wl = wl; 496 #endif 497 498 return 0; 499 errout: 500 wapbl_discard(wl); 501 wapbl_free(wl->wl_wc_scratch); 502 wapbl_free(wl->wl_wc_header); 503 #if WAPBL_UVM_ALLOC 504 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_deallocblks, 505 round_page(sizeof(*wl->wl_deallocblks * 506 wl->wl_dealloclim))); 507 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_dealloclens, 508 round_page(sizeof(*wl->wl_dealloclens * 509 wl->wl_dealloclim))); 510 #else 511 wapbl_free(wl->wl_deallocblks); 512 wapbl_free(wl->wl_dealloclens); 513 #endif 514 wapbl_inodetrk_free(wl); 515 wapbl_free(wl); 516 517 return error; 518 } 519 520 /* 521 * Like wapbl_flush, only discards the transaction 522 * completely 523 */ 524 525 void 526 wapbl_discard(struct wapbl *wl) 527 { 528 struct wapbl_entry *we; 529 struct buf *bp; 530 int i; 531 532 /* 533 * XXX we may consider using upgrade here 534 * if we want to call flush from inside a transaction 535 */ 536 rw_enter(&wl->wl_rwlock, RW_WRITER); 537 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 538 wl->wl_dealloccnt); 539 540 #ifdef WAPBL_DEBUG_PRINT 541 { 542 struct wapbl_entry *we; 543 pid_t pid = -1; 544 lwpid_t lid = -1; 545 if (curproc) 546 pid = curproc->p_pid; 547 if (curlwp) 548 lid = curlwp->l_lid; 549 #ifdef WAPBL_DEBUG_BUFBYTES 550 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 551 ("wapbl_discard: thread %d.%d discarding " 552 "transaction\n" 553 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 554 "deallocs=%d inodes=%d\n" 555 "\terrcnt = %u, reclaimable=%zu reserved=%zu " 556 "unsynced=%zu\n", 557 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 558 wl->wl_bcount, wl->wl_dealloccnt, 559 wl->wl_inohashcnt, wl->wl_error_count, 560 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 561 wl->wl_unsynced_bufbytes)); 562 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 563 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 564 ("\tentry: bufcount = %zu, reclaimable = %zu, " 565 "error = %d, unsynced = %zu\n", 566 we->we_bufcount, we->we_reclaimable_bytes, 567 we->we_error, we->we_unsynced_bufbytes)); 568 } 569 #else /* !WAPBL_DEBUG_BUFBYTES */ 570 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 571 ("wapbl_discard: thread %d.%d discarding transaction\n" 572 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 573 "deallocs=%d inodes=%d\n" 574 "\terrcnt = %u, reclaimable=%zu reserved=%zu\n", 575 pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 576 wl->wl_bcount, wl->wl_dealloccnt, 577 wl->wl_inohashcnt, wl->wl_error_count, 578 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes)); 579 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 580 WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 581 ("\tentry: bufcount = %zu, reclaimable = %zu, " 582 "error = %d\n", 583 we->we_bufcount, we->we_reclaimable_bytes, 584 we->we_error)); 585 } 586 #endif /* !WAPBL_DEBUG_BUFBYTES */ 587 } 588 #endif /* WAPBL_DEBUG_PRINT */ 589 590 for (i = 0; i <= wl->wl_inohashmask; i++) { 591 struct wapbl_ino_head *wih; 592 struct wapbl_ino *wi; 593 594 wih = &wl->wl_inohash[i]; 595 while ((wi = LIST_FIRST(wih)) != NULL) { 596 LIST_REMOVE(wi, wi_hash); 597 pool_put(&wapbl_ino_pool, wi); 598 KASSERT(wl->wl_inohashcnt > 0); 599 wl->wl_inohashcnt--; 600 } 601 } 602 603 /* 604 * clean buffer list 605 */ 606 mutex_enter(&bufcache_lock); 607 mutex_enter(&wl->wl_mtx); 608 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 609 if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { 610 /* 611 * The buffer will be unlocked and 612 * removed from the transaction in brelse 613 */ 614 mutex_exit(&wl->wl_mtx); 615 brelsel(bp, 0); 616 mutex_enter(&wl->wl_mtx); 617 } 618 } 619 mutex_exit(&wl->wl_mtx); 620 mutex_exit(&bufcache_lock); 621 622 /* 623 * Remove references to this wl from wl_entries, free any which 624 * no longer have buffers, others will be freed in wapbl_biodone 625 * when they no longer have any buffers. 626 */ 627 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) != NULL) { 628 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 629 /* XXX should we be accumulating wl_error_count 630 * and increasing reclaimable bytes ? */ 631 we->we_wapbl = NULL; 632 if (we->we_bufcount == 0) { 633 #ifdef WAPBL_DEBUG_BUFBYTES 634 KASSERT(we->we_unsynced_bufbytes == 0); 635 #endif 636 wapbl_free(we); 637 } 638 } 639 640 /* Discard list of deallocs */ 641 wl->wl_dealloccnt = 0; 642 /* XXX should we clear wl_reserved_bytes? */ 643 644 KASSERT(wl->wl_bufbytes == 0); 645 KASSERT(wl->wl_bcount == 0); 646 KASSERT(wl->wl_bufcount == 0); 647 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 648 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 649 KASSERT(wl->wl_inohashcnt == 0); 650 651 rw_exit(&wl->wl_rwlock); 652 } 653 654 int 655 wapbl_stop(struct wapbl *wl, int force) 656 { 657 struct vnode *vp; 658 int error; 659 660 WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_stop called\n")); 661 error = wapbl_flush(wl, 1); 662 if (error) { 663 if (force) 664 wapbl_discard(wl); 665 else 666 return error; 667 } 668 669 /* Unlinked inodes persist after a flush */ 670 if (wl->wl_inohashcnt) { 671 if (force) { 672 wapbl_discard(wl); 673 } else { 674 return EBUSY; 675 } 676 } 677 678 KASSERT(wl->wl_bufbytes == 0); 679 KASSERT(wl->wl_bcount == 0); 680 KASSERT(wl->wl_bufcount == 0); 681 KASSERT(LIST_EMPTY(&wl->wl_bufs)); 682 KASSERT(wl->wl_dealloccnt == 0); 683 KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 684 KASSERT(wl->wl_inohashcnt == 0); 685 686 vp = wl->wl_logvp; 687 688 wapbl_free(wl->wl_wc_scratch); 689 wapbl_free(wl->wl_wc_header); 690 #if WAPBL_UVM_ALLOC 691 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_deallocblks, 692 round_page(sizeof(*wl->wl_deallocblks * 693 wl->wl_dealloclim))); 694 uvm_km_free_wakeup(kernel_map, (vaddr_t) wl->wl_dealloclens, 695 round_page(sizeof(*wl->wl_dealloclens * 696 wl->wl_dealloclim))); 697 #else 698 wapbl_free(wl->wl_deallocblks); 699 wapbl_free(wl->wl_dealloclens); 700 #endif 701 wapbl_inodetrk_free(wl); 702 703 cv_destroy(&wl->wl_reclaimable_cv); 704 mutex_destroy(&wl->wl_mtx); 705 rw_destroy(&wl->wl_rwlock); 706 wapbl_free(wl); 707 708 return 0; 709 } 710 711 static int 712 wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) 713 { 714 struct pstats *pstats = curlwp->l_proc->p_stats; 715 struct buf *bp; 716 int error; 717 718 KASSERT((flags & ~(B_WRITE | B_READ)) == 0); 719 KASSERT(devvp->v_type == VBLK); 720 721 if ((flags & (B_WRITE | B_READ)) == B_WRITE) { 722 mutex_enter(&devvp->v_interlock); 723 devvp->v_numoutput++; 724 mutex_exit(&devvp->v_interlock); 725 pstats->p_ru.ru_oublock++; 726 } else { 727 pstats->p_ru.ru_inblock++; 728 } 729 730 bp = getiobuf(devvp, true); 731 bp->b_flags = flags; 732 bp->b_cflags = BC_BUSY; /* silly & dubious */ 733 bp->b_dev = devvp->v_rdev; 734 bp->b_data = data; 735 bp->b_bufsize = bp->b_resid = bp->b_bcount = len; 736 bp->b_blkno = pbn; 737 738 WAPBL_PRINTF(WAPBL_PRINT_IO, 739 ("wapbl_doio: %s %d bytes at block %"PRId64" on dev 0x%x\n", 740 BUF_ISWRITE(bp) ? "write" : "read", bp->b_bcount, 741 bp->b_blkno, bp->b_dev)); 742 743 VOP_STRATEGY(devvp, bp); 744 745 error = biowait(bp); 746 putiobuf(bp); 747 748 if (error) { 749 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 750 ("wapbl_doio: %s %zu bytes at block %" PRId64 751 " on dev 0x%x failed with error %d\n", 752 (((flags & (B_WRITE | B_READ)) == B_WRITE) ? 753 "write" : "read"), 754 len, pbn, devvp->v_rdev, error)); 755 } 756 757 return error; 758 } 759 760 int 761 wapbl_write(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 762 { 763 764 return wapbl_doio(data, len, devvp, pbn, B_WRITE); 765 } 766 767 int 768 wapbl_read(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 769 { 770 771 return wapbl_doio(data, len, devvp, pbn, B_READ); 772 } 773 774 /* 775 * Off is byte offset returns new offset for next write 776 * handles log wraparound 777 */ 778 static int 779 wapbl_circ_write(struct wapbl *wl, void *data, size_t len, off_t *offp) 780 { 781 size_t slen; 782 off_t off = *offp; 783 int error; 784 785 KDASSERT(((len >> wl->wl_log_dev_bshift) << 786 wl->wl_log_dev_bshift) == len); 787 788 if (off < wl->wl_circ_off) 789 off = wl->wl_circ_off; 790 slen = wl->wl_circ_off + wl->wl_circ_size - off; 791 if (slen < len) { 792 error = wapbl_write(data, slen, wl->wl_devvp, 793 wl->wl_logpbn + (off >> wl->wl_log_dev_bshift)); 794 if (error) 795 return error; 796 data = (uint8_t *)data + slen; 797 len -= slen; 798 off = wl->wl_circ_off; 799 } 800 error = wapbl_write(data, len, wl->wl_devvp, 801 wl->wl_logpbn + (off >> wl->wl_log_dev_bshift)); 802 if (error) 803 return error; 804 off += len; 805 if (off >= wl->wl_circ_off + wl->wl_circ_size) 806 off = wl->wl_circ_off; 807 *offp = off; 808 return 0; 809 } 810 811 /****************************************************************/ 812 813 int 814 wapbl_begin(struct wapbl *wl, const char *file, int line) 815 { 816 int doflush; 817 unsigned lockcount; 818 819 KDASSERT(wl); 820 821 /* 822 * XXX this needs to be made much more sophisticated. 823 * perhaps each wapbl_begin could reserve a specified 824 * number of buffers and bytes. 825 */ 826 mutex_enter(&wl->wl_mtx); 827 lockcount = wl->wl_lock_count; 828 doflush = ((wl->wl_bufbytes + (lockcount * MAXPHYS)) > 829 wl->wl_bufbytes_max / 2) || 830 ((wl->wl_bufcount + (lockcount * 10)) > 831 wl->wl_bufcount_max / 2) || 832 (wapbl_transaction_len(wl) > wl->wl_circ_size / 2) || 833 (wl->wl_dealloccnt >= (wl->wl_dealloclim / 2)); 834 mutex_exit(&wl->wl_mtx); 835 836 if (doflush) { 837 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 838 ("force flush lockcnt=%d bufbytes=%zu " 839 "(max=%zu) bufcount=%zu (max=%zu) " 840 "dealloccnt %d (lim=%d)\n", 841 lockcount, wl->wl_bufbytes, 842 wl->wl_bufbytes_max, wl->wl_bufcount, 843 wl->wl_bufcount_max, 844 wl->wl_dealloccnt, wl->wl_dealloclim)); 845 } 846 847 if (doflush) { 848 int error = wapbl_flush(wl, 0); 849 if (error) 850 return error; 851 } 852 853 rw_enter(&wl->wl_rwlock, RW_READER); 854 mutex_enter(&wl->wl_mtx); 855 wl->wl_lock_count++; 856 mutex_exit(&wl->wl_mtx); 857 858 #if defined(WAPBL_DEBUG_PRINT) 859 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 860 ("wapbl_begin thread %d.%d with bufcount=%zu " 861 "bufbytes=%zu bcount=%zu at %s:%d\n", 862 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 863 wl->wl_bufbytes, wl->wl_bcount, file, line)); 864 #endif 865 866 return 0; 867 } 868 869 void 870 wapbl_end(struct wapbl *wl) 871 { 872 873 #if defined(WAPBL_DEBUG_PRINT) 874 WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 875 ("wapbl_end thread %d.%d with bufcount=%zu " 876 "bufbytes=%zu bcount=%zu\n", 877 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 878 wl->wl_bufbytes, wl->wl_bcount)); 879 #endif 880 881 mutex_enter(&wl->wl_mtx); 882 KASSERT(wl->wl_lock_count > 0); 883 wl->wl_lock_count--; 884 mutex_exit(&wl->wl_mtx); 885 886 rw_exit(&wl->wl_rwlock); 887 } 888 889 void 890 wapbl_add_buf(struct wapbl *wl, struct buf * bp) 891 { 892 893 KASSERT(bp->b_cflags & BC_BUSY); 894 KASSERT(bp->b_vp); 895 896 wapbl_jlock_assert(wl); 897 898 #if 0 899 /* 900 * XXX this might be an issue for swapfiles. 901 * see uvm_swap.c:1702 902 * 903 * XXX2 why require it then? leap of semantics? 904 */ 905 KASSERT((bp->b_cflags & BC_NOCACHE) == 0); 906 #endif 907 908 mutex_enter(&wl->wl_mtx); 909 if (bp->b_flags & B_LOCKED) { 910 LIST_REMOVE(bp, b_wapbllist); 911 WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, 912 ("wapbl_add_buf thread %d.%d re-adding buf %p " 913 "with %d bytes %d bcount\n", 914 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 915 bp->b_bcount)); 916 } else { 917 /* unlocked by dirty buffers shouldn't exist */ 918 KASSERT(!(bp->b_oflags & BO_DELWRI)); 919 wl->wl_bufbytes += bp->b_bufsize; 920 wl->wl_bcount += bp->b_bcount; 921 wl->wl_bufcount++; 922 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 923 ("wapbl_add_buf thread %d.%d adding buf %p " 924 "with %d bytes %d bcount\n", 925 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 926 bp->b_bcount)); 927 } 928 LIST_INSERT_HEAD(&wl->wl_bufs, bp, b_wapbllist); 929 mutex_exit(&wl->wl_mtx); 930 931 bp->b_flags |= B_LOCKED; 932 } 933 934 static void 935 wapbl_remove_buf_locked(struct wapbl * wl, struct buf *bp) 936 { 937 938 KASSERT(mutex_owned(&wl->wl_mtx)); 939 KASSERT(bp->b_cflags & BC_BUSY); 940 wapbl_jlock_assert(wl); 941 942 #if 0 943 /* 944 * XXX this might be an issue for swapfiles. 945 * see uvm_swap.c:1725 946 * 947 * XXXdeux: see above 948 */ 949 KASSERT((bp->b_flags & BC_NOCACHE) == 0); 950 #endif 951 KASSERT(bp->b_flags & B_LOCKED); 952 953 WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 954 ("wapbl_remove_buf thread %d.%d removing buf %p with " 955 "%d bytes %d bcount\n", 956 curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, bp->b_bcount)); 957 958 KASSERT(wl->wl_bufbytes >= bp->b_bufsize); 959 wl->wl_bufbytes -= bp->b_bufsize; 960 KASSERT(wl->wl_bcount >= bp->b_bcount); 961 wl->wl_bcount -= bp->b_bcount; 962 KASSERT(wl->wl_bufcount > 0); 963 wl->wl_bufcount--; 964 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 965 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 966 LIST_REMOVE(bp, b_wapbllist); 967 968 bp->b_flags &= ~B_LOCKED; 969 } 970 971 /* called from brelsel() in vfs_bio among other places */ 972 void 973 wapbl_remove_buf(struct wapbl * wl, struct buf *bp) 974 { 975 976 mutex_enter(&wl->wl_mtx); 977 wapbl_remove_buf_locked(wl, bp); 978 mutex_exit(&wl->wl_mtx); 979 } 980 981 void 982 wapbl_resize_buf(struct wapbl *wl, struct buf *bp, long oldsz, long oldcnt) 983 { 984 985 KASSERT(bp->b_cflags & BC_BUSY); 986 987 /* 988 * XXX: why does this depend on B_LOCKED? otherwise the buf 989 * is not for a transaction? if so, why is this called in the 990 * first place? 991 */ 992 if (bp->b_flags & B_LOCKED) { 993 mutex_enter(&wl->wl_mtx); 994 wl->wl_bufbytes += bp->b_bufsize - oldsz; 995 wl->wl_bcount += bp->b_bcount - oldcnt; 996 mutex_exit(&wl->wl_mtx); 997 } 998 } 999 1000 #endif /* _KERNEL */ 1001 1002 /****************************************************************/ 1003 /* Some utility inlines */ 1004 1005 /* This is used to advance the pointer at old to new value at old+delta */ 1006 static __inline off_t 1007 wapbl_advance(size_t size, size_t off, off_t old, size_t delta) 1008 { 1009 off_t new; 1010 1011 /* Define acceptable ranges for inputs. */ 1012 KASSERT(delta <= size); 1013 KASSERT((old == 0) || (old >= off)); 1014 KASSERT(old < (size + off)); 1015 1016 if ((old == 0) && (delta != 0)) 1017 new = off + delta; 1018 else if ((old + delta) < (size + off)) 1019 new = old + delta; 1020 else 1021 new = (old + delta) - size; 1022 1023 /* Note some interesting axioms */ 1024 KASSERT((delta != 0) || (new == old)); 1025 KASSERT((delta == 0) || (new != 0)); 1026 KASSERT((delta != (size)) || (new == old)); 1027 1028 /* Define acceptable ranges for output. */ 1029 KASSERT((new == 0) || (new >= off)); 1030 KASSERT(new < (size + off)); 1031 return new; 1032 } 1033 1034 static __inline size_t 1035 wapbl_space_used(size_t avail, off_t head, off_t tail) 1036 { 1037 1038 if (tail == 0) { 1039 KASSERT(head == 0); 1040 return 0; 1041 } 1042 return ((head + (avail - 1) - tail) % avail) + 1; 1043 } 1044 1045 static __inline size_t 1046 wapbl_space_free(size_t avail, off_t head, off_t tail) 1047 { 1048 1049 return avail - wapbl_space_used(avail, head, tail); 1050 } 1051 1052 static __inline void 1053 wapbl_advance_head(size_t size, size_t off, size_t delta, off_t *headp, 1054 off_t *tailp) 1055 { 1056 off_t head = *headp; 1057 off_t tail = *tailp; 1058 1059 KASSERT(delta <= wapbl_space_free(size, head, tail)); 1060 head = wapbl_advance(size, off, head, delta); 1061 if ((tail == 0) && (head != 0)) 1062 tail = off; 1063 *headp = head; 1064 *tailp = tail; 1065 } 1066 1067 static __inline void 1068 wapbl_advance_tail(size_t size, size_t off, size_t delta, off_t *headp, 1069 off_t *tailp) 1070 { 1071 off_t head = *headp; 1072 off_t tail = *tailp; 1073 1074 KASSERT(delta <= wapbl_space_used(size, head, tail)); 1075 tail = wapbl_advance(size, off, tail, delta); 1076 if (head == tail) { 1077 head = tail = 0; 1078 } 1079 *headp = head; 1080 *tailp = tail; 1081 } 1082 1083 #ifdef _KERNEL 1084 1085 /****************************************************************/ 1086 1087 /* 1088 * Remove transactions whose buffers are completely flushed to disk. 1089 * Will block until at least minfree space is available. 1090 * only intended to be called from inside wapbl_flush and therefore 1091 * does not protect against commit races with itself or with flush. 1092 */ 1093 static int 1094 wapbl_truncate(struct wapbl *wl, size_t minfree, int waitonly) 1095 { 1096 size_t delta; 1097 size_t avail; 1098 off_t head; 1099 off_t tail; 1100 int error = 0; 1101 1102 KASSERT(minfree <= (wl->wl_circ_size - wl->wl_reserved_bytes)); 1103 KASSERT(rw_write_held(&wl->wl_rwlock)); 1104 1105 mutex_enter(&wl->wl_mtx); 1106 1107 /* 1108 * First check to see if we have to do a commit 1109 * at all. 1110 */ 1111 avail = wapbl_space_free(wl->wl_circ_size, wl->wl_head, wl->wl_tail); 1112 if (minfree < avail) { 1113 mutex_exit(&wl->wl_mtx); 1114 return 0; 1115 } 1116 minfree -= avail; 1117 while ((wl->wl_error_count == 0) && 1118 (wl->wl_reclaimable_bytes < minfree)) { 1119 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1120 ("wapbl_truncate: sleeping on %p wl=%p bytes=%zd " 1121 "minfree=%zd\n", 1122 &wl->wl_reclaimable_bytes, wl, wl->wl_reclaimable_bytes, 1123 minfree)); 1124 1125 cv_wait(&wl->wl_reclaimable_cv, &wl->wl_mtx); 1126 } 1127 if (wl->wl_reclaimable_bytes < minfree) { 1128 KASSERT(wl->wl_error_count); 1129 /* XXX maybe get actual error from buffer instead someday? */ 1130 error = EIO; 1131 } 1132 head = wl->wl_head; 1133 tail = wl->wl_tail; 1134 delta = wl->wl_reclaimable_bytes; 1135 1136 /* If all of of the entries are flushed, then be sure to keep 1137 * the reserved bytes reserved. Watch out for discarded transactions, 1138 * which could leave more bytes reserved than are reclaimable. 1139 */ 1140 if (SIMPLEQ_EMPTY(&wl->wl_entries) && 1141 (delta >= wl->wl_reserved_bytes)) { 1142 delta -= wl->wl_reserved_bytes; 1143 } 1144 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, &head, 1145 &tail); 1146 KDASSERT(wl->wl_reserved_bytes <= 1147 wapbl_space_used(wl->wl_circ_size, head, tail)); 1148 mutex_exit(&wl->wl_mtx); 1149 1150 if (error) 1151 return error; 1152 1153 if (waitonly) 1154 return 0; 1155 1156 /* 1157 * This is where head, tail and delta are unprotected 1158 * from races against itself or flush. This is ok since 1159 * we only call this routine from inside flush itself. 1160 * 1161 * XXX: how can it race against itself when accessed only 1162 * from behind the write-locked rwlock? 1163 */ 1164 error = wapbl_write_commit(wl, head, tail); 1165 if (error) 1166 return error; 1167 1168 wl->wl_head = head; 1169 wl->wl_tail = tail; 1170 1171 mutex_enter(&wl->wl_mtx); 1172 KASSERT(wl->wl_reclaimable_bytes >= delta); 1173 wl->wl_reclaimable_bytes -= delta; 1174 mutex_exit(&wl->wl_mtx); 1175 WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1176 ("wapbl_truncate thread %d.%d truncating %zu bytes\n", 1177 curproc->p_pid, curlwp->l_lid, delta)); 1178 1179 return 0; 1180 } 1181 1182 /****************************************************************/ 1183 1184 void 1185 wapbl_biodone(struct buf *bp) 1186 { 1187 struct wapbl_entry *we = bp->b_private; 1188 struct wapbl *wl = we->we_wapbl; 1189 1190 /* 1191 * Handle possible flushing of buffers after log has been 1192 * decomissioned. 1193 */ 1194 if (!wl) { 1195 KASSERT(we->we_bufcount > 0); 1196 we->we_bufcount--; 1197 #ifdef WAPBL_DEBUG_BUFBYTES 1198 KASSERT(we->we_unsynced_bufbytes >= bp->b_bufsize); 1199 we->we_unsynced_bufbytes -= bp->b_bufsize; 1200 #endif 1201 1202 if (we->we_bufcount == 0) { 1203 #ifdef WAPBL_DEBUG_BUFBYTES 1204 KASSERT(we->we_unsynced_bufbytes == 0); 1205 #endif 1206 wapbl_free(we); 1207 } 1208 1209 brelse(bp, 0); 1210 return; 1211 } 1212 1213 #ifdef ohbother 1214 KDASSERT(bp->b_flags & B_DONE); 1215 KDASSERT(!(bp->b_flags & B_DELWRI)); 1216 KDASSERT(bp->b_flags & B_ASYNC); 1217 KDASSERT(bp->b_flags & B_BUSY); 1218 KDASSERT(!(bp->b_flags & B_LOCKED)); 1219 KDASSERT(!(bp->b_flags & B_READ)); 1220 KDASSERT(!(bp->b_flags & B_INVAL)); 1221 KDASSERT(!(bp->b_flags & B_NOCACHE)); 1222 #endif 1223 1224 if (bp->b_error) { 1225 #ifdef notyet /* Can't currently handle possible dirty buffer reuse */ 1226 XXXpooka: interfaces not fully updated 1227 Note: this was not enabled in the original patch 1228 against netbsd4 either. I don't know if comment 1229 above is true or not. 1230 1231 /* 1232 * If an error occurs, report the error and leave the 1233 * buffer as a delayed write on the LRU queue. 1234 * restarting the write would likely result in 1235 * an error spinloop, so let it be done harmlessly 1236 * by the syncer. 1237 */ 1238 bp->b_flags &= ~(B_DONE); 1239 simple_unlock(&bp->b_interlock); 1240 1241 if (we->we_error == 0) { 1242 mutex_enter(&wl->wl_mtx); 1243 wl->wl_error_count++; 1244 mutex_exit(&wl->wl_mtx); 1245 cv_broadcast(&wl->wl_reclaimable_cv); 1246 } 1247 we->we_error = bp->b_error; 1248 bp->b_error = 0; 1249 brelse(bp); 1250 return; 1251 #else 1252 /* For now, just mark the log permanently errored out */ 1253 1254 mutex_enter(&wl->wl_mtx); 1255 if (wl->wl_error_count == 0) { 1256 wl->wl_error_count++; 1257 cv_broadcast(&wl->wl_reclaimable_cv); 1258 } 1259 mutex_exit(&wl->wl_mtx); 1260 #endif 1261 } 1262 1263 mutex_enter(&wl->wl_mtx); 1264 1265 KASSERT(we->we_bufcount > 0); 1266 we->we_bufcount--; 1267 #ifdef WAPBL_DEBUG_BUFBYTES 1268 KASSERT(we->we_unsynced_bufbytes >= bp->b_bufsize); 1269 we->we_unsynced_bufbytes -= bp->b_bufsize; 1270 KASSERT(wl->wl_unsynced_bufbytes >= bp->b_bufsize); 1271 wl->wl_unsynced_bufbytes -= bp->b_bufsize; 1272 #endif 1273 1274 /* 1275 * If the current transaction can be reclaimed, start 1276 * at the beginning and reclaim any consecutive reclaimable 1277 * transactions. If we successfully reclaim anything, 1278 * then wakeup anyone waiting for the reclaim. 1279 */ 1280 if (we->we_bufcount == 0) { 1281 size_t delta = 0; 1282 int errcnt = 0; 1283 #ifdef WAPBL_DEBUG_BUFBYTES 1284 KDASSERT(we->we_unsynced_bufbytes == 0); 1285 #endif 1286 /* 1287 * clear any posted error, since the buffer it came from 1288 * has successfully flushed by now 1289 */ 1290 while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) && 1291 (we->we_bufcount == 0)) { 1292 delta += we->we_reclaimable_bytes; 1293 if (we->we_error) 1294 errcnt++; 1295 SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 1296 wapbl_free(we); 1297 } 1298 1299 if (delta) { 1300 wl->wl_reclaimable_bytes += delta; 1301 KASSERT(wl->wl_error_count >= errcnt); 1302 wl->wl_error_count -= errcnt; 1303 cv_broadcast(&wl->wl_reclaimable_cv); 1304 } 1305 } 1306 1307 mutex_exit(&wl->wl_mtx); 1308 brelse(bp, 0); 1309 } 1310 1311 /* 1312 * Write transactions to disk + start I/O for contents 1313 */ 1314 int 1315 wapbl_flush(struct wapbl *wl, int waitfor) 1316 { 1317 struct buf *bp; 1318 struct wapbl_entry *we; 1319 off_t off; 1320 off_t head; 1321 off_t tail; 1322 size_t delta = 0; 1323 size_t flushsize; 1324 size_t reserved; 1325 int error = 0; 1326 1327 /* 1328 * Do a quick check to see if a full flush can be skipped 1329 * This assumes that the flush callback does not need to be called 1330 * unless there are other outstanding bufs. 1331 */ 1332 if (!waitfor) { 1333 size_t nbufs; 1334 mutex_enter(&wl->wl_mtx); /* XXX need mutex here to 1335 protect the KASSERTS */ 1336 nbufs = wl->wl_bufcount; 1337 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1338 KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1339 mutex_exit(&wl->wl_mtx); 1340 if (nbufs == 0) 1341 return 0; 1342 } 1343 1344 /* 1345 * XXX we may consider using LK_UPGRADE here 1346 * if we want to call flush from inside a transaction 1347 */ 1348 rw_enter(&wl->wl_rwlock, RW_WRITER); 1349 wl->wl_flush(wl->wl_mount, wl->wl_deallocblks, wl->wl_dealloclens, 1350 wl->wl_dealloccnt); 1351 1352 /* 1353 * Now that we are fully locked and flushed, 1354 * do another check for nothing to do. 1355 */ 1356 if (wl->wl_bufcount == 0) { 1357 goto out; 1358 } 1359 1360 #if 0 1361 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1362 ("wapbl_flush thread %d.%d flushing entries with " 1363 "bufcount=%zu bufbytes=%zu\n", 1364 curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1365 wl->wl_bufbytes)); 1366 #endif 1367 1368 /* Calculate amount of space needed to flush */ 1369 flushsize = wapbl_transaction_len(wl); 1370 if (wapbl_verbose_commit) { 1371 struct timespec ts; 1372 getnanotime(&ts); 1373 printf("%s: %lld.%06ld this transaction = %zu bytes\n", 1374 __func__, (long long)ts.tv_sec, 1375 (long)ts.tv_nsec, flushsize); 1376 } 1377 1378 if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) { 1379 /* 1380 * XXX this could be handled more gracefully, perhaps place 1381 * only a partial transaction in the log and allow the 1382 * remaining to flush without the protection of the journal. 1383 */ 1384 panic("wapbl_flush: current transaction too big to flush\n"); 1385 } 1386 1387 error = wapbl_truncate(wl, flushsize, 0); 1388 if (error) 1389 goto out2; 1390 1391 off = wl->wl_head; 1392 KASSERT((off == 0) || ((off >= wl->wl_circ_off) && 1393 (off < wl->wl_circ_off + wl->wl_circ_size))); 1394 error = wapbl_write_blocks(wl, &off); 1395 if (error) 1396 goto out2; 1397 error = wapbl_write_revocations(wl, &off); 1398 if (error) 1399 goto out2; 1400 error = wapbl_write_inodes(wl, &off); 1401 if (error) 1402 goto out2; 1403 1404 reserved = 0; 1405 if (wl->wl_inohashcnt) 1406 reserved = wapbl_transaction_inodes_len(wl); 1407 1408 head = wl->wl_head; 1409 tail = wl->wl_tail; 1410 1411 wapbl_advance_head(wl->wl_circ_size, wl->wl_circ_off, flushsize, 1412 &head, &tail); 1413 #ifdef WAPBL_DEBUG 1414 if (head != off) { 1415 panic("lost head! head=%"PRIdMAX" tail=%" PRIdMAX 1416 " off=%"PRIdMAX" flush=%zu\n", 1417 (intmax_t)head, (intmax_t)tail, (intmax_t)off, 1418 flushsize); 1419 } 1420 #else 1421 KASSERT(head == off); 1422 #endif 1423 1424 /* Opportunistically move the tail forward if we can */ 1425 if (!wapbl_lazy_truncate) { 1426 mutex_enter(&wl->wl_mtx); 1427 delta = wl->wl_reclaimable_bytes; 1428 mutex_exit(&wl->wl_mtx); 1429 wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, 1430 &head, &tail); 1431 } 1432 1433 error = wapbl_write_commit(wl, head, tail); 1434 if (error) 1435 goto out2; 1436 1437 /* poolme? or kmemme? */ 1438 we = wapbl_calloc(1, sizeof(*we)); 1439 1440 #ifdef WAPBL_DEBUG_BUFBYTES 1441 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1442 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1443 " unsynced=%zu" 1444 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1445 "inodes=%d\n", 1446 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1447 wapbl_space_used(wl->wl_circ_size, head, tail), 1448 wl->wl_unsynced_bufbytes, wl->wl_bufcount, 1449 wl->wl_bufbytes, wl->wl_bcount, wl->wl_dealloccnt, 1450 wl->wl_inohashcnt)); 1451 #else 1452 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1453 ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1454 "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1455 "inodes=%d\n", 1456 curproc->p_pid, curlwp->l_lid, flushsize, delta, 1457 wapbl_space_used(wl->wl_circ_size, head, tail), 1458 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1459 wl->wl_dealloccnt, wl->wl_inohashcnt)); 1460 #endif 1461 1462 1463 mutex_enter(&bufcache_lock); 1464 mutex_enter(&wl->wl_mtx); 1465 1466 wl->wl_reserved_bytes = reserved; 1467 wl->wl_head = head; 1468 wl->wl_tail = tail; 1469 KASSERT(wl->wl_reclaimable_bytes >= delta); 1470 wl->wl_reclaimable_bytes -= delta; 1471 wl->wl_dealloccnt = 0; 1472 #ifdef WAPBL_DEBUG_BUFBYTES 1473 wl->wl_unsynced_bufbytes += wl->wl_bufbytes; 1474 #endif 1475 1476 we->we_wapbl = wl; 1477 we->we_bufcount = wl->wl_bufcount; 1478 #ifdef WAPBL_DEBUG_BUFBYTES 1479 we->we_unsynced_bufbytes = wl->wl_bufbytes; 1480 #endif 1481 we->we_reclaimable_bytes = flushsize; 1482 we->we_error = 0; 1483 SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); 1484 1485 /* 1486 * this flushes bufs in reverse order than they were queued 1487 * it shouldn't matter, but if we care we could use TAILQ instead. 1488 * XXX Note they will get put on the lru queue when they flush 1489 * so we might actually want to change this to preserve order. 1490 */ 1491 while ((bp = LIST_FIRST(&wl->wl_bufs)) != NULL) { 1492 if (bbusy(bp, 0, 0, &wl->wl_mtx)) { 1493 continue; 1494 } 1495 bp->b_iodone = wapbl_biodone; 1496 bp->b_private = we; 1497 bremfree(bp); 1498 wapbl_remove_buf_locked(wl, bp); 1499 mutex_exit(&wl->wl_mtx); 1500 mutex_exit(&bufcache_lock); 1501 bawrite(bp); 1502 mutex_enter(&bufcache_lock); 1503 mutex_enter(&wl->wl_mtx); 1504 } 1505 mutex_exit(&wl->wl_mtx); 1506 mutex_exit(&bufcache_lock); 1507 1508 #if 0 1509 WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1510 ("wapbl_flush thread %d.%d done flushing entries...\n", 1511 curproc->p_pid, curlwp->l_lid)); 1512 #endif 1513 1514 out: 1515 1516 /* 1517 * If the waitfor flag is set, don't return until everything is 1518 * fully flushed and the on disk log is empty. 1519 */ 1520 if (waitfor) { 1521 error = wapbl_truncate(wl, wl->wl_circ_size - 1522 wl->wl_reserved_bytes, wapbl_lazy_truncate); 1523 } 1524 1525 out2: 1526 if (error) { 1527 wl->wl_flush_abort(wl->wl_mount, wl->wl_deallocblks, 1528 wl->wl_dealloclens, wl->wl_dealloccnt); 1529 } 1530 1531 #ifdef WAPBL_DEBUG_PRINT 1532 if (error) { 1533 pid_t pid = -1; 1534 lwpid_t lid = -1; 1535 if (curproc) 1536 pid = curproc->p_pid; 1537 if (curlwp) 1538 lid = curlwp->l_lid; 1539 mutex_enter(&wl->wl_mtx); 1540 #ifdef WAPBL_DEBUG_BUFBYTES 1541 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1542 ("wapbl_flush: thread %d.%d aborted flush: " 1543 "error = %d\n" 1544 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1545 "deallocs=%d inodes=%d\n" 1546 "\terrcnt = %d, reclaimable=%zu reserved=%zu " 1547 "unsynced=%zu\n", 1548 pid, lid, error, wl->wl_bufcount, 1549 wl->wl_bufbytes, wl->wl_bcount, 1550 wl->wl_dealloccnt, wl->wl_inohashcnt, 1551 wl->wl_error_count, wl->wl_reclaimable_bytes, 1552 wl->wl_reserved_bytes, wl->wl_unsynced_bufbytes)); 1553 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1554 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1555 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1556 "error = %d, unsynced = %zu\n", 1557 we->we_bufcount, we->we_reclaimable_bytes, 1558 we->we_error, we->we_unsynced_bufbytes)); 1559 } 1560 #else 1561 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1562 ("wapbl_flush: thread %d.%d aborted flush: " 1563 "error = %d\n" 1564 "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1565 "deallocs=%d inodes=%d\n" 1566 "\terrcnt = %d, reclaimable=%zu reserved=%zu\n", 1567 pid, lid, error, wl->wl_bufcount, 1568 wl->wl_bufbytes, wl->wl_bcount, 1569 wl->wl_dealloccnt, wl->wl_inohashcnt, 1570 wl->wl_error_count, wl->wl_reclaimable_bytes, 1571 wl->wl_reserved_bytes)); 1572 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1573 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1574 ("\tentry: bufcount = %zu, reclaimable = %zu, " 1575 "error = %d\n", we->we_bufcount, 1576 we->we_reclaimable_bytes, we->we_error)); 1577 } 1578 #endif 1579 mutex_exit(&wl->wl_mtx); 1580 } 1581 #endif 1582 1583 rw_exit(&wl->wl_rwlock); 1584 return error; 1585 } 1586 1587 /****************************************************************/ 1588 1589 void 1590 wapbl_jlock_assert(struct wapbl *wl) 1591 { 1592 1593 KASSERT(rw_lock_held(&wl->wl_rwlock)); 1594 } 1595 1596 void 1597 wapbl_junlock_assert(struct wapbl *wl) 1598 { 1599 1600 KASSERT(!rw_write_held(&wl->wl_rwlock)); 1601 } 1602 1603 /****************************************************************/ 1604 1605 /* locks missing */ 1606 void 1607 wapbl_print(struct wapbl *wl, 1608 int full, 1609 void (*pr)(const char *, ...)) 1610 { 1611 struct buf *bp; 1612 struct wapbl_entry *we; 1613 (*pr)("wapbl %p", wl); 1614 (*pr)("\nlogvp = %p, devvp = %p, logpbn = %"PRId64"\n", 1615 wl->wl_logvp, wl->wl_devvp, wl->wl_logpbn); 1616 (*pr)("circ = %zu, header = %zu, head = %"PRIdMAX" tail = %"PRIdMAX"\n", 1617 wl->wl_circ_size, wl->wl_circ_off, 1618 (intmax_t)wl->wl_head, (intmax_t)wl->wl_tail); 1619 (*pr)("fs_dev_bshift = %d, log_dev_bshift = %d\n", 1620 wl->wl_log_dev_bshift, wl->wl_fs_dev_bshift); 1621 #ifdef WAPBL_DEBUG_BUFBYTES 1622 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1623 "reserved = %zu errcnt = %d unsynced = %zu\n", 1624 wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1625 wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1626 wl->wl_error_count, wl->wl_unsynced_bufbytes); 1627 #else 1628 (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 1629 "reserved = %zu errcnt = %d\n", wl->wl_bufcount, wl->wl_bufbytes, 1630 wl->wl_bcount, wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 1631 wl->wl_error_count); 1632 #endif 1633 (*pr)("\tdealloccnt = %d, dealloclim = %d\n", 1634 wl->wl_dealloccnt, wl->wl_dealloclim); 1635 (*pr)("\tinohashcnt = %d, inohashmask = 0x%08x\n", 1636 wl->wl_inohashcnt, wl->wl_inohashmask); 1637 (*pr)("entries:\n"); 1638 SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1639 #ifdef WAPBL_DEBUG_BUFBYTES 1640 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d, " 1641 "unsynced = %zu\n", 1642 we->we_bufcount, we->we_reclaimable_bytes, 1643 we->we_error, we->we_unsynced_bufbytes); 1644 #else 1645 (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d\n", 1646 we->we_bufcount, we->we_reclaimable_bytes, we->we_error); 1647 #endif 1648 } 1649 if (full) { 1650 int cnt = 0; 1651 (*pr)("bufs ="); 1652 LIST_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { 1653 if (!LIST_NEXT(bp, b_wapbllist)) { 1654 (*pr)(" %p", bp); 1655 } else if ((++cnt % 6) == 0) { 1656 (*pr)(" %p,\n\t", bp); 1657 } else { 1658 (*pr)(" %p,", bp); 1659 } 1660 } 1661 (*pr)("\n"); 1662 1663 (*pr)("dealloced blks = "); 1664 { 1665 int i; 1666 cnt = 0; 1667 for (i = 0; i < wl->wl_dealloccnt; i++) { 1668 (*pr)(" %"PRId64":%d,", 1669 wl->wl_deallocblks[i], 1670 wl->wl_dealloclens[i]); 1671 if ((++cnt % 4) == 0) { 1672 (*pr)("\n\t"); 1673 } 1674 } 1675 } 1676 (*pr)("\n"); 1677 1678 (*pr)("registered inodes = "); 1679 { 1680 int i; 1681 cnt = 0; 1682 for (i = 0; i <= wl->wl_inohashmask; i++) { 1683 struct wapbl_ino_head *wih; 1684 struct wapbl_ino *wi; 1685 1686 wih = &wl->wl_inohash[i]; 1687 LIST_FOREACH(wi, wih, wi_hash) { 1688 if (wi->wi_ino == 0) 1689 continue; 1690 (*pr)(" %"PRId32"/0%06"PRIo32",", 1691 wi->wi_ino, wi->wi_mode); 1692 if ((++cnt % 4) == 0) { 1693 (*pr)("\n\t"); 1694 } 1695 } 1696 } 1697 (*pr)("\n"); 1698 } 1699 } 1700 } 1701 1702 #if defined(WAPBL_DEBUG) || defined(DDB) 1703 void 1704 wapbl_dump(struct wapbl *wl) 1705 { 1706 #if defined(WAPBL_DEBUG) 1707 if (!wl) 1708 wl = wapbl_debug_wl; 1709 #endif 1710 if (!wl) 1711 return; 1712 wapbl_print(wl, 1, printf); 1713 } 1714 #endif 1715 1716 /****************************************************************/ 1717 1718 void 1719 wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len) 1720 { 1721 1722 wapbl_jlock_assert(wl); 1723 1724 mutex_enter(&wl->wl_mtx); 1725 /* XXX should eventually instead tie this into resource estimation */ 1726 /* 1727 * XXX this panic needs locking/mutex analysis and the 1728 * ability to cope with the failure. 1729 */ 1730 /* XXX this XXX doesn't have enough XXX */ 1731 if (__predict_false(wl->wl_dealloccnt >= wl->wl_dealloclim)) 1732 panic("wapbl_register_deallocation: out of resources"); 1733 1734 wl->wl_deallocblks[wl->wl_dealloccnt] = blk; 1735 wl->wl_dealloclens[wl->wl_dealloccnt] = len; 1736 wl->wl_dealloccnt++; 1737 WAPBL_PRINTF(WAPBL_PRINT_ALLOC, 1738 ("wapbl_register_deallocation: blk=%"PRId64" len=%d\n", blk, len)); 1739 mutex_exit(&wl->wl_mtx); 1740 } 1741 1742 /****************************************************************/ 1743 1744 static void 1745 wapbl_inodetrk_init(struct wapbl *wl, u_int size) 1746 { 1747 1748 wl->wl_inohash = hashinit(size, HASH_LIST, true, &wl->wl_inohashmask); 1749 if (atomic_inc_uint_nv(&wapbl_ino_pool_refcount) == 1) { 1750 pool_init(&wapbl_ino_pool, sizeof(struct wapbl_ino), 0, 0, 0, 1751 "wapblinopl", &pool_allocator_nointr, IPL_NONE); 1752 } 1753 } 1754 1755 static void 1756 wapbl_inodetrk_free(struct wapbl *wl) 1757 { 1758 1759 /* XXX this KASSERT needs locking/mutex analysis */ 1760 KASSERT(wl->wl_inohashcnt == 0); 1761 hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask); 1762 if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) { 1763 pool_destroy(&wapbl_ino_pool); 1764 } 1765 } 1766 1767 static struct wapbl_ino * 1768 wapbl_inodetrk_get(struct wapbl *wl, ino_t ino) 1769 { 1770 struct wapbl_ino_head *wih; 1771 struct wapbl_ino *wi; 1772 1773 KASSERT(mutex_owned(&wl->wl_mtx)); 1774 1775 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1776 LIST_FOREACH(wi, wih, wi_hash) { 1777 if (ino == wi->wi_ino) 1778 return wi; 1779 } 1780 return 0; 1781 } 1782 1783 void 1784 wapbl_register_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1785 { 1786 struct wapbl_ino_head *wih; 1787 struct wapbl_ino *wi; 1788 1789 wi = pool_get(&wapbl_ino_pool, PR_WAITOK); 1790 1791 mutex_enter(&wl->wl_mtx); 1792 if (wapbl_inodetrk_get(wl, ino) == NULL) { 1793 wi->wi_ino = ino; 1794 wi->wi_mode = mode; 1795 wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 1796 LIST_INSERT_HEAD(wih, wi, wi_hash); 1797 wl->wl_inohashcnt++; 1798 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1799 ("wapbl_register_inode: ino=%"PRId64"\n", ino)); 1800 mutex_exit(&wl->wl_mtx); 1801 } else { 1802 mutex_exit(&wl->wl_mtx); 1803 pool_put(&wapbl_ino_pool, wi); 1804 } 1805 } 1806 1807 void 1808 wapbl_unregister_inode(struct wapbl *wl, ino_t ino, mode_t mode) 1809 { 1810 struct wapbl_ino *wi; 1811 1812 mutex_enter(&wl->wl_mtx); 1813 wi = wapbl_inodetrk_get(wl, ino); 1814 if (wi) { 1815 WAPBL_PRINTF(WAPBL_PRINT_INODE, 1816 ("wapbl_unregister_inode: ino=%"PRId64"\n", ino)); 1817 KASSERT(wl->wl_inohashcnt > 0); 1818 wl->wl_inohashcnt--; 1819 LIST_REMOVE(wi, wi_hash); 1820 mutex_exit(&wl->wl_mtx); 1821 1822 pool_put(&wapbl_ino_pool, wi); 1823 } else { 1824 mutex_exit(&wl->wl_mtx); 1825 } 1826 } 1827 1828 /****************************************************************/ 1829 1830 static __inline size_t 1831 wapbl_transaction_inodes_len(struct wapbl *wl) 1832 { 1833 int blocklen = 1<<wl->wl_log_dev_bshift; 1834 int iph; 1835 1836 /* Calculate number of inodes described in a inodelist header */ 1837 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 1838 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 1839 1840 KASSERT(iph > 0); 1841 1842 return MAX(1, howmany(wl->wl_inohashcnt, iph))*blocklen; 1843 } 1844 1845 1846 /* Calculate amount of space a transaction will take on disk */ 1847 static size_t 1848 wapbl_transaction_len(struct wapbl *wl) 1849 { 1850 int blocklen = 1<<wl->wl_log_dev_bshift; 1851 size_t len; 1852 int bph; 1853 1854 /* Calculate number of blocks described in a blocklist header */ 1855 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1856 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1857 1858 KASSERT(bph > 0); 1859 1860 len = wl->wl_bcount; 1861 len += howmany(wl->wl_bufcount, bph)*blocklen; 1862 len += howmany(wl->wl_dealloccnt, bph)*blocklen; 1863 len += wapbl_transaction_inodes_len(wl); 1864 1865 return len; 1866 } 1867 1868 /* 1869 * Perform commit operation 1870 * 1871 * Note that generation number incrementation needs to 1872 * be protected against racing with other invocations 1873 * of wapbl_commit. This is ok since this routine 1874 * is only invoked from wapbl_flush 1875 */ 1876 static int 1877 wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail) 1878 { 1879 struct wapbl_wc_header *wc = wl->wl_wc_header; 1880 struct timespec ts; 1881 int error; 1882 int force = 1; 1883 1884 if (wapbl_flush_disk_cache) { 1885 /* XXX Calc checksum here, instead we do this for now */ 1886 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, 1887 FWRITE, FSCRED); 1888 if (error) { 1889 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1890 ("wapbl_write_commit: DIOCCACHESYNC on dev 0x%x " 1891 "returned %d\n", wl->wl_devvp->v_rdev, error)); 1892 } 1893 } 1894 1895 wc->wc_head = head; 1896 wc->wc_tail = tail; 1897 wc->wc_checksum = 0; 1898 wc->wc_version = 1; 1899 getnanotime(&ts); 1900 wc->wc_time = ts.tv_sec;; 1901 wc->wc_timensec = ts.tv_nsec; 1902 1903 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 1904 ("wapbl_write_commit: head = %"PRIdMAX "tail = %"PRIdMAX"\n", 1905 (intmax_t)head, (intmax_t)tail)); 1906 1907 /* 1908 * XXX if generation will rollover, then first zero 1909 * over second commit header before trying to write both headers. 1910 */ 1911 1912 error = wapbl_write(wc, wc->wc_len, wl->wl_devvp, 1913 wl->wl_logpbn + wc->wc_generation % 2); 1914 if (error) 1915 return error; 1916 1917 if (wapbl_flush_disk_cache) { 1918 error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, 1919 FWRITE, FSCRED); 1920 if (error) { 1921 WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1922 ("wapbl_write_commit: DIOCCACHESYNC on dev 0x%x " 1923 "returned %d\n", wl->wl_devvp->v_rdev, error)); 1924 } 1925 } 1926 1927 /* 1928 * If the generation number was zero, write it out a second time. 1929 * This handles initialization and generation number rollover 1930 */ 1931 if (wc->wc_generation++ == 0) { 1932 error = wapbl_write_commit(wl, head, tail); 1933 /* 1934 * This panic should be able to be removed if we do the 1935 * zero'ing mentioned above, and we are certain to roll 1936 * back generation number on failure. 1937 */ 1938 if (error) 1939 panic("wapbl_write_commit: error writing duplicate " 1940 "log header: %d\n", error); 1941 } 1942 return 0; 1943 } 1944 1945 /* Returns new offset value */ 1946 static int 1947 wapbl_write_blocks(struct wapbl *wl, off_t *offp) 1948 { 1949 struct wapbl_wc_blocklist *wc = 1950 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 1951 int blocklen = 1<<wl->wl_log_dev_bshift; 1952 int bph; 1953 struct buf *bp; 1954 off_t off = *offp; 1955 int error; 1956 1957 KASSERT(rw_write_held(&wl->wl_rwlock)); 1958 1959 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 1960 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 1961 1962 bp = LIST_FIRST(&wl->wl_bufs); 1963 1964 while (bp) { 1965 int cnt; 1966 struct buf *obp = bp; 1967 1968 KASSERT(bp->b_flags & B_LOCKED); 1969 1970 wc->wc_type = WAPBL_WC_BLOCKS; 1971 wc->wc_len = blocklen; 1972 wc->wc_blkcount = 0; 1973 while (bp && (wc->wc_blkcount < bph)) { 1974 /* 1975 * Make sure all the physical block numbers are up to 1976 * date. If this is not always true on a given 1977 * filesystem, then VOP_BMAP must be called. We 1978 * could call VOP_BMAP here, or else in the filesystem 1979 * specific flush callback, although neither of those 1980 * solutions allow us to take the vnode lock. If a 1981 * filesystem requires that we must take the vnode lock 1982 * to call VOP_BMAP, then we can probably do it in 1983 * bwrite when the vnode lock should already be held 1984 * by the invoking code. 1985 */ 1986 KASSERT((bp->b_vp->v_type == VBLK) || 1987 (bp->b_blkno != bp->b_lblkno)); 1988 KASSERT(bp->b_blkno > 0); 1989 1990 wc->wc_blocks[wc->wc_blkcount].wc_daddr = bp->b_blkno; 1991 wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; 1992 wc->wc_len += bp->b_bcount; 1993 wc->wc_blkcount++; 1994 bp = LIST_NEXT(bp, b_wapbllist); 1995 } 1996 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 1997 ("wapbl_write_blocks: len = %u off = %"PRIdMAX"\n", 1998 wc->wc_len, (intmax_t)off)); 1999 2000 error = wapbl_circ_write(wl, wc, blocklen, &off); 2001 if (error) 2002 return error; 2003 bp = obp; 2004 cnt = 0; 2005 while (bp && (cnt++ < bph)) { 2006 error = wapbl_circ_write(wl, bp->b_data, 2007 bp->b_bcount, &off); 2008 if (error) 2009 return error; 2010 bp = LIST_NEXT(bp, b_wapbllist); 2011 } 2012 } 2013 *offp = off; 2014 return 0; 2015 } 2016 2017 static int 2018 wapbl_write_revocations(struct wapbl *wl, off_t *offp) 2019 { 2020 struct wapbl_wc_blocklist *wc = 2021 (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 2022 int i; 2023 int blocklen = 1<<wl->wl_log_dev_bshift; 2024 int bph; 2025 off_t off = *offp; 2026 int error; 2027 2028 if (wl->wl_dealloccnt == 0) 2029 return 0; 2030 2031 bph = (blocklen - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 2032 sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 2033 2034 i = 0; 2035 while (i < wl->wl_dealloccnt) { 2036 wc->wc_type = WAPBL_WC_REVOCATIONS; 2037 wc->wc_len = blocklen; 2038 wc->wc_blkcount = 0; 2039 while ((i < wl->wl_dealloccnt) && (wc->wc_blkcount < bph)) { 2040 wc->wc_blocks[wc->wc_blkcount].wc_daddr = 2041 wl->wl_deallocblks[i]; 2042 wc->wc_blocks[wc->wc_blkcount].wc_dlen = 2043 wl->wl_dealloclens[i]; 2044 wc->wc_blkcount++; 2045 i++; 2046 } 2047 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2048 ("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", 2049 wc->wc_len, (intmax_t)off)); 2050 error = wapbl_circ_write(wl, wc, blocklen, &off); 2051 if (error) 2052 return error; 2053 } 2054 *offp = off; 2055 return 0; 2056 } 2057 2058 static int 2059 wapbl_write_inodes(struct wapbl *wl, off_t *offp) 2060 { 2061 struct wapbl_wc_inodelist *wc = 2062 (struct wapbl_wc_inodelist *)wl->wl_wc_scratch; 2063 int i; 2064 int blocklen = 1<<wl->wl_log_dev_bshift; 2065 off_t off = *offp; 2066 int error; 2067 2068 struct wapbl_ino_head *wih; 2069 struct wapbl_ino *wi; 2070 int iph; 2071 2072 iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2073 sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2074 2075 i = 0; 2076 wih = &wl->wl_inohash[0]; 2077 wi = 0; 2078 do { 2079 wc->wc_type = WAPBL_WC_INODES; 2080 wc->wc_len = blocklen; 2081 wc->wc_inocnt = 0; 2082 wc->wc_clear = (i == 0); 2083 while ((i < wl->wl_inohashcnt) && (wc->wc_inocnt < iph)) { 2084 while (!wi) { 2085 KASSERT((wih - &wl->wl_inohash[0]) 2086 <= wl->wl_inohashmask); 2087 wi = LIST_FIRST(wih++); 2088 } 2089 wc->wc_inodes[wc->wc_inocnt].wc_inumber = wi->wi_ino; 2090 wc->wc_inodes[wc->wc_inocnt].wc_imode = wi->wi_mode; 2091 wc->wc_inocnt++; 2092 i++; 2093 wi = LIST_NEXT(wi, wi_hash); 2094 } 2095 WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2096 ("wapbl_write_inodes: len = %u off = %"PRIdMAX"\n", 2097 wc->wc_len, (intmax_t)off)); 2098 error = wapbl_circ_write(wl, wc, blocklen, &off); 2099 if (error) 2100 return error; 2101 } while (i < wl->wl_inohashcnt); 2102 2103 *offp = off; 2104 return 0; 2105 } 2106 2107 #endif /* _KERNEL */ 2108 2109 /****************************************************************/ 2110 2111 #ifdef _KERNEL 2112 static struct pool wapbl_blk_pool; 2113 static int wapbl_blk_pool_refcount; 2114 #endif 2115 struct wapbl_blk { 2116 LIST_ENTRY(wapbl_blk) wb_hash; 2117 daddr_t wb_blk; 2118 off_t wb_off; /* Offset of this block in the log */ 2119 }; 2120 #define WAPBL_BLKPOOL_MIN 83 2121 2122 static void 2123 wapbl_blkhash_init(struct wapbl_replay *wr, u_int size) 2124 { 2125 if (size < WAPBL_BLKPOOL_MIN) 2126 size = WAPBL_BLKPOOL_MIN; 2127 KASSERT(wr->wr_blkhash == 0); 2128 #ifdef _KERNEL 2129 wr->wr_blkhash = hashinit(size, HASH_LIST, true, &wr->wr_blkhashmask); 2130 if (atomic_inc_uint_nv(&wapbl_blk_pool_refcount) == 1) { 2131 pool_init(&wapbl_blk_pool, sizeof(struct wapbl_blk), 0, 0, 0, 2132 "wapblblkpl", &pool_allocator_nointr, IPL_NONE); 2133 } 2134 #else /* ! _KERNEL */ 2135 /* Manually implement hashinit */ 2136 { 2137 int i; 2138 unsigned long hashsize; 2139 for (hashsize = 1; hashsize < size; hashsize <<= 1) 2140 continue; 2141 wr->wr_blkhash = wapbl_malloc(hashsize * sizeof(*wr->wr_blkhash)); 2142 for (i = 0; i < hashsize; i++) 2143 LIST_INIT(&wr->wr_blkhash[i]); 2144 wr->wr_blkhashmask = hashsize - 1; 2145 } 2146 #endif /* ! _KERNEL */ 2147 } 2148 2149 static void 2150 wapbl_blkhash_free(struct wapbl_replay *wr) 2151 { 2152 KASSERT(wr->wr_blkhashcnt == 0); 2153 #ifdef _KERNEL 2154 hashdone(wr->wr_blkhash, HASH_LIST, wr->wr_blkhashmask); 2155 if (atomic_dec_uint_nv(&wapbl_blk_pool_refcount) == 0) { 2156 pool_destroy(&wapbl_blk_pool); 2157 } 2158 #else /* ! _KERNEL */ 2159 wapbl_free(wr->wr_blkhash); 2160 #endif /* ! _KERNEL */ 2161 } 2162 2163 static struct wapbl_blk * 2164 wapbl_blkhash_get(struct wapbl_replay *wr, daddr_t blk) 2165 { 2166 struct wapbl_blk_head *wbh; 2167 struct wapbl_blk *wb; 2168 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2169 LIST_FOREACH(wb, wbh, wb_hash) { 2170 if (blk == wb->wb_blk) 2171 return wb; 2172 } 2173 return 0; 2174 } 2175 2176 static void 2177 wapbl_blkhash_ins(struct wapbl_replay *wr, daddr_t blk, off_t off) 2178 { 2179 struct wapbl_blk_head *wbh; 2180 struct wapbl_blk *wb; 2181 wb = wapbl_blkhash_get(wr, blk); 2182 if (wb) { 2183 KASSERT(wb->wb_blk == blk); 2184 wb->wb_off = off; 2185 } else { 2186 #ifdef _KERNEL 2187 wb = pool_get(&wapbl_blk_pool, PR_WAITOK); 2188 #else /* ! _KERNEL */ 2189 wb = wapbl_malloc(sizeof(*wb)); 2190 #endif /* ! _KERNEL */ 2191 wb->wb_blk = blk; 2192 wb->wb_off = off; 2193 wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2194 LIST_INSERT_HEAD(wbh, wb, wb_hash); 2195 wr->wr_blkhashcnt++; 2196 } 2197 } 2198 2199 static void 2200 wapbl_blkhash_rem(struct wapbl_replay *wr, daddr_t blk) 2201 { 2202 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2203 if (wb) { 2204 KASSERT(wr->wr_blkhashcnt > 0); 2205 wr->wr_blkhashcnt--; 2206 LIST_REMOVE(wb, wb_hash); 2207 #ifdef _KERNEL 2208 pool_put(&wapbl_blk_pool, wb); 2209 #else /* ! _KERNEL */ 2210 wapbl_free(wb); 2211 #endif /* ! _KERNEL */ 2212 } 2213 } 2214 2215 static void 2216 wapbl_blkhash_clear(struct wapbl_replay *wr) 2217 { 2218 int i; 2219 for (i = 0; i <= wr->wr_blkhashmask; i++) { 2220 struct wapbl_blk *wb; 2221 2222 while ((wb = LIST_FIRST(&wr->wr_blkhash[i]))) { 2223 KASSERT(wr->wr_blkhashcnt > 0); 2224 wr->wr_blkhashcnt--; 2225 LIST_REMOVE(wb, wb_hash); 2226 #ifdef _KERNEL 2227 pool_put(&wapbl_blk_pool, wb); 2228 #else /* ! _KERNEL */ 2229 wapbl_free(wb); 2230 #endif /* ! _KERNEL */ 2231 } 2232 } 2233 KASSERT(wr->wr_blkhashcnt == 0); 2234 } 2235 2236 /****************************************************************/ 2237 2238 static int 2239 wapbl_circ_read(struct wapbl_replay *wr, void *data, size_t len, off_t *offp) 2240 { 2241 size_t slen; 2242 struct wapbl_wc_header *wc = &wr->wr_wc_header; 2243 off_t off = *offp; 2244 int error; 2245 2246 KASSERT(((len >> wc->wc_log_dev_bshift) << 2247 wc->wc_log_dev_bshift) == len); 2248 if (off < wc->wc_circ_off) 2249 off = wc->wc_circ_off; 2250 slen = wc->wc_circ_off + wc->wc_circ_size - off; 2251 if (slen < len) { 2252 error = wapbl_read(data, slen, wr->wr_devvp, 2253 wr->wr_logpbn + (off >> wc->wc_log_dev_bshift)); 2254 if (error) 2255 return error; 2256 data = (uint8_t *)data + slen; 2257 len -= slen; 2258 off = wc->wc_circ_off; 2259 } 2260 error = wapbl_read(data, len, wr->wr_devvp, 2261 wr->wr_logpbn + (off >> wc->wc_log_dev_bshift)); 2262 if (error) 2263 return error; 2264 off += len; 2265 if (off >= wc->wc_circ_off + wc->wc_circ_size) 2266 off = wc->wc_circ_off; 2267 *offp = off; 2268 return 0; 2269 } 2270 2271 static void 2272 wapbl_circ_advance(struct wapbl_replay *wr, size_t len, off_t *offp) 2273 { 2274 size_t slen; 2275 struct wapbl_wc_header *wc = &wr->wr_wc_header; 2276 off_t off = *offp; 2277 2278 KASSERT(((len >> wc->wc_log_dev_bshift) << 2279 wc->wc_log_dev_bshift) == len); 2280 2281 if (off < wc->wc_circ_off) 2282 off = wc->wc_circ_off; 2283 slen = wc->wc_circ_off + wc->wc_circ_size - off; 2284 if (slen < len) { 2285 len -= slen; 2286 off = wc->wc_circ_off; 2287 } 2288 off += len; 2289 if (off >= wc->wc_circ_off + wc->wc_circ_size) 2290 off = wc->wc_circ_off; 2291 *offp = off; 2292 } 2293 2294 /****************************************************************/ 2295 2296 int 2297 wapbl_replay_start(struct wapbl_replay **wrp, struct vnode *vp, 2298 daddr_t off, size_t count, size_t blksize) 2299 { 2300 struct wapbl_replay *wr; 2301 int error; 2302 struct vnode *devvp; 2303 daddr_t logpbn; 2304 uint8_t *scratch; 2305 struct wapbl_wc_header *wch; 2306 struct wapbl_wc_header *wch2; 2307 /* Use this until we read the actual log header */ 2308 int log_dev_bshift = DEV_BSHIFT; 2309 size_t used; 2310 2311 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2312 ("wapbl_replay_start: vp=%p off=%"PRId64 " count=%zu blksize=%zu\n", 2313 vp, off, count, blksize)); 2314 2315 if (off < 0) 2316 return EINVAL; 2317 2318 if (blksize < DEV_BSIZE) 2319 return EINVAL; 2320 if (blksize % DEV_BSIZE) 2321 return EINVAL; 2322 2323 #ifdef _KERNEL 2324 #if 0 2325 /* XXX vp->v_size isn't reliably set for VBLK devices, 2326 * especially root. However, we might still want to verify 2327 * that the full load is readable */ 2328 if ((off + count) * blksize > vp->v_size) 2329 return EINVAL; 2330 #endif 2331 2332 if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, 0)) != 0) { 2333 return error; 2334 } 2335 #else /* ! _KERNEL */ 2336 devvp = vp; 2337 logpbn = off; 2338 #endif /* ! _KERNEL */ 2339 2340 scratch = wapbl_malloc(MAXBSIZE); 2341 2342 error = wapbl_read(scratch, 2<<log_dev_bshift, devvp, logpbn); 2343 if (error) 2344 goto errout; 2345 2346 wch = (struct wapbl_wc_header *)scratch; 2347 wch2 = 2348 (struct wapbl_wc_header *)(scratch + (1<<log_dev_bshift)); 2349 /* XXX verify checksums and magic numbers */ 2350 if (wch->wc_type != WAPBL_WC_HEADER) { 2351 printf("Unrecognized wapbl magic: 0x%08x\n", wch->wc_type); 2352 error = EFTYPE; 2353 goto errout; 2354 } 2355 2356 if (wch2->wc_generation > wch->wc_generation) 2357 wch = wch2; 2358 2359 wr = wapbl_calloc(1, sizeof(*wr)); 2360 2361 wr->wr_logvp = vp; 2362 wr->wr_devvp = devvp; 2363 wr->wr_logpbn = logpbn; 2364 2365 wr->wr_scratch = scratch; 2366 2367 memcpy(&wr->wr_wc_header, wch, sizeof(wr->wr_wc_header)); 2368 2369 used = wapbl_space_used(wch->wc_circ_size, wch->wc_head, wch->wc_tail); 2370 2371 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2372 ("wapbl_replay: head=%"PRId64" tail=%"PRId64" off=%"PRId64 2373 " len=%"PRId64" used=%zu\n", 2374 wch->wc_head, wch->wc_tail, wch->wc_circ_off, 2375 wch->wc_circ_size, used)); 2376 2377 wapbl_blkhash_init(wr, (used >> wch->wc_fs_dev_bshift)); 2378 error = wapbl_replay_prescan(wr); 2379 if (error) { 2380 wapbl_replay_stop(wr); 2381 wapbl_replay_free(wr); 2382 return error; 2383 } 2384 2385 error = wapbl_replay_get_inodes(wr); 2386 if (error) { 2387 wapbl_replay_stop(wr); 2388 wapbl_replay_free(wr); 2389 return error; 2390 } 2391 2392 *wrp = wr; 2393 return 0; 2394 2395 errout: 2396 wapbl_free(scratch); 2397 return error; 2398 } 2399 2400 void 2401 wapbl_replay_stop(struct wapbl_replay *wr) 2402 { 2403 2404 WAPBL_PRINTF(WAPBL_PRINT_REPLAY, ("wapbl_replay_stop called\n")); 2405 2406 KDASSERT(wapbl_replay_isopen(wr)); 2407 2408 wapbl_free(wr->wr_scratch); 2409 wr->wr_scratch = 0; 2410 2411 wr->wr_logvp = 0; 2412 2413 wapbl_blkhash_clear(wr); 2414 wapbl_blkhash_free(wr); 2415 } 2416 2417 void 2418 wapbl_replay_free(struct wapbl_replay *wr) 2419 { 2420 2421 KDASSERT(!wapbl_replay_isopen(wr)); 2422 2423 if (wr->wr_inodes) 2424 wapbl_free(wr->wr_inodes); 2425 wapbl_free(wr); 2426 } 2427 2428 int 2429 wapbl_replay_isopen1(struct wapbl_replay *wr) 2430 { 2431 2432 return wapbl_replay_isopen(wr); 2433 } 2434 2435 static int 2436 wapbl_replay_prescan(struct wapbl_replay *wr) 2437 { 2438 off_t off; 2439 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2440 int error; 2441 2442 int logblklen = 1<<wch->wc_log_dev_bshift; 2443 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2444 2445 wapbl_blkhash_clear(wr); 2446 2447 off = wch->wc_tail; 2448 while (off != wch->wc_head) { 2449 struct wapbl_wc_null *wcn; 2450 off_t saveoff = off; 2451 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2452 if (error) 2453 goto errout; 2454 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2455 switch (wcn->wc_type) { 2456 case WAPBL_WC_BLOCKS: 2457 { 2458 struct wapbl_wc_blocklist *wc = 2459 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2460 int i; 2461 for (i = 0; i < wc->wc_blkcount; i++) { 2462 int j, n; 2463 /* 2464 * Enter each physical block into the 2465 * hashtable independently 2466 */ 2467 n = wc->wc_blocks[i].wc_dlen >> 2468 wch->wc_fs_dev_bshift; 2469 for (j = 0; j < n; j++) { 2470 wapbl_blkhash_ins(wr, 2471 wc->wc_blocks[i].wc_daddr + j, 2472 off); 2473 wapbl_circ_advance(wr, 2474 fsblklen, &off); 2475 } 2476 } 2477 } 2478 break; 2479 2480 case WAPBL_WC_REVOCATIONS: 2481 { 2482 struct wapbl_wc_blocklist *wc = 2483 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2484 int i; 2485 for (i = 0; i < wc->wc_blkcount; i++) { 2486 int j, n; 2487 /* 2488 * Remove any blocks found from the 2489 * hashtable 2490 */ 2491 n = wc->wc_blocks[i].wc_dlen >> 2492 wch->wc_fs_dev_bshift; 2493 for (j = 0; j < n; j++) { 2494 wapbl_blkhash_rem(wr, 2495 wc->wc_blocks[i].wc_daddr + j); 2496 } 2497 } 2498 } 2499 break; 2500 2501 case WAPBL_WC_INODES: 2502 { 2503 struct wapbl_wc_inodelist *wc = 2504 (struct wapbl_wc_inodelist *)wr->wr_scratch; 2505 /* 2506 * Keep track of where we found this so we 2507 * can use it later 2508 */ 2509 if (wc->wc_clear) { 2510 wr->wr_inodestail = saveoff; 2511 wr->wr_inodescnt = 0; 2512 } 2513 if (wr->wr_inodestail) 2514 wr->wr_inodeshead = off; 2515 wr->wr_inodescnt += wc->wc_inocnt; 2516 } 2517 break; 2518 default: 2519 printf("Unrecognized wapbl type: 0x%08x\n", 2520 wcn->wc_type); 2521 error = EFTYPE; 2522 goto errout; 2523 } 2524 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2525 if (off != saveoff) { 2526 printf("wapbl_replay: corrupted records\n"); 2527 error = EFTYPE; 2528 goto errout; 2529 } 2530 } 2531 return 0; 2532 2533 errout: 2534 wapbl_blkhash_clear(wr); 2535 return error; 2536 } 2537 2538 static int 2539 wapbl_replay_get_inodes(struct wapbl_replay *wr) 2540 { 2541 off_t off; 2542 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2543 int logblklen = 1<<wch->wc_log_dev_bshift; 2544 int cnt= 0; 2545 2546 KDASSERT(wapbl_replay_isopen(wr)); 2547 2548 if (wr->wr_inodescnt == 0) 2549 return 0; 2550 2551 KASSERT(!wr->wr_inodes); 2552 2553 wr->wr_inodes = wapbl_malloc(wr->wr_inodescnt*sizeof(wr->wr_inodes[0])); 2554 2555 off = wr->wr_inodestail; 2556 2557 while (off != wr->wr_inodeshead) { 2558 struct wapbl_wc_null *wcn; 2559 int error; 2560 off_t saveoff = off; 2561 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2562 if (error) { 2563 wapbl_free(wr->wr_inodes); 2564 wr->wr_inodes = 0; 2565 return error; 2566 } 2567 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2568 switch (wcn->wc_type) { 2569 case WAPBL_WC_BLOCKS: 2570 case WAPBL_WC_REVOCATIONS: 2571 break; 2572 case WAPBL_WC_INODES: 2573 { 2574 struct wapbl_wc_inodelist *wc = 2575 (struct wapbl_wc_inodelist *)wr->wr_scratch; 2576 /* 2577 * Keep track of where we found this so we 2578 * can use it later 2579 */ 2580 if (wc->wc_clear) { 2581 cnt = 0; 2582 } 2583 /* This memcpy assumes that wr_inodes is 2584 * laid out the same as wc_inodes. */ 2585 memcpy(&wr->wr_inodes[cnt], wc->wc_inodes, 2586 wc->wc_inocnt*sizeof(wc->wc_inodes[0])); 2587 cnt += wc->wc_inocnt; 2588 } 2589 break; 2590 default: 2591 KASSERT(0); 2592 } 2593 off = saveoff; 2594 wapbl_circ_advance(wr, wcn->wc_len, &off); 2595 } 2596 KASSERT(cnt == wr->wr_inodescnt); 2597 return 0; 2598 } 2599 2600 #ifdef DEBUG 2601 int 2602 wapbl_replay_verify(struct wapbl_replay *wr, struct vnode *fsdevvp) 2603 { 2604 off_t off; 2605 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2606 int mismatchcnt = 0; 2607 int logblklen = 1<<wch->wc_log_dev_bshift; 2608 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2609 void *scratch1 = wapbl_malloc(MAXBSIZE); 2610 void *scratch2 = wapbl_malloc(MAXBSIZE); 2611 int error = 0; 2612 2613 KDASSERT(wapbl_replay_isopen(wr)); 2614 2615 off = wch->wc_tail; 2616 while (off != wch->wc_head) { 2617 struct wapbl_wc_null *wcn; 2618 #ifdef DEBUG 2619 off_t saveoff = off; 2620 #endif 2621 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2622 if (error) 2623 goto out; 2624 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2625 switch (wcn->wc_type) { 2626 case WAPBL_WC_BLOCKS: 2627 { 2628 struct wapbl_wc_blocklist *wc = 2629 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2630 int i; 2631 for (i = 0; i < wc->wc_blkcount; i++) { 2632 int foundcnt = 0; 2633 int dirtycnt = 0; 2634 int j, n; 2635 /* 2636 * Check each physical block into the 2637 * hashtable independently 2638 */ 2639 n = wc->wc_blocks[i].wc_dlen >> 2640 wch->wc_fs_dev_bshift; 2641 for (j = 0; j < n; j++) { 2642 struct wapbl_blk *wb = 2643 wapbl_blkhash_get(wr, 2644 wc->wc_blocks[i].wc_daddr + j); 2645 if (wb && (wb->wb_off == off)) { 2646 foundcnt++; 2647 error = 2648 wapbl_circ_read(wr, 2649 scratch1, fsblklen, 2650 &off); 2651 if (error) 2652 goto out; 2653 error = 2654 wapbl_read(scratch2, 2655 fsblklen, fsdevvp, 2656 wb->wb_blk); 2657 if (error) 2658 goto out; 2659 if (memcmp(scratch1, 2660 scratch2, 2661 fsblklen)) { 2662 printf( 2663 "wapbl_verify: mismatch block %"PRId64" at off %"PRIdMAX"\n", 2664 wb->wb_blk, (intmax_t)off); 2665 dirtycnt++; 2666 mismatchcnt++; 2667 } 2668 } else { 2669 wapbl_circ_advance(wr, 2670 fsblklen, &off); 2671 } 2672 } 2673 #if 0 2674 /* 2675 * If all of the blocks in an entry 2676 * are clean, then remove all of its 2677 * blocks from the hashtable since they 2678 * never will need replay. 2679 */ 2680 if ((foundcnt != 0) && 2681 (dirtycnt == 0)) { 2682 off = saveoff; 2683 wapbl_circ_advance(wr, 2684 logblklen, &off); 2685 for (j = 0; j < n; j++) { 2686 struct wapbl_blk *wb = 2687 wapbl_blkhash_get(wr, 2688 wc->wc_blocks[i].wc_daddr + j); 2689 if (wb && 2690 (wb->wb_off == off)) { 2691 wapbl_blkhash_rem(wr, wb->wb_blk); 2692 } 2693 wapbl_circ_advance(wr, 2694 fsblklen, &off); 2695 } 2696 } 2697 #endif 2698 } 2699 } 2700 break; 2701 case WAPBL_WC_REVOCATIONS: 2702 case WAPBL_WC_INODES: 2703 break; 2704 default: 2705 KASSERT(0); 2706 } 2707 #ifdef DEBUG 2708 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2709 KASSERT(off == saveoff); 2710 #endif 2711 } 2712 out: 2713 wapbl_free(scratch1); 2714 wapbl_free(scratch2); 2715 if (!error && mismatchcnt) 2716 error = EFTYPE; 2717 return error; 2718 } 2719 #endif 2720 2721 int 2722 wapbl_replay_write(struct wapbl_replay *wr, struct vnode *fsdevvp) 2723 { 2724 off_t off; 2725 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2726 int logblklen = 1<<wch->wc_log_dev_bshift; 2727 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2728 void *scratch1 = wapbl_malloc(MAXBSIZE); 2729 int error = 0; 2730 2731 KDASSERT(wapbl_replay_isopen(wr)); 2732 2733 /* 2734 * This parses the journal for replay, although it could 2735 * just as easily walk the hashtable instead. 2736 */ 2737 2738 off = wch->wc_tail; 2739 while (off != wch->wc_head) { 2740 struct wapbl_wc_null *wcn; 2741 #ifdef DEBUG 2742 off_t saveoff = off; 2743 #endif 2744 error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 2745 if (error) 2746 goto out; 2747 wcn = (struct wapbl_wc_null *)wr->wr_scratch; 2748 switch (wcn->wc_type) { 2749 case WAPBL_WC_BLOCKS: 2750 { 2751 struct wapbl_wc_blocklist *wc = 2752 (struct wapbl_wc_blocklist *)wr->wr_scratch; 2753 int i; 2754 for (i = 0; i < wc->wc_blkcount; i++) { 2755 int j, n; 2756 /* 2757 * Check each physical block against 2758 * the hashtable independently 2759 */ 2760 n = wc->wc_blocks[i].wc_dlen >> 2761 wch->wc_fs_dev_bshift; 2762 for (j = 0; j < n; j++) { 2763 struct wapbl_blk *wb = 2764 wapbl_blkhash_get(wr, 2765 wc->wc_blocks[i].wc_daddr + j); 2766 if (wb && (wb->wb_off == off)) { 2767 error = wapbl_circ_read( 2768 wr, scratch1, 2769 fsblklen, &off); 2770 if (error) 2771 goto out; 2772 error = 2773 wapbl_write(scratch1, 2774 fsblklen, fsdevvp, 2775 wb->wb_blk); 2776 if (error) 2777 goto out; 2778 } else { 2779 wapbl_circ_advance(wr, 2780 fsblklen, &off); 2781 } 2782 } 2783 } 2784 } 2785 break; 2786 case WAPBL_WC_REVOCATIONS: 2787 case WAPBL_WC_INODES: 2788 break; 2789 default: 2790 KASSERT(0); 2791 } 2792 #ifdef DEBUG 2793 wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 2794 KASSERT(off == saveoff); 2795 #endif 2796 } 2797 out: 2798 wapbl_free(scratch1); 2799 return error; 2800 } 2801 2802 int 2803 wapbl_replay_read(struct wapbl_replay *wr, void *data, daddr_t blk, long len) 2804 { 2805 struct wapbl_wc_header *wch = &wr->wr_wc_header; 2806 int fsblklen = 1<<wch->wc_fs_dev_bshift; 2807 2808 KDASSERT(wapbl_replay_isopen(wr)); 2809 2810 KASSERT((len % fsblklen) == 0); 2811 2812 while (len != 0) { 2813 struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2814 if (wb) { 2815 off_t off = wb->wb_off; 2816 int error; 2817 error = wapbl_circ_read(wr, data, fsblklen, &off); 2818 if (error) 2819 return error; 2820 } 2821 data = (uint8_t *)data + fsblklen; 2822 len -= fsblklen; 2823 blk++; 2824 } 2825 return 0; 2826 } 2827
Indexes created Thu Oct 02 14:10:14 GMT 2025