vfs_wapbl.c revision 1.52.2.5
1 1.52.2.4 tls /* $NetBSD: vfs_wapbl.c,v 1.52.2.5 2017/12/03 11:38:45 jdolecek Exp $ */ 2 1.2 simonb 3 1.2 simonb /*- 4 1.23 ad * Copyright (c) 2003, 2008, 2009 The NetBSD Foundation, Inc. 5 1.2 simonb * All rights reserved. 6 1.2 simonb * 7 1.2 simonb * This code is derived from software contributed to The NetBSD Foundation 8 1.2 simonb * by Wasabi Systems, Inc. 9 1.2 simonb * 10 1.2 simonb * Redistribution and use in source and binary forms, with or without 11 1.2 simonb * modification, are permitted provided that the following conditions 12 1.2 simonb * are met: 13 1.2 simonb * 1. Redistributions of source code must retain the above copyright 14 1.2 simonb * notice, this list of conditions and the following disclaimer. 15 1.2 simonb * 2. Redistributions in binary form must reproduce the above copyright 16 1.2 simonb * notice, this list of conditions and the following disclaimer in the 17 1.2 simonb * documentation and/or other materials provided with the distribution. 18 1.2 simonb * 19 1.2 simonb * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 1.2 simonb * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 1.2 simonb * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 1.2 simonb * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 1.2 simonb * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 1.2 simonb * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 1.2 simonb * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 1.2 simonb * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 1.2 simonb * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 1.2 simonb * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 1.2 simonb * POSSIBILITY OF SUCH DAMAGE. 30 1.2 simonb */ 31 1.2 simonb 32 1.2 simonb /* 33 1.2 simonb * This implements file system independent write ahead filesystem logging. 34 1.2 simonb */ 35 1.4 joerg 36 1.4 joerg #define WAPBL_INTERNAL 37 1.4 joerg 38 1.2 simonb #include <sys/cdefs.h> 39 1.52.2.4 tls __KERNEL_RCSID(0, "$NetBSD: vfs_wapbl.c,v 1.52.2.5 2017/12/03 11:38:45 jdolecek Exp $"); 40 1.2 simonb 41 1.2 simonb #include <sys/param.h> 42 1.31 mlelstv #include <sys/bitops.h> 43 1.52.2.5 jdolecek #include <sys/time.h> 44 1.52.2.5 jdolecek #include <sys/wapbl.h> 45 1.52.2.5 jdolecek #include <sys/wapbl_replay.h> 46 1.2 simonb 47 1.2 simonb #ifdef _KERNEL 48 1.52.2.5 jdolecek 49 1.52.2.5 jdolecek #include <sys/atomic.h> 50 1.52.2.5 jdolecek #include <sys/conf.h> 51 1.52.2.5 jdolecek #include <sys/evcnt.h> 52 1.52.2.5 jdolecek #include <sys/file.h> 53 1.52.2.5 jdolecek #include <sys/kauth.h> 54 1.52.2.5 jdolecek #include <sys/kernel.h> 55 1.52.2.5 jdolecek #include <sys/module.h> 56 1.52.2.5 jdolecek #include <sys/mount.h> 57 1.52.2.5 jdolecek #include <sys/mutex.h> 58 1.2 simonb #include <sys/namei.h> 59 1.2 simonb #include <sys/proc.h> 60 1.52.2.5 jdolecek #include <sys/resourcevar.h> 61 1.39 christos #include <sys/sysctl.h> 62 1.2 simonb #include <sys/uio.h> 63 1.2 simonb #include <sys/vnode.h> 64 1.2 simonb 65 1.2 simonb #include <miscfs/specfs/specdev.h> 66 1.2 simonb 67 1.51 para #define wapbl_alloc(s) kmem_alloc((s), KM_SLEEP) 68 1.51 para #define wapbl_free(a, s) kmem_free((a), (s)) 69 1.51 para #define wapbl_calloc(n, s) kmem_zalloc((n)*(s), KM_SLEEP) 70 1.2 simonb 71 1.39 christos static struct sysctllog *wapbl_sysctl; 72 1.39 christos static int wapbl_flush_disk_cache = 1; 73 1.39 christos static int wapbl_verbose_commit = 0; 74 1.52.2.5 jdolecek static int wapbl_allow_dpofua = 0; /* switched off by default for now */ 75 1.52.2.5 jdolecek static int wapbl_journal_iobufs = 4; 76 1.39 christos 77 1.52.2.4 tls static inline size_t wapbl_space_free(size_t, off_t, off_t); 78 1.52.2.4 tls 79 1.2 simonb #else /* !_KERNEL */ 80 1.52.2.5 jdolecek 81 1.2 simonb #include <assert.h> 82 1.2 simonb #include <errno.h> 83 1.2 simonb #include <stdbool.h> 84 1.52.2.5 jdolecek #include <stdio.h> 85 1.2 simonb #include <stdlib.h> 86 1.2 simonb #include <string.h> 87 1.2 simonb 88 1.2 simonb #define KDASSERT(x) assert(x) 89 1.2 simonb #define KASSERT(x) assert(x) 90 1.51 para #define wapbl_alloc(s) malloc(s) 91 1.18 yamt #define wapbl_free(a, s) free(a) 92 1.2 simonb #define wapbl_calloc(n, s) calloc((n), (s)) 93 1.2 simonb 94 1.2 simonb #endif /* !_KERNEL */ 95 1.2 simonb 96 1.2 simonb /* 97 1.2 simonb * INTERNAL DATA STRUCTURES 98 1.2 simonb */ 99 1.2 simonb 100 1.52.2.5 jdolecek /* 101 1.2 simonb * This structure holds per-mount log information. 102 1.2 simonb * 103 1.2 simonb * Legend: a = atomic access only 104 1.2 simonb * r = read-only after init 105 1.2 simonb * l = rwlock held 106 1.2 simonb * m = mutex held 107 1.38 hannken * lm = rwlock held writing or mutex held 108 1.2 simonb * u = unlocked access ok 109 1.2 simonb * b = bufcache_lock held 110 1.2 simonb */ 111 1.52.2.5 jdolecek LIST_HEAD(wapbl_ino_head, wapbl_ino); 112 1.2 simonb struct wapbl { 113 1.2 simonb struct vnode *wl_logvp; /* r: log here */ 114 1.2 simonb struct vnode *wl_devvp; /* r: log on this device */ 115 1.2 simonb struct mount *wl_mount; /* r: mountpoint wl is associated with */ 116 1.2 simonb daddr_t wl_logpbn; /* r: Physical block number of start of log */ 117 1.2 simonb int wl_log_dev_bshift; /* r: logarithm of device block size of log 118 1.2 simonb device */ 119 1.2 simonb int wl_fs_dev_bshift; /* r: logarithm of device block size of 120 1.2 simonb filesystem device */ 121 1.2 simonb 122 1.3 yamt unsigned wl_lock_count; /* m: Count of transactions in progress */ 123 1.2 simonb 124 1.2 simonb size_t wl_circ_size; /* r: Number of bytes in buffer of log */ 125 1.2 simonb size_t wl_circ_off; /* r: Number of bytes reserved at start */ 126 1.2 simonb 127 1.2 simonb size_t wl_bufcount_max; /* r: Number of buffers reserved for log */ 128 1.2 simonb size_t wl_bufbytes_max; /* r: Number of buf bytes reserved for log */ 129 1.2 simonb 130 1.2 simonb off_t wl_head; /* l: Byte offset of log head */ 131 1.2 simonb off_t wl_tail; /* l: Byte offset of log tail */ 132 1.2 simonb /* 133 1.52.2.5 jdolecek * WAPBL log layout, stored on wl_devvp at wl_logpbn: 134 1.52.2.5 jdolecek * 135 1.52.2.5 jdolecek * ___________________ wl_circ_size __________________ 136 1.52.2.5 jdolecek * / \ 137 1.52.2.5 jdolecek * +---------+---------+-------+--------------+--------+ 138 1.52.2.5 jdolecek * [ commit0 | commit1 | CCWCW | EEEEEEEEEEEE | CCCWCW ] 139 1.52.2.5 jdolecek * +---------+---------+-------+--------------+--------+ 140 1.52.2.5 jdolecek * wl_circ_off --^ ^-- wl_head ^-- wl_tail 141 1.52.2.5 jdolecek * 142 1.52.2.5 jdolecek * commit0 and commit1 are commit headers. A commit header has 143 1.52.2.5 jdolecek * a generation number, indicating which of the two headers is 144 1.52.2.5 jdolecek * more recent, and an assignment of head and tail pointers. 145 1.52.2.5 jdolecek * The rest is a circular queue of log records, starting at 146 1.52.2.5 jdolecek * the byte offset wl_circ_off. 147 1.52.2.5 jdolecek * 148 1.52.2.5 jdolecek * E marks empty space for records. 149 1.52.2.5 jdolecek * W marks records for block writes issued but waiting. 150 1.52.2.5 jdolecek * C marks completed records. 151 1.52.2.5 jdolecek * 152 1.52.2.5 jdolecek * wapbl_flush writes new records to empty `E' spaces after 153 1.52.2.5 jdolecek * wl_head from the current transaction in memory. 154 1.52.2.5 jdolecek * 155 1.52.2.5 jdolecek * wapbl_truncate advances wl_tail past any completed `C' 156 1.52.2.5 jdolecek * records, freeing them up for use. 157 1.52.2.5 jdolecek * 158 1.52.2.5 jdolecek * head == tail == 0 means log is empty. 159 1.52.2.5 jdolecek * head == tail != 0 means log is full. 160 1.52.2.5 jdolecek * 161 1.52.2.5 jdolecek * See assertions in wapbl_advance() for other boundary 162 1.52.2.5 jdolecek * conditions. 163 1.52.2.5 jdolecek * 164 1.52.2.5 jdolecek * Only wapbl_flush moves the head, except when wapbl_truncate 165 1.52.2.5 jdolecek * sets it to 0 to indicate that the log is empty. 166 1.52.2.5 jdolecek * 167 1.52.2.5 jdolecek * Only wapbl_truncate moves the tail, except when wapbl_flush 168 1.52.2.5 jdolecek * sets it to wl_circ_off to indicate that the log is full. 169 1.2 simonb */ 170 1.2 simonb 171 1.2 simonb struct wapbl_wc_header *wl_wc_header; /* l */ 172 1.2 simonb void *wl_wc_scratch; /* l: scratch space (XXX: por que?!?) */ 173 1.2 simonb 174 1.2 simonb kmutex_t wl_mtx; /* u: short-term lock */ 175 1.2 simonb krwlock_t wl_rwlock; /* u: File system transaction lock */ 176 1.2 simonb 177 1.2 simonb /* 178 1.2 simonb * Must be held while accessing 179 1.2 simonb * wl_count or wl_bufs or head or tail 180 1.2 simonb */ 181 1.2 simonb 182 1.52.2.5 jdolecek #if _KERNEL 183 1.2 simonb /* 184 1.2 simonb * Callback called from within the flush routine to flush any extra 185 1.2 simonb * bits. Note that flush may be skipped without calling this if 186 1.2 simonb * there are no outstanding buffers in the transaction. 187 1.2 simonb */ 188 1.2 simonb wapbl_flush_fn_t wl_flush; /* r */ 189 1.2 simonb wapbl_flush_fn_t wl_flush_abort;/* r */ 190 1.52.2.5 jdolecek 191 1.52.2.5 jdolecek /* Event counters */ 192 1.52.2.5 jdolecek char wl_ev_group[EVCNT_STRING_MAX]; /* r */ 193 1.52.2.5 jdolecek struct evcnt wl_ev_commit; /* l */ 194 1.52.2.5 jdolecek struct evcnt wl_ev_journalwrite; /* l */ 195 1.52.2.5 jdolecek struct evcnt wl_ev_jbufs_bio_nowait; /* l */ 196 1.52.2.5 jdolecek struct evcnt wl_ev_metawrite; /* lm */ 197 1.52.2.5 jdolecek struct evcnt wl_ev_cacheflush; /* l */ 198 1.5 joerg #endif 199 1.2 simonb 200 1.2 simonb size_t wl_bufbytes; /* m: Byte count of pages in wl_bufs */ 201 1.2 simonb size_t wl_bufcount; /* m: Count of buffers in wl_bufs */ 202 1.2 simonb size_t wl_bcount; /* m: Total bcount of wl_bufs */ 203 1.2 simonb 204 1.52.2.5 jdolecek TAILQ_HEAD(, buf) wl_bufs; /* m: Buffers in current transaction */ 205 1.2 simonb 206 1.2 simonb kcondvar_t wl_reclaimable_cv; /* m (obviously) */ 207 1.2 simonb size_t wl_reclaimable_bytes; /* m: Amount of space available for 208 1.2 simonb reclamation by truncate */ 209 1.2 simonb int wl_error_count; /* m: # of wl_entries with errors */ 210 1.2 simonb size_t wl_reserved_bytes; /* never truncate log smaller than this */ 211 1.2 simonb 212 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 213 1.2 simonb size_t wl_unsynced_bufbytes; /* Byte count of unsynced buffers */ 214 1.2 simonb #endif 215 1.2 simonb 216 1.52.2.5 jdolecek #if _KERNEL 217 1.52.2.5 jdolecek int wl_brperjblock; /* r Block records per journal block */ 218 1.52.2.5 jdolecek #endif 219 1.52.2.5 jdolecek 220 1.52.2.5 jdolecek TAILQ_HEAD(, wapbl_dealloc) wl_dealloclist; /* lm: list head */ 221 1.52.2.5 jdolecek int wl_dealloccnt; /* lm: total count */ 222 1.52.2.5 jdolecek int wl_dealloclim; /* r: max count */ 223 1.2 simonb 224 1.2 simonb /* hashtable of inode numbers for allocated but unlinked inodes */ 225 1.2 simonb /* synch ??? */ 226 1.52.2.5 jdolecek struct wapbl_ino_head *wl_inohash; 227 1.2 simonb u_long wl_inohashmask; 228 1.2 simonb int wl_inohashcnt; 229 1.2 simonb 230 1.2 simonb SIMPLEQ_HEAD(, wapbl_entry) wl_entries; /* On disk transaction 231 1.2 simonb accounting */ 232 1.52.2.3 tls 233 1.52.2.5 jdolecek /* buffers for wapbl_buffered_write() */ 234 1.52.2.5 jdolecek TAILQ_HEAD(, buf) wl_iobufs; /* l: Free or filling bufs */ 235 1.52.2.5 jdolecek TAILQ_HEAD(, buf) wl_iobufs_busy; /* l: In-transit bufs */ 236 1.52.2.5 jdolecek 237 1.52.2.5 jdolecek int wl_dkcache; /* r: disk cache flags */ 238 1.52.2.5 jdolecek #define WAPBL_USE_FUA(wl) \ 239 1.52.2.5 jdolecek (wapbl_allow_dpofua && ISSET((wl)->wl_dkcache, DKCACHE_FUA)) 240 1.52.2.5 jdolecek #define WAPBL_JFLAGS(wl) \ 241 1.52.2.5 jdolecek (WAPBL_USE_FUA(wl) ? (wl)->wl_jwrite_flags : 0) 242 1.52.2.5 jdolecek #define WAPBL_JDATA_FLAGS(wl) \ 243 1.52.2.5 jdolecek (WAPBL_JFLAGS(wl) & B_MEDIA_DPO) /* only DPO */ 244 1.52.2.5 jdolecek int wl_jwrite_flags; /* r: journal write flags */ 245 1.2 simonb }; 246 1.2 simonb 247 1.2 simonb #ifdef WAPBL_DEBUG_PRINT 248 1.2 simonb int wapbl_debug_print = WAPBL_DEBUG_PRINT; 249 1.2 simonb #endif 250 1.2 simonb 251 1.2 simonb /****************************************************************/ 252 1.2 simonb #ifdef _KERNEL 253 1.2 simonb 254 1.2 simonb #ifdef WAPBL_DEBUG 255 1.2 simonb struct wapbl *wapbl_debug_wl; 256 1.2 simonb #endif 257 1.2 simonb 258 1.2 simonb static int wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail); 259 1.2 simonb static int wapbl_write_blocks(struct wapbl *wl, off_t *offp); 260 1.2 simonb static int wapbl_write_revocations(struct wapbl *wl, off_t *offp); 261 1.2 simonb static int wapbl_write_inodes(struct wapbl *wl, off_t *offp); 262 1.2 simonb #endif /* _KERNEL */ 263 1.2 simonb 264 1.14 joerg static int wapbl_replay_process(struct wapbl_replay *wr, off_t, off_t); 265 1.2 simonb 266 1.30 uebayasi static inline size_t wapbl_space_used(size_t avail, off_t head, 267 1.2 simonb off_t tail); 268 1.2 simonb 269 1.2 simonb #ifdef _KERNEL 270 1.2 simonb 271 1.51 para static struct pool wapbl_entry_pool; 272 1.52.2.5 jdolecek static struct pool wapbl_dealloc_pool; 273 1.51 para 274 1.2 simonb #define WAPBL_INODETRK_SIZE 83 275 1.2 simonb static int wapbl_ino_pool_refcount; 276 1.2 simonb static struct pool wapbl_ino_pool; 277 1.2 simonb struct wapbl_ino { 278 1.2 simonb LIST_ENTRY(wapbl_ino) wi_hash; 279 1.2 simonb ino_t wi_ino; 280 1.2 simonb mode_t wi_mode; 281 1.2 simonb }; 282 1.2 simonb 283 1.2 simonb static void wapbl_inodetrk_init(struct wapbl *wl, u_int size); 284 1.2 simonb static void wapbl_inodetrk_free(struct wapbl *wl); 285 1.2 simonb static struct wapbl_ino *wapbl_inodetrk_get(struct wapbl *wl, ino_t ino); 286 1.2 simonb 287 1.2 simonb static size_t wapbl_transaction_len(struct wapbl *wl); 288 1.30 uebayasi static inline size_t wapbl_transaction_inodes_len(struct wapbl *wl); 289 1.2 simonb 290 1.52.2.5 jdolecek static void wapbl_deallocation_free(struct wapbl *, struct wapbl_dealloc *, 291 1.52.2.5 jdolecek bool); 292 1.52.2.5 jdolecek 293 1.52.2.5 jdolecek static void wapbl_evcnt_init(struct wapbl *); 294 1.52.2.5 jdolecek static void wapbl_evcnt_free(struct wapbl *); 295 1.52.2.5 jdolecek 296 1.52.2.5 jdolecek static void wapbl_dkcache_init(struct wapbl *); 297 1.52.2.5 jdolecek 298 1.13 joerg #if 0 299 1.4 joerg int wapbl_replay_verify(struct wapbl_replay *, struct vnode *); 300 1.4 joerg #endif 301 1.4 joerg 302 1.4 joerg static int wapbl_replay_isopen1(struct wapbl_replay *); 303 1.4 joerg 304 1.2 simonb struct wapbl_ops wapbl_ops = { 305 1.2 simonb .wo_wapbl_discard = wapbl_discard, 306 1.2 simonb .wo_wapbl_replay_isopen = wapbl_replay_isopen1, 307 1.6 joerg .wo_wapbl_replay_can_read = wapbl_replay_can_read, 308 1.2 simonb .wo_wapbl_replay_read = wapbl_replay_read, 309 1.2 simonb .wo_wapbl_add_buf = wapbl_add_buf, 310 1.2 simonb .wo_wapbl_remove_buf = wapbl_remove_buf, 311 1.2 simonb .wo_wapbl_resize_buf = wapbl_resize_buf, 312 1.2 simonb .wo_wapbl_begin = wapbl_begin, 313 1.2 simonb .wo_wapbl_end = wapbl_end, 314 1.2 simonb .wo_wapbl_junlock_assert= wapbl_junlock_assert, 315 1.2 simonb 316 1.2 simonb /* XXX: the following is only used to say "this is a wapbl buf" */ 317 1.2 simonb .wo_wapbl_biodone = wapbl_biodone, 318 1.2 simonb }; 319 1.2 simonb 320 1.21 yamt static int 321 1.39 christos wapbl_sysctl_init(void) 322 1.39 christos { 323 1.39 christos int rv; 324 1.39 christos const struct sysctlnode *rnode, *cnode; 325 1.39 christos 326 1.39 christos wapbl_sysctl = NULL; 327 1.39 christos 328 1.39 christos rv = sysctl_createv(&wapbl_sysctl, 0, NULL, &rnode, 329 1.39 christos CTLFLAG_PERMANENT, 330 1.39 christos CTLTYPE_NODE, "wapbl", 331 1.39 christos SYSCTL_DESCR("WAPBL journaling options"), 332 1.39 christos NULL, 0, NULL, 0, 333 1.52.2.4 tls CTL_VFS, CTL_CREATE, CTL_EOL); 334 1.39 christos if (rv) 335 1.39 christos return rv; 336 1.39 christos 337 1.39 christos rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 338 1.39 christos CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 339 1.39 christos CTLTYPE_INT, "flush_disk_cache", 340 1.39 christos SYSCTL_DESCR("flush disk cache"), 341 1.39 christos NULL, 0, &wapbl_flush_disk_cache, 0, 342 1.39 christos CTL_CREATE, CTL_EOL); 343 1.39 christos if (rv) 344 1.39 christos return rv; 345 1.39 christos 346 1.39 christos rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 347 1.39 christos CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 348 1.39 christos CTLTYPE_INT, "verbose_commit", 349 1.39 christos SYSCTL_DESCR("show time and size of wapbl log commits"), 350 1.39 christos NULL, 0, &wapbl_verbose_commit, 0, 351 1.39 christos CTL_CREATE, CTL_EOL); 352 1.52.2.5 jdolecek if (rv) 353 1.52.2.5 jdolecek return rv; 354 1.52.2.5 jdolecek 355 1.52.2.5 jdolecek rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 356 1.52.2.5 jdolecek CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 357 1.52.2.5 jdolecek CTLTYPE_INT, "allow_dpofua", 358 1.52.2.5 jdolecek SYSCTL_DESCR("allow use of FUA/DPO instead of cash flush if available"), 359 1.52.2.5 jdolecek NULL, 0, &wapbl_allow_dpofua, 0, 360 1.52.2.5 jdolecek CTL_CREATE, CTL_EOL); 361 1.52.2.5 jdolecek if (rv) 362 1.52.2.5 jdolecek return rv; 363 1.52.2.5 jdolecek 364 1.52.2.5 jdolecek rv = sysctl_createv(&wapbl_sysctl, 0, &rnode, &cnode, 365 1.52.2.5 jdolecek CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 366 1.52.2.5 jdolecek CTLTYPE_INT, "journal_iobufs", 367 1.52.2.5 jdolecek SYSCTL_DESCR("count of bufs used for journal I/O (max async count)"), 368 1.52.2.5 jdolecek NULL, 0, &wapbl_journal_iobufs, 0, 369 1.52.2.5 jdolecek CTL_CREATE, CTL_EOL); 370 1.52.2.5 jdolecek if (rv) 371 1.52.2.5 jdolecek return rv; 372 1.52.2.5 jdolecek 373 1.39 christos return rv; 374 1.39 christos } 375 1.39 christos 376 1.39 christos static void 377 1.39 christos wapbl_init(void) 378 1.39 christos { 379 1.51 para 380 1.51 para pool_init(&wapbl_entry_pool, sizeof(struct wapbl_entry), 0, 0, 0, 381 1.51 para "wapblentrypl", &pool_allocator_kmem, IPL_VM); 382 1.52.2.5 jdolecek pool_init(&wapbl_dealloc_pool, sizeof(struct wapbl_dealloc), 0, 0, 0, 383 1.52.2.5 jdolecek "wapbldealloc", &pool_allocator_nointr, IPL_NONE); 384 1.51 para 385 1.39 christos wapbl_sysctl_init(); 386 1.39 christos } 387 1.39 christos 388 1.39 christos static int 389 1.52.2.5 jdolecek wapbl_fini(void) 390 1.39 christos { 391 1.51 para 392 1.52.2.5 jdolecek if (wapbl_sysctl != NULL) 393 1.52.2.5 jdolecek sysctl_teardown(&wapbl_sysctl); 394 1.51 para 395 1.52.2.5 jdolecek pool_destroy(&wapbl_dealloc_pool); 396 1.51 para pool_destroy(&wapbl_entry_pool); 397 1.51 para 398 1.39 christos return 0; 399 1.39 christos } 400 1.52.2.5 jdolecek 401 1.52.2.5 jdolecek static void 402 1.52.2.5 jdolecek wapbl_evcnt_init(struct wapbl *wl) 403 1.52.2.5 jdolecek { 404 1.52.2.5 jdolecek snprintf(wl->wl_ev_group, sizeof(wl->wl_ev_group), 405 1.52.2.5 jdolecek "wapbl fsid 0x%x/0x%x", 406 1.52.2.5 jdolecek wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[0], 407 1.52.2.5 jdolecek wl->wl_mount->mnt_stat.f_fsidx.__fsid_val[1] 408 1.52.2.5 jdolecek ); 409 1.52.2.5 jdolecek 410 1.52.2.5 jdolecek evcnt_attach_dynamic(&wl->wl_ev_commit, EVCNT_TYPE_MISC, 411 1.52.2.5 jdolecek NULL, wl->wl_ev_group, "commit"); 412 1.52.2.5 jdolecek evcnt_attach_dynamic(&wl->wl_ev_journalwrite, EVCNT_TYPE_MISC, 413 1.52.2.5 jdolecek NULL, wl->wl_ev_group, "journal write total"); 414 1.52.2.5 jdolecek evcnt_attach_dynamic(&wl->wl_ev_jbufs_bio_nowait, EVCNT_TYPE_MISC, 415 1.52.2.5 jdolecek NULL, wl->wl_ev_group, "journal write finished async"); 416 1.52.2.5 jdolecek evcnt_attach_dynamic(&wl->wl_ev_metawrite, EVCNT_TYPE_MISC, 417 1.52.2.5 jdolecek NULL, wl->wl_ev_group, "metadata async write"); 418 1.52.2.5 jdolecek evcnt_attach_dynamic(&wl->wl_ev_cacheflush, EVCNT_TYPE_MISC, 419 1.52.2.5 jdolecek NULL, wl->wl_ev_group, "cache flush"); 420 1.52.2.5 jdolecek } 421 1.52.2.5 jdolecek 422 1.52.2.5 jdolecek static void 423 1.52.2.5 jdolecek wapbl_evcnt_free(struct wapbl *wl) 424 1.52.2.5 jdolecek { 425 1.52.2.5 jdolecek evcnt_detach(&wl->wl_ev_commit); 426 1.52.2.5 jdolecek evcnt_detach(&wl->wl_ev_journalwrite); 427 1.52.2.5 jdolecek evcnt_detach(&wl->wl_ev_jbufs_bio_nowait); 428 1.52.2.5 jdolecek evcnt_detach(&wl->wl_ev_metawrite); 429 1.52.2.5 jdolecek evcnt_detach(&wl->wl_ev_cacheflush); 430 1.52.2.5 jdolecek } 431 1.52.2.5 jdolecek 432 1.52.2.5 jdolecek static void 433 1.52.2.5 jdolecek wapbl_dkcache_init(struct wapbl *wl) 434 1.52.2.5 jdolecek { 435 1.52.2.5 jdolecek int error; 436 1.52.2.5 jdolecek 437 1.52.2.5 jdolecek /* Get disk cache flags */ 438 1.52.2.5 jdolecek error = VOP_IOCTL(wl->wl_devvp, DIOCGCACHE, &wl->wl_dkcache, 439 1.52.2.5 jdolecek FWRITE, FSCRED); 440 1.52.2.5 jdolecek if (error) { 441 1.52.2.5 jdolecek /* behave as if there was a write cache */ 442 1.52.2.5 jdolecek wl->wl_dkcache = DKCACHE_WRITE; 443 1.52.2.5 jdolecek } 444 1.52.2.5 jdolecek 445 1.52.2.5 jdolecek /* Use FUA instead of cache flush if available */ 446 1.52.2.5 jdolecek if (ISSET(wl->wl_dkcache, DKCACHE_FUA)) 447 1.52.2.5 jdolecek wl->wl_jwrite_flags |= B_MEDIA_FUA; 448 1.52.2.5 jdolecek 449 1.52.2.5 jdolecek /* Use DPO for journal writes if available */ 450 1.52.2.5 jdolecek if (ISSET(wl->wl_dkcache, DKCACHE_DPO)) 451 1.52.2.5 jdolecek wl->wl_jwrite_flags |= B_MEDIA_DPO; 452 1.52.2.5 jdolecek } 453 1.39 christos 454 1.39 christos static int 455 1.15 joerg wapbl_start_flush_inodes(struct wapbl *wl, struct wapbl_replay *wr) 456 1.15 joerg { 457 1.15 joerg int error, i; 458 1.15 joerg 459 1.15 joerg WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 460 1.15 joerg ("wapbl_start: reusing log with %d inodes\n", wr->wr_inodescnt)); 461 1.15 joerg 462 1.15 joerg /* 463 1.15 joerg * Its only valid to reuse the replay log if its 464 1.15 joerg * the same as the new log we just opened. 465 1.15 joerg */ 466 1.15 joerg KDASSERT(!wapbl_replay_isopen(wr)); 467 1.47 christos KASSERT(wl->wl_devvp->v_type == VBLK); 468 1.47 christos KASSERT(wr->wr_devvp->v_type == VBLK); 469 1.15 joerg KASSERT(wl->wl_devvp->v_rdev == wr->wr_devvp->v_rdev); 470 1.15 joerg KASSERT(wl->wl_logpbn == wr->wr_logpbn); 471 1.15 joerg KASSERT(wl->wl_circ_size == wr->wr_circ_size); 472 1.15 joerg KASSERT(wl->wl_circ_off == wr->wr_circ_off); 473 1.15 joerg KASSERT(wl->wl_log_dev_bshift == wr->wr_log_dev_bshift); 474 1.15 joerg KASSERT(wl->wl_fs_dev_bshift == wr->wr_fs_dev_bshift); 475 1.15 joerg 476 1.15 joerg wl->wl_wc_header->wc_generation = wr->wr_generation + 1; 477 1.15 joerg 478 1.15 joerg for (i = 0; i < wr->wr_inodescnt; i++) 479 1.15 joerg wapbl_register_inode(wl, wr->wr_inodes[i].wr_inumber, 480 1.15 joerg wr->wr_inodes[i].wr_imode); 481 1.15 joerg 482 1.15 joerg /* Make sure new transaction won't overwrite old inodes list */ 483 1.52.2.5 jdolecek KDASSERT(wapbl_transaction_len(wl) <= 484 1.15 joerg wapbl_space_free(wl->wl_circ_size, wr->wr_inodeshead, 485 1.15 joerg wr->wr_inodestail)); 486 1.15 joerg 487 1.15 joerg wl->wl_head = wl->wl_tail = wr->wr_inodeshead; 488 1.15 joerg wl->wl_reclaimable_bytes = wl->wl_reserved_bytes = 489 1.15 joerg wapbl_transaction_len(wl); 490 1.15 joerg 491 1.15 joerg error = wapbl_write_inodes(wl, &wl->wl_head); 492 1.15 joerg if (error) 493 1.15 joerg return error; 494 1.15 joerg 495 1.15 joerg KASSERT(wl->wl_head != wl->wl_tail); 496 1.15 joerg KASSERT(wl->wl_head != 0); 497 1.15 joerg 498 1.15 joerg return 0; 499 1.15 joerg } 500 1.15 joerg 501 1.2 simonb int 502 1.2 simonb wapbl_start(struct wapbl ** wlp, struct mount *mp, struct vnode *vp, 503 1.2 simonb daddr_t off, size_t count, size_t blksize, struct wapbl_replay *wr, 504 1.2 simonb wapbl_flush_fn_t flushfn, wapbl_flush_fn_t flushabortfn) 505 1.2 simonb { 506 1.2 simonb struct wapbl *wl; 507 1.2 simonb struct vnode *devvp; 508 1.2 simonb daddr_t logpbn; 509 1.2 simonb int error; 510 1.31 mlelstv int log_dev_bshift = ilog2(blksize); 511 1.32 mlelstv int fs_dev_bshift = log_dev_bshift; 512 1.2 simonb int run; 513 1.2 simonb 514 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_start: vp=%p off=%" PRId64 515 1.2 simonb " count=%zu blksize=%zu\n", vp, off, count, blksize)); 516 1.2 simonb 517 1.2 simonb if (log_dev_bshift > fs_dev_bshift) { 518 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_OPEN, 519 1.2 simonb ("wapbl: log device's block size cannot be larger " 520 1.2 simonb "than filesystem's\n")); 521 1.2 simonb /* 522 1.2 simonb * Not currently implemented, although it could be if 523 1.2 simonb * needed someday. 524 1.2 simonb */ 525 1.2 simonb return ENOSYS; 526 1.2 simonb } 527 1.2 simonb 528 1.2 simonb if (off < 0) 529 1.2 simonb return EINVAL; 530 1.2 simonb 531 1.2 simonb if (blksize < DEV_BSIZE) 532 1.2 simonb return EINVAL; 533 1.2 simonb if (blksize % DEV_BSIZE) 534 1.2 simonb return EINVAL; 535 1.2 simonb 536 1.2 simonb /* XXXTODO: verify that the full load is writable */ 537 1.2 simonb 538 1.2 simonb /* 539 1.2 simonb * XXX check for minimum log size 540 1.2 simonb * minimum is governed by minimum amount of space 541 1.2 simonb * to complete a transaction. (probably truncate) 542 1.2 simonb */ 543 1.2 simonb /* XXX for now pick something minimal */ 544 1.52.2.1 tls if ((count * blksize) < mp->mnt_maxphys) { 545 1.2 simonb return ENOSPC; 546 1.2 simonb } 547 1.2 simonb 548 1.2 simonb if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, &run)) != 0) { 549 1.2 simonb return error; 550 1.2 simonb } 551 1.2 simonb 552 1.2 simonb wl = wapbl_calloc(1, sizeof(*wl)); 553 1.2 simonb rw_init(&wl->wl_rwlock); 554 1.2 simonb mutex_init(&wl->wl_mtx, MUTEX_DEFAULT, IPL_NONE); 555 1.2 simonb cv_init(&wl->wl_reclaimable_cv, "wapblrec"); 556 1.52.2.5 jdolecek TAILQ_INIT(&wl->wl_bufs); 557 1.2 simonb SIMPLEQ_INIT(&wl->wl_entries); 558 1.2 simonb 559 1.2 simonb wl->wl_logvp = vp; 560 1.2 simonb wl->wl_devvp = devvp; 561 1.2 simonb wl->wl_mount = mp; 562 1.2 simonb wl->wl_logpbn = logpbn; 563 1.2 simonb wl->wl_log_dev_bshift = log_dev_bshift; 564 1.2 simonb wl->wl_fs_dev_bshift = fs_dev_bshift; 565 1.2 simonb 566 1.2 simonb wl->wl_flush = flushfn; 567 1.2 simonb wl->wl_flush_abort = flushabortfn; 568 1.2 simonb 569 1.2 simonb /* Reserve two log device blocks for the commit headers */ 570 1.2 simonb wl->wl_circ_off = 2<<wl->wl_log_dev_bshift; 571 1.34 mlelstv wl->wl_circ_size = ((count * blksize) - wl->wl_circ_off); 572 1.2 simonb /* truncate the log usage to a multiple of log_dev_bshift */ 573 1.2 simonb wl->wl_circ_size >>= wl->wl_log_dev_bshift; 574 1.2 simonb wl->wl_circ_size <<= wl->wl_log_dev_bshift; 575 1.2 simonb 576 1.2 simonb /* 577 1.2 simonb * wl_bufbytes_max limits the size of the in memory transaction space. 578 1.2 simonb * - Since buffers are allocated and accounted for in units of 579 1.2 simonb * PAGE_SIZE it is required to be a multiple of PAGE_SIZE 580 1.2 simonb * (i.e. 1<<PAGE_SHIFT) 581 1.2 simonb * - Since the log device has to be written in units of 582 1.2 simonb * 1<<wl_log_dev_bshift it is required to be a mulitple of 583 1.2 simonb * 1<<wl_log_dev_bshift. 584 1.2 simonb * - Since filesystem will provide data in units of 1<<wl_fs_dev_bshift, 585 1.2 simonb * it is convenient to be a multiple of 1<<wl_fs_dev_bshift. 586 1.2 simonb * Therefore it must be multiple of the least common multiple of those 587 1.2 simonb * three quantities. Fortunately, all of those quantities are 588 1.2 simonb * guaranteed to be a power of two, and the least common multiple of 589 1.2 simonb * a set of numbers which are all powers of two is simply the maximum 590 1.2 simonb * of those numbers. Finally, the maximum logarithm of a power of two 591 1.2 simonb * is the same as the log of the maximum power of two. So we can do 592 1.2 simonb * the following operations to size wl_bufbytes_max: 593 1.2 simonb */ 594 1.2 simonb 595 1.2 simonb /* XXX fix actual number of pages reserved per filesystem. */ 596 1.2 simonb wl->wl_bufbytes_max = MIN(wl->wl_circ_size, buf_memcalc() / 2); 597 1.2 simonb 598 1.2 simonb /* Round wl_bufbytes_max to the largest power of two constraint */ 599 1.2 simonb wl->wl_bufbytes_max >>= PAGE_SHIFT; 600 1.2 simonb wl->wl_bufbytes_max <<= PAGE_SHIFT; 601 1.2 simonb wl->wl_bufbytes_max >>= wl->wl_log_dev_bshift; 602 1.2 simonb wl->wl_bufbytes_max <<= wl->wl_log_dev_bshift; 603 1.2 simonb wl->wl_bufbytes_max >>= wl->wl_fs_dev_bshift; 604 1.2 simonb wl->wl_bufbytes_max <<= wl->wl_fs_dev_bshift; 605 1.2 simonb 606 1.2 simonb /* XXX maybe use filesystem fragment size instead of 1024 */ 607 1.2 simonb /* XXX fix actual number of buffers reserved per filesystem. */ 608 1.52.2.5 jdolecek wl->wl_bufcount_max = (buf_nbuf() / 2) * 1024; 609 1.52.2.5 jdolecek 610 1.52.2.5 jdolecek wl->wl_brperjblock = ((1<<wl->wl_log_dev_bshift) 611 1.52.2.5 jdolecek - offsetof(struct wapbl_wc_blocklist, wc_blocks)) / 612 1.52.2.5 jdolecek sizeof(((struct wapbl_wc_blocklist *)0)->wc_blocks[0]); 613 1.52.2.5 jdolecek KASSERT(wl->wl_brperjblock > 0); 614 1.2 simonb 615 1.2 simonb /* XXX tie this into resource estimation */ 616 1.41 hannken wl->wl_dealloclim = wl->wl_bufbytes_max / mp->mnt_stat.f_bsize / 2; 617 1.52.2.5 jdolecek TAILQ_INIT(&wl->wl_dealloclist); 618 1.52.2.3 tls 619 1.2 simonb wapbl_inodetrk_init(wl, WAPBL_INODETRK_SIZE); 620 1.2 simonb 621 1.52.2.5 jdolecek wapbl_evcnt_init(wl); 622 1.52.2.5 jdolecek 623 1.52.2.5 jdolecek wapbl_dkcache_init(wl); 624 1.52.2.5 jdolecek 625 1.2 simonb /* Initialize the commit header */ 626 1.2 simonb { 627 1.2 simonb struct wapbl_wc_header *wc; 628 1.14 joerg size_t len = 1 << wl->wl_log_dev_bshift; 629 1.2 simonb wc = wapbl_calloc(1, len); 630 1.2 simonb wc->wc_type = WAPBL_WC_HEADER; 631 1.2 simonb wc->wc_len = len; 632 1.2 simonb wc->wc_circ_off = wl->wl_circ_off; 633 1.2 simonb wc->wc_circ_size = wl->wl_circ_size; 634 1.2 simonb /* XXX wc->wc_fsid */ 635 1.2 simonb wc->wc_log_dev_bshift = wl->wl_log_dev_bshift; 636 1.2 simonb wc->wc_fs_dev_bshift = wl->wl_fs_dev_bshift; 637 1.2 simonb wl->wl_wc_header = wc; 638 1.51 para wl->wl_wc_scratch = wapbl_alloc(len); 639 1.2 simonb } 640 1.2 simonb 641 1.52.2.5 jdolecek TAILQ_INIT(&wl->wl_iobufs); 642 1.52.2.5 jdolecek TAILQ_INIT(&wl->wl_iobufs_busy); 643 1.52.2.5 jdolecek for (int i = 0; i < wapbl_journal_iobufs; i++) { 644 1.52.2.5 jdolecek struct buf *bp; 645 1.52.2.5 jdolecek 646 1.52.2.5 jdolecek if ((bp = geteblk(MAXPHYS)) == NULL) 647 1.52.2.5 jdolecek goto errout; 648 1.52.2.5 jdolecek 649 1.52.2.5 jdolecek mutex_enter(&bufcache_lock); 650 1.52.2.5 jdolecek mutex_enter(devvp->v_interlock); 651 1.52.2.5 jdolecek bgetvp(devvp, bp); 652 1.52.2.5 jdolecek mutex_exit(devvp->v_interlock); 653 1.52.2.5 jdolecek mutex_exit(&bufcache_lock); 654 1.52.2.5 jdolecek 655 1.52.2.5 jdolecek bp->b_dev = devvp->v_rdev; 656 1.52.2.5 jdolecek 657 1.52.2.5 jdolecek TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist); 658 1.52.2.5 jdolecek } 659 1.52.2.5 jdolecek 660 1.2 simonb /* 661 1.2 simonb * if there was an existing set of unlinked but 662 1.2 simonb * allocated inodes, preserve it in the new 663 1.2 simonb * log. 664 1.2 simonb */ 665 1.2 simonb if (wr && wr->wr_inodescnt) { 666 1.15 joerg error = wapbl_start_flush_inodes(wl, wr); 667 1.2 simonb if (error) 668 1.2 simonb goto errout; 669 1.2 simonb } 670 1.2 simonb 671 1.2 simonb error = wapbl_write_commit(wl, wl->wl_head, wl->wl_tail); 672 1.2 simonb if (error) { 673 1.2 simonb goto errout; 674 1.2 simonb } 675 1.2 simonb 676 1.2 simonb *wlp = wl; 677 1.2 simonb #if defined(WAPBL_DEBUG) 678 1.2 simonb wapbl_debug_wl = wl; 679 1.2 simonb #endif 680 1.2 simonb 681 1.2 simonb return 0; 682 1.2 simonb errout: 683 1.2 simonb wapbl_discard(wl); 684 1.18 yamt wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); 685 1.18 yamt wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); 686 1.52.2.5 jdolecek while (!TAILQ_EMPTY(&wl->wl_iobufs)) { 687 1.52.2.5 jdolecek struct buf *bp; 688 1.52.2.5 jdolecek 689 1.52.2.5 jdolecek bp = TAILQ_FIRST(&wl->wl_iobufs); 690 1.52.2.5 jdolecek TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); 691 1.52.2.5 jdolecek brelse(bp, BC_INVAL); 692 1.52.2.5 jdolecek } 693 1.2 simonb wapbl_inodetrk_free(wl); 694 1.18 yamt wapbl_free(wl, sizeof(*wl)); 695 1.2 simonb 696 1.2 simonb return error; 697 1.2 simonb } 698 1.2 simonb 699 1.2 simonb /* 700 1.2 simonb * Like wapbl_flush, only discards the transaction 701 1.2 simonb * completely 702 1.2 simonb */ 703 1.2 simonb 704 1.2 simonb void 705 1.2 simonb wapbl_discard(struct wapbl *wl) 706 1.2 simonb { 707 1.2 simonb struct wapbl_entry *we; 708 1.52.2.5 jdolecek struct wapbl_dealloc *wd; 709 1.2 simonb struct buf *bp; 710 1.2 simonb int i; 711 1.2 simonb 712 1.2 simonb /* 713 1.2 simonb * XXX we may consider using upgrade here 714 1.2 simonb * if we want to call flush from inside a transaction 715 1.2 simonb */ 716 1.2 simonb rw_enter(&wl->wl_rwlock, RW_WRITER); 717 1.52.2.5 jdolecek wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist)); 718 1.2 simonb 719 1.2 simonb #ifdef WAPBL_DEBUG_PRINT 720 1.2 simonb { 721 1.2 simonb pid_t pid = -1; 722 1.2 simonb lwpid_t lid = -1; 723 1.2 simonb if (curproc) 724 1.2 simonb pid = curproc->p_pid; 725 1.2 simonb if (curlwp) 726 1.2 simonb lid = curlwp->l_lid; 727 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 728 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 729 1.2 simonb ("wapbl_discard: thread %d.%d discarding " 730 1.2 simonb "transaction\n" 731 1.2 simonb "\tbufcount=%zu bufbytes=%zu bcount=%zu " 732 1.2 simonb "deallocs=%d inodes=%d\n" 733 1.2 simonb "\terrcnt = %u, reclaimable=%zu reserved=%zu " 734 1.2 simonb "unsynced=%zu\n", 735 1.2 simonb pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 736 1.2 simonb wl->wl_bcount, wl->wl_dealloccnt, 737 1.2 simonb wl->wl_inohashcnt, wl->wl_error_count, 738 1.2 simonb wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 739 1.2 simonb wl->wl_unsynced_bufbytes)); 740 1.2 simonb SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 741 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 742 1.2 simonb ("\tentry: bufcount = %zu, reclaimable = %zu, " 743 1.2 simonb "error = %d, unsynced = %zu\n", 744 1.2 simonb we->we_bufcount, we->we_reclaimable_bytes, 745 1.2 simonb we->we_error, we->we_unsynced_bufbytes)); 746 1.2 simonb } 747 1.2 simonb #else /* !WAPBL_DEBUG_BUFBYTES */ 748 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 749 1.2 simonb ("wapbl_discard: thread %d.%d discarding transaction\n" 750 1.2 simonb "\tbufcount=%zu bufbytes=%zu bcount=%zu " 751 1.2 simonb "deallocs=%d inodes=%d\n" 752 1.2 simonb "\terrcnt = %u, reclaimable=%zu reserved=%zu\n", 753 1.2 simonb pid, lid, wl->wl_bufcount, wl->wl_bufbytes, 754 1.2 simonb wl->wl_bcount, wl->wl_dealloccnt, 755 1.2 simonb wl->wl_inohashcnt, wl->wl_error_count, 756 1.2 simonb wl->wl_reclaimable_bytes, wl->wl_reserved_bytes)); 757 1.2 simonb SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 758 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_DISCARD, 759 1.2 simonb ("\tentry: bufcount = %zu, reclaimable = %zu, " 760 1.2 simonb "error = %d\n", 761 1.2 simonb we->we_bufcount, we->we_reclaimable_bytes, 762 1.2 simonb we->we_error)); 763 1.2 simonb } 764 1.2 simonb #endif /* !WAPBL_DEBUG_BUFBYTES */ 765 1.2 simonb } 766 1.2 simonb #endif /* WAPBL_DEBUG_PRINT */ 767 1.2 simonb 768 1.2 simonb for (i = 0; i <= wl->wl_inohashmask; i++) { 769 1.2 simonb struct wapbl_ino_head *wih; 770 1.2 simonb struct wapbl_ino *wi; 771 1.2 simonb 772 1.2 simonb wih = &wl->wl_inohash[i]; 773 1.2 simonb while ((wi = LIST_FIRST(wih)) != NULL) { 774 1.2 simonb LIST_REMOVE(wi, wi_hash); 775 1.2 simonb pool_put(&wapbl_ino_pool, wi); 776 1.2 simonb KASSERT(wl->wl_inohashcnt > 0); 777 1.2 simonb wl->wl_inohashcnt--; 778 1.2 simonb } 779 1.2 simonb } 780 1.2 simonb 781 1.2 simonb /* 782 1.2 simonb * clean buffer list 783 1.2 simonb */ 784 1.2 simonb mutex_enter(&bufcache_lock); 785 1.2 simonb mutex_enter(&wl->wl_mtx); 786 1.52.2.5 jdolecek while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) { 787 1.2 simonb if (bbusy(bp, 0, 0, &wl->wl_mtx) == 0) { 788 1.2 simonb /* 789 1.2 simonb * The buffer will be unlocked and 790 1.2 simonb * removed from the transaction in brelse 791 1.2 simonb */ 792 1.2 simonb mutex_exit(&wl->wl_mtx); 793 1.2 simonb brelsel(bp, 0); 794 1.2 simonb mutex_enter(&wl->wl_mtx); 795 1.2 simonb } 796 1.2 simonb } 797 1.2 simonb mutex_exit(&wl->wl_mtx); 798 1.2 simonb mutex_exit(&bufcache_lock); 799 1.2 simonb 800 1.2 simonb /* 801 1.2 simonb * Remove references to this wl from wl_entries, free any which 802 1.2 simonb * no longer have buffers, others will be freed in wapbl_biodone 803 1.2 simonb * when they no longer have any buffers. 804 1.2 simonb */ 805 1.2 simonb while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) != NULL) { 806 1.2 simonb SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 807 1.2 simonb /* XXX should we be accumulating wl_error_count 808 1.2 simonb * and increasing reclaimable bytes ? */ 809 1.2 simonb we->we_wapbl = NULL; 810 1.2 simonb if (we->we_bufcount == 0) { 811 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 812 1.2 simonb KASSERT(we->we_unsynced_bufbytes == 0); 813 1.2 simonb #endif 814 1.51 para pool_put(&wapbl_entry_pool, we); 815 1.2 simonb } 816 1.2 simonb } 817 1.2 simonb 818 1.2 simonb /* Discard list of deallocs */ 819 1.52.2.5 jdolecek while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) 820 1.52.2.5 jdolecek wapbl_deallocation_free(wl, wd, true); 821 1.52.2.5 jdolecek 822 1.2 simonb /* XXX should we clear wl_reserved_bytes? */ 823 1.2 simonb 824 1.2 simonb KASSERT(wl->wl_bufbytes == 0); 825 1.2 simonb KASSERT(wl->wl_bcount == 0); 826 1.2 simonb KASSERT(wl->wl_bufcount == 0); 827 1.52.2.5 jdolecek KASSERT(TAILQ_EMPTY(&wl->wl_bufs)); 828 1.2 simonb KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 829 1.2 simonb KASSERT(wl->wl_inohashcnt == 0); 830 1.52.2.5 jdolecek KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist)); 831 1.52.2.5 jdolecek KASSERT(wl->wl_dealloccnt == 0); 832 1.2 simonb 833 1.2 simonb rw_exit(&wl->wl_rwlock); 834 1.2 simonb } 835 1.2 simonb 836 1.2 simonb int 837 1.2 simonb wapbl_stop(struct wapbl *wl, int force) 838 1.2 simonb { 839 1.2 simonb int error; 840 1.2 simonb 841 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_OPEN, ("wapbl_stop called\n")); 842 1.2 simonb error = wapbl_flush(wl, 1); 843 1.2 simonb if (error) { 844 1.2 simonb if (force) 845 1.2 simonb wapbl_discard(wl); 846 1.2 simonb else 847 1.2 simonb return error; 848 1.2 simonb } 849 1.2 simonb 850 1.2 simonb /* Unlinked inodes persist after a flush */ 851 1.2 simonb if (wl->wl_inohashcnt) { 852 1.2 simonb if (force) { 853 1.2 simonb wapbl_discard(wl); 854 1.2 simonb } else { 855 1.2 simonb return EBUSY; 856 1.2 simonb } 857 1.2 simonb } 858 1.2 simonb 859 1.2 simonb KASSERT(wl->wl_bufbytes == 0); 860 1.2 simonb KASSERT(wl->wl_bcount == 0); 861 1.2 simonb KASSERT(wl->wl_bufcount == 0); 862 1.52.2.5 jdolecek KASSERT(TAILQ_EMPTY(&wl->wl_bufs)); 863 1.2 simonb KASSERT(wl->wl_dealloccnt == 0); 864 1.2 simonb KASSERT(SIMPLEQ_EMPTY(&wl->wl_entries)); 865 1.2 simonb KASSERT(wl->wl_inohashcnt == 0); 866 1.52.2.5 jdolecek KASSERT(TAILQ_EMPTY(&wl->wl_dealloclist)); 867 1.52.2.5 jdolecek KASSERT(wl->wl_dealloccnt == 0); 868 1.52.2.5 jdolecek KASSERT(TAILQ_EMPTY(&wl->wl_iobufs_busy)); 869 1.2 simonb 870 1.18 yamt wapbl_free(wl->wl_wc_scratch, wl->wl_wc_header->wc_len); 871 1.18 yamt wapbl_free(wl->wl_wc_header, wl->wl_wc_header->wc_len); 872 1.52.2.5 jdolecek while (!TAILQ_EMPTY(&wl->wl_iobufs)) { 873 1.52.2.5 jdolecek struct buf *bp; 874 1.52.2.5 jdolecek 875 1.52.2.5 jdolecek bp = TAILQ_FIRST(&wl->wl_iobufs); 876 1.52.2.5 jdolecek TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); 877 1.52.2.5 jdolecek brelse(bp, BC_INVAL); 878 1.52.2.5 jdolecek } 879 1.2 simonb wapbl_inodetrk_free(wl); 880 1.2 simonb 881 1.52.2.5 jdolecek wapbl_evcnt_free(wl); 882 1.52.2.5 jdolecek 883 1.2 simonb cv_destroy(&wl->wl_reclaimable_cv); 884 1.2 simonb mutex_destroy(&wl->wl_mtx); 885 1.2 simonb rw_destroy(&wl->wl_rwlock); 886 1.18 yamt wapbl_free(wl, sizeof(*wl)); 887 1.2 simonb 888 1.2 simonb return 0; 889 1.2 simonb } 890 1.2 simonb 891 1.52.2.5 jdolecek /****************************************************************/ 892 1.52.2.5 jdolecek /* 893 1.52.2.5 jdolecek * Unbuffered disk I/O 894 1.52.2.5 jdolecek */ 895 1.52.2.5 jdolecek 896 1.52.2.5 jdolecek static void 897 1.52.2.5 jdolecek wapbl_doio_accounting(struct vnode *devvp, int flags) 898 1.2 simonb { 899 1.2 simonb struct pstats *pstats = curlwp->l_proc->p_stats; 900 1.2 simonb 901 1.2 simonb if ((flags & (B_WRITE | B_READ)) == B_WRITE) { 902 1.45 rmind mutex_enter(devvp->v_interlock); 903 1.2 simonb devvp->v_numoutput++; 904 1.45 rmind mutex_exit(devvp->v_interlock); 905 1.2 simonb pstats->p_ru.ru_oublock++; 906 1.2 simonb } else { 907 1.2 simonb pstats->p_ru.ru_inblock++; 908 1.2 simonb } 909 1.2 simonb 910 1.52.2.5 jdolecek } 911 1.52.2.5 jdolecek 912 1.52.2.5 jdolecek static int 913 1.52.2.5 jdolecek wapbl_doio(void *data, size_t len, struct vnode *devvp, daddr_t pbn, int flags) 914 1.52.2.5 jdolecek { 915 1.52.2.5 jdolecek struct buf *bp; 916 1.52.2.5 jdolecek int error; 917 1.52.2.5 jdolecek 918 1.52.2.5 jdolecek KASSERT(devvp->v_type == VBLK); 919 1.52.2.5 jdolecek 920 1.52.2.5 jdolecek wapbl_doio_accounting(devvp, flags); 921 1.52.2.5 jdolecek 922 1.2 simonb bp = getiobuf(devvp, true); 923 1.2 simonb bp->b_flags = flags; 924 1.52.2.5 jdolecek bp->b_cflags = BC_BUSY; /* mandatory, asserted by biowait() */ 925 1.2 simonb bp->b_dev = devvp->v_rdev; 926 1.2 simonb bp->b_data = data; 927 1.2 simonb bp->b_bufsize = bp->b_resid = bp->b_bcount = len; 928 1.2 simonb bp->b_blkno = pbn; 929 1.52 chs BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 930 1.2 simonb 931 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_IO, 932 1.29 pooka ("wapbl_doio: %s %d bytes at block %"PRId64" on dev 0x%"PRIx64"\n", 933 1.2 simonb BUF_ISWRITE(bp) ? "write" : "read", bp->b_bcount, 934 1.2 simonb bp->b_blkno, bp->b_dev)); 935 1.2 simonb 936 1.2 simonb VOP_STRATEGY(devvp, bp); 937 1.2 simonb 938 1.2 simonb error = biowait(bp); 939 1.2 simonb putiobuf(bp); 940 1.2 simonb 941 1.2 simonb if (error) { 942 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_ERROR, 943 1.2 simonb ("wapbl_doio: %s %zu bytes at block %" PRId64 944 1.29 pooka " on dev 0x%"PRIx64" failed with error %d\n", 945 1.2 simonb (((flags & (B_WRITE | B_READ)) == B_WRITE) ? 946 1.2 simonb "write" : "read"), 947 1.2 simonb len, pbn, devvp->v_rdev, error)); 948 1.2 simonb } 949 1.2 simonb 950 1.2 simonb return error; 951 1.2 simonb } 952 1.2 simonb 953 1.52.2.5 jdolecek /* 954 1.52.2.5 jdolecek * wapbl_write(data, len, devvp, pbn) 955 1.52.2.5 jdolecek * 956 1.52.2.5 jdolecek * Synchronously write len bytes from data to physical block pbn 957 1.52.2.5 jdolecek * on devvp. 958 1.52.2.5 jdolecek */ 959 1.2 simonb int 960 1.2 simonb wapbl_write(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 961 1.2 simonb { 962 1.2 simonb 963 1.2 simonb return wapbl_doio(data, len, devvp, pbn, B_WRITE); 964 1.2 simonb } 965 1.2 simonb 966 1.52.2.5 jdolecek /* 967 1.52.2.5 jdolecek * wapbl_read(data, len, devvp, pbn) 968 1.52.2.5 jdolecek * 969 1.52.2.5 jdolecek * Synchronously read len bytes into data from physical block pbn 970 1.52.2.5 jdolecek * on devvp. 971 1.52.2.5 jdolecek */ 972 1.2 simonb int 973 1.2 simonb wapbl_read(void *data, size_t len, struct vnode *devvp, daddr_t pbn) 974 1.2 simonb { 975 1.2 simonb 976 1.2 simonb return wapbl_doio(data, len, devvp, pbn, B_READ); 977 1.2 simonb } 978 1.2 simonb 979 1.52.2.5 jdolecek /****************************************************************/ 980 1.52.2.5 jdolecek /* 981 1.52.2.5 jdolecek * Buffered disk writes -- try to coalesce writes and emit 982 1.52.2.5 jdolecek * MAXPHYS-aligned blocks. 983 1.52.2.5 jdolecek */ 984 1.52.2.5 jdolecek 985 1.52.2.5 jdolecek /* 986 1.52.2.5 jdolecek * wapbl_buffered_write_async(wl, bp) 987 1.52.2.5 jdolecek * 988 1.52.2.5 jdolecek * Send buffer for asynchronous write. 989 1.52.2.5 jdolecek */ 990 1.52.2.5 jdolecek static void 991 1.52.2.5 jdolecek wapbl_buffered_write_async(struct wapbl *wl, struct buf *bp) 992 1.52.2.5 jdolecek { 993 1.52.2.5 jdolecek wapbl_doio_accounting(wl->wl_devvp, bp->b_flags); 994 1.52.2.5 jdolecek 995 1.52.2.5 jdolecek KASSERT(TAILQ_FIRST(&wl->wl_iobufs) == bp); 996 1.52.2.5 jdolecek TAILQ_REMOVE(&wl->wl_iobufs, bp, b_wapbllist); 997 1.52.2.5 jdolecek 998 1.52.2.5 jdolecek bp->b_flags |= B_WRITE; 999 1.52.2.5 jdolecek bp->b_cflags = BC_BUSY; /* mandatory, asserted by biowait() */ 1000 1.52.2.5 jdolecek bp->b_oflags = 0; 1001 1.52.2.5 jdolecek bp->b_bcount = bp->b_resid; 1002 1.52.2.5 jdolecek BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1003 1.52.2.5 jdolecek 1004 1.52.2.5 jdolecek VOP_STRATEGY(wl->wl_devvp, bp); 1005 1.52.2.5 jdolecek 1006 1.52.2.5 jdolecek wl->wl_ev_journalwrite.ev_count++; 1007 1.52.2.5 jdolecek 1008 1.52.2.5 jdolecek TAILQ_INSERT_TAIL(&wl->wl_iobufs_busy, bp, b_wapbllist); 1009 1.52.2.5 jdolecek } 1010 1.52.2.5 jdolecek 1011 1.2 simonb /* 1012 1.52.2.5 jdolecek * wapbl_buffered_flush(wl) 1013 1.52.2.5 jdolecek * 1014 1.52.2.5 jdolecek * Flush any buffered writes from wapbl_buffered_write. 1015 1.52.2.3 tls */ 1016 1.52.2.3 tls static int 1017 1.52.2.5 jdolecek wapbl_buffered_flush(struct wapbl *wl, bool full) 1018 1.52.2.3 tls { 1019 1.52.2.5 jdolecek int error = 0; 1020 1.52.2.5 jdolecek struct buf *bp, *bnext; 1021 1.52.2.5 jdolecek bool only_done = true, found = false; 1022 1.52.2.3 tls 1023 1.52.2.5 jdolecek /* if there is outstanding buffered write, send it now */ 1024 1.52.2.5 jdolecek if ((bp = TAILQ_FIRST(&wl->wl_iobufs)) && bp->b_resid > 0) 1025 1.52.2.5 jdolecek wapbl_buffered_write_async(wl, bp); 1026 1.52.2.5 jdolecek 1027 1.52.2.5 jdolecek /* wait for I/O to complete */ 1028 1.52.2.5 jdolecek again: 1029 1.52.2.5 jdolecek TAILQ_FOREACH_SAFE(bp, &wl->wl_iobufs_busy, b_wapbllist, bnext) { 1030 1.52.2.5 jdolecek if (!full && only_done) { 1031 1.52.2.5 jdolecek /* skip unfinished */ 1032 1.52.2.5 jdolecek if (!ISSET(bp->b_oflags, BO_DONE)) 1033 1.52.2.5 jdolecek continue; 1034 1.52.2.5 jdolecek } 1035 1.52.2.5 jdolecek 1036 1.52.2.5 jdolecek if (ISSET(bp->b_oflags, BO_DONE)) 1037 1.52.2.5 jdolecek wl->wl_ev_jbufs_bio_nowait.ev_count++; 1038 1.52.2.5 jdolecek 1039 1.52.2.5 jdolecek TAILQ_REMOVE(&wl->wl_iobufs_busy, bp, b_wapbllist); 1040 1.52.2.5 jdolecek error = biowait(bp); 1041 1.52.2.5 jdolecek 1042 1.52.2.5 jdolecek /* reset for reuse */ 1043 1.52.2.5 jdolecek bp->b_blkno = bp->b_resid = bp->b_flags = 0; 1044 1.52.2.5 jdolecek TAILQ_INSERT_TAIL(&wl->wl_iobufs, bp, b_wapbllist); 1045 1.52.2.5 jdolecek found = true; 1046 1.52.2.5 jdolecek 1047 1.52.2.5 jdolecek if (!full) 1048 1.52.2.5 jdolecek break; 1049 1.52.2.5 jdolecek } 1050 1.52.2.3 tls 1051 1.52.2.5 jdolecek if (!found && only_done && !TAILQ_EMPTY(&wl->wl_iobufs_busy)) { 1052 1.52.2.5 jdolecek only_done = false; 1053 1.52.2.5 jdolecek goto again; 1054 1.52.2.5 jdolecek } 1055 1.52.2.3 tls 1056 1.52.2.3 tls return error; 1057 1.52.2.3 tls } 1058 1.52.2.3 tls 1059 1.52.2.3 tls /* 1060 1.52.2.5 jdolecek * wapbl_buffered_write(data, len, wl, pbn) 1061 1.52.2.5 jdolecek * 1062 1.52.2.5 jdolecek * Write len bytes from data to physical block pbn on 1063 1.52.2.5 jdolecek * wl->wl_devvp. The write may not complete until 1064 1.52.2.5 jdolecek * wapbl_buffered_flush. 1065 1.52.2.3 tls */ 1066 1.52.2.3 tls static int 1067 1.52.2.5 jdolecek wapbl_buffered_write(void *data, size_t len, struct wapbl *wl, daddr_t pbn, 1068 1.52.2.5 jdolecek int bflags) 1069 1.52.2.3 tls { 1070 1.52.2.3 tls size_t resid; 1071 1.52.2.5 jdolecek struct buf *bp; 1072 1.52.2.5 jdolecek 1073 1.52.2.5 jdolecek again: 1074 1.52.2.5 jdolecek bp = TAILQ_FIRST(&wl->wl_iobufs); 1075 1.52.2.5 jdolecek 1076 1.52.2.5 jdolecek if (bp == NULL) { 1077 1.52.2.5 jdolecek /* No more buffers, wait for any previous I/O to finish. */ 1078 1.52.2.5 jdolecek wapbl_buffered_flush(wl, false); 1079 1.52.2.5 jdolecek 1080 1.52.2.5 jdolecek bp = TAILQ_FIRST(&wl->wl_iobufs); 1081 1.52.2.5 jdolecek KASSERT(bp != NULL); 1082 1.52.2.5 jdolecek } 1083 1.52.2.3 tls 1084 1.52.2.3 tls /* 1085 1.52.2.3 tls * If not adjacent to buffered data flush first. Disk block 1086 1.52.2.3 tls * address is always valid for non-empty buffer. 1087 1.52.2.3 tls */ 1088 1.52.2.5 jdolecek if ((bp->b_resid > 0 && pbn != bp->b_blkno + btodb(bp->b_resid))) { 1089 1.52.2.5 jdolecek wapbl_buffered_write_async(wl, bp); 1090 1.52.2.5 jdolecek goto again; 1091 1.52.2.3 tls } 1092 1.52.2.5 jdolecek 1093 1.52.2.3 tls /* 1094 1.52.2.3 tls * If this write goes to an empty buffer we have to 1095 1.52.2.3 tls * save the disk block address first. 1096 1.52.2.3 tls */ 1097 1.52.2.5 jdolecek if (bp->b_blkno == 0) { 1098 1.52.2.5 jdolecek bp->b_blkno = pbn; 1099 1.52.2.5 jdolecek bp->b_flags |= bflags; 1100 1.52.2.5 jdolecek } 1101 1.52.2.5 jdolecek 1102 1.52.2.3 tls /* 1103 1.52.2.5 jdolecek * Remaining space so this buffer ends on a buffer size boundary. 1104 1.52.2.3 tls * 1105 1.52.2.3 tls * Cannot become less or equal zero as the buffer would have been 1106 1.52.2.3 tls * flushed on the last call then. 1107 1.52.2.3 tls */ 1108 1.52.2.5 jdolecek resid = bp->b_bufsize - dbtob(bp->b_blkno % btodb(bp->b_bufsize)) - 1109 1.52.2.5 jdolecek bp->b_resid; 1110 1.52.2.3 tls KASSERT(resid > 0); 1111 1.52.2.3 tls KASSERT(dbtob(btodb(resid)) == resid); 1112 1.52.2.5 jdolecek 1113 1.52.2.5 jdolecek if (len < resid) 1114 1.52.2.5 jdolecek resid = len; 1115 1.52.2.5 jdolecek 1116 1.52.2.5 jdolecek memcpy((uint8_t *)bp->b_data + bp->b_resid, data, resid); 1117 1.52.2.5 jdolecek bp->b_resid += resid; 1118 1.52.2.5 jdolecek 1119 1.52.2.3 tls if (len >= resid) { 1120 1.52.2.5 jdolecek /* Just filled the buf, or data did not fit */ 1121 1.52.2.5 jdolecek wapbl_buffered_write_async(wl, bp); 1122 1.52.2.5 jdolecek 1123 1.52.2.3 tls data = (uint8_t *)data + resid; 1124 1.52.2.3 tls len -= resid; 1125 1.52.2.5 jdolecek pbn += btodb(resid); 1126 1.52.2.5 jdolecek 1127 1.52.2.5 jdolecek if (len > 0) 1128 1.52.2.5 jdolecek goto again; 1129 1.52.2.3 tls } 1130 1.52.2.3 tls 1131 1.52.2.3 tls return 0; 1132 1.52.2.3 tls } 1133 1.52.2.3 tls 1134 1.52.2.3 tls /* 1135 1.52.2.5 jdolecek * wapbl_circ_write(wl, data, len, offp) 1136 1.52.2.5 jdolecek * 1137 1.52.2.5 jdolecek * Write len bytes from data to the circular queue of wl, starting 1138 1.52.2.5 jdolecek * at linear byte offset *offp, and returning the new linear byte 1139 1.52.2.5 jdolecek * offset in *offp. 1140 1.52.2.5 jdolecek * 1141 1.52.2.5 jdolecek * If the starting linear byte offset precedes wl->wl_circ_off, 1142 1.52.2.5 jdolecek * the write instead begins at wl->wl_circ_off. XXX WTF? This 1143 1.52.2.5 jdolecek * should be a KASSERT, not a conditional. 1144 1.52.2.5 jdolecek * 1145 1.52.2.5 jdolecek * The write is buffered in wl and must be flushed with 1146 1.52.2.5 jdolecek * wapbl_buffered_flush before it will be submitted to the disk. 1147 1.2 simonb */ 1148 1.2 simonb static int 1149 1.2 simonb wapbl_circ_write(struct wapbl *wl, void *data, size_t len, off_t *offp) 1150 1.2 simonb { 1151 1.2 simonb size_t slen; 1152 1.2 simonb off_t off = *offp; 1153 1.2 simonb int error; 1154 1.34 mlelstv daddr_t pbn; 1155 1.2 simonb 1156 1.2 simonb KDASSERT(((len >> wl->wl_log_dev_bshift) << 1157 1.2 simonb wl->wl_log_dev_bshift) == len); 1158 1.2 simonb 1159 1.2 simonb if (off < wl->wl_circ_off) 1160 1.2 simonb off = wl->wl_circ_off; 1161 1.2 simonb slen = wl->wl_circ_off + wl->wl_circ_size - off; 1162 1.2 simonb if (slen < len) { 1163 1.34 mlelstv pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift); 1164 1.34 mlelstv #ifdef _KERNEL 1165 1.34 mlelstv pbn = btodb(pbn << wl->wl_log_dev_bshift); 1166 1.34 mlelstv #endif 1167 1.52.2.5 jdolecek error = wapbl_buffered_write(data, slen, wl, pbn, 1168 1.52.2.5 jdolecek WAPBL_JDATA_FLAGS(wl)); 1169 1.2 simonb if (error) 1170 1.2 simonb return error; 1171 1.2 simonb data = (uint8_t *)data + slen; 1172 1.2 simonb len -= slen; 1173 1.2 simonb off = wl->wl_circ_off; 1174 1.2 simonb } 1175 1.34 mlelstv pbn = wl->wl_logpbn + (off >> wl->wl_log_dev_bshift); 1176 1.34 mlelstv #ifdef _KERNEL 1177 1.34 mlelstv pbn = btodb(pbn << wl->wl_log_dev_bshift); 1178 1.34 mlelstv #endif 1179 1.52.2.5 jdolecek error = wapbl_buffered_write(data, len, wl, pbn, 1180 1.52.2.5 jdolecek WAPBL_JDATA_FLAGS(wl)); 1181 1.2 simonb if (error) 1182 1.2 simonb return error; 1183 1.2 simonb off += len; 1184 1.2 simonb if (off >= wl->wl_circ_off + wl->wl_circ_size) 1185 1.2 simonb off = wl->wl_circ_off; 1186 1.2 simonb *offp = off; 1187 1.2 simonb return 0; 1188 1.2 simonb } 1189 1.2 simonb 1190 1.2 simonb /****************************************************************/ 1191 1.52.2.5 jdolecek /* 1192 1.52.2.5 jdolecek * WAPBL transactions: entering, adding/removing bufs, and exiting 1193 1.52.2.5 jdolecek */ 1194 1.2 simonb 1195 1.2 simonb int 1196 1.2 simonb wapbl_begin(struct wapbl *wl, const char *file, int line) 1197 1.2 simonb { 1198 1.2 simonb int doflush; 1199 1.2 simonb unsigned lockcount; 1200 1.52.2.1 tls uint32_t maxphys; 1201 1.2 simonb 1202 1.2 simonb KDASSERT(wl); 1203 1.2 simonb 1204 1.2 simonb /* 1205 1.2 simonb * XXX this needs to be made much more sophisticated. 1206 1.2 simonb * perhaps each wapbl_begin could reserve a specified 1207 1.2 simonb * number of buffers and bytes. 1208 1.2 simonb */ 1209 1.2 simonb mutex_enter(&wl->wl_mtx); 1210 1.2 simonb lockcount = wl->wl_lock_count; 1211 1.52.2.1 tls maxphys = wl->wl_mount->mnt_maxphys; 1212 1.52.2.1 tls doflush = ((wl->wl_bufbytes + (lockcount * maxphys)) > 1213 1.2 simonb wl->wl_bufbytes_max / 2) || 1214 1.2 simonb ((wl->wl_bufcount + (lockcount * 10)) > 1215 1.2 simonb wl->wl_bufcount_max / 2) || 1216 1.28 pooka (wapbl_transaction_len(wl) > wl->wl_circ_size / 2) || 1217 1.42 hannken (wl->wl_dealloccnt >= (wl->wl_dealloclim / 2)); 1218 1.2 simonb mutex_exit(&wl->wl_mtx); 1219 1.2 simonb 1220 1.2 simonb if (doflush) { 1221 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1222 1.2 simonb ("force flush lockcnt=%d bufbytes=%zu " 1223 1.28 pooka "(max=%zu) bufcount=%zu (max=%zu) " 1224 1.28 pooka "dealloccnt %d (lim=%d)\n", 1225 1.2 simonb lockcount, wl->wl_bufbytes, 1226 1.2 simonb wl->wl_bufbytes_max, wl->wl_bufcount, 1227 1.28 pooka wl->wl_bufcount_max, 1228 1.28 pooka wl->wl_dealloccnt, wl->wl_dealloclim)); 1229 1.2 simonb } 1230 1.2 simonb 1231 1.2 simonb if (doflush) { 1232 1.2 simonb int error = wapbl_flush(wl, 0); 1233 1.2 simonb if (error) 1234 1.2 simonb return error; 1235 1.2 simonb } 1236 1.2 simonb 1237 1.23 ad rw_enter(&wl->wl_rwlock, RW_READER); 1238 1.2 simonb mutex_enter(&wl->wl_mtx); 1239 1.2 simonb wl->wl_lock_count++; 1240 1.2 simonb mutex_exit(&wl->wl_mtx); 1241 1.2 simonb 1242 1.23 ad #if defined(WAPBL_DEBUG_PRINT) 1243 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 1244 1.2 simonb ("wapbl_begin thread %d.%d with bufcount=%zu " 1245 1.2 simonb "bufbytes=%zu bcount=%zu at %s:%d\n", 1246 1.2 simonb curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1247 1.2 simonb wl->wl_bufbytes, wl->wl_bcount, file, line)); 1248 1.2 simonb #endif 1249 1.2 simonb 1250 1.2 simonb return 0; 1251 1.2 simonb } 1252 1.2 simonb 1253 1.2 simonb void 1254 1.2 simonb wapbl_end(struct wapbl *wl) 1255 1.2 simonb { 1256 1.2 simonb 1257 1.23 ad #if defined(WAPBL_DEBUG_PRINT) 1258 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_TRANSACTION, 1259 1.2 simonb ("wapbl_end thread %d.%d with bufcount=%zu " 1260 1.2 simonb "bufbytes=%zu bcount=%zu\n", 1261 1.2 simonb curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1262 1.2 simonb wl->wl_bufbytes, wl->wl_bcount)); 1263 1.2 simonb #endif 1264 1.2 simonb 1265 1.52.2.5 jdolecek /* 1266 1.52.2.5 jdolecek * XXX this could be handled more gracefully, perhaps place 1267 1.52.2.5 jdolecek * only a partial transaction in the log and allow the 1268 1.52.2.5 jdolecek * remaining to flush without the protection of the journal. 1269 1.52.2.5 jdolecek */ 1270 1.52.2.5 jdolecek KASSERTMSG((wapbl_transaction_len(wl) <= 1271 1.52.2.5 jdolecek (wl->wl_circ_size - wl->wl_reserved_bytes)), 1272 1.52.2.5 jdolecek "wapbl_end: current transaction too big to flush"); 1273 1.40 bouyer 1274 1.2 simonb mutex_enter(&wl->wl_mtx); 1275 1.2 simonb KASSERT(wl->wl_lock_count > 0); 1276 1.2 simonb wl->wl_lock_count--; 1277 1.2 simonb mutex_exit(&wl->wl_mtx); 1278 1.2 simonb 1279 1.2 simonb rw_exit(&wl->wl_rwlock); 1280 1.2 simonb } 1281 1.2 simonb 1282 1.2 simonb void 1283 1.2 simonb wapbl_add_buf(struct wapbl *wl, struct buf * bp) 1284 1.2 simonb { 1285 1.2 simonb 1286 1.2 simonb KASSERT(bp->b_cflags & BC_BUSY); 1287 1.2 simonb KASSERT(bp->b_vp); 1288 1.2 simonb 1289 1.2 simonb wapbl_jlock_assert(wl); 1290 1.2 simonb 1291 1.2 simonb #if 0 1292 1.2 simonb /* 1293 1.2 simonb * XXX this might be an issue for swapfiles. 1294 1.2 simonb * see uvm_swap.c:1702 1295 1.2 simonb * 1296 1.2 simonb * XXX2 why require it then? leap of semantics? 1297 1.2 simonb */ 1298 1.2 simonb KASSERT((bp->b_cflags & BC_NOCACHE) == 0); 1299 1.2 simonb #endif 1300 1.2 simonb 1301 1.2 simonb mutex_enter(&wl->wl_mtx); 1302 1.2 simonb if (bp->b_flags & B_LOCKED) { 1303 1.52.2.5 jdolecek TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist); 1304 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_BUFFER2, 1305 1.2 simonb ("wapbl_add_buf thread %d.%d re-adding buf %p " 1306 1.2 simonb "with %d bytes %d bcount\n", 1307 1.2 simonb curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 1308 1.2 simonb bp->b_bcount)); 1309 1.2 simonb } else { 1310 1.2 simonb /* unlocked by dirty buffers shouldn't exist */ 1311 1.2 simonb KASSERT(!(bp->b_oflags & BO_DELWRI)); 1312 1.2 simonb wl->wl_bufbytes += bp->b_bufsize; 1313 1.2 simonb wl->wl_bcount += bp->b_bcount; 1314 1.2 simonb wl->wl_bufcount++; 1315 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 1316 1.2 simonb ("wapbl_add_buf thread %d.%d adding buf %p " 1317 1.2 simonb "with %d bytes %d bcount\n", 1318 1.2 simonb curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, 1319 1.2 simonb bp->b_bcount)); 1320 1.2 simonb } 1321 1.52.2.5 jdolecek TAILQ_INSERT_TAIL(&wl->wl_bufs, bp, b_wapbllist); 1322 1.2 simonb mutex_exit(&wl->wl_mtx); 1323 1.2 simonb 1324 1.2 simonb bp->b_flags |= B_LOCKED; 1325 1.2 simonb } 1326 1.2 simonb 1327 1.2 simonb static void 1328 1.2 simonb wapbl_remove_buf_locked(struct wapbl * wl, struct buf *bp) 1329 1.2 simonb { 1330 1.2 simonb 1331 1.2 simonb KASSERT(mutex_owned(&wl->wl_mtx)); 1332 1.2 simonb KASSERT(bp->b_cflags & BC_BUSY); 1333 1.2 simonb wapbl_jlock_assert(wl); 1334 1.2 simonb 1335 1.2 simonb #if 0 1336 1.2 simonb /* 1337 1.2 simonb * XXX this might be an issue for swapfiles. 1338 1.2 simonb * see uvm_swap.c:1725 1339 1.2 simonb * 1340 1.2 simonb * XXXdeux: see above 1341 1.2 simonb */ 1342 1.2 simonb KASSERT((bp->b_flags & BC_NOCACHE) == 0); 1343 1.2 simonb #endif 1344 1.2 simonb KASSERT(bp->b_flags & B_LOCKED); 1345 1.2 simonb 1346 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_BUFFER, 1347 1.2 simonb ("wapbl_remove_buf thread %d.%d removing buf %p with " 1348 1.2 simonb "%d bytes %d bcount\n", 1349 1.2 simonb curproc->p_pid, curlwp->l_lid, bp, bp->b_bufsize, bp->b_bcount)); 1350 1.2 simonb 1351 1.2 simonb KASSERT(wl->wl_bufbytes >= bp->b_bufsize); 1352 1.2 simonb wl->wl_bufbytes -= bp->b_bufsize; 1353 1.2 simonb KASSERT(wl->wl_bcount >= bp->b_bcount); 1354 1.2 simonb wl->wl_bcount -= bp->b_bcount; 1355 1.2 simonb KASSERT(wl->wl_bufcount > 0); 1356 1.2 simonb wl->wl_bufcount--; 1357 1.2 simonb KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1358 1.2 simonb KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1359 1.52.2.5 jdolecek TAILQ_REMOVE(&wl->wl_bufs, bp, b_wapbllist); 1360 1.2 simonb 1361 1.2 simonb bp->b_flags &= ~B_LOCKED; 1362 1.2 simonb } 1363 1.2 simonb 1364 1.2 simonb /* called from brelsel() in vfs_bio among other places */ 1365 1.2 simonb void 1366 1.2 simonb wapbl_remove_buf(struct wapbl * wl, struct buf *bp) 1367 1.2 simonb { 1368 1.2 simonb 1369 1.2 simonb mutex_enter(&wl->wl_mtx); 1370 1.2 simonb wapbl_remove_buf_locked(wl, bp); 1371 1.2 simonb mutex_exit(&wl->wl_mtx); 1372 1.2 simonb } 1373 1.2 simonb 1374 1.2 simonb void 1375 1.2 simonb wapbl_resize_buf(struct wapbl *wl, struct buf *bp, long oldsz, long oldcnt) 1376 1.2 simonb { 1377 1.2 simonb 1378 1.2 simonb KASSERT(bp->b_cflags & BC_BUSY); 1379 1.2 simonb 1380 1.2 simonb /* 1381 1.2 simonb * XXX: why does this depend on B_LOCKED? otherwise the buf 1382 1.2 simonb * is not for a transaction? if so, why is this called in the 1383 1.2 simonb * first place? 1384 1.2 simonb */ 1385 1.2 simonb if (bp->b_flags & B_LOCKED) { 1386 1.2 simonb mutex_enter(&wl->wl_mtx); 1387 1.2 simonb wl->wl_bufbytes += bp->b_bufsize - oldsz; 1388 1.2 simonb wl->wl_bcount += bp->b_bcount - oldcnt; 1389 1.2 simonb mutex_exit(&wl->wl_mtx); 1390 1.2 simonb } 1391 1.2 simonb } 1392 1.2 simonb 1393 1.2 simonb #endif /* _KERNEL */ 1394 1.2 simonb 1395 1.2 simonb /****************************************************************/ 1396 1.2 simonb /* Some utility inlines */ 1397 1.2 simonb 1398 1.52.2.5 jdolecek /* 1399 1.52.2.5 jdolecek * wapbl_space_used(avail, head, tail) 1400 1.52.2.5 jdolecek * 1401 1.52.2.5 jdolecek * Number of bytes used in a circular queue of avail total bytes, 1402 1.52.2.5 jdolecek * from tail to head. 1403 1.52.2.5 jdolecek */ 1404 1.52.2.4 tls static inline size_t 1405 1.52.2.4 tls wapbl_space_used(size_t avail, off_t head, off_t tail) 1406 1.52.2.4 tls { 1407 1.52.2.4 tls 1408 1.52.2.4 tls if (tail == 0) { 1409 1.52.2.4 tls KASSERT(head == 0); 1410 1.52.2.4 tls return 0; 1411 1.52.2.4 tls } 1412 1.52.2.4 tls return ((head + (avail - 1) - tail) % avail) + 1; 1413 1.52.2.4 tls } 1414 1.52.2.4 tls 1415 1.52.2.4 tls #ifdef _KERNEL 1416 1.52.2.5 jdolecek /* 1417 1.52.2.5 jdolecek * wapbl_advance(size, off, oldoff, delta) 1418 1.52.2.5 jdolecek * 1419 1.52.2.5 jdolecek * Given a byte offset oldoff into a circular queue of size bytes 1420 1.52.2.5 jdolecek * starting at off, return a new byte offset oldoff + delta into 1421 1.52.2.5 jdolecek * the circular queue. 1422 1.52.2.5 jdolecek */ 1423 1.30 uebayasi static inline off_t 1424 1.52.2.5 jdolecek wapbl_advance(size_t size, size_t off, off_t oldoff, size_t delta) 1425 1.2 simonb { 1426 1.52.2.5 jdolecek off_t newoff; 1427 1.2 simonb 1428 1.2 simonb /* Define acceptable ranges for inputs. */ 1429 1.46 christos KASSERT(delta <= (size_t)size); 1430 1.52.2.5 jdolecek KASSERT((oldoff == 0) || ((size_t)oldoff >= off)); 1431 1.52.2.5 jdolecek KASSERT(oldoff < (off_t)(size + off)); 1432 1.2 simonb 1433 1.52.2.5 jdolecek if ((oldoff == 0) && (delta != 0)) 1434 1.52.2.5 jdolecek newoff = off + delta; 1435 1.52.2.5 jdolecek else if ((oldoff + delta) < (size + off)) 1436 1.52.2.5 jdolecek newoff = oldoff + delta; 1437 1.2 simonb else 1438 1.52.2.5 jdolecek newoff = (oldoff + delta) - size; 1439 1.2 simonb 1440 1.2 simonb /* Note some interesting axioms */ 1441 1.52.2.5 jdolecek KASSERT((delta != 0) || (newoff == oldoff)); 1442 1.52.2.5 jdolecek KASSERT((delta == 0) || (newoff != 0)); 1443 1.52.2.5 jdolecek KASSERT((delta != (size)) || (newoff == oldoff)); 1444 1.2 simonb 1445 1.2 simonb /* Define acceptable ranges for output. */ 1446 1.52.2.5 jdolecek KASSERT((newoff == 0) || ((size_t)newoff >= off)); 1447 1.52.2.5 jdolecek KASSERT((size_t)newoff < (size + off)); 1448 1.52.2.5 jdolecek return newoff; 1449 1.2 simonb } 1450 1.2 simonb 1451 1.52.2.5 jdolecek /* 1452 1.52.2.5 jdolecek * wapbl_space_free(avail, head, tail) 1453 1.52.2.5 jdolecek * 1454 1.52.2.5 jdolecek * Number of bytes free in a circular queue of avail total bytes, 1455 1.52.2.5 jdolecek * in which everything from tail to head is used. 1456 1.52.2.5 jdolecek */ 1457 1.30 uebayasi static inline size_t 1458 1.2 simonb wapbl_space_free(size_t avail, off_t head, off_t tail) 1459 1.2 simonb { 1460 1.2 simonb 1461 1.2 simonb return avail - wapbl_space_used(avail, head, tail); 1462 1.2 simonb } 1463 1.2 simonb 1464 1.52.2.5 jdolecek /* 1465 1.52.2.5 jdolecek * wapbl_advance_head(size, off, delta, headp, tailp) 1466 1.52.2.5 jdolecek * 1467 1.52.2.5 jdolecek * In a circular queue of size bytes starting at off, given the 1468 1.52.2.5 jdolecek * old head and tail offsets *headp and *tailp, store the new head 1469 1.52.2.5 jdolecek * and tail offsets in *headp and *tailp resulting from adding 1470 1.52.2.5 jdolecek * delta bytes of data to the head. 1471 1.52.2.5 jdolecek */ 1472 1.30 uebayasi static inline void 1473 1.2 simonb wapbl_advance_head(size_t size, size_t off, size_t delta, off_t *headp, 1474 1.2 simonb off_t *tailp) 1475 1.2 simonb { 1476 1.2 simonb off_t head = *headp; 1477 1.2 simonb off_t tail = *tailp; 1478 1.2 simonb 1479 1.2 simonb KASSERT(delta <= wapbl_space_free(size, head, tail)); 1480 1.2 simonb head = wapbl_advance(size, off, head, delta); 1481 1.2 simonb if ((tail == 0) && (head != 0)) 1482 1.2 simonb tail = off; 1483 1.2 simonb *headp = head; 1484 1.2 simonb *tailp = tail; 1485 1.2 simonb } 1486 1.2 simonb 1487 1.52.2.5 jdolecek /* 1488 1.52.2.5 jdolecek * wapbl_advance_tail(size, off, delta, headp, tailp) 1489 1.52.2.5 jdolecek * 1490 1.52.2.5 jdolecek * In a circular queue of size bytes starting at off, given the 1491 1.52.2.5 jdolecek * old head and tail offsets *headp and *tailp, store the new head 1492 1.52.2.5 jdolecek * and tail offsets in *headp and *tailp resulting from removing 1493 1.52.2.5 jdolecek * delta bytes of data from the tail. 1494 1.52.2.5 jdolecek */ 1495 1.30 uebayasi static inline void 1496 1.2 simonb wapbl_advance_tail(size_t size, size_t off, size_t delta, off_t *headp, 1497 1.2 simonb off_t *tailp) 1498 1.2 simonb { 1499 1.2 simonb off_t head = *headp; 1500 1.2 simonb off_t tail = *tailp; 1501 1.2 simonb 1502 1.2 simonb KASSERT(delta <= wapbl_space_used(size, head, tail)); 1503 1.2 simonb tail = wapbl_advance(size, off, tail, delta); 1504 1.2 simonb if (head == tail) { 1505 1.2 simonb head = tail = 0; 1506 1.2 simonb } 1507 1.2 simonb *headp = head; 1508 1.2 simonb *tailp = tail; 1509 1.2 simonb } 1510 1.2 simonb 1511 1.2 simonb 1512 1.2 simonb /****************************************************************/ 1513 1.2 simonb 1514 1.2 simonb /* 1515 1.52.2.5 jdolecek * wapbl_truncate(wl, minfree) 1516 1.52.2.5 jdolecek * 1517 1.52.2.5 jdolecek * Wait until at least minfree bytes are available in the log. 1518 1.52.2.5 jdolecek * 1519 1.52.2.5 jdolecek * If it was necessary to wait for writes to complete, 1520 1.52.2.5 jdolecek * advance the circular queue tail to reflect the new write 1521 1.52.2.5 jdolecek * completions and issue a write commit to the log. 1522 1.52.2.5 jdolecek * 1523 1.52.2.5 jdolecek * => Caller must hold wl->wl_rwlock writer lock. 1524 1.2 simonb */ 1525 1.2 simonb static int 1526 1.52.2.5 jdolecek wapbl_truncate(struct wapbl *wl, size_t minfree) 1527 1.2 simonb { 1528 1.2 simonb size_t delta; 1529 1.2 simonb size_t avail; 1530 1.2 simonb off_t head; 1531 1.2 simonb off_t tail; 1532 1.2 simonb int error = 0; 1533 1.2 simonb 1534 1.2 simonb KASSERT(minfree <= (wl->wl_circ_size - wl->wl_reserved_bytes)); 1535 1.2 simonb KASSERT(rw_write_held(&wl->wl_rwlock)); 1536 1.2 simonb 1537 1.2 simonb mutex_enter(&wl->wl_mtx); 1538 1.2 simonb 1539 1.2 simonb /* 1540 1.2 simonb * First check to see if we have to do a commit 1541 1.2 simonb * at all. 1542 1.2 simonb */ 1543 1.2 simonb avail = wapbl_space_free(wl->wl_circ_size, wl->wl_head, wl->wl_tail); 1544 1.2 simonb if (minfree < avail) { 1545 1.2 simonb mutex_exit(&wl->wl_mtx); 1546 1.2 simonb return 0; 1547 1.2 simonb } 1548 1.2 simonb minfree -= avail; 1549 1.2 simonb while ((wl->wl_error_count == 0) && 1550 1.2 simonb (wl->wl_reclaimable_bytes < minfree)) { 1551 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1552 1.2 simonb ("wapbl_truncate: sleeping on %p wl=%p bytes=%zd " 1553 1.2 simonb "minfree=%zd\n", 1554 1.2 simonb &wl->wl_reclaimable_bytes, wl, wl->wl_reclaimable_bytes, 1555 1.2 simonb minfree)); 1556 1.2 simonb 1557 1.2 simonb cv_wait(&wl->wl_reclaimable_cv, &wl->wl_mtx); 1558 1.2 simonb } 1559 1.2 simonb if (wl->wl_reclaimable_bytes < minfree) { 1560 1.2 simonb KASSERT(wl->wl_error_count); 1561 1.2 simonb /* XXX maybe get actual error from buffer instead someday? */ 1562 1.2 simonb error = EIO; 1563 1.2 simonb } 1564 1.2 simonb head = wl->wl_head; 1565 1.2 simonb tail = wl->wl_tail; 1566 1.2 simonb delta = wl->wl_reclaimable_bytes; 1567 1.2 simonb 1568 1.2 simonb /* If all of of the entries are flushed, then be sure to keep 1569 1.2 simonb * the reserved bytes reserved. Watch out for discarded transactions, 1570 1.2 simonb * which could leave more bytes reserved than are reclaimable. 1571 1.2 simonb */ 1572 1.52.2.5 jdolecek if (SIMPLEQ_EMPTY(&wl->wl_entries) && 1573 1.2 simonb (delta >= wl->wl_reserved_bytes)) { 1574 1.2 simonb delta -= wl->wl_reserved_bytes; 1575 1.2 simonb } 1576 1.2 simonb wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, &head, 1577 1.2 simonb &tail); 1578 1.2 simonb KDASSERT(wl->wl_reserved_bytes <= 1579 1.2 simonb wapbl_space_used(wl->wl_circ_size, head, tail)); 1580 1.2 simonb mutex_exit(&wl->wl_mtx); 1581 1.2 simonb 1582 1.2 simonb if (error) 1583 1.2 simonb return error; 1584 1.2 simonb 1585 1.2 simonb /* 1586 1.2 simonb * This is where head, tail and delta are unprotected 1587 1.2 simonb * from races against itself or flush. This is ok since 1588 1.2 simonb * we only call this routine from inside flush itself. 1589 1.2 simonb * 1590 1.2 simonb * XXX: how can it race against itself when accessed only 1591 1.2 simonb * from behind the write-locked rwlock? 1592 1.2 simonb */ 1593 1.2 simonb error = wapbl_write_commit(wl, head, tail); 1594 1.2 simonb if (error) 1595 1.2 simonb return error; 1596 1.2 simonb 1597 1.2 simonb wl->wl_head = head; 1598 1.2 simonb wl->wl_tail = tail; 1599 1.2 simonb 1600 1.2 simonb mutex_enter(&wl->wl_mtx); 1601 1.2 simonb KASSERT(wl->wl_reclaimable_bytes >= delta); 1602 1.2 simonb wl->wl_reclaimable_bytes -= delta; 1603 1.2 simonb mutex_exit(&wl->wl_mtx); 1604 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_TRUNCATE, 1605 1.2 simonb ("wapbl_truncate thread %d.%d truncating %zu bytes\n", 1606 1.2 simonb curproc->p_pid, curlwp->l_lid, delta)); 1607 1.2 simonb 1608 1.2 simonb return 0; 1609 1.2 simonb } 1610 1.2 simonb 1611 1.2 simonb /****************************************************************/ 1612 1.2 simonb 1613 1.2 simonb void 1614 1.2 simonb wapbl_biodone(struct buf *bp) 1615 1.2 simonb { 1616 1.2 simonb struct wapbl_entry *we = bp->b_private; 1617 1.2 simonb struct wapbl *wl = we->we_wapbl; 1618 1.52.2.2 tls #ifdef WAPBL_DEBUG_BUFBYTES 1619 1.52.2.2 tls const int bufsize = bp->b_bufsize; 1620 1.52.2.2 tls #endif 1621 1.2 simonb 1622 1.2 simonb /* 1623 1.2 simonb * Handle possible flushing of buffers after log has been 1624 1.2 simonb * decomissioned. 1625 1.2 simonb */ 1626 1.2 simonb if (!wl) { 1627 1.2 simonb KASSERT(we->we_bufcount > 0); 1628 1.2 simonb we->we_bufcount--; 1629 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1630 1.52.2.2 tls KASSERT(we->we_unsynced_bufbytes >= bufsize); 1631 1.52.2.2 tls we->we_unsynced_bufbytes -= bufsize; 1632 1.2 simonb #endif 1633 1.2 simonb 1634 1.2 simonb if (we->we_bufcount == 0) { 1635 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1636 1.2 simonb KASSERT(we->we_unsynced_bufbytes == 0); 1637 1.2 simonb #endif 1638 1.51 para pool_put(&wapbl_entry_pool, we); 1639 1.2 simonb } 1640 1.2 simonb 1641 1.2 simonb brelse(bp, 0); 1642 1.2 simonb return; 1643 1.2 simonb } 1644 1.2 simonb 1645 1.2 simonb #ifdef ohbother 1646 1.44 uebayasi KDASSERT(bp->b_oflags & BO_DONE); 1647 1.44 uebayasi KDASSERT(!(bp->b_oflags & BO_DELWRI)); 1648 1.2 simonb KDASSERT(bp->b_flags & B_ASYNC); 1649 1.44 uebayasi KDASSERT(bp->b_cflags & BC_BUSY); 1650 1.2 simonb KDASSERT(!(bp->b_flags & B_LOCKED)); 1651 1.2 simonb KDASSERT(!(bp->b_flags & B_READ)); 1652 1.44 uebayasi KDASSERT(!(bp->b_cflags & BC_INVAL)); 1653 1.44 uebayasi KDASSERT(!(bp->b_cflags & BC_NOCACHE)); 1654 1.2 simonb #endif 1655 1.2 simonb 1656 1.2 simonb if (bp->b_error) { 1657 1.2 simonb /* 1658 1.52.2.5 jdolecek * If an error occurs, it would be nice to leave the buffer 1659 1.52.2.5 jdolecek * as a delayed write on the LRU queue so that we can retry 1660 1.52.2.5 jdolecek * it later. But buffercache(9) can't handle dirty buffer 1661 1.52.2.5 jdolecek * reuse, so just mark the log permanently errored out. 1662 1.2 simonb */ 1663 1.2 simonb mutex_enter(&wl->wl_mtx); 1664 1.2 simonb if (wl->wl_error_count == 0) { 1665 1.2 simonb wl->wl_error_count++; 1666 1.2 simonb cv_broadcast(&wl->wl_reclaimable_cv); 1667 1.2 simonb } 1668 1.2 simonb mutex_exit(&wl->wl_mtx); 1669 1.2 simonb } 1670 1.2 simonb 1671 1.52.2.2 tls /* 1672 1.52.2.5 jdolecek * Make sure that the buf doesn't retain the media flags, so that 1673 1.52.2.5 jdolecek * e.g. wapbl_allow_fuadpo has immediate effect on any following I/O. 1674 1.52.2.5 jdolecek * The flags will be set again if needed by another I/O. 1675 1.52.2.5 jdolecek */ 1676 1.52.2.5 jdolecek bp->b_flags &= ~B_MEDIA_FLAGS; 1677 1.52.2.5 jdolecek 1678 1.52.2.5 jdolecek /* 1679 1.52.2.2 tls * Release the buffer here. wapbl_flush() may wait for the 1680 1.52.2.2 tls * log to become empty and we better unbusy the buffer before 1681 1.52.2.2 tls * wapbl_flush() returns. 1682 1.52.2.2 tls */ 1683 1.52.2.2 tls brelse(bp, 0); 1684 1.52.2.2 tls 1685 1.2 simonb mutex_enter(&wl->wl_mtx); 1686 1.2 simonb 1687 1.2 simonb KASSERT(we->we_bufcount > 0); 1688 1.2 simonb we->we_bufcount--; 1689 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1690 1.52.2.2 tls KASSERT(we->we_unsynced_bufbytes >= bufsize); 1691 1.52.2.2 tls we->we_unsynced_bufbytes -= bufsize; 1692 1.52.2.2 tls KASSERT(wl->wl_unsynced_bufbytes >= bufsize); 1693 1.52.2.2 tls wl->wl_unsynced_bufbytes -= bufsize; 1694 1.2 simonb #endif 1695 1.52.2.5 jdolecek wl->wl_ev_metawrite.ev_count++; 1696 1.2 simonb 1697 1.2 simonb /* 1698 1.2 simonb * If the current transaction can be reclaimed, start 1699 1.2 simonb * at the beginning and reclaim any consecutive reclaimable 1700 1.2 simonb * transactions. If we successfully reclaim anything, 1701 1.2 simonb * then wakeup anyone waiting for the reclaim. 1702 1.2 simonb */ 1703 1.2 simonb if (we->we_bufcount == 0) { 1704 1.2 simonb size_t delta = 0; 1705 1.2 simonb int errcnt = 0; 1706 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1707 1.2 simonb KDASSERT(we->we_unsynced_bufbytes == 0); 1708 1.2 simonb #endif 1709 1.2 simonb /* 1710 1.2 simonb * clear any posted error, since the buffer it came from 1711 1.2 simonb * has successfully flushed by now 1712 1.2 simonb */ 1713 1.2 simonb while ((we = SIMPLEQ_FIRST(&wl->wl_entries)) && 1714 1.2 simonb (we->we_bufcount == 0)) { 1715 1.2 simonb delta += we->we_reclaimable_bytes; 1716 1.2 simonb if (we->we_error) 1717 1.2 simonb errcnt++; 1718 1.2 simonb SIMPLEQ_REMOVE_HEAD(&wl->wl_entries, we_entries); 1719 1.51 para pool_put(&wapbl_entry_pool, we); 1720 1.2 simonb } 1721 1.2 simonb 1722 1.2 simonb if (delta) { 1723 1.2 simonb wl->wl_reclaimable_bytes += delta; 1724 1.2 simonb KASSERT(wl->wl_error_count >= errcnt); 1725 1.2 simonb wl->wl_error_count -= errcnt; 1726 1.2 simonb cv_broadcast(&wl->wl_reclaimable_cv); 1727 1.2 simonb } 1728 1.2 simonb } 1729 1.2 simonb 1730 1.2 simonb mutex_exit(&wl->wl_mtx); 1731 1.2 simonb } 1732 1.2 simonb 1733 1.2 simonb /* 1734 1.52.2.5 jdolecek * wapbl_flush(wl, wait) 1735 1.52.2.5 jdolecek * 1736 1.52.2.5 jdolecek * Flush pending block writes, deallocations, and inodes from 1737 1.52.2.5 jdolecek * the current transaction in memory to the log on disk: 1738 1.52.2.5 jdolecek * 1739 1.52.2.5 jdolecek * 1. Call the file system's wl_flush callback to flush any 1740 1.52.2.5 jdolecek * per-file-system pending updates. 1741 1.52.2.5 jdolecek * 2. Wait for enough space in the log for the current transaction. 1742 1.52.2.5 jdolecek * 3. Synchronously write the new log records, advancing the 1743 1.52.2.5 jdolecek * circular queue head. 1744 1.52.2.5 jdolecek * 4. Issue the pending block writes asynchronously, now that they 1745 1.52.2.5 jdolecek * are recorded in the log and can be replayed after crash. 1746 1.52.2.5 jdolecek * 5. If wait is true, wait for all writes to complete and for the 1747 1.52.2.5 jdolecek * log to become empty. 1748 1.52.2.5 jdolecek * 1749 1.52.2.5 jdolecek * On failure, call the file system's wl_flush_abort callback. 1750 1.2 simonb */ 1751 1.2 simonb int 1752 1.2 simonb wapbl_flush(struct wapbl *wl, int waitfor) 1753 1.2 simonb { 1754 1.2 simonb struct buf *bp; 1755 1.2 simonb struct wapbl_entry *we; 1756 1.2 simonb off_t off; 1757 1.2 simonb off_t head; 1758 1.2 simonb off_t tail; 1759 1.2 simonb size_t delta = 0; 1760 1.2 simonb size_t flushsize; 1761 1.2 simonb size_t reserved; 1762 1.2 simonb int error = 0; 1763 1.2 simonb 1764 1.2 simonb /* 1765 1.2 simonb * Do a quick check to see if a full flush can be skipped 1766 1.2 simonb * This assumes that the flush callback does not need to be called 1767 1.2 simonb * unless there are other outstanding bufs. 1768 1.2 simonb */ 1769 1.2 simonb if (!waitfor) { 1770 1.2 simonb size_t nbufs; 1771 1.2 simonb mutex_enter(&wl->wl_mtx); /* XXX need mutex here to 1772 1.2 simonb protect the KASSERTS */ 1773 1.2 simonb nbufs = wl->wl_bufcount; 1774 1.2 simonb KASSERT((wl->wl_bufcount == 0) == (wl->wl_bufbytes == 0)); 1775 1.2 simonb KASSERT((wl->wl_bufcount == 0) == (wl->wl_bcount == 0)); 1776 1.2 simonb mutex_exit(&wl->wl_mtx); 1777 1.2 simonb if (nbufs == 0) 1778 1.2 simonb return 0; 1779 1.2 simonb } 1780 1.2 simonb 1781 1.2 simonb /* 1782 1.2 simonb * XXX we may consider using LK_UPGRADE here 1783 1.2 simonb * if we want to call flush from inside a transaction 1784 1.2 simonb */ 1785 1.2 simonb rw_enter(&wl->wl_rwlock, RW_WRITER); 1786 1.52.2.5 jdolecek wl->wl_flush(wl->wl_mount, TAILQ_FIRST(&wl->wl_dealloclist)); 1787 1.2 simonb 1788 1.2 simonb /* 1789 1.52.2.5 jdolecek * Now that we are exclusively locked and the file system has 1790 1.52.2.5 jdolecek * issued any deferred block writes for this transaction, check 1791 1.52.2.5 jdolecek * whether there are any blocks to write to the log. If not, 1792 1.52.2.5 jdolecek * skip waiting for space or writing any log entries. 1793 1.52.2.5 jdolecek * 1794 1.52.2.5 jdolecek * XXX Shouldn't this also check wl_dealloccnt and 1795 1.52.2.5 jdolecek * wl_inohashcnt? Perhaps wl_dealloccnt doesn't matter if the 1796 1.52.2.5 jdolecek * file system didn't produce any blocks as a consequence of 1797 1.52.2.5 jdolecek * it, but the same does not seem to be so of wl_inohashcnt. 1798 1.2 simonb */ 1799 1.2 simonb if (wl->wl_bufcount == 0) { 1800 1.52.2.5 jdolecek goto wait_out; 1801 1.2 simonb } 1802 1.2 simonb 1803 1.2 simonb #if 0 1804 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1805 1.2 simonb ("wapbl_flush thread %d.%d flushing entries with " 1806 1.2 simonb "bufcount=%zu bufbytes=%zu\n", 1807 1.2 simonb curproc->p_pid, curlwp->l_lid, wl->wl_bufcount, 1808 1.2 simonb wl->wl_bufbytes)); 1809 1.2 simonb #endif 1810 1.2 simonb 1811 1.2 simonb /* Calculate amount of space needed to flush */ 1812 1.2 simonb flushsize = wapbl_transaction_len(wl); 1813 1.39 christos if (wapbl_verbose_commit) { 1814 1.39 christos struct timespec ts; 1815 1.39 christos getnanotime(&ts); 1816 1.43 nakayama printf("%s: %lld.%09ld this transaction = %zu bytes\n", 1817 1.39 christos __func__, (long long)ts.tv_sec, 1818 1.39 christos (long)ts.tv_nsec, flushsize); 1819 1.39 christos } 1820 1.2 simonb 1821 1.2 simonb if (flushsize > (wl->wl_circ_size - wl->wl_reserved_bytes)) { 1822 1.2 simonb /* 1823 1.2 simonb * XXX this could be handled more gracefully, perhaps place 1824 1.2 simonb * only a partial transaction in the log and allow the 1825 1.2 simonb * remaining to flush without the protection of the journal. 1826 1.2 simonb */ 1827 1.52.2.5 jdolecek panic("wapbl_flush: current transaction too big to flush"); 1828 1.2 simonb } 1829 1.2 simonb 1830 1.52.2.5 jdolecek error = wapbl_truncate(wl, flushsize); 1831 1.2 simonb if (error) 1832 1.52.2.5 jdolecek goto out; 1833 1.2 simonb 1834 1.2 simonb off = wl->wl_head; 1835 1.52.2.5 jdolecek KASSERT((off == 0) || (off >= wl->wl_circ_off)); 1836 1.52.2.5 jdolecek KASSERT((off == 0) || (off < wl->wl_circ_off + wl->wl_circ_size)); 1837 1.2 simonb error = wapbl_write_blocks(wl, &off); 1838 1.2 simonb if (error) 1839 1.52.2.5 jdolecek goto out; 1840 1.2 simonb error = wapbl_write_revocations(wl, &off); 1841 1.2 simonb if (error) 1842 1.52.2.5 jdolecek goto out; 1843 1.2 simonb error = wapbl_write_inodes(wl, &off); 1844 1.2 simonb if (error) 1845 1.52.2.5 jdolecek goto out; 1846 1.2 simonb 1847 1.2 simonb reserved = 0; 1848 1.2 simonb if (wl->wl_inohashcnt) 1849 1.2 simonb reserved = wapbl_transaction_inodes_len(wl); 1850 1.2 simonb 1851 1.2 simonb head = wl->wl_head; 1852 1.2 simonb tail = wl->wl_tail; 1853 1.2 simonb 1854 1.2 simonb wapbl_advance_head(wl->wl_circ_size, wl->wl_circ_off, flushsize, 1855 1.2 simonb &head, &tail); 1856 1.52.2.5 jdolecek 1857 1.52.2.5 jdolecek KASSERTMSG(head == off, 1858 1.52.2.5 jdolecek "lost head! head=%"PRIdMAX" tail=%" PRIdMAX 1859 1.52.2.5 jdolecek " off=%"PRIdMAX" flush=%zu", 1860 1.52.2.5 jdolecek (intmax_t)head, (intmax_t)tail, (intmax_t)off, 1861 1.52.2.5 jdolecek flushsize); 1862 1.2 simonb 1863 1.2 simonb /* Opportunistically move the tail forward if we can */ 1864 1.52.2.5 jdolecek mutex_enter(&wl->wl_mtx); 1865 1.52.2.5 jdolecek delta = wl->wl_reclaimable_bytes; 1866 1.52.2.5 jdolecek mutex_exit(&wl->wl_mtx); 1867 1.52.2.5 jdolecek wapbl_advance_tail(wl->wl_circ_size, wl->wl_circ_off, delta, 1868 1.52.2.5 jdolecek &head, &tail); 1869 1.2 simonb 1870 1.2 simonb error = wapbl_write_commit(wl, head, tail); 1871 1.2 simonb if (error) 1872 1.52.2.5 jdolecek goto out; 1873 1.2 simonb 1874 1.51 para we = pool_get(&wapbl_entry_pool, PR_WAITOK); 1875 1.2 simonb 1876 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1877 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1878 1.2 simonb ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1879 1.2 simonb " unsynced=%zu" 1880 1.2 simonb "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1881 1.2 simonb "inodes=%d\n", 1882 1.2 simonb curproc->p_pid, curlwp->l_lid, flushsize, delta, 1883 1.2 simonb wapbl_space_used(wl->wl_circ_size, head, tail), 1884 1.2 simonb wl->wl_unsynced_bufbytes, wl->wl_bufcount, 1885 1.2 simonb wl->wl_bufbytes, wl->wl_bcount, wl->wl_dealloccnt, 1886 1.2 simonb wl->wl_inohashcnt)); 1887 1.2 simonb #else 1888 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1889 1.2 simonb ("wapbl_flush: thread %d.%d head+=%zu tail+=%zu used=%zu" 1890 1.2 simonb "\n\tbufcount=%zu bufbytes=%zu bcount=%zu deallocs=%d " 1891 1.2 simonb "inodes=%d\n", 1892 1.2 simonb curproc->p_pid, curlwp->l_lid, flushsize, delta, 1893 1.2 simonb wapbl_space_used(wl->wl_circ_size, head, tail), 1894 1.2 simonb wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 1895 1.2 simonb wl->wl_dealloccnt, wl->wl_inohashcnt)); 1896 1.2 simonb #endif 1897 1.2 simonb 1898 1.2 simonb 1899 1.2 simonb mutex_enter(&bufcache_lock); 1900 1.2 simonb mutex_enter(&wl->wl_mtx); 1901 1.2 simonb 1902 1.2 simonb wl->wl_reserved_bytes = reserved; 1903 1.2 simonb wl->wl_head = head; 1904 1.2 simonb wl->wl_tail = tail; 1905 1.2 simonb KASSERT(wl->wl_reclaimable_bytes >= delta); 1906 1.2 simonb wl->wl_reclaimable_bytes -= delta; 1907 1.52.2.5 jdolecek KDASSERT(wl->wl_dealloccnt == 0); 1908 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1909 1.2 simonb wl->wl_unsynced_bufbytes += wl->wl_bufbytes; 1910 1.2 simonb #endif 1911 1.2 simonb 1912 1.2 simonb we->we_wapbl = wl; 1913 1.2 simonb we->we_bufcount = wl->wl_bufcount; 1914 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1915 1.2 simonb we->we_unsynced_bufbytes = wl->wl_bufbytes; 1916 1.2 simonb #endif 1917 1.2 simonb we->we_reclaimable_bytes = flushsize; 1918 1.2 simonb we->we_error = 0; 1919 1.2 simonb SIMPLEQ_INSERT_TAIL(&wl->wl_entries, we, we_entries); 1920 1.2 simonb 1921 1.2 simonb /* 1922 1.52.2.5 jdolecek * This flushes bufs in order than they were queued, so the LRU 1923 1.52.2.5 jdolecek * order is preserved. 1924 1.2 simonb */ 1925 1.52.2.5 jdolecek while ((bp = TAILQ_FIRST(&wl->wl_bufs)) != NULL) { 1926 1.2 simonb if (bbusy(bp, 0, 0, &wl->wl_mtx)) { 1927 1.2 simonb continue; 1928 1.2 simonb } 1929 1.2 simonb bp->b_iodone = wapbl_biodone; 1930 1.2 simonb bp->b_private = we; 1931 1.52.2.5 jdolecek 1932 1.2 simonb bremfree(bp); 1933 1.2 simonb wapbl_remove_buf_locked(wl, bp); 1934 1.2 simonb mutex_exit(&wl->wl_mtx); 1935 1.2 simonb mutex_exit(&bufcache_lock); 1936 1.2 simonb bawrite(bp); 1937 1.2 simonb mutex_enter(&bufcache_lock); 1938 1.2 simonb mutex_enter(&wl->wl_mtx); 1939 1.2 simonb } 1940 1.2 simonb mutex_exit(&wl->wl_mtx); 1941 1.2 simonb mutex_exit(&bufcache_lock); 1942 1.2 simonb 1943 1.2 simonb #if 0 1944 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_FLUSH, 1945 1.2 simonb ("wapbl_flush thread %d.%d done flushing entries...\n", 1946 1.2 simonb curproc->p_pid, curlwp->l_lid)); 1947 1.2 simonb #endif 1948 1.2 simonb 1949 1.52.2.5 jdolecek wait_out: 1950 1.2 simonb 1951 1.2 simonb /* 1952 1.2 simonb * If the waitfor flag is set, don't return until everything is 1953 1.2 simonb * fully flushed and the on disk log is empty. 1954 1.2 simonb */ 1955 1.2 simonb if (waitfor) { 1956 1.52.2.5 jdolecek error = wapbl_truncate(wl, wl->wl_circ_size - 1957 1.52.2.5 jdolecek wl->wl_reserved_bytes); 1958 1.2 simonb } 1959 1.2 simonb 1960 1.52.2.5 jdolecek out: 1961 1.2 simonb if (error) { 1962 1.52.2.5 jdolecek wl->wl_flush_abort(wl->wl_mount, 1963 1.52.2.5 jdolecek TAILQ_FIRST(&wl->wl_dealloclist)); 1964 1.2 simonb } 1965 1.2 simonb 1966 1.2 simonb #ifdef WAPBL_DEBUG_PRINT 1967 1.2 simonb if (error) { 1968 1.2 simonb pid_t pid = -1; 1969 1.2 simonb lwpid_t lid = -1; 1970 1.2 simonb if (curproc) 1971 1.2 simonb pid = curproc->p_pid; 1972 1.2 simonb if (curlwp) 1973 1.2 simonb lid = curlwp->l_lid; 1974 1.2 simonb mutex_enter(&wl->wl_mtx); 1975 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 1976 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1977 1.2 simonb ("wapbl_flush: thread %d.%d aborted flush: " 1978 1.2 simonb "error = %d\n" 1979 1.2 simonb "\tbufcount=%zu bufbytes=%zu bcount=%zu " 1980 1.2 simonb "deallocs=%d inodes=%d\n" 1981 1.2 simonb "\terrcnt = %d, reclaimable=%zu reserved=%zu " 1982 1.2 simonb "unsynced=%zu\n", 1983 1.2 simonb pid, lid, error, wl->wl_bufcount, 1984 1.2 simonb wl->wl_bufbytes, wl->wl_bcount, 1985 1.2 simonb wl->wl_dealloccnt, wl->wl_inohashcnt, 1986 1.2 simonb wl->wl_error_count, wl->wl_reclaimable_bytes, 1987 1.2 simonb wl->wl_reserved_bytes, wl->wl_unsynced_bufbytes)); 1988 1.2 simonb SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 1989 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1990 1.2 simonb ("\tentry: bufcount = %zu, reclaimable = %zu, " 1991 1.2 simonb "error = %d, unsynced = %zu\n", 1992 1.2 simonb we->we_bufcount, we->we_reclaimable_bytes, 1993 1.2 simonb we->we_error, we->we_unsynced_bufbytes)); 1994 1.2 simonb } 1995 1.2 simonb #else 1996 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_ERROR, 1997 1.2 simonb ("wapbl_flush: thread %d.%d aborted flush: " 1998 1.2 simonb "error = %d\n" 1999 1.2 simonb "\tbufcount=%zu bufbytes=%zu bcount=%zu " 2000 1.2 simonb "deallocs=%d inodes=%d\n" 2001 1.2 simonb "\terrcnt = %d, reclaimable=%zu reserved=%zu\n", 2002 1.2 simonb pid, lid, error, wl->wl_bufcount, 2003 1.2 simonb wl->wl_bufbytes, wl->wl_bcount, 2004 1.2 simonb wl->wl_dealloccnt, wl->wl_inohashcnt, 2005 1.2 simonb wl->wl_error_count, wl->wl_reclaimable_bytes, 2006 1.2 simonb wl->wl_reserved_bytes)); 2007 1.2 simonb SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 2008 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_ERROR, 2009 1.2 simonb ("\tentry: bufcount = %zu, reclaimable = %zu, " 2010 1.2 simonb "error = %d\n", we->we_bufcount, 2011 1.2 simonb we->we_reclaimable_bytes, we->we_error)); 2012 1.2 simonb } 2013 1.2 simonb #endif 2014 1.2 simonb mutex_exit(&wl->wl_mtx); 2015 1.2 simonb } 2016 1.2 simonb #endif 2017 1.2 simonb 2018 1.2 simonb rw_exit(&wl->wl_rwlock); 2019 1.2 simonb return error; 2020 1.2 simonb } 2021 1.2 simonb 2022 1.2 simonb /****************************************************************/ 2023 1.2 simonb 2024 1.2 simonb void 2025 1.2 simonb wapbl_jlock_assert(struct wapbl *wl) 2026 1.2 simonb { 2027 1.2 simonb 2028 1.23 ad KASSERT(rw_lock_held(&wl->wl_rwlock)); 2029 1.2 simonb } 2030 1.2 simonb 2031 1.2 simonb void 2032 1.2 simonb wapbl_junlock_assert(struct wapbl *wl) 2033 1.2 simonb { 2034 1.2 simonb 2035 1.2 simonb KASSERT(!rw_write_held(&wl->wl_rwlock)); 2036 1.2 simonb } 2037 1.2 simonb 2038 1.2 simonb /****************************************************************/ 2039 1.2 simonb 2040 1.2 simonb /* locks missing */ 2041 1.2 simonb void 2042 1.2 simonb wapbl_print(struct wapbl *wl, 2043 1.2 simonb int full, 2044 1.2 simonb void (*pr)(const char *, ...)) 2045 1.2 simonb { 2046 1.2 simonb struct buf *bp; 2047 1.2 simonb struct wapbl_entry *we; 2048 1.2 simonb (*pr)("wapbl %p", wl); 2049 1.2 simonb (*pr)("\nlogvp = %p, devvp = %p, logpbn = %"PRId64"\n", 2050 1.2 simonb wl->wl_logvp, wl->wl_devvp, wl->wl_logpbn); 2051 1.2 simonb (*pr)("circ = %zu, header = %zu, head = %"PRIdMAX" tail = %"PRIdMAX"\n", 2052 1.2 simonb wl->wl_circ_size, wl->wl_circ_off, 2053 1.2 simonb (intmax_t)wl->wl_head, (intmax_t)wl->wl_tail); 2054 1.2 simonb (*pr)("fs_dev_bshift = %d, log_dev_bshift = %d\n", 2055 1.2 simonb wl->wl_log_dev_bshift, wl->wl_fs_dev_bshift); 2056 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 2057 1.2 simonb (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 2058 1.2 simonb "reserved = %zu errcnt = %d unsynced = %zu\n", 2059 1.2 simonb wl->wl_bufcount, wl->wl_bufbytes, wl->wl_bcount, 2060 1.2 simonb wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 2061 1.2 simonb wl->wl_error_count, wl->wl_unsynced_bufbytes); 2062 1.2 simonb #else 2063 1.2 simonb (*pr)("bufcount = %zu, bufbytes = %zu bcount = %zu reclaimable = %zu " 2064 1.2 simonb "reserved = %zu errcnt = %d\n", wl->wl_bufcount, wl->wl_bufbytes, 2065 1.2 simonb wl->wl_bcount, wl->wl_reclaimable_bytes, wl->wl_reserved_bytes, 2066 1.2 simonb wl->wl_error_count); 2067 1.2 simonb #endif 2068 1.2 simonb (*pr)("\tdealloccnt = %d, dealloclim = %d\n", 2069 1.2 simonb wl->wl_dealloccnt, wl->wl_dealloclim); 2070 1.2 simonb (*pr)("\tinohashcnt = %d, inohashmask = 0x%08x\n", 2071 1.2 simonb wl->wl_inohashcnt, wl->wl_inohashmask); 2072 1.2 simonb (*pr)("entries:\n"); 2073 1.2 simonb SIMPLEQ_FOREACH(we, &wl->wl_entries, we_entries) { 2074 1.2 simonb #ifdef WAPBL_DEBUG_BUFBYTES 2075 1.2 simonb (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d, " 2076 1.2 simonb "unsynced = %zu\n", 2077 1.2 simonb we->we_bufcount, we->we_reclaimable_bytes, 2078 1.2 simonb we->we_error, we->we_unsynced_bufbytes); 2079 1.2 simonb #else 2080 1.2 simonb (*pr)("\tbufcount = %zu, reclaimable = %zu, error = %d\n", 2081 1.2 simonb we->we_bufcount, we->we_reclaimable_bytes, we->we_error); 2082 1.2 simonb #endif 2083 1.2 simonb } 2084 1.2 simonb if (full) { 2085 1.2 simonb int cnt = 0; 2086 1.2 simonb (*pr)("bufs ="); 2087 1.52.2.5 jdolecek TAILQ_FOREACH(bp, &wl->wl_bufs, b_wapbllist) { 2088 1.52.2.5 jdolecek if (!TAILQ_NEXT(bp, b_wapbllist)) { 2089 1.2 simonb (*pr)(" %p", bp); 2090 1.2 simonb } else if ((++cnt % 6) == 0) { 2091 1.2 simonb (*pr)(" %p,\n\t", bp); 2092 1.2 simonb } else { 2093 1.2 simonb (*pr)(" %p,", bp); 2094 1.2 simonb } 2095 1.2 simonb } 2096 1.2 simonb (*pr)("\n"); 2097 1.2 simonb 2098 1.2 simonb (*pr)("dealloced blks = "); 2099 1.2 simonb { 2100 1.52.2.5 jdolecek struct wapbl_dealloc *wd; 2101 1.2 simonb cnt = 0; 2102 1.52.2.5 jdolecek TAILQ_FOREACH(wd, &wl->wl_dealloclist, wd_entries) { 2103 1.2 simonb (*pr)(" %"PRId64":%d,", 2104 1.52.2.5 jdolecek wd->wd_blkno, 2105 1.52.2.5 jdolecek wd->wd_len); 2106 1.2 simonb if ((++cnt % 4) == 0) { 2107 1.2 simonb (*pr)("\n\t"); 2108 1.2 simonb } 2109 1.2 simonb } 2110 1.2 simonb } 2111 1.2 simonb (*pr)("\n"); 2112 1.2 simonb 2113 1.2 simonb (*pr)("registered inodes = "); 2114 1.2 simonb { 2115 1.2 simonb int i; 2116 1.2 simonb cnt = 0; 2117 1.2 simonb for (i = 0; i <= wl->wl_inohashmask; i++) { 2118 1.2 simonb struct wapbl_ino_head *wih; 2119 1.2 simonb struct wapbl_ino *wi; 2120 1.2 simonb 2121 1.2 simonb wih = &wl->wl_inohash[i]; 2122 1.2 simonb LIST_FOREACH(wi, wih, wi_hash) { 2123 1.2 simonb if (wi->wi_ino == 0) 2124 1.2 simonb continue; 2125 1.52.2.3 tls (*pr)(" %"PRIu64"/0%06"PRIo32",", 2126 1.2 simonb wi->wi_ino, wi->wi_mode); 2127 1.2 simonb if ((++cnt % 4) == 0) { 2128 1.2 simonb (*pr)("\n\t"); 2129 1.2 simonb } 2130 1.2 simonb } 2131 1.2 simonb } 2132 1.2 simonb (*pr)("\n"); 2133 1.2 simonb } 2134 1.52.2.5 jdolecek 2135 1.52.2.5 jdolecek (*pr)("iobufs free ="); 2136 1.52.2.5 jdolecek TAILQ_FOREACH(bp, &wl->wl_iobufs, b_wapbllist) { 2137 1.52.2.5 jdolecek if (!TAILQ_NEXT(bp, b_wapbllist)) { 2138 1.52.2.5 jdolecek (*pr)(" %p", bp); 2139 1.52.2.5 jdolecek } else if ((++cnt % 6) == 0) { 2140 1.52.2.5 jdolecek (*pr)(" %p,\n\t", bp); 2141 1.52.2.5 jdolecek } else { 2142 1.52.2.5 jdolecek (*pr)(" %p,", bp); 2143 1.52.2.5 jdolecek } 2144 1.52.2.5 jdolecek } 2145 1.52.2.5 jdolecek (*pr)("\n"); 2146 1.52.2.5 jdolecek 2147 1.52.2.5 jdolecek (*pr)("iobufs busy ="); 2148 1.52.2.5 jdolecek TAILQ_FOREACH(bp, &wl->wl_iobufs_busy, b_wapbllist) { 2149 1.52.2.5 jdolecek if (!TAILQ_NEXT(bp, b_wapbllist)) { 2150 1.52.2.5 jdolecek (*pr)(" %p", bp); 2151 1.52.2.5 jdolecek } else if ((++cnt % 6) == 0) { 2152 1.52.2.5 jdolecek (*pr)(" %p,\n\t", bp); 2153 1.52.2.5 jdolecek } else { 2154 1.52.2.5 jdolecek (*pr)(" %p,", bp); 2155 1.52.2.5 jdolecek } 2156 1.52.2.5 jdolecek } 2157 1.52.2.5 jdolecek (*pr)("\n"); 2158 1.2 simonb } 2159 1.2 simonb } 2160 1.2 simonb 2161 1.2 simonb #if defined(WAPBL_DEBUG) || defined(DDB) 2162 1.2 simonb void 2163 1.2 simonb wapbl_dump(struct wapbl *wl) 2164 1.2 simonb { 2165 1.2 simonb #if defined(WAPBL_DEBUG) 2166 1.2 simonb if (!wl) 2167 1.2 simonb wl = wapbl_debug_wl; 2168 1.2 simonb #endif 2169 1.2 simonb if (!wl) 2170 1.2 simonb return; 2171 1.2 simonb wapbl_print(wl, 1, printf); 2172 1.2 simonb } 2173 1.2 simonb #endif 2174 1.2 simonb 2175 1.2 simonb /****************************************************************/ 2176 1.2 simonb 2177 1.52.2.5 jdolecek int 2178 1.52.2.5 jdolecek wapbl_register_deallocation(struct wapbl *wl, daddr_t blk, int len, bool force, 2179 1.52.2.5 jdolecek void **cookiep) 2180 1.2 simonb { 2181 1.52.2.5 jdolecek struct wapbl_dealloc *wd; 2182 1.52.2.5 jdolecek int error = 0; 2183 1.2 simonb 2184 1.2 simonb wapbl_jlock_assert(wl); 2185 1.2 simonb 2186 1.38 hannken mutex_enter(&wl->wl_mtx); 2187 1.27 pooka 2188 1.52.2.5 jdolecek if (__predict_false(wl->wl_dealloccnt >= wl->wl_dealloclim)) { 2189 1.52.2.5 jdolecek if (!force) { 2190 1.52.2.5 jdolecek error = EAGAIN; 2191 1.52.2.5 jdolecek goto out; 2192 1.52.2.5 jdolecek } 2193 1.52.2.5 jdolecek 2194 1.52.2.5 jdolecek /* 2195 1.52.2.5 jdolecek * Forced registration can only be used when: 2196 1.52.2.5 jdolecek * 1) the caller can't cope with failure 2197 1.52.2.5 jdolecek * 2) the path can be triggered only bounded, small 2198 1.52.2.5 jdolecek * times per transaction 2199 1.52.2.5 jdolecek * If this is not fullfilled, and the path would be triggered 2200 1.52.2.5 jdolecek * many times, this could overflow maximum transaction size 2201 1.52.2.5 jdolecek * and panic later. 2202 1.52.2.5 jdolecek */ 2203 1.52.2.5 jdolecek printf("%s: forced dealloc registration over limit: %d >= %d\n", 2204 1.52.2.5 jdolecek wl->wl_mount->mnt_stat.f_mntonname, 2205 1.52.2.5 jdolecek wl->wl_dealloccnt, wl->wl_dealloclim); 2206 1.52.2.5 jdolecek } 2207 1.52.2.5 jdolecek 2208 1.2 simonb wl->wl_dealloccnt++; 2209 1.38 hannken mutex_exit(&wl->wl_mtx); 2210 1.52.2.5 jdolecek 2211 1.52.2.5 jdolecek wd = pool_get(&wapbl_dealloc_pool, PR_WAITOK); 2212 1.52.2.5 jdolecek wd->wd_blkno = blk; 2213 1.52.2.5 jdolecek wd->wd_len = len; 2214 1.52.2.5 jdolecek 2215 1.52.2.5 jdolecek mutex_enter(&wl->wl_mtx); 2216 1.52.2.5 jdolecek TAILQ_INSERT_TAIL(&wl->wl_dealloclist, wd, wd_entries); 2217 1.52.2.5 jdolecek 2218 1.52.2.5 jdolecek if (cookiep) 2219 1.52.2.5 jdolecek *cookiep = wd; 2220 1.52.2.5 jdolecek 2221 1.52.2.5 jdolecek out: 2222 1.52.2.5 jdolecek mutex_exit(&wl->wl_mtx); 2223 1.52.2.5 jdolecek 2224 1.52.2.5 jdolecek WAPBL_PRINTF(WAPBL_PRINT_ALLOC, 2225 1.52.2.5 jdolecek ("wapbl_register_deallocation: blk=%"PRId64" len=%d error=%d\n", 2226 1.52.2.5 jdolecek blk, len, error)); 2227 1.52.2.5 jdolecek 2228 1.52.2.5 jdolecek return error; 2229 1.52.2.5 jdolecek } 2230 1.52.2.5 jdolecek 2231 1.52.2.5 jdolecek static void 2232 1.52.2.5 jdolecek wapbl_deallocation_free(struct wapbl *wl, struct wapbl_dealloc *wd, 2233 1.52.2.5 jdolecek bool locked) 2234 1.52.2.5 jdolecek { 2235 1.52.2.5 jdolecek KASSERT(!locked 2236 1.52.2.5 jdolecek || rw_lock_held(&wl->wl_rwlock) || mutex_owned(&wl->wl_mtx)); 2237 1.52.2.5 jdolecek 2238 1.52.2.5 jdolecek if (!locked) 2239 1.52.2.5 jdolecek mutex_enter(&wl->wl_mtx); 2240 1.52.2.5 jdolecek 2241 1.52.2.5 jdolecek TAILQ_REMOVE(&wl->wl_dealloclist, wd, wd_entries); 2242 1.52.2.5 jdolecek wl->wl_dealloccnt--; 2243 1.52.2.5 jdolecek 2244 1.52.2.5 jdolecek if (!locked) 2245 1.52.2.5 jdolecek mutex_exit(&wl->wl_mtx); 2246 1.52.2.5 jdolecek 2247 1.52.2.5 jdolecek pool_put(&wapbl_dealloc_pool, wd); 2248 1.52.2.5 jdolecek } 2249 1.52.2.5 jdolecek 2250 1.52.2.5 jdolecek void 2251 1.52.2.5 jdolecek wapbl_unregister_deallocation(struct wapbl *wl, void *cookie) 2252 1.52.2.5 jdolecek { 2253 1.52.2.5 jdolecek KASSERT(cookie != NULL); 2254 1.52.2.5 jdolecek wapbl_deallocation_free(wl, cookie, false); 2255 1.2 simonb } 2256 1.2 simonb 2257 1.2 simonb /****************************************************************/ 2258 1.2 simonb 2259 1.2 simonb static void 2260 1.2 simonb wapbl_inodetrk_init(struct wapbl *wl, u_int size) 2261 1.2 simonb { 2262 1.2 simonb 2263 1.2 simonb wl->wl_inohash = hashinit(size, HASH_LIST, true, &wl->wl_inohashmask); 2264 1.2 simonb if (atomic_inc_uint_nv(&wapbl_ino_pool_refcount) == 1) { 2265 1.2 simonb pool_init(&wapbl_ino_pool, sizeof(struct wapbl_ino), 0, 0, 0, 2266 1.2 simonb "wapblinopl", &pool_allocator_nointr, IPL_NONE); 2267 1.2 simonb } 2268 1.2 simonb } 2269 1.2 simonb 2270 1.2 simonb static void 2271 1.2 simonb wapbl_inodetrk_free(struct wapbl *wl) 2272 1.2 simonb { 2273 1.2 simonb 2274 1.2 simonb /* XXX this KASSERT needs locking/mutex analysis */ 2275 1.2 simonb KASSERT(wl->wl_inohashcnt == 0); 2276 1.2 simonb hashdone(wl->wl_inohash, HASH_LIST, wl->wl_inohashmask); 2277 1.2 simonb if (atomic_dec_uint_nv(&wapbl_ino_pool_refcount) == 0) { 2278 1.2 simonb pool_destroy(&wapbl_ino_pool); 2279 1.2 simonb } 2280 1.2 simonb } 2281 1.2 simonb 2282 1.2 simonb static struct wapbl_ino * 2283 1.2 simonb wapbl_inodetrk_get(struct wapbl *wl, ino_t ino) 2284 1.2 simonb { 2285 1.2 simonb struct wapbl_ino_head *wih; 2286 1.2 simonb struct wapbl_ino *wi; 2287 1.2 simonb 2288 1.2 simonb KASSERT(mutex_owned(&wl->wl_mtx)); 2289 1.2 simonb 2290 1.2 simonb wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 2291 1.2 simonb LIST_FOREACH(wi, wih, wi_hash) { 2292 1.2 simonb if (ino == wi->wi_ino) 2293 1.2 simonb return wi; 2294 1.2 simonb } 2295 1.2 simonb return 0; 2296 1.2 simonb } 2297 1.2 simonb 2298 1.2 simonb void 2299 1.2 simonb wapbl_register_inode(struct wapbl *wl, ino_t ino, mode_t mode) 2300 1.2 simonb { 2301 1.2 simonb struct wapbl_ino_head *wih; 2302 1.2 simonb struct wapbl_ino *wi; 2303 1.2 simonb 2304 1.2 simonb wi = pool_get(&wapbl_ino_pool, PR_WAITOK); 2305 1.2 simonb 2306 1.2 simonb mutex_enter(&wl->wl_mtx); 2307 1.2 simonb if (wapbl_inodetrk_get(wl, ino) == NULL) { 2308 1.2 simonb wi->wi_ino = ino; 2309 1.2 simonb wi->wi_mode = mode; 2310 1.2 simonb wih = &wl->wl_inohash[ino & wl->wl_inohashmask]; 2311 1.2 simonb LIST_INSERT_HEAD(wih, wi, wi_hash); 2312 1.2 simonb wl->wl_inohashcnt++; 2313 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_INODE, 2314 1.2 simonb ("wapbl_register_inode: ino=%"PRId64"\n", ino)); 2315 1.2 simonb mutex_exit(&wl->wl_mtx); 2316 1.2 simonb } else { 2317 1.2 simonb mutex_exit(&wl->wl_mtx); 2318 1.2 simonb pool_put(&wapbl_ino_pool, wi); 2319 1.2 simonb } 2320 1.2 simonb } 2321 1.2 simonb 2322 1.2 simonb void 2323 1.2 simonb wapbl_unregister_inode(struct wapbl *wl, ino_t ino, mode_t mode) 2324 1.2 simonb { 2325 1.2 simonb struct wapbl_ino *wi; 2326 1.2 simonb 2327 1.2 simonb mutex_enter(&wl->wl_mtx); 2328 1.2 simonb wi = wapbl_inodetrk_get(wl, ino); 2329 1.2 simonb if (wi) { 2330 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_INODE, 2331 1.2 simonb ("wapbl_unregister_inode: ino=%"PRId64"\n", ino)); 2332 1.2 simonb KASSERT(wl->wl_inohashcnt > 0); 2333 1.2 simonb wl->wl_inohashcnt--; 2334 1.2 simonb LIST_REMOVE(wi, wi_hash); 2335 1.2 simonb mutex_exit(&wl->wl_mtx); 2336 1.2 simonb 2337 1.2 simonb pool_put(&wapbl_ino_pool, wi); 2338 1.2 simonb } else { 2339 1.2 simonb mutex_exit(&wl->wl_mtx); 2340 1.2 simonb } 2341 1.2 simonb } 2342 1.2 simonb 2343 1.2 simonb /****************************************************************/ 2344 1.2 simonb 2345 1.52.2.5 jdolecek /* 2346 1.52.2.5 jdolecek * wapbl_transaction_inodes_len(wl) 2347 1.52.2.5 jdolecek * 2348 1.52.2.5 jdolecek * Calculate the number of bytes required for inode registration 2349 1.52.2.5 jdolecek * log records in wl. 2350 1.52.2.5 jdolecek */ 2351 1.30 uebayasi static inline size_t 2352 1.2 simonb wapbl_transaction_inodes_len(struct wapbl *wl) 2353 1.2 simonb { 2354 1.2 simonb int blocklen = 1<<wl->wl_log_dev_bshift; 2355 1.2 simonb int iph; 2356 1.2 simonb 2357 1.2 simonb /* Calculate number of inodes described in a inodelist header */ 2358 1.2 simonb iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2359 1.2 simonb sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2360 1.2 simonb 2361 1.2 simonb KASSERT(iph > 0); 2362 1.2 simonb 2363 1.39 christos return MAX(1, howmany(wl->wl_inohashcnt, iph)) * blocklen; 2364 1.2 simonb } 2365 1.2 simonb 2366 1.2 simonb 2367 1.52.2.5 jdolecek /* 2368 1.52.2.5 jdolecek * wapbl_transaction_len(wl) 2369 1.52.2.5 jdolecek * 2370 1.52.2.5 jdolecek * Calculate number of bytes required for all log records in wl. 2371 1.52.2.5 jdolecek */ 2372 1.2 simonb static size_t 2373 1.2 simonb wapbl_transaction_len(struct wapbl *wl) 2374 1.2 simonb { 2375 1.2 simonb int blocklen = 1<<wl->wl_log_dev_bshift; 2376 1.2 simonb size_t len; 2377 1.2 simonb 2378 1.2 simonb /* Calculate number of blocks described in a blocklist header */ 2379 1.2 simonb len = wl->wl_bcount; 2380 1.52.2.5 jdolecek len += howmany(wl->wl_bufcount, wl->wl_brperjblock) * blocklen; 2381 1.52.2.5 jdolecek len += howmany(wl->wl_dealloccnt, wl->wl_brperjblock) * blocklen; 2382 1.2 simonb len += wapbl_transaction_inodes_len(wl); 2383 1.2 simonb 2384 1.2 simonb return len; 2385 1.2 simonb } 2386 1.2 simonb 2387 1.2 simonb /* 2388 1.52.2.5 jdolecek * wapbl_cache_sync(wl, msg) 2389 1.52.2.5 jdolecek * 2390 1.52.2.5 jdolecek * Issue DIOCCACHESYNC to wl->wl_devvp. 2391 1.52.2.5 jdolecek * 2392 1.52.2.5 jdolecek * If sysctl(vfs.wapbl.verbose_commit) >= 2, print a message 2393 1.52.2.5 jdolecek * including msg about the duration of the cache sync. 2394 1.48 yamt */ 2395 1.48 yamt static int 2396 1.48 yamt wapbl_cache_sync(struct wapbl *wl, const char *msg) 2397 1.48 yamt { 2398 1.48 yamt const bool verbose = wapbl_verbose_commit >= 2; 2399 1.48 yamt struct bintime start_time; 2400 1.48 yamt int force = 1; 2401 1.48 yamt int error; 2402 1.48 yamt 2403 1.52.2.5 jdolecek /* Skip full cache sync if disabled */ 2404 1.48 yamt if (!wapbl_flush_disk_cache) { 2405 1.48 yamt return 0; 2406 1.48 yamt } 2407 1.48 yamt if (verbose) { 2408 1.48 yamt bintime(&start_time); 2409 1.48 yamt } 2410 1.48 yamt error = VOP_IOCTL(wl->wl_devvp, DIOCCACHESYNC, &force, 2411 1.48 yamt FWRITE, FSCRED); 2412 1.48 yamt if (error) { 2413 1.48 yamt WAPBL_PRINTF(WAPBL_PRINT_ERROR, 2414 1.52.2.5 jdolecek ("wapbl_cache_sync: DIOCCACHESYNC on dev 0x%jx " 2415 1.52.2.5 jdolecek "returned %d\n", (uintmax_t)wl->wl_devvp->v_rdev, error)); 2416 1.48 yamt } 2417 1.48 yamt if (verbose) { 2418 1.48 yamt struct bintime d; 2419 1.48 yamt struct timespec ts; 2420 1.48 yamt 2421 1.48 yamt bintime(&d); 2422 1.48 yamt bintime_sub(&d, &start_time); 2423 1.48 yamt bintime2timespec(&d, &ts); 2424 1.48 yamt printf("wapbl_cache_sync: %s: dev 0x%jx %ju.%09lu\n", 2425 1.48 yamt msg, (uintmax_t)wl->wl_devvp->v_rdev, 2426 1.48 yamt (uintmax_t)ts.tv_sec, ts.tv_nsec); 2427 1.48 yamt } 2428 1.52.2.5 jdolecek 2429 1.52.2.5 jdolecek wl->wl_ev_cacheflush.ev_count++; 2430 1.52.2.5 jdolecek 2431 1.48 yamt return error; 2432 1.48 yamt } 2433 1.48 yamt 2434 1.48 yamt /* 2435 1.52.2.5 jdolecek * wapbl_write_commit(wl, head, tail) 2436 1.2 simonb * 2437 1.52.2.5 jdolecek * Issue a disk cache sync to wait for all pending writes to the 2438 1.52.2.5 jdolecek * log to complete, and then synchronously commit the current 2439 1.52.2.5 jdolecek * circular queue head and tail to the log, in the next of two 2440 1.52.2.5 jdolecek * locations for commit headers on disk. 2441 1.52.2.5 jdolecek * 2442 1.52.2.5 jdolecek * Increment the generation number. If the generation number 2443 1.52.2.5 jdolecek * rolls over to zero, then a subsequent commit would appear to 2444 1.52.2.5 jdolecek * have an older generation than this one -- in that case, issue a 2445 1.52.2.5 jdolecek * duplicate commit to avoid this. 2446 1.52.2.5 jdolecek * 2447 1.52.2.5 jdolecek * => Caller must have exclusive access to wl, either by holding 2448 1.52.2.5 jdolecek * wl->wl_rwlock for writer or by being wapbl_start before anyone 2449 1.52.2.5 jdolecek * else has seen wl. 2450 1.2 simonb */ 2451 1.2 simonb static int 2452 1.2 simonb wapbl_write_commit(struct wapbl *wl, off_t head, off_t tail) 2453 1.2 simonb { 2454 1.2 simonb struct wapbl_wc_header *wc = wl->wl_wc_header; 2455 1.2 simonb struct timespec ts; 2456 1.2 simonb int error; 2457 1.34 mlelstv daddr_t pbn; 2458 1.2 simonb 2459 1.52.2.5 jdolecek error = wapbl_buffered_flush(wl, true); 2460 1.52.2.3 tls if (error) 2461 1.52.2.3 tls return error; 2462 1.49 yamt /* 2463 1.52.2.5 jdolecek * Flush disk cache to ensure that blocks we've written are actually 2464 1.49 yamt * written to the stable storage before the commit header. 2465 1.52.2.5 jdolecek * This flushes to disk not only journal blocks, but also all 2466 1.52.2.5 jdolecek * metadata blocks, written asynchronously since previous commit. 2467 1.49 yamt * 2468 1.49 yamt * XXX Calc checksum here, instead we do this for now 2469 1.49 yamt */ 2470 1.48 yamt wapbl_cache_sync(wl, "1"); 2471 1.2 simonb 2472 1.2 simonb wc->wc_head = head; 2473 1.2 simonb wc->wc_tail = tail; 2474 1.2 simonb wc->wc_checksum = 0; 2475 1.2 simonb wc->wc_version = 1; 2476 1.2 simonb getnanotime(&ts); 2477 1.17 yamt wc->wc_time = ts.tv_sec; 2478 1.2 simonb wc->wc_timensec = ts.tv_nsec; 2479 1.2 simonb 2480 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2481 1.2 simonb ("wapbl_write_commit: head = %"PRIdMAX "tail = %"PRIdMAX"\n", 2482 1.2 simonb (intmax_t)head, (intmax_t)tail)); 2483 1.2 simonb 2484 1.2 simonb /* 2485 1.49 yamt * write the commit header. 2486 1.49 yamt * 2487 1.2 simonb * XXX if generation will rollover, then first zero 2488 1.2 simonb * over second commit header before trying to write both headers. 2489 1.2 simonb */ 2490 1.2 simonb 2491 1.34 mlelstv pbn = wl->wl_logpbn + (wc->wc_generation % 2); 2492 1.34 mlelstv #ifdef _KERNEL 2493 1.34 mlelstv pbn = btodb(pbn << wc->wc_log_dev_bshift); 2494 1.34 mlelstv #endif 2495 1.52.2.5 jdolecek error = wapbl_buffered_write(wc, wc->wc_len, wl, pbn, WAPBL_JFLAGS(wl)); 2496 1.52.2.3 tls if (error) 2497 1.52.2.3 tls return error; 2498 1.52.2.5 jdolecek error = wapbl_buffered_flush(wl, true); 2499 1.2 simonb if (error) 2500 1.2 simonb return error; 2501 1.2 simonb 2502 1.49 yamt /* 2503 1.52.2.5 jdolecek * Flush disk cache to ensure that the commit header is actually 2504 1.52.2.5 jdolecek * written before meta data blocks. Commit block is written using 2505 1.52.2.5 jdolecek * FUA when enabled, in that case this flush is not needed. 2506 1.49 yamt */ 2507 1.52.2.5 jdolecek if (!WAPBL_USE_FUA(wl)) 2508 1.52.2.5 jdolecek wapbl_cache_sync(wl, "2"); 2509 1.2 simonb 2510 1.2 simonb /* 2511 1.2 simonb * If the generation number was zero, write it out a second time. 2512 1.2 simonb * This handles initialization and generation number rollover 2513 1.2 simonb */ 2514 1.2 simonb if (wc->wc_generation++ == 0) { 2515 1.2 simonb error = wapbl_write_commit(wl, head, tail); 2516 1.2 simonb /* 2517 1.2 simonb * This panic should be able to be removed if we do the 2518 1.2 simonb * zero'ing mentioned above, and we are certain to roll 2519 1.2 simonb * back generation number on failure. 2520 1.2 simonb */ 2521 1.2 simonb if (error) 2522 1.2 simonb panic("wapbl_write_commit: error writing duplicate " 2523 1.52.2.5 jdolecek "log header: %d", error); 2524 1.2 simonb } 2525 1.52.2.5 jdolecek 2526 1.52.2.5 jdolecek wl->wl_ev_commit.ev_count++; 2527 1.52.2.5 jdolecek 2528 1.2 simonb return 0; 2529 1.2 simonb } 2530 1.2 simonb 2531 1.52.2.5 jdolecek /* 2532 1.52.2.5 jdolecek * wapbl_write_blocks(wl, offp) 2533 1.52.2.5 jdolecek * 2534 1.52.2.5 jdolecek * Write all pending physical blocks in the current transaction 2535 1.52.2.5 jdolecek * from wapbl_add_buf to the log on disk, adding to the circular 2536 1.52.2.5 jdolecek * queue head at byte offset *offp, and returning the new head's 2537 1.52.2.5 jdolecek * byte offset in *offp. 2538 1.52.2.5 jdolecek */ 2539 1.2 simonb static int 2540 1.2 simonb wapbl_write_blocks(struct wapbl *wl, off_t *offp) 2541 1.2 simonb { 2542 1.2 simonb struct wapbl_wc_blocklist *wc = 2543 1.2 simonb (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 2544 1.2 simonb int blocklen = 1<<wl->wl_log_dev_bshift; 2545 1.2 simonb struct buf *bp; 2546 1.2 simonb off_t off = *offp; 2547 1.2 simonb int error; 2548 1.7 joerg size_t padding; 2549 1.2 simonb 2550 1.2 simonb KASSERT(rw_write_held(&wl->wl_rwlock)); 2551 1.2 simonb 2552 1.52.2.5 jdolecek bp = TAILQ_FIRST(&wl->wl_bufs); 2553 1.2 simonb 2554 1.2 simonb while (bp) { 2555 1.2 simonb int cnt; 2556 1.2 simonb struct buf *obp = bp; 2557 1.2 simonb 2558 1.2 simonb KASSERT(bp->b_flags & B_LOCKED); 2559 1.2 simonb 2560 1.2 simonb wc->wc_type = WAPBL_WC_BLOCKS; 2561 1.2 simonb wc->wc_len = blocklen; 2562 1.2 simonb wc->wc_blkcount = 0; 2563 1.52.2.5 jdolecek while (bp && (wc->wc_blkcount < wl->wl_brperjblock)) { 2564 1.2 simonb /* 2565 1.2 simonb * Make sure all the physical block numbers are up to 2566 1.2 simonb * date. If this is not always true on a given 2567 1.2 simonb * filesystem, then VOP_BMAP must be called. We 2568 1.2 simonb * could call VOP_BMAP here, or else in the filesystem 2569 1.2 simonb * specific flush callback, although neither of those 2570 1.2 simonb * solutions allow us to take the vnode lock. If a 2571 1.2 simonb * filesystem requires that we must take the vnode lock 2572 1.2 simonb * to call VOP_BMAP, then we can probably do it in 2573 1.2 simonb * bwrite when the vnode lock should already be held 2574 1.2 simonb * by the invoking code. 2575 1.2 simonb */ 2576 1.2 simonb KASSERT((bp->b_vp->v_type == VBLK) || 2577 1.2 simonb (bp->b_blkno != bp->b_lblkno)); 2578 1.2 simonb KASSERT(bp->b_blkno > 0); 2579 1.2 simonb 2580 1.2 simonb wc->wc_blocks[wc->wc_blkcount].wc_daddr = bp->b_blkno; 2581 1.2 simonb wc->wc_blocks[wc->wc_blkcount].wc_dlen = bp->b_bcount; 2582 1.2 simonb wc->wc_len += bp->b_bcount; 2583 1.2 simonb wc->wc_blkcount++; 2584 1.52.2.5 jdolecek bp = TAILQ_NEXT(bp, b_wapbllist); 2585 1.2 simonb } 2586 1.7 joerg if (wc->wc_len % blocklen != 0) { 2587 1.7 joerg padding = blocklen - wc->wc_len % blocklen; 2588 1.7 joerg wc->wc_len += padding; 2589 1.7 joerg } else { 2590 1.7 joerg padding = 0; 2591 1.7 joerg } 2592 1.7 joerg 2593 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2594 1.7 joerg ("wapbl_write_blocks: len = %u (padding %zu) off = %"PRIdMAX"\n", 2595 1.7 joerg wc->wc_len, padding, (intmax_t)off)); 2596 1.2 simonb 2597 1.2 simonb error = wapbl_circ_write(wl, wc, blocklen, &off); 2598 1.2 simonb if (error) 2599 1.2 simonb return error; 2600 1.2 simonb bp = obp; 2601 1.2 simonb cnt = 0; 2602 1.52.2.5 jdolecek while (bp && (cnt++ < wl->wl_brperjblock)) { 2603 1.2 simonb error = wapbl_circ_write(wl, bp->b_data, 2604 1.2 simonb bp->b_bcount, &off); 2605 1.2 simonb if (error) 2606 1.2 simonb return error; 2607 1.52.2.5 jdolecek bp = TAILQ_NEXT(bp, b_wapbllist); 2608 1.2 simonb } 2609 1.7 joerg if (padding) { 2610 1.7 joerg void *zero; 2611 1.52.2.5 jdolecek 2612 1.51 para zero = wapbl_alloc(padding); 2613 1.7 joerg memset(zero, 0, padding); 2614 1.7 joerg error = wapbl_circ_write(wl, zero, padding, &off); 2615 1.18 yamt wapbl_free(zero, padding); 2616 1.7 joerg if (error) 2617 1.7 joerg return error; 2618 1.7 joerg } 2619 1.2 simonb } 2620 1.2 simonb *offp = off; 2621 1.2 simonb return 0; 2622 1.2 simonb } 2623 1.2 simonb 2624 1.52.2.5 jdolecek /* 2625 1.52.2.5 jdolecek * wapbl_write_revocations(wl, offp) 2626 1.52.2.5 jdolecek * 2627 1.52.2.5 jdolecek * Write all pending deallocations in the current transaction from 2628 1.52.2.5 jdolecek * wapbl_register_deallocation to the log on disk, adding to the 2629 1.52.2.5 jdolecek * circular queue's head at byte offset *offp, and returning the 2630 1.52.2.5 jdolecek * new head's byte offset in *offp. 2631 1.52.2.5 jdolecek */ 2632 1.2 simonb static int 2633 1.2 simonb wapbl_write_revocations(struct wapbl *wl, off_t *offp) 2634 1.2 simonb { 2635 1.2 simonb struct wapbl_wc_blocklist *wc = 2636 1.2 simonb (struct wapbl_wc_blocklist *)wl->wl_wc_scratch; 2637 1.52.2.5 jdolecek struct wapbl_dealloc *wd, *lwd; 2638 1.2 simonb int blocklen = 1<<wl->wl_log_dev_bshift; 2639 1.2 simonb off_t off = *offp; 2640 1.2 simonb int error; 2641 1.2 simonb 2642 1.52.2.5 jdolecek KASSERT(rw_write_held(&wl->wl_rwlock)); 2643 1.52.2.5 jdolecek 2644 1.2 simonb if (wl->wl_dealloccnt == 0) 2645 1.2 simonb return 0; 2646 1.2 simonb 2647 1.52.2.5 jdolecek while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) { 2648 1.2 simonb wc->wc_type = WAPBL_WC_REVOCATIONS; 2649 1.2 simonb wc->wc_len = blocklen; 2650 1.2 simonb wc->wc_blkcount = 0; 2651 1.52.2.5 jdolecek while (wd && (wc->wc_blkcount < wl->wl_brperjblock)) { 2652 1.2 simonb wc->wc_blocks[wc->wc_blkcount].wc_daddr = 2653 1.52.2.5 jdolecek wd->wd_blkno; 2654 1.2 simonb wc->wc_blocks[wc->wc_blkcount].wc_dlen = 2655 1.52.2.5 jdolecek wd->wd_len; 2656 1.2 simonb wc->wc_blkcount++; 2657 1.52.2.5 jdolecek 2658 1.52.2.5 jdolecek wd = TAILQ_NEXT(wd, wd_entries); 2659 1.2 simonb } 2660 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2661 1.2 simonb ("wapbl_write_revocations: len = %u off = %"PRIdMAX"\n", 2662 1.2 simonb wc->wc_len, (intmax_t)off)); 2663 1.2 simonb error = wapbl_circ_write(wl, wc, blocklen, &off); 2664 1.2 simonb if (error) 2665 1.2 simonb return error; 2666 1.52.2.5 jdolecek 2667 1.52.2.5 jdolecek /* free all successfully written deallocs */ 2668 1.52.2.5 jdolecek lwd = wd; 2669 1.52.2.5 jdolecek while ((wd = TAILQ_FIRST(&wl->wl_dealloclist)) != NULL) { 2670 1.52.2.5 jdolecek if (wd == lwd) 2671 1.52.2.5 jdolecek break; 2672 1.52.2.5 jdolecek wapbl_deallocation_free(wl, wd, true); 2673 1.52.2.5 jdolecek } 2674 1.2 simonb } 2675 1.2 simonb *offp = off; 2676 1.2 simonb return 0; 2677 1.2 simonb } 2678 1.2 simonb 2679 1.52.2.5 jdolecek /* 2680 1.52.2.5 jdolecek * wapbl_write_inodes(wl, offp) 2681 1.52.2.5 jdolecek * 2682 1.52.2.5 jdolecek * Write all pending inode allocations in the current transaction 2683 1.52.2.5 jdolecek * from wapbl_register_inode to the log on disk, adding to the 2684 1.52.2.5 jdolecek * circular queue's head at byte offset *offp and returning the 2685 1.52.2.5 jdolecek * new head's byte offset in *offp. 2686 1.52.2.5 jdolecek */ 2687 1.2 simonb static int 2688 1.2 simonb wapbl_write_inodes(struct wapbl *wl, off_t *offp) 2689 1.2 simonb { 2690 1.2 simonb struct wapbl_wc_inodelist *wc = 2691 1.2 simonb (struct wapbl_wc_inodelist *)wl->wl_wc_scratch; 2692 1.2 simonb int i; 2693 1.14 joerg int blocklen = 1 << wl->wl_log_dev_bshift; 2694 1.2 simonb off_t off = *offp; 2695 1.2 simonb int error; 2696 1.2 simonb 2697 1.2 simonb struct wapbl_ino_head *wih; 2698 1.2 simonb struct wapbl_ino *wi; 2699 1.2 simonb int iph; 2700 1.2 simonb 2701 1.2 simonb iph = (blocklen - offsetof(struct wapbl_wc_inodelist, wc_inodes)) / 2702 1.2 simonb sizeof(((struct wapbl_wc_inodelist *)0)->wc_inodes[0]); 2703 1.2 simonb 2704 1.2 simonb i = 0; 2705 1.2 simonb wih = &wl->wl_inohash[0]; 2706 1.2 simonb wi = 0; 2707 1.2 simonb do { 2708 1.2 simonb wc->wc_type = WAPBL_WC_INODES; 2709 1.2 simonb wc->wc_len = blocklen; 2710 1.2 simonb wc->wc_inocnt = 0; 2711 1.2 simonb wc->wc_clear = (i == 0); 2712 1.2 simonb while ((i < wl->wl_inohashcnt) && (wc->wc_inocnt < iph)) { 2713 1.2 simonb while (!wi) { 2714 1.2 simonb KASSERT((wih - &wl->wl_inohash[0]) 2715 1.2 simonb <= wl->wl_inohashmask); 2716 1.2 simonb wi = LIST_FIRST(wih++); 2717 1.2 simonb } 2718 1.2 simonb wc->wc_inodes[wc->wc_inocnt].wc_inumber = wi->wi_ino; 2719 1.2 simonb wc->wc_inodes[wc->wc_inocnt].wc_imode = wi->wi_mode; 2720 1.2 simonb wc->wc_inocnt++; 2721 1.2 simonb i++; 2722 1.2 simonb wi = LIST_NEXT(wi, wi_hash); 2723 1.2 simonb } 2724 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_WRITE, 2725 1.2 simonb ("wapbl_write_inodes: len = %u off = %"PRIdMAX"\n", 2726 1.2 simonb wc->wc_len, (intmax_t)off)); 2727 1.2 simonb error = wapbl_circ_write(wl, wc, blocklen, &off); 2728 1.2 simonb if (error) 2729 1.2 simonb return error; 2730 1.2 simonb } while (i < wl->wl_inohashcnt); 2731 1.52.2.5 jdolecek 2732 1.2 simonb *offp = off; 2733 1.2 simonb return 0; 2734 1.2 simonb } 2735 1.2 simonb 2736 1.2 simonb #endif /* _KERNEL */ 2737 1.2 simonb 2738 1.2 simonb /****************************************************************/ 2739 1.2 simonb 2740 1.2 simonb struct wapbl_blk { 2741 1.2 simonb LIST_ENTRY(wapbl_blk) wb_hash; 2742 1.2 simonb daddr_t wb_blk; 2743 1.2 simonb off_t wb_off; /* Offset of this block in the log */ 2744 1.2 simonb }; 2745 1.2 simonb #define WAPBL_BLKPOOL_MIN 83 2746 1.2 simonb 2747 1.2 simonb static void 2748 1.2 simonb wapbl_blkhash_init(struct wapbl_replay *wr, u_int size) 2749 1.2 simonb { 2750 1.2 simonb if (size < WAPBL_BLKPOOL_MIN) 2751 1.2 simonb size = WAPBL_BLKPOOL_MIN; 2752 1.2 simonb KASSERT(wr->wr_blkhash == 0); 2753 1.2 simonb #ifdef _KERNEL 2754 1.2 simonb wr->wr_blkhash = hashinit(size, HASH_LIST, true, &wr->wr_blkhashmask); 2755 1.2 simonb #else /* ! _KERNEL */ 2756 1.2 simonb /* Manually implement hashinit */ 2757 1.2 simonb { 2758 1.25 lukem unsigned long i, hashsize; 2759 1.2 simonb for (hashsize = 1; hashsize < size; hashsize <<= 1) 2760 1.2 simonb continue; 2761 1.51 para wr->wr_blkhash = wapbl_alloc(hashsize * sizeof(*wr->wr_blkhash)); 2762 1.37 drochner for (i = 0; i < hashsize; i++) 2763 1.2 simonb LIST_INIT(&wr->wr_blkhash[i]); 2764 1.2 simonb wr->wr_blkhashmask = hashsize - 1; 2765 1.2 simonb } 2766 1.2 simonb #endif /* ! _KERNEL */ 2767 1.2 simonb } 2768 1.2 simonb 2769 1.2 simonb static void 2770 1.2 simonb wapbl_blkhash_free(struct wapbl_replay *wr) 2771 1.2 simonb { 2772 1.2 simonb KASSERT(wr->wr_blkhashcnt == 0); 2773 1.2 simonb #ifdef _KERNEL 2774 1.2 simonb hashdone(wr->wr_blkhash, HASH_LIST, wr->wr_blkhashmask); 2775 1.2 simonb #else /* ! _KERNEL */ 2776 1.18 yamt wapbl_free(wr->wr_blkhash, 2777 1.18 yamt (wr->wr_blkhashmask + 1) * sizeof(*wr->wr_blkhash)); 2778 1.2 simonb #endif /* ! _KERNEL */ 2779 1.2 simonb } 2780 1.2 simonb 2781 1.2 simonb static struct wapbl_blk * 2782 1.2 simonb wapbl_blkhash_get(struct wapbl_replay *wr, daddr_t blk) 2783 1.2 simonb { 2784 1.2 simonb struct wapbl_blk_head *wbh; 2785 1.2 simonb struct wapbl_blk *wb; 2786 1.2 simonb wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2787 1.2 simonb LIST_FOREACH(wb, wbh, wb_hash) { 2788 1.2 simonb if (blk == wb->wb_blk) 2789 1.2 simonb return wb; 2790 1.2 simonb } 2791 1.2 simonb return 0; 2792 1.2 simonb } 2793 1.2 simonb 2794 1.2 simonb static void 2795 1.2 simonb wapbl_blkhash_ins(struct wapbl_replay *wr, daddr_t blk, off_t off) 2796 1.2 simonb { 2797 1.2 simonb struct wapbl_blk_head *wbh; 2798 1.2 simonb struct wapbl_blk *wb; 2799 1.2 simonb wb = wapbl_blkhash_get(wr, blk); 2800 1.2 simonb if (wb) { 2801 1.2 simonb KASSERT(wb->wb_blk == blk); 2802 1.2 simonb wb->wb_off = off; 2803 1.2 simonb } else { 2804 1.51 para wb = wapbl_alloc(sizeof(*wb)); 2805 1.2 simonb wb->wb_blk = blk; 2806 1.2 simonb wb->wb_off = off; 2807 1.2 simonb wbh = &wr->wr_blkhash[blk & wr->wr_blkhashmask]; 2808 1.2 simonb LIST_INSERT_HEAD(wbh, wb, wb_hash); 2809 1.2 simonb wr->wr_blkhashcnt++; 2810 1.2 simonb } 2811 1.2 simonb } 2812 1.2 simonb 2813 1.2 simonb static void 2814 1.2 simonb wapbl_blkhash_rem(struct wapbl_replay *wr, daddr_t blk) 2815 1.2 simonb { 2816 1.2 simonb struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 2817 1.2 simonb if (wb) { 2818 1.2 simonb KASSERT(wr->wr_blkhashcnt > 0); 2819 1.2 simonb wr->wr_blkhashcnt--; 2820 1.2 simonb LIST_REMOVE(wb, wb_hash); 2821 1.18 yamt wapbl_free(wb, sizeof(*wb)); 2822 1.2 simonb } 2823 1.2 simonb } 2824 1.2 simonb 2825 1.2 simonb static void 2826 1.2 simonb wapbl_blkhash_clear(struct wapbl_replay *wr) 2827 1.2 simonb { 2828 1.25 lukem unsigned long i; 2829 1.2 simonb for (i = 0; i <= wr->wr_blkhashmask; i++) { 2830 1.2 simonb struct wapbl_blk *wb; 2831 1.2 simonb 2832 1.2 simonb while ((wb = LIST_FIRST(&wr->wr_blkhash[i]))) { 2833 1.2 simonb KASSERT(wr->wr_blkhashcnt > 0); 2834 1.2 simonb wr->wr_blkhashcnt--; 2835 1.2 simonb LIST_REMOVE(wb, wb_hash); 2836 1.18 yamt wapbl_free(wb, sizeof(*wb)); 2837 1.2 simonb } 2838 1.2 simonb } 2839 1.2 simonb KASSERT(wr->wr_blkhashcnt == 0); 2840 1.2 simonb } 2841 1.2 simonb 2842 1.2 simonb /****************************************************************/ 2843 1.2 simonb 2844 1.52.2.5 jdolecek /* 2845 1.52.2.5 jdolecek * wapbl_circ_read(wr, data, len, offp) 2846 1.52.2.5 jdolecek * 2847 1.52.2.5 jdolecek * Read len bytes into data from the circular queue of wr, 2848 1.52.2.5 jdolecek * starting at the linear byte offset *offp, and returning the new 2849 1.52.2.5 jdolecek * linear byte offset in *offp. 2850 1.52.2.5 jdolecek * 2851 1.52.2.5 jdolecek * If the starting linear byte offset precedes wr->wr_circ_off, 2852 1.52.2.5 jdolecek * the read instead begins at wr->wr_circ_off. XXX WTF? This 2853 1.52.2.5 jdolecek * should be a KASSERT, not a conditional. 2854 1.52.2.5 jdolecek */ 2855 1.2 simonb static int 2856 1.2 simonb wapbl_circ_read(struct wapbl_replay *wr, void *data, size_t len, off_t *offp) 2857 1.2 simonb { 2858 1.2 simonb size_t slen; 2859 1.2 simonb off_t off = *offp; 2860 1.2 simonb int error; 2861 1.34 mlelstv daddr_t pbn; 2862 1.2 simonb 2863 1.14 joerg KASSERT(((len >> wr->wr_log_dev_bshift) << 2864 1.14 joerg wr->wr_log_dev_bshift) == len); 2865 1.34 mlelstv 2866 1.14 joerg if (off < wr->wr_circ_off) 2867 1.14 joerg off = wr->wr_circ_off; 2868 1.14 joerg slen = wr->wr_circ_off + wr->wr_circ_size - off; 2869 1.2 simonb if (slen < len) { 2870 1.34 mlelstv pbn = wr->wr_logpbn + (off >> wr->wr_log_dev_bshift); 2871 1.34 mlelstv #ifdef _KERNEL 2872 1.34 mlelstv pbn = btodb(pbn << wr->wr_log_dev_bshift); 2873 1.34 mlelstv #endif 2874 1.34 mlelstv error = wapbl_read(data, slen, wr->wr_devvp, pbn); 2875 1.2 simonb if (error) 2876 1.2 simonb return error; 2877 1.2 simonb data = (uint8_t *)data + slen; 2878 1.2 simonb len -= slen; 2879 1.14 joerg off = wr->wr_circ_off; 2880 1.2 simonb } 2881 1.34 mlelstv pbn = wr->wr_logpbn + (off >> wr->wr_log_dev_bshift); 2882 1.34 mlelstv #ifdef _KERNEL 2883 1.34 mlelstv pbn = btodb(pbn << wr->wr_log_dev_bshift); 2884 1.34 mlelstv #endif 2885 1.34 mlelstv error = wapbl_read(data, len, wr->wr_devvp, pbn); 2886 1.2 simonb if (error) 2887 1.2 simonb return error; 2888 1.2 simonb off += len; 2889 1.14 joerg if (off >= wr->wr_circ_off + wr->wr_circ_size) 2890 1.14 joerg off = wr->wr_circ_off; 2891 1.2 simonb *offp = off; 2892 1.2 simonb return 0; 2893 1.2 simonb } 2894 1.2 simonb 2895 1.52.2.5 jdolecek /* 2896 1.52.2.5 jdolecek * wapbl_circ_advance(wr, len, offp) 2897 1.52.2.5 jdolecek * 2898 1.52.2.5 jdolecek * Compute the linear byte offset of the circular queue of wr that 2899 1.52.2.5 jdolecek * is len bytes past *offp, and store it in *offp. 2900 1.52.2.5 jdolecek * 2901 1.52.2.5 jdolecek * This is as if wapbl_circ_read, but without actually reading 2902 1.52.2.5 jdolecek * anything. 2903 1.52.2.5 jdolecek * 2904 1.52.2.5 jdolecek * If the starting linear byte offset precedes wr->wr_circ_off, it 2905 1.52.2.5 jdolecek * is taken to be wr->wr_circ_off instead. XXX WTF? This should 2906 1.52.2.5 jdolecek * be a KASSERT, not a conditional. 2907 1.52.2.5 jdolecek */ 2908 1.2 simonb static void 2909 1.2 simonb wapbl_circ_advance(struct wapbl_replay *wr, size_t len, off_t *offp) 2910 1.2 simonb { 2911 1.2 simonb size_t slen; 2912 1.2 simonb off_t off = *offp; 2913 1.2 simonb 2914 1.14 joerg KASSERT(((len >> wr->wr_log_dev_bshift) << 2915 1.14 joerg wr->wr_log_dev_bshift) == len); 2916 1.2 simonb 2917 1.14 joerg if (off < wr->wr_circ_off) 2918 1.14 joerg off = wr->wr_circ_off; 2919 1.14 joerg slen = wr->wr_circ_off + wr->wr_circ_size - off; 2920 1.2 simonb if (slen < len) { 2921 1.2 simonb len -= slen; 2922 1.14 joerg off = wr->wr_circ_off; 2923 1.2 simonb } 2924 1.2 simonb off += len; 2925 1.14 joerg if (off >= wr->wr_circ_off + wr->wr_circ_size) 2926 1.14 joerg off = wr->wr_circ_off; 2927 1.2 simonb *offp = off; 2928 1.2 simonb } 2929 1.2 simonb 2930 1.2 simonb /****************************************************************/ 2931 1.2 simonb 2932 1.2 simonb int 2933 1.2 simonb wapbl_replay_start(struct wapbl_replay **wrp, struct vnode *vp, 2934 1.2 simonb daddr_t off, size_t count, size_t blksize) 2935 1.2 simonb { 2936 1.2 simonb struct wapbl_replay *wr; 2937 1.2 simonb int error; 2938 1.2 simonb struct vnode *devvp; 2939 1.2 simonb daddr_t logpbn; 2940 1.2 simonb uint8_t *scratch; 2941 1.2 simonb struct wapbl_wc_header *wch; 2942 1.2 simonb struct wapbl_wc_header *wch2; 2943 1.2 simonb /* Use this until we read the actual log header */ 2944 1.31 mlelstv int log_dev_bshift = ilog2(blksize); 2945 1.2 simonb size_t used; 2946 1.34 mlelstv daddr_t pbn; 2947 1.2 simonb 2948 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 2949 1.2 simonb ("wapbl_replay_start: vp=%p off=%"PRId64 " count=%zu blksize=%zu\n", 2950 1.2 simonb vp, off, count, blksize)); 2951 1.2 simonb 2952 1.2 simonb if (off < 0) 2953 1.2 simonb return EINVAL; 2954 1.2 simonb 2955 1.2 simonb if (blksize < DEV_BSIZE) 2956 1.2 simonb return EINVAL; 2957 1.2 simonb if (blksize % DEV_BSIZE) 2958 1.2 simonb return EINVAL; 2959 1.2 simonb 2960 1.2 simonb #ifdef _KERNEL 2961 1.2 simonb #if 0 2962 1.2 simonb /* XXX vp->v_size isn't reliably set for VBLK devices, 2963 1.2 simonb * especially root. However, we might still want to verify 2964 1.2 simonb * that the full load is readable */ 2965 1.2 simonb if ((off + count) * blksize > vp->v_size) 2966 1.2 simonb return EINVAL; 2967 1.2 simonb #endif 2968 1.2 simonb if ((error = VOP_BMAP(vp, off, &devvp, &logpbn, 0)) != 0) { 2969 1.2 simonb return error; 2970 1.2 simonb } 2971 1.2 simonb #else /* ! _KERNEL */ 2972 1.2 simonb devvp = vp; 2973 1.2 simonb logpbn = off; 2974 1.2 simonb #endif /* ! _KERNEL */ 2975 1.2 simonb 2976 1.51 para scratch = wapbl_alloc(MAXBSIZE); 2977 1.2 simonb 2978 1.34 mlelstv pbn = logpbn; 2979 1.34 mlelstv #ifdef _KERNEL 2980 1.34 mlelstv pbn = btodb(pbn << log_dev_bshift); 2981 1.34 mlelstv #endif 2982 1.34 mlelstv error = wapbl_read(scratch, 2<<log_dev_bshift, devvp, pbn); 2983 1.2 simonb if (error) 2984 1.2 simonb goto errout; 2985 1.2 simonb 2986 1.2 simonb wch = (struct wapbl_wc_header *)scratch; 2987 1.2 simonb wch2 = 2988 1.2 simonb (struct wapbl_wc_header *)(scratch + (1<<log_dev_bshift)); 2989 1.2 simonb /* XXX verify checksums and magic numbers */ 2990 1.2 simonb if (wch->wc_type != WAPBL_WC_HEADER) { 2991 1.2 simonb printf("Unrecognized wapbl magic: 0x%08x\n", wch->wc_type); 2992 1.2 simonb error = EFTYPE; 2993 1.2 simonb goto errout; 2994 1.2 simonb } 2995 1.2 simonb 2996 1.2 simonb if (wch2->wc_generation > wch->wc_generation) 2997 1.2 simonb wch = wch2; 2998 1.2 simonb 2999 1.2 simonb wr = wapbl_calloc(1, sizeof(*wr)); 3000 1.2 simonb 3001 1.2 simonb wr->wr_logvp = vp; 3002 1.2 simonb wr->wr_devvp = devvp; 3003 1.2 simonb wr->wr_logpbn = logpbn; 3004 1.2 simonb 3005 1.2 simonb wr->wr_scratch = scratch; 3006 1.2 simonb 3007 1.14 joerg wr->wr_log_dev_bshift = wch->wc_log_dev_bshift; 3008 1.14 joerg wr->wr_fs_dev_bshift = wch->wc_fs_dev_bshift; 3009 1.14 joerg wr->wr_circ_off = wch->wc_circ_off; 3010 1.14 joerg wr->wr_circ_size = wch->wc_circ_size; 3011 1.14 joerg wr->wr_generation = wch->wc_generation; 3012 1.2 simonb 3013 1.2 simonb used = wapbl_space_used(wch->wc_circ_size, wch->wc_head, wch->wc_tail); 3014 1.2 simonb 3015 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_REPLAY, 3016 1.2 simonb ("wapbl_replay: head=%"PRId64" tail=%"PRId64" off=%"PRId64 3017 1.2 simonb " len=%"PRId64" used=%zu\n", 3018 1.2 simonb wch->wc_head, wch->wc_tail, wch->wc_circ_off, 3019 1.2 simonb wch->wc_circ_size, used)); 3020 1.2 simonb 3021 1.2 simonb wapbl_blkhash_init(wr, (used >> wch->wc_fs_dev_bshift)); 3022 1.11 joerg 3023 1.14 joerg error = wapbl_replay_process(wr, wch->wc_head, wch->wc_tail); 3024 1.2 simonb if (error) { 3025 1.2 simonb wapbl_replay_stop(wr); 3026 1.2 simonb wapbl_replay_free(wr); 3027 1.2 simonb return error; 3028 1.2 simonb } 3029 1.2 simonb 3030 1.2 simonb *wrp = wr; 3031 1.2 simonb return 0; 3032 1.2 simonb 3033 1.2 simonb errout: 3034 1.18 yamt wapbl_free(scratch, MAXBSIZE); 3035 1.2 simonb return error; 3036 1.2 simonb } 3037 1.2 simonb 3038 1.2 simonb void 3039 1.2 simonb wapbl_replay_stop(struct wapbl_replay *wr) 3040 1.2 simonb { 3041 1.2 simonb 3042 1.4 joerg if (!wapbl_replay_isopen(wr)) 3043 1.4 joerg return; 3044 1.4 joerg 3045 1.2 simonb WAPBL_PRINTF(WAPBL_PRINT_REPLAY, ("wapbl_replay_stop called\n")); 3046 1.2 simonb 3047 1.18 yamt wapbl_free(wr->wr_scratch, MAXBSIZE); 3048 1.18 yamt wr->wr_scratch = NULL; 3049 1.2 simonb 3050 1.18 yamt wr->wr_logvp = NULL; 3051 1.2 simonb 3052 1.2 simonb wapbl_blkhash_clear(wr); 3053 1.2 simonb wapbl_blkhash_free(wr); 3054 1.2 simonb } 3055 1.2 simonb 3056 1.2 simonb void 3057 1.2 simonb wapbl_replay_free(struct wapbl_replay *wr) 3058 1.2 simonb { 3059 1.2 simonb 3060 1.2 simonb KDASSERT(!wapbl_replay_isopen(wr)); 3061 1.2 simonb 3062 1.2 simonb if (wr->wr_inodes) 3063 1.18 yamt wapbl_free(wr->wr_inodes, 3064 1.18 yamt wr->wr_inodescnt * sizeof(wr->wr_inodes[0])); 3065 1.18 yamt wapbl_free(wr, sizeof(*wr)); 3066 1.2 simonb } 3067 1.2 simonb 3068 1.4 joerg #ifdef _KERNEL 3069 1.2 simonb int 3070 1.2 simonb wapbl_replay_isopen1(struct wapbl_replay *wr) 3071 1.2 simonb { 3072 1.2 simonb 3073 1.2 simonb return wapbl_replay_isopen(wr); 3074 1.2 simonb } 3075 1.4 joerg #endif 3076 1.2 simonb 3077 1.52.2.5 jdolecek /* 3078 1.52.2.5 jdolecek * calculate the disk address for the i'th block in the wc_blockblist 3079 1.52.2.5 jdolecek * offset by j blocks of size blen. 3080 1.52.2.5 jdolecek * 3081 1.52.2.5 jdolecek * wc_daddr is always a kernel disk address in DEV_BSIZE units that 3082 1.52.2.5 jdolecek * was written to the journal. 3083 1.52.2.5 jdolecek * 3084 1.52.2.5 jdolecek * The kernel needs that address plus the offset in DEV_BSIZE units. 3085 1.52.2.5 jdolecek * 3086 1.52.2.5 jdolecek * Userland needs that address plus the offset in blen units. 3087 1.52.2.5 jdolecek * 3088 1.52.2.5 jdolecek */ 3089 1.52.2.5 jdolecek static daddr_t 3090 1.52.2.5 jdolecek wapbl_block_daddr(struct wapbl_wc_blocklist *wc, int i, int j, int blen) 3091 1.52.2.5 jdolecek { 3092 1.52.2.5 jdolecek daddr_t pbn; 3093 1.52.2.5 jdolecek 3094 1.52.2.5 jdolecek #ifdef _KERNEL 3095 1.52.2.5 jdolecek pbn = wc->wc_blocks[i].wc_daddr + btodb(j * blen); 3096 1.52.2.5 jdolecek #else 3097 1.52.2.5 jdolecek pbn = dbtob(wc->wc_blocks[i].wc_daddr) / blen + j; 3098 1.52.2.5 jdolecek #endif 3099 1.52.2.5 jdolecek 3100 1.52.2.5 jdolecek return pbn; 3101 1.52.2.5 jdolecek } 3102 1.52.2.5 jdolecek 3103 1.10 joerg static void 3104 1.10 joerg wapbl_replay_process_blocks(struct wapbl_replay *wr, off_t *offp) 3105 1.10 joerg { 3106 1.10 joerg struct wapbl_wc_blocklist *wc = 3107 1.10 joerg (struct wapbl_wc_blocklist *)wr->wr_scratch; 3108 1.14 joerg int fsblklen = 1 << wr->wr_fs_dev_bshift; 3109 1.10 joerg int i, j, n; 3110 1.10 joerg 3111 1.10 joerg for (i = 0; i < wc->wc_blkcount; i++) { 3112 1.10 joerg /* 3113 1.10 joerg * Enter each physical block into the hashtable independently. 3114 1.10 joerg */ 3115 1.14 joerg n = wc->wc_blocks[i].wc_dlen >> wr->wr_fs_dev_bshift; 3116 1.10 joerg for (j = 0; j < n; j++) { 3117 1.52.2.5 jdolecek wapbl_blkhash_ins(wr, wapbl_block_daddr(wc, i, j, fsblklen), 3118 1.10 joerg *offp); 3119 1.10 joerg wapbl_circ_advance(wr, fsblklen, offp); 3120 1.10 joerg } 3121 1.10 joerg } 3122 1.10 joerg } 3123 1.10 joerg 3124 1.10 joerg static void 3125 1.10 joerg wapbl_replay_process_revocations(struct wapbl_replay *wr) 3126 1.10 joerg { 3127 1.10 joerg struct wapbl_wc_blocklist *wc = 3128 1.10 joerg (struct wapbl_wc_blocklist *)wr->wr_scratch; 3129 1.34 mlelstv int fsblklen = 1 << wr->wr_fs_dev_bshift; 3130 1.10 joerg int i, j, n; 3131 1.10 joerg 3132 1.10 joerg for (i = 0; i < wc->wc_blkcount; i++) { 3133 1.10 joerg /* 3134 1.10 joerg * Remove any blocks found from the hashtable. 3135 1.10 joerg */ 3136 1.14 joerg n = wc->wc_blocks[i].wc_dlen >> wr->wr_fs_dev_bshift; 3137 1.10 joerg for (j = 0; j < n; j++) 3138 1.52.2.5 jdolecek wapbl_blkhash_rem(wr, wapbl_block_daddr(wc, i, j, fsblklen)); 3139 1.10 joerg } 3140 1.10 joerg } 3141 1.10 joerg 3142 1.10 joerg static void 3143 1.10 joerg wapbl_replay_process_inodes(struct wapbl_replay *wr, off_t oldoff, off_t newoff) 3144 1.10 joerg { 3145 1.10 joerg struct wapbl_wc_inodelist *wc = 3146 1.10 joerg (struct wapbl_wc_inodelist *)wr->wr_scratch; 3147 1.18 yamt void *new_inodes; 3148 1.18 yamt const size_t oldsize = wr->wr_inodescnt * sizeof(wr->wr_inodes[0]); 3149 1.18 yamt 3150 1.18 yamt KASSERT(sizeof(wr->wr_inodes[0]) == sizeof(wc->wc_inodes[0])); 3151 1.18 yamt 3152 1.10 joerg /* 3153 1.10 joerg * Keep track of where we found this so location won't be 3154 1.10 joerg * overwritten. 3155 1.10 joerg */ 3156 1.10 joerg if (wc->wc_clear) { 3157 1.10 joerg wr->wr_inodestail = oldoff; 3158 1.10 joerg wr->wr_inodescnt = 0; 3159 1.12 joerg if (wr->wr_inodes != NULL) { 3160 1.18 yamt wapbl_free(wr->wr_inodes, oldsize); 3161 1.12 joerg wr->wr_inodes = NULL; 3162 1.12 joerg } 3163 1.10 joerg } 3164 1.10 joerg wr->wr_inodeshead = newoff; 3165 1.10 joerg if (wc->wc_inocnt == 0) 3166 1.10 joerg return; 3167 1.10 joerg 3168 1.51 para new_inodes = wapbl_alloc((wr->wr_inodescnt + wc->wc_inocnt) * 3169 1.18 yamt sizeof(wr->wr_inodes[0])); 3170 1.18 yamt if (wr->wr_inodes != NULL) { 3171 1.18 yamt memcpy(new_inodes, wr->wr_inodes, oldsize); 3172 1.18 yamt wapbl_free(wr->wr_inodes, oldsize); 3173 1.18 yamt } 3174 1.18 yamt wr->wr_inodes = new_inodes; 3175 1.10 joerg memcpy(&wr->wr_inodes[wr->wr_inodescnt], wc->wc_inodes, 3176 1.18 yamt wc->wc_inocnt * sizeof(wr->wr_inodes[0])); 3177 1.10 joerg wr->wr_inodescnt += wc->wc_inocnt; 3178 1.10 joerg } 3179 1.10 joerg 3180 1.2 simonb static int 3181 1.14 joerg wapbl_replay_process(struct wapbl_replay *wr, off_t head, off_t tail) 3182 1.2 simonb { 3183 1.2 simonb off_t off; 3184 1.2 simonb int error; 3185 1.2 simonb 3186 1.14 joerg int logblklen = 1 << wr->wr_log_dev_bshift; 3187 1.2 simonb 3188 1.2 simonb wapbl_blkhash_clear(wr); 3189 1.2 simonb 3190 1.14 joerg off = tail; 3191 1.14 joerg while (off != head) { 3192 1.2 simonb struct wapbl_wc_null *wcn; 3193 1.2 simonb off_t saveoff = off; 3194 1.2 simonb error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 3195 1.2 simonb if (error) 3196 1.2 simonb goto errout; 3197 1.2 simonb wcn = (struct wapbl_wc_null *)wr->wr_scratch; 3198 1.2 simonb switch (wcn->wc_type) { 3199 1.2 simonb case WAPBL_WC_BLOCKS: 3200 1.10 joerg wapbl_replay_process_blocks(wr, &off); 3201 1.2 simonb break; 3202 1.2 simonb 3203 1.2 simonb case WAPBL_WC_REVOCATIONS: 3204 1.10 joerg wapbl_replay_process_revocations(wr); 3205 1.2 simonb break; 3206 1.2 simonb 3207 1.2 simonb case WAPBL_WC_INODES: 3208 1.10 joerg wapbl_replay_process_inodes(wr, saveoff, off); 3209 1.2 simonb break; 3210 1.10 joerg 3211 1.2 simonb default: 3212 1.2 simonb printf("Unrecognized wapbl type: 0x%08x\n", 3213 1.2 simonb wcn->wc_type); 3214 1.2 simonb error = EFTYPE; 3215 1.2 simonb goto errout; 3216 1.2 simonb } 3217 1.2 simonb wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 3218 1.2 simonb if (off != saveoff) { 3219 1.2 simonb printf("wapbl_replay: corrupted records\n"); 3220 1.2 simonb error = EFTYPE; 3221 1.2 simonb goto errout; 3222 1.2 simonb } 3223 1.2 simonb } 3224 1.2 simonb return 0; 3225 1.2 simonb 3226 1.2 simonb errout: 3227 1.2 simonb wapbl_blkhash_clear(wr); 3228 1.2 simonb return error; 3229 1.2 simonb } 3230 1.2 simonb 3231 1.13 joerg #if 0 3232 1.2 simonb int 3233 1.2 simonb wapbl_replay_verify(struct wapbl_replay *wr, struct vnode *fsdevvp) 3234 1.2 simonb { 3235 1.2 simonb off_t off; 3236 1.2 simonb int mismatchcnt = 0; 3237 1.14 joerg int logblklen = 1 << wr->wr_log_dev_bshift; 3238 1.14 joerg int fsblklen = 1 << wr->wr_fs_dev_bshift; 3239 1.51 para void *scratch1 = wapbl_alloc(MAXBSIZE); 3240 1.51 para void *scratch2 = wapbl_alloc(MAXBSIZE); 3241 1.2 simonb int error = 0; 3242 1.2 simonb 3243 1.2 simonb KDASSERT(wapbl_replay_isopen(wr)); 3244 1.2 simonb 3245 1.2 simonb off = wch->wc_tail; 3246 1.2 simonb while (off != wch->wc_head) { 3247 1.2 simonb struct wapbl_wc_null *wcn; 3248 1.2 simonb #ifdef DEBUG 3249 1.2 simonb off_t saveoff = off; 3250 1.2 simonb #endif 3251 1.2 simonb error = wapbl_circ_read(wr, wr->wr_scratch, logblklen, &off); 3252 1.2 simonb if (error) 3253 1.2 simonb goto out; 3254 1.2 simonb wcn = (struct wapbl_wc_null *)wr->wr_scratch; 3255 1.2 simonb switch (wcn->wc_type) { 3256 1.2 simonb case WAPBL_WC_BLOCKS: 3257 1.2 simonb { 3258 1.2 simonb struct wapbl_wc_blocklist *wc = 3259 1.2 simonb (struct wapbl_wc_blocklist *)wr->wr_scratch; 3260 1.2 simonb int i; 3261 1.2 simonb for (i = 0; i < wc->wc_blkcount; i++) { 3262 1.2 simonb int foundcnt = 0; 3263 1.2 simonb int dirtycnt = 0; 3264 1.2 simonb int j, n; 3265 1.2 simonb /* 3266 1.2 simonb * Check each physical block into the 3267 1.2 simonb * hashtable independently 3268 1.2 simonb */ 3269 1.2 simonb n = wc->wc_blocks[i].wc_dlen >> 3270 1.2 simonb wch->wc_fs_dev_bshift; 3271 1.2 simonb for (j = 0; j < n; j++) { 3272 1.2 simonb struct wapbl_blk *wb = 3273 1.2 simonb wapbl_blkhash_get(wr, 3274 1.52.2.5 jdolecek wapbl_block_daddr(wc, i, j, fsblklen)); 3275 1.2 simonb if (wb && (wb->wb_off == off)) { 3276 1.2 simonb foundcnt++; 3277 1.2 simonb error = 3278 1.2 simonb wapbl_circ_read(wr, 3279 1.2 simonb scratch1, fsblklen, 3280 1.2 simonb &off); 3281 1.2 simonb if (error) 3282 1.2 simonb goto out; 3283 1.2 simonb error = 3284 1.2 simonb wapbl_read(scratch2, 3285 1.2 simonb fsblklen, fsdevvp, 3286 1.2 simonb wb->wb_blk); 3287 1.2 simonb if (error) 3288 1.2 simonb goto out; 3289 1.2 simonb if (memcmp(scratch1, 3290 1.2 simonb scratch2, 3291 1.2 simonb fsblklen)) { 3292 1.2 simonb printf( 3293 1.2 simonb "wapbl_verify: mismatch block %"PRId64" at off %"PRIdMAX"\n", 3294 1.2 simonb wb->wb_blk, (intmax_t)off); 3295 1.2 simonb dirtycnt++; 3296 1.2 simonb mismatchcnt++; 3297 1.2 simonb } 3298 1.2 simonb } else { 3299 1.2 simonb wapbl_circ_advance(wr, 3300 1.2 simonb fsblklen, &off); 3301 1.2 simonb } 3302 1.2 simonb } 3303 1.2 simonb #if 0 3304 1.2 simonb /* 3305 1.2 simonb * If all of the blocks in an entry 3306 1.2 simonb * are clean, then remove all of its 3307 1.2 simonb * blocks from the hashtable since they 3308 1.2 simonb * never will need replay. 3309 1.2 simonb */ 3310 1.2 simonb if ((foundcnt != 0) && 3311 1.2 simonb (dirtycnt == 0)) { 3312 1.2 simonb off = saveoff; 3313 1.2 simonb wapbl_circ_advance(wr, 3314 1.2 simonb logblklen, &off); 3315 1.2 simonb for (j = 0; j < n; j++) { 3316 1.2 simonb struct wapbl_blk *wb = 3317 1.2 simonb wapbl_blkhash_get(wr, 3318 1.52.2.5 jdolecek wapbl_block_daddr(wc, i, j, fsblklen)); 3319 1.2 simonb if (wb && 3320 1.2 simonb (wb->wb_off == off)) { 3321 1.2 simonb wapbl_blkhash_rem(wr, wb->wb_blk); 3322 1.2 simonb } 3323 1.2 simonb wapbl_circ_advance(wr, 3324 1.2 simonb fsblklen, &off); 3325 1.2 simonb } 3326 1.2 simonb } 3327 1.2 simonb #endif 3328 1.2 simonb } 3329 1.2 simonb } 3330 1.2 simonb break; 3331 1.2 simonb case WAPBL_WC_REVOCATIONS: 3332 1.2 simonb case WAPBL_WC_INODES: 3333 1.2 simonb break; 3334 1.2 simonb default: 3335 1.2 simonb KASSERT(0); 3336 1.2 simonb } 3337 1.2 simonb #ifdef DEBUG 3338 1.2 simonb wapbl_circ_advance(wr, wcn->wc_len, &saveoff); 3339 1.2 simonb KASSERT(off == saveoff); 3340 1.2 simonb #endif 3341 1.2 simonb } 3342 1.2 simonb out: 3343 1.18 yamt wapbl_free(scratch1, MAXBSIZE); 3344 1.18 yamt wapbl_free(scratch2, MAXBSIZE); 3345 1.2 simonb if (!error && mismatchcnt) 3346 1.2 simonb error = EFTYPE; 3347 1.2 simonb return error; 3348 1.2 simonb } 3349 1.2 simonb #endif 3350 1.2 simonb 3351 1.2 simonb int 3352 1.2 simonb wapbl_replay_write(struct wapbl_replay *wr, struct vnode *fsdevvp) 3353 1.2 simonb { 3354 1.9 joerg struct wapbl_blk *wb; 3355 1.9 joerg size_t i; 3356 1.2 simonb off_t off; 3357 1.9 joerg void *scratch; 3358 1.2 simonb int error = 0; 3359 1.14 joerg int fsblklen = 1 << wr->wr_fs_dev_bshift; 3360 1.2 simonb 3361 1.2 simonb KDASSERT(wapbl_replay_isopen(wr)); 3362 1.2 simonb 3363 1.51 para scratch = wapbl_alloc(MAXBSIZE); 3364 1.2 simonb 3365 1.37 drochner for (i = 0; i <= wr->wr_blkhashmask; ++i) { 3366 1.9 joerg LIST_FOREACH(wb, &wr->wr_blkhash[i], wb_hash) { 3367 1.9 joerg off = wb->wb_off; 3368 1.9 joerg error = wapbl_circ_read(wr, scratch, fsblklen, &off); 3369 1.9 joerg if (error) 3370 1.9 joerg break; 3371 1.9 joerg error = wapbl_write(scratch, fsblklen, fsdevvp, 3372 1.9 joerg wb->wb_blk); 3373 1.9 joerg if (error) 3374 1.9 joerg break; 3375 1.2 simonb } 3376 1.2 simonb } 3377 1.9 joerg 3378 1.18 yamt wapbl_free(scratch, MAXBSIZE); 3379 1.2 simonb return error; 3380 1.2 simonb } 3381 1.2 simonb 3382 1.2 simonb int 3383 1.6 joerg wapbl_replay_can_read(struct wapbl_replay *wr, daddr_t blk, long len) 3384 1.6 joerg { 3385 1.14 joerg int fsblklen = 1 << wr->wr_fs_dev_bshift; 3386 1.6 joerg 3387 1.6 joerg KDASSERT(wapbl_replay_isopen(wr)); 3388 1.6 joerg KASSERT((len % fsblklen) == 0); 3389 1.6 joerg 3390 1.6 joerg while (len != 0) { 3391 1.6 joerg struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 3392 1.6 joerg if (wb) 3393 1.6 joerg return 1; 3394 1.6 joerg len -= fsblklen; 3395 1.6 joerg } 3396 1.6 joerg return 0; 3397 1.6 joerg } 3398 1.6 joerg 3399 1.6 joerg int 3400 1.2 simonb wapbl_replay_read(struct wapbl_replay *wr, void *data, daddr_t blk, long len) 3401 1.2 simonb { 3402 1.14 joerg int fsblklen = 1 << wr->wr_fs_dev_bshift; 3403 1.2 simonb 3404 1.2 simonb KDASSERT(wapbl_replay_isopen(wr)); 3405 1.2 simonb 3406 1.2 simonb KASSERT((len % fsblklen) == 0); 3407 1.2 simonb 3408 1.2 simonb while (len != 0) { 3409 1.2 simonb struct wapbl_blk *wb = wapbl_blkhash_get(wr, blk); 3410 1.2 simonb if (wb) { 3411 1.2 simonb off_t off = wb->wb_off; 3412 1.2 simonb int error; 3413 1.2 simonb error = wapbl_circ_read(wr, data, fsblklen, &off); 3414 1.2 simonb if (error) 3415 1.2 simonb return error; 3416 1.2 simonb } 3417 1.2 simonb data = (uint8_t *)data + fsblklen; 3418 1.2 simonb len -= fsblklen; 3419 1.2 simonb blk++; 3420 1.2 simonb } 3421 1.2 simonb return 0; 3422 1.2 simonb } 3423 1.35 pooka 3424 1.36 pooka #ifdef _KERNEL 3425 1.52.2.5 jdolecek 3426 1.35 pooka MODULE(MODULE_CLASS_VFS, wapbl, NULL); 3427 1.35 pooka 3428 1.35 pooka static int 3429 1.35 pooka wapbl_modcmd(modcmd_t cmd, void *arg) 3430 1.35 pooka { 3431 1.35 pooka 3432 1.35 pooka switch (cmd) { 3433 1.35 pooka case MODULE_CMD_INIT: 3434 1.39 christos wapbl_init(); 3435 1.35 pooka return 0; 3436 1.35 pooka case MODULE_CMD_FINI: 3437 1.52.2.5 jdolecek return wapbl_fini(); 3438 1.35 pooka default: 3439 1.35 pooka return ENOTTY; 3440 1.35 pooka } 3441 1.35 pooka } 3442 1.36 pooka #endif /* _KERNEL */ 3443
Indexes created Fri Oct 03 02:09:57 GMT 2025