lfs_segment.c revision 1.199 1 /* $NetBSD: lfs_segment.c,v 1.199 2007/04/17 01:16:46 perseant Exp $ */
2
3 /*-
4 * Copyright (c) 1999, 2000, 2001, 2002, 2003 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Konrad E. Schroder <perseant (at) hhhh.org>.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by the NetBSD
21 * Foundation, Inc. and its contributors.
22 * 4. Neither the name of The NetBSD Foundation nor the names of its
23 * contributors may be used to endorse or promote products derived
24 * from this software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
37 */
38 /*
39 * Copyright (c) 1991, 1993
40 * The Regents of the University of California. All rights reserved.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)lfs_segment.c 8.10 (Berkeley) 6/10/95
67 */
68
69 #include <sys/cdefs.h>
70 __KERNEL_RCSID(0, "$NetBSD: lfs_segment.c,v 1.199 2007/04/17 01:16:46 perseant Exp $");
71
72 #ifdef DEBUG
73 # define vndebug(vp, str) do { \
74 if (VTOI(vp)->i_flag & IN_CLEANING) \
75 DLOG((DLOG_WVNODE, "not writing ino %d because %s (op %d)\n", \
76 VTOI(vp)->i_number, (str), op)); \
77 } while(0)
78 #else
79 # define vndebug(vp, str)
80 #endif
81 #define ivndebug(vp, str) \
82 DLOG((DLOG_WVNODE, "ino %d: %s\n", VTOI(vp)->i_number, (str)))
83
84 #if defined(_KERNEL_OPT)
85 #include "opt_ddb.h"
86 #endif
87
88 #include <sys/param.h>
89 #include <sys/systm.h>
90 #include <sys/namei.h>
91 #include <sys/kernel.h>
92 #include <sys/resourcevar.h>
93 #include <sys/file.h>
94 #include <sys/stat.h>
95 #include <sys/buf.h>
96 #include <sys/proc.h>
97 #include <sys/vnode.h>
98 #include <sys/mount.h>
99 #include <sys/kauth.h>
100 #include <sys/syslog.h>
101
102 #include <miscfs/specfs/specdev.h>
103 #include <miscfs/fifofs/fifo.h>
104
105 #include <ufs/ufs/inode.h>
106 #include <ufs/ufs/dir.h>
107 #include <ufs/ufs/ufsmount.h>
108 #include <ufs/ufs/ufs_extern.h>
109
110 #include <ufs/lfs/lfs.h>
111 #include <ufs/lfs/lfs_extern.h>
112
113 #include <uvm/uvm.h>
114 #include <uvm/uvm_extern.h>
115
116 MALLOC_DEFINE(M_SEGMENT, "LFS segment", "Segment for LFS");
117
118 extern int count_lock_queue(void);
119 extern struct simplelock vnode_free_list_slock; /* XXX */
120 extern struct simplelock bqueue_slock; /* XXX */
121
122 static void lfs_generic_callback(struct buf *, void (*)(struct buf *));
123 static void lfs_free_aiodone(struct buf *);
124 static void lfs_super_aiodone(struct buf *);
125 static void lfs_cluster_aiodone(struct buf *);
126 static void lfs_cluster_callback(struct buf *);
127
128 /*
129 * Determine if it's OK to start a partial in this segment, or if we need
130 * to go on to a new segment.
131 */
132 #define LFS_PARTIAL_FITS(fs) \
133 ((fs)->lfs_fsbpseg - ((fs)->lfs_offset - (fs)->lfs_curseg) > \
134 fragstofsb((fs), (fs)->lfs_frag))
135
136 /*
137 * Figure out whether we should do a checkpoint write or go ahead with
138 * an ordinary write.
139 */
140 #define LFS_SHOULD_CHECKPOINT(fs, flags) \
141 ((flags & SEGM_CLEAN) == 0 && \
142 ((fs->lfs_nactive > LFS_MAX_ACTIVE || \
143 (flags & SEGM_CKP) || \
144 fs->lfs_nclean < LFS_MAX_ACTIVE)))
145
146 int lfs_match_fake(struct lfs *, struct buf *);
147 void lfs_newseg(struct lfs *);
148 /* XXX ondisk32 */
149 void lfs_shellsort(struct buf **, int32_t *, int, int);
150 void lfs_supercallback(struct buf *);
151 void lfs_updatemeta(struct segment *);
152 void lfs_writesuper(struct lfs *, daddr_t);
153 int lfs_writevnodes(struct lfs *fs, struct mount *mp,
154 struct segment *sp, int dirops);
155
156 int lfs_allclean_wakeup; /* Cleaner wakeup address. */
157 int lfs_writeindir = 1; /* whether to flush indir on non-ckp */
158 int lfs_clean_vnhead = 0; /* Allow freeing to head of vn list */
159 int lfs_dirvcount = 0; /* # active dirops */
160
161 /* Statistics Counters */
162 int lfs_dostats = 1;
163 struct lfs_stats lfs_stats;
164
165 /* op values to lfs_writevnodes */
166 #define VN_REG 0
167 #define VN_DIROP 1
168 #define VN_EMPTY 2
169 #define VN_CLEAN 3
170
171 /*
172 * XXX KS - Set modification time on the Ifile, so the cleaner can
173 * read the fs mod time off of it. We don't set IN_UPDATE here,
174 * since we don't really need this to be flushed to disk (and in any
175 * case that wouldn't happen to the Ifile until we checkpoint).
176 */
177 void
178 lfs_imtime(struct lfs *fs)
179 {
180 struct timespec ts;
181 struct inode *ip;
182
183 ASSERT_MAYBE_SEGLOCK(fs);
184 vfs_timestamp(&ts);
185 ip = VTOI(fs->lfs_ivnode);
186 ip->i_ffs1_mtime = ts.tv_sec;
187 ip->i_ffs1_mtimensec = ts.tv_nsec;
188 }
189
190 /*
191 * Ifile and meta data blocks are not marked busy, so segment writes MUST be
192 * single threaded. Currently, there are two paths into lfs_segwrite, sync()
193 * and getnewbuf(). They both mark the file system busy. Lfs_vflush()
194 * explicitly marks the file system busy. So lfs_segwrite is safe. I think.
195 */
196
197 #define IS_FLUSHING(fs,vp) ((fs)->lfs_flushvp == (vp))
198
199 int
200 lfs_vflush(struct vnode *vp)
201 {
202 struct inode *ip;
203 struct lfs *fs;
204 struct segment *sp;
205 struct buf *bp, *nbp, *tbp, *tnbp;
206 int error, s;
207 int flushed;
208 int relock;
209 int loopcount;
210
211 ip = VTOI(vp);
212 fs = VFSTOUFS(vp->v_mount)->um_lfs;
213 relock = 0;
214
215 top:
216 ASSERT_NO_SEGLOCK(fs);
217 if (ip->i_flag & IN_CLEANING) {
218 ivndebug(vp,"vflush/in_cleaning");
219 LFS_CLR_UINO(ip, IN_CLEANING);
220 LFS_SET_UINO(ip, IN_MODIFIED);
221
222 /*
223 * Toss any cleaning buffers that have real counterparts
224 * to avoid losing new data.
225 */
226 s = splbio();
227 for (bp = LIST_FIRST(&vp->v_dirtyblkhd); bp; bp = nbp) {
228 nbp = LIST_NEXT(bp, b_vnbufs);
229 if (!LFS_IS_MALLOC_BUF(bp))
230 continue;
231 /*
232 * Look for pages matching the range covered
233 * by cleaning blocks. It's okay if more dirty
234 * pages appear, so long as none disappear out
235 * from under us.
236 */
237 if (bp->b_lblkno > 0 && vp->v_type == VREG &&
238 vp != fs->lfs_ivnode) {
239 struct vm_page *pg;
240 voff_t off;
241
242 simple_lock(&vp->v_interlock);
243 for (off = lblktosize(fs, bp->b_lblkno);
244 off < lblktosize(fs, bp->b_lblkno + 1);
245 off += PAGE_SIZE) {
246 pg = uvm_pagelookup(&vp->v_uobj, off);
247 if (pg == NULL)
248 continue;
249 if ((pg->flags & PG_CLEAN) == 0 ||
250 pmap_is_modified(pg)) {
251 fs->lfs_avail += btofsb(fs,
252 bp->b_bcount);
253 wakeup(&fs->lfs_avail);
254 lfs_freebuf(fs, bp);
255 bp = NULL;
256 simple_unlock(&vp->v_interlock);
257 goto nextbp;
258 }
259 }
260 simple_unlock(&vp->v_interlock);
261 }
262 for (tbp = LIST_FIRST(&vp->v_dirtyblkhd); tbp;
263 tbp = tnbp)
264 {
265 tnbp = LIST_NEXT(tbp, b_vnbufs);
266 if (tbp->b_vp == bp->b_vp
267 && tbp->b_lblkno == bp->b_lblkno
268 && tbp != bp)
269 {
270 fs->lfs_avail += btofsb(fs,
271 bp->b_bcount);
272 wakeup(&fs->lfs_avail);
273 lfs_freebuf(fs, bp);
274 bp = NULL;
275 break;
276 }
277 }
278 nextbp:
279 ;
280 }
281 splx(s);
282 }
283
284 /* If the node is being written, wait until that is done */
285 simple_lock(&vp->v_interlock);
286 s = splbio();
287 if (WRITEINPROG(vp)) {
288 ivndebug(vp,"vflush/writeinprog");
289 ltsleep(vp, (PRIBIO+1), "lfs_vw", 0, &vp->v_interlock);
290 }
291 splx(s);
292 simple_unlock(&vp->v_interlock);
293
294 /* Protect against VXLOCK deadlock in vinvalbuf() */
295 lfs_seglock(fs, SEGM_SYNC);
296
297 /* If we're supposed to flush a freed inode, just toss it */
298 if (ip->i_lfs_iflags & LFSI_DELETED) {
299 DLOG((DLOG_VNODE, "lfs_vflush: ino %d freed, not flushing\n",
300 ip->i_number));
301 s = splbio();
302 /* Drain v_numoutput */
303 simple_lock(&global_v_numoutput_slock);
304 while (vp->v_numoutput > 0) {
305 vp->v_flag |= VBWAIT;
306 ltsleep(&vp->v_numoutput, PRIBIO + 1, "lfs_vf4", 0,
307 &global_v_numoutput_slock);
308 }
309 simple_unlock(&global_v_numoutput_slock);
310 KASSERT(vp->v_numoutput == 0);
311
312 for (bp = LIST_FIRST(&vp->v_dirtyblkhd); bp; bp = nbp) {
313 nbp = LIST_NEXT(bp, b_vnbufs);
314
315 KASSERT((bp->b_flags & B_GATHERED) == 0);
316 if (bp->b_flags & B_DELWRI) { /* XXX always true? */
317 fs->lfs_avail += btofsb(fs, bp->b_bcount);
318 wakeup(&fs->lfs_avail);
319 }
320 /* Copied from lfs_writeseg */
321 if (bp->b_flags & B_CALL) {
322 biodone(bp);
323 } else {
324 bremfree(bp);
325 LFS_UNLOCK_BUF(bp);
326 bp->b_flags &= ~(B_ERROR | B_READ | B_DELWRI |
327 B_GATHERED);
328 bp->b_flags |= B_DONE;
329 reassignbuf(bp, vp);
330 brelse(bp);
331 }
332 }
333 splx(s);
334 LFS_CLR_UINO(ip, IN_CLEANING);
335 LFS_CLR_UINO(ip, IN_MODIFIED | IN_ACCESSED);
336 ip->i_flag &= ~IN_ALLMOD;
337 DLOG((DLOG_VNODE, "lfs_vflush: done not flushing ino %d\n",
338 ip->i_number));
339 lfs_segunlock(fs);
340
341 KASSERT(LIST_FIRST(&vp->v_dirtyblkhd) == NULL);
342
343 return 0;
344 }
345
346 fs->lfs_flushvp = vp;
347 if (LFS_SHOULD_CHECKPOINT(fs, fs->lfs_sp->seg_flags)) {
348 error = lfs_segwrite(vp->v_mount, SEGM_CKP | SEGM_SYNC);
349 fs->lfs_flushvp = NULL;
350 KASSERT(fs->lfs_flushvp_fakevref == 0);
351 lfs_segunlock(fs);
352
353 /* Make sure that any pending buffers get written */
354 s = splbio();
355 simple_lock(&global_v_numoutput_slock);
356 while (vp->v_numoutput > 0) {
357 vp->v_flag |= VBWAIT;
358 ltsleep(&vp->v_numoutput, PRIBIO + 1, "lfs_vf3", 0,
359 &global_v_numoutput_slock);
360 }
361 simple_unlock(&global_v_numoutput_slock);
362 splx(s);
363
364 KASSERT(LIST_FIRST(&vp->v_dirtyblkhd) == NULL);
365 KASSERT(vp->v_numoutput == 0);
366
367 return error;
368 }
369 sp = fs->lfs_sp;
370
371 flushed = 0;
372 if (VPISEMPTY(vp)) {
373 lfs_writevnodes(fs, vp->v_mount, sp, VN_EMPTY);
374 ++flushed;
375 } else if ((ip->i_flag & IN_CLEANING) &&
376 (fs->lfs_sp->seg_flags & SEGM_CLEAN)) {
377 ivndebug(vp,"vflush/clean");
378 lfs_writevnodes(fs, vp->v_mount, sp, VN_CLEAN);
379 ++flushed;
380 } else if (lfs_dostats) {
381 if (!VPISEMPTY(vp) || (VTOI(vp)->i_flag & IN_ALLMOD))
382 ++lfs_stats.vflush_invoked;
383 ivndebug(vp,"vflush");
384 }
385
386 #ifdef DIAGNOSTIC
387 if (vp->v_flag & VDIROP) {
388 DLOG((DLOG_VNODE, "lfs_vflush: flushing VDIROP\n"));
389 /* panic("lfs_vflush: VDIROP being flushed...this can\'t happen"); */
390 }
391 if (vp->v_usecount < 0) {
392 printf("usecount=%ld\n", (long)vp->v_usecount);
393 panic("lfs_vflush: usecount<0");
394 }
395 #endif
396
397 do {
398 loopcount = 0;
399 do {
400 if (LIST_FIRST(&vp->v_dirtyblkhd) != NULL) {
401 relock = lfs_writefile(fs, sp, vp);
402 if (relock) {
403 /*
404 * Might have to wait for the
405 * cleaner to run; but we're
406 * still not done with this vnode.
407 */
408 KDASSERT(ip->i_number != LFS_IFILE_INUM);
409 lfs_writeinode(fs, sp, ip);
410 LFS_SET_UINO(ip, IN_MODIFIED);
411 lfs_writeseg(fs, sp);
412 lfs_segunlock(fs);
413 lfs_segunlock_relock(fs);
414 goto top;
415 }
416 }
417 /*
418 * If we begin a new segment in the middle of writing
419 * the Ifile, it creates an inconsistent checkpoint,
420 * since the Ifile information for the new segment
421 * is not up-to-date. Take care of this here by
422 * sending the Ifile through again in case there
423 * are newly dirtied blocks. But wait, there's more!
424 * This second Ifile write could *also* cross a segment
425 * boundary, if the first one was large. The second
426 * one is guaranteed to be no more than 8 blocks,
427 * though (two segment blocks and supporting indirects)
428 * so the third write *will not* cross the boundary.
429 */
430 if (vp == fs->lfs_ivnode) {
431 lfs_writefile(fs, sp, vp);
432 lfs_writefile(fs, sp, vp);
433 }
434 #ifdef DEBUG
435 if (++loopcount > 2)
436 log(LOG_NOTICE, "lfs_vflush: looping count=%d\n", loopcount);
437 #endif
438 } while (lfs_writeinode(fs, sp, ip));
439 } while (lfs_writeseg(fs, sp) && ip->i_number == LFS_IFILE_INUM);
440
441 if (lfs_dostats) {
442 ++lfs_stats.nwrites;
443 if (sp->seg_flags & SEGM_SYNC)
444 ++lfs_stats.nsync_writes;
445 if (sp->seg_flags & SEGM_CKP)
446 ++lfs_stats.ncheckpoints;
447 }
448 /*
449 * If we were called from somewhere that has already held the seglock
450 * (e.g., lfs_markv()), the lfs_segunlock will not wait for
451 * the write to complete because we are still locked.
452 * Since lfs_vflush() must return the vnode with no dirty buffers,
453 * we must explicitly wait, if that is the case.
454 *
455 * We compare the iocount against 1, not 0, because it is
456 * artificially incremented by lfs_seglock().
457 */
458 simple_lock(&fs->lfs_interlock);
459 if (fs->lfs_seglock > 1) {
460 while (fs->lfs_iocount > 1)
461 (void)ltsleep(&fs->lfs_iocount, PRIBIO + 1,
462 "lfs_vflush", 0, &fs->lfs_interlock);
463 }
464 simple_unlock(&fs->lfs_interlock);
465
466 lfs_segunlock(fs);
467
468 /* Wait for these buffers to be recovered by aiodoned */
469 s = splbio();
470 simple_lock(&global_v_numoutput_slock);
471 while (vp->v_numoutput > 0) {
472 vp->v_flag |= VBWAIT;
473 ltsleep(&vp->v_numoutput, PRIBIO + 1, "lfs_vf2", 0,
474 &global_v_numoutput_slock);
475 }
476 simple_unlock(&global_v_numoutput_slock);
477 splx(s);
478
479 KASSERT(LIST_FIRST(&vp->v_dirtyblkhd) == NULL);
480 KASSERT(vp->v_numoutput == 0);
481
482 fs->lfs_flushvp = NULL;
483 KASSERT(fs->lfs_flushvp_fakevref == 0);
484
485 return (0);
486 }
487
488 int
489 lfs_writevnodes(struct lfs *fs, struct mount *mp, struct segment *sp, int op)
490 {
491 struct inode *ip;
492 struct vnode *vp;
493 int inodes_written = 0, only_cleaning;
494 int error = 0;
495
496 ASSERT_SEGLOCK(fs);
497 loop:
498 /* start at last (newest) vnode. */
499 TAILQ_FOREACH_REVERSE(vp, &mp->mnt_vnodelist, vnodelst, v_mntvnodes) {
500 /*
501 * If the vnode that we are about to sync is no longer
502 * associated with this mount point, start over.
503 */
504 if (vp->v_mount != mp) {
505 DLOG((DLOG_VNODE, "lfs_writevnodes: starting over\n"));
506 /*
507 * After this, pages might be busy
508 * due to our own previous putpages.
509 * Start actual segment write here to avoid deadlock.
510 */
511 (void)lfs_writeseg(fs, sp);
512 goto loop;
513 }
514
515 if (vp->v_type == VNON) {
516 continue;
517 }
518
519 ip = VTOI(vp);
520 if ((op == VN_DIROP && !(vp->v_flag & VDIROP)) ||
521 (op != VN_DIROP && op != VN_CLEAN &&
522 (vp->v_flag & VDIROP))) {
523 vndebug(vp,"dirop");
524 continue;
525 }
526
527 if (op == VN_EMPTY && !VPISEMPTY(vp)) {
528 vndebug(vp,"empty");
529 continue;
530 }
531
532 if (op == VN_CLEAN && ip->i_number != LFS_IFILE_INUM
533 && vp != fs->lfs_flushvp
534 && !(ip->i_flag & IN_CLEANING)) {
535 vndebug(vp,"cleaning");
536 continue;
537 }
538
539 if (lfs_vref(vp)) {
540 vndebug(vp,"vref");
541 continue;
542 }
543
544 only_cleaning = 0;
545 /*
546 * Write the inode/file if dirty and it's not the IFILE.
547 */
548 if ((ip->i_flag & IN_ALLMOD) || !VPISEMPTY(vp)) {
549 only_cleaning =
550 ((ip->i_flag & IN_ALLMOD) == IN_CLEANING);
551
552 if (ip->i_number != LFS_IFILE_INUM) {
553 error = lfs_writefile(fs, sp, vp);
554 if (error) {
555 lfs_vunref(vp);
556 if (error == EAGAIN) {
557 /*
558 * This error from lfs_putpages
559 * indicates we need to drop
560 * the segment lock and start
561 * over after the cleaner has
562 * had a chance to run.
563 */
564 lfs_writeinode(fs, sp, ip);
565 lfs_writeseg(fs, sp);
566 if (!VPISEMPTY(vp) &&
567 !WRITEINPROG(vp) &&
568 !(ip->i_flag & IN_ALLMOD))
569 LFS_SET_UINO(ip, IN_MODIFIED);
570 break;
571 }
572 error = 0; /* XXX not quite right */
573 continue;
574 }
575
576 if (!VPISEMPTY(vp)) {
577 if (WRITEINPROG(vp)) {
578 ivndebug(vp,"writevnodes/write2");
579 } else if (!(ip->i_flag & IN_ALLMOD)) {
580 LFS_SET_UINO(ip, IN_MODIFIED);
581 }
582 }
583 (void) lfs_writeinode(fs, sp, ip);
584 inodes_written++;
585 }
586 }
587
588 if (lfs_clean_vnhead && only_cleaning)
589 lfs_vunref_head(vp);
590 else
591 lfs_vunref(vp);
592 }
593 return error;
594 }
595
596 /*
597 * Do a checkpoint.
598 */
599 int
600 lfs_segwrite(struct mount *mp, int flags)
601 {
602 struct buf *bp;
603 struct inode *ip;
604 struct lfs *fs;
605 struct segment *sp;
606 struct vnode *vp;
607 SEGUSE *segusep;
608 int do_ckp, did_ckp, error, s;
609 unsigned n, segleft, maxseg, sn, i, curseg;
610 int writer_set = 0;
611 int dirty;
612 int redo;
613 int um_error;
614 int loopcount;
615
616 fs = VFSTOUFS(mp)->um_lfs;
617 ASSERT_MAYBE_SEGLOCK(fs);
618
619 if (fs->lfs_ronly)
620 return EROFS;
621
622 lfs_imtime(fs);
623
624 /*
625 * Allocate a segment structure and enough space to hold pointers to
626 * the maximum possible number of buffers which can be described in a
627 * single summary block.
628 */
629 do_ckp = LFS_SHOULD_CHECKPOINT(fs, flags);
630
631 lfs_seglock(fs, flags | (do_ckp ? SEGM_CKP : 0));
632 sp = fs->lfs_sp;
633 if (sp->seg_flags & (SEGM_CLEAN | SEGM_CKP))
634 do_ckp = 1;
635
636 /*
637 * If lfs_flushvp is non-NULL, we are called from lfs_vflush,
638 * in which case we have to flush *all* buffers off of this vnode.
639 * We don't care about other nodes, but write any non-dirop nodes
640 * anyway in anticipation of another getnewvnode().
641 *
642 * If we're cleaning we only write cleaning and ifile blocks, and
643 * no dirops, since otherwise we'd risk corruption in a crash.
644 */
645 if (sp->seg_flags & SEGM_CLEAN)
646 lfs_writevnodes(fs, mp, sp, VN_CLEAN);
647 else if (!(sp->seg_flags & SEGM_FORCE_CKP)) {
648 do {
649 um_error = lfs_writevnodes(fs, mp, sp, VN_REG);
650 if (do_ckp ||
651 (sp->seg_flags & SEGM_W_DIROPS && !fs->lfs_dirops)) {
652 if (!writer_set) {
653 lfs_writer_enter(fs, "lfs writer");
654 writer_set = 1;
655 }
656 error = lfs_writevnodes(fs, mp, sp, VN_DIROP);
657 if (um_error == 0)
658 um_error = error;
659 /* In case writevnodes errored out */
660 lfs_flush_dirops(fs);
661 ((SEGSUM *)(sp->segsum))->ss_flags &= ~(SS_CONT);
662 lfs_finalize_fs_seguse(fs);
663 }
664 if (do_ckp && um_error) {
665 lfs_segunlock_relock(fs);
666 sp = fs->lfs_sp;
667 }
668 } while (do_ckp && um_error != 0);
669 }
670
671 /*
672 * If we are doing a checkpoint, mark everything since the
673 * last checkpoint as no longer ACTIVE.
674 */
675 if (do_ckp || fs->lfs_doifile) {
676 segleft = fs->lfs_nseg;
677 curseg = 0;
678 for (n = 0; n < fs->lfs_segtabsz; n++) {
679 dirty = 0;
680 if (bread(fs->lfs_ivnode,
681 fs->lfs_cleansz + n, fs->lfs_bsize, NOCRED, &bp))
682 panic("lfs_segwrite: ifile read");
683 segusep = (SEGUSE *)bp->b_data;
684 maxseg = min(segleft, fs->lfs_sepb);
685 for (i = 0; i < maxseg; i++) {
686 sn = curseg + i;
687 if (sn != dtosn(fs, fs->lfs_curseg) &&
688 segusep->su_flags & SEGUSE_ACTIVE) {
689 segusep->su_flags &= ~SEGUSE_ACTIVE;
690 --fs->lfs_nactive;
691 ++dirty;
692 }
693 fs->lfs_suflags[fs->lfs_activesb][sn] =
694 segusep->su_flags;
695 if (fs->lfs_version > 1)
696 ++segusep;
697 else
698 segusep = (SEGUSE *)
699 ((SEGUSE_V1 *)segusep + 1);
700 }
701
702 if (dirty)
703 error = LFS_BWRITE_LOG(bp); /* Ifile */
704 else
705 brelse(bp);
706 segleft -= fs->lfs_sepb;
707 curseg += fs->lfs_sepb;
708 }
709 }
710
711 LOCK_ASSERT(LFS_SEGLOCK_HELD(fs));
712
713 did_ckp = 0;
714 if (do_ckp || fs->lfs_doifile) {
715 vp = fs->lfs_ivnode;
716 vn_lock(vp, LK_EXCLUSIVE);
717 loopcount = 0;
718 do {
719 #ifdef DEBUG
720 LFS_ENTER_LOG("pretend", __FILE__, __LINE__, 0, 0, curproc->p_pid);
721 #endif
722 simple_lock(&fs->lfs_interlock);
723 fs->lfs_flags &= ~LFS_IFDIRTY;
724 simple_unlock(&fs->lfs_interlock);
725
726 ip = VTOI(vp);
727
728 if (LIST_FIRST(&vp->v_dirtyblkhd) != NULL) {
729 /*
730 * Ifile has no pages, so we don't need
731 * to check error return here.
732 */
733 lfs_writefile(fs, sp, vp);
734 /*
735 * Ensure the Ifile takes the current segment
736 * into account. See comment in lfs_vflush.
737 */
738 lfs_writefile(fs, sp, vp);
739 lfs_writefile(fs, sp, vp);
740 }
741
742 if (ip->i_flag & IN_ALLMOD)
743 ++did_ckp;
744 #if 0
745 redo = (do_ckp ? lfs_writeinode(fs, sp, ip) : 0);
746 #else
747 redo = lfs_writeinode(fs, sp, ip);
748 #endif
749 redo += lfs_writeseg(fs, sp);
750 simple_lock(&fs->lfs_interlock);
751 redo += (fs->lfs_flags & LFS_IFDIRTY);
752 simple_unlock(&fs->lfs_interlock);
753 #ifdef DEBUG
754 if (++loopcount > 2)
755 log(LOG_NOTICE, "lfs_segwrite: looping count=%d\n",
756 loopcount);
757 #endif
758 } while (redo && do_ckp);
759
760 /*
761 * Unless we are unmounting, the Ifile may continue to have
762 * dirty blocks even after a checkpoint, due to changes to
763 * inodes' atime. If we're checkpointing, it's "impossible"
764 * for other parts of the Ifile to be dirty after the loop
765 * above, since we hold the segment lock.
766 */
767 s = splbio();
768 if (LIST_EMPTY(&vp->v_dirtyblkhd)) {
769 LFS_CLR_UINO(ip, IN_ALLMOD);
770 }
771 #ifdef DIAGNOSTIC
772 else if (do_ckp) {
773 int do_panic = 0;
774 LIST_FOREACH(bp, &vp->v_dirtyblkhd, b_vnbufs) {
775 if (bp->b_lblkno < fs->lfs_cleansz +
776 fs->lfs_segtabsz &&
777 !(bp->b_flags & B_GATHERED)) {
778 printf("ifile lbn %ld still dirty (flags %lx)\n",
779 (long)bp->b_lblkno,
780 (long)bp->b_flags);
781 ++do_panic;
782 }
783 }
784 if (do_panic)
785 panic("dirty blocks");
786 }
787 #endif
788 splx(s);
789 VOP_UNLOCK(vp, 0);
790 } else {
791 (void) lfs_writeseg(fs, sp);
792 }
793
794 /* Note Ifile no longer needs to be written */
795 fs->lfs_doifile = 0;
796 if (writer_set)
797 lfs_writer_leave(fs);
798
799 /*
800 * If we didn't write the Ifile, we didn't really do anything.
801 * That means that (1) there is a checkpoint on disk and (2)
802 * nothing has changed since it was written.
803 *
804 * Take the flags off of the segment so that lfs_segunlock
805 * doesn't have to write the superblock either.
806 */
807 if (do_ckp && !did_ckp) {
808 sp->seg_flags &= ~SEGM_CKP;
809 }
810
811 if (lfs_dostats) {
812 ++lfs_stats.nwrites;
813 if (sp->seg_flags & SEGM_SYNC)
814 ++lfs_stats.nsync_writes;
815 if (sp->seg_flags & SEGM_CKP)
816 ++lfs_stats.ncheckpoints;
817 }
818 lfs_segunlock(fs);
819 return (0);
820 }
821
822 /*
823 * Write the dirty blocks associated with a vnode.
824 */
825 int
826 lfs_writefile(struct lfs *fs, struct segment *sp, struct vnode *vp)
827 {
828 struct finfo *fip;
829 struct inode *ip;
830 int i, frag;
831 int error;
832
833 ASSERT_SEGLOCK(fs);
834 error = 0;
835 ip = VTOI(vp);
836
837 fip = sp->fip;
838 lfs_acquire_finfo(fs, ip->i_number, ip->i_gen);
839
840 if (vp->v_flag & VDIROP)
841 ((SEGSUM *)(sp->segsum))->ss_flags |= (SS_DIROP|SS_CONT);
842
843 if (sp->seg_flags & SEGM_CLEAN) {
844 lfs_gather(fs, sp, vp, lfs_match_fake);
845 /*
846 * For a file being flushed, we need to write *all* blocks.
847 * This means writing the cleaning blocks first, and then
848 * immediately following with any non-cleaning blocks.
849 * The same is true of the Ifile since checkpoints assume
850 * that all valid Ifile blocks are written.
851 */
852 if (IS_FLUSHING(fs, vp) || vp == fs->lfs_ivnode) {
853 lfs_gather(fs, sp, vp, lfs_match_data);
854 /*
855 * Don't call VOP_PUTPAGES: if we're flushing,
856 * we've already done it, and the Ifile doesn't
857 * use the page cache.
858 */
859 }
860 } else {
861 lfs_gather(fs, sp, vp, lfs_match_data);
862 /*
863 * If we're flushing, we've already called VOP_PUTPAGES
864 * so don't do it again. Otherwise, we want to write
865 * everything we've got.
866 */
867 if (!IS_FLUSHING(fs, vp)) {
868 simple_lock(&vp->v_interlock);
869 error = VOP_PUTPAGES(vp, 0, 0,
870 PGO_CLEANIT | PGO_ALLPAGES | PGO_LOCKED);
871 }
872 }
873
874 /*
875 * It may not be necessary to write the meta-data blocks at this point,
876 * as the roll-forward recovery code should be able to reconstruct the
877 * list.
878 *
879 * We have to write them anyway, though, under two conditions: (1) the
880 * vnode is being flushed (for reuse by vinvalbuf); or (2) we are
881 * checkpointing.
882 *
883 * BUT if we are cleaning, we might have indirect blocks that refer to
884 * new blocks not being written yet, in addition to fragments being
885 * moved out of a cleaned segment. If that is the case, don't
886 * write the indirect blocks, or the finfo will have a small block
887 * in the middle of it!
888 * XXX in this case isn't the inode size wrong too?
889 */
890 frag = 0;
891 if (sp->seg_flags & SEGM_CLEAN) {
892 for (i = 0; i < NDADDR; i++)
893 if (ip->i_lfs_fragsize[i] > 0 &&
894 ip->i_lfs_fragsize[i] < fs->lfs_bsize)
895 ++frag;
896 }
897 #ifdef DIAGNOSTIC
898 if (frag > 1)
899 panic("lfs_writefile: more than one fragment!");
900 #endif
901 if (IS_FLUSHING(fs, vp) ||
902 (frag == 0 && (lfs_writeindir || (sp->seg_flags & SEGM_CKP)))) {
903 lfs_gather(fs, sp, vp, lfs_match_indir);
904 lfs_gather(fs, sp, vp, lfs_match_dindir);
905 lfs_gather(fs, sp, vp, lfs_match_tindir);
906 }
907 fip = sp->fip;
908 lfs_release_finfo(fs);
909
910 return error;
911 }
912
913 /*
914 * Update segment accounting to reflect this inode's change of address.
915 */
916 static int
917 lfs_update_iaddr(struct lfs *fs, struct segment *sp, struct inode *ip, daddr_t ndaddr)
918 {
919 struct buf *bp;
920 daddr_t daddr;
921 IFILE *ifp;
922 SEGUSE *sup;
923 ino_t ino;
924 int redo_ifile, error;
925 u_int32_t sn;
926
927 redo_ifile = 0;
928
929 /*
930 * If updating the ifile, update the super-block. Update the disk
931 * address and access times for this inode in the ifile.
932 */
933 ino = ip->i_number;
934 if (ino == LFS_IFILE_INUM) {
935 daddr = fs->lfs_idaddr;
936 fs->lfs_idaddr = dbtofsb(fs, ndaddr);
937 } else {
938 LFS_IENTRY(ifp, fs, ino, bp);
939 daddr = ifp->if_daddr;
940 ifp->if_daddr = dbtofsb(fs, ndaddr);
941 error = LFS_BWRITE_LOG(bp); /* Ifile */
942 }
943
944 /*
945 * If this is the Ifile and lfs_offset is set to the first block
946 * in the segment, dirty the new segment's accounting block
947 * (XXX should already be dirty?) and tell the caller to do it again.
948 */
949 if (ip->i_number == LFS_IFILE_INUM) {
950 sn = dtosn(fs, fs->lfs_offset);
951 if (sntod(fs, sn) + btofsb(fs, fs->lfs_sumsize) ==
952 fs->lfs_offset) {
953 LFS_SEGENTRY(sup, fs, sn, bp);
954 KASSERT(bp->b_flags & B_DELWRI);
955 LFS_WRITESEGENTRY(sup, fs, sn, bp);
956 /* fs->lfs_flags |= LFS_IFDIRTY; */
957 redo_ifile |= 1;
958 }
959 }
960
961 /*
962 * The inode's last address should not be in the current partial
963 * segment, except under exceptional circumstances (lfs_writevnodes
964 * had to start over, and in the meantime more blocks were written
965 * to a vnode). Both inodes will be accounted to this segment
966 * in lfs_writeseg so we need to subtract the earlier version
967 * here anyway. The segment count can temporarily dip below
968 * zero here; keep track of how many duplicates we have in
969 * "dupino" so we don't panic below.
970 */
971 if (daddr >= fs->lfs_lastpseg && daddr <= fs->lfs_offset) {
972 ++sp->ndupino;
973 DLOG((DLOG_SEG, "lfs_writeinode: last inode addr in current pseg "
974 "(ino %d daddr 0x%llx) ndupino=%d\n", ino,
975 (long long)daddr, sp->ndupino));
976 }
977 /*
978 * Account the inode: it no longer belongs to its former segment,
979 * though it will not belong to the new segment until that segment
980 * is actually written.
981 */
982 if (daddr != LFS_UNUSED_DADDR) {
983 u_int32_t oldsn = dtosn(fs, daddr);
984 #ifdef DIAGNOSTIC
985 int ndupino = (sp->seg_number == oldsn) ? sp->ndupino : 0;
986 #endif
987 LFS_SEGENTRY(sup, fs, oldsn, bp);
988 #ifdef DIAGNOSTIC
989 if (sup->su_nbytes +
990 sizeof (struct ufs1_dinode) * ndupino
991 < sizeof (struct ufs1_dinode)) {
992 printf("lfs_writeinode: negative bytes "
993 "(segment %" PRIu32 " short by %d, "
994 "oldsn=%" PRIu32 ", cursn=%" PRIu32
995 ", daddr=%" PRId64 ", su_nbytes=%u, "
996 "ndupino=%d)\n",
997 dtosn(fs, daddr),
998 (int)sizeof (struct ufs1_dinode) *
999 (1 - sp->ndupino) - sup->su_nbytes,
1000 oldsn, sp->seg_number, daddr,
1001 (unsigned int)sup->su_nbytes,
1002 sp->ndupino);
1003 panic("lfs_writeinode: negative bytes");
1004 sup->su_nbytes = sizeof (struct ufs1_dinode);
1005 }
1006 #endif
1007 DLOG((DLOG_SU, "seg %d -= %d for ino %d inode\n",
1008 dtosn(fs, daddr), sizeof (struct ufs1_dinode), ino));
1009 sup->su_nbytes -= sizeof (struct ufs1_dinode);
1010 redo_ifile |=
1011 (ino == LFS_IFILE_INUM && !(bp->b_flags & B_GATHERED));
1012 if (redo_ifile) {
1013 simple_lock(&fs->lfs_interlock);
1014 fs->lfs_flags |= LFS_IFDIRTY;
1015 simple_unlock(&fs->lfs_interlock);
1016 /* Don't double-account */
1017 fs->lfs_idaddr = 0x0;
1018 }
1019 LFS_WRITESEGENTRY(sup, fs, oldsn, bp); /* Ifile */
1020 }
1021
1022 return redo_ifile;
1023 }
1024
1025 int
1026 lfs_writeinode(struct lfs *fs, struct segment *sp, struct inode *ip)
1027 {
1028 struct buf *bp;
1029 struct ufs1_dinode *cdp;
1030 daddr_t daddr;
1031 int32_t *daddrp; /* XXX ondisk32 */
1032 int i, ndx;
1033 int redo_ifile = 0;
1034 int gotblk = 0;
1035 int count;
1036
1037 ASSERT_SEGLOCK(fs);
1038 if (!(ip->i_flag & IN_ALLMOD))
1039 return (0);
1040
1041 /* Can't write ifile when writer is not set */
1042 KASSERT(ip->i_number != LFS_IFILE_INUM || fs->lfs_writer > 0 ||
1043 (sp->seg_flags & SEGM_CLEAN));
1044
1045 /*
1046 * If this is the Ifile, see if writing it here will generate a
1047 * temporary misaccounting. If it will, do the accounting and write
1048 * the blocks, postponing the inode write until the accounting is
1049 * solid.
1050 */
1051 count = 0;
1052 while (ip->i_number == LFS_IFILE_INUM) {
1053 int redo = 0;
1054
1055 if (sp->idp == NULL && sp->ibp == NULL &&
1056 (sp->seg_bytes_left < fs->lfs_ibsize ||
1057 sp->sum_bytes_left < sizeof(int32_t))) {
1058 (void) lfs_writeseg(fs, sp);
1059 continue;
1060 }
1061
1062 /* Look for dirty Ifile blocks */
1063 LIST_FOREACH(bp, &fs->lfs_ivnode->v_dirtyblkhd, b_vnbufs) {
1064 if (!(bp->b_flags & B_GATHERED)) {
1065 redo = 1;
1066 break;
1067 }
1068 }
1069
1070 if (redo == 0)
1071 redo = lfs_update_iaddr(fs, sp, ip, 0x0);
1072 if (redo == 0)
1073 break;
1074
1075 if (sp->idp) {
1076 sp->idp->di_inumber = 0;
1077 sp->idp = NULL;
1078 }
1079 ++count;
1080 if (count > 2)
1081 log(LOG_NOTICE, "lfs_writeinode: looping count=%d\n", count);
1082 lfs_writefile(fs, sp, fs->lfs_ivnode);
1083 }
1084
1085 /* Allocate a new inode block if necessary. */
1086 if ((ip->i_number != LFS_IFILE_INUM || sp->idp == NULL) &&
1087 sp->ibp == NULL) {
1088 /* Allocate a new segment if necessary. */
1089 if (sp->seg_bytes_left < fs->lfs_ibsize ||
1090 sp->sum_bytes_left < sizeof(int32_t))
1091 (void) lfs_writeseg(fs, sp);
1092
1093 /* Get next inode block. */
1094 daddr = fs->lfs_offset;
1095 fs->lfs_offset += btofsb(fs, fs->lfs_ibsize);
1096 sp->ibp = *sp->cbpp++ =
1097 getblk(VTOI(fs->lfs_ivnode)->i_devvp,
1098 fsbtodb(fs, daddr), fs->lfs_ibsize, 0, 0);
1099 gotblk++;
1100
1101 /* Zero out inode numbers */
1102 for (i = 0; i < INOPB(fs); ++i)
1103 ((struct ufs1_dinode *)sp->ibp->b_data)[i].di_inumber =
1104 0;
1105
1106 ++sp->start_bpp;
1107 fs->lfs_avail -= btofsb(fs, fs->lfs_ibsize);
1108 /* Set remaining space counters. */
1109 sp->seg_bytes_left -= fs->lfs_ibsize;
1110 sp->sum_bytes_left -= sizeof(int32_t);
1111 ndx = fs->lfs_sumsize / sizeof(int32_t) -
1112 sp->ninodes / INOPB(fs) - 1;
1113 ((int32_t *)(sp->segsum))[ndx] = daddr;
1114 }
1115
1116 /* Check VDIROP in case there is a new file with no data blocks */
1117 if (ITOV(ip)->v_flag & VDIROP)
1118 ((SEGSUM *)(sp->segsum))->ss_flags |= (SS_DIROP|SS_CONT);
1119
1120 /* Update the inode times and copy the inode onto the inode page. */
1121 /* XXX kludge --- don't redirty the ifile just to put times on it */
1122 if (ip->i_number != LFS_IFILE_INUM)
1123 LFS_ITIMES(ip, NULL, NULL, NULL);
1124
1125 /*
1126 * If this is the Ifile, and we've already written the Ifile in this
1127 * partial segment, just overwrite it (it's not on disk yet) and
1128 * continue.
1129 *
1130 * XXX we know that the bp that we get the second time around has
1131 * already been gathered.
1132 */
1133 if (ip->i_number == LFS_IFILE_INUM && sp->idp) {
1134 *(sp->idp) = *ip->i_din.ffs1_din;
1135 ip->i_lfs_osize = ip->i_size;
1136 return 0;
1137 }
1138
1139 bp = sp->ibp;
1140 cdp = ((struct ufs1_dinode *)bp->b_data) + (sp->ninodes % INOPB(fs));
1141 *cdp = *ip->i_din.ffs1_din;
1142
1143 /*
1144 * If cleaning, link counts and directory file sizes cannot change,
1145 * since those would be directory operations---even if the file
1146 * we are writing is marked VDIROP we should write the old values.
1147 * If we're not cleaning, of course, update the values so we get
1148 * current values the next time we clean.
1149 */
1150 if (sp->seg_flags & SEGM_CLEAN) {
1151 if (ITOV(ip)->v_flag & VDIROP) {
1152 cdp->di_nlink = ip->i_lfs_odnlink;
1153 /* if (ITOV(ip)->v_type == VDIR) */
1154 cdp->di_size = ip->i_lfs_osize;
1155 }
1156 } else {
1157 ip->i_lfs_odnlink = cdp->di_nlink;
1158 ip->i_lfs_osize = ip->i_size;
1159 }
1160
1161
1162 /* We can finish the segment accounting for truncations now */
1163 lfs_finalize_ino_seguse(fs, ip);
1164
1165 /*
1166 * If we are cleaning, ensure that we don't write UNWRITTEN disk
1167 * addresses to disk; possibly change the on-disk record of
1168 * the inode size, either by reverting to the previous size
1169 * (in the case of cleaning) or by verifying the inode's block
1170 * holdings (in the case of files being allocated as they are being
1171 * written).
1172 * XXX By not writing UNWRITTEN blocks, we are making the lfs_avail
1173 * XXX count on disk wrong by the same amount. We should be
1174 * XXX able to "borrow" from lfs_avail and return it after the
1175 * XXX Ifile is written. See also in lfs_writeseg.
1176 */
1177
1178 /* Check file size based on highest allocated block */
1179 if (((ip->i_ffs1_mode & IFMT) == IFREG ||
1180 (ip->i_ffs1_mode & IFMT) == IFDIR) &&
1181 ip->i_size > ((ip->i_lfs_hiblk + 1) << fs->lfs_bshift)) {
1182 cdp->di_size = (ip->i_lfs_hiblk + 1) << fs->lfs_bshift;
1183 DLOG((DLOG_SEG, "lfs_writeinode: ino %d size %" PRId64 " -> %"
1184 PRId64 "\n", (int)ip->i_number, ip->i_size, cdp->di_size));
1185 }
1186 if (ip->i_lfs_effnblks != ip->i_ffs1_blocks) {
1187 DLOG((DLOG_SEG, "lfs_writeinode: cleansing ino %d eff %d != nblk %d)"
1188 " at %x\n", ip->i_number, ip->i_lfs_effnblks,
1189 ip->i_ffs1_blocks, fs->lfs_offset));
1190 for (daddrp = cdp->di_db; daddrp < cdp->di_ib + NIADDR;
1191 daddrp++) {
1192 if (*daddrp == UNWRITTEN) {
1193 DLOG((DLOG_SEG, "lfs_writeinode: wiping UNWRITTEN\n"));
1194 *daddrp = 0;
1195 }
1196 }
1197 }
1198
1199 #ifdef DIAGNOSTIC
1200 /*
1201 * Check dinode held blocks against dinode size.
1202 * This should be identical to the check in lfs_vget().
1203 */
1204 for (i = (cdp->di_size + fs->lfs_bsize - 1) >> fs->lfs_bshift;
1205 i < NDADDR; i++) {
1206 KASSERT(i >= 0);
1207 if ((cdp->di_mode & IFMT) == IFLNK)
1208 continue;
1209 if (((cdp->di_mode & IFMT) == IFBLK ||
1210 (cdp->di_mode & IFMT) == IFCHR) && i == 0)
1211 continue;
1212 if (cdp->di_db[i] != 0) {
1213 # ifdef DEBUG
1214 lfs_dump_dinode(cdp);
1215 # endif
1216 panic("writing inconsistent inode");
1217 }
1218 }
1219 #endif /* DIAGNOSTIC */
1220
1221 if (ip->i_flag & IN_CLEANING)
1222 LFS_CLR_UINO(ip, IN_CLEANING);
1223 else {
1224 /* XXX IN_ALLMOD */
1225 LFS_CLR_UINO(ip, IN_ACCESSED | IN_ACCESS | IN_CHANGE |
1226 IN_UPDATE | IN_MODIFY);
1227 if (ip->i_lfs_effnblks == ip->i_ffs1_blocks)
1228 LFS_CLR_UINO(ip, IN_MODIFIED);
1229 else {
1230 DLOG((DLOG_VNODE, "lfs_writeinode: ino %d: real "
1231 "blks=%d, eff=%d\n", ip->i_number,
1232 ip->i_ffs1_blocks, ip->i_lfs_effnblks));
1233 }
1234 }
1235
1236 if (ip->i_number == LFS_IFILE_INUM) {
1237 /* We know sp->idp == NULL */
1238 sp->idp = ((struct ufs1_dinode *)bp->b_data) +
1239 (sp->ninodes % INOPB(fs));
1240
1241 /* Not dirty any more */
1242 simple_lock(&fs->lfs_interlock);
1243 fs->lfs_flags &= ~LFS_IFDIRTY;
1244 simple_unlock(&fs->lfs_interlock);
1245 }
1246
1247 if (gotblk) {
1248 LFS_LOCK_BUF(bp);
1249 brelse(bp);
1250 }
1251
1252 /* Increment inode count in segment summary block. */
1253 ++((SEGSUM *)(sp->segsum))->ss_ninos;
1254
1255 /* If this page is full, set flag to allocate a new page. */
1256 if (++sp->ninodes % INOPB(fs) == 0)
1257 sp->ibp = NULL;
1258
1259 redo_ifile = lfs_update_iaddr(fs, sp, ip, bp->b_blkno);
1260
1261 KASSERT(redo_ifile == 0);
1262 return (redo_ifile);
1263 }
1264
1265 int
1266 lfs_gatherblock(struct segment *sp, struct buf *bp, int *sptr)
1267 {
1268 struct lfs *fs;
1269 int vers;
1270 int j, blksinblk;
1271
1272 ASSERT_SEGLOCK(sp->fs);
1273 /*
1274 * If full, finish this segment. We may be doing I/O, so
1275 * release and reacquire the splbio().
1276 */
1277 #ifdef DIAGNOSTIC
1278 if (sp->vp == NULL)
1279 panic ("lfs_gatherblock: Null vp in segment");
1280 #endif
1281 fs = sp->fs;
1282 blksinblk = howmany(bp->b_bcount, fs->lfs_bsize);
1283 if (sp->sum_bytes_left < sizeof(int32_t) * blksinblk ||
1284 sp->seg_bytes_left < bp->b_bcount) {
1285 if (sptr)
1286 splx(*sptr);
1287 lfs_updatemeta(sp);
1288
1289 vers = sp->fip->fi_version;
1290 (void) lfs_writeseg(fs, sp);
1291
1292 /* Add the current file to the segment summary. */
1293 lfs_acquire_finfo(fs, VTOI(sp->vp)->i_number, vers);
1294
1295 if (sptr)
1296 *sptr = splbio();
1297 return (1);
1298 }
1299
1300 if (bp->b_flags & B_GATHERED) {
1301 DLOG((DLOG_SEG, "lfs_gatherblock: already gathered! Ino %d,"
1302 " lbn %" PRId64 "\n",
1303 sp->fip->fi_ino, bp->b_lblkno));
1304 return (0);
1305 }
1306
1307 /* Insert into the buffer list, update the FINFO block. */
1308 bp->b_flags |= B_GATHERED;
1309
1310 *sp->cbpp++ = bp;
1311 for (j = 0; j < blksinblk; j++) {
1312 sp->fip->fi_blocks[sp->fip->fi_nblocks++] = bp->b_lblkno + j;
1313 /* This block's accounting moves from lfs_favail to lfs_avail */
1314 lfs_deregister_block(sp->vp, bp->b_lblkno + j);
1315 }
1316
1317 sp->sum_bytes_left -= sizeof(int32_t) * blksinblk;
1318 sp->seg_bytes_left -= bp->b_bcount;
1319 return (0);
1320 }
1321
1322 int
1323 lfs_gather(struct lfs *fs, struct segment *sp, struct vnode *vp,
1324 int (*match)(struct lfs *, struct buf *))
1325 {
1326 struct buf *bp, *nbp;
1327 int s, count = 0;
1328
1329 ASSERT_SEGLOCK(fs);
1330 if (vp->v_type == VBLK)
1331 return 0;
1332 KASSERT(sp->vp == NULL);
1333 sp->vp = vp;
1334 s = splbio();
1335
1336 #ifndef LFS_NO_BACKBUF_HACK
1337 /* This is a hack to see if ordering the blocks in LFS makes a difference. */
1338 # define BUF_OFFSET \
1339 (((char *)&LIST_NEXT(bp, b_vnbufs)) - (char *)bp)
1340 # define BACK_BUF(BP) \
1341 ((struct buf *)(((char *)(BP)->b_vnbufs.le_prev) - BUF_OFFSET))
1342 # define BEG_OF_LIST \
1343 ((struct buf *)(((char *)&LIST_FIRST(&vp->v_dirtyblkhd)) - BUF_OFFSET))
1344
1345 loop:
1346 /* Find last buffer. */
1347 for (bp = LIST_FIRST(&vp->v_dirtyblkhd);
1348 bp && LIST_NEXT(bp, b_vnbufs) != NULL;
1349 bp = LIST_NEXT(bp, b_vnbufs))
1350 /* nothing */;
1351 for (; bp && bp != BEG_OF_LIST; bp = nbp) {
1352 nbp = BACK_BUF(bp);
1353 #else /* LFS_NO_BACKBUF_HACK */
1354 loop:
1355 for (bp = LIST_FIRST(&vp->v_dirtyblkhd); bp; bp = nbp) {
1356 nbp = LIST_NEXT(bp, b_vnbufs);
1357 #endif /* LFS_NO_BACKBUF_HACK */
1358 if ((bp->b_flags & (B_BUSY|B_GATHERED)) || !match(fs, bp)) {
1359 #ifdef DEBUG
1360 if (vp == fs->lfs_ivnode &&
1361 (bp->b_flags & (B_BUSY|B_GATHERED)) == B_BUSY)
1362 log(LOG_NOTICE, "lfs_gather: ifile lbn %"
1363 PRId64 " busy (%x) at 0x%x",
1364 bp->b_lblkno, bp->b_flags,
1365 (unsigned)fs->lfs_offset);
1366 #endif
1367 continue;
1368 }
1369 #ifdef DIAGNOSTIC
1370 # ifdef LFS_USE_B_INVAL
1371 if ((bp->b_flags & (B_CALL|B_INVAL)) == B_INVAL) {
1372 DLOG((DLOG_SEG, "lfs_gather: lbn %" PRId64
1373 " is B_INVAL\n", bp->b_lblkno));
1374 VOP_PRINT(bp->b_vp);
1375 }
1376 # endif /* LFS_USE_B_INVAL */
1377 if (!(bp->b_flags & B_DELWRI))
1378 panic("lfs_gather: bp not B_DELWRI");
1379 if (!(bp->b_flags & B_LOCKED)) {
1380 DLOG((DLOG_SEG, "lfs_gather: lbn %" PRId64
1381 " blk %" PRId64 " not B_LOCKED\n",
1382 bp->b_lblkno,
1383 dbtofsb(fs, bp->b_blkno)));
1384 VOP_PRINT(bp->b_vp);
1385 panic("lfs_gather: bp not B_LOCKED");
1386 }
1387 #endif
1388 if (lfs_gatherblock(sp, bp, &s)) {
1389 goto loop;
1390 }
1391 count++;
1392 }
1393 splx(s);
1394 lfs_updatemeta(sp);
1395 KASSERT(sp->vp == vp);
1396 sp->vp = NULL;
1397 return count;
1398 }
1399
1400 #if DEBUG
1401 # define DEBUG_OOFF(n) do { \
1402 if (ooff == 0) { \
1403 DLOG((DLOG_SEG, "lfs_updatemeta[%d]: warning: writing " \
1404 "ino %d lbn %" PRId64 " at 0x%" PRIx32 \
1405 ", was 0x0 (or %" PRId64 ")\n", \
1406 (n), ip->i_number, lbn, ndaddr, daddr)); \
1407 } \
1408 } while (0)
1409 #else
1410 # define DEBUG_OOFF(n)
1411 #endif
1412
1413 /*
1414 * Change the given block's address to ndaddr, finding its previous
1415 * location using ufs_bmaparray().
1416 *
1417 * Account for this change in the segment table.
1418 *
1419 * called with sp == NULL by roll-forwarding code.
1420 */
1421 void
1422 lfs_update_single(struct lfs *fs, struct segment *sp,
1423 struct vnode *vp, daddr_t lbn, int32_t ndaddr, int size)
1424 {
1425 SEGUSE *sup;
1426 struct buf *bp;
1427 struct indir a[NIADDR + 2], *ap;
1428 struct inode *ip;
1429 daddr_t daddr, ooff;
1430 int num, error;
1431 int bb, osize, obb;
1432
1433 ASSERT_SEGLOCK(fs);
1434 KASSERT(sp == NULL || sp->vp == vp);
1435 ip = VTOI(vp);
1436
1437 error = ufs_bmaparray(vp, lbn, &daddr, a, &num, NULL, NULL);
1438 if (error)
1439 panic("lfs_updatemeta: ufs_bmaparray returned %d", error);
1440
1441 daddr = (daddr_t)((int32_t)daddr); /* XXX ondisk32 */
1442 KASSERT(daddr <= LFS_MAX_DADDR);
1443 if (daddr > 0)
1444 daddr = dbtofsb(fs, daddr);
1445
1446 bb = fragstofsb(fs, numfrags(fs, size));
1447 switch (num) {
1448 case 0:
1449 ooff = ip->i_ffs1_db[lbn];
1450 DEBUG_OOFF(0);
1451 if (ooff == UNWRITTEN)
1452 ip->i_ffs1_blocks += bb;
1453 else {
1454 /* possible fragment truncation or extension */
1455 obb = btofsb(fs, ip->i_lfs_fragsize[lbn]);
1456 ip->i_ffs1_blocks += (bb - obb);
1457 }
1458 ip->i_ffs1_db[lbn] = ndaddr;
1459 break;
1460 case 1:
1461 ooff = ip->i_ffs1_ib[a[0].in_off];
1462 DEBUG_OOFF(1);
1463 if (ooff == UNWRITTEN)
1464 ip->i_ffs1_blocks += bb;
1465 ip->i_ffs1_ib[a[0].in_off] = ndaddr;
1466 break;
1467 default:
1468 ap = &a[num - 1];
1469 if (bread(vp, ap->in_lbn, fs->lfs_bsize, NOCRED, &bp))
1470 panic("lfs_updatemeta: bread bno %" PRId64,
1471 ap->in_lbn);
1472
1473 /* XXX ondisk32 */
1474 ooff = ((int32_t *)bp->b_data)[ap->in_off];
1475 DEBUG_OOFF(num);
1476 if (ooff == UNWRITTEN)
1477 ip->i_ffs1_blocks += bb;
1478 /* XXX ondisk32 */
1479 ((int32_t *)bp->b_data)[ap->in_off] = ndaddr;
1480 (void) VOP_BWRITE(bp);
1481 }
1482
1483 KASSERT(ooff == 0 || ooff == UNWRITTEN || ooff == daddr);
1484
1485 /* Update hiblk when extending the file */
1486 if (lbn > ip->i_lfs_hiblk)
1487 ip->i_lfs_hiblk = lbn;
1488
1489 /*
1490 * Though we'd rather it couldn't, this *can* happen right now
1491 * if cleaning blocks and regular blocks coexist.
1492 */
1493 /* KASSERT(daddr < fs->lfs_lastpseg || daddr > ndaddr); */
1494
1495 /*
1496 * Update segment usage information, based on old size
1497 * and location.
1498 */
1499 if (daddr > 0) {
1500 u_int32_t oldsn = dtosn(fs, daddr);
1501 #ifdef DIAGNOSTIC
1502 int ndupino;
1503
1504 if (sp && sp->seg_number == oldsn) {
1505 ndupino = sp->ndupino;
1506 } else {
1507 ndupino = 0;
1508 }
1509 #endif
1510 KASSERT(oldsn < fs->lfs_nseg);
1511 if (lbn >= 0 && lbn < NDADDR)
1512 osize = ip->i_lfs_fragsize[lbn];
1513 else
1514 osize = fs->lfs_bsize;
1515 LFS_SEGENTRY(sup, fs, oldsn, bp);
1516 #ifdef DIAGNOSTIC
1517 if (sup->su_nbytes + sizeof (struct ufs1_dinode) * ndupino
1518 < osize) {
1519 printf("lfs_updatemeta: negative bytes "
1520 "(segment %" PRIu32 " short by %" PRId64
1521 ")\n", dtosn(fs, daddr),
1522 (int64_t)osize -
1523 (sizeof (struct ufs1_dinode) * ndupino +
1524 sup->su_nbytes));
1525 printf("lfs_updatemeta: ino %llu, lbn %" PRId64
1526 ", addr = 0x%" PRIx64 "\n",
1527 (unsigned long long)ip->i_number, lbn, daddr);
1528 printf("lfs_updatemeta: ndupino=%d\n", ndupino);
1529 panic("lfs_updatemeta: negative bytes");
1530 sup->su_nbytes = osize -
1531 sizeof (struct ufs1_dinode) * ndupino;
1532 }
1533 #endif
1534 DLOG((DLOG_SU, "seg %" PRIu32 " -= %d for ino %d lbn %" PRId64
1535 " db 0x%" PRIx64 "\n",
1536 dtosn(fs, daddr), osize,
1537 ip->i_number, lbn, daddr));
1538 sup->su_nbytes -= osize;
1539 if (!(bp->b_flags & B_GATHERED)) {
1540 simple_lock(&fs->lfs_interlock);
1541 fs->lfs_flags |= LFS_IFDIRTY;
1542 simple_unlock(&fs->lfs_interlock);
1543 }
1544 LFS_WRITESEGENTRY(sup, fs, oldsn, bp);
1545 }
1546 /*
1547 * Now that this block has a new address, and its old
1548 * segment no longer owns it, we can forget about its
1549 * old size.
1550 */
1551 if (lbn >= 0 && lbn < NDADDR)
1552 ip->i_lfs_fragsize[lbn] = size;
1553 }
1554
1555 /*
1556 * Update the metadata that points to the blocks listed in the FINFO
1557 * array.
1558 */
1559 void
1560 lfs_updatemeta(struct segment *sp)
1561 {
1562 struct buf *sbp;
1563 struct lfs *fs;
1564 struct vnode *vp;
1565 daddr_t lbn;
1566 int i, nblocks, num;
1567 int bb;
1568 int bytesleft, size;
1569
1570 ASSERT_SEGLOCK(sp->fs);
1571 vp = sp->vp;
1572 nblocks = &sp->fip->fi_blocks[sp->fip->fi_nblocks] - sp->start_lbp;
1573 KASSERT(nblocks >= 0);
1574 KASSERT(vp != NULL);
1575 if (nblocks == 0)
1576 return;
1577
1578 /*
1579 * This count may be high due to oversize blocks from lfs_gop_write.
1580 * Correct for this. (XXX we should be able to keep track of these.)
1581 */
1582 fs = sp->fs;
1583 for (i = 0; i < nblocks; i++) {
1584 if (sp->start_bpp[i] == NULL) {
1585 DLOG((DLOG_SEG, "lfs_updatemeta: nblocks = %d, not %d\n", i, nblocks));
1586 nblocks = i;
1587 break;
1588 }
1589 num = howmany(sp->start_bpp[i]->b_bcount, fs->lfs_bsize);
1590 KASSERT(sp->start_bpp[i]->b_lblkno >= 0 || num == 1);
1591 nblocks -= num - 1;
1592 }
1593
1594 KASSERT(vp->v_type == VREG ||
1595 nblocks == &sp->fip->fi_blocks[sp->fip->fi_nblocks] - sp->start_lbp);
1596 KASSERT(nblocks == sp->cbpp - sp->start_bpp);
1597
1598 /*
1599 * Sort the blocks.
1600 *
1601 * We have to sort even if the blocks come from the
1602 * cleaner, because there might be other pending blocks on the
1603 * same inode...and if we don't sort, and there are fragments
1604 * present, blocks may be written in the wrong place.
1605 */
1606 lfs_shellsort(sp->start_bpp, sp->start_lbp, nblocks, fs->lfs_bsize);
1607
1608 /*
1609 * Record the length of the last block in case it's a fragment.
1610 * If there are indirect blocks present, they sort last. An
1611 * indirect block will be lfs_bsize and its presence indicates
1612 * that you cannot have fragments.
1613 *
1614 * XXX This last is a lie. A cleaned fragment can coexist with
1615 * XXX a later indirect block. This will continue to be
1616 * XXX true until lfs_markv is fixed to do everything with
1617 * XXX fake blocks (including fake inodes and fake indirect blocks).
1618 */
1619 sp->fip->fi_lastlength = ((sp->start_bpp[nblocks - 1]->b_bcount - 1) &
1620 fs->lfs_bmask) + 1;
1621
1622 /*
1623 * Assign disk addresses, and update references to the logical
1624 * block and the segment usage information.
1625 */
1626 for (i = nblocks; i--; ++sp->start_bpp) {
1627 sbp = *sp->start_bpp;
1628 lbn = *sp->start_lbp;
1629 KASSERT(sbp->b_lblkno == lbn);
1630
1631 sbp->b_blkno = fsbtodb(fs, fs->lfs_offset);
1632
1633 /*
1634 * If we write a frag in the wrong place, the cleaner won't
1635 * be able to correctly identify its size later, and the
1636 * segment will be uncleanable. (Even worse, it will assume
1637 * that the indirect block that actually ends the list
1638 * is of a smaller size!)
1639 */
1640 if ((sbp->b_bcount & fs->lfs_bmask) && i != 0)
1641 panic("lfs_updatemeta: fragment is not last block");
1642
1643 /*
1644 * For each subblock in this possibly oversized block,
1645 * update its address on disk.
1646 */
1647 KASSERT(lbn >= 0 || sbp->b_bcount == fs->lfs_bsize);
1648 KASSERT(vp == sbp->b_vp);
1649 for (bytesleft = sbp->b_bcount; bytesleft > 0;
1650 bytesleft -= fs->lfs_bsize) {
1651 size = MIN(bytesleft, fs->lfs_bsize);
1652 bb = fragstofsb(fs, numfrags(fs, size));
1653 lbn = *sp->start_lbp++;
1654 lfs_update_single(fs, sp, sp->vp, lbn, fs->lfs_offset,
1655 size);
1656 fs->lfs_offset += bb;
1657 }
1658
1659 }
1660
1661 /* This inode has been modified */
1662 LFS_SET_UINO(VTOI(vp), IN_MODIFIED);
1663 }
1664
1665 /*
1666 * Move lfs_offset to a segment earlier than sn.
1667 */
1668 int
1669 lfs_rewind(struct lfs *fs, int newsn)
1670 {
1671 int sn, osn, isdirty;
1672 struct buf *bp;
1673 SEGUSE *sup;
1674
1675 ASSERT_SEGLOCK(fs);
1676
1677 osn = dtosn(fs, fs->lfs_offset);
1678 if (osn < newsn)
1679 return 0;
1680
1681 /* lfs_avail eats the remaining space in this segment */
1682 fs->lfs_avail -= fs->lfs_fsbpseg - (fs->lfs_offset - fs->lfs_curseg);
1683
1684 /* Find a low-numbered segment */
1685 for (sn = 0; sn < fs->lfs_nseg; ++sn) {
1686 LFS_SEGENTRY(sup, fs, sn, bp);
1687 isdirty = sup->su_flags & SEGUSE_DIRTY;
1688 brelse(bp);
1689
1690 if (!isdirty)
1691 break;
1692 }
1693 if (sn == fs->lfs_nseg)
1694 panic("lfs_rewind: no clean segments");
1695 if (newsn >= 0 && sn >= newsn)
1696 return ENOENT;
1697 fs->lfs_nextseg = sn;
1698 lfs_newseg(fs);
1699 fs->lfs_offset = fs->lfs_curseg;
1700
1701 return 0;
1702 }
1703
1704 /*
1705 * Start a new partial segment.
1706 *
1707 * Return 1 when we entered to a new segment.
1708 * Otherwise, return 0.
1709 */
1710 int
1711 lfs_initseg(struct lfs *fs)
1712 {
1713 struct segment *sp = fs->lfs_sp;
1714 SEGSUM *ssp;
1715 struct buf *sbp; /* buffer for SEGSUM */
1716 int repeat = 0; /* return value */
1717
1718 ASSERT_SEGLOCK(fs);
1719 /* Advance to the next segment. */
1720 if (!LFS_PARTIAL_FITS(fs)) {
1721 SEGUSE *sup;
1722 struct buf *bp;
1723
1724 /* lfs_avail eats the remaining space */
1725 fs->lfs_avail -= fs->lfs_fsbpseg - (fs->lfs_offset -
1726 fs->lfs_curseg);
1727 /* Wake up any cleaning procs waiting on this file system. */
1728 lfs_wakeup_cleaner(fs);
1729 lfs_newseg(fs);
1730 repeat = 1;
1731 fs->lfs_offset = fs->lfs_curseg;
1732
1733 sp->seg_number = dtosn(fs, fs->lfs_curseg);
1734 sp->seg_bytes_left = fsbtob(fs, fs->lfs_fsbpseg);
1735
1736 /*
1737 * If the segment contains a superblock, update the offset
1738 * and summary address to skip over it.
1739 */
1740 LFS_SEGENTRY(sup, fs, sp->seg_number, bp);
1741 if (sup->su_flags & SEGUSE_SUPERBLOCK) {
1742 fs->lfs_offset += btofsb(fs, LFS_SBPAD);
1743 sp->seg_bytes_left -= LFS_SBPAD;
1744 }
1745 brelse(bp);
1746 /* Segment zero could also contain the labelpad */
1747 if (fs->lfs_version > 1 && sp->seg_number == 0 &&
1748 fs->lfs_start < btofsb(fs, LFS_LABELPAD)) {
1749 fs->lfs_offset +=
1750 btofsb(fs, LFS_LABELPAD) - fs->lfs_start;
1751 sp->seg_bytes_left -=
1752 LFS_LABELPAD - fsbtob(fs, fs->lfs_start);
1753 }
1754 } else {
1755 sp->seg_number = dtosn(fs, fs->lfs_curseg);
1756 sp->seg_bytes_left = fsbtob(fs, fs->lfs_fsbpseg -
1757 (fs->lfs_offset - fs->lfs_curseg));
1758 }
1759 fs->lfs_lastpseg = fs->lfs_offset;
1760
1761 /* Record first address of this partial segment */
1762 if (sp->seg_flags & SEGM_CLEAN) {
1763 fs->lfs_cleanint[fs->lfs_cleanind] = fs->lfs_offset;
1764 if (++fs->lfs_cleanind >= LFS_MAX_CLEANIND) {
1765 /* "1" is the artificial inc in lfs_seglock */
1766 simple_lock(&fs->lfs_interlock);
1767 while (fs->lfs_iocount > 1) {
1768 ltsleep(&fs->lfs_iocount, PRIBIO + 1,
1769 "lfs_initseg", 0, &fs->lfs_interlock);
1770 }
1771 simple_unlock(&fs->lfs_interlock);
1772 fs->lfs_cleanind = 0;
1773 }
1774 }
1775
1776 sp->fs = fs;
1777 sp->ibp = NULL;
1778 sp->idp = NULL;
1779 sp->ninodes = 0;
1780 sp->ndupino = 0;
1781
1782 sp->cbpp = sp->bpp;
1783
1784 /* Get a new buffer for SEGSUM */
1785 sbp = lfs_newbuf(fs, VTOI(fs->lfs_ivnode)->i_devvp,
1786 fsbtodb(fs, fs->lfs_offset), fs->lfs_sumsize, LFS_NB_SUMMARY);
1787
1788 /* ... and enter it into the buffer list. */
1789 *sp->cbpp = sbp;
1790 sp->cbpp++;
1791 fs->lfs_offset += btofsb(fs, fs->lfs_sumsize);
1792
1793 sp->start_bpp = sp->cbpp;
1794
1795 /* Set point to SEGSUM, initialize it. */
1796 ssp = sp->segsum = sbp->b_data;
1797 memset(ssp, 0, fs->lfs_sumsize);
1798 ssp->ss_next = fs->lfs_nextseg;
1799 ssp->ss_nfinfo = ssp->ss_ninos = 0;
1800 ssp->ss_magic = SS_MAGIC;
1801
1802 /* Set pointer to first FINFO, initialize it. */
1803 sp->fip = (struct finfo *)((char *)sp->segsum + SEGSUM_SIZE(fs));
1804 sp->fip->fi_nblocks = 0;
1805 sp->start_lbp = &sp->fip->fi_blocks[0];
1806 sp->fip->fi_lastlength = 0;
1807
1808 sp->seg_bytes_left -= fs->lfs_sumsize;
1809 sp->sum_bytes_left = fs->lfs_sumsize - SEGSUM_SIZE(fs);
1810
1811 return (repeat);
1812 }
1813
1814 /*
1815 * Remove SEGUSE_INVAL from all segments.
1816 */
1817 void
1818 lfs_unset_inval_all(struct lfs *fs)
1819 {
1820 SEGUSE *sup;
1821 struct buf *bp;
1822 int i;
1823
1824 for (i = 0; i < fs->lfs_nseg; i++) {
1825 LFS_SEGENTRY(sup, fs, i, bp);
1826 if (sup->su_flags & SEGUSE_INVAL) {
1827 sup->su_flags &= ~SEGUSE_INVAL;
1828 LFS_WRITESEGENTRY(sup, fs, i, bp);
1829 } else
1830 brelse(bp);
1831 }
1832 }
1833
1834 /*
1835 * Return the next segment to write.
1836 */
1837 void
1838 lfs_newseg(struct lfs *fs)
1839 {
1840 CLEANERINFO *cip;
1841 SEGUSE *sup;
1842 struct buf *bp;
1843 int curseg, isdirty, sn, skip_inval;
1844
1845 ASSERT_SEGLOCK(fs);
1846
1847 /* Honor LFCNWRAPSTOP */
1848 simple_lock(&fs->lfs_interlock);
1849 while (fs->lfs_nextseg < fs->lfs_curseg && fs->lfs_nowrap) {
1850 if (fs->lfs_wrappass) {
1851 log(LOG_NOTICE, "%s: wrappass=%d\n",
1852 fs->lfs_fsmnt, fs->lfs_wrappass);
1853 fs->lfs_wrappass = 0;
1854 break;
1855 }
1856 fs->lfs_wrapstatus = LFS_WRAP_WAITING;
1857 wakeup(&fs->lfs_nowrap);
1858 log(LOG_NOTICE, "%s: waiting at log wrap\n", fs->lfs_fsmnt);
1859 ltsleep(&fs->lfs_wrappass, PVFS, "newseg", 10 * hz,
1860 &fs->lfs_interlock);
1861 }
1862 fs->lfs_wrapstatus = LFS_WRAP_GOING;
1863 simple_unlock(&fs->lfs_interlock);
1864
1865 LFS_SEGENTRY(sup, fs, dtosn(fs, fs->lfs_nextseg), bp);
1866 DLOG((DLOG_SU, "lfs_newseg: seg %d := 0 in newseg\n",
1867 dtosn(fs, fs->lfs_nextseg)));
1868 sup->su_flags |= SEGUSE_DIRTY | SEGUSE_ACTIVE;
1869 sup->su_nbytes = 0;
1870 sup->su_nsums = 0;
1871 sup->su_ninos = 0;
1872 LFS_WRITESEGENTRY(sup, fs, dtosn(fs, fs->lfs_nextseg), bp);
1873
1874 LFS_CLEANERINFO(cip, fs, bp);
1875 --cip->clean;
1876 ++cip->dirty;
1877 fs->lfs_nclean = cip->clean;
1878 LFS_SYNC_CLEANERINFO(cip, fs, bp, 1);
1879
1880 fs->lfs_lastseg = fs->lfs_curseg;
1881 fs->lfs_curseg = fs->lfs_nextseg;
1882 skip_inval = 1;
1883 for (sn = curseg = dtosn(fs, fs->lfs_curseg) + fs->lfs_interleave;;) {
1884 sn = (sn + 1) % fs->lfs_nseg;
1885
1886 if (sn == curseg) {
1887 if (skip_inval)
1888 skip_inval = 0;
1889 else
1890 panic("lfs_nextseg: no clean segments");
1891 }
1892 LFS_SEGENTRY(sup, fs, sn, bp);
1893 isdirty = sup->su_flags & (SEGUSE_DIRTY | (skip_inval ? SEGUSE_INVAL : 0));
1894 /* Check SEGUSE_EMPTY as we go along */
1895 if (isdirty && sup->su_nbytes == 0 &&
1896 !(sup->su_flags & SEGUSE_EMPTY))
1897 LFS_WRITESEGENTRY(sup, fs, sn, bp);
1898 else
1899 brelse(bp);
1900
1901 if (!isdirty)
1902 break;
1903 }
1904 if (skip_inval == 0)
1905 lfs_unset_inval_all(fs);
1906
1907 ++fs->lfs_nactive;
1908 fs->lfs_nextseg = sntod(fs, sn);
1909 if (lfs_dostats) {
1910 ++lfs_stats.segsused;
1911 }
1912 }
1913
1914 static struct buf *
1915 lfs_newclusterbuf(struct lfs *fs, struct vnode *vp, daddr_t addr,
1916 int n)
1917 {
1918 struct lfs_cluster *cl;
1919 struct buf **bpp, *bp;
1920
1921 ASSERT_SEGLOCK(fs);
1922 cl = (struct lfs_cluster *)pool_get(&fs->lfs_clpool, PR_WAITOK);
1923 bpp = (struct buf **)pool_get(&fs->lfs_bpppool, PR_WAITOK);
1924 memset(cl, 0, sizeof(*cl));
1925 cl->fs = fs;
1926 cl->bpp = bpp;
1927 cl->bufcount = 0;
1928 cl->bufsize = 0;
1929
1930 /* If this segment is being written synchronously, note that */
1931 if (fs->lfs_sp->seg_flags & SEGM_SYNC) {
1932 cl->flags |= LFS_CL_SYNC;
1933 cl->seg = fs->lfs_sp;
1934 ++cl->seg->seg_iocount;
1935 }
1936
1937 /* Get an empty buffer header, or maybe one with something on it */
1938 bp = getiobuf();
1939 bp->b_flags = B_BUSY | B_CALL;
1940 bp->b_dev = NODEV;
1941 bp->b_blkno = bp->b_lblkno = addr;
1942 bp->b_iodone = lfs_cluster_callback;
1943 bp->b_private = cl;
1944 bp->b_vp = vp;
1945
1946 return bp;
1947 }
1948
1949 int
1950 lfs_writeseg(struct lfs *fs, struct segment *sp)
1951 {
1952 struct buf **bpp, *bp, *cbp, *newbp;
1953 SEGUSE *sup;
1954 SEGSUM *ssp;
1955 int i, s;
1956 int do_again, nblocks, byteoffset;
1957 size_t el_size;
1958 struct lfs_cluster *cl;
1959 u_short ninos;
1960 struct vnode *devvp;
1961 char *p = NULL;
1962 struct vnode *vp;
1963 int32_t *daddrp; /* XXX ondisk32 */
1964 int changed;
1965 u_int32_t sum;
1966 #ifdef DEBUG
1967 FINFO *fip;
1968 int findex;
1969 #endif
1970
1971 ASSERT_SEGLOCK(fs);
1972
1973 ssp = (SEGSUM *)sp->segsum;
1974
1975 /*
1976 * If there are no buffers other than the segment summary to write,
1977 * don't do anything. If we are the end of a dirop sequence, however,
1978 * write the empty segment summary anyway, to help out the
1979 * roll-forward agent.
1980 */
1981 if ((nblocks = sp->cbpp - sp->bpp) == 1) {
1982 if ((ssp->ss_flags & (SS_DIROP | SS_CONT)) != SS_DIROP)
1983 return 0;
1984 }
1985
1986 /* Note if partial segment is being written by the cleaner */
1987 if (sp->seg_flags & SEGM_CLEAN)
1988 ssp->ss_flags |= SS_CLEAN;
1989
1990 devvp = VTOI(fs->lfs_ivnode)->i_devvp;
1991
1992 /* Update the segment usage information. */
1993 LFS_SEGENTRY(sup, fs, sp->seg_number, bp);
1994
1995 /* Loop through all blocks, except the segment summary. */
1996 for (bpp = sp->bpp; ++bpp < sp->cbpp; ) {
1997 if ((*bpp)->b_vp != devvp) {
1998 sup->su_nbytes += (*bpp)->b_bcount;
1999 DLOG((DLOG_SU, "seg %" PRIu32 " += %ld for ino %d"
2000 " lbn %" PRId64 " db 0x%" PRIx64 "\n",
2001 sp->seg_number, (*bpp)->b_bcount,
2002 VTOI((*bpp)->b_vp)->i_number, (*bpp)->b_lblkno,
2003 (*bpp)->b_blkno));
2004 }
2005 }
2006
2007 #ifdef DEBUG
2008 /* Check for zero-length and zero-version FINFO entries. */
2009 fip = (struct finfo *)((char *)ssp + SEGSUM_SIZE(fs));
2010 for (findex = 0; findex < ssp->ss_nfinfo; findex++) {
2011 KDASSERT(fip->fi_nblocks > 0);
2012 KDASSERT(fip->fi_version > 0);
2013 fip = (FINFO *)((char *)fip + FINFOSIZE +
2014 sizeof(int32_t) * fip->fi_nblocks);
2015 }
2016 #endif /* DEBUG */
2017
2018 ninos = (ssp->ss_ninos + INOPB(fs) - 1) / INOPB(fs);
2019 DLOG((DLOG_SU, "seg %d += %d for %d inodes\n",
2020 sp->seg_number, ssp->ss_ninos * sizeof (struct ufs1_dinode),
2021 ssp->ss_ninos));
2022 sup->su_nbytes += ssp->ss_ninos * sizeof (struct ufs1_dinode);
2023 /* sup->su_nbytes += fs->lfs_sumsize; */
2024 if (fs->lfs_version == 1)
2025 sup->su_olastmod = time_second;
2026 else
2027 sup->su_lastmod = time_second;
2028 sup->su_ninos += ninos;
2029 ++sup->su_nsums;
2030 fs->lfs_avail -= btofsb(fs, fs->lfs_sumsize);
2031
2032 do_again = !(bp->b_flags & B_GATHERED);
2033 LFS_WRITESEGENTRY(sup, fs, sp->seg_number, bp); /* Ifile */
2034
2035 /*
2036 * Mark blocks B_BUSY, to prevent then from being changed between
2037 * the checksum computation and the actual write.
2038 *
2039 * If we are cleaning, check indirect blocks for UNWRITTEN, and if
2040 * there are any, replace them with copies that have UNASSIGNED
2041 * instead.
2042 */
2043 for (bpp = sp->bpp, i = nblocks - 1; i--;) {
2044 ++bpp;
2045 bp = *bpp;
2046 if (bp->b_flags & B_CALL) { /* UBC or malloced buffer */
2047 bp->b_flags |= B_BUSY;
2048 continue;
2049 }
2050
2051 simple_lock(&bp->b_interlock);
2052 s = splbio();
2053 while (bp->b_flags & B_BUSY) {
2054 DLOG((DLOG_SEG, "lfs_writeseg: avoiding potential"
2055 " data summary corruption for ino %d, lbn %"
2056 PRId64 "\n",
2057 VTOI(bp->b_vp)->i_number, bp->b_lblkno));
2058 bp->b_flags |= B_WANTED;
2059 ltsleep(bp, (PRIBIO + 1), "lfs_writeseg", 0,
2060 &bp->b_interlock);
2061 splx(s);
2062 s = splbio();
2063 }
2064 bp->b_flags |= B_BUSY;
2065 splx(s);
2066 simple_unlock(&bp->b_interlock);
2067
2068 /*
2069 * Check and replace indirect block UNWRITTEN bogosity.
2070 * XXX See comment in lfs_writefile.
2071 */
2072 if (bp->b_lblkno < 0 && bp->b_vp != devvp && bp->b_vp &&
2073 VTOI(bp->b_vp)->i_ffs1_blocks !=
2074 VTOI(bp->b_vp)->i_lfs_effnblks) {
2075 DLOG((DLOG_VNODE, "lfs_writeseg: cleansing ino %d (%d != %d)\n",
2076 VTOI(bp->b_vp)->i_number,
2077 VTOI(bp->b_vp)->i_lfs_effnblks,
2078 VTOI(bp->b_vp)->i_ffs1_blocks));
2079 /* Make a copy we'll make changes to */
2080 newbp = lfs_newbuf(fs, bp->b_vp, bp->b_lblkno,
2081 bp->b_bcount, LFS_NB_IBLOCK);
2082 newbp->b_blkno = bp->b_blkno;
2083 memcpy(newbp->b_data, bp->b_data,
2084 newbp->b_bcount);
2085
2086 changed = 0;
2087 /* XXX ondisk32 */
2088 for (daddrp = (int32_t *)(newbp->b_data);
2089 daddrp < (int32_t *)((char *)newbp->b_data +
2090 newbp->b_bcount); daddrp++) {
2091 if (*daddrp == UNWRITTEN) {
2092 ++changed;
2093 *daddrp = 0;
2094 }
2095 }
2096 /*
2097 * Get rid of the old buffer. Don't mark it clean,
2098 * though, if it still has dirty data on it.
2099 */
2100 if (changed) {
2101 DLOG((DLOG_SEG, "lfs_writeseg: replacing UNWRITTEN(%d):"
2102 " bp = %p newbp = %p\n", changed, bp,
2103 newbp));
2104 *bpp = newbp;
2105 bp->b_flags &= ~(B_ERROR | B_GATHERED);
2106 if (bp->b_flags & B_CALL) {
2107 DLOG((DLOG_SEG, "lfs_writeseg: "
2108 "indir bp should not be B_CALL\n"));
2109 s = splbio();
2110 biodone(bp);
2111 splx(s);
2112 bp = NULL;
2113 } else {
2114 /* Still on free list, leave it there */
2115 s = splbio();
2116 bp->b_flags &= ~B_BUSY;
2117 if (bp->b_flags & B_WANTED)
2118 wakeup(bp);
2119 splx(s);
2120 /*
2121 * We have to re-decrement lfs_avail
2122 * since this block is going to come
2123 * back around to us in the next
2124 * segment.
2125 */
2126 fs->lfs_avail -=
2127 btofsb(fs, bp->b_bcount);
2128 }
2129 } else {
2130 lfs_freebuf(fs, newbp);
2131 }
2132 }
2133 }
2134 /*
2135 * Compute checksum across data and then across summary; the first
2136 * block (the summary block) is skipped. Set the create time here
2137 * so that it's guaranteed to be later than the inode mod times.
2138 */
2139 sum = 0;
2140 if (fs->lfs_version == 1)
2141 el_size = sizeof(u_long);
2142 else
2143 el_size = sizeof(u_int32_t);
2144 for (bpp = sp->bpp, i = nblocks - 1; i--; ) {
2145 ++bpp;
2146 /* Loop through gop_write cluster blocks */
2147 for (byteoffset = 0; byteoffset < (*bpp)->b_bcount;
2148 byteoffset += fs->lfs_bsize) {
2149 #ifdef LFS_USE_B_INVAL
2150 if (((*bpp)->b_flags & (B_CALL | B_INVAL)) ==
2151 (B_CALL | B_INVAL)) {
2152 if (copyin((void *)(*bpp)->b_saveaddr +
2153 byteoffset, dp, el_size)) {
2154 panic("lfs_writeseg: copyin failed [1]:"
2155 " ino %d blk %" PRId64,
2156 VTOI((*bpp)->b_vp)->i_number,
2157 (*bpp)->b_lblkno);
2158 }
2159 } else
2160 #endif /* LFS_USE_B_INVAL */
2161 {
2162 sum = lfs_cksum_part((char *)
2163 (*bpp)->b_data + byteoffset, el_size, sum);
2164 }
2165 }
2166 }
2167 if (fs->lfs_version == 1)
2168 ssp->ss_ocreate = time_second;
2169 else {
2170 ssp->ss_create = time_second;
2171 ssp->ss_serial = ++fs->lfs_serial;
2172 ssp->ss_ident = fs->lfs_ident;
2173 }
2174 ssp->ss_datasum = lfs_cksum_fold(sum);
2175 ssp->ss_sumsum = cksum(&ssp->ss_datasum,
2176 fs->lfs_sumsize - sizeof(ssp->ss_sumsum));
2177
2178 simple_lock(&fs->lfs_interlock);
2179 fs->lfs_bfree -= (btofsb(fs, ninos * fs->lfs_ibsize) +
2180 btofsb(fs, fs->lfs_sumsize));
2181 fs->lfs_dmeta += (btofsb(fs, ninos * fs->lfs_ibsize) +
2182 btofsb(fs, fs->lfs_sumsize));
2183 simple_unlock(&fs->lfs_interlock);
2184
2185 /*
2186 * When we simply write the blocks we lose a rotation for every block
2187 * written. To avoid this problem, we cluster the buffers into a
2188 * chunk and write the chunk. MAXPHYS is the largest size I/O
2189 * devices can handle, use that for the size of the chunks.
2190 *
2191 * Blocks that are already clusters (from GOP_WRITE), however, we
2192 * don't bother to copy into other clusters.
2193 */
2194
2195 #define CHUNKSIZE MAXPHYS
2196
2197 if (devvp == NULL)
2198 panic("devvp is NULL");
2199 for (bpp = sp->bpp, i = nblocks; i;) {
2200 cbp = lfs_newclusterbuf(fs, devvp, (*bpp)->b_blkno, i);
2201 cl = cbp->b_private;
2202
2203 cbp->b_flags |= B_ASYNC | B_BUSY;
2204 cbp->b_bcount = 0;
2205
2206 #if defined(DEBUG) && defined(DIAGNOSTIC)
2207 if (bpp - sp->bpp > (fs->lfs_sumsize - SEGSUM_SIZE(fs))
2208 / sizeof(int32_t)) {
2209 panic("lfs_writeseg: real bpp overwrite");
2210 }
2211 if (bpp - sp->bpp > segsize(fs) / fs->lfs_fsize) {
2212 panic("lfs_writeseg: theoretical bpp overwrite");
2213 }
2214 #endif
2215
2216 /*
2217 * Construct the cluster.
2218 */
2219 simple_lock(&fs->lfs_interlock);
2220 ++fs->lfs_iocount;
2221 simple_unlock(&fs->lfs_interlock);
2222 while (i && cbp->b_bcount < CHUNKSIZE) {
2223 bp = *bpp;
2224
2225 if (bp->b_bcount > (CHUNKSIZE - cbp->b_bcount))
2226 break;
2227 if (cbp->b_bcount > 0 && !(cl->flags & LFS_CL_MALLOC))
2228 break;
2229
2230 /* Clusters from GOP_WRITE are expedited */
2231 if (bp->b_bcount > fs->lfs_bsize) {
2232 if (cbp->b_bcount > 0)
2233 /* Put in its own buffer */
2234 break;
2235 else {
2236 cbp->b_data = bp->b_data;
2237 }
2238 } else if (cbp->b_bcount == 0) {
2239 p = cbp->b_data = lfs_malloc(fs, CHUNKSIZE,
2240 LFS_NB_CLUSTER);
2241 cl->flags |= LFS_CL_MALLOC;
2242 }
2243 #ifdef DIAGNOSTIC
2244 if (dtosn(fs, dbtofsb(fs, bp->b_blkno +
2245 btodb(bp->b_bcount - 1))) !=
2246 sp->seg_number) {
2247 printf("blk size %d daddr %" PRIx64
2248 " not in seg %d\n",
2249 bp->b_bcount, bp->b_blkno,
2250 sp->seg_number);
2251 panic("segment overwrite");
2252 }
2253 #endif
2254
2255 #ifdef LFS_USE_B_INVAL
2256 /*
2257 * Fake buffers from the cleaner are marked as B_INVAL.
2258 * We need to copy the data from user space rather than
2259 * from the buffer indicated.
2260 * XXX == what do I do on an error?
2261 */
2262 if ((bp->b_flags & (B_CALL|B_INVAL)) ==
2263 (B_CALL|B_INVAL)) {
2264 if (copyin(bp->b_saveaddr, p, bp->b_bcount))
2265 panic("lfs_writeseg: "
2266 "copyin failed [2]");
2267 } else
2268 #endif /* LFS_USE_B_INVAL */
2269 if (cl->flags & LFS_CL_MALLOC) {
2270 /* copy data into our cluster. */
2271 memcpy(p, bp->b_data, bp->b_bcount);
2272 p += bp->b_bcount;
2273 }
2274
2275 cbp->b_bcount += bp->b_bcount;
2276 cl->bufsize += bp->b_bcount;
2277
2278 bp->b_flags &= ~(B_ERROR | B_READ | B_DELWRI | B_DONE);
2279 cl->bpp[cl->bufcount++] = bp;
2280 vp = bp->b_vp;
2281 s = splbio();
2282 reassignbuf(bp, vp);
2283 V_INCR_NUMOUTPUT(vp);
2284 splx(s);
2285
2286 bpp++;
2287 i--;
2288 }
2289 if (fs->lfs_sp->seg_flags & SEGM_SYNC)
2290 BIO_SETPRIO(cbp, BPRIO_TIMECRITICAL);
2291 else
2292 BIO_SETPRIO(cbp, BPRIO_TIMELIMITED);
2293 s = splbio();
2294 V_INCR_NUMOUTPUT(devvp);
2295 splx(s);
2296 VOP_STRATEGY(devvp, cbp);
2297 curproc->p_stats->p_ru.ru_oublock++;
2298 }
2299
2300 if (lfs_dostats) {
2301 ++lfs_stats.psegwrites;
2302 lfs_stats.blocktot += nblocks - 1;
2303 if (fs->lfs_sp->seg_flags & SEGM_SYNC)
2304 ++lfs_stats.psyncwrites;
2305 if (fs->lfs_sp->seg_flags & SEGM_CLEAN) {
2306 ++lfs_stats.pcleanwrites;
2307 lfs_stats.cleanblocks += nblocks - 1;
2308 }
2309 }
2310
2311 return (lfs_initseg(fs) || do_again);
2312 }
2313
2314 void
2315 lfs_writesuper(struct lfs *fs, daddr_t daddr)
2316 {
2317 struct buf *bp;
2318 int s;
2319 struct vnode *devvp = VTOI(fs->lfs_ivnode)->i_devvp;
2320
2321 ASSERT_MAYBE_SEGLOCK(fs);
2322 #ifdef DIAGNOSTIC
2323 KASSERT(fs->lfs_magic == LFS_MAGIC);
2324 #endif
2325 /*
2326 * If we can write one superblock while another is in
2327 * progress, we risk not having a complete checkpoint if we crash.
2328 * So, block here if a superblock write is in progress.
2329 */
2330 simple_lock(&fs->lfs_interlock);
2331 s = splbio();
2332 while (fs->lfs_sbactive) {
2333 ltsleep(&fs->lfs_sbactive, PRIBIO+1, "lfs sb", 0,
2334 &fs->lfs_interlock);
2335 }
2336 fs->lfs_sbactive = daddr;
2337 splx(s);
2338 simple_unlock(&fs->lfs_interlock);
2339
2340 /* Set timestamp of this version of the superblock */
2341 if (fs->lfs_version == 1)
2342 fs->lfs_otstamp = time_second;
2343 fs->lfs_tstamp = time_second;
2344
2345 /* Checksum the superblock and copy it into a buffer. */
2346 fs->lfs_cksum = lfs_sb_cksum(&(fs->lfs_dlfs));
2347 bp = lfs_newbuf(fs, devvp,
2348 fsbtodb(fs, daddr), LFS_SBPAD, LFS_NB_SBLOCK);
2349 memset((char *)bp->b_data + sizeof(struct dlfs), 0,
2350 LFS_SBPAD - sizeof(struct dlfs));
2351 *(struct dlfs *)bp->b_data = fs->lfs_dlfs;
2352
2353 bp->b_flags |= B_BUSY | B_CALL | B_ASYNC;
2354 bp->b_flags &= ~(B_DONE | B_ERROR | B_READ | B_DELWRI);
2355 bp->b_iodone = lfs_supercallback;
2356
2357 if (fs->lfs_sp != NULL && fs->lfs_sp->seg_flags & SEGM_SYNC)
2358 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
2359 else
2360 BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
2361 curproc->p_stats->p_ru.ru_oublock++;
2362 s = splbio();
2363 V_INCR_NUMOUTPUT(bp->b_vp);
2364 splx(s);
2365 simple_lock(&fs->lfs_interlock);
2366 ++fs->lfs_iocount;
2367 simple_unlock(&fs->lfs_interlock);
2368 VOP_STRATEGY(devvp, bp);
2369 }
2370
2371 /*
2372 * Logical block number match routines used when traversing the dirty block
2373 * chain.
2374 */
2375 int
2376 lfs_match_fake(struct lfs *fs, struct buf *bp)
2377 {
2378
2379 ASSERT_SEGLOCK(fs);
2380 return LFS_IS_MALLOC_BUF(bp);
2381 }
2382
2383 #if 0
2384 int
2385 lfs_match_real(struct lfs *fs, struct buf *bp)
2386 {
2387
2388 ASSERT_SEGLOCK(fs);
2389 return (lfs_match_data(fs, bp) && !lfs_match_fake(fs, bp));
2390 }
2391 #endif
2392
2393 int
2394 lfs_match_data(struct lfs *fs, struct buf *bp)
2395 {
2396
2397 ASSERT_SEGLOCK(fs);
2398 return (bp->b_lblkno >= 0);
2399 }
2400
2401 int
2402 lfs_match_indir(struct lfs *fs, struct buf *bp)
2403 {
2404 daddr_t lbn;
2405
2406 ASSERT_SEGLOCK(fs);
2407 lbn = bp->b_lblkno;
2408 return (lbn < 0 && (-lbn - NDADDR) % NINDIR(fs) == 0);
2409 }
2410
2411 int
2412 lfs_match_dindir(struct lfs *fs, struct buf *bp)
2413 {
2414 daddr_t lbn;
2415
2416 ASSERT_SEGLOCK(fs);
2417 lbn = bp->b_lblkno;
2418 return (lbn < 0 && (-lbn - NDADDR) % NINDIR(fs) == 1);
2419 }
2420
2421 int
2422 lfs_match_tindir(struct lfs *fs, struct buf *bp)
2423 {
2424 daddr_t lbn;
2425
2426 ASSERT_SEGLOCK(fs);
2427 lbn = bp->b_lblkno;
2428 return (lbn < 0 && (-lbn - NDADDR) % NINDIR(fs) == 2);
2429 }
2430
2431 static void
2432 lfs_free_aiodone(struct buf *bp)
2433 {
2434 struct lfs *fs;
2435
2436 fs = bp->b_private;
2437 ASSERT_NO_SEGLOCK(fs);
2438 lfs_freebuf(fs, bp);
2439 }
2440
2441 static void
2442 lfs_super_aiodone(struct buf *bp)
2443 {
2444 struct lfs *fs;
2445
2446 fs = bp->b_private;
2447 ASSERT_NO_SEGLOCK(fs);
2448 simple_lock(&fs->lfs_interlock);
2449 fs->lfs_sbactive = 0;
2450 if (--fs->lfs_iocount <= 1)
2451 wakeup(&fs->lfs_iocount);
2452 simple_unlock(&fs->lfs_interlock);
2453 wakeup(&fs->lfs_sbactive);
2454 lfs_freebuf(fs, bp);
2455 }
2456
2457 static void
2458 lfs_cluster_aiodone(struct buf *bp)
2459 {
2460 struct lfs_cluster *cl;
2461 struct lfs *fs;
2462 struct buf *tbp, *fbp;
2463 struct vnode *vp, *devvp;
2464 struct inode *ip;
2465 int s, error=0;
2466
2467 if (bp->b_flags & B_ERROR)
2468 error = bp->b_error;
2469
2470 cl = bp->b_private;
2471 fs = cl->fs;
2472 devvp = VTOI(fs->lfs_ivnode)->i_devvp;
2473 ASSERT_NO_SEGLOCK(fs);
2474
2475 /* Put the pages back, and release the buffer */
2476 while (cl->bufcount--) {
2477 tbp = cl->bpp[cl->bufcount];
2478 KASSERT(tbp->b_flags & B_BUSY);
2479 if (error) {
2480 tbp->b_flags |= B_ERROR;
2481 tbp->b_error = error;
2482 }
2483
2484 /*
2485 * We're done with tbp. If it has not been re-dirtied since
2486 * the cluster was written, free it. Otherwise, keep it on
2487 * the locked list to be written again.
2488 */
2489 vp = tbp->b_vp;
2490
2491 tbp->b_flags &= ~B_GATHERED;
2492
2493 LFS_BCLEAN_LOG(fs, tbp);
2494
2495 if (!(tbp->b_flags & B_CALL)) {
2496 KASSERT(tbp->b_flags & B_LOCKED);
2497 s = splbio();
2498 simple_lock(&bqueue_slock);
2499 bremfree(tbp);
2500 simple_unlock(&bqueue_slock);
2501 if (vp)
2502 reassignbuf(tbp, vp);
2503 splx(s);
2504 tbp->b_flags |= B_ASYNC; /* for biodone */
2505 }
2506
2507 if ((tbp->b_flags & (B_LOCKED | B_DELWRI)) == B_LOCKED)
2508 LFS_UNLOCK_BUF(tbp);
2509
2510 if (tbp->b_flags & B_DONE) {
2511 DLOG((DLOG_SEG, "blk %d biodone already (flags %lx)\n",
2512 cl->bufcount, (long)tbp->b_flags));
2513 }
2514
2515 if ((tbp->b_flags & B_CALL) && !LFS_IS_MALLOC_BUF(tbp)) {
2516 /*
2517 * A buffer from the page daemon.
2518 * We use the same iodone as it does,
2519 * so we must manually disassociate its
2520 * buffers from the vp.
2521 */
2522 if (tbp->b_vp) {
2523 /* This is just silly */
2524 s = splbio();
2525 brelvp(tbp);
2526 tbp->b_vp = vp;
2527 splx(s);
2528 }
2529 /* Put it back the way it was */
2530 tbp->b_flags |= B_ASYNC;
2531 /* Master buffers have B_AGE */
2532 if (tbp->b_private == tbp)
2533 tbp->b_flags |= B_AGE;
2534 }
2535 s = splbio();
2536 biodone(tbp);
2537
2538 /*
2539 * If this is the last block for this vnode, but
2540 * there are other blocks on its dirty list,
2541 * set IN_MODIFIED/IN_CLEANING depending on what
2542 * sort of block. Only do this for our mount point,
2543 * not for, e.g., inode blocks that are attached to
2544 * the devvp.
2545 * XXX KS - Shouldn't we set *both* if both types
2546 * of blocks are present (traverse the dirty list?)
2547 */
2548 simple_lock(&global_v_numoutput_slock);
2549 if (vp != devvp && vp->v_numoutput == 0 &&
2550 (fbp = LIST_FIRST(&vp->v_dirtyblkhd)) != NULL) {
2551 ip = VTOI(vp);
2552 DLOG((DLOG_SEG, "lfs_cluster_aiodone: mark ino %d\n",
2553 ip->i_number));
2554 if (LFS_IS_MALLOC_BUF(fbp))
2555 LFS_SET_UINO(ip, IN_CLEANING);
2556 else
2557 LFS_SET_UINO(ip, IN_MODIFIED);
2558 }
2559 simple_unlock(&global_v_numoutput_slock);
2560 splx(s);
2561 wakeup(vp);
2562 }
2563
2564 /* Fix up the cluster buffer, and release it */
2565 if (cl->flags & LFS_CL_MALLOC)
2566 lfs_free(fs, bp->b_data, LFS_NB_CLUSTER);
2567 putiobuf(bp);
2568
2569 /* Note i/o done */
2570 if (cl->flags & LFS_CL_SYNC) {
2571 if (--cl->seg->seg_iocount == 0)
2572 wakeup(&cl->seg->seg_iocount);
2573 }
2574 simple_lock(&fs->lfs_interlock);
2575 #ifdef DIAGNOSTIC
2576 if (fs->lfs_iocount == 0)
2577 panic("lfs_cluster_aiodone: zero iocount");
2578 #endif
2579 if (--fs->lfs_iocount <= 1)
2580 wakeup(&fs->lfs_iocount);
2581 simple_unlock(&fs->lfs_interlock);
2582
2583 pool_put(&fs->lfs_bpppool, cl->bpp);
2584 cl->bpp = NULL;
2585 pool_put(&fs->lfs_clpool, cl);
2586 }
2587
2588 static void
2589 lfs_generic_callback(struct buf *bp, void (*aiodone)(struct buf *))
2590 {
2591 /* reset b_iodone for when this is a single-buf i/o. */
2592 bp->b_iodone = aiodone;
2593
2594 workqueue_enqueue(uvm.aiodone_queue, &bp->b_work);
2595 }
2596
2597 static void
2598 lfs_cluster_callback(struct buf *bp)
2599 {
2600
2601 lfs_generic_callback(bp, lfs_cluster_aiodone);
2602 }
2603
2604 void
2605 lfs_supercallback(struct buf *bp)
2606 {
2607
2608 lfs_generic_callback(bp, lfs_super_aiodone);
2609 }
2610
2611 /*
2612 * The only buffers that are going to hit these functions are the
2613 * segment write blocks, or the segment summaries, or the superblocks.
2614 *
2615 * All of the above are created by lfs_newbuf, and so do not need to be
2616 * released via brelse.
2617 */
2618 void
2619 lfs_callback(struct buf *bp)
2620 {
2621
2622 lfs_generic_callback(bp, lfs_free_aiodone);
2623 }
2624
2625 /*
2626 * Shellsort (diminishing increment sort) from Data Structures and
2627 * Algorithms, Aho, Hopcraft and Ullman, 1983 Edition, page 290;
2628 * see also Knuth Vol. 3, page 84. The increments are selected from
2629 * formula (8), page 95. Roughly O(N^3/2).
2630 */
2631 /*
2632 * This is our own private copy of shellsort because we want to sort
2633 * two parallel arrays (the array of buffer pointers and the array of
2634 * logical block numbers) simultaneously. Note that we cast the array
2635 * of logical block numbers to a unsigned in this routine so that the
2636 * negative block numbers (meta data blocks) sort AFTER the data blocks.
2637 */
2638
2639 void
2640 lfs_shellsort(struct buf **bp_array, int32_t *lb_array, int nmemb, int size)
2641 {
2642 static int __rsshell_increments[] = { 4, 1, 0 };
2643 int incr, *incrp, t1, t2;
2644 struct buf *bp_temp;
2645
2646 #ifdef DEBUG
2647 incr = 0;
2648 for (t1 = 0; t1 < nmemb; t1++) {
2649 for (t2 = 0; t2 * size < bp_array[t1]->b_bcount; t2++) {
2650 if (lb_array[incr++] != bp_array[t1]->b_lblkno + t2) {
2651 /* dump before panic */
2652 printf("lfs_shellsort: nmemb=%d, size=%d\n",
2653 nmemb, size);
2654 incr = 0;
2655 for (t1 = 0; t1 < nmemb; t1++) {
2656 const struct buf *bp = bp_array[t1];
2657
2658 printf("bp[%d]: lbn=%" PRIu64 ", size=%"
2659 PRIu64 "\n", t1,
2660 (uint64_t)bp->b_bcount,
2661 (uint64_t)bp->b_lblkno);
2662 printf("lbns:");
2663 for (t2 = 0; t2 * size < bp->b_bcount;
2664 t2++) {
2665 printf(" %" PRId32,
2666 lb_array[incr++]);
2667 }
2668 printf("\n");
2669 }
2670 panic("lfs_shellsort: inconsistent input");
2671 }
2672 }
2673 }
2674 #endif
2675
2676 for (incrp = __rsshell_increments; (incr = *incrp++) != 0;)
2677 for (t1 = incr; t1 < nmemb; ++t1)
2678 for (t2 = t1 - incr; t2 >= 0;)
2679 if ((u_int32_t)bp_array[t2]->b_lblkno >
2680 (u_int32_t)bp_array[t2 + incr]->b_lblkno) {
2681 bp_temp = bp_array[t2];
2682 bp_array[t2] = bp_array[t2 + incr];
2683 bp_array[t2 + incr] = bp_temp;
2684 t2 -= incr;
2685 } else
2686 break;
2687
2688 /* Reform the list of logical blocks */
2689 incr = 0;
2690 for (t1 = 0; t1 < nmemb; t1++) {
2691 for (t2 = 0; t2 * size < bp_array[t1]->b_bcount; t2++) {
2692 lb_array[incr++] = bp_array[t1]->b_lblkno + t2;
2693 }
2694 }
2695 }
2696
2697 /*
2698 * Call vget with LK_NOWAIT. If we are the one who holds VXLOCK/VFREEING,
2699 * however, we must press on. Just fake success in that case.
2700 */
2701 int
2702 lfs_vref(struct vnode *vp)
2703 {
2704 int error;
2705 struct lfs *fs;
2706
2707 fs = VTOI(vp)->i_lfs;
2708
2709 ASSERT_MAYBE_SEGLOCK(fs);
2710
2711 /*
2712 * If we return 1 here during a flush, we risk vinvalbuf() not
2713 * being able to flush all of the pages from this vnode, which
2714 * will cause it to panic. So, return 0 if a flush is in progress.
2715 */
2716 error = vget(vp, LK_NOWAIT);
2717 if (error == EBUSY && IS_FLUSHING(VTOI(vp)->i_lfs, vp)) {
2718 ++fs->lfs_flushvp_fakevref;
2719 return 0;
2720 }
2721 return error;
2722 }
2723
2724 /*
2725 * This is vrele except that we do not want to VOP_INACTIVE this vnode. We
2726 * inline vrele here to avoid the vn_lock and VOP_INACTIVE call at the end.
2727 */
2728 void
2729 lfs_vunref(struct vnode *vp)
2730 {
2731 struct lfs *fs;
2732
2733 fs = VTOI(vp)->i_lfs;
2734 ASSERT_MAYBE_SEGLOCK(fs);
2735
2736 /*
2737 * Analogous to lfs_vref, if the node is flushing, fake it.
2738 */
2739 if (IS_FLUSHING(fs, vp) && fs->lfs_flushvp_fakevref) {
2740 --fs->lfs_flushvp_fakevref;
2741 return;
2742 }
2743
2744 simple_lock(&vp->v_interlock);
2745 #ifdef DIAGNOSTIC
2746 if (vp->v_usecount <= 0) {
2747 printf("lfs_vunref: inum is %llu\n", (unsigned long long)
2748 VTOI(vp)->i_number);
2749 printf("lfs_vunref: flags are 0x%lx\n", (u_long)vp->v_flag);
2750 printf("lfs_vunref: usecount = %ld\n", (long)vp->v_usecount);
2751 panic("lfs_vunref: v_usecount < 0");
2752 }
2753 #endif
2754 vp->v_usecount--;
2755 if (vp->v_usecount > 0) {
2756 simple_unlock(&vp->v_interlock);
2757 return;
2758 }
2759 /*
2760 * insert at tail of LRU list
2761 */
2762 simple_lock(&vnode_free_list_slock);
2763 if (vp->v_holdcnt > 0)
2764 TAILQ_INSERT_TAIL(&vnode_hold_list, vp, v_freelist);
2765 else
2766 TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
2767 simple_unlock(&vnode_free_list_slock);
2768 simple_unlock(&vp->v_interlock);
2769 }
2770
2771 /*
2772 * We use this when we have vnodes that were loaded in solely for cleaning.
2773 * There is no reason to believe that these vnodes will be referenced again
2774 * soon, since the cleaning process is unrelated to normal filesystem
2775 * activity. Putting cleaned vnodes at the tail of the list has the effect
2776 * of flushing the vnode LRU. So, put vnodes that were loaded only for
2777 * cleaning at the head of the list, instead.
2778 */
2779 void
2780 lfs_vunref_head(struct vnode *vp)
2781 {
2782
2783 ASSERT_SEGLOCK(VTOI(vp)->i_lfs);
2784 simple_lock(&vp->v_interlock);
2785 #ifdef DIAGNOSTIC
2786 if (vp->v_usecount == 0) {
2787 panic("lfs_vunref: v_usecount<0");
2788 }
2789 #endif
2790 vp->v_usecount--;
2791 if (vp->v_usecount > 0) {
2792 simple_unlock(&vp->v_interlock);
2793 return;
2794 }
2795 /*
2796 * insert at head of LRU list
2797 */
2798 simple_lock(&vnode_free_list_slock);
2799 if (vp->v_holdcnt > 0)
2800 TAILQ_INSERT_TAIL(&vnode_hold_list, vp, v_freelist);
2801 else
2802 TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_freelist);
2803 simple_unlock(&vnode_free_list_slock);
2804 simple_unlock(&vp->v_interlock);
2805 }
2806
2807
2808 /*
2809 * Set up an FINFO entry for a new file. The fip pointer is assumed to
2810 * point at uninitialized space.
2811 */
2812 void
2813 lfs_acquire_finfo(struct lfs *fs, ino_t ino, int vers)
2814 {
2815 struct segment *sp = fs->lfs_sp;
2816
2817 KASSERT(vers > 0);
2818
2819 if (sp->seg_bytes_left < fs->lfs_bsize ||
2820 sp->sum_bytes_left < sizeof(struct finfo))
2821 (void) lfs_writeseg(fs, fs->lfs_sp);
2822
2823 sp->sum_bytes_left -= FINFOSIZE;
2824 ++((SEGSUM *)(sp->segsum))->ss_nfinfo;
2825 sp->fip->fi_nblocks = 0;
2826 sp->fip->fi_ino = ino;
2827 sp->fip->fi_version = vers;
2828 }
2829
2830 /*
2831 * Release the FINFO entry, either clearing out an unused entry or
2832 * advancing us to the next available entry.
2833 */
2834 void
2835 lfs_release_finfo(struct lfs *fs)
2836 {
2837 struct segment *sp = fs->lfs_sp;
2838
2839 if (sp->fip->fi_nblocks != 0) {
2840 sp->fip = (FINFO*)((char *)sp->fip + FINFOSIZE +
2841 sizeof(int32_t) * sp->fip->fi_nblocks);
2842 sp->start_lbp = &sp->fip->fi_blocks[0];
2843 } else {
2844 sp->sum_bytes_left += FINFOSIZE;
2845 --((SEGSUM *)(sp->segsum))->ss_nfinfo;
2846 }
2847 }
2848