uvm_swap.c revision 1.46.2.10 1 /* $NetBSD: uvm_swap.c,v 1.46.2.10 2002/06/20 03:50:46 nathanw Exp $ */
2
3 /*
4 * Copyright (c) 1995, 1996, 1997 Matthew R. Green
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp
31 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.46.2.10 2002/06/20 03:50:46 nathanw Exp $");
36
37 #include "fs_nfs.h"
38 #include "opt_uvmhist.h"
39 #include "opt_compat_netbsd.h"
40 #include "opt_ddb.h"
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/buf.h>
45 #include <sys/conf.h>
46 #include <sys/lwp.h>
47 #include <sys/proc.h>
48 #include <sys/namei.h>
49 #include <sys/disklabel.h>
50 #include <sys/errno.h>
51 #include <sys/kernel.h>
52 #include <sys/malloc.h>
53 #include <sys/vnode.h>
54 #include <sys/file.h>
55 #include <sys/extent.h>
56 #include <sys/mount.h>
57 #include <sys/pool.h>
58 #include <sys/sa.h>
59 #include <sys/syscallargs.h>
60 #include <sys/swap.h>
61
62 #include <uvm/uvm.h>
63
64 #include <miscfs/specfs/specdev.h>
65
66 /*
67 * uvm_swap.c: manage configuration and i/o to swap space.
68 */
69
70 /*
71 * swap space is managed in the following way:
72 *
73 * each swap partition or file is described by a "swapdev" structure.
74 * each "swapdev" structure contains a "swapent" structure which contains
75 * information that is passed up to the user (via system calls).
76 *
77 * each swap partition is assigned a "priority" (int) which controls
78 * swap parition usage.
79 *
80 * the system maintains a global data structure describing all swap
81 * partitions/files. there is a sorted LIST of "swappri" structures
82 * which describe "swapdev"'s at that priority. this LIST is headed
83 * by the "swap_priority" global var. each "swappri" contains a
84 * CIRCLEQ of "swapdev" structures at that priority.
85 *
86 * locking:
87 * - swap_syscall_lock (sleep lock): this lock serializes the swapctl
88 * system call and prevents the swap priority list from changing
89 * while we are in the middle of a system call (e.g. SWAP_STATS).
90 * - uvm.swap_data_lock (simple_lock): this lock protects all swap data
91 * structures including the priority list, the swapdev structures,
92 * and the swapmap extent.
93 *
94 * each swap device has the following info:
95 * - swap device in use (could be disabled, preventing future use)
96 * - swap enabled (allows new allocations on swap)
97 * - map info in /dev/drum
98 * - vnode pointer
99 * for swap files only:
100 * - block size
101 * - max byte count in buffer
102 * - buffer
103 *
104 * userland controls and configures swap with the swapctl(2) system call.
105 * the sys_swapctl performs the following operations:
106 * [1] SWAP_NSWAP: returns the number of swap devices currently configured
107 * [2] SWAP_STATS: given a pointer to an array of swapent structures
108 * (passed in via "arg") of a size passed in via "misc" ... we load
109 * the current swap config into the array. The actual work is done
110 * in the uvm_swap_stats(9) function.
111 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a
112 * priority in "misc", start swapping on it.
113 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device
114 * [5] SWAP_CTL: changes the priority of a swap device (new priority in
115 * "misc")
116 */
117
118 /*
119 * swapdev: describes a single swap partition/file
120 *
121 * note the following should be true:
122 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks]
123 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel]
124 */
125 struct swapdev {
126 struct oswapent swd_ose;
127 #define swd_dev swd_ose.ose_dev /* device id */
128 #define swd_flags swd_ose.ose_flags /* flags:inuse/enable/fake */
129 #define swd_priority swd_ose.ose_priority /* our priority */
130 /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */
131 char *swd_path; /* saved pathname of device */
132 int swd_pathlen; /* length of pathname */
133 int swd_npages; /* #pages we can use */
134 int swd_npginuse; /* #pages in use */
135 int swd_npgbad; /* #pages bad */
136 int swd_drumoffset; /* page0 offset in drum */
137 int swd_drumsize; /* #pages in drum */
138 struct extent *swd_ex; /* extent for this swapdev */
139 char swd_exname[12]; /* name of extent above */
140 struct vnode *swd_vp; /* backing vnode */
141 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */
142
143 int swd_bsize; /* blocksize (bytes) */
144 int swd_maxactive; /* max active i/o reqs */
145 struct buf_queue swd_tab; /* buffer list */
146 int swd_active; /* number of active buffers */
147 };
148
149 /*
150 * swap device priority entry; the list is kept sorted on `spi_priority'.
151 */
152 struct swappri {
153 int spi_priority; /* priority */
154 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev;
155 /* circleq of swapdevs at this priority */
156 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */
157 };
158
159 /*
160 * The following two structures are used to keep track of data transfers
161 * on swap devices associated with regular files.
162 * NOTE: this code is more or less a copy of vnd.c; we use the same
163 * structure names here to ease porting..
164 */
165 struct vndxfer {
166 struct buf *vx_bp; /* Pointer to parent buffer */
167 struct swapdev *vx_sdp;
168 int vx_error;
169 int vx_pending; /* # of pending aux buffers */
170 int vx_flags;
171 #define VX_BUSY 1
172 #define VX_DEAD 2
173 };
174
175 struct vndbuf {
176 struct buf vb_buf;
177 struct vndxfer *vb_xfer;
178 };
179
180
181 /*
182 * We keep a of pool vndbuf's and vndxfer structures.
183 */
184 static struct pool vndxfer_pool;
185 static struct pool vndbuf_pool;
186
187 #define getvndxfer(vnx) do { \
188 int s = splbio(); \
189 vnx = pool_get(&vndxfer_pool, PR_WAITOK); \
190 splx(s); \
191 } while (0)
192
193 #define putvndxfer(vnx) { \
194 pool_put(&vndxfer_pool, (void *)(vnx)); \
195 }
196
197 #define getvndbuf(vbp) do { \
198 int s = splbio(); \
199 vbp = pool_get(&vndbuf_pool, PR_WAITOK); \
200 splx(s); \
201 } while (0)
202
203 #define putvndbuf(vbp) { \
204 pool_put(&vndbuf_pool, (void *)(vbp)); \
205 }
206
207 /* /dev/drum */
208 bdev_decl(sw);
209 cdev_decl(sw);
210
211 /*
212 * local variables
213 */
214 static struct extent *swapmap; /* controls the mapping of /dev/drum */
215
216 /* list of all active swap devices [by priority] */
217 LIST_HEAD(swap_priority, swappri);
218 static struct swap_priority swap_priority;
219
220 /* locks */
221 struct lock swap_syscall_lock;
222
223 /*
224 * prototypes
225 */
226 static struct swapdev *swapdrum_getsdp __P((int));
227
228 static struct swapdev *swaplist_find __P((struct vnode *, int));
229 static void swaplist_insert __P((struct swapdev *,
230 struct swappri *, int));
231 static void swaplist_trim __P((void));
232
233 static int swap_on __P((struct proc *, struct swapdev *));
234 static int swap_off __P((struct proc *, struct swapdev *));
235
236 static void sw_reg_strategy __P((struct swapdev *, struct buf *, int));
237 static void sw_reg_iodone __P((struct buf *));
238 static void sw_reg_start __P((struct swapdev *));
239
240 static int uvm_swap_io __P((struct vm_page **, int, int, int));
241
242 /*
243 * uvm_swap_init: init the swap system data structures and locks
244 *
245 * => called at boot time from init_main.c after the filesystems
246 * are brought up (which happens after uvm_init())
247 */
248 void
249 uvm_swap_init()
250 {
251 UVMHIST_FUNC("uvm_swap_init");
252
253 UVMHIST_CALLED(pdhist);
254 /*
255 * first, init the swap list, its counter, and its lock.
256 * then get a handle on the vnode for /dev/drum by using
257 * the its dev_t number ("swapdev", from MD conf.c).
258 */
259
260 LIST_INIT(&swap_priority);
261 uvmexp.nswapdev = 0;
262 lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0);
263 simple_lock_init(&uvm.swap_data_lock);
264
265 if (bdevvp(swapdev, &swapdev_vp))
266 panic("uvm_swap_init: can't get vnode for swap device");
267
268 /*
269 * create swap block resource map to map /dev/drum. the range
270 * from 1 to INT_MAX allows 2 gigablocks of swap space. note
271 * that block 0 is reserved (used to indicate an allocation
272 * failure, or no allocation).
273 */
274 swapmap = extent_create("swapmap", 1, INT_MAX,
275 M_VMSWAP, 0, 0, EX_NOWAIT);
276 if (swapmap == 0)
277 panic("uvm_swap_init: extent_create failed");
278
279 /*
280 * allocate pools for structures used for swapping to files.
281 */
282
283 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0,
284 "swp vnx", NULL);
285
286 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0,
287 "swp vnd", NULL);
288
289 /*
290 * done!
291 */
292 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0);
293 }
294
295 /*
296 * swaplist functions: functions that operate on the list of swap
297 * devices on the system.
298 */
299
300 /*
301 * swaplist_insert: insert swap device "sdp" into the global list
302 *
303 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
304 * => caller must provide a newly malloc'd swappri structure (we will
305 * FREE it if we don't need it... this it to prevent malloc blocking
306 * here while adding swap)
307 */
308 static void
309 swaplist_insert(sdp, newspp, priority)
310 struct swapdev *sdp;
311 struct swappri *newspp;
312 int priority;
313 {
314 struct swappri *spp, *pspp;
315 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist);
316
317 /*
318 * find entry at or after which to insert the new device.
319 */
320 pspp = NULL;
321 LIST_FOREACH(spp, &swap_priority, spi_swappri) {
322 if (priority <= spp->spi_priority)
323 break;
324 pspp = spp;
325 }
326
327 /*
328 * new priority?
329 */
330 if (spp == NULL || spp->spi_priority != priority) {
331 spp = newspp; /* use newspp! */
332 UVMHIST_LOG(pdhist, "created new swappri = %d",
333 priority, 0, 0, 0);
334
335 spp->spi_priority = priority;
336 CIRCLEQ_INIT(&spp->spi_swapdev);
337
338 if (pspp)
339 LIST_INSERT_AFTER(pspp, spp, spi_swappri);
340 else
341 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri);
342 } else {
343 /* we don't need a new priority structure, free it */
344 FREE(newspp, M_VMSWAP);
345 }
346
347 /*
348 * priority found (or created). now insert on the priority's
349 * circleq list and bump the total number of swapdevs.
350 */
351 sdp->swd_priority = priority;
352 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
353 uvmexp.nswapdev++;
354 }
355
356 /*
357 * swaplist_find: find and optionally remove a swap device from the
358 * global list.
359 *
360 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
361 * => we return the swapdev we found (and removed)
362 */
363 static struct swapdev *
364 swaplist_find(vp, remove)
365 struct vnode *vp;
366 boolean_t remove;
367 {
368 struct swapdev *sdp;
369 struct swappri *spp;
370
371 /*
372 * search the lists for the requested vp
373 */
374
375 LIST_FOREACH(spp, &swap_priority, spi_swappri) {
376 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
377 if (sdp->swd_vp == vp) {
378 if (remove) {
379 CIRCLEQ_REMOVE(&spp->spi_swapdev,
380 sdp, swd_next);
381 uvmexp.nswapdev--;
382 }
383 return(sdp);
384 }
385 }
386 }
387 return (NULL);
388 }
389
390
391 /*
392 * swaplist_trim: scan priority list for empty priority entries and kill
393 * them.
394 *
395 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
396 */
397 static void
398 swaplist_trim()
399 {
400 struct swappri *spp, *nextspp;
401
402 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) {
403 nextspp = LIST_NEXT(spp, spi_swappri);
404 if (CIRCLEQ_FIRST(&spp->spi_swapdev) !=
405 (void *)&spp->spi_swapdev)
406 continue;
407 LIST_REMOVE(spp, spi_swappri);
408 free(spp, M_VMSWAP);
409 }
410 }
411
412 /*
413 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back
414 * to the "swapdev" that maps that section of the drum.
415 *
416 * => each swapdev takes one big contig chunk of the drum
417 * => caller must hold uvm.swap_data_lock
418 */
419 static struct swapdev *
420 swapdrum_getsdp(pgno)
421 int pgno;
422 {
423 struct swapdev *sdp;
424 struct swappri *spp;
425
426 LIST_FOREACH(spp, &swap_priority, spi_swappri) {
427 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
428 if (sdp->swd_flags & SWF_FAKE)
429 continue;
430 if (pgno >= sdp->swd_drumoffset &&
431 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) {
432 return sdp;
433 }
434 }
435 }
436 return NULL;
437 }
438
439
440 /*
441 * sys_swapctl: main entry point for swapctl(2) system call
442 * [with two helper functions: swap_on and swap_off]
443 */
444 int
445 sys_swapctl(l, v, retval)
446 struct lwp *l;
447 void *v;
448 register_t *retval;
449 {
450 struct sys_swapctl_args /* {
451 syscallarg(int) cmd;
452 syscallarg(void *) arg;
453 syscallarg(int) misc;
454 } */ *uap = (struct sys_swapctl_args *)v;
455 struct proc *p = l->l_proc;
456 struct vnode *vp;
457 struct nameidata nd;
458 struct swappri *spp;
459 struct swapdev *sdp;
460 struct swapent *sep;
461 char userpath[PATH_MAX + 1];
462 size_t len;
463 int error, misc;
464 int priority;
465 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist);
466
467 misc = SCARG(uap, misc);
468
469 /*
470 * ensure serialized syscall access by grabbing the swap_syscall_lock
471 */
472 lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL);
473
474 /*
475 * we handle the non-priv NSWAP and STATS request first.
476 *
477 * SWAP_NSWAP: return number of config'd swap devices
478 * [can also be obtained with uvmexp sysctl]
479 */
480 if (SCARG(uap, cmd) == SWAP_NSWAP) {
481 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev,
482 0, 0, 0);
483 *retval = uvmexp.nswapdev;
484 error = 0;
485 goto out;
486 }
487
488 /*
489 * SWAP_STATS: get stats on current # of configured swap devs
490 *
491 * note that the swap_priority list can't change as long
492 * as we are holding the swap_syscall_lock. we don't want
493 * to grab the uvm.swap_data_lock because we may fault&sleep during
494 * copyout() and we don't want to be holding that lock then!
495 */
496 if (SCARG(uap, cmd) == SWAP_STATS
497 #if defined(COMPAT_13)
498 || SCARG(uap, cmd) == SWAP_OSTATS
499 #endif
500 ) {
501 misc = MIN(uvmexp.nswapdev, misc);
502 #if defined(COMPAT_13)
503 if (SCARG(uap, cmd) == SWAP_OSTATS)
504 len = sizeof(struct oswapent) * misc;
505 else
506 #endif
507 len = sizeof(struct swapent) * misc;
508 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK);
509
510 uvm_swap_stats(SCARG(uap, cmd), sep, misc, retval);
511 error = copyout(sep, (void *)SCARG(uap, arg), len);
512
513 free(sep, M_TEMP);
514 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0);
515 goto out;
516 }
517 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) {
518 dev_t *devp = (dev_t *)SCARG(uap, arg);
519
520 error = copyout(&dumpdev, devp, sizeof(dumpdev));
521 goto out;
522 }
523
524 /*
525 * all other requests require superuser privs. verify.
526 */
527 if ((error = suser(p->p_ucred, &p->p_acflag)))
528 goto out;
529
530 /*
531 * at this point we expect a path name in arg. we will
532 * use namei() to gain a vnode reference (vref), and lock
533 * the vnode (VOP_LOCK).
534 *
535 * XXX: a NULL arg means use the root vnode pointer (e.g. for
536 * miniroot)
537 */
538 if (SCARG(uap, arg) == NULL) {
539 vp = rootvp; /* miniroot */
540 if (vget(vp, LK_EXCLUSIVE)) {
541 error = EBUSY;
542 goto out;
543 }
544 if (SCARG(uap, cmd) == SWAP_ON &&
545 copystr("miniroot", userpath, sizeof userpath, &len))
546 panic("swapctl: miniroot copy failed");
547 } else {
548 int space;
549 char *where;
550
551 if (SCARG(uap, cmd) == SWAP_ON) {
552 if ((error = copyinstr(SCARG(uap, arg), userpath,
553 sizeof userpath, &len)))
554 goto out;
555 space = UIO_SYSSPACE;
556 where = userpath;
557 } else {
558 space = UIO_USERSPACE;
559 where = (char *)SCARG(uap, arg);
560 }
561 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p);
562 if ((error = namei(&nd)))
563 goto out;
564 vp = nd.ni_vp;
565 }
566 /* note: "vp" is referenced and locked */
567
568 error = 0; /* assume no error */
569 switch(SCARG(uap, cmd)) {
570
571 case SWAP_DUMPDEV:
572 if (vp->v_type != VBLK) {
573 error = ENOTBLK;
574 break;
575 }
576 dumpdev = vp->v_rdev;
577 break;
578
579 case SWAP_CTL:
580 /*
581 * get new priority, remove old entry (if any) and then
582 * reinsert it in the correct place. finally, prune out
583 * any empty priority structures.
584 */
585 priority = SCARG(uap, misc);
586 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
587 simple_lock(&uvm.swap_data_lock);
588 if ((sdp = swaplist_find(vp, 1)) == NULL) {
589 error = ENOENT;
590 } else {
591 swaplist_insert(sdp, spp, priority);
592 swaplist_trim();
593 }
594 simple_unlock(&uvm.swap_data_lock);
595 if (error)
596 free(spp, M_VMSWAP);
597 break;
598
599 case SWAP_ON:
600
601 /*
602 * check for duplicates. if none found, then insert a
603 * dummy entry on the list to prevent someone else from
604 * trying to enable this device while we are working on
605 * it.
606 */
607
608 priority = SCARG(uap, misc);
609 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK);
610 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
611 simple_lock(&uvm.swap_data_lock);
612 if (swaplist_find(vp, 0) != NULL) {
613 error = EBUSY;
614 simple_unlock(&uvm.swap_data_lock);
615 free(sdp, M_VMSWAP);
616 free(spp, M_VMSWAP);
617 break;
618 }
619 memset(sdp, 0, sizeof(*sdp));
620 sdp->swd_flags = SWF_FAKE; /* placeholder only */
621 sdp->swd_vp = vp;
622 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV;
623 BUFQ_INIT(&sdp->swd_tab);
624
625 swaplist_insert(sdp, spp, priority);
626 simple_unlock(&uvm.swap_data_lock);
627
628 sdp->swd_pathlen = len;
629 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK);
630 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0)
631 panic("swapctl: copystr");
632
633 /*
634 * we've now got a FAKE placeholder in the swap list.
635 * now attempt to enable swap on it. if we fail, undo
636 * what we've done and kill the fake entry we just inserted.
637 * if swap_on is a success, it will clear the SWF_FAKE flag
638 */
639
640 if ((error = swap_on(p, sdp)) != 0) {
641 simple_lock(&uvm.swap_data_lock);
642 (void) swaplist_find(vp, 1); /* kill fake entry */
643 swaplist_trim();
644 simple_unlock(&uvm.swap_data_lock);
645 free(sdp->swd_path, M_VMSWAP);
646 free(sdp, M_VMSWAP);
647 break;
648 }
649 break;
650
651 case SWAP_OFF:
652 simple_lock(&uvm.swap_data_lock);
653 if ((sdp = swaplist_find(vp, 0)) == NULL) {
654 simple_unlock(&uvm.swap_data_lock);
655 error = ENXIO;
656 break;
657 }
658
659 /*
660 * If a device isn't in use or enabled, we
661 * can't stop swapping from it (again).
662 */
663 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) {
664 simple_unlock(&uvm.swap_data_lock);
665 error = EBUSY;
666 break;
667 }
668
669 /*
670 * do the real work.
671 */
672 error = swap_off(p, sdp);
673 break;
674
675 default:
676 error = EINVAL;
677 }
678
679 /*
680 * done! release the ref gained by namei() and unlock.
681 */
682 vput(vp);
683
684 out:
685 lockmgr(&swap_syscall_lock, LK_RELEASE, NULL);
686
687 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0);
688 return (error);
689 }
690
691 /*
692 * swap_stats: implements swapctl(SWAP_STATS). The function is kept
693 * away from sys_swapctl() in order to allow COMPAT_* swapctl()
694 * emulation to use it directly without going through sys_swapctl().
695 * The problem with using sys_swapctl() there is that it involves
696 * copying the swapent array to the stackgap, and this array's size
697 * is not known at build time. Hence it would not be possible to
698 * ensure it would fit in the stackgap in any case.
699 */
700 void
701 uvm_swap_stats(cmd, sep, sec, retval)
702 int cmd;
703 struct swapent *sep;
704 int sec;
705 register_t *retval;
706 {
707 struct swappri *spp;
708 struct swapdev *sdp;
709 int count = 0;
710
711 LIST_FOREACH(spp, &swap_priority, spi_swappri) {
712 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev);
713 sdp != (void *)&spp->spi_swapdev && sec-- > 0;
714 sdp = CIRCLEQ_NEXT(sdp, swd_next)) {
715 /*
716 * backwards compatibility for system call.
717 * note that we use 'struct oswapent' as an
718 * overlay into both 'struct swapdev' and
719 * the userland 'struct swapent', as we
720 * want to retain backwards compatibility
721 * with NetBSD 1.3.
722 */
723 sdp->swd_ose.ose_inuse =
724 btodb((u_int64_t)sdp->swd_npginuse <<
725 PAGE_SHIFT);
726 (void)memcpy(sep, &sdp->swd_ose,
727 sizeof(struct oswapent));
728
729 /* now copy out the path if necessary */
730 #if defined(COMPAT_13)
731 if (cmd == SWAP_STATS)
732 #endif
733 (void)memcpy(&sep->se_path, sdp->swd_path,
734 sdp->swd_pathlen);
735
736 count++;
737 #if defined(COMPAT_13)
738 if (cmd == SWAP_OSTATS)
739 sep = (struct swapent *)
740 ((struct oswapent *)sep + 1);
741 else
742 #endif
743 sep++;
744 }
745 }
746
747 *retval = count;
748 return;
749 }
750
751 /*
752 * swap_on: attempt to enable a swapdev for swapping. note that the
753 * swapdev is already on the global list, but disabled (marked
754 * SWF_FAKE).
755 *
756 * => we avoid the start of the disk (to protect disk labels)
757 * => we also avoid the miniroot, if we are swapping to root.
758 * => caller should leave uvm.swap_data_lock unlocked, we may lock it
759 * if needed.
760 */
761 static int
762 swap_on(p, sdp)
763 struct proc *p;
764 struct swapdev *sdp;
765 {
766 static int count = 0; /* static */
767 struct vnode *vp;
768 int error, npages, nblocks, size;
769 long addr;
770 u_long result;
771 struct vattr va;
772 #ifdef NFS
773 extern int (**nfsv2_vnodeop_p) __P((void *));
774 #endif /* NFS */
775 dev_t dev;
776 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist);
777
778 /*
779 * we want to enable swapping on sdp. the swd_vp contains
780 * the vnode we want (locked and ref'd), and the swd_dev
781 * contains the dev_t of the file, if it a block device.
782 */
783
784 vp = sdp->swd_vp;
785 dev = sdp->swd_dev;
786
787 /*
788 * open the swap file (mostly useful for block device files to
789 * let device driver know what is up).
790 *
791 * we skip the open/close for root on swap because the root
792 * has already been opened when root was mounted (mountroot).
793 */
794 if (vp != rootvp) {
795 if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p)))
796 return (error);
797 }
798
799 /* XXX this only works for block devices */
800 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0);
801
802 /*
803 * we now need to determine the size of the swap area. for
804 * block specials we can call the d_psize function.
805 * for normal files, we must stat [get attrs].
806 *
807 * we put the result in nblks.
808 * for normal files, we also want the filesystem block size
809 * (which we get with statfs).
810 */
811 switch (vp->v_type) {
812 case VBLK:
813 if (bdevsw[major(dev)].d_psize == 0 ||
814 (nblocks = (*bdevsw[major(dev)].d_psize)(dev)) == -1) {
815 error = ENXIO;
816 goto bad;
817 }
818 break;
819
820 case VREG:
821 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p)))
822 goto bad;
823 nblocks = (int)btodb(va.va_size);
824 if ((error =
825 VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0)
826 goto bad;
827
828 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize;
829 /*
830 * limit the max # of outstanding I/O requests we issue
831 * at any one time. take it easy on NFS servers.
832 */
833 #ifdef NFS
834 if (vp->v_op == nfsv2_vnodeop_p)
835 sdp->swd_maxactive = 2; /* XXX */
836 else
837 #endif /* NFS */
838 sdp->swd_maxactive = 8; /* XXX */
839 break;
840
841 default:
842 error = ENXIO;
843 goto bad;
844 }
845
846 /*
847 * save nblocks in a safe place and convert to pages.
848 */
849
850 sdp->swd_ose.ose_nblks = nblocks;
851 npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT;
852
853 /*
854 * for block special files, we want to make sure that leave
855 * the disklabel and bootblocks alone, so we arrange to skip
856 * over them (arbitrarily choosing to skip PAGE_SIZE bytes).
857 * note that because of this the "size" can be less than the
858 * actual number of blocks on the device.
859 */
860 if (vp->v_type == VBLK) {
861 /* we use pages 1 to (size - 1) [inclusive] */
862 size = npages - 1;
863 addr = 1;
864 } else {
865 /* we use pages 0 to (size - 1) [inclusive] */
866 size = npages;
867 addr = 0;
868 }
869
870 /*
871 * make sure we have enough blocks for a reasonable sized swap
872 * area. we want at least one page.
873 */
874
875 if (size < 1) {
876 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0);
877 error = EINVAL;
878 goto bad;
879 }
880
881 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0);
882
883 /*
884 * now we need to allocate an extent to manage this swap device
885 */
886 snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x",
887 count++);
888
889 /* note that extent_create's 3rd arg is inclusive, thus "- 1" */
890 sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP,
891 0, 0, EX_WAITOK);
892 /* allocate the `saved' region from the extent so it won't be used */
893 if (addr) {
894 if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK))
895 panic("disklabel region");
896 }
897
898 /*
899 * if the vnode we are swapping to is the root vnode
900 * (i.e. we are swapping to the miniroot) then we want
901 * to make sure we don't overwrite it. do a statfs to
902 * find its size and skip over it.
903 */
904 if (vp == rootvp) {
905 struct mount *mp;
906 struct statfs *sp;
907 int rootblocks, rootpages;
908
909 mp = rootvnode->v_mount;
910 sp = &mp->mnt_stat;
911 rootblocks = sp->f_blocks * btodb(sp->f_bsize);
912 /*
913 * XXX: sp->f_blocks isn't the total number of
914 * blocks in the filesystem, it's the number of
915 * data blocks. so, our rootblocks almost
916 * definitely underestimates the total size
917 * of the filesystem - how badly depends on the
918 * details of the filesystem type. there isn't
919 * an obvious way to deal with this cleanly
920 * and perfectly, so for now we just pad our
921 * rootblocks estimate with an extra 5 percent.
922 */
923 rootblocks += (rootblocks >> 5) +
924 (rootblocks >> 6) +
925 (rootblocks >> 7);
926 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT;
927 if (rootpages > size)
928 panic("swap_on: miniroot larger than swap?");
929
930 if (extent_alloc_region(sdp->swd_ex, addr,
931 rootpages, EX_WAITOK))
932 panic("swap_on: unable to preserve miniroot");
933
934 size -= rootpages;
935 printf("Preserved %d pages of miniroot ", rootpages);
936 printf("leaving %d pages of swap\n", size);
937 }
938
939 /*
940 * try to add anons to reflect the new swap space.
941 */
942
943 error = uvm_anon_add(size);
944 if (error) {
945 goto bad;
946 }
947
948 /*
949 * add a ref to vp to reflect usage as a swap device.
950 */
951 vref(vp);
952
953 /*
954 * now add the new swapdev to the drum and enable.
955 */
956 if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY,
957 EX_WAITOK, &result))
958 panic("swapdrum_add");
959
960 sdp->swd_drumoffset = (int)result;
961 sdp->swd_drumsize = npages;
962 sdp->swd_npages = size;
963 simple_lock(&uvm.swap_data_lock);
964 sdp->swd_flags &= ~SWF_FAKE; /* going live */
965 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE);
966 uvmexp.swpages += size;
967 simple_unlock(&uvm.swap_data_lock);
968 return (0);
969
970 /*
971 * failure: clean up and return error.
972 */
973
974 bad:
975 if (sdp->swd_ex) {
976 extent_destroy(sdp->swd_ex);
977 }
978 if (vp != rootvp) {
979 (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p);
980 }
981 return (error);
982 }
983
984 /*
985 * swap_off: stop swapping on swapdev
986 *
987 * => swap data should be locked, we will unlock.
988 */
989 static int
990 swap_off(p, sdp)
991 struct proc *p;
992 struct swapdev *sdp;
993 {
994 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist);
995 UVMHIST_LOG(pdhist, " dev=%x", sdp->swd_dev,0,0,0);
996
997 /* disable the swap area being removed */
998 sdp->swd_flags &= ~SWF_ENABLE;
999 simple_unlock(&uvm.swap_data_lock);
1000
1001 /*
1002 * the idea is to find all the pages that are paged out to this
1003 * device, and page them all in. in uvm, swap-backed pageable
1004 * memory can take two forms: aobjs and anons. call the
1005 * swapoff hook for each subsystem to bring in pages.
1006 */
1007
1008 if (uao_swap_off(sdp->swd_drumoffset,
1009 sdp->swd_drumoffset + sdp->swd_drumsize) ||
1010 anon_swap_off(sdp->swd_drumoffset,
1011 sdp->swd_drumoffset + sdp->swd_drumsize)) {
1012
1013 simple_lock(&uvm.swap_data_lock);
1014 sdp->swd_flags |= SWF_ENABLE;
1015 simple_unlock(&uvm.swap_data_lock);
1016 return ENOMEM;
1017 }
1018 KASSERT(sdp->swd_npginuse == sdp->swd_npgbad);
1019
1020 /*
1021 * done with the vnode.
1022 * drop our ref on the vnode before calling VOP_CLOSE()
1023 * so that spec_close() can tell if this is the last close.
1024 */
1025 vrele(sdp->swd_vp);
1026 if (sdp->swd_vp != rootvp) {
1027 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p);
1028 }
1029
1030 /* remove anons from the system */
1031 uvm_anon_remove(sdp->swd_npages);
1032
1033 simple_lock(&uvm.swap_data_lock);
1034 uvmexp.swpages -= sdp->swd_npages;
1035
1036 if (swaplist_find(sdp->swd_vp, 1) == NULL)
1037 panic("swap_off: swapdev not in list\n");
1038 swaplist_trim();
1039 simple_unlock(&uvm.swap_data_lock);
1040
1041 /*
1042 * free all resources!
1043 */
1044 extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize,
1045 EX_WAITOK);
1046 extent_destroy(sdp->swd_ex);
1047 free(sdp, M_VMSWAP);
1048 return (0);
1049 }
1050
1051 /*
1052 * /dev/drum interface and i/o functions
1053 */
1054
1055 /*
1056 * swread: the read function for the drum (just a call to physio)
1057 */
1058 /*ARGSUSED*/
1059 int
1060 swread(dev, uio, ioflag)
1061 dev_t dev;
1062 struct uio *uio;
1063 int ioflag;
1064 {
1065 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist);
1066
1067 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0);
1068 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio));
1069 }
1070
1071 /*
1072 * swwrite: the write function for the drum (just a call to physio)
1073 */
1074 /*ARGSUSED*/
1075 int
1076 swwrite(dev, uio, ioflag)
1077 dev_t dev;
1078 struct uio *uio;
1079 int ioflag;
1080 {
1081 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist);
1082
1083 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0);
1084 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio));
1085 }
1086
1087 /*
1088 * swstrategy: perform I/O on the drum
1089 *
1090 * => we must map the i/o request from the drum to the correct swapdev.
1091 */
1092 void
1093 swstrategy(bp)
1094 struct buf *bp;
1095 {
1096 struct swapdev *sdp;
1097 struct vnode *vp;
1098 int s, pageno, bn;
1099 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist);
1100
1101 /*
1102 * convert block number to swapdev. note that swapdev can't
1103 * be yanked out from under us because we are holding resources
1104 * in it (i.e. the blocks we are doing I/O on).
1105 */
1106 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT;
1107 simple_lock(&uvm.swap_data_lock);
1108 sdp = swapdrum_getsdp(pageno);
1109 simple_unlock(&uvm.swap_data_lock);
1110 if (sdp == NULL) {
1111 bp->b_error = EINVAL;
1112 bp->b_flags |= B_ERROR;
1113 biodone(bp);
1114 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0);
1115 return;
1116 }
1117
1118 /*
1119 * convert drum page number to block number on this swapdev.
1120 */
1121
1122 pageno -= sdp->swd_drumoffset; /* page # on swapdev */
1123 bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */
1124
1125 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld",
1126 ((bp->b_flags & B_READ) == 0) ? "write" : "read",
1127 sdp->swd_drumoffset, bn, bp->b_bcount);
1128
1129 /*
1130 * for block devices we finish up here.
1131 * for regular files we have to do more work which we delegate
1132 * to sw_reg_strategy().
1133 */
1134
1135 switch (sdp->swd_vp->v_type) {
1136 default:
1137 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type);
1138
1139 case VBLK:
1140
1141 /*
1142 * must convert "bp" from an I/O on /dev/drum to an I/O
1143 * on the swapdev (sdp).
1144 */
1145 s = splbio();
1146 bp->b_blkno = bn; /* swapdev block number */
1147 vp = sdp->swd_vp; /* swapdev vnode pointer */
1148 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */
1149
1150 /*
1151 * if we are doing a write, we have to redirect the i/o on
1152 * drum's v_numoutput counter to the swapdevs.
1153 */
1154 if ((bp->b_flags & B_READ) == 0) {
1155 vwakeup(bp); /* kills one 'v_numoutput' on drum */
1156 vp->v_numoutput++; /* put it on swapdev */
1157 }
1158
1159 /*
1160 * finally plug in swapdev vnode and start I/O
1161 */
1162 bp->b_vp = vp;
1163 splx(s);
1164 VOP_STRATEGY(bp);
1165 return;
1166
1167 case VREG:
1168 /*
1169 * delegate to sw_reg_strategy function.
1170 */
1171 sw_reg_strategy(sdp, bp, bn);
1172 return;
1173 }
1174 /* NOTREACHED */
1175 }
1176
1177 /*
1178 * sw_reg_strategy: handle swap i/o to regular files
1179 */
1180 static void
1181 sw_reg_strategy(sdp, bp, bn)
1182 struct swapdev *sdp;
1183 struct buf *bp;
1184 int bn;
1185 {
1186 struct vnode *vp;
1187 struct vndxfer *vnx;
1188 daddr_t nbn;
1189 caddr_t addr;
1190 off_t byteoff;
1191 int s, off, nra, error, sz, resid;
1192 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist);
1193
1194 /*
1195 * allocate a vndxfer head for this transfer and point it to
1196 * our buffer.
1197 */
1198 getvndxfer(vnx);
1199 vnx->vx_flags = VX_BUSY;
1200 vnx->vx_error = 0;
1201 vnx->vx_pending = 0;
1202 vnx->vx_bp = bp;
1203 vnx->vx_sdp = sdp;
1204
1205 /*
1206 * setup for main loop where we read filesystem blocks into
1207 * our buffer.
1208 */
1209 error = 0;
1210 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */
1211 addr = bp->b_data; /* current position in buffer */
1212 byteoff = dbtob((u_int64_t)bn);
1213
1214 for (resid = bp->b_resid; resid; resid -= sz) {
1215 struct vndbuf *nbp;
1216
1217 /*
1218 * translate byteoffset into block number. return values:
1219 * vp = vnode of underlying device
1220 * nbn = new block number (on underlying vnode dev)
1221 * nra = num blocks we can read-ahead (excludes requested
1222 * block)
1223 */
1224 nra = 0;
1225 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize,
1226 &vp, &nbn, &nra);
1227
1228 if (error == 0 && nbn == (daddr_t)-1) {
1229 /*
1230 * this used to just set error, but that doesn't
1231 * do the right thing. Instead, it causes random
1232 * memory errors. The panic() should remain until
1233 * this condition doesn't destabilize the system.
1234 */
1235 #if 1
1236 panic("sw_reg_strategy: swap to sparse file");
1237 #else
1238 error = EIO; /* failure */
1239 #endif
1240 }
1241
1242 /*
1243 * punt if there was an error or a hole in the file.
1244 * we must wait for any i/o ops we have already started
1245 * to finish before returning.
1246 *
1247 * XXX we could deal with holes here but it would be
1248 * a hassle (in the write case).
1249 */
1250 if (error) {
1251 s = splbio();
1252 vnx->vx_error = error; /* pass error up */
1253 goto out;
1254 }
1255
1256 /*
1257 * compute the size ("sz") of this transfer (in bytes).
1258 */
1259 off = byteoff % sdp->swd_bsize;
1260 sz = (1 + nra) * sdp->swd_bsize - off;
1261 if (sz > resid)
1262 sz = resid;
1263
1264 UVMHIST_LOG(pdhist, "sw_reg_strategy: "
1265 "vp %p/%p offset 0x%x/0x%x",
1266 sdp->swd_vp, vp, byteoff, nbn);
1267
1268 /*
1269 * now get a buf structure. note that the vb_buf is
1270 * at the front of the nbp structure so that you can
1271 * cast pointers between the two structure easily.
1272 */
1273 getvndbuf(nbp);
1274 nbp->vb_buf.b_flags = bp->b_flags | B_CALL;
1275 nbp->vb_buf.b_bcount = sz;
1276 nbp->vb_buf.b_bufsize = sz;
1277 nbp->vb_buf.b_error = 0;
1278 nbp->vb_buf.b_data = addr;
1279 nbp->vb_buf.b_lblkno = 0;
1280 nbp->vb_buf.b_blkno = nbn + btodb(off);
1281 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno;
1282 nbp->vb_buf.b_iodone = sw_reg_iodone;
1283 nbp->vb_buf.b_vp = vp;
1284 if (vp->v_type == VBLK) {
1285 nbp->vb_buf.b_dev = vp->v_rdev;
1286 }
1287 LIST_INIT(&nbp->vb_buf.b_dep);
1288
1289 nbp->vb_xfer = vnx; /* patch it back in to vnx */
1290
1291 /*
1292 * Just sort by block number
1293 */
1294 s = splbio();
1295 if (vnx->vx_error != 0) {
1296 putvndbuf(nbp);
1297 goto out;
1298 }
1299 vnx->vx_pending++;
1300
1301 /* sort it in and start I/O if we are not over our limit */
1302 disksort_blkno(&sdp->swd_tab, &nbp->vb_buf);
1303 sw_reg_start(sdp);
1304 splx(s);
1305
1306 /*
1307 * advance to the next I/O
1308 */
1309 byteoff += sz;
1310 addr += sz;
1311 }
1312
1313 s = splbio();
1314
1315 out: /* Arrive here at splbio */
1316 vnx->vx_flags &= ~VX_BUSY;
1317 if (vnx->vx_pending == 0) {
1318 if (vnx->vx_error != 0) {
1319 bp->b_error = vnx->vx_error;
1320 bp->b_flags |= B_ERROR;
1321 }
1322 putvndxfer(vnx);
1323 biodone(bp);
1324 }
1325 splx(s);
1326 }
1327
1328 /*
1329 * sw_reg_start: start an I/O request on the requested swapdev
1330 *
1331 * => reqs are sorted by disksort (above)
1332 */
1333 static void
1334 sw_reg_start(sdp)
1335 struct swapdev *sdp;
1336 {
1337 struct buf *bp;
1338 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist);
1339
1340 /* recursion control */
1341 if ((sdp->swd_flags & SWF_BUSY) != 0)
1342 return;
1343
1344 sdp->swd_flags |= SWF_BUSY;
1345
1346 while (sdp->swd_active < sdp->swd_maxactive) {
1347 bp = BUFQ_FIRST(&sdp->swd_tab);
1348 if (bp == NULL)
1349 break;
1350 BUFQ_REMOVE(&sdp->swd_tab, bp);
1351 sdp->swd_active++;
1352
1353 UVMHIST_LOG(pdhist,
1354 "sw_reg_start: bp %p vp %p blkno %p cnt %lx",
1355 bp, bp->b_vp, bp->b_blkno, bp->b_bcount);
1356 if ((bp->b_flags & B_READ) == 0)
1357 bp->b_vp->v_numoutput++;
1358
1359 VOP_STRATEGY(bp);
1360 }
1361 sdp->swd_flags &= ~SWF_BUSY;
1362 }
1363
1364 /*
1365 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup
1366 *
1367 * => note that we can recover the vndbuf struct by casting the buf ptr
1368 */
1369 static void
1370 sw_reg_iodone(bp)
1371 struct buf *bp;
1372 {
1373 struct vndbuf *vbp = (struct vndbuf *) bp;
1374 struct vndxfer *vnx = vbp->vb_xfer;
1375 struct buf *pbp = vnx->vx_bp; /* parent buffer */
1376 struct swapdev *sdp = vnx->vx_sdp;
1377 int s, resid;
1378 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist);
1379
1380 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p",
1381 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data);
1382 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx",
1383 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0);
1384
1385 /*
1386 * protect vbp at splbio and update.
1387 */
1388
1389 s = splbio();
1390 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid;
1391 pbp->b_resid -= resid;
1392 vnx->vx_pending--;
1393
1394 if (vbp->vb_buf.b_error) {
1395 UVMHIST_LOG(pdhist, " got error=%d !",
1396 vbp->vb_buf.b_error, 0, 0, 0);
1397
1398 /* pass error upward */
1399 vnx->vx_error = vbp->vb_buf.b_error;
1400 }
1401
1402 /*
1403 * kill vbp structure
1404 */
1405 putvndbuf(vbp);
1406
1407 /*
1408 * wrap up this transaction if it has run to completion or, in
1409 * case of an error, when all auxiliary buffers have returned.
1410 */
1411 if (vnx->vx_error != 0) {
1412 /* pass error upward */
1413 pbp->b_flags |= B_ERROR;
1414 pbp->b_error = vnx->vx_error;
1415 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) {
1416 putvndxfer(vnx);
1417 biodone(pbp);
1418 }
1419 } else if (pbp->b_resid == 0) {
1420 KASSERT(vnx->vx_pending == 0);
1421 if ((vnx->vx_flags & VX_BUSY) == 0) {
1422 UVMHIST_LOG(pdhist, " iodone error=%d !",
1423 pbp, vnx->vx_error, 0, 0);
1424 putvndxfer(vnx);
1425 biodone(pbp);
1426 }
1427 }
1428
1429 /*
1430 * done! start next swapdev I/O if one is pending
1431 */
1432 sdp->swd_active--;
1433 sw_reg_start(sdp);
1434 splx(s);
1435 }
1436
1437
1438 /*
1439 * uvm_swap_alloc: allocate space on swap
1440 *
1441 * => allocation is done "round robin" down the priority list, as we
1442 * allocate in a priority we "rotate" the circle queue.
1443 * => space can be freed with uvm_swap_free
1444 * => we return the page slot number in /dev/drum (0 == invalid slot)
1445 * => we lock uvm.swap_data_lock
1446 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM
1447 */
1448 int
1449 uvm_swap_alloc(nslots, lessok)
1450 int *nslots; /* IN/OUT */
1451 boolean_t lessok;
1452 {
1453 struct swapdev *sdp;
1454 struct swappri *spp;
1455 u_long result;
1456 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist);
1457
1458 /*
1459 * no swap devices configured yet? definite failure.
1460 */
1461 if (uvmexp.nswapdev < 1)
1462 return 0;
1463
1464 /*
1465 * lock data lock, convert slots into blocks, and enter loop
1466 */
1467 simple_lock(&uvm.swap_data_lock);
1468
1469 ReTry: /* XXXMRG */
1470 LIST_FOREACH(spp, &swap_priority, spi_swappri) {
1471 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
1472 /* if it's not enabled, then we can't swap from it */
1473 if ((sdp->swd_flags & SWF_ENABLE) == 0)
1474 continue;
1475 if (sdp->swd_npginuse + *nslots > sdp->swd_npages)
1476 continue;
1477 if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN,
1478 EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT,
1479 &result) != 0) {
1480 continue;
1481 }
1482
1483 /*
1484 * successful allocation! now rotate the circleq.
1485 */
1486 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next);
1487 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
1488 sdp->swd_npginuse += *nslots;
1489 uvmexp.swpginuse += *nslots;
1490 simple_unlock(&uvm.swap_data_lock);
1491 /* done! return drum slot number */
1492 UVMHIST_LOG(pdhist,
1493 "success! returning %d slots starting at %d",
1494 *nslots, result + sdp->swd_drumoffset, 0, 0);
1495 return (result + sdp->swd_drumoffset);
1496 }
1497 }
1498
1499 /* XXXMRG: BEGIN HACK */
1500 if (*nslots > 1 && lessok) {
1501 *nslots = 1;
1502 goto ReTry; /* XXXMRG: ugh! extent should support this for us */
1503 }
1504 /* XXXMRG: END HACK */
1505
1506 simple_unlock(&uvm.swap_data_lock);
1507 return 0;
1508 }
1509
1510 /*
1511 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors
1512 *
1513 * => we lock uvm.swap_data_lock
1514 */
1515 void
1516 uvm_swap_markbad(startslot, nslots)
1517 int startslot;
1518 int nslots;
1519 {
1520 struct swapdev *sdp;
1521 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist);
1522
1523 simple_lock(&uvm.swap_data_lock);
1524 sdp = swapdrum_getsdp(startslot);
1525
1526 /*
1527 * we just keep track of how many pages have been marked bad
1528 * in this device, to make everything add up in swap_off().
1529 * we assume here that the range of slots will all be within
1530 * one swap device.
1531 */
1532
1533 sdp->swd_npgbad += nslots;
1534 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0);
1535 simple_unlock(&uvm.swap_data_lock);
1536 }
1537
1538 /*
1539 * uvm_swap_free: free swap slots
1540 *
1541 * => this can be all or part of an allocation made by uvm_swap_alloc
1542 * => we lock uvm.swap_data_lock
1543 */
1544 void
1545 uvm_swap_free(startslot, nslots)
1546 int startslot;
1547 int nslots;
1548 {
1549 struct swapdev *sdp;
1550 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist);
1551
1552 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots,
1553 startslot, 0, 0);
1554
1555 /*
1556 * ignore attempts to free the "bad" slot.
1557 */
1558
1559 if (startslot == SWSLOT_BAD) {
1560 return;
1561 }
1562
1563 /*
1564 * convert drum slot offset back to sdp, free the blocks
1565 * in the extent, and return. must hold pri lock to do
1566 * lookup and access the extent.
1567 */
1568
1569 simple_lock(&uvm.swap_data_lock);
1570 sdp = swapdrum_getsdp(startslot);
1571 KASSERT(uvmexp.nswapdev >= 1);
1572 KASSERT(sdp != NULL);
1573 KASSERT(sdp->swd_npginuse >= nslots);
1574 if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots,
1575 EX_MALLOCOK|EX_NOWAIT) != 0) {
1576 printf("warning: resource shortage: %d pages of swap lost\n",
1577 nslots);
1578 }
1579 sdp->swd_npginuse -= nslots;
1580 uvmexp.swpginuse -= nslots;
1581 simple_unlock(&uvm.swap_data_lock);
1582 }
1583
1584 /*
1585 * uvm_swap_put: put any number of pages into a contig place on swap
1586 *
1587 * => can be sync or async
1588 */
1589
1590 int
1591 uvm_swap_put(swslot, ppsp, npages, flags)
1592 int swslot;
1593 struct vm_page **ppsp;
1594 int npages;
1595 int flags;
1596 {
1597 int error;
1598
1599 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE |
1600 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
1601 return error;
1602 }
1603
1604 /*
1605 * uvm_swap_get: get a single page from swap
1606 *
1607 * => usually a sync op (from fault)
1608 */
1609
1610 int
1611 uvm_swap_get(page, swslot, flags)
1612 struct vm_page *page;
1613 int swslot, flags;
1614 {
1615 int error;
1616
1617 uvmexp.nswget++;
1618 KASSERT(flags & PGO_SYNCIO);
1619 if (swslot == SWSLOT_BAD) {
1620 return EIO;
1621 }
1622 error = uvm_swap_io(&page, swslot, 1, B_READ |
1623 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
1624 if (error == 0) {
1625
1626 /*
1627 * this page is no longer only in swap.
1628 */
1629
1630 simple_lock(&uvm.swap_data_lock);
1631 KASSERT(uvmexp.swpgonly > 0);
1632 uvmexp.swpgonly--;
1633 simple_unlock(&uvm.swap_data_lock);
1634 }
1635 return error;
1636 }
1637
1638 /*
1639 * uvm_swap_io: do an i/o operation to swap
1640 */
1641
1642 static int
1643 uvm_swap_io(pps, startslot, npages, flags)
1644 struct vm_page **pps;
1645 int startslot, npages, flags;
1646 {
1647 daddr_t startblk;
1648 struct buf *bp;
1649 vaddr_t kva;
1650 int error, s, mapinflags;
1651 boolean_t write, async;
1652 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist);
1653
1654 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d",
1655 startslot, npages, flags, 0);
1656
1657 write = (flags & B_READ) == 0;
1658 async = (flags & B_ASYNC) != 0;
1659
1660 /*
1661 * convert starting drum slot to block number
1662 */
1663
1664 startblk = btodb((u_int64_t)startslot << PAGE_SHIFT);
1665
1666 /*
1667 * first, map the pages into the kernel.
1668 */
1669
1670 mapinflags = !write ?
1671 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ :
1672 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE;
1673 kva = uvm_pagermapin(pps, npages, mapinflags);
1674
1675 /*
1676 * now allocate a buf for the i/o.
1677 */
1678
1679 s = splbio();
1680 bp = pool_get(&bufpool, PR_WAITOK);
1681 splx(s);
1682
1683 /*
1684 * fill in the bp/sbp. we currently route our i/o through
1685 * /dev/drum's vnode [swapdev_vp].
1686 */
1687
1688 bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC));
1689 bp->b_proc = &proc0; /* XXX */
1690 bp->b_vnbufs.le_next = NOLIST;
1691 bp->b_data = (caddr_t)kva;
1692 bp->b_blkno = startblk;
1693 bp->b_vp = swapdev_vp;
1694 bp->b_dev = swapdev_vp->v_rdev;
1695 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT;
1696 LIST_INIT(&bp->b_dep);
1697
1698 /*
1699 * bump v_numoutput (counter of number of active outputs).
1700 */
1701
1702 if (write) {
1703 s = splbio();
1704 swapdev_vp->v_numoutput++;
1705 splx(s);
1706 }
1707
1708 /*
1709 * for async ops we must set up the iodone handler.
1710 */
1711
1712 if (async) {
1713 bp->b_flags |= B_CALL;
1714 bp->b_iodone = uvm_aio_biodone;
1715 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0);
1716 }
1717 UVMHIST_LOG(pdhist,
1718 "about to start io: data = %p blkno = 0x%x, bcount = %ld",
1719 bp->b_data, bp->b_blkno, bp->b_bcount, 0);
1720
1721 /*
1722 * now we start the I/O, and if async, return.
1723 */
1724
1725 VOP_STRATEGY(bp);
1726 if (async)
1727 return 0;
1728
1729 /*
1730 * must be sync i/o. wait for it to finish
1731 */
1732
1733 error = biowait(bp);
1734
1735 /*
1736 * kill the pager mapping
1737 */
1738
1739 uvm_pagermapout(kva, npages);
1740
1741 /*
1742 * now dispose of the buf and we're done.
1743 */
1744
1745 s = splbio();
1746 if (write)
1747 vwakeup(bp);
1748 pool_put(&bufpool, bp);
1749 splx(s);
1750 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0);
1751 return (error);
1752 }
1753