xref: /src/sys/uvm/uvm_pdaemon.c

/*	$NetBSD: uvm_pdaemon.c,v 1.141 2026/08/23 22:08:41 riastradh Exp $	*/

/*
 * Copyright (c) 1997 Charles D. Cranor and Washington University.
 * Copyright (c) 1991, 1993, The Regents of the University of California.
 *
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * The Mach Operating System project at Carnegie-Mellon University.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)vm_pageout.c        8.5 (Berkeley) 2/14/94
 * from: Id: uvm_pdaemon.c,v 1.1.2.32 1998/02/06 05:26:30 chs Exp
 *
 *
 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
 * All rights reserved.
 *
 * Permission to use, copy, modify and distribute this software and
 * its documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 *
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 *
 * Carnegie Mellon requests users of this software to return to
 *
 *  Software Distribution Coordinator  or  Software.Distribution (at) CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 *
 * any improvements or extensions that they make and grant Carnegie the
 * rights to redistribute these changes.
 */

/*
 * uvm_pdaemon.c: the page daemon
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.141 2026/08/23 22:08:41 riastradh Exp $");

#include "opt_uvmhist.h"
#include "opt_readahead.h"

#define	__RWLOCK_PRIVATE

#include <sys/param.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/pool.h>
#include <sys/buf.h>
#include <sys/module.h>
#include <sys/atomic.h>
#include <sys/kthread.h>

#include <uvm/uvm.h>
#include <uvm/uvm_pdpolicy.h>
#include <uvm/uvm_pgflcache.h>

#ifdef UVMHIST
#ifndef UVMHIST_PDHIST_SIZE
#define UVMHIST_PDHIST_SIZE 100
#endif
static struct kern_history_ent pdhistbuf[UVMHIST_PDHIST_SIZE];
UVMHIST_DEFINE(pdhist) = UVMHIST_INITIALIZER(pdhisthist, pdhistbuf);
#endif

/*
 * UVMPD_NUMDIRTYREACTS is how many dirty pages the pagedaemon will reactivate
 * in a pass thru the inactive list when swap is full.  the value should be
 * "small"... if it's too large we'll cycle the active pages thru the inactive
 * queue too quickly to for them to be referenced and avoid being freed.
 */

#define	UVMPD_NUMDIRTYREACTS	16

/*
 * local prototypes
 */

static void	uvmpd_scan(void);
static void	uvmpd_scan_queue(void);
static void	uvmpd_tune(void);
static void	uvmpd_pool_drain_thread(void *);
static void	uvmpd_pool_drain_wakeup(void);

/*
 * State coordinating sleeping in uvm_wait with wakeups in uvm_pageout
 * (page daemon) and uvm_pageout_done (paging completion handler)
 */
static kmutex_t uvmpd_waiter_lock __cacheline_aligned; /* IPL_SOFTBIO */
static volatile unsigned uvm_pagedaemon_waiters __cacheline_aligned;
static volatile unsigned uvm_waiter_wakeup __cacheline_aligned;

/* State for the pool drainer thread */
static kmutex_t uvmpd_pool_drain_lock; /* IPL_NONE */
static kcondvar_t uvmpd_pool_drain_cv;
static bool uvmpd_pool_drain_run = false;

/*
 * State for requesting page daemon wakeups, which may be triggered
 * from:
 *
 * - hard interrupt context when NOSLEEP allocations fail
 * - soft interrupt context when uvm_pageout_done hasn't done enough
 * - thread context in uvm_wait when SLEEP allocations fail
 */
static kmutex_t uvmpd_kick_lock __cacheline_aligned; /* IPL_VM */
static volatile unsigned uvmpd_sleeping __cacheline_aligned;
static volatile unsigned uvmpd_wakeup __cacheline_aligned;

/*
 * XXX hack to avoid hangs when large processes fork.
 */
u_int uvm_extrapages;

#if __HAVE_ATOMIC64_LOADSTORE

static volatile uint64_t uvmpd_gen;

static uint64_t
uvmpd_read_gen(void)
{

	return atomic_load_acquire(&uvmpd_gen);
}

static void
uvmpd_advance_gen(void)
{

	KASSERT(mutex_owned(&uvmpd_waiter_lock));
	atomic_store_release(&uvmpd_gen, atomic_load_relaxed(&uvmpd_gen) + 1);
}

#else  /* !__HAVE_ATOMIC64_LOADSTORE */

static volatile struct {
#if _BYTE_ORDER == _BIG_ENDIAN
	uint32_t	hi, lo;
#else
	uint32_t	lo, hi;
#endif
} uvmpd_gen;

static uint64_t
uvmpd_read_gen(void)
{
	uint32_t hi, lo;

top:	while (__predict_false((hi = atomic_load_relaxed(&uvmpd_gen.hi)) ==
		0xffffffff))
		SPINLOCK_BACKOFF_HOOK;
	membar_consumer();	/* membar_producer in uvmpd_advance_gen */
	/* load-acquire matches store-release in uvmpd_advance_gen */
	lo = atomic_load_acquire(&uvmpd_gen.lo);
	if (__predict_false(hi != atomic_load_relaxed(&uvmpd_gen.hi)))
		goto top;

	return (uint64_t)hi << 32 | lo;
}

static void
uvmpd_advance_gen(void)
{
	uint32_t hi, lo;

	KASSERT(mutex_owned(&uvmpd_waiter_lock));

	lo = atomic_load_relaxed(&uvmpd_gen.lo);
	if (__predict_false(++lo == 0)) {
		hi = atomic_load_relaxed(&uvmpd_gen.hi) + 1; /* carry */
		atomic_store_relaxed(&uvmpd_gen.hi, 0xffffffff);
		/* store-release matches load-acquire in uvmpd_read_gen */
		atomic_store_release(&uvmpd_gen.lo, lo);
		membar_producer(); /* membar_consumer in uvmpd_read_gen */
		atomic_store_relaxed(&uvmpd_gen.hi, hi);
	} else {
		/* store-release matches load-acquire in uvmpd_read_gen */
		atomic_store_release(&uvmpd_gen.lo, lo);
	}
}

#endif

static void
uvmpd_wake_pagedaemon(void)
{

	/*
	 * Record that a wakeup is pending.  If there was already a
	 * wakeup pending, some other thread will take care of the
	 * rest, so stop here.
	 *
	 * membar_release matches membar_acquire in uvm_pageout.
	 */
	membar_release();
	if (atomic_swap_uint(&uvmpd_wakeup, 1))
		return;

	/*
	 * Ensure store-before-load ordering so we announce intent to
	 * wake up before we check whether the page daemon has
	 * announced intent to sleep.
	 *
	 * Matches membar_sync in uvm_pageout between storing
	 * uvmpd_sleeping := 1 and loading uvmpd_wakeup.
	 */
	membar_sync();

	/*
	 * If _after_ we have told the page daemon to wake up, it has
	 * still not announced intent to sleep, nothing more to do --
	 * it's busy working, and it will read uvmpd_wakeup on the next
	 * iteration and avoid going to sleep.
	 */
	if (!atomic_load_relaxed(&uvmpd_sleeping))
		return;

	/*
	 * The page daemon has already decided to go to sleep, and it
	 * may be too late for us to stop it.  Wake it up.
	 */
	mutex_spin_enter(&uvmpd_kick_lock);
	wakeup(&uvm.pagedaemon);
	mutex_spin_exit(&uvmpd_kick_lock);
}

static void
uvmpd_wake_waiters(void)
{

	/*
	 * Record that a wakeup is pending.  If there was already a
	 * wakeup pending, some other thread will take care of the
	 * rest, so stop here.
	 *
	 * membar_release matches membar_acquire in uvm_wait.
	 */
	membar_release();
	if (atomic_swap_uint(&uvm_waiter_wakeup, 1))
		return;

	/*
	 * Ensure store-before-load ordering so we announce intent to
	 * wake up before we check whether there are any waiters
	 * waiting.
	 *
	 * Matches membar_sync in uvm_wait.
	 */
	membar_sync();

	/*
	 * If _after_ we have told the waiters to wake up, there still
	 * are none, nothing more to do -- whoever calls uvm_wait next
	 * will read uvm_waiter_wakeup and avoid going to sleep.
	 */
	if (atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0)
		return;

	/*
	 * uvm_wait waiters are either asleep or about to go to sleep,
	 * and it may be too late to stop them, so we have to take the
	 * lock to wake them up.
	 *
	 * Make sure to advance the generation so that logic still in
	 * the ellipsis of
	 *
	 * top:	ticket = uvm_wait_prepare();
	 *	...
	 *	if (allocfailed) {
	 *		uvm_wait("wmesg", ticket);
	 *		goto top;
	 *	}
	 *
	 * will wake up and not sleep forever when it hits uvm_wait.
	 */
	mutex_enter(&uvmpd_waiter_lock);
	uvmpd_advance_gen();
	if (atomic_load_relaxed(&uvm_pagedaemon_waiters) != 0) {
		atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
		wakeup(&uvmexp.free);
	}
	mutex_exit(&uvmpd_waiter_lock);
}

/*
 * uvm_wait_prepare: return a ticket for uvm_wait
 *
 * => caller must:
 *    1. get a ticket _first_,
 *    2. _then_ try allocation, and
 *    3. uvm_wait with the ticket if it fails
 */

uint64_t
uvm_wait_prepare(void)
{
	uint64_t ticket;

	ticket = uvmpd_read_gen();

	return ticket;
}

/*
 * uvm_wait: wait (sleep) for the page daemon to free some pages
 *
 * => should be called with all locks released
 * => should _not_ be called by the page daemon (to avoid deadlock)
 * => caller must have acquired a ticket with uvm_wait_prepare,
 *    before trying an allocation that we now wait for
 */

void
uvm_wait(const char *wmsg, uint64_t ticket)
{
	uint64_t gen = ticket;
	int timo = 0;

	if (uvm.pagedaemon_lwp == NULL)
		panic("out of memory before the pagedaemon thread exists");

	/*
	 * Fast path: if the page daemon wakeup generation has already
	 * advanced since the caller began uvm_wait_prepare, don't add
	 * to contention on uvmpd_waiter_lock; we already know the
	 * caller can retry its allocation.
	 */
	if (gen != uvmpd_read_gen())
		return;

	mutex_enter(&uvmpd_waiter_lock);

	/*
	 * If the page daemon wakeup generation has advanced since the
	 * caller began uvm_wait_prepare, don't sleep; the caller can
	 * already retry its allocation.
	 */
	if (gen != uvmpd_read_gen()) {
		mutex_exit(&uvmpd_waiter_lock);
		return;
	}

	/*
	 * check for page daemon going to sleep (waiting for itself)
	 */
	if (uvm_lwp_is_pagedaemon(curlwp) && uvmexp.paging == 0) {
		/*
		 * now we have a problem: the pagedaemon wants to go to
		 * sleep until it frees more memory.   but how can it
		 * free more memory if it is asleep?  that is a deadlock.
		 * we have two options:
		 *  [1] panic now
		 *  [2] put a timeout on the sleep, thus causing the
		 *      pagedaemon to only pause (rather than sleep forever)
		 *
		 * note that option [2] will only help us if we get lucky
		 * and some other process on the system breaks the deadlock
		 * by exiting or freeing memory (thus allowing the pagedaemon
		 * to continue).  for now we panic if DEBUG is defined,
		 * otherwise we hope for the best with option [2] (better
		 * yet, this should never happen in the first place!).
		 */

		printf("pagedaemon: deadlock detected!\n");
		timo = hz >> 3;		/* set timeout */
#if defined(DEBUG)
		/* DEBUG: panic so we can debug it */
		panic("pagedaemon deadlock");
#endif
	}

	/*
	 * Notify uvmpd_wake_waiters that there are waiters before we
	 * check whether there's a pending unlocked wakeup.
	 *
	 * membar_sync here matches membar_sync in uvmpd_wake_waiters.
	 *
	 * If there's pending unlocked wakeup, consume it by advancing
	 * the wakeup generation and waking any other waiters -- that
	 * can be done only with the lock.  Generally there should be
	 * no other waiters at this point -- if there is a pending
	 * wakeup, any other call to uvm_wait would have consumed it
	 * and so can't be waiting; if another call to uvm_wait didn't
	 * find a pending wakeup, then any subsequent uvm_pageout or
	 * uvm_pageout_done wakeup should have noticed that call and
	 * issued wakeup(&uvmexp.free) -- but it takes a more effort to
	 * prove that it is safe to elide wakeup(&uvmexp.free) than it
	 * takes to just call it anyway here.
	 *
	 * The wakeup might be spurious by now, but the caller has to
	 * cope with spurious wakeups anyway.
	 */
	atomic_store_relaxed(&uvm_pagedaemon_waiters,
	    atomic_load_relaxed(&uvm_pagedaemon_waiters) + 1);
	membar_sync();
	if (atomic_swap_uint(&uvm_waiter_wakeup, 0)) {
		/*
		 * membar_acquire matches membar_release in
		 * uvmpd_wake_waiters.
		 */
		membar_acquire();
		uvmpd_advance_gen();
		atomic_store_relaxed(&uvm_pagedaemon_waiters, 0);
		wakeup(&uvmexp.free);
		mutex_exit(&uvmpd_waiter_lock);
		return;
	}

	uvmpd_wake_pagedaemon();	/* wake the daemon! */
	UVM_UNLOCK_AND_WAIT(&uvmexp.free, &uvmpd_waiter_lock, false, wmsg,
	    timo);
}

/*
 * uvm_kick_pdaemon: perform checks to determine if we need to
 * give the pagedaemon a nudge, and do so if necessary.
 */

void
uvm_kick_pdaemon(void)
{
	int fpages = uvm_availmem(false);

	if (fpages + uvmexp.paging < uvmexp.freemin ||
	    (fpages + uvmexp.paging < uvmexp.freetarg &&
	     uvmpdpol_needsscan_p()) ||
	     uvm_km_va_starved_p()) {
		uvmpd_wake_pagedaemon();
	}
}

/*
 * _uvm_lwp_is_pagedaemon: returns true of the specified lwp is
 * a pagedaemon lwp.
 */

bool
_uvm_lwp_is_pagedaemon(struct lwp *l)
{
	return _uvm_lwp_is_pagedaemon_test(l);
}

/*
 * uvmpd_tune: tune paging parameters
 *
 * => called when ever memory is added (or removed?) to the system
 */

static void
uvmpd_tune(void)
{
	int val;

	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/*
	 * try to keep 0.5% of available RAM free, but limit to between
	 * 128k and 1024k per-CPU.  XXX: what are these values good for?
	 */
	val = uvmexp.npages / 200;
	val = MAX(val, (128*1024) >> PAGE_SHIFT);
	val = MIN(val, (1024*1024) >> PAGE_SHIFT);
	val *= ncpu;

	/* Make sure there's always a user page free. */
	if (val < uvmexp.reserve_kernel + 1)
		val = uvmexp.reserve_kernel + 1;
	uvmexp.freemin = val;

	/* Calculate free target. */
	val = (uvmexp.freemin * 4) / 3;
	if (val <= uvmexp.freemin)
		val = uvmexp.freemin + 1;
	uvmexp.freetarg = val + atomic_swap_uint(&uvm_extrapages, 0);

	uvmexp.wiredmax = uvmexp.npages / 3;
	UVMHIST_LOG(pdhist, "<- done, freemin=%jd, freetarg=%jd, wiredmax=%jd",
	      uvmexp.freemin, uvmexp.freetarg, uvmexp.wiredmax, 0);
}

/*
 * uvm_pageout: the main loop for the pagedaemon
 */

void
uvm_pageout(void *arg)
{
	int npages = 0;
	int extrapages = 0;
	int fpages;

	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	UVMHIST_LOG(pdhist,"<starting uvm pagedaemon>", 0, 0, 0, 0);

	mutex_init(&uvmpd_kick_lock, MUTEX_DEFAULT, IPL_VM);
	mutex_init(&uvmpd_waiter_lock, MUTEX_DEFAULT, IPL_SOFTBIO);
	mutex_init(&uvmpd_pool_drain_lock, MUTEX_DEFAULT, IPL_NONE);
	cv_init(&uvmpd_pool_drain_cv, "pooldrain");

	/* Create the pool drainer kernel thread. */
	if (kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL,
	    uvmpd_pool_drain_thread, NULL, NULL, "pooldrain"))
		panic("fork pooldrain");

	/*
	 * ensure correct priority and set paging parameters...
	 */

	uvm.pagedaemon_lwp = curlwp;
	npages = uvmexp.npages;
	uvmpd_tune();

	/*
	 * main loop
	 */

	for (;;) {
		bool needsscan, needsfree, kmem_va_starved;

		kmem_va_starved = uvm_km_va_starved_p();

		mutex_spin_enter(&uvmpd_kick_lock);
		if ((atomic_load_relaxed(&uvm_pagedaemon_waiters) == 0 ||
			uvmexp.paging > 0) &&
		    !kmem_va_starved) {
			UVMHIST_LOG(pdhist,"  <<SLEEPING>>",0,0,0,0);

			/*
			 * Notify uvm_kick_pdaemon that we're about to
			 * go to sleep.  Then check whether it has
			 * already asked us to wake up again before it
			 * puts contention on uvmpd_kick_lock.  If,
			 * after we have announced our intent to sleep,
			 * there's still no pending wakeup, go to sleep
			 * and release uvm_pdlock.
			 *
			 * membar_sync here matches membar_sync in
			 * uvmpd_wake_pagedaemon.
			 *
			 * Once we're not asleep -- either because
			 * uvmpd_wakeup = 1 or because of a concurrent
			 * wakeup(&uvm.pagedaemon) -- set
			 * uvmpd_sleeping back to 0 again so that while
			 * we work, uvmpd_wake_pagedaemon need not take
			 * uvmpd_kick_lock in order to make sure we
			 * wake up next time.
			 */
			atomic_store_relaxed(&uvmpd_sleeping, 1);
			membar_sync();
			if (atomic_swap_uint(&uvmpd_wakeup, 0)) {
				/*
				 * membar_release matches
				 * membar_acquire in
				 * uvmpd_wake_pagedaemon.
				 */
				membar_acquire();
				mutex_spin_exit(&uvmpd_kick_lock);
			} else {
				UVM_UNLOCK_AND_WAIT(&uvm.pagedaemon,
				    &uvmpd_kick_lock, false, "pgdaemon", 0);
			}
			atomic_store_relaxed(&uvmpd_sleeping, 0);
			uvmexp.pdwoke++;

			UVMHIST_LOG(pdhist,"  <<WOKE UP>>",0,0,0,0);
		} else {
			mutex_spin_exit(&uvmpd_kick_lock);
		}

		/*
		 * now recompute inactive count
		 */

		if (npages != uvmexp.npages || extrapages != uvm_extrapages) {
			npages = uvmexp.npages;
			extrapages = uvm_extrapages;
			uvmpd_tune();
		}

		uvmpdpol_tune();

		/*
		 * Estimate a hint.  Note that bufmem are returned to
		 * system only when entire pool page is empty.
		 */
		fpages = uvm_availmem(false);
		UVMHIST_LOG(pdhist,"  free/ftarg=%jd/%jd",
		    fpages, uvmexp.freetarg, 0,0);

		needsfree = fpages + uvmexp.paging < uvmexp.freetarg;
		needsscan = needsfree || uvmpdpol_needsscan_p();

		/*
		 * scan if needed
		 */
		if (needsscan) {
			uvmpd_scan();
		}

		/*
		 * if there's any free memory to be had,
		 * wake up any waiters.
		 */
		if (uvm_availmem(false) > uvmexp.reserve_kernel ||
		    uvmexp.paging == 0) {
			uvmpd_wake_waiters();
		}

		/*
		 * scan done.  if we don't need free memory, we're done.
		 */

		if (!needsfree && !kmem_va_starved)
			continue;

		/*
		 * kick the pool drainer thread.
		 */

		uvmpd_pool_drain_wakeup();
	}
	/*NOTREACHED*/
}

void
uvm_pageout_start(int npages)
{

	atomic_add_int(&uvmexp.paging, npages);
}

void
uvm_pageout_done(int npages)
{

	KASSERT(atomic_load_relaxed(&uvmexp.paging) >= npages);

	if (npages == 0) {
		return;
	}

	atomic_add_int(&uvmexp.paging, -npages);

	/*
	 * wake up either of pagedaemon or LWPs waiting for it.
	 */
	if (uvm_availmem(false) <= uvmexp.reserve_kernel) {
		uvmpd_wake_pagedaemon();
	} else {
		uvmpd_wake_waiters();
	}
}

static krwlock_t *
uvmpd_page_owner_lock(struct vm_page *pg)
{
	struct uvm_object *uobj = pg->uobject;
	struct vm_anon *anon = pg->uanon;
	krwlock_t *slock;

	KASSERT(mutex_owned(&pg->interlock));

#ifdef DEBUG
	if (uobj == (void *)0xdeadbeef || anon == (void *)0xdeadbeef) {
		return NULL;
	}
#endif
	if (uobj != NULL) {
		slock = uobj->vmobjlock;
		KASSERTMSG(slock != NULL, "pg %p uobj %p, NULL lock", pg, uobj);
	} else if (anon != NULL) {
		slock = anon->an_lock;
		KASSERTMSG(slock != NULL, "pg %p anon %p, NULL lock", pg, anon);
	} else {
		slock = NULL;
	}
	return slock;
}

/*
 * uvmpd_trylockowner: trylock the page's owner.
 *
 * => called with page interlock held.
 * => resolve orphaned O->A loaned page.
 * => return the locked mutex on success.  otherwise, return NULL.
 */

krwlock_t *
uvmpd_trylockowner(struct vm_page *pg)
{
	krwlock_t *slock, *heldslock = NULL;

	KASSERT(mutex_owned(&pg->interlock));

	slock = uvmpd_page_owner_lock(pg);
	if (slock == NULL) {
		/* Page may be in state of flux - ignore. */
		mutex_exit(&pg->interlock);
		return NULL;
	}

	if (rw_tryenter(slock, RW_WRITER)) {
		goto success;
	}

	/*
	 * The try-lock didn't work, so now do a blocking lock after
	 * dropping the page interlock.  Prevent the owner lock from
	 * being freed by taking a hold on it first.
	 */

	rw_obj_hold(slock);
	mutex_exit(&pg->interlock);
	rw_enter(slock, RW_WRITER);
	heldslock = slock;

	/*
	 * Now we hold some owner lock.  Check if the lock we hold
	 * is still the lock for the owner of the page.
	 * If it is then return it, otherwise release it and return NULL.
	 */

	mutex_enter(&pg->interlock);
	slock = uvmpd_page_owner_lock(pg);
	if (heldslock != slock) {
		rw_exit(heldslock);
		slock = NULL;
	} else {
success:
		/*
		 * Set PG_ANON if it isn't set already.
		 */
		if (pg->uobject == NULL && (pg->flags & PG_ANON) == 0) {
			KASSERT(pg->loan_count > 0);
			pg->loan_count--;
			pg->flags |= PG_ANON;
			/* anon now owns it */
		}
	}
	mutex_exit(&pg->interlock);
	if (heldslock != NULL) {
		rw_obj_free(heldslock);
	}
	return slock;
}

#if defined(VMSWAP)
struct swapcluster {
	int swc_slot;
	int swc_nallocated;
	int swc_nused;
	struct vm_page *swc_pages[howmany(MAXPHYS, MIN_PAGE_SIZE)];
};

static void
swapcluster_init(struct swapcluster *swc)
{

	swc->swc_slot = 0;
	swc->swc_nused = 0;
}

static int
swapcluster_allocslots(struct swapcluster *swc)
{
	int slot;
	int npages;

	if (swc->swc_slot != 0) {
		return 0;
	}

	/* Even with strange MAXPHYS, the shift
	   implicitly rounds down to a page. */
	npages = MAXPHYS >> PAGE_SHIFT;
	slot = uvm_swap_alloc(&npages, true);
	if (slot == 0) {
		return ENOMEM;
	}
	swc->swc_slot = slot;
	swc->swc_nallocated = npages;
	swc->swc_nused = 0;

	return 0;
}

static int
swapcluster_add(struct swapcluster *swc, struct vm_page *pg)
{
	int slot;
	struct uvm_object *uobj;

	KASSERT(swc->swc_slot != 0);
	KASSERT(swc->swc_nused < swc->swc_nallocated);
	KASSERT((pg->flags & PG_SWAPBACKED) != 0);

	slot = swc->swc_slot + swc->swc_nused;
	uobj = pg->uobject;
	if (uobj == NULL) {
		KASSERT(rw_write_held(pg->uanon->an_lock));
		pg->uanon->an_swslot = slot;
	} else {
		int result;

		KASSERT(rw_write_held(uobj->vmobjlock));
		result = uao_set_swslot(uobj, pg->offset >> PAGE_SHIFT, slot);
		if (result == -1) {
			return ENOMEM;
		}
	}
	swc->swc_pages[swc->swc_nused] = pg;
	swc->swc_nused++;

	return 0;
}

static void
swapcluster_flush(struct swapcluster *swc, bool now)
{
	int slot;
	int nused;
	int nallocated;
	int error __diagused;

	if (swc->swc_slot == 0) {
		return;
	}
	KASSERT(swc->swc_nused <= swc->swc_nallocated);

	slot = swc->swc_slot;
	nused = swc->swc_nused;
	nallocated = swc->swc_nallocated;

	/*
	 * if this is the final pageout we could have a few
	 * unused swap blocks.  if so, free them now.
	 */

	if (nused < nallocated) {
		if (!now) {
			return;
		}
		uvm_swap_free(slot + nused, nallocated - nused);
	}

	/*
	 * now start the pageout.
	 */

	if (nused > 0) {
		uvmexp.pdpageouts++;
		uvm_pageout_start(nused);
		error = uvm_swap_put(slot, swc->swc_pages, nused, 0);
		KASSERT(error == 0 || error == ENOMEM);
	}

	/*
	 * zero swslot to indicate that we are
	 * no longer building a swap-backed cluster.
	 */

	swc->swc_slot = 0;
	swc->swc_nused = 0;
}

static int
swapcluster_nused(struct swapcluster *swc)
{

	return swc->swc_nused;
}

/*
 * uvmpd_dropswap: free any swap allocated to this page.
 *
 * => called with owner locked.
 * => return true if a page had an associated slot.
 */

bool
uvmpd_dropswap(struct vm_page *pg)
{
	bool result = false;
	struct vm_anon *anon = pg->uanon;

	if ((pg->flags & PG_ANON) && anon->an_swslot) {
		uvm_swap_free(anon->an_swslot, 1);
		anon->an_swslot = 0;
		uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
		result = true;
	} else if (pg->flags & PG_AOBJ) {
		int slot = uao_set_swslot(pg->uobject,
		    pg->offset >> PAGE_SHIFT, 0);
		if (slot) {
			uvm_swap_free(slot, 1);
			uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
			result = true;
		}
	}

	return result;
}

#endif /* defined(VMSWAP) */

/*
 * uvmpd_scan_queue: scan an replace candidate list for pages
 * to clean or free.
 *
 * => we work on meeting our free target by converting inactive pages
 *    into free pages.
 * => we handle the building of swap-backed clusters
 */

static void
uvmpd_scan_queue(void)
{
	struct vm_page *p;
	struct uvm_object *uobj;
	struct vm_anon *anon;
#if defined(VMSWAP)
	struct swapcluster swc;
#endif /* defined(VMSWAP) */
	int dirtyreacts;
	krwlock_t *slock;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/*
	 * swslot is non-zero if we are building a swap cluster.  we want
	 * to stay in the loop while we have a page to scan or we have
	 * a swap-cluster to build.
	 */

#if defined(VMSWAP)
	swapcluster_init(&swc);
#endif /* defined(VMSWAP) */

	dirtyreacts = 0;
	uvmpdpol_scaninit();

	while (/* CONSTCOND */ 1) {

		/*
		 * see if we've met the free target.
		 */

		if (uvm_availmem(false) + uvmexp.paging
#if defined(VMSWAP)
		    + swapcluster_nused(&swc)
#endif /* defined(VMSWAP) */
		    >= uvmexp.freetarg << 2 ||
		    dirtyreacts == UVMPD_NUMDIRTYREACTS) {
			UVMHIST_LOG(pdhist,"  met free target: "
				    "exit loop", 0, 0, 0, 0);
			break;
		}

		/*
		 * first we have the pdpolicy select a victim page
		 * and attempt to lock the object that the page
		 * belongs to.  if our attempt fails we skip on to
		 * the next page (no harm done).  it is important to
		 * "try" locking the object as we are locking in the
		 * wrong order (pageq -> object) and we don't want to
		 * deadlock.
		 *
		 * the only time we expect to see an ownerless page
		 * (i.e. a page with no uobject and !PG_ANON) is if an
		 * anon has loaned a page from a uvm_object and the
		 * uvm_object has dropped the ownership.  in that
		 * case, the anon can "take over" the loaned page
		 * and make it its own.
		 */

		p = uvmpdpol_selectvictim(&slock);
		if (p == NULL) {
			break;
		}
		KASSERT(uvmpdpol_pageisqueued_p(p));
		KASSERT(uvm_page_owner_locked_p(p, true));
		KASSERT(p->wire_count == 0);

		/*
		 * we are below target and have a new page to consider.
		 */

		anon = p->uanon;
		uobj = p->uobject;

		if (p->flags & PG_BUSY) {
			rw_exit(slock);
			uvmexp.pdbusy++;
			continue;
		}

		/* does the page belong to an object? */
		if (uobj != NULL) {
			uvmexp.pdobscan++;
		} else {
#if defined(VMSWAP)
			KASSERT(anon != NULL);
			uvmexp.pdanscan++;
#else /* defined(VMSWAP) */
			panic("%s: anon", __func__);
#endif /* defined(VMSWAP) */
		}


		/*
		 * we now have the object locked.
		 * if the page is not swap-backed, call the object's
		 * pager to flush and free the page.
		 */

#if defined(READAHEAD_STATS)
		if ((p->flags & PG_READAHEAD) != 0) {
			p->flags &= ~PG_READAHEAD;
			uvm_ra_miss.ev_count++;
		}
#endif /* defined(READAHEAD_STATS) */

		if ((p->flags & PG_SWAPBACKED) == 0) {
			KASSERT(uobj != NULL);
			(void) (uobj->pgops->pgo_put)(uobj, p->offset,
			    p->offset + PAGE_SIZE, PGO_CLEANIT|PGO_FREE);
			continue;
		}

		/*
		 * the page is swap-backed.  remove all the permissions
		 * from the page so we can sync the modified info
		 * without any race conditions.  if the page is clean
		 * we can free it now and continue.
		 */

		pmap_page_protect(p, VM_PROT_NONE);
		if (uvm_pagegetdirty(p) == UVM_PAGE_STATUS_UNKNOWN) {
			if (pmap_clear_modify(p)) {
				uvm_pagemarkdirty(p, UVM_PAGE_STATUS_DIRTY);
			} else {
				uvm_pagemarkdirty(p, UVM_PAGE_STATUS_CLEAN);
			}
		}
		if (uvm_pagegetdirty(p) != UVM_PAGE_STATUS_DIRTY) {
			int slot;
			int pageidx;

			pageidx = p->offset >> PAGE_SHIFT;
			uvm_pagefree(p);
			atomic_inc_uint(&uvmexp.pdfreed);

			/*
			 * for anons, we need to remove the page
			 * from the anon ourselves.  for aobjs,
			 * pagefree did that for us.
			 */

			if (anon) {
				KASSERT(anon->an_swslot != 0);
				anon->an_page = NULL;
				slot = anon->an_swslot;
			} else {
				slot = uao_find_swslot(uobj, pageidx);
			}
			if (slot > 0) {
				/* this page is now only in swap. */
				atomic_inc_uint(&uvmexp.swpgonly);
			}
			rw_exit(slock);
			continue;
		}

#if defined(VMSWAP)
		/*
		 * this page is dirty, skip it if we'll have met our
		 * free target when all the current pageouts complete.
		 */

		if (uvm_availmem(false) + uvmexp.paging >
		    uvmexp.freetarg << 2) {
			rw_exit(slock);
			continue;
		}

		/*
		 * free any swap space allocated to the page since
		 * we'll have to write it again with its new data.
		 */

		uvmpd_dropswap(p);

		/*
		 * start new swap pageout cluster (if necessary).
		 *
		 * if swap is full reactivate this page so that
		 * we eventually cycle all pages through the
		 * inactive queue.
		 */

		if (swapcluster_allocslots(&swc)) {
			dirtyreacts++;
			uvm_pagelock(p);
			uvm_pageactivate(p);
			uvm_pageunlock(p);
			rw_exit(slock);
			continue;
		}

		/*
		 * at this point, we're definitely going reuse this
		 * page.  mark the page busy and delayed-free.
		 * we should remove the page from the page queues
		 * so we don't ever look at it again.
		 * adjust counters and such.
		 */

		p->flags |= PG_BUSY;
		UVM_PAGE_OWN(p, "scan_queue");
		p->flags |= PG_PAGEOUT;
		uvmexp.pgswapout++;

		uvm_pagelock(p);
		uvm_pagedequeue(p);
		uvm_pageunlock(p);

		/*
		 * add the new page to the cluster.
		 */

		if (swapcluster_add(&swc, p)) {
			p->flags &= ~(PG_BUSY|PG_PAGEOUT);
			UVM_PAGE_OWN(p, NULL);
			dirtyreacts++;
			uvm_pagelock(p);
			uvm_pageactivate(p);
			uvm_pageunlock(p);
			rw_exit(slock);
			continue;
		}
		rw_exit(slock);

		swapcluster_flush(&swc, false);

#else /* defined(VMSWAP) */
		uvm_pagelock(p);
		uvm_pageactivate(p);
		uvm_pageunlock(p);
		rw_exit(slock);
#endif /* defined(VMSWAP) */
	}

	uvmpdpol_scanfini();

#if defined(VMSWAP)
	swapcluster_flush(&swc, true);
#endif /* defined(VMSWAP) */
}

/*
 * uvmpd_scan: scan the page queues and attempt to meet our targets.
 */

static void
uvmpd_scan(void)
{
	int swap_shortage, pages_freed, fpages;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	uvmexp.pdrevs++;

	/*
	 * work on meeting our targets.   first we work on our free target
	 * by converting inactive pages into free pages.  then we work on
	 * meeting our inactive target by converting active pages to
	 * inactive ones.
	 */

	UVMHIST_LOG(pdhist, "  starting 'free' loop",0,0,0,0);

	pages_freed = uvmexp.pdfreed;
	uvmpd_scan_queue();
	pages_freed = uvmexp.pdfreed - pages_freed;

	/*
	 * detect if we're not going to be able to page anything out
	 * until we free some swap resources from active pages.
	 */

	swap_shortage = 0;
	fpages = uvm_availmem(false);
	if (fpages < uvmexp.freetarg &&
	    uvmexp.swpginuse >= uvmexp.swpgavail &&
	    !uvm_swapisfull() &&
	    pages_freed == 0) {
		swap_shortage = uvmexp.freetarg - fpages;
	}

	uvmpdpol_balancequeue(swap_shortage);

	/*
	 * if still below the minimum target, try unloading kernel
	 * modules.
	 */

	if (uvm_availmem(false) < uvmexp.freemin) {
		module_thread_kick();
	}
}

/*
 * uvm_reclaimable: decide whether to wait for pagedaemon.
 *
 * => return true if it seems to be worth to do uvm_wait.
 *
 * XXX should be tunable.
 * XXX should consider pools, etc?
 */

bool
uvm_reclaimable(void)
{
	int filepages;
	int active, inactive;

	/*
	 * if swap is not full, no problem.
	 */

	if (!uvm_swapisfull()) {
		return true;
	}

	/*
	 * file-backed pages can be reclaimed even when swap is full.
	 * if we have more than 1/16 of pageable memory or 5MB, try to reclaim.
	 * NB: filepages calculation does not exclude EXECPAGES - intentional.
	 *
	 * XXX assume the worst case, ie. all wired pages are file-backed.
	 *
	 * XXX should consider about other reclaimable memory.
	 * XXX ie. pools, traditional buffer cache.
	 */

	cpu_count_sync(false);
	filepages = (int)(cpu_count_get(CPU_COUNT_FILECLEAN) +
	    cpu_count_get(CPU_COUNT_FILEUNKNOWN) +
	    cpu_count_get(CPU_COUNT_FILEDIRTY) - uvmexp.wired);
	uvm_estimatepageable(&active, &inactive);
	if (filepages >= MIN((active + inactive) >> 4,
	    5 * 1024 * 1024 >> PAGE_SHIFT)) {
		return true;
	}

	/*
	 * kill the process, fail allocation, etc..
	 */

	return false;
}

void
uvm_estimatepageable(int *active, int *inactive)
{

	uvmpdpol_estimatepageable(active, inactive);
}


/*
 * Use a separate thread for draining pools.
 * This work can't done from the main pagedaemon thread because
 * some pool allocators need to take vm_map locks.
 */

static void
uvmpd_pool_drain_thread(void *arg)
{
	struct pool *firstpool, *curpool;
	int bufcnt, lastslept;
	bool cycled;

	firstpool = NULL;
	cycled = true;
	for (;;) {
		/*
		 * sleep until awoken by the pagedaemon.
		 */
		mutex_enter(&uvmpd_pool_drain_lock);
		if (!uvmpd_pool_drain_run) {
			lastslept = getticks();
			cv_wait(&uvmpd_pool_drain_cv, &uvmpd_pool_drain_lock);
			if (getticks() != lastslept) {
				cycled = false;
				firstpool = NULL;
			}
		}
		uvmpd_pool_drain_run = false;
		mutex_exit(&uvmpd_pool_drain_lock);

		/*
		 * rate limit draining, otherwise in desperate circumstances
		 * this can totally saturate the system with xcall activity.
		 */
		if (cycled) {
			kpause("uvmpdlmt", false, 1, NULL);
			cycled = false;
			firstpool = NULL;
		}

		/*
		 * drain and temporarily disable the freelist cache.
		 */
		uvm_pgflcache_pause();

		/*
		 * kill unused metadata buffers.
		 */
		bufcnt = uvmexp.freetarg - uvm_availmem(false);
		if (bufcnt < 0)
			bufcnt = 0;

		mutex_enter(&bufcache_lock);
		buf_drain(bufcnt << PAGE_SHIFT);
		mutex_exit(&bufcache_lock);

		/*
		 * drain a pool, and then re-enable the freelist cache.
		 */
		(void)pool_drain(&curpool);
		KASSERT(curpool != NULL);
		if (firstpool == NULL) {
			firstpool = curpool;
		} else if (firstpool == curpool) {
			cycled = true;
		}
		uvm_pgflcache_resume();
	}
	/*NOTREACHED*/
}

static void
uvmpd_pool_drain_wakeup(void)
{

	mutex_enter(&uvmpd_pool_drain_lock);
	uvmpd_pool_drain_run = true;
	cv_signal(&uvmpd_pool_drain_cv);
	mutex_exit(&uvmpd_pool_drain_lock);
}