Home | History | Annotate | Line # | Download | only in kern
subr_pcu.c revision 1.13
      1  1.13   matt /*	$NetBSD: subr_pcu.c,v 1.13 2012/12/26 18:30:23 matt Exp $	*/
      2   1.1  rmind 
      3   1.1  rmind /*-
      4   1.1  rmind  * Copyright (c) 2011 The NetBSD Foundation, Inc.
      5   1.1  rmind  * All rights reserved.
      6   1.1  rmind  *
      7   1.1  rmind  * This code is derived from software contributed to The NetBSD Foundation
      8   1.1  rmind  * by Mindaugas Rasiukevicius.
      9   1.1  rmind  *
     10   1.1  rmind  * Redistribution and use in source and binary forms, with or without
     11   1.1  rmind  * modification, are permitted provided that the following conditions
     12   1.1  rmind  * are met:
     13   1.1  rmind  * 1. Redistributions of source code must retain the above copyright
     14   1.1  rmind  *    notice, this list of conditions and the following disclaimer.
     15   1.1  rmind  * 2. Redistributions in binary form must reproduce the above copyright
     16   1.1  rmind  *    notice, this list of conditions and the following disclaimer in the
     17   1.1  rmind  *    documentation and/or other materials provided with the distribution.
     18   1.1  rmind  *
     19   1.1  rmind  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20   1.1  rmind  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21   1.1  rmind  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22   1.1  rmind  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23   1.1  rmind  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24   1.1  rmind  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25   1.1  rmind  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26   1.1  rmind  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27   1.1  rmind  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28   1.1  rmind  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29   1.1  rmind  * POSSIBILITY OF SUCH DAMAGE.
     30   1.1  rmind  */
     31   1.1  rmind 
     32   1.1  rmind /*
     33   1.1  rmind  * Per CPU Unit (PCU) - is an interface to manage synchronization of any
     34   1.1  rmind  * per CPU context (unit) tied with LWP context.  Typical use: FPU state.
     35   1.1  rmind  *
     36   1.1  rmind  * Concurrency notes:
     37   1.1  rmind  *
     38   1.1  rmind  *	PCU state may be loaded only by the current LWP, that is, curlwp.
     39   1.1  rmind  *	Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id].
     40   1.1  rmind  *
     41   1.1  rmind  *	Request for a PCU release can be from owner LWP (whether PCU state
     42   1.1  rmind  *	is on current CPU or remote CPU) or any other LWP running on that
     43   1.1  rmind  *	CPU (in such case, owner LWP is on a remote CPU or sleeping).
     44   1.1  rmind  *
     45   1.1  rmind  *	In any case, PCU state can only be changed from the running CPU.
     46   1.1  rmind  *	If said PCU state is on the remote CPU, a cross-call will be sent
     47   1.1  rmind  *	by the owner LWP.  Therefore struct cpu_info::ci_pcu_curlwp[id]
     48   1.1  rmind  *	may only be changed by current CPU, and lwp_t::l_pcu_cpu[id] may
     49   1.1  rmind  *	only be unset by the CPU which has PCU state loaded.
     50   1.1  rmind  *
     51   1.1  rmind  *	There is a race condition: LWP may have a PCU state on a remote CPU,
     52   1.1  rmind  *	which it requests to be released via cross-call.  At the same time,
     53   1.1  rmind  *	other LWP on remote CPU might release existing PCU state and load
     54   1.1  rmind  *	its own one.  Cross-call may arrive after this and release different
     55   1.1  rmind  *	PCU state than intended.  In such case, such LWP would re-load its
     56   1.1  rmind  *	PCU state again.
     57   1.1  rmind  */
     58   1.1  rmind 
     59   1.1  rmind #include <sys/cdefs.h>
     60  1.13   matt __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.13 2012/12/26 18:30:23 matt Exp $");
     61   1.1  rmind 
     62   1.1  rmind #include <sys/param.h>
     63   1.1  rmind #include <sys/cpu.h>
     64   1.1  rmind #include <sys/lwp.h>
     65   1.1  rmind #include <sys/pcu.h>
     66   1.1  rmind #include <sys/xcall.h>
     67   1.1  rmind 
     68   1.3   matt #if PCU_UNIT_COUNT > 0
     69   1.3   matt 
     70  1.13   matt static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int);
     71  1.13   matt static void pcu_cpu_op(const pcu_ops_t *, const int);
     72  1.13   matt static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int);
     73  1.13   matt 
     74  1.13   matt __CTASSERT(PCU_KERNEL == 1);
     75  1.13   matt 
     76  1.13   matt #define	PCU_SAVE	(PCU_LOADED << 1) /* Save PCU state to the LWP. */
     77  1.13   matt #define	PCU_RELEASE	(PCU_SAVE << 1)	/* Release PCU state on the CPU. */
     78  1.13   matt #define	PCU_CLAIM	(PCU_RELEASE << 1)	/* CLAIM a PCU for a LWP. */
     79   1.1  rmind 
     80   1.4  rmind /* XXX */
     81   1.4  rmind extern const pcu_ops_t * const	pcu_ops_md_defs[];
     82   1.4  rmind 
     83  1.11   yamt /*
     84  1.11   yamt  * pcu_switchpoint: release PCU state if the LWP is being run on another CPU.
     85  1.11   yamt  *
     86  1.11   yamt  * On each context switches, called by mi_switch() with IPL_SCHED.
     87  1.11   yamt  * 'l' is an LWP which is just we switched to.  (the new curlwp)
     88  1.11   yamt  */
     89  1.11   yamt 
     90   1.1  rmind void
     91   1.4  rmind pcu_switchpoint(lwp_t *l)
     92   1.1  rmind {
     93  1.13   matt 	const uint32_t pcu_kernel_inuse = l->l_pcu_used[PCU_KERNEL];
     94  1.13   matt 	uint32_t pcu_user_inuse = l->l_pcu_used[PCU_USER];
     95   1.4  rmind 	/* int s; */
     96   1.1  rmind 
     97  1.12   matt 	KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp);
     98   1.4  rmind 
     99  1.13   matt 	if (__predict_false(pcu_kernel_inuse != 0)) {
    100  1.13   matt 		for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
    101  1.13   matt 			if ((pcu_kernel_inuse & (1 << id)) == 0) {
    102  1.13   matt 				continue;
    103  1.13   matt 			}
    104  1.13   matt 			struct cpu_info * const pcu_ci = l->l_pcu_cpu[id];
    105  1.13   matt 			if (pcu_ci == NULL || pcu_ci == l->l_cpu) {
    106  1.13   matt 				continue;
    107  1.13   matt 			}
    108  1.13   matt 			const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
    109  1.13   matt 			/*
    110  1.13   matt 			 * Steal the PCU away from the current owner and
    111  1.13   matt 			 * take ownership of it.
    112  1.13   matt 			 */
    113  1.13   matt 			pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE);
    114  1.13   matt 			pcu_do_op(pcu, l, PCU_KERNEL | PCU_CLAIM | PCU_RELOAD);
    115  1.13   matt 			pcu_user_inuse &= ~(1 << id);
    116  1.13   matt 		}
    117  1.13   matt 	}
    118  1.13   matt 
    119  1.13   matt 	if (__predict_true(pcu_user_inuse == 0)) {
    120   1.4  rmind 		/* PCUs are not in use. */
    121   1.4  rmind 		return;
    122   1.4  rmind 	}
    123  1.11   yamt 	/* commented out as we know we are already at IPL_SCHED */
    124   1.4  rmind 	/* s = splsoftclock(); */
    125  1.13   matt 	for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
    126  1.13   matt 		if ((pcu_user_inuse & (1 << id)) == 0) {
    127   1.4  rmind 			continue;
    128   1.4  rmind 		}
    129   1.5   matt 		struct cpu_info * const pcu_ci = l->l_pcu_cpu[id];
    130   1.4  rmind 		if (pcu_ci == NULL || pcu_ci == l->l_cpu) {
    131   1.4  rmind 			continue;
    132   1.4  rmind 		}
    133   1.4  rmind 		const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
    134  1.13   matt 		pcu->pcu_state_release(l, 0);
    135   1.4  rmind 	}
    136   1.4  rmind 	/* splx(s); */
    137   1.1  rmind }
    138   1.1  rmind 
    139  1.11   yamt /*
    140  1.11   yamt  * pcu_discard_all: discard PCU state of the given LWP.
    141  1.11   yamt  *
    142  1.11   yamt  * Used by exec and LWP exit.
    143  1.11   yamt  */
    144  1.11   yamt 
    145   1.7   matt void
    146   1.7   matt pcu_discard_all(lwp_t *l)
    147   1.7   matt {
    148  1.13   matt 	const uint32_t pcu_inuse = l->l_pcu_used[PCU_USER];
    149   1.7   matt 
    150   1.8   matt 	KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_inuse == 0));
    151  1.13   matt 	KASSERT(l->l_pcu_used[PCU_KERNEL] == 0);
    152   1.7   matt 
    153   1.7   matt 	if (__predict_true(pcu_inuse == 0)) {
    154   1.7   matt 		/* PCUs are not in use. */
    155   1.7   matt 		return;
    156   1.7   matt 	}
    157   1.7   matt 	const int s = splsoftclock();
    158   1.7   matt 	for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
    159   1.7   matt 		if ((pcu_inuse & (1 << id)) == 0) {
    160   1.7   matt 			continue;
    161   1.7   matt 		}
    162   1.7   matt 		if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
    163   1.7   matt 			continue;
    164   1.7   matt 		}
    165   1.7   matt 		const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
    166   1.7   matt 		/*
    167   1.7   matt 		 * We aren't releasing since this LWP isn't giving up PCU,
    168   1.7   matt 		 * just saving it.
    169   1.7   matt 		 */
    170   1.7   matt 		pcu_lwp_op(pcu, l, PCU_RELEASE);
    171   1.7   matt 	}
    172  1.13   matt 	l->l_pcu_used[PCU_USER] = 0;
    173   1.7   matt 	splx(s);
    174   1.7   matt }
    175   1.7   matt 
    176  1.11   yamt /*
    177  1.11   yamt  * pcu_save_all: save PCU state of the given LWP so that eg. coredump can
    178  1.11   yamt  * examine it.
    179  1.11   yamt  */
    180  1.11   yamt 
    181   1.7   matt void
    182   1.7   matt pcu_save_all(lwp_t *l)
    183   1.7   matt {
    184  1.13   matt 	const uint32_t pcu_inuse = l->l_pcu_used[PCU_USER];
    185  1.11   yamt 	/*
    186  1.11   yamt 	 * Unless LW_WCORE, we aren't releasing since this LWP isn't giving
    187  1.11   yamt 	 * up PCU, just saving it.
    188  1.11   yamt 	 */
    189   1.9   matt 	const int flags = PCU_SAVE | (l->l_flag & LW_WCORE ? PCU_RELEASE : 0);
    190   1.7   matt 
    191   1.9   matt 	/*
    192   1.9   matt 	 * Normally we save for the current LWP, but sometimes we get called
    193   1.9   matt 	 * with a different LWP (forking a system LWP or doing a coredump of
    194   1.9   matt 	 * a process with multiple threads) and we need to deal with that.
    195   1.9   matt 	 */
    196   1.9   matt 	KASSERT(l == curlwp
    197   1.9   matt 	    || (((l->l_flag & LW_SYSTEM)
    198   1.9   matt 		 || (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED))
    199   1.9   matt 	        && pcu_inuse == 0));
    200  1.13   matt 	KASSERT(l->l_pcu_used[PCU_KERNEL] == 0);
    201   1.7   matt 
    202   1.7   matt 	if (__predict_true(pcu_inuse == 0)) {
    203   1.7   matt 		/* PCUs are not in use. */
    204   1.7   matt 		return;
    205   1.7   matt 	}
    206   1.7   matt 	const int s = splsoftclock();
    207   1.7   matt 	for (u_int id = 0; id < PCU_UNIT_COUNT; id++) {
    208   1.7   matt 		if ((pcu_inuse & (1 << id)) == 0) {
    209   1.7   matt 			continue;
    210   1.7   matt 		}
    211   1.7   matt 		if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
    212   1.7   matt 			continue;
    213   1.7   matt 		}
    214   1.7   matt 		const pcu_ops_t * const pcu = pcu_ops_md_defs[id];
    215   1.9   matt 		pcu_lwp_op(pcu, l, flags);
    216   1.7   matt 	}
    217   1.7   matt 	splx(s);
    218   1.7   matt }
    219   1.7   matt 
    220   1.1  rmind /*
    221   1.4  rmind  * pcu_do_op: save/release PCU state on the current CPU.
    222   1.1  rmind  *
    223   1.1  rmind  * => Must be called at IPL_SOFTCLOCK or from the soft-interrupt.
    224   1.1  rmind  */
    225   1.4  rmind static inline void
    226   1.4  rmind pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags)
    227   1.4  rmind {
    228   1.4  rmind 	struct cpu_info * const ci = curcpu();
    229   1.4  rmind 	const u_int id = pcu->pcu_id;
    230  1.13   matt 	u_int state_flags = flags & (PCU_KERNEL|PCU_RELOAD|PCU_ENABLE);
    231  1.13   matt 	uint32_t id_mask = 1 << id;
    232  1.13   matt 	const bool kernel_p = (l->l_pcu_used[PCU_KERNEL] & id_mask) != 0;
    233   1.4  rmind 
    234  1.13   matt 	KASSERT(l->l_pcu_cpu[id] == (flags & PCU_CLAIM ? NULL : ci));
    235   1.4  rmind 
    236   1.4  rmind 	if (flags & PCU_SAVE) {
    237  1.13   matt 		pcu->pcu_state_save(l, (kernel_p ? PCU_KERNEL : 0));
    238   1.4  rmind 	}
    239   1.4  rmind 	if (flags & PCU_RELEASE) {
    240  1.13   matt 		pcu->pcu_state_release(l, state_flags);
    241  1.13   matt 		if (flags & PCU_KERNEL) {
    242  1.13   matt 			l->l_pcu_used[PCU_KERNEL] &= ~id_mask;
    243  1.13   matt 		}
    244   1.4  rmind 		ci->ci_pcu_curlwp[id] = NULL;
    245   1.4  rmind 		l->l_pcu_cpu[id] = NULL;
    246   1.4  rmind 	}
    247  1.13   matt 	if (flags & PCU_CLAIM) {
    248  1.13   matt 		if (l->l_pcu_used[(flags & PCU_KERNEL)] & id_mask)
    249  1.13   matt 			state_flags |= PCU_LOADED;
    250  1.13   matt 		pcu->pcu_state_load(l, state_flags);
    251  1.13   matt 		l->l_pcu_cpu[id] = ci;
    252  1.13   matt 		ci->ci_pcu_curlwp[id] = l;
    253  1.13   matt 		l->l_pcu_used[flags & PCU_KERNEL] |= id_mask;
    254  1.13   matt 	}
    255  1.13   matt 	if (flags == PCU_KERNEL) {
    256  1.13   matt 		KASSERT(ci->ci_pcu_curlwp[id] == l);
    257  1.13   matt 		pcu->pcu_state_save(l, 0);
    258  1.13   matt 		l->l_pcu_used[PCU_KERNEL] |= id_mask;
    259  1.13   matt 	}
    260   1.4  rmind }
    261   1.4  rmind 
    262   1.4  rmind /*
    263   1.6   matt  * pcu_cpu_op: helper routine to call pcu_do_op() via xcall(9) or
    264   1.6   matt  * by pcu_load.
    265   1.4  rmind  */
    266   1.1  rmind static void
    267   1.1  rmind pcu_cpu_op(const pcu_ops_t *pcu, const int flags)
    268   1.1  rmind {
    269   1.1  rmind 	const u_int id = pcu->pcu_id;
    270   1.4  rmind 	lwp_t * const l = curcpu()->ci_pcu_curlwp[id];
    271   1.4  rmind 
    272   1.6   matt 	//KASSERT(cpu_softintr_p());
    273   1.1  rmind 
    274   1.1  rmind 	/* If no state - nothing to do. */
    275   1.1  rmind 	if (l == NULL) {
    276   1.1  rmind 		return;
    277   1.1  rmind 	}
    278   1.4  rmind 	pcu_do_op(pcu, l, flags);
    279   1.1  rmind }
    280   1.1  rmind 
    281   1.1  rmind /*
    282   1.1  rmind  * pcu_lwp_op: perform PCU state save, release or both operations on LWP.
    283   1.1  rmind  */
    284   1.1  rmind static void
    285  1.13   matt pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags)
    286   1.1  rmind {
    287   1.1  rmind 	const u_int id = pcu->pcu_id;
    288   1.1  rmind 	struct cpu_info *ci;
    289   1.1  rmind 	uint64_t where;
    290   1.1  rmind 	int s;
    291   1.1  rmind 
    292   1.1  rmind 	/*
    293   1.1  rmind 	 * Caller should have re-checked if there is any state to manage.
    294   1.1  rmind 	 * Block the interrupts and inspect again, since cross-call sent
    295   1.1  rmind 	 * by remote CPU could have changed the state.
    296   1.1  rmind 	 */
    297   1.1  rmind 	s = splsoftclock();
    298   1.1  rmind 	ci = l->l_pcu_cpu[id];
    299   1.1  rmind 	if (ci == curcpu()) {
    300   1.1  rmind 		/*
    301   1.1  rmind 		 * State is on the current CPU - just perform the operations.
    302   1.1  rmind 		 */
    303  1.13   matt 		KASSERT((flags & PCU_CLAIM) == 0);
    304   1.6   matt 		KASSERTMSG(ci->ci_pcu_curlwp[id] == l,
    305  1.10    jym 		    "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)",
    306  1.10    jym 		     __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l);
    307   1.4  rmind 		pcu_do_op(pcu, l, flags);
    308   1.1  rmind 		splx(s);
    309   1.1  rmind 		return;
    310   1.1  rmind 	}
    311   1.1  rmind 
    312   1.1  rmind 	if (__predict_false(ci == NULL)) {
    313  1.13   matt 		if (flags & PCU_CLAIM) {
    314  1.13   matt 			pcu_do_op(pcu, l, flags);
    315  1.13   matt 		}
    316   1.1  rmind 		/* Cross-call has won the race - no state to manage. */
    317  1.13   matt 		splx(s);
    318   1.1  rmind 		return;
    319   1.1  rmind 	}
    320   1.1  rmind 
    321  1.13   matt 	splx(s);
    322  1.13   matt 
    323   1.1  rmind 	/*
    324   1.1  rmind 	 * State is on the remote CPU - perform the operations there.
    325   1.1  rmind 	 * Note: there is a race condition; see description in the top.
    326   1.1  rmind 	 */
    327   1.1  rmind 	where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op,
    328   1.1  rmind 	    __UNCONST(pcu), (void *)(uintptr_t)flags, ci);
    329   1.1  rmind 	xc_wait(where);
    330   1.1  rmind 
    331   1.1  rmind 	KASSERT((flags & PCU_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL);
    332   1.1  rmind }
    333   1.1  rmind 
    334   1.1  rmind /*
    335   1.1  rmind  * pcu_load: load/initialize the PCU state of current LWP on current CPU.
    336   1.1  rmind  */
    337   1.1  rmind void
    338   1.1  rmind pcu_load(const pcu_ops_t *pcu)
    339   1.1  rmind {
    340   1.1  rmind 	const u_int id = pcu->pcu_id;
    341   1.1  rmind 	struct cpu_info *ci, *curci;
    342   1.5   matt 	lwp_t * const l = curlwp;
    343   1.1  rmind 	uint64_t where;
    344   1.1  rmind 	int s;
    345   1.1  rmind 
    346   1.1  rmind 	KASSERT(!cpu_intr_p() && !cpu_softintr_p());
    347   1.1  rmind 
    348   1.1  rmind 	s = splsoftclock();
    349   1.1  rmind 	curci = curcpu();
    350   1.1  rmind 	ci = l->l_pcu_cpu[id];
    351   1.1  rmind 
    352   1.1  rmind 	/* Does this CPU already have our PCU state loaded? */
    353   1.1  rmind 	if (ci == curci) {
    354   1.1  rmind 		KASSERT(curci->ci_pcu_curlwp[id] == l);
    355  1.13   matt 		pcu->pcu_state_load(l, PCU_ENABLE);	/* Re-enable */
    356   1.1  rmind 		splx(s);
    357   1.1  rmind 		return;
    358   1.1  rmind 	}
    359   1.1  rmind 
    360   1.1  rmind 	/* If PCU state of this LWP is on the remote CPU - save it there. */
    361   1.1  rmind 	if (ci) {
    362   1.1  rmind 		splx(s);
    363   1.1  rmind 		/* Note: there is a race; see description in the top. */
    364   1.1  rmind 		where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op,
    365   1.1  rmind 		    __UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci);
    366   1.1  rmind 		xc_wait(where);
    367   1.1  rmind 
    368   1.1  rmind 		/* Enter IPL_SOFTCLOCK and re-fetch the current CPU. */
    369   1.1  rmind 		s = splsoftclock();
    370   1.1  rmind 		curci = curcpu();
    371   1.1  rmind 	}
    372   1.1  rmind 	KASSERT(l->l_pcu_cpu[id] == NULL);
    373   1.1  rmind 
    374   1.1  rmind 	/* Save the PCU state on the current CPU, if there is any. */
    375   1.6   matt 	pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE);
    376   1.1  rmind 	KASSERT(curci->ci_pcu_curlwp[id] == NULL);
    377   1.1  rmind 
    378   1.1  rmind 	/*
    379   1.1  rmind 	 * Finally, load the state for this LWP on this CPU.  Indicate to
    380   1.1  rmind 	 * load function whether PCU was used before.  Note the usage.
    381   1.1  rmind 	 */
    382  1.13   matt 	pcu_do_op(pcu, l, PCU_CLAIM | PCU_ENABLE | PCU_RELOAD);
    383   1.1  rmind 	splx(s);
    384   1.1  rmind }
    385   1.1  rmind 
    386   1.1  rmind /*
    387   1.1  rmind  * pcu_discard: discard the PCU state of current LWP.
    388   1.1  rmind  */
    389   1.1  rmind void
    390   1.1  rmind pcu_discard(const pcu_ops_t *pcu)
    391   1.1  rmind {
    392   1.1  rmind 	const u_int id = pcu->pcu_id;
    393   1.5   matt 	lwp_t * const l = curlwp;
    394   1.1  rmind 
    395   1.1  rmind 	KASSERT(!cpu_intr_p() && !cpu_softintr_p());
    396   1.1  rmind 
    397   1.1  rmind 	if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
    398   1.1  rmind 		return;
    399   1.1  rmind 	}
    400   1.1  rmind 	pcu_lwp_op(pcu, l, PCU_RELEASE);
    401  1.13   matt 	l->l_pcu_used[PCU_USER] &= ~(1 << id);
    402   1.1  rmind }
    403   1.1  rmind 
    404   1.1  rmind /*
    405   1.1  rmind  * pcu_save_lwp: save PCU state to the given LWP.
    406   1.1  rmind  */
    407   1.1  rmind void
    408   1.4  rmind pcu_save(const pcu_ops_t *pcu)
    409   1.1  rmind {
    410   1.1  rmind 	const u_int id = pcu->pcu_id;
    411   1.4  rmind 	lwp_t * const l = curlwp;
    412   1.1  rmind 
    413   1.1  rmind 	KASSERT(!cpu_intr_p() && !cpu_softintr_p());
    414   1.1  rmind 
    415   1.1  rmind 	if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
    416   1.1  rmind 		return;
    417   1.1  rmind 	}
    418   1.1  rmind 	pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE);
    419   1.1  rmind }
    420   1.1  rmind 
    421   1.1  rmind /*
    422   1.1  rmind  * pcu_used: return true if PCU was used (pcu_load() case) by the LWP.
    423   1.1  rmind  */
    424   1.1  rmind bool
    425   1.4  rmind pcu_used_p(const pcu_ops_t *pcu)
    426   1.1  rmind {
    427   1.1  rmind 	const u_int id = pcu->pcu_id;
    428   1.4  rmind 	lwp_t * const l = curlwp;
    429   1.1  rmind 
    430  1.13   matt 	return l->l_pcu_used[0] & (1 << id);
    431  1.13   matt }
    432  1.13   matt 
    433  1.13   matt void
    434  1.13   matt pcu_kernel_acquire(const pcu_ops_t *pcu)
    435  1.13   matt {
    436  1.13   matt 	struct cpu_info * const ci = curcpu();
    437  1.13   matt 	lwp_t * const l = curlwp;
    438  1.13   matt 	const u_int id = pcu->pcu_id;
    439  1.13   matt 
    440  1.13   matt 	/*
    441  1.13   matt 	 * If we own the PCU, save our user state.
    442  1.13   matt 	 */
    443  1.13   matt 	if (ci == l->l_pcu_cpu[id]) {
    444  1.13   matt 		pcu_lwp_op(pcu, l, PCU_KERNEL);
    445  1.13   matt 		return;
    446  1.13   matt 	}
    447  1.13   matt 	if (ci->ci_data.cpu_pcu_curlwp[id] != NULL) {
    448  1.13   matt 		/*
    449  1.13   matt 		 * The PCU is owned by another LWP so save its state.
    450  1.13   matt 		 */
    451  1.13   matt 		pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE);
    452  1.13   matt 	}
    453  1.13   matt 	/*
    454  1.13   matt 	 * Mark the PCU as hijacked and take ownership of it.
    455  1.13   matt 	 */
    456  1.13   matt 	printf("!");
    457  1.13   matt 	pcu_lwp_op(pcu, l, PCU_KERNEL | PCU_CLAIM | PCU_ENABLE | PCU_RELOAD);
    458  1.13   matt }
    459  1.13   matt 
    460  1.13   matt void
    461  1.13   matt pcu_kernel_release(const pcu_ops_t *pcu)
    462  1.13   matt {
    463  1.13   matt 	lwp_t * const l = curlwp;
    464  1.13   matt 
    465  1.13   matt 	KASSERT(l->l_pcu_used[PCU_KERNEL] & (1 << pcu->pcu_id));
    466  1.13   matt 
    467  1.13   matt 	/*
    468  1.13   matt 	 * Release the PCU, if the curlwp wants to use it, it will have incur
    469  1.13   matt 	 * a trap to reenable it.
    470  1.13   matt 	 */
    471  1.13   matt 	pcu_lwp_op(pcu, l, PCU_KERNEL | PCU_RELEASE);
    472   1.1  rmind }
    473   1.3   matt 
    474   1.3   matt #endif /* PCU_UNIT_COUNT > 0 */
    475