xref: /src/sys/arch/arm/arm32/cpuswitch.S

/*	$NetBSD: cpuswitch.S,v 1.35 2003/06/23 09:05:22 chris Exp $	*/

/*
 * Copyright 2003 Wasabi Systems, Inc.
 * All rights reserved.
 *
 * Written by Steve C. Woodford for Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed for the NetBSD Project by
 *      Wasabi Systems, Inc.
 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
 *    or promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
/*
 * Copyright (c) 1994-1998 Mark Brinicombe.
 * Copyright (c) 1994 Brini.
 * All rights reserved.
 *
 * This code is derived from software written for Brini by Mark Brinicombe
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by Brini.
 * 4. The name of the company nor the name of the author may be used to
 *    endorse or promote products derived from this software without specific
 *    prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * RiscBSD kernel project
 *
 * cpuswitch.S
 *
 * cpu switching functions
 *
 * Created      : 15/10/94
 */

#include "opt_armfpe.h"
#include "opt_arm32_pmap.h"
#include "opt_multiprocessor.h"

#include "assym.h"
#include <machine/param.h>
#include <machine/cpu.h>
#include <machine/frame.h>
#include <machine/asm.h>

/* LINTSTUB: include <sys/param.h> */

#undef IRQdisable
#undef IRQenable

/*
 * New experimental definitions of IRQdisable and IRQenable
 * These keep FIQ's enabled since FIQ's are special.
 */

#define IRQdisable \
	mrs	r14, cpsr ; \
	orr	r14, r14, #(I32_bit) ; \
	msr	cpsr_c, r14 ; \

#define IRQenable \
	mrs	r14, cpsr ; \
	bic	r14, r14, #(I32_bit) ; \
	msr	cpsr_c, r14 ; \

/*
 * These are used for switching the translation table/DACR.
 * Since the vector page can be invalid for a short time, we must
 * disable both regular IRQs *and* FIQs.
 *
 * XXX: This is not necessary if the vector table is relocated.
 */
#define IRQdisableALL \
	mrs	r14, cpsr ; \
	orr	r14, r14, #(I32_bit | F32_bit) ; \
	msr	cpsr_c, r14

#define IRQenableALL \
	mrs	r14, cpsr ; \
	bic	r14, r14, #(I32_bit | F32_bit) ; \
	msr	cpsr_c, r14

	.text

.Lwhichqs:
	.word	_C_LABEL(sched_whichqs)

.Lqs:
	.word	_C_LABEL(sched_qs)

/*
 * cpuswitch()
 *
 * preforms a process context switch.
 * This function has several entry points
 */

#ifdef MULTIPROCESSOR
.Lcpu_info_store:
	.word	_C_LABEL(cpu_info_store)
.Lcurlwp:
	/* FIXME: This is bogus in the general case. */
	.word	_C_LABEL(cpu_info_store) + CI_CURLWP

.Lcurpcb:
	.word	_C_LABEL(cpu_info_store) + CI_CURPCB
#else
.Lcurlwp:
	.word	_C_LABEL(curlwp)

.Lcurpcb:
	.word	_C_LABEL(curpcb)
#endif

.Lwant_resched:
	.word	_C_LABEL(want_resched)

.Lcpufuncs:
	.word	_C_LABEL(cpufuncs)

#ifndef MULTIPROCESSOR
	.data
	.global	_C_LABEL(curpcb)
_C_LABEL(curpcb):
	.word	0x00000000
	.text
#endif

.Lblock_userspace_access:
	.word	_C_LABEL(block_userspace_access)

.Lcpu_do_powersave:
	.word	_C_LABEL(cpu_do_powersave)

.Lpmap_kernel_cstate:
	.word	(kernel_pmap_store + PMAP_CSTATE)

.Llast_cache_state_ptr:
	.word	_C_LABEL(pmap_cache_state)

/*
 * Idle loop, exercised while waiting for a process to wake up.
 *
 * NOTE: When we jump back to .Lswitch_search, we must have a
 * pointer to whichqs in r7, which is what it is when we arrive
 * here.
 */
/* LINTSTUB: Ignore */
ASENTRY_NP(idle)
#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
	bl	_C_LABEL(sched_unlock_idle)
#endif
	ldr	r3, .Lcpu_do_powersave

	/* Enable interrupts */
	IRQenable

	/* If we don't want to sleep, use a simpler loop. */
	ldr	r3, [r3]		/* r3 = cpu_do_powersave */
	teq	r3, #0
	bne	2f

	/* Non-powersave idle. */
1:	/* should maybe do uvm pageidlezero stuff here */
	ldr	r3, [r7]		/* r3 = whichqs */
	teq	r3, #0x00000000
	bne	.Lswitch_search
	b	1b

2:	/* Powersave idle. */
	ldr	r4, .Lcpufuncs
3:	ldr	r3, [r7]		/* r3 = whichqs */
	teq	r3, #0x00000000
	bne	.Lswitch_search

	/* if saving power, don't want to pageidlezero */
	mov	r0, #0
	adr	lr, 3b
	ldr	pc, [r4, #(CF_SLEEP)]
	/* loops back around */


/*
 * Find a new lwp to run, save the current context and
 * load the new context
 *
 * Arguments:
 *	r0	'struct lwp *' of the current LWP
 */

ENTRY(cpu_switch)
/*
 * Local register usage. Some of these registers are out of date.
 * r1 = oldlwp
 * r2 = spl level
 * r3 = whichqs
 * r4 = queue
 * r5 = &qs[queue]
 * r6 = newlwp
 * r7 = scratch
 */
	stmfd	sp!, {r4-r7, lr}

	/*
	 * Indicate that there is no longer a valid process (curlwp = 0).
	 * Zero the current PCB pointer while we're at it.
	 */
	ldr	r7, .Lcurlwp
	ldr	r6, .Lcurpcb
	mov	r2, #0x00000000
	str	r2, [r7]		/* curproc = NULL */
	str	r2, [r6]		/* curpcb = NULL */

	/* stash the old proc while we call functions */
	mov	r5, r0

#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
	/* release the sched_lock before handling interrupts */
	bl	_C_LABEL(sched_unlock_idle)
#endif

	/* Lower the spl level to spl0 and get the current spl level. */
#ifdef __NEWINTR
	mov	r0, #(IPL_NONE)
	bl	_C_LABEL(_spllower)
#else /* ! __NEWINTR */
	mov	r0, #(_SPL_0)
	bl	_C_LABEL(splx)
#endif /* __NEWINTR */

	/* Push the old spl level onto the stack */
	str	r0, [sp, #-0x0004]!

	/* First phase : find a new lwp */

	ldr	r7, .Lwhichqs

	/* rem: r5 = old lwp */
	/* rem: r7 = &whichqs */

.Lswitch_search:
	IRQdisable
#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
	bl	_C_LABEL(sched_lock_idle)
#endif

	/* Do we have any active queues  */
	ldr	r3, [r7]

	/* If not we must idle until we do. */
	teq	r3, #0x00000000
	beq	_ASM_LABEL(idle)

	/* put old proc back in r1 */
	mov	r1, r5

	/* rem: r1 = old lwp */
	/* rem: r3 = whichqs */
	/* rem: interrupts are disabled */

	/*
	 * We have found an active queue. Currently we do not know which queue
	 * is active just that one of them is.
	 */
	/* this is the ffs algorithm devised by d.seal and posted to
	 * comp.sys.arm on 16 Feb 1994.
	 */
 	rsb	r5, r3, #0
 	ands	r0, r3, r5

	adr	r5, .Lcpu_switch_ffs_table

				    /* X = R0 */
	orr	r4, r0, r0, lsl #4  /* r4 = X * 0x11 */
	orr	r4, r4, r4, lsl #6  /* r4 = X * 0x451 */
	rsb	r4, r4, r4, lsl #16 /* r4 = X * 0x0450fbaf */

	/* used further down, saves SA stall */
	ldr	r6, .Lqs

	/* now lookup in table indexed on top 6 bits of a4 */
	ldrb	r4, [ r5, r4, lsr #26 ]

	/* rem: r0 = bit mask of chosen queue (1 << r4) */
	/* rem: r1 = old lwp */
	/* rem: r3 = whichqs */
	/* rem: r4 = queue number */
	/* rem: interrupts are disabled */

	/* Get the address of the queue (&qs[queue]) */
	add	r5, r6, r4, lsl #3

	/*
	 * Get the lwp from the queue and place the next process in
	 * the queue at the head. This basically unlinks the lwp at
	 * the head of the queue.
	 */
	ldr	r6, [r5, #(L_FORW)]

	/* rem: r6 = new lwp */
	ldr	r7, [r6, #(L_FORW)]
	str	r7, [r5, #(L_FORW)]

	/*
	 * Test to see if the queue is now empty. If the head of the queue
	 * points to the queue itself then there are no more lwps in
	 * the queue. We can therefore clear the queue not empty flag held
	 * in r3.
	 */

	teq	r5, r7
	biceq	r3, r3, r0

	/* rem: r0 = bit mask of chosen queue (1 << r4) - NOT NEEDED AN MORE */

	/* Fix the back pointer for the lwp now at the head of the queue. */
	ldr	r0, [r6, #(L_BACK)]
	str	r0, [r7, #(L_BACK)]

	/* Update the RAM copy of the queue not empty flags word. */
	ldr	r7, .Lwhichqs
	str	r3, [r7]

	/* rem: r1 = old lwp */
	/* rem: r3 = whichqs - NOT NEEDED ANY MORE */
	/* rem: r4 = queue number - NOT NEEDED ANY MORE */
	/* rem: r6 = new lwp */
	/* rem: interrupts are disabled */

	/* Clear the want_resched flag */
	ldr	r7, .Lwant_resched
	mov	r0, #0x00000000
	str	r0, [r7]

	/*
	 * Clear the back pointer of the lwp we have removed from
	 * the head of the queue. The new lwp is isolated now.
	 */
	str	r0, [r6, #(L_BACK)]

#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
	/*
	 * unlock the sched_lock, but leave interrupts off, for now.
	 */
	mov	r7, r1
	bl	_C_LABEL(sched_unlock_idle)
	mov	r1, r7
#endif

.Lswitch_resume:
#ifdef MULTIPROCESSOR
	/* XXX use curcpu() */
	ldr	r0, .Lcpu_info_store
	str	r0, [r6, #(L_CPU)]
#else
	/* l->l_cpu initialized in fork1() for single-processor */
#endif

	/* Process is now on a processor. */
	mov	r0, #LSONPROC			/* l->l_stat = LSONPROC */
	str	r0, [r6, #(L_STAT)]

	/* We have a new curlwp now so make a note it */
	ldr	r7, .Lcurlwp
	str	r6, [r7]

	/* Hook in a new pcb */
	ldr	r7, .Lcurpcb
	ldr	r0, [r6, #(L_ADDR)]
	str	r0, [r7]

	/* At this point we can allow IRQ's again. */
	IRQenable

	/* rem: r1 = old lwp */
	/* rem: r4 = return value */
	/* rem: r6 = new process */
	/* rem: interrupts are enabled */

	/*
	 * If the new process is the same as the process that called
	 * cpu_switch() then we do not need to save and restore any
	 * contexts. This means we can make a quick exit.
	 * The test is simple if curlwp on entry (now in r1) is the
	 * same as the proc removed from the queue we can jump to the exit.
	 */
	teq	r1, r6
	moveq	r4, #0x00000000		/* default to "didn't switch" */
	beq	.Lswitch_return

	/*
	 * At this point, we are guaranteed to be switching to
	 * a new lwp.
	 */
	mov	r4, #0x00000001

	/* Remember the old lwp in r0 */
	mov	r0, r1

	/*
	 * If the old lwp on entry to cpu_switch was zero then the
	 * process that called it was exiting. This means that we do
	 * not need to save the current context. Instead we can jump
	 * straight to restoring the context for the new process.
	 */
	teq	r0, #0x00000000
	beq	.Lswitch_exited

	/* rem: r0 = old lwp */
	/* rem: r4 = return value */
	/* rem: r6 = new process */
	/* rem: interrupts are enabled */

	/* Stage two : Save old context */

	/* Get the user structure for the old lwp. */
	ldr	r1, [r0, #(L_ADDR)]

	/* Save all the registers in the old lwp's pcb */
	add	r7, r1, #(PCB_R8)
	stmia	r7, {r8-r13}

	/*
	 * NOTE: We can now use r8-r13 until it is time to restore
	 * them for the new process.
	 */

	/* Remember the old PCB. */
	mov	r8, r1

	/* r1 now free! */

	/* Get the user structure for the new process in r9 */
	ldr	r9, [r6, #(L_ADDR)]

	/*
	 * This can be optimised... We know we want to go from SVC32
	 * mode to UND32 mode
	 */
        mrs	r3, cpsr
	bic	r2, r3, #(PSR_MODE)
	orr	r2, r2, #(PSR_UND32_MODE | I32_bit)
        msr	cpsr_c, r2

	str	sp, [r8, #(PCB_UND_SP)]

        msr	cpsr_c, r3		/* Restore the old mode */

	/* rem: r0 = old lwp */
	/* rem: r4 = return value */
	/* rem: r6 = new process */
	/* rem: r8 = old PCB */
	/* rem: r9 = new PCB */
	/* rem: interrupts are enabled */

	/* What else needs to be saved  Only FPA stuff when that is supported */

	/* Third phase : restore saved context */

	/* rem: r0 = old lwp */
	/* rem: r4 = return value */
	/* rem: r6 = new lwp */
	/* rem: r8 = old PCB */
	/* rem: r9 = new PCB */
	/* rem: interrupts are enabled */

	/*
	 * Get the new L1 table pointer into r11.  If we're switching to
	 * an LWP with the same address space as the outgoing one, we can
	 * skip the cache purge and the TTB load.
	 *
	 * To avoid data dep stalls that would happen anyway, we try
	 * and get some useful work done in the mean time.
	 */
	ldr	r10, [r8, #(PCB_PAGEDIR)]	/* r10 = old L1 */
	ldr	r11, [r9, #(PCB_PAGEDIR)]	/* r11 = new L1 */

	ldr	r0, [r8, #(PCB_DACR)]		/* r0 = old DACR */
	ldr	r1, [r9, #(PCB_DACR)]		/* r1 = new DACR */
	ldr	r8, [r9, #(PCB_CSTATE)]		/* r8 = &new_pmap->pm_cstate */
	ldr	r5, .Llast_cache_state_ptr	/* Previous thread's cstate */

	teq	r10, r11			/* Same L1? */
	ldr	r5, [r5]
	cmpeq	r0, r1				/* Same DACR? */
	beq	.Lcs_context_switched		/* yes! */

	ldr	r3, .Lblock_userspace_access
	mov	r12, #0
	cmp	r5, #0				/* No last vm? (switch_exit) */
	beq	.Lcs_cache_purge_skipped	/* No, we can skip cache flsh */

	mov	r2, #DOMAIN_CLIENT
	cmp	r1, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */
	beq	.Lcs_cache_purge_skipped	/* Yup. Don't flush cache */

	cmp	r5, r8				/* Same userland VM space? */
	ldrneb	r12, [r5, #(CS_CACHE_ID)]	/* Last VM space cache state */

	/*
	 * We're definately switching to a new userland VM space,
	 * and the previous userland VM space has yet to be flushed
	 * from the cache/tlb.
	 *
	 * r12 holds the previous VM space's cs_cache_id state
	 */
	tst	r12, #0xff			/* Test cs_cache_id */
	beq	.Lcs_cache_purge_skipped	/* VM space is not in cache */

	/*
	 * Definately need to flush the cache.
	 * Mark the old VM space as NOT being resident in the cache.
	 */
	mov	r2, #0x00000000
	strb	r2, [r5, #(CS_CACHE_ID)]
	strb	r2, [r5, #(CS_CACHE_D)]

	/*
	 * Don't allow user space access between the purge and the switch.
	 */
	mov	r2, #0x00000001
	str	r2, [r3]

	stmfd	sp!, {r0-r3}
	ldr	r1, .Lcpufuncs
	mov	lr, pc
	ldr	pc, [r1, #CF_IDCACHE_WBINV_ALL]
	ldmfd	sp!, {r0-r3}

.Lcs_cache_purge_skipped:
	/* rem: r1 = new DACR */
	/* rem: r3 = &block_userspace_access */
	/* rem: r4 = return value */
	/* rem: r5 = &old_pmap->pm_cstate (or NULL) */
	/* rem: r6 = new lwp */
	/* rem: r8 = &new_pmap->pm_cstate */
	/* rem: r9 = new PCB */
	/* rem: r10 = old L1 */
	/* rem: r11 = new L1 */

	mov	r2, #0x00000000
	ldr	r7, [r9, #(PCB_PL1VEC)]

	/*
	 * At this point we need to kill IRQ's again.
	 *
	 * XXXSCW: Don't need to block FIQs if vectors have been relocated
	 */
	IRQdisableALL

	/*
	 * Interrupts are disabled so we can allow user space accesses again
	 * as none will occur until interrupts are re-enabled after the
	 * switch.
	 */
	str	r2, [r3]

	/*
	 * Ensure the vector table is accessible by fixing up the L1
	 */
	cmp	r7, #0			/* No need to fixup vector table? */
	ldrne	r2, [r7]		/* But if yes, fetch current value */
	ldrne	r0, [r9, #(PCB_L1VEC)]	/* Fetch new vector_page value */
	mcr	p15, 0, r1, c3, c0, 0	/* Update DACR for new context */
	cmpne	r2, r0			/* Stuffing the same value? */
#ifndef PMAP_INCLUDE_PTE_SYNC
	strne	r0, [r7]		/* Nope, update it */
#else
	beq	.Lcs_same_vector
	str	r0, [r7]		/* Otherwise, update it */

	/*
	 * Need to sync the cache to make sure that last store is
	 * visible to the MMU.
	 */
	ldr	r2, .Lcpufuncs
	mov	r0, r7
	mov	r1, #4
	mov	lr, pc
	ldr	pc, [r2, #CF_DCACHE_WB_RANGE]

.Lcs_same_vector:
#endif /* PMAP_INCLUDE_PTE_SYNC */

	cmp	r10, r11		/* Switching to the same L1? */
	ldr	r10, .Lcpufuncs
	beq	.Lcs_same_l1		/* Yup. */

	/*
	 * Do a full context switch, including full TLB flush.
	 */
	mov	r0, r11
	mov	lr, pc
	ldr	pc, [r10, #CF_CONTEXT_SWITCH]

	/*
	 * Mark the old VM space as NOT being resident in the TLB
	 */
	mov	r2, #0x00000000
	cmp	r5, #0
	strneh	r2, [r5, #(CS_TLB_ID)]
	b	.Lcs_context_switched

	/*
	 * We're switching to a different process in the same L1.
	 * In this situation, we only need to flush the TLB for the
	 * vector_page mapping, and even then only if r7 is non-NULL.
	 */
.Lcs_same_l1:
	cmp	r7, #0
	movne	r0, #0			/* We *know* vector_page's VA is 0x0 */
	movne	lr, pc
	ldrne	pc, [r10, #CF_TLB_FLUSHID_SE]

.Lcs_context_switched:
	/* rem: r8 = &new_pmap->pm_cstate */

	/* XXXSCW: Safe to re-enable FIQs here */

	/*
	 * The new VM space is live in the cache and TLB.
	 * Update its cache/tlb state, and if it's not the kernel
	 * pmap, update the 'last cache state' pointer.
	 */
	mov	r2, #-1
	ldr	r5, .Lpmap_kernel_cstate
	ldr	r0, .Llast_cache_state_ptr
	str	r2, [r8, #(CS_ALL)]
	cmp	r5, r8
	strne	r8, [r0]

	/* rem: r4 = return value */
	/* rem: r6 = new lwp */
	/* rem: r9 = new PCB */

	/*
	 * This can be optimised... We know we want to go from SVC32
	 * mode to UND32 mode
	 */
        mrs	r3, cpsr
	bic	r2, r3, #(PSR_MODE)
	orr	r2, r2, #(PSR_UND32_MODE)
        msr	cpsr_c, r2

	ldr	sp, [r9, #(PCB_UND_SP)]

        msr	cpsr_c, r3		/* Restore the old mode */

	/* Restore all the save registers */
	add	r7, r9, #PCB_R8
	ldmia	r7, {r8-r13}

	sub	r7, r7, #PCB_R8		/* restore PCB pointer */

	ldr	r5, [r6, #(L_PROC)]	/* fetch the proc for below */

	/* rem: r4 = return value */
	/* rem: r5 = new lwp's proc */
	/* rem: r6 = new lwp */
	/* rem: r7 = new pcb */

#ifdef ARMFPE
	add	r0, r7, #(USER_SIZE) & 0x00ff
	add	r0, r0, #(USER_SIZE) & 0xff00
	bl	_C_LABEL(arm_fpe_core_changecontext)
#endif

	/* We can enable interrupts again */
	IRQenableALL

	/* rem: r4 = return value */
	/* rem: r5 = new lwp's proc */
	/* rem: r6 = new lwp */
	/* rem: r7 = new PCB */

	/*
	 * Check for restartable atomic sequences (RAS).
	 */

	ldr	r2, [r5, #(P_NRAS)]
	ldr	r4, [r7, #(PCB_TF)]	/* r4 = trapframe (used below) */
	teq	r2, #0			/* p->p_nras == 0? */
	bne	.Lswitch_do_ras		/* no, check for one */

.Lswitch_return:

	/* Get the spl level from the stack and update the current spl level */
	ldr	r0, [sp], #0x0004
	bl	_C_LABEL(splx)

	/* cpu_switch returns 1 == switched, 0 == didn't switch */
	mov	r0, r4

	/*
	 * Pull the registers that got pushed when either savectx() or
	 * cpu_switch() was called and return.
	 */
	ldmfd	sp!, {r4-r7, pc}

.Lswitch_do_ras:
	ldr	r1, [r4, #(TF_PC)]	/* second ras_lookup() arg */
	mov	r0, r5			/* first ras_lookup() arg */
	bl	_C_LABEL(ras_lookup)
	cmn	r0, #1			/* -1 means "not in a RAS" */
	strne	r0, [r4, #(TF_PC)]
	b	.Lswitch_return

.Lswitch_exited:
	/*
	 * We skip the cache purge because switch_exit() already did it.
	 * Load up registers the way .Lcs_cache_purge_skipped expects.
	 * Userpsace access already blocked by switch_exit().
	 */
	ldr	r9, [r6, #(L_ADDR)]		/* r9 = new PCB */
	ldr	r3, .Lblock_userspace_access
	mrc	p15, 0, r10, c2, c0, 0		/* r10 = old L1 */
	mov	r5, #0				/* No previous cache state */
	ldr	r1, [r9, #(PCB_DACR)]		/* r1 = new DACR */
	ldr	r8, [r9, #(PCB_CSTATE)]		/* r8 = new cache state */
	ldr	r11, [r9, #(PCB_PAGEDIR)]	/* r11 = new L1 */
	b	.Lcs_cache_purge_skipped

/*
 * cpu_switchto(struct lwp *current, struct lwp *next)
 * Switch to the specified next LWP
 * Arguments:
 *
 *	r0	'struct lwp *' of the current LWP
 *	r1	'struct lwp *' of the LWP to switch to
 */
ENTRY(cpu_switchto)
	stmfd	sp!, {r4-r7, lr}

	/* Lower the spl level to spl0 and get the current spl level. */
	mov	r6, r0		/* save old lwp */
	mov	r5, r1		/* save new lwp */

#if defined(LOCKDEBUG)
	/* release the sched_lock before handling interrupts */
	bl	_C_LABEL(sched_unlock_idle)
#endif

#ifdef __NEWINTR
	mov	r0, #(IPL_NONE)
	bl	_C_LABEL(_spllower)
#else /* ! __NEWINTR */
#ifdef spl0
	mov	r0, #(_SPL_0)
	bl	_C_LABEL(splx)
#else
	bl	_C_LABEL(spl0)
#endif /* spl0 */
#endif /* __NEWINTR */

	/* Push the old spl level onto the stack */
	str	r0, [sp, #-0x0004]!

	IRQdisable
#if defined(LOCKDEBUG)
	bl	_C_LABEL(sched_lock_idle)
#endif

	mov	r0, r6		/* restore old lwp */
	mov	r1, r5		/* restore new lwp */

	/* rem: r0 = old lwp */
	/* rem: r1 = new lwp */
	/* rem: interrupts are disabled */

	/*
	 * Okay, set up registers the way cpu_switch() wants them,
	 * and jump into the middle of it (where we bring up the
	 * new process).
	 */
	mov	r6, r1			/* r6 = new lwp */
#if defined(LOCKDEBUG)
	mov	r5, r0			/* preserve old lwp */
	bl	_C_LABEL(sched_unlock_idle)
	mov	r1, r5			/* r1 = old lwp */
#else
	mov	r1, r0			/* r1 = old lwp */
#endif
	b	.Lswitch_resume

/*
 * void switch_exit(struct lwp *l, struct lwp *l0, void (*exit)(struct lwp *));
 * Switch to lwp0's saved context and deallocate the address space and kernel
 * stack for l.  Then jump into cpu_switch(), as if we were in lwp0 all along.
 */

/* LINTSTUB: Func: void switch_exit(struct lwp *l, struct lwp *l0, void (*func)(struct lwp *)) */
ENTRY(switch_exit)
	/*
	 * The process is going away, so we can use callee-saved
	 * registers here without having to save them.
	 */

	mov	r4, r0
	ldr	r0, .Lcurlwp

	mov	r5, r1
	ldr	r1, .Lblock_userspace_access

	mov	r6, r2

	/*
	 * r4 = lwp
	 * r5 = lwp0
	 * r6 = exit func
	 */

	mov	r2, #0x00000000		/* curlwp = NULL */
	str	r2, [r0]

	/*
	 * We're about to clear both the cache and the TLB.
	 * Make sure to zap the 'last cache state' pointer since the
	 * pmap might be about to go away. Also ensure the outgoing
	 * VM space's cache state is marked as NOT resident in the
	 * cache, and that lwp0's cache state IS resident.
	 */
	ldr	r7, [r4, #(L_ADDR)]		/* r7 = old lwp's PCB */
	ldr	r0, .Llast_cache_state_ptr	/* Last userland cache state */
	ldr	r9, [r7, #(PCB_CSTATE)]		/* Fetch cache state pointer */
	ldr	r3, [r5, #(L_ADDR)]		/* r3 = lwp0's PCB */
	str	r2, [r0]			/* No previous cache state */
	str	r2, [r9, #(CS_ALL)]		/* Zap old lwp's cache state */
	ldr	r3, [r3, #(PCB_CSTATE)]		/* lwp0's cache state */
	mov	r2, #-1
	str	r2, [r3, #(CS_ALL)]		/* lwp0 is in da cache! */

	/*
	 * Don't allow user space access between the purge and the switch.
	 */
	mov	r2, #0x00000001
	str	r2, [r1]

	/* Switch to lwp0 context */

	ldr	r9, .Lcpufuncs
	mov	lr, pc
	ldr	pc, [r9, #CF_IDCACHE_WBINV_ALL]

	ldr	r0, [r7, #(PCB_PL1VEC)]
	ldr	r1, [r7, #(PCB_DACR)]

	/*
	 * r0 = Pointer to L1 slot for vector_page (or NULL)
	 * r1 = lwp0's DACR
	 * r4 = lwp we're switching from
	 * r5 = lwp0
	 * r6 = exit func
	 * r7 = lwp0's PCB
	 * r9 = cpufuncs
	 */

	IRQdisableALL

	/*
	 * Ensure the vector table is accessible by fixing up lwp0's L1
	 */
	cmp	r0, #0			/* No need to fixup vector table? */
	ldrne	r3, [r0]		/* But if yes, fetch current value */
	ldrne	r2, [r7, #(PCB_L1VEC)]	/* Fetch new vector_page value */
	mcr	p15, 0, r1, c3, c0, 0	/* Update DACR for lwp0's context */
	cmpne	r3, r2			/* Stuffing the same value? */
	strne	r2, [r0]		/* Store if not. */

#ifdef PMAP_INCLUDE_PTE_SYNC
	/*
	 * Need to sync the cache to make sure that last store is
	 * visible to the MMU.
	 */
	movne	r1, #4
	movne	lr, pc
	ldrne	pc, [r9, #CF_DCACHE_WB_RANGE]
#endif /* PMAP_INCLUDE_PTE_SYNC */

	/*
	 * Note: We don't do the same optimisation as cpu_switch() with
	 * respect to avoiding flushing the TLB if we're switching to
	 * the same L1 since this process' VM space may be about to go
	 * away, so we don't want *any* turds left in the TLB.
	 */

	/* Switch the memory to the new process */
	ldr	r0, [r7, #(PCB_PAGEDIR)]
	mov	lr, pc
	ldr	pc, [r9, #CF_CONTEXT_SWITCH]

	ldr	r0, .Lcurpcb

	/* Restore all the save registers */
	add	r1, r7, #PCB_R8
	ldmia	r1, {r8-r13}

	str	r7, [r0]	/* curpcb = lwp0's PCB */

	IRQenableALL

	/*
	 * Schedule the vmspace and stack to be freed.
	 */
	mov	r0, r4			/* {lwp_}exit2(l) */
	mov	lr, pc
	mov	pc, r6

	ldr	r7, .Lwhichqs		/* r7 = &whichqs */
	mov	r5, #0x00000000		/* r5 = old lwp = NULL */
	b	.Lswitch_search

/* LINTSTUB: Func: void savectx(struct pcb *pcb) */
ENTRY(savectx)
	/*
	 * r0 = pcb
	 */

	/* Push registers.*/
	stmfd	sp!, {r4-r7, lr}

	/* Store all the registers in the process's pcb */
	add	r2, r0, #(PCB_R8)
	stmia	r2, {r8-r13}

	/* Pull the regs of the stack */
	ldmfd	sp!, {r4-r7, pc}

ENTRY(proc_trampoline)
#ifdef MULTIPROCESSOR
	bl	_C_LABEL(proc_trampoline_mp)
#endif
	mov	r0, r5
	mov	r1, sp
	mov	lr, pc
	mov	pc, r4

	/* Kill irq's */
        mrs     r0, cpsr
        orr     r0, r0, #(I32_bit)
        msr     cpsr_c, r0

	PULLFRAME

	movs	pc, lr			/* Exit */

	.type .Lcpu_switch_ffs_table, _ASM_TYPE_OBJECT;
.Lcpu_switch_ffs_table:
/* same as ffs table but all nums are -1 from that */
/*               0   1   2   3   4   5   6   7           */
	.byte	 0,  0,  1, 12,  2,  6,  0, 13  /*  0- 7 */
	.byte	 3,  0,  7,  0,  0,  0,  0, 14  /*  8-15 */
	.byte	10,  4,  0,  0,  8,  0,  0, 25  /* 16-23 */
	.byte	 0,  0,  0,  0,  0, 21, 27, 15  /* 24-31 */
	.byte	31, 11,  5,  0,  0,  0,  0,  0	/* 32-39 */
	.byte	 9,  0,  0, 24,  0,  0, 20, 26  /* 40-47 */
	.byte	30,  0,  0,  0,  0, 23,  0, 19  /* 48-55 */
	.byte   29,  0, 22, 18, 28, 17, 16,  0  /* 56-63 */