cpuswitch.S revision 1.15 1 /* $NetBSD: cpuswitch.S,v 1.15 2002/08/16 15:25:54 thorpej Exp $ */
2
3 /*
4 * Copyright (c) 1994-1998 Mark Brinicombe.
5 * Copyright (c) 1994 Brini.
6 * All rights reserved.
7 *
8 * This code is derived from software written for Brini by Mark Brinicombe
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by Brini.
21 * 4. The name of the company nor the name of the author may be used to
22 * endorse or promote products derived from this software without specific
23 * prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
26 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
27 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
29 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
30 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
31 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 *
37 * RiscBSD kernel project
38 *
39 * cpuswitch.S
40 *
41 * cpu switching functions
42 *
43 * Created : 15/10/94
44 */
45
46 #include "opt_armfpe.h"
47
48 #include "assym.h"
49 #include <machine/param.h>
50 #include <machine/cpu.h>
51 #include <machine/frame.h>
52 #include <machine/asm.h>
53
54 #undef IRQdisable
55 #undef IRQenable
56
57 /*
58 * New experimental definitions of IRQdisable and IRQenable
59 * These keep FIQ's enabled since FIQ's are special.
60 */
61
62 #define IRQdisable \
63 mrs r14, cpsr ; \
64 orr r14, r14, #(I32_bit) ; \
65 msr cpsr_c, r14 ; \
66
67 #define IRQenable \
68 mrs r14, cpsr ; \
69 bic r14, r14, #(I32_bit) ; \
70 msr cpsr_c, r14 ; \
71
72 /*
73 * setrunqueue() and remrunqueue()
74 *
75 * Functions to add and remove a process for the run queue.
76 */
77
78 .text
79
80 Lwhichqs:
81 .word _C_LABEL(sched_whichqs)
82
83 Lqs:
84 .word _C_LABEL(sched_qs)
85
86 /*
87 * On entry
88 * r0 = process
89 */
90
91 ENTRY(setrunqueue)
92 /*
93 * Local register usage
94 * r0 = process
95 * r1 = queue
96 * r2 = &qs[queue] and temp
97 * r3 = temp
98 * r12 = whichqs
99 */
100 #ifdef DIAGNOSTIC
101 ldr r1, [r0, #(P_BACK)]
102 teq r1, #0x00000000
103 bne .Lsetrunqueue_erg
104
105 ldr r1, [r0, #(P_WCHAN)]
106 teq r1, #0x00000000
107 bne .Lsetrunqueue_erg
108 #endif
109
110 /* Get the priority of the queue */
111 ldrb r1, [r0, #(P_PRIORITY)]
112
113 /* Indicate that there is a process on this queue */
114 ldr r12, Lwhichqs
115 mov r1, r1, lsr #2
116 ldr r2, [r12]
117 mov r3, #0x00000001
118 mov r3, r3, lsl r1
119 orr r2, r2, r3
120 str r2, [r12]
121
122 /* Get the address of the queue */
123 ldr r2, Lqs
124 add r1, r2, r1, lsl # 3
125
126 /* Hook the process in */
127 str r1, [r0, #(P_FORW)]
128 ldr r2, [r1, #(P_BACK)]
129
130 str r0, [r1, #(P_BACK)]
131 #ifdef DIAGNOSTIC
132 teq r2, #0x00000000
133 beq .Lsetrunqueue_erg
134 #endif
135 str r0, [r2, #(P_FORW)]
136 str r2, [r0, #(P_BACK)]
137
138 mov pc, lr
139
140 #ifdef DIAGNOSTIC
141 .Lsetrunqueue_erg:
142 mov r2, r1
143 mov r1, r0
144 add r0, pc, #Ltext1 - . - 8
145 bl _C_LABEL(printf)
146
147 ldr r2, Lqs
148 ldr r1, [r2]
149 add r0, pc, #Ltext2 - . - 8
150 b _C_LABEL(panic)
151
152 Ltext1:
153 .asciz "setrunqueue : %08x %08x\n"
154 Ltext2:
155 .asciz "setrunqueue : [qs]=%08x qs=%08x\n"
156 .align 0
157 #endif
158
159 /*
160 * On entry
161 * r0 = process
162 */
163
164 ENTRY(remrunqueue)
165 /*
166 * Local register usage
167 * r0 = oldproc
168 * r1 = queue
169 * r2 = &qs[queue] and scratch
170 * r3 = scratch
171 * r12 = whichqs
172 */
173
174 /* Get the priority of the queue */
175 ldrb r1, [r0, #(P_PRIORITY)]
176 mov r1, r1, lsr #2
177
178 /* Unhook the process */
179 ldr r2, [r0, #(P_FORW)]
180 ldr r3, [r0, #(P_BACK)]
181
182 str r3, [r2, #(P_BACK)]
183 str r2, [r3, #(P_FORW)]
184
185 /* If the queue is now empty clear the queue not empty flag */
186 teq r2, r3
187
188 /* This could be reworked to avoid the use of r4 */
189 ldreq r12, Lwhichqs
190 moveq r3, #0x00000001
191 ldreq r2, [r12]
192 moveq r3, r3, lsl r1
193 biceq r2, r2, r3
194 streq r2, [r12]
195
196 /* Remove the back pointer for the process */
197 mov r1, #0x00000000
198 str r1, [r0, #(P_BACK)]
199
200 mov pc, lr
201
202
203 /*
204 * cpuswitch()
205 *
206 * preforms a process context switch.
207 * This function has several entry points
208 */
209
210 Lcurproc:
211 .word _C_LABEL(curproc)
212
213 Lcurpcb:
214 .word _C_LABEL(curpcb)
215
216 Lwant_resched:
217 .word _C_LABEL(want_resched)
218
219 Lcpufuncs:
220 .word _C_LABEL(cpufuncs)
221
222 .data
223 .global _C_LABEL(curpcb)
224 _C_LABEL(curpcb):
225 .word 0x00000000
226 .text
227
228 Lblock_userspace_access:
229 .word _C_LABEL(block_userspace_access)
230
231 .Lcpu_do_powersave:
232 .word _C_LABEL(cpu_do_powersave)
233
234 /*
235 * Idle loop, exercised while waiting for a process to wake up.
236 */
237 /* LINTSTUB: Ignore */
238 ASENTRY_NP(idle)
239 #if defined(LOCKDEBUG)
240 bl _C_LABEL(sched_unlock_idle)
241 #endif
242 /* Enable interrupts */
243 IRQenable
244
245 /* If we don't want to sleep, use a simpler loop. */
246 ldr r3, .Lcpu_do_powersave
247 ldr r7, Lwhichqs /* r7 = &whichqs */
248 ldr r3, [r3]
249 teq r3, #0
250 beq .Lidle_nosleep
251
252 /* Powersave idle. */
253 ldr r4, Lcpufuncs
254 .Lidle_sleep:
255 ldr r3, [r7]
256 teq r3, #0x00000000
257 bne .Lswitch_search
258
259 /* if saving power, don't want to pageidlezero */
260 mov r0, #0
261 add lr, pc, #.Lidle_sleep - . - 8
262 ldr pc, [r4, #(CF_SLEEP)]
263 /* loops back around */
264
265 /* Non-powersave idle. */
266 .Lidle_nosleep:
267 /* should maybe do uvm pageidlezero stuff here */
268 ldr r3, [r7]
269 teq r3, #0x00000000
270 bne .Lswitch_search
271 b .Lidle_nosleep
272
273 /*
274 * Find a new process to run, save the current context and
275 * load the new context
276 */
277
278 ENTRY(cpu_switch)
279 /*
280 * Local register usage. Some of these registers are out of date.
281 * r1 = oldproc
282 * r2 = spl level
283 * r3 = whichqs
284 * r4 = queue
285 * r5 = &qs[queue]
286 * r6 = newproc
287 * r7 = scratch
288 */
289 stmfd sp!, {r4-r7, lr}
290
291 /*
292 * Get the current process and indicate that there is no longer
293 * a valid process (curproc = 0). Zero the current PCB pointer
294 * while we're at it.
295 */
296 ldr r7, Lcurproc
297 ldr r6, Lcurpcb
298 mov r0, #0x00000000
299 ldr r1, [r7] /* r1 = curproc */
300 str r0, [r7] /* curproc = NULL */
301 str r0, [r6] /* curpcb = NULL */
302
303 /* stash the old proc while we call functions */
304 mov r5, r1
305
306 #if defined(LOCKDEBUG)
307 /* release the sched_lock before handling interrupts */
308 bl _C_LABEL(sched_unlock_idle)
309 #endif
310
311 /* Lower the spl level to spl0 and get the current spl level. */
312 #ifdef __NEWINTR
313 mov r0, #(IPL_NONE)
314 bl _C_LABEL(_spllower)
315 #else /* ! __NEWINTR */
316 #ifdef spl0
317 mov r0, #(_SPL_0)
318 bl _C_LABEL(splx)
319 #else
320 bl _C_LABEL(spl0)
321 #endif /* spl0 */
322 #endif /* __NEWINTR */
323
324 /* Push the old spl level onto the stack */
325 str r0, [sp, #-0x0004]!
326
327 /* First phase : find a new process */
328
329 /* rem: r5 = old proc */
330
331 .Lswitch_search:
332 IRQdisable
333 #if defined(LOCKDEBUG)
334 bl _C_LABEL(sched_lock_idle)
335 #endif
336
337 /* Do we have any active queues */
338 ldr r7, Lwhichqs
339 ldr r3, [r7]
340
341 /* If not we must idle until we do. */
342 teq r3, #0x00000000
343 beq _ASM_LABEL(idle)
344
345 /* put old proc back in r1 */
346 mov r1, r5
347
348 /* rem: r1 = old proc */
349 /* rem: r3 = whichqs */
350 /* rem: interrupts are disabled */
351
352 /*
353 * We have found an active queue. Currently we do not know which queue
354 * is active just that one of them is.
355 */
356 /* this is the ffs algorithm devised by d.seal and posted to
357 * comp.sys.arm on 16 Feb 1994.
358 */
359 rsb r5, r3, #0
360 ands r0, r3, r5
361
362 adr r5, Lcpu_switch_ffs_table
363
364 /* X = R0 */
365 orr r4, r0, r0, lsl #4 /* r4 = X * 0x11 */
366 orr r4, r4, r4, lsl #6 /* r4 = X * 0x451 */
367 rsb r4, r4, r4, lsl #16 /* r4 = X * 0x0450fbaf */
368
369 /* used further down, saves SA stall */
370 ldr r6, Lqs
371
372 /* now lookup in table indexed on top 6 bits of a4 */
373 ldrb r4, [ r5, r4, lsr #26 ]
374
375 /* rem: r0 = bit mask of chosen queue (1 << r4) */
376 /* rem: r1 = old proc */
377 /* rem: r3 = whichqs */
378 /* rem: r4 = queue number */
379 /* rem: interrupts are disabled */
380
381 /* Get the address of the queue (&qs[queue]) */
382 add r5, r6, r4, lsl #3
383
384 /*
385 * Get the process from the queue and place the next process in
386 * the queue at the head. This basically unlinks the process at
387 * the head of the queue.
388 */
389 ldr r6, [r5, #(P_FORW)]
390
391 /* rem: r6 = new process */
392 ldr r7, [r6, #(P_FORW)]
393 str r7, [r5, #(P_FORW)]
394
395 /*
396 * Test to see if the queue is now empty. If the head of the queue
397 * points to the queue itself then there are no more processes in
398 * the queue. We can therefore clear the queue not empty flag held
399 * in r3.
400 */
401
402 teq r5, r7
403 biceq r3, r3, r0
404
405 /* rem: r0 = bit mask of chosen queue (1 << r4) - NOT NEEDED AN MORE */
406
407 /* Fix the back pointer for the process now at the head of the queue. */
408 ldr r0, [r6, #(P_BACK)]
409 str r0, [r7, #(P_BACK)]
410
411 /* Update the RAM copy of the queue not empty flags word. */
412 ldr r7, Lwhichqs
413 str r3, [r7]
414
415 /* rem: r1 = old proc */
416 /* rem: r3 = whichqs - NOT NEEDED ANY MORE */
417 /* rem: r4 = queue number - NOT NEEDED ANY MORE */
418 /* rem: r6 = new process */
419 /* rem: interrupts are disabled */
420
421 /* Clear the want_resched flag */
422 ldr r7, Lwant_resched
423 mov r0, #0x00000000
424 str r0, [r7]
425
426 /*
427 * Clear the back pointer of the process we have removed from
428 * the head of the queue. The new process is isolated now.
429 */
430 str r0, [r6, #(P_BACK)]
431
432 #if defined(LOCKDEBUG)
433 /*
434 * unlock the sched_lock, but leave interrupts off, for now.
435 */
436 mov r7, r1
437 bl _C_LABEL(sched_unlock_idle)
438 mov r1, r7
439 #endif
440
441 /* p->p_cpu initialized in fork1() for single-processor */
442
443 /* Process is now on a processor. */
444 mov r0, #SONPROC /* p->p_stat = SONPROC */
445 strb r0, [r6, #(P_STAT)]
446
447 /* We have a new curproc now so make a note it */
448 ldr r7, Lcurproc
449 str r6, [r7]
450
451 /* Hook in a new pcb */
452 ldr r7, Lcurpcb
453 ldr r0, [r6, #(P_ADDR)]
454 str r0, [r7]
455
456 /* At this point we can allow IRQ's again. */
457 IRQenable
458
459 /* rem: r1 = old proc */
460 /* rem: r6 = new process */
461 /* rem: interrupts are enabled */
462
463 /*
464 * If the new process is the same as the process that called
465 * cpu_switch() then we do not need to save and restore any
466 * contexts. This means we can make a quick exit.
467 * The test is simple if curproc on entry (now in r1) is the
468 * same as the proc removed from the queue we can jump to the exit.
469 */
470 teq r1, r6
471 beq .Lswitch_return
472
473 /* Remember the old process in r0 */
474 mov r0, r1
475
476 /*
477 * If the curproc on entry to cpu_switch was zero then the
478 * process that called it was exiting. This means that we do
479 * not need to save the current context. Instead we can jump
480 * straight to restoring the context for the new process.
481 */
482 teq r0, #0x00000000
483 beq .Lswitch_exited
484
485 /* rem: r0 = old proc */
486 /* rem: r6 = new process */
487 /* rem: interrupts are enabled */
488
489 /* Stage two : Save old context */
490
491 /* Get the user structure for the old process. */
492 ldr r1, [r0, #(P_ADDR)]
493
494 /* Save all the registers in the old process's pcb */
495 add r7, r1, #(PCB_R8)
496 stmia r7, {r8-r13}
497
498 /*
499 * This can be optimised... We know we want to go from SVC32
500 * mode to UND32 mode
501 */
502 mrs r3, cpsr
503 bic r2, r3, #(PSR_MODE)
504 orr r2, r2, #(PSR_UND32_MODE | I32_bit)
505 msr cpsr_c, r2
506
507 str sp, [r1, #(PCB_UND_SP)]
508
509 msr cpsr_c, r3 /* Restore the old mode */
510
511 /* rem: r0 = old proc */
512 /* rem: r1 = old pcb */
513 /* rem: r6 = new process */
514 /* rem: interrupts are enabled */
515
516 /* What else needs to be saved Only FPA stuff when that is supported */
517
518 /* r1 now free! */
519
520 /* Third phase : restore saved context */
521
522 /* rem: r0 = old proc */
523 /* rem: r6 = new process */
524 /* rem: interrupts are enabled */
525
526 /*
527 * Don't allow user space access between the purge and the switch.
528 */
529 ldr r3, Lblock_userspace_access
530 mov r1, #0x00000001
531 mov r2, #0x00000000
532 str r1, [r3]
533
534 stmfd sp!, {r0-r3}
535 ldr r1, Lcpufuncs
536 add lr, pc, #.Lcs_cache_purged - . - 8
537 ldr pc, [r1, #CF_IDCACHE_WBINV_ALL]
538
539 .Lcs_cache_purged:
540 ldmfd sp!, {r0-r3}
541
542 .Lcs_cache_purge_skipped:
543 /* At this point we need to kill IRQ's again. */
544 IRQdisable
545
546 /*
547 * Interrupts are disabled so we can allow user space accesses again
548 * as none will occur until interrupts are re-enabled after the
549 * switch.
550 */
551 str r2, [r3]
552
553 /* Get the user structure for the new process in r1 */
554 ldr r1, [r6, #(P_ADDR)]
555
556 /* Get the pagedir physical address for the process. */
557 ldr r0, [r1, #(PCB_PAGEDIR)]
558
559 /* Switch the memory to the new process */
560 ldr r3, Lcpufuncs
561 add lr, pc, #.Lcs_context_switched - . - 8
562 ldr pc, [r3, #CF_CONTEXT_SWITCH]
563
564 .Lcs_context_switched:
565 /*
566 * This can be optimised... We know we want to go from SVC32
567 * mode to UND32 mode
568 */
569 mrs r3, cpsr
570 bic r2, r3, #(PSR_MODE)
571 orr r2, r2, #(PSR_UND32_MODE)
572 msr cpsr_c, r2
573
574 ldr sp, [r1, #(PCB_UND_SP)]
575
576 msr cpsr_c, r3 /* Restore the old mode */
577
578 /* Restore all the save registers */
579 add r7, r1, #PCB_R8
580 ldmia r7, {r8-r13}
581
582 #ifdef ARMFPE
583 add r0, r1, #(USER_SIZE) & 0x00ff
584 add r0, r0, #(USER_SIZE) & 0xff00
585 bl _C_LABEL(arm_fpe_core_changecontext)
586 #endif
587
588 /* We can enable interrupts again */
589 IRQenable
590
591 .Lswitch_return:
592
593 /* Get the spl level from the stack and update the current spl level */
594 ldr r0, [sp], #0x0004
595 bl _C_LABEL(splx)
596
597 /* cpu_switch returns the proc it switched to. */
598 mov r0, r6
599
600 /*
601 * Pull the registers that got pushed when either savectx() or
602 * cpu_switch() was called and return.
603 */
604 ldmfd sp!, {r4-r7, pc}
605
606 .Lswitch_exited:
607 /*
608 * We skip the cache purge because switch_exit() already did
609 * it. Load up registers the way Lcs_cache_purge_skipped
610 * expects. Userspace access already blocked in switch_exit().
611 */
612 ldr r3, Lblock_userspace_access
613 mov r2, #0x00000000
614 b .Lcs_cache_purge_skipped
615
616 Lproc0:
617 .word _C_LABEL(proc0)
618
619 Lkernel_map:
620 .word _C_LABEL(kernel_map)
621
622 /*
623 * void switch_exit(struct proc *p, struct proc *p0);
624 * Switch to proc0's saved context and deallocate the address space and kernel
625 * stack for p. Then jump into cpu_switch(), as if we were in proc0 all along.
626 */
627
628 /* LINTSTUB: Func: void switch_exit(struct proc *p, struct proc *p0) */
629 ENTRY(switch_exit)
630 /*
631 * r0 = proc
632 * r1 = proc0
633 */
634
635 mov r3, r0
636
637 /* In case we fault */
638 ldr r0, Lcurproc
639 mov r2, #0x00000000
640 str r2, [r0]
641
642 /* ldr r0, Lcurpcb
643 str r2, [r0]*/
644
645 /*
646 * Don't allow user space access between the purge and the switch.
647 */
648 ldr r0, Lblock_userspace_access
649 mov r2, #0x00000001
650 str r2, [r0]
651
652 /* Switch to proc0 context */
653
654 stmfd sp!, {r0-r3}
655
656 ldr r0, Lcpufuncs
657 add lr, pc, #.Lse_cache_purged - . - 8
658 ldr pc, [r0, #CF_IDCACHE_WBINV_ALL]
659
660 .Lse_cache_purged:
661 ldmfd sp!, {r0-r3}
662
663 IRQdisable
664
665 ldr r2, [r1, #(P_ADDR)]
666 ldr r0, [r2, #(PCB_PAGEDIR)]
667
668 /* Switch the memory to the new process */
669 ldr r4, Lcpufuncs
670 add lr, pc, #.Lse_context_switched - . - 8
671 ldr pc, [r4, #CF_CONTEXT_SWITCH]
672
673 .Lse_context_switched:
674 /* Restore all the save registers */
675 add r7, r2, #PCB_R8
676 ldmia r7, {r8-r13}
677
678 /* This is not really needed ! */
679 /* Yes it is for the su and fu routines */
680 ldr r0, Lcurpcb
681 str r2, [r0]
682
683 IRQenable
684
685 /* str r3, [sp, #-0x0004]!*/
686
687 /*
688 * Schedule the vmspace and stack to be freed.
689 */
690 mov r0, r3 /* exit2(p) */
691 bl _C_LABEL(exit2)
692
693 /* Paranoia */
694 ldr r1, Lcurproc
695 mov r0, #0x00000000
696 str r0, [r1]
697
698 mov r5, #0x00000000 /* r5 = old proc = NULL */
699 b .Lswitch_search
700
701 /* LINTSTUB: Func: void savectx(struct pcb *pcb) */
702 ENTRY(savectx)
703 /*
704 * r0 = pcb
705 */
706
707 /* Push registers.*/
708 stmfd sp!, {r4-r7, lr}
709
710 /* Store all the registers in the process's pcb */
711 add r2, r0, #(PCB_R8)
712 stmia r2, {r8-r13}
713
714 /* Pull the regs of the stack */
715 ldmfd sp!, {r4-r7, pc}
716
717 ENTRY(proc_trampoline)
718 add lr, pc, #(.Ltrampoline_return - . - 8)
719 mov r0, r5
720 mov r1, sp
721 mov pc, r4
722
723 .Ltrampoline_return:
724 /* Kill irq's */
725 mrs r0, cpsr
726 orr r0, r0, #(I32_bit)
727 msr cpsr_c, r0
728
729 PULLFRAME
730
731 movs pc, lr /* Exit */
732
733 .type Lcpu_switch_ffs_table, _ASM_TYPE_OBJECT;
734 Lcpu_switch_ffs_table:
735 /* same as ffs table but all nums are -1 from that */
736 /* 0 1 2 3 4 5 6 7 */
737 .byte 0, 0, 1, 12, 2, 6, 0, 13 /* 0- 7 */
738 .byte 3, 0, 7, 0, 0, 0, 0, 14 /* 8-15 */
739 .byte 10, 4, 0, 0, 8, 0, 0, 25 /* 16-23 */
740 .byte 0, 0, 0, 0, 0, 21, 27, 15 /* 24-31 */
741 .byte 31, 11, 5, 0, 0, 0, 0, 0 /* 32-39 */
742 .byte 9, 0, 0, 24, 0, 0, 20, 26 /* 40-47 */
743 .byte 30, 0, 0, 0, 0, 23, 0, 19 /* 48-55 */
744 .byte 29, 0, 22, 18, 28, 17, 16, 0 /* 56-63 */
745
746 /* End of cpuswitch.S */
747