cpu.c revision 1.4 1 /* $NetBSD: cpu.c,v 1.4 2007/10/18 15:28:38 yamt Exp $ */
2
3 /*-
4 * Copyright (c) 2000 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by RedBack Networks Inc.
9 *
10 * Author: Bill Sommerfeld
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by the NetBSD
23 * Foundation, Inc. and its contributors.
24 * 4. Neither the name of The NetBSD Foundation nor the names of its
25 * contributors may be used to endorse or promote products derived
26 * from this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
29 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
30 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
31 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
32 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 * POSSIBILITY OF SUCH DAMAGE.
39 */
40
41 /*
42 * Copyright (c) 1999 Stefan Grefen
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. All advertising materials mentioning features or use of this software
53 * must display the following acknowledgement:
54 * This product includes software developed by the NetBSD
55 * Foundation, Inc. and its contributors.
56 * 4. Neither the name of The NetBSD Foundation nor the names of its
57 * contributors may be used to endorse or promote products derived
58 * from this software without specific prior written permission.
59 *
60 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
61 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
62 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
63 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
64 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
65 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
66 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
67 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
68 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
69 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
70 * SUCH DAMAGE.
71 */
72
73 #include <sys/cdefs.h>
74 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.4 2007/10/18 15:28:38 yamt Exp $");
75
76 #include "opt_ddb.h"
77 #include "opt_multiprocessor.h"
78 #include "opt_mpbios.h" /* for MPDEBUG */
79 #include "opt_mtrr.h"
80
81 #include "lapic.h"
82 #include "ioapic.h"
83
84 #include <sys/param.h>
85 #include <sys/proc.h>
86 #include <sys/user.h>
87 #include <sys/systm.h>
88 #include <sys/device.h>
89 #include <sys/malloc.h>
90
91 #include <uvm/uvm_extern.h>
92
93 #include <machine/cpu.h>
94 #include <machine/cpufunc.h>
95 #include <machine/cpuvar.h>
96 #include <machine/pmap.h>
97 #include <machine/vmparam.h>
98 #include <machine/mpbiosvar.h>
99 #include <machine/pcb.h>
100 #include <machine/specialreg.h>
101 #include <machine/segments.h>
102 #include <machine/gdt.h>
103 #include <machine/mtrr.h>
104 #include <machine/pio.h>
105
106 #ifdef i386
107 #include <machine/tlog.h>
108 #endif
109
110 #if NLAPIC > 0
111 #include <machine/apicvar.h>
112 #include <machine/i82489reg.h>
113 #include <machine/i82489var.h>
114 #endif
115
116 #if NIOAPIC > 0
117 #include <machine/i82093var.h>
118 #endif
119
120 #include <dev/ic/mc146818reg.h>
121 #include <i386/isa/nvram.h>
122 #include <dev/isa/isareg.h>
123
124 int cpu_match(struct device *, struct cfdata *, void *);
125 void cpu_attach(struct device *, struct device *, void *);
126
127 struct cpu_softc {
128 struct device sc_dev; /* device tree glue */
129 struct cpu_info *sc_info; /* pointer to CPU info */
130 };
131
132 int mp_cpu_start(struct cpu_info *);
133 void mp_cpu_start_cleanup(struct cpu_info *);
134 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
135 mp_cpu_start_cleanup };
136
137
138 CFATTACH_DECL(cpu, sizeof(struct cpu_softc),
139 cpu_match, cpu_attach, NULL, NULL);
140
141 /*
142 * Statically-allocated CPU info for the primary CPU (or the only
143 * CPU, on uniprocessors). The CPU info list is initialized to
144 * point at it.
145 */
146 #ifdef TRAPLOG
147 struct tlog tlog_primary;
148 #endif
149 struct cpu_info cpu_info_primary = {
150 .ci_dev = 0,
151 .ci_self = &cpu_info_primary,
152 .ci_idepth = -1,
153 .ci_curlwp = &lwp0,
154 #ifdef TRAPLOG
155 .ci_tlog_base = &tlog_primary,
156 #endif /* !TRAPLOG */
157 };
158
159 struct cpu_info *cpu_info_list = &cpu_info_primary;
160
161 static void cpu_set_tss_gates(struct cpu_info *ci);
162
163 #ifdef i386
164 static void cpu_init_tss(struct i386tss *, void *, void *);
165 #endif
166
167 uint32_t cpus_attached = 0;
168
169 extern char x86_64_doubleflt_stack[];
170
171 #ifdef MULTIPROCESSOR
172 /*
173 * Array of CPU info structures. Must be statically-allocated because
174 * curproc, etc. are used early.
175 */
176 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary, };
177
178 uint32_t cpus_running = 0;
179
180 void cpu_hatch(void *);
181 static void cpu_boot_secondary(struct cpu_info *ci);
182 static void cpu_start_secondary(struct cpu_info *ci);
183 static void cpu_copy_trampoline(void);
184
185 /*
186 * Runs once per boot once multiprocessor goo has been detected and
187 * the local APIC on the boot processor has been mapped.
188 *
189 * Called from lapic_boot_init() (from mpbios_scan()).
190 */
191 void
192 cpu_init_first()
193 {
194 int cpunum = lapic_cpu_number();
195
196 if (cpunum != 0) {
197 cpu_info[0] = NULL;
198 cpu_info[cpunum] = &cpu_info_primary;
199 }
200
201 cpu_info_primary.ci_cpuid = cpunum;
202 cpu_copy_trampoline();
203 }
204 #endif
205
206 int
207 cpu_match(struct device *parent, struct cfdata *match,
208 void *aux)
209 {
210
211 return 1;
212 }
213
214 static void
215 cpu_vm_init(struct cpu_info *ci)
216 {
217 int ncolors = 2, i;
218
219 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
220 struct x86_cache_info *cai;
221 int tcolors;
222
223 cai = &ci->ci_cinfo[i];
224
225 tcolors = atop(cai->cai_totalsize);
226 switch(cai->cai_associativity) {
227 case 0xff:
228 tcolors = 1; /* fully associative */
229 break;
230 case 0:
231 case 1:
232 break;
233 default:
234 tcolors /= cai->cai_associativity;
235 }
236 ncolors = max(ncolors, tcolors);
237 }
238
239 /*
240 * Knowing the size of the largest cache on this CPU, re-color
241 * our pages.
242 */
243 if (ncolors <= uvmexp.ncolors)
244 return;
245 aprint_verbose("%s: %d page colors\n", ci->ci_dev->dv_xname, ncolors);
246 uvm_page_recolor(ncolors);
247 }
248
249
250 void
251 cpu_attach(struct device *parent, struct device *self, void *aux)
252 {
253 struct cpu_softc *sc = (void *) self;
254 struct cpu_attach_args *caa = aux;
255 struct cpu_info *ci;
256 #if defined(MULTIPROCESSOR)
257 int cpunum = caa->cpu_number;
258 #endif
259
260 /*
261 * If we're an Application Processor, allocate a cpu_info
262 * structure, otherwise use the primary's.
263 */
264 if (caa->cpu_role == CPU_ROLE_AP) {
265 aprint_naive(": Application Processor\n");
266 ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK);
267 memset(ci, 0, sizeof(*ci));
268 #if defined(MULTIPROCESSOR)
269 if (cpu_info[cpunum] != NULL) {
270 printf("\n");
271 panic("cpu at apic id %d already attached?", cpunum);
272 }
273 cpu_info[cpunum] = ci;
274 #endif
275 #ifdef TRAPLOG
276 ci->ci_tlog_base = malloc(sizeof(struct tlog),
277 M_DEVBUF, M_WAITOK);
278 #endif
279 } else {
280 aprint_naive(": %s Processor\n",
281 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
282 ci = &cpu_info_primary;
283 #if defined(MULTIPROCESSOR)
284 if (cpunum != lapic_cpu_number()) {
285 printf("\n");
286 panic("%s: running CPU is at apic %d"
287 " instead of at expected %d",
288 sc->sc_dev.dv_xname, lapic_cpu_number(), cpunum);
289 }
290 #endif
291 }
292
293 ci->ci_self = ci;
294 sc->sc_info = ci;
295
296 ci->ci_dev = self;
297 ci->ci_apicid = caa->cpu_number;
298 #ifdef MULTIPROCESSOR
299 ci->ci_cpuid = ci->ci_apicid;
300 #else
301 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */
302 #endif
303 ci->ci_cpumask = (1 << ci->ci_cpuid);
304 ci->ci_func = caa->cpu_func;
305
306 if (caa->cpu_role == CPU_ROLE_AP) {
307 #ifdef MULTIPROCESSOR
308 int error;
309
310 error = mi_cpu_attach(ci);
311 if (error != 0) {
312 aprint_normal("\n");
313 aprint_error("%s: mi_cpu_attach failed with %d\n",
314 sc->sc_dev.dv_xname, error);
315 return;
316 }
317 #endif
318 } else {
319 KASSERT(ci->ci_data.cpu_idlelwp != NULL);
320 }
321
322 pmap_reference(pmap_kernel());
323 ci->ci_pmap = pmap_kernel();
324 ci->ci_tlbstate = TLBSTATE_STALE;
325
326 /* further PCB init done later. */
327
328 switch (caa->cpu_role) {
329 case CPU_ROLE_SP:
330 aprint_normal(": (uniprocessor)\n");
331 ci->ci_flags |= CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY;
332 cpu_intr_init(ci);
333 identifycpu(ci);
334 cpu_init(ci);
335 cpu_set_tss_gates(ci);
336 pmap_cpu_init_late(ci);
337 break;
338
339 case CPU_ROLE_BP:
340 aprint_normal(": (boot processor)\n");
341 ci->ci_flags |= CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY;
342 cpu_intr_init(ci);
343 identifycpu(ci);
344 cpu_init(ci);
345 cpu_set_tss_gates(ci);
346 pmap_cpu_init_late(ci);
347 #if NLAPIC > 0
348 /*
349 * Enable local apic
350 */
351 lapic_enable();
352 lapic_calibrate_timer(ci);
353 #endif
354 #if NIOAPIC > 0
355 ioapic_bsp_id = caa->cpu_number;
356 #endif
357 break;
358
359 case CPU_ROLE_AP:
360 /*
361 * report on an AP
362 */
363 aprint_normal(": (application processor)\n");
364
365 #if defined(MULTIPROCESSOR)
366 cpu_intr_init(ci);
367 gdt_alloc_cpu(ci);
368 cpu_set_tss_gates(ci);
369 pmap_cpu_init_early(ci);
370 pmap_cpu_init_late(ci);
371 cpu_start_secondary(ci);
372 if (ci->ci_flags & CPUF_PRESENT) {
373 identifycpu(ci);
374 ci->ci_next = cpu_info_list->ci_next;
375 cpu_info_list->ci_next = ci;
376 }
377 #else
378 aprint_normal("%s: not started\n", sc->sc_dev.dv_xname);
379 #endif
380 break;
381
382 default:
383 printf("\n");
384 panic("unknown processor type??\n");
385 }
386 cpu_vm_init(ci);
387
388 cpus_attached |= ci->ci_cpumask;
389
390 #if defined(MULTIPROCESSOR)
391 if (mp_verbose) {
392 struct lwp *l = ci->ci_data.cpu_idlelwp;
393
394 aprint_verbose(
395 "%s: idle lwp at %p, idle sp at %p\n",
396 sc->sc_dev.dv_xname, l,
397 #ifdef i386
398 (void *)l->l_addr->u_pcb.pcb_esp
399 #else
400 (void *)l->l_addr->u_pcb.pcb_rsp
401 #endif
402 );
403 }
404 #endif
405 }
406
407 /*
408 * Initialize the processor appropriately.
409 */
410
411 void
412 cpu_init(ci)
413 struct cpu_info *ci;
414 {
415 /* configure the CPU if needed */
416 if (ci->cpu_setup != NULL)
417 (*ci->cpu_setup)(ci);
418
419 #ifdef i386
420 /*
421 * On a 486 or above, enable ring 0 write protection.
422 */
423 if (ci->ci_cpu_class >= CPUCLASS_486)
424 lcr0(rcr0() | CR0_WP);
425 #else
426 lcr0(rcr0() | CR0_WP);
427 #endif
428
429 /*
430 * On a P6 or above, enable global TLB caching if the
431 * hardware supports it.
432 */
433 if (cpu_feature & CPUID_PGE)
434 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */
435
436 /*
437 * If we have FXSAVE/FXRESTOR, use them.
438 */
439 if (cpu_feature & CPUID_FXSR) {
440 lcr4(rcr4() | CR4_OSFXSR);
441
442 /*
443 * If we have SSE/SSE2, enable XMM exceptions.
444 */
445 if (cpu_feature & (CPUID_SSE|CPUID_SSE2))
446 lcr4(rcr4() | CR4_OSXMMEXCPT);
447 }
448
449 #ifdef MTRR
450 /*
451 * On a P6 or above, initialize MTRR's if the hardware supports them.
452 */
453 if (cpu_feature & CPUID_MTRR) {
454 if ((ci->ci_flags & CPUF_AP) == 0)
455 i686_mtrr_init_first();
456 mtrr_init_cpu(ci);
457 }
458
459 #ifdef i386
460 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
461 /*
462 * Must be a K6-2 Step >= 7 or a K6-III.
463 */
464 if (CPUID2FAMILY(ci->ci_signature) == 5) {
465 if (CPUID2MODEL(ci->ci_signature) > 8 ||
466 (CPUID2MODEL(ci->ci_signature) == 8 &&
467 CPUID2STEPPING(ci->ci_signature) >= 7)) {
468 mtrr_funcs = &k6_mtrr_funcs;
469 k6_mtrr_init_first();
470 mtrr_init_cpu(ci);
471 }
472 }
473 }
474 #endif /* i386 */
475 #endif /* MTRR */
476
477 #ifdef MULTIPROCESSOR
478 ci->ci_flags |= CPUF_RUNNING;
479 cpus_running |= ci->ci_cpumask;
480 #endif
481 }
482
483 bool x86_mp_online;
484
485 #ifdef MULTIPROCESSOR
486 void
487 cpu_boot_secondary_processors()
488 {
489 struct cpu_info *ci;
490 u_long i;
491
492 for (i=0; i < X86_MAXPROCS; i++) {
493 ci = cpu_info[i];
494 if (ci == NULL)
495 continue;
496 if (ci->ci_data.cpu_idlelwp == NULL)
497 continue;
498 if ((ci->ci_flags & CPUF_PRESENT) == 0)
499 continue;
500 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
501 continue;
502 cpu_boot_secondary(ci);
503 }
504
505 x86_mp_online = true;
506 }
507
508 static void
509 cpu_init_idle_lwp(struct cpu_info *ci)
510 {
511 struct lwp *l = ci->ci_data.cpu_idlelwp;
512 struct pcb *pcb = &l->l_addr->u_pcb;
513
514 pcb->pcb_cr0 = rcr0();
515 }
516
517 void
518 cpu_init_idle_lwps()
519 {
520 struct cpu_info *ci;
521 u_long i;
522
523 for (i = 0; i < X86_MAXPROCS; i++) {
524 ci = cpu_info[i];
525 if (ci == NULL)
526 continue;
527 if (ci->ci_data.cpu_idlelwp == NULL)
528 continue;
529 if ((ci->ci_flags & CPUF_PRESENT) == 0)
530 continue;
531 cpu_init_idle_lwp(ci);
532 }
533 }
534
535 void
536 cpu_start_secondary(ci)
537 struct cpu_info *ci;
538 {
539 int i;
540 struct pmap *kpm = pmap_kernel();
541 extern paddr_t mp_pdirpa;
542
543 #ifdef __x86_64__
544 /*
545 * The initial PML4 pointer must be below 4G, so if the
546 * current one isn't, use a "bounce buffer"
547 *
548 * XXX move elsewhere, not per CPU.
549 */
550 if (kpm->pm_pdirpa > 0xffffffff) {
551 extern vaddr_t lo32_vaddr;
552 extern paddr_t lo32_paddr;
553 memcpy((void *)lo32_vaddr, kpm->pm_pdir, PAGE_SIZE);
554 mp_pdirpa = lo32_paddr;
555 } else
556 #endif
557 mp_pdirpa = kpm->pm_pdirpa;
558
559 ci->ci_flags |= CPUF_AP;
560
561 aprint_debug("%s: starting\n", ci->ci_dev->dv_xname);
562
563 ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
564 CPU_STARTUP(ci);
565
566 /*
567 * wait for it to become ready
568 */
569 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i>0;i--) {
570 delay(10);
571 }
572 if (! (ci->ci_flags & CPUF_PRESENT)) {
573 aprint_error("%s: failed to become ready\n",
574 ci->ci_dev->dv_xname);
575 #if defined(MPDEBUG) && defined(DDB)
576 printf("dropping into debugger; continue from here to resume boot\n");
577 Debugger();
578 #endif
579 }
580
581 CPU_START_CLEANUP(ci);
582 }
583
584 void
585 cpu_boot_secondary(ci)
586 struct cpu_info *ci;
587 {
588 int i;
589
590 ci->ci_flags |= CPUF_GO; /* XXX atomic */
591
592 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i>0;i--) {
593 delay(10);
594 }
595 if (! (ci->ci_flags & CPUF_RUNNING)) {
596 aprint_error("%s: failed to start\n", ci->ci_dev->dv_xname);
597 #if defined(MPDEBUG) && defined(DDB)
598 printf("dropping into debugger; continue from here to resume boot\n");
599 Debugger();
600 #endif
601 }
602 }
603
604 /*
605 * The CPU ends up here when its ready to run
606 * This is called from code in mptramp.s; at this point, we are running
607 * in the idle pcb/idle stack of the new CPU. When this function returns,
608 * this processor will enter the idle loop and start looking for work.
609 *
610 * XXX should share some of this with init386 in machdep.c
611 */
612 void
613 cpu_hatch(void *v)
614 {
615 struct cpu_info *ci = (struct cpu_info *)v;
616 int s;
617
618 #ifdef __x86_64__
619 cpu_init_msrs(ci);
620 #endif
621 cpu_probe_features(ci);
622 cpu_feature &= ci->ci_feature_flags;
623 cpu_feature2 &= ci->ci_feature2_flags;
624
625 #ifdef DEBUG
626 if (ci->ci_flags & CPUF_PRESENT)
627 panic("%s: already running!?", ci->ci_dev->dv_xname);
628 #endif
629
630 ci->ci_flags |= CPUF_PRESENT;
631
632 lapic_enable();
633 lapic_initclocks();
634
635 while ((ci->ci_flags & CPUF_GO) == 0)
636 delay(10);
637 #ifdef DEBUG
638 if (ci->ci_flags & CPUF_RUNNING)
639 panic("%s: already running!?", ci->ci_dev->dv_xname);
640 #endif
641
642 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0);
643 cpu_init_idt();
644 lapic_set_lvt();
645 gdt_init_cpu(ci);
646
647 #ifdef i386
648 npxinit(ci);
649 #else
650 fpuinit(ci);
651 #endif
652 lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
653
654 cpu_init(ci);
655
656 s = splhigh();
657 #ifdef i386
658 lapic_tpr = 0;
659 #else
660 lcr8(0);
661 #endif
662 x86_enable_intr();
663 splx(s);
664
665 aprint_debug("%s: CPU %ld running\n", ci->ci_dev->dv_xname,
666 (long)ci->ci_cpuid);
667 }
668
669 #if defined(DDB)
670
671 #include <ddb/db_output.h>
672 #include <machine/db_machdep.h>
673
674 /*
675 * Dump CPU information from ddb.
676 */
677 void
678 cpu_debug_dump(void)
679 {
680 struct cpu_info *ci;
681 CPU_INFO_ITERATOR cii;
682
683 db_printf("addr dev id flags ipis curproc fpcurproc\n");
684 for (CPU_INFO_FOREACH(cii, ci)) {
685 db_printf("%p %s %ld %x %x %10p %10p\n",
686 ci,
687 ci->ci_dev == NULL ? "BOOT" : ci->ci_dev->dv_xname,
688 (long)ci->ci_cpuid,
689 ci->ci_flags, ci->ci_ipis,
690 ci->ci_curlwp,
691 ci->ci_fpcurlwp);
692 }
693 }
694 #endif
695
696 static void
697 cpu_copy_trampoline()
698 {
699 /*
700 * Copy boot code.
701 */
702 extern u_char cpu_spinup_trampoline[];
703 extern u_char cpu_spinup_trampoline_end[];
704 pmap_kenter_pa((vaddr_t)MP_TRAMPOLINE, /* virtual */
705 (paddr_t)MP_TRAMPOLINE, /* physical */
706 VM_PROT_ALL); /* protection */
707 pmap_update(pmap_kernel());
708 memcpy((void *)MP_TRAMPOLINE,
709 cpu_spinup_trampoline,
710 cpu_spinup_trampoline_end-cpu_spinup_trampoline);
711 }
712
713 #endif
714
715 #ifdef i386
716 static void
717 cpu_init_tss(struct i386tss *tss, void *stack, void *func)
718 {
719 memset(tss, 0, sizeof *tss);
720 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
721 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
722 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
723 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
724 tss->tss_gs = tss->__tss_es = tss->__tss_ds =
725 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
726 tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
727 tss->tss_esp = (int)((char *)stack + USPACE - 16);
728 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
729 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */
730 tss->__tss_eip = (int)func;
731 }
732
733 /* XXX */
734 #define IDTVEC(name) __CONCAT(X, name)
735 typedef void (vector)(void);
736 extern vector IDTVEC(tss_trap08);
737 #ifdef DDB
738 extern vector Xintrddbipi;
739 extern int ddb_vec;
740 #endif
741
742 static void
743 cpu_set_tss_gates(struct cpu_info *ci)
744 {
745 struct segment_descriptor sd;
746
747 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
748 UVM_KMF_WIRED);
749 cpu_init_tss(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
750 IDTVEC(tss_trap08));
751 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
752 SDT_SYS386TSS, SEL_KPL, 0, 0);
753 ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
754 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
755 GSEL(GTRAPTSS_SEL, SEL_KPL));
756
757 #if defined(DDB) && defined(MULTIPROCESSOR)
758 /*
759 * Set up separate handler for the DDB IPI, so that it doesn't
760 * stomp on a possibly corrupted stack.
761 *
762 * XXX overwriting the gate set in db_machine_init.
763 * Should rearrange the code so that it's set only once.
764 */
765 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
766 UVM_KMF_WIRED);
767 cpu_init_tss(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack,
768 Xintrddbipi);
769
770 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
771 SDT_SYS386TSS, SEL_KPL, 0, 0);
772 ci->ci_gdt[GIPITSS_SEL].sd = sd;
773
774 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
775 GSEL(GIPITSS_SEL, SEL_KPL));
776 #endif
777 }
778 #else
779 static void
780 cpu_set_tss_gates(struct cpu_info *ci)
781 {
782
783 }
784 #endif /* i386 */
785
786
787 int
788 mp_cpu_start(struct cpu_info *ci)
789 {
790 #if NLAPIC > 0
791 int error;
792 #endif
793 unsigned short dwordptr[2];
794
795 /*
796 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
797 */
798
799 outb(IO_RTC, NVRAM_RESET);
800 outb(IO_RTC+1, NVRAM_RESET_JUMP);
801
802 /*
803 * "and the warm reset vector (DWORD based at 40:67) to point
804 * to the AP startup code ..."
805 */
806
807 dwordptr[0] = 0;
808 dwordptr[1] = MP_TRAMPOLINE >> 4;
809
810 pmap_kenter_pa(0, 0, VM_PROT_READ|VM_PROT_WRITE);
811 pmap_update(pmap_kernel());
812 memcpy((uint8_t *)0x467, dwordptr, 4);
813 pmap_kremove(0, PAGE_SIZE);
814 pmap_update(pmap_kernel());
815
816 #if NLAPIC > 0
817 /*
818 * ... prior to executing the following sequence:"
819 */
820
821 if (ci->ci_flags & CPUF_AP) {
822 if ((error = x86_ipi_init(ci->ci_apicid)) != 0)
823 return error;
824
825 delay(10000);
826
827 if (cpu_feature & CPUID_APIC) {
828
829 if ((error = x86_ipi(MP_TRAMPOLINE/PAGE_SIZE,
830 ci->ci_apicid,
831 LAPIC_DLMODE_STARTUP)) != 0)
832 return error;
833 delay(200);
834
835 if ((error = x86_ipi(MP_TRAMPOLINE/PAGE_SIZE,
836 ci->ci_apicid,
837 LAPIC_DLMODE_STARTUP)) != 0)
838 return error;
839 delay(200);
840 }
841 }
842 #endif
843 return 0;
844 }
845
846 void
847 mp_cpu_start_cleanup(struct cpu_info *ci)
848 {
849 /*
850 * Ensure the NVRAM reset byte contains something vaguely sane.
851 */
852
853 outb(IO_RTC, NVRAM_RESET);
854 outb(IO_RTC+1, NVRAM_RESET_RST);
855 }
856
857 #ifdef __x86_64__
858 typedef void (vector)(void);
859 extern vector Xsyscall, Xsyscall32;
860
861 void
862 cpu_init_msrs(struct cpu_info *ci)
863 {
864 wrmsr(MSR_STAR,
865 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
866 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
867 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
868 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
869 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
870
871 wrmsr(MSR_FSBASE, 0);
872 wrmsr(MSR_GSBASE, (u_int64_t)ci);
873 wrmsr(MSR_KERNELGSBASE, 0);
874
875 if (cpu_feature & CPUID_NOX)
876 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
877 }
878 #endif /* __x86_64__ */
879