cpu.c revision 1.12 1 /* $NetBSD: cpu.c,v 1.12 2008/04/17 12:24:44 cegger Exp $ */
2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */
3
4 /*-
5 * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by RedBack Networks Inc.
10 *
11 * Author: Bill Sommerfeld
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. All advertising materials mentioning features or use of this software
22 * must display the following acknowledgement:
23 * This product includes software developed by the NetBSD
24 * Foundation, Inc. and its contributors.
25 * 4. Neither the name of The NetBSD Foundation nor the names of its
26 * contributors may be used to endorse or promote products derived
27 * from this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
30 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
31 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
32 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
33 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
36 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
37 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39 * POSSIBILITY OF SUCH DAMAGE.
40 */
41
42 /*
43 * Copyright (c) 1999 Stefan Grefen
44 *
45 * Redistribution and use in source and binary forms, with or without
46 * modification, are permitted provided that the following conditions
47 * are met:
48 * 1. Redistributions of source code must retain the above copyright
49 * notice, this list of conditions and the following disclaimer.
50 * 2. Redistributions in binary form must reproduce the above copyright
51 * notice, this list of conditions and the following disclaimer in the
52 * documentation and/or other materials provided with the distribution.
53 * 3. All advertising materials mentioning features or use of this software
54 * must display the following acknowledgement:
55 * This product includes software developed by the NetBSD
56 * Foundation, Inc. and its contributors.
57 * 4. Neither the name of The NetBSD Foundation nor the names of its
58 * contributors may be used to endorse or promote products derived
59 * from this software without specific prior written permission.
60 *
61 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
62 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
63 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
64 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
65 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
66 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
67 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
68 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
69 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
70 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
71 * SUCH DAMAGE.
72 */
73
74 #include <sys/cdefs.h>
75 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.12 2008/04/17 12:24:44 cegger Exp $");
76
77 #include "opt_ddb.h"
78 #include "opt_multiprocessor.h"
79 #include "opt_mpbios.h" /* for MPDEBUG */
80 #include "opt_mtrr.h"
81 #include "opt_xen.h"
82
83 #include "lapic.h"
84 #include "ioapic.h"
85
86 #include <sys/param.h>
87 #include <sys/proc.h>
88 #include <sys/user.h>
89 #include <sys/systm.h>
90 #include <sys/device.h>
91 #include <sys/malloc.h>
92 #include <sys/cpu.h>
93 #include <sys/atomic.h>
94
95 #include <uvm/uvm_extern.h>
96
97 #include <machine/cpufunc.h>
98 #include <machine/cpuvar.h>
99 #include <machine/pmap.h>
100 #include <machine/vmparam.h>
101 #include <machine/mpbiosvar.h>
102 #include <machine/pcb.h>
103 #include <machine/specialreg.h>
104 #include <machine/segments.h>
105 #include <machine/gdt.h>
106 #include <machine/mtrr.h>
107 #include <machine/pio.h>
108
109 #ifdef XEN3
110 #include <xen/vcpuvar.h>
111 #endif
112
113 #if NLAPIC > 0
114 #include <machine/apicvar.h>
115 #include <machine/i82489reg.h>
116 #include <machine/i82489var.h>
117 #endif
118
119 #if NIOAPIC > 0
120 #include <machine/i82093var.h>
121 #endif
122
123 #include <dev/ic/mc146818reg.h>
124 #include <dev/isa/isareg.h>
125
126 int cpu_match(device_t, cfdata_t, void *);
127 void cpu_attach(device_t, device_t, void *);
128 #ifdef XEN3
129 int vcpu_match(device_t, cfdata_t, void *);
130 void vcpu_attach(device_t, device_t, void *);
131 #endif
132 void cpu_attach_common(device_t, device_t, void *);
133 void cpu_offline_md(void);
134
135 struct cpu_softc {
136 device_t sc_dev; /* device tree glue */
137 struct cpu_info *sc_info; /* pointer to CPU info */
138 };
139
140 int mp_cpu_start(struct cpu_info *, paddr_t);
141 void mp_cpu_start_cleanup(struct cpu_info *);
142 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
143 mp_cpu_start_cleanup };
144
145 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc),
146 cpu_match, cpu_attach, NULL, NULL);
147 #ifdef XEN3
148 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
149 vcpu_match, vcpu_attach, NULL, NULL);
150 #endif
151
152 /*
153 * Statically-allocated CPU info for the primary CPU (or the only
154 * CPU, on uniprocessors). The CPU info list is initialized to
155 * point at it.
156 */
157 #ifdef TRAPLOG
158 #include <machine/tlog.h>
159 struct tlog tlog_primary;
160 #endif
161 struct cpu_info cpu_info_primary = {
162 .ci_dev = 0,
163 .ci_self = &cpu_info_primary,
164 .ci_idepth = -1,
165 .ci_curlwp = &lwp0,
166 #ifdef TRAPLOG
167 .ci_tlog = &tlog_primary,
168 #endif
169
170 };
171 struct cpu_info phycpu_info_primary = {
172 .ci_dev = 0,
173 .ci_self = &phycpu_info_primary,
174 };
175
176 struct cpu_info *cpu_info_list = &cpu_info_primary;
177
178 static void cpu_set_tss_gates(struct cpu_info *ci);
179
180 uint32_t cpus_attached = 0;
181 uint32_t cpus_running = 0;
182
183 bool x86_mp_online;
184 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
185
186 struct cpu_info *phycpu_info[X86_MAXPROCS] = { &cpu_info_primary };
187
188 #ifdef MULTIPROCESSOR
189 /*
190 * Array of CPU info structures. Must be statically-allocated because
191 * curproc, etc. are used early.
192 */
193 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary };
194
195 void cpu_hatch(void *);
196 static void cpu_boot_secondary(struct cpu_info *ci);
197 static void cpu_start_secondary(struct cpu_info *ci);
198 static void cpu_copy_trampoline(void);
199
200 /*
201 * Runs once per boot once multiprocessor goo has been detected and
202 * the local APIC on the boot processor has been mapped.
203 *
204 * Called from lapic_boot_init() (from mpbios_scan()).
205 */
206 void
207 cpu_init_first(void)
208 {
209 int cpunum = lapic_cpu_number();
210
211 if (cpunum != 0) {
212 cpu_info[0] = NULL;
213 cpu_info[cpunum] = &cpu_info_primary;
214 }
215
216 cpu_copy_trampoline();
217 }
218 #endif
219
220 int
221 cpu_match(device_t parent, cfdata_t match, void *aux)
222 {
223
224 return 1;
225 }
226
227 void
228 cpu_attach(device_t parent, device_t self, void *aux)
229 {
230 #ifdef XEN3
231 struct cpu_softc *sc = device_private(self);
232 struct cpu_attach_args *caa = aux;
233 struct cpu_info *ci;
234 int cpunum = caa->cpu_number;
235
236 sc->sc_dev = self;
237
238 /*
239 * If we're an Application Processor, allocate a cpu_info
240 * structure, otherwise use the primary's.
241 */
242 if (caa->cpu_role == CPU_ROLE_AP) {
243 ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK | M_ZERO);
244 if (phycpu_info[cpunum] != NULL)
245 panic("cpu at apic id %d already attached?", cpunum);
246 phycpu_info[cpunum] = ci;
247 } else {
248 ci = &phycpu_info_primary;
249 if (cpunum != 0) {
250 phycpu_info[0] = NULL;
251 phycpu_info[cpunum] = ci;
252 }
253 }
254
255 ci->ci_self = ci;
256 sc->sc_info = ci;
257
258 ci->ci_dev = self;
259 ci->ci_apicid = caa->cpu_number;
260 ci->ci_cpuid = ci->ci_apicid;
261
262 printf(": ");
263 switch (caa->cpu_role) {
264 case CPU_ROLE_SP:
265 printf("(uniprocessor)\n");
266 ci->ci_flags |= CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY;
267 break;
268
269 case CPU_ROLE_BP:
270 printf("(boot processor)\n");
271 ci->ci_flags |= CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY;
272 #if NIOAPIC > 0
273 ioapic_bsp_id = caa->cpu_number;
274 #endif
275 break;
276
277 case CPU_ROLE_AP:
278 /*
279 * report on an AP
280 */
281 printf("(application processor)\n");
282 break;
283
284 default:
285 panic("unknown processor type??\n");
286 }
287 return;
288 #else
289 cpu_attach_common(parent, self, aux);
290 #endif
291 }
292
293 #ifdef XEN3
294 int
295 vcpu_match(device_t parent, cfdata_t match, void *aux)
296 {
297 struct vcpu_attach_args *vcaa = aux;
298
299 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0)
300 return 1;
301 return 0;
302 }
303
304 void
305 vcpu_attach(device_t parent, device_t self, void *aux)
306 {
307 struct vcpu_attach_args *vcaa = aux;
308
309 cpu_attach_common(parent, self, &vcaa->vcaa_caa);
310 }
311 #endif
312
313 static void
314 cpu_vm_init(struct cpu_info *ci)
315 {
316 int ncolors = 2, i;
317
318 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
319 struct x86_cache_info *cai;
320 int tcolors;
321
322 cai = &ci->ci_cinfo[i];
323
324 tcolors = atop(cai->cai_totalsize);
325 switch(cai->cai_associativity) {
326 case 0xff:
327 tcolors = 1; /* fully associative */
328 break;
329 case 0:
330 case 1:
331 break;
332 default:
333 tcolors /= cai->cai_associativity;
334 }
335 ncolors = max(ncolors, tcolors);
336 }
337
338 /*
339 * Knowing the size of the largest cache on this CPU, re-color
340 * our pages.
341 */
342 if (ncolors <= uvmexp.ncolors)
343 return;
344 printf("%s: %d page colors\n", device_xname(ci->ci_dev), ncolors);
345 uvm_page_recolor(ncolors);
346 }
347
348 void
349 cpu_attach_common(device_t parent, device_t self, void *aux)
350 {
351 struct cpu_softc *sc = device_private(self);
352 struct cpu_attach_args *caa = aux;
353 struct cpu_info *ci;
354 uintptr_t ptr;
355 #if defined(MULTIPROCESSOR)
356 int cpunum = caa->cpu_number;
357 #endif
358
359 sc->sc_dev = self;
360
361 /*
362 * If we're an Application Processor, allocate a cpu_info
363 * structure, otherwise use the primary's.
364 */
365 if (caa->cpu_role == CPU_ROLE_AP) {
366 aprint_naive(": Application Processor\n");
367 ptr = (uintptr_t)malloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
368 M_DEVBUF, M_WAITOK);
369 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) &
370 ~(CACHE_LINE_SIZE - 1));
371 memset(ci, 0, sizeof(*ci));
372 #if defined(MULTIPROCESSOR)
373 if (cpu_info[cpunum] != NULL)
374 panic("cpu at apic id %d already attached?", cpunum);
375 cpu_info[cpunum] = ci;
376 #endif
377 #ifdef TRAPLOG
378 ci->ci_tlog_base = malloc(sizeof(struct tlog),
379 M_DEVBUF, M_WAITOK);
380 #endif
381 } else {
382 aprint_naive(": %s Processor\n",
383 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
384 ci = &cpu_info_primary;
385 #if defined(MULTIPROCESSOR)
386 if (cpunum != lapic_cpu_number()) {
387 panic("%s: running CPU is at apic %d"
388 " instead of at expected %d",
389 device_xname(sc->sc_dev), lapic_cpu_number(), cpunum);
390 }
391 #endif
392 }
393
394 ci->ci_self = ci;
395 sc->sc_info = ci;
396
397 ci->ci_dev = self;
398 ci->ci_apicid = caa->cpu_number;
399 #ifdef MULTIPROCESSOR
400 ci->ci_cpuid = ci->ci_apicid;
401 #else
402 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */
403 #endif
404 ci->ci_cpumask = (1 << ci->ci_cpuid);
405 ci->ci_func = caa->cpu_func;
406
407 if (caa->cpu_role == CPU_ROLE_AP) {
408 #if defined(MULTIPROCESSOR)
409 int error;
410
411 error = mi_cpu_attach(ci);
412 if (error != 0) {
413 aprint_normal("\n");
414 aprint_error_dev(sc->sc_dev, "mi_cpu_attach failed with %d\n",
415 error);
416 return;
417 }
418 #endif
419 } else {
420 KASSERT(ci->ci_data.cpu_idlelwp != NULL);
421 }
422
423 pmap_reference(pmap_kernel());
424 ci->ci_pmap = pmap_kernel();
425 ci->ci_tlbstate = TLBSTATE_STALE;
426
427 /* further PCB init done later. */
428
429 switch (caa->cpu_role) {
430 case CPU_ROLE_SP:
431 aprint_normal(": (uniprocessor)\n");
432 atomic_or_32(&ci->ci_flags,
433 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY);
434 cpu_intr_init(ci);
435 identifycpu(ci);
436 cpu_init(ci);
437 cpu_set_tss_gates(ci);
438 pmap_cpu_init_late(ci);
439 #if 0
440 x86_errata();
441 #endif
442 break;
443
444 case CPU_ROLE_BP:
445 aprint_normal("apid %d (boot processor)\n", caa->cpu_number);
446 atomic_or_32(&ci->ci_flags,
447 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY);
448 cpu_intr_init(ci);
449 identifycpu(ci);
450 cpu_init(ci);
451 cpu_set_tss_gates(ci);
452 pmap_cpu_init_late(ci);
453 #if NLAPIC > 0
454 /*
455 * Enable local apic
456 */
457 lapic_enable();
458 lapic_set_lvt();
459 lapic_calibrate_timer(ci);
460 #endif
461 #if NIOAPIC > 0
462 ioapic_bsp_id = caa->cpu_number;
463 #endif
464 #if 0
465 x86_errata();
466 #endif
467 break;
468
469 case CPU_ROLE_AP:
470 /*
471 * report on an AP
472 */
473 aprint_normal("apid %d (application processor)\n", caa->cpu_number);
474
475 #if defined(MULTIPROCESSOR)
476 cpu_intr_init(ci);
477 gdt_alloc_cpu(ci);
478 cpu_set_tss_gates(ci);
479 pmap_cpu_init_early(ci);
480 pmap_cpu_init_late(ci);
481 cpu_start_secondary(ci);
482 if (ci->ci_flags & CPUF_PRESENT) {
483 identifycpu(ci);
484 ci->ci_next = cpu_info_list->ci_next;
485 cpu_info_list->ci_next = ci;
486 }
487 #else
488 aprint_normal_dev(sc->sc_dev, "not started\n");
489 #endif
490 break;
491
492 default:
493 aprint_normal("\n");
494 panic("unknown processor type??\n");
495 }
496 cpu_vm_init(ci);
497
498 cpus_attached |= (1 << ci->ci_cpuid);
499
500 #if 0
501 if (!pmf_device_register(self, cpu_suspend, cpu_resume))
502 aprint_error_dev(self, "couldn't establish power handler\n");
503 #endif
504
505 #if defined(MULTIPROCESSOR)
506 if (mp_verbose) {
507 struct lwp *l = ci->ci_data.cpu_idlelwp;
508
509 aprint_verbose_dev(sc->sc_dev, "idle lwp at %p, idle sp at 0x%p\n",
510 l,
511 #ifdef i386
512 (void *)l->l_addr->u_pcb.pcb_esp
513 #else
514 (void *)l->l_addr->u_pcb.pcb_rsp
515 #endif
516 );
517
518 }
519 #endif
520 }
521
522 /*
523 * Initialize the processor appropriately.
524 */
525
526 void
527 cpu_init(struct cpu_info *ci)
528 {
529 /* configure the CPU if needed */
530 if (ci->cpu_setup != NULL)
531 (*ci->cpu_setup)(ci);
532
533 /*
534 * On a P6 or above, enable global TLB caching if the
535 * hardware supports it.
536 */
537 if (cpu_feature & CPUID_PGE)
538 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */
539
540 #ifdef XXXMTRR
541 /*
542 * On a P6 or above, initialize MTRR's if the hardware supports them.
543 */
544 if (cpu_feature & CPUID_MTRR) {
545 if ((ci->ci_flags & CPUF_AP) == 0)
546 i686_mtrr_init_first();
547 mtrr_init_cpu(ci);
548 }
549 #endif
550 /*
551 * If we have FXSAVE/FXRESTOR, use them.
552 */
553 if (cpu_feature & CPUID_FXSR) {
554 lcr4(rcr4() | CR4_OSFXSR);
555
556 /*
557 * If we have SSE/SSE2, enable XMM exceptions.
558 */
559 if (cpu_feature & (CPUID_SSE|CPUID_SSE2))
560 lcr4(rcr4() | CR4_OSXMMEXCPT);
561 }
562
563 #ifdef MULTIPROCESSOR
564 atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
565 atomic_or_32(&cpus_running, ci->ci_cpumask);
566 #endif
567 }
568
569
570 #ifdef MULTIPROCESSOR
571 void
572 cpu_boot_secondary_processors(void)
573 {
574 struct cpu_info *ci;
575 u_long i;
576
577 for (i = 0; i < X86_MAXPROCS; i++) {
578 ci = cpu_info[i];
579 if (ci == NULL)
580 continue;
581 if (ci->ci_data.cpu_idlelwp == NULL)
582 continue;
583 if ((ci->ci_flags & CPUF_PRESENT) == 0)
584 continue;
585 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
586 continue;
587 cpu_boot_secondary(ci);
588 }
589
590 x86_mp_online = true;
591 }
592
593 static void
594 cpu_init_idle_lwp(struct cpu_info *ci)
595 {
596 struct lwp *l = ci->ci_data.cpu_idlelwp;
597 struct pcb *pcb = &l->l_addr->u_pcb;
598
599 pcb->pcb_cr0 = rcr0();
600 }
601
602 void
603 cpu_init_idle_lwps(void)
604 {
605 struct cpu_info *ci;
606 u_long i;
607
608 for (i = 0; i < X86_MAXPROCS; i++) {
609 ci = cpu_info[i];
610 if (ci == NULL)
611 continue;
612 if (ci->ci_data.cpu_idlelwp == NULL)
613 continue;
614 if ((ci->ci_flags & CPUF_PRESENT) == 0)
615 continue;
616 cpu_init_idle_lwp(ci);
617 }
618 }
619
620 void
621 cpu_start_secondary(struct cpu_info *ci)
622 {
623 int i;
624 struct pmap *kpm = pmap_kernel();
625 extern uint32_t mp_pdirpa;
626
627 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */
628
629 atomic_or_32(&ci->ci_flags, CPUF_AP);
630
631 aprint_debug_dev(ci->ci_dev, "starting\n");
632
633 ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
634 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0)
635 return;
636
637 /*
638 * wait for it to become ready
639 */
640 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
641 #ifdef MPDEBUG
642 extern int cpu_trace[3];
643 static int otrace[3];
644 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
645 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
646 cpu_trace[0], cpu_trace[1], cpu_trace[2]);
647 memcpy(otrace, cpu_trace, sizeof(otrace));
648 }
649 #endif
650 delay(10);
651 }
652 if ((ci->ci_flags & CPUF_PRESENT) == 0) {
653 aprint_error_dev(ci->ci_dev, "failed to become ready\n");
654 #if defined(MPDEBUG) && defined(DDB)
655 printf("dropping into debugger; continue from here to resume boot\n");
656 Debugger();
657 #endif
658 }
659
660 CPU_START_CLEANUP(ci);
661 }
662
663 void
664 cpu_boot_secondary(struct cpu_info *ci)
665 {
666 int i;
667
668 atomic_or_32(&ci->ci_flags, CPUF_GO);
669 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
670 delay(10);
671 }
672 if ((ci->ci_flags & CPUF_RUNNING) == 0) {
673 aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
674 #if defined(MPDEBUG) && defined(DDB)
675 printf("dropping into debugger; continue from here to resume boot\n");
676 Debugger();
677 #endif
678 }
679 }
680
681 /*
682 * The CPU ends up here when its ready to run
683 * This is called from code in mptramp.s; at this point, we are running
684 * in the idle pcb/idle stack of the new CPU. When this function returns,
685 * this processor will enter the idle loop and start looking for work.
686 *
687 * XXX should share some of this with init386 in machdep.c
688 */
689 void
690 cpu_hatch(void *v)
691 {
692 struct cpu_info *ci = (struct cpu_info *)v;
693 int s, i;
694 uint32_t blacklist_features;
695
696 #ifdef __x86_64__
697 cpu_init_msrs(ci, true);
698 #endif
699
700 cpu_probe_features(ci);
701 cpu_feature &= ci->ci_feature_flags;
702 cpu_feature2 &= ci->ci_feature2_flags;
703
704 /* not on Xen... */
705 blacklist_features = ~(CPUID_PGE|CPUID_PSE|CPUID_MTRR|CPUID_FXSR|CPUID_NOX); /* XXX add CPUID_SVM */
706
707 cpu_feature &= blacklist_features;
708
709 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
710 atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
711 while ((ci->ci_flags & CPUF_GO) == 0) {
712 /* Don't use delay, boot CPU may be patching the text. */
713 for (i = 10000; i != 0; i--)
714 x86_pause();
715 }
716
717 /* Because the text may have been patched in x86_patch(). */
718 wbinvd();
719 x86_flush();
720
721 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
722
723 lcr3(pmap_kernel()->pm_pdirpa);
724 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa;
725 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0);
726 cpu_init_idt();
727 gdt_init_cpu(ci);
728 lapic_enable();
729 lapic_set_lvt();
730 lapic_initclocks();
731
732 #ifdef i386
733 npxinit(ci);
734 #else
735 fpuinit(ci);
736 #endif
737
738 lldt(GSEL(GLDT_SEL, SEL_KPL));
739 ltr(ci->ci_tss_sel);
740
741 cpu_init(ci);
742 cpu_get_tsc_freq(ci);
743
744 s = splhigh();
745 #ifdef i386
746 lapic_tpr = 0;
747 #else
748 lcr8(0);
749 #endif
750 x86_enable_intr();
751 splx(s);
752 #if 0
753 x86_errata();
754 #endif
755
756 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n",
757 (long)ci->ci_cpuid);
758 }
759
760 #if defined(DDB)
761
762 #include <ddb/db_output.h>
763 #include <machine/db_machdep.h>
764
765 /*
766 * Dump CPU information from ddb.
767 */
768 void
769 cpu_debug_dump(void)
770 {
771 struct cpu_info *ci;
772 CPU_INFO_ITERATOR cii;
773
774 db_printf("addr dev id flags ipis curproc fpcurproc\n");
775 for (CPU_INFO_FOREACH(cii, ci)) {
776 db_printf("%p %s %ld %x %x %10p %10p\n",
777 ci,
778 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
779 (long)ci->ci_cpuid,
780 ci->ci_flags, ci->ci_ipis,
781 ci->ci_curlwp,
782 ci->ci_fpcurlwp);
783 }
784 }
785 #endif
786
787 static void
788 cpu_copy_trampoline(void)
789 {
790 /*
791 * Copy boot code.
792 */
793 extern u_char cpu_spinup_trampoline[];
794 extern u_char cpu_spinup_trampoline_end[];
795
796 vaddr_t mp_trampoline_vaddr;
797
798 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
799 UVM_KMF_VAONLY);
800
801 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
802 VM_PROT_READ | VM_PROT_WRITE);
803 pmap_update(pmap_kernel());
804 memcpy((void *)mp_trampoline_vaddr,
805 cpu_spinup_trampoline,
806 cpu_spinup_trampoline_end - cpu_spinup_trampoline);
807
808 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
809 pmap_update(pmap_kernel());
810 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
811 }
812
813 #endif
814
815 #ifdef i386
816 #if 0
817 static void
818 tss_init(struct i386tss *tss, void *stack, void *func)
819 {
820 memset(tss, 0, sizeof *tss);
821 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
822 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
823 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
824 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
825 tss->tss_gs = tss->__tss_es = tss->__tss_ds =
826 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
827 tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
828 tss->tss_esp = (int)((char *)stack + USPACE - 16);
829 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
830 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */
831 tss->__tss_eip = (int)func;
832 }
833 #endif
834
835 /* XXX */
836 #define IDTVEC(name) __CONCAT(X, name)
837 typedef void (vector)(void);
838 extern vector IDTVEC(tss_trap08);
839 #ifdef DDB
840 extern vector Xintrddbipi;
841 extern int ddb_vec;
842 #endif
843
844 static void
845 cpu_set_tss_gates(struct cpu_info *ci)
846 {
847 #if 0
848 struct segment_descriptor sd;
849
850 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
851 UVM_KMF_WIRED);
852 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
853 IDTVEC(tss_trap08));
854 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
855 SDT_SYS386TSS, SEL_KPL, 0, 0);
856 ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
857 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
858 GSEL(GTRAPTSS_SEL, SEL_KPL));
859 #endif
860
861 #if defined(DDB) && defined(MULTIPROCESSOR)
862 /*
863 * Set up separate handler for the DDB IPI, so that it doesn't
864 * stomp on a possibly corrupted stack.
865 *
866 * XXX overwriting the gate set in db_machine_init.
867 * Should rearrange the code so that it's set only once.
868 */
869 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
870 UVM_KMF_WIRED);
871 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack,
872 Xintrddbipi);
873
874 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
875 SDT_SYS386TSS, SEL_KPL, 0, 0);
876 ci->ci_gdt[GIPITSS_SEL].sd = sd;
877
878 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
879 GSEL(GIPITSS_SEL, SEL_KPL));
880 #endif
881 }
882 #else
883 static void
884 cpu_set_tss_gates(struct cpu_info *ci)
885 {
886
887 }
888 #endif /* i386 */
889
890 int
891 mp_cpu_start(struct cpu_info *ci, paddr_t target)
892 {
893 #if 0
894 #if NLAPIC > 0
895 int error;
896 #endif
897 unsigned short dwordptr[2];
898
899 /*
900 * Bootstrap code must be addressable in real mode
901 * and it must be page aligned.
902 */
903 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
904
905 /*
906 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
907 */
908
909 outb(IO_RTC, NVRAM_RESET);
910 outb(IO_RTC+1, NVRAM_RESET_JUMP);
911
912 /*
913 * "and the warm reset vector (DWORD based at 40:67) to point
914 * to the AP startup code ..."
915 */
916
917 dwordptr[0] = 0;
918 dwordptr[1] = target >> 4;
919
920 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE);
921 memcpy ((uint8_t *) 0x467, dwordptr, 4);
922 pmap_kremove (0, PAGE_SIZE);
923
924 #if NLAPIC > 0
925 /*
926 * ... prior to executing the following sequence:"
927 */
928
929 if (ci->ci_flags & CPUF_AP) {
930 if ((error = x86_ipi_init(ci->ci_apicid)) != 0)
931 return error;
932
933 delay(10000);
934
935 if (cpu_feature & CPUID_APIC) {
936 error = x86_ipi_init(ci->ci_apicid);
937 if (error != 0) {
938 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
939 __func__);
940 return error;
941 }
942
943 delay(10000);
944
945 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid,
946 LAPIC_DLMODE_STARTUP);
947 if (error != 0) {
948 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
949 __func__);
950 return error;
951 }
952 delay(200);
953
954 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid,
955 LAPIC_DLMODE_STARTUP);
956 if (error != 0) {
957 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n",
958 __func__);
959 return error;
960 }
961 delay(200);
962 }
963 }
964 #endif
965 #endif /* 0 */
966 return 0;
967 }
968
969 void
970 mp_cpu_start_cleanup(struct cpu_info *ci)
971 {
972 #if 0
973 /*
974 * Ensure the NVRAM reset byte contains something vaguely sane.
975 */
976
977 outb(IO_RTC, NVRAM_RESET);
978 outb(IO_RTC+1, NVRAM_RESET_RST);
979 #endif
980 }
981
982 #ifdef __x86_64__
983
984 void
985 cpu_init_msrs(struct cpu_info *ci, bool full)
986 {
987 if (full) {
988 HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
989 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
990 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
991 }
992 }
993 #endif /* __x86_64__ */
994
995 void
996 cpu_offline_md(void)
997 {
998 int s;
999
1000 s = splhigh();
1001 #ifdef __i386__
1002 npxsave_cpu(true);
1003 #else
1004 fpusave_cpu(true);
1005 #endif
1006 splx(s);
1007 }
1008
1009 #if 0
1010 /* XXX joerg restructure and restart CPUs individually */
1011 static bool
1012 cpu_suspend(device_t dv PMF_FN_ARGS)
1013 {
1014 struct cpu_softc *sc = device_private(dv);
1015 struct cpu_info *ci = sc->sc_info;
1016 int err;
1017
1018 if (ci->ci_flags & CPUF_PRIMARY)
1019 return true;
1020 if (ci->ci_data.cpu_idlelwp == NULL)
1021 return true;
1022 if ((ci->ci_flags & CPUF_PRESENT) == 0)
1023 return true;
1024
1025 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
1026
1027 if (sc->sc_wasonline) {
1028 mutex_enter(&cpu_lock);
1029 err = cpu_setonline(ci, false);
1030 mutex_exit(&cpu_lock);
1031
1032 if (err)
1033 return false;
1034 }
1035
1036 return true;
1037 }
1038
1039 static bool
1040 cpu_resume(device_t dv PMF_FN_ARGS)
1041 {
1042 struct cpu_softc *sc = device_private(dv);
1043 struct cpu_info *ci = sc->sc_info;
1044 int err = 0;
1045
1046 if (ci->ci_flags & CPUF_PRIMARY)
1047 return true;
1048 if (ci->ci_data.cpu_idlelwp == NULL)
1049 return true;
1050 if ((ci->ci_flags & CPUF_PRESENT) == 0)
1051 return true;
1052
1053 if (sc->sc_wasonline) {
1054 mutex_enter(&cpu_lock);
1055 err = cpu_setonline(ci, true);
1056 mutex_exit(&cpu_lock);
1057 }
1058
1059 return err == 0;
1060 }
1061 #endif
1062
1063 void
1064 cpu_get_tsc_freq(struct cpu_info *ci)
1065 {
1066 #ifdef XEN3
1067 const volatile vcpu_time_info_t *tinfo =
1068 &HYPERVISOR_shared_info->vcpu_info[0].time;
1069 delay(1000000);
1070 uint64_t freq = 1000000000ULL << 32;
1071 freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
1072 if ( tinfo->tsc_shift < 0 )
1073 freq = freq << -tinfo->tsc_shift;
1074 else
1075 freq = freq >> tinfo->tsc_shift;
1076 ci->ci_tsc_freq = freq;
1077 #else
1078 /* XXX this needs to read the shared_info of the CPU being probed.. */
1079 ci->ci_tsc_freq = HYPERVISOR_shared_info->cpu_freq;
1080 #endif /* XEN3 */
1081 }
1082