cpu.c revision 1.93 1 /* $NetBSD: cpu.c,v 1.93 2011/09/28 15:38:21 jruoho Exp $ */
2
3 /*-
4 * Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c) 1999 Stefan Grefen
34 *
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
37 * are met:
38 * 1. Redistributions of source code must retain the above copyright
39 * notice, this list of conditions and the following disclaimer.
40 * 2. Redistributions in binary form must reproduce the above copyright
41 * notice, this list of conditions and the following disclaimer in the
42 * documentation and/or other materials provided with the distribution.
43 * 3. All advertising materials mentioning features or use of this software
44 * must display the following acknowledgement:
45 * This product includes software developed by the NetBSD
46 * Foundation, Inc. and its contributors.
47 * 4. Neither the name of The NetBSD Foundation nor the names of its
48 * contributors may be used to endorse or promote products derived
49 * from this software without specific prior written permission.
50 *
51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
61 * SUCH DAMAGE.
62 */
63
64 #include <sys/cdefs.h>
65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.93 2011/09/28 15:38:21 jruoho Exp $");
66
67 #include "opt_ddb.h"
68 #include "opt_mpbios.h" /* for MPDEBUG */
69 #include "opt_mtrr.h"
70
71 #include "lapic.h"
72 #include "ioapic.h"
73
74 #ifdef i386
75 #include "npx.h"
76 #endif
77
78 #include <sys/param.h>
79 #include <sys/proc.h>
80 #include <sys/systm.h>
81 #include <sys/device.h>
82 #include <sys/kmem.h>
83 #include <sys/cpu.h>
84 #include <sys/cpufreq.h>
85 #include <sys/atomic.h>
86 #include <sys/reboot.h>
87
88 #include <uvm/uvm.h>
89
90 #include <machine/cpufunc.h>
91 #include <machine/cpuvar.h>
92 #include <machine/pmap.h>
93 #include <machine/vmparam.h>
94 #include <machine/mpbiosvar.h>
95 #include <machine/pcb.h>
96 #include <machine/specialreg.h>
97 #include <machine/segments.h>
98 #include <machine/gdt.h>
99 #include <machine/mtrr.h>
100 #include <machine/pio.h>
101 #include <machine/cpu_counter.h>
102
103 #ifdef i386
104 #include <machine/tlog.h>
105 #endif
106
107 #include <machine/apicvar.h>
108 #include <machine/i82489reg.h>
109 #include <machine/i82489var.h>
110
111 #include <dev/ic/mc146818reg.h>
112 #include <i386/isa/nvram.h>
113 #include <dev/isa/isareg.h>
114
115 #include "tsc.h"
116
117 #if MAXCPUS > 32
118 #error cpu_info contains 32bit bitmasks
119 #endif
120
121 static int cpu_match(device_t, cfdata_t, void *);
122 static void cpu_attach(device_t, device_t, void *);
123 static void cpu_defer(device_t);
124 static int cpu_rescan(device_t, const char *, const int *);
125 static void cpu_childdetached(device_t, device_t);
126 static bool cpu_suspend(device_t, const pmf_qual_t *);
127 static bool cpu_resume(device_t, const pmf_qual_t *);
128 static bool cpu_shutdown(device_t, int);
129
130 struct cpu_softc {
131 device_t sc_dev; /* device tree glue */
132 struct cpu_info *sc_info; /* pointer to CPU info */
133 bool sc_wasonline;
134 };
135
136 int mp_cpu_start(struct cpu_info *, paddr_t);
137 void mp_cpu_start_cleanup(struct cpu_info *);
138 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
139 mp_cpu_start_cleanup };
140
141
142 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
143 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
144
145 /*
146 * Statically-allocated CPU info for the primary CPU (or the only
147 * CPU, on uniprocessors). The CPU info list is initialized to
148 * point at it.
149 */
150 #ifdef TRAPLOG
151 struct tlog tlog_primary;
152 #endif
153 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
154 .ci_dev = 0,
155 .ci_self = &cpu_info_primary,
156 .ci_idepth = -1,
157 .ci_curlwp = &lwp0,
158 .ci_curldt = -1,
159 .ci_cpumask = 1,
160 #ifdef TRAPLOG
161 .ci_tlog_base = &tlog_primary,
162 #endif /* !TRAPLOG */
163 };
164
165 struct cpu_info *cpu_info_list = &cpu_info_primary;
166
167 static void cpu_set_tss_gates(struct cpu_info *);
168
169 #ifdef i386
170 static void tss_init(struct i386tss *, void *, void *);
171 #endif
172
173 static void cpu_init_idle_lwp(struct cpu_info *);
174
175 uint32_t cpus_attached = 0;
176 uint32_t cpus_running = 1;
177
178 uint32_t cpu_feature[5]; /* X86 CPUID feature bits
179 * [0] basic features %edx
180 * [1] basic features %ecx
181 * [2] extended features %edx
182 * [3] extended features %ecx
183 * [4] VIA padlock features
184 */
185
186 extern char x86_64_doubleflt_stack[];
187
188 bool x86_mp_online;
189 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
190 static vaddr_t cmos_data_mapping;
191 struct cpu_info *cpu_starting;
192
193 void cpu_hatch(void *);
194 static void cpu_boot_secondary(struct cpu_info *ci);
195 static void cpu_start_secondary(struct cpu_info *ci);
196 static void cpu_copy_trampoline(void);
197
198 /*
199 * Runs once per boot once multiprocessor goo has been detected and
200 * the local APIC on the boot processor has been mapped.
201 *
202 * Called from lapic_boot_init() (from mpbios_scan()).
203 */
204 void
205 cpu_init_first(void)
206 {
207
208 cpu_info_primary.ci_cpuid = lapic_cpu_number();
209 cpu_copy_trampoline();
210
211 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY);
212 if (cmos_data_mapping == 0)
213 panic("No KVA for page 0");
214 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
215 pmap_update(pmap_kernel());
216 }
217
218 static int
219 cpu_match(device_t parent, cfdata_t match, void *aux)
220 {
221
222 return 1;
223 }
224
225 static void
226 cpu_vm_init(struct cpu_info *ci)
227 {
228 int ncolors = 2, i;
229
230 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
231 struct x86_cache_info *cai;
232 int tcolors;
233
234 cai = &ci->ci_cinfo[i];
235
236 tcolors = atop(cai->cai_totalsize);
237 switch(cai->cai_associativity) {
238 case 0xff:
239 tcolors = 1; /* fully associative */
240 break;
241 case 0:
242 case 1:
243 break;
244 default:
245 tcolors /= cai->cai_associativity;
246 }
247 ncolors = max(ncolors, tcolors);
248 /*
249 * If the desired number of colors is not a power of
250 * two, it won't be good. Find the greatest power of
251 * two which is an even divisor of the number of colors,
252 * to preserve even coloring of pages.
253 */
254 if (ncolors & (ncolors - 1) ) {
255 int try, picked = 1;
256 for (try = 1; try < ncolors; try *= 2) {
257 if (ncolors % try == 0) picked = try;
258 }
259 if (picked == 1) {
260 panic("desired number of cache colors %d is "
261 " > 1, but not even!", ncolors);
262 }
263 ncolors = picked;
264 }
265 }
266
267 /*
268 * Knowing the size of the largest cache on this CPU, re-color
269 * our pages.
270 */
271 if (ncolors <= uvmexp.ncolors)
272 return;
273 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
274 uvm_page_recolor(ncolors);
275 }
276
277
278 static void
279 cpu_attach(device_t parent, device_t self, void *aux)
280 {
281 struct cpu_softc *sc = device_private(self);
282 struct cpu_attach_args *caa = aux;
283 struct cpu_info *ci;
284 uintptr_t ptr;
285 int cpunum = caa->cpu_number;
286 static bool again;
287
288 sc->sc_dev = self;
289
290 if (cpus_attached == ~0) {
291 aprint_error(": increase MAXCPUS\n");
292 return;
293 }
294
295 /*
296 * If we're an Application Processor, allocate a cpu_info
297 * structure, otherwise use the primary's.
298 */
299 if (caa->cpu_role == CPU_ROLE_AP) {
300 if ((boothowto & RB_MD1) != 0) {
301 aprint_error(": multiprocessor boot disabled\n");
302 if (!pmf_device_register(self, NULL, NULL))
303 aprint_error_dev(self,
304 "couldn't establish power handler\n");
305 return;
306 }
307 aprint_naive(": Application Processor\n");
308 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
309 KM_SLEEP);
310 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
311 ci->ci_curldt = -1;
312 #ifdef TRAPLOG
313 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
314 #endif
315 } else {
316 aprint_naive(": %s Processor\n",
317 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
318 ci = &cpu_info_primary;
319 if (cpunum != lapic_cpu_number()) {
320 /* XXX should be done earlier. */
321 uint32_t reg;
322 aprint_verbose("\n");
323 aprint_verbose_dev(self, "running CPU at apic %d"
324 " instead of at expected %d", lapic_cpu_number(),
325 cpunum);
326 reg = i82489_readreg(LAPIC_ID);
327 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
328 (cpunum << LAPIC_ID_SHIFT));
329 }
330 if (cpunum != lapic_cpu_number()) {
331 aprint_error_dev(self, "unable to reset apic id\n");
332 }
333 }
334
335 ci->ci_self = ci;
336 sc->sc_info = ci;
337 ci->ci_dev = self;
338 ci->ci_acpiid = caa->cpu_id;
339 ci->ci_cpuid = caa->cpu_number;
340 ci->ci_func = caa->cpu_func;
341
342 /* Must be before mi_cpu_attach(). */
343 cpu_vm_init(ci);
344
345 if (caa->cpu_role == CPU_ROLE_AP) {
346 int error;
347
348 error = mi_cpu_attach(ci);
349 if (error != 0) {
350 aprint_normal("\n");
351 aprint_error_dev(self,
352 "mi_cpu_attach failed with %d\n", error);
353 return;
354 }
355 cpu_init_tss(ci);
356 } else {
357 KASSERT(ci->ci_data.cpu_idlelwp != NULL);
358 }
359
360 ci->ci_cpumask = (1 << cpu_index(ci));
361 pmap_reference(pmap_kernel());
362 ci->ci_pmap = pmap_kernel();
363 ci->ci_tlbstate = TLBSTATE_STALE;
364
365 /*
366 * Boot processor may not be attached first, but the below
367 * must be done to allow booting other processors.
368 */
369 if (!again) {
370 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
371 /* Basic init. */
372 cpu_intr_init(ci);
373 cpu_get_tsc_freq(ci);
374 cpu_init(ci);
375 cpu_set_tss_gates(ci);
376 pmap_cpu_init_late(ci);
377 if (caa->cpu_role != CPU_ROLE_SP) {
378 /* Enable lapic. */
379 lapic_enable();
380 lapic_set_lvt();
381 lapic_calibrate_timer(ci);
382 }
383 /* Make sure DELAY() is initialized. */
384 DELAY(1);
385 again = true;
386 }
387
388 /* further PCB init done later. */
389
390 switch (caa->cpu_role) {
391 case CPU_ROLE_SP:
392 atomic_or_32(&ci->ci_flags, CPUF_SP);
393 cpu_identify(ci);
394 x86_errata();
395 x86_cpu_idle_init();
396 break;
397
398 case CPU_ROLE_BP:
399 atomic_or_32(&ci->ci_flags, CPUF_BSP);
400 cpu_identify(ci);
401 x86_errata();
402 x86_cpu_idle_init();
403 break;
404
405 case CPU_ROLE_AP:
406 /*
407 * report on an AP
408 */
409 cpu_intr_init(ci);
410 gdt_alloc_cpu(ci);
411 cpu_set_tss_gates(ci);
412 pmap_cpu_init_late(ci);
413 cpu_start_secondary(ci);
414 if (ci->ci_flags & CPUF_PRESENT) {
415 struct cpu_info *tmp;
416
417 cpu_identify(ci);
418 tmp = cpu_info_list;
419 while (tmp->ci_next)
420 tmp = tmp->ci_next;
421
422 tmp->ci_next = ci;
423 }
424 break;
425
426 default:
427 aprint_normal("\n");
428 panic("unknown processor type??\n");
429 }
430
431 pat_init(ci);
432 atomic_or_32(&cpus_attached, ci->ci_cpumask);
433
434 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown))
435 aprint_error_dev(self, "couldn't establish power handler\n");
436
437 if (mp_verbose) {
438 struct lwp *l = ci->ci_data.cpu_idlelwp;
439 struct pcb *pcb = lwp_getpcb(l);
440
441 aprint_verbose_dev(self,
442 "idle lwp at %p, idle sp at %p\n",
443 l,
444 #ifdef i386
445 (void *)pcb->pcb_esp
446 #else
447 (void *)pcb->pcb_rsp
448 #endif
449 );
450 }
451
452 /*
453 * Postpone the "cpufeaturebus" scan.
454 * It is safe to scan the pseudo-bus
455 * only after all CPUs have attached.
456 */
457 (void)config_defer(self, cpu_defer);
458 }
459
460 static void
461 cpu_defer(device_t self)
462 {
463 cpu_rescan(self, NULL, NULL);
464 }
465
466 static int
467 cpu_rescan(device_t self, const char *ifattr, const int *locators)
468 {
469 struct cpu_softc *sc = device_private(self);
470 struct cpufeature_attach_args cfaa;
471 struct cpu_info *ci = sc->sc_info;
472
473 memset(&cfaa, 0, sizeof(cfaa));
474 cfaa.ci = ci;
475
476 if (ifattr_match(ifattr, "cpufeaturebus")) {
477
478 if (ci->ci_frequency == NULL) {
479 cfaa.name = "frequency";
480 ci->ci_frequency = config_found_ia(self,
481 "cpufeaturebus", &cfaa, NULL);
482 }
483
484 if (ci->ci_padlock == NULL) {
485 cfaa.name = "padlock";
486 ci->ci_padlock = config_found_ia(self,
487 "cpufeaturebus", &cfaa, NULL);
488 }
489
490 if (ci->ci_temperature == NULL) {
491 cfaa.name = "temperature";
492 ci->ci_temperature = config_found_ia(self,
493 "cpufeaturebus", &cfaa, NULL);
494 }
495 }
496
497 return 0;
498 }
499
500 static void
501 cpu_childdetached(device_t self, device_t child)
502 {
503 struct cpu_softc *sc = device_private(self);
504 struct cpu_info *ci = sc->sc_info;
505
506 if (ci->ci_frequency == child)
507 ci->ci_frequency = NULL;
508
509 if (ci->ci_padlock == child)
510 ci->ci_padlock = NULL;
511
512 if (ci->ci_temperature == child)
513 ci->ci_temperature = NULL;
514 }
515
516 /*
517 * Initialize the processor appropriately.
518 */
519
520 void
521 cpu_init(struct cpu_info *ci)
522 {
523
524 lcr0(rcr0() | CR0_WP);
525
526 /*
527 * On a P6 or above, enable global TLB caching if the
528 * hardware supports it.
529 */
530 if (cpu_feature[0] & CPUID_PGE)
531 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */
532
533 /*
534 * If we have FXSAVE/FXRESTOR, use them.
535 */
536 if (cpu_feature[0] & CPUID_FXSR) {
537 lcr4(rcr4() | CR4_OSFXSR);
538
539 /*
540 * If we have SSE/SSE2, enable XMM exceptions.
541 */
542 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
543 lcr4(rcr4() | CR4_OSXMMEXCPT);
544 }
545
546 #ifdef MTRR
547 /*
548 * On a P6 or above, initialize MTRR's if the hardware supports them.
549 */
550 if (cpu_feature[0] & CPUID_MTRR) {
551 if ((ci->ci_flags & CPUF_AP) == 0)
552 i686_mtrr_init_first();
553 mtrr_init_cpu(ci);
554 }
555
556 #ifdef i386
557 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
558 /*
559 * Must be a K6-2 Step >= 7 or a K6-III.
560 */
561 if (CPUID2FAMILY(ci->ci_signature) == 5) {
562 if (CPUID2MODEL(ci->ci_signature) > 8 ||
563 (CPUID2MODEL(ci->ci_signature) == 8 &&
564 CPUID2STEPPING(ci->ci_signature) >= 7)) {
565 mtrr_funcs = &k6_mtrr_funcs;
566 k6_mtrr_init_first();
567 mtrr_init_cpu(ci);
568 }
569 }
570 }
571 #endif /* i386 */
572 #endif /* MTRR */
573
574 atomic_or_32(&cpus_running, ci->ci_cpumask);
575
576 if (ci != &cpu_info_primary) {
577 /* Synchronize TSC again, and check for drift. */
578 wbinvd();
579 atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
580 tsc_sync_ap(ci);
581 } else {
582 atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
583 }
584 }
585
586 void
587 cpu_boot_secondary_processors(void)
588 {
589 struct cpu_info *ci;
590 u_long i;
591
592 /* Now that we know the number of CPUs, patch the text segment. */
593 x86_patch(false);
594
595 for (i=0; i < maxcpus; i++) {
596 ci = cpu_lookup(i);
597 if (ci == NULL)
598 continue;
599 if (ci->ci_data.cpu_idlelwp == NULL)
600 continue;
601 if ((ci->ci_flags & CPUF_PRESENT) == 0)
602 continue;
603 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
604 continue;
605 cpu_boot_secondary(ci);
606 }
607
608 x86_mp_online = true;
609
610 /* Now that we know about the TSC, attach the timecounter. */
611 tsc_tc_init();
612
613 /* Enable zeroing of pages in the idle loop if we have SSE2. */
614 vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0);
615 }
616
617 static void
618 cpu_init_idle_lwp(struct cpu_info *ci)
619 {
620 struct lwp *l = ci->ci_data.cpu_idlelwp;
621 struct pcb *pcb = lwp_getpcb(l);
622
623 pcb->pcb_cr0 = rcr0();
624 }
625
626 void
627 cpu_init_idle_lwps(void)
628 {
629 struct cpu_info *ci;
630 u_long i;
631
632 for (i = 0; i < maxcpus; i++) {
633 ci = cpu_lookup(i);
634 if (ci == NULL)
635 continue;
636 if (ci->ci_data.cpu_idlelwp == NULL)
637 continue;
638 if ((ci->ci_flags & CPUF_PRESENT) == 0)
639 continue;
640 cpu_init_idle_lwp(ci);
641 }
642 }
643
644 void
645 cpu_start_secondary(struct cpu_info *ci)
646 {
647 extern paddr_t mp_pdirpa;
648 u_long psl;
649 int i;
650
651 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr);
652 atomic_or_32(&ci->ci_flags, CPUF_AP);
653 ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
654 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) {
655 return;
656 }
657
658 /*
659 * Wait for it to become ready. Setting cpu_starting opens the
660 * initial gate and allows the AP to start soft initialization.
661 */
662 KASSERT(cpu_starting == NULL);
663 cpu_starting = ci;
664 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
665 #ifdef MPDEBUG
666 extern int cpu_trace[3];
667 static int otrace[3];
668 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
669 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
670 cpu_trace[0], cpu_trace[1], cpu_trace[2]);
671 memcpy(otrace, cpu_trace, sizeof(otrace));
672 }
673 #endif
674 i8254_delay(10);
675 }
676
677 if ((ci->ci_flags & CPUF_PRESENT) == 0) {
678 aprint_error_dev(ci->ci_dev, "failed to become ready\n");
679 #if defined(MPDEBUG) && defined(DDB)
680 printf("dropping into debugger; continue from here to resume boot\n");
681 Debugger();
682 #endif
683 } else {
684 /*
685 * Synchronize time stamp counters. Invalidate cache and do
686 * twice to try and minimize possible cache effects. Disable
687 * interrupts to try and rule out any external interference.
688 */
689 psl = x86_read_psl();
690 x86_disable_intr();
691 wbinvd();
692 tsc_sync_bp(ci);
693 x86_write_psl(psl);
694 }
695
696 CPU_START_CLEANUP(ci);
697 cpu_starting = NULL;
698 }
699
700 void
701 cpu_boot_secondary(struct cpu_info *ci)
702 {
703 int64_t drift;
704 u_long psl;
705 int i;
706
707 atomic_or_32(&ci->ci_flags, CPUF_GO);
708 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
709 i8254_delay(10);
710 }
711 if ((ci->ci_flags & CPUF_RUNNING) == 0) {
712 aprint_error_dev(ci->ci_dev, "failed to start\n");
713 #if defined(MPDEBUG) && defined(DDB)
714 printf("dropping into debugger; continue from here to resume boot\n");
715 Debugger();
716 #endif
717 } else {
718 /* Synchronize TSC again, check for drift. */
719 drift = ci->ci_data.cpu_cc_skew;
720 psl = x86_read_psl();
721 x86_disable_intr();
722 wbinvd();
723 tsc_sync_bp(ci);
724 x86_write_psl(psl);
725 drift -= ci->ci_data.cpu_cc_skew;
726 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n",
727 (long long)ci->ci_data.cpu_cc_skew, (long long)drift);
728 tsc_sync_drift(drift);
729 }
730 }
731
732 /*
733 * The CPU ends up here when its ready to run
734 * This is called from code in mptramp.s; at this point, we are running
735 * in the idle pcb/idle stack of the new CPU. When this function returns,
736 * this processor will enter the idle loop and start looking for work.
737 */
738 void
739 cpu_hatch(void *v)
740 {
741 struct cpu_info *ci = (struct cpu_info *)v;
742 struct pcb *pcb;
743 int s, i;
744
745 cpu_init_msrs(ci, true);
746 cpu_probe(ci);
747
748 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq;
749 /* cpu_get_tsc_freq(ci); */
750
751 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
752
753 /*
754 * Synchronize time stamp counters. Invalidate cache and do twice
755 * to try and minimize possible cache effects. Note that interrupts
756 * are off at this point.
757 */
758 wbinvd();
759 atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
760 tsc_sync_ap(ci);
761
762 /*
763 * Wait to be brought online. Use 'monitor/mwait' if available,
764 * in order to make the TSC drift as much as possible. so that
765 * we can detect it later. If not available, try 'pause'.
766 * We'd like to use 'hlt', but we have interrupts off.
767 */
768 while ((ci->ci_flags & CPUF_GO) == 0) {
769 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) {
770 x86_monitor(&ci->ci_flags, 0, 0);
771 if ((ci->ci_flags & CPUF_GO) != 0) {
772 continue;
773 }
774 x86_mwait(0, 0);
775 } else {
776 for (i = 10000; i != 0; i--) {
777 x86_pause();
778 }
779 }
780 }
781
782 /* Because the text may have been patched in x86_patch(). */
783 wbinvd();
784 x86_flush();
785 tlbflushg();
786
787 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
788
789 #ifdef PAE
790 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir;
791 for (i = 0 ; i < PDP_SIZE; i++) {
792 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V;
793 }
794 lcr3(ci->ci_pae_l3_pdirpa);
795 #else
796 lcr3(pmap_pdirpa(pmap_kernel(), 0));
797 #endif
798
799 pcb = lwp_getpcb(curlwp);
800 pcb->pcb_cr3 = rcr3();
801 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
802 lcr0(pcb->pcb_cr0);
803
804 cpu_init_idt();
805 gdt_init_cpu(ci);
806 lapic_enable();
807 lapic_set_lvt();
808 lapic_initclocks();
809
810 #ifdef i386
811 #if NNPX > 0
812 npxinit(ci);
813 #endif
814 #else
815 fpuinit(ci);
816 #endif
817 lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
818 ltr(ci->ci_tss_sel);
819
820 cpu_init(ci);
821 cpu_get_tsc_freq(ci);
822
823 s = splhigh();
824 #ifdef i386
825 lapic_tpr = 0;
826 #else
827 lcr8(0);
828 #endif
829 x86_enable_intr();
830 splx(s);
831 x86_errata();
832
833 aprint_debug_dev(ci->ci_dev, "running\n");
834 }
835
836 #if defined(DDB)
837
838 #include <ddb/db_output.h>
839 #include <machine/db_machdep.h>
840
841 /*
842 * Dump CPU information from ddb.
843 */
844 void
845 cpu_debug_dump(void)
846 {
847 struct cpu_info *ci;
848 CPU_INFO_ITERATOR cii;
849
850 db_printf("addr dev id flags ipis curlwp fpcurlwp\n");
851 for (CPU_INFO_FOREACH(cii, ci)) {
852 db_printf("%p %s %ld %x %x %10p %10p\n",
853 ci,
854 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
855 (long)ci->ci_cpuid,
856 ci->ci_flags, ci->ci_ipis,
857 ci->ci_curlwp,
858 ci->ci_fpcurlwp);
859 }
860 }
861 #endif
862
863 static void
864 cpu_copy_trampoline(void)
865 {
866 /*
867 * Copy boot code.
868 */
869 extern u_char cpu_spinup_trampoline[];
870 extern u_char cpu_spinup_trampoline_end[];
871
872 vaddr_t mp_trampoline_vaddr;
873
874 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
875 UVM_KMF_VAONLY);
876
877 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
878 VM_PROT_READ | VM_PROT_WRITE, 0);
879 pmap_update(pmap_kernel());
880 memcpy((void *)mp_trampoline_vaddr,
881 cpu_spinup_trampoline,
882 cpu_spinup_trampoline_end - cpu_spinup_trampoline);
883
884 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
885 pmap_update(pmap_kernel());
886 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
887 }
888
889 #ifdef i386
890 static void
891 tss_init(struct i386tss *tss, void *stack, void *func)
892 {
893 KASSERT(curcpu()->ci_pmap == pmap_kernel());
894
895 memset(tss, 0, sizeof *tss);
896 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
897 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
898 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
899 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
900 tss->tss_gs = tss->__tss_es = tss->__tss_ds =
901 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
902 /* %cr3 contains the value associated to pmap_kernel */
903 tss->tss_cr3 = rcr3();
904 tss->tss_esp = (int)((char *)stack + USPACE - 16);
905 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
906 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */
907 tss->__tss_eip = (int)func;
908 }
909
910 /* XXX */
911 #define IDTVEC(name) __CONCAT(X, name)
912 typedef void (vector)(void);
913 extern vector IDTVEC(tss_trap08);
914 #ifdef DDB
915 extern vector Xintrddbipi;
916 extern int ddb_vec;
917 #endif
918
919 static void
920 cpu_set_tss_gates(struct cpu_info *ci)
921 {
922 struct segment_descriptor sd;
923
924 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
925 UVM_KMF_WIRED);
926 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
927 IDTVEC(tss_trap08));
928 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
929 SDT_SYS386TSS, SEL_KPL, 0, 0);
930 ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
931 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
932 GSEL(GTRAPTSS_SEL, SEL_KPL));
933
934 #if defined(DDB)
935 /*
936 * Set up separate handler for the DDB IPI, so that it doesn't
937 * stomp on a possibly corrupted stack.
938 *
939 * XXX overwriting the gate set in db_machine_init.
940 * Should rearrange the code so that it's set only once.
941 */
942 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
943 UVM_KMF_WIRED);
944 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi);
945
946 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
947 SDT_SYS386TSS, SEL_KPL, 0, 0);
948 ci->ci_gdt[GIPITSS_SEL].sd = sd;
949
950 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
951 GSEL(GIPITSS_SEL, SEL_KPL));
952 #endif
953 }
954 #else
955 static void
956 cpu_set_tss_gates(struct cpu_info *ci)
957 {
958
959 }
960 #endif /* i386 */
961
962 int
963 mp_cpu_start(struct cpu_info *ci, paddr_t target)
964 {
965 unsigned short dwordptr[2];
966 int error;
967
968 /*
969 * Bootstrap code must be addressable in real mode
970 * and it must be page aligned.
971 */
972 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
973
974 /*
975 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
976 */
977
978 outb(IO_RTC, NVRAM_RESET);
979 outb(IO_RTC+1, NVRAM_RESET_JUMP);
980
981 /*
982 * "and the warm reset vector (DWORD based at 40:67) to point
983 * to the AP startup code ..."
984 */
985
986 dwordptr[0] = 0;
987 dwordptr[1] = target >> 4;
988
989 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4);
990
991 if ((cpu_feature[0] & CPUID_APIC) == 0) {
992 aprint_error("mp_cpu_start: CPU does not have APIC\n");
993 return ENODEV;
994 }
995
996 /*
997 * ... prior to executing the following sequence:". We'll also add in
998 * local cache flush, in case the BIOS has left the AP with its cache
999 * disabled. It may not be able to cope with MP coherency.
1000 */
1001 wbinvd();
1002
1003 if (ci->ci_flags & CPUF_AP) {
1004 error = x86_ipi_init(ci->ci_cpuid);
1005 if (error != 0) {
1006 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
1007 __func__);
1008 return error;
1009 }
1010 i8254_delay(10000);
1011
1012 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
1013 if (error != 0) {
1014 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
1015 __func__);
1016 return error;
1017 }
1018 i8254_delay(200);
1019
1020 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
1021 if (error != 0) {
1022 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n",
1023 __func__);
1024 return error;
1025 }
1026 i8254_delay(200);
1027 }
1028
1029 return 0;
1030 }
1031
1032 void
1033 mp_cpu_start_cleanup(struct cpu_info *ci)
1034 {
1035 /*
1036 * Ensure the NVRAM reset byte contains something vaguely sane.
1037 */
1038
1039 outb(IO_RTC, NVRAM_RESET);
1040 outb(IO_RTC+1, NVRAM_RESET_RST);
1041 }
1042
1043 #ifdef __x86_64__
1044 typedef void (vector)(void);
1045 extern vector Xsyscall, Xsyscall32;
1046 #endif
1047
1048 void
1049 cpu_init_msrs(struct cpu_info *ci, bool full)
1050 {
1051 #ifdef __x86_64__
1052 wrmsr(MSR_STAR,
1053 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1054 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
1055 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
1056 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
1057 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
1058
1059 if (full) {
1060 wrmsr(MSR_FSBASE, 0);
1061 wrmsr(MSR_GSBASE, (uint64_t)ci);
1062 wrmsr(MSR_KERNELGSBASE, 0);
1063 }
1064 #endif /* __x86_64__ */
1065
1066 if (cpu_feature[2] & CPUID_NOX)
1067 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
1068 }
1069
1070 void
1071 cpu_offline_md(void)
1072 {
1073 int s;
1074
1075 s = splhigh();
1076 #ifdef i386
1077 #if NNPX > 0
1078 npxsave_cpu(true);
1079 #endif
1080 #else
1081 fpusave_cpu(true);
1082 #endif
1083 splx(s);
1084 }
1085
1086 /* XXX joerg restructure and restart CPUs individually */
1087 static bool
1088 cpu_suspend(device_t dv, const pmf_qual_t *qual)
1089 {
1090 struct cpu_softc *sc = device_private(dv);
1091 struct cpu_info *ci = sc->sc_info;
1092 int err;
1093
1094 if ((ci->ci_flags & CPUF_PRESENT) == 0)
1095 return true;
1096
1097 cpufreq_suspend(ci);
1098
1099 if ((ci->ci_flags & CPUF_PRIMARY) != 0)
1100 return true;
1101
1102 if (ci->ci_data.cpu_idlelwp == NULL)
1103 return true;
1104
1105 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
1106
1107 if (sc->sc_wasonline) {
1108 mutex_enter(&cpu_lock);
1109 err = cpu_setstate(ci, false);
1110 mutex_exit(&cpu_lock);
1111
1112 if (err != 0)
1113 return false;
1114 }
1115
1116 return true;
1117 }
1118
1119 static bool
1120 cpu_resume(device_t dv, const pmf_qual_t *qual)
1121 {
1122 struct cpu_softc *sc = device_private(dv);
1123 struct cpu_info *ci = sc->sc_info;
1124 int err = 0;
1125
1126 if ((ci->ci_flags & CPUF_PRESENT) == 0)
1127 return true;
1128
1129 if ((ci->ci_flags & CPUF_PRIMARY) != 0)
1130 goto out;
1131
1132 if (ci->ci_data.cpu_idlelwp == NULL)
1133 goto out;
1134
1135 if (sc->sc_wasonline) {
1136 mutex_enter(&cpu_lock);
1137 err = cpu_setstate(ci, true);
1138 mutex_exit(&cpu_lock);
1139 }
1140
1141 out:
1142 if (err != 0)
1143 return false;
1144
1145 cpufreq_resume(ci);
1146
1147 return true;
1148 }
1149
1150 static bool
1151 cpu_shutdown(device_t dv, int how)
1152 {
1153 struct cpu_softc *sc = device_private(dv);
1154 struct cpu_info *ci = sc->sc_info;
1155
1156 if (ci->ci_flags & CPUF_BSP)
1157 return false;
1158
1159 return cpu_suspend(dv, NULL);
1160 }
1161
1162 void
1163 cpu_get_tsc_freq(struct cpu_info *ci)
1164 {
1165 uint64_t last_tsc;
1166
1167 if (cpu_hascounter()) {
1168 last_tsc = cpu_counter_serializing();
1169 i8254_delay(100000);
1170 ci->ci_data.cpu_cc_freq =
1171 (cpu_counter_serializing() - last_tsc) * 10;
1172 }
1173 }
1174
1175 void
1176 x86_cpu_idle_mwait(void)
1177 {
1178 struct cpu_info *ci = curcpu();
1179
1180 KASSERT(ci->ci_ilevel == IPL_NONE);
1181
1182 x86_monitor(&ci->ci_want_resched, 0, 0);
1183 if (__predict_false(ci->ci_want_resched)) {
1184 return;
1185 }
1186 x86_mwait(0, 0);
1187 }
1188
1189 void
1190 x86_cpu_idle_halt(void)
1191 {
1192 struct cpu_info *ci = curcpu();
1193
1194 KASSERT(ci->ci_ilevel == IPL_NONE);
1195
1196 x86_disable_intr();
1197 if (!__predict_false(ci->ci_want_resched)) {
1198 x86_stihlt();
1199 } else {
1200 x86_enable_intr();
1201 }
1202 }
1203
1204 /*
1205 * Loads pmap for the current CPU.
1206 */
1207 void
1208 cpu_load_pmap(struct pmap *pmap)
1209 {
1210 #ifdef PAE
1211 int i, s;
1212 struct cpu_info *ci;
1213
1214 s = splvm(); /* just to be safe */
1215 ci = curcpu();
1216 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir;
1217 for (i = 0 ; i < PDP_SIZE; i++) {
1218 l3_pd[i] = pmap->pm_pdirpa[i] | PG_V;
1219 }
1220 splx(s);
1221 tlbflush();
1222 #else /* PAE */
1223 lcr3(pmap_pdirpa(pmap, 0));
1224 #endif /* PAE */
1225 }
1226
1227 /*
1228 * Notify all other cpus to halt.
1229 */
1230
1231 void
1232 cpu_broadcast_halt(void)
1233 {
1234 x86_broadcast_ipi(X86_IPI_HALT);
1235 }
1236
1237 /*
1238 * Send a dummy ipi to a cpu to force it to run splraise()/spllower()
1239 */
1240
1241 void
1242 cpu_kick(struct cpu_info *ci)
1243 {
1244 x86_send_ipi(ci, 0);
1245 }
1246