cpu.c revision 1.136 1 /* $NetBSD: cpu.c,v 1.136 2017/09/28 17:48:20 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c) 1999 Stefan Grefen
34 *
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
37 * are met:
38 * 1. Redistributions of source code must retain the above copyright
39 * notice, this list of conditions and the following disclaimer.
40 * 2. Redistributions in binary form must reproduce the above copyright
41 * notice, this list of conditions and the following disclaimer in the
42 * documentation and/or other materials provided with the distribution.
43 * 3. All advertising materials mentioning features or use of this software
44 * must display the following acknowledgement:
45 * This product includes software developed by the NetBSD
46 * Foundation, Inc. and its contributors.
47 * 4. Neither the name of The NetBSD Foundation nor the names of its
48 * contributors may be used to endorse or promote products derived
49 * from this software without specific prior written permission.
50 *
51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
61 * SUCH DAMAGE.
62 */
63
64 #include <sys/cdefs.h>
65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.136 2017/09/28 17:48:20 maxv Exp $");
66
67 #include "opt_ddb.h"
68 #include "opt_mpbios.h" /* for MPDEBUG */
69 #include "opt_mtrr.h"
70 #include "opt_multiprocessor.h"
71
72 #include "lapic.h"
73 #include "ioapic.h"
74
75 #include <sys/param.h>
76 #include <sys/proc.h>
77 #include <sys/systm.h>
78 #include <sys/device.h>
79 #include <sys/kmem.h>
80 #include <sys/cpu.h>
81 #include <sys/cpufreq.h>
82 #include <sys/idle.h>
83 #include <sys/atomic.h>
84 #include <sys/reboot.h>
85
86 #include <uvm/uvm.h>
87
88 #include "acpica.h" /* for NACPICA, for mp_verbose */
89
90 #include <machine/cpufunc.h>
91 #include <machine/cpuvar.h>
92 #include <machine/pmap.h>
93 #include <machine/vmparam.h>
94 #if defined(MULTIPROCESSOR)
95 #include <machine/mpbiosvar.h>
96 #endif
97 #include <machine/mpconfig.h> /* for mp_verbose */
98 #include <machine/pcb.h>
99 #include <machine/specialreg.h>
100 #include <machine/segments.h>
101 #include <machine/gdt.h>
102 #include <machine/mtrr.h>
103 #include <machine/pio.h>
104 #include <machine/cpu_counter.h>
105
106 #include <x86/fpu.h>
107
108 #if NLAPIC > 0
109 #include <machine/apicvar.h>
110 #include <machine/i82489reg.h>
111 #include <machine/i82489var.h>
112 #endif
113
114 #include <dev/ic/mc146818reg.h>
115 #include <i386/isa/nvram.h>
116 #include <dev/isa/isareg.h>
117
118 #include "tsc.h"
119
120 static int cpu_match(device_t, cfdata_t, void *);
121 static void cpu_attach(device_t, device_t, void *);
122 static void cpu_defer(device_t);
123 static int cpu_rescan(device_t, const char *, const int *);
124 static void cpu_childdetached(device_t, device_t);
125 static bool cpu_stop(device_t);
126 static bool cpu_suspend(device_t, const pmf_qual_t *);
127 static bool cpu_resume(device_t, const pmf_qual_t *);
128 static bool cpu_shutdown(device_t, int);
129
130 struct cpu_softc {
131 device_t sc_dev; /* device tree glue */
132 struct cpu_info *sc_info; /* pointer to CPU info */
133 bool sc_wasonline;
134 };
135
136 #ifdef MULTIPROCESSOR
137 int mp_cpu_start(struct cpu_info *, paddr_t);
138 void mp_cpu_start_cleanup(struct cpu_info *);
139 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
140 mp_cpu_start_cleanup };
141 #endif
142
143
144 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
145 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
146
147 /*
148 * Statically-allocated CPU info for the primary CPU (or the only
149 * CPU, on uniprocessors). The CPU info list is initialized to
150 * point at it.
151 */
152 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
153 .ci_dev = 0,
154 .ci_self = &cpu_info_primary,
155 .ci_idepth = -1,
156 .ci_curlwp = &lwp0,
157 .ci_curldt = -1,
158 };
159
160 struct cpu_info *cpu_info_list = &cpu_info_primary;
161
162 #ifdef i386
163 void cpu_set_tss_gates(struct cpu_info *);
164 #endif
165
166 static void cpu_init_idle_lwp(struct cpu_info *);
167
168 uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */
169 /* [0] basic features cpuid.1:%edx
170 * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits)
171 * [2] extended features cpuid:80000001:%edx
172 * [3] extended features cpuid:80000001:%ecx
173 * [4] VIA padlock features
174 * [5] structured extended features cpuid.7:%ebx
175 * [6] structured extended features cpuid.7:%ecx
176 */
177
178 #ifdef MULTIPROCESSOR
179 bool x86_mp_online;
180 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
181 #endif
182 #if NLAPIC > 0
183 static vaddr_t cmos_data_mapping;
184 #endif
185 struct cpu_info *cpu_starting;
186
187 #ifdef MULTIPROCESSOR
188 void cpu_hatch(void *);
189 static void cpu_boot_secondary(struct cpu_info *ci);
190 static void cpu_start_secondary(struct cpu_info *ci);
191 #endif
192 #if NLAPIC > 0
193 static void cpu_copy_trampoline(paddr_t);
194 #endif
195
196 /*
197 * Runs once per boot once multiprocessor goo has been detected and
198 * the local APIC on the boot processor has been mapped.
199 *
200 * Called from lapic_boot_init() (from mpbios_scan()).
201 */
202 #if NLAPIC > 0
203 void
204 cpu_init_first(void)
205 {
206
207 cpu_info_primary.ci_cpuid = lapic_cpu_number();
208
209 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY);
210 if (cmos_data_mapping == 0)
211 panic("No KVA for page 0");
212 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
213 pmap_update(pmap_kernel());
214 }
215 #endif
216
217 static int
218 cpu_match(device_t parent, cfdata_t match, void *aux)
219 {
220
221 return 1;
222 }
223
224 static void
225 cpu_vm_init(struct cpu_info *ci)
226 {
227 int ncolors = 2, i;
228
229 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
230 struct x86_cache_info *cai;
231 int tcolors;
232
233 cai = &ci->ci_cinfo[i];
234
235 tcolors = atop(cai->cai_totalsize);
236 switch(cai->cai_associativity) {
237 case 0xff:
238 tcolors = 1; /* fully associative */
239 break;
240 case 0:
241 case 1:
242 break;
243 default:
244 tcolors /= cai->cai_associativity;
245 }
246 ncolors = max(ncolors, tcolors);
247 /*
248 * If the desired number of colors is not a power of
249 * two, it won't be good. Find the greatest power of
250 * two which is an even divisor of the number of colors,
251 * to preserve even coloring of pages.
252 */
253 if (ncolors & (ncolors - 1) ) {
254 int try, picked = 1;
255 for (try = 1; try < ncolors; try *= 2) {
256 if (ncolors % try == 0) picked = try;
257 }
258 if (picked == 1) {
259 panic("desired number of cache colors %d is "
260 " > 1, but not even!", ncolors);
261 }
262 ncolors = picked;
263 }
264 }
265
266 /*
267 * Knowing the size of the largest cache on this CPU, potentially
268 * re-color our pages.
269 */
270 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
271 uvm_page_recolor(ncolors);
272
273 pmap_tlb_cpu_init(ci);
274 #ifndef __HAVE_DIRECT_MAP
275 pmap_vpage_cpu_init(ci);
276 #endif
277 }
278
279 static void
280 cpu_attach(device_t parent, device_t self, void *aux)
281 {
282 struct cpu_softc *sc = device_private(self);
283 struct cpu_attach_args *caa = aux;
284 struct cpu_info *ci;
285 uintptr_t ptr;
286 #if NLAPIC > 0
287 int cpunum = caa->cpu_number;
288 #endif
289 static bool again;
290
291 sc->sc_dev = self;
292
293 if (ncpu == maxcpus) {
294 #ifndef _LP64
295 aprint_error(": too many CPUs, please use NetBSD/amd64\n");
296 #else
297 aprint_error(": too many CPUs\n");
298 #endif
299 return;
300 }
301
302 /*
303 * If we're an Application Processor, allocate a cpu_info
304 * structure, otherwise use the primary's.
305 */
306 if (caa->cpu_role == CPU_ROLE_AP) {
307 if ((boothowto & RB_MD1) != 0) {
308 aprint_error(": multiprocessor boot disabled\n");
309 if (!pmf_device_register(self, NULL, NULL))
310 aprint_error_dev(self,
311 "couldn't establish power handler\n");
312 return;
313 }
314 aprint_naive(": Application Processor\n");
315 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
316 KM_SLEEP);
317 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
318 ci->ci_curldt = -1;
319 } else {
320 aprint_naive(": %s Processor\n",
321 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
322 ci = &cpu_info_primary;
323 #if NLAPIC > 0
324 if (cpunum != lapic_cpu_number()) {
325 /* XXX should be done earlier. */
326 uint32_t reg;
327 aprint_verbose("\n");
328 aprint_verbose_dev(self, "running CPU at apic %d"
329 " instead of at expected %d", lapic_cpu_number(),
330 cpunum);
331 reg = lapic_readreg(LAPIC_ID);
332 lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
333 (cpunum << LAPIC_ID_SHIFT));
334 }
335 if (cpunum != lapic_cpu_number()) {
336 aprint_error_dev(self, "unable to reset apic id\n");
337 }
338 #endif
339 }
340
341 ci->ci_self = ci;
342 sc->sc_info = ci;
343 ci->ci_dev = self;
344 ci->ci_acpiid = caa->cpu_id;
345 ci->ci_cpuid = caa->cpu_number;
346 ci->ci_func = caa->cpu_func;
347 aprint_normal("\n");
348
349 /* Must be before mi_cpu_attach(). */
350 cpu_vm_init(ci);
351
352 if (caa->cpu_role == CPU_ROLE_AP) {
353 int error;
354
355 error = mi_cpu_attach(ci);
356 if (error != 0) {
357 aprint_error_dev(self,
358 "mi_cpu_attach failed with %d\n", error);
359 return;
360 }
361 cpu_init_tss(ci);
362 } else {
363 KASSERT(ci->ci_data.cpu_idlelwp != NULL);
364 }
365
366 pmap_reference(pmap_kernel());
367 ci->ci_pmap = pmap_kernel();
368 ci->ci_tlbstate = TLBSTATE_STALE;
369
370 /*
371 * Boot processor may not be attached first, but the below
372 * must be done to allow booting other processors.
373 */
374 if (!again) {
375 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
376 /* Basic init. */
377 cpu_intr_init(ci);
378 cpu_get_tsc_freq(ci);
379 cpu_init(ci);
380 #ifdef i386
381 cpu_set_tss_gates(ci);
382 #endif
383 pmap_cpu_init_late(ci);
384 #if NLAPIC > 0
385 if (caa->cpu_role != CPU_ROLE_SP) {
386 /* Enable lapic. */
387 lapic_enable();
388 lapic_set_lvt();
389 lapic_calibrate_timer(ci);
390 }
391 #endif
392 /* Make sure DELAY() is initialized. */
393 DELAY(1);
394 again = true;
395 }
396
397 /* further PCB init done later. */
398
399 switch (caa->cpu_role) {
400 case CPU_ROLE_SP:
401 atomic_or_32(&ci->ci_flags, CPUF_SP);
402 cpu_identify(ci);
403 x86_errata();
404 x86_cpu_idle_init();
405 break;
406
407 case CPU_ROLE_BP:
408 atomic_or_32(&ci->ci_flags, CPUF_BSP);
409 cpu_identify(ci);
410 x86_errata();
411 x86_cpu_idle_init();
412 break;
413
414 #ifdef MULTIPROCESSOR
415 case CPU_ROLE_AP:
416 /*
417 * report on an AP
418 */
419 cpu_intr_init(ci);
420 gdt_alloc_cpu(ci);
421 #ifdef i386
422 cpu_set_tss_gates(ci);
423 #endif
424 pmap_cpu_init_late(ci);
425 cpu_start_secondary(ci);
426 if (ci->ci_flags & CPUF_PRESENT) {
427 struct cpu_info *tmp;
428
429 cpu_identify(ci);
430 tmp = cpu_info_list;
431 while (tmp->ci_next)
432 tmp = tmp->ci_next;
433
434 tmp->ci_next = ci;
435 }
436 break;
437 #endif
438
439 default:
440 panic("unknown processor type??\n");
441 }
442
443 pat_init(ci);
444
445 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown))
446 aprint_error_dev(self, "couldn't establish power handler\n");
447
448 #ifdef MULTIPROCESSOR
449 if (mp_verbose) {
450 struct lwp *l = ci->ci_data.cpu_idlelwp;
451 struct pcb *pcb = lwp_getpcb(l);
452
453 aprint_verbose_dev(self,
454 "idle lwp at %p, idle sp at %p\n",
455 l,
456 #ifdef i386
457 (void *)pcb->pcb_esp
458 #else
459 (void *)pcb->pcb_rsp
460 #endif
461 );
462 }
463 #endif
464
465 /*
466 * Postpone the "cpufeaturebus" scan.
467 * It is safe to scan the pseudo-bus
468 * only after all CPUs have attached.
469 */
470 (void)config_defer(self, cpu_defer);
471 }
472
473 static void
474 cpu_defer(device_t self)
475 {
476 cpu_rescan(self, NULL, NULL);
477 }
478
479 static int
480 cpu_rescan(device_t self, const char *ifattr, const int *locators)
481 {
482 struct cpu_softc *sc = device_private(self);
483 struct cpufeature_attach_args cfaa;
484 struct cpu_info *ci = sc->sc_info;
485
486 memset(&cfaa, 0, sizeof(cfaa));
487 cfaa.ci = ci;
488
489 if (ifattr_match(ifattr, "cpufeaturebus")) {
490 if (ci->ci_frequency == NULL) {
491 cfaa.name = "frequency";
492 ci->ci_frequency = config_found_ia(self,
493 "cpufeaturebus", &cfaa, NULL);
494 }
495
496 if (ci->ci_padlock == NULL) {
497 cfaa.name = "padlock";
498 ci->ci_padlock = config_found_ia(self,
499 "cpufeaturebus", &cfaa, NULL);
500 }
501
502 if (ci->ci_temperature == NULL) {
503 cfaa.name = "temperature";
504 ci->ci_temperature = config_found_ia(self,
505 "cpufeaturebus", &cfaa, NULL);
506 }
507
508 if (ci->ci_vm == NULL) {
509 cfaa.name = "vm";
510 ci->ci_vm = config_found_ia(self,
511 "cpufeaturebus", &cfaa, NULL);
512 }
513 }
514
515 return 0;
516 }
517
518 static void
519 cpu_childdetached(device_t self, device_t child)
520 {
521 struct cpu_softc *sc = device_private(self);
522 struct cpu_info *ci = sc->sc_info;
523
524 if (ci->ci_frequency == child)
525 ci->ci_frequency = NULL;
526
527 if (ci->ci_padlock == child)
528 ci->ci_padlock = NULL;
529
530 if (ci->ci_temperature == child)
531 ci->ci_temperature = NULL;
532
533 if (ci->ci_vm == child)
534 ci->ci_vm = NULL;
535 }
536
537 /*
538 * Initialize the processor appropriately.
539 */
540
541 void
542 cpu_init(struct cpu_info *ci)
543 {
544 uint32_t cr4 = 0;
545
546 lcr0(rcr0() | CR0_WP);
547
548 /*
549 * On a P6 or above, enable global TLB caching if the
550 * hardware supports it.
551 */
552 if (cpu_feature[0] & CPUID_PGE)
553 cr4 |= CR4_PGE; /* enable global TLB caching */
554
555 /*
556 * If we have FXSAVE/FXRESTOR, use them.
557 */
558 if (cpu_feature[0] & CPUID_FXSR) {
559 cr4 |= CR4_OSFXSR;
560
561 /*
562 * If we have SSE/SSE2, enable XMM exceptions.
563 */
564 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
565 cr4 |= CR4_OSXMMEXCPT;
566 }
567
568 /* If xsave is supported, enable it */
569 if (cpu_feature[1] & CPUID2_XSAVE)
570 cr4 |= CR4_OSXSAVE;
571
572 /* If SMEP is supported, enable it */
573 if (cpu_feature[5] & CPUID_SEF_SMEP)
574 cr4 |= CR4_SMEP;
575
576 if (cr4) {
577 cr4 |= rcr4();
578 lcr4(cr4);
579 }
580
581 /* If xsave is enabled, enable all fpu features */
582 if (cr4 & CR4_OSXSAVE)
583 wrxcr(0, x86_xsave_features & XCR0_FPU);
584
585 #ifdef MTRR
586 /*
587 * On a P6 or above, initialize MTRR's if the hardware supports them.
588 */
589 if (cpu_feature[0] & CPUID_MTRR) {
590 if ((ci->ci_flags & CPUF_AP) == 0)
591 i686_mtrr_init_first();
592 mtrr_init_cpu(ci);
593 }
594
595 #ifdef i386
596 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
597 /*
598 * Must be a K6-2 Step >= 7 or a K6-III.
599 */
600 if (CPUID_TO_FAMILY(ci->ci_signature) == 5) {
601 if (CPUID_TO_MODEL(ci->ci_signature) > 8 ||
602 (CPUID_TO_MODEL(ci->ci_signature) == 8 &&
603 CPUID_TO_STEPPING(ci->ci_signature) >= 7)) {
604 mtrr_funcs = &k6_mtrr_funcs;
605 k6_mtrr_init_first();
606 mtrr_init_cpu(ci);
607 }
608 }
609 }
610 #endif /* i386 */
611 #endif /* MTRR */
612
613 if (ci != &cpu_info_primary) {
614 /* Synchronize TSC again, and check for drift. */
615 wbinvd();
616 atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
617 tsc_sync_ap(ci);
618 } else {
619 atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
620 }
621 }
622
623 #ifdef MULTIPROCESSOR
624 void
625 cpu_boot_secondary_processors(void)
626 {
627 struct cpu_info *ci;
628 kcpuset_t *cpus;
629 u_long i;
630
631 /* Now that we know the number of CPUs, patch the text segment. */
632 x86_patch(false);
633
634 kcpuset_create(&cpus, true);
635 kcpuset_set(cpus, cpu_index(curcpu()));
636 for (i = 0; i < maxcpus; i++) {
637 ci = cpu_lookup(i);
638 if (ci == NULL)
639 continue;
640 if (ci->ci_data.cpu_idlelwp == NULL)
641 continue;
642 if ((ci->ci_flags & CPUF_PRESENT) == 0)
643 continue;
644 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
645 continue;
646 cpu_boot_secondary(ci);
647 kcpuset_set(cpus, cpu_index(ci));
648 }
649 while (!kcpuset_match(cpus, kcpuset_running))
650 ;
651 kcpuset_destroy(cpus);
652
653 x86_mp_online = true;
654
655 /* Now that we know about the TSC, attach the timecounter. */
656 tsc_tc_init();
657
658 /* Enable zeroing of pages in the idle loop if we have SSE2. */
659 vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0);
660 }
661 #endif
662
663 static void
664 cpu_init_idle_lwp(struct cpu_info *ci)
665 {
666 struct lwp *l = ci->ci_data.cpu_idlelwp;
667 struct pcb *pcb = lwp_getpcb(l);
668
669 pcb->pcb_cr0 = rcr0();
670 }
671
672 void
673 cpu_init_idle_lwps(void)
674 {
675 struct cpu_info *ci;
676 u_long i;
677
678 for (i = 0; i < maxcpus; i++) {
679 ci = cpu_lookup(i);
680 if (ci == NULL)
681 continue;
682 if (ci->ci_data.cpu_idlelwp == NULL)
683 continue;
684 if ((ci->ci_flags & CPUF_PRESENT) == 0)
685 continue;
686 cpu_init_idle_lwp(ci);
687 }
688 }
689
690 #ifdef MULTIPROCESSOR
691 void
692 cpu_start_secondary(struct cpu_info *ci)
693 {
694 paddr_t mp_pdirpa;
695 u_long psl;
696 int i;
697
698 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr);
699 cpu_copy_trampoline(mp_pdirpa);
700
701 atomic_or_32(&ci->ci_flags, CPUF_AP);
702 ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
703 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) {
704 return;
705 }
706
707 /*
708 * Wait for it to become ready. Setting cpu_starting opens the
709 * initial gate and allows the AP to start soft initialization.
710 */
711 KASSERT(cpu_starting == NULL);
712 cpu_starting = ci;
713 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
714 i8254_delay(10);
715 }
716
717 if ((ci->ci_flags & CPUF_PRESENT) == 0) {
718 aprint_error_dev(ci->ci_dev, "failed to become ready\n");
719 #if defined(MPDEBUG) && defined(DDB)
720 printf("dropping into debugger; continue from here to resume boot\n");
721 Debugger();
722 #endif
723 } else {
724 /*
725 * Synchronize time stamp counters. Invalidate cache and do
726 * twice to try and minimize possible cache effects. Disable
727 * interrupts to try and rule out any external interference.
728 */
729 psl = x86_read_psl();
730 x86_disable_intr();
731 wbinvd();
732 tsc_sync_bp(ci);
733 x86_write_psl(psl);
734 }
735
736 CPU_START_CLEANUP(ci);
737 cpu_starting = NULL;
738 }
739
740 void
741 cpu_boot_secondary(struct cpu_info *ci)
742 {
743 int64_t drift;
744 u_long psl;
745 int i;
746
747 atomic_or_32(&ci->ci_flags, CPUF_GO);
748 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
749 i8254_delay(10);
750 }
751 if ((ci->ci_flags & CPUF_RUNNING) == 0) {
752 aprint_error_dev(ci->ci_dev, "failed to start\n");
753 #if defined(MPDEBUG) && defined(DDB)
754 printf("dropping into debugger; continue from here to resume boot\n");
755 Debugger();
756 #endif
757 } else {
758 /* Synchronize TSC again, check for drift. */
759 drift = ci->ci_data.cpu_cc_skew;
760 psl = x86_read_psl();
761 x86_disable_intr();
762 wbinvd();
763 tsc_sync_bp(ci);
764 x86_write_psl(psl);
765 drift -= ci->ci_data.cpu_cc_skew;
766 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n",
767 (long long)ci->ci_data.cpu_cc_skew, (long long)drift);
768 tsc_sync_drift(drift);
769 }
770 }
771
772 /*
773 * The CPU ends up here when it's ready to run.
774 * This is called from code in mptramp.s; at this point, we are running
775 * in the idle pcb/idle stack of the new CPU. When this function returns,
776 * this processor will enter the idle loop and start looking for work.
777 */
778 void
779 cpu_hatch(void *v)
780 {
781 struct cpu_info *ci = (struct cpu_info *)v;
782 struct pcb *pcb;
783 int s, i;
784
785 cpu_init_msrs(ci, true);
786 cpu_probe(ci);
787
788 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq;
789 /* cpu_get_tsc_freq(ci); */
790
791 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
792
793 /*
794 * Synchronize time stamp counters. Invalidate cache and do twice
795 * to try and minimize possible cache effects. Note that interrupts
796 * are off at this point.
797 */
798 wbinvd();
799 atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
800 tsc_sync_ap(ci);
801
802 /*
803 * Wait to be brought online. Use 'monitor/mwait' if available,
804 * in order to make the TSC drift as much as possible. so that
805 * we can detect it later. If not available, try 'pause'.
806 * We'd like to use 'hlt', but we have interrupts off.
807 */
808 while ((ci->ci_flags & CPUF_GO) == 0) {
809 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) {
810 x86_monitor(&ci->ci_flags, 0, 0);
811 if ((ci->ci_flags & CPUF_GO) != 0) {
812 continue;
813 }
814 x86_mwait(0, 0);
815 } else {
816 /*
817 * XXX The loop repetition count could be a lot higher, but
818 * XXX currently qemu emulator takes a _very_long_time_ to
819 * XXX execute the pause instruction. So for now, use a low
820 * XXX value to allow the cpu to hatch before timing out.
821 */
822 for (i = 50; i != 0; i--) {
823 x86_pause();
824 }
825 }
826 }
827
828 /* Because the text may have been patched in x86_patch(). */
829 wbinvd();
830 x86_flush();
831 tlbflushg();
832
833 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
834
835 #ifdef PAE
836 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir;
837 for (i = 0 ; i < PDP_SIZE; i++) {
838 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V;
839 }
840 lcr3(ci->ci_pae_l3_pdirpa);
841 #else
842 lcr3(pmap_pdirpa(pmap_kernel(), 0));
843 #endif
844
845 pcb = lwp_getpcb(curlwp);
846 pcb->pcb_cr3 = rcr3();
847 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
848 lcr0(pcb->pcb_cr0);
849
850 cpu_init_idt();
851 gdt_init_cpu(ci);
852 #if NLAPIC > 0
853 lapic_enable();
854 lapic_set_lvt();
855 lapic_initclocks();
856 #endif
857
858 fpuinit(ci);
859 lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
860 ltr(ci->ci_tss_sel);
861
862 cpu_init(ci);
863 cpu_get_tsc_freq(ci);
864
865 s = splhigh();
866 lapic_write_tpri(0);
867 x86_enable_intr();
868 splx(s);
869 x86_errata();
870
871 aprint_debug_dev(ci->ci_dev, "running\n");
872
873 idle_loop(NULL);
874 KASSERT(false);
875 }
876 #endif
877
878 #if defined(DDB)
879
880 #include <ddb/db_output.h>
881 #include <machine/db_machdep.h>
882
883 /*
884 * Dump CPU information from ddb.
885 */
886 void
887 cpu_debug_dump(void)
888 {
889 struct cpu_info *ci;
890 CPU_INFO_ITERATOR cii;
891
892 db_printf("addr dev id flags ipis curlwp fpcurlwp\n");
893 for (CPU_INFO_FOREACH(cii, ci)) {
894 db_printf("%p %s %ld %x %x %10p %10p\n",
895 ci,
896 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
897 (long)ci->ci_cpuid,
898 ci->ci_flags, ci->ci_ipis,
899 ci->ci_curlwp,
900 ci->ci_fpcurlwp);
901 }
902 }
903 #endif
904
905 #if NLAPIC > 0
906 static void
907 cpu_copy_trampoline(paddr_t pdir_pa)
908 {
909 extern uint32_t nox_flag;
910 extern u_char cpu_spinup_trampoline[];
911 extern u_char cpu_spinup_trampoline_end[];
912 vaddr_t mp_trampoline_vaddr;
913 struct {
914 uint32_t large;
915 uint32_t nox;
916 uint32_t pdir;
917 } smp_data;
918 CTASSERT(sizeof(smp_data) == 3 * 4);
919
920 smp_data.large = (pmap_largepages != 0);
921 smp_data.nox = nox_flag;
922 smp_data.pdir = (uint32_t)(pdir_pa & 0xFFFFFFFF);
923
924 /* Enter the physical address */
925 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
926 UVM_KMF_VAONLY);
927 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
928 VM_PROT_READ | VM_PROT_WRITE, 0);
929 pmap_update(pmap_kernel());
930
931 /* Copy boot code */
932 memcpy((void *)mp_trampoline_vaddr,
933 cpu_spinup_trampoline,
934 cpu_spinup_trampoline_end - cpu_spinup_trampoline);
935
936 /* Copy smp_data at the end */
937 memcpy((void *)(mp_trampoline_vaddr + PAGE_SIZE - sizeof(smp_data)),
938 &smp_data, sizeof(smp_data));
939
940 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
941 pmap_update(pmap_kernel());
942 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
943 }
944 #endif
945
946 #ifdef MULTIPROCESSOR
947 int
948 mp_cpu_start(struct cpu_info *ci, paddr_t target)
949 {
950 unsigned short dwordptr[2];
951 int error;
952
953 /*
954 * Bootstrap code must be addressable in real mode
955 * and it must be page aligned.
956 */
957 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
958
959 /*
960 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
961 */
962
963 outb(IO_RTC, NVRAM_RESET);
964 outb(IO_RTC+1, NVRAM_RESET_JUMP);
965
966 /*
967 * "and the warm reset vector (DWORD based at 40:67) to point
968 * to the AP startup code ..."
969 */
970
971 dwordptr[0] = 0;
972 dwordptr[1] = target >> 4;
973
974 #if NLAPIC > 0
975 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4);
976 #endif
977
978 if ((cpu_feature[0] & CPUID_APIC) == 0) {
979 aprint_error("mp_cpu_start: CPU does not have APIC\n");
980 return ENODEV;
981 }
982
983 /*
984 * ... prior to executing the following sequence:". We'll also add in
985 * local cache flush, in case the BIOS has left the AP with its cache
986 * disabled. It may not be able to cope with MP coherency.
987 */
988 wbinvd();
989
990 if (ci->ci_flags & CPUF_AP) {
991 error = x86_ipi_init(ci->ci_cpuid);
992 if (error != 0) {
993 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
994 __func__);
995 return error;
996 }
997 i8254_delay(10000);
998
999 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
1000 if (error != 0) {
1001 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
1002 __func__);
1003 return error;
1004 }
1005 i8254_delay(200);
1006
1007 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
1008 if (error != 0) {
1009 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n",
1010 __func__);
1011 return error;
1012 }
1013 i8254_delay(200);
1014 }
1015
1016 return 0;
1017 }
1018
1019 void
1020 mp_cpu_start_cleanup(struct cpu_info *ci)
1021 {
1022 /*
1023 * Ensure the NVRAM reset byte contains something vaguely sane.
1024 */
1025
1026 outb(IO_RTC, NVRAM_RESET);
1027 outb(IO_RTC+1, NVRAM_RESET_RST);
1028 }
1029 #endif
1030
1031 #ifdef __x86_64__
1032 typedef void (vector)(void);
1033 extern vector Xsyscall, Xsyscall32;
1034 #endif
1035
1036 void
1037 cpu_init_msrs(struct cpu_info *ci, bool full)
1038 {
1039 #ifdef __x86_64__
1040 wrmsr(MSR_STAR,
1041 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1042 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
1043 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
1044 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
1045 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
1046
1047 if (full) {
1048 wrmsr(MSR_FSBASE, 0);
1049 wrmsr(MSR_GSBASE, (uint64_t)ci);
1050 wrmsr(MSR_KERNELGSBASE, 0);
1051 }
1052 #endif /* __x86_64__ */
1053
1054 if (cpu_feature[2] & CPUID_NOX)
1055 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
1056 }
1057
1058 void
1059 cpu_offline_md(void)
1060 {
1061 int s;
1062
1063 s = splhigh();
1064 fpusave_cpu(true);
1065 splx(s);
1066 }
1067
1068 /* XXX joerg restructure and restart CPUs individually */
1069 static bool
1070 cpu_stop(device_t dv)
1071 {
1072 struct cpu_softc *sc = device_private(dv);
1073 struct cpu_info *ci = sc->sc_info;
1074 int err;
1075
1076 KASSERT((ci->ci_flags & CPUF_PRESENT) != 0);
1077
1078 if ((ci->ci_flags & CPUF_PRIMARY) != 0)
1079 return true;
1080
1081 if (ci->ci_data.cpu_idlelwp == NULL)
1082 return true;
1083
1084 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
1085
1086 if (sc->sc_wasonline) {
1087 mutex_enter(&cpu_lock);
1088 err = cpu_setstate(ci, false);
1089 mutex_exit(&cpu_lock);
1090
1091 if (err != 0)
1092 return false;
1093 }
1094
1095 return true;
1096 }
1097
1098 static bool
1099 cpu_suspend(device_t dv, const pmf_qual_t *qual)
1100 {
1101 struct cpu_softc *sc = device_private(dv);
1102 struct cpu_info *ci = sc->sc_info;
1103
1104 if ((ci->ci_flags & CPUF_PRESENT) == 0)
1105 return true;
1106 else {
1107 cpufreq_suspend(ci);
1108 }
1109
1110 return cpu_stop(dv);
1111 }
1112
1113 static bool
1114 cpu_resume(device_t dv, const pmf_qual_t *qual)
1115 {
1116 struct cpu_softc *sc = device_private(dv);
1117 struct cpu_info *ci = sc->sc_info;
1118 int err = 0;
1119
1120 if ((ci->ci_flags & CPUF_PRESENT) == 0)
1121 return true;
1122
1123 if ((ci->ci_flags & CPUF_PRIMARY) != 0)
1124 goto out;
1125
1126 if (ci->ci_data.cpu_idlelwp == NULL)
1127 goto out;
1128
1129 if (sc->sc_wasonline) {
1130 mutex_enter(&cpu_lock);
1131 err = cpu_setstate(ci, true);
1132 mutex_exit(&cpu_lock);
1133 }
1134
1135 out:
1136 if (err != 0)
1137 return false;
1138
1139 cpufreq_resume(ci);
1140
1141 return true;
1142 }
1143
1144 static bool
1145 cpu_shutdown(device_t dv, int how)
1146 {
1147 struct cpu_softc *sc = device_private(dv);
1148 struct cpu_info *ci = sc->sc_info;
1149
1150 if ((ci->ci_flags & CPUF_BSP) != 0)
1151 return false;
1152
1153 if ((ci->ci_flags & CPUF_PRESENT) == 0)
1154 return true;
1155
1156 return cpu_stop(dv);
1157 }
1158
1159 void
1160 cpu_get_tsc_freq(struct cpu_info *ci)
1161 {
1162 uint64_t last_tsc;
1163
1164 if (cpu_hascounter()) {
1165 last_tsc = cpu_counter_serializing();
1166 i8254_delay(100000);
1167 ci->ci_data.cpu_cc_freq =
1168 (cpu_counter_serializing() - last_tsc) * 10;
1169 }
1170 }
1171
1172 void
1173 x86_cpu_idle_mwait(void)
1174 {
1175 struct cpu_info *ci = curcpu();
1176
1177 KASSERT(ci->ci_ilevel == IPL_NONE);
1178
1179 x86_monitor(&ci->ci_want_resched, 0, 0);
1180 if (__predict_false(ci->ci_want_resched)) {
1181 return;
1182 }
1183 x86_mwait(0, 0);
1184 }
1185
1186 void
1187 x86_cpu_idle_halt(void)
1188 {
1189 struct cpu_info *ci = curcpu();
1190
1191 KASSERT(ci->ci_ilevel == IPL_NONE);
1192
1193 x86_disable_intr();
1194 if (!__predict_false(ci->ci_want_resched)) {
1195 x86_stihlt();
1196 } else {
1197 x86_enable_intr();
1198 }
1199 }
1200
1201 /*
1202 * Loads pmap for the current CPU.
1203 */
1204 void
1205 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap)
1206 {
1207 #ifdef PAE
1208 struct cpu_info *ci = curcpu();
1209 bool interrupts_enabled;
1210 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir;
1211 int i;
1212
1213 /*
1214 * disable interrupts to block TLB shootdowns, which can reload cr3.
1215 * while this doesn't block NMIs, it's probably ok as NMIs unlikely
1216 * reload cr3.
1217 */
1218 interrupts_enabled = (x86_read_flags() & PSL_I) != 0;
1219 if (interrupts_enabled)
1220 x86_disable_intr();
1221
1222 for (i = 0 ; i < PDP_SIZE; i++) {
1223 l3_pd[i] = pmap->pm_pdirpa[i] | PG_V;
1224 }
1225
1226 if (interrupts_enabled)
1227 x86_enable_intr();
1228 tlbflush();
1229 #else /* PAE */
1230 lcr3(pmap_pdirpa(pmap, 0));
1231 #endif /* PAE */
1232 }
1233
1234 /*
1235 * Notify all other cpus to halt.
1236 */
1237
1238 void
1239 cpu_broadcast_halt(void)
1240 {
1241 x86_broadcast_ipi(X86_IPI_HALT);
1242 }
1243
1244 /*
1245 * Send a dummy ipi to a cpu to force it to run splraise()/spllower()
1246 */
1247
1248 void
1249 cpu_kick(struct cpu_info *ci)
1250 {
1251 x86_send_ipi(ci, 0);
1252 }
1253