cpu.c revision 1.51 1 1.51 uebayasi /* $NetBSD: cpu.c,v 1.51 2010/11/06 11:46:04 uebayasi Exp $ */
2 1.2 bouyer /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */
3 1.2 bouyer
4 1.2 bouyer /*-
5 1.2 bouyer * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 1.19 joerg * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
7 1.2 bouyer * All rights reserved.
8 1.2 bouyer *
9 1.2 bouyer * This code is derived from software contributed to The NetBSD Foundation
10 1.2 bouyer * by RedBack Networks Inc.
11 1.2 bouyer *
12 1.2 bouyer * Author: Bill Sommerfeld
13 1.2 bouyer *
14 1.2 bouyer * Redistribution and use in source and binary forms, with or without
15 1.2 bouyer * modification, are permitted provided that the following conditions
16 1.2 bouyer * are met:
17 1.2 bouyer * 1. Redistributions of source code must retain the above copyright
18 1.2 bouyer * notice, this list of conditions and the following disclaimer.
19 1.2 bouyer * 2. Redistributions in binary form must reproduce the above copyright
20 1.2 bouyer * notice, this list of conditions and the following disclaimer in the
21 1.2 bouyer * documentation and/or other materials provided with the distribution.
22 1.2 bouyer *
23 1.2 bouyer * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
24 1.2 bouyer * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
25 1.2 bouyer * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26 1.2 bouyer * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
27 1.2 bouyer * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 1.2 bouyer * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 1.2 bouyer * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 1.2 bouyer * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 1.2 bouyer * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 1.2 bouyer * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 1.2 bouyer * POSSIBILITY OF SUCH DAMAGE.
34 1.2 bouyer */
35 1.2 bouyer
36 1.2 bouyer /*
37 1.2 bouyer * Copyright (c) 1999 Stefan Grefen
38 1.2 bouyer *
39 1.2 bouyer * Redistribution and use in source and binary forms, with or without
40 1.2 bouyer * modification, are permitted provided that the following conditions
41 1.2 bouyer * are met:
42 1.2 bouyer * 1. Redistributions of source code must retain the above copyright
43 1.2 bouyer * notice, this list of conditions and the following disclaimer.
44 1.2 bouyer * 2. Redistributions in binary form must reproduce the above copyright
45 1.2 bouyer * notice, this list of conditions and the following disclaimer in the
46 1.2 bouyer * documentation and/or other materials provided with the distribution.
47 1.2 bouyer * 3. All advertising materials mentioning features or use of this software
48 1.2 bouyer * must display the following acknowledgement:
49 1.2 bouyer * This product includes software developed by the NetBSD
50 1.2 bouyer * Foundation, Inc. and its contributors.
51 1.2 bouyer * 4. Neither the name of The NetBSD Foundation nor the names of its
52 1.2 bouyer * contributors may be used to endorse or promote products derived
53 1.2 bouyer * from this software without specific prior written permission.
54 1.2 bouyer *
55 1.2 bouyer * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
56 1.2 bouyer * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
57 1.2 bouyer * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
58 1.2 bouyer * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
59 1.2 bouyer * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
60 1.2 bouyer * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
61 1.2 bouyer * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
62 1.2 bouyer * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
63 1.2 bouyer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
64 1.2 bouyer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
65 1.2 bouyer * SUCH DAMAGE.
66 1.2 bouyer */
67 1.2 bouyer
68 1.2 bouyer #include <sys/cdefs.h>
69 1.51 uebayasi __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.51 2010/11/06 11:46:04 uebayasi Exp $");
70 1.2 bouyer
71 1.2 bouyer #include "opt_ddb.h"
72 1.2 bouyer #include "opt_multiprocessor.h"
73 1.2 bouyer #include "opt_mpbios.h" /* for MPDEBUG */
74 1.2 bouyer #include "opt_mtrr.h"
75 1.2 bouyer #include "opt_xen.h"
76 1.2 bouyer
77 1.2 bouyer #include "lapic.h"
78 1.2 bouyer #include "ioapic.h"
79 1.2 bouyer
80 1.2 bouyer #include <sys/param.h>
81 1.2 bouyer #include <sys/proc.h>
82 1.2 bouyer #include <sys/systm.h>
83 1.2 bouyer #include <sys/device.h>
84 1.31 cegger #include <sys/kmem.h>
85 1.11 cegger #include <sys/cpu.h>
86 1.11 cegger #include <sys/atomic.h>
87 1.32 cegger #include <sys/reboot.h>
88 1.2 bouyer
89 1.51 uebayasi #include <uvm/uvm.h>
90 1.2 bouyer
91 1.2 bouyer #include <machine/cpufunc.h>
92 1.2 bouyer #include <machine/cpuvar.h>
93 1.2 bouyer #include <machine/pmap.h>
94 1.2 bouyer #include <machine/vmparam.h>
95 1.2 bouyer #include <machine/mpbiosvar.h>
96 1.2 bouyer #include <machine/pcb.h>
97 1.2 bouyer #include <machine/specialreg.h>
98 1.2 bouyer #include <machine/segments.h>
99 1.2 bouyer #include <machine/gdt.h>
100 1.2 bouyer #include <machine/mtrr.h>
101 1.2 bouyer #include <machine/pio.h>
102 1.2 bouyer
103 1.2 bouyer #include <xen/vcpuvar.h>
104 1.2 bouyer
105 1.2 bouyer #if NLAPIC > 0
106 1.2 bouyer #include <machine/apicvar.h>
107 1.2 bouyer #include <machine/i82489reg.h>
108 1.2 bouyer #include <machine/i82489var.h>
109 1.2 bouyer #endif
110 1.2 bouyer
111 1.2 bouyer #include <dev/ic/mc146818reg.h>
112 1.2 bouyer #include <dev/isa/isareg.h>
113 1.2 bouyer
114 1.38 cegger #if MAXCPUS > 32
115 1.38 cegger #error cpu_info contains 32bit bitmasks
116 1.38 cegger #endif
117 1.27 ad
118 1.10 cegger int cpu_match(device_t, cfdata_t, void *);
119 1.10 cegger void cpu_attach(device_t, device_t, void *);
120 1.10 cegger int vcpu_match(device_t, cfdata_t, void *);
121 1.10 cegger void vcpu_attach(device_t, device_t, void *);
122 1.10 cegger void cpu_attach_common(device_t, device_t, void *);
123 1.8 dogcow void cpu_offline_md(void);
124 1.2 bouyer
125 1.2 bouyer struct cpu_softc {
126 1.10 cegger device_t sc_dev; /* device tree glue */
127 1.2 bouyer struct cpu_info *sc_info; /* pointer to CPU info */
128 1.32 cegger bool sc_wasonline;
129 1.2 bouyer };
130 1.2 bouyer
131 1.5 joerg int mp_cpu_start(struct cpu_info *, paddr_t);
132 1.2 bouyer void mp_cpu_start_cleanup(struct cpu_info *);
133 1.2 bouyer const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
134 1.2 bouyer mp_cpu_start_cleanup };
135 1.2 bouyer
136 1.10 cegger CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc),
137 1.2 bouyer cpu_match, cpu_attach, NULL, NULL);
138 1.10 cegger CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
139 1.2 bouyer vcpu_match, vcpu_attach, NULL, NULL);
140 1.2 bouyer
141 1.2 bouyer /*
142 1.2 bouyer * Statically-allocated CPU info for the primary CPU (or the only
143 1.2 bouyer * CPU, on uniprocessors). The CPU info list is initialized to
144 1.2 bouyer * point at it.
145 1.2 bouyer */
146 1.2 bouyer #ifdef TRAPLOG
147 1.2 bouyer #include <machine/tlog.h>
148 1.2 bouyer struct tlog tlog_primary;
149 1.2 bouyer #endif
150 1.38 cegger struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
151 1.7 bouyer .ci_dev = 0,
152 1.2 bouyer .ci_self = &cpu_info_primary,
153 1.4 bouyer .ci_idepth = -1,
154 1.2 bouyer .ci_curlwp = &lwp0,
155 1.25 ad .ci_curldt = -1,
156 1.2 bouyer #ifdef TRAPLOG
157 1.2 bouyer .ci_tlog = &tlog_primary,
158 1.2 bouyer #endif
159 1.2 bouyer
160 1.2 bouyer };
161 1.38 cegger struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
162 1.7 bouyer .ci_dev = 0,
163 1.2 bouyer .ci_self = &phycpu_info_primary,
164 1.2 bouyer };
165 1.2 bouyer
166 1.2 bouyer struct cpu_info *cpu_info_list = &cpu_info_primary;
167 1.38 cegger struct cpu_info *phycpu_info_list = &phycpu_info_primary;
168 1.2 bouyer
169 1.2 bouyer static void cpu_set_tss_gates(struct cpu_info *ci);
170 1.2 bouyer
171 1.11 cegger uint32_t cpus_attached = 0;
172 1.11 cegger uint32_t cpus_running = 0;
173 1.11 cegger
174 1.38 cegger uint32_t phycpus_attached = 0;
175 1.38 cegger uint32_t phycpus_running = 0;
176 1.38 cegger
177 1.43 jym uint32_t cpu_feature[5]; /* X86 CPUID feature bits
178 1.43 jym * [0] basic features %edx
179 1.43 jym * [1] basic features %ecx
180 1.43 jym * [2] extended features %edx
181 1.43 jym * [3] extended features %ecx
182 1.43 jym * [4] VIA padlock features
183 1.43 jym */
184 1.43 jym
185 1.11 cegger bool x86_mp_online;
186 1.11 cegger paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
187 1.2 bouyer
188 1.38 cegger #if defined(MULTIPROCESSOR)
189 1.2 bouyer void cpu_hatch(void *);
190 1.2 bouyer static void cpu_boot_secondary(struct cpu_info *ci);
191 1.2 bouyer static void cpu_start_secondary(struct cpu_info *ci);
192 1.2 bouyer static void cpu_copy_trampoline(void);
193 1.2 bouyer
194 1.2 bouyer /*
195 1.2 bouyer * Runs once per boot once multiprocessor goo has been detected and
196 1.2 bouyer * the local APIC on the boot processor has been mapped.
197 1.2 bouyer *
198 1.2 bouyer * Called from lapic_boot_init() (from mpbios_scan()).
199 1.2 bouyer */
200 1.2 bouyer void
201 1.10 cegger cpu_init_first(void)
202 1.2 bouyer {
203 1.2 bouyer
204 1.38 cegger cpu_info_primary.ci_cpuid = lapic_cpu_number();
205 1.2 bouyer cpu_copy_trampoline();
206 1.2 bouyer }
207 1.38 cegger #endif /* MULTIPROCESSOR */
208 1.2 bouyer
209 1.2 bouyer int
210 1.10 cegger cpu_match(device_t parent, cfdata_t match, void *aux)
211 1.2 bouyer {
212 1.2 bouyer
213 1.2 bouyer return 1;
214 1.2 bouyer }
215 1.2 bouyer
216 1.2 bouyer void
217 1.10 cegger cpu_attach(device_t parent, device_t self, void *aux)
218 1.2 bouyer {
219 1.10 cegger struct cpu_softc *sc = device_private(self);
220 1.2 bouyer struct cpu_attach_args *caa = aux;
221 1.2 bouyer struct cpu_info *ci;
222 1.34 cegger uintptr_t ptr;
223 1.38 cegger static bool again = false;
224 1.2 bouyer
225 1.10 cegger sc->sc_dev = self;
226 1.10 cegger
227 1.38 cegger if (phycpus_attached == ~0) {
228 1.34 cegger aprint_error(": increase MAXCPUS\n");
229 1.34 cegger return;
230 1.34 cegger }
231 1.34 cegger
232 1.2 bouyer /*
233 1.2 bouyer * If we're an Application Processor, allocate a cpu_info
234 1.2 bouyer * structure, otherwise use the primary's.
235 1.2 bouyer */
236 1.2 bouyer if (caa->cpu_role == CPU_ROLE_AP) {
237 1.32 cegger if ((boothowto & RB_MD1) != 0) {
238 1.32 cegger aprint_error(": multiprocessor boot disabled\n");
239 1.32 cegger if (!pmf_device_register(self, NULL, NULL))
240 1.32 cegger aprint_error_dev(self,
241 1.32 cegger "couldn't establish power handler\n");
242 1.32 cegger return;
243 1.32 cegger }
244 1.32 cegger aprint_naive(": Application Processor\n");
245 1.34 cegger ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
246 1.34 cegger KM_SLEEP);
247 1.42 jym ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
248 1.24 ad ci->ci_curldt = -1;
249 1.2 bouyer } else {
250 1.32 cegger aprint_naive(": %s Processor\n",
251 1.32 cegger caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
252 1.2 bouyer ci = &phycpu_info_primary;
253 1.2 bouyer }
254 1.2 bouyer
255 1.2 bouyer ci->ci_self = ci;
256 1.2 bouyer sc->sc_info = ci;
257 1.2 bouyer
258 1.2 bouyer ci->ci_dev = self;
259 1.50 jruoho ci->ci_acpiid = caa->cpu_id;
260 1.23 ad ci->ci_cpuid = caa->cpu_number;
261 1.16 cegger ci->ci_vcpu = NULL;
262 1.2 bouyer
263 1.38 cegger /*
264 1.38 cegger * Boot processor may not be attached first, but the below
265 1.38 cegger * must be done to allow booting other processors.
266 1.38 cegger */
267 1.38 cegger if (!again) {
268 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
269 1.38 cegger /* Basic init */
270 1.38 cegger again = true;
271 1.38 cegger }
272 1.38 cegger
273 1.2 bouyer printf(": ");
274 1.2 bouyer switch (caa->cpu_role) {
275 1.2 bouyer case CPU_ROLE_SP:
276 1.2 bouyer printf("(uniprocessor)\n");
277 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_SP);
278 1.2 bouyer break;
279 1.2 bouyer
280 1.2 bouyer case CPU_ROLE_BP:
281 1.2 bouyer printf("(boot processor)\n");
282 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_BSP);
283 1.2 bouyer break;
284 1.2 bouyer
285 1.2 bouyer case CPU_ROLE_AP:
286 1.2 bouyer /*
287 1.2 bouyer * report on an AP
288 1.2 bouyer */
289 1.2 bouyer printf("(application processor)\n");
290 1.38 cegger if (ci->ci_flags & CPUF_PRESENT) {
291 1.38 cegger struct cpu_info *tmp;
292 1.38 cegger
293 1.38 cegger tmp = phycpu_info_list;
294 1.38 cegger while (tmp->ci_next)
295 1.38 cegger tmp = tmp->ci_next;
296 1.38 cegger
297 1.38 cegger tmp->ci_next = ci;
298 1.38 cegger }
299 1.2 bouyer break;
300 1.2 bouyer
301 1.2 bouyer default:
302 1.2 bouyer panic("unknown processor type??\n");
303 1.2 bouyer }
304 1.34 cegger
305 1.38 cegger atomic_or_32(&phycpus_attached, ci->ci_cpumask);
306 1.34 cegger
307 1.2 bouyer return;
308 1.2 bouyer }
309 1.2 bouyer
310 1.2 bouyer int
311 1.10 cegger vcpu_match(device_t parent, cfdata_t match, void *aux)
312 1.2 bouyer {
313 1.2 bouyer struct vcpu_attach_args *vcaa = aux;
314 1.2 bouyer
315 1.2 bouyer if (strcmp(vcaa->vcaa_name, match->cf_name) == 0)
316 1.2 bouyer return 1;
317 1.2 bouyer return 0;
318 1.2 bouyer }
319 1.2 bouyer
320 1.2 bouyer void
321 1.10 cegger vcpu_attach(device_t parent, device_t self, void *aux)
322 1.2 bouyer {
323 1.2 bouyer struct vcpu_attach_args *vcaa = aux;
324 1.2 bouyer
325 1.2 bouyer cpu_attach_common(parent, self, &vcaa->vcaa_caa);
326 1.2 bouyer }
327 1.2 bouyer
328 1.2 bouyer static void
329 1.2 bouyer cpu_vm_init(struct cpu_info *ci)
330 1.2 bouyer {
331 1.2 bouyer int ncolors = 2, i;
332 1.2 bouyer
333 1.2 bouyer for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
334 1.2 bouyer struct x86_cache_info *cai;
335 1.2 bouyer int tcolors;
336 1.2 bouyer
337 1.2 bouyer cai = &ci->ci_cinfo[i];
338 1.2 bouyer
339 1.2 bouyer tcolors = atop(cai->cai_totalsize);
340 1.2 bouyer switch(cai->cai_associativity) {
341 1.2 bouyer case 0xff:
342 1.2 bouyer tcolors = 1; /* fully associative */
343 1.2 bouyer break;
344 1.2 bouyer case 0:
345 1.2 bouyer case 1:
346 1.2 bouyer break;
347 1.2 bouyer default:
348 1.2 bouyer tcolors /= cai->cai_associativity;
349 1.2 bouyer }
350 1.2 bouyer ncolors = max(ncolors, tcolors);
351 1.2 bouyer }
352 1.2 bouyer
353 1.2 bouyer /*
354 1.2 bouyer * Knowing the size of the largest cache on this CPU, re-color
355 1.2 bouyer * our pages.
356 1.2 bouyer */
357 1.2 bouyer if (ncolors <= uvmexp.ncolors)
358 1.2 bouyer return;
359 1.28 bouyer aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
360 1.2 bouyer uvm_page_recolor(ncolors);
361 1.2 bouyer }
362 1.2 bouyer
363 1.2 bouyer void
364 1.11 cegger cpu_attach_common(device_t parent, device_t self, void *aux)
365 1.2 bouyer {
366 1.10 cegger struct cpu_softc *sc = device_private(self);
367 1.2 bouyer struct cpu_attach_args *caa = aux;
368 1.2 bouyer struct cpu_info *ci;
369 1.12 cegger uintptr_t ptr;
370 1.2 bouyer int cpunum = caa->cpu_number;
371 1.38 cegger static bool again = false;
372 1.2 bouyer
373 1.10 cegger sc->sc_dev = self;
374 1.10 cegger
375 1.2 bouyer /*
376 1.2 bouyer * If we're an Application Processor, allocate a cpu_info
377 1.2 bouyer * structure, otherwise use the primary's.
378 1.2 bouyer */
379 1.2 bouyer if (caa->cpu_role == CPU_ROLE_AP) {
380 1.12 cegger aprint_naive(": Application Processor\n");
381 1.31 cegger ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
382 1.31 cegger KM_SLEEP);
383 1.42 jym ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
384 1.12 cegger memset(ci, 0, sizeof(*ci));
385 1.2 bouyer #ifdef TRAPLOG
386 1.31 cegger ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
387 1.2 bouyer #endif
388 1.2 bouyer } else {
389 1.12 cegger aprint_naive(": %s Processor\n",
390 1.12 cegger caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
391 1.2 bouyer ci = &cpu_info_primary;
392 1.38 cegger #if NLAPIC > 0
393 1.38 cegger if (cpunum != lapic_cpu_number()) {
394 1.38 cegger /* XXX should be done earlier */
395 1.38 cegger uint32_t reg;
396 1.38 cegger aprint_verbose("\n");
397 1.38 cegger aprint_verbose_dev(self, "running CPU at apic %d"
398 1.38 cegger " instead of at expected %d", lapic_cpu_number(),
399 1.38 cegger cpunum);
400 1.38 cegger reg = i82489_readreg(LAPIC_ID);
401 1.38 cegger i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
402 1.38 cegger (cpunum << LAPIC_ID_SHIFT));
403 1.38 cegger }
404 1.2 bouyer if (cpunum != lapic_cpu_number()) {
405 1.38 cegger aprint_error_dev(self, "unable to reset apic id\n");
406 1.2 bouyer }
407 1.2 bouyer #endif
408 1.2 bouyer }
409 1.2 bouyer
410 1.2 bouyer ci->ci_self = ci;
411 1.2 bouyer sc->sc_info = ci;
412 1.2 bouyer ci->ci_dev = self;
413 1.23 ad ci->ci_cpuid = cpunum;
414 1.16 cegger
415 1.16 cegger KASSERT(HYPERVISOR_shared_info != NULL);
416 1.16 cegger ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
417 1.16 cegger
418 1.2 bouyer ci->ci_func = caa->cpu_func;
419 1.2 bouyer
420 1.38 cegger /* Must be called before mi_cpu_attach(). */
421 1.38 cegger cpu_vm_init(ci);
422 1.38 cegger
423 1.2 bouyer if (caa->cpu_role == CPU_ROLE_AP) {
424 1.2 bouyer int error;
425 1.2 bouyer
426 1.2 bouyer error = mi_cpu_attach(ci);
427 1.2 bouyer if (error != 0) {
428 1.2 bouyer aprint_normal("\n");
429 1.38 cegger aprint_error_dev(self,
430 1.38 cegger "mi_cpu_attach failed with %d\n", error);
431 1.2 bouyer return;
432 1.2 bouyer }
433 1.2 bouyer } else {
434 1.2 bouyer KASSERT(ci->ci_data.cpu_idlelwp != NULL);
435 1.2 bouyer }
436 1.2 bouyer
437 1.23 ad ci->ci_cpumask = (1 << cpu_index(ci));
438 1.2 bouyer pmap_reference(pmap_kernel());
439 1.2 bouyer ci->ci_pmap = pmap_kernel();
440 1.2 bouyer ci->ci_tlbstate = TLBSTATE_STALE;
441 1.2 bouyer
442 1.38 cegger /*
443 1.38 cegger * Boot processor may not be attached first, but the below
444 1.38 cegger * must be done to allow booting other processors.
445 1.38 cegger */
446 1.38 cegger if (!again) {
447 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
448 1.38 cegger /* Basic init. */
449 1.38 cegger cpu_intr_init(ci);
450 1.38 cegger cpu_get_tsc_freq(ci);
451 1.38 cegger cpu_init(ci);
452 1.38 cegger cpu_set_tss_gates(ci);
453 1.38 cegger pmap_cpu_init_late(ci);
454 1.38 cegger #if NLAPIC > 0
455 1.38 cegger if (caa->cpu_role != CPU_ROLE_SP) {
456 1.38 cegger /* Enable lapic. */
457 1.38 cegger lapic_enable();
458 1.38 cegger lapic_set_lvt();
459 1.38 cegger lapic_calibrate_timer();
460 1.38 cegger }
461 1.38 cegger #endif
462 1.38 cegger /* Make sure DELAY() is initialized. */
463 1.38 cegger DELAY(1);
464 1.38 cegger again = true;
465 1.38 cegger }
466 1.38 cegger
467 1.2 bouyer /* further PCB init done later. */
468 1.2 bouyer
469 1.2 bouyer switch (caa->cpu_role) {
470 1.2 bouyer case CPU_ROLE_SP:
471 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_SP);
472 1.21 ad cpu_identify(ci);
473 1.12 cegger #if 0
474 1.12 cegger x86_errata();
475 1.12 cegger #endif
476 1.38 cegger x86_cpu_idle_init();
477 1.2 bouyer break;
478 1.2 bouyer
479 1.2 bouyer case CPU_ROLE_BP:
480 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_BSP);
481 1.21 ad cpu_identify(ci);
482 1.2 bouyer cpu_init(ci);
483 1.14 bouyer #if 0
484 1.12 cegger x86_errata();
485 1.12 cegger #endif
486 1.38 cegger x86_cpu_idle_init();
487 1.2 bouyer break;
488 1.2 bouyer
489 1.2 bouyer case CPU_ROLE_AP:
490 1.2 bouyer /*
491 1.2 bouyer * report on an AP
492 1.2 bouyer */
493 1.2 bouyer
494 1.2 bouyer #if defined(MULTIPROCESSOR)
495 1.2 bouyer cpu_intr_init(ci);
496 1.2 bouyer gdt_alloc_cpu(ci);
497 1.2 bouyer cpu_set_tss_gates(ci);
498 1.12 cegger pmap_cpu_init_early(ci);
499 1.12 cegger pmap_cpu_init_late(ci);
500 1.2 bouyer cpu_start_secondary(ci);
501 1.2 bouyer if (ci->ci_flags & CPUF_PRESENT) {
502 1.30 cegger struct cpu_info *tmp;
503 1.30 cegger
504 1.2 bouyer identifycpu(ci);
505 1.30 cegger tmp = cpu_info_list;
506 1.30 cegger while (tmp->ci_next)
507 1.30 cegger tmp = tmp->ci_next;
508 1.30 cegger
509 1.30 cegger tmp->ci_next = ci;
510 1.2 bouyer }
511 1.2 bouyer #else
512 1.38 cegger aprint_error_dev(self, "not started\n");
513 1.2 bouyer #endif
514 1.2 bouyer break;
515 1.2 bouyer
516 1.2 bouyer default:
517 1.12 cegger aprint_normal("\n");
518 1.2 bouyer panic("unknown processor type??\n");
519 1.2 bouyer }
520 1.2 bouyer
521 1.46 cegger pat_init(ci);
522 1.34 cegger atomic_or_32(&cpus_attached, ci->ci_cpumask);
523 1.2 bouyer
524 1.12 cegger #if 0
525 1.12 cegger if (!pmf_device_register(self, cpu_suspend, cpu_resume))
526 1.12 cegger aprint_error_dev(self, "couldn't establish power handler\n");
527 1.12 cegger #endif
528 1.12 cegger
529 1.2 bouyer #if defined(MULTIPROCESSOR)
530 1.2 bouyer if (mp_verbose) {
531 1.2 bouyer struct lwp *l = ci->ci_data.cpu_idlelwp;
532 1.37 rmind struct pcb *pcb = lwp_getpcb(l);
533 1.2 bouyer
534 1.38 cegger aprint_verbose_dev(self,
535 1.38 cegger "idle lwp at %p, idle sp at 0x%p\n",
536 1.12 cegger l,
537 1.12 cegger #ifdef i386
538 1.37 rmind (void *)pcb->pcb_esp
539 1.12 cegger #else
540 1.37 rmind (void *)pcb->pcb_rsp
541 1.12 cegger #endif
542 1.12 cegger );
543 1.12 cegger
544 1.2 bouyer }
545 1.2 bouyer #endif
546 1.2 bouyer }
547 1.2 bouyer
548 1.2 bouyer /*
549 1.2 bouyer * Initialize the processor appropriately.
550 1.2 bouyer */
551 1.2 bouyer
552 1.2 bouyer void
553 1.10 cegger cpu_init(struct cpu_info *ci)
554 1.2 bouyer {
555 1.2 bouyer
556 1.2 bouyer /*
557 1.2 bouyer * On a P6 or above, enable global TLB caching if the
558 1.2 bouyer * hardware supports it.
559 1.2 bouyer */
560 1.43 jym if (cpu_feature[0] & CPUID_PGE)
561 1.2 bouyer lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */
562 1.2 bouyer
563 1.2 bouyer #ifdef XXXMTRR
564 1.2 bouyer /*
565 1.2 bouyer * On a P6 or above, initialize MTRR's if the hardware supports them.
566 1.2 bouyer */
567 1.43 jym if (cpu_feature[0] & CPUID_MTRR) {
568 1.2 bouyer if ((ci->ci_flags & CPUF_AP) == 0)
569 1.2 bouyer i686_mtrr_init_first();
570 1.2 bouyer mtrr_init_cpu(ci);
571 1.2 bouyer }
572 1.2 bouyer #endif
573 1.2 bouyer /*
574 1.2 bouyer * If we have FXSAVE/FXRESTOR, use them.
575 1.2 bouyer */
576 1.43 jym if (cpu_feature[0] & CPUID_FXSR) {
577 1.2 bouyer lcr4(rcr4() | CR4_OSFXSR);
578 1.2 bouyer
579 1.2 bouyer /*
580 1.2 bouyer * If we have SSE/SSE2, enable XMM exceptions.
581 1.2 bouyer */
582 1.43 jym if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
583 1.2 bouyer lcr4(rcr4() | CR4_OSXMMEXCPT);
584 1.2 bouyer }
585 1.2 bouyer
586 1.47 jym #ifdef __x86_64__
587 1.47 jym /* No user PGD mapped for this CPU yet */
588 1.47 jym ci->ci_xen_current_user_pgd = 0;
589 1.47 jym #endif
590 1.47 jym
591 1.34 cegger atomic_or_32(&cpus_running, ci->ci_cpumask);
592 1.11 cegger atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
593 1.2 bouyer }
594 1.2 bouyer
595 1.2 bouyer
596 1.2 bouyer #ifdef MULTIPROCESSOR
597 1.2 bouyer void
598 1.10 cegger cpu_boot_secondary_processors(void)
599 1.2 bouyer {
600 1.2 bouyer struct cpu_info *ci;
601 1.2 bouyer u_long i;
602 1.2 bouyer
603 1.38 cegger for (i = 0; i < maxcpus; i++) {
604 1.38 cegger ci = cpu_lookup(i);
605 1.2 bouyer if (ci == NULL)
606 1.2 bouyer continue;
607 1.2 bouyer if (ci->ci_data.cpu_idlelwp == NULL)
608 1.2 bouyer continue;
609 1.2 bouyer if ((ci->ci_flags & CPUF_PRESENT) == 0)
610 1.2 bouyer continue;
611 1.2 bouyer if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
612 1.2 bouyer continue;
613 1.2 bouyer cpu_boot_secondary(ci);
614 1.2 bouyer }
615 1.11 cegger
616 1.11 cegger x86_mp_online = true;
617 1.2 bouyer }
618 1.2 bouyer
619 1.2 bouyer static void
620 1.2 bouyer cpu_init_idle_lwp(struct cpu_info *ci)
621 1.2 bouyer {
622 1.2 bouyer struct lwp *l = ci->ci_data.cpu_idlelwp;
623 1.37 rmind struct pcb *pcb = lwp_getpcb(l);
624 1.2 bouyer
625 1.2 bouyer pcb->pcb_cr0 = rcr0();
626 1.2 bouyer }
627 1.2 bouyer
628 1.2 bouyer void
629 1.10 cegger cpu_init_idle_lwps(void)
630 1.2 bouyer {
631 1.2 bouyer struct cpu_info *ci;
632 1.2 bouyer u_long i;
633 1.2 bouyer
634 1.38 cegger for (i = 0; i < maxcpus; i++) {
635 1.38 cegger ci = cpu_lookup(i);
636 1.2 bouyer if (ci == NULL)
637 1.2 bouyer continue;
638 1.2 bouyer if (ci->ci_data.cpu_idlelwp == NULL)
639 1.2 bouyer continue;
640 1.2 bouyer if ((ci->ci_flags & CPUF_PRESENT) == 0)
641 1.2 bouyer continue;
642 1.2 bouyer cpu_init_idle_lwp(ci);
643 1.2 bouyer }
644 1.2 bouyer }
645 1.2 bouyer
646 1.2 bouyer void
647 1.10 cegger cpu_start_secondary(struct cpu_info *ci)
648 1.2 bouyer {
649 1.2 bouyer int i;
650 1.2 bouyer struct pmap *kpm = pmap_kernel();
651 1.11 cegger extern uint32_t mp_pdirpa;
652 1.2 bouyer
653 1.2 bouyer mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */
654 1.2 bouyer
655 1.11 cegger atomic_or_32(&ci->ci_flags, CPUF_AP);
656 1.2 bouyer
657 1.11 cegger aprint_debug_dev(ci->ci_dev, "starting\n");
658 1.2 bouyer
659 1.2 bouyer ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
660 1.11 cegger if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0)
661 1.11 cegger return;
662 1.2 bouyer
663 1.2 bouyer /*
664 1.2 bouyer * wait for it to become ready
665 1.2 bouyer */
666 1.11 cegger for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
667 1.11 cegger #ifdef MPDEBUG
668 1.11 cegger extern int cpu_trace[3];
669 1.11 cegger static int otrace[3];
670 1.11 cegger if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
671 1.11 cegger aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
672 1.11 cegger cpu_trace[0], cpu_trace[1], cpu_trace[2]);
673 1.11 cegger memcpy(otrace, cpu_trace, sizeof(otrace));
674 1.11 cegger }
675 1.11 cegger #endif
676 1.2 bouyer delay(10);
677 1.2 bouyer }
678 1.11 cegger if ((ci->ci_flags & CPUF_PRESENT) == 0) {
679 1.9 cegger aprint_error_dev(ci->ci_dev, "failed to become ready\n");
680 1.2 bouyer #if defined(MPDEBUG) && defined(DDB)
681 1.2 bouyer printf("dropping into debugger; continue from here to resume boot\n");
682 1.2 bouyer Debugger();
683 1.2 bouyer #endif
684 1.2 bouyer }
685 1.2 bouyer
686 1.2 bouyer CPU_START_CLEANUP(ci);
687 1.2 bouyer }
688 1.2 bouyer
689 1.2 bouyer void
690 1.10 cegger cpu_boot_secondary(struct cpu_info *ci)
691 1.2 bouyer {
692 1.2 bouyer int i;
693 1.2 bouyer
694 1.11 cegger atomic_or_32(&ci->ci_flags, CPUF_GO);
695 1.11 cegger for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
696 1.2 bouyer delay(10);
697 1.2 bouyer }
698 1.11 cegger if ((ci->ci_flags & CPUF_RUNNING) == 0) {
699 1.11 cegger aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
700 1.2 bouyer #if defined(MPDEBUG) && defined(DDB)
701 1.2 bouyer printf("dropping into debugger; continue from here to resume boot\n");
702 1.2 bouyer Debugger();
703 1.2 bouyer #endif
704 1.2 bouyer }
705 1.2 bouyer }
706 1.2 bouyer
707 1.2 bouyer /*
708 1.2 bouyer * The CPU ends up here when its ready to run
709 1.2 bouyer * This is called from code in mptramp.s; at this point, we are running
710 1.2 bouyer * in the idle pcb/idle stack of the new CPU. When this function returns,
711 1.2 bouyer * this processor will enter the idle loop and start looking for work.
712 1.2 bouyer *
713 1.2 bouyer * XXX should share some of this with init386 in machdep.c
714 1.2 bouyer */
715 1.2 bouyer void
716 1.2 bouyer cpu_hatch(void *v)
717 1.2 bouyer {
718 1.2 bouyer struct cpu_info *ci = (struct cpu_info *)v;
719 1.37 rmind struct pcb *pcb;
720 1.11 cegger int s, i;
721 1.11 cegger
722 1.21 ad cpu_probe(ci);
723 1.11 cegger
724 1.43 jym cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST;
725 1.43 jym cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST;
726 1.2 bouyer
727 1.43 jym cpu_init_msrs(ci, true);
728 1.2 bouyer
729 1.11 cegger KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
730 1.11 cegger atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
731 1.11 cegger while ((ci->ci_flags & CPUF_GO) == 0) {
732 1.11 cegger /* Don't use delay, boot CPU may be patching the text. */
733 1.11 cegger for (i = 10000; i != 0; i--)
734 1.11 cegger x86_pause();
735 1.11 cegger }
736 1.2 bouyer
737 1.11 cegger /* Because the text may have been patched in x86_patch(). */
738 1.11 cegger wbinvd();
739 1.11 cegger x86_flush();
740 1.2 bouyer
741 1.11 cegger KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
742 1.2 bouyer
743 1.37 rmind pcb = lwp_getpcb(curlwp);
744 1.12 cegger lcr3(pmap_kernel()->pm_pdirpa);
745 1.37 rmind pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa;
746 1.37 rmind pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
747 1.37 rmind lcr0(pcb->pcb_cr0);
748 1.37 rmind
749 1.2 bouyer cpu_init_idt();
750 1.11 cegger gdt_init_cpu(ci);
751 1.11 cegger lapic_enable();
752 1.2 bouyer lapic_set_lvt();
753 1.11 cegger lapic_initclocks();
754 1.11 cegger
755 1.12 cegger #ifdef i386
756 1.2 bouyer npxinit(ci);
757 1.12 cegger #else
758 1.12 cegger fpuinit(ci);
759 1.12 cegger #endif
760 1.2 bouyer
761 1.2 bouyer lldt(GSEL(GLDT_SEL, SEL_KPL));
762 1.12 cegger ltr(ci->ci_tss_sel);
763 1.2 bouyer
764 1.2 bouyer cpu_init(ci);
765 1.11 cegger cpu_get_tsc_freq(ci);
766 1.2 bouyer
767 1.2 bouyer s = splhigh();
768 1.11 cegger #ifdef i386
769 1.2 bouyer lapic_tpr = 0;
770 1.11 cegger #else
771 1.11 cegger lcr8(0);
772 1.11 cegger #endif
773 1.11 cegger x86_enable_intr();
774 1.11 cegger splx(s);
775 1.12 cegger #if 0
776 1.11 cegger x86_errata();
777 1.11 cegger #endif
778 1.2 bouyer
779 1.11 cegger aprint_debug_dev(ci->ci_dev, "CPU %ld running\n",
780 1.11 cegger (long)ci->ci_cpuid);
781 1.2 bouyer }
782 1.2 bouyer
783 1.2 bouyer #if defined(DDB)
784 1.2 bouyer
785 1.2 bouyer #include <ddb/db_output.h>
786 1.2 bouyer #include <machine/db_machdep.h>
787 1.2 bouyer
788 1.2 bouyer /*
789 1.2 bouyer * Dump CPU information from ddb.
790 1.2 bouyer */
791 1.2 bouyer void
792 1.2 bouyer cpu_debug_dump(void)
793 1.2 bouyer {
794 1.2 bouyer struct cpu_info *ci;
795 1.2 bouyer CPU_INFO_ITERATOR cii;
796 1.2 bouyer
797 1.13 yamt db_printf("addr dev id flags ipis curlwp fpcurlwp\n");
798 1.2 bouyer for (CPU_INFO_FOREACH(cii, ci)) {
799 1.2 bouyer db_printf("%p %s %ld %x %x %10p %10p\n",
800 1.2 bouyer ci,
801 1.9 cegger ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
802 1.12 cegger (long)ci->ci_cpuid,
803 1.2 bouyer ci->ci_flags, ci->ci_ipis,
804 1.2 bouyer ci->ci_curlwp,
805 1.2 bouyer ci->ci_fpcurlwp);
806 1.2 bouyer }
807 1.2 bouyer }
808 1.38 cegger #endif /* DDB */
809 1.2 bouyer
810 1.2 bouyer static void
811 1.10 cegger cpu_copy_trampoline(void)
812 1.2 bouyer {
813 1.2 bouyer /*
814 1.2 bouyer * Copy boot code.
815 1.2 bouyer */
816 1.2 bouyer extern u_char cpu_spinup_trampoline[];
817 1.2 bouyer extern u_char cpu_spinup_trampoline_end[];
818 1.11 cegger
819 1.11 cegger vaddr_t mp_trampoline_vaddr;
820 1.11 cegger
821 1.11 cegger mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
822 1.11 cegger UVM_KMF_VAONLY);
823 1.11 cegger
824 1.11 cegger pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
825 1.36 cegger VM_PROT_READ | VM_PROT_WRITE, 0);
826 1.11 cegger pmap_update(pmap_kernel());
827 1.11 cegger memcpy((void *)mp_trampoline_vaddr,
828 1.11 cegger cpu_spinup_trampoline,
829 1.11 cegger cpu_spinup_trampoline_end - cpu_spinup_trampoline);
830 1.11 cegger
831 1.11 cegger pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
832 1.11 cegger pmap_update(pmap_kernel());
833 1.11 cegger uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
834 1.2 bouyer }
835 1.2 bouyer
836 1.38 cegger #endif /* MULTIPROCESSOR */
837 1.2 bouyer
838 1.11 cegger #ifdef i386
839 1.11 cegger #if 0
840 1.11 cegger static void
841 1.11 cegger tss_init(struct i386tss *tss, void *stack, void *func)
842 1.11 cegger {
843 1.11 cegger memset(tss, 0, sizeof *tss);
844 1.11 cegger tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
845 1.11 cegger tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
846 1.11 cegger tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
847 1.11 cegger tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
848 1.11 cegger tss->tss_gs = tss->__tss_es = tss->__tss_ds =
849 1.11 cegger tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
850 1.11 cegger tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
851 1.11 cegger tss->tss_esp = (int)((char *)stack + USPACE - 16);
852 1.11 cegger tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
853 1.11 cegger tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */
854 1.11 cegger tss->__tss_eip = (int)func;
855 1.11 cegger }
856 1.11 cegger #endif
857 1.2 bouyer
858 1.2 bouyer /* XXX */
859 1.2 bouyer #define IDTVEC(name) __CONCAT(X, name)
860 1.2 bouyer typedef void (vector)(void);
861 1.2 bouyer extern vector IDTVEC(tss_trap08);
862 1.2 bouyer #ifdef DDB
863 1.2 bouyer extern vector Xintrddbipi;
864 1.2 bouyer extern int ddb_vec;
865 1.2 bouyer #endif
866 1.2 bouyer
867 1.2 bouyer static void
868 1.2 bouyer cpu_set_tss_gates(struct cpu_info *ci)
869 1.2 bouyer {
870 1.11 cegger #if 0
871 1.11 cegger struct segment_descriptor sd;
872 1.11 cegger
873 1.11 cegger ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
874 1.11 cegger UVM_KMF_WIRED);
875 1.11 cegger tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
876 1.11 cegger IDTVEC(tss_trap08));
877 1.11 cegger setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
878 1.11 cegger SDT_SYS386TSS, SEL_KPL, 0, 0);
879 1.11 cegger ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
880 1.11 cegger setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
881 1.11 cegger GSEL(GTRAPTSS_SEL, SEL_KPL));
882 1.11 cegger #endif
883 1.11 cegger
884 1.2 bouyer #if defined(DDB) && defined(MULTIPROCESSOR)
885 1.2 bouyer /*
886 1.2 bouyer * Set up separate handler for the DDB IPI, so that it doesn't
887 1.2 bouyer * stomp on a possibly corrupted stack.
888 1.2 bouyer *
889 1.2 bouyer * XXX overwriting the gate set in db_machine_init.
890 1.2 bouyer * Should rearrange the code so that it's set only once.
891 1.2 bouyer */
892 1.2 bouyer ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
893 1.2 bouyer UVM_KMF_WIRED);
894 1.6 yamt tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack,
895 1.2 bouyer Xintrddbipi);
896 1.2 bouyer
897 1.2 bouyer setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
898 1.2 bouyer SDT_SYS386TSS, SEL_KPL, 0, 0);
899 1.2 bouyer ci->ci_gdt[GIPITSS_SEL].sd = sd;
900 1.2 bouyer
901 1.2 bouyer setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
902 1.2 bouyer GSEL(GIPITSS_SEL, SEL_KPL));
903 1.2 bouyer #endif
904 1.2 bouyer }
905 1.11 cegger #else
906 1.11 cegger static void
907 1.11 cegger cpu_set_tss_gates(struct cpu_info *ci)
908 1.11 cegger {
909 1.11 cegger
910 1.11 cegger }
911 1.11 cegger #endif /* i386 */
912 1.2 bouyer
913 1.2 bouyer int
914 1.5 joerg mp_cpu_start(struct cpu_info *ci, paddr_t target)
915 1.2 bouyer {
916 1.2 bouyer #if 0
917 1.2 bouyer #if NLAPIC > 0
918 1.2 bouyer int error;
919 1.2 bouyer #endif
920 1.2 bouyer unsigned short dwordptr[2];
921 1.2 bouyer
922 1.2 bouyer /*
923 1.11 cegger * Bootstrap code must be addressable in real mode
924 1.11 cegger * and it must be page aligned.
925 1.11 cegger */
926 1.11 cegger KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
927 1.11 cegger
928 1.11 cegger /*
929 1.2 bouyer * "The BSP must initialize CMOS shutdown code to 0Ah ..."
930 1.2 bouyer */
931 1.2 bouyer
932 1.2 bouyer outb(IO_RTC, NVRAM_RESET);
933 1.2 bouyer outb(IO_RTC+1, NVRAM_RESET_JUMP);
934 1.2 bouyer
935 1.2 bouyer /*
936 1.2 bouyer * "and the warm reset vector (DWORD based at 40:67) to point
937 1.2 bouyer * to the AP startup code ..."
938 1.2 bouyer */
939 1.2 bouyer
940 1.2 bouyer dwordptr[0] = 0;
941 1.5 joerg dwordptr[1] = target >> 4;
942 1.2 bouyer
943 1.36 cegger pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
944 1.45 rmind pmap_update(pmap_kernel());
945 1.45 rmind
946 1.11 cegger memcpy ((uint8_t *) 0x467, dwordptr, 4);
947 1.45 rmind
948 1.2 bouyer pmap_kremove (0, PAGE_SIZE);
949 1.45 rmind pmap_update(pmap_kernel());
950 1.2 bouyer
951 1.2 bouyer #if NLAPIC > 0
952 1.2 bouyer /*
953 1.2 bouyer * ... prior to executing the following sequence:"
954 1.2 bouyer */
955 1.2 bouyer
956 1.2 bouyer if (ci->ci_flags & CPUF_AP) {
957 1.23 ad if ((error = x86_ipi_init(ci->ci_cpuid)) != 0)
958 1.2 bouyer return error;
959 1.2 bouyer
960 1.2 bouyer delay(10000);
961 1.2 bouyer
962 1.2 bouyer if (cpu_feature & CPUID_APIC) {
963 1.23 ad error = x86_ipi_init(ci->ci_cpuid);
964 1.11 cegger if (error != 0) {
965 1.11 cegger aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
966 1.11 cegger __func__);
967 1.11 cegger return error;
968 1.11 cegger }
969 1.11 cegger
970 1.11 cegger delay(10000);
971 1.2 bouyer
972 1.23 ad error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid,
973 1.11 cegger LAPIC_DLMODE_STARTUP);
974 1.11 cegger if (error != 0) {
975 1.11 cegger aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
976 1.11 cegger __func__);
977 1.2 bouyer return error;
978 1.11 cegger }
979 1.2 bouyer delay(200);
980 1.2 bouyer
981 1.23 ad error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid,
982 1.11 cegger LAPIC_DLMODE_STARTUP);
983 1.11 cegger if (error != 0) {
984 1.11 cegger aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n",
985 1.11 cegger __func__);
986 1.2 bouyer return error;
987 1.11 cegger }
988 1.2 bouyer delay(200);
989 1.2 bouyer }
990 1.2 bouyer }
991 1.2 bouyer #endif
992 1.2 bouyer #endif /* 0 */
993 1.2 bouyer return 0;
994 1.2 bouyer }
995 1.2 bouyer
996 1.2 bouyer void
997 1.2 bouyer mp_cpu_start_cleanup(struct cpu_info *ci)
998 1.2 bouyer {
999 1.2 bouyer #if 0
1000 1.2 bouyer /*
1001 1.2 bouyer * Ensure the NVRAM reset byte contains something vaguely sane.
1002 1.2 bouyer */
1003 1.2 bouyer
1004 1.2 bouyer outb(IO_RTC, NVRAM_RESET);
1005 1.2 bouyer outb(IO_RTC+1, NVRAM_RESET_RST);
1006 1.2 bouyer #endif
1007 1.2 bouyer }
1008 1.2 bouyer
1009 1.2 bouyer void
1010 1.3 bouyer cpu_init_msrs(struct cpu_info *ci, bool full)
1011 1.2 bouyer {
1012 1.43 jym #ifdef __x86_64__
1013 1.3 bouyer if (full) {
1014 1.3 bouyer HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
1015 1.11 cegger HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
1016 1.3 bouyer HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
1017 1.3 bouyer }
1018 1.43 jym #endif /* __x86_64__ */
1019 1.44 jym
1020 1.44 jym if (cpu_feature[2] & CPUID_NOX)
1021 1.44 jym wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
1022 1.2 bouyer }
1023 1.2 bouyer
1024 1.11 cegger void
1025 1.11 cegger cpu_offline_md(void)
1026 1.11 cegger {
1027 1.11 cegger int s;
1028 1.11 cegger
1029 1.11 cegger s = splhigh();
1030 1.11 cegger #ifdef __i386__
1031 1.11 cegger npxsave_cpu(true);
1032 1.11 cegger #else
1033 1.11 cegger fpusave_cpu(true);
1034 1.11 cegger #endif
1035 1.11 cegger splx(s);
1036 1.11 cegger }
1037 1.11 cegger
1038 1.11 cegger #if 0
1039 1.11 cegger /* XXX joerg restructure and restart CPUs individually */
1040 1.11 cegger static bool
1041 1.41 dyoung cpu_suspend(device_t dv, const pmf_qual_t *qual)
1042 1.11 cegger {
1043 1.11 cegger struct cpu_softc *sc = device_private(dv);
1044 1.11 cegger struct cpu_info *ci = sc->sc_info;
1045 1.11 cegger int err;
1046 1.11 cegger
1047 1.11 cegger if (ci->ci_flags & CPUF_PRIMARY)
1048 1.11 cegger return true;
1049 1.11 cegger if (ci->ci_data.cpu_idlelwp == NULL)
1050 1.11 cegger return true;
1051 1.11 cegger if ((ci->ci_flags & CPUF_PRESENT) == 0)
1052 1.11 cegger return true;
1053 1.11 cegger
1054 1.11 cegger sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
1055 1.11 cegger
1056 1.11 cegger if (sc->sc_wasonline) {
1057 1.11 cegger mutex_enter(&cpu_lock);
1058 1.29 rmind err = cpu_setstate(ci, false);
1059 1.11 cegger mutex_exit(&cpu_lock);
1060 1.11 cegger
1061 1.11 cegger if (err)
1062 1.11 cegger return false;
1063 1.11 cegger }
1064 1.11 cegger
1065 1.11 cegger return true;
1066 1.11 cegger }
1067 1.11 cegger
1068 1.11 cegger static bool
1069 1.41 dyoung cpu_resume(device_t dv, const pmf_qual_t *qual)
1070 1.11 cegger {
1071 1.11 cegger struct cpu_softc *sc = device_private(dv);
1072 1.11 cegger struct cpu_info *ci = sc->sc_info;
1073 1.11 cegger int err = 0;
1074 1.11 cegger
1075 1.11 cegger if (ci->ci_flags & CPUF_PRIMARY)
1076 1.11 cegger return true;
1077 1.11 cegger if (ci->ci_data.cpu_idlelwp == NULL)
1078 1.11 cegger return true;
1079 1.11 cegger if ((ci->ci_flags & CPUF_PRESENT) == 0)
1080 1.11 cegger return true;
1081 1.11 cegger
1082 1.11 cegger if (sc->sc_wasonline) {
1083 1.11 cegger mutex_enter(&cpu_lock);
1084 1.29 rmind err = cpu_setstate(ci, true);
1085 1.11 cegger mutex_exit(&cpu_lock);
1086 1.11 cegger }
1087 1.11 cegger
1088 1.11 cegger return err == 0;
1089 1.11 cegger }
1090 1.11 cegger #endif
1091 1.11 cegger
1092 1.2 bouyer void
1093 1.2 bouyer cpu_get_tsc_freq(struct cpu_info *ci)
1094 1.2 bouyer {
1095 1.16 cegger const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
1096 1.2 bouyer delay(1000000);
1097 1.2 bouyer uint64_t freq = 1000000000ULL << 32;
1098 1.2 bouyer freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
1099 1.2 bouyer if ( tinfo->tsc_shift < 0 )
1100 1.2 bouyer freq = freq << -tinfo->tsc_shift;
1101 1.2 bouyer else
1102 1.2 bouyer freq = freq >> tinfo->tsc_shift;
1103 1.20 ad ci->ci_data.cpu_cc_freq = freq;
1104 1.2 bouyer }
1105 1.19 joerg
1106 1.19 joerg void
1107 1.19 joerg x86_cpu_idle_xen(void)
1108 1.19 joerg {
1109 1.19 joerg struct cpu_info *ci = curcpu();
1110 1.19 joerg
1111 1.19 joerg KASSERT(ci->ci_ilevel == IPL_NONE);
1112 1.19 joerg
1113 1.19 joerg x86_disable_intr();
1114 1.19 joerg if (!__predict_false(ci->ci_want_resched)) {
1115 1.19 joerg idle_block();
1116 1.19 joerg } else {
1117 1.19 joerg x86_enable_intr();
1118 1.19 joerg }
1119 1.19 joerg }
1120 1.47 jym
1121 1.47 jym /*
1122 1.47 jym * Loads pmap for the current CPU.
1123 1.47 jym */
1124 1.47 jym void
1125 1.47 jym cpu_load_pmap(struct pmap *pmap)
1126 1.47 jym {
1127 1.47 jym #ifdef i386
1128 1.47 jym #ifdef PAE
1129 1.47 jym int i, s;
1130 1.47 jym struct cpu_info *ci;
1131 1.47 jym
1132 1.47 jym s = splvm(); /* just to be safe */
1133 1.47 jym ci = curcpu();
1134 1.47 jym paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);
1135 1.47 jym /* don't update the kernel L3 slot */
1136 1.47 jym for (i = 0 ; i < PDP_SIZE - 1; i++) {
1137 1.47 jym xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t),
1138 1.47 jym xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V);
1139 1.47 jym }
1140 1.47 jym splx(s);
1141 1.47 jym tlbflush();
1142 1.47 jym #else /* PAE */
1143 1.47 jym lcr3(pmap_pdirpa(pmap, 0));
1144 1.47 jym #endif /* PAE */
1145 1.47 jym #endif /* i386 */
1146 1.47 jym
1147 1.47 jym #ifdef __x86_64__
1148 1.47 jym int i, s;
1149 1.47 jym pd_entry_t *old_pgd, *new_pgd;
1150 1.47 jym paddr_t addr;
1151 1.47 jym struct cpu_info *ci;
1152 1.47 jym
1153 1.47 jym /* kernel pmap always in cr3 and should never go in user cr3 */
1154 1.47 jym if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) {
1155 1.47 jym ci = curcpu();
1156 1.47 jym /*
1157 1.47 jym * Map user space address in kernel space and load
1158 1.47 jym * user cr3
1159 1.47 jym */
1160 1.47 jym s = splvm();
1161 1.47 jym new_pgd = pmap->pm_pdir;
1162 1.47 jym old_pgd = pmap_kernel()->pm_pdir;
1163 1.47 jym addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0));
1164 1.47 jym for (i = 0; i < PDIR_SLOT_PTE;
1165 1.47 jym i++, addr += sizeof(pd_entry_t)) {
1166 1.47 jym if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V))
1167 1.47 jym xpq_queue_pte_update(addr, new_pgd[i]);
1168 1.47 jym }
1169 1.47 jym tlbflush();
1170 1.47 jym xen_set_user_pgd(pmap_pdirpa(pmap, 0));
1171 1.47 jym ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);
1172 1.47 jym splx(s);
1173 1.47 jym }
1174 1.47 jym #endif /* __x86_64__ */
1175 1.47 jym }
1176