cpu.c revision 1.65 1 1.65 jym /* $NetBSD: cpu.c,v 1.65 2011/09/20 00:12:24 jym Exp $ */
2 1.2 bouyer /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */
3 1.2 bouyer
4 1.2 bouyer /*-
5 1.2 bouyer * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 1.19 joerg * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
7 1.2 bouyer * All rights reserved.
8 1.2 bouyer *
9 1.2 bouyer * This code is derived from software contributed to The NetBSD Foundation
10 1.2 bouyer * by RedBack Networks Inc.
11 1.2 bouyer *
12 1.2 bouyer * Author: Bill Sommerfeld
13 1.2 bouyer *
14 1.2 bouyer * Redistribution and use in source and binary forms, with or without
15 1.2 bouyer * modification, are permitted provided that the following conditions
16 1.2 bouyer * are met:
17 1.2 bouyer * 1. Redistributions of source code must retain the above copyright
18 1.2 bouyer * notice, this list of conditions and the following disclaimer.
19 1.2 bouyer * 2. Redistributions in binary form must reproduce the above copyright
20 1.2 bouyer * notice, this list of conditions and the following disclaimer in the
21 1.2 bouyer * documentation and/or other materials provided with the distribution.
22 1.2 bouyer *
23 1.2 bouyer * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
24 1.2 bouyer * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
25 1.2 bouyer * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26 1.2 bouyer * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
27 1.2 bouyer * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 1.2 bouyer * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 1.2 bouyer * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 1.2 bouyer * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 1.2 bouyer * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 1.2 bouyer * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 1.2 bouyer * POSSIBILITY OF SUCH DAMAGE.
34 1.2 bouyer */
35 1.2 bouyer
36 1.2 bouyer /*
37 1.2 bouyer * Copyright (c) 1999 Stefan Grefen
38 1.2 bouyer *
39 1.2 bouyer * Redistribution and use in source and binary forms, with or without
40 1.2 bouyer * modification, are permitted provided that the following conditions
41 1.2 bouyer * are met:
42 1.2 bouyer * 1. Redistributions of source code must retain the above copyright
43 1.2 bouyer * notice, this list of conditions and the following disclaimer.
44 1.2 bouyer * 2. Redistributions in binary form must reproduce the above copyright
45 1.2 bouyer * notice, this list of conditions and the following disclaimer in the
46 1.2 bouyer * documentation and/or other materials provided with the distribution.
47 1.2 bouyer * 3. All advertising materials mentioning features or use of this software
48 1.2 bouyer * must display the following acknowledgement:
49 1.2 bouyer * This product includes software developed by the NetBSD
50 1.2 bouyer * Foundation, Inc. and its contributors.
51 1.2 bouyer * 4. Neither the name of The NetBSD Foundation nor the names of its
52 1.2 bouyer * contributors may be used to endorse or promote products derived
53 1.2 bouyer * from this software without specific prior written permission.
54 1.2 bouyer *
55 1.2 bouyer * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
56 1.2 bouyer * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
57 1.2 bouyer * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
58 1.2 bouyer * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
59 1.2 bouyer * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
60 1.2 bouyer * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
61 1.2 bouyer * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
62 1.2 bouyer * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
63 1.2 bouyer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
64 1.2 bouyer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
65 1.2 bouyer * SUCH DAMAGE.
66 1.2 bouyer */
67 1.2 bouyer
68 1.2 bouyer #include <sys/cdefs.h>
69 1.65 jym __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.65 2011/09/20 00:12:24 jym Exp $");
70 1.2 bouyer
71 1.2 bouyer #include "opt_ddb.h"
72 1.2 bouyer #include "opt_multiprocessor.h"
73 1.2 bouyer #include "opt_mpbios.h" /* for MPDEBUG */
74 1.2 bouyer #include "opt_mtrr.h"
75 1.2 bouyer #include "opt_xen.h"
76 1.2 bouyer
77 1.2 bouyer #include "lapic.h"
78 1.2 bouyer #include "ioapic.h"
79 1.2 bouyer
80 1.2 bouyer #include <sys/param.h>
81 1.2 bouyer #include <sys/proc.h>
82 1.2 bouyer #include <sys/systm.h>
83 1.2 bouyer #include <sys/device.h>
84 1.31 cegger #include <sys/kmem.h>
85 1.11 cegger #include <sys/cpu.h>
86 1.11 cegger #include <sys/atomic.h>
87 1.32 cegger #include <sys/reboot.h>
88 1.62 cherry #include <sys/idle.h>
89 1.2 bouyer
90 1.51 uebayasi #include <uvm/uvm.h>
91 1.2 bouyer
92 1.2 bouyer #include <machine/cpufunc.h>
93 1.2 bouyer #include <machine/cpuvar.h>
94 1.2 bouyer #include <machine/pmap.h>
95 1.2 bouyer #include <machine/vmparam.h>
96 1.2 bouyer #include <machine/mpbiosvar.h>
97 1.2 bouyer #include <machine/pcb.h>
98 1.2 bouyer #include <machine/specialreg.h>
99 1.2 bouyer #include <machine/segments.h>
100 1.2 bouyer #include <machine/gdt.h>
101 1.2 bouyer #include <machine/mtrr.h>
102 1.2 bouyer #include <machine/pio.h>
103 1.2 bouyer
104 1.62 cherry #ifdef i386
105 1.62 cherry #include <machine/npx.h>
106 1.62 cherry #else
107 1.62 cherry #include <machine/fpu.h>
108 1.62 cherry #endif
109 1.62 cherry
110 1.62 cherry #include <xen/xen.h>
111 1.62 cherry #include <xen/xen3-public/vcpu.h>
112 1.2 bouyer #include <xen/vcpuvar.h>
113 1.2 bouyer
114 1.2 bouyer #if NLAPIC > 0
115 1.2 bouyer #include <machine/apicvar.h>
116 1.2 bouyer #include <machine/i82489reg.h>
117 1.2 bouyer #include <machine/i82489var.h>
118 1.2 bouyer #endif
119 1.2 bouyer
120 1.2 bouyer #include <dev/ic/mc146818reg.h>
121 1.2 bouyer #include <dev/isa/isareg.h>
122 1.2 bouyer
123 1.38 cegger #if MAXCPUS > 32
124 1.38 cegger #error cpu_info contains 32bit bitmasks
125 1.38 cegger #endif
126 1.27 ad
127 1.56 jruoho static int cpu_match(device_t, cfdata_t, void *);
128 1.56 jruoho static void cpu_attach(device_t, device_t, void *);
129 1.56 jruoho static void cpu_defer(device_t);
130 1.56 jruoho static int cpu_rescan(device_t, const char *, const int *);
131 1.56 jruoho static void cpu_childdetached(device_t, device_t);
132 1.56 jruoho static int vcpu_match(device_t, cfdata_t, void *);
133 1.56 jruoho static void vcpu_attach(device_t, device_t, void *);
134 1.56 jruoho static void cpu_attach_common(device_t, device_t, void *);
135 1.56 jruoho void cpu_offline_md(void);
136 1.2 bouyer
137 1.2 bouyer struct cpu_softc {
138 1.10 cegger device_t sc_dev; /* device tree glue */
139 1.2 bouyer struct cpu_info *sc_info; /* pointer to CPU info */
140 1.32 cegger bool sc_wasonline;
141 1.2 bouyer };
142 1.2 bouyer
143 1.62 cherry int mp_cpu_start(struct cpu_info *, vaddr_t);
144 1.2 bouyer void mp_cpu_start_cleanup(struct cpu_info *);
145 1.2 bouyer const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
146 1.2 bouyer mp_cpu_start_cleanup };
147 1.2 bouyer
148 1.53 jruoho CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
149 1.53 jruoho cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
150 1.53 jruoho
151 1.10 cegger CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
152 1.2 bouyer vcpu_match, vcpu_attach, NULL, NULL);
153 1.2 bouyer
154 1.2 bouyer /*
155 1.2 bouyer * Statically-allocated CPU info for the primary CPU (or the only
156 1.2 bouyer * CPU, on uniprocessors). The CPU info list is initialized to
157 1.2 bouyer * point at it.
158 1.2 bouyer */
159 1.2 bouyer #ifdef TRAPLOG
160 1.2 bouyer #include <machine/tlog.h>
161 1.2 bouyer struct tlog tlog_primary;
162 1.2 bouyer #endif
163 1.38 cegger struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
164 1.7 bouyer .ci_dev = 0,
165 1.2 bouyer .ci_self = &cpu_info_primary,
166 1.4 bouyer .ci_idepth = -1,
167 1.2 bouyer .ci_curlwp = &lwp0,
168 1.25 ad .ci_curldt = -1,
169 1.58 rmind .ci_cpumask = 1,
170 1.2 bouyer #ifdef TRAPLOG
171 1.2 bouyer .ci_tlog = &tlog_primary,
172 1.2 bouyer #endif
173 1.2 bouyer
174 1.2 bouyer };
175 1.38 cegger struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
176 1.7 bouyer .ci_dev = 0,
177 1.2 bouyer .ci_self = &phycpu_info_primary,
178 1.2 bouyer };
179 1.2 bouyer
180 1.2 bouyer struct cpu_info *cpu_info_list = &cpu_info_primary;
181 1.38 cegger struct cpu_info *phycpu_info_list = &phycpu_info_primary;
182 1.2 bouyer
183 1.58 rmind uint32_t cpus_attached = 1;
184 1.60 rmind uint32_t cpus_running = 1;
185 1.11 cegger
186 1.38 cegger uint32_t phycpus_attached = 0;
187 1.38 cegger uint32_t phycpus_running = 0;
188 1.38 cegger
189 1.43 jym uint32_t cpu_feature[5]; /* X86 CPUID feature bits
190 1.43 jym * [0] basic features %edx
191 1.43 jym * [1] basic features %ecx
192 1.43 jym * [2] extended features %edx
193 1.43 jym * [3] extended features %ecx
194 1.43 jym * [4] VIA padlock features
195 1.43 jym */
196 1.43 jym
197 1.11 cegger bool x86_mp_online;
198 1.11 cegger paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
199 1.2 bouyer
200 1.38 cegger #if defined(MULTIPROCESSOR)
201 1.2 bouyer void cpu_hatch(void *);
202 1.2 bouyer static void cpu_boot_secondary(struct cpu_info *ci);
203 1.2 bouyer static void cpu_start_secondary(struct cpu_info *ci);
204 1.38 cegger #endif /* MULTIPROCESSOR */
205 1.2 bouyer
206 1.56 jruoho static int
207 1.10 cegger cpu_match(device_t parent, cfdata_t match, void *aux)
208 1.2 bouyer {
209 1.2 bouyer
210 1.2 bouyer return 1;
211 1.2 bouyer }
212 1.2 bouyer
213 1.56 jruoho static void
214 1.10 cegger cpu_attach(device_t parent, device_t self, void *aux)
215 1.2 bouyer {
216 1.10 cegger struct cpu_softc *sc = device_private(self);
217 1.2 bouyer struct cpu_attach_args *caa = aux;
218 1.2 bouyer struct cpu_info *ci;
219 1.34 cegger uintptr_t ptr;
220 1.52 bouyer static int nphycpu = 0;
221 1.2 bouyer
222 1.10 cegger sc->sc_dev = self;
223 1.10 cegger
224 1.38 cegger if (phycpus_attached == ~0) {
225 1.34 cegger aprint_error(": increase MAXCPUS\n");
226 1.34 cegger return;
227 1.34 cegger }
228 1.34 cegger
229 1.2 bouyer /*
230 1.2 bouyer * If we're an Application Processor, allocate a cpu_info
231 1.52 bouyer * If we're the first attached CPU use the primary cpu_info,
232 1.52 bouyer * otherwise allocate a new one
233 1.2 bouyer */
234 1.52 bouyer aprint_naive("\n");
235 1.52 bouyer aprint_normal("\n");
236 1.52 bouyer if (nphycpu > 0) {
237 1.52 bouyer struct cpu_info *tmp;
238 1.34 cegger ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
239 1.34 cegger KM_SLEEP);
240 1.42 jym ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
241 1.24 ad ci->ci_curldt = -1;
242 1.52 bouyer
243 1.52 bouyer tmp = phycpu_info_list;
244 1.52 bouyer while (tmp->ci_next)
245 1.52 bouyer tmp = tmp->ci_next;
246 1.52 bouyer
247 1.52 bouyer tmp->ci_next = ci;
248 1.2 bouyer } else {
249 1.2 bouyer ci = &phycpu_info_primary;
250 1.2 bouyer }
251 1.2 bouyer
252 1.2 bouyer ci->ci_self = ci;
253 1.2 bouyer sc->sc_info = ci;
254 1.2 bouyer
255 1.2 bouyer ci->ci_dev = self;
256 1.50 jruoho ci->ci_acpiid = caa->cpu_id;
257 1.23 ad ci->ci_cpuid = caa->cpu_number;
258 1.16 cegger ci->ci_vcpu = NULL;
259 1.52 bouyer ci->ci_index = nphycpu++;
260 1.52 bouyer ci->ci_cpumask = (1 << cpu_index(ci));
261 1.2 bouyer
262 1.52 bouyer atomic_or_32(&phycpus_attached, ci->ci_cpumask);
263 1.38 cegger
264 1.52 bouyer if (!pmf_device_register(self, NULL, NULL))
265 1.52 bouyer aprint_error_dev(self, "couldn't establish power handler\n");
266 1.34 cegger
267 1.56 jruoho (void)config_defer(self, cpu_defer);
268 1.56 jruoho }
269 1.56 jruoho
270 1.56 jruoho static void
271 1.56 jruoho cpu_defer(device_t self)
272 1.56 jruoho {
273 1.56 jruoho cpu_rescan(self, NULL, NULL);
274 1.2 bouyer }
275 1.2 bouyer
276 1.56 jruoho static int
277 1.53 jruoho cpu_rescan(device_t self, const char *ifattr, const int *locators)
278 1.53 jruoho {
279 1.53 jruoho struct cpu_softc *sc = device_private(self);
280 1.53 jruoho struct cpufeature_attach_args cfaa;
281 1.53 jruoho struct cpu_info *ci = sc->sc_info;
282 1.53 jruoho
283 1.53 jruoho memset(&cfaa, 0, sizeof(cfaa));
284 1.53 jruoho cfaa.ci = ci;
285 1.53 jruoho
286 1.53 jruoho if (ifattr_match(ifattr, "cpufeaturebus")) {
287 1.53 jruoho
288 1.53 jruoho if (ci->ci_frequency == NULL) {
289 1.55 jruoho cfaa.name = "frequency";
290 1.54 jruoho ci->ci_frequency = config_found_ia(self,
291 1.54 jruoho "cpufeaturebus", &cfaa, NULL);
292 1.54 jruoho }
293 1.53 jruoho }
294 1.53 jruoho
295 1.53 jruoho return 0;
296 1.53 jruoho }
297 1.53 jruoho
298 1.56 jruoho static void
299 1.53 jruoho cpu_childdetached(device_t self, device_t child)
300 1.53 jruoho {
301 1.53 jruoho struct cpu_softc *sc = device_private(self);
302 1.53 jruoho struct cpu_info *ci = sc->sc_info;
303 1.53 jruoho
304 1.53 jruoho if (ci->ci_frequency == child)
305 1.53 jruoho ci->ci_frequency = NULL;
306 1.53 jruoho }
307 1.53 jruoho
308 1.56 jruoho static int
309 1.10 cegger vcpu_match(device_t parent, cfdata_t match, void *aux)
310 1.2 bouyer {
311 1.2 bouyer struct vcpu_attach_args *vcaa = aux;
312 1.62 cherry struct vcpu_runstate_info vcr;
313 1.62 cherry int error;
314 1.62 cherry
315 1.62 cherry if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) {
316 1.62 cherry error = HYPERVISOR_vcpu_op(VCPUOP_get_runstate_info,
317 1.62 cherry vcaa->vcaa_caa.cpu_number,
318 1.62 cherry &vcr);
319 1.62 cherry switch (error) {
320 1.62 cherry case 0:
321 1.62 cherry return 1;
322 1.62 cherry case -ENOENT:
323 1.62 cherry return 0;
324 1.62 cherry default:
325 1.62 cherry panic("Unknown hypervisor error %d returned on vcpu runstate probe\n", error);
326 1.62 cherry }
327 1.62 cherry }
328 1.2 bouyer
329 1.2 bouyer return 0;
330 1.2 bouyer }
331 1.2 bouyer
332 1.56 jruoho static void
333 1.10 cegger vcpu_attach(device_t parent, device_t self, void *aux)
334 1.2 bouyer {
335 1.2 bouyer struct vcpu_attach_args *vcaa = aux;
336 1.2 bouyer
337 1.62 cherry KASSERT(vcaa->vcaa_caa.cpu_func == NULL);
338 1.62 cherry vcaa->vcaa_caa.cpu_func = &mp_cpu_funcs;
339 1.2 bouyer cpu_attach_common(parent, self, &vcaa->vcaa_caa);
340 1.65 jym
341 1.65 jym if (!pmf_device_register(self, NULL, NULL))
342 1.65 jym aprint_error_dev(self, "couldn't establish power handler\n");
343 1.2 bouyer }
344 1.2 bouyer
345 1.62 cherry static int
346 1.62 cherry vcpu_is_up(struct cpu_info *ci)
347 1.62 cherry {
348 1.62 cherry KASSERT(ci != NULL);
349 1.62 cherry return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_cpuid, NULL);
350 1.62 cherry }
351 1.62 cherry
352 1.2 bouyer static void
353 1.2 bouyer cpu_vm_init(struct cpu_info *ci)
354 1.2 bouyer {
355 1.2 bouyer int ncolors = 2, i;
356 1.2 bouyer
357 1.2 bouyer for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
358 1.2 bouyer struct x86_cache_info *cai;
359 1.2 bouyer int tcolors;
360 1.2 bouyer
361 1.2 bouyer cai = &ci->ci_cinfo[i];
362 1.2 bouyer
363 1.2 bouyer tcolors = atop(cai->cai_totalsize);
364 1.2 bouyer switch(cai->cai_associativity) {
365 1.2 bouyer case 0xff:
366 1.2 bouyer tcolors = 1; /* fully associative */
367 1.2 bouyer break;
368 1.2 bouyer case 0:
369 1.2 bouyer case 1:
370 1.2 bouyer break;
371 1.2 bouyer default:
372 1.2 bouyer tcolors /= cai->cai_associativity;
373 1.2 bouyer }
374 1.2 bouyer ncolors = max(ncolors, tcolors);
375 1.2 bouyer }
376 1.2 bouyer
377 1.2 bouyer /*
378 1.2 bouyer * Knowing the size of the largest cache on this CPU, re-color
379 1.2 bouyer * our pages.
380 1.2 bouyer */
381 1.2 bouyer if (ncolors <= uvmexp.ncolors)
382 1.2 bouyer return;
383 1.28 bouyer aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
384 1.2 bouyer uvm_page_recolor(ncolors);
385 1.2 bouyer }
386 1.2 bouyer
387 1.56 jruoho static void
388 1.11 cegger cpu_attach_common(device_t parent, device_t self, void *aux)
389 1.2 bouyer {
390 1.10 cegger struct cpu_softc *sc = device_private(self);
391 1.2 bouyer struct cpu_attach_args *caa = aux;
392 1.2 bouyer struct cpu_info *ci;
393 1.12 cegger uintptr_t ptr;
394 1.2 bouyer int cpunum = caa->cpu_number;
395 1.38 cegger static bool again = false;
396 1.2 bouyer
397 1.10 cegger sc->sc_dev = self;
398 1.10 cegger
399 1.2 bouyer /*
400 1.2 bouyer * If we're an Application Processor, allocate a cpu_info
401 1.2 bouyer * structure, otherwise use the primary's.
402 1.2 bouyer */
403 1.2 bouyer if (caa->cpu_role == CPU_ROLE_AP) {
404 1.12 cegger aprint_naive(": Application Processor\n");
405 1.31 cegger ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
406 1.31 cegger KM_SLEEP);
407 1.42 jym ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
408 1.12 cegger memset(ci, 0, sizeof(*ci));
409 1.2 bouyer #ifdef TRAPLOG
410 1.31 cegger ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
411 1.2 bouyer #endif
412 1.2 bouyer } else {
413 1.12 cegger aprint_naive(": %s Processor\n",
414 1.12 cegger caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
415 1.2 bouyer ci = &cpu_info_primary;
416 1.2 bouyer }
417 1.2 bouyer
418 1.2 bouyer ci->ci_self = ci;
419 1.2 bouyer sc->sc_info = ci;
420 1.2 bouyer ci->ci_dev = self;
421 1.23 ad ci->ci_cpuid = cpunum;
422 1.16 cegger
423 1.16 cegger KASSERT(HYPERVISOR_shared_info != NULL);
424 1.16 cegger ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
425 1.16 cegger
426 1.62 cherry KASSERT(ci->ci_func == 0);
427 1.2 bouyer ci->ci_func = caa->cpu_func;
428 1.2 bouyer
429 1.38 cegger /* Must be called before mi_cpu_attach(). */
430 1.38 cegger cpu_vm_init(ci);
431 1.38 cegger
432 1.2 bouyer if (caa->cpu_role == CPU_ROLE_AP) {
433 1.2 bouyer int error;
434 1.2 bouyer
435 1.2 bouyer error = mi_cpu_attach(ci);
436 1.62 cherry
437 1.62 cherry KASSERT(ci->ci_data.cpu_idlelwp != NULL);
438 1.2 bouyer if (error != 0) {
439 1.2 bouyer aprint_normal("\n");
440 1.38 cegger aprint_error_dev(self,
441 1.38 cegger "mi_cpu_attach failed with %d\n", error);
442 1.2 bouyer return;
443 1.2 bouyer }
444 1.62 cherry
445 1.2 bouyer } else {
446 1.2 bouyer KASSERT(ci->ci_data.cpu_idlelwp != NULL);
447 1.2 bouyer }
448 1.2 bouyer
449 1.23 ad ci->ci_cpumask = (1 << cpu_index(ci));
450 1.2 bouyer pmap_reference(pmap_kernel());
451 1.2 bouyer ci->ci_pmap = pmap_kernel();
452 1.2 bouyer ci->ci_tlbstate = TLBSTATE_STALE;
453 1.2 bouyer
454 1.38 cegger /*
455 1.38 cegger * Boot processor may not be attached first, but the below
456 1.38 cegger * must be done to allow booting other processors.
457 1.38 cegger */
458 1.38 cegger if (!again) {
459 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
460 1.38 cegger /* Basic init. */
461 1.38 cegger cpu_intr_init(ci);
462 1.38 cegger cpu_get_tsc_freq(ci);
463 1.38 cegger cpu_init(ci);
464 1.62 cherry pmap_cpu_init_late(ci); /* XXX: cosmetic */
465 1.62 cherry
466 1.62 cherry /* Every processor needs to init it's own ipi h/w (similar to lapic) */
467 1.62 cherry xen_ipi_init();
468 1.62 cherry /* XXX: clock_init() */
469 1.62 cherry
470 1.38 cegger /* Make sure DELAY() is initialized. */
471 1.38 cegger DELAY(1);
472 1.38 cegger again = true;
473 1.38 cegger }
474 1.38 cegger
475 1.2 bouyer /* further PCB init done later. */
476 1.2 bouyer
477 1.2 bouyer switch (caa->cpu_role) {
478 1.2 bouyer case CPU_ROLE_SP:
479 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_SP);
480 1.21 ad cpu_identify(ci);
481 1.12 cegger #if 0
482 1.12 cegger x86_errata();
483 1.12 cegger #endif
484 1.38 cegger x86_cpu_idle_init();
485 1.62 cherry
486 1.2 bouyer break;
487 1.2 bouyer
488 1.2 bouyer case CPU_ROLE_BP:
489 1.38 cegger atomic_or_32(&ci->ci_flags, CPUF_BSP);
490 1.21 ad cpu_identify(ci);
491 1.2 bouyer cpu_init(ci);
492 1.14 bouyer #if 0
493 1.12 cegger x86_errata();
494 1.12 cegger #endif
495 1.38 cegger x86_cpu_idle_init();
496 1.62 cherry
497 1.2 bouyer break;
498 1.2 bouyer
499 1.2 bouyer case CPU_ROLE_AP:
500 1.62 cherry atomic_or_32(&ci->ci_flags, CPUF_AP);
501 1.62 cherry
502 1.2 bouyer /*
503 1.2 bouyer * report on an AP
504 1.2 bouyer */
505 1.2 bouyer
506 1.2 bouyer #if defined(MULTIPROCESSOR)
507 1.62 cherry /* interrupt handler stack */
508 1.2 bouyer cpu_intr_init(ci);
509 1.62 cherry
510 1.62 cherry /* Setup per-cpu memory for gdt */
511 1.2 bouyer gdt_alloc_cpu(ci);
512 1.62 cherry
513 1.62 cherry pmap_cpu_init_late(ci);
514 1.2 bouyer cpu_start_secondary(ci);
515 1.62 cherry
516 1.2 bouyer if (ci->ci_flags & CPUF_PRESENT) {
517 1.30 cegger struct cpu_info *tmp;
518 1.30 cegger
519 1.62 cherry cpu_identify(ci);
520 1.30 cegger tmp = cpu_info_list;
521 1.30 cegger while (tmp->ci_next)
522 1.30 cegger tmp = tmp->ci_next;
523 1.30 cegger
524 1.30 cegger tmp->ci_next = ci;
525 1.2 bouyer }
526 1.2 bouyer #else
527 1.62 cherry aprint_error(": not started\n");
528 1.2 bouyer #endif
529 1.2 bouyer break;
530 1.2 bouyer
531 1.2 bouyer default:
532 1.12 cegger aprint_normal("\n");
533 1.2 bouyer panic("unknown processor type??\n");
534 1.2 bouyer }
535 1.2 bouyer
536 1.46 cegger pat_init(ci);
537 1.34 cegger atomic_or_32(&cpus_attached, ci->ci_cpumask);
538 1.2 bouyer
539 1.12 cegger #if 0
540 1.12 cegger if (!pmf_device_register(self, cpu_suspend, cpu_resume))
541 1.12 cegger aprint_error_dev(self, "couldn't establish power handler\n");
542 1.12 cegger #endif
543 1.12 cegger
544 1.62 cherry #ifdef MPVERBOSE
545 1.2 bouyer if (mp_verbose) {
546 1.2 bouyer struct lwp *l = ci->ci_data.cpu_idlelwp;
547 1.37 rmind struct pcb *pcb = lwp_getpcb(l);
548 1.2 bouyer
549 1.38 cegger aprint_verbose_dev(self,
550 1.38 cegger "idle lwp at %p, idle sp at 0x%p\n",
551 1.12 cegger l,
552 1.12 cegger #ifdef i386
553 1.37 rmind (void *)pcb->pcb_esp
554 1.62 cherry #else /* i386 */
555 1.37 rmind (void *)pcb->pcb_rsp
556 1.62 cherry #endif /* i386 */
557 1.12 cegger );
558 1.12 cegger
559 1.2 bouyer }
560 1.62 cherry #endif /* MPVERBOSE */
561 1.2 bouyer }
562 1.2 bouyer
563 1.2 bouyer /*
564 1.2 bouyer * Initialize the processor appropriately.
565 1.2 bouyer */
566 1.2 bouyer
567 1.2 bouyer void
568 1.10 cegger cpu_init(struct cpu_info *ci)
569 1.2 bouyer {
570 1.2 bouyer
571 1.2 bouyer /*
572 1.2 bouyer * On a P6 or above, enable global TLB caching if the
573 1.2 bouyer * hardware supports it.
574 1.2 bouyer */
575 1.43 jym if (cpu_feature[0] & CPUID_PGE)
576 1.2 bouyer lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */
577 1.2 bouyer
578 1.2 bouyer #ifdef XXXMTRR
579 1.2 bouyer /*
580 1.2 bouyer * On a P6 or above, initialize MTRR's if the hardware supports them.
581 1.2 bouyer */
582 1.43 jym if (cpu_feature[0] & CPUID_MTRR) {
583 1.2 bouyer if ((ci->ci_flags & CPUF_AP) == 0)
584 1.2 bouyer i686_mtrr_init_first();
585 1.2 bouyer mtrr_init_cpu(ci);
586 1.2 bouyer }
587 1.2 bouyer #endif
588 1.2 bouyer /*
589 1.2 bouyer * If we have FXSAVE/FXRESTOR, use them.
590 1.2 bouyer */
591 1.43 jym if (cpu_feature[0] & CPUID_FXSR) {
592 1.2 bouyer lcr4(rcr4() | CR4_OSFXSR);
593 1.2 bouyer
594 1.2 bouyer /*
595 1.2 bouyer * If we have SSE/SSE2, enable XMM exceptions.
596 1.2 bouyer */
597 1.43 jym if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
598 1.2 bouyer lcr4(rcr4() | CR4_OSXMMEXCPT);
599 1.2 bouyer }
600 1.2 bouyer
601 1.47 jym #ifdef __x86_64__
602 1.47 jym /* No user PGD mapped for this CPU yet */
603 1.47 jym ci->ci_xen_current_user_pgd = 0;
604 1.47 jym #endif
605 1.47 jym
606 1.34 cegger atomic_or_32(&cpus_running, ci->ci_cpumask);
607 1.11 cegger atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
608 1.62 cherry
609 1.62 cherry /* XXX: register vcpu_register_runstate_memory_area, and figure out how to make sure this VCPU is running ? */
610 1.2 bouyer }
611 1.2 bouyer
612 1.2 bouyer
613 1.2 bouyer #ifdef MULTIPROCESSOR
614 1.62 cherry
615 1.2 bouyer void
616 1.10 cegger cpu_boot_secondary_processors(void)
617 1.2 bouyer {
618 1.2 bouyer struct cpu_info *ci;
619 1.2 bouyer u_long i;
620 1.38 cegger for (i = 0; i < maxcpus; i++) {
621 1.38 cegger ci = cpu_lookup(i);
622 1.2 bouyer if (ci == NULL)
623 1.2 bouyer continue;
624 1.2 bouyer if (ci->ci_data.cpu_idlelwp == NULL)
625 1.2 bouyer continue;
626 1.2 bouyer if ((ci->ci_flags & CPUF_PRESENT) == 0)
627 1.2 bouyer continue;
628 1.2 bouyer if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
629 1.2 bouyer continue;
630 1.2 bouyer cpu_boot_secondary(ci);
631 1.2 bouyer }
632 1.11 cegger
633 1.11 cegger x86_mp_online = true;
634 1.2 bouyer }
635 1.2 bouyer
636 1.2 bouyer static void
637 1.2 bouyer cpu_init_idle_lwp(struct cpu_info *ci)
638 1.2 bouyer {
639 1.2 bouyer struct lwp *l = ci->ci_data.cpu_idlelwp;
640 1.37 rmind struct pcb *pcb = lwp_getpcb(l);
641 1.2 bouyer
642 1.2 bouyer pcb->pcb_cr0 = rcr0();
643 1.2 bouyer }
644 1.2 bouyer
645 1.2 bouyer void
646 1.10 cegger cpu_init_idle_lwps(void)
647 1.2 bouyer {
648 1.2 bouyer struct cpu_info *ci;
649 1.2 bouyer u_long i;
650 1.2 bouyer
651 1.38 cegger for (i = 0; i < maxcpus; i++) {
652 1.38 cegger ci = cpu_lookup(i);
653 1.2 bouyer if (ci == NULL)
654 1.2 bouyer continue;
655 1.2 bouyer if (ci->ci_data.cpu_idlelwp == NULL)
656 1.2 bouyer continue;
657 1.2 bouyer if ((ci->ci_flags & CPUF_PRESENT) == 0)
658 1.2 bouyer continue;
659 1.2 bouyer cpu_init_idle_lwp(ci);
660 1.2 bouyer }
661 1.2 bouyer }
662 1.2 bouyer
663 1.62 cherry static void
664 1.10 cegger cpu_start_secondary(struct cpu_info *ci)
665 1.2 bouyer {
666 1.2 bouyer int i;
667 1.2 bouyer
668 1.11 cegger aprint_debug_dev(ci->ci_dev, "starting\n");
669 1.2 bouyer
670 1.2 bouyer ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
671 1.62 cherry
672 1.62 cherry if (CPU_STARTUP(ci, (vaddr_t) cpu_hatch) != 0) {
673 1.11 cegger return;
674 1.62 cherry }
675 1.2 bouyer
676 1.2 bouyer /*
677 1.2 bouyer * wait for it to become ready
678 1.2 bouyer */
679 1.11 cegger for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
680 1.2 bouyer delay(10);
681 1.2 bouyer }
682 1.11 cegger if ((ci->ci_flags & CPUF_PRESENT) == 0) {
683 1.9 cegger aprint_error_dev(ci->ci_dev, "failed to become ready\n");
684 1.2 bouyer #if defined(MPDEBUG) && defined(DDB)
685 1.2 bouyer printf("dropping into debugger; continue from here to resume boot\n");
686 1.2 bouyer Debugger();
687 1.2 bouyer #endif
688 1.2 bouyer }
689 1.2 bouyer
690 1.2 bouyer CPU_START_CLEANUP(ci);
691 1.2 bouyer }
692 1.2 bouyer
693 1.2 bouyer void
694 1.10 cegger cpu_boot_secondary(struct cpu_info *ci)
695 1.2 bouyer {
696 1.2 bouyer int i;
697 1.11 cegger atomic_or_32(&ci->ci_flags, CPUF_GO);
698 1.11 cegger for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
699 1.2 bouyer delay(10);
700 1.2 bouyer }
701 1.11 cegger if ((ci->ci_flags & CPUF_RUNNING) == 0) {
702 1.11 cegger aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
703 1.2 bouyer #if defined(MPDEBUG) && defined(DDB)
704 1.2 bouyer printf("dropping into debugger; continue from here to resume boot\n");
705 1.2 bouyer Debugger();
706 1.2 bouyer #endif
707 1.2 bouyer }
708 1.2 bouyer }
709 1.2 bouyer
710 1.2 bouyer /*
711 1.62 cherry * APs end up here immediately after initialisation and VCPUOP_up in
712 1.62 cherry * mp_cpu_start().
713 1.62 cherry * At this point, we are running in the idle pcb/idle stack of the new
714 1.62 cherry * CPU. This function jumps to the idle loop and starts looking for
715 1.62 cherry * work.
716 1.2 bouyer */
717 1.62 cherry extern void x86_64_tls_switch(struct lwp *);
718 1.2 bouyer void
719 1.2 bouyer cpu_hatch(void *v)
720 1.2 bouyer {
721 1.2 bouyer struct cpu_info *ci = (struct cpu_info *)v;
722 1.37 rmind struct pcb *pcb;
723 1.11 cegger int s, i;
724 1.11 cegger
725 1.62 cherry /* Setup TLS and kernel GS/FS */
726 1.62 cherry cpu_init_msrs(ci, true);
727 1.62 cherry cpu_init_idt();
728 1.62 cherry gdt_init_cpu(ci);
729 1.62 cherry
730 1.21 ad cpu_probe(ci);
731 1.11 cegger
732 1.62 cherry atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
733 1.2 bouyer
734 1.11 cegger while ((ci->ci_flags & CPUF_GO) == 0) {
735 1.11 cegger /* Don't use delay, boot CPU may be patching the text. */
736 1.11 cegger for (i = 10000; i != 0; i--)
737 1.11 cegger x86_pause();
738 1.11 cegger }
739 1.2 bouyer
740 1.11 cegger /* Because the text may have been patched in x86_patch(). */
741 1.11 cegger x86_flush();
742 1.58 rmind tlbflushg();
743 1.2 bouyer
744 1.11 cegger KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
745 1.2 bouyer
746 1.37 rmind pcb = lwp_getpcb(curlwp);
747 1.62 cherry pcb->pcb_cr3 = pmap_pdirpa(pmap_kernel(), 0); /* XXX: consider using pmap_load() ? */
748 1.37 rmind pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
749 1.37 rmind
750 1.62 cherry xen_ipi_init();
751 1.62 cherry
752 1.62 cherry xen_initclocks();
753 1.62 cherry
754 1.62 cherry /* XXX: lapic_initclocks(); */
755 1.11 cegger
756 1.62 cherry #ifdef __x86_64__
757 1.12 cegger fpuinit(ci);
758 1.12 cegger #endif
759 1.2 bouyer
760 1.2 bouyer lldt(GSEL(GLDT_SEL, SEL_KPL));
761 1.2 bouyer
762 1.2 bouyer cpu_init(ci);
763 1.11 cegger cpu_get_tsc_freq(ci);
764 1.2 bouyer
765 1.2 bouyer s = splhigh();
766 1.11 cegger x86_enable_intr();
767 1.11 cegger splx(s);
768 1.12 cegger #if 0
769 1.11 cegger x86_errata();
770 1.11 cegger #endif
771 1.2 bouyer
772 1.62 cherry aprint_debug_dev(ci->ci_dev, "running\n");
773 1.62 cherry
774 1.62 cherry cpu_switchto(NULL, ci->ci_data.cpu_idlelwp, true);
775 1.62 cherry
776 1.62 cherry panic("switch to idle_loop context returned!\n");
777 1.62 cherry /* NOTREACHED */
778 1.2 bouyer }
779 1.2 bouyer
780 1.2 bouyer #if defined(DDB)
781 1.2 bouyer
782 1.2 bouyer #include <ddb/db_output.h>
783 1.2 bouyer #include <machine/db_machdep.h>
784 1.2 bouyer
785 1.2 bouyer /*
786 1.2 bouyer * Dump CPU information from ddb.
787 1.2 bouyer */
788 1.2 bouyer void
789 1.2 bouyer cpu_debug_dump(void)
790 1.2 bouyer {
791 1.2 bouyer struct cpu_info *ci;
792 1.2 bouyer CPU_INFO_ITERATOR cii;
793 1.2 bouyer
794 1.13 yamt db_printf("addr dev id flags ipis curlwp fpcurlwp\n");
795 1.2 bouyer for (CPU_INFO_FOREACH(cii, ci)) {
796 1.2 bouyer db_printf("%p %s %ld %x %x %10p %10p\n",
797 1.2 bouyer ci,
798 1.9 cegger ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
799 1.12 cegger (long)ci->ci_cpuid,
800 1.2 bouyer ci->ci_flags, ci->ci_ipis,
801 1.2 bouyer ci->ci_curlwp,
802 1.2 bouyer ci->ci_fpcurlwp);
803 1.2 bouyer }
804 1.2 bouyer }
805 1.38 cegger #endif /* DDB */
806 1.2 bouyer
807 1.62 cherry #endif /* MULTIPROCESSOR */
808 1.62 cherry
809 1.62 cherry extern void hypervisor_callback(void);
810 1.62 cherry extern void failsafe_callback(void);
811 1.62 cherry #ifdef __x86_64__
812 1.62 cherry typedef void (vector)(void);
813 1.62 cherry extern vector Xsyscall, Xsyscall32;
814 1.62 cherry #endif
815 1.62 cherry
816 1.62 cherry /*
817 1.62 cherry * Setup the "trampoline". On Xen, we setup nearly all cpu context
818 1.62 cherry * outside a trampoline, so we prototype and call targetip like so:
819 1.62 cherry * void targetip(struct cpu_info *);
820 1.62 cherry */
821 1.62 cherry
822 1.2 bouyer static void
823 1.62 cherry gdt_prepframes(paddr_t *frames, vaddr_t base, uint32_t entries)
824 1.2 bouyer {
825 1.62 cherry int i;
826 1.62 cherry for (i = 0; i < roundup(entries, PAGE_SIZE) >> PAGE_SHIFT; i++) {
827 1.62 cherry
828 1.62 cherry frames[i] = ((paddr_t) xpmap_ptetomach(
829 1.62 cherry (pt_entry_t *) (base + (i << PAGE_SHIFT))))
830 1.62 cherry >> PAGE_SHIFT;
831 1.62 cherry
832 1.62 cherry /* Mark Read-only */
833 1.62 cherry pmap_pte_clearbits(kvtopte(base + (i << PAGE_SHIFT)),
834 1.62 cherry PG_RW);
835 1.62 cherry }
836 1.62 cherry }
837 1.62 cherry
838 1.62 cherry #ifdef __x86_64__
839 1.62 cherry extern char *ldtstore; /* XXX: Xen MP todo */
840 1.62 cherry
841 1.62 cherry static void
842 1.62 cherry xen_init_amd64_vcpuctxt(struct cpu_info *ci,
843 1.62 cherry struct vcpu_guest_context *initctx,
844 1.62 cherry void targetrip(struct cpu_info *))
845 1.62 cherry {
846 1.62 cherry /* page frames to point at GDT */
847 1.62 cherry extern int gdt_size;
848 1.62 cherry paddr_t frames[16];
849 1.62 cherry psize_t gdt_ents;
850 1.62 cherry
851 1.62 cherry struct lwp *l;
852 1.62 cherry struct pcb *pcb;
853 1.62 cherry
854 1.62 cherry volatile struct vcpu_info *vci;
855 1.62 cherry
856 1.62 cherry KASSERT(ci != NULL);
857 1.62 cherry KASSERT(ci != &cpu_info_primary);
858 1.62 cherry KASSERT(initctx != NULL);
859 1.62 cherry KASSERT(targetrip != NULL);
860 1.62 cherry
861 1.62 cherry memset(initctx, 0, sizeof *initctx);
862 1.62 cherry
863 1.62 cherry gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT; /* XXX: re-investigate roundup(gdt_size... ) for gdt_ents. */
864 1.62 cherry KASSERT(gdt_ents <= 16);
865 1.62 cherry
866 1.62 cherry gdt_prepframes(frames, (vaddr_t) ci->ci_gdt, gdt_ents);
867 1.62 cherry
868 1.62 cherry /* XXX: The stuff in here is amd64 specific. move to mptramp.[Sc] ? */
869 1.62 cherry
870 1.62 cherry /* Initialise the vcpu context: We use idle_loop()'s pcb context. */
871 1.11 cegger
872 1.62 cherry l = ci->ci_data.cpu_idlelwp;
873 1.11 cegger
874 1.62 cherry KASSERT(l != NULL);
875 1.62 cherry pcb = lwp_getpcb(l);
876 1.62 cherry KASSERT(pcb != NULL);
877 1.11 cegger
878 1.62 cherry /* resume with interrupts off */
879 1.62 cherry vci = ci->ci_vcpu;
880 1.62 cherry vci->evtchn_upcall_mask = 1;
881 1.62 cherry xen_mb();
882 1.2 bouyer
883 1.62 cherry /* resume in kernel-mode */
884 1.62 cherry initctx->flags = VGCF_in_kernel | VGCF_online;
885 1.2 bouyer
886 1.62 cherry /* Stack and entry points:
887 1.62 cherry * We arrange for the stack frame for cpu_hatch() to
888 1.62 cherry * appear as a callee frame of lwp_trampoline(). Being a
889 1.62 cherry * leaf frame prevents trampling on any of the MD stack setup
890 1.62 cherry * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
891 1.62 cherry */
892 1.2 bouyer
893 1.62 cherry initctx->user_regs.rdi = (uint64_t) ci; /* targetrip(ci); */
894 1.62 cherry initctx->user_regs.rip = (vaddr_t) targetrip;
895 1.2 bouyer
896 1.62 cherry initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
897 1.11 cegger
898 1.62 cherry initctx->user_regs.rflags = pcb->pcb_flags;
899 1.62 cherry initctx->user_regs.rsp = pcb->pcb_rsp;
900 1.11 cegger
901 1.62 cherry /* Data segments */
902 1.62 cherry initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
903 1.62 cherry initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
904 1.62 cherry initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
905 1.62 cherry
906 1.62 cherry /* GDT */
907 1.62 cherry memcpy(initctx->gdt_frames, frames, sizeof frames);
908 1.62 cherry initctx->gdt_ents = gdt_ents;
909 1.62 cherry
910 1.62 cherry /* LDT */
911 1.62 cherry initctx->ldt_base = (unsigned long) ldtstore;
912 1.62 cherry initctx->ldt_ents = LDT_SIZE >> 3;
913 1.62 cherry
914 1.62 cherry /* Kernel context state */
915 1.62 cherry initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
916 1.62 cherry initctx->kernel_sp = pcb->pcb_rsp0;
917 1.62 cherry initctx->ctrlreg[0] = pcb->pcb_cr0;
918 1.62 cherry initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
919 1.62 cherry initctx->ctrlreg[2] = pcb->pcb_cr2; /* XXX: */
920 1.62 cherry /*
921 1.62 cherry * Use pmap_kernel() L4 PD directly, until we setup the
922 1.62 cherry * per-cpu L4 PD in pmap_cpu_init_late()
923 1.2 bouyer */
924 1.62 cherry initctx->ctrlreg[3] = xpmap_ptom(pcb->pcb_cr3);
925 1.62 cherry initctx->ctrlreg[4] = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT;
926 1.2 bouyer
927 1.62 cherry
928 1.62 cherry /* Xen callbacks */
929 1.62 cherry initctx->event_callback_eip = (unsigned long) hypervisor_callback;
930 1.62 cherry initctx->failsafe_callback_eip = (unsigned long) failsafe_callback;
931 1.62 cherry initctx->syscall_callback_eip = (unsigned long) Xsyscall;
932 1.62 cherry
933 1.62 cherry return;
934 1.2 bouyer }
935 1.62 cherry #else /* i386 */
936 1.62 cherry extern union descriptor *ldt;
937 1.62 cherry extern void Xsyscall(void);
938 1.62 cherry
939 1.11 cegger static void
940 1.62 cherry xen_init_i386_vcpuctxt(struct cpu_info *ci,
941 1.62 cherry struct vcpu_guest_context *initctx,
942 1.62 cherry void targeteip(struct cpu_info *))
943 1.62 cherry {
944 1.62 cherry /* page frames to point at GDT */
945 1.62 cherry extern int gdt_size;
946 1.62 cherry paddr_t frames[16];
947 1.62 cherry psize_t gdt_ents;
948 1.62 cherry
949 1.62 cherry struct lwp *l;
950 1.62 cherry struct pcb *pcb;
951 1.62 cherry
952 1.62 cherry volatile struct vcpu_info *vci;
953 1.62 cherry
954 1.62 cherry KASSERT(ci != NULL);
955 1.62 cherry KASSERT(ci != &cpu_info_primary);
956 1.62 cherry KASSERT(initctx != NULL);
957 1.62 cherry KASSERT(targeteip != NULL);
958 1.62 cherry
959 1.62 cherry memset(initctx, 0, sizeof *initctx);
960 1.11 cegger
961 1.62 cherry gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT; /* XXX: re-investigate roundup(gdt_size... ) for gdt_ents. */
962 1.62 cherry KASSERT(gdt_ents <= 16);
963 1.2 bouyer
964 1.62 cherry gdt_prepframes(frames, (vaddr_t) ci->ci_gdt, gdt_ents);
965 1.2 bouyer
966 1.62 cherry /*
967 1.62 cherry * Initialise the vcpu context:
968 1.62 cherry * We use this cpu's idle_loop() pcb context.
969 1.11 cegger */
970 1.11 cegger
971 1.62 cherry l = ci->ci_data.cpu_idlelwp;
972 1.62 cherry
973 1.62 cherry KASSERT(l != NULL);
974 1.62 cherry pcb = lwp_getpcb(l);
975 1.62 cherry KASSERT(pcb != NULL);
976 1.62 cherry
977 1.62 cherry /* resume with interrupts off */
978 1.62 cherry vci = ci->ci_vcpu;
979 1.62 cherry vci->evtchn_upcall_mask = 1;
980 1.62 cherry xen_mb();
981 1.62 cherry
982 1.62 cherry /* resume in kernel-mode */
983 1.62 cherry initctx->flags = VGCF_in_kernel | VGCF_online;
984 1.62 cherry
985 1.62 cherry /* Stack frame setup for cpu_hatch():
986 1.62 cherry * We arrange for the stack frame for cpu_hatch() to
987 1.62 cherry * appear as a callee frame of lwp_trampoline(). Being a
988 1.62 cherry * leaf frame prevents trampling on any of the MD stack setup
989 1.62 cherry * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
990 1.2 bouyer */
991 1.2 bouyer
992 1.62 cherry initctx->user_regs.esp = pcb->pcb_esp - 4; /* Leave word for
993 1.62 cherry arg1 */
994 1.62 cherry { /* targeteip(ci); */
995 1.62 cherry uint32_t *arg = (uint32_t *) initctx->user_regs.esp;
996 1.62 cherry arg[1] = (uint32_t) ci; /* arg1 */
997 1.62 cherry
998 1.62 cherry }
999 1.2 bouyer
1000 1.62 cherry initctx->user_regs.eip = (vaddr_t) targeteip;
1001 1.62 cherry initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
1002 1.62 cherry initctx->user_regs.eflags |= pcb->pcb_iopl;
1003 1.62 cherry
1004 1.62 cherry /* Data segments */
1005 1.62 cherry initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
1006 1.62 cherry initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
1007 1.62 cherry initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
1008 1.62 cherry initctx->user_regs.fs = GSEL(GDATA_SEL, SEL_KPL);
1009 1.62 cherry
1010 1.62 cherry /* GDT */
1011 1.62 cherry memcpy(initctx->gdt_frames, frames, sizeof frames);
1012 1.62 cherry initctx->gdt_ents = gdt_ents;
1013 1.62 cherry
1014 1.62 cherry /* LDT */
1015 1.62 cherry initctx->ldt_base = (unsigned long) ldt;
1016 1.62 cherry initctx->ldt_ents = NLDT;
1017 1.62 cherry
1018 1.62 cherry /* Kernel context state */
1019 1.62 cherry initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
1020 1.62 cherry initctx->kernel_sp = pcb->pcb_esp0;
1021 1.62 cherry initctx->ctrlreg[0] = pcb->pcb_cr0;
1022 1.62 cherry initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
1023 1.62 cherry initctx->ctrlreg[2] = pcb->pcb_cr2; /* XXX: */
1024 1.62 cherry /*
1025 1.62 cherry * Use pmap_kernel() L4 PD directly, until we setup the
1026 1.62 cherry * per-cpu L4 PD in pmap_cpu_init_late()
1027 1.2 bouyer */
1028 1.62 cherry initctx->ctrlreg[3] = xpmap_ptom(pcb->pcb_cr3);
1029 1.62 cherry initctx->ctrlreg[4] = /* CR4_PAE | */CR4_OSFXSR | CR4_OSXMMEXCPT;
1030 1.2 bouyer
1031 1.2 bouyer
1032 1.62 cherry /* Xen callbacks */
1033 1.62 cherry initctx->event_callback_eip = (unsigned long) hypervisor_callback;
1034 1.62 cherry initctx->event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
1035 1.62 cherry initctx->failsafe_callback_eip = (unsigned long) failsafe_callback;
1036 1.62 cherry initctx->failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
1037 1.45 rmind
1038 1.62 cherry return;
1039 1.62 cherry }
1040 1.62 cherry #endif /* __x86_64__ */
1041 1.45 rmind
1042 1.62 cherry int
1043 1.62 cherry mp_cpu_start(struct cpu_info *ci, vaddr_t target)
1044 1.62 cherry {
1045 1.62 cherry
1046 1.62 cherry int hyperror;
1047 1.62 cherry struct vcpu_guest_context vcpuctx;
1048 1.2 bouyer
1049 1.62 cherry KASSERT(ci != NULL);
1050 1.62 cherry KASSERT(ci != &cpu_info_primary);
1051 1.62 cherry KASSERT(ci->ci_flags & CPUF_AP);
1052 1.62 cherry
1053 1.62 cherry #ifdef __x86_64__
1054 1.62 cherry xen_init_amd64_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
1055 1.62 cherry #else /* i386 */
1056 1.62 cherry xen_init_i386_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
1057 1.62 cherry #endif /* __x86_64__ */
1058 1.62 cherry
1059 1.62 cherry /* Initialise the given vcpu to execute cpu_hatch(ci); */
1060 1.62 cherry if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_cpuid, &vcpuctx))) {
1061 1.62 cherry aprint_error(": context initialisation failed. errno = %d\n", hyperror);
1062 1.62 cherry return hyperror;
1063 1.62 cherry }
1064 1.62 cherry
1065 1.62 cherry /* Start it up */
1066 1.62 cherry
1067 1.62 cherry /* First bring it down - the Xen documentation conveniently omits this slight detail. */
1068 1.62 cherry if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_cpuid, NULL))) {
1069 1.62 cherry aprint_error(": VCPUOP_down hypervisor command failed. errno = %d\n", hyperror);
1070 1.62 cherry return hyperror;
1071 1.62 cherry }
1072 1.62 cherry
1073 1.62 cherry if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_up, ci->ci_cpuid, NULL))) {
1074 1.62 cherry aprint_error(": VCPUOP_up hypervisor command failed. errno = %d\n", hyperror);
1075 1.62 cherry return hyperror;
1076 1.62 cherry }
1077 1.2 bouyer
1078 1.62 cherry if (!vcpu_is_up(ci)) {
1079 1.62 cherry aprint_error(": did not come up\n");
1080 1.62 cherry return -1;
1081 1.2 bouyer }
1082 1.62 cherry
1083 1.2 bouyer return 0;
1084 1.2 bouyer }
1085 1.2 bouyer
1086 1.2 bouyer void
1087 1.2 bouyer mp_cpu_start_cleanup(struct cpu_info *ci)
1088 1.2 bouyer {
1089 1.2 bouyer #if 0
1090 1.2 bouyer /*
1091 1.2 bouyer * Ensure the NVRAM reset byte contains something vaguely sane.
1092 1.2 bouyer */
1093 1.2 bouyer
1094 1.2 bouyer outb(IO_RTC, NVRAM_RESET);
1095 1.2 bouyer outb(IO_RTC+1, NVRAM_RESET_RST);
1096 1.2 bouyer #endif
1097 1.62 cherry if (vcpu_is_up(ci)) {
1098 1.62 cherry aprint_debug_dev(ci->ci_dev, "is started.\n");
1099 1.62 cherry }
1100 1.62 cherry else {
1101 1.62 cherry aprint_error_dev(ci->ci_dev, "did not start up.\n");
1102 1.62 cherry }
1103 1.62 cherry
1104 1.2 bouyer }
1105 1.2 bouyer
1106 1.2 bouyer void
1107 1.3 bouyer cpu_init_msrs(struct cpu_info *ci, bool full)
1108 1.2 bouyer {
1109 1.43 jym #ifdef __x86_64__
1110 1.3 bouyer if (full) {
1111 1.3 bouyer HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
1112 1.11 cegger HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
1113 1.3 bouyer HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
1114 1.3 bouyer }
1115 1.43 jym #endif /* __x86_64__ */
1116 1.44 jym
1117 1.44 jym if (cpu_feature[2] & CPUID_NOX)
1118 1.44 jym wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
1119 1.62 cherry
1120 1.2 bouyer }
1121 1.2 bouyer
1122 1.11 cegger void
1123 1.11 cegger cpu_offline_md(void)
1124 1.11 cegger {
1125 1.11 cegger int s;
1126 1.11 cegger
1127 1.11 cegger s = splhigh();
1128 1.11 cegger #ifdef __i386__
1129 1.11 cegger npxsave_cpu(true);
1130 1.11 cegger #else
1131 1.11 cegger fpusave_cpu(true);
1132 1.11 cegger #endif
1133 1.11 cegger splx(s);
1134 1.11 cegger }
1135 1.11 cegger
1136 1.11 cegger #if 0
1137 1.11 cegger /* XXX joerg restructure and restart CPUs individually */
1138 1.11 cegger static bool
1139 1.41 dyoung cpu_suspend(device_t dv, const pmf_qual_t *qual)
1140 1.11 cegger {
1141 1.11 cegger struct cpu_softc *sc = device_private(dv);
1142 1.11 cegger struct cpu_info *ci = sc->sc_info;
1143 1.11 cegger int err;
1144 1.11 cegger
1145 1.11 cegger if (ci->ci_flags & CPUF_PRIMARY)
1146 1.11 cegger return true;
1147 1.11 cegger if (ci->ci_data.cpu_idlelwp == NULL)
1148 1.11 cegger return true;
1149 1.11 cegger if ((ci->ci_flags & CPUF_PRESENT) == 0)
1150 1.11 cegger return true;
1151 1.11 cegger
1152 1.11 cegger sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
1153 1.11 cegger
1154 1.11 cegger if (sc->sc_wasonline) {
1155 1.11 cegger mutex_enter(&cpu_lock);
1156 1.29 rmind err = cpu_setstate(ci, false);
1157 1.11 cegger mutex_exit(&cpu_lock);
1158 1.11 cegger
1159 1.11 cegger if (err)
1160 1.11 cegger return false;
1161 1.11 cegger }
1162 1.11 cegger
1163 1.11 cegger return true;
1164 1.11 cegger }
1165 1.11 cegger
1166 1.11 cegger static bool
1167 1.41 dyoung cpu_resume(device_t dv, const pmf_qual_t *qual)
1168 1.11 cegger {
1169 1.11 cegger struct cpu_softc *sc = device_private(dv);
1170 1.11 cegger struct cpu_info *ci = sc->sc_info;
1171 1.11 cegger int err = 0;
1172 1.11 cegger
1173 1.11 cegger if (ci->ci_flags & CPUF_PRIMARY)
1174 1.11 cegger return true;
1175 1.11 cegger if (ci->ci_data.cpu_idlelwp == NULL)
1176 1.11 cegger return true;
1177 1.11 cegger if ((ci->ci_flags & CPUF_PRESENT) == 0)
1178 1.11 cegger return true;
1179 1.11 cegger
1180 1.11 cegger if (sc->sc_wasonline) {
1181 1.11 cegger mutex_enter(&cpu_lock);
1182 1.29 rmind err = cpu_setstate(ci, true);
1183 1.11 cegger mutex_exit(&cpu_lock);
1184 1.11 cegger }
1185 1.11 cegger
1186 1.11 cegger return err == 0;
1187 1.11 cegger }
1188 1.11 cegger #endif
1189 1.11 cegger
1190 1.2 bouyer void
1191 1.2 bouyer cpu_get_tsc_freq(struct cpu_info *ci)
1192 1.2 bouyer {
1193 1.62 cherry uint32_t vcpu_tversion;
1194 1.16 cegger const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
1195 1.62 cherry
1196 1.62 cherry vcpu_tversion = tinfo->version;
1197 1.62 cherry while (tinfo->version == vcpu_tversion); /* Wait for a time update. XXX: timeout ? */
1198 1.62 cherry
1199 1.2 bouyer uint64_t freq = 1000000000ULL << 32;
1200 1.2 bouyer freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
1201 1.2 bouyer if ( tinfo->tsc_shift < 0 )
1202 1.2 bouyer freq = freq << -tinfo->tsc_shift;
1203 1.2 bouyer else
1204 1.2 bouyer freq = freq >> tinfo->tsc_shift;
1205 1.20 ad ci->ci_data.cpu_cc_freq = freq;
1206 1.2 bouyer }
1207 1.19 joerg
1208 1.19 joerg void
1209 1.19 joerg x86_cpu_idle_xen(void)
1210 1.19 joerg {
1211 1.19 joerg struct cpu_info *ci = curcpu();
1212 1.62 cherry
1213 1.19 joerg KASSERT(ci->ci_ilevel == IPL_NONE);
1214 1.19 joerg
1215 1.19 joerg x86_disable_intr();
1216 1.19 joerg if (!__predict_false(ci->ci_want_resched)) {
1217 1.19 joerg idle_block();
1218 1.19 joerg } else {
1219 1.19 joerg x86_enable_intr();
1220 1.19 joerg }
1221 1.19 joerg }
1222 1.47 jym
1223 1.47 jym /*
1224 1.47 jym * Loads pmap for the current CPU.
1225 1.47 jym */
1226 1.47 jym void
1227 1.47 jym cpu_load_pmap(struct pmap *pmap)
1228 1.47 jym {
1229 1.47 jym #ifdef i386
1230 1.47 jym #ifdef PAE
1231 1.47 jym int i, s;
1232 1.47 jym struct cpu_info *ci;
1233 1.47 jym
1234 1.47 jym s = splvm(); /* just to be safe */
1235 1.62 cherry xpq_queue_lock();
1236 1.47 jym ci = curcpu();
1237 1.47 jym paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);
1238 1.47 jym /* don't update the kernel L3 slot */
1239 1.47 jym for (i = 0 ; i < PDP_SIZE - 1; i++) {
1240 1.47 jym xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t),
1241 1.47 jym xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V);
1242 1.47 jym }
1243 1.62 cherry xpq_queue_unlock();
1244 1.47 jym splx(s);
1245 1.47 jym tlbflush();
1246 1.47 jym #else /* PAE */
1247 1.47 jym lcr3(pmap_pdirpa(pmap, 0));
1248 1.47 jym #endif /* PAE */
1249 1.47 jym #endif /* i386 */
1250 1.47 jym
1251 1.47 jym #ifdef __x86_64__
1252 1.47 jym int i, s;
1253 1.47 jym pd_entry_t *old_pgd, *new_pgd;
1254 1.47 jym paddr_t addr;
1255 1.47 jym struct cpu_info *ci;
1256 1.47 jym
1257 1.47 jym /* kernel pmap always in cr3 and should never go in user cr3 */
1258 1.47 jym if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) {
1259 1.47 jym ci = curcpu();
1260 1.47 jym /*
1261 1.47 jym * Map user space address in kernel space and load
1262 1.47 jym * user cr3
1263 1.47 jym */
1264 1.47 jym s = splvm();
1265 1.47 jym new_pgd = pmap->pm_pdir;
1266 1.47 jym old_pgd = pmap_kernel()->pm_pdir;
1267 1.47 jym addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0));
1268 1.47 jym for (i = 0; i < PDIR_SLOT_PTE;
1269 1.47 jym i++, addr += sizeof(pd_entry_t)) {
1270 1.47 jym if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V))
1271 1.47 jym xpq_queue_pte_update(addr, new_pgd[i]);
1272 1.47 jym }
1273 1.47 jym xen_set_user_pgd(pmap_pdirpa(pmap, 0));
1274 1.47 jym ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);
1275 1.59 rmind tlbflush();
1276 1.47 jym splx(s);
1277 1.47 jym }
1278 1.47 jym #endif /* __x86_64__ */
1279 1.47 jym }
1280 1.61 cherry
1281 1.61 cherry /*
1282 1.61 cherry * Notify all other cpus to halt.
1283 1.61 cherry */
1284 1.61 cherry
1285 1.61 cherry void
1286 1.61 cherry cpu_broadcast_halt(void)
1287 1.61 cherry {
1288 1.61 cherry xen_broadcast_ipi(XEN_IPI_HALT);
1289 1.61 cherry }
1290 1.61 cherry
1291 1.61 cherry /*
1292 1.61 cherry * Send a dummy ipi to a cpu.
1293 1.61 cherry */
1294 1.61 cherry
1295 1.61 cherry void
1296 1.61 cherry cpu_kick(struct cpu_info *ci)
1297 1.61 cherry {
1298 1.64 dholland (void)xen_send_ipi(ci, XEN_IPI_KICK);
1299 1.61 cherry }
1300