Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.78
      1  1.78    cherry /*	$NetBSD: cpu.c,v 1.78 2012/01/28 07:19:17 cherry Exp $	*/
      2   1.2    bouyer /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp  */
      3   1.2    bouyer 
      4   1.2    bouyer /*-
      5   1.2    bouyer  * Copyright (c) 2000 The NetBSD Foundation, Inc.
      6  1.19     joerg  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
      7   1.2    bouyer  * All rights reserved.
      8   1.2    bouyer  *
      9   1.2    bouyer  * This code is derived from software contributed to The NetBSD Foundation
     10   1.2    bouyer  * by RedBack Networks Inc.
     11   1.2    bouyer  *
     12   1.2    bouyer  * Author: Bill Sommerfeld
     13   1.2    bouyer  *
     14   1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     15   1.2    bouyer  * modification, are permitted provided that the following conditions
     16   1.2    bouyer  * are met:
     17   1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     18   1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     19   1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     20   1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     21   1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     22   1.2    bouyer  *
     23   1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     24   1.2    bouyer  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     25   1.2    bouyer  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     26   1.2    bouyer  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     27   1.2    bouyer  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     28   1.2    bouyer  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     29   1.2    bouyer  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     30   1.2    bouyer  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     31   1.2    bouyer  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     32   1.2    bouyer  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     33   1.2    bouyer  * POSSIBILITY OF SUCH DAMAGE.
     34   1.2    bouyer  */
     35   1.2    bouyer 
     36   1.2    bouyer /*
     37   1.2    bouyer  * Copyright (c) 1999 Stefan Grefen
     38   1.2    bouyer  *
     39   1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     40   1.2    bouyer  * modification, are permitted provided that the following conditions
     41   1.2    bouyer  * are met:
     42   1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     43   1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     44   1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     45   1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     46   1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     47   1.2    bouyer  * 3. All advertising materials mentioning features or use of this software
     48   1.2    bouyer  *    must display the following acknowledgement:
     49   1.2    bouyer  *      This product includes software developed by the NetBSD
     50   1.2    bouyer  *      Foundation, Inc. and its contributors.
     51   1.2    bouyer  * 4. Neither the name of The NetBSD Foundation nor the names of its
     52   1.2    bouyer  *    contributors may be used to endorse or promote products derived
     53   1.2    bouyer  *    from this software without specific prior written permission.
     54   1.2    bouyer  *
     55   1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     56   1.2    bouyer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     57   1.2    bouyer  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     58   1.2    bouyer  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     59   1.2    bouyer  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     60   1.2    bouyer  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     61   1.2    bouyer  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     62   1.2    bouyer  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     63   1.2    bouyer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     64   1.2    bouyer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     65   1.2    bouyer  * SUCH DAMAGE.
     66   1.2    bouyer  */
     67   1.2    bouyer 
     68   1.2    bouyer #include <sys/cdefs.h>
     69  1.78    cherry __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.78 2012/01/28 07:19:17 cherry Exp $");
     70   1.2    bouyer 
     71   1.2    bouyer #include "opt_ddb.h"
     72   1.2    bouyer #include "opt_multiprocessor.h"
     73   1.2    bouyer #include "opt_mpbios.h"		/* for MPDEBUG */
     74   1.2    bouyer #include "opt_mtrr.h"
     75   1.2    bouyer #include "opt_xen.h"
     76   1.2    bouyer 
     77   1.2    bouyer #include "lapic.h"
     78   1.2    bouyer #include "ioapic.h"
     79   1.2    bouyer 
     80   1.2    bouyer #include <sys/param.h>
     81   1.2    bouyer #include <sys/proc.h>
     82   1.2    bouyer #include <sys/systm.h>
     83   1.2    bouyer #include <sys/device.h>
     84  1.31    cegger #include <sys/kmem.h>
     85  1.11    cegger #include <sys/cpu.h>
     86  1.66    jruoho #include <sys/cpufreq.h>
     87  1.11    cegger #include <sys/atomic.h>
     88  1.32    cegger #include <sys/reboot.h>
     89  1.62    cherry #include <sys/idle.h>
     90   1.2    bouyer 
     91  1.51  uebayasi #include <uvm/uvm.h>
     92   1.2    bouyer 
     93   1.2    bouyer #include <machine/cpufunc.h>
     94   1.2    bouyer #include <machine/cpuvar.h>
     95   1.2    bouyer #include <machine/pmap.h>
     96   1.2    bouyer #include <machine/vmparam.h>
     97   1.2    bouyer #include <machine/mpbiosvar.h>
     98   1.2    bouyer #include <machine/pcb.h>
     99   1.2    bouyer #include <machine/specialreg.h>
    100   1.2    bouyer #include <machine/segments.h>
    101   1.2    bouyer #include <machine/gdt.h>
    102   1.2    bouyer #include <machine/mtrr.h>
    103   1.2    bouyer #include <machine/pio.h>
    104   1.2    bouyer 
    105  1.62    cherry #ifdef i386
    106  1.62    cherry #include <machine/npx.h>
    107  1.62    cherry #else
    108  1.62    cherry #include <machine/fpu.h>
    109  1.62    cherry #endif
    110  1.62    cherry 
    111  1.62    cherry #include <xen/xen.h>
    112  1.71    cegger #include <xen/xen-public/vcpu.h>
    113   1.2    bouyer #include <xen/vcpuvar.h>
    114   1.2    bouyer 
    115   1.2    bouyer #if NLAPIC > 0
    116   1.2    bouyer #include <machine/apicvar.h>
    117   1.2    bouyer #include <machine/i82489reg.h>
    118   1.2    bouyer #include <machine/i82489var.h>
    119   1.2    bouyer #endif
    120   1.2    bouyer 
    121   1.2    bouyer #include <dev/ic/mc146818reg.h>
    122   1.2    bouyer #include <dev/isa/isareg.h>
    123   1.2    bouyer 
    124  1.38    cegger #if MAXCPUS > 32
    125  1.38    cegger #error cpu_info contains 32bit bitmasks
    126  1.38    cegger #endif
    127  1.27        ad 
    128  1.56    jruoho static int	cpu_match(device_t, cfdata_t, void *);
    129  1.56    jruoho static void	cpu_attach(device_t, device_t, void *);
    130  1.56    jruoho static void	cpu_defer(device_t);
    131  1.56    jruoho static int	cpu_rescan(device_t, const char *, const int *);
    132  1.56    jruoho static void	cpu_childdetached(device_t, device_t);
    133  1.56    jruoho static int	vcpu_match(device_t, cfdata_t, void *);
    134  1.56    jruoho static void	vcpu_attach(device_t, device_t, void *);
    135  1.56    jruoho static void	cpu_attach_common(device_t, device_t, void *);
    136  1.56    jruoho void		cpu_offline_md(void);
    137   1.2    bouyer 
    138   1.2    bouyer struct cpu_softc {
    139  1.10    cegger 	device_t sc_dev;		/* device tree glue */
    140   1.2    bouyer 	struct cpu_info *sc_info;	/* pointer to CPU info */
    141  1.32    cegger 	bool sc_wasonline;
    142   1.2    bouyer };
    143   1.2    bouyer 
    144  1.62    cherry int mp_cpu_start(struct cpu_info *, vaddr_t);
    145   1.2    bouyer void mp_cpu_start_cleanup(struct cpu_info *);
    146   1.2    bouyer const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    147   1.2    bouyer 				      mp_cpu_start_cleanup };
    148   1.2    bouyer 
    149  1.53    jruoho CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
    150  1.53    jruoho     cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
    151  1.53    jruoho 
    152  1.10    cegger CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
    153   1.2    bouyer     vcpu_match, vcpu_attach, NULL, NULL);
    154   1.2    bouyer 
    155   1.2    bouyer /*
    156   1.2    bouyer  * Statically-allocated CPU info for the primary CPU (or the only
    157   1.2    bouyer  * CPU, on uniprocessors).  The CPU info list is initialized to
    158   1.2    bouyer  * point at it.
    159   1.2    bouyer  */
    160   1.2    bouyer #ifdef TRAPLOG
    161   1.2    bouyer #include <machine/tlog.h>
    162   1.2    bouyer struct tlog tlog_primary;
    163   1.2    bouyer #endif
    164  1.38    cegger struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    165   1.7    bouyer 	.ci_dev = 0,
    166   1.2    bouyer 	.ci_self = &cpu_info_primary,
    167   1.4    bouyer 	.ci_idepth = -1,
    168   1.2    bouyer 	.ci_curlwp = &lwp0,
    169  1.25        ad 	.ci_curldt = -1,
    170  1.58     rmind 	.ci_cpumask = 1,
    171   1.2    bouyer #ifdef TRAPLOG
    172   1.2    bouyer 	.ci_tlog = &tlog_primary,
    173   1.2    bouyer #endif
    174   1.2    bouyer 
    175   1.2    bouyer };
    176  1.38    cegger struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    177   1.7    bouyer 	.ci_dev = 0,
    178   1.2    bouyer 	.ci_self = &phycpu_info_primary,
    179   1.2    bouyer };
    180   1.2    bouyer 
    181   1.2    bouyer struct cpu_info *cpu_info_list = &cpu_info_primary;
    182  1.38    cegger struct cpu_info *phycpu_info_list = &phycpu_info_primary;
    183   1.2    bouyer 
    184  1.58     rmind uint32_t cpus_attached = 1;
    185  1.60     rmind uint32_t cpus_running = 1;
    186  1.11    cegger 
    187  1.38    cegger uint32_t phycpus_attached = 0;
    188  1.38    cegger uint32_t phycpus_running = 0;
    189  1.38    cegger 
    190  1.43       jym uint32_t cpu_feature[5]; /* X86 CPUID feature bits
    191  1.43       jym 			  *	[0] basic features %edx
    192  1.43       jym 			  *	[1] basic features %ecx
    193  1.43       jym 			  *	[2] extended features %edx
    194  1.43       jym 			  *	[3] extended features %ecx
    195  1.43       jym 			  *	[4] VIA padlock features
    196  1.43       jym 			  */
    197  1.43       jym 
    198  1.11    cegger bool x86_mp_online;
    199  1.11    cegger paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    200   1.2    bouyer 
    201  1.38    cegger #if defined(MULTIPROCESSOR)
    202   1.2    bouyer void    	cpu_hatch(void *);
    203   1.2    bouyer static void    	cpu_boot_secondary(struct cpu_info *ci);
    204   1.2    bouyer static void    	cpu_start_secondary(struct cpu_info *ci);
    205  1.38    cegger #endif	/* MULTIPROCESSOR */
    206   1.2    bouyer 
    207  1.56    jruoho static int
    208  1.10    cegger cpu_match(device_t parent, cfdata_t match, void *aux)
    209   1.2    bouyer {
    210   1.2    bouyer 
    211   1.2    bouyer 	return 1;
    212   1.2    bouyer }
    213   1.2    bouyer 
    214  1.56    jruoho static void
    215  1.10    cegger cpu_attach(device_t parent, device_t self, void *aux)
    216   1.2    bouyer {
    217  1.10    cegger 	struct cpu_softc *sc = device_private(self);
    218   1.2    bouyer 	struct cpu_attach_args *caa = aux;
    219   1.2    bouyer 	struct cpu_info *ci;
    220  1.34    cegger 	uintptr_t ptr;
    221  1.52    bouyer 	static int nphycpu = 0;
    222   1.2    bouyer 
    223  1.10    cegger 	sc->sc_dev = self;
    224  1.10    cegger 
    225  1.38    cegger 	if (phycpus_attached == ~0) {
    226  1.34    cegger 		aprint_error(": increase MAXCPUS\n");
    227  1.34    cegger 		return;
    228  1.34    cegger 	}
    229  1.34    cegger 
    230   1.2    bouyer 	/*
    231   1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    232  1.52    bouyer 	 * If we're the first attached CPU use the primary cpu_info,
    233  1.52    bouyer 	 * otherwise allocate a new one
    234   1.2    bouyer 	 */
    235  1.52    bouyer 	aprint_naive("\n");
    236  1.52    bouyer 	aprint_normal("\n");
    237  1.52    bouyer 	if (nphycpu > 0) {
    238  1.52    bouyer 		struct cpu_info *tmp;
    239  1.34    cegger 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    240  1.34    cegger 		    KM_SLEEP);
    241  1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    242  1.24        ad 		ci->ci_curldt = -1;
    243  1.52    bouyer 
    244  1.52    bouyer 		tmp = phycpu_info_list;
    245  1.52    bouyer 		while (tmp->ci_next)
    246  1.52    bouyer 			tmp = tmp->ci_next;
    247  1.52    bouyer 
    248  1.52    bouyer 		tmp->ci_next = ci;
    249   1.2    bouyer 	} else {
    250   1.2    bouyer 		ci = &phycpu_info_primary;
    251   1.2    bouyer 	}
    252   1.2    bouyer 
    253   1.2    bouyer 	ci->ci_self = ci;
    254   1.2    bouyer 	sc->sc_info = ci;
    255   1.2    bouyer 
    256   1.2    bouyer 	ci->ci_dev = self;
    257  1.50    jruoho 	ci->ci_acpiid = caa->cpu_id;
    258  1.23        ad 	ci->ci_cpuid = caa->cpu_number;
    259  1.16    cegger 	ci->ci_vcpu = NULL;
    260  1.52    bouyer 	ci->ci_index = nphycpu++;
    261  1.52    bouyer 	ci->ci_cpumask = (1 << cpu_index(ci));
    262   1.2    bouyer 
    263  1.52    bouyer 	atomic_or_32(&phycpus_attached, ci->ci_cpumask);
    264  1.38    cegger 
    265  1.52    bouyer 	if (!pmf_device_register(self, NULL, NULL))
    266  1.52    bouyer 		aprint_error_dev(self, "couldn't establish power handler\n");
    267  1.34    cegger 
    268  1.56    jruoho 	(void)config_defer(self, cpu_defer);
    269  1.56    jruoho }
    270  1.56    jruoho 
    271  1.56    jruoho static void
    272  1.56    jruoho cpu_defer(device_t self)
    273  1.56    jruoho {
    274  1.56    jruoho 	cpu_rescan(self, NULL, NULL);
    275   1.2    bouyer }
    276   1.2    bouyer 
    277  1.56    jruoho static int
    278  1.53    jruoho cpu_rescan(device_t self, const char *ifattr, const int *locators)
    279  1.53    jruoho {
    280  1.53    jruoho 	struct cpu_softc *sc = device_private(self);
    281  1.53    jruoho 	struct cpufeature_attach_args cfaa;
    282  1.53    jruoho 	struct cpu_info *ci = sc->sc_info;
    283  1.53    jruoho 
    284  1.53    jruoho 	memset(&cfaa, 0, sizeof(cfaa));
    285  1.53    jruoho 	cfaa.ci = ci;
    286  1.53    jruoho 
    287  1.53    jruoho 	if (ifattr_match(ifattr, "cpufeaturebus")) {
    288  1.53    jruoho 
    289  1.53    jruoho 		if (ci->ci_frequency == NULL) {
    290  1.55    jruoho 			cfaa.name = "frequency";
    291  1.54    jruoho 			ci->ci_frequency = config_found_ia(self,
    292  1.54    jruoho 			    "cpufeaturebus", &cfaa, NULL);
    293  1.54    jruoho 		}
    294  1.53    jruoho 	}
    295  1.53    jruoho 
    296  1.53    jruoho 	return 0;
    297  1.53    jruoho }
    298  1.53    jruoho 
    299  1.56    jruoho static void
    300  1.53    jruoho cpu_childdetached(device_t self, device_t child)
    301  1.53    jruoho {
    302  1.53    jruoho 	struct cpu_softc *sc = device_private(self);
    303  1.53    jruoho 	struct cpu_info *ci = sc->sc_info;
    304  1.53    jruoho 
    305  1.53    jruoho 	if (ci->ci_frequency == child)
    306  1.53    jruoho 		ci->ci_frequency = NULL;
    307  1.53    jruoho }
    308  1.53    jruoho 
    309  1.56    jruoho static int
    310  1.10    cegger vcpu_match(device_t parent, cfdata_t match, void *aux)
    311   1.2    bouyer {
    312   1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    313  1.62    cherry 	struct vcpu_runstate_info vcr;
    314  1.62    cherry 	int error;
    315  1.62    cherry 
    316  1.62    cherry 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) {
    317  1.62    cherry 		error = HYPERVISOR_vcpu_op(VCPUOP_get_runstate_info,
    318  1.62    cherry 					   vcaa->vcaa_caa.cpu_number,
    319  1.62    cherry 					   &vcr);
    320  1.62    cherry 		switch (error) {
    321  1.62    cherry 		case 0:
    322  1.62    cherry 			return 1;
    323  1.62    cherry 		case -ENOENT:
    324  1.62    cherry 			return 0;
    325  1.62    cherry 		default:
    326  1.62    cherry 			panic("Unknown hypervisor error %d returned on vcpu runstate probe\n", error);
    327  1.62    cherry 		}
    328  1.62    cherry 	}
    329   1.2    bouyer 
    330   1.2    bouyer 	return 0;
    331   1.2    bouyer }
    332   1.2    bouyer 
    333  1.56    jruoho static void
    334  1.10    cegger vcpu_attach(device_t parent, device_t self, void *aux)
    335   1.2    bouyer {
    336   1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    337   1.2    bouyer 
    338  1.62    cherry 	KASSERT(vcaa->vcaa_caa.cpu_func == NULL);
    339  1.62    cherry 	vcaa->vcaa_caa.cpu_func = &mp_cpu_funcs;
    340   1.2    bouyer 	cpu_attach_common(parent, self, &vcaa->vcaa_caa);
    341  1.65       jym 
    342  1.65       jym 	if (!pmf_device_register(self, NULL, NULL))
    343  1.65       jym 		aprint_error_dev(self, "couldn't establish power handler\n");
    344   1.2    bouyer }
    345   1.2    bouyer 
    346  1.62    cherry static int
    347  1.62    cherry vcpu_is_up(struct cpu_info *ci)
    348  1.62    cherry {
    349  1.62    cherry 	KASSERT(ci != NULL);
    350  1.62    cherry 	return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_cpuid, NULL);
    351  1.62    cherry }
    352  1.62    cherry 
    353   1.2    bouyer static void
    354   1.2    bouyer cpu_vm_init(struct cpu_info *ci)
    355   1.2    bouyer {
    356   1.2    bouyer 	int ncolors = 2, i;
    357   1.2    bouyer 
    358   1.2    bouyer 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    359   1.2    bouyer 		struct x86_cache_info *cai;
    360   1.2    bouyer 		int tcolors;
    361   1.2    bouyer 
    362   1.2    bouyer 		cai = &ci->ci_cinfo[i];
    363   1.2    bouyer 
    364   1.2    bouyer 		tcolors = atop(cai->cai_totalsize);
    365   1.2    bouyer 		switch(cai->cai_associativity) {
    366   1.2    bouyer 		case 0xff:
    367   1.2    bouyer 			tcolors = 1; /* fully associative */
    368   1.2    bouyer 			break;
    369   1.2    bouyer 		case 0:
    370   1.2    bouyer 		case 1:
    371   1.2    bouyer 			break;
    372   1.2    bouyer 		default:
    373   1.2    bouyer 			tcolors /= cai->cai_associativity;
    374   1.2    bouyer 		}
    375   1.2    bouyer 		ncolors = max(ncolors, tcolors);
    376   1.2    bouyer 	}
    377   1.2    bouyer 
    378   1.2    bouyer 	/*
    379  1.67       mrg 	 * Knowing the size of the largest cache on this CPU, potentially
    380  1.67       mrg 	 * re-color our pages.
    381   1.2    bouyer 	 */
    382  1.28    bouyer 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    383   1.2    bouyer 	uvm_page_recolor(ncolors);
    384   1.2    bouyer }
    385   1.2    bouyer 
    386  1.56    jruoho static void
    387  1.11    cegger cpu_attach_common(device_t parent, device_t self, void *aux)
    388   1.2    bouyer {
    389  1.10    cegger 	struct cpu_softc *sc = device_private(self);
    390   1.2    bouyer 	struct cpu_attach_args *caa = aux;
    391   1.2    bouyer 	struct cpu_info *ci;
    392  1.12    cegger 	uintptr_t ptr;
    393   1.2    bouyer 	int cpunum = caa->cpu_number;
    394  1.38    cegger 	static bool again = false;
    395   1.2    bouyer 
    396  1.10    cegger 	sc->sc_dev = self;
    397  1.10    cegger 
    398   1.2    bouyer 	/*
    399   1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    400   1.2    bouyer 	 * structure, otherwise use the primary's.
    401   1.2    bouyer 	 */
    402   1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    403  1.12    cegger 		aprint_naive(": Application Processor\n");
    404  1.31    cegger 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    405  1.31    cegger 		    KM_SLEEP);
    406  1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    407  1.12    cegger 		memset(ci, 0, sizeof(*ci));
    408   1.2    bouyer #ifdef TRAPLOG
    409  1.31    cegger 		ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
    410   1.2    bouyer #endif
    411   1.2    bouyer 	} else {
    412  1.12    cegger 		aprint_naive(": %s Processor\n",
    413  1.12    cegger 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    414   1.2    bouyer 		ci = &cpu_info_primary;
    415   1.2    bouyer 	}
    416   1.2    bouyer 
    417   1.2    bouyer 	ci->ci_self = ci;
    418   1.2    bouyer 	sc->sc_info = ci;
    419   1.2    bouyer 	ci->ci_dev = self;
    420  1.23        ad 	ci->ci_cpuid = cpunum;
    421  1.16    cegger 
    422  1.16    cegger 	KASSERT(HYPERVISOR_shared_info != NULL);
    423  1.16    cegger 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
    424  1.16    cegger 
    425  1.62    cherry 	KASSERT(ci->ci_func == 0);
    426   1.2    bouyer 	ci->ci_func = caa->cpu_func;
    427   1.2    bouyer 
    428  1.38    cegger 	/* Must be called before mi_cpu_attach(). */
    429  1.38    cegger 	cpu_vm_init(ci);
    430  1.38    cegger 
    431   1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    432   1.2    bouyer 		int error;
    433   1.2    bouyer 
    434   1.2    bouyer 		error = mi_cpu_attach(ci);
    435  1.62    cherry 
    436  1.62    cherry 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    437   1.2    bouyer 		if (error != 0) {
    438   1.2    bouyer 			aprint_normal("\n");
    439  1.38    cegger 			aprint_error_dev(self,
    440  1.38    cegger 			    "mi_cpu_attach failed with %d\n", error);
    441   1.2    bouyer 			return;
    442   1.2    bouyer 		}
    443  1.62    cherry 
    444   1.2    bouyer 	} else {
    445   1.2    bouyer 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    446   1.2    bouyer 	}
    447   1.2    bouyer 
    448  1.23        ad 	ci->ci_cpumask = (1 << cpu_index(ci));
    449   1.2    bouyer 	pmap_reference(pmap_kernel());
    450   1.2    bouyer 	ci->ci_pmap = pmap_kernel();
    451   1.2    bouyer 	ci->ci_tlbstate = TLBSTATE_STALE;
    452   1.2    bouyer 
    453  1.38    cegger 	/*
    454  1.38    cegger 	 * Boot processor may not be attached first, but the below
    455  1.38    cegger 	 * must be done to allow booting other processors.
    456  1.38    cegger 	 */
    457  1.38    cegger 	if (!again) {
    458  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    459  1.38    cegger 		/* Basic init. */
    460  1.38    cegger 		cpu_intr_init(ci);
    461  1.38    cegger 		cpu_get_tsc_freq(ci);
    462  1.38    cegger 		cpu_init(ci);
    463  1.78    cherry 		pmap_cpu_init_late(ci);
    464  1.62    cherry 
    465  1.62    cherry 		/* Every processor needs to init it's own ipi h/w (similar to lapic) */
    466  1.62    cherry 		xen_ipi_init();
    467  1.62    cherry 		/* XXX: clock_init() */
    468  1.62    cherry 
    469  1.38    cegger 		/* Make sure DELAY() is initialized. */
    470  1.38    cegger 		DELAY(1);
    471  1.38    cegger 		again = true;
    472  1.38    cegger 	}
    473  1.38    cegger 
    474   1.2    bouyer 	/* further PCB init done later. */
    475   1.2    bouyer 
    476   1.2    bouyer 	switch (caa->cpu_role) {
    477   1.2    bouyer 	case CPU_ROLE_SP:
    478  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    479  1.21        ad 		cpu_identify(ci);
    480  1.12    cegger #if 0
    481  1.12    cegger 		x86_errata();
    482  1.12    cegger #endif
    483  1.38    cegger 		x86_cpu_idle_init();
    484  1.62    cherry 
    485   1.2    bouyer 		break;
    486   1.2    bouyer 
    487   1.2    bouyer 	case CPU_ROLE_BP:
    488  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    489  1.21        ad 		cpu_identify(ci);
    490   1.2    bouyer 		cpu_init(ci);
    491  1.14    bouyer #if 0
    492  1.12    cegger 		x86_errata();
    493  1.12    cegger #endif
    494  1.38    cegger 		x86_cpu_idle_init();
    495  1.62    cherry 
    496   1.2    bouyer 		break;
    497   1.2    bouyer 
    498   1.2    bouyer 	case CPU_ROLE_AP:
    499  1.62    cherry 		atomic_or_32(&ci->ci_flags, CPUF_AP);
    500  1.62    cherry 
    501   1.2    bouyer 		/*
    502   1.2    bouyer 		 * report on an AP
    503   1.2    bouyer 		 */
    504   1.2    bouyer 
    505   1.2    bouyer #if defined(MULTIPROCESSOR)
    506  1.62    cherry 		/* interrupt handler stack */
    507   1.2    bouyer 		cpu_intr_init(ci);
    508  1.62    cherry 
    509  1.62    cherry 		/* Setup per-cpu memory for gdt */
    510   1.2    bouyer 		gdt_alloc_cpu(ci);
    511  1.62    cherry 
    512  1.62    cherry 		pmap_cpu_init_late(ci);
    513   1.2    bouyer 		cpu_start_secondary(ci);
    514  1.62    cherry 
    515   1.2    bouyer 		if (ci->ci_flags & CPUF_PRESENT) {
    516  1.30    cegger 			struct cpu_info *tmp;
    517  1.30    cegger 
    518  1.62    cherry 			cpu_identify(ci);
    519  1.30    cegger 			tmp = cpu_info_list;
    520  1.30    cegger 			while (tmp->ci_next)
    521  1.30    cegger 				tmp = tmp->ci_next;
    522  1.30    cegger 
    523  1.30    cegger 			tmp->ci_next = ci;
    524   1.2    bouyer 		}
    525   1.2    bouyer #else
    526  1.62    cherry 		aprint_error(": not started\n");
    527   1.2    bouyer #endif
    528   1.2    bouyer 		break;
    529   1.2    bouyer 
    530   1.2    bouyer 	default:
    531  1.12    cegger 		aprint_normal("\n");
    532   1.2    bouyer 		panic("unknown processor type??\n");
    533   1.2    bouyer 	}
    534   1.2    bouyer 
    535  1.46    cegger 	pat_init(ci);
    536  1.34    cegger 	atomic_or_32(&cpus_attached, ci->ci_cpumask);
    537   1.2    bouyer 
    538  1.62    cherry #ifdef MPVERBOSE
    539   1.2    bouyer 	if (mp_verbose) {
    540   1.2    bouyer 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    541  1.37     rmind 		struct pcb *pcb = lwp_getpcb(l);
    542   1.2    bouyer 
    543  1.38    cegger 		aprint_verbose_dev(self,
    544  1.38    cegger 		    "idle lwp at %p, idle sp at 0x%p\n",
    545  1.12    cegger 		    l,
    546  1.12    cegger #ifdef i386
    547  1.37     rmind 		    (void *)pcb->pcb_esp
    548  1.62    cherry #else /* i386 */
    549  1.37     rmind 		    (void *)pcb->pcb_rsp
    550  1.62    cherry #endif /* i386 */
    551  1.12    cegger 		);
    552  1.12    cegger 
    553   1.2    bouyer 	}
    554  1.62    cherry #endif /* MPVERBOSE */
    555   1.2    bouyer }
    556   1.2    bouyer 
    557   1.2    bouyer /*
    558   1.2    bouyer  * Initialize the processor appropriately.
    559   1.2    bouyer  */
    560   1.2    bouyer 
    561   1.2    bouyer void
    562  1.10    cegger cpu_init(struct cpu_info *ci)
    563   1.2    bouyer {
    564   1.2    bouyer 
    565   1.2    bouyer 	/*
    566   1.2    bouyer 	 * On a P6 or above, enable global TLB caching if the
    567   1.2    bouyer 	 * hardware supports it.
    568   1.2    bouyer 	 */
    569  1.43       jym 	if (cpu_feature[0] & CPUID_PGE)
    570   1.2    bouyer 		lcr4(rcr4() | CR4_PGE);	/* enable global TLB caching */
    571   1.2    bouyer 
    572   1.2    bouyer #ifdef XXXMTRR
    573   1.2    bouyer 	/*
    574   1.2    bouyer 	 * On a P6 or above, initialize MTRR's if the hardware supports them.
    575   1.2    bouyer 	 */
    576  1.43       jym 	if (cpu_feature[0] & CPUID_MTRR) {
    577   1.2    bouyer 		if ((ci->ci_flags & CPUF_AP) == 0)
    578   1.2    bouyer 			i686_mtrr_init_first();
    579   1.2    bouyer 		mtrr_init_cpu(ci);
    580   1.2    bouyer 	}
    581   1.2    bouyer #endif
    582   1.2    bouyer 	/*
    583   1.2    bouyer 	 * If we have FXSAVE/FXRESTOR, use them.
    584   1.2    bouyer 	 */
    585  1.43       jym 	if (cpu_feature[0] & CPUID_FXSR) {
    586   1.2    bouyer 		lcr4(rcr4() | CR4_OSFXSR);
    587   1.2    bouyer 
    588   1.2    bouyer 		/*
    589   1.2    bouyer 		 * If we have SSE/SSE2, enable XMM exceptions.
    590   1.2    bouyer 		 */
    591  1.43       jym 		if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
    592   1.2    bouyer 			lcr4(rcr4() | CR4_OSXMMEXCPT);
    593   1.2    bouyer 	}
    594   1.2    bouyer 
    595  1.47       jym #ifdef __x86_64__
    596  1.47       jym 	/* No user PGD mapped for this CPU yet */
    597  1.47       jym 	ci->ci_xen_current_user_pgd = 0;
    598  1.47       jym #endif
    599  1.47       jym 
    600  1.34    cegger 	atomic_or_32(&cpus_running, ci->ci_cpumask);
    601  1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    602  1.62    cherry 
    603  1.62    cherry 	/* XXX: register vcpu_register_runstate_memory_area, and figure out how to make sure this VCPU is running ? */
    604   1.2    bouyer }
    605   1.2    bouyer 
    606   1.2    bouyer 
    607   1.2    bouyer #ifdef MULTIPROCESSOR
    608  1.62    cherry 
    609   1.2    bouyer void
    610  1.10    cegger cpu_boot_secondary_processors(void)
    611   1.2    bouyer {
    612   1.2    bouyer 	struct cpu_info *ci;
    613   1.2    bouyer 	u_long i;
    614  1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    615  1.38    cegger 		ci = cpu_lookup(i);
    616   1.2    bouyer 		if (ci == NULL)
    617   1.2    bouyer 			continue;
    618   1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    619   1.2    bouyer 			continue;
    620   1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    621   1.2    bouyer 			continue;
    622   1.2    bouyer 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    623   1.2    bouyer 			continue;
    624   1.2    bouyer 		cpu_boot_secondary(ci);
    625   1.2    bouyer 	}
    626  1.11    cegger 
    627  1.11    cegger 	x86_mp_online = true;
    628   1.2    bouyer }
    629   1.2    bouyer 
    630   1.2    bouyer static void
    631   1.2    bouyer cpu_init_idle_lwp(struct cpu_info *ci)
    632   1.2    bouyer {
    633   1.2    bouyer 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    634  1.37     rmind 	struct pcb *pcb = lwp_getpcb(l);
    635   1.2    bouyer 
    636   1.2    bouyer 	pcb->pcb_cr0 = rcr0();
    637   1.2    bouyer }
    638   1.2    bouyer 
    639   1.2    bouyer void
    640  1.10    cegger cpu_init_idle_lwps(void)
    641   1.2    bouyer {
    642   1.2    bouyer 	struct cpu_info *ci;
    643   1.2    bouyer 	u_long i;
    644   1.2    bouyer 
    645  1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    646  1.38    cegger 		ci = cpu_lookup(i);
    647   1.2    bouyer 		if (ci == NULL)
    648   1.2    bouyer 			continue;
    649   1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    650   1.2    bouyer 			continue;
    651   1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    652   1.2    bouyer 			continue;
    653   1.2    bouyer 		cpu_init_idle_lwp(ci);
    654   1.2    bouyer 	}
    655   1.2    bouyer }
    656   1.2    bouyer 
    657  1.62    cherry static void
    658  1.10    cegger cpu_start_secondary(struct cpu_info *ci)
    659   1.2    bouyer {
    660   1.2    bouyer 	int i;
    661   1.2    bouyer 
    662  1.11    cegger 	aprint_debug_dev(ci->ci_dev, "starting\n");
    663   1.2    bouyer 
    664   1.2    bouyer 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    665  1.62    cherry 
    666  1.62    cherry 	if (CPU_STARTUP(ci, (vaddr_t) cpu_hatch) != 0) {
    667  1.11    cegger 		return;
    668  1.62    cherry 	}
    669   1.2    bouyer 
    670   1.2    bouyer 	/*
    671   1.2    bouyer 	 * wait for it to become ready
    672   1.2    bouyer 	 */
    673  1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    674   1.2    bouyer 		delay(10);
    675   1.2    bouyer 	}
    676  1.11    cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    677   1.9    cegger 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    678   1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    679   1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    680   1.2    bouyer 		Debugger();
    681   1.2    bouyer #endif
    682   1.2    bouyer 	}
    683   1.2    bouyer 
    684   1.2    bouyer 	CPU_START_CLEANUP(ci);
    685   1.2    bouyer }
    686   1.2    bouyer 
    687   1.2    bouyer void
    688  1.10    cegger cpu_boot_secondary(struct cpu_info *ci)
    689   1.2    bouyer {
    690   1.2    bouyer 	int i;
    691  1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    692  1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    693   1.2    bouyer 		delay(10);
    694   1.2    bouyer 	}
    695  1.11    cegger 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    696  1.11    cegger 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
    697   1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    698   1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    699   1.2    bouyer 		Debugger();
    700   1.2    bouyer #endif
    701   1.2    bouyer 	}
    702   1.2    bouyer }
    703   1.2    bouyer 
    704   1.2    bouyer /*
    705  1.62    cherry  * APs end up here immediately after initialisation and VCPUOP_up in
    706  1.62    cherry  * mp_cpu_start().
    707  1.62    cherry  * At this point, we are running in the idle pcb/idle stack of the new
    708  1.62    cherry  * CPU.  This function jumps to the idle loop and starts looking for
    709  1.62    cherry  * work.
    710   1.2    bouyer  */
    711  1.62    cherry extern void x86_64_tls_switch(struct lwp *);
    712   1.2    bouyer void
    713   1.2    bouyer cpu_hatch(void *v)
    714   1.2    bouyer {
    715   1.2    bouyer 	struct cpu_info *ci = (struct cpu_info *)v;
    716  1.37     rmind 	struct pcb *pcb;
    717  1.11    cegger 	int s, i;
    718  1.11    cegger 
    719  1.62    cherry 	/* Setup TLS and kernel GS/FS */
    720  1.62    cherry 	cpu_init_msrs(ci, true);
    721  1.62    cherry 	cpu_init_idt();
    722  1.62    cherry 	gdt_init_cpu(ci);
    723  1.62    cherry 
    724  1.21        ad 	cpu_probe(ci);
    725  1.11    cegger 
    726  1.62    cherry 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    727   1.2    bouyer 
    728  1.11    cegger 	while ((ci->ci_flags & CPUF_GO) == 0) {
    729  1.11    cegger 		/* Don't use delay, boot CPU may be patching the text. */
    730  1.11    cegger 		for (i = 10000; i != 0; i--)
    731  1.11    cegger 			x86_pause();
    732  1.11    cegger 	}
    733   1.2    bouyer 
    734  1.11    cegger 	/* Because the text may have been patched in x86_patch(). */
    735  1.11    cegger 	x86_flush();
    736  1.58     rmind 	tlbflushg();
    737   1.2    bouyer 
    738  1.11    cegger 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    739   1.2    bouyer 
    740  1.37     rmind 	pcb = lwp_getpcb(curlwp);
    741  1.62    cherry 	pcb->pcb_cr3 = pmap_pdirpa(pmap_kernel(), 0); /* XXX: consider using pmap_load() ? */
    742  1.37     rmind 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
    743  1.37     rmind 
    744  1.62    cherry 	xen_ipi_init();
    745  1.62    cherry 
    746  1.62    cherry 	xen_initclocks();
    747  1.62    cherry 
    748  1.62    cherry 	/* XXX: lapic_initclocks(); */
    749  1.11    cegger 
    750  1.62    cherry #ifdef __x86_64__
    751  1.12    cegger 	fpuinit(ci);
    752  1.12    cegger #endif
    753   1.2    bouyer 
    754   1.2    bouyer 	lldt(GSEL(GLDT_SEL, SEL_KPL));
    755   1.2    bouyer 
    756   1.2    bouyer 	cpu_init(ci);
    757  1.11    cegger 	cpu_get_tsc_freq(ci);
    758   1.2    bouyer 
    759   1.2    bouyer 	s = splhigh();
    760  1.11    cegger 	x86_enable_intr();
    761  1.11    cegger 	splx(s);
    762  1.12    cegger #if 0
    763  1.11    cegger 	x86_errata();
    764  1.11    cegger #endif
    765   1.2    bouyer 
    766  1.62    cherry 	aprint_debug_dev(ci->ci_dev, "running\n");
    767  1.62    cherry 
    768  1.62    cherry 	cpu_switchto(NULL, ci->ci_data.cpu_idlelwp, true);
    769  1.62    cherry 
    770  1.62    cherry 	panic("switch to idle_loop context returned!\n");
    771  1.62    cherry 	/* NOTREACHED */
    772   1.2    bouyer }
    773   1.2    bouyer 
    774   1.2    bouyer #if defined(DDB)
    775   1.2    bouyer 
    776   1.2    bouyer #include <ddb/db_output.h>
    777   1.2    bouyer #include <machine/db_machdep.h>
    778   1.2    bouyer 
    779   1.2    bouyer /*
    780   1.2    bouyer  * Dump CPU information from ddb.
    781   1.2    bouyer  */
    782   1.2    bouyer void
    783   1.2    bouyer cpu_debug_dump(void)
    784   1.2    bouyer {
    785   1.2    bouyer 	struct cpu_info *ci;
    786   1.2    bouyer 	CPU_INFO_ITERATOR cii;
    787   1.2    bouyer 
    788  1.13      yamt 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    789   1.2    bouyer 	for (CPU_INFO_FOREACH(cii, ci)) {
    790   1.2    bouyer 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    791   1.2    bouyer 		    ci,
    792   1.9    cegger 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    793  1.12    cegger 		    (long)ci->ci_cpuid,
    794   1.2    bouyer 		    ci->ci_flags, ci->ci_ipis,
    795   1.2    bouyer 		    ci->ci_curlwp,
    796   1.2    bouyer 		    ci->ci_fpcurlwp);
    797   1.2    bouyer 	}
    798   1.2    bouyer }
    799  1.38    cegger #endif /* DDB */
    800   1.2    bouyer 
    801  1.62    cherry #endif /* MULTIPROCESSOR */
    802  1.62    cherry 
    803  1.62    cherry extern void hypervisor_callback(void);
    804  1.62    cherry extern void failsafe_callback(void);
    805  1.62    cherry #ifdef __x86_64__
    806  1.62    cherry typedef void (vector)(void);
    807  1.62    cherry extern vector Xsyscall, Xsyscall32;
    808  1.62    cherry #endif
    809  1.62    cherry 
    810  1.62    cherry /*
    811  1.62    cherry  * Setup the "trampoline". On Xen, we setup nearly all cpu context
    812  1.62    cherry  * outside a trampoline, so we prototype and call targetip like so:
    813  1.62    cherry  * void targetip(struct cpu_info *);
    814  1.62    cherry  */
    815  1.62    cherry 
    816   1.2    bouyer static void
    817  1.62    cherry gdt_prepframes(paddr_t *frames, vaddr_t base, uint32_t entries)
    818   1.2    bouyer {
    819  1.62    cherry 	int i;
    820  1.62    cherry 	for (i = 0; i < roundup(entries, PAGE_SIZE) >> PAGE_SHIFT; i++) {
    821  1.62    cherry 
    822  1.62    cherry 		frames[i] = ((paddr_t) xpmap_ptetomach(
    823  1.62    cherry 				(pt_entry_t *) (base + (i << PAGE_SHIFT))))
    824  1.62    cherry 			>> PAGE_SHIFT;
    825  1.62    cherry 
    826  1.62    cherry 		/* Mark Read-only */
    827  1.62    cherry 		pmap_pte_clearbits(kvtopte(base + (i << PAGE_SHIFT)),
    828  1.62    cherry 		    PG_RW);
    829  1.62    cherry 	}
    830  1.62    cherry }
    831  1.62    cherry 
    832  1.62    cherry #ifdef __x86_64__
    833  1.62    cherry extern char *ldtstore; /* XXX: Xen MP todo */
    834  1.62    cherry 
    835  1.62    cherry static void
    836  1.62    cherry xen_init_amd64_vcpuctxt(struct cpu_info *ci,
    837  1.62    cherry 			struct vcpu_guest_context *initctx,
    838  1.62    cherry 			void targetrip(struct cpu_info *))
    839  1.62    cherry {
    840  1.62    cherry 	/* page frames to point at GDT */
    841  1.62    cherry 	extern int gdt_size;
    842  1.62    cherry 	paddr_t frames[16];
    843  1.62    cherry 	psize_t gdt_ents;
    844  1.62    cherry 
    845  1.62    cherry 	struct lwp *l;
    846  1.62    cherry 	struct pcb *pcb;
    847  1.62    cherry 
    848  1.62    cherry 	volatile struct vcpu_info *vci;
    849  1.62    cherry 
    850  1.62    cherry 	KASSERT(ci != NULL);
    851  1.62    cherry 	KASSERT(ci != &cpu_info_primary);
    852  1.62    cherry 	KASSERT(initctx != NULL);
    853  1.62    cherry 	KASSERT(targetrip != NULL);
    854  1.62    cherry 
    855  1.62    cherry 	memset(initctx, 0, sizeof *initctx);
    856  1.62    cherry 
    857  1.62    cherry 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT; /* XXX: re-investigate roundup(gdt_size... ) for gdt_ents. */
    858  1.62    cherry 	KASSERT(gdt_ents <= 16);
    859  1.62    cherry 
    860  1.62    cherry 	gdt_prepframes(frames, (vaddr_t) ci->ci_gdt, gdt_ents);
    861  1.62    cherry 
    862  1.62    cherry 	/* XXX: The stuff in here is amd64 specific. move to mptramp.[Sc] ? */
    863  1.62    cherry 
    864  1.62    cherry 	/* Initialise the vcpu context: We use idle_loop()'s pcb context. */
    865  1.11    cegger 
    866  1.62    cherry 	l = ci->ci_data.cpu_idlelwp;
    867  1.11    cegger 
    868  1.62    cherry 	KASSERT(l != NULL);
    869  1.62    cherry 	pcb = lwp_getpcb(l);
    870  1.62    cherry 	KASSERT(pcb != NULL);
    871  1.11    cegger 
    872  1.62    cherry 	/* resume with interrupts off */
    873  1.62    cherry 	vci = ci->ci_vcpu;
    874  1.62    cherry 	vci->evtchn_upcall_mask = 1;
    875  1.62    cherry 	xen_mb();
    876   1.2    bouyer 
    877  1.62    cherry 	/* resume in kernel-mode */
    878  1.62    cherry 	initctx->flags = VGCF_in_kernel | VGCF_online;
    879   1.2    bouyer 
    880  1.62    cherry 	/* Stack and entry points:
    881  1.62    cherry 	 * We arrange for the stack frame for cpu_hatch() to
    882  1.62    cherry 	 * appear as a callee frame of lwp_trampoline(). Being a
    883  1.62    cherry 	 * leaf frame prevents trampling on any of the MD stack setup
    884  1.62    cherry 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    885  1.62    cherry 	 */
    886   1.2    bouyer 
    887  1.62    cherry 	initctx->user_regs.rdi = (uint64_t) ci; /* targetrip(ci); */
    888  1.62    cherry 	initctx->user_regs.rip = (vaddr_t) targetrip;
    889   1.2    bouyer 
    890  1.62    cherry 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    891  1.11    cegger 
    892  1.62    cherry 	initctx->user_regs.rflags = pcb->pcb_flags;
    893  1.62    cherry 	initctx->user_regs.rsp = pcb->pcb_rsp;
    894  1.11    cegger 
    895  1.62    cherry 	/* Data segments */
    896  1.62    cherry 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
    897  1.62    cherry 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
    898  1.62    cherry 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
    899  1.62    cherry 
    900  1.62    cherry 	/* GDT */
    901  1.62    cherry 	memcpy(initctx->gdt_frames, frames, sizeof frames);
    902  1.62    cherry 	initctx->gdt_ents = gdt_ents;
    903  1.62    cherry 
    904  1.62    cherry 	/* LDT */
    905  1.62    cherry 	initctx->ldt_base = (unsigned long) ldtstore;
    906  1.62    cherry 	initctx->ldt_ents = LDT_SIZE >> 3;
    907  1.62    cherry 
    908  1.62    cherry 	/* Kernel context state */
    909  1.62    cherry 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
    910  1.62    cherry 	initctx->kernel_sp = pcb->pcb_rsp0;
    911  1.62    cherry 	initctx->ctrlreg[0] = pcb->pcb_cr0;
    912  1.62    cherry 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
    913  1.62    cherry 	initctx->ctrlreg[2] = pcb->pcb_cr2; /* XXX: */
    914  1.62    cherry 	/*
    915  1.62    cherry 	 * Use pmap_kernel() L4 PD directly, until we setup the
    916  1.62    cherry 	 * per-cpu L4 PD in pmap_cpu_init_late()
    917   1.2    bouyer 	 */
    918  1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_kpm_pdirpa)));
    919  1.62    cherry 	initctx->ctrlreg[4] = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT;
    920   1.2    bouyer 
    921  1.62    cherry 
    922  1.62    cherry 	/* Xen callbacks */
    923  1.62    cherry 	initctx->event_callback_eip = (unsigned long) hypervisor_callback;
    924  1.62    cherry 	initctx->failsafe_callback_eip = (unsigned long) failsafe_callback;
    925  1.62    cherry 	initctx->syscall_callback_eip = (unsigned long) Xsyscall;
    926  1.62    cherry 
    927  1.62    cherry 	return;
    928   1.2    bouyer }
    929  1.62    cherry #else /* i386 */
    930  1.62    cherry extern union descriptor *ldt;
    931  1.62    cherry extern void Xsyscall(void);
    932  1.62    cherry 
    933  1.11    cegger static void
    934  1.62    cherry xen_init_i386_vcpuctxt(struct cpu_info *ci,
    935  1.62    cherry 			struct vcpu_guest_context *initctx,
    936  1.62    cherry 			void targeteip(struct cpu_info *))
    937  1.62    cherry {
    938  1.62    cherry 	/* page frames to point at GDT */
    939  1.62    cherry 	extern int gdt_size;
    940  1.62    cherry 	paddr_t frames[16];
    941  1.62    cherry 	psize_t gdt_ents;
    942  1.62    cherry 
    943  1.62    cherry 	struct lwp *l;
    944  1.62    cherry 	struct pcb *pcb;
    945  1.62    cherry 
    946  1.62    cherry 	volatile struct vcpu_info *vci;
    947  1.62    cherry 
    948  1.62    cherry 	KASSERT(ci != NULL);
    949  1.62    cherry 	KASSERT(ci != &cpu_info_primary);
    950  1.62    cherry 	KASSERT(initctx != NULL);
    951  1.62    cherry 	KASSERT(targeteip != NULL);
    952  1.62    cherry 
    953  1.62    cherry 	memset(initctx, 0, sizeof *initctx);
    954  1.11    cegger 
    955  1.62    cherry 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT; /* XXX: re-investigate roundup(gdt_size... ) for gdt_ents. */
    956  1.62    cherry 	KASSERT(gdt_ents <= 16);
    957   1.2    bouyer 
    958  1.62    cherry 	gdt_prepframes(frames, (vaddr_t) ci->ci_gdt, gdt_ents);
    959   1.2    bouyer 
    960  1.62    cherry 	/*
    961  1.62    cherry 	 * Initialise the vcpu context:
    962  1.62    cherry 	 * We use this cpu's idle_loop() pcb context.
    963  1.11    cegger 	 */
    964  1.11    cegger 
    965  1.62    cherry 	l = ci->ci_data.cpu_idlelwp;
    966  1.62    cherry 
    967  1.62    cherry 	KASSERT(l != NULL);
    968  1.62    cherry 	pcb = lwp_getpcb(l);
    969  1.62    cherry 	KASSERT(pcb != NULL);
    970  1.62    cherry 
    971  1.62    cherry 	/* resume with interrupts off */
    972  1.62    cherry 	vci = ci->ci_vcpu;
    973  1.62    cherry 	vci->evtchn_upcall_mask = 1;
    974  1.62    cherry 	xen_mb();
    975  1.62    cherry 
    976  1.62    cherry 	/* resume in kernel-mode */
    977  1.62    cherry 	initctx->flags = VGCF_in_kernel | VGCF_online;
    978  1.62    cherry 
    979  1.62    cherry 	/* Stack frame setup for cpu_hatch():
    980  1.62    cherry 	 * We arrange for the stack frame for cpu_hatch() to
    981  1.62    cherry 	 * appear as a callee frame of lwp_trampoline(). Being a
    982  1.62    cherry 	 * leaf frame prevents trampling on any of the MD stack setup
    983  1.62    cherry 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    984   1.2    bouyer 	 */
    985   1.2    bouyer 
    986  1.62    cherry 	initctx->user_regs.esp = pcb->pcb_esp - 4; /* Leave word for
    987  1.62    cherry 						      arg1 */
    988  1.62    cherry 	{ /* targeteip(ci); */
    989  1.62    cherry 		uint32_t *arg = (uint32_t *) initctx->user_regs.esp;
    990  1.62    cherry 		arg[1] = (uint32_t) ci; /* arg1 */
    991  1.62    cherry 
    992  1.62    cherry 	}
    993   1.2    bouyer 
    994  1.62    cherry 	initctx->user_regs.eip = (vaddr_t) targeteip;
    995  1.62    cherry 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    996  1.62    cherry 	initctx->user_regs.eflags |= pcb->pcb_iopl;
    997  1.62    cherry 
    998  1.62    cherry 	/* Data segments */
    999  1.62    cherry 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
   1000  1.62    cherry 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
   1001  1.62    cherry 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
   1002  1.62    cherry 	initctx->user_regs.fs = GSEL(GDATA_SEL, SEL_KPL);
   1003  1.62    cherry 
   1004  1.62    cherry 	/* GDT */
   1005  1.62    cherry 	memcpy(initctx->gdt_frames, frames, sizeof frames);
   1006  1.62    cherry 	initctx->gdt_ents = gdt_ents;
   1007  1.62    cherry 
   1008  1.62    cherry 	/* LDT */
   1009  1.62    cherry 	initctx->ldt_base = (unsigned long) ldt;
   1010  1.62    cherry 	initctx->ldt_ents = NLDT;
   1011  1.62    cherry 
   1012  1.62    cherry 	/* Kernel context state */
   1013  1.62    cherry 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
   1014  1.62    cherry 	initctx->kernel_sp = pcb->pcb_esp0;
   1015  1.62    cherry 	initctx->ctrlreg[0] = pcb->pcb_cr0;
   1016  1.62    cherry 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
   1017  1.62    cherry 	initctx->ctrlreg[2] = pcb->pcb_cr2; /* XXX: */
   1018  1.70    cherry #ifdef PAE
   1019  1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_pae_l3_pdirpa)));
   1020  1.70    cherry #else /* PAE */
   1021  1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(pcb->pcb_cr3)));
   1022  1.70    cherry #endif /* PAE */
   1023  1.62    cherry 	initctx->ctrlreg[4] = /* CR4_PAE |  */CR4_OSFXSR | CR4_OSXMMEXCPT;
   1024   1.2    bouyer 
   1025   1.2    bouyer 
   1026  1.62    cherry 	/* Xen callbacks */
   1027  1.62    cherry 	initctx->event_callback_eip = (unsigned long) hypervisor_callback;
   1028  1.62    cherry 	initctx->event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
   1029  1.62    cherry 	initctx->failsafe_callback_eip = (unsigned long) failsafe_callback;
   1030  1.62    cherry 	initctx->failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
   1031  1.45     rmind 
   1032  1.62    cherry 	return;
   1033  1.62    cherry }
   1034  1.62    cherry #endif /* __x86_64__ */
   1035  1.45     rmind 
   1036  1.62    cherry int
   1037  1.62    cherry mp_cpu_start(struct cpu_info *ci, vaddr_t target)
   1038  1.62    cherry {
   1039  1.62    cherry 
   1040  1.62    cherry 	int hyperror;
   1041  1.62    cherry 	struct vcpu_guest_context vcpuctx;
   1042   1.2    bouyer 
   1043  1.62    cherry 	KASSERT(ci != NULL);
   1044  1.62    cherry 	KASSERT(ci != &cpu_info_primary);
   1045  1.62    cherry 	KASSERT(ci->ci_flags & CPUF_AP);
   1046  1.62    cherry 
   1047  1.62    cherry #ifdef __x86_64__
   1048  1.62    cherry 	xen_init_amd64_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
   1049  1.62    cherry #else  /* i386 */
   1050  1.62    cherry 	xen_init_i386_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
   1051  1.62    cherry #endif /* __x86_64__ */
   1052  1.62    cherry 
   1053  1.62    cherry 	/* Initialise the given vcpu to execute cpu_hatch(ci); */
   1054  1.62    cherry 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_cpuid, &vcpuctx))) {
   1055  1.62    cherry 		aprint_error(": context initialisation failed. errno = %d\n", hyperror);
   1056  1.62    cherry 		return hyperror;
   1057  1.62    cherry 	}
   1058  1.62    cherry 
   1059  1.62    cherry 	/* Start it up */
   1060  1.62    cherry 
   1061  1.70    cherry 	/* First bring it down */
   1062  1.62    cherry 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_cpuid, NULL))) {
   1063  1.62    cherry 		aprint_error(": VCPUOP_down hypervisor command failed. errno = %d\n", hyperror);
   1064  1.62    cherry 		return hyperror;
   1065  1.62    cherry 	}
   1066  1.62    cherry 
   1067  1.62    cherry 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_up, ci->ci_cpuid, NULL))) {
   1068  1.62    cherry 		aprint_error(": VCPUOP_up hypervisor command failed. errno = %d\n", hyperror);
   1069  1.62    cherry 		return hyperror;
   1070  1.62    cherry 	}
   1071   1.2    bouyer 
   1072  1.62    cherry 	if (!vcpu_is_up(ci)) {
   1073  1.62    cherry 		aprint_error(": did not come up\n");
   1074  1.62    cherry 		return -1;
   1075   1.2    bouyer 	}
   1076  1.62    cherry 
   1077   1.2    bouyer 	return 0;
   1078   1.2    bouyer }
   1079   1.2    bouyer 
   1080   1.2    bouyer void
   1081   1.2    bouyer mp_cpu_start_cleanup(struct cpu_info *ci)
   1082   1.2    bouyer {
   1083   1.2    bouyer #if 0
   1084   1.2    bouyer 	/*
   1085   1.2    bouyer 	 * Ensure the NVRAM reset byte contains something vaguely sane.
   1086   1.2    bouyer 	 */
   1087   1.2    bouyer 
   1088   1.2    bouyer 	outb(IO_RTC, NVRAM_RESET);
   1089   1.2    bouyer 	outb(IO_RTC+1, NVRAM_RESET_RST);
   1090   1.2    bouyer #endif
   1091  1.62    cherry 	if (vcpu_is_up(ci)) {
   1092  1.62    cherry 		aprint_debug_dev(ci->ci_dev, "is started.\n");
   1093  1.62    cherry 	}
   1094  1.62    cherry 	else {
   1095  1.62    cherry 		aprint_error_dev(ci->ci_dev, "did not start up.\n");
   1096  1.62    cherry 	}
   1097  1.62    cherry 
   1098   1.2    bouyer }
   1099   1.2    bouyer 
   1100  1.69    cherry /* curcpu() uses %fs - shim for until cpu_init_msrs(), below */
   1101  1.69    cherry static struct cpu_info *cpu_primary(void)
   1102  1.69    cherry {
   1103  1.69    cherry 	return &cpu_info_primary;
   1104  1.69    cherry }
   1105  1.72    cherry /* XXX: rename to something more generic. users other than xpq exist */
   1106  1.69    cherry struct cpu_info	* (*xpq_cpu)(void) = cpu_primary;
   1107  1.69    cherry 
   1108   1.2    bouyer void
   1109   1.3    bouyer cpu_init_msrs(struct cpu_info *ci, bool full)
   1110   1.2    bouyer {
   1111  1.43       jym #ifdef __x86_64__
   1112   1.3    bouyer 	if (full) {
   1113   1.3    bouyer 		HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
   1114  1.11    cegger 		HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
   1115   1.3    bouyer 		HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
   1116  1.69    cherry 		xpq_cpu = x86_curcpu;
   1117   1.3    bouyer 	}
   1118  1.43       jym #endif	/* __x86_64__ */
   1119  1.44       jym 
   1120  1.44       jym 	if (cpu_feature[2] & CPUID_NOX)
   1121  1.44       jym 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
   1122  1.62    cherry 
   1123   1.2    bouyer }
   1124   1.2    bouyer 
   1125  1.11    cegger void
   1126  1.11    cegger cpu_offline_md(void)
   1127  1.11    cegger {
   1128  1.11    cegger         int s;
   1129  1.11    cegger 
   1130  1.11    cegger         s = splhigh();
   1131  1.11    cegger #ifdef __i386__
   1132  1.11    cegger         npxsave_cpu(true);
   1133  1.11    cegger #else
   1134  1.11    cegger         fpusave_cpu(true);
   1135  1.11    cegger #endif
   1136  1.11    cegger         splx(s);
   1137  1.11    cegger }
   1138  1.11    cegger 
   1139   1.2    bouyer void
   1140   1.2    bouyer cpu_get_tsc_freq(struct cpu_info *ci)
   1141   1.2    bouyer {
   1142  1.62    cherry 	uint32_t vcpu_tversion;
   1143  1.16    cegger 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
   1144  1.62    cherry 
   1145  1.62    cherry 	vcpu_tversion = tinfo->version;
   1146  1.62    cherry 	while (tinfo->version == vcpu_tversion); /* Wait for a time update. XXX: timeout ? */
   1147  1.62    cherry 
   1148   1.2    bouyer 	uint64_t freq = 1000000000ULL << 32;
   1149   1.2    bouyer 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
   1150   1.2    bouyer 	if ( tinfo->tsc_shift < 0 )
   1151   1.2    bouyer 		freq = freq << -tinfo->tsc_shift;
   1152   1.2    bouyer 	else
   1153   1.2    bouyer 		freq = freq >> tinfo->tsc_shift;
   1154  1.20        ad 	ci->ci_data.cpu_cc_freq = freq;
   1155   1.2    bouyer }
   1156  1.19     joerg 
   1157  1.19     joerg void
   1158  1.19     joerg x86_cpu_idle_xen(void)
   1159  1.19     joerg {
   1160  1.19     joerg 	struct cpu_info *ci = curcpu();
   1161  1.62    cherry 
   1162  1.19     joerg 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1163  1.19     joerg 
   1164  1.19     joerg 	x86_disable_intr();
   1165  1.19     joerg 	if (!__predict_false(ci->ci_want_resched)) {
   1166  1.19     joerg 		idle_block();
   1167  1.19     joerg 	} else {
   1168  1.19     joerg 		x86_enable_intr();
   1169  1.19     joerg 	}
   1170  1.19     joerg }
   1171  1.47       jym 
   1172  1.47       jym /*
   1173  1.47       jym  * Loads pmap for the current CPU.
   1174  1.47       jym  */
   1175  1.47       jym void
   1176  1.47       jym cpu_load_pmap(struct pmap *pmap)
   1177  1.47       jym {
   1178  1.47       jym #ifdef i386
   1179  1.47       jym #ifdef PAE
   1180  1.47       jym 	int i, s;
   1181  1.47       jym 	struct cpu_info *ci;
   1182  1.47       jym 
   1183  1.47       jym 	s = splvm(); /* just to be safe */
   1184  1.47       jym 	ci = curcpu();
   1185  1.47       jym 	paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);
   1186  1.47       jym 	/* don't update the kernel L3 slot */
   1187  1.47       jym 	for (i = 0 ; i < PDP_SIZE - 1; i++) {
   1188  1.47       jym 		xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t),
   1189  1.47       jym 		    xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V);
   1190  1.47       jym 	}
   1191  1.47       jym 	splx(s);
   1192  1.47       jym 	tlbflush();
   1193  1.47       jym #else /* PAE */
   1194  1.47       jym 	lcr3(pmap_pdirpa(pmap, 0));
   1195  1.47       jym #endif /* PAE */
   1196  1.47       jym #endif /* i386 */
   1197  1.47       jym 
   1198  1.47       jym #ifdef __x86_64__
   1199  1.47       jym 	int i, s;
   1200  1.70    cherry 	pd_entry_t *new_pgd;
   1201  1.47       jym 	struct cpu_info *ci;
   1202  1.70    cherry 	paddr_t l4_pd_ma;
   1203  1.47       jym 
   1204  1.70    cherry 	ci = curcpu();
   1205  1.70    cherry 	l4_pd_ma = xpmap_ptom_masked(ci->ci_kpm_pdirpa);
   1206  1.70    cherry 
   1207  1.70    cherry 	/*
   1208  1.70    cherry 	 * Map user space address in kernel space and load
   1209  1.70    cherry 	 * user cr3
   1210  1.70    cherry 	 */
   1211  1.70    cherry 	s = splvm();
   1212  1.70    cherry 	new_pgd = pmap->pm_pdir;
   1213  1.70    cherry 
   1214  1.70    cherry 	/* Copy user pmap L4 PDEs (in user addr. range) to per-cpu L4 */
   1215  1.70    cherry 	for (i = 0; i < PDIR_SLOT_PTE; i++) {
   1216  1.70    cherry 		xpq_queue_pte_update(l4_pd_ma + i * sizeof(pd_entry_t), new_pgd[i]);
   1217  1.70    cherry 	}
   1218  1.70    cherry 
   1219  1.70    cherry 	if (__predict_true(pmap != pmap_kernel())) {
   1220  1.47       jym 		xen_set_user_pgd(pmap_pdirpa(pmap, 0));
   1221  1.47       jym 		ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);
   1222  1.47       jym 	}
   1223  1.70    cherry 	else {
   1224  1.70    cherry 		xpq_queue_pt_switch(l4_pd_ma);
   1225  1.70    cherry 		ci->ci_xen_current_user_pgd = 0;
   1226  1.70    cherry 	}
   1227  1.70    cherry 
   1228  1.70    cherry 	tlbflush();
   1229  1.70    cherry 	splx(s);
   1230  1.70    cherry 
   1231  1.47       jym #endif /* __x86_64__ */
   1232  1.47       jym }
   1233  1.61    cherry 
   1234  1.70    cherry  /*
   1235  1.70    cherry   * pmap_cpu_init_late: perform late per-CPU initialization.
   1236  1.70    cherry   * Short note about percpu PDIR pages:
   1237  1.70    cherry   * Both the PAE and __x86_64__ architectures have per-cpu PDIR
   1238  1.70    cherry   * tables. This is to get around Xen's pagetable setup constraints for
   1239  1.70    cherry   * PAE (multiple L3[3]s cannot point to the same L2 - Xen
   1240  1.70    cherry   * will refuse to pin a table setup this way.) and for multiple cpus
   1241  1.70    cherry   * to map in different user pmaps on __x86_64__ (see: cpu_load_pmap())
   1242  1.70    cherry   *
   1243  1.70    cherry   * What this means for us is that the PDIR of the pmap_kernel() is
   1244  1.70    cherry   * considered to be a canonical "SHADOW" PDIR with the following
   1245  1.70    cherry   * properties:
   1246  1.70    cherry   * - Its recursive mapping points to itself
   1247  1.70    cherry   * - per-cpu recurseive mappings point to themselves
   1248  1.70    cherry   * - per-cpu L4 pages' kernel entries are expected to be in sync with
   1249  1.70    cherry   *   the shadow
   1250  1.70    cherry   * - APDP_PDE_SHADOW accesses the shadow pdir
   1251  1.70    cherry   * - APDP_PDE accesses the per-cpu pdir
   1252  1.70    cherry   * - alternate mappings are considered per-cpu - however, x86 pmap
   1253  1.70    cherry   *   currently partially consults the shadow - this works because the
   1254  1.70    cherry   *   shadow PDE is updated together with the per-cpu entry (see:
   1255  1.70    cherry   *   xen_pmap.c: pmap_map_ptes(), and the pmap is locked while the
   1256  1.70    cherry   * alternate ptes are mapped in.
   1257  1.70    cherry   */
   1258  1.70    cherry 
   1259  1.70    cherry void
   1260  1.70    cherry pmap_cpu_init_late(struct cpu_info *ci)
   1261  1.70    cherry {
   1262  1.70    cherry #if defined(PAE) || defined(__x86_64__)
   1263  1.70    cherry 	/*
   1264  1.70    cherry 	 * The BP has already its own PD page allocated during early
   1265  1.70    cherry 	 * MD startup.
   1266  1.70    cherry 	 */
   1267  1.70    cherry 
   1268  1.78    cherry #if defined(__x86_64__)
   1269  1.78    cherry 	/* Setup per-cpu normal_pdes */
   1270  1.78    cherry 	int i;
   1271  1.78    cherry 	extern pd_entry_t * const normal_pdes[];
   1272  1.78    cherry 	for (i = 0;i < PTP_LEVELS - 1;i++) {
   1273  1.78    cherry 		ci->ci_normal_pdes[i] = normal_pdes[i];
   1274  1.78    cherry 	}
   1275  1.78    cherry #endif /* __x86_64__ */
   1276  1.78    cherry 
   1277  1.70    cherry 	if (ci == &cpu_info_primary)
   1278  1.70    cherry 		return;
   1279  1.70    cherry 
   1280  1.70    cherry 	KASSERT(ci != NULL);
   1281  1.70    cherry 
   1282  1.70    cherry #if defined(PAE)
   1283  1.73    cherry 	cpu_alloc_l3_page(ci);
   1284  1.70    cherry 	KASSERT(ci->ci_pae_l3_pdirpa != 0);
   1285  1.70    cherry 
   1286  1.70    cherry 	/* Initialise L2 entries 0 - 2: Point them to pmap_kernel() */
   1287  1.73    cherry 	int i;
   1288  1.75    cherry 	for (i = 0 ; i < PDP_SIZE - 1; i++) {
   1289  1.73    cherry 		ci->ci_pae_l3_pdir[i] =
   1290  1.73    cherry 		    xpmap_ptom_masked(pmap_kernel()->pm_pdirpa[i]) | PG_V;
   1291  1.73    cherry 	}
   1292  1.70    cherry #endif /* PAE */
   1293  1.70    cherry 
   1294  1.70    cherry 	ci->ci_kpm_pdir = (pd_entry_t *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
   1295  1.70    cherry 	    UVM_KMF_WIRED | UVM_KMF_ZERO | UVM_KMF_NOWAIT);
   1296  1.70    cherry 
   1297  1.70    cherry 	if (ci->ci_kpm_pdir == NULL) {
   1298  1.70    cherry 		panic("%s: failed to allocate L4 per-cpu PD for CPU %d\n",
   1299  1.70    cherry 		      __func__, cpu_index(ci));
   1300  1.70    cherry 	}
   1301  1.70    cherry 	ci->ci_kpm_pdirpa = vtophys((vaddr_t) ci->ci_kpm_pdir);
   1302  1.70    cherry 	KASSERT(ci->ci_kpm_pdirpa != 0);
   1303  1.70    cherry 
   1304  1.70    cherry #if defined(__x86_64__)
   1305  1.70    cherry 	/*
   1306  1.70    cherry 	 * Copy over the pmap_kernel() shadow L4 entries
   1307  1.70    cherry 	 */
   1308  1.70    cherry 
   1309  1.70    cherry 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir, PAGE_SIZE);
   1310  1.70    cherry 
   1311  1.70    cherry 	/* Recursive kernel mapping */
   1312  1.70    cherry 	ci->ci_kpm_pdir[PDIR_SLOT_PTE] = xpmap_ptom_masked(ci->ci_kpm_pdirpa) | PG_k | PG_V;
   1313  1.70    cherry #elif defined(PAE)
   1314  1.70    cherry 	/* Copy over the pmap_kernel() shadow L2 entries that map the kernel */
   1315  1.70    cherry 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir + PDIR_SLOT_KERN, nkptp[PTP_LEVELS - 1] * sizeof(pd_entry_t));
   1316  1.70    cherry #endif /* __x86_64__ else PAE */
   1317  1.70    cherry 
   1318  1.70    cherry 	/* Xen wants R/O */
   1319  1.70    cherry 	pmap_kenter_pa((vaddr_t)ci->ci_kpm_pdir, ci->ci_kpm_pdirpa,
   1320  1.70    cherry 	    VM_PROT_READ, 0);
   1321  1.70    cherry 
   1322  1.70    cherry #if defined(PAE)
   1323  1.70    cherry 	/* Initialise L3 entry 3. This mapping is shared across all
   1324  1.70    cherry 	 * pmaps and is static, ie; loading a new pmap will not update
   1325  1.70    cherry 	 * this entry.
   1326  1.70    cherry 	 */
   1327  1.70    cherry 
   1328  1.70    cherry 	ci->ci_pae_l3_pdir[3] = xpmap_ptom_masked(ci->ci_kpm_pdirpa) | PG_k | PG_V;
   1329  1.70    cherry 
   1330  1.70    cherry 	/* Mark L3 R/O (Xen wants this) */
   1331  1.70    cherry 	pmap_kenter_pa((vaddr_t)ci->ci_pae_l3_pdir, ci->ci_pae_l3_pdirpa,
   1332  1.70    cherry 		VM_PROT_READ, 0);
   1333  1.70    cherry 
   1334  1.70    cherry 	xpq_queue_pin_l3_table(xpmap_ptom_masked(ci->ci_pae_l3_pdirpa));
   1335  1.70    cherry 
   1336  1.70    cherry #elif defined(__x86_64__)
   1337  1.70    cherry 	xpq_queue_pin_l4_table(xpmap_ptom_masked(ci->ci_kpm_pdirpa));
   1338  1.78    cherry #endif /* PAE , __x86_64__ */
   1339  1.70    cherry #endif /* defined(PAE) || defined(__x86_64__) */
   1340  1.70    cherry }
   1341  1.70    cherry 
   1342  1.61    cherry /*
   1343  1.61    cherry  * Notify all other cpus to halt.
   1344  1.61    cherry  */
   1345  1.61    cherry 
   1346  1.61    cherry void
   1347  1.61    cherry cpu_broadcast_halt(void)
   1348  1.61    cherry {
   1349  1.61    cherry 	xen_broadcast_ipi(XEN_IPI_HALT);
   1350  1.61    cherry }
   1351  1.61    cherry 
   1352  1.61    cherry /*
   1353  1.61    cherry  * Send a dummy ipi to a cpu.
   1354  1.61    cherry  */
   1355  1.61    cherry 
   1356  1.61    cherry void
   1357  1.61    cherry cpu_kick(struct cpu_info *ci)
   1358  1.61    cherry {
   1359  1.64  dholland 	(void)xen_send_ipi(ci, XEN_IPI_KICK);
   1360  1.61    cherry }
   1361