Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.122
      1  1.121      maxv /*	$NetBSD: cpu.c,v 1.122 2018/06/23 10:30:22 jdolecek Exp $	*/
      2    1.2    bouyer 
      3    1.2    bouyer /*-
      4    1.2    bouyer  * Copyright (c) 2000 The NetBSD Foundation, Inc.
      5   1.19     joerg  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
      6    1.2    bouyer  * All rights reserved.
      7    1.2    bouyer  *
      8    1.2    bouyer  * This code is derived from software contributed to The NetBSD Foundation
      9    1.2    bouyer  * by RedBack Networks Inc.
     10    1.2    bouyer  *
     11    1.2    bouyer  * Author: Bill Sommerfeld
     12    1.2    bouyer  *
     13    1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     14    1.2    bouyer  * modification, are permitted provided that the following conditions
     15    1.2    bouyer  * are met:
     16    1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     17    1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     18    1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     19    1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     20    1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     21    1.2    bouyer  *
     22    1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23    1.2    bouyer  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24    1.2    bouyer  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25    1.2    bouyer  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26    1.2    bouyer  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27    1.2    bouyer  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28    1.2    bouyer  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29    1.2    bouyer  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30    1.2    bouyer  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31    1.2    bouyer  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32    1.2    bouyer  * POSSIBILITY OF SUCH DAMAGE.
     33    1.2    bouyer  */
     34    1.2    bouyer 
     35    1.2    bouyer /*
     36    1.2    bouyer  * Copyright (c) 1999 Stefan Grefen
     37    1.2    bouyer  *
     38    1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     39    1.2    bouyer  * modification, are permitted provided that the following conditions
     40    1.2    bouyer  * are met:
     41    1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     42    1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     43    1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     44    1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     45    1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     46    1.2    bouyer  * 3. All advertising materials mentioning features or use of this software
     47    1.2    bouyer  *    must display the following acknowledgement:
     48    1.2    bouyer  *      This product includes software developed by the NetBSD
     49    1.2    bouyer  *      Foundation, Inc. and its contributors.
     50    1.2    bouyer  * 4. Neither the name of The NetBSD Foundation nor the names of its
     51    1.2    bouyer  *    contributors may be used to endorse or promote products derived
     52    1.2    bouyer  *    from this software without specific prior written permission.
     53    1.2    bouyer  *
     54    1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     55    1.2    bouyer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     56    1.2    bouyer  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     57    1.2    bouyer  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     58    1.2    bouyer  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     59    1.2    bouyer  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     60    1.2    bouyer  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     61    1.2    bouyer  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     62    1.2    bouyer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     63    1.2    bouyer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     64    1.2    bouyer  * SUCH DAMAGE.
     65    1.2    bouyer  */
     66    1.2    bouyer 
     67    1.2    bouyer #include <sys/cdefs.h>
     68  1.121      maxv __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.122 2018/06/23 10:30:22 jdolecek Exp $");
     69    1.2    bouyer 
     70    1.2    bouyer #include "opt_ddb.h"
     71    1.2    bouyer #include "opt_multiprocessor.h"
     72    1.2    bouyer #include "opt_mpbios.h"		/* for MPDEBUG */
     73    1.2    bouyer #include "opt_mtrr.h"
     74    1.2    bouyer #include "opt_xen.h"
     75    1.2    bouyer 
     76    1.2    bouyer #include "lapic.h"
     77    1.2    bouyer #include "ioapic.h"
     78    1.2    bouyer 
     79    1.2    bouyer #include <sys/param.h>
     80    1.2    bouyer #include <sys/proc.h>
     81    1.2    bouyer #include <sys/systm.h>
     82    1.2    bouyer #include <sys/device.h>
     83   1.31    cegger #include <sys/kmem.h>
     84   1.11    cegger #include <sys/cpu.h>
     85   1.66    jruoho #include <sys/cpufreq.h>
     86   1.11    cegger #include <sys/atomic.h>
     87   1.32    cegger #include <sys/reboot.h>
     88   1.62    cherry #include <sys/idle.h>
     89    1.2    bouyer 
     90   1.51  uebayasi #include <uvm/uvm.h>
     91    1.2    bouyer 
     92  1.114  riastrad #include <machine/cpu.h>
     93    1.2    bouyer #include <machine/cpufunc.h>
     94    1.2    bouyer #include <machine/cpuvar.h>
     95    1.2    bouyer #include <machine/pmap.h>
     96    1.2    bouyer #include <machine/vmparam.h>
     97    1.2    bouyer #include <machine/mpbiosvar.h>
     98    1.2    bouyer #include <machine/pcb.h>
     99    1.2    bouyer #include <machine/specialreg.h>
    100    1.2    bouyer #include <machine/segments.h>
    101    1.2    bouyer #include <machine/gdt.h>
    102    1.2    bouyer #include <machine/mtrr.h>
    103    1.2    bouyer #include <machine/pio.h>
    104    1.2    bouyer 
    105   1.97       dsl #include <x86/fpu.h>
    106   1.62    cherry 
    107   1.62    cherry #include <xen/xen.h>
    108   1.71    cegger #include <xen/xen-public/vcpu.h>
    109    1.2    bouyer #include <xen/vcpuvar.h>
    110    1.2    bouyer 
    111    1.2    bouyer #if NLAPIC > 0
    112    1.2    bouyer #include <machine/apicvar.h>
    113    1.2    bouyer #include <machine/i82489reg.h>
    114    1.2    bouyer #include <machine/i82489var.h>
    115    1.2    bouyer #endif
    116    1.2    bouyer 
    117    1.2    bouyer #include <dev/ic/mc146818reg.h>
    118    1.2    bouyer #include <dev/isa/isareg.h>
    119    1.2    bouyer 
    120   1.56    jruoho static int	cpu_match(device_t, cfdata_t, void *);
    121   1.56    jruoho static void	cpu_attach(device_t, device_t, void *);
    122   1.56    jruoho static void	cpu_defer(device_t);
    123   1.56    jruoho static int	cpu_rescan(device_t, const char *, const int *);
    124   1.56    jruoho static void	cpu_childdetached(device_t, device_t);
    125   1.56    jruoho static int	vcpu_match(device_t, cfdata_t, void *);
    126   1.56    jruoho static void	vcpu_attach(device_t, device_t, void *);
    127   1.56    jruoho static void	cpu_attach_common(device_t, device_t, void *);
    128   1.56    jruoho void		cpu_offline_md(void);
    129    1.2    bouyer 
    130    1.2    bouyer struct cpu_softc {
    131   1.10    cegger 	device_t sc_dev;		/* device tree glue */
    132    1.2    bouyer 	struct cpu_info *sc_info;	/* pointer to CPU info */
    133   1.32    cegger 	bool sc_wasonline;
    134    1.2    bouyer };
    135    1.2    bouyer 
    136   1.62    cherry int mp_cpu_start(struct cpu_info *, vaddr_t);
    137    1.2    bouyer void mp_cpu_start_cleanup(struct cpu_info *);
    138    1.2    bouyer const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    139    1.2    bouyer 				      mp_cpu_start_cleanup };
    140    1.2    bouyer 
    141   1.53    jruoho CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
    142   1.53    jruoho     cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
    143   1.53    jruoho 
    144   1.10    cegger CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
    145    1.2    bouyer     vcpu_match, vcpu_attach, NULL, NULL);
    146    1.2    bouyer 
    147    1.2    bouyer /*
    148    1.2    bouyer  * Statically-allocated CPU info for the primary CPU (or the only
    149    1.2    bouyer  * CPU, on uniprocessors).  The CPU info list is initialized to
    150    1.2    bouyer  * point at it.
    151    1.2    bouyer  */
    152   1.38    cegger struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    153    1.7    bouyer 	.ci_dev = 0,
    154    1.2    bouyer 	.ci_self = &cpu_info_primary,
    155    1.4    bouyer 	.ci_idepth = -1,
    156    1.2    bouyer 	.ci_curlwp = &lwp0,
    157   1.25        ad 	.ci_curldt = -1,
    158    1.2    bouyer };
    159   1.38    cegger struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    160    1.7    bouyer 	.ci_dev = 0,
    161    1.2    bouyer 	.ci_self = &phycpu_info_primary,
    162    1.2    bouyer };
    163    1.2    bouyer 
    164    1.2    bouyer struct cpu_info *cpu_info_list = &cpu_info_primary;
    165   1.38    cegger struct cpu_info *phycpu_info_list = &phycpu_info_primary;
    166    1.2    bouyer 
    167  1.107      maxv uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits
    168   1.43       jym 			  *	[0] basic features %edx
    169   1.43       jym 			  *	[1] basic features %ecx
    170   1.43       jym 			  *	[2] extended features %edx
    171   1.43       jym 			  *	[3] extended features %ecx
    172   1.43       jym 			  *	[4] VIA padlock features
    173  1.102  christos 			  *	[5] structured extended features cpuid.7:%ebx
    174  1.102  christos 			  *	[6] structured extended features cpuid.7:%ecx
    175   1.43       jym 			  */
    176   1.43       jym 
    177   1.11    cegger bool x86_mp_online;
    178   1.11    cegger paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    179    1.2    bouyer 
    180   1.38    cegger #if defined(MULTIPROCESSOR)
    181    1.2    bouyer void    	cpu_hatch(void *);
    182    1.2    bouyer static void    	cpu_boot_secondary(struct cpu_info *ci);
    183    1.2    bouyer static void    	cpu_start_secondary(struct cpu_info *ci);
    184   1.38    cegger #endif	/* MULTIPROCESSOR */
    185    1.2    bouyer 
    186   1.56    jruoho static int
    187   1.10    cegger cpu_match(device_t parent, cfdata_t match, void *aux)
    188    1.2    bouyer {
    189    1.2    bouyer 
    190    1.2    bouyer 	return 1;
    191    1.2    bouyer }
    192    1.2    bouyer 
    193   1.56    jruoho static void
    194   1.10    cegger cpu_attach(device_t parent, device_t self, void *aux)
    195    1.2    bouyer {
    196   1.10    cegger 	struct cpu_softc *sc = device_private(self);
    197    1.2    bouyer 	struct cpu_attach_args *caa = aux;
    198    1.2    bouyer 	struct cpu_info *ci;
    199   1.34    cegger 	uintptr_t ptr;
    200   1.52    bouyer 	static int nphycpu = 0;
    201    1.2    bouyer 
    202   1.10    cegger 	sc->sc_dev = self;
    203   1.10    cegger 
    204    1.2    bouyer 	/*
    205    1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    206   1.52    bouyer 	 * If we're the first attached CPU use the primary cpu_info,
    207   1.52    bouyer 	 * otherwise allocate a new one
    208    1.2    bouyer 	 */
    209   1.52    bouyer 	aprint_naive("\n");
    210   1.52    bouyer 	aprint_normal("\n");
    211   1.52    bouyer 	if (nphycpu > 0) {
    212   1.52    bouyer 		struct cpu_info *tmp;
    213   1.34    cegger 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    214   1.34    cegger 		    KM_SLEEP);
    215   1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    216   1.24        ad 		ci->ci_curldt = -1;
    217   1.52    bouyer 
    218   1.52    bouyer 		tmp = phycpu_info_list;
    219   1.52    bouyer 		while (tmp->ci_next)
    220   1.52    bouyer 			tmp = tmp->ci_next;
    221   1.52    bouyer 
    222   1.52    bouyer 		tmp->ci_next = ci;
    223    1.2    bouyer 	} else {
    224    1.2    bouyer 		ci = &phycpu_info_primary;
    225    1.2    bouyer 	}
    226    1.2    bouyer 
    227    1.2    bouyer 	ci->ci_self = ci;
    228    1.2    bouyer 	sc->sc_info = ci;
    229    1.2    bouyer 
    230    1.2    bouyer 	ci->ci_dev = self;
    231   1.50    jruoho 	ci->ci_acpiid = caa->cpu_id;
    232   1.23        ad 	ci->ci_cpuid = caa->cpu_number;
    233   1.16    cegger 	ci->ci_vcpu = NULL;
    234   1.52    bouyer 	ci->ci_index = nphycpu++;
    235    1.2    bouyer 
    236   1.52    bouyer 	if (!pmf_device_register(self, NULL, NULL))
    237   1.52    bouyer 		aprint_error_dev(self, "couldn't establish power handler\n");
    238   1.34    cegger 
    239   1.56    jruoho 	(void)config_defer(self, cpu_defer);
    240   1.56    jruoho }
    241   1.56    jruoho 
    242   1.56    jruoho static void
    243   1.56    jruoho cpu_defer(device_t self)
    244   1.56    jruoho {
    245   1.56    jruoho 	cpu_rescan(self, NULL, NULL);
    246    1.2    bouyer }
    247    1.2    bouyer 
    248   1.56    jruoho static int
    249   1.53    jruoho cpu_rescan(device_t self, const char *ifattr, const int *locators)
    250   1.53    jruoho {
    251   1.53    jruoho 	struct cpu_softc *sc = device_private(self);
    252   1.53    jruoho 	struct cpufeature_attach_args cfaa;
    253   1.53    jruoho 	struct cpu_info *ci = sc->sc_info;
    254   1.53    jruoho 
    255   1.53    jruoho 	memset(&cfaa, 0, sizeof(cfaa));
    256   1.53    jruoho 	cfaa.ci = ci;
    257   1.53    jruoho 
    258   1.53    jruoho 	if (ifattr_match(ifattr, "cpufeaturebus")) {
    259   1.53    jruoho 
    260   1.53    jruoho 		if (ci->ci_frequency == NULL) {
    261   1.55    jruoho 			cfaa.name = "frequency";
    262   1.54    jruoho 			ci->ci_frequency = config_found_ia(self,
    263   1.54    jruoho 			    "cpufeaturebus", &cfaa, NULL);
    264   1.54    jruoho 		}
    265   1.53    jruoho 	}
    266   1.53    jruoho 
    267   1.53    jruoho 	return 0;
    268   1.53    jruoho }
    269   1.53    jruoho 
    270   1.56    jruoho static void
    271   1.53    jruoho cpu_childdetached(device_t self, device_t child)
    272   1.53    jruoho {
    273   1.53    jruoho 	struct cpu_softc *sc = device_private(self);
    274   1.53    jruoho 	struct cpu_info *ci = sc->sc_info;
    275   1.53    jruoho 
    276   1.53    jruoho 	if (ci->ci_frequency == child)
    277   1.53    jruoho 		ci->ci_frequency = NULL;
    278   1.53    jruoho }
    279   1.53    jruoho 
    280   1.56    jruoho static int
    281   1.10    cegger vcpu_match(device_t parent, cfdata_t match, void *aux)
    282    1.2    bouyer {
    283    1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    284   1.62    cherry 	struct vcpu_runstate_info vcr;
    285   1.62    cherry 	int error;
    286   1.62    cherry 
    287   1.62    cherry 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) {
    288   1.62    cherry 		error = HYPERVISOR_vcpu_op(VCPUOP_get_runstate_info,
    289  1.105      maxv 		    vcaa->vcaa_caa.cpu_number, &vcr);
    290   1.62    cherry 		switch (error) {
    291   1.62    cherry 		case 0:
    292   1.62    cherry 			return 1;
    293   1.62    cherry 		case -ENOENT:
    294   1.62    cherry 			return 0;
    295   1.62    cherry 		default:
    296   1.62    cherry 			panic("Unknown hypervisor error %d returned on vcpu runstate probe\n", error);
    297   1.62    cherry 		}
    298   1.62    cherry 	}
    299    1.2    bouyer 
    300    1.2    bouyer 	return 0;
    301    1.2    bouyer }
    302    1.2    bouyer 
    303   1.56    jruoho static void
    304   1.10    cegger vcpu_attach(device_t parent, device_t self, void *aux)
    305    1.2    bouyer {
    306    1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    307    1.2    bouyer 
    308   1.62    cherry 	KASSERT(vcaa->vcaa_caa.cpu_func == NULL);
    309   1.62    cherry 	vcaa->vcaa_caa.cpu_func = &mp_cpu_funcs;
    310    1.2    bouyer 	cpu_attach_common(parent, self, &vcaa->vcaa_caa);
    311   1.65       jym 
    312   1.65       jym 	if (!pmf_device_register(self, NULL, NULL))
    313   1.65       jym 		aprint_error_dev(self, "couldn't establish power handler\n");
    314    1.2    bouyer }
    315    1.2    bouyer 
    316   1.62    cherry static int
    317   1.62    cherry vcpu_is_up(struct cpu_info *ci)
    318   1.62    cherry {
    319   1.62    cherry 	KASSERT(ci != NULL);
    320   1.62    cherry 	return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_cpuid, NULL);
    321   1.62    cherry }
    322   1.62    cherry 
    323    1.2    bouyer static void
    324    1.2    bouyer cpu_vm_init(struct cpu_info *ci)
    325    1.2    bouyer {
    326    1.2    bouyer 	int ncolors = 2, i;
    327    1.2    bouyer 
    328    1.2    bouyer 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    329    1.2    bouyer 		struct x86_cache_info *cai;
    330    1.2    bouyer 		int tcolors;
    331    1.2    bouyer 
    332    1.2    bouyer 		cai = &ci->ci_cinfo[i];
    333    1.2    bouyer 
    334    1.2    bouyer 		tcolors = atop(cai->cai_totalsize);
    335  1.105      maxv 		switch (cai->cai_associativity) {
    336    1.2    bouyer 		case 0xff:
    337    1.2    bouyer 			tcolors = 1; /* fully associative */
    338    1.2    bouyer 			break;
    339    1.2    bouyer 		case 0:
    340    1.2    bouyer 		case 1:
    341    1.2    bouyer 			break;
    342    1.2    bouyer 		default:
    343    1.2    bouyer 			tcolors /= cai->cai_associativity;
    344    1.2    bouyer 		}
    345    1.2    bouyer 		ncolors = max(ncolors, tcolors);
    346    1.2    bouyer 	}
    347    1.2    bouyer 
    348    1.2    bouyer 	/*
    349   1.67       mrg 	 * Knowing the size of the largest cache on this CPU, potentially
    350   1.67       mrg 	 * re-color our pages.
    351    1.2    bouyer 	 */
    352   1.28    bouyer 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    353    1.2    bouyer 	uvm_page_recolor(ncolors);
    354   1.91     rmind 	pmap_tlb_cpu_init(ci);
    355  1.109      maxv #ifndef __HAVE_DIRECT_MAP
    356  1.109      maxv 	pmap_vpage_cpu_init(ci);
    357  1.109      maxv #endif
    358    1.2    bouyer }
    359    1.2    bouyer 
    360   1.56    jruoho static void
    361   1.11    cegger cpu_attach_common(device_t parent, device_t self, void *aux)
    362    1.2    bouyer {
    363   1.10    cegger 	struct cpu_softc *sc = device_private(self);
    364    1.2    bouyer 	struct cpu_attach_args *caa = aux;
    365    1.2    bouyer 	struct cpu_info *ci;
    366   1.12    cegger 	uintptr_t ptr;
    367    1.2    bouyer 	int cpunum = caa->cpu_number;
    368   1.38    cegger 	static bool again = false;
    369    1.2    bouyer 
    370   1.10    cegger 	sc->sc_dev = self;
    371   1.10    cegger 
    372    1.2    bouyer 	/*
    373    1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    374    1.2    bouyer 	 * structure, otherwise use the primary's.
    375    1.2    bouyer 	 */
    376    1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    377   1.12    cegger 		aprint_naive(": Application Processor\n");
    378   1.31    cegger 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    379   1.31    cegger 		    KM_SLEEP);
    380   1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    381   1.12    cegger 		memset(ci, 0, sizeof(*ci));
    382  1.117    bouyer 		cpu_init_tss(ci);
    383    1.2    bouyer 	} else {
    384   1.12    cegger 		aprint_naive(": %s Processor\n",
    385   1.12    cegger 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    386    1.2    bouyer 		ci = &cpu_info_primary;
    387    1.2    bouyer 	}
    388    1.2    bouyer 
    389    1.2    bouyer 	ci->ci_self = ci;
    390    1.2    bouyer 	sc->sc_info = ci;
    391    1.2    bouyer 	ci->ci_dev = self;
    392   1.23        ad 	ci->ci_cpuid = cpunum;
    393   1.16    cegger 
    394   1.16    cegger 	KASSERT(HYPERVISOR_shared_info != NULL);
    395   1.89    bouyer 	KASSERT(cpunum < XEN_LEGACY_MAX_VCPUS);
    396   1.16    cegger 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
    397   1.16    cegger 
    398   1.62    cherry 	KASSERT(ci->ci_func == 0);
    399    1.2    bouyer 	ci->ci_func = caa->cpu_func;
    400  1.101   msaitoh 	aprint_normal("\n");
    401    1.2    bouyer 
    402   1.38    cegger 	/* Must be called before mi_cpu_attach(). */
    403   1.38    cegger 	cpu_vm_init(ci);
    404   1.38    cegger 
    405    1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    406    1.2    bouyer 		int error;
    407    1.2    bouyer 
    408    1.2    bouyer 		error = mi_cpu_attach(ci);
    409   1.62    cherry 
    410   1.62    cherry 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    411    1.2    bouyer 		if (error != 0) {
    412   1.38    cegger 			aprint_error_dev(self,
    413   1.38    cegger 			    "mi_cpu_attach failed with %d\n", error);
    414    1.2    bouyer 			return;
    415    1.2    bouyer 		}
    416   1.62    cherry 
    417    1.2    bouyer 	} else {
    418    1.2    bouyer 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    419    1.2    bouyer 	}
    420    1.2    bouyer 
    421   1.89    bouyer 	KASSERT(ci->ci_cpuid == ci->ci_index);
    422  1.100    bouyer #ifdef __x86_64__
    423  1.100    bouyer 	/* No user PGD mapped for this CPU yet */
    424  1.100    bouyer 	ci->ci_xen_current_user_pgd = 0;
    425  1.100    bouyer #endif
    426  1.100    bouyer #if defined(__x86_64__) || defined(PAE)
    427  1.100    bouyer 	mutex_init(&ci->ci_kpm_mtx, MUTEX_DEFAULT, IPL_VM);
    428  1.100    bouyer #endif
    429    1.2    bouyer 	pmap_reference(pmap_kernel());
    430    1.2    bouyer 	ci->ci_pmap = pmap_kernel();
    431    1.2    bouyer 	ci->ci_tlbstate = TLBSTATE_STALE;
    432    1.2    bouyer 
    433   1.38    cegger 	/*
    434   1.38    cegger 	 * Boot processor may not be attached first, but the below
    435   1.38    cegger 	 * must be done to allow booting other processors.
    436   1.38    cegger 	 */
    437   1.38    cegger 	if (!again) {
    438   1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    439   1.38    cegger 		/* Basic init. */
    440   1.38    cegger 		cpu_intr_init(ci);
    441   1.38    cegger 		cpu_get_tsc_freq(ci);
    442   1.38    cegger 		cpu_init(ci);
    443   1.78    cherry 		pmap_cpu_init_late(ci);
    444   1.62    cherry 
    445   1.99       snj 		/* Every processor needs to init its own ipi h/w (similar to lapic) */
    446   1.62    cherry 		xen_ipi_init();
    447   1.62    cherry 
    448   1.38    cegger 		/* Make sure DELAY() is initialized. */
    449   1.38    cegger 		DELAY(1);
    450   1.38    cegger 		again = true;
    451   1.38    cegger 	}
    452   1.38    cegger 
    453    1.2    bouyer 	/* further PCB init done later. */
    454    1.2    bouyer 
    455    1.2    bouyer 	switch (caa->cpu_role) {
    456    1.2    bouyer 	case CPU_ROLE_SP:
    457   1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    458   1.21        ad 		cpu_identify(ci);
    459   1.38    cegger 		x86_cpu_idle_init();
    460    1.2    bouyer 		break;
    461    1.2    bouyer 
    462    1.2    bouyer 	case CPU_ROLE_BP:
    463   1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    464   1.21        ad 		cpu_identify(ci);
    465   1.38    cegger 		x86_cpu_idle_init();
    466    1.2    bouyer 		break;
    467    1.2    bouyer 
    468    1.2    bouyer 	case CPU_ROLE_AP:
    469   1.62    cherry 		atomic_or_32(&ci->ci_flags, CPUF_AP);
    470   1.62    cherry 
    471    1.2    bouyer 		/*
    472    1.2    bouyer 		 * report on an AP
    473    1.2    bouyer 		 */
    474    1.2    bouyer 
    475    1.2    bouyer #if defined(MULTIPROCESSOR)
    476   1.62    cherry 		/* interrupt handler stack */
    477    1.2    bouyer 		cpu_intr_init(ci);
    478   1.62    cherry 
    479   1.62    cherry 		/* Setup per-cpu memory for gdt */
    480    1.2    bouyer 		gdt_alloc_cpu(ci);
    481   1.62    cherry 
    482   1.62    cherry 		pmap_cpu_init_late(ci);
    483    1.2    bouyer 		cpu_start_secondary(ci);
    484   1.62    cherry 
    485    1.2    bouyer 		if (ci->ci_flags & CPUF_PRESENT) {
    486   1.30    cegger 			struct cpu_info *tmp;
    487   1.30    cegger 
    488   1.62    cherry 			cpu_identify(ci);
    489   1.30    cegger 			tmp = cpu_info_list;
    490   1.30    cegger 			while (tmp->ci_next)
    491   1.30    cegger 				tmp = tmp->ci_next;
    492   1.30    cegger 
    493   1.30    cegger 			tmp->ci_next = ci;
    494    1.2    bouyer 		}
    495    1.2    bouyer #else
    496  1.101   msaitoh 		aprint_error_dev(ci->ci_dev, "not started\n");
    497    1.2    bouyer #endif
    498    1.2    bouyer 		break;
    499    1.2    bouyer 
    500    1.2    bouyer 	default:
    501    1.2    bouyer 		panic("unknown processor type??\n");
    502    1.2    bouyer 	}
    503    1.2    bouyer 
    504   1.62    cherry #ifdef MPVERBOSE
    505    1.2    bouyer 	if (mp_verbose) {
    506    1.2    bouyer 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    507   1.37     rmind 		struct pcb *pcb = lwp_getpcb(l);
    508    1.2    bouyer 
    509   1.38    cegger 		aprint_verbose_dev(self,
    510   1.38    cegger 		    "idle lwp at %p, idle sp at 0x%p\n",
    511   1.12    cegger 		    l,
    512   1.12    cegger #ifdef i386
    513   1.37     rmind 		    (void *)pcb->pcb_esp
    514  1.105      maxv #else
    515   1.37     rmind 		    (void *)pcb->pcb_rsp
    516  1.105      maxv #endif
    517   1.12    cegger 		);
    518   1.12    cegger 
    519    1.2    bouyer 	}
    520   1.62    cherry #endif /* MPVERBOSE */
    521    1.2    bouyer }
    522    1.2    bouyer 
    523    1.2    bouyer /*
    524    1.2    bouyer  * Initialize the processor appropriately.
    525    1.2    bouyer  */
    526    1.2    bouyer 
    527    1.2    bouyer void
    528   1.10    cegger cpu_init(struct cpu_info *ci)
    529    1.2    bouyer {
    530  1.122  jdolecek 	uint32_t cr4 = 0;
    531    1.2    bouyer 
    532    1.2    bouyer 	/*
    533    1.2    bouyer 	 * If we have FXSAVE/FXRESTOR, use them.
    534    1.2    bouyer 	 */
    535   1.43       jym 	if (cpu_feature[0] & CPUID_FXSR) {
    536  1.122  jdolecek 		cr4 |= CR4_OSFXSR;
    537    1.2    bouyer 
    538    1.2    bouyer 		/*
    539    1.2    bouyer 		 * If we have SSE/SSE2, enable XMM exceptions.
    540    1.2    bouyer 		 */
    541   1.43       jym 		if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
    542  1.122  jdolecek 			cr4 |= CR4_OSXMMEXCPT;
    543  1.122  jdolecek 	}
    544  1.122  jdolecek 
    545  1.122  jdolecek 	/* If xsave is supported, enable it */
    546  1.122  jdolecek 	if (cpu_feature[1] & CPUID2_XSAVE && x86_fpu_save >= FPU_SAVE_XSAVE)
    547  1.122  jdolecek 		cr4 |= CR4_OSXSAVE;
    548  1.122  jdolecek 
    549  1.122  jdolecek 	if (cr4) {
    550  1.122  jdolecek 		cr4 |= rcr4();
    551  1.122  jdolecek 		lcr4(cr4);
    552    1.2    bouyer 	}
    553    1.2    bouyer 
    554  1.116      maxv 	if (x86_fpu_save >= FPU_SAVE_FXSAVE) {
    555  1.120      maxv 		fpuinit_mxcsr_mask();
    556  1.118  jdolecek 	}
    557  1.118  jdolecek 
    558  1.122  jdolecek 	/*
    559  1.122  jdolecek 	 * Changing CR4 register may change cpuid values. For example, setting
    560  1.122  jdolecek 	 * CR4_OSXSAVE sets CPUID2_OSXSAVE. The CPUID2_OSXSAVE is in
    561  1.122  jdolecek 	 * ci_feat_val[1], so update it.
    562  1.122  jdolecek 	 * XXX Other than ci_feat_val[1] might be changed.
    563  1.122  jdolecek 	 */
    564  1.122  jdolecek 	if (cpuid_level >= 1) {
    565  1.122  jdolecek 		u_int descs[4];
    566  1.122  jdolecek 
    567  1.122  jdolecek 		x86_cpuid(1, descs);
    568  1.122  jdolecek 		ci->ci_feat_val[1] = descs[2];
    569  1.122  jdolecek 	}
    570  1.122  jdolecek 
    571  1.122  jdolecek 	/* If xsave is enabled, enable all fpu features */
    572  1.122  jdolecek 	if (cr4 & CR4_OSXSAVE) {
    573  1.122  jdolecek 		wrxcr(0, x86_xsave_features & XCR0_FPU);
    574  1.122  jdolecek 	}
    575  1.122  jdolecek 
    576   1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    577    1.2    bouyer }
    578    1.2    bouyer 
    579    1.2    bouyer 
    580    1.2    bouyer #ifdef MULTIPROCESSOR
    581   1.62    cherry 
    582    1.2    bouyer void
    583   1.10    cegger cpu_boot_secondary_processors(void)
    584    1.2    bouyer {
    585    1.2    bouyer 	struct cpu_info *ci;
    586    1.2    bouyer 	u_long i;
    587   1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    588   1.38    cegger 		ci = cpu_lookup(i);
    589    1.2    bouyer 		if (ci == NULL)
    590    1.2    bouyer 			continue;
    591    1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    592    1.2    bouyer 			continue;
    593    1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    594    1.2    bouyer 			continue;
    595    1.2    bouyer 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    596    1.2    bouyer 			continue;
    597    1.2    bouyer 		cpu_boot_secondary(ci);
    598    1.2    bouyer 	}
    599   1.11    cegger 
    600   1.11    cegger 	x86_mp_online = true;
    601    1.2    bouyer }
    602    1.2    bouyer 
    603    1.2    bouyer static void
    604    1.2    bouyer cpu_init_idle_lwp(struct cpu_info *ci)
    605    1.2    bouyer {
    606    1.2    bouyer 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    607   1.37     rmind 	struct pcb *pcb = lwp_getpcb(l);
    608    1.2    bouyer 
    609    1.2    bouyer 	pcb->pcb_cr0 = rcr0();
    610    1.2    bouyer }
    611    1.2    bouyer 
    612    1.2    bouyer void
    613   1.10    cegger cpu_init_idle_lwps(void)
    614    1.2    bouyer {
    615    1.2    bouyer 	struct cpu_info *ci;
    616    1.2    bouyer 	u_long i;
    617    1.2    bouyer 
    618   1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    619   1.38    cegger 		ci = cpu_lookup(i);
    620    1.2    bouyer 		if (ci == NULL)
    621    1.2    bouyer 			continue;
    622    1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    623    1.2    bouyer 			continue;
    624    1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    625    1.2    bouyer 			continue;
    626    1.2    bouyer 		cpu_init_idle_lwp(ci);
    627    1.2    bouyer 	}
    628    1.2    bouyer }
    629    1.2    bouyer 
    630   1.62    cherry static void
    631   1.10    cegger cpu_start_secondary(struct cpu_info *ci)
    632    1.2    bouyer {
    633    1.2    bouyer 	int i;
    634    1.2    bouyer 
    635   1.11    cegger 	aprint_debug_dev(ci->ci_dev, "starting\n");
    636    1.2    bouyer 
    637    1.2    bouyer 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    638   1.62    cherry 
    639   1.62    cherry 	if (CPU_STARTUP(ci, (vaddr_t) cpu_hatch) != 0) {
    640   1.11    cegger 		return;
    641   1.62    cherry 	}
    642    1.2    bouyer 
    643    1.2    bouyer 	/*
    644    1.2    bouyer 	 * wait for it to become ready
    645    1.2    bouyer 	 */
    646   1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    647    1.2    bouyer 		delay(10);
    648    1.2    bouyer 	}
    649   1.11    cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    650    1.9    cegger 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    651    1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    652    1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    653    1.2    bouyer 		Debugger();
    654    1.2    bouyer #endif
    655    1.2    bouyer 	}
    656    1.2    bouyer 
    657    1.2    bouyer 	CPU_START_CLEANUP(ci);
    658    1.2    bouyer }
    659    1.2    bouyer 
    660    1.2    bouyer void
    661   1.10    cegger cpu_boot_secondary(struct cpu_info *ci)
    662    1.2    bouyer {
    663    1.2    bouyer 	int i;
    664   1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    665   1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    666    1.2    bouyer 		delay(10);
    667    1.2    bouyer 	}
    668   1.11    cegger 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    669   1.11    cegger 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
    670    1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    671    1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    672    1.2    bouyer 		Debugger();
    673    1.2    bouyer #endif
    674    1.2    bouyer 	}
    675    1.2    bouyer }
    676    1.2    bouyer 
    677    1.2    bouyer /*
    678   1.62    cherry  * APs end up here immediately after initialisation and VCPUOP_up in
    679   1.62    cherry  * mp_cpu_start().
    680   1.62    cherry  * At this point, we are running in the idle pcb/idle stack of the new
    681   1.62    cherry  * CPU.  This function jumps to the idle loop and starts looking for
    682   1.62    cherry  * work.
    683    1.2    bouyer  */
    684   1.62    cherry extern void x86_64_tls_switch(struct lwp *);
    685    1.2    bouyer void
    686    1.2    bouyer cpu_hatch(void *v)
    687    1.2    bouyer {
    688    1.2    bouyer 	struct cpu_info *ci = (struct cpu_info *)v;
    689   1.37     rmind 	struct pcb *pcb;
    690   1.11    cegger 	int s, i;
    691   1.11    cegger 
    692   1.62    cherry 	/* Setup TLS and kernel GS/FS */
    693   1.62    cherry 	cpu_init_msrs(ci, true);
    694   1.62    cherry 	cpu_init_idt();
    695   1.62    cherry 	gdt_init_cpu(ci);
    696   1.62    cherry 
    697   1.21        ad 	cpu_probe(ci);
    698   1.11    cegger 
    699   1.62    cherry 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    700    1.2    bouyer 
    701   1.11    cegger 	while ((ci->ci_flags & CPUF_GO) == 0) {
    702   1.11    cegger 		/* Don't use delay, boot CPU may be patching the text. */
    703   1.11    cegger 		for (i = 10000; i != 0; i--)
    704   1.11    cegger 			x86_pause();
    705   1.11    cegger 	}
    706    1.2    bouyer 
    707   1.11    cegger 	/* Because the text may have been patched in x86_patch(). */
    708   1.11    cegger 	x86_flush();
    709   1.58     rmind 	tlbflushg();
    710    1.2    bouyer 
    711   1.11    cegger 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    712    1.2    bouyer 
    713   1.37     rmind 	pcb = lwp_getpcb(curlwp);
    714   1.85    cherry 	pcb->pcb_cr3 = pmap_pdirpa(pmap_kernel(), 0);
    715   1.37     rmind 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
    716   1.37     rmind 
    717   1.62    cherry 	xen_ipi_init();
    718   1.62    cherry 
    719   1.62    cherry 	xen_initclocks();
    720  1.105      maxv 
    721   1.62    cherry #ifdef __x86_64__
    722   1.12    cegger 	fpuinit(ci);
    723   1.12    cegger #endif
    724    1.2    bouyer 
    725    1.2    bouyer 	lldt(GSEL(GLDT_SEL, SEL_KPL));
    726    1.2    bouyer 
    727    1.2    bouyer 	cpu_init(ci);
    728   1.11    cegger 	cpu_get_tsc_freq(ci);
    729    1.2    bouyer 
    730    1.2    bouyer 	s = splhigh();
    731   1.11    cegger 	x86_enable_intr();
    732   1.11    cegger 	splx(s);
    733    1.2    bouyer 
    734   1.62    cherry 	aprint_debug_dev(ci->ci_dev, "running\n");
    735   1.62    cherry 
    736   1.62    cherry 	cpu_switchto(NULL, ci->ci_data.cpu_idlelwp, true);
    737   1.62    cherry 
    738   1.91     rmind 	idle_loop(NULL);
    739   1.91     rmind 	KASSERT(false);
    740    1.2    bouyer }
    741    1.2    bouyer 
    742    1.2    bouyer #if defined(DDB)
    743    1.2    bouyer 
    744    1.2    bouyer #include <ddb/db_output.h>
    745    1.2    bouyer #include <machine/db_machdep.h>
    746    1.2    bouyer 
    747    1.2    bouyer /*
    748    1.2    bouyer  * Dump CPU information from ddb.
    749    1.2    bouyer  */
    750    1.2    bouyer void
    751    1.2    bouyer cpu_debug_dump(void)
    752    1.2    bouyer {
    753    1.2    bouyer 	struct cpu_info *ci;
    754    1.2    bouyer 	CPU_INFO_ITERATOR cii;
    755    1.2    bouyer 
    756   1.95  christos 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    757    1.2    bouyer 	for (CPU_INFO_FOREACH(cii, ci)) {
    758   1.95  christos 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    759    1.2    bouyer 		    ci,
    760    1.9    cegger 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    761   1.12    cegger 		    (long)ci->ci_cpuid,
    762    1.2    bouyer 		    ci->ci_flags, ci->ci_ipis,
    763   1.95  christos 		    ci->ci_curlwp,
    764   1.95  christos 		    ci->ci_fpcurlwp);
    765    1.2    bouyer 	}
    766    1.2    bouyer }
    767   1.38    cegger #endif /* DDB */
    768    1.2    bouyer 
    769   1.62    cherry #endif /* MULTIPROCESSOR */
    770   1.62    cherry 
    771   1.62    cherry extern void hypervisor_callback(void);
    772   1.62    cherry extern void failsafe_callback(void);
    773   1.62    cherry #ifdef __x86_64__
    774   1.62    cherry typedef void (vector)(void);
    775   1.62    cherry extern vector Xsyscall, Xsyscall32;
    776   1.62    cherry #endif
    777   1.62    cherry 
    778   1.62    cherry /*
    779   1.62    cherry  * Setup the "trampoline". On Xen, we setup nearly all cpu context
    780   1.62    cherry  * outside a trampoline, so we prototype and call targetip like so:
    781   1.62    cherry  * void targetip(struct cpu_info *);
    782   1.62    cherry  */
    783   1.62    cherry 
    784    1.2    bouyer static void
    785   1.62    cherry gdt_prepframes(paddr_t *frames, vaddr_t base, uint32_t entries)
    786    1.2    bouyer {
    787  1.104   msaitoh 	int i;
    788  1.111    bouyer 	for (i = 0; i < entries; i++) {
    789  1.105      maxv 		frames[i] = ((paddr_t)xpmap_ptetomach(
    790  1.105      maxv 		    (pt_entry_t *)(base + (i << PAGE_SHIFT)))) >> PAGE_SHIFT;
    791   1.62    cherry 
    792   1.62    cherry 		/* Mark Read-only */
    793   1.62    cherry 		pmap_pte_clearbits(kvtopte(base + (i << PAGE_SHIFT)),
    794   1.62    cherry 		    PG_RW);
    795   1.62    cherry 	}
    796   1.62    cherry }
    797   1.62    cherry 
    798   1.62    cherry #ifdef __x86_64__
    799   1.85    cherry extern char *ldtstore;
    800   1.62    cherry 
    801   1.62    cherry static void
    802  1.105      maxv xen_init_amd64_vcpuctxt(struct cpu_info *ci, struct vcpu_guest_context *initctx,
    803  1.105      maxv     void targetrip(struct cpu_info *))
    804   1.62    cherry {
    805   1.62    cherry 	/* page frames to point at GDT */
    806   1.62    cherry 	extern int gdt_size;
    807   1.62    cherry 	paddr_t frames[16];
    808   1.62    cherry 	psize_t gdt_ents;
    809   1.62    cherry 
    810   1.62    cherry 	struct lwp *l;
    811   1.62    cherry 	struct pcb *pcb;
    812   1.62    cherry 
    813   1.62    cherry 	volatile struct vcpu_info *vci;
    814   1.62    cherry 
    815   1.62    cherry 	KASSERT(ci != NULL);
    816   1.62    cherry 	KASSERT(ci != &cpu_info_primary);
    817   1.62    cherry 	KASSERT(initctx != NULL);
    818   1.62    cherry 	KASSERT(targetrip != NULL);
    819   1.62    cherry 
    820  1.105      maxv 	memset(initctx, 0, sizeof(*initctx));
    821   1.62    cherry 
    822  1.104   msaitoh 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT;
    823   1.62    cherry 	KASSERT(gdt_ents <= 16);
    824   1.62    cherry 
    825  1.105      maxv 	gdt_prepframes(frames, (vaddr_t)ci->ci_gdt, gdt_ents);
    826   1.62    cherry 
    827   1.62    cherry 	/* Initialise the vcpu context: We use idle_loop()'s pcb context. */
    828   1.11    cegger 
    829   1.62    cherry 	l = ci->ci_data.cpu_idlelwp;
    830   1.11    cegger 
    831   1.62    cherry 	KASSERT(l != NULL);
    832   1.62    cherry 	pcb = lwp_getpcb(l);
    833   1.62    cherry 	KASSERT(pcb != NULL);
    834   1.11    cegger 
    835   1.62    cherry 	/* resume with interrupts off */
    836   1.62    cherry 	vci = ci->ci_vcpu;
    837   1.62    cherry 	vci->evtchn_upcall_mask = 1;
    838   1.62    cherry 	xen_mb();
    839    1.2    bouyer 
    840   1.62    cherry 	/* resume in kernel-mode */
    841   1.62    cherry 	initctx->flags = VGCF_in_kernel | VGCF_online;
    842    1.2    bouyer 
    843   1.62    cherry 	/* Stack and entry points:
    844   1.62    cherry 	 * We arrange for the stack frame for cpu_hatch() to
    845   1.62    cherry 	 * appear as a callee frame of lwp_trampoline(). Being a
    846   1.62    cherry 	 * leaf frame prevents trampling on any of the MD stack setup
    847   1.62    cherry 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    848   1.62    cherry 	 */
    849    1.2    bouyer 
    850   1.62    cherry 	initctx->user_regs.rdi = (uint64_t) ci; /* targetrip(ci); */
    851   1.62    cherry 	initctx->user_regs.rip = (vaddr_t) targetrip;
    852    1.2    bouyer 
    853   1.62    cherry 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    854   1.11    cegger 
    855   1.62    cherry 	initctx->user_regs.rflags = pcb->pcb_flags;
    856   1.62    cherry 	initctx->user_regs.rsp = pcb->pcb_rsp;
    857   1.11    cegger 
    858   1.62    cherry 	/* Data segments */
    859   1.62    cherry 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
    860   1.62    cherry 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
    861   1.62    cherry 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
    862   1.62    cherry 
    863   1.62    cherry 	/* GDT */
    864  1.105      maxv 	memcpy(initctx->gdt_frames, frames, sizeof(frames));
    865   1.62    cherry 	initctx->gdt_ents = gdt_ents;
    866   1.62    cherry 
    867   1.62    cherry 	/* LDT */
    868  1.105      maxv 	initctx->ldt_base = (unsigned long)ldtstore;
    869   1.62    cherry 	initctx->ldt_ents = LDT_SIZE >> 3;
    870   1.62    cherry 
    871   1.62    cherry 	/* Kernel context state */
    872   1.62    cherry 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
    873   1.62    cherry 	initctx->kernel_sp = pcb->pcb_rsp0;
    874   1.62    cherry 	initctx->ctrlreg[0] = pcb->pcb_cr0;
    875   1.62    cherry 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
    876  1.105      maxv 	initctx->ctrlreg[2] = (vaddr_t)targetrip;
    877  1.105      maxv 	/*
    878   1.62    cherry 	 * Use pmap_kernel() L4 PD directly, until we setup the
    879   1.62    cherry 	 * per-cpu L4 PD in pmap_cpu_init_late()
    880    1.2    bouyer 	 */
    881   1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_kpm_pdirpa)));
    882   1.62    cherry 	initctx->ctrlreg[4] = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT;
    883    1.2    bouyer 
    884   1.62    cherry 	/* Xen callbacks */
    885  1.105      maxv 	initctx->event_callback_eip = (unsigned long)hypervisor_callback;
    886  1.105      maxv 	initctx->failsafe_callback_eip = (unsigned long)failsafe_callback;
    887  1.105      maxv 	initctx->syscall_callback_eip = (unsigned long)Xsyscall;
    888   1.62    cherry 
    889   1.62    cherry 	return;
    890    1.2    bouyer }
    891   1.62    cherry #else /* i386 */
    892  1.108      maxv extern union descriptor *ldtstore;
    893   1.62    cherry extern void Xsyscall(void);
    894   1.62    cherry 
    895   1.11    cegger static void
    896  1.105      maxv xen_init_i386_vcpuctxt(struct cpu_info *ci, struct vcpu_guest_context *initctx,
    897  1.105      maxv     void targeteip(struct cpu_info *))
    898   1.62    cherry {
    899   1.62    cherry 	/* page frames to point at GDT */
    900   1.62    cherry 	extern int gdt_size;
    901   1.62    cherry 	paddr_t frames[16];
    902   1.62    cherry 	psize_t gdt_ents;
    903   1.62    cherry 
    904   1.62    cherry 	struct lwp *l;
    905   1.62    cherry 	struct pcb *pcb;
    906   1.62    cherry 
    907   1.62    cherry 	volatile struct vcpu_info *vci;
    908   1.62    cherry 
    909   1.62    cherry 	KASSERT(ci != NULL);
    910   1.62    cherry 	KASSERT(ci != &cpu_info_primary);
    911   1.62    cherry 	KASSERT(initctx != NULL);
    912   1.62    cherry 	KASSERT(targeteip != NULL);
    913   1.62    cherry 
    914  1.105      maxv 	memset(initctx, 0, sizeof(*initctx));
    915   1.11    cegger 
    916   1.85    cherry 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT;
    917   1.62    cherry 	KASSERT(gdt_ents <= 16);
    918    1.2    bouyer 
    919  1.105      maxv 	gdt_prepframes(frames, (vaddr_t)ci->ci_gdt, gdt_ents);
    920    1.2    bouyer 
    921   1.62    cherry 	/*
    922   1.62    cherry 	 * Initialise the vcpu context:
    923   1.62    cherry 	 * We use this cpu's idle_loop() pcb context.
    924   1.11    cegger 	 */
    925   1.11    cegger 
    926   1.62    cherry 	l = ci->ci_data.cpu_idlelwp;
    927   1.62    cherry 
    928   1.62    cherry 	KASSERT(l != NULL);
    929   1.62    cherry 	pcb = lwp_getpcb(l);
    930   1.62    cherry 	KASSERT(pcb != NULL);
    931   1.62    cherry 
    932   1.62    cherry 	/* resume with interrupts off */
    933   1.62    cherry 	vci = ci->ci_vcpu;
    934   1.62    cherry 	vci->evtchn_upcall_mask = 1;
    935   1.62    cherry 	xen_mb();
    936   1.62    cherry 
    937   1.62    cherry 	/* resume in kernel-mode */
    938   1.62    cherry 	initctx->flags = VGCF_in_kernel | VGCF_online;
    939   1.62    cherry 
    940   1.62    cherry 	/* Stack frame setup for cpu_hatch():
    941   1.62    cherry 	 * We arrange for the stack frame for cpu_hatch() to
    942   1.62    cherry 	 * appear as a callee frame of lwp_trampoline(). Being a
    943   1.62    cherry 	 * leaf frame prevents trampling on any of the MD stack setup
    944   1.62    cherry 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    945    1.2    bouyer 	 */
    946    1.2    bouyer 
    947   1.62    cherry 	initctx->user_regs.esp = pcb->pcb_esp - 4; /* Leave word for
    948   1.62    cherry 						      arg1 */
    949  1.105      maxv 	{
    950  1.105      maxv 		/* targeteip(ci); */
    951  1.105      maxv 		uint32_t *arg = (uint32_t *)initctx->user_regs.esp;
    952  1.105      maxv 		arg[1] = (uint32_t)ci; /* arg1 */
    953   1.62    cherry 	}
    954    1.2    bouyer 
    955  1.105      maxv 	initctx->user_regs.eip = (vaddr_t)targeteip;
    956   1.62    cherry 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    957   1.62    cherry 	initctx->user_regs.eflags |= pcb->pcb_iopl;
    958   1.62    cherry 
    959   1.62    cherry 	/* Data segments */
    960   1.62    cherry 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
    961   1.62    cherry 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
    962   1.62    cherry 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
    963   1.62    cherry 	initctx->user_regs.fs = GSEL(GDATA_SEL, SEL_KPL);
    964   1.62    cherry 
    965   1.62    cherry 	/* GDT */
    966  1.105      maxv 	memcpy(initctx->gdt_frames, frames, sizeof(frames));
    967   1.62    cherry 	initctx->gdt_ents = gdt_ents;
    968   1.62    cherry 
    969   1.62    cherry 	/* LDT */
    970  1.108      maxv 	initctx->ldt_base = (unsigned long)ldtstore;
    971   1.62    cherry 	initctx->ldt_ents = NLDT;
    972   1.62    cherry 
    973   1.62    cherry 	/* Kernel context state */
    974   1.62    cherry 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
    975   1.62    cherry 	initctx->kernel_sp = pcb->pcb_esp0;
    976   1.62    cherry 	initctx->ctrlreg[0] = pcb->pcb_cr0;
    977   1.62    cherry 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
    978  1.105      maxv 	initctx->ctrlreg[2] = (vaddr_t)targeteip;
    979   1.70    cherry #ifdef PAE
    980   1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_pae_l3_pdirpa)));
    981  1.105      maxv #else
    982   1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(pcb->pcb_cr3)));
    983  1.105      maxv #endif
    984  1.105      maxv 	initctx->ctrlreg[4] = /* CR4_PAE | */CR4_OSFXSR | CR4_OSXMMEXCPT;
    985    1.2    bouyer 
    986   1.62    cherry 	/* Xen callbacks */
    987  1.105      maxv 	initctx->event_callback_eip = (unsigned long)hypervisor_callback;
    988   1.62    cherry 	initctx->event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
    989  1.105      maxv 	initctx->failsafe_callback_eip = (unsigned long)failsafe_callback;
    990   1.62    cherry 	initctx->failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
    991   1.45     rmind 
    992   1.62    cherry 	return;
    993   1.62    cherry }
    994   1.62    cherry #endif /* __x86_64__ */
    995   1.45     rmind 
    996   1.62    cherry int
    997   1.62    cherry mp_cpu_start(struct cpu_info *ci, vaddr_t target)
    998   1.62    cherry {
    999   1.62    cherry 	int hyperror;
   1000   1.62    cherry 	struct vcpu_guest_context vcpuctx;
   1001    1.2    bouyer 
   1002   1.62    cherry 	KASSERT(ci != NULL);
   1003   1.62    cherry 	KASSERT(ci != &cpu_info_primary);
   1004   1.62    cherry 	KASSERT(ci->ci_flags & CPUF_AP);
   1005   1.62    cherry 
   1006   1.62    cherry #ifdef __x86_64__
   1007   1.62    cherry 	xen_init_amd64_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
   1008  1.105      maxv #else
   1009   1.62    cherry 	xen_init_i386_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
   1010  1.105      maxv #endif
   1011   1.62    cherry 
   1012   1.62    cherry 	/* Initialise the given vcpu to execute cpu_hatch(ci); */
   1013   1.62    cherry 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_cpuid, &vcpuctx))) {
   1014   1.62    cherry 		aprint_error(": context initialisation failed. errno = %d\n", hyperror);
   1015   1.62    cherry 		return hyperror;
   1016   1.62    cherry 	}
   1017   1.62    cherry 
   1018   1.62    cherry 	/* Start it up */
   1019   1.62    cherry 
   1020   1.70    cherry 	/* First bring it down */
   1021   1.62    cherry 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_cpuid, NULL))) {
   1022   1.62    cherry 		aprint_error(": VCPUOP_down hypervisor command failed. errno = %d\n", hyperror);
   1023   1.62    cherry 		return hyperror;
   1024   1.62    cherry 	}
   1025   1.62    cherry 
   1026   1.62    cherry 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_up, ci->ci_cpuid, NULL))) {
   1027   1.62    cherry 		aprint_error(": VCPUOP_up hypervisor command failed. errno = %d\n", hyperror);
   1028   1.62    cherry 		return hyperror;
   1029   1.62    cherry 	}
   1030    1.2    bouyer 
   1031   1.62    cherry 	if (!vcpu_is_up(ci)) {
   1032   1.62    cherry 		aprint_error(": did not come up\n");
   1033   1.62    cherry 		return -1;
   1034    1.2    bouyer 	}
   1035   1.62    cherry 
   1036    1.2    bouyer 	return 0;
   1037    1.2    bouyer }
   1038    1.2    bouyer 
   1039    1.2    bouyer void
   1040    1.2    bouyer mp_cpu_start_cleanup(struct cpu_info *ci)
   1041    1.2    bouyer {
   1042   1.62    cherry 	if (vcpu_is_up(ci)) {
   1043   1.62    cherry 		aprint_debug_dev(ci->ci_dev, "is started.\n");
   1044  1.105      maxv 	} else {
   1045   1.62    cherry 		aprint_error_dev(ci->ci_dev, "did not start up.\n");
   1046   1.62    cherry 	}
   1047    1.2    bouyer }
   1048    1.2    bouyer 
   1049    1.2    bouyer void
   1050    1.3    bouyer cpu_init_msrs(struct cpu_info *ci, bool full)
   1051    1.2    bouyer {
   1052   1.43       jym #ifdef __x86_64__
   1053    1.3    bouyer 	if (full) {
   1054  1.105      maxv 		HYPERVISOR_set_segment_base(SEGBASE_FS, 0);
   1055  1.105      maxv 		HYPERVISOR_set_segment_base(SEGBASE_GS_KERNEL, (uint64_t)ci);
   1056  1.105      maxv 		HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 0);
   1057    1.3    bouyer 	}
   1058  1.105      maxv #endif
   1059   1.44       jym 
   1060   1.44       jym 	if (cpu_feature[2] & CPUID_NOX)
   1061   1.44       jym 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
   1062    1.2    bouyer }
   1063    1.2    bouyer 
   1064   1.95  christos void
   1065   1.95  christos cpu_offline_md(void)
   1066   1.95  christos {
   1067  1.105      maxv 	int s;
   1068   1.95  christos 
   1069  1.105      maxv 	s = splhigh();
   1070  1.105      maxv 	fpusave_cpu(true);
   1071  1.105      maxv 	splx(s);
   1072   1.95  christos }
   1073   1.95  christos 
   1074  1.105      maxv void
   1075    1.2    bouyer cpu_get_tsc_freq(struct cpu_info *ci)
   1076    1.2    bouyer {
   1077   1.62    cherry 	uint32_t vcpu_tversion;
   1078   1.16    cegger 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
   1079   1.62    cherry 
   1080   1.62    cherry 	vcpu_tversion = tinfo->version;
   1081   1.62    cherry 	while (tinfo->version == vcpu_tversion); /* Wait for a time update. XXX: timeout ? */
   1082   1.62    cherry 
   1083    1.2    bouyer 	uint64_t freq = 1000000000ULL << 32;
   1084    1.2    bouyer 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
   1085  1.105      maxv 	if (tinfo->tsc_shift < 0)
   1086    1.2    bouyer 		freq = freq << -tinfo->tsc_shift;
   1087    1.2    bouyer 	else
   1088    1.2    bouyer 		freq = freq >> tinfo->tsc_shift;
   1089   1.20        ad 	ci->ci_data.cpu_cc_freq = freq;
   1090    1.2    bouyer }
   1091   1.19     joerg 
   1092   1.19     joerg void
   1093   1.19     joerg x86_cpu_idle_xen(void)
   1094   1.19     joerg {
   1095   1.19     joerg 	struct cpu_info *ci = curcpu();
   1096   1.62    cherry 
   1097   1.19     joerg 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1098   1.19     joerg 
   1099   1.19     joerg 	x86_disable_intr();
   1100   1.19     joerg 	if (!__predict_false(ci->ci_want_resched)) {
   1101   1.19     joerg 		idle_block();
   1102   1.19     joerg 	} else {
   1103   1.19     joerg 		x86_enable_intr();
   1104   1.19     joerg 	}
   1105   1.19     joerg }
   1106   1.47       jym 
   1107   1.47       jym /*
   1108   1.47       jym  * Loads pmap for the current CPU.
   1109   1.47       jym  */
   1110   1.47       jym void
   1111   1.81    bouyer cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap)
   1112   1.47       jym {
   1113   1.84    cherry 	KASSERT(pmap != pmap_kernel());
   1114   1.91     rmind 
   1115   1.81    bouyer #if defined(__x86_64__) || defined(PAE)
   1116   1.81    bouyer 	struct cpu_info *ci = curcpu();
   1117   1.92     rmind 	cpuid_t cid = cpu_index(ci);
   1118   1.81    bouyer 
   1119   1.81    bouyer 	mutex_enter(&ci->ci_kpm_mtx);
   1120   1.93       jym 	/* make new pmap visible to xen_kpm_sync() */
   1121   1.92     rmind 	kcpuset_atomic_set(pmap->pm_xen_ptp_cpus, cid);
   1122   1.81    bouyer #endif
   1123  1.105      maxv 
   1124   1.47       jym #ifdef i386
   1125   1.47       jym #ifdef PAE
   1126   1.81    bouyer 	{
   1127   1.81    bouyer 		int i;
   1128   1.81    bouyer 		paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);
   1129   1.81    bouyer 		/* don't update the kernel L3 slot */
   1130   1.81    bouyer 		for (i = 0 ; i < PDP_SIZE - 1; i++) {
   1131   1.81    bouyer 			xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t),
   1132   1.81    bouyer 			    xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V);
   1133   1.81    bouyer 		}
   1134   1.81    bouyer 		tlbflush();
   1135   1.47       jym 	}
   1136   1.47       jym #else /* PAE */
   1137   1.47       jym 	lcr3(pmap_pdirpa(pmap, 0));
   1138   1.47       jym #endif /* PAE */
   1139   1.47       jym #endif /* i386 */
   1140   1.47       jym 
   1141   1.47       jym #ifdef __x86_64__
   1142   1.81    bouyer 	{
   1143   1.81    bouyer 		int i;
   1144   1.81    bouyer 		pd_entry_t *new_pgd;
   1145   1.81    bouyer 		paddr_t l4_pd_ma;
   1146   1.81    bouyer 
   1147   1.81    bouyer 		l4_pd_ma = xpmap_ptom_masked(ci->ci_kpm_pdirpa);
   1148   1.47       jym 
   1149   1.81    bouyer 		/*
   1150   1.81    bouyer 		 * Map user space address in kernel space and load
   1151   1.81    bouyer 		 * user cr3
   1152   1.81    bouyer 		 */
   1153   1.81    bouyer 		new_pgd = pmap->pm_pdir;
   1154   1.81    bouyer 		KASSERT(pmap == ci->ci_pmap);
   1155   1.70    cherry 
   1156   1.81    bouyer 		/* Copy user pmap L4 PDEs (in user addr. range) to per-cpu L4 */
   1157   1.81    bouyer 		for (i = 0; i < PDIR_SLOT_PTE; i++) {
   1158   1.81    bouyer 			KASSERT(pmap != pmap_kernel() || new_pgd[i] == 0);
   1159   1.81    bouyer 			if (ci->ci_kpm_pdir[i] != new_pgd[i]) {
   1160   1.81    bouyer 				xpq_queue_pte_update(
   1161  1.105      maxv 				    l4_pd_ma + i * sizeof(pd_entry_t),
   1162   1.81    bouyer 				    new_pgd[i]);
   1163   1.81    bouyer 			}
   1164   1.81    bouyer 		}
   1165   1.70    cherry 
   1166   1.84    cherry 		xen_set_user_pgd(pmap_pdirpa(pmap, 0));
   1167   1.84    cherry 		ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);
   1168   1.70    cherry 
   1169   1.81    bouyer 		tlbflush();
   1170   1.70    cherry 	}
   1171  1.105      maxv #endif /* __x86_64__ */
   1172   1.70    cherry 
   1173   1.81    bouyer #if defined(__x86_64__) || defined(PAE)
   1174   1.93       jym 	/* old pmap no longer visible to xen_kpm_sync() */
   1175   1.92     rmind 	if (oldpmap != pmap_kernel()) {
   1176   1.92     rmind 		kcpuset_atomic_clear(oldpmap->pm_xen_ptp_cpus, cid);
   1177   1.92     rmind 	}
   1178   1.81    bouyer 	mutex_exit(&ci->ci_kpm_mtx);
   1179   1.81    bouyer #endif
   1180   1.47       jym }
   1181   1.61    cherry 
   1182  1.105      maxv /*
   1183  1.105      maxv  * pmap_cpu_init_late: perform late per-CPU initialization.
   1184  1.105      maxv  *
   1185  1.105      maxv  * Short note about percpu PDIR pages. Both the PAE and __x86_64__ architectures
   1186  1.105      maxv  * have per-cpu PDIR tables, for two different reasons:
   1187  1.105      maxv  *  - on PAE, this is to get around Xen's pagetable setup constraints (multiple
   1188  1.105      maxv  *    L3[3]s cannot point to the same L2 - Xen will refuse to pin a table set up
   1189  1.105      maxv  *    this way).
   1190  1.105      maxv  *  - on __x86_64__, this is for multiple CPUs to map in different user pmaps
   1191  1.105      maxv  *    (see cpu_load_pmap()).
   1192  1.105      maxv  *
   1193  1.105      maxv  * What this means for us is that the PDIR of the pmap_kernel() is considered
   1194  1.105      maxv  * to be a canonical "SHADOW" PDIR with the following properties:
   1195  1.105      maxv  *  - its recursive mapping points to itself
   1196  1.105      maxv  *  - per-cpu recursive mappings point to themselves on __x86_64__
   1197  1.105      maxv  *  - per-cpu L4 pages' kernel entries are expected to be in sync with
   1198  1.105      maxv  *    the shadow
   1199  1.105      maxv  */
   1200   1.70    cherry 
   1201   1.70    cherry void
   1202   1.70    cherry pmap_cpu_init_late(struct cpu_info *ci)
   1203   1.70    cherry {
   1204   1.70    cherry #if defined(PAE) || defined(__x86_64__)
   1205   1.70    cherry 	/*
   1206   1.70    cherry 	 * The BP has already its own PD page allocated during early
   1207   1.70    cherry 	 * MD startup.
   1208   1.70    cherry 	 */
   1209   1.70    cherry 
   1210   1.78    cherry #if defined(__x86_64__)
   1211   1.78    cherry 	/* Setup per-cpu normal_pdes */
   1212   1.78    cherry 	int i;
   1213   1.78    cherry 	extern pd_entry_t * const normal_pdes[];
   1214   1.78    cherry 	for (i = 0;i < PTP_LEVELS - 1;i++) {
   1215   1.78    cherry 		ci->ci_normal_pdes[i] = normal_pdes[i];
   1216   1.78    cherry 	}
   1217   1.78    cherry #endif /* __x86_64__ */
   1218   1.78    cherry 
   1219   1.70    cherry 	if (ci == &cpu_info_primary)
   1220   1.70    cherry 		return;
   1221   1.70    cherry 
   1222   1.70    cherry 	KASSERT(ci != NULL);
   1223   1.70    cherry 
   1224   1.70    cherry #if defined(PAE)
   1225   1.73    cherry 	cpu_alloc_l3_page(ci);
   1226   1.70    cherry 	KASSERT(ci->ci_pae_l3_pdirpa != 0);
   1227   1.70    cherry 
   1228   1.70    cherry 	/* Initialise L2 entries 0 - 2: Point them to pmap_kernel() */
   1229   1.73    cherry 	int i;
   1230   1.75    cherry 	for (i = 0 ; i < PDP_SIZE - 1; i++) {
   1231   1.73    cherry 		ci->ci_pae_l3_pdir[i] =
   1232   1.73    cherry 		    xpmap_ptom_masked(pmap_kernel()->pm_pdirpa[i]) | PG_V;
   1233   1.73    cherry 	}
   1234   1.70    cherry #endif /* PAE */
   1235   1.70    cherry 
   1236   1.70    cherry 	ci->ci_kpm_pdir = (pd_entry_t *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
   1237   1.70    cherry 	    UVM_KMF_WIRED | UVM_KMF_ZERO | UVM_KMF_NOWAIT);
   1238   1.70    cherry 
   1239   1.70    cherry 	if (ci->ci_kpm_pdir == NULL) {
   1240   1.70    cherry 		panic("%s: failed to allocate L4 per-cpu PD for CPU %d\n",
   1241  1.105      maxv 		    __func__, cpu_index(ci));
   1242   1.70    cherry 	}
   1243  1.105      maxv 	ci->ci_kpm_pdirpa = vtophys((vaddr_t)ci->ci_kpm_pdir);
   1244   1.70    cherry 	KASSERT(ci->ci_kpm_pdirpa != 0);
   1245   1.70    cherry 
   1246   1.70    cherry #if defined(__x86_64__)
   1247  1.106      maxv 	extern pt_entry_t xpmap_pg_nx;
   1248   1.70    cherry 
   1249  1.106      maxv 	/* Copy over the pmap_kernel() shadow L4 entries */
   1250   1.70    cherry 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir, PAGE_SIZE);
   1251   1.70    cherry 
   1252   1.70    cherry 	/* Recursive kernel mapping */
   1253  1.105      maxv 	ci->ci_kpm_pdir[PDIR_SLOT_PTE] = xpmap_ptom_masked(ci->ci_kpm_pdirpa)
   1254  1.110      maxv 	    | PG_V | xpmap_pg_nx;
   1255   1.70    cherry #elif defined(PAE)
   1256  1.106      maxv 	/* Copy over the pmap_kernel() shadow L2 entries */
   1257  1.105      maxv 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir + PDIR_SLOT_KERN,
   1258  1.105      maxv 	    nkptp[PTP_LEVELS - 1] * sizeof(pd_entry_t));
   1259  1.106      maxv #endif
   1260   1.70    cherry 
   1261  1.105      maxv 	/* Xen wants a RO pdir. */
   1262   1.83    bouyer 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_kpm_pdir,
   1263   1.83    bouyer 	    (vaddr_t)ci->ci_kpm_pdir + PAGE_SIZE, VM_PROT_READ);
   1264   1.83    bouyer 	pmap_update(pmap_kernel());
   1265   1.70    cherry #if defined(PAE)
   1266  1.105      maxv 	/*
   1267  1.105      maxv 	 * Initialize L3 entry 3. This mapping is shared across all pmaps and is
   1268  1.105      maxv 	 * static, ie: loading a new pmap will not update this entry.
   1269   1.70    cherry 	 */
   1270  1.110      maxv 	ci->ci_pae_l3_pdir[3] = xpmap_ptom_masked(ci->ci_kpm_pdirpa) | PG_V;
   1271   1.70    cherry 
   1272  1.105      maxv 	/* Xen wants a RO L3. */
   1273   1.83    bouyer 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_pae_l3_pdir,
   1274   1.83    bouyer 	    (vaddr_t)ci->ci_pae_l3_pdir + PAGE_SIZE, VM_PROT_READ);
   1275   1.83    bouyer 	pmap_update(pmap_kernel());
   1276   1.70    cherry 
   1277   1.70    cherry 	xpq_queue_pin_l3_table(xpmap_ptom_masked(ci->ci_pae_l3_pdirpa));
   1278   1.70    cherry 
   1279  1.105      maxv #elif defined(__x86_64__)
   1280   1.70    cherry 	xpq_queue_pin_l4_table(xpmap_ptom_masked(ci->ci_kpm_pdirpa));
   1281   1.78    cherry #endif /* PAE , __x86_64__ */
   1282   1.70    cherry #endif /* defined(PAE) || defined(__x86_64__) */
   1283   1.70    cherry }
   1284   1.70    cherry 
   1285   1.61    cherry /*
   1286   1.61    cherry  * Notify all other cpus to halt.
   1287   1.61    cherry  */
   1288   1.61    cherry 
   1289   1.61    cherry void
   1290   1.61    cherry cpu_broadcast_halt(void)
   1291   1.61    cherry {
   1292   1.61    cherry 	xen_broadcast_ipi(XEN_IPI_HALT);
   1293   1.61    cherry }
   1294   1.61    cherry 
   1295   1.61    cherry /*
   1296   1.61    cherry  * Send a dummy ipi to a cpu.
   1297   1.61    cherry  */
   1298   1.61    cherry 
   1299   1.61    cherry void
   1300   1.61    cherry cpu_kick(struct cpu_info *ci)
   1301   1.61    cherry {
   1302   1.64  dholland 	(void)xen_send_ipi(ci, XEN_IPI_KICK);
   1303   1.61    cherry }
   1304