Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.145
      1  1.145  riastrad /*	$NetBSD: cpu.c,v 1.145 2023/02/25 00:35:01 riastradh Exp $	*/
      2    1.2    bouyer 
      3    1.2    bouyer /*-
      4    1.2    bouyer  * Copyright (c) 2000 The NetBSD Foundation, Inc.
      5   1.19     joerg  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
      6    1.2    bouyer  * All rights reserved.
      7    1.2    bouyer  *
      8    1.2    bouyer  * This code is derived from software contributed to The NetBSD Foundation
      9    1.2    bouyer  * by RedBack Networks Inc.
     10    1.2    bouyer  *
     11    1.2    bouyer  * Author: Bill Sommerfeld
     12    1.2    bouyer  *
     13    1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     14    1.2    bouyer  * modification, are permitted provided that the following conditions
     15    1.2    bouyer  * are met:
     16    1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     17    1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     18    1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     19    1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     20    1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     21    1.2    bouyer  *
     22    1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     23    1.2    bouyer  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     24    1.2    bouyer  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     25    1.2    bouyer  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     26    1.2    bouyer  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27    1.2    bouyer  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28    1.2    bouyer  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29    1.2    bouyer  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30    1.2    bouyer  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31    1.2    bouyer  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32    1.2    bouyer  * POSSIBILITY OF SUCH DAMAGE.
     33    1.2    bouyer  */
     34    1.2    bouyer 
     35    1.2    bouyer /*
     36    1.2    bouyer  * Copyright (c) 1999 Stefan Grefen
     37    1.2    bouyer  *
     38    1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     39    1.2    bouyer  * modification, are permitted provided that the following conditions
     40    1.2    bouyer  * are met:
     41    1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     42    1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     43    1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     44    1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     45    1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     46    1.2    bouyer  * 3. All advertising materials mentioning features or use of this software
     47    1.2    bouyer  *    must display the following acknowledgement:
     48    1.2    bouyer  *      This product includes software developed by the NetBSD
     49    1.2    bouyer  *      Foundation, Inc. and its contributors.
     50    1.2    bouyer  * 4. Neither the name of The NetBSD Foundation nor the names of its
     51    1.2    bouyer  *    contributors may be used to endorse or promote products derived
     52    1.2    bouyer  *    from this software without specific prior written permission.
     53    1.2    bouyer  *
     54    1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     55    1.2    bouyer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     56    1.2    bouyer  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     57    1.2    bouyer  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     58    1.2    bouyer  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     59    1.2    bouyer  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     60    1.2    bouyer  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     61    1.2    bouyer  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     62    1.2    bouyer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     63    1.2    bouyer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     64    1.2    bouyer  * SUCH DAMAGE.
     65    1.2    bouyer  */
     66    1.2    bouyer 
     67    1.2    bouyer #include <sys/cdefs.h>
     68  1.145  riastrad __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.145 2023/02/25 00:35:01 riastradh Exp $");
     69    1.2    bouyer 
     70    1.2    bouyer #include "opt_ddb.h"
     71    1.2    bouyer #include "opt_multiprocessor.h"
     72    1.2    bouyer #include "opt_mpbios.h"		/* for MPDEBUG */
     73    1.2    bouyer #include "opt_mtrr.h"
     74    1.2    bouyer #include "opt_xen.h"
     75    1.2    bouyer 
     76    1.2    bouyer #include "lapic.h"
     77    1.2    bouyer #include "ioapic.h"
     78    1.2    bouyer 
     79    1.2    bouyer #include <sys/param.h>
     80    1.2    bouyer #include <sys/proc.h>
     81    1.2    bouyer #include <sys/systm.h>
     82    1.2    bouyer #include <sys/device.h>
     83   1.31    cegger #include <sys/kmem.h>
     84   1.11    cegger #include <sys/cpu.h>
     85   1.66    jruoho #include <sys/cpufreq.h>
     86   1.11    cegger #include <sys/atomic.h>
     87   1.32    cegger #include <sys/reboot.h>
     88   1.62    cherry #include <sys/idle.h>
     89    1.2    bouyer 
     90   1.51  uebayasi #include <uvm/uvm.h>
     91    1.2    bouyer 
     92  1.114  riastrad #include <machine/cpu.h>
     93    1.2    bouyer #include <machine/cpufunc.h>
     94    1.2    bouyer #include <machine/cpuvar.h>
     95    1.2    bouyer #include <machine/pmap.h>
     96  1.142  riastrad #include <machine/pmap_private.h>
     97    1.2    bouyer #include <machine/vmparam.h>
     98    1.2    bouyer #include <machine/mpbiosvar.h>
     99    1.2    bouyer #include <machine/pcb.h>
    100    1.2    bouyer #include <machine/specialreg.h>
    101    1.2    bouyer #include <machine/segments.h>
    102    1.2    bouyer #include <machine/gdt.h>
    103    1.2    bouyer #include <machine/mtrr.h>
    104    1.2    bouyer #include <machine/pio.h>
    105    1.2    bouyer 
    106   1.97       dsl #include <x86/fpu.h>
    107   1.62    cherry 
    108   1.62    cherry #include <xen/xen.h>
    109  1.128    cherry #include <xen/include/public/vcpu.h>
    110    1.2    bouyer #include <xen/vcpuvar.h>
    111    1.2    bouyer 
    112    1.2    bouyer #if NLAPIC > 0
    113    1.2    bouyer #include <machine/apicvar.h>
    114    1.2    bouyer #include <machine/i82489reg.h>
    115    1.2    bouyer #include <machine/i82489var.h>
    116    1.2    bouyer #endif
    117    1.2    bouyer 
    118    1.2    bouyer #include <dev/ic/mc146818reg.h>
    119    1.2    bouyer #include <dev/isa/isareg.h>
    120    1.2    bouyer 
    121   1.56    jruoho static int	cpu_match(device_t, cfdata_t, void *);
    122   1.56    jruoho static void	cpu_attach(device_t, device_t, void *);
    123   1.56    jruoho static void	cpu_defer(device_t);
    124   1.56    jruoho static int	cpu_rescan(device_t, const char *, const int *);
    125   1.56    jruoho static void	cpu_childdetached(device_t, device_t);
    126   1.56    jruoho static int	vcpu_match(device_t, cfdata_t, void *);
    127   1.56    jruoho static void	vcpu_attach(device_t, device_t, void *);
    128   1.56    jruoho static void	cpu_attach_common(device_t, device_t, void *);
    129   1.56    jruoho void		cpu_offline_md(void);
    130    1.2    bouyer 
    131    1.2    bouyer struct cpu_softc {
    132   1.10    cegger 	device_t sc_dev;		/* device tree glue */
    133    1.2    bouyer 	struct cpu_info *sc_info;	/* pointer to CPU info */
    134   1.32    cegger 	bool sc_wasonline;
    135    1.2    bouyer };
    136    1.2    bouyer 
    137   1.62    cherry int mp_cpu_start(struct cpu_info *, vaddr_t);
    138    1.2    bouyer void mp_cpu_start_cleanup(struct cpu_info *);
    139    1.2    bouyer const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    140    1.2    bouyer 				      mp_cpu_start_cleanup };
    141    1.2    bouyer 
    142   1.53    jruoho CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
    143   1.53    jruoho     cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
    144   1.53    jruoho 
    145   1.10    cegger CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
    146    1.2    bouyer     vcpu_match, vcpu_attach, NULL, NULL);
    147    1.2    bouyer 
    148    1.2    bouyer /*
    149    1.2    bouyer  * Statically-allocated CPU info for the primary CPU (or the only
    150    1.2    bouyer  * CPU, on uniprocessors).  The CPU info list is initialized to
    151    1.2    bouyer  * point at it.
    152    1.2    bouyer  */
    153   1.38    cegger struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    154    1.7    bouyer 	.ci_dev = 0,
    155    1.2    bouyer 	.ci_self = &cpu_info_primary,
    156    1.4    bouyer 	.ci_idepth = -1,
    157    1.2    bouyer 	.ci_curlwp = &lwp0,
    158   1.25        ad 	.ci_curldt = -1,
    159    1.2    bouyer };
    160   1.38    cegger struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    161    1.7    bouyer 	.ci_dev = 0,
    162    1.2    bouyer 	.ci_self = &phycpu_info_primary,
    163    1.2    bouyer };
    164    1.2    bouyer 
    165    1.2    bouyer struct cpu_info *cpu_info_list = &cpu_info_primary;
    166   1.38    cegger struct cpu_info *phycpu_info_list = &phycpu_info_primary;
    167    1.2    bouyer 
    168  1.107      maxv uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits
    169   1.43       jym 			  *	[0] basic features %edx
    170   1.43       jym 			  *	[1] basic features %ecx
    171   1.43       jym 			  *	[2] extended features %edx
    172   1.43       jym 			  *	[3] extended features %ecx
    173   1.43       jym 			  *	[4] VIA padlock features
    174  1.102  christos 			  *	[5] structured extended features cpuid.7:%ebx
    175  1.102  christos 			  *	[6] structured extended features cpuid.7:%ecx
    176   1.43       jym 			  */
    177   1.43       jym 
    178   1.11    cegger bool x86_mp_online;
    179   1.11    cegger paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    180    1.2    bouyer 
    181   1.38    cegger #if defined(MULTIPROCESSOR)
    182    1.2    bouyer void    	cpu_hatch(void *);
    183    1.2    bouyer static void    	cpu_boot_secondary(struct cpu_info *ci);
    184    1.2    bouyer static void    	cpu_start_secondary(struct cpu_info *ci);
    185   1.38    cegger #endif	/* MULTIPROCESSOR */
    186    1.2    bouyer 
    187   1.56    jruoho static int
    188   1.10    cegger cpu_match(device_t parent, cfdata_t match, void *aux)
    189    1.2    bouyer {
    190    1.2    bouyer 
    191    1.2    bouyer 	return 1;
    192    1.2    bouyer }
    193    1.2    bouyer 
    194   1.56    jruoho static void
    195   1.10    cegger cpu_attach(device_t parent, device_t self, void *aux)
    196    1.2    bouyer {
    197   1.10    cegger 	struct cpu_softc *sc = device_private(self);
    198    1.2    bouyer 	struct cpu_attach_args *caa = aux;
    199    1.2    bouyer 	struct cpu_info *ci;
    200   1.34    cegger 	uintptr_t ptr;
    201   1.52    bouyer 	static int nphycpu = 0;
    202    1.2    bouyer 
    203   1.10    cegger 	sc->sc_dev = self;
    204   1.10    cegger 
    205    1.2    bouyer 	/*
    206    1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    207   1.52    bouyer 	 * If we're the first attached CPU use the primary cpu_info,
    208   1.52    bouyer 	 * otherwise allocate a new one
    209    1.2    bouyer 	 */
    210   1.52    bouyer 	aprint_naive("\n");
    211   1.52    bouyer 	aprint_normal("\n");
    212   1.52    bouyer 	if (nphycpu > 0) {
    213   1.52    bouyer 		struct cpu_info *tmp;
    214   1.34    cegger 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    215   1.34    cegger 		    KM_SLEEP);
    216   1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    217   1.24        ad 		ci->ci_curldt = -1;
    218   1.52    bouyer 
    219   1.52    bouyer 		tmp = phycpu_info_list;
    220   1.52    bouyer 		while (tmp->ci_next)
    221   1.52    bouyer 			tmp = tmp->ci_next;
    222   1.52    bouyer 
    223   1.52    bouyer 		tmp->ci_next = ci;
    224    1.2    bouyer 	} else {
    225    1.2    bouyer 		ci = &phycpu_info_primary;
    226    1.2    bouyer 	}
    227    1.2    bouyer 
    228    1.2    bouyer 	ci->ci_self = ci;
    229    1.2    bouyer 	sc->sc_info = ci;
    230    1.2    bouyer 
    231    1.2    bouyer 	ci->ci_dev = self;
    232   1.50    jruoho 	ci->ci_acpiid = caa->cpu_id;
    233   1.23        ad 	ci->ci_cpuid = caa->cpu_number;
    234   1.16    cegger 	ci->ci_vcpu = NULL;
    235   1.52    bouyer 	ci->ci_index = nphycpu++;
    236  1.138  jdolecek 	ci->ci_kfpu_spl = -1;
    237    1.2    bouyer 
    238   1.52    bouyer 	if (!pmf_device_register(self, NULL, NULL))
    239   1.52    bouyer 		aprint_error_dev(self, "couldn't establish power handler\n");
    240   1.34    cegger 
    241   1.56    jruoho 	(void)config_defer(self, cpu_defer);
    242   1.56    jruoho }
    243   1.56    jruoho 
    244   1.56    jruoho static void
    245   1.56    jruoho cpu_defer(device_t self)
    246   1.56    jruoho {
    247   1.56    jruoho 	cpu_rescan(self, NULL, NULL);
    248    1.2    bouyer }
    249    1.2    bouyer 
    250   1.56    jruoho static int
    251   1.53    jruoho cpu_rescan(device_t self, const char *ifattr, const int *locators)
    252   1.53    jruoho {
    253   1.53    jruoho 	struct cpu_softc *sc = device_private(self);
    254   1.53    jruoho 	struct cpufeature_attach_args cfaa;
    255   1.53    jruoho 	struct cpu_info *ci = sc->sc_info;
    256   1.53    jruoho 
    257   1.53    jruoho 	memset(&cfaa, 0, sizeof(cfaa));
    258   1.53    jruoho 	cfaa.ci = ci;
    259   1.53    jruoho 
    260   1.53    jruoho 	if (ifattr_match(ifattr, "cpufeaturebus")) {
    261   1.53    jruoho 
    262   1.53    jruoho 		if (ci->ci_frequency == NULL) {
    263   1.55    jruoho 			cfaa.name = "frequency";
    264  1.140   thorpej 			ci->ci_frequency =
    265  1.140   thorpej 			    config_found(self, &cfaa, NULL,
    266  1.141   thorpej 					 CFARGS(.iattr = "cpufeaturebus"));
    267   1.54    jruoho 		}
    268   1.53    jruoho 	}
    269   1.53    jruoho 
    270   1.53    jruoho 	return 0;
    271   1.53    jruoho }
    272   1.53    jruoho 
    273   1.56    jruoho static void
    274   1.53    jruoho cpu_childdetached(device_t self, device_t child)
    275   1.53    jruoho {
    276   1.53    jruoho 	struct cpu_softc *sc = device_private(self);
    277   1.53    jruoho 	struct cpu_info *ci = sc->sc_info;
    278   1.53    jruoho 
    279   1.53    jruoho 	if (ci->ci_frequency == child)
    280   1.53    jruoho 		ci->ci_frequency = NULL;
    281   1.53    jruoho }
    282   1.53    jruoho 
    283   1.56    jruoho static int
    284   1.10    cegger vcpu_match(device_t parent, cfdata_t match, void *aux)
    285    1.2    bouyer {
    286    1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    287   1.62    cherry 	struct vcpu_runstate_info vcr;
    288   1.62    cherry 	int error;
    289   1.62    cherry 
    290   1.62    cherry 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) {
    291   1.62    cherry 		error = HYPERVISOR_vcpu_op(VCPUOP_get_runstate_info,
    292  1.105      maxv 		    vcaa->vcaa_caa.cpu_number, &vcr);
    293   1.62    cherry 		switch (error) {
    294   1.62    cherry 		case 0:
    295   1.62    cherry 			return 1;
    296   1.62    cherry 		case -ENOENT:
    297   1.62    cherry 			return 0;
    298   1.62    cherry 		default:
    299   1.62    cherry 			panic("Unknown hypervisor error %d returned on vcpu runstate probe\n", error);
    300   1.62    cherry 		}
    301   1.62    cherry 	}
    302    1.2    bouyer 
    303    1.2    bouyer 	return 0;
    304    1.2    bouyer }
    305    1.2    bouyer 
    306   1.56    jruoho static void
    307   1.10    cegger vcpu_attach(device_t parent, device_t self, void *aux)
    308    1.2    bouyer {
    309    1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    310    1.2    bouyer 
    311   1.62    cherry 	KASSERT(vcaa->vcaa_caa.cpu_func == NULL);
    312   1.62    cherry 	vcaa->vcaa_caa.cpu_func = &mp_cpu_funcs;
    313    1.2    bouyer 	cpu_attach_common(parent, self, &vcaa->vcaa_caa);
    314   1.65       jym 
    315   1.65       jym 	if (!pmf_device_register(self, NULL, NULL))
    316   1.65       jym 		aprint_error_dev(self, "couldn't establish power handler\n");
    317    1.2    bouyer }
    318    1.2    bouyer 
    319   1.62    cherry static int
    320   1.62    cherry vcpu_is_up(struct cpu_info *ci)
    321   1.62    cherry {
    322   1.62    cherry 	KASSERT(ci != NULL);
    323  1.135    bouyer 	return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_vcpuid, NULL);
    324   1.62    cherry }
    325   1.62    cherry 
    326    1.2    bouyer static void
    327    1.2    bouyer cpu_vm_init(struct cpu_info *ci)
    328    1.2    bouyer {
    329    1.2    bouyer 	int ncolors = 2, i;
    330    1.2    bouyer 
    331    1.2    bouyer 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    332    1.2    bouyer 		struct x86_cache_info *cai;
    333    1.2    bouyer 		int tcolors;
    334    1.2    bouyer 
    335    1.2    bouyer 		cai = &ci->ci_cinfo[i];
    336    1.2    bouyer 
    337    1.2    bouyer 		tcolors = atop(cai->cai_totalsize);
    338  1.105      maxv 		switch (cai->cai_associativity) {
    339    1.2    bouyer 		case 0xff:
    340    1.2    bouyer 			tcolors = 1; /* fully associative */
    341    1.2    bouyer 			break;
    342    1.2    bouyer 		case 0:
    343    1.2    bouyer 		case 1:
    344    1.2    bouyer 			break;
    345    1.2    bouyer 		default:
    346    1.2    bouyer 			tcolors /= cai->cai_associativity;
    347    1.2    bouyer 		}
    348  1.127  riastrad 		ncolors = uimax(ncolors, tcolors);
    349    1.2    bouyer 	}
    350    1.2    bouyer 
    351    1.2    bouyer 	/*
    352   1.67       mrg 	 * Knowing the size of the largest cache on this CPU, potentially
    353   1.67       mrg 	 * re-color our pages.
    354    1.2    bouyer 	 */
    355   1.28    bouyer 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    356    1.2    bouyer 	uvm_page_recolor(ncolors);
    357   1.91     rmind 	pmap_tlb_cpu_init(ci);
    358  1.109      maxv #ifndef __HAVE_DIRECT_MAP
    359  1.109      maxv 	pmap_vpage_cpu_init(ci);
    360  1.109      maxv #endif
    361    1.2    bouyer }
    362    1.2    bouyer 
    363   1.56    jruoho static void
    364   1.11    cegger cpu_attach_common(device_t parent, device_t self, void *aux)
    365    1.2    bouyer {
    366   1.10    cegger 	struct cpu_softc *sc = device_private(self);
    367    1.2    bouyer 	struct cpu_attach_args *caa = aux;
    368    1.2    bouyer 	struct cpu_info *ci;
    369   1.12    cegger 	uintptr_t ptr;
    370    1.2    bouyer 	int cpunum = caa->cpu_number;
    371   1.38    cegger 	static bool again = false;
    372    1.2    bouyer 
    373   1.10    cegger 	sc->sc_dev = self;
    374   1.10    cegger 
    375    1.2    bouyer 	/*
    376    1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    377    1.2    bouyer 	 * structure, otherwise use the primary's.
    378    1.2    bouyer 	 */
    379    1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    380   1.12    cegger 		aprint_naive(": Application Processor\n");
    381   1.31    cegger 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    382   1.31    cegger 		    KM_SLEEP);
    383   1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    384   1.12    cegger 		memset(ci, 0, sizeof(*ci));
    385  1.117    bouyer 		cpu_init_tss(ci);
    386    1.2    bouyer 	} else {
    387   1.12    cegger 		aprint_naive(": %s Processor\n",
    388   1.12    cegger 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    389    1.2    bouyer 		ci = &cpu_info_primary;
    390    1.2    bouyer 	}
    391    1.2    bouyer 
    392    1.2    bouyer 	ci->ci_self = ci;
    393    1.2    bouyer 	sc->sc_info = ci;
    394    1.2    bouyer 	ci->ci_dev = self;
    395   1.23        ad 	ci->ci_cpuid = cpunum;
    396  1.135    bouyer 	ci->ci_vcpuid = cpunum;
    397  1.138  jdolecek 	ci->ci_kfpu_spl = -1;
    398   1.16    cegger 
    399   1.16    cegger 	KASSERT(HYPERVISOR_shared_info != NULL);
    400   1.89    bouyer 	KASSERT(cpunum < XEN_LEGACY_MAX_VCPUS);
    401   1.16    cegger 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
    402   1.16    cegger 
    403   1.62    cherry 	KASSERT(ci->ci_func == 0);
    404    1.2    bouyer 	ci->ci_func = caa->cpu_func;
    405  1.101   msaitoh 	aprint_normal("\n");
    406    1.2    bouyer 
    407   1.38    cegger 	/* Must be called before mi_cpu_attach(). */
    408   1.38    cegger 	cpu_vm_init(ci);
    409   1.38    cegger 
    410    1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    411    1.2    bouyer 		int error;
    412    1.2    bouyer 
    413    1.2    bouyer 		error = mi_cpu_attach(ci);
    414   1.62    cherry 
    415   1.62    cherry 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    416    1.2    bouyer 		if (error != 0) {
    417   1.38    cegger 			aprint_error_dev(self,
    418   1.38    cegger 			    "mi_cpu_attach failed with %d\n", error);
    419    1.2    bouyer 			return;
    420    1.2    bouyer 		}
    421   1.62    cherry 
    422    1.2    bouyer 	} else {
    423    1.2    bouyer 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    424    1.2    bouyer 	}
    425    1.2    bouyer 
    426   1.89    bouyer 	KASSERT(ci->ci_cpuid == ci->ci_index);
    427  1.100    bouyer #ifdef __x86_64__
    428  1.100    bouyer 	/* No user PGD mapped for this CPU yet */
    429  1.100    bouyer 	ci->ci_xen_current_user_pgd = 0;
    430  1.100    bouyer #endif
    431  1.100    bouyer 	mutex_init(&ci->ci_kpm_mtx, MUTEX_DEFAULT, IPL_VM);
    432    1.2    bouyer 	pmap_reference(pmap_kernel());
    433    1.2    bouyer 	ci->ci_pmap = pmap_kernel();
    434    1.2    bouyer 	ci->ci_tlbstate = TLBSTATE_STALE;
    435    1.2    bouyer 
    436   1.38    cegger 	/*
    437   1.38    cegger 	 * Boot processor may not be attached first, but the below
    438   1.38    cegger 	 * must be done to allow booting other processors.
    439   1.38    cegger 	 */
    440   1.38    cegger 	if (!again) {
    441   1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    442   1.38    cegger 		/* Basic init. */
    443   1.38    cegger 		cpu_intr_init(ci);
    444   1.38    cegger 		cpu_get_tsc_freq(ci);
    445   1.38    cegger 		cpu_init(ci);
    446   1.78    cherry 		pmap_cpu_init_late(ci);
    447   1.62    cherry 
    448   1.99       snj 		/* Every processor needs to init its own ipi h/w (similar to lapic) */
    449   1.62    cherry 		xen_ipi_init();
    450   1.62    cherry 
    451   1.38    cegger 		/* Make sure DELAY() is initialized. */
    452   1.38    cegger 		DELAY(1);
    453   1.38    cegger 		again = true;
    454   1.38    cegger 	}
    455   1.38    cegger 
    456    1.2    bouyer 	/* further PCB init done later. */
    457    1.2    bouyer 
    458    1.2    bouyer 	switch (caa->cpu_role) {
    459    1.2    bouyer 	case CPU_ROLE_SP:
    460   1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    461   1.21        ad 		cpu_identify(ci);
    462   1.38    cegger 		x86_cpu_idle_init();
    463    1.2    bouyer 		break;
    464    1.2    bouyer 
    465    1.2    bouyer 	case CPU_ROLE_BP:
    466   1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    467   1.21        ad 		cpu_identify(ci);
    468   1.38    cegger 		x86_cpu_idle_init();
    469    1.2    bouyer 		break;
    470    1.2    bouyer 
    471    1.2    bouyer 	case CPU_ROLE_AP:
    472   1.62    cherry 		atomic_or_32(&ci->ci_flags, CPUF_AP);
    473   1.62    cherry 
    474    1.2    bouyer 		/*
    475    1.2    bouyer 		 * report on an AP
    476    1.2    bouyer 		 */
    477    1.2    bouyer 
    478    1.2    bouyer #if defined(MULTIPROCESSOR)
    479   1.62    cherry 		/* interrupt handler stack */
    480    1.2    bouyer 		cpu_intr_init(ci);
    481   1.62    cherry 
    482  1.139  yamaguch 		/* Setup per-cpu memory for idt */
    483  1.139  yamaguch 		idt_vec_init_cpu_md(&ci->ci_idtvec, cpu_index(ci));
    484  1.139  yamaguch 
    485   1.62    cherry 		/* Setup per-cpu memory for gdt */
    486    1.2    bouyer 		gdt_alloc_cpu(ci);
    487   1.62    cherry 
    488   1.62    cherry 		pmap_cpu_init_late(ci);
    489    1.2    bouyer 		cpu_start_secondary(ci);
    490   1.62    cherry 
    491    1.2    bouyer 		if (ci->ci_flags & CPUF_PRESENT) {
    492   1.30    cegger 			struct cpu_info *tmp;
    493   1.30    cegger 
    494   1.62    cherry 			cpu_identify(ci);
    495   1.30    cegger 			tmp = cpu_info_list;
    496   1.30    cegger 			while (tmp->ci_next)
    497   1.30    cegger 				tmp = tmp->ci_next;
    498   1.30    cegger 
    499   1.30    cegger 			tmp->ci_next = ci;
    500    1.2    bouyer 		}
    501    1.2    bouyer #else
    502  1.101   msaitoh 		aprint_error_dev(ci->ci_dev, "not started\n");
    503    1.2    bouyer #endif
    504    1.2    bouyer 		break;
    505    1.2    bouyer 
    506    1.2    bouyer 	default:
    507    1.2    bouyer 		panic("unknown processor type??\n");
    508    1.2    bouyer 	}
    509    1.2    bouyer 
    510   1.62    cherry #ifdef MPVERBOSE
    511    1.2    bouyer 	if (mp_verbose) {
    512    1.2    bouyer 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    513   1.37     rmind 		struct pcb *pcb = lwp_getpcb(l);
    514    1.2    bouyer 
    515   1.38    cegger 		aprint_verbose_dev(self,
    516  1.133       rin 		    "idle lwp at %p, idle sp at %p\n",
    517   1.12    cegger 		    l,
    518   1.12    cegger #ifdef i386
    519   1.37     rmind 		    (void *)pcb->pcb_esp
    520  1.105      maxv #else
    521   1.37     rmind 		    (void *)pcb->pcb_rsp
    522  1.105      maxv #endif
    523   1.12    cegger 		);
    524  1.145  riastrad 
    525    1.2    bouyer 	}
    526   1.62    cherry #endif /* MPVERBOSE */
    527    1.2    bouyer }
    528    1.2    bouyer 
    529    1.2    bouyer /*
    530    1.2    bouyer  * Initialize the processor appropriately.
    531    1.2    bouyer  */
    532    1.2    bouyer 
    533    1.2    bouyer void
    534   1.10    cegger cpu_init(struct cpu_info *ci)
    535    1.2    bouyer {
    536  1.122  jdolecek 	uint32_t cr4 = 0;
    537    1.2    bouyer 
    538    1.2    bouyer 	/*
    539    1.2    bouyer 	 * If we have FXSAVE/FXRESTOR, use them.
    540    1.2    bouyer 	 */
    541   1.43       jym 	if (cpu_feature[0] & CPUID_FXSR) {
    542  1.122  jdolecek 		cr4 |= CR4_OSFXSR;
    543    1.2    bouyer 
    544    1.2    bouyer 		/*
    545    1.2    bouyer 		 * If we have SSE/SSE2, enable XMM exceptions.
    546    1.2    bouyer 		 */
    547   1.43       jym 		if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
    548  1.122  jdolecek 			cr4 |= CR4_OSXMMEXCPT;
    549  1.122  jdolecek 	}
    550  1.122  jdolecek 
    551  1.122  jdolecek 	/* If xsave is supported, enable it */
    552  1.122  jdolecek 	if (cpu_feature[1] & CPUID2_XSAVE && x86_fpu_save >= FPU_SAVE_XSAVE)
    553  1.122  jdolecek 		cr4 |= CR4_OSXSAVE;
    554  1.122  jdolecek 
    555  1.122  jdolecek 	if (cr4) {
    556  1.122  jdolecek 		cr4 |= rcr4();
    557  1.122  jdolecek 		lcr4(cr4);
    558    1.2    bouyer 	}
    559    1.2    bouyer 
    560  1.116      maxv 	if (x86_fpu_save >= FPU_SAVE_FXSAVE) {
    561  1.120      maxv 		fpuinit_mxcsr_mask();
    562  1.118  jdolecek 	}
    563  1.118  jdolecek 
    564  1.122  jdolecek 	/*
    565  1.122  jdolecek 	 * Changing CR4 register may change cpuid values. For example, setting
    566  1.122  jdolecek 	 * CR4_OSXSAVE sets CPUID2_OSXSAVE. The CPUID2_OSXSAVE is in
    567  1.122  jdolecek 	 * ci_feat_val[1], so update it.
    568  1.122  jdolecek 	 * XXX Other than ci_feat_val[1] might be changed.
    569  1.122  jdolecek 	 */
    570  1.122  jdolecek 	if (cpuid_level >= 1) {
    571  1.122  jdolecek 		u_int descs[4];
    572  1.122  jdolecek 
    573  1.122  jdolecek 		x86_cpuid(1, descs);
    574  1.122  jdolecek 		ci->ci_feat_val[1] = descs[2];
    575  1.122  jdolecek 	}
    576  1.122  jdolecek 
    577  1.122  jdolecek 	/* If xsave is enabled, enable all fpu features */
    578  1.122  jdolecek 	if (cr4 & CR4_OSXSAVE) {
    579  1.122  jdolecek 		wrxcr(0, x86_xsave_features & XCR0_FPU);
    580  1.122  jdolecek 	}
    581  1.122  jdolecek 
    582   1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    583    1.2    bouyer }
    584    1.2    bouyer 
    585    1.2    bouyer 
    586    1.2    bouyer #ifdef MULTIPROCESSOR
    587   1.62    cherry 
    588    1.2    bouyer void
    589   1.10    cegger cpu_boot_secondary_processors(void)
    590    1.2    bouyer {
    591    1.2    bouyer 	struct cpu_info *ci;
    592  1.123    bouyer 	kcpuset_t *cpus;
    593    1.2    bouyer 	u_long i;
    594  1.123    bouyer 
    595  1.123    bouyer 	kcpuset_create(&cpus, true);
    596  1.123    bouyer 	kcpuset_set(cpus, cpu_index(curcpu()));
    597   1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    598   1.38    cegger 		ci = cpu_lookup(i);
    599    1.2    bouyer 		if (ci == NULL)
    600    1.2    bouyer 			continue;
    601    1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    602    1.2    bouyer 			continue;
    603    1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    604    1.2    bouyer 			continue;
    605    1.2    bouyer 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    606    1.2    bouyer 			continue;
    607    1.2    bouyer 		cpu_boot_secondary(ci);
    608  1.123    bouyer 		kcpuset_set(cpus, cpu_index(ci));
    609    1.2    bouyer 	}
    610  1.123    bouyer 	while (!kcpuset_match(cpus, kcpuset_running))
    611  1.123    bouyer 		;
    612  1.123    bouyer 	kcpuset_destroy(cpus);
    613   1.11    cegger 
    614   1.11    cegger 	x86_mp_online = true;
    615    1.2    bouyer }
    616    1.2    bouyer 
    617    1.2    bouyer static void
    618    1.2    bouyer cpu_init_idle_lwp(struct cpu_info *ci)
    619    1.2    bouyer {
    620    1.2    bouyer 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    621   1.37     rmind 	struct pcb *pcb = lwp_getpcb(l);
    622    1.2    bouyer 
    623    1.2    bouyer 	pcb->pcb_cr0 = rcr0();
    624    1.2    bouyer }
    625    1.2    bouyer 
    626    1.2    bouyer void
    627   1.10    cegger cpu_init_idle_lwps(void)
    628    1.2    bouyer {
    629    1.2    bouyer 	struct cpu_info *ci;
    630    1.2    bouyer 	u_long i;
    631    1.2    bouyer 
    632   1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    633   1.38    cegger 		ci = cpu_lookup(i);
    634    1.2    bouyer 		if (ci == NULL)
    635    1.2    bouyer 			continue;
    636    1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    637    1.2    bouyer 			continue;
    638    1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    639    1.2    bouyer 			continue;
    640    1.2    bouyer 		cpu_init_idle_lwp(ci);
    641    1.2    bouyer 	}
    642    1.2    bouyer }
    643    1.2    bouyer 
    644   1.62    cherry static void
    645   1.10    cegger cpu_start_secondary(struct cpu_info *ci)
    646    1.2    bouyer {
    647    1.2    bouyer 	int i;
    648    1.2    bouyer 
    649   1.11    cegger 	aprint_debug_dev(ci->ci_dev, "starting\n");
    650    1.2    bouyer 
    651    1.2    bouyer 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    652   1.62    cherry 
    653   1.62    cherry 	if (CPU_STARTUP(ci, (vaddr_t) cpu_hatch) != 0) {
    654   1.11    cegger 		return;
    655   1.62    cherry 	}
    656    1.2    bouyer 
    657    1.2    bouyer 	/*
    658    1.2    bouyer 	 * wait for it to become ready
    659    1.2    bouyer 	 */
    660   1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    661    1.2    bouyer 		delay(10);
    662    1.2    bouyer 	}
    663   1.11    cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    664    1.9    cegger 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    665    1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    666    1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    667    1.2    bouyer 		Debugger();
    668    1.2    bouyer #endif
    669    1.2    bouyer 	}
    670    1.2    bouyer 
    671    1.2    bouyer 	CPU_START_CLEANUP(ci);
    672    1.2    bouyer }
    673    1.2    bouyer 
    674    1.2    bouyer void
    675   1.10    cegger cpu_boot_secondary(struct cpu_info *ci)
    676    1.2    bouyer {
    677    1.2    bouyer 	int i;
    678   1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    679   1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    680    1.2    bouyer 		delay(10);
    681    1.2    bouyer 	}
    682   1.11    cegger 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    683   1.11    cegger 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
    684    1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    685    1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    686    1.2    bouyer 		Debugger();
    687    1.2    bouyer #endif
    688    1.2    bouyer 	}
    689    1.2    bouyer }
    690    1.2    bouyer 
    691    1.2    bouyer /*
    692   1.62    cherry  * APs end up here immediately after initialisation and VCPUOP_up in
    693  1.145  riastrad  * mp_cpu_start().
    694   1.62    cherry  * At this point, we are running in the idle pcb/idle stack of the new
    695   1.62    cherry  * CPU.  This function jumps to the idle loop and starts looking for
    696  1.145  riastrad  * work.
    697    1.2    bouyer  */
    698   1.62    cherry extern void x86_64_tls_switch(struct lwp *);
    699    1.2    bouyer void
    700    1.2    bouyer cpu_hatch(void *v)
    701    1.2    bouyer {
    702    1.2    bouyer 	struct cpu_info *ci = (struct cpu_info *)v;
    703   1.37     rmind 	struct pcb *pcb;
    704   1.11    cegger 	int s, i;
    705   1.11    cegger 
    706   1.62    cherry 	/* Setup TLS and kernel GS/FS */
    707   1.62    cherry 	cpu_init_msrs(ci, true);
    708  1.139  yamaguch 	cpu_init_idt(ci);
    709   1.62    cherry 	gdt_init_cpu(ci);
    710   1.62    cherry 
    711   1.21        ad 	cpu_probe(ci);
    712   1.11    cegger 
    713   1.62    cherry 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    714    1.2    bouyer 
    715   1.11    cegger 	while ((ci->ci_flags & CPUF_GO) == 0) {
    716   1.11    cegger 		/* Don't use delay, boot CPU may be patching the text. */
    717   1.11    cegger 		for (i = 10000; i != 0; i--)
    718   1.11    cegger 			x86_pause();
    719   1.11    cegger 	}
    720    1.2    bouyer 
    721   1.11    cegger 	/* Because the text may have been patched in x86_patch(). */
    722   1.11    cegger 	x86_flush();
    723   1.58     rmind 	tlbflushg();
    724    1.2    bouyer 
    725   1.11    cegger 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    726    1.2    bouyer 
    727  1.132    bouyer 	KASSERT(ci->ci_curlwp == ci->ci_data.cpu_idlelwp);
    728  1.132    bouyer 	KASSERT(curlwp == ci->ci_data.cpu_idlelwp);
    729   1.37     rmind 	pcb = lwp_getpcb(curlwp);
    730   1.85    cherry 	pcb->pcb_cr3 = pmap_pdirpa(pmap_kernel(), 0);
    731   1.37     rmind 
    732   1.62    cherry 	xen_ipi_init();
    733   1.62    cherry 
    734  1.136        ad 	xen_initclocks();
    735  1.105      maxv 
    736   1.62    cherry #ifdef __x86_64__
    737   1.12    cegger 	fpuinit(ci);
    738   1.12    cegger #endif
    739    1.2    bouyer 
    740    1.2    bouyer 	lldt(GSEL(GLDT_SEL, SEL_KPL));
    741    1.2    bouyer 
    742    1.2    bouyer 	cpu_init(ci);
    743   1.11    cegger 	cpu_get_tsc_freq(ci);
    744    1.2    bouyer 
    745    1.2    bouyer 	s = splhigh();
    746   1.11    cegger 	x86_enable_intr();
    747   1.11    cegger 	splx(s);
    748    1.2    bouyer 
    749   1.62    cherry 	aprint_debug_dev(ci->ci_dev, "running\n");
    750   1.62    cherry 
    751  1.132    bouyer 	KASSERT(ci->ci_curlwp == ci->ci_data.cpu_idlelwp);
    752   1.91     rmind 	idle_loop(NULL);
    753   1.91     rmind 	KASSERT(false);
    754    1.2    bouyer }
    755    1.2    bouyer 
    756    1.2    bouyer #if defined(DDB)
    757    1.2    bouyer 
    758    1.2    bouyer #include <ddb/db_output.h>
    759    1.2    bouyer #include <machine/db_machdep.h>
    760    1.2    bouyer 
    761    1.2    bouyer /*
    762    1.2    bouyer  * Dump CPU information from ddb.
    763    1.2    bouyer  */
    764    1.2    bouyer void
    765    1.2    bouyer cpu_debug_dump(void)
    766    1.2    bouyer {
    767    1.2    bouyer 	struct cpu_info *ci;
    768    1.2    bouyer 	CPU_INFO_ITERATOR cii;
    769    1.2    bouyer 
    770  1.130      maxv 	db_printf("addr		dev	id	flags	ipis	curlwp\n");
    771    1.2    bouyer 	for (CPU_INFO_FOREACH(cii, ci)) {
    772  1.130      maxv 		db_printf("%p	%s	%ld	%x	%x	%10p\n",
    773    1.2    bouyer 		    ci,
    774    1.9    cegger 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    775  1.135    bouyer 		    (long)ci->ci_vcpuid,
    776    1.2    bouyer 		    ci->ci_flags, ci->ci_ipis,
    777  1.130      maxv 		    ci->ci_curlwp);
    778    1.2    bouyer 	}
    779    1.2    bouyer }
    780   1.38    cegger #endif /* DDB */
    781    1.2    bouyer 
    782   1.62    cherry #endif /* MULTIPROCESSOR */
    783   1.62    cherry 
    784   1.62    cherry extern void hypervisor_callback(void);
    785   1.62    cherry extern void failsafe_callback(void);
    786   1.62    cherry #ifdef __x86_64__
    787   1.62    cherry typedef void (vector)(void);
    788   1.62    cherry extern vector Xsyscall, Xsyscall32;
    789   1.62    cherry #endif
    790   1.62    cherry 
    791   1.62    cherry /*
    792   1.62    cherry  * Setup the "trampoline". On Xen, we setup nearly all cpu context
    793   1.62    cherry  * outside a trampoline, so we prototype and call targetip like so:
    794   1.62    cherry  * void targetip(struct cpu_info *);
    795   1.62    cherry  */
    796   1.62    cherry 
    797    1.2    bouyer static void
    798   1.62    cherry gdt_prepframes(paddr_t *frames, vaddr_t base, uint32_t entries)
    799    1.2    bouyer {
    800  1.104   msaitoh 	int i;
    801  1.111    bouyer 	for (i = 0; i < entries; i++) {
    802  1.105      maxv 		frames[i] = ((paddr_t)xpmap_ptetomach(
    803  1.105      maxv 		    (pt_entry_t *)(base + (i << PAGE_SHIFT)))) >> PAGE_SHIFT;
    804   1.62    cherry 
    805   1.62    cherry 		/* Mark Read-only */
    806   1.62    cherry 		pmap_pte_clearbits(kvtopte(base + (i << PAGE_SHIFT)),
    807  1.129      maxv 		    PTE_W);
    808   1.62    cherry 	}
    809   1.62    cherry }
    810   1.62    cherry 
    811   1.62    cherry #ifdef __x86_64__
    812   1.85    cherry extern char *ldtstore;
    813   1.62    cherry 
    814   1.62    cherry static void
    815  1.105      maxv xen_init_amd64_vcpuctxt(struct cpu_info *ci, struct vcpu_guest_context *initctx,
    816  1.105      maxv     void targetrip(struct cpu_info *))
    817   1.62    cherry {
    818   1.62    cherry 	/* page frames to point at GDT */
    819   1.62    cherry 	extern int gdt_size;
    820   1.62    cherry 	paddr_t frames[16];
    821   1.62    cherry 	psize_t gdt_ents;
    822   1.62    cherry 
    823   1.62    cherry 	struct lwp *l;
    824   1.62    cherry 	struct pcb *pcb;
    825   1.62    cherry 
    826   1.62    cherry 	volatile struct vcpu_info *vci;
    827   1.62    cherry 
    828   1.62    cherry 	KASSERT(ci != NULL);
    829   1.62    cherry 	KASSERT(ci != &cpu_info_primary);
    830   1.62    cherry 	KASSERT(initctx != NULL);
    831   1.62    cherry 	KASSERT(targetrip != NULL);
    832   1.62    cherry 
    833  1.105      maxv 	memset(initctx, 0, sizeof(*initctx));
    834   1.62    cherry 
    835  1.104   msaitoh 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT;
    836   1.62    cherry 	KASSERT(gdt_ents <= 16);
    837   1.62    cherry 
    838  1.105      maxv 	gdt_prepframes(frames, (vaddr_t)ci->ci_gdt, gdt_ents);
    839   1.62    cherry 
    840   1.62    cherry 	/* Initialise the vcpu context: We use idle_loop()'s pcb context. */
    841   1.11    cegger 
    842   1.62    cherry 	l = ci->ci_data.cpu_idlelwp;
    843   1.11    cegger 
    844   1.62    cherry 	KASSERT(l != NULL);
    845   1.62    cherry 	pcb = lwp_getpcb(l);
    846   1.62    cherry 	KASSERT(pcb != NULL);
    847   1.11    cegger 
    848   1.62    cherry 	/* resume with interrupts off */
    849   1.62    cherry 	vci = ci->ci_vcpu;
    850   1.62    cherry 	vci->evtchn_upcall_mask = 1;
    851  1.144  riastrad 	__insn_barrier();
    852    1.2    bouyer 
    853   1.62    cherry 	/* resume in kernel-mode */
    854   1.62    cherry 	initctx->flags = VGCF_in_kernel | VGCF_online;
    855    1.2    bouyer 
    856   1.62    cherry 	/* Stack and entry points:
    857   1.62    cherry 	 * We arrange for the stack frame for cpu_hatch() to
    858   1.62    cherry 	 * appear as a callee frame of lwp_trampoline(). Being a
    859   1.62    cherry 	 * leaf frame prevents trampling on any of the MD stack setup
    860   1.62    cherry 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    861   1.62    cherry 	 */
    862    1.2    bouyer 
    863   1.62    cherry 	initctx->user_regs.rdi = (uint64_t) ci; /* targetrip(ci); */
    864   1.62    cherry 	initctx->user_regs.rip = (vaddr_t) targetrip;
    865    1.2    bouyer 
    866   1.62    cherry 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    867   1.11    cegger 
    868   1.62    cherry 	initctx->user_regs.rflags = pcb->pcb_flags;
    869   1.62    cherry 	initctx->user_regs.rsp = pcb->pcb_rsp;
    870   1.11    cegger 
    871   1.62    cherry 	/* Data segments */
    872   1.62    cherry 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
    873   1.62    cherry 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
    874   1.62    cherry 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
    875   1.62    cherry 
    876   1.62    cherry 	/* GDT */
    877  1.105      maxv 	memcpy(initctx->gdt_frames, frames, sizeof(frames));
    878   1.62    cherry 	initctx->gdt_ents = gdt_ents;
    879   1.62    cherry 
    880   1.62    cherry 	/* LDT */
    881  1.105      maxv 	initctx->ldt_base = (unsigned long)ldtstore;
    882   1.62    cherry 	initctx->ldt_ents = LDT_SIZE >> 3;
    883   1.62    cherry 
    884   1.62    cherry 	/* Kernel context state */
    885   1.62    cherry 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
    886   1.62    cherry 	initctx->kernel_sp = pcb->pcb_rsp0;
    887   1.62    cherry 	initctx->ctrlreg[0] = pcb->pcb_cr0;
    888   1.62    cherry 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
    889  1.105      maxv 	initctx->ctrlreg[2] = (vaddr_t)targetrip;
    890  1.105      maxv 	/*
    891   1.62    cherry 	 * Use pmap_kernel() L4 PD directly, until we setup the
    892   1.62    cherry 	 * per-cpu L4 PD in pmap_cpu_init_late()
    893    1.2    bouyer 	 */
    894   1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_kpm_pdirpa)));
    895   1.62    cherry 	initctx->ctrlreg[4] = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT;
    896    1.2    bouyer 
    897   1.62    cherry 	/* Xen callbacks */
    898  1.105      maxv 	initctx->event_callback_eip = (unsigned long)hypervisor_callback;
    899  1.105      maxv 	initctx->failsafe_callback_eip = (unsigned long)failsafe_callback;
    900  1.105      maxv 	initctx->syscall_callback_eip = (unsigned long)Xsyscall;
    901   1.62    cherry 
    902   1.62    cherry 	return;
    903    1.2    bouyer }
    904   1.62    cherry #else /* i386 */
    905  1.108      maxv extern union descriptor *ldtstore;
    906   1.62    cherry extern void Xsyscall(void);
    907   1.62    cherry 
    908   1.11    cegger static void
    909  1.105      maxv xen_init_i386_vcpuctxt(struct cpu_info *ci, struct vcpu_guest_context *initctx,
    910  1.105      maxv     void targeteip(struct cpu_info *))
    911   1.62    cherry {
    912   1.62    cherry 	/* page frames to point at GDT */
    913   1.62    cherry 	extern int gdt_size;
    914   1.62    cherry 	paddr_t frames[16];
    915   1.62    cherry 	psize_t gdt_ents;
    916   1.62    cherry 
    917   1.62    cherry 	struct lwp *l;
    918   1.62    cherry 	struct pcb *pcb;
    919   1.62    cherry 
    920   1.62    cherry 	volatile struct vcpu_info *vci;
    921   1.62    cherry 
    922   1.62    cherry 	KASSERT(ci != NULL);
    923   1.62    cherry 	KASSERT(ci != &cpu_info_primary);
    924   1.62    cherry 	KASSERT(initctx != NULL);
    925   1.62    cherry 	KASSERT(targeteip != NULL);
    926   1.62    cherry 
    927  1.105      maxv 	memset(initctx, 0, sizeof(*initctx));
    928   1.11    cegger 
    929   1.85    cherry 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT;
    930   1.62    cherry 	KASSERT(gdt_ents <= 16);
    931    1.2    bouyer 
    932  1.105      maxv 	gdt_prepframes(frames, (vaddr_t)ci->ci_gdt, gdt_ents);
    933    1.2    bouyer 
    934  1.145  riastrad 	/*
    935  1.145  riastrad 	 * Initialise the vcpu context:
    936   1.62    cherry 	 * We use this cpu's idle_loop() pcb context.
    937   1.11    cegger 	 */
    938   1.11    cegger 
    939   1.62    cherry 	l = ci->ci_data.cpu_idlelwp;
    940   1.62    cherry 
    941   1.62    cherry 	KASSERT(l != NULL);
    942   1.62    cherry 	pcb = lwp_getpcb(l);
    943   1.62    cherry 	KASSERT(pcb != NULL);
    944   1.62    cherry 
    945   1.62    cherry 	/* resume with interrupts off */
    946   1.62    cherry 	vci = ci->ci_vcpu;
    947   1.62    cherry 	vci->evtchn_upcall_mask = 1;
    948  1.144  riastrad 	__insn_barrier();
    949   1.62    cherry 
    950   1.62    cherry 	/* resume in kernel-mode */
    951   1.62    cherry 	initctx->flags = VGCF_in_kernel | VGCF_online;
    952   1.62    cherry 
    953   1.62    cherry 	/* Stack frame setup for cpu_hatch():
    954   1.62    cherry 	 * We arrange for the stack frame for cpu_hatch() to
    955   1.62    cherry 	 * appear as a callee frame of lwp_trampoline(). Being a
    956   1.62    cherry 	 * leaf frame prevents trampling on any of the MD stack setup
    957   1.62    cherry 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    958    1.2    bouyer 	 */
    959    1.2    bouyer 
    960   1.62    cherry 	initctx->user_regs.esp = pcb->pcb_esp - 4; /* Leave word for
    961   1.62    cherry 						      arg1 */
    962  1.105      maxv 	{
    963  1.105      maxv 		/* targeteip(ci); */
    964  1.105      maxv 		uint32_t *arg = (uint32_t *)initctx->user_regs.esp;
    965  1.105      maxv 		arg[1] = (uint32_t)ci; /* arg1 */
    966   1.62    cherry 	}
    967    1.2    bouyer 
    968  1.105      maxv 	initctx->user_regs.eip = (vaddr_t)targeteip;
    969   1.62    cherry 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    970   1.62    cherry 	initctx->user_regs.eflags |= pcb->pcb_iopl;
    971   1.62    cherry 
    972   1.62    cherry 	/* Data segments */
    973   1.62    cherry 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
    974   1.62    cherry 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
    975   1.62    cherry 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
    976   1.62    cherry 	initctx->user_regs.fs = GSEL(GDATA_SEL, SEL_KPL);
    977   1.62    cherry 
    978   1.62    cherry 	/* GDT */
    979  1.105      maxv 	memcpy(initctx->gdt_frames, frames, sizeof(frames));
    980   1.62    cherry 	initctx->gdt_ents = gdt_ents;
    981   1.62    cherry 
    982   1.62    cherry 	/* LDT */
    983  1.108      maxv 	initctx->ldt_base = (unsigned long)ldtstore;
    984   1.62    cherry 	initctx->ldt_ents = NLDT;
    985   1.62    cherry 
    986   1.62    cherry 	/* Kernel context state */
    987   1.62    cherry 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
    988   1.62    cherry 	initctx->kernel_sp = pcb->pcb_esp0;
    989   1.62    cherry 	initctx->ctrlreg[0] = pcb->pcb_cr0;
    990   1.62    cherry 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
    991  1.105      maxv 	initctx->ctrlreg[2] = (vaddr_t)targeteip;
    992   1.70    cherry 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_pae_l3_pdirpa)));
    993  1.105      maxv 	initctx->ctrlreg[4] = /* CR4_PAE | */CR4_OSFXSR | CR4_OSXMMEXCPT;
    994    1.2    bouyer 
    995   1.62    cherry 	/* Xen callbacks */
    996  1.105      maxv 	initctx->event_callback_eip = (unsigned long)hypervisor_callback;
    997   1.62    cherry 	initctx->event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
    998  1.105      maxv 	initctx->failsafe_callback_eip = (unsigned long)failsafe_callback;
    999   1.62    cherry 	initctx->failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
   1000   1.45     rmind 
   1001   1.62    cherry 	return;
   1002   1.62    cherry }
   1003   1.62    cherry #endif /* __x86_64__ */
   1004   1.45     rmind 
   1005   1.62    cherry int
   1006   1.62    cherry mp_cpu_start(struct cpu_info *ci, vaddr_t target)
   1007   1.62    cherry {
   1008   1.62    cherry 	int hyperror;
   1009  1.137  jdolecek 	struct vcpu_guest_context *vcpuctx;
   1010    1.2    bouyer 
   1011   1.62    cherry 	KASSERT(ci != NULL);
   1012   1.62    cherry 	KASSERT(ci != &cpu_info_primary);
   1013   1.62    cherry 	KASSERT(ci->ci_flags & CPUF_AP);
   1014   1.62    cherry 
   1015  1.137  jdolecek 	vcpuctx = kmem_alloc(sizeof(*vcpuctx), KM_SLEEP);
   1016  1.137  jdolecek 
   1017   1.62    cherry #ifdef __x86_64__
   1018  1.137  jdolecek 	xen_init_amd64_vcpuctxt(ci, vcpuctx, (void (*)(struct cpu_info *))target);
   1019  1.105      maxv #else
   1020  1.137  jdolecek 	xen_init_i386_vcpuctxt(ci, vcpuctx, (void (*)(struct cpu_info *))target);
   1021  1.105      maxv #endif
   1022   1.62    cherry 
   1023   1.62    cherry 	/* Initialise the given vcpu to execute cpu_hatch(ci); */
   1024  1.137  jdolecek 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_vcpuid, vcpuctx))) {
   1025   1.62    cherry 		aprint_error(": context initialisation failed. errno = %d\n", hyperror);
   1026  1.137  jdolecek 		goto out;
   1027   1.62    cherry 	}
   1028   1.62    cherry 
   1029   1.62    cherry 	/* Start it up */
   1030   1.62    cherry 
   1031   1.70    cherry 	/* First bring it down */
   1032  1.135    bouyer 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_vcpuid, NULL))) {
   1033   1.62    cherry 		aprint_error(": VCPUOP_down hypervisor command failed. errno = %d\n", hyperror);
   1034  1.137  jdolecek 		goto out;
   1035   1.62    cherry 	}
   1036   1.62    cherry 
   1037  1.135    bouyer 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_up, ci->ci_vcpuid, NULL))) {
   1038   1.62    cherry 		aprint_error(": VCPUOP_up hypervisor command failed. errno = %d\n", hyperror);
   1039  1.137  jdolecek 		goto out;
   1040   1.62    cherry 	}
   1041    1.2    bouyer 
   1042   1.62    cherry 	if (!vcpu_is_up(ci)) {
   1043   1.62    cherry 		aprint_error(": did not come up\n");
   1044  1.137  jdolecek 		hyperror = -1;
   1045  1.137  jdolecek 		goto out;
   1046    1.2    bouyer 	}
   1047   1.62    cherry 
   1048  1.137  jdolecek out:
   1049  1.137  jdolecek 	kmem_free(vcpuctx, sizeof(*vcpuctx));
   1050  1.137  jdolecek 	return hyperror;
   1051    1.2    bouyer }
   1052    1.2    bouyer 
   1053    1.2    bouyer void
   1054    1.2    bouyer mp_cpu_start_cleanup(struct cpu_info *ci)
   1055    1.2    bouyer {
   1056   1.62    cherry 	if (vcpu_is_up(ci)) {
   1057   1.62    cherry 		aprint_debug_dev(ci->ci_dev, "is started.\n");
   1058  1.105      maxv 	} else {
   1059   1.62    cherry 		aprint_error_dev(ci->ci_dev, "did not start up.\n");
   1060   1.62    cherry 	}
   1061    1.2    bouyer }
   1062    1.2    bouyer 
   1063    1.2    bouyer void
   1064    1.3    bouyer cpu_init_msrs(struct cpu_info *ci, bool full)
   1065    1.2    bouyer {
   1066   1.43       jym #ifdef __x86_64__
   1067    1.3    bouyer 	if (full) {
   1068  1.105      maxv 		HYPERVISOR_set_segment_base(SEGBASE_FS, 0);
   1069  1.105      maxv 		HYPERVISOR_set_segment_base(SEGBASE_GS_KERNEL, (uint64_t)ci);
   1070  1.105      maxv 		HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 0);
   1071    1.3    bouyer 	}
   1072  1.105      maxv #endif
   1073   1.44       jym 
   1074   1.44       jym 	if (cpu_feature[2] & CPUID_NOX)
   1075   1.44       jym 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
   1076    1.2    bouyer }
   1077    1.2    bouyer 
   1078   1.95  christos void
   1079   1.95  christos cpu_offline_md(void)
   1080   1.95  christos {
   1081  1.130      maxv 	return;
   1082   1.95  christos }
   1083   1.95  christos 
   1084  1.105      maxv void
   1085    1.2    bouyer cpu_get_tsc_freq(struct cpu_info *ci)
   1086    1.2    bouyer {
   1087   1.62    cherry 	uint32_t vcpu_tversion;
   1088   1.16    cegger 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
   1089   1.62    cherry 
   1090   1.62    cherry 	vcpu_tversion = tinfo->version;
   1091   1.62    cherry 	while (tinfo->version == vcpu_tversion); /* Wait for a time update. XXX: timeout ? */
   1092   1.62    cherry 
   1093    1.2    bouyer 	uint64_t freq = 1000000000ULL << 32;
   1094    1.2    bouyer 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
   1095  1.105      maxv 	if (tinfo->tsc_shift < 0)
   1096    1.2    bouyer 		freq = freq << -tinfo->tsc_shift;
   1097    1.2    bouyer 	else
   1098    1.2    bouyer 		freq = freq >> tinfo->tsc_shift;
   1099   1.20        ad 	ci->ci_data.cpu_cc_freq = freq;
   1100    1.2    bouyer }
   1101   1.19     joerg 
   1102   1.47       jym /*
   1103   1.47       jym  * Loads pmap for the current CPU.
   1104   1.47       jym  */
   1105   1.47       jym void
   1106   1.81    bouyer cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap)
   1107   1.47       jym {
   1108   1.81    bouyer 	struct cpu_info *ci = curcpu();
   1109   1.92     rmind 	cpuid_t cid = cpu_index(ci);
   1110  1.125      maxv 	int i;
   1111   1.81    bouyer 
   1112  1.143  riastrad 	KASSERT(kpreempt_disabled());
   1113  1.124      maxv 	KASSERT(pmap != pmap_kernel());
   1114  1.124      maxv 
   1115   1.81    bouyer 	mutex_enter(&ci->ci_kpm_mtx);
   1116   1.93       jym 	/* make new pmap visible to xen_kpm_sync() */
   1117   1.92     rmind 	kcpuset_atomic_set(pmap->pm_xen_ptp_cpus, cid);
   1118  1.105      maxv 
   1119   1.47       jym #ifdef __x86_64__
   1120  1.125      maxv 	pd_entry_t *new_pgd;
   1121  1.125      maxv 	paddr_t l4_pd_ma;
   1122   1.81    bouyer 
   1123  1.125      maxv 	l4_pd_ma = xpmap_ptom_masked(ci->ci_kpm_pdirpa);
   1124   1.47       jym 
   1125  1.125      maxv 	/*
   1126  1.125      maxv 	 * Map user space address in kernel space and load
   1127  1.125      maxv 	 * user cr3
   1128  1.125      maxv 	 */
   1129  1.125      maxv 	new_pgd = pmap->pm_pdir;
   1130  1.125      maxv 	KASSERT(pmap == ci->ci_pmap);
   1131   1.70    cherry 
   1132  1.125      maxv 	/* Copy user pmap L4 PDEs (in user addr. range) to per-cpu L4 */
   1133  1.126      maxv 	for (i = 0; i < PDIR_SLOT_USERLIM; i++) {
   1134  1.125      maxv 		KASSERT(pmap != pmap_kernel() || new_pgd[i] == 0);
   1135  1.125      maxv 		if (ci->ci_kpm_pdir[i] != new_pgd[i]) {
   1136  1.125      maxv 			xpq_queue_pte_update(l4_pd_ma + i * sizeof(pd_entry_t),
   1137  1.125      maxv 			    new_pgd[i]);
   1138   1.81    bouyer 		}
   1139  1.125      maxv 	}
   1140   1.70    cherry 
   1141  1.125      maxv 	xen_set_user_pgd(pmap_pdirpa(pmap, 0));
   1142  1.125      maxv 	ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);
   1143  1.125      maxv #else
   1144  1.125      maxv 	paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);
   1145  1.125      maxv 	/* don't update the kernel L3 slot */
   1146  1.125      maxv 	for (i = 0; i < PDP_SIZE - 1; i++) {
   1147  1.125      maxv 		xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t),
   1148  1.129      maxv 		    xpmap_ptom(pmap->pm_pdirpa[i]) | PTE_P);
   1149   1.70    cherry 	}
   1150  1.124      maxv #endif
   1151   1.70    cherry 
   1152  1.125      maxv 	tlbflush();
   1153  1.125      maxv 
   1154   1.93       jym 	/* old pmap no longer visible to xen_kpm_sync() */
   1155   1.92     rmind 	if (oldpmap != pmap_kernel()) {
   1156   1.92     rmind 		kcpuset_atomic_clear(oldpmap->pm_xen_ptp_cpus, cid);
   1157   1.92     rmind 	}
   1158   1.81    bouyer 	mutex_exit(&ci->ci_kpm_mtx);
   1159   1.47       jym }
   1160   1.61    cherry 
   1161  1.105      maxv /*
   1162  1.105      maxv  * pmap_cpu_init_late: perform late per-CPU initialization.
   1163  1.145  riastrad  *
   1164  1.105      maxv  * Short note about percpu PDIR pages. Both the PAE and __x86_64__ architectures
   1165  1.105      maxv  * have per-cpu PDIR tables, for two different reasons:
   1166  1.105      maxv  *  - on PAE, this is to get around Xen's pagetable setup constraints (multiple
   1167  1.105      maxv  *    L3[3]s cannot point to the same L2 - Xen will refuse to pin a table set up
   1168  1.105      maxv  *    this way).
   1169  1.105      maxv  *  - on __x86_64__, this is for multiple CPUs to map in different user pmaps
   1170  1.105      maxv  *    (see cpu_load_pmap()).
   1171  1.105      maxv  *
   1172  1.105      maxv  * What this means for us is that the PDIR of the pmap_kernel() is considered
   1173  1.145  riastrad  * to be a canonical "SHADOW" PDIR with the following properties:
   1174  1.105      maxv  *  - its recursive mapping points to itself
   1175  1.105      maxv  *  - per-cpu recursive mappings point to themselves on __x86_64__
   1176  1.105      maxv  *  - per-cpu L4 pages' kernel entries are expected to be in sync with
   1177  1.105      maxv  *    the shadow
   1178  1.105      maxv  */
   1179   1.70    cherry 
   1180   1.70    cherry void
   1181   1.70    cherry pmap_cpu_init_late(struct cpu_info *ci)
   1182   1.70    cherry {
   1183  1.125      maxv 	int i;
   1184  1.125      maxv 
   1185   1.70    cherry 	/*
   1186   1.70    cherry 	 * The BP has already its own PD page allocated during early
   1187   1.70    cherry 	 * MD startup.
   1188   1.70    cherry 	 */
   1189   1.70    cherry 
   1190  1.124      maxv #ifdef __x86_64__
   1191   1.78    cherry 	/* Setup per-cpu normal_pdes */
   1192   1.78    cherry 	extern pd_entry_t * const normal_pdes[];
   1193   1.78    cherry 	for (i = 0;i < PTP_LEVELS - 1;i++) {
   1194   1.78    cherry 		ci->ci_normal_pdes[i] = normal_pdes[i];
   1195   1.78    cherry 	}
   1196  1.124      maxv #endif
   1197   1.78    cherry 
   1198   1.70    cherry 	if (ci == &cpu_info_primary)
   1199   1.70    cherry 		return;
   1200   1.70    cherry 
   1201   1.70    cherry 	KASSERT(ci != NULL);
   1202   1.70    cherry 
   1203  1.124      maxv #if defined(i386)
   1204   1.73    cherry 	cpu_alloc_l3_page(ci);
   1205   1.70    cherry 	KASSERT(ci->ci_pae_l3_pdirpa != 0);
   1206   1.70    cherry 
   1207   1.70    cherry 	/* Initialise L2 entries 0 - 2: Point them to pmap_kernel() */
   1208  1.125      maxv 	for (i = 0; i < PDP_SIZE - 1; i++) {
   1209   1.73    cherry 		ci->ci_pae_l3_pdir[i] =
   1210  1.129      maxv 		    xpmap_ptom_masked(pmap_kernel()->pm_pdirpa[i]) | PTE_P;
   1211   1.73    cherry 	}
   1212  1.124      maxv #endif
   1213   1.70    cherry 
   1214   1.70    cherry 	ci->ci_kpm_pdir = (pd_entry_t *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
   1215   1.70    cherry 	    UVM_KMF_WIRED | UVM_KMF_ZERO | UVM_KMF_NOWAIT);
   1216   1.70    cherry 
   1217   1.70    cherry 	if (ci->ci_kpm_pdir == NULL) {
   1218   1.70    cherry 		panic("%s: failed to allocate L4 per-cpu PD for CPU %d\n",
   1219  1.105      maxv 		    __func__, cpu_index(ci));
   1220   1.70    cherry 	}
   1221  1.105      maxv 	ci->ci_kpm_pdirpa = vtophys((vaddr_t)ci->ci_kpm_pdir);
   1222   1.70    cherry 	KASSERT(ci->ci_kpm_pdirpa != 0);
   1223   1.70    cherry 
   1224  1.124      maxv #ifdef __x86_64__
   1225  1.106      maxv 	extern pt_entry_t xpmap_pg_nx;
   1226   1.70    cherry 
   1227  1.106      maxv 	/* Copy over the pmap_kernel() shadow L4 entries */
   1228   1.70    cherry 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir, PAGE_SIZE);
   1229   1.70    cherry 
   1230   1.70    cherry 	/* Recursive kernel mapping */
   1231  1.105      maxv 	ci->ci_kpm_pdir[PDIR_SLOT_PTE] = xpmap_ptom_masked(ci->ci_kpm_pdirpa)
   1232  1.129      maxv 	    | PTE_P | xpmap_pg_nx;
   1233  1.124      maxv #else
   1234  1.106      maxv 	/* Copy over the pmap_kernel() shadow L2 entries */
   1235  1.105      maxv 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir + PDIR_SLOT_KERN,
   1236  1.105      maxv 	    nkptp[PTP_LEVELS - 1] * sizeof(pd_entry_t));
   1237  1.106      maxv #endif
   1238   1.70    cherry 
   1239  1.105      maxv 	/* Xen wants a RO pdir. */
   1240   1.83    bouyer 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_kpm_pdir,
   1241   1.83    bouyer 	    (vaddr_t)ci->ci_kpm_pdir + PAGE_SIZE, VM_PROT_READ);
   1242   1.83    bouyer 	pmap_update(pmap_kernel());
   1243  1.124      maxv 
   1244  1.124      maxv #ifdef __x86_64__
   1245  1.124      maxv 	xpq_queue_pin_l4_table(xpmap_ptom_masked(ci->ci_kpm_pdirpa));
   1246  1.124      maxv #else
   1247  1.105      maxv 	/*
   1248  1.105      maxv 	 * Initialize L3 entry 3. This mapping is shared across all pmaps and is
   1249  1.105      maxv 	 * static, ie: loading a new pmap will not update this entry.
   1250   1.70    cherry 	 */
   1251  1.129      maxv 	ci->ci_pae_l3_pdir[3] = xpmap_ptom_masked(ci->ci_kpm_pdirpa) | PTE_P;
   1252   1.70    cherry 
   1253  1.105      maxv 	/* Xen wants a RO L3. */
   1254   1.83    bouyer 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_pae_l3_pdir,
   1255   1.83    bouyer 	    (vaddr_t)ci->ci_pae_l3_pdir + PAGE_SIZE, VM_PROT_READ);
   1256   1.83    bouyer 	pmap_update(pmap_kernel());
   1257   1.70    cherry 
   1258   1.70    cherry 	xpq_queue_pin_l3_table(xpmap_ptom_masked(ci->ci_pae_l3_pdirpa));
   1259  1.124      maxv #endif
   1260   1.70    cherry }
   1261   1.70    cherry 
   1262   1.61    cherry /*
   1263   1.61    cherry  * Notify all other cpus to halt.
   1264   1.61    cherry  */
   1265   1.61    cherry 
   1266   1.61    cherry void
   1267   1.61    cherry cpu_broadcast_halt(void)
   1268   1.61    cherry {
   1269   1.61    cherry 	xen_broadcast_ipi(XEN_IPI_HALT);
   1270   1.61    cherry }
   1271   1.61    cherry 
   1272   1.61    cherry /*
   1273  1.131        ad  * Send a dummy ipi to a cpu, and raise an AST on the running LWP.
   1274   1.61    cherry  */
   1275   1.61    cherry 
   1276   1.61    cherry void
   1277   1.61    cherry cpu_kick(struct cpu_info *ci)
   1278   1.61    cherry {
   1279  1.131        ad 	(void)xen_send_ipi(ci, XEN_IPI_AST);
   1280   1.61    cherry }
   1281