Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.51
      1  1.51  uebayasi /*	$NetBSD: cpu.c,v 1.51 2010/11/06 11:46:04 uebayasi Exp $	*/
      2   1.2    bouyer /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp  */
      3   1.2    bouyer 
      4   1.2    bouyer /*-
      5   1.2    bouyer  * Copyright (c) 2000 The NetBSD Foundation, Inc.
      6  1.19     joerg  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
      7   1.2    bouyer  * All rights reserved.
      8   1.2    bouyer  *
      9   1.2    bouyer  * This code is derived from software contributed to The NetBSD Foundation
     10   1.2    bouyer  * by RedBack Networks Inc.
     11   1.2    bouyer  *
     12   1.2    bouyer  * Author: Bill Sommerfeld
     13   1.2    bouyer  *
     14   1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     15   1.2    bouyer  * modification, are permitted provided that the following conditions
     16   1.2    bouyer  * are met:
     17   1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     18   1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     19   1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     20   1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     21   1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     22   1.2    bouyer  *
     23   1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     24   1.2    bouyer  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     25   1.2    bouyer  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     26   1.2    bouyer  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     27   1.2    bouyer  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     28   1.2    bouyer  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     29   1.2    bouyer  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     30   1.2    bouyer  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     31   1.2    bouyer  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     32   1.2    bouyer  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     33   1.2    bouyer  * POSSIBILITY OF SUCH DAMAGE.
     34   1.2    bouyer  */
     35   1.2    bouyer 
     36   1.2    bouyer /*
     37   1.2    bouyer  * Copyright (c) 1999 Stefan Grefen
     38   1.2    bouyer  *
     39   1.2    bouyer  * Redistribution and use in source and binary forms, with or without
     40   1.2    bouyer  * modification, are permitted provided that the following conditions
     41   1.2    bouyer  * are met:
     42   1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     43   1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     44   1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     45   1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     46   1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     47   1.2    bouyer  * 3. All advertising materials mentioning features or use of this software
     48   1.2    bouyer  *    must display the following acknowledgement:
     49   1.2    bouyer  *      This product includes software developed by the NetBSD
     50   1.2    bouyer  *      Foundation, Inc. and its contributors.
     51   1.2    bouyer  * 4. Neither the name of The NetBSD Foundation nor the names of its
     52   1.2    bouyer  *    contributors may be used to endorse or promote products derived
     53   1.2    bouyer  *    from this software without specific prior written permission.
     54   1.2    bouyer  *
     55   1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     56   1.2    bouyer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     57   1.2    bouyer  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     58   1.2    bouyer  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     59   1.2    bouyer  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     60   1.2    bouyer  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     61   1.2    bouyer  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     62   1.2    bouyer  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     63   1.2    bouyer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     64   1.2    bouyer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     65   1.2    bouyer  * SUCH DAMAGE.
     66   1.2    bouyer  */
     67   1.2    bouyer 
     68   1.2    bouyer #include <sys/cdefs.h>
     69  1.51  uebayasi __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.51 2010/11/06 11:46:04 uebayasi Exp $");
     70   1.2    bouyer 
     71   1.2    bouyer #include "opt_ddb.h"
     72   1.2    bouyer #include "opt_multiprocessor.h"
     73   1.2    bouyer #include "opt_mpbios.h"		/* for MPDEBUG */
     74   1.2    bouyer #include "opt_mtrr.h"
     75   1.2    bouyer #include "opt_xen.h"
     76   1.2    bouyer 
     77   1.2    bouyer #include "lapic.h"
     78   1.2    bouyer #include "ioapic.h"
     79   1.2    bouyer 
     80   1.2    bouyer #include <sys/param.h>
     81   1.2    bouyer #include <sys/proc.h>
     82   1.2    bouyer #include <sys/systm.h>
     83   1.2    bouyer #include <sys/device.h>
     84  1.31    cegger #include <sys/kmem.h>
     85  1.11    cegger #include <sys/cpu.h>
     86  1.11    cegger #include <sys/atomic.h>
     87  1.32    cegger #include <sys/reboot.h>
     88   1.2    bouyer 
     89  1.51  uebayasi #include <uvm/uvm.h>
     90   1.2    bouyer 
     91   1.2    bouyer #include <machine/cpufunc.h>
     92   1.2    bouyer #include <machine/cpuvar.h>
     93   1.2    bouyer #include <machine/pmap.h>
     94   1.2    bouyer #include <machine/vmparam.h>
     95   1.2    bouyer #include <machine/mpbiosvar.h>
     96   1.2    bouyer #include <machine/pcb.h>
     97   1.2    bouyer #include <machine/specialreg.h>
     98   1.2    bouyer #include <machine/segments.h>
     99   1.2    bouyer #include <machine/gdt.h>
    100   1.2    bouyer #include <machine/mtrr.h>
    101   1.2    bouyer #include <machine/pio.h>
    102   1.2    bouyer 
    103   1.2    bouyer #include <xen/vcpuvar.h>
    104   1.2    bouyer 
    105   1.2    bouyer #if NLAPIC > 0
    106   1.2    bouyer #include <machine/apicvar.h>
    107   1.2    bouyer #include <machine/i82489reg.h>
    108   1.2    bouyer #include <machine/i82489var.h>
    109   1.2    bouyer #endif
    110   1.2    bouyer 
    111   1.2    bouyer #include <dev/ic/mc146818reg.h>
    112   1.2    bouyer #include <dev/isa/isareg.h>
    113   1.2    bouyer 
    114  1.38    cegger #if MAXCPUS > 32
    115  1.38    cegger #error cpu_info contains 32bit bitmasks
    116  1.38    cegger #endif
    117  1.27        ad 
    118  1.10    cegger int     cpu_match(device_t, cfdata_t, void *);
    119  1.10    cegger void    cpu_attach(device_t, device_t, void *);
    120  1.10    cegger int     vcpu_match(device_t, cfdata_t, void *);
    121  1.10    cegger void    vcpu_attach(device_t, device_t, void *);
    122  1.10    cegger void    cpu_attach_common(device_t, device_t, void *);
    123   1.8    dogcow void	cpu_offline_md(void);
    124   1.2    bouyer 
    125   1.2    bouyer struct cpu_softc {
    126  1.10    cegger 	device_t sc_dev;		/* device tree glue */
    127   1.2    bouyer 	struct cpu_info *sc_info;	/* pointer to CPU info */
    128  1.32    cegger 	bool sc_wasonline;
    129   1.2    bouyer };
    130   1.2    bouyer 
    131   1.5     joerg int mp_cpu_start(struct cpu_info *, paddr_t);
    132   1.2    bouyer void mp_cpu_start_cleanup(struct cpu_info *);
    133   1.2    bouyer const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    134   1.2    bouyer 				      mp_cpu_start_cleanup };
    135   1.2    bouyer 
    136  1.10    cegger CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc),
    137   1.2    bouyer     cpu_match, cpu_attach, NULL, NULL);
    138  1.10    cegger CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
    139   1.2    bouyer     vcpu_match, vcpu_attach, NULL, NULL);
    140   1.2    bouyer 
    141   1.2    bouyer /*
    142   1.2    bouyer  * Statically-allocated CPU info for the primary CPU (or the only
    143   1.2    bouyer  * CPU, on uniprocessors).  The CPU info list is initialized to
    144   1.2    bouyer  * point at it.
    145   1.2    bouyer  */
    146   1.2    bouyer #ifdef TRAPLOG
    147   1.2    bouyer #include <machine/tlog.h>
    148   1.2    bouyer struct tlog tlog_primary;
    149   1.2    bouyer #endif
    150  1.38    cegger struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    151   1.7    bouyer 	.ci_dev = 0,
    152   1.2    bouyer 	.ci_self = &cpu_info_primary,
    153   1.4    bouyer 	.ci_idepth = -1,
    154   1.2    bouyer 	.ci_curlwp = &lwp0,
    155  1.25        ad 	.ci_curldt = -1,
    156   1.2    bouyer #ifdef TRAPLOG
    157   1.2    bouyer 	.ci_tlog = &tlog_primary,
    158   1.2    bouyer #endif
    159   1.2    bouyer 
    160   1.2    bouyer };
    161  1.38    cegger struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    162   1.7    bouyer 	.ci_dev = 0,
    163   1.2    bouyer 	.ci_self = &phycpu_info_primary,
    164   1.2    bouyer };
    165   1.2    bouyer 
    166   1.2    bouyer struct cpu_info *cpu_info_list = &cpu_info_primary;
    167  1.38    cegger struct cpu_info *phycpu_info_list = &phycpu_info_primary;
    168   1.2    bouyer 
    169   1.2    bouyer static void	cpu_set_tss_gates(struct cpu_info *ci);
    170   1.2    bouyer 
    171  1.11    cegger uint32_t cpus_attached = 0;
    172  1.11    cegger uint32_t cpus_running = 0;
    173  1.11    cegger 
    174  1.38    cegger uint32_t phycpus_attached = 0;
    175  1.38    cegger uint32_t phycpus_running = 0;
    176  1.38    cegger 
    177  1.43       jym uint32_t cpu_feature[5]; /* X86 CPUID feature bits
    178  1.43       jym 			  *	[0] basic features %edx
    179  1.43       jym 			  *	[1] basic features %ecx
    180  1.43       jym 			  *	[2] extended features %edx
    181  1.43       jym 			  *	[3] extended features %ecx
    182  1.43       jym 			  *	[4] VIA padlock features
    183  1.43       jym 			  */
    184  1.43       jym 
    185  1.11    cegger bool x86_mp_online;
    186  1.11    cegger paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    187   1.2    bouyer 
    188  1.38    cegger #if defined(MULTIPROCESSOR)
    189   1.2    bouyer void    	cpu_hatch(void *);
    190   1.2    bouyer static void    	cpu_boot_secondary(struct cpu_info *ci);
    191   1.2    bouyer static void    	cpu_start_secondary(struct cpu_info *ci);
    192   1.2    bouyer static void	cpu_copy_trampoline(void);
    193   1.2    bouyer 
    194   1.2    bouyer /*
    195   1.2    bouyer  * Runs once per boot once multiprocessor goo has been detected and
    196   1.2    bouyer  * the local APIC on the boot processor has been mapped.
    197   1.2    bouyer  *
    198   1.2    bouyer  * Called from lapic_boot_init() (from mpbios_scan()).
    199   1.2    bouyer  */
    200   1.2    bouyer void
    201  1.10    cegger cpu_init_first(void)
    202   1.2    bouyer {
    203   1.2    bouyer 
    204  1.38    cegger 	cpu_info_primary.ci_cpuid = lapic_cpu_number();
    205   1.2    bouyer 	cpu_copy_trampoline();
    206   1.2    bouyer }
    207  1.38    cegger #endif	/* MULTIPROCESSOR */
    208   1.2    bouyer 
    209   1.2    bouyer int
    210  1.10    cegger cpu_match(device_t parent, cfdata_t match, void *aux)
    211   1.2    bouyer {
    212   1.2    bouyer 
    213   1.2    bouyer 	return 1;
    214   1.2    bouyer }
    215   1.2    bouyer 
    216   1.2    bouyer void
    217  1.10    cegger cpu_attach(device_t parent, device_t self, void *aux)
    218   1.2    bouyer {
    219  1.10    cegger 	struct cpu_softc *sc = device_private(self);
    220   1.2    bouyer 	struct cpu_attach_args *caa = aux;
    221   1.2    bouyer 	struct cpu_info *ci;
    222  1.34    cegger 	uintptr_t ptr;
    223  1.38    cegger 	static bool again = false;
    224   1.2    bouyer 
    225  1.10    cegger 	sc->sc_dev = self;
    226  1.10    cegger 
    227  1.38    cegger 	if (phycpus_attached == ~0) {
    228  1.34    cegger 		aprint_error(": increase MAXCPUS\n");
    229  1.34    cegger 		return;
    230  1.34    cegger 	}
    231  1.34    cegger 
    232   1.2    bouyer 	/*
    233   1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    234   1.2    bouyer 	 * structure, otherwise use the primary's.
    235   1.2    bouyer 	 */
    236   1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    237  1.32    cegger 		if ((boothowto & RB_MD1) != 0) {
    238  1.32    cegger 			aprint_error(": multiprocessor boot disabled\n");
    239  1.32    cegger 			if (!pmf_device_register(self, NULL, NULL))
    240  1.32    cegger 				aprint_error_dev(self,
    241  1.32    cegger 				   "couldn't establish power handler\n");
    242  1.32    cegger 			return;
    243  1.32    cegger 		}
    244  1.32    cegger 		aprint_naive(": Application Processor\n");
    245  1.34    cegger 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    246  1.34    cegger 		    KM_SLEEP);
    247  1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    248  1.24        ad 		ci->ci_curldt = -1;
    249   1.2    bouyer 	} else {
    250  1.32    cegger 		aprint_naive(": %s Processor\n",
    251  1.32    cegger 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    252   1.2    bouyer 		ci = &phycpu_info_primary;
    253   1.2    bouyer 	}
    254   1.2    bouyer 
    255   1.2    bouyer 	ci->ci_self = ci;
    256   1.2    bouyer 	sc->sc_info = ci;
    257   1.2    bouyer 
    258   1.2    bouyer 	ci->ci_dev = self;
    259  1.50    jruoho 	ci->ci_acpiid = caa->cpu_id;
    260  1.23        ad 	ci->ci_cpuid = caa->cpu_number;
    261  1.16    cegger 	ci->ci_vcpu = NULL;
    262   1.2    bouyer 
    263  1.38    cegger 	/*
    264  1.38    cegger 	 * Boot processor may not be attached first, but the below
    265  1.38    cegger 	 * must be done to allow booting other processors.
    266  1.38    cegger 	 */
    267  1.38    cegger 	if (!again) {
    268  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    269  1.38    cegger 		/* Basic init */
    270  1.38    cegger 		again = true;
    271  1.38    cegger 	}
    272  1.38    cegger 
    273   1.2    bouyer 	printf(": ");
    274   1.2    bouyer 	switch (caa->cpu_role) {
    275   1.2    bouyer 	case CPU_ROLE_SP:
    276   1.2    bouyer 		printf("(uniprocessor)\n");
    277  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    278   1.2    bouyer 		break;
    279   1.2    bouyer 
    280   1.2    bouyer 	case CPU_ROLE_BP:
    281   1.2    bouyer 		printf("(boot processor)\n");
    282  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    283   1.2    bouyer 		break;
    284   1.2    bouyer 
    285   1.2    bouyer 	case CPU_ROLE_AP:
    286   1.2    bouyer 		/*
    287   1.2    bouyer 		 * report on an AP
    288   1.2    bouyer 		 */
    289   1.2    bouyer 		printf("(application processor)\n");
    290  1.38    cegger 		if (ci->ci_flags & CPUF_PRESENT) {
    291  1.38    cegger 			struct cpu_info *tmp;
    292  1.38    cegger 
    293  1.38    cegger 			tmp = phycpu_info_list;
    294  1.38    cegger 			while (tmp->ci_next)
    295  1.38    cegger 				tmp = tmp->ci_next;
    296  1.38    cegger 
    297  1.38    cegger 			tmp->ci_next = ci;
    298  1.38    cegger 		}
    299   1.2    bouyer 		break;
    300   1.2    bouyer 
    301   1.2    bouyer 	default:
    302   1.2    bouyer 		panic("unknown processor type??\n");
    303   1.2    bouyer 	}
    304  1.34    cegger 
    305  1.38    cegger 	atomic_or_32(&phycpus_attached, ci->ci_cpumask);
    306  1.34    cegger 
    307   1.2    bouyer 	return;
    308   1.2    bouyer }
    309   1.2    bouyer 
    310   1.2    bouyer int
    311  1.10    cegger vcpu_match(device_t parent, cfdata_t match, void *aux)
    312   1.2    bouyer {
    313   1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    314   1.2    bouyer 
    315   1.2    bouyer 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0)
    316   1.2    bouyer 		return 1;
    317   1.2    bouyer 	return 0;
    318   1.2    bouyer }
    319   1.2    bouyer 
    320   1.2    bouyer void
    321  1.10    cegger vcpu_attach(device_t parent, device_t self, void *aux)
    322   1.2    bouyer {
    323   1.2    bouyer 	struct vcpu_attach_args *vcaa = aux;
    324   1.2    bouyer 
    325   1.2    bouyer 	cpu_attach_common(parent, self, &vcaa->vcaa_caa);
    326   1.2    bouyer }
    327   1.2    bouyer 
    328   1.2    bouyer static void
    329   1.2    bouyer cpu_vm_init(struct cpu_info *ci)
    330   1.2    bouyer {
    331   1.2    bouyer 	int ncolors = 2, i;
    332   1.2    bouyer 
    333   1.2    bouyer 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    334   1.2    bouyer 		struct x86_cache_info *cai;
    335   1.2    bouyer 		int tcolors;
    336   1.2    bouyer 
    337   1.2    bouyer 		cai = &ci->ci_cinfo[i];
    338   1.2    bouyer 
    339   1.2    bouyer 		tcolors = atop(cai->cai_totalsize);
    340   1.2    bouyer 		switch(cai->cai_associativity) {
    341   1.2    bouyer 		case 0xff:
    342   1.2    bouyer 			tcolors = 1; /* fully associative */
    343   1.2    bouyer 			break;
    344   1.2    bouyer 		case 0:
    345   1.2    bouyer 		case 1:
    346   1.2    bouyer 			break;
    347   1.2    bouyer 		default:
    348   1.2    bouyer 			tcolors /= cai->cai_associativity;
    349   1.2    bouyer 		}
    350   1.2    bouyer 		ncolors = max(ncolors, tcolors);
    351   1.2    bouyer 	}
    352   1.2    bouyer 
    353   1.2    bouyer 	/*
    354   1.2    bouyer 	 * Knowing the size of the largest cache on this CPU, re-color
    355   1.2    bouyer 	 * our pages.
    356   1.2    bouyer 	 */
    357   1.2    bouyer 	if (ncolors <= uvmexp.ncolors)
    358   1.2    bouyer 		return;
    359  1.28    bouyer 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    360   1.2    bouyer 	uvm_page_recolor(ncolors);
    361   1.2    bouyer }
    362   1.2    bouyer 
    363   1.2    bouyer void
    364  1.11    cegger cpu_attach_common(device_t parent, device_t self, void *aux)
    365   1.2    bouyer {
    366  1.10    cegger 	struct cpu_softc *sc = device_private(self);
    367   1.2    bouyer 	struct cpu_attach_args *caa = aux;
    368   1.2    bouyer 	struct cpu_info *ci;
    369  1.12    cegger 	uintptr_t ptr;
    370   1.2    bouyer 	int cpunum = caa->cpu_number;
    371  1.38    cegger 	static bool again = false;
    372   1.2    bouyer 
    373  1.10    cegger 	sc->sc_dev = self;
    374  1.10    cegger 
    375   1.2    bouyer 	/*
    376   1.2    bouyer 	 * If we're an Application Processor, allocate a cpu_info
    377   1.2    bouyer 	 * structure, otherwise use the primary's.
    378   1.2    bouyer 	 */
    379   1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    380  1.12    cegger 		aprint_naive(": Application Processor\n");
    381  1.31    cegger 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    382  1.31    cegger 		    KM_SLEEP);
    383  1.42       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    384  1.12    cegger 		memset(ci, 0, sizeof(*ci));
    385   1.2    bouyer #ifdef TRAPLOG
    386  1.31    cegger 		ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
    387   1.2    bouyer #endif
    388   1.2    bouyer 	} else {
    389  1.12    cegger 		aprint_naive(": %s Processor\n",
    390  1.12    cegger 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    391   1.2    bouyer 		ci = &cpu_info_primary;
    392  1.38    cegger #if NLAPIC > 0
    393  1.38    cegger 		if (cpunum != lapic_cpu_number()) {
    394  1.38    cegger 			/* XXX should be done earlier */
    395  1.38    cegger 			uint32_t reg;
    396  1.38    cegger 			aprint_verbose("\n");
    397  1.38    cegger 			aprint_verbose_dev(self, "running CPU at apic %d"
    398  1.38    cegger 			    " instead of at expected %d", lapic_cpu_number(),
    399  1.38    cegger 			    cpunum);
    400  1.38    cegger 			reg = i82489_readreg(LAPIC_ID);
    401  1.38    cegger 			i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
    402  1.38    cegger 			    (cpunum << LAPIC_ID_SHIFT));
    403  1.38    cegger 		}
    404   1.2    bouyer 		if (cpunum != lapic_cpu_number()) {
    405  1.38    cegger 			aprint_error_dev(self, "unable to reset apic id\n");
    406   1.2    bouyer 		}
    407   1.2    bouyer #endif
    408   1.2    bouyer 	}
    409   1.2    bouyer 
    410   1.2    bouyer 	ci->ci_self = ci;
    411   1.2    bouyer 	sc->sc_info = ci;
    412   1.2    bouyer 	ci->ci_dev = self;
    413  1.23        ad 	ci->ci_cpuid = cpunum;
    414  1.16    cegger 
    415  1.16    cegger 	KASSERT(HYPERVISOR_shared_info != NULL);
    416  1.16    cegger 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
    417  1.16    cegger 
    418   1.2    bouyer 	ci->ci_func = caa->cpu_func;
    419   1.2    bouyer 
    420  1.38    cegger 	/* Must be called before mi_cpu_attach(). */
    421  1.38    cegger 	cpu_vm_init(ci);
    422  1.38    cegger 
    423   1.2    bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    424   1.2    bouyer 		int error;
    425   1.2    bouyer 
    426   1.2    bouyer 		error = mi_cpu_attach(ci);
    427   1.2    bouyer 		if (error != 0) {
    428   1.2    bouyer 			aprint_normal("\n");
    429  1.38    cegger 			aprint_error_dev(self,
    430  1.38    cegger 			    "mi_cpu_attach failed with %d\n", error);
    431   1.2    bouyer 			return;
    432   1.2    bouyer 		}
    433   1.2    bouyer 	} else {
    434   1.2    bouyer 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    435   1.2    bouyer 	}
    436   1.2    bouyer 
    437  1.23        ad 	ci->ci_cpumask = (1 << cpu_index(ci));
    438   1.2    bouyer 	pmap_reference(pmap_kernel());
    439   1.2    bouyer 	ci->ci_pmap = pmap_kernel();
    440   1.2    bouyer 	ci->ci_tlbstate = TLBSTATE_STALE;
    441   1.2    bouyer 
    442  1.38    cegger 	/*
    443  1.38    cegger 	 * Boot processor may not be attached first, but the below
    444  1.38    cegger 	 * must be done to allow booting other processors.
    445  1.38    cegger 	 */
    446  1.38    cegger 	if (!again) {
    447  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    448  1.38    cegger 		/* Basic init. */
    449  1.38    cegger 		cpu_intr_init(ci);
    450  1.38    cegger 		cpu_get_tsc_freq(ci);
    451  1.38    cegger 		cpu_init(ci);
    452  1.38    cegger 		cpu_set_tss_gates(ci);
    453  1.38    cegger 		pmap_cpu_init_late(ci);
    454  1.38    cegger #if NLAPIC > 0
    455  1.38    cegger 		if (caa->cpu_role != CPU_ROLE_SP) {
    456  1.38    cegger 			/* Enable lapic. */
    457  1.38    cegger 			lapic_enable();
    458  1.38    cegger 			lapic_set_lvt();
    459  1.38    cegger 			lapic_calibrate_timer();
    460  1.38    cegger 		}
    461  1.38    cegger #endif
    462  1.38    cegger 		/* Make sure DELAY() is initialized. */
    463  1.38    cegger 		DELAY(1);
    464  1.38    cegger 		again = true;
    465  1.38    cegger 	}
    466  1.38    cegger 
    467   1.2    bouyer 	/* further PCB init done later. */
    468   1.2    bouyer 
    469   1.2    bouyer 	switch (caa->cpu_role) {
    470   1.2    bouyer 	case CPU_ROLE_SP:
    471  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    472  1.21        ad 		cpu_identify(ci);
    473  1.12    cegger #if 0
    474  1.12    cegger 		x86_errata();
    475  1.12    cegger #endif
    476  1.38    cegger 		x86_cpu_idle_init();
    477   1.2    bouyer 		break;
    478   1.2    bouyer 
    479   1.2    bouyer 	case CPU_ROLE_BP:
    480  1.38    cegger 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    481  1.21        ad 		cpu_identify(ci);
    482   1.2    bouyer 		cpu_init(ci);
    483  1.14    bouyer #if 0
    484  1.12    cegger 		x86_errata();
    485  1.12    cegger #endif
    486  1.38    cegger 		x86_cpu_idle_init();
    487   1.2    bouyer 		break;
    488   1.2    bouyer 
    489   1.2    bouyer 	case CPU_ROLE_AP:
    490   1.2    bouyer 		/*
    491   1.2    bouyer 		 * report on an AP
    492   1.2    bouyer 		 */
    493   1.2    bouyer 
    494   1.2    bouyer #if defined(MULTIPROCESSOR)
    495   1.2    bouyer 		cpu_intr_init(ci);
    496   1.2    bouyer 		gdt_alloc_cpu(ci);
    497   1.2    bouyer 		cpu_set_tss_gates(ci);
    498  1.12    cegger 		pmap_cpu_init_early(ci);
    499  1.12    cegger 		pmap_cpu_init_late(ci);
    500   1.2    bouyer 		cpu_start_secondary(ci);
    501   1.2    bouyer 		if (ci->ci_flags & CPUF_PRESENT) {
    502  1.30    cegger 			struct cpu_info *tmp;
    503  1.30    cegger 
    504   1.2    bouyer 			identifycpu(ci);
    505  1.30    cegger 			tmp = cpu_info_list;
    506  1.30    cegger 			while (tmp->ci_next)
    507  1.30    cegger 				tmp = tmp->ci_next;
    508  1.30    cegger 
    509  1.30    cegger 			tmp->ci_next = ci;
    510   1.2    bouyer 		}
    511   1.2    bouyer #else
    512  1.38    cegger 		aprint_error_dev(self, "not started\n");
    513   1.2    bouyer #endif
    514   1.2    bouyer 		break;
    515   1.2    bouyer 
    516   1.2    bouyer 	default:
    517  1.12    cegger 		aprint_normal("\n");
    518   1.2    bouyer 		panic("unknown processor type??\n");
    519   1.2    bouyer 	}
    520   1.2    bouyer 
    521  1.46    cegger 	pat_init(ci);
    522  1.34    cegger 	atomic_or_32(&cpus_attached, ci->ci_cpumask);
    523   1.2    bouyer 
    524  1.12    cegger #if 0
    525  1.12    cegger 	if (!pmf_device_register(self, cpu_suspend, cpu_resume))
    526  1.12    cegger 		aprint_error_dev(self, "couldn't establish power handler\n");
    527  1.12    cegger #endif
    528  1.12    cegger 
    529   1.2    bouyer #if defined(MULTIPROCESSOR)
    530   1.2    bouyer 	if (mp_verbose) {
    531   1.2    bouyer 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    532  1.37     rmind 		struct pcb *pcb = lwp_getpcb(l);
    533   1.2    bouyer 
    534  1.38    cegger 		aprint_verbose_dev(self,
    535  1.38    cegger 		    "idle lwp at %p, idle sp at 0x%p\n",
    536  1.12    cegger 		    l,
    537  1.12    cegger #ifdef i386
    538  1.37     rmind 		    (void *)pcb->pcb_esp
    539  1.12    cegger #else
    540  1.37     rmind 		    (void *)pcb->pcb_rsp
    541  1.12    cegger #endif
    542  1.12    cegger 		);
    543  1.12    cegger 
    544   1.2    bouyer 	}
    545   1.2    bouyer #endif
    546   1.2    bouyer }
    547   1.2    bouyer 
    548   1.2    bouyer /*
    549   1.2    bouyer  * Initialize the processor appropriately.
    550   1.2    bouyer  */
    551   1.2    bouyer 
    552   1.2    bouyer void
    553  1.10    cegger cpu_init(struct cpu_info *ci)
    554   1.2    bouyer {
    555   1.2    bouyer 
    556   1.2    bouyer 	/*
    557   1.2    bouyer 	 * On a P6 or above, enable global TLB caching if the
    558   1.2    bouyer 	 * hardware supports it.
    559   1.2    bouyer 	 */
    560  1.43       jym 	if (cpu_feature[0] & CPUID_PGE)
    561   1.2    bouyer 		lcr4(rcr4() | CR4_PGE);	/* enable global TLB caching */
    562   1.2    bouyer 
    563   1.2    bouyer #ifdef XXXMTRR
    564   1.2    bouyer 	/*
    565   1.2    bouyer 	 * On a P6 or above, initialize MTRR's if the hardware supports them.
    566   1.2    bouyer 	 */
    567  1.43       jym 	if (cpu_feature[0] & CPUID_MTRR) {
    568   1.2    bouyer 		if ((ci->ci_flags & CPUF_AP) == 0)
    569   1.2    bouyer 			i686_mtrr_init_first();
    570   1.2    bouyer 		mtrr_init_cpu(ci);
    571   1.2    bouyer 	}
    572   1.2    bouyer #endif
    573   1.2    bouyer 	/*
    574   1.2    bouyer 	 * If we have FXSAVE/FXRESTOR, use them.
    575   1.2    bouyer 	 */
    576  1.43       jym 	if (cpu_feature[0] & CPUID_FXSR) {
    577   1.2    bouyer 		lcr4(rcr4() | CR4_OSFXSR);
    578   1.2    bouyer 
    579   1.2    bouyer 		/*
    580   1.2    bouyer 		 * If we have SSE/SSE2, enable XMM exceptions.
    581   1.2    bouyer 		 */
    582  1.43       jym 		if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
    583   1.2    bouyer 			lcr4(rcr4() | CR4_OSXMMEXCPT);
    584   1.2    bouyer 	}
    585   1.2    bouyer 
    586  1.47       jym #ifdef __x86_64__
    587  1.47       jym 	/* No user PGD mapped for this CPU yet */
    588  1.47       jym 	ci->ci_xen_current_user_pgd = 0;
    589  1.47       jym #endif
    590  1.47       jym 
    591  1.34    cegger 	atomic_or_32(&cpus_running, ci->ci_cpumask);
    592  1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    593   1.2    bouyer }
    594   1.2    bouyer 
    595   1.2    bouyer 
    596   1.2    bouyer #ifdef MULTIPROCESSOR
    597   1.2    bouyer void
    598  1.10    cegger cpu_boot_secondary_processors(void)
    599   1.2    bouyer {
    600   1.2    bouyer 	struct cpu_info *ci;
    601   1.2    bouyer 	u_long i;
    602   1.2    bouyer 
    603  1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    604  1.38    cegger 		ci = cpu_lookup(i);
    605   1.2    bouyer 		if (ci == NULL)
    606   1.2    bouyer 			continue;
    607   1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    608   1.2    bouyer 			continue;
    609   1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    610   1.2    bouyer 			continue;
    611   1.2    bouyer 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    612   1.2    bouyer 			continue;
    613   1.2    bouyer 		cpu_boot_secondary(ci);
    614   1.2    bouyer 	}
    615  1.11    cegger 
    616  1.11    cegger 	x86_mp_online = true;
    617   1.2    bouyer }
    618   1.2    bouyer 
    619   1.2    bouyer static void
    620   1.2    bouyer cpu_init_idle_lwp(struct cpu_info *ci)
    621   1.2    bouyer {
    622   1.2    bouyer 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    623  1.37     rmind 	struct pcb *pcb = lwp_getpcb(l);
    624   1.2    bouyer 
    625   1.2    bouyer 	pcb->pcb_cr0 = rcr0();
    626   1.2    bouyer }
    627   1.2    bouyer 
    628   1.2    bouyer void
    629  1.10    cegger cpu_init_idle_lwps(void)
    630   1.2    bouyer {
    631   1.2    bouyer 	struct cpu_info *ci;
    632   1.2    bouyer 	u_long i;
    633   1.2    bouyer 
    634  1.38    cegger 	for (i = 0; i < maxcpus; i++) {
    635  1.38    cegger 		ci = cpu_lookup(i);
    636   1.2    bouyer 		if (ci == NULL)
    637   1.2    bouyer 			continue;
    638   1.2    bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    639   1.2    bouyer 			continue;
    640   1.2    bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    641   1.2    bouyer 			continue;
    642   1.2    bouyer 		cpu_init_idle_lwp(ci);
    643   1.2    bouyer 	}
    644   1.2    bouyer }
    645   1.2    bouyer 
    646   1.2    bouyer void
    647  1.10    cegger cpu_start_secondary(struct cpu_info *ci)
    648   1.2    bouyer {
    649   1.2    bouyer 	int i;
    650   1.2    bouyer 	struct pmap *kpm = pmap_kernel();
    651  1.11    cegger 	extern uint32_t mp_pdirpa;
    652   1.2    bouyer 
    653   1.2    bouyer 	mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */
    654   1.2    bouyer 
    655  1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_AP);
    656   1.2    bouyer 
    657  1.11    cegger 	aprint_debug_dev(ci->ci_dev, "starting\n");
    658   1.2    bouyer 
    659   1.2    bouyer 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    660  1.11    cegger 	if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0)
    661  1.11    cegger 		return;
    662   1.2    bouyer 
    663   1.2    bouyer 	/*
    664   1.2    bouyer 	 * wait for it to become ready
    665   1.2    bouyer 	 */
    666  1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    667  1.11    cegger #ifdef MPDEBUG
    668  1.11    cegger 		extern int cpu_trace[3];
    669  1.11    cegger 		static int otrace[3];
    670  1.11    cegger 		if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
    671  1.11    cegger 			aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
    672  1.11    cegger 				cpu_trace[0], cpu_trace[1], cpu_trace[2]);
    673  1.11    cegger 			memcpy(otrace, cpu_trace, sizeof(otrace));
    674  1.11    cegger 		}
    675  1.11    cegger #endif
    676   1.2    bouyer 		delay(10);
    677   1.2    bouyer 	}
    678  1.11    cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    679   1.9    cegger 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    680   1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    681   1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    682   1.2    bouyer 		Debugger();
    683   1.2    bouyer #endif
    684   1.2    bouyer 	}
    685   1.2    bouyer 
    686   1.2    bouyer 	CPU_START_CLEANUP(ci);
    687   1.2    bouyer }
    688   1.2    bouyer 
    689   1.2    bouyer void
    690  1.10    cegger cpu_boot_secondary(struct cpu_info *ci)
    691   1.2    bouyer {
    692   1.2    bouyer 	int i;
    693   1.2    bouyer 
    694  1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    695  1.11    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    696   1.2    bouyer 		delay(10);
    697   1.2    bouyer 	}
    698  1.11    cegger 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    699  1.11    cegger 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
    700   1.2    bouyer #if defined(MPDEBUG) && defined(DDB)
    701   1.2    bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    702   1.2    bouyer 		Debugger();
    703   1.2    bouyer #endif
    704   1.2    bouyer 	}
    705   1.2    bouyer }
    706   1.2    bouyer 
    707   1.2    bouyer /*
    708   1.2    bouyer  * The CPU ends up here when its ready to run
    709   1.2    bouyer  * This is called from code in mptramp.s; at this point, we are running
    710   1.2    bouyer  * in the idle pcb/idle stack of the new CPU.  When this function returns,
    711   1.2    bouyer  * this processor will enter the idle loop and start looking for work.
    712   1.2    bouyer  *
    713   1.2    bouyer  * XXX should share some of this with init386 in machdep.c
    714   1.2    bouyer  */
    715   1.2    bouyer void
    716   1.2    bouyer cpu_hatch(void *v)
    717   1.2    bouyer {
    718   1.2    bouyer 	struct cpu_info *ci = (struct cpu_info *)v;
    719  1.37     rmind 	struct pcb *pcb;
    720  1.11    cegger 	int s, i;
    721  1.11    cegger 
    722  1.21        ad 	cpu_probe(ci);
    723  1.11    cegger 
    724  1.43       jym 	cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST;
    725  1.43       jym 	cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST;
    726   1.2    bouyer 
    727  1.43       jym         cpu_init_msrs(ci, true);
    728   1.2    bouyer 
    729  1.11    cegger 	KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
    730  1.11    cegger 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    731  1.11    cegger 	while ((ci->ci_flags & CPUF_GO) == 0) {
    732  1.11    cegger 		/* Don't use delay, boot CPU may be patching the text. */
    733  1.11    cegger 		for (i = 10000; i != 0; i--)
    734  1.11    cegger 			x86_pause();
    735  1.11    cegger 	}
    736   1.2    bouyer 
    737  1.11    cegger 	/* Because the text may have been patched in x86_patch(). */
    738  1.11    cegger 	wbinvd();
    739  1.11    cegger 	x86_flush();
    740   1.2    bouyer 
    741  1.11    cegger 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    742   1.2    bouyer 
    743  1.37     rmind 	pcb = lwp_getpcb(curlwp);
    744  1.12    cegger 	lcr3(pmap_kernel()->pm_pdirpa);
    745  1.37     rmind 	pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa;
    746  1.37     rmind 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
    747  1.37     rmind 	lcr0(pcb->pcb_cr0);
    748  1.37     rmind 
    749   1.2    bouyer 	cpu_init_idt();
    750  1.11    cegger 	gdt_init_cpu(ci);
    751  1.11    cegger 	lapic_enable();
    752   1.2    bouyer 	lapic_set_lvt();
    753  1.11    cegger 	lapic_initclocks();
    754  1.11    cegger 
    755  1.12    cegger #ifdef i386
    756   1.2    bouyer 	npxinit(ci);
    757  1.12    cegger #else
    758  1.12    cegger 	fpuinit(ci);
    759  1.12    cegger #endif
    760   1.2    bouyer 
    761   1.2    bouyer 	lldt(GSEL(GLDT_SEL, SEL_KPL));
    762  1.12    cegger 	ltr(ci->ci_tss_sel);
    763   1.2    bouyer 
    764   1.2    bouyer 	cpu_init(ci);
    765  1.11    cegger 	cpu_get_tsc_freq(ci);
    766   1.2    bouyer 
    767   1.2    bouyer 	s = splhigh();
    768  1.11    cegger #ifdef i386
    769   1.2    bouyer 	lapic_tpr = 0;
    770  1.11    cegger #else
    771  1.11    cegger 	lcr8(0);
    772  1.11    cegger #endif
    773  1.11    cegger 	x86_enable_intr();
    774  1.11    cegger 	splx(s);
    775  1.12    cegger #if 0
    776  1.11    cegger 	x86_errata();
    777  1.11    cegger #endif
    778   1.2    bouyer 
    779  1.11    cegger 	aprint_debug_dev(ci->ci_dev, "CPU %ld running\n",
    780  1.11    cegger 		(long)ci->ci_cpuid);
    781   1.2    bouyer }
    782   1.2    bouyer 
    783   1.2    bouyer #if defined(DDB)
    784   1.2    bouyer 
    785   1.2    bouyer #include <ddb/db_output.h>
    786   1.2    bouyer #include <machine/db_machdep.h>
    787   1.2    bouyer 
    788   1.2    bouyer /*
    789   1.2    bouyer  * Dump CPU information from ddb.
    790   1.2    bouyer  */
    791   1.2    bouyer void
    792   1.2    bouyer cpu_debug_dump(void)
    793   1.2    bouyer {
    794   1.2    bouyer 	struct cpu_info *ci;
    795   1.2    bouyer 	CPU_INFO_ITERATOR cii;
    796   1.2    bouyer 
    797  1.13      yamt 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    798   1.2    bouyer 	for (CPU_INFO_FOREACH(cii, ci)) {
    799   1.2    bouyer 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    800   1.2    bouyer 		    ci,
    801   1.9    cegger 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    802  1.12    cegger 		    (long)ci->ci_cpuid,
    803   1.2    bouyer 		    ci->ci_flags, ci->ci_ipis,
    804   1.2    bouyer 		    ci->ci_curlwp,
    805   1.2    bouyer 		    ci->ci_fpcurlwp);
    806   1.2    bouyer 	}
    807   1.2    bouyer }
    808  1.38    cegger #endif /* DDB */
    809   1.2    bouyer 
    810   1.2    bouyer static void
    811  1.10    cegger cpu_copy_trampoline(void)
    812   1.2    bouyer {
    813   1.2    bouyer 	/*
    814   1.2    bouyer 	 * Copy boot code.
    815   1.2    bouyer 	 */
    816   1.2    bouyer 	extern u_char cpu_spinup_trampoline[];
    817   1.2    bouyer 	extern u_char cpu_spinup_trampoline_end[];
    818  1.11    cegger 
    819  1.11    cegger 	vaddr_t mp_trampoline_vaddr;
    820  1.11    cegger 
    821  1.11    cegger 	mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    822  1.11    cegger 		UVM_KMF_VAONLY);
    823  1.11    cegger 
    824  1.11    cegger 	pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
    825  1.36    cegger 		VM_PROT_READ | VM_PROT_WRITE, 0);
    826  1.11    cegger 	pmap_update(pmap_kernel());
    827  1.11    cegger 	memcpy((void *)mp_trampoline_vaddr,
    828  1.11    cegger 		cpu_spinup_trampoline,
    829  1.11    cegger 		cpu_spinup_trampoline_end - cpu_spinup_trampoline);
    830  1.11    cegger 
    831  1.11    cegger 	pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
    832  1.11    cegger 	pmap_update(pmap_kernel());
    833  1.11    cegger 	uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
    834   1.2    bouyer }
    835   1.2    bouyer 
    836  1.38    cegger #endif /* MULTIPROCESSOR */
    837   1.2    bouyer 
    838  1.11    cegger #ifdef i386
    839  1.11    cegger #if 0
    840  1.11    cegger static void
    841  1.11    cegger tss_init(struct i386tss *tss, void *stack, void *func)
    842  1.11    cegger {
    843  1.11    cegger 	memset(tss, 0, sizeof *tss);
    844  1.11    cegger 	tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
    845  1.11    cegger 	tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
    846  1.11    cegger 	tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
    847  1.11    cegger 	tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
    848  1.11    cegger 	tss->tss_gs = tss->__tss_es = tss->__tss_ds =
    849  1.11    cegger 	    tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
    850  1.11    cegger 	tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
    851  1.11    cegger 	tss->tss_esp = (int)((char *)stack + USPACE - 16);
    852  1.11    cegger 	tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
    853  1.11    cegger 	tss->__tss_eflags = PSL_MBO | PSL_NT;   /* XXX not needed? */
    854  1.11    cegger 	tss->__tss_eip = (int)func;
    855  1.11    cegger }
    856  1.11    cegger #endif
    857   1.2    bouyer 
    858   1.2    bouyer /* XXX */
    859   1.2    bouyer #define IDTVEC(name)	__CONCAT(X, name)
    860   1.2    bouyer typedef void (vector)(void);
    861   1.2    bouyer extern vector IDTVEC(tss_trap08);
    862   1.2    bouyer #ifdef DDB
    863   1.2    bouyer extern vector Xintrddbipi;
    864   1.2    bouyer extern int ddb_vec;
    865   1.2    bouyer #endif
    866   1.2    bouyer 
    867   1.2    bouyer static void
    868   1.2    bouyer cpu_set_tss_gates(struct cpu_info *ci)
    869   1.2    bouyer {
    870  1.11    cegger #if 0
    871  1.11    cegger 	struct segment_descriptor sd;
    872  1.11    cegger 
    873  1.11    cegger 	ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    874  1.11    cegger 	    UVM_KMF_WIRED);
    875  1.11    cegger 	tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
    876  1.11    cegger 	    IDTVEC(tss_trap08));
    877  1.11    cegger 	setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
    878  1.11    cegger 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    879  1.11    cegger 	ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
    880  1.11    cegger 	setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    881  1.11    cegger 	    GSEL(GTRAPTSS_SEL, SEL_KPL));
    882  1.11    cegger #endif
    883  1.11    cegger 
    884   1.2    bouyer #if defined(DDB) && defined(MULTIPROCESSOR)
    885   1.2    bouyer 	/*
    886   1.2    bouyer 	 * Set up separate handler for the DDB IPI, so that it doesn't
    887   1.2    bouyer 	 * stomp on a possibly corrupted stack.
    888   1.2    bouyer 	 *
    889   1.2    bouyer 	 * XXX overwriting the gate set in db_machine_init.
    890   1.2    bouyer 	 * Should rearrange the code so that it's set only once.
    891   1.2    bouyer 	 */
    892   1.2    bouyer 	ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    893   1.2    bouyer 	    UVM_KMF_WIRED);
    894   1.6      yamt 	tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack,
    895   1.2    bouyer 	    Xintrddbipi);
    896   1.2    bouyer 
    897   1.2    bouyer 	setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
    898   1.2    bouyer 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    899   1.2    bouyer 	ci->ci_gdt[GIPITSS_SEL].sd = sd;
    900   1.2    bouyer 
    901   1.2    bouyer 	setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    902   1.2    bouyer 	    GSEL(GIPITSS_SEL, SEL_KPL));
    903   1.2    bouyer #endif
    904   1.2    bouyer }
    905  1.11    cegger #else
    906  1.11    cegger static void
    907  1.11    cegger cpu_set_tss_gates(struct cpu_info *ci)
    908  1.11    cegger {
    909  1.11    cegger 
    910  1.11    cegger }
    911  1.11    cegger #endif	/* i386 */
    912   1.2    bouyer 
    913   1.2    bouyer int
    914   1.5     joerg mp_cpu_start(struct cpu_info *ci, paddr_t target)
    915   1.2    bouyer {
    916   1.2    bouyer #if 0
    917   1.2    bouyer #if NLAPIC > 0
    918   1.2    bouyer 	int error;
    919   1.2    bouyer #endif
    920   1.2    bouyer 	unsigned short dwordptr[2];
    921   1.2    bouyer 
    922   1.2    bouyer 	/*
    923  1.11    cegger 	 * Bootstrap code must be addressable in real mode
    924  1.11    cegger 	 * and it must be page aligned.
    925  1.11    cegger 	 */
    926  1.11    cegger 	KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
    927  1.11    cegger 
    928  1.11    cegger 	/*
    929   1.2    bouyer 	 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
    930   1.2    bouyer 	 */
    931   1.2    bouyer 
    932   1.2    bouyer 	outb(IO_RTC, NVRAM_RESET);
    933   1.2    bouyer 	outb(IO_RTC+1, NVRAM_RESET_JUMP);
    934   1.2    bouyer 
    935   1.2    bouyer 	/*
    936   1.2    bouyer 	 * "and the warm reset vector (DWORD based at 40:67) to point
    937   1.2    bouyer 	 * to the AP startup code ..."
    938   1.2    bouyer 	 */
    939   1.2    bouyer 
    940   1.2    bouyer 	dwordptr[0] = 0;
    941   1.5     joerg 	dwordptr[1] = target >> 4;
    942   1.2    bouyer 
    943  1.36    cegger 	pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
    944  1.45     rmind 	pmap_update(pmap_kernel());
    945  1.45     rmind 
    946  1.11    cegger 	memcpy ((uint8_t *) 0x467, dwordptr, 4);
    947  1.45     rmind 
    948   1.2    bouyer 	pmap_kremove (0, PAGE_SIZE);
    949  1.45     rmind 	pmap_update(pmap_kernel());
    950   1.2    bouyer 
    951   1.2    bouyer #if NLAPIC > 0
    952   1.2    bouyer 	/*
    953   1.2    bouyer 	 * ... prior to executing the following sequence:"
    954   1.2    bouyer 	 */
    955   1.2    bouyer 
    956   1.2    bouyer 	if (ci->ci_flags & CPUF_AP) {
    957  1.23        ad 		if ((error = x86_ipi_init(ci->ci_cpuid)) != 0)
    958   1.2    bouyer 			return error;
    959   1.2    bouyer 
    960   1.2    bouyer 		delay(10000);
    961   1.2    bouyer 
    962   1.2    bouyer 		if (cpu_feature & CPUID_APIC) {
    963  1.23        ad 			error = x86_ipi_init(ci->ci_cpuid);
    964  1.11    cegger 			if (error != 0) {
    965  1.11    cegger 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
    966  1.11    cegger 						__func__);
    967  1.11    cegger 				return error;
    968  1.11    cegger 			}
    969  1.11    cegger 
    970  1.11    cegger 			delay(10000);
    971   1.2    bouyer 
    972  1.23        ad 			error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid,
    973  1.11    cegger 					LAPIC_DLMODE_STARTUP);
    974  1.11    cegger 			if (error != 0) {
    975  1.11    cegger 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
    976  1.11    cegger 						__func__);
    977   1.2    bouyer 				return error;
    978  1.11    cegger 			}
    979   1.2    bouyer 			delay(200);
    980   1.2    bouyer 
    981  1.23        ad 			error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid,
    982  1.11    cegger 					LAPIC_DLMODE_STARTUP);
    983  1.11    cegger 			if (error != 0) {
    984  1.11    cegger 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n",
    985  1.11    cegger 						__func__);
    986   1.2    bouyer 				return error;
    987  1.11    cegger 			}
    988   1.2    bouyer 			delay(200);
    989   1.2    bouyer 		}
    990   1.2    bouyer 	}
    991   1.2    bouyer #endif
    992   1.2    bouyer #endif /* 0 */
    993   1.2    bouyer 	return 0;
    994   1.2    bouyer }
    995   1.2    bouyer 
    996   1.2    bouyer void
    997   1.2    bouyer mp_cpu_start_cleanup(struct cpu_info *ci)
    998   1.2    bouyer {
    999   1.2    bouyer #if 0
   1000   1.2    bouyer 	/*
   1001   1.2    bouyer 	 * Ensure the NVRAM reset byte contains something vaguely sane.
   1002   1.2    bouyer 	 */
   1003   1.2    bouyer 
   1004   1.2    bouyer 	outb(IO_RTC, NVRAM_RESET);
   1005   1.2    bouyer 	outb(IO_RTC+1, NVRAM_RESET_RST);
   1006   1.2    bouyer #endif
   1007   1.2    bouyer }
   1008   1.2    bouyer 
   1009   1.2    bouyer void
   1010   1.3    bouyer cpu_init_msrs(struct cpu_info *ci, bool full)
   1011   1.2    bouyer {
   1012  1.43       jym #ifdef __x86_64__
   1013   1.3    bouyer 	if (full) {
   1014   1.3    bouyer 		HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
   1015  1.11    cegger 		HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
   1016   1.3    bouyer 		HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
   1017   1.3    bouyer 	}
   1018  1.43       jym #endif	/* __x86_64__ */
   1019  1.44       jym 
   1020  1.44       jym 	if (cpu_feature[2] & CPUID_NOX)
   1021  1.44       jym 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
   1022   1.2    bouyer }
   1023   1.2    bouyer 
   1024  1.11    cegger void
   1025  1.11    cegger cpu_offline_md(void)
   1026  1.11    cegger {
   1027  1.11    cegger         int s;
   1028  1.11    cegger 
   1029  1.11    cegger         s = splhigh();
   1030  1.11    cegger #ifdef __i386__
   1031  1.11    cegger         npxsave_cpu(true);
   1032  1.11    cegger #else
   1033  1.11    cegger         fpusave_cpu(true);
   1034  1.11    cegger #endif
   1035  1.11    cegger         splx(s);
   1036  1.11    cegger }
   1037  1.11    cegger 
   1038  1.11    cegger #if 0
   1039  1.11    cegger /* XXX joerg restructure and restart CPUs individually */
   1040  1.11    cegger static bool
   1041  1.41    dyoung cpu_suspend(device_t dv, const pmf_qual_t *qual)
   1042  1.11    cegger {
   1043  1.11    cegger 	struct cpu_softc *sc = device_private(dv);
   1044  1.11    cegger 	struct cpu_info *ci = sc->sc_info;
   1045  1.11    cegger 	int err;
   1046  1.11    cegger 
   1047  1.11    cegger 	if (ci->ci_flags & CPUF_PRIMARY)
   1048  1.11    cegger 		return true;
   1049  1.11    cegger 	if (ci->ci_data.cpu_idlelwp == NULL)
   1050  1.11    cegger 		return true;
   1051  1.11    cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1052  1.11    cegger 		return true;
   1053  1.11    cegger 
   1054  1.11    cegger 	sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
   1055  1.11    cegger 
   1056  1.11    cegger 	if (sc->sc_wasonline) {
   1057  1.11    cegger 		mutex_enter(&cpu_lock);
   1058  1.29     rmind 		err = cpu_setstate(ci, false);
   1059  1.11    cegger 		mutex_exit(&cpu_lock);
   1060  1.11    cegger 
   1061  1.11    cegger 		if (err)
   1062  1.11    cegger 			return false;
   1063  1.11    cegger 	}
   1064  1.11    cegger 
   1065  1.11    cegger 	return true;
   1066  1.11    cegger }
   1067  1.11    cegger 
   1068  1.11    cegger static bool
   1069  1.41    dyoung cpu_resume(device_t dv, const pmf_qual_t *qual)
   1070  1.11    cegger {
   1071  1.11    cegger 	struct cpu_softc *sc = device_private(dv);
   1072  1.11    cegger 	struct cpu_info *ci = sc->sc_info;
   1073  1.11    cegger 	int err = 0;
   1074  1.11    cegger 
   1075  1.11    cegger 	if (ci->ci_flags & CPUF_PRIMARY)
   1076  1.11    cegger 		return true;
   1077  1.11    cegger 	if (ci->ci_data.cpu_idlelwp == NULL)
   1078  1.11    cegger 		return true;
   1079  1.11    cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1080  1.11    cegger 		return true;
   1081  1.11    cegger 
   1082  1.11    cegger 	if (sc->sc_wasonline) {
   1083  1.11    cegger 		mutex_enter(&cpu_lock);
   1084  1.29     rmind 		err = cpu_setstate(ci, true);
   1085  1.11    cegger 		mutex_exit(&cpu_lock);
   1086  1.11    cegger 	}
   1087  1.11    cegger 
   1088  1.11    cegger 	return err == 0;
   1089  1.11    cegger }
   1090  1.11    cegger #endif
   1091  1.11    cegger 
   1092   1.2    bouyer void
   1093   1.2    bouyer cpu_get_tsc_freq(struct cpu_info *ci)
   1094   1.2    bouyer {
   1095  1.16    cegger 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
   1096   1.2    bouyer 	delay(1000000);
   1097   1.2    bouyer 	uint64_t freq = 1000000000ULL << 32;
   1098   1.2    bouyer 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
   1099   1.2    bouyer 	if ( tinfo->tsc_shift < 0 )
   1100   1.2    bouyer 		freq = freq << -tinfo->tsc_shift;
   1101   1.2    bouyer 	else
   1102   1.2    bouyer 		freq = freq >> tinfo->tsc_shift;
   1103  1.20        ad 	ci->ci_data.cpu_cc_freq = freq;
   1104   1.2    bouyer }
   1105  1.19     joerg 
   1106  1.19     joerg void
   1107  1.19     joerg x86_cpu_idle_xen(void)
   1108  1.19     joerg {
   1109  1.19     joerg 	struct cpu_info *ci = curcpu();
   1110  1.19     joerg 
   1111  1.19     joerg 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1112  1.19     joerg 
   1113  1.19     joerg 	x86_disable_intr();
   1114  1.19     joerg 	if (!__predict_false(ci->ci_want_resched)) {
   1115  1.19     joerg 		idle_block();
   1116  1.19     joerg 	} else {
   1117  1.19     joerg 		x86_enable_intr();
   1118  1.19     joerg 	}
   1119  1.19     joerg }
   1120  1.47       jym 
   1121  1.47       jym /*
   1122  1.47       jym  * Loads pmap for the current CPU.
   1123  1.47       jym  */
   1124  1.47       jym void
   1125  1.47       jym cpu_load_pmap(struct pmap *pmap)
   1126  1.47       jym {
   1127  1.47       jym #ifdef i386
   1128  1.47       jym #ifdef PAE
   1129  1.47       jym 	int i, s;
   1130  1.47       jym 	struct cpu_info *ci;
   1131  1.47       jym 
   1132  1.47       jym 	s = splvm(); /* just to be safe */
   1133  1.47       jym 	ci = curcpu();
   1134  1.47       jym 	paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);
   1135  1.47       jym 	/* don't update the kernel L3 slot */
   1136  1.47       jym 	for (i = 0 ; i < PDP_SIZE - 1; i++) {
   1137  1.47       jym 		xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t),
   1138  1.47       jym 		    xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V);
   1139  1.47       jym 	}
   1140  1.47       jym 	splx(s);
   1141  1.47       jym 	tlbflush();
   1142  1.47       jym #else /* PAE */
   1143  1.47       jym 	lcr3(pmap_pdirpa(pmap, 0));
   1144  1.47       jym #endif /* PAE */
   1145  1.47       jym #endif /* i386 */
   1146  1.47       jym 
   1147  1.47       jym #ifdef __x86_64__
   1148  1.47       jym 	int i, s;
   1149  1.47       jym 	pd_entry_t *old_pgd, *new_pgd;
   1150  1.47       jym 	paddr_t addr;
   1151  1.47       jym 	struct cpu_info *ci;
   1152  1.47       jym 
   1153  1.47       jym 	/* kernel pmap always in cr3 and should never go in user cr3 */
   1154  1.47       jym 	if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) {
   1155  1.47       jym 		ci = curcpu();
   1156  1.47       jym 		/*
   1157  1.47       jym 		 * Map user space address in kernel space and load
   1158  1.47       jym 		 * user cr3
   1159  1.47       jym 		 */
   1160  1.47       jym 		s = splvm();
   1161  1.47       jym 		new_pgd = pmap->pm_pdir;
   1162  1.47       jym 		old_pgd = pmap_kernel()->pm_pdir;
   1163  1.47       jym 		addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0));
   1164  1.47       jym 		for (i = 0; i < PDIR_SLOT_PTE;
   1165  1.47       jym 		    i++, addr += sizeof(pd_entry_t)) {
   1166  1.47       jym 			if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V))
   1167  1.47       jym 				xpq_queue_pte_update(addr, new_pgd[i]);
   1168  1.47       jym 		}
   1169  1.47       jym 		tlbflush();
   1170  1.47       jym 		xen_set_user_pgd(pmap_pdirpa(pmap, 0));
   1171  1.47       jym 		ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);
   1172  1.47       jym 		splx(s);
   1173  1.47       jym 	}
   1174  1.47       jym #endif /* __x86_64__ */
   1175  1.47       jym }
   1176