Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.67
      1  1.67       jym /*	$NetBSD: cpu.c,v 1.67 2010/02/09 23:47:10 jym Exp $	*/
      2   1.2        ad 
      3   1.2        ad /*-
      4  1.38        ad  * Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc.
      5   1.2        ad  * All rights reserved.
      6   1.2        ad  *
      7   1.2        ad  * This code is derived from software contributed to The NetBSD Foundation
      8  1.11        ad  * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
      9   1.2        ad  *
     10   1.2        ad  * Redistribution and use in source and binary forms, with or without
     11   1.2        ad  * modification, are permitted provided that the following conditions
     12   1.2        ad  * are met:
     13   1.2        ad  * 1. Redistributions of source code must retain the above copyright
     14   1.2        ad  *    notice, this list of conditions and the following disclaimer.
     15   1.2        ad  * 2. Redistributions in binary form must reproduce the above copyright
     16   1.2        ad  *    notice, this list of conditions and the following disclaimer in the
     17   1.2        ad  *    documentation and/or other materials provided with the distribution.
     18   1.2        ad  *
     19   1.2        ad  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20   1.2        ad  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21   1.2        ad  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22   1.2        ad  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23   1.2        ad  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24   1.2        ad  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25   1.2        ad  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26   1.2        ad  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27   1.2        ad  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28   1.2        ad  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29   1.2        ad  * POSSIBILITY OF SUCH DAMAGE.
     30   1.2        ad  */
     31   1.2        ad 
     32   1.2        ad /*
     33   1.2        ad  * Copyright (c) 1999 Stefan Grefen
     34   1.2        ad  *
     35   1.2        ad  * Redistribution and use in source and binary forms, with or without
     36   1.2        ad  * modification, are permitted provided that the following conditions
     37   1.2        ad  * are met:
     38   1.2        ad  * 1. Redistributions of source code must retain the above copyright
     39   1.2        ad  *    notice, this list of conditions and the following disclaimer.
     40   1.2        ad  * 2. Redistributions in binary form must reproduce the above copyright
     41   1.2        ad  *    notice, this list of conditions and the following disclaimer in the
     42   1.2        ad  *    documentation and/or other materials provided with the distribution.
     43   1.2        ad  * 3. All advertising materials mentioning features or use of this software
     44   1.2        ad  *    must display the following acknowledgement:
     45   1.2        ad  *      This product includes software developed by the NetBSD
     46   1.2        ad  *      Foundation, Inc. and its contributors.
     47   1.2        ad  * 4. Neither the name of The NetBSD Foundation nor the names of its
     48   1.2        ad  *    contributors may be used to endorse or promote products derived
     49   1.2        ad  *    from this software without specific prior written permission.
     50   1.2        ad  *
     51   1.2        ad  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     52   1.2        ad  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     53   1.2        ad  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     54   1.2        ad  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     55   1.2        ad  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     56   1.2        ad  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     57   1.2        ad  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     58   1.2        ad  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     59   1.2        ad  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     60   1.2        ad  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     61   1.2        ad  * SUCH DAMAGE.
     62   1.2        ad  */
     63   1.2        ad 
     64   1.2        ad #include <sys/cdefs.h>
     65  1.67       jym __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.67 2010/02/09 23:47:10 jym Exp $");
     66   1.2        ad 
     67   1.2        ad #include "opt_ddb.h"
     68   1.2        ad #include "opt_mpbios.h"		/* for MPDEBUG */
     69   1.2        ad #include "opt_mtrr.h"
     70   1.2        ad 
     71   1.2        ad #include "lapic.h"
     72   1.2        ad #include "ioapic.h"
     73   1.2        ad 
     74  1.62    bouyer #ifdef i386
     75  1.62    bouyer #include "npx.h"
     76  1.62    bouyer #endif
     77  1.62    bouyer 
     78   1.2        ad #include <sys/param.h>
     79   1.2        ad #include <sys/proc.h>
     80   1.2        ad #include <sys/systm.h>
     81   1.2        ad #include <sys/device.h>
     82  1.61    cegger #include <sys/kmem.h>
     83   1.9        ad #include <sys/cpu.h>
     84   1.9        ad #include <sys/atomic.h>
     85  1.35        ad #include <sys/reboot.h>
     86   1.2        ad 
     87   1.2        ad #include <uvm/uvm_extern.h>
     88   1.2        ad 
     89   1.2        ad #include <machine/cpufunc.h>
     90   1.2        ad #include <machine/cpuvar.h>
     91   1.2        ad #include <machine/pmap.h>
     92   1.2        ad #include <machine/vmparam.h>
     93   1.2        ad #include <machine/mpbiosvar.h>
     94   1.2        ad #include <machine/pcb.h>
     95   1.2        ad #include <machine/specialreg.h>
     96   1.2        ad #include <machine/segments.h>
     97   1.2        ad #include <machine/gdt.h>
     98   1.2        ad #include <machine/mtrr.h>
     99   1.2        ad #include <machine/pio.h>
    100  1.38        ad #include <machine/cpu_counter.h>
    101   1.2        ad 
    102   1.2        ad #ifdef i386
    103   1.2        ad #include <machine/tlog.h>
    104   1.2        ad #endif
    105   1.2        ad 
    106   1.2        ad #include <machine/apicvar.h>
    107   1.2        ad #include <machine/i82489reg.h>
    108   1.2        ad #include <machine/i82489var.h>
    109   1.2        ad 
    110   1.2        ad #include <dev/ic/mc146818reg.h>
    111   1.2        ad #include <i386/isa/nvram.h>
    112   1.2        ad #include <dev/isa/isareg.h>
    113   1.2        ad 
    114  1.38        ad #include "tsc.h"
    115  1.38        ad 
    116  1.54        ad #if MAXCPUS > 32
    117  1.54        ad #error cpu_info contains 32bit bitmasks
    118  1.54        ad #endif
    119  1.54        ad 
    120  1.23      cube int     cpu_match(device_t, cfdata_t, void *);
    121  1.23      cube void    cpu_attach(device_t, device_t, void *);
    122   1.2        ad 
    123  1.66    dyoung static bool	cpu_suspend(device_t, pmf_qual_t);
    124  1.66    dyoung static bool	cpu_resume(device_t, pmf_qual_t);
    125  1.12  jmcneill 
    126   1.2        ad struct cpu_softc {
    127  1.23      cube 	device_t sc_dev;		/* device tree glue */
    128   1.2        ad 	struct cpu_info *sc_info;	/* pointer to CPU info */
    129  1.20  jmcneill 	bool sc_wasonline;
    130   1.2        ad };
    131   1.2        ad 
    132  1.14     joerg int mp_cpu_start(struct cpu_info *, paddr_t);
    133   1.2        ad void mp_cpu_start_cleanup(struct cpu_info *);
    134   1.2        ad const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    135   1.2        ad 					    mp_cpu_start_cleanup };
    136   1.2        ad 
    137   1.2        ad 
    138  1.23      cube CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc),
    139   1.2        ad     cpu_match, cpu_attach, NULL, NULL);
    140   1.2        ad 
    141   1.2        ad /*
    142   1.2        ad  * Statically-allocated CPU info for the primary CPU (or the only
    143   1.2        ad  * CPU, on uniprocessors).  The CPU info list is initialized to
    144   1.2        ad  * point at it.
    145   1.2        ad  */
    146   1.2        ad #ifdef TRAPLOG
    147   1.2        ad struct tlog tlog_primary;
    148   1.2        ad #endif
    149  1.21        ad struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    150   1.2        ad 	.ci_dev = 0,
    151   1.2        ad 	.ci_self = &cpu_info_primary,
    152   1.2        ad 	.ci_idepth = -1,
    153   1.2        ad 	.ci_curlwp = &lwp0,
    154  1.43        ad 	.ci_curldt = -1,
    155   1.2        ad #ifdef TRAPLOG
    156   1.2        ad 	.ci_tlog_base = &tlog_primary,
    157   1.2        ad #endif /* !TRAPLOG */
    158   1.2        ad };
    159   1.2        ad 
    160   1.2        ad struct cpu_info *cpu_info_list = &cpu_info_primary;
    161   1.2        ad 
    162  1.12  jmcneill static void	cpu_set_tss_gates(struct cpu_info *);
    163   1.2        ad 
    164   1.2        ad #ifdef i386
    165  1.15      yamt static void	tss_init(struct i386tss *, void *, void *);
    166   1.2        ad #endif
    167   1.2        ad 
    168  1.12  jmcneill static void	cpu_init_idle_lwp(struct cpu_info *);
    169  1.12  jmcneill 
    170   1.2        ad uint32_t cpus_attached = 0;
    171   1.9        ad uint32_t cpus_running = 0;
    172   1.2        ad 
    173   1.2        ad extern char x86_64_doubleflt_stack[];
    174   1.2        ad 
    175  1.12  jmcneill bool x86_mp_online;
    176  1.12  jmcneill paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    177  1.14     joerg static vaddr_t cmos_data_mapping;
    178  1.45        ad struct cpu_info *cpu_starting;
    179   1.2        ad 
    180   1.2        ad void    	cpu_hatch(void *);
    181   1.2        ad static void    	cpu_boot_secondary(struct cpu_info *ci);
    182   1.2        ad static void    	cpu_start_secondary(struct cpu_info *ci);
    183   1.2        ad static void	cpu_copy_trampoline(void);
    184   1.2        ad 
    185   1.2        ad /*
    186   1.2        ad  * Runs once per boot once multiprocessor goo has been detected and
    187   1.2        ad  * the local APIC on the boot processor has been mapped.
    188   1.2        ad  *
    189   1.2        ad  * Called from lapic_boot_init() (from mpbios_scan()).
    190   1.2        ad  */
    191   1.2        ad void
    192   1.9        ad cpu_init_first(void)
    193   1.2        ad {
    194   1.2        ad 
    195  1.45        ad 	cpu_info_primary.ci_cpuid = lapic_cpu_number();
    196   1.2        ad 	cpu_copy_trampoline();
    197  1.14     joerg 
    198  1.14     joerg 	cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY);
    199  1.14     joerg 	if (cmos_data_mapping == 0)
    200  1.14     joerg 		panic("No KVA for page 0");
    201  1.64    cegger 	pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
    202  1.14     joerg 	pmap_update(pmap_kernel());
    203   1.2        ad }
    204   1.2        ad 
    205   1.2        ad int
    206  1.23      cube cpu_match(device_t parent, cfdata_t match, void *aux)
    207   1.2        ad {
    208   1.2        ad 
    209   1.2        ad 	return 1;
    210   1.2        ad }
    211   1.2        ad 
    212   1.2        ad static void
    213   1.2        ad cpu_vm_init(struct cpu_info *ci)
    214   1.2        ad {
    215   1.2        ad 	int ncolors = 2, i;
    216   1.2        ad 
    217   1.2        ad 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    218   1.2        ad 		struct x86_cache_info *cai;
    219   1.2        ad 		int tcolors;
    220   1.2        ad 
    221   1.2        ad 		cai = &ci->ci_cinfo[i];
    222   1.2        ad 
    223   1.2        ad 		tcolors = atop(cai->cai_totalsize);
    224   1.2        ad 		switch(cai->cai_associativity) {
    225   1.2        ad 		case 0xff:
    226   1.2        ad 			tcolors = 1; /* fully associative */
    227   1.2        ad 			break;
    228   1.2        ad 		case 0:
    229   1.2        ad 		case 1:
    230   1.2        ad 			break;
    231   1.2        ad 		default:
    232   1.2        ad 			tcolors /= cai->cai_associativity;
    233   1.2        ad 		}
    234   1.2        ad 		ncolors = max(ncolors, tcolors);
    235  1.32       tls 		/*
    236  1.32       tls 		 * If the desired number of colors is not a power of
    237  1.32       tls 		 * two, it won't be good.  Find the greatest power of
    238  1.32       tls 		 * two which is an even divisor of the number of colors,
    239  1.32       tls 		 * to preserve even coloring of pages.
    240  1.32       tls 		 */
    241  1.32       tls 		if (ncolors & (ncolors - 1) ) {
    242  1.32       tls 			int try, picked = 1;
    243  1.32       tls 			for (try = 1; try < ncolors; try *= 2) {
    244  1.32       tls 				if (ncolors % try == 0) picked = try;
    245  1.32       tls 			}
    246  1.32       tls 			if (picked == 1) {
    247  1.32       tls 				panic("desired number of cache colors %d is "
    248  1.32       tls 			      	" > 1, but not even!", ncolors);
    249  1.32       tls 			}
    250  1.32       tls 			ncolors = picked;
    251  1.32       tls 		}
    252   1.2        ad 	}
    253   1.2        ad 
    254   1.2        ad 	/*
    255   1.2        ad 	 * Knowing the size of the largest cache on this CPU, re-color
    256   1.2        ad 	 * our pages.
    257   1.2        ad 	 */
    258   1.2        ad 	if (ncolors <= uvmexp.ncolors)
    259   1.2        ad 		return;
    260  1.52        ad 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    261   1.2        ad 	uvm_page_recolor(ncolors);
    262   1.2        ad }
    263   1.2        ad 
    264   1.2        ad 
    265   1.2        ad void
    266  1.23      cube cpu_attach(device_t parent, device_t self, void *aux)
    267   1.2        ad {
    268  1.23      cube 	struct cpu_softc *sc = device_private(self);
    269   1.2        ad 	struct cpu_attach_args *caa = aux;
    270   1.2        ad 	struct cpu_info *ci;
    271  1.21        ad 	uintptr_t ptr;
    272   1.2        ad 	int cpunum = caa->cpu_number;
    273  1.51        ad 	static bool again;
    274   1.2        ad 
    275  1.23      cube 	sc->sc_dev = self;
    276  1.23      cube 
    277  1.48        ad 	if (cpus_attached == ~0) {
    278  1.54        ad 		aprint_error(": increase MAXCPUS\n");
    279  1.48        ad 		return;
    280  1.48        ad 	}
    281  1.48        ad 
    282   1.2        ad 	/*
    283   1.2        ad 	 * If we're an Application Processor, allocate a cpu_info
    284   1.2        ad 	 * structure, otherwise use the primary's.
    285   1.2        ad 	 */
    286   1.2        ad 	if (caa->cpu_role == CPU_ROLE_AP) {
    287  1.36        ad 		if ((boothowto & RB_MD1) != 0) {
    288  1.35        ad 			aprint_error(": multiprocessor boot disabled\n");
    289  1.56  jmcneill 			if (!pmf_device_register(self, NULL, NULL))
    290  1.56  jmcneill 				aprint_error_dev(self,
    291  1.56  jmcneill 				    "couldn't establish power handler\n");
    292  1.35        ad 			return;
    293  1.35        ad 		}
    294   1.2        ad 		aprint_naive(": Application Processor\n");
    295  1.61    cegger 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    296  1.61    cegger 		    KM_SLEEP);
    297  1.67       jym 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    298   1.2        ad 		memset(ci, 0, sizeof(*ci));
    299  1.43        ad 		ci->ci_curldt = -1;
    300   1.2        ad #ifdef TRAPLOG
    301  1.61    cegger 		ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
    302   1.2        ad #endif
    303   1.2        ad 	} else {
    304   1.2        ad 		aprint_naive(": %s Processor\n",
    305   1.2        ad 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    306   1.2        ad 		ci = &cpu_info_primary;
    307   1.2        ad 		if (cpunum != lapic_cpu_number()) {
    308  1.51        ad 			/* XXX should be done earlier. */
    309  1.39        ad 			uint32_t reg;
    310  1.39        ad 			aprint_verbose("\n");
    311  1.47        ad 			aprint_verbose_dev(self, "running CPU at apic %d"
    312  1.47        ad 			    " instead of at expected %d", lapic_cpu_number(),
    313  1.23      cube 			    cpunum);
    314  1.39        ad 			reg = i82489_readreg(LAPIC_ID);
    315  1.39        ad 			i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
    316  1.39        ad 			    (cpunum << LAPIC_ID_SHIFT));
    317   1.2        ad 		}
    318  1.47        ad 		if (cpunum != lapic_cpu_number()) {
    319  1.47        ad 			aprint_error_dev(self, "unable to reset apic id\n");
    320  1.47        ad 		}
    321   1.2        ad 	}
    322   1.2        ad 
    323   1.2        ad 	ci->ci_self = ci;
    324   1.2        ad 	sc->sc_info = ci;
    325   1.2        ad 	ci->ci_dev = self;
    326  1.42        ad 	ci->ci_cpuid = caa->cpu_number;
    327   1.2        ad 	ci->ci_func = caa->cpu_func;
    328   1.2        ad 
    329  1.55        ad 	/* Must be before mi_cpu_attach(). */
    330  1.55        ad 	cpu_vm_init(ci);
    331  1.55        ad 
    332   1.2        ad 	if (caa->cpu_role == CPU_ROLE_AP) {
    333   1.2        ad 		int error;
    334   1.2        ad 
    335   1.2        ad 		error = mi_cpu_attach(ci);
    336   1.2        ad 		if (error != 0) {
    337   1.2        ad 			aprint_normal("\n");
    338  1.47        ad 			aprint_error_dev(self,
    339  1.30    cegger 			    "mi_cpu_attach failed with %d\n", error);
    340   1.2        ad 			return;
    341   1.2        ad 		}
    342  1.15      yamt 		cpu_init_tss(ci);
    343   1.2        ad 	} else {
    344   1.2        ad 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    345   1.2        ad 	}
    346   1.2        ad 
    347  1.42        ad 	ci->ci_cpumask = (1 << cpu_index(ci));
    348   1.2        ad 	pmap_reference(pmap_kernel());
    349   1.2        ad 	ci->ci_pmap = pmap_kernel();
    350   1.2        ad 	ci->ci_tlbstate = TLBSTATE_STALE;
    351   1.2        ad 
    352  1.51        ad 	/*
    353  1.51        ad 	 * Boot processor may not be attached first, but the below
    354  1.51        ad 	 * must be done to allow booting other processors.
    355  1.51        ad 	 */
    356  1.51        ad 	if (!again) {
    357  1.51        ad 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    358  1.51        ad 		/* Basic init. */
    359   1.2        ad 		cpu_intr_init(ci);
    360  1.40        ad 		cpu_get_tsc_freq(ci);
    361   1.2        ad 		cpu_init(ci);
    362   1.2        ad 		cpu_set_tss_gates(ci);
    363   1.2        ad 		pmap_cpu_init_late(ci);
    364  1.51        ad 		if (caa->cpu_role != CPU_ROLE_SP) {
    365  1.51        ad 			/* Enable lapic. */
    366  1.51        ad 			lapic_enable();
    367  1.51        ad 			lapic_set_lvt();
    368  1.51        ad 			lapic_calibrate_timer(ci);
    369  1.51        ad 		}
    370  1.51        ad 		/* Make sure DELAY() is initialized. */
    371  1.51        ad 		DELAY(1);
    372  1.51        ad 		again = true;
    373  1.51        ad 	}
    374  1.51        ad 
    375  1.51        ad 	/* further PCB init done later. */
    376  1.51        ad 
    377  1.51        ad 	switch (caa->cpu_role) {
    378  1.51        ad 	case CPU_ROLE_SP:
    379  1.51        ad 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    380  1.51        ad 		cpu_identify(ci);
    381  1.53        ad 		x86_errata();
    382  1.37     joerg 		x86_cpu_idle_init();
    383   1.2        ad 		break;
    384   1.2        ad 
    385   1.2        ad 	case CPU_ROLE_BP:
    386  1.51        ad 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    387  1.40        ad 		cpu_identify(ci);
    388  1.53        ad 		x86_errata();
    389  1.37     joerg 		x86_cpu_idle_init();
    390   1.2        ad 		break;
    391   1.2        ad 
    392   1.2        ad 	case CPU_ROLE_AP:
    393   1.2        ad 		/*
    394   1.2        ad 		 * report on an AP
    395   1.2        ad 		 */
    396   1.2        ad 		cpu_intr_init(ci);
    397   1.2        ad 		gdt_alloc_cpu(ci);
    398   1.2        ad 		cpu_set_tss_gates(ci);
    399   1.2        ad 		pmap_cpu_init_early(ci);
    400   1.2        ad 		pmap_cpu_init_late(ci);
    401   1.2        ad 		cpu_start_secondary(ci);
    402   1.2        ad 		if (ci->ci_flags & CPUF_PRESENT) {
    403  1.59    cegger 			struct cpu_info *tmp;
    404  1.59    cegger 
    405  1.40        ad 			cpu_identify(ci);
    406  1.59    cegger 			tmp = cpu_info_list;
    407  1.59    cegger 			while (tmp->ci_next)
    408  1.59    cegger 				tmp = tmp->ci_next;
    409  1.59    cegger 
    410  1.59    cegger 			tmp->ci_next = ci;
    411   1.2        ad 		}
    412   1.2        ad 		break;
    413   1.2        ad 
    414   1.2        ad 	default:
    415  1.28    cegger 		aprint_normal("\n");
    416   1.2        ad 		panic("unknown processor type??\n");
    417   1.2        ad 	}
    418  1.51        ad 
    419  1.47        ad 	atomic_or_32(&cpus_attached, ci->ci_cpumask);
    420   1.2        ad 
    421  1.12  jmcneill 	if (!pmf_device_register(self, cpu_suspend, cpu_resume))
    422  1.12  jmcneill 		aprint_error_dev(self, "couldn't establish power handler\n");
    423  1.12  jmcneill 
    424   1.2        ad 	if (mp_verbose) {
    425   1.2        ad 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    426  1.65     rmind 		struct pcb *pcb = lwp_getpcb(l);
    427   1.2        ad 
    428  1.47        ad 		aprint_verbose_dev(self,
    429  1.28    cegger 		    "idle lwp at %p, idle sp at %p\n",
    430  1.28    cegger 		    l,
    431   1.2        ad #ifdef i386
    432  1.65     rmind 		    (void *)pcb->pcb_esp
    433   1.2        ad #else
    434  1.65     rmind 		    (void *)pcb->pcb_rsp
    435   1.2        ad #endif
    436   1.2        ad 		);
    437   1.2        ad 	}
    438   1.2        ad }
    439   1.2        ad 
    440   1.2        ad /*
    441   1.2        ad  * Initialize the processor appropriately.
    442   1.2        ad  */
    443   1.2        ad 
    444   1.2        ad void
    445   1.9        ad cpu_init(struct cpu_info *ci)
    446   1.2        ad {
    447   1.2        ad 
    448   1.2        ad 	lcr0(rcr0() | CR0_WP);
    449   1.2        ad 
    450   1.2        ad 	/*
    451   1.2        ad 	 * On a P6 or above, enable global TLB caching if the
    452   1.2        ad 	 * hardware supports it.
    453   1.2        ad 	 */
    454   1.2        ad 	if (cpu_feature & CPUID_PGE)
    455   1.2        ad 		lcr4(rcr4() | CR4_PGE);	/* enable global TLB caching */
    456   1.2        ad 
    457   1.2        ad 	/*
    458   1.2        ad 	 * If we have FXSAVE/FXRESTOR, use them.
    459   1.2        ad 	 */
    460   1.2        ad 	if (cpu_feature & CPUID_FXSR) {
    461   1.2        ad 		lcr4(rcr4() | CR4_OSFXSR);
    462   1.2        ad 
    463   1.2        ad 		/*
    464   1.2        ad 		 * If we have SSE/SSE2, enable XMM exceptions.
    465   1.2        ad 		 */
    466   1.2        ad 		if (cpu_feature & (CPUID_SSE|CPUID_SSE2))
    467   1.2        ad 			lcr4(rcr4() | CR4_OSXMMEXCPT);
    468   1.2        ad 	}
    469   1.2        ad 
    470   1.2        ad #ifdef MTRR
    471   1.2        ad 	/*
    472   1.2        ad 	 * On a P6 or above, initialize MTRR's if the hardware supports them.
    473   1.2        ad 	 */
    474   1.2        ad 	if (cpu_feature & CPUID_MTRR) {
    475   1.2        ad 		if ((ci->ci_flags & CPUF_AP) == 0)
    476   1.2        ad 			i686_mtrr_init_first();
    477   1.2        ad 		mtrr_init_cpu(ci);
    478   1.2        ad 	}
    479   1.2        ad 
    480   1.2        ad #ifdef i386
    481   1.2        ad 	if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
    482   1.2        ad 		/*
    483   1.2        ad 		 * Must be a K6-2 Step >= 7 or a K6-III.
    484   1.2        ad 		 */
    485   1.2        ad 		if (CPUID2FAMILY(ci->ci_signature) == 5) {
    486   1.2        ad 			if (CPUID2MODEL(ci->ci_signature) > 8 ||
    487   1.2        ad 			    (CPUID2MODEL(ci->ci_signature) == 8 &&
    488   1.2        ad 			     CPUID2STEPPING(ci->ci_signature) >= 7)) {
    489   1.2        ad 				mtrr_funcs = &k6_mtrr_funcs;
    490   1.2        ad 				k6_mtrr_init_first();
    491   1.2        ad 				mtrr_init_cpu(ci);
    492   1.2        ad 			}
    493   1.2        ad 		}
    494   1.2        ad 	}
    495   1.2        ad #endif	/* i386 */
    496   1.2        ad #endif /* MTRR */
    497   1.2        ad 
    498   1.9        ad 	atomic_or_32(&cpus_running, ci->ci_cpumask);
    499   1.9        ad 
    500  1.38        ad 	if (ci != &cpu_info_primary) {
    501  1.38        ad 		/* Synchronize TSC again, and check for drift. */
    502  1.38        ad 		wbinvd();
    503  1.38        ad 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    504  1.38        ad 		tsc_sync_ap(ci);
    505  1.38        ad 	} else {
    506  1.38        ad 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    507  1.38        ad 	}
    508   1.2        ad }
    509   1.2        ad 
    510   1.2        ad void
    511  1.12  jmcneill cpu_boot_secondary_processors(void)
    512   1.2        ad {
    513   1.2        ad 	struct cpu_info *ci;
    514   1.2        ad 	u_long i;
    515   1.2        ad 
    516   1.5        ad 	/* Now that we know the number of CPUs, patch the text segment. */
    517  1.60        ad 	x86_patch(false);
    518   1.5        ad 
    519  1.54        ad 	for (i=0; i < maxcpus; i++) {
    520  1.57        ad 		ci = cpu_lookup(i);
    521   1.2        ad 		if (ci == NULL)
    522   1.2        ad 			continue;
    523   1.2        ad 		if (ci->ci_data.cpu_idlelwp == NULL)
    524   1.2        ad 			continue;
    525   1.2        ad 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    526   1.2        ad 			continue;
    527   1.2        ad 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    528   1.2        ad 			continue;
    529   1.2        ad 		cpu_boot_secondary(ci);
    530   1.2        ad 	}
    531   1.2        ad 
    532   1.2        ad 	x86_mp_online = true;
    533  1.38        ad 
    534  1.38        ad 	/* Now that we know about the TSC, attach the timecounter. */
    535  1.38        ad 	tsc_tc_init();
    536  1.55        ad 
    537  1.55        ad 	/* Enable zeroing of pages in the idle loop if we have SSE2. */
    538  1.55        ad 	vm_page_zero_enable = ((cpu_feature & CPUID_SSE2) != 0);
    539   1.2        ad }
    540   1.2        ad 
    541   1.2        ad static void
    542   1.2        ad cpu_init_idle_lwp(struct cpu_info *ci)
    543   1.2        ad {
    544   1.2        ad 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    545  1.65     rmind 	struct pcb *pcb = lwp_getpcb(l);
    546   1.2        ad 
    547   1.2        ad 	pcb->pcb_cr0 = rcr0();
    548   1.2        ad }
    549   1.2        ad 
    550   1.2        ad void
    551  1.12  jmcneill cpu_init_idle_lwps(void)
    552   1.2        ad {
    553   1.2        ad 	struct cpu_info *ci;
    554   1.2        ad 	u_long i;
    555   1.2        ad 
    556  1.54        ad 	for (i = 0; i < maxcpus; i++) {
    557  1.57        ad 		ci = cpu_lookup(i);
    558   1.2        ad 		if (ci == NULL)
    559   1.2        ad 			continue;
    560   1.2        ad 		if (ci->ci_data.cpu_idlelwp == NULL)
    561   1.2        ad 			continue;
    562   1.2        ad 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    563   1.2        ad 			continue;
    564   1.2        ad 		cpu_init_idle_lwp(ci);
    565   1.2        ad 	}
    566   1.2        ad }
    567   1.2        ad 
    568   1.2        ad void
    569  1.12  jmcneill cpu_start_secondary(struct cpu_info *ci)
    570   1.2        ad {
    571  1.38        ad 	extern paddr_t mp_pdirpa;
    572  1.38        ad 	u_long psl;
    573   1.2        ad 	int i;
    574   1.2        ad 
    575  1.12  jmcneill 	mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr);
    576   1.9        ad 	atomic_or_32(&ci->ci_flags, CPUF_AP);
    577   1.2        ad 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    578  1.45        ad 	if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) {
    579  1.25        ad 		return;
    580  1.45        ad 	}
    581   1.2        ad 
    582   1.2        ad 	/*
    583  1.50        ad 	 * Wait for it to become ready.   Setting cpu_starting opens the
    584  1.50        ad 	 * initial gate and allows the AP to start soft initialization.
    585   1.2        ad 	 */
    586  1.50        ad 	KASSERT(cpu_starting == NULL);
    587  1.50        ad 	cpu_starting = ci;
    588  1.26    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    589  1.24        ad #ifdef MPDEBUG
    590  1.24        ad 		extern int cpu_trace[3];
    591  1.24        ad 		static int otrace[3];
    592  1.24        ad 		if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
    593  1.26    cegger 			aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
    594  1.26    cegger 			    cpu_trace[0], cpu_trace[1], cpu_trace[2]);
    595  1.24        ad 			memcpy(otrace, cpu_trace, sizeof(otrace));
    596  1.24        ad 		}
    597  1.24        ad #endif
    598  1.11        ad 		i8254_delay(10);
    599   1.2        ad 	}
    600  1.38        ad 
    601   1.9        ad 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    602  1.26    cegger 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    603   1.2        ad #if defined(MPDEBUG) && defined(DDB)
    604   1.2        ad 		printf("dropping into debugger; continue from here to resume boot\n");
    605   1.2        ad 		Debugger();
    606   1.2        ad #endif
    607  1.38        ad 	} else {
    608  1.38        ad 		/*
    609  1.38        ad 		 * Synchronize time stamp counters.  Invalidate cache and do twice
    610  1.38        ad 		 * to try and minimize possible cache effects.  Disable interrupts
    611  1.38        ad 		 * to try and rule out any external interference.
    612  1.38        ad 		 */
    613  1.38        ad 		psl = x86_read_psl();
    614  1.38        ad 		x86_disable_intr();
    615  1.38        ad 		wbinvd();
    616  1.38        ad 		tsc_sync_bp(ci);
    617  1.38        ad 		x86_write_psl(psl);
    618   1.2        ad 	}
    619   1.2        ad 
    620   1.2        ad 	CPU_START_CLEANUP(ci);
    621  1.45        ad 	cpu_starting = NULL;
    622   1.2        ad }
    623   1.2        ad 
    624   1.2        ad void
    625  1.12  jmcneill cpu_boot_secondary(struct cpu_info *ci)
    626   1.2        ad {
    627  1.38        ad 	int64_t drift;
    628  1.38        ad 	u_long psl;
    629   1.2        ad 	int i;
    630   1.2        ad 
    631   1.9        ad 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    632  1.26    cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    633  1.11        ad 		i8254_delay(10);
    634   1.2        ad 	}
    635   1.9        ad 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    636  1.26    cegger 		aprint_error_dev(ci->ci_dev, "failed to start\n");
    637   1.2        ad #if defined(MPDEBUG) && defined(DDB)
    638   1.2        ad 		printf("dropping into debugger; continue from here to resume boot\n");
    639   1.2        ad 		Debugger();
    640   1.2        ad #endif
    641  1.38        ad 	} else {
    642  1.38        ad 		/* Synchronize TSC again, check for drift. */
    643  1.38        ad 		drift = ci->ci_data.cpu_cc_skew;
    644  1.38        ad 		psl = x86_read_psl();
    645  1.38        ad 		x86_disable_intr();
    646  1.38        ad 		wbinvd();
    647  1.38        ad 		tsc_sync_bp(ci);
    648  1.38        ad 		x86_write_psl(psl);
    649  1.38        ad 		drift -= ci->ci_data.cpu_cc_skew;
    650  1.38        ad 		aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n",
    651  1.38        ad 		    (long long)ci->ci_data.cpu_cc_skew, (long long)drift);
    652  1.38        ad 		tsc_sync_drift(drift);
    653   1.2        ad 	}
    654   1.2        ad }
    655   1.2        ad 
    656   1.2        ad /*
    657   1.2        ad  * The CPU ends up here when its ready to run
    658   1.2        ad  * This is called from code in mptramp.s; at this point, we are running
    659   1.2        ad  * in the idle pcb/idle stack of the new CPU.  When this function returns,
    660   1.2        ad  * this processor will enter the idle loop and start looking for work.
    661   1.2        ad  */
    662   1.2        ad void
    663   1.2        ad cpu_hatch(void *v)
    664   1.2        ad {
    665   1.2        ad 	struct cpu_info *ci = (struct cpu_info *)v;
    666  1.65     rmind 	struct pcb *pcb;
    667   1.6        ad 	int s, i;
    668   1.2        ad 
    669   1.2        ad #ifdef __x86_64__
    670  1.12  jmcneill 	cpu_init_msrs(ci, true);
    671   1.2        ad #endif
    672  1.40        ad 	cpu_probe(ci);
    673  1.46        ad 
    674  1.46        ad 	ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq;
    675  1.46        ad 	/* cpu_get_tsc_freq(ci); */
    676  1.38        ad 
    677   1.8        ad 	KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
    678  1.38        ad 
    679  1.38        ad 	/*
    680  1.38        ad 	 * Synchronize time stamp counters.  Invalidate cache and do twice
    681  1.38        ad 	 * to try and minimize possible cache effects.  Note that interrupts
    682  1.38        ad 	 * are off at this point.
    683  1.38        ad 	 */
    684  1.38        ad 	wbinvd();
    685   1.9        ad 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    686  1.38        ad 	tsc_sync_ap(ci);
    687  1.38        ad 
    688  1.38        ad 	/*
    689  1.38        ad 	 * Wait to be brought online.  Use 'monitor/mwait' if available,
    690  1.38        ad 	 * in order to make the TSC drift as much as possible. so that
    691  1.38        ad 	 * we can detect it later.  If not available, try 'pause'.
    692  1.38        ad 	 * We'd like to use 'hlt', but we have interrupts off.
    693  1.38        ad 	 */
    694   1.6        ad 	while ((ci->ci_flags & CPUF_GO) == 0) {
    695  1.38        ad 		if ((ci->ci_feature2_flags & CPUID2_MONITOR) != 0) {
    696  1.38        ad 			x86_monitor(&ci->ci_flags, 0, 0);
    697  1.38        ad 			if ((ci->ci_flags & CPUF_GO) != 0) {
    698  1.38        ad 				continue;
    699  1.38        ad 			}
    700  1.38        ad 			x86_mwait(0, 0);
    701  1.38        ad 		} else {
    702  1.38        ad 			for (i = 10000; i != 0; i--) {
    703  1.38        ad 				x86_pause();
    704  1.38        ad 			}
    705  1.38        ad 		}
    706   1.6        ad 	}
    707   1.5        ad 
    708  1.26    cegger 	/* Because the text may have been patched in x86_patch(). */
    709   1.5        ad 	wbinvd();
    710   1.5        ad 	x86_flush();
    711   1.5        ad 
    712   1.8        ad 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    713   1.2        ad 
    714  1.12  jmcneill 	lcr3(pmap_kernel()->pm_pdirpa);
    715  1.65     rmind 	pcb = lwp_getpcb(curlwp);
    716  1.65     rmind 	pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa;
    717  1.65     rmind 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
    718  1.65     rmind 	lcr0(pcb->pcb_cr0);
    719  1.65     rmind 
    720   1.2        ad 	cpu_init_idt();
    721   1.8        ad 	gdt_init_cpu(ci);
    722   1.8        ad 	lapic_enable();
    723   1.2        ad 	lapic_set_lvt();
    724   1.8        ad 	lapic_initclocks();
    725   1.2        ad 
    726   1.2        ad #ifdef i386
    727  1.62    bouyer #if NNPX > 0
    728   1.2        ad 	npxinit(ci);
    729  1.62    bouyer #endif
    730   1.2        ad #else
    731   1.2        ad 	fpuinit(ci);
    732   1.4      yamt #endif
    733   1.2        ad 	lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
    734  1.15      yamt 	ltr(ci->ci_tss_sel);
    735   1.2        ad 
    736   1.2        ad 	cpu_init(ci);
    737   1.7        ad 	cpu_get_tsc_freq(ci);
    738   1.2        ad 
    739   1.2        ad 	s = splhigh();
    740   1.2        ad #ifdef i386
    741   1.2        ad 	lapic_tpr = 0;
    742   1.2        ad #else
    743   1.2        ad 	lcr8(0);
    744   1.2        ad #endif
    745   1.3        ad 	x86_enable_intr();
    746   1.2        ad 	splx(s);
    747   1.6        ad 	x86_errata();
    748   1.2        ad 
    749  1.42        ad 	aprint_debug_dev(ci->ci_dev, "running\n");
    750   1.2        ad }
    751   1.2        ad 
    752   1.2        ad #if defined(DDB)
    753   1.2        ad 
    754   1.2        ad #include <ddb/db_output.h>
    755   1.2        ad #include <machine/db_machdep.h>
    756   1.2        ad 
    757   1.2        ad /*
    758   1.2        ad  * Dump CPU information from ddb.
    759   1.2        ad  */
    760   1.2        ad void
    761   1.2        ad cpu_debug_dump(void)
    762   1.2        ad {
    763   1.2        ad 	struct cpu_info *ci;
    764   1.2        ad 	CPU_INFO_ITERATOR cii;
    765   1.2        ad 
    766  1.29      yamt 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    767   1.2        ad 	for (CPU_INFO_FOREACH(cii, ci)) {
    768   1.2        ad 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    769   1.2        ad 		    ci,
    770  1.27    cegger 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    771   1.2        ad 		    (long)ci->ci_cpuid,
    772   1.2        ad 		    ci->ci_flags, ci->ci_ipis,
    773   1.2        ad 		    ci->ci_curlwp,
    774   1.2        ad 		    ci->ci_fpcurlwp);
    775   1.2        ad 	}
    776   1.2        ad }
    777   1.2        ad #endif
    778   1.2        ad 
    779   1.2        ad static void
    780  1.12  jmcneill cpu_copy_trampoline(void)
    781   1.2        ad {
    782   1.2        ad 	/*
    783   1.2        ad 	 * Copy boot code.
    784   1.2        ad 	 */
    785   1.2        ad 	extern u_char cpu_spinup_trampoline[];
    786   1.2        ad 	extern u_char cpu_spinup_trampoline_end[];
    787  1.12  jmcneill 
    788  1.12  jmcneill 	vaddr_t mp_trampoline_vaddr;
    789  1.12  jmcneill 
    790  1.12  jmcneill 	mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    791  1.12  jmcneill 	    UVM_KMF_VAONLY);
    792  1.12  jmcneill 
    793  1.12  jmcneill 	pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
    794  1.64    cegger 	    VM_PROT_READ | VM_PROT_WRITE, 0);
    795   1.2        ad 	pmap_update(pmap_kernel());
    796  1.12  jmcneill 	memcpy((void *)mp_trampoline_vaddr,
    797   1.2        ad 	    cpu_spinup_trampoline,
    798  1.26    cegger 	    cpu_spinup_trampoline_end - cpu_spinup_trampoline);
    799  1.12  jmcneill 
    800  1.12  jmcneill 	pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
    801  1.12  jmcneill 	pmap_update(pmap_kernel());
    802  1.12  jmcneill 	uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
    803   1.2        ad }
    804   1.2        ad 
    805   1.2        ad #ifdef i386
    806   1.2        ad static void
    807  1.15      yamt tss_init(struct i386tss *tss, void *stack, void *func)
    808   1.2        ad {
    809   1.2        ad 	memset(tss, 0, sizeof *tss);
    810   1.2        ad 	tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
    811   1.2        ad 	tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
    812   1.2        ad 	tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
    813   1.2        ad 	tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
    814   1.2        ad 	tss->tss_gs = tss->__tss_es = tss->__tss_ds =
    815   1.2        ad 	    tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
    816   1.2        ad 	tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
    817   1.2        ad 	tss->tss_esp = (int)((char *)stack + USPACE - 16);
    818   1.2        ad 	tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
    819   1.2        ad 	tss->__tss_eflags = PSL_MBO | PSL_NT;	/* XXX not needed? */
    820   1.2        ad 	tss->__tss_eip = (int)func;
    821   1.2        ad }
    822   1.2        ad 
    823   1.2        ad /* XXX */
    824   1.2        ad #define IDTVEC(name)	__CONCAT(X, name)
    825   1.2        ad typedef void (vector)(void);
    826   1.2        ad extern vector IDTVEC(tss_trap08);
    827   1.2        ad #ifdef DDB
    828   1.2        ad extern vector Xintrddbipi;
    829   1.2        ad extern int ddb_vec;
    830   1.2        ad #endif
    831   1.2        ad 
    832   1.2        ad static void
    833   1.2        ad cpu_set_tss_gates(struct cpu_info *ci)
    834   1.2        ad {
    835   1.2        ad 	struct segment_descriptor sd;
    836   1.2        ad 
    837   1.2        ad 	ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    838   1.2        ad 	    UVM_KMF_WIRED);
    839  1.15      yamt 	tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
    840   1.2        ad 	    IDTVEC(tss_trap08));
    841   1.2        ad 	setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
    842   1.2        ad 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    843   1.2        ad 	ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
    844   1.2        ad 	setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    845   1.2        ad 	    GSEL(GTRAPTSS_SEL, SEL_KPL));
    846   1.2        ad 
    847  1.44        ad #if defined(DDB)
    848   1.2        ad 	/*
    849   1.2        ad 	 * Set up separate handler for the DDB IPI, so that it doesn't
    850   1.2        ad 	 * stomp on a possibly corrupted stack.
    851   1.2        ad 	 *
    852   1.2        ad 	 * XXX overwriting the gate set in db_machine_init.
    853   1.2        ad 	 * Should rearrange the code so that it's set only once.
    854   1.2        ad 	 */
    855   1.2        ad 	ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    856   1.2        ad 	    UVM_KMF_WIRED);
    857  1.15      yamt 	tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi);
    858   1.2        ad 
    859   1.2        ad 	setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
    860   1.2        ad 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    861   1.2        ad 	ci->ci_gdt[GIPITSS_SEL].sd = sd;
    862   1.2        ad 
    863   1.2        ad 	setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    864   1.2        ad 	    GSEL(GIPITSS_SEL, SEL_KPL));
    865   1.2        ad #endif
    866   1.2        ad }
    867   1.2        ad #else
    868   1.2        ad static void
    869   1.2        ad cpu_set_tss_gates(struct cpu_info *ci)
    870   1.2        ad {
    871   1.2        ad 
    872   1.2        ad }
    873   1.2        ad #endif	/* i386 */
    874   1.2        ad 
    875   1.2        ad int
    876  1.14     joerg mp_cpu_start(struct cpu_info *ci, paddr_t target)
    877   1.2        ad {
    878  1.44        ad 	unsigned short dwordptr[2];
    879   1.2        ad 	int error;
    880  1.14     joerg 
    881  1.14     joerg 	/*
    882  1.14     joerg 	 * Bootstrap code must be addressable in real mode
    883  1.14     joerg 	 * and it must be page aligned.
    884  1.14     joerg 	 */
    885  1.14     joerg 	KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
    886   1.2        ad 
    887   1.2        ad 	/*
    888   1.2        ad 	 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
    889   1.2        ad 	 */
    890   1.2        ad 
    891   1.2        ad 	outb(IO_RTC, NVRAM_RESET);
    892   1.2        ad 	outb(IO_RTC+1, NVRAM_RESET_JUMP);
    893   1.2        ad 
    894   1.2        ad 	/*
    895   1.2        ad 	 * "and the warm reset vector (DWORD based at 40:67) to point
    896   1.2        ad 	 * to the AP startup code ..."
    897   1.2        ad 	 */
    898   1.2        ad 
    899   1.2        ad 	dwordptr[0] = 0;
    900  1.14     joerg 	dwordptr[1] = target >> 4;
    901   1.2        ad 
    902  1.25        ad 	memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4);
    903   1.2        ad 
    904  1.25        ad 	if ((cpu_feature & CPUID_APIC) == 0) {
    905  1.25        ad 		aprint_error("mp_cpu_start: CPU does not have APIC\n");
    906  1.25        ad 		return ENODEV;
    907  1.25        ad 	}
    908  1.25        ad 
    909   1.2        ad 	/*
    910  1.51        ad 	 * ... prior to executing the following sequence:".  We'll also add in
    911  1.51        ad 	 * local cache flush, in case the BIOS has left the AP with its cache
    912  1.51        ad 	 * disabled.  It may not be able to cope with MP coherency.
    913   1.2        ad 	 */
    914  1.51        ad 	wbinvd();
    915   1.2        ad 
    916   1.2        ad 	if (ci->ci_flags & CPUF_AP) {
    917  1.42        ad 		error = x86_ipi_init(ci->ci_cpuid);
    918  1.26    cegger 		if (error != 0) {
    919  1.26    cegger 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
    920  1.50        ad 			    __func__);
    921   1.2        ad 			return error;
    922  1.25        ad 		}
    923  1.11        ad 		i8254_delay(10000);
    924   1.2        ad 
    925  1.50        ad 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
    926  1.26    cegger 		if (error != 0) {
    927  1.26    cegger 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
    928  1.50        ad 			    __func__);
    929  1.25        ad 			return error;
    930  1.25        ad 		}
    931  1.25        ad 		i8254_delay(200);
    932   1.2        ad 
    933  1.50        ad 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
    934  1.26    cegger 		if (error != 0) {
    935  1.26    cegger 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n",
    936  1.50        ad 			    __func__);
    937  1.25        ad 			return error;
    938   1.2        ad 		}
    939  1.25        ad 		i8254_delay(200);
    940   1.2        ad 	}
    941  1.44        ad 
    942   1.2        ad 	return 0;
    943   1.2        ad }
    944   1.2        ad 
    945   1.2        ad void
    946   1.2        ad mp_cpu_start_cleanup(struct cpu_info *ci)
    947   1.2        ad {
    948   1.2        ad 	/*
    949   1.2        ad 	 * Ensure the NVRAM reset byte contains something vaguely sane.
    950   1.2        ad 	 */
    951   1.2        ad 
    952   1.2        ad 	outb(IO_RTC, NVRAM_RESET);
    953   1.2        ad 	outb(IO_RTC+1, NVRAM_RESET_RST);
    954   1.2        ad }
    955   1.2        ad 
    956   1.2        ad #ifdef __x86_64__
    957   1.2        ad typedef void (vector)(void);
    958   1.2        ad extern vector Xsyscall, Xsyscall32;
    959   1.2        ad 
    960   1.2        ad void
    961  1.12  jmcneill cpu_init_msrs(struct cpu_info *ci, bool full)
    962   1.2        ad {
    963   1.2        ad 	wrmsr(MSR_STAR,
    964   1.2        ad 	    ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
    965   1.2        ad 	    ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
    966   1.2        ad 	wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
    967   1.2        ad 	wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
    968   1.2        ad 	wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
    969   1.2        ad 
    970  1.12  jmcneill 	if (full) {
    971  1.12  jmcneill 		wrmsr(MSR_FSBASE, 0);
    972  1.27    cegger 		wrmsr(MSR_GSBASE, (uint64_t)ci);
    973  1.12  jmcneill 		wrmsr(MSR_KERNELGSBASE, 0);
    974  1.12  jmcneill 	}
    975   1.2        ad 
    976   1.2        ad 	if (cpu_feature & CPUID_NOX)
    977   1.2        ad 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
    978   1.2        ad }
    979   1.2        ad #endif	/* __x86_64__ */
    980   1.7        ad 
    981  1.18     joerg void
    982  1.18     joerg cpu_offline_md(void)
    983  1.18     joerg {
    984  1.18     joerg 	int s;
    985  1.18     joerg 
    986  1.18     joerg 	s = splhigh();
    987  1.62    bouyer #ifdef i386
    988  1.62    bouyer #if NNPX > 0
    989  1.18     joerg 	npxsave_cpu(true);
    990  1.62    bouyer #endif
    991  1.18     joerg #else
    992  1.18     joerg 	fpusave_cpu(true);
    993  1.18     joerg #endif
    994  1.18     joerg 	splx(s);
    995  1.18     joerg }
    996  1.18     joerg 
    997  1.12  jmcneill /* XXX joerg restructure and restart CPUs individually */
    998  1.12  jmcneill static bool
    999  1.66    dyoung cpu_suspend(device_t dv, pmf_qual_t qual)
   1000  1.12  jmcneill {
   1001  1.12  jmcneill 	struct cpu_softc *sc = device_private(dv);
   1002  1.12  jmcneill 	struct cpu_info *ci = sc->sc_info;
   1003  1.18     joerg 	int err;
   1004  1.12  jmcneill 
   1005  1.13     joerg 	if (ci->ci_flags & CPUF_PRIMARY)
   1006  1.12  jmcneill 		return true;
   1007  1.12  jmcneill 	if (ci->ci_data.cpu_idlelwp == NULL)
   1008  1.12  jmcneill 		return true;
   1009  1.12  jmcneill 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1010  1.12  jmcneill 		return true;
   1011  1.12  jmcneill 
   1012  1.20  jmcneill 	sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
   1013  1.17     joerg 
   1014  1.20  jmcneill 	if (sc->sc_wasonline) {
   1015  1.20  jmcneill 		mutex_enter(&cpu_lock);
   1016  1.58     rmind 		err = cpu_setstate(ci, false);
   1017  1.20  jmcneill 		mutex_exit(&cpu_lock);
   1018  1.20  jmcneill 
   1019  1.20  jmcneill 		if (err)
   1020  1.20  jmcneill 			return false;
   1021  1.20  jmcneill 	}
   1022  1.17     joerg 
   1023  1.17     joerg 	return true;
   1024  1.12  jmcneill }
   1025  1.12  jmcneill 
   1026  1.12  jmcneill static bool
   1027  1.66    dyoung cpu_resume(device_t dv, pmf_qual_t qual)
   1028  1.12  jmcneill {
   1029  1.12  jmcneill 	struct cpu_softc *sc = device_private(dv);
   1030  1.12  jmcneill 	struct cpu_info *ci = sc->sc_info;
   1031  1.20  jmcneill 	int err = 0;
   1032  1.12  jmcneill 
   1033  1.13     joerg 	if (ci->ci_flags & CPUF_PRIMARY)
   1034  1.12  jmcneill 		return true;
   1035  1.12  jmcneill 	if (ci->ci_data.cpu_idlelwp == NULL)
   1036  1.12  jmcneill 		return true;
   1037  1.12  jmcneill 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1038  1.12  jmcneill 		return true;
   1039  1.12  jmcneill 
   1040  1.20  jmcneill 	if (sc->sc_wasonline) {
   1041  1.20  jmcneill 		mutex_enter(&cpu_lock);
   1042  1.58     rmind 		err = cpu_setstate(ci, true);
   1043  1.20  jmcneill 		mutex_exit(&cpu_lock);
   1044  1.20  jmcneill 	}
   1045  1.13     joerg 
   1046  1.13     joerg 	return err == 0;
   1047  1.12  jmcneill }
   1048  1.12  jmcneill 
   1049   1.7        ad void
   1050   1.7        ad cpu_get_tsc_freq(struct cpu_info *ci)
   1051   1.7        ad {
   1052   1.7        ad 	uint64_t last_tsc;
   1053   1.7        ad 
   1054   1.7        ad 	if (ci->ci_feature_flags & CPUID_TSC) {
   1055  1.45        ad 		last_tsc = rdmsr(MSR_TSC);
   1056   1.7        ad 		i8254_delay(100000);
   1057  1.45        ad 		ci->ci_data.cpu_cc_freq = (rdmsr(MSR_TSC) - last_tsc) * 10;
   1058   1.7        ad 	}
   1059   1.7        ad }
   1060  1.37     joerg 
   1061  1.37     joerg void
   1062  1.37     joerg x86_cpu_idle_mwait(void)
   1063  1.37     joerg {
   1064  1.37     joerg 	struct cpu_info *ci = curcpu();
   1065  1.37     joerg 
   1066  1.37     joerg 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1067  1.37     joerg 
   1068  1.37     joerg 	x86_monitor(&ci->ci_want_resched, 0, 0);
   1069  1.37     joerg 	if (__predict_false(ci->ci_want_resched)) {
   1070  1.37     joerg 		return;
   1071  1.37     joerg 	}
   1072  1.37     joerg 	x86_mwait(0, 0);
   1073  1.37     joerg }
   1074  1.37     joerg 
   1075  1.37     joerg void
   1076  1.37     joerg x86_cpu_idle_halt(void)
   1077  1.37     joerg {
   1078  1.37     joerg 	struct cpu_info *ci = curcpu();
   1079  1.37     joerg 
   1080  1.37     joerg 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1081  1.37     joerg 
   1082  1.37     joerg 	x86_disable_intr();
   1083  1.37     joerg 	if (!__predict_false(ci->ci_want_resched)) {
   1084  1.37     joerg 		x86_stihlt();
   1085  1.37     joerg 	} else {
   1086  1.37     joerg 		x86_enable_intr();
   1087  1.37     joerg 	}
   1088  1.37     joerg }
   1089