Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.23
      1  1.23      ad /*	$NetBSD: cpu.c,v 1.23 2008/05/11 15:59:51 ad Exp $	*/
      2   1.2  bouyer /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp  */
      3   1.2  bouyer 
      4   1.2  bouyer /*-
      5   1.2  bouyer  * Copyright (c) 2000 The NetBSD Foundation, Inc.
      6  1.19   joerg  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
      7   1.2  bouyer  * All rights reserved.
      8   1.2  bouyer  *
      9   1.2  bouyer  * This code is derived from software contributed to The NetBSD Foundation
     10   1.2  bouyer  * by RedBack Networks Inc.
     11   1.2  bouyer  *
     12   1.2  bouyer  * Author: Bill Sommerfeld
     13   1.2  bouyer  *
     14   1.2  bouyer  * Redistribution and use in source and binary forms, with or without
     15   1.2  bouyer  * modification, are permitted provided that the following conditions
     16   1.2  bouyer  * are met:
     17   1.2  bouyer  * 1. Redistributions of source code must retain the above copyright
     18   1.2  bouyer  *    notice, this list of conditions and the following disclaimer.
     19   1.2  bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     20   1.2  bouyer  *    notice, this list of conditions and the following disclaimer in the
     21   1.2  bouyer  *    documentation and/or other materials provided with the distribution.
     22   1.2  bouyer  *
     23   1.2  bouyer  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     24   1.2  bouyer  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     25   1.2  bouyer  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     26   1.2  bouyer  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     27   1.2  bouyer  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     28   1.2  bouyer  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     29   1.2  bouyer  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     30   1.2  bouyer  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     31   1.2  bouyer  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     32   1.2  bouyer  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     33   1.2  bouyer  * POSSIBILITY OF SUCH DAMAGE.
     34   1.2  bouyer  */
     35   1.2  bouyer 
     36   1.2  bouyer /*
     37   1.2  bouyer  * Copyright (c) 1999 Stefan Grefen
     38   1.2  bouyer  *
     39   1.2  bouyer  * Redistribution and use in source and binary forms, with or without
     40   1.2  bouyer  * modification, are permitted provided that the following conditions
     41   1.2  bouyer  * are met:
     42   1.2  bouyer  * 1. Redistributions of source code must retain the above copyright
     43   1.2  bouyer  *    notice, this list of conditions and the following disclaimer.
     44   1.2  bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     45   1.2  bouyer  *    notice, this list of conditions and the following disclaimer in the
     46   1.2  bouyer  *    documentation and/or other materials provided with the distribution.
     47   1.2  bouyer  * 3. All advertising materials mentioning features or use of this software
     48   1.2  bouyer  *    must display the following acknowledgement:
     49   1.2  bouyer  *      This product includes software developed by the NetBSD
     50   1.2  bouyer  *      Foundation, Inc. and its contributors.
     51   1.2  bouyer  * 4. Neither the name of The NetBSD Foundation nor the names of its
     52   1.2  bouyer  *    contributors may be used to endorse or promote products derived
     53   1.2  bouyer  *    from this software without specific prior written permission.
     54   1.2  bouyer  *
     55   1.2  bouyer  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     56   1.2  bouyer  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     57   1.2  bouyer  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     58   1.2  bouyer  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     59   1.2  bouyer  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     60   1.2  bouyer  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     61   1.2  bouyer  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     62   1.2  bouyer  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     63   1.2  bouyer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     64   1.2  bouyer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     65   1.2  bouyer  * SUCH DAMAGE.
     66   1.2  bouyer  */
     67   1.2  bouyer 
     68   1.2  bouyer #include <sys/cdefs.h>
     69  1.23      ad __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.23 2008/05/11 15:59:51 ad Exp $");
     70   1.2  bouyer 
     71   1.2  bouyer #include "opt_ddb.h"
     72   1.2  bouyer #include "opt_multiprocessor.h"
     73   1.2  bouyer #include "opt_mpbios.h"		/* for MPDEBUG */
     74   1.2  bouyer #include "opt_mtrr.h"
     75   1.2  bouyer #include "opt_xen.h"
     76   1.2  bouyer 
     77   1.2  bouyer #include "lapic.h"
     78   1.2  bouyer #include "ioapic.h"
     79   1.2  bouyer 
     80   1.2  bouyer #include <sys/param.h>
     81   1.2  bouyer #include <sys/proc.h>
     82   1.2  bouyer #include <sys/user.h>
     83   1.2  bouyer #include <sys/systm.h>
     84   1.2  bouyer #include <sys/device.h>
     85   1.2  bouyer #include <sys/malloc.h>
     86  1.11  cegger #include <sys/cpu.h>
     87  1.11  cegger #include <sys/atomic.h>
     88   1.2  bouyer 
     89   1.2  bouyer #include <uvm/uvm_extern.h>
     90   1.2  bouyer 
     91   1.2  bouyer #include <machine/cpufunc.h>
     92   1.2  bouyer #include <machine/cpuvar.h>
     93   1.2  bouyer #include <machine/pmap.h>
     94   1.2  bouyer #include <machine/vmparam.h>
     95   1.2  bouyer #include <machine/mpbiosvar.h>
     96   1.2  bouyer #include <machine/pcb.h>
     97   1.2  bouyer #include <machine/specialreg.h>
     98   1.2  bouyer #include <machine/segments.h>
     99   1.2  bouyer #include <machine/gdt.h>
    100   1.2  bouyer #include <machine/mtrr.h>
    101   1.2  bouyer #include <machine/pio.h>
    102   1.2  bouyer 
    103   1.2  bouyer #ifdef XEN3
    104   1.2  bouyer #include <xen/vcpuvar.h>
    105   1.2  bouyer #endif
    106   1.2  bouyer 
    107   1.2  bouyer #if NLAPIC > 0
    108   1.2  bouyer #include <machine/apicvar.h>
    109   1.2  bouyer #include <machine/i82489reg.h>
    110   1.2  bouyer #include <machine/i82489var.h>
    111   1.2  bouyer #endif
    112   1.2  bouyer 
    113   1.2  bouyer #include <dev/ic/mc146818reg.h>
    114   1.2  bouyer #include <dev/isa/isareg.h>
    115   1.2  bouyer 
    116  1.10  cegger int     cpu_match(device_t, cfdata_t, void *);
    117  1.10  cegger void    cpu_attach(device_t, device_t, void *);
    118   1.2  bouyer #ifdef XEN3
    119  1.10  cegger int     vcpu_match(device_t, cfdata_t, void *);
    120  1.10  cegger void    vcpu_attach(device_t, device_t, void *);
    121   1.2  bouyer #endif
    122  1.10  cegger void    cpu_attach_common(device_t, device_t, void *);
    123   1.8  dogcow void	cpu_offline_md(void);
    124   1.2  bouyer 
    125   1.2  bouyer struct cpu_softc {
    126  1.10  cegger 	device_t sc_dev;		/* device tree glue */
    127   1.2  bouyer 	struct cpu_info *sc_info;	/* pointer to CPU info */
    128   1.2  bouyer };
    129   1.2  bouyer 
    130   1.5   joerg int mp_cpu_start(struct cpu_info *, paddr_t);
    131   1.2  bouyer void mp_cpu_start_cleanup(struct cpu_info *);
    132   1.2  bouyer const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    133   1.2  bouyer 				      mp_cpu_start_cleanup };
    134   1.2  bouyer 
    135  1.10  cegger CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc),
    136   1.2  bouyer     cpu_match, cpu_attach, NULL, NULL);
    137   1.2  bouyer #ifdef XEN3
    138  1.10  cegger CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
    139   1.2  bouyer     vcpu_match, vcpu_attach, NULL, NULL);
    140   1.2  bouyer #endif
    141   1.2  bouyer 
    142   1.2  bouyer /*
    143   1.2  bouyer  * Statically-allocated CPU info for the primary CPU (or the only
    144   1.2  bouyer  * CPU, on uniprocessors).  The CPU info list is initialized to
    145   1.2  bouyer  * point at it.
    146   1.2  bouyer  */
    147   1.2  bouyer #ifdef TRAPLOG
    148   1.2  bouyer #include <machine/tlog.h>
    149   1.2  bouyer struct tlog tlog_primary;
    150   1.2  bouyer #endif
    151   1.2  bouyer struct cpu_info cpu_info_primary = {
    152   1.7  bouyer 	.ci_dev = 0,
    153   1.2  bouyer 	.ci_self = &cpu_info_primary,
    154   1.4  bouyer 	.ci_idepth = -1,
    155   1.2  bouyer 	.ci_curlwp = &lwp0,
    156   1.2  bouyer #ifdef TRAPLOG
    157   1.2  bouyer 	.ci_tlog = &tlog_primary,
    158   1.2  bouyer #endif
    159   1.2  bouyer 
    160   1.2  bouyer };
    161   1.2  bouyer struct cpu_info phycpu_info_primary = {
    162   1.7  bouyer 	.ci_dev = 0,
    163   1.2  bouyer 	.ci_self = &phycpu_info_primary,
    164   1.2  bouyer };
    165   1.2  bouyer 
    166   1.2  bouyer struct cpu_info *cpu_info_list = &cpu_info_primary;
    167   1.2  bouyer 
    168   1.2  bouyer static void	cpu_set_tss_gates(struct cpu_info *ci);
    169   1.2  bouyer 
    170  1.11  cegger uint32_t cpus_attached = 0;
    171  1.11  cegger uint32_t cpus_running = 0;
    172  1.11  cegger 
    173  1.11  cegger bool x86_mp_online;
    174  1.11  cegger paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    175   1.2  bouyer 
    176   1.2  bouyer struct cpu_info *phycpu_info[X86_MAXPROCS] = { &cpu_info_primary };
    177   1.2  bouyer 
    178   1.2  bouyer #ifdef MULTIPROCESSOR
    179   1.2  bouyer /*
    180   1.2  bouyer  * Array of CPU info structures.  Must be statically-allocated because
    181   1.2  bouyer  * curproc, etc. are used early.
    182   1.2  bouyer  */
    183   1.2  bouyer struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary };
    184   1.2  bouyer 
    185   1.2  bouyer void    	cpu_hatch(void *);
    186   1.2  bouyer static void    	cpu_boot_secondary(struct cpu_info *ci);
    187   1.2  bouyer static void    	cpu_start_secondary(struct cpu_info *ci);
    188   1.2  bouyer static void	cpu_copy_trampoline(void);
    189   1.2  bouyer 
    190   1.2  bouyer /*
    191   1.2  bouyer  * Runs once per boot once multiprocessor goo has been detected and
    192   1.2  bouyer  * the local APIC on the boot processor has been mapped.
    193   1.2  bouyer  *
    194   1.2  bouyer  * Called from lapic_boot_init() (from mpbios_scan()).
    195   1.2  bouyer  */
    196   1.2  bouyer void
    197  1.10  cegger cpu_init_first(void)
    198   1.2  bouyer {
    199   1.2  bouyer 	int cpunum = lapic_cpu_number();
    200   1.2  bouyer 
    201   1.2  bouyer 	if (cpunum != 0) {
    202   1.2  bouyer 		cpu_info[0] = NULL;
    203   1.2  bouyer 		cpu_info[cpunum] = &cpu_info_primary;
    204   1.2  bouyer 	}
    205   1.2  bouyer 
    206   1.2  bouyer 	cpu_copy_trampoline();
    207   1.2  bouyer }
    208   1.2  bouyer #endif
    209   1.2  bouyer 
    210   1.2  bouyer int
    211  1.10  cegger cpu_match(device_t parent, cfdata_t match, void *aux)
    212   1.2  bouyer {
    213   1.2  bouyer 
    214   1.2  bouyer 	return 1;
    215   1.2  bouyer }
    216   1.2  bouyer 
    217   1.2  bouyer void
    218  1.10  cegger cpu_attach(device_t parent, device_t self, void *aux)
    219   1.2  bouyer {
    220   1.2  bouyer #ifdef XEN3
    221  1.10  cegger 	struct cpu_softc *sc = device_private(self);
    222   1.2  bouyer 	struct cpu_attach_args *caa = aux;
    223   1.2  bouyer 	struct cpu_info *ci;
    224   1.2  bouyer 	int cpunum = caa->cpu_number;
    225   1.2  bouyer 
    226  1.10  cegger 	sc->sc_dev = self;
    227  1.10  cegger 
    228   1.2  bouyer 	/*
    229   1.2  bouyer 	 * If we're an Application Processor, allocate a cpu_info
    230   1.2  bouyer 	 * structure, otherwise use the primary's.
    231   1.2  bouyer 	 */
    232   1.2  bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    233   1.2  bouyer 		ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK | M_ZERO);
    234   1.2  bouyer 		if (phycpu_info[cpunum] != NULL)
    235   1.2  bouyer 			panic("cpu at apic id %d already attached?", cpunum);
    236   1.2  bouyer 		phycpu_info[cpunum] = ci;
    237   1.2  bouyer 	} else {
    238   1.2  bouyer 		ci = &phycpu_info_primary;
    239   1.2  bouyer 		if (cpunum != 0) {
    240   1.2  bouyer 			phycpu_info[0] = NULL;
    241   1.2  bouyer 			phycpu_info[cpunum] = ci;
    242   1.2  bouyer 		}
    243   1.2  bouyer 	}
    244   1.2  bouyer 
    245   1.2  bouyer 	ci->ci_self = ci;
    246   1.2  bouyer 	sc->sc_info = ci;
    247   1.2  bouyer 
    248   1.2  bouyer 	ci->ci_dev = self;
    249  1.23      ad 	ci->ci_cpuid = caa->cpu_number;
    250  1.16  cegger 	ci->ci_vcpu = NULL;
    251   1.2  bouyer 
    252   1.2  bouyer 	printf(": ");
    253   1.2  bouyer 	switch (caa->cpu_role) {
    254   1.2  bouyer 	case CPU_ROLE_SP:
    255   1.2  bouyer 		printf("(uniprocessor)\n");
    256   1.2  bouyer 		ci->ci_flags |= CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY;
    257   1.2  bouyer 		break;
    258   1.2  bouyer 
    259   1.2  bouyer 	case CPU_ROLE_BP:
    260   1.2  bouyer 		printf("(boot processor)\n");
    261   1.2  bouyer 		ci->ci_flags |= CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY;
    262   1.2  bouyer 		break;
    263   1.2  bouyer 
    264   1.2  bouyer 	case CPU_ROLE_AP:
    265   1.2  bouyer 		/*
    266   1.2  bouyer 		 * report on an AP
    267   1.2  bouyer 		 */
    268   1.2  bouyer 		printf("(application processor)\n");
    269   1.2  bouyer 		break;
    270   1.2  bouyer 
    271   1.2  bouyer 	default:
    272   1.2  bouyer 		panic("unknown processor type??\n");
    273   1.2  bouyer 	}
    274   1.2  bouyer 	return;
    275   1.2  bouyer #else
    276   1.2  bouyer 	cpu_attach_common(parent, self, aux);
    277   1.2  bouyer #endif
    278   1.2  bouyer }
    279   1.2  bouyer 
    280   1.2  bouyer #ifdef XEN3
    281   1.2  bouyer int
    282  1.10  cegger vcpu_match(device_t parent, cfdata_t match, void *aux)
    283   1.2  bouyer {
    284   1.2  bouyer 	struct vcpu_attach_args *vcaa = aux;
    285   1.2  bouyer 
    286   1.2  bouyer 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0)
    287   1.2  bouyer 		return 1;
    288   1.2  bouyer 	return 0;
    289   1.2  bouyer }
    290   1.2  bouyer 
    291   1.2  bouyer void
    292  1.10  cegger vcpu_attach(device_t parent, device_t self, void *aux)
    293   1.2  bouyer {
    294   1.2  bouyer 	struct vcpu_attach_args *vcaa = aux;
    295   1.2  bouyer 
    296   1.2  bouyer 	cpu_attach_common(parent, self, &vcaa->vcaa_caa);
    297   1.2  bouyer }
    298   1.2  bouyer #endif
    299   1.2  bouyer 
    300   1.2  bouyer static void
    301   1.2  bouyer cpu_vm_init(struct cpu_info *ci)
    302   1.2  bouyer {
    303   1.2  bouyer 	int ncolors = 2, i;
    304   1.2  bouyer 
    305   1.2  bouyer 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    306   1.2  bouyer 		struct x86_cache_info *cai;
    307   1.2  bouyer 		int tcolors;
    308   1.2  bouyer 
    309   1.2  bouyer 		cai = &ci->ci_cinfo[i];
    310   1.2  bouyer 
    311   1.2  bouyer 		tcolors = atop(cai->cai_totalsize);
    312   1.2  bouyer 		switch(cai->cai_associativity) {
    313   1.2  bouyer 		case 0xff:
    314   1.2  bouyer 			tcolors = 1; /* fully associative */
    315   1.2  bouyer 			break;
    316   1.2  bouyer 		case 0:
    317   1.2  bouyer 		case 1:
    318   1.2  bouyer 			break;
    319   1.2  bouyer 		default:
    320   1.2  bouyer 			tcolors /= cai->cai_associativity;
    321   1.2  bouyer 		}
    322   1.2  bouyer 		ncolors = max(ncolors, tcolors);
    323   1.2  bouyer 	}
    324   1.2  bouyer 
    325   1.2  bouyer 	/*
    326   1.2  bouyer 	 * Knowing the size of the largest cache on this CPU, re-color
    327   1.2  bouyer 	 * our pages.
    328   1.2  bouyer 	 */
    329   1.2  bouyer 	if (ncolors <= uvmexp.ncolors)
    330   1.2  bouyer 		return;
    331   1.9  cegger 	printf("%s: %d page colors\n", device_xname(ci->ci_dev), ncolors);
    332   1.2  bouyer 	uvm_page_recolor(ncolors);
    333   1.2  bouyer }
    334   1.2  bouyer 
    335   1.2  bouyer void
    336  1.11  cegger cpu_attach_common(device_t parent, device_t self, void *aux)
    337   1.2  bouyer {
    338  1.10  cegger 	struct cpu_softc *sc = device_private(self);
    339   1.2  bouyer 	struct cpu_attach_args *caa = aux;
    340   1.2  bouyer 	struct cpu_info *ci;
    341  1.12  cegger 	uintptr_t ptr;
    342   1.2  bouyer 	int cpunum = caa->cpu_number;
    343   1.2  bouyer 
    344  1.10  cegger 	sc->sc_dev = self;
    345  1.10  cegger 
    346   1.2  bouyer 	/*
    347   1.2  bouyer 	 * If we're an Application Processor, allocate a cpu_info
    348   1.2  bouyer 	 * structure, otherwise use the primary's.
    349   1.2  bouyer 	 */
    350   1.2  bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    351  1.17  cegger 		if (cpunum >= X86_MAXPROCS) {
    352  1.17  cegger 			aprint_error(": apic id %d ignored, "
    353  1.17  cegger 				"please increase X86_MAXPROCS\n", cpunum);
    354  1.17  cegger 		}
    355  1.17  cegger 
    356  1.12  cegger 		aprint_naive(": Application Processor\n");
    357  1.12  cegger 		ptr = (uintptr_t)malloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    358  1.12  cegger 		    M_DEVBUF, M_WAITOK);
    359  1.12  cegger 		ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) &
    360  1.12  cegger 		    ~(CACHE_LINE_SIZE - 1));
    361  1.12  cegger 		memset(ci, 0, sizeof(*ci));
    362   1.2  bouyer #if defined(MULTIPROCESSOR)
    363   1.2  bouyer 		if (cpu_info[cpunum] != NULL)
    364   1.2  bouyer 			panic("cpu at apic id %d already attached?", cpunum);
    365   1.2  bouyer 		cpu_info[cpunum] = ci;
    366   1.2  bouyer #endif
    367   1.2  bouyer #ifdef TRAPLOG
    368   1.2  bouyer 		ci->ci_tlog_base = malloc(sizeof(struct tlog),
    369   1.2  bouyer 		    M_DEVBUF, M_WAITOK);
    370   1.2  bouyer #endif
    371   1.2  bouyer 	} else {
    372  1.12  cegger 		aprint_naive(": %s Processor\n",
    373  1.12  cegger 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    374   1.2  bouyer 		ci = &cpu_info_primary;
    375   1.2  bouyer #if defined(MULTIPROCESSOR)
    376   1.2  bouyer 		if (cpunum != lapic_cpu_number()) {
    377   1.2  bouyer 			panic("%s: running CPU is at apic %d"
    378   1.2  bouyer 			    " instead of at expected %d",
    379   1.9  cegger 			    device_xname(sc->sc_dev), lapic_cpu_number(), cpunum);
    380   1.2  bouyer 		}
    381   1.2  bouyer #endif
    382   1.2  bouyer 	}
    383   1.2  bouyer 
    384   1.2  bouyer 	ci->ci_self = ci;
    385   1.2  bouyer 	sc->sc_info = ci;
    386   1.2  bouyer 
    387   1.2  bouyer 	ci->ci_dev = self;
    388  1.23      ad 	ci->ci_cpuid = cpunum;
    389  1.16  cegger 
    390  1.16  cegger 	KASSERT(HYPERVISOR_shared_info != NULL);
    391  1.16  cegger 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
    392  1.16  cegger 
    393   1.2  bouyer 	ci->ci_func = caa->cpu_func;
    394   1.2  bouyer 
    395   1.2  bouyer 	if (caa->cpu_role == CPU_ROLE_AP) {
    396   1.2  bouyer #if defined(MULTIPROCESSOR)
    397   1.2  bouyer 		int error;
    398   1.2  bouyer 
    399   1.2  bouyer 		error = mi_cpu_attach(ci);
    400   1.2  bouyer 		if (error != 0) {
    401   1.2  bouyer 			aprint_normal("\n");
    402  1.10  cegger 			aprint_error_dev(sc->sc_dev, "mi_cpu_attach failed with %d\n",
    403   1.9  cegger 			    error);
    404   1.2  bouyer 			return;
    405   1.2  bouyer 		}
    406   1.2  bouyer #endif
    407   1.2  bouyer 	} else {
    408   1.2  bouyer 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    409   1.2  bouyer 	}
    410   1.2  bouyer 
    411  1.23      ad 	ci->ci_cpumask = (1 << cpu_index(ci));
    412   1.2  bouyer 	pmap_reference(pmap_kernel());
    413   1.2  bouyer 	ci->ci_pmap = pmap_kernel();
    414   1.2  bouyer 	ci->ci_tlbstate = TLBSTATE_STALE;
    415   1.2  bouyer 
    416   1.2  bouyer 	/* further PCB init done later. */
    417   1.2  bouyer 
    418   1.2  bouyer 	switch (caa->cpu_role) {
    419   1.2  bouyer 	case CPU_ROLE_SP:
    420  1.12  cegger 		aprint_normal(": (uniprocessor)\n");
    421  1.12  cegger 		atomic_or_32(&ci->ci_flags,
    422  1.12  cegger 		     CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY);
    423   1.2  bouyer 		cpu_intr_init(ci);
    424  1.21      ad 		cpu_get_tsc_freq(ci);
    425  1.21      ad 		cpu_identify(ci);
    426   1.2  bouyer 		cpu_init(ci);
    427   1.2  bouyer 		cpu_set_tss_gates(ci);
    428  1.12  cegger 		pmap_cpu_init_late(ci);
    429  1.12  cegger #if 0
    430  1.12  cegger 		x86_errata();
    431  1.12  cegger #endif
    432   1.2  bouyer 		break;
    433   1.2  bouyer 
    434   1.2  bouyer 	case CPU_ROLE_BP:
    435  1.12  cegger 		aprint_normal("apid %d (boot processor)\n", caa->cpu_number);
    436  1.12  cegger 		atomic_or_32(&ci->ci_flags,
    437  1.12  cegger 		    CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY);
    438   1.2  bouyer 		cpu_intr_init(ci);
    439  1.21      ad 		cpu_get_tsc_freq(ci);
    440  1.21      ad 		cpu_identify(ci);
    441   1.2  bouyer 		cpu_init(ci);
    442   1.2  bouyer 		cpu_set_tss_gates(ci);
    443  1.12  cegger 		pmap_cpu_init_late(ci);
    444  1.12  cegger #if NLAPIC > 0
    445  1.12  cegger 		/*
    446  1.12  cegger 		 * Enable local apic
    447  1.12  cegger 		 */
    448  1.12  cegger 		lapic_enable();
    449  1.12  cegger 		lapic_set_lvt();
    450  1.12  cegger 		lapic_calibrate_timer(ci);
    451  1.12  cegger #endif
    452  1.14  bouyer #if 0
    453  1.12  cegger 		x86_errata();
    454  1.12  cegger #endif
    455   1.2  bouyer 		break;
    456   1.2  bouyer 
    457   1.2  bouyer 	case CPU_ROLE_AP:
    458   1.2  bouyer 		/*
    459   1.2  bouyer 		 * report on an AP
    460   1.2  bouyer 		 */
    461  1.12  cegger 		aprint_normal("apid %d (application processor)\n", caa->cpu_number);
    462   1.2  bouyer 
    463   1.2  bouyer #if defined(MULTIPROCESSOR)
    464   1.2  bouyer 		cpu_intr_init(ci);
    465   1.2  bouyer 		gdt_alloc_cpu(ci);
    466   1.2  bouyer 		cpu_set_tss_gates(ci);
    467  1.12  cegger 		pmap_cpu_init_early(ci);
    468  1.12  cegger 		pmap_cpu_init_late(ci);
    469   1.2  bouyer 		cpu_start_secondary(ci);
    470   1.2  bouyer 		if (ci->ci_flags & CPUF_PRESENT) {
    471   1.2  bouyer 			identifycpu(ci);
    472   1.2  bouyer 			ci->ci_next = cpu_info_list->ci_next;
    473   1.2  bouyer 			cpu_info_list->ci_next = ci;
    474   1.2  bouyer 		}
    475   1.2  bouyer #else
    476  1.12  cegger 		aprint_normal_dev(sc->sc_dev, "not started\n");
    477   1.2  bouyer #endif
    478   1.2  bouyer 		break;
    479   1.2  bouyer 
    480   1.2  bouyer 	default:
    481  1.12  cegger 		aprint_normal("\n");
    482   1.2  bouyer 		panic("unknown processor type??\n");
    483   1.2  bouyer 	}
    484   1.2  bouyer 	cpu_vm_init(ci);
    485   1.2  bouyer 
    486   1.2  bouyer 	cpus_attached |= (1 << ci->ci_cpuid);
    487   1.2  bouyer 
    488  1.12  cegger #if 0
    489  1.12  cegger 	if (!pmf_device_register(self, cpu_suspend, cpu_resume))
    490  1.12  cegger 		aprint_error_dev(self, "couldn't establish power handler\n");
    491  1.12  cegger #endif
    492  1.12  cegger 
    493   1.2  bouyer #if defined(MULTIPROCESSOR)
    494   1.2  bouyer 	if (mp_verbose) {
    495   1.2  bouyer 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    496   1.2  bouyer 
    497  1.12  cegger 		aprint_verbose_dev(sc->sc_dev, "idle lwp at %p, idle sp at 0x%p\n",
    498  1.12  cegger 		    l,
    499  1.12  cegger #ifdef i386
    500  1.12  cegger 		    (void *)l->l_addr->u_pcb.pcb_esp
    501  1.12  cegger #else
    502  1.12  cegger 		    (void *)l->l_addr->u_pcb.pcb_rsp
    503  1.12  cegger #endif
    504  1.12  cegger 		);
    505  1.12  cegger 
    506   1.2  bouyer 	}
    507   1.2  bouyer #endif
    508   1.2  bouyer }
    509   1.2  bouyer 
    510   1.2  bouyer /*
    511   1.2  bouyer  * Initialize the processor appropriately.
    512   1.2  bouyer  */
    513   1.2  bouyer 
    514   1.2  bouyer void
    515  1.10  cegger cpu_init(struct cpu_info *ci)
    516   1.2  bouyer {
    517   1.2  bouyer 
    518   1.2  bouyer 	/*
    519   1.2  bouyer 	 * On a P6 or above, enable global TLB caching if the
    520   1.2  bouyer 	 * hardware supports it.
    521   1.2  bouyer 	 */
    522   1.2  bouyer 	if (cpu_feature & CPUID_PGE)
    523   1.2  bouyer 		lcr4(rcr4() | CR4_PGE);	/* enable global TLB caching */
    524   1.2  bouyer 
    525   1.2  bouyer #ifdef XXXMTRR
    526   1.2  bouyer 	/*
    527   1.2  bouyer 	 * On a P6 or above, initialize MTRR's if the hardware supports them.
    528   1.2  bouyer 	 */
    529   1.2  bouyer 	if (cpu_feature & CPUID_MTRR) {
    530   1.2  bouyer 		if ((ci->ci_flags & CPUF_AP) == 0)
    531   1.2  bouyer 			i686_mtrr_init_first();
    532   1.2  bouyer 		mtrr_init_cpu(ci);
    533   1.2  bouyer 	}
    534   1.2  bouyer #endif
    535   1.2  bouyer 	/*
    536   1.2  bouyer 	 * If we have FXSAVE/FXRESTOR, use them.
    537   1.2  bouyer 	 */
    538   1.2  bouyer 	if (cpu_feature & CPUID_FXSR) {
    539   1.2  bouyer 		lcr4(rcr4() | CR4_OSFXSR);
    540   1.2  bouyer 
    541   1.2  bouyer 		/*
    542   1.2  bouyer 		 * If we have SSE/SSE2, enable XMM exceptions.
    543   1.2  bouyer 		 */
    544   1.2  bouyer 		if (cpu_feature & (CPUID_SSE|CPUID_SSE2))
    545   1.2  bouyer 			lcr4(rcr4() | CR4_OSXMMEXCPT);
    546   1.2  bouyer 	}
    547   1.2  bouyer 
    548   1.2  bouyer #ifdef MULTIPROCESSOR
    549  1.11  cegger 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    550  1.11  cegger 	atomic_or_32(&cpus_running, ci->ci_cpumask);
    551   1.2  bouyer #endif
    552   1.2  bouyer }
    553   1.2  bouyer 
    554   1.2  bouyer 
    555   1.2  bouyer #ifdef MULTIPROCESSOR
    556   1.2  bouyer void
    557  1.10  cegger cpu_boot_secondary_processors(void)
    558   1.2  bouyer {
    559   1.2  bouyer 	struct cpu_info *ci;
    560   1.2  bouyer 	u_long i;
    561   1.2  bouyer 
    562  1.11  cegger 	for (i = 0; i < X86_MAXPROCS; i++) {
    563   1.2  bouyer 		ci = cpu_info[i];
    564   1.2  bouyer 		if (ci == NULL)
    565   1.2  bouyer 			continue;
    566   1.2  bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    567   1.2  bouyer 			continue;
    568   1.2  bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    569   1.2  bouyer 			continue;
    570   1.2  bouyer 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    571   1.2  bouyer 			continue;
    572   1.2  bouyer 		cpu_boot_secondary(ci);
    573   1.2  bouyer 	}
    574  1.11  cegger 
    575  1.11  cegger 	x86_mp_online = true;
    576   1.2  bouyer }
    577   1.2  bouyer 
    578   1.2  bouyer static void
    579   1.2  bouyer cpu_init_idle_lwp(struct cpu_info *ci)
    580   1.2  bouyer {
    581   1.2  bouyer 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    582   1.2  bouyer 	struct pcb *pcb = &l->l_addr->u_pcb;
    583   1.2  bouyer 
    584   1.2  bouyer 	pcb->pcb_cr0 = rcr0();
    585   1.2  bouyer }
    586   1.2  bouyer 
    587   1.2  bouyer void
    588  1.10  cegger cpu_init_idle_lwps(void)
    589   1.2  bouyer {
    590   1.2  bouyer 	struct cpu_info *ci;
    591   1.2  bouyer 	u_long i;
    592   1.2  bouyer 
    593   1.2  bouyer 	for (i = 0; i < X86_MAXPROCS; i++) {
    594   1.2  bouyer 		ci = cpu_info[i];
    595   1.2  bouyer 		if (ci == NULL)
    596   1.2  bouyer 			continue;
    597   1.2  bouyer 		if (ci->ci_data.cpu_idlelwp == NULL)
    598   1.2  bouyer 			continue;
    599   1.2  bouyer 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    600   1.2  bouyer 			continue;
    601   1.2  bouyer 		cpu_init_idle_lwp(ci);
    602   1.2  bouyer 	}
    603   1.2  bouyer }
    604   1.2  bouyer 
    605   1.2  bouyer void
    606  1.10  cegger cpu_start_secondary(struct cpu_info *ci)
    607   1.2  bouyer {
    608   1.2  bouyer 	int i;
    609   1.2  bouyer 	struct pmap *kpm = pmap_kernel();
    610  1.11  cegger 	extern uint32_t mp_pdirpa;
    611   1.2  bouyer 
    612   1.2  bouyer 	mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */
    613   1.2  bouyer 
    614  1.11  cegger 	atomic_or_32(&ci->ci_flags, CPUF_AP);
    615   1.2  bouyer 
    616  1.11  cegger 	aprint_debug_dev(ci->ci_dev, "starting\n");
    617   1.2  bouyer 
    618   1.2  bouyer 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    619  1.11  cegger 	if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0)
    620  1.11  cegger 		return;
    621   1.2  bouyer 
    622   1.2  bouyer 	/*
    623   1.2  bouyer 	 * wait for it to become ready
    624   1.2  bouyer 	 */
    625  1.11  cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    626  1.11  cegger #ifdef MPDEBUG
    627  1.11  cegger 		extern int cpu_trace[3];
    628  1.11  cegger 		static int otrace[3];
    629  1.11  cegger 		if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
    630  1.11  cegger 			aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
    631  1.11  cegger 				cpu_trace[0], cpu_trace[1], cpu_trace[2]);
    632  1.11  cegger 			memcpy(otrace, cpu_trace, sizeof(otrace));
    633  1.11  cegger 		}
    634  1.11  cegger #endif
    635   1.2  bouyer 		delay(10);
    636   1.2  bouyer 	}
    637  1.11  cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    638   1.9  cegger 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    639   1.2  bouyer #if defined(MPDEBUG) && defined(DDB)
    640   1.2  bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    641   1.2  bouyer 		Debugger();
    642   1.2  bouyer #endif
    643   1.2  bouyer 	}
    644   1.2  bouyer 
    645   1.2  bouyer 	CPU_START_CLEANUP(ci);
    646   1.2  bouyer }
    647   1.2  bouyer 
    648   1.2  bouyer void
    649  1.10  cegger cpu_boot_secondary(struct cpu_info *ci)
    650   1.2  bouyer {
    651   1.2  bouyer 	int i;
    652   1.2  bouyer 
    653  1.11  cegger 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    654  1.11  cegger 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    655   1.2  bouyer 		delay(10);
    656   1.2  bouyer 	}
    657  1.11  cegger 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    658  1.11  cegger 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
    659   1.2  bouyer #if defined(MPDEBUG) && defined(DDB)
    660   1.2  bouyer 		printf("dropping into debugger; continue from here to resume boot\n");
    661   1.2  bouyer 		Debugger();
    662   1.2  bouyer #endif
    663   1.2  bouyer 	}
    664   1.2  bouyer }
    665   1.2  bouyer 
    666   1.2  bouyer /*
    667   1.2  bouyer  * The CPU ends up here when its ready to run
    668   1.2  bouyer  * This is called from code in mptramp.s; at this point, we are running
    669   1.2  bouyer  * in the idle pcb/idle stack of the new CPU.  When this function returns,
    670   1.2  bouyer  * this processor will enter the idle loop and start looking for work.
    671   1.2  bouyer  *
    672   1.2  bouyer  * XXX should share some of this with init386 in machdep.c
    673   1.2  bouyer  */
    674   1.2  bouyer void
    675   1.2  bouyer cpu_hatch(void *v)
    676   1.2  bouyer {
    677   1.2  bouyer 	struct cpu_info *ci = (struct cpu_info *)v;
    678  1.11  cegger 	int s, i;
    679  1.11  cegger 	uint32_t blacklist_features;
    680  1.11  cegger 
    681   1.2  bouyer #ifdef __x86_64__
    682  1.11  cegger         cpu_init_msrs(ci, true);
    683   1.2  bouyer #endif
    684   1.2  bouyer 
    685  1.21      ad 	cpu_probe(ci);
    686  1.11  cegger 
    687   1.2  bouyer 	/* not on Xen... */
    688  1.11  cegger 	blacklist_features = ~(CPUID_PGE|CPUID_PSE|CPUID_MTRR|CPUID_FXSR|CPUID_NOX); /* XXX add CPUID_SVM */
    689   1.2  bouyer 
    690  1.11  cegger 	cpu_feature &= blacklist_features;
    691   1.2  bouyer 
    692  1.11  cegger 	KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
    693  1.11  cegger 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    694  1.11  cegger 	while ((ci->ci_flags & CPUF_GO) == 0) {
    695  1.11  cegger 		/* Don't use delay, boot CPU may be patching the text. */
    696  1.11  cegger 		for (i = 10000; i != 0; i--)
    697  1.11  cegger 			x86_pause();
    698  1.11  cegger 	}
    699   1.2  bouyer 
    700  1.11  cegger 	/* Because the text may have been patched in x86_patch(). */
    701  1.11  cegger 	wbinvd();
    702  1.11  cegger 	x86_flush();
    703   1.2  bouyer 
    704  1.11  cegger 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    705   1.2  bouyer 
    706  1.12  cegger 	lcr3(pmap_kernel()->pm_pdirpa);
    707  1.12  cegger 	curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa;
    708   1.2  bouyer 	lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0);
    709   1.2  bouyer 	cpu_init_idt();
    710  1.11  cegger 	gdt_init_cpu(ci);
    711  1.11  cegger 	lapic_enable();
    712   1.2  bouyer 	lapic_set_lvt();
    713  1.11  cegger 	lapic_initclocks();
    714  1.11  cegger 
    715  1.12  cegger #ifdef i386
    716   1.2  bouyer 	npxinit(ci);
    717  1.12  cegger #else
    718  1.12  cegger 	fpuinit(ci);
    719  1.12  cegger #endif
    720   1.2  bouyer 
    721   1.2  bouyer 	lldt(GSEL(GLDT_SEL, SEL_KPL));
    722  1.12  cegger 	ltr(ci->ci_tss_sel);
    723   1.2  bouyer 
    724   1.2  bouyer 	cpu_init(ci);
    725  1.11  cegger 	cpu_get_tsc_freq(ci);
    726   1.2  bouyer 
    727   1.2  bouyer 	s = splhigh();
    728  1.11  cegger #ifdef i386
    729   1.2  bouyer 	lapic_tpr = 0;
    730  1.11  cegger #else
    731  1.11  cegger 	lcr8(0);
    732  1.11  cegger #endif
    733  1.11  cegger 	x86_enable_intr();
    734  1.11  cegger 	splx(s);
    735  1.12  cegger #if 0
    736  1.11  cegger 	x86_errata();
    737  1.11  cegger #endif
    738   1.2  bouyer 
    739  1.11  cegger 	aprint_debug_dev(ci->ci_dev, "CPU %ld running\n",
    740  1.11  cegger 		(long)ci->ci_cpuid);
    741   1.2  bouyer }
    742   1.2  bouyer 
    743   1.2  bouyer #if defined(DDB)
    744   1.2  bouyer 
    745   1.2  bouyer #include <ddb/db_output.h>
    746   1.2  bouyer #include <machine/db_machdep.h>
    747   1.2  bouyer 
    748   1.2  bouyer /*
    749   1.2  bouyer  * Dump CPU information from ddb.
    750   1.2  bouyer  */
    751   1.2  bouyer void
    752   1.2  bouyer cpu_debug_dump(void)
    753   1.2  bouyer {
    754   1.2  bouyer 	struct cpu_info *ci;
    755   1.2  bouyer 	CPU_INFO_ITERATOR cii;
    756   1.2  bouyer 
    757  1.13    yamt 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    758   1.2  bouyer 	for (CPU_INFO_FOREACH(cii, ci)) {
    759   1.2  bouyer 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    760   1.2  bouyer 		    ci,
    761   1.9  cegger 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    762  1.12  cegger 		    (long)ci->ci_cpuid,
    763   1.2  bouyer 		    ci->ci_flags, ci->ci_ipis,
    764   1.2  bouyer 		    ci->ci_curlwp,
    765   1.2  bouyer 		    ci->ci_fpcurlwp);
    766   1.2  bouyer 	}
    767   1.2  bouyer }
    768   1.2  bouyer #endif
    769   1.2  bouyer 
    770   1.2  bouyer static void
    771  1.10  cegger cpu_copy_trampoline(void)
    772   1.2  bouyer {
    773   1.2  bouyer 	/*
    774   1.2  bouyer 	 * Copy boot code.
    775   1.2  bouyer 	 */
    776   1.2  bouyer 	extern u_char cpu_spinup_trampoline[];
    777   1.2  bouyer 	extern u_char cpu_spinup_trampoline_end[];
    778  1.11  cegger 
    779  1.11  cegger 	vaddr_t mp_trampoline_vaddr;
    780  1.11  cegger 
    781  1.11  cegger 	mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    782  1.11  cegger 		UVM_KMF_VAONLY);
    783  1.11  cegger 
    784  1.11  cegger 	pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
    785  1.11  cegger 		VM_PROT_READ | VM_PROT_WRITE);
    786  1.11  cegger 	pmap_update(pmap_kernel());
    787  1.11  cegger 	memcpy((void *)mp_trampoline_vaddr,
    788  1.11  cegger 		cpu_spinup_trampoline,
    789  1.11  cegger 		cpu_spinup_trampoline_end - cpu_spinup_trampoline);
    790  1.11  cegger 
    791  1.11  cegger 	pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
    792  1.11  cegger 	pmap_update(pmap_kernel());
    793  1.11  cegger 	uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
    794   1.2  bouyer }
    795   1.2  bouyer 
    796   1.2  bouyer #endif
    797   1.2  bouyer 
    798  1.11  cegger #ifdef i386
    799  1.11  cegger #if 0
    800  1.11  cegger static void
    801  1.11  cegger tss_init(struct i386tss *tss, void *stack, void *func)
    802  1.11  cegger {
    803  1.11  cegger 	memset(tss, 0, sizeof *tss);
    804  1.11  cegger 	tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
    805  1.11  cegger 	tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
    806  1.11  cegger 	tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
    807  1.11  cegger 	tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
    808  1.11  cegger 	tss->tss_gs = tss->__tss_es = tss->__tss_ds =
    809  1.11  cegger 	    tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
    810  1.11  cegger 	tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
    811  1.11  cegger 	tss->tss_esp = (int)((char *)stack + USPACE - 16);
    812  1.11  cegger 	tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
    813  1.11  cegger 	tss->__tss_eflags = PSL_MBO | PSL_NT;   /* XXX not needed? */
    814  1.11  cegger 	tss->__tss_eip = (int)func;
    815  1.11  cegger }
    816  1.11  cegger #endif
    817   1.2  bouyer 
    818   1.2  bouyer /* XXX */
    819   1.2  bouyer #define IDTVEC(name)	__CONCAT(X, name)
    820   1.2  bouyer typedef void (vector)(void);
    821   1.2  bouyer extern vector IDTVEC(tss_trap08);
    822   1.2  bouyer #ifdef DDB
    823   1.2  bouyer extern vector Xintrddbipi;
    824   1.2  bouyer extern int ddb_vec;
    825   1.2  bouyer #endif
    826   1.2  bouyer 
    827   1.2  bouyer static void
    828   1.2  bouyer cpu_set_tss_gates(struct cpu_info *ci)
    829   1.2  bouyer {
    830  1.11  cegger #if 0
    831  1.11  cegger 	struct segment_descriptor sd;
    832  1.11  cegger 
    833  1.11  cegger 	ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    834  1.11  cegger 	    UVM_KMF_WIRED);
    835  1.11  cegger 	tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
    836  1.11  cegger 	    IDTVEC(tss_trap08));
    837  1.11  cegger 	setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
    838  1.11  cegger 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    839  1.11  cegger 	ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
    840  1.11  cegger 	setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    841  1.11  cegger 	    GSEL(GTRAPTSS_SEL, SEL_KPL));
    842  1.11  cegger #endif
    843  1.11  cegger 
    844   1.2  bouyer #if defined(DDB) && defined(MULTIPROCESSOR)
    845   1.2  bouyer 	/*
    846   1.2  bouyer 	 * Set up separate handler for the DDB IPI, so that it doesn't
    847   1.2  bouyer 	 * stomp on a possibly corrupted stack.
    848   1.2  bouyer 	 *
    849   1.2  bouyer 	 * XXX overwriting the gate set in db_machine_init.
    850   1.2  bouyer 	 * Should rearrange the code so that it's set only once.
    851   1.2  bouyer 	 */
    852   1.2  bouyer 	ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    853   1.2  bouyer 	    UVM_KMF_WIRED);
    854   1.6    yamt 	tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack,
    855   1.2  bouyer 	    Xintrddbipi);
    856   1.2  bouyer 
    857   1.2  bouyer 	setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
    858   1.2  bouyer 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    859   1.2  bouyer 	ci->ci_gdt[GIPITSS_SEL].sd = sd;
    860   1.2  bouyer 
    861   1.2  bouyer 	setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    862   1.2  bouyer 	    GSEL(GIPITSS_SEL, SEL_KPL));
    863   1.2  bouyer #endif
    864   1.2  bouyer }
    865  1.11  cegger #else
    866  1.11  cegger static void
    867  1.11  cegger cpu_set_tss_gates(struct cpu_info *ci)
    868  1.11  cegger {
    869  1.11  cegger 
    870  1.11  cegger }
    871  1.11  cegger #endif	/* i386 */
    872   1.2  bouyer 
    873   1.2  bouyer int
    874   1.5   joerg mp_cpu_start(struct cpu_info *ci, paddr_t target)
    875   1.2  bouyer {
    876   1.2  bouyer #if 0
    877   1.2  bouyer #if NLAPIC > 0
    878   1.2  bouyer 	int error;
    879   1.2  bouyer #endif
    880   1.2  bouyer 	unsigned short dwordptr[2];
    881   1.2  bouyer 
    882   1.2  bouyer 	/*
    883  1.11  cegger 	 * Bootstrap code must be addressable in real mode
    884  1.11  cegger 	 * and it must be page aligned.
    885  1.11  cegger 	 */
    886  1.11  cegger 	KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
    887  1.11  cegger 
    888  1.11  cegger 	/*
    889   1.2  bouyer 	 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
    890   1.2  bouyer 	 */
    891   1.2  bouyer 
    892   1.2  bouyer 	outb(IO_RTC, NVRAM_RESET);
    893   1.2  bouyer 	outb(IO_RTC+1, NVRAM_RESET_JUMP);
    894   1.2  bouyer 
    895   1.2  bouyer 	/*
    896   1.2  bouyer 	 * "and the warm reset vector (DWORD based at 40:67) to point
    897   1.2  bouyer 	 * to the AP startup code ..."
    898   1.2  bouyer 	 */
    899   1.2  bouyer 
    900   1.2  bouyer 	dwordptr[0] = 0;
    901   1.5   joerg 	dwordptr[1] = target >> 4;
    902   1.2  bouyer 
    903   1.2  bouyer 	pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE);
    904  1.11  cegger 	memcpy ((uint8_t *) 0x467, dwordptr, 4);
    905   1.2  bouyer 	pmap_kremove (0, PAGE_SIZE);
    906   1.2  bouyer 
    907   1.2  bouyer #if NLAPIC > 0
    908   1.2  bouyer 	/*
    909   1.2  bouyer 	 * ... prior to executing the following sequence:"
    910   1.2  bouyer 	 */
    911   1.2  bouyer 
    912   1.2  bouyer 	if (ci->ci_flags & CPUF_AP) {
    913  1.23      ad 		if ((error = x86_ipi_init(ci->ci_cpuid)) != 0)
    914   1.2  bouyer 			return error;
    915   1.2  bouyer 
    916   1.2  bouyer 		delay(10000);
    917   1.2  bouyer 
    918   1.2  bouyer 		if (cpu_feature & CPUID_APIC) {
    919  1.23      ad 			error = x86_ipi_init(ci->ci_cpuid);
    920  1.11  cegger 			if (error != 0) {
    921  1.11  cegger 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
    922  1.11  cegger 						__func__);
    923  1.11  cegger 				return error;
    924  1.11  cegger 			}
    925  1.11  cegger 
    926  1.11  cegger 			delay(10000);
    927   1.2  bouyer 
    928  1.23      ad 			error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid,
    929  1.11  cegger 					LAPIC_DLMODE_STARTUP);
    930  1.11  cegger 			if (error != 0) {
    931  1.11  cegger 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
    932  1.11  cegger 						__func__);
    933   1.2  bouyer 				return error;
    934  1.11  cegger 			}
    935   1.2  bouyer 			delay(200);
    936   1.2  bouyer 
    937  1.23      ad 			error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid,
    938  1.11  cegger 					LAPIC_DLMODE_STARTUP);
    939  1.11  cegger 			if (error != 0) {
    940  1.11  cegger 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n",
    941  1.11  cegger 						__func__);
    942   1.2  bouyer 				return error;
    943  1.11  cegger 			}
    944   1.2  bouyer 			delay(200);
    945   1.2  bouyer 		}
    946   1.2  bouyer 	}
    947   1.2  bouyer #endif
    948   1.2  bouyer #endif /* 0 */
    949   1.2  bouyer 	return 0;
    950   1.2  bouyer }
    951   1.2  bouyer 
    952   1.2  bouyer void
    953   1.2  bouyer mp_cpu_start_cleanup(struct cpu_info *ci)
    954   1.2  bouyer {
    955   1.2  bouyer #if 0
    956   1.2  bouyer 	/*
    957   1.2  bouyer 	 * Ensure the NVRAM reset byte contains something vaguely sane.
    958   1.2  bouyer 	 */
    959   1.2  bouyer 
    960   1.2  bouyer 	outb(IO_RTC, NVRAM_RESET);
    961   1.2  bouyer 	outb(IO_RTC+1, NVRAM_RESET_RST);
    962   1.2  bouyer #endif
    963   1.2  bouyer }
    964   1.2  bouyer 
    965   1.2  bouyer #ifdef __x86_64__
    966   1.2  bouyer 
    967   1.2  bouyer void
    968   1.3  bouyer cpu_init_msrs(struct cpu_info *ci, bool full)
    969   1.2  bouyer {
    970   1.3  bouyer 	if (full) {
    971   1.3  bouyer 		HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
    972  1.11  cegger 		HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
    973   1.3  bouyer 		HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
    974   1.3  bouyer 	}
    975   1.2  bouyer }
    976   1.2  bouyer #endif	/* __x86_64__ */
    977   1.2  bouyer 
    978  1.11  cegger void
    979  1.11  cegger cpu_offline_md(void)
    980  1.11  cegger {
    981  1.11  cegger         int s;
    982  1.11  cegger 
    983  1.11  cegger         s = splhigh();
    984  1.11  cegger #ifdef __i386__
    985  1.11  cegger         npxsave_cpu(true);
    986  1.11  cegger #else
    987  1.11  cegger         fpusave_cpu(true);
    988  1.11  cegger #endif
    989  1.11  cegger         splx(s);
    990  1.11  cegger }
    991  1.11  cegger 
    992  1.11  cegger #if 0
    993  1.11  cegger /* XXX joerg restructure and restart CPUs individually */
    994  1.11  cegger static bool
    995  1.11  cegger cpu_suspend(device_t dv PMF_FN_ARGS)
    996  1.11  cegger {
    997  1.11  cegger 	struct cpu_softc *sc = device_private(dv);
    998  1.11  cegger 	struct cpu_info *ci = sc->sc_info;
    999  1.11  cegger 	int err;
   1000  1.11  cegger 
   1001  1.11  cegger 	if (ci->ci_flags & CPUF_PRIMARY)
   1002  1.11  cegger 		return true;
   1003  1.11  cegger 	if (ci->ci_data.cpu_idlelwp == NULL)
   1004  1.11  cegger 		return true;
   1005  1.11  cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1006  1.11  cegger 		return true;
   1007  1.11  cegger 
   1008  1.11  cegger 	sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
   1009  1.11  cegger 
   1010  1.11  cegger 	if (sc->sc_wasonline) {
   1011  1.11  cegger 		mutex_enter(&cpu_lock);
   1012  1.11  cegger 		err = cpu_setonline(ci, false);
   1013  1.11  cegger 		mutex_exit(&cpu_lock);
   1014  1.11  cegger 
   1015  1.11  cegger 		if (err)
   1016  1.11  cegger 			return false;
   1017  1.11  cegger 	}
   1018  1.11  cegger 
   1019  1.11  cegger 	return true;
   1020  1.11  cegger }
   1021  1.11  cegger 
   1022  1.11  cegger static bool
   1023  1.11  cegger cpu_resume(device_t dv PMF_FN_ARGS)
   1024  1.11  cegger {
   1025  1.11  cegger 	struct cpu_softc *sc = device_private(dv);
   1026  1.11  cegger 	struct cpu_info *ci = sc->sc_info;
   1027  1.11  cegger 	int err = 0;
   1028  1.11  cegger 
   1029  1.11  cegger 	if (ci->ci_flags & CPUF_PRIMARY)
   1030  1.11  cegger 		return true;
   1031  1.11  cegger 	if (ci->ci_data.cpu_idlelwp == NULL)
   1032  1.11  cegger 		return true;
   1033  1.11  cegger 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1034  1.11  cegger 		return true;
   1035  1.11  cegger 
   1036  1.11  cegger 	if (sc->sc_wasonline) {
   1037  1.11  cegger 		mutex_enter(&cpu_lock);
   1038  1.11  cegger 		err = cpu_setonline(ci, true);
   1039  1.11  cegger 		mutex_exit(&cpu_lock);
   1040  1.11  cegger 	}
   1041  1.11  cegger 
   1042  1.11  cegger 	return err == 0;
   1043  1.11  cegger }
   1044  1.11  cegger #endif
   1045  1.11  cegger 
   1046   1.2  bouyer void
   1047   1.2  bouyer cpu_get_tsc_freq(struct cpu_info *ci)
   1048   1.2  bouyer {
   1049   1.2  bouyer #ifdef XEN3
   1050  1.16  cegger 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
   1051   1.2  bouyer 	delay(1000000);
   1052   1.2  bouyer 	uint64_t freq = 1000000000ULL << 32;
   1053   1.2  bouyer 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
   1054   1.2  bouyer 	if ( tinfo->tsc_shift < 0 )
   1055   1.2  bouyer 		freq = freq << -tinfo->tsc_shift;
   1056   1.2  bouyer 	else
   1057   1.2  bouyer 		freq = freq >> tinfo->tsc_shift;
   1058  1.20      ad 	ci->ci_data.cpu_cc_freq = freq;
   1059   1.2  bouyer #else
   1060  1.16  cegger 	/* Xen2 */
   1061   1.2  bouyer 	/* XXX this needs to read the shared_info of the CPU being probed.. */
   1062  1.20      ad 	ci->ci_data.cpu_cc_freq = HYPERVISOR_shared_info->cpu_freq;
   1063   1.2  bouyer #endif /* XEN3 */
   1064   1.2  bouyer }
   1065  1.19   joerg 
   1066  1.19   joerg void
   1067  1.19   joerg x86_cpu_idle_xen(void)
   1068  1.19   joerg {
   1069  1.19   joerg 	struct cpu_info *ci = curcpu();
   1070  1.19   joerg 
   1071  1.19   joerg 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1072  1.19   joerg 
   1073  1.19   joerg 	x86_disable_intr();
   1074  1.19   joerg 	if (!__predict_false(ci->ci_want_resched)) {
   1075  1.19   joerg 		idle_block();
   1076  1.19   joerg 	} else {
   1077  1.19   joerg 		x86_enable_intr();
   1078  1.19   joerg 	}
   1079  1.19   joerg }
   1080