Home | History | Annotate | Line # | Download | only in rumpkern
rump.c revision 1.349
      1 /*	$NetBSD: rump.c,v 1.349 2020/06/11 00:33:30 kamil Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  *
     15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     16  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     17  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     18  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     21  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     25  * SUCH DAMAGE.
     26  */
     27 
     28 #include <sys/cdefs.h>
     29 __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.349 2020/06/11 00:33:30 kamil Exp $");
     30 
     31 #include <sys/systm.h>
     32 #define ELFSIZE ARCH_ELFSIZE
     33 
     34 #include <sys/param.h>
     35 #include <sys/atomic.h>
     36 #include <sys/buf.h>
     37 #include <sys/callout.h>
     38 #include <sys/conf.h>
     39 #include <sys/cpu.h>
     40 #include <sys/device.h>
     41 #include <sys/evcnt.h>
     42 #include <sys/event.h>
     43 #include <sys/exec_elf.h>
     44 #include <sys/filedesc.h>
     45 #include <sys/iostat.h>
     46 #include <sys/kauth.h>
     47 #include <sys/kcpuset.h>
     48 #include <sys/kernel.h>
     49 #include <sys/kmem.h>
     50 #include <sys/kprintf.h>
     51 #include <sys/kthread.h>
     52 #include <sys/ksyms.h>
     53 #include <sys/msgbuf.h>
     54 #include <sys/module.h>
     55 #include <sys/module_hook.h>
     56 #include <sys/namei.h>
     57 #include <sys/once.h>
     58 #include <sys/percpu.h>
     59 #include <sys/pipe.h>
     60 #include <sys/pool.h>
     61 #include <sys/queue.h>
     62 #include <sys/reboot.h>
     63 #include <sys/resourcevar.h>
     64 #include <sys/select.h>
     65 #include <sys/sysctl.h>
     66 #include <sys/syscall.h>
     67 #include <sys/syscallvar.h>
     68 #include <sys/threadpool.h>
     69 #include <sys/timetc.h>
     70 #include <sys/tty.h>
     71 #include <sys/uidinfo.h>
     72 #include <sys/vmem.h>
     73 #include <sys/xcall.h>
     74 #include <sys/cprng.h>
     75 #include <sys/rnd.h>
     76 #include <sys/ktrace.h>
     77 #include <sys/pserialize.h>
     78 #include <sys/psref.h>
     79 
     80 #include <rump-sys/kern.h>
     81 #include <rump-sys/dev.h>
     82 #include <rump-sys/net.h>
     83 #include <rump-sys/vfs.h>
     84 
     85 #include <rump/rumpuser.h>
     86 
     87 #include <secmodel/suser/suser.h>
     88 
     89 #include <prop/proplib.h>
     90 
     91 #include <uvm/uvm_extern.h>
     92 #include <uvm/uvm_readahead.h>
     93 
     94 char machine[] = MACHINE;
     95 char machine_arch[] = MACHINE_ARCH;
     96 
     97 struct proc *initproc;
     98 
     99 struct device rump_rootdev = {
    100 	.dv_class = DV_VIRTUAL
    101 };
    102 
    103 #ifdef RUMP_WITHOUT_THREADS
    104 int rump_threads = 0;
    105 #else
    106 int rump_threads = 1;
    107 #endif
    108 
    109 static void rump_component_addlocal(void);
    110 static struct lwp *bootlwp;
    111 
    112 /* 16k should be enough for std rump needs */
    113 static  char rump_msgbuf[16*1024] __aligned(256);
    114 
    115 bool rump_ttycomponent = false;
    116 
    117 extern pool_cache_t pnbuf_cache;
    118 
    119 static int rump_inited;
    120 
    121 void (*rump_vfs_drainbufs)(int) = (void *)nullop;
    122 int  (*rump_vfs_makeonedevnode)(dev_t, const char *,
    123 				devmajor_t, devminor_t) = (void *)nullop;
    124 int  (*rump_vfs_makedevnodes)(dev_t, const char *, char,
    125 			      devmajor_t, devminor_t, int) = (void *)nullop;
    126 int  (*rump_vfs_makesymlink)(const char *, const char *) = (void *)nullop;
    127 
    128 rump_proc_vfs_init_fn rump_proc_vfs_init = (void *)nullop;
    129 rump_proc_vfs_release_fn rump_proc_vfs_release = (void *)nullop;
    130 
    131 static void add_linkedin_modules(const struct modinfo *const *, size_t);
    132 static void add_static_evcnt(struct evcnt *);
    133 
    134 static pid_t rspo_wrap_getpid(void) {
    135 	return rump_sysproxy_hyp_getpid();
    136 }
    137 static int rspo_wrap_syscall(int num, void *arg, long *retval) {
    138 	return rump_sysproxy_hyp_syscall(num, arg, retval);
    139 }
    140 static int rspo_wrap_rfork(void *priv, int flag, const char *comm) {
    141 	return rump_sysproxy_hyp_rfork(priv, flag, comm);
    142 }
    143 static void rspo_wrap_lwpexit(void) {
    144 	rump_sysproxy_hyp_lwpexit();
    145 }
    146 static void rspo_wrap_execnotify(const char *comm) {
    147 	rump_sysproxy_hyp_execnotify(comm);
    148 }
    149 static const struct rumpuser_hyperup hyp = {
    150 	.hyp_schedule		= rump_schedule,
    151 	.hyp_unschedule		= rump_unschedule,
    152 	.hyp_backend_unschedule	= rump_user_unschedule,
    153 	.hyp_backend_schedule	= rump_user_schedule,
    154 	.hyp_lwproc_switch	= rump_lwproc_switch,
    155 	.hyp_lwproc_release	= rump_lwproc_releaselwp,
    156 	.hyp_lwproc_newlwp	= rump_lwproc_newlwp,
    157 	.hyp_lwproc_curlwp	= rump_lwproc_curlwp,
    158 
    159 	.hyp_getpid		= rspo_wrap_getpid,
    160 	.hyp_syscall		= rspo_wrap_syscall,
    161 	.hyp_lwproc_rfork	= rspo_wrap_rfork,
    162 	.hyp_lwpexit		= rspo_wrap_lwpexit,
    163 	.hyp_execnotify		= rspo_wrap_execnotify,
    164 };
    165 struct rump_sysproxy_ops rump_sysproxy_ops = {
    166 	.rspo_copyin		= (void *)enxio,
    167 	.rspo_copyinstr 	= (void *)enxio,
    168 	.rspo_copyout	 	= (void *)enxio,
    169 	.rspo_copyoutstr 	= (void *)enxio,
    170 	.rspo_anonmmap 		= (void *)enxio,
    171 	.rspo_raise 		= (void *)enxio,
    172 	.rspo_fini 		= (void *)enxio,
    173 	.rspo_hyp_getpid 	= (void *)enxio,
    174 	.rspo_hyp_syscall 	= (void *)enxio,
    175 	.rspo_hyp_rfork 	= (void *)enxio,
    176 	.rspo_hyp_lwpexit 	= (void *)enxio,
    177 	.rspo_hyp_execnotify 	= (void *)enxio,
    178 };
    179 
    180 int
    181 rump_daemonize_begin(void)
    182 {
    183 
    184 	if (rump_inited)
    185 		return EALREADY;
    186 
    187 	return rumpuser_daemonize_begin();
    188 }
    189 
    190 int
    191 rump_daemonize_done(int error)
    192 {
    193 
    194 	return rumpuser_daemonize_done(error);
    195 }
    196 
    197 #ifdef RUMP_USE_CTOR
    198 
    199 /* sysctl bootstrap handling */
    200 struct sysctl_boot_chain sysctl_boot_chain \
    201     = LIST_HEAD_INITIALIZER(sysctl_boot_chain);
    202 __link_set_add_text(sysctl_funcs,voidop); /* ensure linkset is non-empty */
    203 
    204 #else /* RUMP_USE_CTOR */
    205 
    206 RUMP_COMPONENT(RUMP_COMPONENT_POSTINIT)
    207 {
    208 	__link_set_decl(rump_components, struct rump_component);
    209 
    210 	/*
    211 	 * Trick compiler into generating references so that statically
    212 	 * linked rump kernels are generated with the link set symbols.
    213 	 */
    214 	asm("" :: "r"(__start_link_set_rump_components));
    215 	asm("" :: "r"(__stop_link_set_rump_components));
    216 }
    217 
    218 #endif /* RUMP_USE_CTOR */
    219 
    220 int
    221 rump_init(void)
    222 {
    223 	char buf[256];
    224 	struct timespec bts;
    225 	int64_t sec;
    226 	long nsec;
    227 	struct lwp *l, *initlwp;
    228 	int i, numcpu;
    229 
    230 	/* not reentrant */
    231 	if (rump_inited)
    232 		return 0;
    233 	else if (rump_inited == -1)
    234 		panic("rump_init: host process restart required");
    235 	else
    236 		rump_inited = 1;
    237 
    238 	/* initialize hypervisor */
    239 	if (rumpuser_init(RUMPUSER_VERSION, &hyp) != 0) {
    240 		rumpuser_dprintf("rumpuser init failed\n");
    241 		return EINVAL;
    242 	}
    243 
    244 	/* init minimal lwp/cpu context */
    245 	rump_lwproc_init();
    246 	l = &lwp0;
    247 	l->l_cpu = l->l_target_cpu = &rump_bootcpu;
    248 	rump_lwproc_curlwp_set(l);
    249 
    250 	/* retrieve env vars which affect the early stage of bootstrap */
    251 	if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) {
    252 		rump_threads = *buf != '0';
    253 	}
    254 	if (rumpuser_getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) {
    255 		if (*buf != '0')
    256 			boothowto = AB_VERBOSE;
    257 	}
    258 
    259 	if (rumpuser_getparam(RUMPUSER_PARAM_NCPU, buf, sizeof(buf)) != 0)
    260 		panic("mandatory hypervisor configuration (NCPU) missing");
    261 	numcpu = strtoll(buf, NULL, 10);
    262 	if (numcpu < 1) {
    263 		panic("rump kernels are not lightweight enough for \"%d\" CPUs",
    264 		    numcpu);
    265 	}
    266 
    267 	rump_thread_init();
    268 	rump_cpus_bootstrap(&numcpu);
    269 
    270 	rumpuser_clock_gettime(RUMPUSER_CLOCK_RELWALL, &sec, &nsec);
    271 	bts.tv_sec = sec;
    272 	bts.tv_nsec = nsec;
    273 
    274 	initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf));
    275 	aprint_verbose("%s%s", copyright, version);
    276 
    277 	rump_intr_init(numcpu);
    278 
    279 	rump_tsleep_init();
    280 
    281 	rumpuser_mutex_init(&rump_giantlock, RUMPUSER_MTX_SPIN);
    282 	ksyms_init();
    283 	uvm_init();
    284 	evcnt_init();
    285 
    286 	kcpuset_sysinit();
    287 	once_init();
    288 	kernconfig_lock_init();
    289 	prop_kern_init();
    290 
    291 	kmem_init();
    292 
    293 	uvm_ra_init();
    294 	uao_init();
    295 
    296 	mutex_obj_init();
    297 	rw_obj_init();
    298 	callout_startup();
    299 
    300 	kprintf_init();
    301 	percpu_init();
    302 	pserialize_init();
    303 
    304 	kauth_init();
    305 
    306 	secmodel_init();
    307 	sysctl_init();
    308 	/*
    309 	 * The above call to sysctl_init() only initializes sysctl nodes
    310 	 * from link sets.  Initialize sysctls in case we used ctors.
    311 	 */
    312 #ifdef RUMP_USE_CTOR
    313 	{
    314 		struct sysctl_setup_chain *ssc;
    315 
    316 		while ((ssc = LIST_FIRST(&sysctl_boot_chain)) != NULL) {
    317 			LIST_REMOVE(ssc, ssc_entries);
    318 			ssc->ssc_func(NULL);
    319 		}
    320 	}
    321 #endif /* RUMP_USE_CTOR */
    322 
    323 	rnd_init();
    324 	rump_hyperentropy_init();
    325 
    326 	procinit();
    327 	proc0_init();
    328 	uid_init();
    329 	chgproccnt(0, 1);
    330 
    331 	l->l_proc = &proc0;
    332 	lwp_update_creds(l);
    333 
    334 	lwpinit_specificdata();
    335 	lwp_initspecific(&lwp0);
    336 
    337 	/* Must be called after lwpinit_specificdata */
    338 	psref_init();
    339 
    340 	threadpools_init();
    341 
    342 	loginit();
    343 
    344 	rump_biglock_init();
    345 
    346 	rump_scheduler_init(numcpu);
    347 	/* revert temporary context and schedule a semireal context */
    348 	rump_lwproc_curlwp_clear(l);
    349 	initproc = &proc0; /* borrow proc0 before we get initproc started */
    350 	rump_schedule();
    351 	bootlwp = curlwp;
    352 
    353 	inittimecounter();
    354 	ntp_init();
    355 
    356 #ifdef KTRACE
    357 	ktrinit();
    358 #endif
    359 
    360 	tc_setclock(&bts);
    361 
    362 	extern krwlock_t exec_lock;
    363 	rw_init(&exec_lock);
    364 
    365 	/* we are mostly go.  do per-cpu subsystem init */
    366 	for (i = 0; i < numcpu; i++) {
    367 		struct cpu_info *ci = cpu_lookup(i);
    368 
    369 		/* attach non-bootstrap CPUs */
    370 		if (i > 0) {
    371 			rump_cpu_attach(ci);
    372 			ncpu++;
    373 		}
    374 		snprintf(ci->ci_cpuname, sizeof ci->ci_cpuname, "cpu%d", i);
    375 
    376 		callout_init_cpu(ci);
    377 		softint_init(ci);
    378 		xc_init_cpu(ci);
    379 		pool_cache_cpu_init(ci);
    380 		selsysinit(ci);
    381 		percpu_init_cpu(ci);
    382 
    383 		TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
    384 		__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
    385 
    386 		aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i);
    387 	}
    388 	ncpuonline = ncpu;
    389 
    390 	/* Once all CPUs are detected, initialize the per-CPU cprng_fast.  */
    391 	cprng_init();
    392 	cprng_fast_init();
    393 
    394 	mp_online = true;
    395 
    396 	/* CPUs are up.  allow kernel threads to run */
    397 	rump_thread_allow(NULL);
    398 
    399 	rnd_init_softint();
    400 
    401 	kqueue_init();
    402 	iostat_init();
    403 	fd_sys_init();
    404 	module_init();
    405 	module_hook_init();
    406 	devsw_init();
    407 	pipe_init();
    408 	resource_init();
    409 	procinit_sysctl();
    410 	time_init();
    411 	time_init2();
    412 	config_init();
    413 
    414 	/* start page baroness */
    415 	if (rump_threads) {
    416 		if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL,
    417 		    uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0)
    418 			panic("pagedaemon create failed");
    419 	} else
    420 		uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */
    421 
    422 	/* process dso's */
    423 	rumpuser_dl_bootstrap(add_linkedin_modules,
    424 	    rump_kernelfsym_load, rump_component_load, add_static_evcnt);
    425 
    426 	rump_component_addlocal();
    427 	rump_component_init(RUMP_COMPONENT_KERN);
    428 
    429 	/* initialize factions, if present */
    430 	rump_component_init(RUMP__FACTION_VFS);
    431 	/* pnbuf_cache is used even without vfs */
    432 	if (rump_component_count(RUMP__FACTION_VFS) == 0) {
    433 		pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl",
    434 		    NULL, IPL_NONE, NULL, NULL, NULL);
    435 	}
    436 	rump_component_init(RUMP__FACTION_NET);
    437 	rump_component_init(RUMP__FACTION_DEV);
    438 	KASSERT(rump_component_count(RUMP__FACTION_VFS) <= 1
    439 	    && rump_component_count(RUMP__FACTION_NET) <= 1
    440 	    && rump_component_count(RUMP__FACTION_DEV) <= 1);
    441 
    442 	rump_component_init(RUMP_COMPONENT_KERN_VFS);
    443 
    444 	/*
    445 	 * if we initialized the tty component above, the tyttymtx is
    446 	 * now initialized.  otherwise, we need to initialize it.
    447 	 */
    448 	if (!rump_ttycomponent)
    449 		mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM);
    450 
    451 	cold = 0;
    452 
    453 	sysctl_finalize();
    454 
    455 	module_init_class(MODULE_CLASS_ANY);
    456 
    457 	if (rumpuser_getparam(RUMPUSER_PARAM_HOSTNAME,
    458 	    hostname, MAXHOSTNAMELEN) != 0) {
    459 		panic("mandatory hypervisor configuration (HOSTNAME) missing");
    460 	}
    461 	hostnamelen = strlen(hostname);
    462 
    463 	sigemptyset(&sigcantmask);
    464 
    465 	if (rump_threads)
    466 		vmem_rehash_start();
    467 
    468 	/*
    469 	 * Create init (proc 1), used to attach implicit threads in rump.
    470 	 * (note: must be done after vfsinit to get cwdi)
    471 	 */
    472 	initlwp = rump__lwproc_alloclwp(NULL);
    473 	mutex_enter(&proc_lock);
    474 	initproc = proc_find_raw(1);
    475 	mutex_exit(&proc_lock);
    476 	if (initproc == NULL)
    477 		panic("where in the world is initproc?");
    478 	strlcpy(initproc->p_comm, "rumplocal", sizeof(initproc->p_comm));
    479 
    480 	rump_component_init(RUMP_COMPONENT_POSTINIT);
    481 
    482 	/* load syscalls */
    483 	rump_component_init(RUMP_COMPONENT_SYSCALL);
    484 
    485 	/* component inits done */
    486 	bootlwp = NULL;
    487 
    488 	/* open 0/1/2 for init */
    489 	KASSERT(rump_lwproc_curlwp() == NULL);
    490 	rump_lwproc_switch(initlwp);
    491 	rump_consdev_init();
    492 	rump_lwproc_switch(NULL);
    493 
    494 	/* release cpu */
    495 	rump_unschedule();
    496 
    497 	return 0;
    498 }
    499 /* historic compat */
    500 __strong_alias(rump__init,rump_init);
    501 
    502 static int compcounter[RUMP_COMPONENT_MAX];
    503 static int compinited[RUMP_COMPONENT_MAX];
    504 
    505 /*
    506  * Yea, this is O(n^2), but we're only looking at a handful of components.
    507  * Components are always initialized from the thread that called rump_init().
    508  */
    509 static LIST_HEAD(, rump_component) rchead = LIST_HEAD_INITIALIZER(rchead);
    510 
    511 #ifdef RUMP_USE_CTOR
    512 struct modinfo_boot_chain modinfo_boot_chain \
    513     = LIST_HEAD_INITIALIZER(modinfo_boot_chain);
    514 
    515 static void
    516 rump_component_addlocal(void)
    517 {
    518 	struct modinfo_chain *mc;
    519 
    520 	while ((mc = LIST_FIRST(&modinfo_boot_chain)) != NULL) {
    521 		LIST_REMOVE(mc, mc_entries);
    522 		module_builtin_add(&mc->mc_info, 1, false);
    523 	}
    524 }
    525 
    526 #else /* RUMP_USE_CTOR */
    527 
    528 static void
    529 rump_component_addlocal(void)
    530 {
    531 	__link_set_decl(rump_components, struct rump_component);
    532 	struct rump_component *const *rc;
    533 
    534 	__link_set_foreach(rc, rump_components) {
    535 		rump_component_load(*rc);
    536 	}
    537 }
    538 #endif /* RUMP_USE_CTOR */
    539 
    540 void
    541 rump_component_load(const struct rump_component *rc_const)
    542 {
    543 	struct rump_component *rc, *rc_iter;
    544 
    545 	/* time for rump component loading and unloading has passed */
    546 	if (!cold)
    547 		return;
    548 
    549 	/*
    550 	 * XXX: this is ok since the "const" was removed from the
    551 	 * definition of RUMP_COMPONENT().
    552 	 *
    553 	 * However, to preserve the hypercall interface, the const
    554 	 * remains here.  This can be fixed in the next hypercall revision.
    555 	 */
    556 	rc = __UNCONST(rc_const);
    557 
    558 	KASSERT(!rump_inited || curlwp == bootlwp);
    559 
    560 	LIST_FOREACH(rc_iter, &rchead, rc_entries) {
    561 		if (rc_iter == rc)
    562 			return;
    563 	}
    564 
    565 	LIST_INSERT_HEAD(&rchead, rc, rc_entries);
    566 	KASSERT(rc->rc_type < RUMP_COMPONENT_MAX);
    567 	compcounter[rc->rc_type]++;
    568 }
    569 
    570 void
    571 rump_component_unload(struct rump_component *rc)
    572 {
    573 
    574 	/*
    575 	 * Checking for cold is enough because rump_init() both
    576 	 * flips it and handles component loading.
    577 	 */
    578 	if (!cold)
    579 		return;
    580 
    581 	LIST_REMOVE(rc, rc_entries);
    582 }
    583 
    584 int
    585 rump_component_count(enum rump_component_type type)
    586 {
    587 
    588 	KASSERT(curlwp == bootlwp);
    589 	KASSERT(type < RUMP_COMPONENT_MAX);
    590 	return compcounter[type];
    591 }
    592 
    593 void
    594 rump_component_init(enum rump_component_type type)
    595 {
    596 	const struct rump_component *rc, *rc_safe;
    597 
    598 	KASSERT(curlwp == bootlwp);
    599 	KASSERT(!compinited[type]);
    600 	LIST_FOREACH_SAFE(rc, &rchead, rc_entries, rc_safe) {
    601 		if (rc->rc_type == type) {
    602 			rc->rc_init();
    603 			LIST_REMOVE(rc, rc_entries);
    604 		}
    605 	}
    606 	compinited[type] = 1;
    607 }
    608 
    609 /*
    610  * Initialize a module which has already been loaded and linked
    611  * with dlopen(). This is fundamentally the same as a builtin module.
    612  *
    613  * XXX: this interface does not really work in the RUMP_USE_CTOR case,
    614  * but I'm not sure it's anything to cry about.  In feeling blue,
    615  * things could somehow be handled via modinfo_boot_chain.
    616  */
    617 int
    618 rump_module_init(const struct modinfo * const *mip, size_t nmodinfo)
    619 {
    620 
    621 	return module_builtin_add(mip, nmodinfo, true);
    622 }
    623 
    624 /*
    625  * Finish module (flawless victory, fatality!).
    626  */
    627 int
    628 rump_module_fini(const struct modinfo *mi)
    629 {
    630 
    631 	return module_builtin_remove(mi, true);
    632 }
    633 
    634 /*
    635  * Add loaded and linked module to the builtin list.  It will
    636  * later be initialized with module_init_class().
    637  */
    638 
    639 static void
    640 add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo)
    641 {
    642 
    643 	module_builtin_add(mip, nmodinfo, false);
    644 }
    645 
    646 /*
    647  * Add an evcnt.
    648  */
    649 static void
    650 add_static_evcnt(struct evcnt *ev)
    651 {
    652 
    653 	evcnt_attach_static(ev);
    654 }
    655 
    656 int
    657 rump_kernelfsym_load(void *symtab, uint64_t symsize,
    658 	char *strtab, uint64_t strsize)
    659 {
    660 	static int inited = 0;
    661 	Elf64_Ehdr ehdr;
    662 
    663 	if (inited)
    664 		return EBUSY;
    665 	inited = 1;
    666 
    667 	/*
    668 	 * Use 64bit header since it's bigger.  Shouldn't make a
    669 	 * difference, since we're passing in all zeroes anyway.
    670 	 */
    671 	memset(&ehdr, 0, sizeof(ehdr));
    672 	ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize);
    673 
    674 	return 0;
    675 }
    676 
    677 int
    678 rump_boot_gethowto()
    679 {
    680 
    681 	return boothowto;
    682 }
    683 
    684 void
    685 rump_boot_sethowto(int howto)
    686 {
    687 
    688 	boothowto = howto;
    689 }
    690 
    691 int
    692 rump_getversion(void)
    693 {
    694 
    695 	return __NetBSD_Version__;
    696 }
    697 /* compat */
    698 __strong_alias(rump_pub_getversion,rump_getversion);
    699 
    700 /*
    701  * Note: may be called unscheduled.  Not fully safe since no locking
    702  * of allevents (currently that's not even available).
    703  */
    704 void
    705 rump_printevcnts()
    706 {
    707 	struct evcnt *ev;
    708 
    709 	TAILQ_FOREACH(ev, &allevents, ev_list)
    710 		rumpuser_dprintf("%s / %s: %" PRIu64 "\n",
    711 		    ev->ev_group, ev->ev_name, ev->ev_count);
    712 }
    713 
    714 /*
    715  * If you use this interface ... well ... all bets are off.
    716  * The original purpose is for the p2k fs server library to be
    717  * able to use the same pid/lid for VOPs as the host kernel.
    718  */
    719 void
    720 rump_allbetsareoff_setid(pid_t pid, int lid)
    721 {
    722 	struct lwp *l = curlwp;
    723 	struct proc *p = l->l_proc;
    724 
    725 	l->l_lid = lid;
    726 	p->p_pid = pid;
    727 }
    728 
    729 static void
    730 ipiemu(void *a1, void *a2)
    731 {
    732 
    733 	xc__highpri_intr(NULL);
    734 }
    735 
    736 void
    737 rump_xc_highpri(struct cpu_info *ci)
    738 {
    739 
    740 	if (ci)
    741 		xc_unicast(0, ipiemu, NULL, NULL, ci);
    742 	else
    743 		xc_broadcast(0, ipiemu, NULL, NULL);
    744 }
    745 
    746 int
    747 rump_syscall(int num, void *data, size_t dlen, register_t *retval)
    748 {
    749 	struct proc *p;
    750 	struct emul *e;
    751 	struct sysent *callp;
    752 	const int *etrans = NULL;
    753 	int rv;
    754 
    755 	rump_schedule();
    756 	p = curproc;
    757 	e = p->p_emul;
    758 #ifndef __HAVE_MINIMAL_EMUL
    759 	num &= e->e_nsysent - 1;
    760 #else
    761 	num &= SYS_NSYSENT - 1;
    762 #endif
    763 	callp = e->e_sysent + num;
    764 
    765 	rv = sy_invoke(callp, curlwp, data, retval, num);
    766 
    767 	/*
    768 	 * I hope that (!__HAVE_MINIMAL_EMUL || __HAVE_SYSCALL_INTERN) is
    769 	 * an invariant ...
    770 	 */
    771 #if !defined(__HAVE_MINIMAL_EMUL)
    772 	etrans = e->e_errno;
    773 #elif defined(__HAVE_SYSCALL_INTERN)
    774 	etrans = p->p_emuldata;
    775 #endif
    776 
    777 	if (etrans) {
    778 		rv = etrans[rv];
    779 		/*
    780 		 * XXX: small hack since Linux etrans vectors on some
    781 		 * archs contain negative errnos, but rump_syscalls
    782 		 * uses the -1 + errno ABI.  Note that these
    783 		 * negative values are always the result of translation,
    784 		 * otherwise the above translation method would not
    785 		 * work very well.
    786 		 */
    787 		if (rv < 0)
    788 			rv = -rv;
    789 	}
    790 	rump_unschedule();
    791 
    792 	return rv;
    793 }
    794 
    795 void
    796 rump_syscall_boot_establish(const struct rump_onesyscall *calls, size_t ncall)
    797 {
    798 	struct sysent *callp;
    799 	size_t i;
    800 
    801 	for (i = 0; i < ncall; i++) {
    802 		callp = rump_sysent + calls[i].ros_num;
    803 		KASSERT(bootlwp != NULL
    804 		    && callp->sy_call == (sy_call_t *)(void *)enosys);
    805 		callp->sy_call = calls[i].ros_handler;
    806 	}
    807 }
    808 
    809 struct rump_boot_etfs *ebstart;
    810 void
    811 rump_boot_etfs_register(struct rump_boot_etfs *eb)
    812 {
    813 
    814 	/*
    815 	 * Could use atomics, but, since caller would need to synchronize
    816 	 * against calling rump_init() anyway, easier to just specify the
    817 	 * interface as "caller serializes".  This solve-by-specification
    818 	 * approach avoids the grey area of using atomics before rump_init()
    819 	 * runs.
    820 	 */
    821 	eb->_eb_next = ebstart;
    822 	eb->eb_status = -1;
    823 	ebstart = eb;
    824 }
    825