Home | History | Annotate | Line # | Download | only in rumpkern
rump.c revision 1.243
      1  1.243  pgoyette /*	$NetBSD: rump.c,v 1.243 2012/08/16 17:25:36 pgoyette Exp $	*/
      2    1.1     pooka 
      3    1.1     pooka /*
      4  1.233     pooka  * Copyright (c) 2007-2011 Antti Kantee.  All Rights Reserved.
      5    1.1     pooka  *
      6    1.1     pooka  * Redistribution and use in source and binary forms, with or without
      7    1.1     pooka  * modification, are permitted provided that the following conditions
      8    1.1     pooka  * are met:
      9    1.1     pooka  * 1. Redistributions of source code must retain the above copyright
     10    1.1     pooka  *    notice, this list of conditions and the following disclaimer.
     11    1.1     pooka  * 2. Redistributions in binary form must reproduce the above copyright
     12    1.1     pooka  *    notice, this list of conditions and the following disclaimer in the
     13    1.1     pooka  *    documentation and/or other materials provided with the distribution.
     14    1.1     pooka  *
     15    1.1     pooka  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     16    1.1     pooka  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     17    1.1     pooka  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     18    1.1     pooka  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     19    1.1     pooka  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     20    1.1     pooka  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     21    1.1     pooka  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     22    1.1     pooka  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     23    1.1     pooka  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     24    1.1     pooka  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     25    1.1     pooka  * SUCH DAMAGE.
     26    1.1     pooka  */
     27    1.1     pooka 
     28   1.78     pooka #include <sys/cdefs.h>
     29  1.243  pgoyette __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.243 2012/08/16 17:25:36 pgoyette Exp $");
     30  1.165     pooka 
     31  1.165     pooka #include <sys/systm.h>
     32  1.165     pooka #define ELFSIZE ARCH_ELFSIZE
     33   1.78     pooka 
     34    1.1     pooka #include <sys/param.h>
     35   1.64     pooka #include <sys/atomic.h>
     36   1.75     pooka #include <sys/buf.h>
     37   1.66     pooka #include <sys/callout.h>
     38   1.79     pooka #include <sys/conf.h>
     39   1.27     pooka #include <sys/cpu.h>
     40  1.144     pooka #include <sys/device.h>
     41  1.100     pooka #include <sys/evcnt.h>
     42   1.98     pooka #include <sys/event.h>
     43  1.137     pooka #include <sys/exec_elf.h>
     44    1.1     pooka #include <sys/filedesc.h>
     45   1.72     pooka #include <sys/iostat.h>
     46    1.1     pooka #include <sys/kauth.h>
     47   1.80     pooka #include <sys/kernel.h>
     48   1.14     pooka #include <sys/kmem.h>
     49   1.81     pooka #include <sys/kprintf.h>
     50  1.175     pooka #include <sys/kthread.h>
     51   1.82     pooka #include <sys/ksyms.h>
     52   1.81     pooka #include <sys/msgbuf.h>
     53   1.49    simonb #include <sys/module.h>
     54  1.234     pooka #include <sys/namei.h>
     55   1.71     pooka #include <sys/once.h>
     56   1.65     pooka #include <sys/percpu.h>
     57  1.139     pooka #include <sys/pipe.h>
     58  1.162     pooka #include <sys/pool.h>
     59    1.1     pooka #include <sys/queue.h>
     60  1.121     pooka #include <sys/reboot.h>
     61    1.1     pooka #include <sys/resourcevar.h>
     62   1.27     pooka #include <sys/select.h>
     63   1.87     pooka #include <sys/sysctl.h>
     64   1.96     pooka #include <sys/syscall.h>
     65  1.169     pooka #include <sys/syscallvar.h>
     66  1.160     pooka #include <sys/timetc.h>
     67   1.87     pooka #include <sys/tty.h>
     68   1.68     pooka #include <sys/uidinfo.h>
     69   1.89     pooka #include <sys/vmem.h>
     70  1.143     pooka #include <sys/xcall.h>
     71  1.216     pooka #include <sys/simplelock.h>
     72  1.235       tls #include <sys/cprng.h>
     73    1.1     pooka 
     74   1.48     pooka #include <rump/rumpuser.h>
     75   1.48     pooka 
     76  1.117      elad #include <secmodel/suser/suser.h>
     77  1.101     pooka 
     78  1.114     pooka #include <prop/proplib.h>
     79  1.114     pooka 
     80  1.174     pooka #include <uvm/uvm_extern.h>
     81  1.136     pooka #include <uvm/uvm_readahead.h>
     82  1.136     pooka 
     83    1.8     pooka #include "rump_private.h"
     84   1.71     pooka #include "rump_net_private.h"
     85   1.75     pooka #include "rump_vfs_private.h"
     86  1.112     pooka #include "rump_dev_private.h"
     87    1.1     pooka 
     88  1.164     pooka char machine[] = MACHINE;
     89    1.1     pooka 
     90  1.193     pooka struct proc *initproc;
     91  1.101     pooka 
     92  1.144     pooka struct device rump_rootdev = {
     93  1.144     pooka 	.dv_class = DV_VIRTUAL
     94  1.144     pooka };
     95  1.144     pooka 
     96   1.54     pooka #ifdef RUMP_WITHOUT_THREADS
     97   1.54     pooka int rump_threads = 0;
     98   1.54     pooka #else
     99   1.54     pooka int rump_threads = 1;
    100   1.54     pooka #endif
    101   1.54     pooka 
    102  1.199     pooka static int rump_proxy_syscall(int, void *, register_t *);
    103  1.226     pooka static int rump_proxy_rfork(void *, int, const char *);
    104  1.232     pooka static void rump_proxy_lwpexit(void);
    105  1.230     pooka static void rump_proxy_execnotify(const char *);
    106  1.199     pooka 
    107  1.156     pooka static char rump_msgbuf[16*1024]; /* 16k should be enough for std rump needs */
    108  1.156     pooka 
    109  1.217     pooka #ifdef LOCKDEBUG
    110  1.217     pooka const int rump_lockdebug = 1;
    111  1.217     pooka #else
    112  1.217     pooka const int rump_lockdebug = 0;
    113  1.217     pooka #endif
    114  1.218     pooka bool rump_ttycomponent = false;
    115  1.217     pooka 
    116   1.14     pooka static void
    117   1.14     pooka rump_aiodone_worker(struct work *wk, void *dummy)
    118   1.14     pooka {
    119   1.14     pooka 	struct buf *bp = (struct buf *)wk;
    120   1.14     pooka 
    121   1.14     pooka 	KASSERT(&bp->b_work == wk);
    122   1.14     pooka 	bp->b_iodone(bp);
    123   1.14     pooka }
    124   1.14     pooka 
    125   1.51     pooka static int rump_inited;
    126   1.27     pooka 
    127  1.162     pooka /*
    128  1.162     pooka  * Make sure pnbuf_cache is available even without vfs
    129  1.162     pooka  */
    130  1.162     pooka int rump_initpnbufpool(void);
    131  1.162     pooka int rump_initpnbufpool(void)
    132  1.162     pooka {
    133  1.162     pooka 
    134  1.162     pooka         pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl",
    135  1.162     pooka 	    NULL, IPL_NONE, NULL, NULL, NULL);
    136  1.162     pooka 	return EOPNOTSUPP;
    137  1.162     pooka }
    138  1.162     pooka 
    139  1.126     pooka int rump__unavailable(void);
    140  1.126     pooka int rump__unavailable() {return EOPNOTSUPP;}
    141   1.86     pooka __weak_alias(rump_net_init,rump__unavailable);
    142  1.162     pooka __weak_alias(rump_vfs_init,rump_initpnbufpool);
    143  1.112     pooka __weak_alias(rump_dev_init,rump__unavailable);
    144   1.75     pooka 
    145  1.121     pooka __weak_alias(rump_vfs_fini,rump__unavailable);
    146  1.121     pooka 
    147  1.113     pooka __weak_alias(biodone,rump__unavailable);
    148  1.126     pooka __weak_alias(sopoll,rump__unavailable);
    149  1.113     pooka 
    150  1.188     pooka __weak_alias(rump_vfs_drainbufs,rump__unavailable);
    151  1.188     pooka 
    152   1.92     pooka void rump__unavailable_vfs_panic(void);
    153   1.92     pooka void rump__unavailable_vfs_panic() {panic("vfs component not available");}
    154  1.132     pooka __weak_alias(usermount_common_policy,rump__unavailable_vfs_panic);
    155   1.92     pooka 
    156  1.211     pooka /* easier to write vfs-less clients */
    157  1.211     pooka __weak_alias(rump_pub_etfs_register,rump__unavailable);
    158  1.228     pooka __weak_alias(rump_pub_etfs_register_withsize,rump__unavailable);
    159  1.211     pooka __weak_alias(rump_pub_etfs_remove,rump__unavailable);
    160  1.211     pooka 
    161  1.147     pooka rump_proc_vfs_init_fn rump_proc_vfs_init;
    162  1.147     pooka rump_proc_vfs_release_fn rump_proc_vfs_release;
    163   1.71     pooka 
    164  1.155     pooka static void add_linkedin_modules(const struct modinfo *const *, size_t);
    165  1.155     pooka 
    166  1.158     pooka /*
    167  1.158     pooka  * Create kern.hostname.  why only this you ask.  well, init_sysctl
    168  1.158     pooka  * is a kitchen sink in need of some gardening.  but i want to use
    169  1.158     pooka  * kern.hostname today.
    170  1.158     pooka  */
    171  1.158     pooka static void
    172  1.158     pooka mksysctls(void)
    173  1.158     pooka {
    174  1.158     pooka 
    175  1.158     pooka 	sysctl_createv(NULL, 0, NULL, NULL,
    176  1.158     pooka 	    CTLFLAG_PERMANENT, CTLTYPE_NODE, "kern", NULL,
    177  1.158     pooka 	    NULL, 0, NULL, 0, CTL_KERN, CTL_EOL);
    178  1.158     pooka 
    179  1.158     pooka 	/* XXX: setting hostnamelen is missing */
    180  1.158     pooka 	sysctl_createv(NULL, 0, NULL, NULL,
    181  1.158     pooka 	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRING, "hostname",
    182  1.158     pooka 	    SYSCTL_DESCR("System hostname"), NULL, 0,
    183  1.241       dsl 	    hostname, MAXHOSTNAMELEN, CTL_KERN, KERN_HOSTNAME, CTL_EOL);
    184  1.158     pooka }
    185  1.158     pooka 
    186  1.201     pooka /* there's no convenient kernel entry point for this, so just craft out own */
    187  1.201     pooka static pid_t
    188  1.201     pooka spgetpid(void)
    189  1.201     pooka {
    190  1.201     pooka 
    191  1.201     pooka 	return curproc->p_pid;
    192  1.201     pooka }
    193  1.201     pooka 
    194  1.197     pooka static const struct rumpuser_sp_ops spops = {
    195  1.197     pooka 	.spop_schedule		= rump_schedule,
    196  1.197     pooka 	.spop_unschedule	= rump_unschedule,
    197  1.197     pooka 	.spop_lwproc_switch	= rump_lwproc_switch,
    198  1.197     pooka 	.spop_lwproc_release	= rump_lwproc_releaselwp,
    199  1.214     pooka 	.spop_lwproc_rfork	= rump_proxy_rfork,
    200  1.201     pooka 	.spop_lwproc_newlwp	= rump_lwproc_newlwp,
    201  1.197     pooka 	.spop_lwproc_curlwp	= rump_lwproc_curlwp,
    202  1.232     pooka 	.spop_lwpexit		= rump_proxy_lwpexit,
    203  1.199     pooka 	.spop_syscall		= rump_proxy_syscall,
    204  1.230     pooka 	.spop_execnotify	= rump_proxy_execnotify,
    205  1.201     pooka 	.spop_getpid		= spgetpid,
    206  1.197     pooka };
    207  1.197     pooka 
    208   1.61     pooka int
    209  1.207     pooka rump_daemonize_begin(void)
    210  1.207     pooka {
    211  1.207     pooka 
    212  1.207     pooka 	if (rump_inited)
    213  1.207     pooka 		return EALREADY;
    214  1.207     pooka 
    215  1.207     pooka 	return rumpuser_daemonize_begin();
    216  1.207     pooka }
    217  1.207     pooka 
    218  1.207     pooka int
    219  1.207     pooka rump_daemonize_done(int error)
    220  1.207     pooka {
    221  1.207     pooka 
    222  1.207     pooka 	return rumpuser_daemonize_done(error);
    223  1.207     pooka }
    224  1.207     pooka 
    225  1.207     pooka int
    226   1.83     pooka rump__init(int rump_version)
    227    1.1     pooka {
    228   1.36     pooka 	char buf[256];
    229  1.160     pooka 	struct timespec ts;
    230  1.157     pooka 	uint64_t sec, nsec;
    231   1.14     pooka 	struct lwp *l;
    232  1.166     pooka 	int i, numcpu;
    233    1.1     pooka 	int error;
    234    1.1     pooka 
    235  1.121     pooka 	/* not reentrant */
    236   1.27     pooka 	if (rump_inited)
    237   1.61     pooka 		return 0;
    238  1.121     pooka 	else if (rump_inited == -1)
    239  1.121     pooka 		panic("rump_init: host process restart required");
    240  1.121     pooka 	else
    241  1.121     pooka 		rump_inited = 1;
    242   1.27     pooka 
    243  1.177     pooka 	if (rumpuser_getversion() != RUMPUSER_VERSION) {
    244  1.177     pooka 		/* let's hope the ABI of rumpuser_dprintf is the same ;) */
    245  1.177     pooka 		rumpuser_dprintf("rumpuser version mismatch: %d vs. %d\n",
    246  1.177     pooka 		    rumpuser_getversion(), RUMPUSER_VERSION);
    247  1.177     pooka 		return EPROGMISMATCH;
    248  1.177     pooka 	}
    249  1.177     pooka 
    250  1.149     pooka 	if (rumpuser_getenv("RUMP_VERBOSE", buf, sizeof(buf), &error) == 0) {
    251  1.149     pooka 		if (*buf != '0')
    252  1.149     pooka 			boothowto = AB_VERBOSE;
    253  1.149     pooka 	}
    254  1.149     pooka 
    255  1.167     pooka 	if (rumpuser_getenv("RUMP_NCPU", buf, sizeof(buf), &error) == 0)
    256  1.167     pooka 		error = 0;
    257  1.167     pooka 	if (error == 0) {
    258  1.167     pooka 		numcpu = strtoll(buf, NULL, 10);
    259  1.167     pooka 		if (numcpu < 1)
    260  1.167     pooka 			numcpu = 1;
    261  1.167     pooka 	} else {
    262  1.167     pooka 		numcpu = rumpuser_getnhostcpu();
    263  1.167     pooka 	}
    264  1.204     pooka 	rump_cpus_bootstrap(&numcpu);
    265  1.165     pooka 
    266  1.157     pooka 	rumpuser_gettime(&sec, &nsec, &error);
    267  1.157     pooka 	boottime.tv_sec = sec;
    268  1.157     pooka 	boottime.tv_nsec = nsec;
    269  1.157     pooka 
    270  1.156     pooka 	initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf));
    271  1.156     pooka 	aprint_verbose("%s%s", copyright, version);
    272  1.148     pooka 
    273   1.61     pooka 	if (rump_version != RUMP_VERSION) {
    274   1.61     pooka 		printf("rump version mismatch, %d vs. %d\n",
    275   1.61     pooka 		    rump_version, RUMP_VERSION);
    276   1.61     pooka 		return EPROGMISMATCH;
    277   1.61     pooka 	}
    278   1.61     pooka 
    279   1.54     pooka 	if (rumpuser_getenv("RUMP_THREADS", buf, sizeof(buf), &error) == 0) {
    280   1.54     pooka 		rump_threads = *buf != '0';
    281   1.54     pooka 	}
    282  1.124     pooka 	rumpuser_thrinit(rump_user_schedule, rump_user_unschedule,
    283  1.124     pooka 	    rump_threads);
    284  1.187     pooka 	rump_intr_init(numcpu);
    285  1.173     pooka 	rump_tsleep_init();
    286   1.36     pooka 
    287  1.131     pooka 	/* init minimal lwp/cpu context */
    288  1.131     pooka 	l = &lwp0;
    289  1.131     pooka 	l->l_lid = 1;
    290  1.172     pooka 	l->l_cpu = l->l_target_cpu = rump_cpu;
    291  1.178     pooka 	l->l_fd = &filedesc0;
    292  1.131     pooka 	rumpuser_set_curlwp(l);
    293  1.131     pooka 
    294  1.208     pooka 	rumpuser_mutex_init(&rump_giantlock);
    295   1.82     pooka 	ksyms_init();
    296  1.174     pooka 	uvm_init();
    297  1.100     pooka 	evcnt_init();
    298   1.89     pooka 
    299   1.89     pooka 	once_init();
    300  1.181  pgoyette 	kernconfig_lock_init();
    301  1.114     pooka 	prop_kern_init();
    302   1.63     pooka 
    303   1.52     pooka 	kmem_init();
    304  1.102     pooka 
    305  1.136     pooka 	uvm_ra_init();
    306  1.185     pooka 	uao_init();
    307  1.136     pooka 
    308  1.133     pooka 	mutex_obj_init();
    309  1.143     pooka 	callout_startup();
    310  1.133     pooka 
    311   1.81     pooka 	kprintf_init();
    312  1.243  pgoyette 	pserialize_init();
    313   1.81     pooka 	loginit();
    314   1.52     pooka 
    315   1.59     pooka 	kauth_init();
    316   1.59     pooka 
    317  1.238       jym 	secmodel_init();
    318  1.238       jym 
    319  1.240     njoly 	rnd_init();
    320  1.240     njoly 
    321  1.236     njoly 	/*
    322  1.237       tls 	 * Create the kernel cprng.  Yes, it's currently stubbed out
    323  1.237       tls 	 * to arc4random() for RUMP, but this won't always be so.
    324  1.236     njoly 	 */
    325  1.237       tls 	kern_cprng = cprng_strong_create("kernel", IPL_VM,
    326  1.237       tls 					 CPRNG_INIT_ANY|CPRNG_REKEY_ANY);
    327  1.236     njoly 
    328  1.184     pooka 	procinit();
    329  1.184     pooka 	proc0_init();
    330  1.242  christos 	sysctl_init();
    331  1.192     pooka 	uid_init();
    332  1.192     pooka 	chgproccnt(0, 1);
    333  1.184     pooka 
    334  1.178     pooka 	l->l_proc = &proc0;
    335  1.184     pooka 	lwp_update_creds(l);
    336  1.176     pooka 
    337  1.159     pooka 	lwpinit_specificdata();
    338  1.176     pooka 	lwp_initspecific(&lwp0);
    339   1.36     pooka 
    340  1.186     pooka 	rump_scheduler_init(numcpu);
    341  1.193     pooka 	/* revert temporary context and schedule a semireal context */
    342  1.131     pooka 	rumpuser_set_curlwp(NULL);
    343  1.193     pooka 	initproc = &proc0; /* borrow proc0 before we get initproc started */
    344  1.124     pooka 	rump_schedule();
    345  1.124     pooka 
    346  1.150     pooka 	percpu_init();
    347  1.160     pooka 	inittimecounter();
    348  1.160     pooka 	ntp_init();
    349  1.160     pooka 
    350  1.160     pooka 	rumpuser_gettime(&sec, &nsec, &error);
    351  1.160     pooka 	ts.tv_sec = sec;
    352  1.160     pooka 	ts.tv_nsec = nsec;
    353  1.160     pooka 	tc_setclock(&ts);
    354  1.150     pooka 
    355  1.143     pooka 	/* we are mostly go.  do per-cpu subsystem init */
    356  1.186     pooka 	for (i = 0; i < numcpu; i++) {
    357  1.143     pooka 		struct cpu_info *ci = cpu_lookup(i);
    358  1.143     pooka 
    359  1.186     pooka 		/* attach non-bootstrap CPUs */
    360  1.186     pooka 		if (i > 0) {
    361  1.186     pooka 			rump_cpu_attach(ci);
    362  1.186     pooka 			ncpu++;
    363  1.186     pooka 		}
    364  1.186     pooka 
    365  1.143     pooka 		callout_init_cpu(ci);
    366  1.143     pooka 		softint_init(ci);
    367  1.143     pooka 		xc_init_cpu(ci);
    368  1.143     pooka 		pool_cache_cpu_init(ci);
    369  1.143     pooka 		selsysinit(ci);
    370  1.150     pooka 		percpu_init_cpu(ci);
    371  1.182     pooka 
    372  1.216     pooka 		TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
    373  1.216     pooka 		__cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
    374  1.216     pooka 
    375  1.182     pooka 		aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i);
    376  1.143     pooka 	}
    377   1.43     pooka 
    378  1.191     pooka 	mksysctls();
    379   1.98     pooka 	kqueue_init();
    380   1.72     pooka 	iostat_init();
    381   1.43     pooka 	fd_sys_init();
    382   1.44        ad 	module_init();
    383   1.79     pooka 	devsw_init();
    384  1.139     pooka 	pipe_init();
    385  1.162     pooka 	resource_init();
    386  1.224     pooka 	procinit_sysctl();
    387    1.1     pooka 
    388  1.175     pooka 	/* start page baroness */
    389  1.175     pooka 	if (rump_threads) {
    390  1.175     pooka 		if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL,
    391  1.175     pooka 		    uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0)
    392  1.175     pooka 			panic("pagedaemon create failed");
    393  1.175     pooka 	} else
    394  1.175     pooka 		uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */
    395  1.175     pooka 
    396  1.175     pooka 	/* process dso's */
    397  1.155     pooka 	rumpuser_dl_bootstrap(add_linkedin_modules, rump_kernelfsym_load);
    398  1.155     pooka 
    399  1.179     pooka 	rump_component_init(RUMP_COMPONENT_KERN);
    400  1.179     pooka 
    401   1.75     pooka 	/* these do nothing if not present */
    402   1.75     pooka 	rump_vfs_init();
    403   1.75     pooka 	rump_net_init();
    404  1.112     pooka 	rump_dev_init();
    405  1.179     pooka 
    406  1.179     pooka 	rump_component_init(RUMP_COMPONENT_KERN_VFS);
    407  1.179     pooka 
    408  1.218     pooka 	/*
    409  1.218     pooka 	 * if we initialized the tty component above, the tyttymtx is
    410  1.218     pooka 	 * now initialized.  otherwise, we need to initialize it.
    411  1.218     pooka 	 */
    412  1.218     pooka 	if (!rump_ttycomponent)
    413  1.218     pooka 		mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM);
    414  1.218     pooka 
    415  1.112     pooka 	cold = 0;
    416   1.31     pooka 
    417   1.14     pooka 	/* aieeeedondest */
    418   1.54     pooka 	if (rump_threads) {
    419   1.54     pooka 		if (workqueue_create(&uvm.aiodone_queue, "aiodoned",
    420  1.110     pooka 		    rump_aiodone_worker, NULL, 0, 0, WQ_MPSAFE))
    421   1.54     pooka 			panic("aiodoned");
    422   1.54     pooka 	}
    423   1.14     pooka 
    424  1.115     pooka 	sysctl_finalize();
    425  1.115     pooka 
    426  1.155     pooka 	module_init_class(MODULE_CLASS_ANY);
    427  1.140     pooka 
    428    1.1     pooka 	rumpuser_gethostname(hostname, MAXHOSTNAMELEN, &error);
    429    1.1     pooka 	hostnamelen = strlen(hostname);
    430   1.24     pooka 
    431   1.24     pooka 	sigemptyset(&sigcantmask);
    432   1.27     pooka 
    433   1.89     pooka 	if (rump_threads)
    434   1.89     pooka 		vmem_rehash_start();
    435   1.89     pooka 
    436  1.193     pooka 	/*
    437  1.193     pooka 	 * Create init, used to attach implicit threads in rump.
    438  1.193     pooka 	 * (note: must be done after vfsinit to get cwdi)
    439  1.193     pooka 	 */
    440  1.193     pooka 	(void)rump__lwproc_alloclwp(NULL); /* dummy thread for initproc */
    441  1.193     pooka 	mutex_enter(proc_lock);
    442  1.193     pooka 	initproc = proc_find_raw(1);
    443  1.193     pooka 	mutex_exit(proc_lock);
    444  1.193     pooka 	if (initproc == NULL)
    445  1.193     pooka 		panic("where in the world is initproc?");
    446  1.193     pooka 
    447  1.213     pooka 	/*
    448  1.213     pooka 	 * Adjust syscall vector in case factions were dlopen()'d
    449  1.213     pooka 	 * before calling rump_init().
    450  1.213     pooka 	 * (modules will handle dynamic syscalls the usual way)
    451  1.213     pooka 	 *
    452  1.213     pooka 	 * Note: this will adjust the function vectors of
    453  1.213     pooka 	 * syscalls which use a funcalias (getpid etc.), but
    454  1.213     pooka 	 * it makes no difference.
    455  1.213     pooka 	 */
    456  1.213     pooka 	for (i = 0; i < SYS_NSYSENT; i++) {
    457  1.213     pooka 		void *sym;
    458  1.213     pooka 
    459  1.213     pooka 		if (rump_sysent[i].sy_flags & SYCALL_NOSYS ||
    460  1.213     pooka 		    *syscallnames[i] == '#' ||
    461  1.213     pooka 		    rump_sysent[i].sy_call == sys_nomodule)
    462  1.213     pooka 			continue;
    463  1.213     pooka 
    464  1.222     pooka 		/*
    465  1.222     pooka 		 * deal with compat wrappers.  makesyscalls.sh should
    466  1.222     pooka 		 * generate the necessary info instead of this hack,
    467  1.222     pooka 		 * though.  ugly, fix it later.
    468  1.222     pooka 		 */
    469  1.222     pooka #define CPFX "compat_"
    470  1.222     pooka #define CPFXLEN (sizeof(CPFX)-1)
    471  1.222     pooka 		if (strncmp(syscallnames[i], CPFX, CPFXLEN) == 0) {
    472  1.222     pooka 			const char *p = syscallnames[i] + CPFXLEN;
    473  1.222     pooka 			size_t namelen;
    474  1.222     pooka 
    475  1.222     pooka 			/* skip version number */
    476  1.222     pooka 			while (*p >= '0' && *p <= '9')
    477  1.222     pooka 				p++;
    478  1.222     pooka 			if (p == syscallnames[i] + CPFXLEN || *p != '_')
    479  1.222     pooka 				panic("invalid syscall name %s\n",
    480  1.222     pooka 				    syscallnames[i]);
    481  1.222     pooka 
    482  1.222     pooka 			/* skip over the next underscore */
    483  1.222     pooka 			p++;
    484  1.222     pooka 			namelen = p + (sizeof("rumpns_")-1) - syscallnames[i];
    485  1.222     pooka 
    486  1.222     pooka 			strcpy(buf, "rumpns_");
    487  1.222     pooka 			strcat(buf, syscallnames[i]);
    488  1.222     pooka 			/* XXX: no strncat in the kernel */
    489  1.222     pooka 			strcpy(buf+namelen, "sys_");
    490  1.222     pooka 			strcat(buf, p);
    491  1.222     pooka #undef CPFX
    492  1.222     pooka #undef CPFXLEN
    493  1.222     pooka 		} else {
    494  1.222     pooka 			sprintf(buf, "rumpns_sys_%s", syscallnames[i]);
    495  1.222     pooka 		}
    496  1.213     pooka 		if ((sym = rumpuser_dl_globalsym(buf)) != NULL
    497  1.213     pooka 		    && sym != rump_sysent[i].sy_call) {
    498  1.213     pooka #if 0
    499  1.213     pooka 			rumpuser_dprintf("adjusting %s: %p (old %p)\n",
    500  1.213     pooka 			    syscallnames[i], sym, rump_sysent[i].sy_call);
    501  1.213     pooka #endif
    502  1.213     pooka 			rump_sysent[i].sy_call = sym;
    503  1.213     pooka 		}
    504  1.213     pooka 	}
    505  1.213     pooka 
    506  1.193     pooka 	/* release cpu */
    507  1.124     pooka 	rump_unschedule();
    508  1.116     pooka 
    509    1.2     pooka 	return 0;
    510    1.2     pooka }
    511    1.2     pooka 
    512  1.207     pooka int
    513  1.207     pooka rump_init_server(const char *url)
    514  1.207     pooka {
    515  1.207     pooka 
    516  1.212     pooka 	return rumpuser_sp_init(url, &spops, ostype, osrelease, MACHINE);
    517  1.207     pooka }
    518  1.207     pooka 
    519  1.121     pooka void
    520  1.203     pooka cpu_reboot(int howto, char *bootstr)
    521  1.121     pooka {
    522  1.209     pooka 	int ruhow = 0;
    523  1.220     pooka 	void *finiarg;
    524  1.209     pooka 
    525  1.209     pooka 	printf("rump kernel halting...\n");
    526  1.220     pooka 
    527  1.220     pooka 	if (!RUMP_LOCALPROC_P(curproc))
    528  1.220     pooka 		finiarg = curproc->p_vmspace->vm_map.pmap;
    529  1.220     pooka 	else
    530  1.220     pooka 		finiarg = NULL;
    531  1.121     pooka 
    532  1.121     pooka 	/* dump means we really take the dive here */
    533  1.121     pooka 	if ((howto & RB_DUMP) || panicstr) {
    534  1.209     pooka 		ruhow = RUMPUSER_PANIC;
    535  1.209     pooka 		goto out;
    536  1.121     pooka 	}
    537  1.121     pooka 
    538  1.121     pooka 	/* try to sync */
    539  1.121     pooka 	if (!((howto & RB_NOSYNC) || panicstr)) {
    540  1.121     pooka 		rump_vfs_fini();
    541  1.121     pooka 	}
    542  1.121     pooka 
    543  1.121     pooka 	/* your wish is my command */
    544  1.121     pooka 	if (howto & RB_HALT) {
    545  1.203     pooka 		printf("rump kernel halted\n");
    546  1.227    bouyer 		rumpuser_sp_fini(finiarg);
    547  1.121     pooka 		for (;;) {
    548  1.121     pooka 			uint64_t sec = 5, nsec = 0;
    549  1.121     pooka 			int error;
    550  1.121     pooka 
    551  1.121     pooka 			rumpuser_nanosleep(&sec, &nsec, &error);
    552  1.121     pooka 		}
    553  1.121     pooka 	}
    554  1.203     pooka 
    555  1.203     pooka 	/* this function is __dead, we must exit */
    556  1.209     pooka  out:
    557  1.209     pooka 	printf("halted\n");
    558  1.227    bouyer 	rumpuser_sp_fini(finiarg);
    559  1.209     pooka 	rumpuser_exit(ruhow);
    560  1.121     pooka }
    561  1.121     pooka 
    562    1.8     pooka struct uio *
    563  1.123     pooka rump_uio_setup(void *buf, size_t bufsize, off_t offset, enum rump_uiorw rw)
    564    1.8     pooka {
    565    1.8     pooka 	struct uio *uio;
    566    1.8     pooka 	enum uio_rw uiorw;
    567    1.8     pooka 
    568    1.8     pooka 	switch (rw) {
    569    1.8     pooka 	case RUMPUIO_READ:
    570    1.8     pooka 		uiorw = UIO_READ;
    571    1.8     pooka 		break;
    572    1.8     pooka 	case RUMPUIO_WRITE:
    573    1.8     pooka 		uiorw = UIO_WRITE;
    574    1.8     pooka 		break;
    575   1.11     pooka 	default:
    576   1.11     pooka 		panic("%s: invalid rw %d", __func__, rw);
    577    1.8     pooka 	}
    578    1.8     pooka 
    579   1.28     pooka 	uio = kmem_alloc(sizeof(struct uio), KM_SLEEP);
    580   1.28     pooka 	uio->uio_iov = kmem_alloc(sizeof(struct iovec), KM_SLEEP);
    581    1.8     pooka 
    582    1.8     pooka 	uio->uio_iov->iov_base = buf;
    583    1.8     pooka 	uio->uio_iov->iov_len = bufsize;
    584    1.8     pooka 
    585    1.8     pooka 	uio->uio_iovcnt = 1;
    586    1.8     pooka 	uio->uio_offset = offset;
    587    1.8     pooka 	uio->uio_resid = bufsize;
    588    1.8     pooka 	uio->uio_rw = uiorw;
    589  1.198     pooka 	UIO_SETUP_SYSSPACE(uio);
    590    1.8     pooka 
    591    1.8     pooka 	return uio;
    592    1.8     pooka }
    593    1.8     pooka 
    594    1.8     pooka size_t
    595  1.123     pooka rump_uio_getresid(struct uio *uio)
    596    1.8     pooka {
    597    1.8     pooka 
    598    1.8     pooka 	return uio->uio_resid;
    599    1.8     pooka }
    600    1.8     pooka 
    601    1.8     pooka off_t
    602  1.123     pooka rump_uio_getoff(struct uio *uio)
    603    1.8     pooka {
    604    1.8     pooka 
    605    1.8     pooka 	return uio->uio_offset;
    606    1.8     pooka }
    607    1.8     pooka 
    608    1.8     pooka size_t
    609  1.123     pooka rump_uio_free(struct uio *uio)
    610    1.8     pooka {
    611    1.8     pooka 	size_t resid;
    612    1.8     pooka 
    613    1.8     pooka 	resid = uio->uio_resid;
    614   1.28     pooka 	kmem_free(uio->uio_iov, sizeof(*uio->uio_iov));
    615   1.28     pooka 	kmem_free(uio, sizeof(*uio));
    616    1.8     pooka 
    617    1.8     pooka 	return resid;
    618    1.8     pooka }
    619    1.8     pooka 
    620   1.59     pooka kauth_cred_t
    621  1.123     pooka rump_cred_create(uid_t uid, gid_t gid, size_t ngroups, gid_t *groups)
    622   1.59     pooka {
    623   1.59     pooka 	kauth_cred_t cred;
    624   1.59     pooka 	int rv;
    625   1.59     pooka 
    626   1.59     pooka 	cred = kauth_cred_alloc();
    627   1.59     pooka 	kauth_cred_setuid(cred, uid);
    628   1.59     pooka 	kauth_cred_seteuid(cred, uid);
    629   1.59     pooka 	kauth_cred_setsvuid(cred, uid);
    630   1.59     pooka 	kauth_cred_setgid(cred, gid);
    631   1.59     pooka 	kauth_cred_setgid(cred, gid);
    632   1.59     pooka 	kauth_cred_setegid(cred, gid);
    633   1.59     pooka 	kauth_cred_setsvgid(cred, gid);
    634   1.59     pooka 	rv = kauth_cred_setgroups(cred, groups, ngroups, 0, UIO_SYSSPACE);
    635   1.59     pooka 	/* oh this is silly.  and by "this" I mean kauth_cred_setgroups() */
    636   1.59     pooka 	assert(rv == 0);
    637   1.59     pooka 
    638   1.59     pooka 	return cred;
    639   1.59     pooka }
    640   1.59     pooka 
    641   1.59     pooka void
    642  1.123     pooka rump_cred_put(kauth_cred_t cred)
    643   1.59     pooka {
    644   1.59     pooka 
    645   1.59     pooka 	kauth_cred_free(cred);
    646   1.59     pooka }
    647   1.59     pooka 
    648  1.154     pooka static int compcounter[RUMP_COMPONENT_MAX];
    649  1.154     pooka 
    650  1.154     pooka static void
    651  1.154     pooka rump_component_init_cb(struct rump_component *rc, int type)
    652  1.154     pooka {
    653  1.154     pooka 
    654  1.154     pooka 	KASSERT(type < RUMP_COMPONENT_MAX);
    655  1.154     pooka 	if (rc->rc_type == type) {
    656  1.154     pooka 		rc->rc_init();
    657  1.154     pooka 		compcounter[type]++;
    658  1.154     pooka 	}
    659  1.154     pooka }
    660  1.154     pooka 
    661  1.154     pooka int
    662  1.154     pooka rump_component_count(enum rump_component_type type)
    663  1.154     pooka {
    664  1.154     pooka 
    665  1.154     pooka 	KASSERT(type <= RUMP_COMPONENT_MAX);
    666  1.154     pooka 	return compcounter[type];
    667  1.154     pooka }
    668  1.154     pooka 
    669  1.154     pooka void
    670  1.154     pooka rump_component_init(enum rump_component_type type)
    671  1.154     pooka {
    672  1.154     pooka 
    673  1.154     pooka 	rumpuser_dl_component_init(type, rump_component_init_cb);
    674  1.154     pooka }
    675  1.154     pooka 
    676  1.155     pooka /*
    677  1.155     pooka  * Initialize a module which has already been loaded and linked
    678  1.155     pooka  * with dlopen(). This is fundamentally the same as a builtin module.
    679  1.155     pooka  */
    680   1.76     pooka int
    681  1.155     pooka rump_module_init(const struct modinfo * const *mip, size_t nmodinfo)
    682   1.76     pooka {
    683   1.76     pooka 
    684  1.155     pooka 	return module_builtin_add(mip, nmodinfo, true);
    685  1.106     pooka }
    686  1.106     pooka 
    687  1.155     pooka /*
    688  1.155     pooka  * Finish module (flawless victory, fatality!).
    689  1.155     pooka  */
    690  1.106     pooka int
    691  1.155     pooka rump_module_fini(const struct modinfo *mi)
    692  1.106     pooka {
    693  1.120     pooka 
    694  1.155     pooka 	return module_builtin_remove(mi, true);
    695  1.155     pooka }
    696  1.155     pooka 
    697  1.155     pooka /*
    698  1.155     pooka  * Add loaded and linked module to the builtin list.  It will
    699  1.155     pooka  * later be initialized with module_init_class().
    700  1.155     pooka  */
    701  1.155     pooka 
    702  1.155     pooka static void
    703  1.155     pooka add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo)
    704  1.155     pooka {
    705  1.106     pooka 
    706  1.155     pooka 	module_builtin_add(mip, nmodinfo, false);
    707   1.76     pooka }
    708   1.76     pooka 
    709  1.137     pooka int
    710  1.137     pooka rump_kernelfsym_load(void *symtab, uint64_t symsize,
    711  1.137     pooka 	char *strtab, uint64_t strsize)
    712  1.137     pooka {
    713  1.137     pooka 	static int inited = 0;
    714  1.137     pooka 	Elf64_Ehdr ehdr;
    715  1.137     pooka 
    716  1.137     pooka 	if (inited)
    717  1.137     pooka 		return EBUSY;
    718  1.137     pooka 	inited = 1;
    719  1.137     pooka 
    720  1.137     pooka 	/*
    721  1.137     pooka 	 * Use 64bit header since it's bigger.  Shouldn't make a
    722  1.137     pooka 	 * difference, since we're passing in all zeroes anyway.
    723  1.137     pooka 	 */
    724  1.137     pooka 	memset(&ehdr, 0, sizeof(ehdr));
    725  1.137     pooka 	ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize);
    726  1.137     pooka 
    727  1.137     pooka 	return 0;
    728  1.137     pooka }
    729  1.137     pooka 
    730  1.199     pooka static int
    731  1.199     pooka rump_proxy_syscall(int num, void *arg, register_t *retval)
    732   1.95     pooka {
    733   1.95     pooka 	struct lwp *l;
    734   1.95     pooka 	struct sysent *callp;
    735  1.124     pooka 	int rv;
    736   1.95     pooka 
    737   1.95     pooka 	if (__predict_false(num >= SYS_NSYSENT))
    738   1.95     pooka 		return ENOSYS;
    739   1.95     pooka 
    740   1.95     pooka 	callp = rump_sysent + num;
    741  1.125     pooka 	l = curlwp;
    742  1.191     pooka 	rv = sy_call(callp, l, (void *)arg, retval);
    743  1.191     pooka 
    744  1.191     pooka 	return rv;
    745  1.191     pooka }
    746  1.191     pooka 
    747  1.206     pooka static int
    748  1.226     pooka rump_proxy_rfork(void *priv, int flags, const char *comm)
    749  1.206     pooka {
    750  1.206     pooka 	struct vmspace *newspace;
    751  1.226     pooka 	struct proc *p;
    752  1.206     pooka 	int error;
    753  1.206     pooka 
    754  1.214     pooka 	if ((error = rump_lwproc_rfork(flags)) != 0)
    755  1.206     pooka 		return error;
    756  1.206     pooka 
    757  1.206     pooka 	/*
    758  1.206     pooka 	 * Since it's a proxy proc, adjust the vmspace.
    759  1.206     pooka 	 * Refcount will eternally be 1.
    760  1.206     pooka 	 */
    761  1.226     pooka 	p = curproc;
    762  1.229     pooka 	newspace = kmem_zalloc(sizeof(*newspace), KM_SLEEP);
    763  1.206     pooka 	newspace->vm_refcnt = 1;
    764  1.206     pooka 	newspace->vm_map.pmap = priv;
    765  1.226     pooka 	KASSERT(p->p_vmspace == vmspace_kernel());
    766  1.226     pooka 	p->p_vmspace = newspace;
    767  1.226     pooka 	if (comm)
    768  1.226     pooka 		strlcpy(p->p_comm, comm, sizeof(p->p_comm));
    769  1.206     pooka 
    770  1.206     pooka 	return 0;
    771  1.206     pooka }
    772  1.206     pooka 
    773  1.232     pooka /*
    774  1.232     pooka  * Order all lwps in a process to exit.  does *not* wait for them to drain.
    775  1.232     pooka  */
    776  1.219     pooka static void
    777  1.232     pooka rump_proxy_lwpexit(void)
    778  1.219     pooka {
    779  1.219     pooka 	struct proc *p = curproc;
    780  1.219     pooka 	uint64_t where;
    781  1.219     pooka 	struct lwp *l;
    782  1.219     pooka 
    783  1.219     pooka 	mutex_enter(p->p_lock);
    784  1.219     pooka 	/*
    785  1.232     pooka 	 * First pass: mark all lwps in the process with LW_RUMP_QEXIT
    786  1.219     pooka 	 * so that they know they should exit.
    787  1.219     pooka 	 */
    788  1.219     pooka 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    789  1.219     pooka 		if (l == curlwp)
    790  1.219     pooka 			continue;
    791  1.232     pooka 		l->l_flag |= LW_RUMP_QEXIT;
    792  1.219     pooka 	}
    793  1.219     pooka 	mutex_exit(p->p_lock);
    794  1.219     pooka 
    795  1.219     pooka 	/*
    796  1.219     pooka 	 * Next, make sure everyone on all CPUs sees our status
    797  1.219     pooka 	 * update.  This keeps threads inside cv_wait() and makes
    798  1.219     pooka 	 * sure we don't access a stale cv pointer later when
    799  1.219     pooka 	 * we wake up the threads.
    800  1.219     pooka 	 */
    801  1.219     pooka 
    802  1.219     pooka 	where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
    803  1.219     pooka 	xc_wait(where);
    804  1.219     pooka 
    805  1.219     pooka 	/*
    806  1.219     pooka 	 * Ok, all lwps are either:
    807  1.219     pooka 	 *  1) not in the cv code
    808  1.219     pooka 	 *  2) sleeping on l->l_private
    809  1.219     pooka 	 *  3) sleeping on p->p_waitcv
    810  1.219     pooka 	 *
    811  1.232     pooka 	 * Either way, l_private is stable until we set PS_RUMP_LWPEXIT
    812  1.232     pooka 	 * in p->p_sflag.
    813  1.219     pooka 	 */
    814  1.219     pooka 
    815  1.219     pooka 	mutex_enter(p->p_lock);
    816  1.219     pooka 	LIST_FOREACH(l, &p->p_lwps, l_sibling) {
    817  1.219     pooka 		if (l->l_private)
    818  1.219     pooka 			cv_broadcast(l->l_private);
    819  1.219     pooka 	}
    820  1.232     pooka 	p->p_sflag |= PS_RUMP_LWPEXIT;
    821  1.219     pooka 	cv_broadcast(&p->p_waitcv);
    822  1.219     pooka 	mutex_exit(p->p_lock);
    823  1.232     pooka }
    824  1.219     pooka 
    825  1.232     pooka /*
    826  1.232     pooka  * Notify process that all threads have been drained and exec is complete.
    827  1.232     pooka  */
    828  1.232     pooka static void
    829  1.232     pooka rump_proxy_execnotify(const char *comm)
    830  1.232     pooka {
    831  1.232     pooka 	struct proc *p = curproc;
    832  1.232     pooka 
    833  1.232     pooka 	fd_closeexec();
    834  1.232     pooka 	mutex_enter(p->p_lock);
    835  1.232     pooka 	KASSERT(p->p_nlwps == 1 && p->p_sflag & PS_RUMP_LWPEXIT);
    836  1.232     pooka 	p->p_sflag &= ~PS_RUMP_LWPEXIT;
    837  1.232     pooka 	mutex_exit(p->p_lock);
    838  1.232     pooka 	strlcpy(p->p_comm, comm, sizeof(p->p_comm));
    839  1.219     pooka }
    840  1.219     pooka 
    841  1.191     pooka int
    842  1.130     pooka rump_boot_gethowto()
    843  1.130     pooka {
    844  1.130     pooka 
    845  1.130     pooka 	return boothowto;
    846  1.130     pooka }
    847  1.130     pooka 
    848  1.130     pooka void
    849  1.130     pooka rump_boot_sethowto(int howto)
    850  1.130     pooka {
    851  1.130     pooka 
    852  1.130     pooka 	boothowto = howto;
    853  1.130     pooka }
    854  1.130     pooka 
    855  1.109     pooka int
    856  1.124     pooka rump_getversion(void)
    857  1.109     pooka {
    858  1.109     pooka 
    859  1.109     pooka 	return __NetBSD_Version__;
    860  1.109     pooka }
    861  1.170     pooka 
    862  1.170     pooka /*
    863  1.170     pooka  * Note: may be called unscheduled.  Not fully safe since no locking
    864  1.170     pooka  * of allevents (currently that's not even available).
    865  1.170     pooka  */
    866  1.170     pooka void
    867  1.170     pooka rump_printevcnts()
    868  1.170     pooka {
    869  1.170     pooka 	struct evcnt *ev;
    870  1.170     pooka 
    871  1.170     pooka 	TAILQ_FOREACH(ev, &allevents, ev_list)
    872  1.170     pooka 		rumpuser_dprintf("%s / %s: %" PRIu64 "\n",
    873  1.170     pooka 		    ev->ev_group, ev->ev_name, ev->ev_count);
    874  1.170     pooka }
    875  1.184     pooka 
    876  1.184     pooka /*
    877  1.184     pooka  * If you use this interface ... well ... all bets are off.
    878  1.184     pooka  * The original purpose is for the p2k fs server library to be
    879  1.184     pooka  * able to use the same pid/lid for VOPs as the host kernel.
    880  1.184     pooka  */
    881  1.184     pooka void
    882  1.184     pooka rump_allbetsareoff_setid(pid_t pid, int lid)
    883  1.184     pooka {
    884  1.184     pooka 	struct lwp *l = curlwp;
    885  1.184     pooka 	struct proc *p = l->l_proc;
    886  1.184     pooka 
    887  1.184     pooka 	l->l_lid = lid;
    888  1.184     pooka 	p->p_pid = pid;
    889  1.184     pooka }
    890