rump.c revision 1.235 1 1.235 tls /* $NetBSD: rump.c,v 1.235 2011/11/19 22:51:30 tls Exp $ */
2 1.1 pooka
3 1.1 pooka /*
4 1.233 pooka * Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved.
5 1.1 pooka *
6 1.1 pooka * Redistribution and use in source and binary forms, with or without
7 1.1 pooka * modification, are permitted provided that the following conditions
8 1.1 pooka * are met:
9 1.1 pooka * 1. Redistributions of source code must retain the above copyright
10 1.1 pooka * notice, this list of conditions and the following disclaimer.
11 1.1 pooka * 2. Redistributions in binary form must reproduce the above copyright
12 1.1 pooka * notice, this list of conditions and the following disclaimer in the
13 1.1 pooka * documentation and/or other materials provided with the distribution.
14 1.1 pooka *
15 1.1 pooka * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
16 1.1 pooka * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 1.1 pooka * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 1.1 pooka * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 1.1 pooka * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 1.1 pooka * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 1.1 pooka * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 1.1 pooka * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 1.1 pooka * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 1.1 pooka * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 1.1 pooka * SUCH DAMAGE.
26 1.1 pooka */
27 1.1 pooka
28 1.78 pooka #include <sys/cdefs.h>
29 1.235 tls __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.235 2011/11/19 22:51:30 tls Exp $");
30 1.165 pooka
31 1.165 pooka #include <sys/systm.h>
32 1.165 pooka #define ELFSIZE ARCH_ELFSIZE
33 1.78 pooka
34 1.1 pooka #include <sys/param.h>
35 1.64 pooka #include <sys/atomic.h>
36 1.75 pooka #include <sys/buf.h>
37 1.66 pooka #include <sys/callout.h>
38 1.79 pooka #include <sys/conf.h>
39 1.27 pooka #include <sys/cpu.h>
40 1.144 pooka #include <sys/device.h>
41 1.100 pooka #include <sys/evcnt.h>
42 1.98 pooka #include <sys/event.h>
43 1.137 pooka #include <sys/exec_elf.h>
44 1.1 pooka #include <sys/filedesc.h>
45 1.72 pooka #include <sys/iostat.h>
46 1.1 pooka #include <sys/kauth.h>
47 1.80 pooka #include <sys/kernel.h>
48 1.14 pooka #include <sys/kmem.h>
49 1.81 pooka #include <sys/kprintf.h>
50 1.175 pooka #include <sys/kthread.h>
51 1.82 pooka #include <sys/ksyms.h>
52 1.81 pooka #include <sys/msgbuf.h>
53 1.49 simonb #include <sys/module.h>
54 1.234 pooka #include <sys/namei.h>
55 1.71 pooka #include <sys/once.h>
56 1.65 pooka #include <sys/percpu.h>
57 1.139 pooka #include <sys/pipe.h>
58 1.162 pooka #include <sys/pool.h>
59 1.1 pooka #include <sys/queue.h>
60 1.121 pooka #include <sys/reboot.h>
61 1.1 pooka #include <sys/resourcevar.h>
62 1.27 pooka #include <sys/select.h>
63 1.87 pooka #include <sys/sysctl.h>
64 1.96 pooka #include <sys/syscall.h>
65 1.169 pooka #include <sys/syscallvar.h>
66 1.160 pooka #include <sys/timetc.h>
67 1.87 pooka #include <sys/tty.h>
68 1.68 pooka #include <sys/uidinfo.h>
69 1.89 pooka #include <sys/vmem.h>
70 1.143 pooka #include <sys/xcall.h>
71 1.216 pooka #include <sys/simplelock.h>
72 1.235 tls #include <sys/cprng.h>
73 1.1 pooka
74 1.48 pooka #include <rump/rumpuser.h>
75 1.48 pooka
76 1.117 elad #include <secmodel/suser/suser.h>
77 1.101 pooka
78 1.114 pooka #include <prop/proplib.h>
79 1.114 pooka
80 1.174 pooka #include <uvm/uvm_extern.h>
81 1.136 pooka #include <uvm/uvm_readahead.h>
82 1.136 pooka
83 1.8 pooka #include "rump_private.h"
84 1.71 pooka #include "rump_net_private.h"
85 1.75 pooka #include "rump_vfs_private.h"
86 1.112 pooka #include "rump_dev_private.h"
87 1.1 pooka
88 1.164 pooka char machine[] = MACHINE;
89 1.1 pooka
90 1.193 pooka struct proc *initproc;
91 1.101 pooka
92 1.144 pooka struct device rump_rootdev = {
93 1.144 pooka .dv_class = DV_VIRTUAL
94 1.144 pooka };
95 1.144 pooka
96 1.54 pooka #ifdef RUMP_WITHOUT_THREADS
97 1.54 pooka int rump_threads = 0;
98 1.54 pooka #else
99 1.54 pooka int rump_threads = 1;
100 1.54 pooka #endif
101 1.54 pooka
102 1.199 pooka static int rump_proxy_syscall(int, void *, register_t *);
103 1.226 pooka static int rump_proxy_rfork(void *, int, const char *);
104 1.232 pooka static void rump_proxy_lwpexit(void);
105 1.230 pooka static void rump_proxy_execnotify(const char *);
106 1.199 pooka
107 1.156 pooka static char rump_msgbuf[16*1024]; /* 16k should be enough for std rump needs */
108 1.156 pooka
109 1.217 pooka #ifdef LOCKDEBUG
110 1.217 pooka const int rump_lockdebug = 1;
111 1.217 pooka #else
112 1.217 pooka const int rump_lockdebug = 0;
113 1.217 pooka #endif
114 1.218 pooka bool rump_ttycomponent = false;
115 1.217 pooka
116 1.14 pooka static void
117 1.14 pooka rump_aiodone_worker(struct work *wk, void *dummy)
118 1.14 pooka {
119 1.14 pooka struct buf *bp = (struct buf *)wk;
120 1.14 pooka
121 1.14 pooka KASSERT(&bp->b_work == wk);
122 1.14 pooka bp->b_iodone(bp);
123 1.14 pooka }
124 1.14 pooka
125 1.51 pooka static int rump_inited;
126 1.27 pooka
127 1.162 pooka /*
128 1.162 pooka * Make sure pnbuf_cache is available even without vfs
129 1.162 pooka */
130 1.162 pooka int rump_initpnbufpool(void);
131 1.162 pooka int rump_initpnbufpool(void)
132 1.162 pooka {
133 1.162 pooka
134 1.162 pooka pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl",
135 1.162 pooka NULL, IPL_NONE, NULL, NULL, NULL);
136 1.162 pooka return EOPNOTSUPP;
137 1.162 pooka }
138 1.162 pooka
139 1.126 pooka int rump__unavailable(void);
140 1.126 pooka int rump__unavailable() {return EOPNOTSUPP;}
141 1.86 pooka __weak_alias(rump_net_init,rump__unavailable);
142 1.162 pooka __weak_alias(rump_vfs_init,rump_initpnbufpool);
143 1.112 pooka __weak_alias(rump_dev_init,rump__unavailable);
144 1.75 pooka
145 1.121 pooka __weak_alias(rump_vfs_fini,rump__unavailable);
146 1.121 pooka
147 1.113 pooka __weak_alias(biodone,rump__unavailable);
148 1.126 pooka __weak_alias(sopoll,rump__unavailable);
149 1.113 pooka
150 1.188 pooka __weak_alias(rump_vfs_drainbufs,rump__unavailable);
151 1.188 pooka
152 1.92 pooka void rump__unavailable_vfs_panic(void);
153 1.92 pooka void rump__unavailable_vfs_panic() {panic("vfs component not available");}
154 1.132 pooka __weak_alias(usermount_common_policy,rump__unavailable_vfs_panic);
155 1.92 pooka
156 1.211 pooka /* easier to write vfs-less clients */
157 1.211 pooka __weak_alias(rump_pub_etfs_register,rump__unavailable);
158 1.228 pooka __weak_alias(rump_pub_etfs_register_withsize,rump__unavailable);
159 1.211 pooka __weak_alias(rump_pub_etfs_remove,rump__unavailable);
160 1.211 pooka
161 1.147 pooka rump_proc_vfs_init_fn rump_proc_vfs_init;
162 1.147 pooka rump_proc_vfs_release_fn rump_proc_vfs_release;
163 1.71 pooka
164 1.155 pooka static void add_linkedin_modules(const struct modinfo *const *, size_t);
165 1.155 pooka
166 1.151 pooka static void __noinline
167 1.105 pooka messthestack(void)
168 1.105 pooka {
169 1.151 pooka volatile uint32_t mess[64];
170 1.105 pooka uint64_t d1, d2;
171 1.105 pooka int i, error;
172 1.105 pooka
173 1.105 pooka for (i = 0; i < 64; i++) {
174 1.105 pooka rumpuser_gettime(&d1, &d2, &error);
175 1.105 pooka mess[i] = d2;
176 1.105 pooka }
177 1.105 pooka }
178 1.105 pooka
179 1.158 pooka /*
180 1.158 pooka * Create kern.hostname. why only this you ask. well, init_sysctl
181 1.158 pooka * is a kitchen sink in need of some gardening. but i want to use
182 1.158 pooka * kern.hostname today.
183 1.158 pooka */
184 1.158 pooka static void
185 1.158 pooka mksysctls(void)
186 1.158 pooka {
187 1.158 pooka
188 1.158 pooka sysctl_createv(NULL, 0, NULL, NULL,
189 1.158 pooka CTLFLAG_PERMANENT, CTLTYPE_NODE, "kern", NULL,
190 1.158 pooka NULL, 0, NULL, 0, CTL_KERN, CTL_EOL);
191 1.158 pooka
192 1.158 pooka /* XXX: setting hostnamelen is missing */
193 1.158 pooka sysctl_createv(NULL, 0, NULL, NULL,
194 1.158 pooka CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRING, "hostname",
195 1.158 pooka SYSCTL_DESCR("System hostname"), NULL, 0,
196 1.158 pooka &hostname, MAXHOSTNAMELEN, CTL_KERN, KERN_HOSTNAME, CTL_EOL);
197 1.158 pooka }
198 1.158 pooka
199 1.201 pooka /* there's no convenient kernel entry point for this, so just craft out own */
200 1.201 pooka static pid_t
201 1.201 pooka spgetpid(void)
202 1.201 pooka {
203 1.201 pooka
204 1.201 pooka return curproc->p_pid;
205 1.201 pooka }
206 1.201 pooka
207 1.197 pooka static const struct rumpuser_sp_ops spops = {
208 1.197 pooka .spop_schedule = rump_schedule,
209 1.197 pooka .spop_unschedule = rump_unschedule,
210 1.197 pooka .spop_lwproc_switch = rump_lwproc_switch,
211 1.197 pooka .spop_lwproc_release = rump_lwproc_releaselwp,
212 1.214 pooka .spop_lwproc_rfork = rump_proxy_rfork,
213 1.201 pooka .spop_lwproc_newlwp = rump_lwproc_newlwp,
214 1.197 pooka .spop_lwproc_curlwp = rump_lwproc_curlwp,
215 1.232 pooka .spop_lwpexit = rump_proxy_lwpexit,
216 1.199 pooka .spop_syscall = rump_proxy_syscall,
217 1.230 pooka .spop_execnotify = rump_proxy_execnotify,
218 1.201 pooka .spop_getpid = spgetpid,
219 1.197 pooka };
220 1.197 pooka
221 1.61 pooka int
222 1.207 pooka rump_daemonize_begin(void)
223 1.207 pooka {
224 1.207 pooka
225 1.207 pooka if (rump_inited)
226 1.207 pooka return EALREADY;
227 1.207 pooka
228 1.207 pooka return rumpuser_daemonize_begin();
229 1.207 pooka }
230 1.207 pooka
231 1.207 pooka int
232 1.207 pooka rump_daemonize_done(int error)
233 1.207 pooka {
234 1.207 pooka
235 1.207 pooka return rumpuser_daemonize_done(error);
236 1.207 pooka }
237 1.207 pooka
238 1.207 pooka int
239 1.83 pooka rump__init(int rump_version)
240 1.1 pooka {
241 1.36 pooka char buf[256];
242 1.160 pooka struct timespec ts;
243 1.157 pooka uint64_t sec, nsec;
244 1.14 pooka struct lwp *l;
245 1.166 pooka int i, numcpu;
246 1.1 pooka int error;
247 1.1 pooka
248 1.121 pooka /* not reentrant */
249 1.27 pooka if (rump_inited)
250 1.61 pooka return 0;
251 1.121 pooka else if (rump_inited == -1)
252 1.121 pooka panic("rump_init: host process restart required");
253 1.121 pooka else
254 1.121 pooka rump_inited = 1;
255 1.27 pooka
256 1.177 pooka if (rumpuser_getversion() != RUMPUSER_VERSION) {
257 1.177 pooka /* let's hope the ABI of rumpuser_dprintf is the same ;) */
258 1.177 pooka rumpuser_dprintf("rumpuser version mismatch: %d vs. %d\n",
259 1.177 pooka rumpuser_getversion(), RUMPUSER_VERSION);
260 1.177 pooka return EPROGMISMATCH;
261 1.177 pooka }
262 1.177 pooka
263 1.149 pooka if (rumpuser_getenv("RUMP_VERBOSE", buf, sizeof(buf), &error) == 0) {
264 1.149 pooka if (*buf != '0')
265 1.149 pooka boothowto = AB_VERBOSE;
266 1.149 pooka }
267 1.149 pooka
268 1.167 pooka if (rumpuser_getenv("RUMP_NCPU", buf, sizeof(buf), &error) == 0)
269 1.167 pooka error = 0;
270 1.167 pooka if (error == 0) {
271 1.167 pooka numcpu = strtoll(buf, NULL, 10);
272 1.167 pooka if (numcpu < 1)
273 1.167 pooka numcpu = 1;
274 1.167 pooka } else {
275 1.167 pooka numcpu = rumpuser_getnhostcpu();
276 1.167 pooka }
277 1.204 pooka rump_cpus_bootstrap(&numcpu);
278 1.165 pooka
279 1.157 pooka rumpuser_gettime(&sec, &nsec, &error);
280 1.157 pooka boottime.tv_sec = sec;
281 1.157 pooka boottime.tv_nsec = nsec;
282 1.157 pooka
283 1.156 pooka initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf));
284 1.156 pooka aprint_verbose("%s%s", copyright, version);
285 1.148 pooka
286 1.105 pooka /*
287 1.105 pooka * Seed arc4random() with a "reasonable" amount of randomness.
288 1.105 pooka * Yes, this is a quick kludge which depends on the arc4random
289 1.105 pooka * implementation.
290 1.105 pooka */
291 1.105 pooka messthestack();
292 1.235 tls cprng_fast32();
293 1.105 pooka
294 1.61 pooka if (rump_version != RUMP_VERSION) {
295 1.61 pooka printf("rump version mismatch, %d vs. %d\n",
296 1.61 pooka rump_version, RUMP_VERSION);
297 1.61 pooka return EPROGMISMATCH;
298 1.61 pooka }
299 1.61 pooka
300 1.54 pooka if (rumpuser_getenv("RUMP_THREADS", buf, sizeof(buf), &error) == 0) {
301 1.54 pooka rump_threads = *buf != '0';
302 1.54 pooka }
303 1.124 pooka rumpuser_thrinit(rump_user_schedule, rump_user_unschedule,
304 1.124 pooka rump_threads);
305 1.187 pooka rump_intr_init(numcpu);
306 1.173 pooka rump_tsleep_init();
307 1.36 pooka
308 1.131 pooka /* init minimal lwp/cpu context */
309 1.131 pooka l = &lwp0;
310 1.131 pooka l->l_lid = 1;
311 1.172 pooka l->l_cpu = l->l_target_cpu = rump_cpu;
312 1.178 pooka l->l_fd = &filedesc0;
313 1.131 pooka rumpuser_set_curlwp(l);
314 1.131 pooka
315 1.208 pooka rumpuser_mutex_init(&rump_giantlock);
316 1.82 pooka ksyms_init();
317 1.174 pooka uvm_init();
318 1.100 pooka evcnt_init();
319 1.89 pooka
320 1.89 pooka once_init();
321 1.181 pgoyette kernconfig_lock_init();
322 1.114 pooka prop_kern_init();
323 1.63 pooka
324 1.90 pooka pool_subsystem_init();
325 1.52 pooka kmem_init();
326 1.102 pooka
327 1.136 pooka uvm_ra_init();
328 1.185 pooka uao_init();
329 1.136 pooka
330 1.133 pooka mutex_obj_init();
331 1.143 pooka callout_startup();
332 1.133 pooka
333 1.81 pooka kprintf_init();
334 1.81 pooka loginit();
335 1.52 pooka
336 1.59 pooka kauth_init();
337 1.59 pooka
338 1.184 pooka procinit();
339 1.184 pooka proc0_init();
340 1.192 pooka uid_init();
341 1.192 pooka chgproccnt(0, 1);
342 1.184 pooka
343 1.178 pooka l->l_proc = &proc0;
344 1.184 pooka lwp_update_creds(l);
345 1.176 pooka
346 1.159 pooka lwpinit_specificdata();
347 1.176 pooka lwp_initspecific(&lwp0);
348 1.36 pooka
349 1.186 pooka rump_scheduler_init(numcpu);
350 1.193 pooka /* revert temporary context and schedule a semireal context */
351 1.131 pooka rumpuser_set_curlwp(NULL);
352 1.193 pooka initproc = &proc0; /* borrow proc0 before we get initproc started */
353 1.124 pooka rump_schedule();
354 1.124 pooka
355 1.150 pooka percpu_init();
356 1.160 pooka inittimecounter();
357 1.160 pooka ntp_init();
358 1.160 pooka
359 1.160 pooka rumpuser_gettime(&sec, &nsec, &error);
360 1.160 pooka ts.tv_sec = sec;
361 1.160 pooka ts.tv_nsec = nsec;
362 1.160 pooka tc_setclock(&ts);
363 1.150 pooka
364 1.143 pooka /* we are mostly go. do per-cpu subsystem init */
365 1.186 pooka for (i = 0; i < numcpu; i++) {
366 1.143 pooka struct cpu_info *ci = cpu_lookup(i);
367 1.143 pooka
368 1.186 pooka /* attach non-bootstrap CPUs */
369 1.186 pooka if (i > 0) {
370 1.186 pooka rump_cpu_attach(ci);
371 1.186 pooka ncpu++;
372 1.186 pooka }
373 1.186 pooka
374 1.143 pooka callout_init_cpu(ci);
375 1.143 pooka softint_init(ci);
376 1.143 pooka xc_init_cpu(ci);
377 1.143 pooka pool_cache_cpu_init(ci);
378 1.143 pooka selsysinit(ci);
379 1.150 pooka percpu_init_cpu(ci);
380 1.182 pooka
381 1.216 pooka TAILQ_INIT(&ci->ci_data.cpu_ld_locks);
382 1.216 pooka __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock);
383 1.216 pooka
384 1.182 pooka aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i);
385 1.143 pooka }
386 1.43 pooka
387 1.115 pooka sysctl_init();
388 1.191 pooka mksysctls();
389 1.98 pooka kqueue_init();
390 1.72 pooka iostat_init();
391 1.43 pooka fd_sys_init();
392 1.44 ad module_init();
393 1.79 pooka devsw_init();
394 1.139 pooka pipe_init();
395 1.162 pooka resource_init();
396 1.224 pooka procinit_sysctl();
397 1.1 pooka
398 1.175 pooka /* start page baroness */
399 1.175 pooka if (rump_threads) {
400 1.175 pooka if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL,
401 1.175 pooka uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0)
402 1.175 pooka panic("pagedaemon create failed");
403 1.175 pooka } else
404 1.175 pooka uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */
405 1.175 pooka
406 1.175 pooka /* process dso's */
407 1.155 pooka rumpuser_dl_bootstrap(add_linkedin_modules, rump_kernelfsym_load);
408 1.155 pooka
409 1.179 pooka rump_component_init(RUMP_COMPONENT_KERN);
410 1.179 pooka
411 1.75 pooka /* these do nothing if not present */
412 1.75 pooka rump_vfs_init();
413 1.75 pooka rump_net_init();
414 1.112 pooka rump_dev_init();
415 1.179 pooka
416 1.179 pooka rump_component_init(RUMP_COMPONENT_KERN_VFS);
417 1.179 pooka
418 1.218 pooka /*
419 1.218 pooka * if we initialized the tty component above, the tyttymtx is
420 1.218 pooka * now initialized. otherwise, we need to initialize it.
421 1.218 pooka */
422 1.218 pooka if (!rump_ttycomponent)
423 1.218 pooka mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM);
424 1.218 pooka
425 1.112 pooka cold = 0;
426 1.31 pooka
427 1.14 pooka /* aieeeedondest */
428 1.54 pooka if (rump_threads) {
429 1.54 pooka if (workqueue_create(&uvm.aiodone_queue, "aiodoned",
430 1.110 pooka rump_aiodone_worker, NULL, 0, 0, WQ_MPSAFE))
431 1.54 pooka panic("aiodoned");
432 1.54 pooka }
433 1.14 pooka
434 1.115 pooka sysctl_finalize();
435 1.115 pooka
436 1.155 pooka module_init_class(MODULE_CLASS_ANY);
437 1.140 pooka
438 1.1 pooka rumpuser_gethostname(hostname, MAXHOSTNAMELEN, &error);
439 1.1 pooka hostnamelen = strlen(hostname);
440 1.24 pooka
441 1.24 pooka sigemptyset(&sigcantmask);
442 1.27 pooka
443 1.89 pooka if (rump_threads)
444 1.89 pooka vmem_rehash_start();
445 1.89 pooka
446 1.193 pooka /*
447 1.193 pooka * Create init, used to attach implicit threads in rump.
448 1.193 pooka * (note: must be done after vfsinit to get cwdi)
449 1.193 pooka */
450 1.193 pooka (void)rump__lwproc_alloclwp(NULL); /* dummy thread for initproc */
451 1.193 pooka mutex_enter(proc_lock);
452 1.193 pooka initproc = proc_find_raw(1);
453 1.193 pooka mutex_exit(proc_lock);
454 1.193 pooka if (initproc == NULL)
455 1.193 pooka panic("where in the world is initproc?");
456 1.193 pooka
457 1.213 pooka /*
458 1.213 pooka * Adjust syscall vector in case factions were dlopen()'d
459 1.213 pooka * before calling rump_init().
460 1.213 pooka * (modules will handle dynamic syscalls the usual way)
461 1.213 pooka *
462 1.213 pooka * Note: this will adjust the function vectors of
463 1.213 pooka * syscalls which use a funcalias (getpid etc.), but
464 1.213 pooka * it makes no difference.
465 1.213 pooka */
466 1.213 pooka for (i = 0; i < SYS_NSYSENT; i++) {
467 1.213 pooka void *sym;
468 1.213 pooka
469 1.213 pooka if (rump_sysent[i].sy_flags & SYCALL_NOSYS ||
470 1.213 pooka *syscallnames[i] == '#' ||
471 1.213 pooka rump_sysent[i].sy_call == sys_nomodule)
472 1.213 pooka continue;
473 1.213 pooka
474 1.222 pooka /*
475 1.222 pooka * deal with compat wrappers. makesyscalls.sh should
476 1.222 pooka * generate the necessary info instead of this hack,
477 1.222 pooka * though. ugly, fix it later.
478 1.222 pooka */
479 1.222 pooka #define CPFX "compat_"
480 1.222 pooka #define CPFXLEN (sizeof(CPFX)-1)
481 1.222 pooka if (strncmp(syscallnames[i], CPFX, CPFXLEN) == 0) {
482 1.222 pooka const char *p = syscallnames[i] + CPFXLEN;
483 1.222 pooka size_t namelen;
484 1.222 pooka
485 1.222 pooka /* skip version number */
486 1.222 pooka while (*p >= '0' && *p <= '9')
487 1.222 pooka p++;
488 1.222 pooka if (p == syscallnames[i] + CPFXLEN || *p != '_')
489 1.222 pooka panic("invalid syscall name %s\n",
490 1.222 pooka syscallnames[i]);
491 1.222 pooka
492 1.222 pooka /* skip over the next underscore */
493 1.222 pooka p++;
494 1.222 pooka namelen = p + (sizeof("rumpns_")-1) - syscallnames[i];
495 1.222 pooka
496 1.222 pooka strcpy(buf, "rumpns_");
497 1.222 pooka strcat(buf, syscallnames[i]);
498 1.222 pooka /* XXX: no strncat in the kernel */
499 1.222 pooka strcpy(buf+namelen, "sys_");
500 1.222 pooka strcat(buf, p);
501 1.222 pooka #undef CPFX
502 1.222 pooka #undef CPFXLEN
503 1.222 pooka } else {
504 1.222 pooka sprintf(buf, "rumpns_sys_%s", syscallnames[i]);
505 1.222 pooka }
506 1.213 pooka if ((sym = rumpuser_dl_globalsym(buf)) != NULL
507 1.213 pooka && sym != rump_sysent[i].sy_call) {
508 1.213 pooka #if 0
509 1.213 pooka rumpuser_dprintf("adjusting %s: %p (old %p)\n",
510 1.213 pooka syscallnames[i], sym, rump_sysent[i].sy_call);
511 1.213 pooka #endif
512 1.213 pooka rump_sysent[i].sy_call = sym;
513 1.213 pooka }
514 1.213 pooka }
515 1.213 pooka
516 1.193 pooka /* release cpu */
517 1.124 pooka rump_unschedule();
518 1.116 pooka
519 1.2 pooka return 0;
520 1.2 pooka }
521 1.2 pooka
522 1.207 pooka int
523 1.207 pooka rump_init_server(const char *url)
524 1.207 pooka {
525 1.207 pooka
526 1.212 pooka return rumpuser_sp_init(url, &spops, ostype, osrelease, MACHINE);
527 1.207 pooka }
528 1.207 pooka
529 1.121 pooka void
530 1.203 pooka cpu_reboot(int howto, char *bootstr)
531 1.121 pooka {
532 1.209 pooka int ruhow = 0;
533 1.220 pooka void *finiarg;
534 1.209 pooka
535 1.209 pooka printf("rump kernel halting...\n");
536 1.220 pooka
537 1.220 pooka if (!RUMP_LOCALPROC_P(curproc))
538 1.220 pooka finiarg = curproc->p_vmspace->vm_map.pmap;
539 1.220 pooka else
540 1.220 pooka finiarg = NULL;
541 1.121 pooka
542 1.121 pooka /* dump means we really take the dive here */
543 1.121 pooka if ((howto & RB_DUMP) || panicstr) {
544 1.209 pooka ruhow = RUMPUSER_PANIC;
545 1.209 pooka goto out;
546 1.121 pooka }
547 1.121 pooka
548 1.121 pooka /* try to sync */
549 1.121 pooka if (!((howto & RB_NOSYNC) || panicstr)) {
550 1.121 pooka rump_vfs_fini();
551 1.121 pooka }
552 1.121 pooka
553 1.121 pooka /* your wish is my command */
554 1.121 pooka if (howto & RB_HALT) {
555 1.203 pooka printf("rump kernel halted\n");
556 1.227 bouyer rumpuser_sp_fini(finiarg);
557 1.121 pooka for (;;) {
558 1.121 pooka uint64_t sec = 5, nsec = 0;
559 1.121 pooka int error;
560 1.121 pooka
561 1.121 pooka rumpuser_nanosleep(&sec, &nsec, &error);
562 1.121 pooka }
563 1.121 pooka }
564 1.203 pooka
565 1.203 pooka /* this function is __dead, we must exit */
566 1.209 pooka out:
567 1.209 pooka printf("halted\n");
568 1.227 bouyer rumpuser_sp_fini(finiarg);
569 1.209 pooka rumpuser_exit(ruhow);
570 1.121 pooka }
571 1.121 pooka
572 1.8 pooka struct uio *
573 1.123 pooka rump_uio_setup(void *buf, size_t bufsize, off_t offset, enum rump_uiorw rw)
574 1.8 pooka {
575 1.8 pooka struct uio *uio;
576 1.8 pooka enum uio_rw uiorw;
577 1.8 pooka
578 1.8 pooka switch (rw) {
579 1.8 pooka case RUMPUIO_READ:
580 1.8 pooka uiorw = UIO_READ;
581 1.8 pooka break;
582 1.8 pooka case RUMPUIO_WRITE:
583 1.8 pooka uiorw = UIO_WRITE;
584 1.8 pooka break;
585 1.11 pooka default:
586 1.11 pooka panic("%s: invalid rw %d", __func__, rw);
587 1.8 pooka }
588 1.8 pooka
589 1.28 pooka uio = kmem_alloc(sizeof(struct uio), KM_SLEEP);
590 1.28 pooka uio->uio_iov = kmem_alloc(sizeof(struct iovec), KM_SLEEP);
591 1.8 pooka
592 1.8 pooka uio->uio_iov->iov_base = buf;
593 1.8 pooka uio->uio_iov->iov_len = bufsize;
594 1.8 pooka
595 1.8 pooka uio->uio_iovcnt = 1;
596 1.8 pooka uio->uio_offset = offset;
597 1.8 pooka uio->uio_resid = bufsize;
598 1.8 pooka uio->uio_rw = uiorw;
599 1.198 pooka UIO_SETUP_SYSSPACE(uio);
600 1.8 pooka
601 1.8 pooka return uio;
602 1.8 pooka }
603 1.8 pooka
604 1.8 pooka size_t
605 1.123 pooka rump_uio_getresid(struct uio *uio)
606 1.8 pooka {
607 1.8 pooka
608 1.8 pooka return uio->uio_resid;
609 1.8 pooka }
610 1.8 pooka
611 1.8 pooka off_t
612 1.123 pooka rump_uio_getoff(struct uio *uio)
613 1.8 pooka {
614 1.8 pooka
615 1.8 pooka return uio->uio_offset;
616 1.8 pooka }
617 1.8 pooka
618 1.8 pooka size_t
619 1.123 pooka rump_uio_free(struct uio *uio)
620 1.8 pooka {
621 1.8 pooka size_t resid;
622 1.8 pooka
623 1.8 pooka resid = uio->uio_resid;
624 1.28 pooka kmem_free(uio->uio_iov, sizeof(*uio->uio_iov));
625 1.28 pooka kmem_free(uio, sizeof(*uio));
626 1.8 pooka
627 1.8 pooka return resid;
628 1.8 pooka }
629 1.8 pooka
630 1.59 pooka kauth_cred_t
631 1.123 pooka rump_cred_create(uid_t uid, gid_t gid, size_t ngroups, gid_t *groups)
632 1.59 pooka {
633 1.59 pooka kauth_cred_t cred;
634 1.59 pooka int rv;
635 1.59 pooka
636 1.59 pooka cred = kauth_cred_alloc();
637 1.59 pooka kauth_cred_setuid(cred, uid);
638 1.59 pooka kauth_cred_seteuid(cred, uid);
639 1.59 pooka kauth_cred_setsvuid(cred, uid);
640 1.59 pooka kauth_cred_setgid(cred, gid);
641 1.59 pooka kauth_cred_setgid(cred, gid);
642 1.59 pooka kauth_cred_setegid(cred, gid);
643 1.59 pooka kauth_cred_setsvgid(cred, gid);
644 1.59 pooka rv = kauth_cred_setgroups(cred, groups, ngroups, 0, UIO_SYSSPACE);
645 1.59 pooka /* oh this is silly. and by "this" I mean kauth_cred_setgroups() */
646 1.59 pooka assert(rv == 0);
647 1.59 pooka
648 1.59 pooka return cred;
649 1.59 pooka }
650 1.59 pooka
651 1.59 pooka void
652 1.123 pooka rump_cred_put(kauth_cred_t cred)
653 1.59 pooka {
654 1.59 pooka
655 1.59 pooka kauth_cred_free(cred);
656 1.59 pooka }
657 1.59 pooka
658 1.154 pooka static int compcounter[RUMP_COMPONENT_MAX];
659 1.154 pooka
660 1.154 pooka static void
661 1.154 pooka rump_component_init_cb(struct rump_component *rc, int type)
662 1.154 pooka {
663 1.154 pooka
664 1.154 pooka KASSERT(type < RUMP_COMPONENT_MAX);
665 1.154 pooka if (rc->rc_type == type) {
666 1.154 pooka rc->rc_init();
667 1.154 pooka compcounter[type]++;
668 1.154 pooka }
669 1.154 pooka }
670 1.154 pooka
671 1.154 pooka int
672 1.154 pooka rump_component_count(enum rump_component_type type)
673 1.154 pooka {
674 1.154 pooka
675 1.154 pooka KASSERT(type <= RUMP_COMPONENT_MAX);
676 1.154 pooka return compcounter[type];
677 1.154 pooka }
678 1.154 pooka
679 1.154 pooka void
680 1.154 pooka rump_component_init(enum rump_component_type type)
681 1.154 pooka {
682 1.154 pooka
683 1.154 pooka rumpuser_dl_component_init(type, rump_component_init_cb);
684 1.154 pooka }
685 1.154 pooka
686 1.155 pooka /*
687 1.155 pooka * Initialize a module which has already been loaded and linked
688 1.155 pooka * with dlopen(). This is fundamentally the same as a builtin module.
689 1.155 pooka */
690 1.76 pooka int
691 1.155 pooka rump_module_init(const struct modinfo * const *mip, size_t nmodinfo)
692 1.76 pooka {
693 1.76 pooka
694 1.155 pooka return module_builtin_add(mip, nmodinfo, true);
695 1.106 pooka }
696 1.106 pooka
697 1.155 pooka /*
698 1.155 pooka * Finish module (flawless victory, fatality!).
699 1.155 pooka */
700 1.106 pooka int
701 1.155 pooka rump_module_fini(const struct modinfo *mi)
702 1.106 pooka {
703 1.120 pooka
704 1.155 pooka return module_builtin_remove(mi, true);
705 1.155 pooka }
706 1.155 pooka
707 1.155 pooka /*
708 1.155 pooka * Add loaded and linked module to the builtin list. It will
709 1.155 pooka * later be initialized with module_init_class().
710 1.155 pooka */
711 1.155 pooka
712 1.155 pooka static void
713 1.155 pooka add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo)
714 1.155 pooka {
715 1.106 pooka
716 1.155 pooka module_builtin_add(mip, nmodinfo, false);
717 1.76 pooka }
718 1.76 pooka
719 1.137 pooka int
720 1.137 pooka rump_kernelfsym_load(void *symtab, uint64_t symsize,
721 1.137 pooka char *strtab, uint64_t strsize)
722 1.137 pooka {
723 1.137 pooka static int inited = 0;
724 1.137 pooka Elf64_Ehdr ehdr;
725 1.137 pooka
726 1.137 pooka if (inited)
727 1.137 pooka return EBUSY;
728 1.137 pooka inited = 1;
729 1.137 pooka
730 1.137 pooka /*
731 1.137 pooka * Use 64bit header since it's bigger. Shouldn't make a
732 1.137 pooka * difference, since we're passing in all zeroes anyway.
733 1.137 pooka */
734 1.137 pooka memset(&ehdr, 0, sizeof(ehdr));
735 1.137 pooka ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize);
736 1.137 pooka
737 1.137 pooka return 0;
738 1.137 pooka }
739 1.137 pooka
740 1.199 pooka static int
741 1.199 pooka rump_proxy_syscall(int num, void *arg, register_t *retval)
742 1.95 pooka {
743 1.95 pooka struct lwp *l;
744 1.95 pooka struct sysent *callp;
745 1.124 pooka int rv;
746 1.95 pooka
747 1.95 pooka if (__predict_false(num >= SYS_NSYSENT))
748 1.95 pooka return ENOSYS;
749 1.95 pooka
750 1.95 pooka callp = rump_sysent + num;
751 1.125 pooka l = curlwp;
752 1.191 pooka rv = sy_call(callp, l, (void *)arg, retval);
753 1.191 pooka
754 1.191 pooka return rv;
755 1.191 pooka }
756 1.191 pooka
757 1.206 pooka static int
758 1.226 pooka rump_proxy_rfork(void *priv, int flags, const char *comm)
759 1.206 pooka {
760 1.206 pooka struct vmspace *newspace;
761 1.226 pooka struct proc *p;
762 1.206 pooka int error;
763 1.206 pooka
764 1.214 pooka if ((error = rump_lwproc_rfork(flags)) != 0)
765 1.206 pooka return error;
766 1.206 pooka
767 1.206 pooka /*
768 1.206 pooka * Since it's a proxy proc, adjust the vmspace.
769 1.206 pooka * Refcount will eternally be 1.
770 1.206 pooka */
771 1.226 pooka p = curproc;
772 1.229 pooka newspace = kmem_zalloc(sizeof(*newspace), KM_SLEEP);
773 1.206 pooka newspace->vm_refcnt = 1;
774 1.206 pooka newspace->vm_map.pmap = priv;
775 1.226 pooka KASSERT(p->p_vmspace == vmspace_kernel());
776 1.226 pooka p->p_vmspace = newspace;
777 1.226 pooka if (comm)
778 1.226 pooka strlcpy(p->p_comm, comm, sizeof(p->p_comm));
779 1.206 pooka
780 1.206 pooka return 0;
781 1.206 pooka }
782 1.206 pooka
783 1.232 pooka /*
784 1.232 pooka * Order all lwps in a process to exit. does *not* wait for them to drain.
785 1.232 pooka */
786 1.219 pooka static void
787 1.232 pooka rump_proxy_lwpexit(void)
788 1.219 pooka {
789 1.219 pooka struct proc *p = curproc;
790 1.219 pooka uint64_t where;
791 1.219 pooka struct lwp *l;
792 1.219 pooka
793 1.219 pooka mutex_enter(p->p_lock);
794 1.219 pooka /*
795 1.232 pooka * First pass: mark all lwps in the process with LW_RUMP_QEXIT
796 1.219 pooka * so that they know they should exit.
797 1.219 pooka */
798 1.219 pooka LIST_FOREACH(l, &p->p_lwps, l_sibling) {
799 1.219 pooka if (l == curlwp)
800 1.219 pooka continue;
801 1.232 pooka l->l_flag |= LW_RUMP_QEXIT;
802 1.219 pooka }
803 1.219 pooka mutex_exit(p->p_lock);
804 1.219 pooka
805 1.219 pooka /*
806 1.219 pooka * Next, make sure everyone on all CPUs sees our status
807 1.219 pooka * update. This keeps threads inside cv_wait() and makes
808 1.219 pooka * sure we don't access a stale cv pointer later when
809 1.219 pooka * we wake up the threads.
810 1.219 pooka */
811 1.219 pooka
812 1.219 pooka where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
813 1.219 pooka xc_wait(where);
814 1.219 pooka
815 1.219 pooka /*
816 1.219 pooka * Ok, all lwps are either:
817 1.219 pooka * 1) not in the cv code
818 1.219 pooka * 2) sleeping on l->l_private
819 1.219 pooka * 3) sleeping on p->p_waitcv
820 1.219 pooka *
821 1.232 pooka * Either way, l_private is stable until we set PS_RUMP_LWPEXIT
822 1.232 pooka * in p->p_sflag.
823 1.219 pooka */
824 1.219 pooka
825 1.219 pooka mutex_enter(p->p_lock);
826 1.219 pooka LIST_FOREACH(l, &p->p_lwps, l_sibling) {
827 1.219 pooka if (l->l_private)
828 1.219 pooka cv_broadcast(l->l_private);
829 1.219 pooka }
830 1.232 pooka p->p_sflag |= PS_RUMP_LWPEXIT;
831 1.219 pooka cv_broadcast(&p->p_waitcv);
832 1.219 pooka mutex_exit(p->p_lock);
833 1.232 pooka }
834 1.219 pooka
835 1.232 pooka /*
836 1.232 pooka * Notify process that all threads have been drained and exec is complete.
837 1.232 pooka */
838 1.232 pooka static void
839 1.232 pooka rump_proxy_execnotify(const char *comm)
840 1.232 pooka {
841 1.232 pooka struct proc *p = curproc;
842 1.232 pooka
843 1.232 pooka fd_closeexec();
844 1.232 pooka mutex_enter(p->p_lock);
845 1.232 pooka KASSERT(p->p_nlwps == 1 && p->p_sflag & PS_RUMP_LWPEXIT);
846 1.232 pooka p->p_sflag &= ~PS_RUMP_LWPEXIT;
847 1.232 pooka mutex_exit(p->p_lock);
848 1.232 pooka strlcpy(p->p_comm, comm, sizeof(p->p_comm));
849 1.219 pooka }
850 1.219 pooka
851 1.191 pooka int
852 1.130 pooka rump_boot_gethowto()
853 1.130 pooka {
854 1.130 pooka
855 1.130 pooka return boothowto;
856 1.130 pooka }
857 1.130 pooka
858 1.130 pooka void
859 1.130 pooka rump_boot_sethowto(int howto)
860 1.130 pooka {
861 1.130 pooka
862 1.130 pooka boothowto = howto;
863 1.130 pooka }
864 1.130 pooka
865 1.109 pooka int
866 1.124 pooka rump_getversion(void)
867 1.109 pooka {
868 1.109 pooka
869 1.109 pooka return __NetBSD_Version__;
870 1.109 pooka }
871 1.170 pooka
872 1.170 pooka /*
873 1.170 pooka * Note: may be called unscheduled. Not fully safe since no locking
874 1.170 pooka * of allevents (currently that's not even available).
875 1.170 pooka */
876 1.170 pooka void
877 1.170 pooka rump_printevcnts()
878 1.170 pooka {
879 1.170 pooka struct evcnt *ev;
880 1.170 pooka
881 1.170 pooka TAILQ_FOREACH(ev, &allevents, ev_list)
882 1.170 pooka rumpuser_dprintf("%s / %s: %" PRIu64 "\n",
883 1.170 pooka ev->ev_group, ev->ev_name, ev->ev_count);
884 1.170 pooka }
885 1.184 pooka
886 1.184 pooka /*
887 1.184 pooka * If you use this interface ... well ... all bets are off.
888 1.184 pooka * The original purpose is for the p2k fs server library to be
889 1.184 pooka * able to use the same pid/lid for VOPs as the host kernel.
890 1.184 pooka */
891 1.184 pooka void
892 1.184 pooka rump_allbetsareoff_setid(pid_t pid, int lid)
893 1.184 pooka {
894 1.184 pooka struct lwp *l = curlwp;
895 1.184 pooka struct proc *p = l->l_proc;
896 1.184 pooka
897 1.184 pooka l->l_lid = lid;
898 1.184 pooka p->p_pid = pid;
899 1.184 pooka }
900