sys_aio.c revision 1.31.2.1
1 /* $NetBSD: sys_aio.c,v 1.31.2.1 2010/08/17 06:47:30 uebayasi Exp $ */ 2 3 /* 4 * Copyright (c) 2007 Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Implementation of POSIX asynchronous I/O. 31 * Defined in the Base Definitions volume of IEEE Std 1003.1-2001. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.31.2.1 2010/08/17 06:47:30 uebayasi Exp $"); 36 37 #ifdef _KERNEL_OPT 38 #include "opt_ddb.h" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/condvar.h> 43 #include <sys/file.h> 44 #include <sys/filedesc.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/lwp.h> 48 #include <sys/mutex.h> 49 #include <sys/pool.h> 50 #include <sys/proc.h> 51 #include <sys/queue.h> 52 #include <sys/signal.h> 53 #include <sys/signalvar.h> 54 #include <sys/syscall.h> 55 #include <sys/syscallargs.h> 56 #include <sys/syscallvar.h> 57 #include <sys/sysctl.h> 58 #include <sys/systm.h> 59 #include <sys/types.h> 60 #include <sys/vnode.h> 61 #include <sys/atomic.h> 62 #include <sys/module.h> 63 #include <sys/buf.h> 64 65 #include <uvm/uvm_extern.h> 66 67 MODULE(MODULE_CLASS_MISC, aio, NULL); 68 69 /* 70 * System-wide limits and counter of AIO operations. 71 */ 72 u_int aio_listio_max = AIO_LISTIO_MAX; 73 static u_int aio_max = AIO_MAX; 74 static u_int aio_jobs_count; 75 76 static struct sysctllog *aio_sysctl; 77 static struct pool aio_job_pool; 78 static struct pool aio_lio_pool; 79 static void * aio_ehook; 80 81 static void aio_worker(void *); 82 static void aio_process(struct aio_job *); 83 static void aio_sendsig(struct proc *, struct sigevent *); 84 static int aio_enqueue_job(int, void *, struct lio_req *); 85 static void aio_exit(proc_t *, void *); 86 87 static int sysctl_aio_listio_max(SYSCTLFN_PROTO); 88 static int sysctl_aio_max(SYSCTLFN_PROTO); 89 static int sysctl_aio_init(void); 90 91 static const struct syscall_package aio_syscalls[] = { 92 { SYS_aio_cancel, 0, (sy_call_t *)sys_aio_cancel }, 93 { SYS_aio_error, 0, (sy_call_t *)sys_aio_error }, 94 { SYS_aio_fsync, 0, (sy_call_t *)sys_aio_fsync }, 95 { SYS_aio_read, 0, (sy_call_t *)sys_aio_read }, 96 { SYS_aio_return, 0, (sy_call_t *)sys_aio_return }, 97 { SYS___aio_suspend50, 0, (sy_call_t *)sys___aio_suspend50 }, 98 { SYS_aio_write, 0, (sy_call_t *)sys_aio_write }, 99 { SYS_lio_listio, 0, (sy_call_t *)sys_lio_listio }, 100 { 0, 0, NULL }, 101 }; 102 103 /* 104 * Tear down all AIO state. 105 */ 106 static int 107 aio_fini(bool interface) 108 { 109 int error; 110 proc_t *p; 111 112 if (interface) { 113 /* Stop syscall activity. */ 114 error = syscall_disestablish(NULL, aio_syscalls); 115 if (error != 0) 116 return error; 117 /* Abort if any processes are using AIO. */ 118 mutex_enter(proc_lock); 119 PROCLIST_FOREACH(p, &allproc) { 120 if (p->p_aio != NULL) 121 break; 122 } 123 mutex_exit(proc_lock); 124 if (p != NULL) { 125 error = syscall_establish(NULL, aio_syscalls); 126 KASSERT(error == 0); 127 return EBUSY; 128 } 129 } 130 if (aio_sysctl != NULL) 131 sysctl_teardown(&aio_sysctl); 132 133 KASSERT(aio_jobs_count == 0); 134 exithook_disestablish(aio_ehook); 135 pool_destroy(&aio_job_pool); 136 pool_destroy(&aio_lio_pool); 137 return 0; 138 } 139 140 /* 141 * Initialize global AIO state. 142 */ 143 static int 144 aio_init(void) 145 { 146 int error; 147 148 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 149 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 150 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 151 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 152 aio_ehook = exithook_establish(aio_exit, NULL); 153 154 error = sysctl_aio_init(); 155 if (error != 0) { 156 (void)aio_fini(false); 157 return error; 158 } 159 error = syscall_establish(NULL, aio_syscalls); 160 if (error != 0) 161 (void)aio_fini(false); 162 return error; 163 } 164 165 /* 166 * Module interface. 167 */ 168 static int 169 aio_modcmd(modcmd_t cmd, void *arg) 170 { 171 172 switch (cmd) { 173 case MODULE_CMD_INIT: 174 return aio_init(); 175 case MODULE_CMD_FINI: 176 return aio_fini(true); 177 default: 178 return ENOTTY; 179 } 180 } 181 182 /* 183 * Initialize Asynchronous I/O data structures for the process. 184 */ 185 static int 186 aio_procinit(struct proc *p) 187 { 188 struct aioproc *aio; 189 struct lwp *l; 190 int error; 191 vaddr_t uaddr; 192 193 /* Allocate and initialize AIO structure */ 194 aio = kmem_zalloc(sizeof(struct aioproc), KM_SLEEP); 195 if (aio == NULL) 196 return EAGAIN; 197 198 /* Initialize queue and their synchronization structures */ 199 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 200 cv_init(&aio->aio_worker_cv, "aiowork"); 201 cv_init(&aio->done_cv, "aiodone"); 202 TAILQ_INIT(&aio->jobs_queue); 203 204 /* 205 * Create an AIO worker thread. 206 * XXX: Currently, AIO thread is not protected against user's actions. 207 */ 208 uaddr = uvm_uarea_alloc(); 209 if (uaddr == 0) { 210 aio_exit(p, aio); 211 return EAGAIN; 212 } 213 error = lwp_create(curlwp, p, uaddr, 0, NULL, 0, aio_worker, 214 NULL, &l, curlwp->l_class); 215 if (error != 0) { 216 uvm_uarea_free(uaddr); 217 aio_exit(p, aio); 218 return error; 219 } 220 221 /* Recheck if we are really first */ 222 mutex_enter(p->p_lock); 223 if (p->p_aio) { 224 mutex_exit(p->p_lock); 225 aio_exit(p, aio); 226 lwp_exit(l); 227 return 0; 228 } 229 p->p_aio = aio; 230 231 /* Complete the initialization of thread, and run it */ 232 aio->aio_worker = l; 233 lwp_lock(l); 234 l->l_stat = LSRUN; 235 l->l_priority = MAXPRI_USER; 236 sched_enqueue(l, false); 237 lwp_unlock(l); 238 mutex_exit(p->p_lock); 239 240 return 0; 241 } 242 243 /* 244 * Exit of Asynchronous I/O subsystem of process. 245 */ 246 static void 247 aio_exit(struct proc *p, void *cookie) 248 { 249 struct aio_job *a_job; 250 struct aioproc *aio; 251 252 if (cookie != NULL) 253 aio = cookie; 254 else if ((aio = p->p_aio) == NULL) 255 return; 256 257 /* Free AIO queue */ 258 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 259 a_job = TAILQ_FIRST(&aio->jobs_queue); 260 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 261 pool_put(&aio_job_pool, a_job); 262 atomic_dec_uint(&aio_jobs_count); 263 } 264 265 /* Destroy and free the entire AIO data structure */ 266 cv_destroy(&aio->aio_worker_cv); 267 cv_destroy(&aio->done_cv); 268 mutex_destroy(&aio->aio_mtx); 269 kmem_free(aio, sizeof(struct aioproc)); 270 } 271 272 /* 273 * AIO worker thread and processor. 274 */ 275 static void 276 aio_worker(void *arg) 277 { 278 struct proc *p = curlwp->l_proc; 279 struct aioproc *aio = p->p_aio; 280 struct aio_job *a_job; 281 struct lio_req *lio; 282 sigset_t oss, nss; 283 int error, refcnt; 284 285 /* 286 * Make an empty signal mask, so it 287 * handles only SIGKILL and SIGSTOP. 288 */ 289 sigfillset(&nss); 290 mutex_enter(p->p_lock); 291 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 292 mutex_exit(p->p_lock); 293 KASSERT(error == 0); 294 295 for (;;) { 296 /* 297 * Loop for each job in the queue. If there 298 * are no jobs then sleep. 299 */ 300 mutex_enter(&aio->aio_mtx); 301 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 302 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 303 /* 304 * Thread was interrupted - check for 305 * pending exit or suspend. 306 */ 307 mutex_exit(&aio->aio_mtx); 308 lwp_userret(curlwp); 309 mutex_enter(&aio->aio_mtx); 310 } 311 } 312 313 /* Take the job from the queue */ 314 aio->curjob = a_job; 315 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 316 317 atomic_dec_uint(&aio_jobs_count); 318 aio->jobs_count--; 319 320 mutex_exit(&aio->aio_mtx); 321 322 /* Process an AIO operation */ 323 aio_process(a_job); 324 325 /* Copy data structure back to the user-space */ 326 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 327 sizeof(struct aiocb)); 328 329 mutex_enter(&aio->aio_mtx); 330 aio->curjob = NULL; 331 332 /* Decrease a reference counter, if there is a LIO structure */ 333 lio = a_job->lio; 334 refcnt = (lio != NULL ? --lio->refcnt : -1); 335 336 /* Notify all suspenders */ 337 cv_broadcast(&aio->done_cv); 338 mutex_exit(&aio->aio_mtx); 339 340 /* Send a signal, if any */ 341 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 342 343 /* Destroy the LIO structure */ 344 if (refcnt == 0) { 345 aio_sendsig(p, &lio->sig); 346 pool_put(&aio_lio_pool, lio); 347 } 348 349 /* Destroy the job */ 350 pool_put(&aio_job_pool, a_job); 351 } 352 353 /* NOTREACHED */ 354 } 355 356 static void 357 aio_process(struct aio_job *a_job) 358 { 359 struct proc *p = curlwp->l_proc; 360 struct aiocb *aiocbp = &a_job->aiocbp; 361 struct file *fp; 362 int fd = aiocbp->aio_fildes; 363 int error = 0; 364 365 KASSERT(a_job->aio_op != 0); 366 367 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 368 struct iovec aiov; 369 struct uio auio; 370 371 if (aiocbp->aio_nbytes > SSIZE_MAX) { 372 error = EINVAL; 373 goto done; 374 } 375 376 fp = fd_getfile(fd); 377 if (fp == NULL) { 378 error = EBADF; 379 goto done; 380 } 381 382 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 383 aiov.iov_len = aiocbp->aio_nbytes; 384 auio.uio_iov = &aiov; 385 auio.uio_iovcnt = 1; 386 auio.uio_resid = aiocbp->aio_nbytes; 387 auio.uio_vmspace = p->p_vmspace; 388 389 if (a_job->aio_op & AIO_READ) { 390 /* 391 * Perform a Read operation 392 */ 393 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 394 395 if ((fp->f_flag & FREAD) == 0) { 396 fd_putfile(fd); 397 error = EBADF; 398 goto done; 399 } 400 auio.uio_rw = UIO_READ; 401 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 402 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 403 } else { 404 /* 405 * Perform a Write operation 406 */ 407 KASSERT(a_job->aio_op & AIO_WRITE); 408 409 if ((fp->f_flag & FWRITE) == 0) { 410 fd_putfile(fd); 411 error = EBADF; 412 goto done; 413 } 414 auio.uio_rw = UIO_WRITE; 415 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 416 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 417 } 418 fd_putfile(fd); 419 420 /* Store the result value */ 421 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 422 a_job->aiocbp._retval = (error == 0) ? 423 a_job->aiocbp.aio_nbytes : -1; 424 425 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 426 /* 427 * Perform a file Sync operation 428 */ 429 struct vnode *vp; 430 431 if ((error = fd_getvnode(fd, &fp)) != 0) 432 goto done; 433 434 if ((fp->f_flag & FWRITE) == 0) { 435 fd_putfile(fd); 436 error = EBADF; 437 goto done; 438 } 439 440 vp = (struct vnode *)fp->f_data; 441 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 442 if (a_job->aio_op & AIO_DSYNC) { 443 error = VOP_FSYNC(vp, fp->f_cred, 444 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0); 445 } else if (a_job->aio_op & AIO_SYNC) { 446 error = VOP_FSYNC(vp, fp->f_cred, 447 FSYNC_WAIT, 0, 0); 448 } 449 VOP_UNLOCK(vp); 450 fd_putfile(fd); 451 452 /* Store the result value */ 453 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 454 455 } else 456 panic("aio_process: invalid operation code\n"); 457 458 done: 459 /* Job is done, set the error, if any */ 460 a_job->aiocbp._errno = error; 461 a_job->aiocbp._state = JOB_DONE; 462 } 463 464 /* 465 * Send AIO signal. 466 */ 467 static void 468 aio_sendsig(struct proc *p, struct sigevent *sig) 469 { 470 ksiginfo_t ksi; 471 472 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 473 return; 474 475 KSI_INIT(&ksi); 476 ksi.ksi_signo = sig->sigev_signo; 477 ksi.ksi_code = SI_ASYNCIO; 478 ksi.ksi_value = sig->sigev_value; 479 mutex_enter(proc_lock); 480 kpsignal(p, &ksi, NULL); 481 mutex_exit(proc_lock); 482 } 483 484 /* 485 * Enqueue the job. 486 */ 487 static int 488 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 489 { 490 struct proc *p = curlwp->l_proc; 491 struct aioproc *aio; 492 struct aio_job *a_job; 493 struct aiocb aiocbp; 494 struct sigevent *sig; 495 int error; 496 497 /* Non-accurate check for the limit */ 498 if (aio_jobs_count + 1 > aio_max) 499 return EAGAIN; 500 501 /* Get the data structure from user-space */ 502 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 503 if (error) 504 return error; 505 506 /* Check if signal is set, and validate it */ 507 sig = &aiocbp.aio_sigevent; 508 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 509 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 510 return EINVAL; 511 512 /* Buffer and byte count */ 513 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 514 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 515 return EINVAL; 516 517 /* Check the opcode, if LIO_NOP - simply ignore */ 518 if (op == AIO_LIO) { 519 KASSERT(lio != NULL); 520 if (aiocbp.aio_lio_opcode == LIO_WRITE) 521 op = AIO_WRITE; 522 else if (aiocbp.aio_lio_opcode == LIO_READ) 523 op = AIO_READ; 524 else 525 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 526 } else { 527 KASSERT(lio == NULL); 528 } 529 530 /* 531 * Look for already existing job. If found - the job is in-progress. 532 * According to POSIX this is invalid, so return the error. 533 */ 534 aio = p->p_aio; 535 if (aio) { 536 mutex_enter(&aio->aio_mtx); 537 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 538 if (a_job->aiocb_uptr != aiocb_uptr) 539 continue; 540 mutex_exit(&aio->aio_mtx); 541 return EINVAL; 542 } 543 mutex_exit(&aio->aio_mtx); 544 } 545 546 /* 547 * Check if AIO structure is initialized, if not - initialize it. 548 * In LIO case, we did that already. We will recheck this with 549 * the lock in aio_procinit(). 550 */ 551 if (lio == NULL && p->p_aio == NULL) 552 if (aio_procinit(p)) 553 return EAGAIN; 554 aio = p->p_aio; 555 556 /* 557 * Set the state with errno, and copy data 558 * structure back to the user-space. 559 */ 560 aiocbp._state = JOB_WIP; 561 aiocbp._errno = EINPROGRESS; 562 aiocbp._retval = -1; 563 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 564 if (error) 565 return error; 566 567 /* Allocate and initialize a new AIO job */ 568 a_job = pool_get(&aio_job_pool, PR_WAITOK); 569 memset(a_job, 0, sizeof(struct aio_job)); 570 571 /* 572 * Set the data. 573 * Store the user-space pointer for searching. Since we 574 * are storing only per proc pointers - it is safe. 575 */ 576 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 577 a_job->aiocb_uptr = aiocb_uptr; 578 a_job->aio_op |= op; 579 a_job->lio = lio; 580 581 /* 582 * Add the job to the queue, update the counters, and 583 * notify the AIO worker thread to handle the job. 584 */ 585 mutex_enter(&aio->aio_mtx); 586 587 /* Fail, if the limit was reached */ 588 if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max || 589 aio->jobs_count >= aio_listio_max) { 590 atomic_dec_uint(&aio_jobs_count); 591 mutex_exit(&aio->aio_mtx); 592 pool_put(&aio_job_pool, a_job); 593 return EAGAIN; 594 } 595 596 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 597 aio->jobs_count++; 598 if (lio) 599 lio->refcnt++; 600 cv_signal(&aio->aio_worker_cv); 601 602 mutex_exit(&aio->aio_mtx); 603 604 /* 605 * One would handle the errors only with aio_error() function. 606 * This way is appropriate according to POSIX. 607 */ 608 return 0; 609 } 610 611 /* 612 * Syscall functions. 613 */ 614 615 int 616 sys_aio_cancel(struct lwp *l, const struct sys_aio_cancel_args *uap, 617 register_t *retval) 618 { 619 /* { 620 syscallarg(int) fildes; 621 syscallarg(struct aiocb *) aiocbp; 622 } */ 623 struct proc *p = l->l_proc; 624 struct aioproc *aio; 625 struct aio_job *a_job; 626 struct aiocb *aiocbp_ptr; 627 struct lio_req *lio; 628 struct filedesc *fdp = p->p_fd; 629 unsigned int cn, errcnt, fildes; 630 fdtab_t *dt; 631 632 TAILQ_HEAD(, aio_job) tmp_jobs_list; 633 634 /* Check for invalid file descriptor */ 635 fildes = (unsigned int)SCARG(uap, fildes); 636 dt = fdp->fd_dt; 637 if (fildes >= dt->dt_nfiles) 638 return EBADF; 639 if (dt->dt_ff[fildes] == NULL || dt->dt_ff[fildes]->ff_file == NULL) 640 return EBADF; 641 642 /* Check if AIO structure is initialized */ 643 if (p->p_aio == NULL) { 644 *retval = AIO_NOTCANCELED; 645 return 0; 646 } 647 648 aio = p->p_aio; 649 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 650 651 mutex_enter(&aio->aio_mtx); 652 653 /* Cancel the jobs, and remove them from the queue */ 654 cn = 0; 655 TAILQ_INIT(&tmp_jobs_list); 656 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 657 if (aiocbp_ptr) { 658 if (aiocbp_ptr != a_job->aiocb_uptr) 659 continue; 660 if (fildes != a_job->aiocbp.aio_fildes) { 661 mutex_exit(&aio->aio_mtx); 662 return EBADF; 663 } 664 } else if (a_job->aiocbp.aio_fildes != fildes) 665 continue; 666 667 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 668 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 669 670 /* Decrease the counters */ 671 atomic_dec_uint(&aio_jobs_count); 672 aio->jobs_count--; 673 lio = a_job->lio; 674 if (lio != NULL && --lio->refcnt != 0) 675 a_job->lio = NULL; 676 677 cn++; 678 if (aiocbp_ptr) 679 break; 680 } 681 682 /* There are canceled jobs */ 683 if (cn) 684 *retval = AIO_CANCELED; 685 686 /* We cannot cancel current job */ 687 a_job = aio->curjob; 688 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 689 (a_job->aiocb_uptr == aiocbp_ptr))) 690 *retval = AIO_NOTCANCELED; 691 692 mutex_exit(&aio->aio_mtx); 693 694 /* Free the jobs after the lock */ 695 errcnt = 0; 696 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 697 a_job = TAILQ_FIRST(&tmp_jobs_list); 698 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 699 /* Set the errno and copy structures back to the user-space */ 700 a_job->aiocbp._errno = ECANCELED; 701 a_job->aiocbp._state = JOB_DONE; 702 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 703 sizeof(struct aiocb))) 704 errcnt++; 705 /* Send a signal if any */ 706 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 707 if (a_job->lio) { 708 lio = a_job->lio; 709 aio_sendsig(p, &lio->sig); 710 pool_put(&aio_lio_pool, lio); 711 } 712 pool_put(&aio_job_pool, a_job); 713 } 714 715 if (errcnt) 716 return EFAULT; 717 718 /* Set a correct return value */ 719 if (*retval == 0) 720 *retval = AIO_ALLDONE; 721 722 return 0; 723 } 724 725 int 726 sys_aio_error(struct lwp *l, const struct sys_aio_error_args *uap, 727 register_t *retval) 728 { 729 /* { 730 syscallarg(const struct aiocb *) aiocbp; 731 } */ 732 struct proc *p = l->l_proc; 733 struct aioproc *aio = p->p_aio; 734 struct aiocb aiocbp; 735 int error; 736 737 if (aio == NULL) 738 return EINVAL; 739 740 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 741 if (error) 742 return error; 743 744 if (aiocbp._state == JOB_NONE) 745 return EINVAL; 746 747 *retval = aiocbp._errno; 748 749 return 0; 750 } 751 752 int 753 sys_aio_fsync(struct lwp *l, const struct sys_aio_fsync_args *uap, 754 register_t *retval) 755 { 756 /* { 757 syscallarg(int) op; 758 syscallarg(struct aiocb *) aiocbp; 759 } */ 760 int op = SCARG(uap, op); 761 762 if ((op != O_DSYNC) && (op != O_SYNC)) 763 return EINVAL; 764 765 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 766 767 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 768 } 769 770 int 771 sys_aio_read(struct lwp *l, const struct sys_aio_read_args *uap, 772 register_t *retval) 773 { 774 /* { 775 syscallarg(struct aiocb *) aiocbp; 776 } */ 777 778 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 779 } 780 781 int 782 sys_aio_return(struct lwp *l, const struct sys_aio_return_args *uap, 783 register_t *retval) 784 { 785 /* { 786 syscallarg(struct aiocb *) aiocbp; 787 } */ 788 struct proc *p = l->l_proc; 789 struct aioproc *aio = p->p_aio; 790 struct aiocb aiocbp; 791 int error; 792 793 if (aio == NULL) 794 return EINVAL; 795 796 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 797 if (error) 798 return error; 799 800 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 801 return EINVAL; 802 803 *retval = aiocbp._retval; 804 805 /* Reset the internal variables */ 806 aiocbp._errno = 0; 807 aiocbp._retval = -1; 808 aiocbp._state = JOB_NONE; 809 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 810 811 return error; 812 } 813 814 int 815 sys___aio_suspend50(struct lwp *l, const struct sys___aio_suspend50_args *uap, 816 register_t *retval) 817 { 818 /* { 819 syscallarg(const struct aiocb *const[]) list; 820 syscallarg(int) nent; 821 syscallarg(const struct timespec *) timeout; 822 } */ 823 struct aiocb **list; 824 struct timespec ts; 825 int error, nent; 826 827 nent = SCARG(uap, nent); 828 if (nent <= 0 || nent > aio_listio_max) 829 return EAGAIN; 830 831 if (SCARG(uap, timeout)) { 832 /* Convert timespec to ticks */ 833 error = copyin(SCARG(uap, timeout), &ts, 834 sizeof(struct timespec)); 835 if (error) 836 return error; 837 } 838 839 list = kmem_alloc(nent * sizeof(*list), KM_SLEEP); 840 error = copyin(SCARG(uap, list), list, nent * sizeof(*list)); 841 if (error) 842 goto out; 843 error = aio_suspend1(l, list, nent, SCARG(uap, timeout) ? &ts : NULL); 844 out: 845 kmem_free(list, nent * sizeof(*list)); 846 return error; 847 } 848 849 int 850 aio_suspend1(struct lwp *l, struct aiocb **aiocbp_list, int nent, 851 struct timespec *ts) 852 { 853 struct proc *p = l->l_proc; 854 struct aioproc *aio; 855 struct aio_job *a_job; 856 int i, error, timo; 857 858 if (p->p_aio == NULL) 859 return EAGAIN; 860 aio = p->p_aio; 861 862 if (ts) { 863 timo = mstohz((ts->tv_sec * 1000) + (ts->tv_nsec / 1000000)); 864 if (timo == 0 && ts->tv_sec == 0 && ts->tv_nsec > 0) 865 timo = 1; 866 if (timo <= 0) 867 return EAGAIN; 868 } else 869 timo = 0; 870 871 mutex_enter(&aio->aio_mtx); 872 for (;;) { 873 for (i = 0; i < nent; i++) { 874 875 /* Skip NULL entries */ 876 if (aiocbp_list[i] == NULL) 877 continue; 878 879 /* Skip current job */ 880 if (aio->curjob) { 881 a_job = aio->curjob; 882 if (a_job->aiocb_uptr == aiocbp_list[i]) 883 continue; 884 } 885 886 /* Look for a job in the queue */ 887 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 888 if (a_job->aiocb_uptr == aiocbp_list[i]) 889 break; 890 891 if (a_job == NULL) { 892 struct aiocb aiocbp; 893 894 mutex_exit(&aio->aio_mtx); 895 896 /* Check if the job is done. */ 897 error = copyin(aiocbp_list[i], &aiocbp, 898 sizeof(struct aiocb)); 899 if (error == 0 && aiocbp._state != JOB_DONE) { 900 mutex_enter(&aio->aio_mtx); 901 continue; 902 } 903 return error; 904 } 905 } 906 907 /* Wait for a signal or when timeout occurs */ 908 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 909 if (error) { 910 if (error == EWOULDBLOCK) 911 error = EAGAIN; 912 break; 913 } 914 } 915 mutex_exit(&aio->aio_mtx); 916 return error; 917 } 918 919 int 920 sys_aio_write(struct lwp *l, const struct sys_aio_write_args *uap, 921 register_t *retval) 922 { 923 /* { 924 syscallarg(struct aiocb *) aiocbp; 925 } */ 926 927 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 928 } 929 930 int 931 sys_lio_listio(struct lwp *l, const struct sys_lio_listio_args *uap, 932 register_t *retval) 933 { 934 /* { 935 syscallarg(int) mode; 936 syscallarg(struct aiocb *const[]) list; 937 syscallarg(int) nent; 938 syscallarg(struct sigevent *) sig; 939 } */ 940 struct proc *p = l->l_proc; 941 struct aioproc *aio; 942 struct aiocb **aiocbp_list; 943 struct lio_req *lio; 944 int i, error, errcnt, mode, nent; 945 946 mode = SCARG(uap, mode); 947 nent = SCARG(uap, nent); 948 949 /* Non-accurate checks for the limit and invalid values */ 950 if (nent < 1 || nent > aio_listio_max) 951 return EINVAL; 952 if (aio_jobs_count + nent > aio_max) 953 return EAGAIN; 954 955 /* Check if AIO structure is initialized, if not - initialize it */ 956 if (p->p_aio == NULL) 957 if (aio_procinit(p)) 958 return EAGAIN; 959 aio = p->p_aio; 960 961 /* Create a LIO structure */ 962 lio = pool_get(&aio_lio_pool, PR_WAITOK); 963 lio->refcnt = 1; 964 error = 0; 965 966 switch (mode) { 967 case LIO_WAIT: 968 memset(&lio->sig, 0, sizeof(struct sigevent)); 969 break; 970 case LIO_NOWAIT: 971 /* Check for signal, validate it */ 972 if (SCARG(uap, sig)) { 973 struct sigevent *sig = &lio->sig; 974 975 error = copyin(SCARG(uap, sig), &lio->sig, 976 sizeof(struct sigevent)); 977 if (error == 0 && 978 (sig->sigev_signo < 0 || 979 sig->sigev_signo >= NSIG || 980 sig->sigev_notify < SIGEV_NONE || 981 sig->sigev_notify > SIGEV_SA)) 982 error = EINVAL; 983 } else 984 memset(&lio->sig, 0, sizeof(struct sigevent)); 985 break; 986 default: 987 error = EINVAL; 988 break; 989 } 990 991 if (error != 0) { 992 pool_put(&aio_lio_pool, lio); 993 return error; 994 } 995 996 /* Get the list from user-space */ 997 aiocbp_list = kmem_alloc(nent * sizeof(*aiocbp_list), KM_SLEEP); 998 error = copyin(SCARG(uap, list), aiocbp_list, 999 nent * sizeof(*aiocbp_list)); 1000 if (error) { 1001 mutex_enter(&aio->aio_mtx); 1002 goto err; 1003 } 1004 1005 /* Enqueue all jobs */ 1006 errcnt = 0; 1007 for (i = 0; i < nent; i++) { 1008 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 1009 /* 1010 * According to POSIX, in such error case it may 1011 * fail with other I/O operations initiated. 1012 */ 1013 if (error) 1014 errcnt++; 1015 } 1016 1017 mutex_enter(&aio->aio_mtx); 1018 1019 /* Return an error, if any */ 1020 if (errcnt) { 1021 error = EIO; 1022 goto err; 1023 } 1024 1025 if (mode == LIO_WAIT) { 1026 /* 1027 * Wait for AIO completion. In such case, 1028 * the LIO structure will be freed here. 1029 */ 1030 while (lio->refcnt > 1 && error == 0) 1031 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 1032 if (error) 1033 error = EINTR; 1034 } 1035 1036 err: 1037 if (--lio->refcnt != 0) 1038 lio = NULL; 1039 mutex_exit(&aio->aio_mtx); 1040 if (lio != NULL) { 1041 aio_sendsig(p, &lio->sig); 1042 pool_put(&aio_lio_pool, lio); 1043 } 1044 kmem_free(aiocbp_list, nent * sizeof(*aiocbp_list)); 1045 return error; 1046 } 1047 1048 /* 1049 * SysCtl 1050 */ 1051 1052 static int 1053 sysctl_aio_listio_max(SYSCTLFN_ARGS) 1054 { 1055 struct sysctlnode node; 1056 int error, newsize; 1057 1058 node = *rnode; 1059 node.sysctl_data = &newsize; 1060 1061 newsize = aio_listio_max; 1062 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1063 if (error || newp == NULL) 1064 return error; 1065 1066 if (newsize < 1 || newsize > aio_max) 1067 return EINVAL; 1068 aio_listio_max = newsize; 1069 1070 return 0; 1071 } 1072 1073 static int 1074 sysctl_aio_max(SYSCTLFN_ARGS) 1075 { 1076 struct sysctlnode node; 1077 int error, newsize; 1078 1079 node = *rnode; 1080 node.sysctl_data = &newsize; 1081 1082 newsize = aio_max; 1083 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1084 if (error || newp == NULL) 1085 return error; 1086 1087 if (newsize < 1 || newsize < aio_listio_max) 1088 return EINVAL; 1089 aio_max = newsize; 1090 1091 return 0; 1092 } 1093 1094 static int 1095 sysctl_aio_init(void) 1096 { 1097 int rv; 1098 1099 aio_sysctl = NULL; 1100 1101 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1102 CTLFLAG_PERMANENT, 1103 CTLTYPE_NODE, "kern", NULL, 1104 NULL, 0, NULL, 0, 1105 CTL_KERN, CTL_EOL); 1106 1107 if (rv != 0) 1108 return rv; 1109 1110 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1111 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1112 CTLTYPE_INT, "posix_aio", 1113 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1114 "Asynchronous I/O option to which the " 1115 "system attempts to conform"), 1116 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1117 CTL_KERN, CTL_CREATE, CTL_EOL); 1118 1119 if (rv != 0) 1120 return rv; 1121 1122 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1123 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1124 CTLTYPE_INT, "aio_listio_max", 1125 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1126 "operations in a single list I/O call"), 1127 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1128 CTL_KERN, CTL_CREATE, CTL_EOL); 1129 1130 if (rv != 0) 1131 return rv; 1132 1133 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1134 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1135 CTLTYPE_INT, "aio_max", 1136 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1137 "operations"), 1138 sysctl_aio_max, 0, &aio_max, 0, 1139 CTL_KERN, CTL_CREATE, CTL_EOL); 1140 1141 return rv; 1142 } 1143 1144 /* 1145 * Debugging 1146 */ 1147 #if defined(DDB) 1148 void 1149 aio_print_jobs(void (*pr)(const char *, ...)) 1150 { 1151 struct proc *p = (curlwp == NULL ? NULL : curlwp->l_proc); 1152 struct aioproc *aio; 1153 struct aio_job *a_job; 1154 struct aiocb *aiocbp; 1155 1156 if (p == NULL) { 1157 (*pr)("AIO: We are not in the processes right now.\n"); 1158 return; 1159 } 1160 1161 aio = p->p_aio; 1162 if (aio == NULL) { 1163 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1164 return; 1165 } 1166 1167 (*pr)("AIO: PID = %d\n", p->p_pid); 1168 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1169 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1170 1171 if (aio->curjob) { 1172 a_job = aio->curjob; 1173 (*pr)("\nAIO current job:\n"); 1174 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1175 a_job->aio_op, a_job->aiocbp._errno, 1176 a_job->aiocbp._state, a_job->aiocb_uptr); 1177 aiocbp = &a_job->aiocbp; 1178 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1179 aiocbp->aio_fildes, aiocbp->aio_offset, 1180 aiocbp->aio_buf, aiocbp->aio_nbytes); 1181 } 1182 1183 (*pr)("\nAIO queue:\n"); 1184 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1185 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1186 a_job->aio_op, a_job->aiocbp._errno, 1187 a_job->aiocbp._state, a_job->aiocb_uptr); 1188 aiocbp = &a_job->aiocbp; 1189 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1190 aiocbp->aio_fildes, aiocbp->aio_offset, 1191 aiocbp->aio_buf, aiocbp->aio_nbytes); 1192 } 1193 } 1194 #endif /* defined(DDB) */ 1195
Indexes created Tue Sep 30 20:09:53 GMT 2025