sys_aio.c revision 1.20
1 /* $NetBSD: sys_aio.c,v 1.20 2008/11/14 23:33:45 ad Exp $ */ 2 3 /* 4 * Copyright (c) 2007, Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Implementation of POSIX asynchronous I/O. 31 * Defined in the Base Definitions volume of IEEE Std 1003.1-2001. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.20 2008/11/14 23:33:45 ad Exp $"); 36 37 #ifdef _KERNEL_OPT 38 #include "opt_ddb.h" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/condvar.h> 43 #include <sys/file.h> 44 #include <sys/filedesc.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/lwp.h> 48 #include <sys/mutex.h> 49 #include <sys/pool.h> 50 #include <sys/proc.h> 51 #include <sys/queue.h> 52 #include <sys/signal.h> 53 #include <sys/signalvar.h> 54 #include <sys/syscall.h> 55 #include <sys/syscallargs.h> 56 #include <sys/syscallvar.h> 57 #include <sys/sysctl.h> 58 #include <sys/systm.h> 59 #include <sys/types.h> 60 #include <sys/vnode.h> 61 #include <sys/atomic.h> 62 #include <sys/module.h> 63 64 #include <uvm/uvm_extern.h> 65 66 MODULE(MODULE_CLASS_MISC, aio, NULL); 67 68 /* 69 * System-wide limits and counter of AIO operations. 70 */ 71 static u_int aio_listio_max = AIO_LISTIO_MAX; 72 static u_int aio_max = AIO_MAX; 73 static u_int aio_jobs_count; 74 75 static struct pool aio_job_pool; 76 static struct pool aio_lio_pool; 77 static void *aio_ehook; 78 79 /* Prototypes */ 80 void aio_worker(void *); 81 static void aio_process(struct aio_job *); 82 static void aio_sendsig(struct proc *, struct sigevent *); 83 static int aio_enqueue_job(int, void *, struct lio_req *); 84 static void aio_exit(proc_t *, void *); 85 86 static const struct syscall_package aio_syscalls[] = { 87 { SYS_aio_cancel, 0, (sy_call_t *)sys_aio_cancel }, 88 { SYS_aio_error, 0, (sy_call_t *)sys_aio_error }, 89 { SYS_aio_fsync, 0, (sy_call_t *)sys_aio_fsync }, 90 { SYS_aio_read, 0, (sy_call_t *)sys_aio_read }, 91 { SYS_aio_return, 0, (sy_call_t *)sys_aio_return }, 92 { SYS_aio_suspend, 0, (sy_call_t *)sys_aio_suspend }, 93 { SYS_aio_write, 0, (sy_call_t *)sys_aio_write }, 94 { SYS_lio_listio, 0, (sy_call_t *)sys_lio_listio }, 95 { 0, 0, NULL }, 96 }; 97 98 /* 99 * Tear down all AIO state. 100 */ 101 static int 102 aio_fini(bool interface) 103 { 104 int error; 105 proc_t *p; 106 107 if (interface) { 108 /* Stop syscall activity. */ 109 error = syscall_disestablish(NULL, aio_syscalls); 110 if (error != 0) 111 return error; 112 /* Abort if any processes are using AIO. */ 113 mutex_enter(proc_lock); 114 PROCLIST_FOREACH(p, &allproc) { 115 if (p->p_aio != NULL) 116 break; 117 } 118 mutex_exit(proc_lock); 119 if (p != NULL) { 120 error = syscall_establish(NULL, aio_syscalls); 121 KASSERT(error == 0); 122 return EBUSY; 123 } 124 } 125 KASSERT(aio_jobs_count == 0); 126 exithook_disestablish(aio_ehook); 127 pool_destroy(&aio_job_pool); 128 pool_destroy(&aio_lio_pool); 129 return 0; 130 } 131 132 /* 133 * Initialize global AIO state. 134 */ 135 static int 136 aio_init(void) 137 { 138 int error; 139 140 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 141 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 142 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 143 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 144 aio_ehook = exithook_establish(aio_exit, NULL); 145 error = syscall_establish(NULL, aio_syscalls); 146 if (error != 0) 147 aio_fini(false); 148 return error; 149 } 150 151 /* 152 * Module interface. 153 */ 154 static int 155 aio_modcmd(modcmd_t cmd, void *arg) 156 { 157 158 switch (cmd) { 159 case MODULE_CMD_INIT: 160 return aio_init(); 161 case MODULE_CMD_FINI: 162 return aio_fini(true); 163 default: 164 return ENOTTY; 165 } 166 } 167 168 /* 169 * Initialize Asynchronous I/O data structures for the process. 170 */ 171 static int 172 aio_procinit(struct proc *p) 173 { 174 struct aioproc *aio; 175 struct lwp *l; 176 int error; 177 bool inmem; 178 vaddr_t uaddr; 179 180 /* Allocate and initialize AIO structure */ 181 aio = kmem_zalloc(sizeof(struct aioproc), KM_SLEEP); 182 if (aio == NULL) 183 return EAGAIN; 184 185 /* Initialize queue and their synchronization structures */ 186 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 187 cv_init(&aio->aio_worker_cv, "aiowork"); 188 cv_init(&aio->done_cv, "aiodone"); 189 TAILQ_INIT(&aio->jobs_queue); 190 191 /* 192 * Create an AIO worker thread. 193 * XXX: Currently, AIO thread is not protected against user's actions. 194 */ 195 inmem = uvm_uarea_alloc(&uaddr); 196 if (uaddr == 0) { 197 aio_exit(p, aio); 198 return EAGAIN; 199 } 200 error = lwp_create(curlwp, p, uaddr, inmem, 0, NULL, 0, aio_worker, 201 NULL, &l, curlwp->l_class); 202 if (error != 0) { 203 uvm_uarea_free(uaddr, curcpu()); 204 aio_exit(p, aio); 205 return error; 206 } 207 208 /* Recheck if we are really first */ 209 mutex_enter(p->p_lock); 210 if (p->p_aio) { 211 mutex_exit(p->p_lock); 212 aio_exit(p, aio); 213 lwp_exit(l); 214 return 0; 215 } 216 p->p_aio = aio; 217 218 /* Complete the initialization of thread, and run it */ 219 aio->aio_worker = l; 220 p->p_nrlwps++; 221 lwp_lock(l); 222 l->l_stat = LSRUN; 223 l->l_priority = MAXPRI_USER; 224 sched_enqueue(l, false); 225 lwp_unlock(l); 226 mutex_exit(p->p_lock); 227 228 return 0; 229 } 230 231 /* 232 * Exit of Asynchronous I/O subsystem of process. 233 */ 234 static void 235 aio_exit(struct proc *p, void *cookie) 236 { 237 struct aio_job *a_job; 238 struct aioproc *aio; 239 240 if (cookie != NULL) 241 aio = cookie; 242 else if ((aio = p->p_aio) == NULL) 243 return; 244 245 /* Free AIO queue */ 246 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 247 a_job = TAILQ_FIRST(&aio->jobs_queue); 248 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 249 pool_put(&aio_job_pool, a_job); 250 atomic_dec_uint(&aio_jobs_count); 251 } 252 253 /* Destroy and free the entire AIO data structure */ 254 cv_destroy(&aio->aio_worker_cv); 255 cv_destroy(&aio->done_cv); 256 mutex_destroy(&aio->aio_mtx); 257 kmem_free(aio, sizeof(struct aioproc)); 258 } 259 260 /* 261 * AIO worker thread and processor. 262 */ 263 void 264 aio_worker(void *arg) 265 { 266 struct proc *p = curlwp->l_proc; 267 struct aioproc *aio = p->p_aio; 268 struct aio_job *a_job; 269 struct lio_req *lio; 270 sigset_t oss, nss; 271 int error, refcnt; 272 273 /* 274 * Make an empty signal mask, so it 275 * handles only SIGKILL and SIGSTOP. 276 */ 277 sigfillset(&nss); 278 mutex_enter(p->p_lock); 279 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 280 mutex_exit(p->p_lock); 281 KASSERT(error == 0); 282 283 for (;;) { 284 /* 285 * Loop for each job in the queue. If there 286 * are no jobs then sleep. 287 */ 288 mutex_enter(&aio->aio_mtx); 289 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 290 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 291 /* 292 * Thread was interrupted - check for 293 * pending exit or suspend. 294 */ 295 mutex_exit(&aio->aio_mtx); 296 lwp_userret(curlwp); 297 mutex_enter(&aio->aio_mtx); 298 } 299 } 300 301 /* Take the job from the queue */ 302 aio->curjob = a_job; 303 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 304 305 atomic_dec_uint(&aio_jobs_count); 306 aio->jobs_count--; 307 308 mutex_exit(&aio->aio_mtx); 309 310 /* Process an AIO operation */ 311 aio_process(a_job); 312 313 /* Copy data structure back to the user-space */ 314 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 315 sizeof(struct aiocb)); 316 317 mutex_enter(&aio->aio_mtx); 318 aio->curjob = NULL; 319 320 /* Decrease a reference counter, if there is a LIO structure */ 321 lio = a_job->lio; 322 refcnt = (lio != NULL ? --lio->refcnt : -1); 323 324 /* Notify all suspenders */ 325 cv_broadcast(&aio->done_cv); 326 mutex_exit(&aio->aio_mtx); 327 328 /* Send a signal, if any */ 329 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 330 331 /* Destroy the LIO structure */ 332 if (refcnt == 0) { 333 aio_sendsig(p, &lio->sig); 334 pool_put(&aio_lio_pool, lio); 335 } 336 337 /* Destroy the the job */ 338 pool_put(&aio_job_pool, a_job); 339 } 340 341 /* NOTREACHED */ 342 } 343 344 static void 345 aio_process(struct aio_job *a_job) 346 { 347 struct proc *p = curlwp->l_proc; 348 struct aiocb *aiocbp = &a_job->aiocbp; 349 struct file *fp; 350 int fd = aiocbp->aio_fildes; 351 int error = 0; 352 353 KASSERT(a_job->aio_op != 0); 354 355 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 356 struct iovec aiov; 357 struct uio auio; 358 359 if (aiocbp->aio_nbytes > SSIZE_MAX) { 360 error = EINVAL; 361 goto done; 362 } 363 364 fp = fd_getfile(fd); 365 if (fp == NULL) { 366 error = EBADF; 367 goto done; 368 } 369 370 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 371 aiov.iov_len = aiocbp->aio_nbytes; 372 auio.uio_iov = &aiov; 373 auio.uio_iovcnt = 1; 374 auio.uio_resid = aiocbp->aio_nbytes; 375 auio.uio_vmspace = p->p_vmspace; 376 377 if (a_job->aio_op & AIO_READ) { 378 /* 379 * Perform a Read operation 380 */ 381 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 382 383 if ((fp->f_flag & FREAD) == 0) { 384 fd_putfile(fd); 385 error = EBADF; 386 goto done; 387 } 388 auio.uio_rw = UIO_READ; 389 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 390 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 391 } else { 392 /* 393 * Perform a Write operation 394 */ 395 KASSERT(a_job->aio_op & AIO_WRITE); 396 397 if ((fp->f_flag & FWRITE) == 0) { 398 fd_putfile(fd); 399 error = EBADF; 400 goto done; 401 } 402 auio.uio_rw = UIO_WRITE; 403 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 404 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 405 } 406 fd_putfile(fd); 407 408 /* Store the result value */ 409 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 410 a_job->aiocbp._retval = (error == 0) ? 411 a_job->aiocbp.aio_nbytes : -1; 412 413 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 414 /* 415 * Perform a file Sync operation 416 */ 417 struct vnode *vp; 418 419 if ((error = fd_getvnode(fd, &fp)) != 0) 420 goto done; 421 422 if ((fp->f_flag & FWRITE) == 0) { 423 fd_putfile(fd); 424 error = EBADF; 425 goto done; 426 } 427 428 vp = (struct vnode *)fp->f_data; 429 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 430 if (a_job->aio_op & AIO_DSYNC) { 431 error = VOP_FSYNC(vp, fp->f_cred, 432 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0); 433 } else if (a_job->aio_op & AIO_SYNC) { 434 error = VOP_FSYNC(vp, fp->f_cred, 435 FSYNC_WAIT, 0, 0); 436 if (error == 0 && bioopsp != NULL && 437 vp->v_mount && 438 (vp->v_mount->mnt_flag & MNT_SOFTDEP)) 439 bioopsp->io_fsync(vp, 0); 440 } 441 VOP_UNLOCK(vp, 0); 442 fd_putfile(fd); 443 444 /* Store the result value */ 445 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 446 447 } else 448 panic("aio_process: invalid operation code\n"); 449 450 done: 451 /* Job is done, set the error, if any */ 452 a_job->aiocbp._errno = error; 453 a_job->aiocbp._state = JOB_DONE; 454 } 455 456 /* 457 * Send AIO signal. 458 */ 459 static void 460 aio_sendsig(struct proc *p, struct sigevent *sig) 461 { 462 ksiginfo_t ksi; 463 464 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 465 return; 466 467 KSI_INIT(&ksi); 468 ksi.ksi_signo = sig->sigev_signo; 469 ksi.ksi_code = SI_ASYNCIO; 470 ksi.ksi_value = sig->sigev_value; 471 mutex_enter(proc_lock); 472 kpsignal(p, &ksi, NULL); 473 mutex_exit(proc_lock); 474 } 475 476 /* 477 * Enqueue the job. 478 */ 479 static int 480 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 481 { 482 struct proc *p = curlwp->l_proc; 483 struct aioproc *aio; 484 struct aio_job *a_job; 485 struct aiocb aiocbp; 486 struct sigevent *sig; 487 int error; 488 489 /* Non-accurate check for the limit */ 490 if (aio_jobs_count + 1 > aio_max) 491 return EAGAIN; 492 493 /* Get the data structure from user-space */ 494 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 495 if (error) 496 return error; 497 498 /* Check if signal is set, and validate it */ 499 sig = &aiocbp.aio_sigevent; 500 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 501 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 502 return EINVAL; 503 504 /* Buffer and byte count */ 505 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 506 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 507 return EINVAL; 508 509 /* Check the opcode, if LIO_NOP - simply ignore */ 510 if (op == AIO_LIO) { 511 KASSERT(lio != NULL); 512 if (aiocbp.aio_lio_opcode == LIO_WRITE) 513 op = AIO_WRITE; 514 else if (aiocbp.aio_lio_opcode == LIO_READ) 515 op = AIO_READ; 516 else 517 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 518 } else { 519 KASSERT(lio == NULL); 520 } 521 522 /* 523 * Look for already existing job. If found - the job is in-progress. 524 * According to POSIX this is invalid, so return the error. 525 */ 526 aio = p->p_aio; 527 if (aio) { 528 mutex_enter(&aio->aio_mtx); 529 if (aio->curjob) { 530 a_job = aio->curjob; 531 if (a_job->aiocb_uptr == aiocb_uptr) { 532 mutex_exit(&aio->aio_mtx); 533 return EINVAL; 534 } 535 } 536 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 537 if (a_job->aiocb_uptr != aiocb_uptr) 538 continue; 539 mutex_exit(&aio->aio_mtx); 540 return EINVAL; 541 } 542 mutex_exit(&aio->aio_mtx); 543 } 544 545 /* 546 * Check if AIO structure is initialized, if not - initialize it. 547 * In LIO case, we did that already. We will recheck this with 548 * the lock in aio_procinit(). 549 */ 550 if (lio == NULL && p->p_aio == NULL) 551 if (aio_procinit(p)) 552 return EAGAIN; 553 aio = p->p_aio; 554 555 /* 556 * Set the state with errno, and copy data 557 * structure back to the user-space. 558 */ 559 aiocbp._state = JOB_WIP; 560 aiocbp._errno = EINPROGRESS; 561 aiocbp._retval = -1; 562 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 563 if (error) 564 return error; 565 566 /* Allocate and initialize a new AIO job */ 567 a_job = pool_get(&aio_job_pool, PR_WAITOK); 568 memset(a_job, 0, sizeof(struct aio_job)); 569 570 /* 571 * Set the data. 572 * Store the user-space pointer for searching. Since we 573 * are storing only per proc pointers - it is safe. 574 */ 575 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 576 a_job->aiocb_uptr = aiocb_uptr; 577 a_job->aio_op |= op; 578 a_job->lio = lio; 579 580 /* 581 * Add the job to the queue, update the counters, and 582 * notify the AIO worker thread to handle the job. 583 */ 584 mutex_enter(&aio->aio_mtx); 585 586 /* Fail, if the limit was reached */ 587 if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max || 588 aio->jobs_count >= aio_listio_max) { 589 atomic_dec_uint(&aio_jobs_count); 590 mutex_exit(&aio->aio_mtx); 591 pool_put(&aio_job_pool, a_job); 592 return EAGAIN; 593 } 594 595 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 596 aio->jobs_count++; 597 if (lio) 598 lio->refcnt++; 599 cv_signal(&aio->aio_worker_cv); 600 601 mutex_exit(&aio->aio_mtx); 602 603 /* 604 * One would handle the errors only with aio_error() function. 605 * This way is appropriate according to POSIX. 606 */ 607 return 0; 608 } 609 610 /* 611 * Syscall functions. 612 */ 613 614 int 615 sys_aio_cancel(struct lwp *l, const struct sys_aio_cancel_args *uap, register_t *retval) 616 { 617 /* { 618 syscallarg(int) fildes; 619 syscallarg(struct aiocb *) aiocbp; 620 } */ 621 struct proc *p = l->l_proc; 622 struct aioproc *aio; 623 struct aio_job *a_job; 624 struct aiocb *aiocbp_ptr; 625 struct lio_req *lio; 626 struct filedesc *fdp = p->p_fd; 627 unsigned int cn, errcnt, fildes; 628 629 TAILQ_HEAD(, aio_job) tmp_jobs_list; 630 631 /* Check for invalid file descriptor */ 632 fildes = (unsigned int)SCARG(uap, fildes); 633 if (fildes >= fdp->fd_nfiles) 634 return EBADF; 635 membar_consumer(); 636 if (fdp->fd_ofiles[fildes] == NULL || fdp->fd_ofiles[fildes]->ff_file == NULL) 637 return EBADF; 638 639 /* Check if AIO structure is initialized */ 640 if (p->p_aio == NULL) { 641 *retval = AIO_NOTCANCELED; 642 return 0; 643 } 644 645 aio = p->p_aio; 646 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 647 648 mutex_enter(&aio->aio_mtx); 649 650 /* Cancel the jobs, and remove them from the queue */ 651 cn = 0; 652 TAILQ_INIT(&tmp_jobs_list); 653 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 654 if (aiocbp_ptr) { 655 if (aiocbp_ptr != a_job->aiocb_uptr) 656 continue; 657 if (fildes != a_job->aiocbp.aio_fildes) { 658 mutex_exit(&aio->aio_mtx); 659 return EBADF; 660 } 661 } else if (a_job->aiocbp.aio_fildes != fildes) 662 continue; 663 664 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 665 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 666 667 /* Decrease the counters */ 668 atomic_dec_uint(&aio_jobs_count); 669 aio->jobs_count--; 670 lio = a_job->lio; 671 if (lio != NULL && --lio->refcnt != 0) 672 a_job->lio = NULL; 673 674 cn++; 675 if (aiocbp_ptr) 676 break; 677 } 678 679 /* There are canceled jobs */ 680 if (cn) 681 *retval = AIO_CANCELED; 682 683 /* We cannot cancel current job */ 684 a_job = aio->curjob; 685 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 686 (a_job->aiocb_uptr == aiocbp_ptr))) 687 *retval = AIO_NOTCANCELED; 688 689 mutex_exit(&aio->aio_mtx); 690 691 /* Free the jobs after the lock */ 692 errcnt = 0; 693 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 694 a_job = TAILQ_FIRST(&tmp_jobs_list); 695 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 696 /* Set the errno and copy structures back to the user-space */ 697 a_job->aiocbp._errno = ECANCELED; 698 a_job->aiocbp._state = JOB_DONE; 699 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 700 sizeof(struct aiocb))) 701 errcnt++; 702 /* Send a signal if any */ 703 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 704 if (a_job->lio) { 705 lio = a_job->lio; 706 aio_sendsig(p, &lio->sig); 707 pool_put(&aio_lio_pool, lio); 708 } 709 pool_put(&aio_job_pool, a_job); 710 } 711 712 if (errcnt) 713 return EFAULT; 714 715 /* Set a correct return value */ 716 if (*retval == 0) 717 *retval = AIO_ALLDONE; 718 719 return 0; 720 } 721 722 int 723 sys_aio_error(struct lwp *l, const struct sys_aio_error_args *uap, register_t *retval) 724 { 725 /* { 726 syscallarg(const struct aiocb *) aiocbp; 727 } */ 728 struct proc *p = l->l_proc; 729 struct aioproc *aio = p->p_aio; 730 struct aiocb aiocbp; 731 int error; 732 733 if (aio == NULL) 734 return EINVAL; 735 736 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 737 if (error) 738 return error; 739 740 if (aiocbp._state == JOB_NONE) 741 return EINVAL; 742 743 *retval = aiocbp._errno; 744 745 return 0; 746 } 747 748 int 749 sys_aio_fsync(struct lwp *l, const struct sys_aio_fsync_args *uap, register_t *retval) 750 { 751 /* { 752 syscallarg(int) op; 753 syscallarg(struct aiocb *) aiocbp; 754 } */ 755 int op = SCARG(uap, op); 756 757 if ((op != O_DSYNC) && (op != O_SYNC)) 758 return EINVAL; 759 760 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 761 762 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 763 } 764 765 int 766 sys_aio_read(struct lwp *l, const struct sys_aio_read_args *uap, register_t *retval) 767 { 768 /* { 769 syscallarg(struct aiocb *) aiocbp; 770 } */ 771 772 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 773 } 774 775 int 776 sys_aio_return(struct lwp *l, const struct sys_aio_return_args *uap, register_t *retval) 777 { 778 /* { 779 syscallarg(struct aiocb *) aiocbp; 780 } */ 781 struct proc *p = l->l_proc; 782 struct aioproc *aio = p->p_aio; 783 struct aiocb aiocbp; 784 int error; 785 786 if (aio == NULL) 787 return EINVAL; 788 789 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 790 if (error) 791 return error; 792 793 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 794 return EINVAL; 795 796 *retval = aiocbp._retval; 797 798 /* Reset the internal variables */ 799 aiocbp._errno = 0; 800 aiocbp._retval = -1; 801 aiocbp._state = JOB_NONE; 802 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 803 804 return error; 805 } 806 807 int 808 sys_aio_suspend(struct lwp *l, const struct sys_aio_suspend_args *uap, register_t *retval) 809 { 810 /* { 811 syscallarg(const struct aiocb *const[]) list; 812 syscallarg(int) nent; 813 syscallarg(const struct timespec *) timeout; 814 } */ 815 struct proc *p = l->l_proc; 816 struct aioproc *aio; 817 struct aio_job *a_job; 818 struct aiocb **aiocbp_list; 819 struct timespec ts; 820 int i, error, nent, timo; 821 822 if (p->p_aio == NULL) 823 return EAGAIN; 824 aio = p->p_aio; 825 826 nent = SCARG(uap, nent); 827 if (nent <= 0 || nent > aio_listio_max) 828 return EAGAIN; 829 830 if (SCARG(uap, timeout)) { 831 /* Convert timespec to ticks */ 832 error = copyin(SCARG(uap, timeout), &ts, 833 sizeof(struct timespec)); 834 if (error) 835 return error; 836 timo = mstohz((ts.tv_sec * 1000) + (ts.tv_nsec / 1000000)); 837 if (timo == 0 && ts.tv_sec == 0 && ts.tv_nsec > 0) 838 timo = 1; 839 if (timo <= 0) 840 return EAGAIN; 841 } else 842 timo = 0; 843 844 /* Get the list from user-space */ 845 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 846 error = copyin(SCARG(uap, list), aiocbp_list, 847 nent * sizeof(struct aiocb)); 848 if (error) { 849 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 850 return error; 851 } 852 853 mutex_enter(&aio->aio_mtx); 854 for (;;) { 855 856 for (i = 0; i < nent; i++) { 857 858 /* Skip NULL entries */ 859 if (aiocbp_list[i] == NULL) 860 continue; 861 862 /* Skip current job */ 863 if (aio->curjob) { 864 a_job = aio->curjob; 865 if (a_job->aiocb_uptr == aiocbp_list[i]) 866 continue; 867 } 868 869 /* Look for a job in the queue */ 870 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 871 if (a_job->aiocb_uptr == aiocbp_list[i]) 872 break; 873 874 if (a_job == NULL) { 875 struct aiocb aiocbp; 876 877 mutex_exit(&aio->aio_mtx); 878 879 error = copyin(aiocbp_list[i], &aiocbp, 880 sizeof(struct aiocb)); 881 if (error == 0 && aiocbp._state != JOB_DONE) { 882 mutex_enter(&aio->aio_mtx); 883 continue; 884 } 885 886 kmem_free(aiocbp_list, 887 nent * sizeof(struct aio_job)); 888 return error; 889 } 890 } 891 892 /* Wait for a signal or when timeout occurs */ 893 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 894 if (error) { 895 if (error == EWOULDBLOCK) 896 error = EAGAIN; 897 break; 898 } 899 } 900 mutex_exit(&aio->aio_mtx); 901 902 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 903 return error; 904 } 905 906 int 907 sys_aio_write(struct lwp *l, const struct sys_aio_write_args *uap, register_t *retval) 908 { 909 /* { 910 syscallarg(struct aiocb *) aiocbp; 911 } */ 912 913 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 914 } 915 916 int 917 sys_lio_listio(struct lwp *l, const struct sys_lio_listio_args *uap, register_t *retval) 918 { 919 /* { 920 syscallarg(int) mode; 921 syscallarg(struct aiocb *const[]) list; 922 syscallarg(int) nent; 923 syscallarg(struct sigevent *) sig; 924 } */ 925 struct proc *p = l->l_proc; 926 struct aioproc *aio; 927 struct aiocb **aiocbp_list; 928 struct lio_req *lio; 929 int i, error, errcnt, mode, nent; 930 931 mode = SCARG(uap, mode); 932 nent = SCARG(uap, nent); 933 934 /* Non-accurate checks for the limit and invalid values */ 935 if (nent < 1 || nent > aio_listio_max) 936 return EINVAL; 937 if (aio_jobs_count + nent > aio_max) 938 return EAGAIN; 939 940 /* Check if AIO structure is initialized, if not - initialize it */ 941 if (p->p_aio == NULL) 942 if (aio_procinit(p)) 943 return EAGAIN; 944 aio = p->p_aio; 945 946 /* Create a LIO structure */ 947 lio = pool_get(&aio_lio_pool, PR_WAITOK); 948 lio->refcnt = 1; 949 error = 0; 950 951 switch (mode) { 952 case LIO_WAIT: 953 memset(&lio->sig, 0, sizeof(struct sigevent)); 954 break; 955 case LIO_NOWAIT: 956 /* Check for signal, validate it */ 957 if (SCARG(uap, sig)) { 958 struct sigevent *sig = &lio->sig; 959 960 error = copyin(SCARG(uap, sig), &lio->sig, 961 sizeof(struct sigevent)); 962 if (error == 0 && 963 (sig->sigev_signo < 0 || 964 sig->sigev_signo >= NSIG || 965 sig->sigev_notify < SIGEV_NONE || 966 sig->sigev_notify > SIGEV_SA)) 967 error = EINVAL; 968 } else 969 memset(&lio->sig, 0, sizeof(struct sigevent)); 970 break; 971 default: 972 error = EINVAL; 973 break; 974 } 975 976 if (error != 0) { 977 pool_put(&aio_lio_pool, lio); 978 return error; 979 } 980 981 /* Get the list from user-space */ 982 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 983 error = copyin(SCARG(uap, list), aiocbp_list, 984 nent * sizeof(struct aiocb)); 985 if (error) { 986 mutex_enter(&aio->aio_mtx); 987 goto err; 988 } 989 990 /* Enqueue all jobs */ 991 errcnt = 0; 992 for (i = 0; i < nent; i++) { 993 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 994 /* 995 * According to POSIX, in such error case it may 996 * fail with other I/O operations initiated. 997 */ 998 if (error) 999 errcnt++; 1000 } 1001 1002 mutex_enter(&aio->aio_mtx); 1003 1004 /* Return an error, if any */ 1005 if (errcnt) { 1006 error = EIO; 1007 goto err; 1008 } 1009 1010 if (mode == LIO_WAIT) { 1011 /* 1012 * Wait for AIO completion. In such case, 1013 * the LIO structure will be freed here. 1014 */ 1015 while (lio->refcnt > 1 && error == 0) 1016 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 1017 if (error) 1018 error = EINTR; 1019 } 1020 1021 err: 1022 if (--lio->refcnt != 0) 1023 lio = NULL; 1024 mutex_exit(&aio->aio_mtx); 1025 if (lio != NULL) { 1026 aio_sendsig(p, &lio->sig); 1027 pool_put(&aio_lio_pool, lio); 1028 } 1029 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 1030 return error; 1031 } 1032 1033 /* 1034 * SysCtl 1035 */ 1036 1037 static int 1038 sysctl_aio_listio_max(SYSCTLFN_ARGS) 1039 { 1040 struct sysctlnode node; 1041 int error, newsize; 1042 1043 node = *rnode; 1044 node.sysctl_data = &newsize; 1045 1046 newsize = aio_listio_max; 1047 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1048 if (error || newp == NULL) 1049 return error; 1050 1051 if (newsize < 1 || newsize > aio_max) 1052 return EINVAL; 1053 aio_listio_max = newsize; 1054 1055 return 0; 1056 } 1057 1058 static int 1059 sysctl_aio_max(SYSCTLFN_ARGS) 1060 { 1061 struct sysctlnode node; 1062 int error, newsize; 1063 1064 node = *rnode; 1065 node.sysctl_data = &newsize; 1066 1067 newsize = aio_max; 1068 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1069 if (error || newp == NULL) 1070 return error; 1071 1072 if (newsize < 1 || newsize < aio_listio_max) 1073 return EINVAL; 1074 aio_max = newsize; 1075 1076 return 0; 1077 } 1078 1079 SYSCTL_SETUP(sysctl_aio_setup, "sysctl aio setup") 1080 { 1081 1082 sysctl_createv(clog, 0, NULL, NULL, 1083 CTLFLAG_PERMANENT, 1084 CTLTYPE_NODE, "kern", NULL, 1085 NULL, 0, NULL, 0, 1086 CTL_KERN, CTL_EOL); 1087 sysctl_createv(clog, 0, NULL, NULL, 1088 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1089 CTLTYPE_INT, "posix_aio", 1090 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1091 "Asynchronous I/O option to which the " 1092 "system attempts to conform"), 1093 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1094 CTL_KERN, CTL_CREATE, CTL_EOL); 1095 sysctl_createv(clog, 0, NULL, NULL, 1096 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1097 CTLTYPE_INT, "aio_listio_max", 1098 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1099 "operations in a single list I/O call"), 1100 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1101 CTL_KERN, CTL_CREATE, CTL_EOL); 1102 sysctl_createv(clog, 0, NULL, NULL, 1103 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1104 CTLTYPE_INT, "aio_max", 1105 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1106 "operations"), 1107 sysctl_aio_max, 0, &aio_max, 0, 1108 CTL_KERN, CTL_CREATE, CTL_EOL); 1109 } 1110 1111 /* 1112 * Debugging 1113 */ 1114 #if defined(DDB) 1115 void 1116 aio_print_jobs(void (*pr)(const char *, ...)) 1117 { 1118 struct proc *p = (curlwp == NULL ? NULL : curlwp->l_proc); 1119 struct aioproc *aio; 1120 struct aio_job *a_job; 1121 struct aiocb *aiocbp; 1122 1123 if (p == NULL) { 1124 (*pr)("AIO: We are not in the processes right now.\n"); 1125 return; 1126 } 1127 1128 aio = p->p_aio; 1129 if (aio == NULL) { 1130 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1131 return; 1132 } 1133 1134 (*pr)("AIO: PID = %d\n", p->p_pid); 1135 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1136 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1137 1138 if (aio->curjob) { 1139 a_job = aio->curjob; 1140 (*pr)("\nAIO current job:\n"); 1141 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1142 a_job->aio_op, a_job->aiocbp._errno, 1143 a_job->aiocbp._state, a_job->aiocb_uptr); 1144 aiocbp = &a_job->aiocbp; 1145 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1146 aiocbp->aio_fildes, aiocbp->aio_offset, 1147 aiocbp->aio_buf, aiocbp->aio_nbytes); 1148 } 1149 1150 (*pr)("\nAIO queue:\n"); 1151 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1152 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1153 a_job->aio_op, a_job->aiocbp._errno, 1154 a_job->aiocbp._state, a_job->aiocb_uptr); 1155 aiocbp = &a_job->aiocbp; 1156 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1157 aiocbp->aio_fildes, aiocbp->aio_offset, 1158 aiocbp->aio_buf, aiocbp->aio_nbytes); 1159 } 1160 } 1161 #endif /* defined(DDB) */ 1162
Indexes created Wed Sep 24 05:09:52 GMT 2025