sys_aio.c revision 1.18.2.1
1 /* $NetBSD: sys_aio.c,v 1.18.2.1 2009/05/04 08:13:48 yamt Exp $ */ 2 3 /* 4 * Copyright (c) 2007, Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Implementation of POSIX asynchronous I/O. 31 * Defined in the Base Definitions volume of IEEE Std 1003.1-2001. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.18.2.1 2009/05/04 08:13:48 yamt Exp $"); 36 37 #ifdef _KERNEL_OPT 38 #include "opt_ddb.h" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/condvar.h> 43 #include <sys/file.h> 44 #include <sys/filedesc.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/lwp.h> 48 #include <sys/mutex.h> 49 #include <sys/pool.h> 50 #include <sys/proc.h> 51 #include <sys/queue.h> 52 #include <sys/signal.h> 53 #include <sys/signalvar.h> 54 #include <sys/syscall.h> 55 #include <sys/syscallargs.h> 56 #include <sys/syscallvar.h> 57 #include <sys/sysctl.h> 58 #include <sys/systm.h> 59 #include <sys/types.h> 60 #include <sys/vnode.h> 61 #include <sys/atomic.h> 62 #include <sys/module.h> 63 #include <sys/buf.h> 64 65 #include <uvm/uvm_extern.h> 66 67 MODULE(MODULE_CLASS_MISC, aio, NULL); 68 69 /* 70 * System-wide limits and counter of AIO operations. 71 */ 72 u_int aio_listio_max = AIO_LISTIO_MAX; 73 static u_int aio_max = AIO_MAX; 74 static u_int aio_jobs_count; 75 76 static struct pool aio_job_pool; 77 static struct pool aio_lio_pool; 78 static void *aio_ehook; 79 80 /* Prototypes */ 81 void aio_worker(void *); 82 static void aio_process(struct aio_job *); 83 static void aio_sendsig(struct proc *, struct sigevent *); 84 static int aio_enqueue_job(int, void *, struct lio_req *); 85 static void aio_exit(proc_t *, void *); 86 87 static const struct syscall_package aio_syscalls[] = { 88 { SYS_aio_cancel, 0, (sy_call_t *)sys_aio_cancel }, 89 { SYS_aio_error, 0, (sy_call_t *)sys_aio_error }, 90 { SYS_aio_fsync, 0, (sy_call_t *)sys_aio_fsync }, 91 { SYS_aio_read, 0, (sy_call_t *)sys_aio_read }, 92 { SYS_aio_return, 0, (sy_call_t *)sys_aio_return }, 93 { SYS___aio_suspend50, 0, (sy_call_t *)sys___aio_suspend50 }, 94 { SYS_aio_write, 0, (sy_call_t *)sys_aio_write }, 95 { SYS_lio_listio, 0, (sy_call_t *)sys_lio_listio }, 96 { 0, 0, NULL }, 97 }; 98 99 /* 100 * Tear down all AIO state. 101 */ 102 static int 103 aio_fini(bool interface) 104 { 105 int error; 106 proc_t *p; 107 108 if (interface) { 109 /* Stop syscall activity. */ 110 error = syscall_disestablish(NULL, aio_syscalls); 111 if (error != 0) 112 return error; 113 /* Abort if any processes are using AIO. */ 114 mutex_enter(proc_lock); 115 PROCLIST_FOREACH(p, &allproc) { 116 if (p->p_aio != NULL) 117 break; 118 } 119 mutex_exit(proc_lock); 120 if (p != NULL) { 121 error = syscall_establish(NULL, aio_syscalls); 122 KASSERT(error == 0); 123 return EBUSY; 124 } 125 } 126 KASSERT(aio_jobs_count == 0); 127 exithook_disestablish(aio_ehook); 128 pool_destroy(&aio_job_pool); 129 pool_destroy(&aio_lio_pool); 130 return 0; 131 } 132 133 /* 134 * Initialize global AIO state. 135 */ 136 static int 137 aio_init(void) 138 { 139 int error; 140 141 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 142 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 143 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 144 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 145 aio_ehook = exithook_establish(aio_exit, NULL); 146 error = syscall_establish(NULL, aio_syscalls); 147 if (error != 0) 148 aio_fini(false); 149 return error; 150 } 151 152 /* 153 * Module interface. 154 */ 155 static int 156 aio_modcmd(modcmd_t cmd, void *arg) 157 { 158 159 switch (cmd) { 160 case MODULE_CMD_INIT: 161 return aio_init(); 162 case MODULE_CMD_FINI: 163 return aio_fini(true); 164 default: 165 return ENOTTY; 166 } 167 } 168 169 /* 170 * Initialize Asynchronous I/O data structures for the process. 171 */ 172 static int 173 aio_procinit(struct proc *p) 174 { 175 struct aioproc *aio; 176 struct lwp *l; 177 int error; 178 bool inmem; 179 vaddr_t uaddr; 180 181 /* Allocate and initialize AIO structure */ 182 aio = kmem_zalloc(sizeof(struct aioproc), KM_SLEEP); 183 if (aio == NULL) 184 return EAGAIN; 185 186 /* Initialize queue and their synchronization structures */ 187 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 188 cv_init(&aio->aio_worker_cv, "aiowork"); 189 cv_init(&aio->done_cv, "aiodone"); 190 TAILQ_INIT(&aio->jobs_queue); 191 192 /* 193 * Create an AIO worker thread. 194 * XXX: Currently, AIO thread is not protected against user's actions. 195 */ 196 inmem = uvm_uarea_alloc(&uaddr); 197 if (uaddr == 0) { 198 aio_exit(p, aio); 199 return EAGAIN; 200 } 201 error = lwp_create(curlwp, p, uaddr, inmem, 0, NULL, 0, aio_worker, 202 NULL, &l, curlwp->l_class); 203 if (error != 0) { 204 uvm_uarea_free(uaddr, curcpu()); 205 aio_exit(p, aio); 206 return error; 207 } 208 209 /* Recheck if we are really first */ 210 mutex_enter(p->p_lock); 211 if (p->p_aio) { 212 mutex_exit(p->p_lock); 213 aio_exit(p, aio); 214 lwp_exit(l); 215 return 0; 216 } 217 p->p_aio = aio; 218 219 /* Complete the initialization of thread, and run it */ 220 aio->aio_worker = l; 221 p->p_nrlwps++; 222 lwp_lock(l); 223 l->l_stat = LSRUN; 224 l->l_priority = MAXPRI_USER; 225 sched_enqueue(l, false); 226 lwp_unlock(l); 227 mutex_exit(p->p_lock); 228 229 return 0; 230 } 231 232 /* 233 * Exit of Asynchronous I/O subsystem of process. 234 */ 235 static void 236 aio_exit(struct proc *p, void *cookie) 237 { 238 struct aio_job *a_job; 239 struct aioproc *aio; 240 241 if (cookie != NULL) 242 aio = cookie; 243 else if ((aio = p->p_aio) == NULL) 244 return; 245 246 /* Free AIO queue */ 247 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 248 a_job = TAILQ_FIRST(&aio->jobs_queue); 249 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 250 pool_put(&aio_job_pool, a_job); 251 atomic_dec_uint(&aio_jobs_count); 252 } 253 254 /* Destroy and free the entire AIO data structure */ 255 cv_destroy(&aio->aio_worker_cv); 256 cv_destroy(&aio->done_cv); 257 mutex_destroy(&aio->aio_mtx); 258 kmem_free(aio, sizeof(struct aioproc)); 259 } 260 261 /* 262 * AIO worker thread and processor. 263 */ 264 void 265 aio_worker(void *arg) 266 { 267 struct proc *p = curlwp->l_proc; 268 struct aioproc *aio = p->p_aio; 269 struct aio_job *a_job; 270 struct lio_req *lio; 271 sigset_t oss, nss; 272 int error, refcnt; 273 274 /* 275 * Make an empty signal mask, so it 276 * handles only SIGKILL and SIGSTOP. 277 */ 278 sigfillset(&nss); 279 mutex_enter(p->p_lock); 280 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 281 mutex_exit(p->p_lock); 282 KASSERT(error == 0); 283 284 for (;;) { 285 /* 286 * Loop for each job in the queue. If there 287 * are no jobs then sleep. 288 */ 289 mutex_enter(&aio->aio_mtx); 290 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 291 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 292 /* 293 * Thread was interrupted - check for 294 * pending exit or suspend. 295 */ 296 mutex_exit(&aio->aio_mtx); 297 lwp_userret(curlwp); 298 mutex_enter(&aio->aio_mtx); 299 } 300 } 301 302 /* Take the job from the queue */ 303 aio->curjob = a_job; 304 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 305 306 atomic_dec_uint(&aio_jobs_count); 307 aio->jobs_count--; 308 309 mutex_exit(&aio->aio_mtx); 310 311 /* Process an AIO operation */ 312 aio_process(a_job); 313 314 /* Copy data structure back to the user-space */ 315 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 316 sizeof(struct aiocb)); 317 318 mutex_enter(&aio->aio_mtx); 319 aio->curjob = NULL; 320 321 /* Decrease a reference counter, if there is a LIO structure */ 322 lio = a_job->lio; 323 refcnt = (lio != NULL ? --lio->refcnt : -1); 324 325 /* Notify all suspenders */ 326 cv_broadcast(&aio->done_cv); 327 mutex_exit(&aio->aio_mtx); 328 329 /* Send a signal, if any */ 330 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 331 332 /* Destroy the LIO structure */ 333 if (refcnt == 0) { 334 aio_sendsig(p, &lio->sig); 335 pool_put(&aio_lio_pool, lio); 336 } 337 338 /* Destroy the the job */ 339 pool_put(&aio_job_pool, a_job); 340 } 341 342 /* NOTREACHED */ 343 } 344 345 static void 346 aio_process(struct aio_job *a_job) 347 { 348 struct proc *p = curlwp->l_proc; 349 struct aiocb *aiocbp = &a_job->aiocbp; 350 struct file *fp; 351 int fd = aiocbp->aio_fildes; 352 int error = 0; 353 354 KASSERT(a_job->aio_op != 0); 355 356 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 357 struct iovec aiov; 358 struct uio auio; 359 360 if (aiocbp->aio_nbytes > SSIZE_MAX) { 361 error = EINVAL; 362 goto done; 363 } 364 365 fp = fd_getfile(fd); 366 if (fp == NULL) { 367 error = EBADF; 368 goto done; 369 } 370 371 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 372 aiov.iov_len = aiocbp->aio_nbytes; 373 auio.uio_iov = &aiov; 374 auio.uio_iovcnt = 1; 375 auio.uio_resid = aiocbp->aio_nbytes; 376 auio.uio_vmspace = p->p_vmspace; 377 378 if (a_job->aio_op & AIO_READ) { 379 /* 380 * Perform a Read operation 381 */ 382 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 383 384 if ((fp->f_flag & FREAD) == 0) { 385 fd_putfile(fd); 386 error = EBADF; 387 goto done; 388 } 389 auio.uio_rw = UIO_READ; 390 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 391 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 392 } else { 393 /* 394 * Perform a Write operation 395 */ 396 KASSERT(a_job->aio_op & AIO_WRITE); 397 398 if ((fp->f_flag & FWRITE) == 0) { 399 fd_putfile(fd); 400 error = EBADF; 401 goto done; 402 } 403 auio.uio_rw = UIO_WRITE; 404 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 405 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 406 } 407 fd_putfile(fd); 408 409 /* Store the result value */ 410 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 411 a_job->aiocbp._retval = (error == 0) ? 412 a_job->aiocbp.aio_nbytes : -1; 413 414 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 415 /* 416 * Perform a file Sync operation 417 */ 418 struct vnode *vp; 419 420 if ((error = fd_getvnode(fd, &fp)) != 0) 421 goto done; 422 423 if ((fp->f_flag & FWRITE) == 0) { 424 fd_putfile(fd); 425 error = EBADF; 426 goto done; 427 } 428 429 vp = (struct vnode *)fp->f_data; 430 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 431 if (a_job->aio_op & AIO_DSYNC) { 432 error = VOP_FSYNC(vp, fp->f_cred, 433 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0); 434 } else if (a_job->aio_op & AIO_SYNC) { 435 error = VOP_FSYNC(vp, fp->f_cred, 436 FSYNC_WAIT, 0, 0); 437 } 438 VOP_UNLOCK(vp, 0); 439 fd_putfile(fd); 440 441 /* Store the result value */ 442 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 443 444 } else 445 panic("aio_process: invalid operation code\n"); 446 447 done: 448 /* Job is done, set the error, if any */ 449 a_job->aiocbp._errno = error; 450 a_job->aiocbp._state = JOB_DONE; 451 } 452 453 /* 454 * Send AIO signal. 455 */ 456 static void 457 aio_sendsig(struct proc *p, struct sigevent *sig) 458 { 459 ksiginfo_t ksi; 460 461 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 462 return; 463 464 KSI_INIT(&ksi); 465 ksi.ksi_signo = sig->sigev_signo; 466 ksi.ksi_code = SI_ASYNCIO; 467 ksi.ksi_value = sig->sigev_value; 468 mutex_enter(proc_lock); 469 kpsignal(p, &ksi, NULL); 470 mutex_exit(proc_lock); 471 } 472 473 /* 474 * Enqueue the job. 475 */ 476 static int 477 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 478 { 479 struct proc *p = curlwp->l_proc; 480 struct aioproc *aio; 481 struct aio_job *a_job; 482 struct aiocb aiocbp; 483 struct sigevent *sig; 484 int error; 485 486 /* Non-accurate check for the limit */ 487 if (aio_jobs_count + 1 > aio_max) 488 return EAGAIN; 489 490 /* Get the data structure from user-space */ 491 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 492 if (error) 493 return error; 494 495 /* Check if signal is set, and validate it */ 496 sig = &aiocbp.aio_sigevent; 497 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 498 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 499 return EINVAL; 500 501 /* Buffer and byte count */ 502 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 503 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 504 return EINVAL; 505 506 /* Check the opcode, if LIO_NOP - simply ignore */ 507 if (op == AIO_LIO) { 508 KASSERT(lio != NULL); 509 if (aiocbp.aio_lio_opcode == LIO_WRITE) 510 op = AIO_WRITE; 511 else if (aiocbp.aio_lio_opcode == LIO_READ) 512 op = AIO_READ; 513 else 514 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 515 } else { 516 KASSERT(lio == NULL); 517 } 518 519 /* 520 * Look for already existing job. If found - the job is in-progress. 521 * According to POSIX this is invalid, so return the error. 522 */ 523 aio = p->p_aio; 524 if (aio) { 525 mutex_enter(&aio->aio_mtx); 526 if (aio->curjob) { 527 a_job = aio->curjob; 528 if (a_job->aiocb_uptr == aiocb_uptr) { 529 mutex_exit(&aio->aio_mtx); 530 return EINVAL; 531 } 532 } 533 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 534 if (a_job->aiocb_uptr != aiocb_uptr) 535 continue; 536 mutex_exit(&aio->aio_mtx); 537 return EINVAL; 538 } 539 mutex_exit(&aio->aio_mtx); 540 } 541 542 /* 543 * Check if AIO structure is initialized, if not - initialize it. 544 * In LIO case, we did that already. We will recheck this with 545 * the lock in aio_procinit(). 546 */ 547 if (lio == NULL && p->p_aio == NULL) 548 if (aio_procinit(p)) 549 return EAGAIN; 550 aio = p->p_aio; 551 552 /* 553 * Set the state with errno, and copy data 554 * structure back to the user-space. 555 */ 556 aiocbp._state = JOB_WIP; 557 aiocbp._errno = EINPROGRESS; 558 aiocbp._retval = -1; 559 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 560 if (error) 561 return error; 562 563 /* Allocate and initialize a new AIO job */ 564 a_job = pool_get(&aio_job_pool, PR_WAITOK); 565 memset(a_job, 0, sizeof(struct aio_job)); 566 567 /* 568 * Set the data. 569 * Store the user-space pointer for searching. Since we 570 * are storing only per proc pointers - it is safe. 571 */ 572 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 573 a_job->aiocb_uptr = aiocb_uptr; 574 a_job->aio_op |= op; 575 a_job->lio = lio; 576 577 /* 578 * Add the job to the queue, update the counters, and 579 * notify the AIO worker thread to handle the job. 580 */ 581 mutex_enter(&aio->aio_mtx); 582 583 /* Fail, if the limit was reached */ 584 if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max || 585 aio->jobs_count >= aio_listio_max) { 586 atomic_dec_uint(&aio_jobs_count); 587 mutex_exit(&aio->aio_mtx); 588 pool_put(&aio_job_pool, a_job); 589 return EAGAIN; 590 } 591 592 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 593 aio->jobs_count++; 594 if (lio) 595 lio->refcnt++; 596 cv_signal(&aio->aio_worker_cv); 597 598 mutex_exit(&aio->aio_mtx); 599 600 /* 601 * One would handle the errors only with aio_error() function. 602 * This way is appropriate according to POSIX. 603 */ 604 return 0; 605 } 606 607 /* 608 * Syscall functions. 609 */ 610 611 int 612 sys_aio_cancel(struct lwp *l, const struct sys_aio_cancel_args *uap, register_t *retval) 613 { 614 /* { 615 syscallarg(int) fildes; 616 syscallarg(struct aiocb *) aiocbp; 617 } */ 618 struct proc *p = l->l_proc; 619 struct aioproc *aio; 620 struct aio_job *a_job; 621 struct aiocb *aiocbp_ptr; 622 struct lio_req *lio; 623 struct filedesc *fdp = p->p_fd; 624 unsigned int cn, errcnt, fildes; 625 626 TAILQ_HEAD(, aio_job) tmp_jobs_list; 627 628 /* Check for invalid file descriptor */ 629 fildes = (unsigned int)SCARG(uap, fildes); 630 if (fildes >= fdp->fd_nfiles) 631 return EBADF; 632 membar_consumer(); 633 if (fdp->fd_ofiles[fildes] == NULL || fdp->fd_ofiles[fildes]->ff_file == NULL) 634 return EBADF; 635 636 /* Check if AIO structure is initialized */ 637 if (p->p_aio == NULL) { 638 *retval = AIO_NOTCANCELED; 639 return 0; 640 } 641 642 aio = p->p_aio; 643 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 644 645 mutex_enter(&aio->aio_mtx); 646 647 /* Cancel the jobs, and remove them from the queue */ 648 cn = 0; 649 TAILQ_INIT(&tmp_jobs_list); 650 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 651 if (aiocbp_ptr) { 652 if (aiocbp_ptr != a_job->aiocb_uptr) 653 continue; 654 if (fildes != a_job->aiocbp.aio_fildes) { 655 mutex_exit(&aio->aio_mtx); 656 return EBADF; 657 } 658 } else if (a_job->aiocbp.aio_fildes != fildes) 659 continue; 660 661 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 662 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 663 664 /* Decrease the counters */ 665 atomic_dec_uint(&aio_jobs_count); 666 aio->jobs_count--; 667 lio = a_job->lio; 668 if (lio != NULL && --lio->refcnt != 0) 669 a_job->lio = NULL; 670 671 cn++; 672 if (aiocbp_ptr) 673 break; 674 } 675 676 /* There are canceled jobs */ 677 if (cn) 678 *retval = AIO_CANCELED; 679 680 /* We cannot cancel current job */ 681 a_job = aio->curjob; 682 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 683 (a_job->aiocb_uptr == aiocbp_ptr))) 684 *retval = AIO_NOTCANCELED; 685 686 mutex_exit(&aio->aio_mtx); 687 688 /* Free the jobs after the lock */ 689 errcnt = 0; 690 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 691 a_job = TAILQ_FIRST(&tmp_jobs_list); 692 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 693 /* Set the errno and copy structures back to the user-space */ 694 a_job->aiocbp._errno = ECANCELED; 695 a_job->aiocbp._state = JOB_DONE; 696 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 697 sizeof(struct aiocb))) 698 errcnt++; 699 /* Send a signal if any */ 700 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 701 if (a_job->lio) { 702 lio = a_job->lio; 703 aio_sendsig(p, &lio->sig); 704 pool_put(&aio_lio_pool, lio); 705 } 706 pool_put(&aio_job_pool, a_job); 707 } 708 709 if (errcnt) 710 return EFAULT; 711 712 /* Set a correct return value */ 713 if (*retval == 0) 714 *retval = AIO_ALLDONE; 715 716 return 0; 717 } 718 719 int 720 sys_aio_error(struct lwp *l, const struct sys_aio_error_args *uap, register_t *retval) 721 { 722 /* { 723 syscallarg(const struct aiocb *) aiocbp; 724 } */ 725 struct proc *p = l->l_proc; 726 struct aioproc *aio = p->p_aio; 727 struct aiocb aiocbp; 728 int error; 729 730 if (aio == NULL) 731 return EINVAL; 732 733 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 734 if (error) 735 return error; 736 737 if (aiocbp._state == JOB_NONE) 738 return EINVAL; 739 740 *retval = aiocbp._errno; 741 742 return 0; 743 } 744 745 int 746 sys_aio_fsync(struct lwp *l, const struct sys_aio_fsync_args *uap, register_t *retval) 747 { 748 /* { 749 syscallarg(int) op; 750 syscallarg(struct aiocb *) aiocbp; 751 } */ 752 int op = SCARG(uap, op); 753 754 if ((op != O_DSYNC) && (op != O_SYNC)) 755 return EINVAL; 756 757 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 758 759 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 760 } 761 762 int 763 sys_aio_read(struct lwp *l, const struct sys_aio_read_args *uap, register_t *retval) 764 { 765 /* { 766 syscallarg(struct aiocb *) aiocbp; 767 } */ 768 769 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 770 } 771 772 int 773 sys_aio_return(struct lwp *l, const struct sys_aio_return_args *uap, register_t *retval) 774 { 775 /* { 776 syscallarg(struct aiocb *) aiocbp; 777 } */ 778 struct proc *p = l->l_proc; 779 struct aioproc *aio = p->p_aio; 780 struct aiocb aiocbp; 781 int error; 782 783 if (aio == NULL) 784 return EINVAL; 785 786 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 787 if (error) 788 return error; 789 790 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 791 return EINVAL; 792 793 *retval = aiocbp._retval; 794 795 /* Reset the internal variables */ 796 aiocbp._errno = 0; 797 aiocbp._retval = -1; 798 aiocbp._state = JOB_NONE; 799 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 800 801 return error; 802 } 803 804 int 805 sys___aio_suspend50(struct lwp *l, const struct sys___aio_suspend50_args *uap, 806 register_t *retval) 807 { 808 /* { 809 syscallarg(const struct aiocb *const[]) list; 810 syscallarg(int) nent; 811 syscallarg(const struct timespec *) timeout; 812 } */ 813 struct aiocb **list; 814 struct timespec ts; 815 int error, nent; 816 817 nent = SCARG(uap, nent); 818 if (nent <= 0 || nent > aio_listio_max) 819 return EAGAIN; 820 821 if (SCARG(uap, timeout)) { 822 /* Convert timespec to ticks */ 823 error = copyin(SCARG(uap, timeout), &ts, 824 sizeof(struct timespec)); 825 if (error) 826 return error; 827 } 828 list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 829 error = copyin(SCARG(uap, list), list, nent * sizeof(struct aiocb)); 830 if (error) 831 goto out; 832 error = aio_suspend1(l, list, nent, SCARG(uap, timeout) ? &ts : NULL); 833 out: 834 kmem_free(list, nent * sizeof(struct aio_job)); 835 return error; 836 } 837 838 int 839 aio_suspend1(struct lwp *l, struct aiocb **aiocbp_list, int nent, 840 struct timespec *ts) 841 { 842 struct proc *p = l->l_proc; 843 struct aioproc *aio; 844 struct aio_job *a_job; 845 int i, error, timo; 846 847 if (p->p_aio == NULL) 848 return EAGAIN; 849 aio = p->p_aio; 850 851 if (ts) { 852 timo = mstohz((ts->tv_sec * 1000) + (ts->tv_nsec / 1000000)); 853 if (timo == 0 && ts->tv_sec == 0 && ts->tv_nsec > 0) 854 timo = 1; 855 if (timo <= 0) 856 return EAGAIN; 857 } else 858 timo = 0; 859 860 /* Get the list from user-space */ 861 862 mutex_enter(&aio->aio_mtx); 863 for (;;) { 864 865 for (i = 0; i < nent; i++) { 866 867 /* Skip NULL entries */ 868 if (aiocbp_list[i] == NULL) 869 continue; 870 871 /* Skip current job */ 872 if (aio->curjob) { 873 a_job = aio->curjob; 874 if (a_job->aiocb_uptr == aiocbp_list[i]) 875 continue; 876 } 877 878 /* Look for a job in the queue */ 879 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 880 if (a_job->aiocb_uptr == aiocbp_list[i]) 881 break; 882 883 if (a_job == NULL) { 884 struct aiocb aiocbp; 885 886 mutex_exit(&aio->aio_mtx); 887 888 error = copyin(aiocbp_list[i], &aiocbp, 889 sizeof(struct aiocb)); 890 if (error == 0 && aiocbp._state != JOB_DONE) { 891 mutex_enter(&aio->aio_mtx); 892 continue; 893 } 894 895 kmem_free(aiocbp_list, 896 nent * sizeof(struct aio_job)); 897 return error; 898 } 899 } 900 901 /* Wait for a signal or when timeout occurs */ 902 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 903 if (error) { 904 if (error == EWOULDBLOCK) 905 error = EAGAIN; 906 break; 907 } 908 } 909 mutex_exit(&aio->aio_mtx); 910 return error; 911 } 912 913 int 914 sys_aio_write(struct lwp *l, const struct sys_aio_write_args *uap, register_t *retval) 915 { 916 /* { 917 syscallarg(struct aiocb *) aiocbp; 918 } */ 919 920 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 921 } 922 923 int 924 sys_lio_listio(struct lwp *l, const struct sys_lio_listio_args *uap, register_t *retval) 925 { 926 /* { 927 syscallarg(int) mode; 928 syscallarg(struct aiocb *const[]) list; 929 syscallarg(int) nent; 930 syscallarg(struct sigevent *) sig; 931 } */ 932 struct proc *p = l->l_proc; 933 struct aioproc *aio; 934 struct aiocb **aiocbp_list; 935 struct lio_req *lio; 936 int i, error, errcnt, mode, nent; 937 938 mode = SCARG(uap, mode); 939 nent = SCARG(uap, nent); 940 941 /* Non-accurate checks for the limit and invalid values */ 942 if (nent < 1 || nent > aio_listio_max) 943 return EINVAL; 944 if (aio_jobs_count + nent > aio_max) 945 return EAGAIN; 946 947 /* Check if AIO structure is initialized, if not - initialize it */ 948 if (p->p_aio == NULL) 949 if (aio_procinit(p)) 950 return EAGAIN; 951 aio = p->p_aio; 952 953 /* Create a LIO structure */ 954 lio = pool_get(&aio_lio_pool, PR_WAITOK); 955 lio->refcnt = 1; 956 error = 0; 957 958 switch (mode) { 959 case LIO_WAIT: 960 memset(&lio->sig, 0, sizeof(struct sigevent)); 961 break; 962 case LIO_NOWAIT: 963 /* Check for signal, validate it */ 964 if (SCARG(uap, sig)) { 965 struct sigevent *sig = &lio->sig; 966 967 error = copyin(SCARG(uap, sig), &lio->sig, 968 sizeof(struct sigevent)); 969 if (error == 0 && 970 (sig->sigev_signo < 0 || 971 sig->sigev_signo >= NSIG || 972 sig->sigev_notify < SIGEV_NONE || 973 sig->sigev_notify > SIGEV_SA)) 974 error = EINVAL; 975 } else 976 memset(&lio->sig, 0, sizeof(struct sigevent)); 977 break; 978 default: 979 error = EINVAL; 980 break; 981 } 982 983 if (error != 0) { 984 pool_put(&aio_lio_pool, lio); 985 return error; 986 } 987 988 /* Get the list from user-space */ 989 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 990 error = copyin(SCARG(uap, list), aiocbp_list, 991 nent * sizeof(struct aiocb)); 992 if (error) { 993 mutex_enter(&aio->aio_mtx); 994 goto err; 995 } 996 997 /* Enqueue all jobs */ 998 errcnt = 0; 999 for (i = 0; i < nent; i++) { 1000 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 1001 /* 1002 * According to POSIX, in such error case it may 1003 * fail with other I/O operations initiated. 1004 */ 1005 if (error) 1006 errcnt++; 1007 } 1008 1009 mutex_enter(&aio->aio_mtx); 1010 1011 /* Return an error, if any */ 1012 if (errcnt) { 1013 error = EIO; 1014 goto err; 1015 } 1016 1017 if (mode == LIO_WAIT) { 1018 /* 1019 * Wait for AIO completion. In such case, 1020 * the LIO structure will be freed here. 1021 */ 1022 while (lio->refcnt > 1 && error == 0) 1023 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 1024 if (error) 1025 error = EINTR; 1026 } 1027 1028 err: 1029 if (--lio->refcnt != 0) 1030 lio = NULL; 1031 mutex_exit(&aio->aio_mtx); 1032 if (lio != NULL) { 1033 aio_sendsig(p, &lio->sig); 1034 pool_put(&aio_lio_pool, lio); 1035 } 1036 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 1037 return error; 1038 } 1039 1040 /* 1041 * SysCtl 1042 */ 1043 1044 static int 1045 sysctl_aio_listio_max(SYSCTLFN_ARGS) 1046 { 1047 struct sysctlnode node; 1048 int error, newsize; 1049 1050 node = *rnode; 1051 node.sysctl_data = &newsize; 1052 1053 newsize = aio_listio_max; 1054 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1055 if (error || newp == NULL) 1056 return error; 1057 1058 if (newsize < 1 || newsize > aio_max) 1059 return EINVAL; 1060 aio_listio_max = newsize; 1061 1062 return 0; 1063 } 1064 1065 static int 1066 sysctl_aio_max(SYSCTLFN_ARGS) 1067 { 1068 struct sysctlnode node; 1069 int error, newsize; 1070 1071 node = *rnode; 1072 node.sysctl_data = &newsize; 1073 1074 newsize = aio_max; 1075 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1076 if (error || newp == NULL) 1077 return error; 1078 1079 if (newsize < 1 || newsize < aio_listio_max) 1080 return EINVAL; 1081 aio_max = newsize; 1082 1083 return 0; 1084 } 1085 1086 SYSCTL_SETUP(sysctl_aio_setup, "sysctl aio setup") 1087 { 1088 1089 sysctl_createv(clog, 0, NULL, NULL, 1090 CTLFLAG_PERMANENT, 1091 CTLTYPE_NODE, "kern", NULL, 1092 NULL, 0, NULL, 0, 1093 CTL_KERN, CTL_EOL); 1094 sysctl_createv(clog, 0, NULL, NULL, 1095 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1096 CTLTYPE_INT, "posix_aio", 1097 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1098 "Asynchronous I/O option to which the " 1099 "system attempts to conform"), 1100 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1101 CTL_KERN, CTL_CREATE, CTL_EOL); 1102 sysctl_createv(clog, 0, NULL, NULL, 1103 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1104 CTLTYPE_INT, "aio_listio_max", 1105 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1106 "operations in a single list I/O call"), 1107 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1108 CTL_KERN, CTL_CREATE, CTL_EOL); 1109 sysctl_createv(clog, 0, NULL, NULL, 1110 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1111 CTLTYPE_INT, "aio_max", 1112 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1113 "operations"), 1114 sysctl_aio_max, 0, &aio_max, 0, 1115 CTL_KERN, CTL_CREATE, CTL_EOL); 1116 } 1117 1118 /* 1119 * Debugging 1120 */ 1121 #if defined(DDB) 1122 void 1123 aio_print_jobs(void (*pr)(const char *, ...)) 1124 { 1125 struct proc *p = (curlwp == NULL ? NULL : curlwp->l_proc); 1126 struct aioproc *aio; 1127 struct aio_job *a_job; 1128 struct aiocb *aiocbp; 1129 1130 if (p == NULL) { 1131 (*pr)("AIO: We are not in the processes right now.\n"); 1132 return; 1133 } 1134 1135 aio = p->p_aio; 1136 if (aio == NULL) { 1137 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1138 return; 1139 } 1140 1141 (*pr)("AIO: PID = %d\n", p->p_pid); 1142 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1143 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1144 1145 if (aio->curjob) { 1146 a_job = aio->curjob; 1147 (*pr)("\nAIO current job:\n"); 1148 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1149 a_job->aio_op, a_job->aiocbp._errno, 1150 a_job->aiocbp._state, a_job->aiocb_uptr); 1151 aiocbp = &a_job->aiocbp; 1152 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1153 aiocbp->aio_fildes, aiocbp->aio_offset, 1154 aiocbp->aio_buf, aiocbp->aio_nbytes); 1155 } 1156 1157 (*pr)("\nAIO queue:\n"); 1158 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1159 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1160 a_job->aio_op, a_job->aiocbp._errno, 1161 a_job->aiocbp._state, a_job->aiocb_uptr); 1162 aiocbp = &a_job->aiocbp; 1163 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1164 aiocbp->aio_fildes, aiocbp->aio_offset, 1165 aiocbp->aio_buf, aiocbp->aio_nbytes); 1166 } 1167 } 1168 #endif /* defined(DDB) */ 1169
Indexes created Tue Sep 30 20:09:53 GMT 2025