sys_aio.c revision 1.5.2.8
1 /* $NetBSD: sys_aio.c,v 1.5.2.8 2007/10/16 10:53:25 ad Exp $ */ 2 3 /* 4 * Copyright (c) 2007, Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 17 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 19 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * TODO: 30 * 1. Additional work for VCHR and maybe VBLK devices. 31 * 2. Consider making the job-finding O(n) per one file descriptor. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.5.2.8 2007/10/16 10:53:25 ad Exp $"); 36 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/condvar.h> 41 #include <sys/file.h> 42 #include <sys/filedesc.h> 43 #include <sys/kernel.h> 44 #include <sys/kmem.h> 45 #include <sys/lwp.h> 46 #include <sys/mutex.h> 47 #include <sys/pool.h> 48 #include <sys/proc.h> 49 #include <sys/queue.h> 50 #include <sys/signal.h> 51 #include <sys/signalvar.h> 52 #include <sys/syscallargs.h> 53 #include <sys/sysctl.h> 54 #include <sys/systm.h> 55 #include <sys/types.h> 56 #include <sys/vnode.h> 57 58 #include <uvm/uvm_extern.h> 59 60 /* 61 * System-wide limits and counter of AIO operations. 62 * XXXSMP: We should spin-lock it, or modify atomically. 63 */ 64 static u_int aio_listio_max = AIO_LISTIO_MAX; 65 static u_int aio_max = AIO_MAX; 66 static u_int aio_jobs_count; 67 68 static struct pool aio_job_pool; 69 static struct pool aio_lio_pool; 70 71 /* Prototypes */ 72 void aio_worker(void *); 73 static void aio_process(struct aio_job *); 74 static void aio_sendsig(struct proc *, struct sigevent *); 75 static int aio_enqueue_job(int, void *, struct lio_req *); 76 77 /* 78 * Initialize the AIO system. 79 */ 80 void 81 aio_sysinit(void) 82 { 83 84 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 85 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 86 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 87 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 88 } 89 90 /* 91 * Initialize Asynchronous I/O data structures for the process. 92 */ 93 int 94 aio_init(struct proc *p) 95 { 96 struct aioproc *aio; 97 struct lwp *l; 98 bool inmem; 99 vaddr_t uaddr; 100 101 /* Allocate and initialize AIO structure */ 102 aio = kmem_zalloc(sizeof(struct aioproc), KM_NOSLEEP); 103 if (aio == NULL) 104 return EAGAIN; 105 106 /* Initialize queue and their synchronization structures */ 107 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 108 cv_init(&aio->aio_worker_cv, "aiowork"); 109 cv_init(&aio->done_cv, "aiodone"); 110 TAILQ_INIT(&aio->jobs_queue); 111 112 /* 113 * Create an AIO worker thread. 114 * XXX: Currently, AIO thread is not protected against user's actions. 115 */ 116 inmem = uvm_uarea_alloc(&uaddr); 117 if (uaddr == 0) { 118 aio_exit(p, aio); 119 return EAGAIN; 120 } 121 if (newlwp(curlwp, p, uaddr, inmem, 0, NULL, 0, 122 aio_worker, NULL, &l)) { 123 uvm_uarea_free(uaddr); 124 aio_exit(p, aio); 125 return EAGAIN; 126 } 127 128 /* Recheck if we are really first */ 129 mutex_enter(&p->p_mutex); 130 if (p->p_aio) { 131 mutex_exit(&p->p_mutex); 132 aio_exit(p, aio); 133 lwp_exit(l); 134 return 0; 135 } 136 p->p_aio = aio; 137 mutex_exit(&p->p_mutex); 138 139 /* Complete the initialization of thread, and run it */ 140 mutex_enter(&p->p_smutex); 141 aio->aio_worker = l; 142 p->p_nrlwps++; 143 lwp_lock(l); 144 l->l_stat = LSRUN; 145 l->l_usrpri = PRI_KERNEL - 1; 146 l->l_priority = l->l_usrpri. 147 sched_enqueue(l, false); 148 lwp_unlock(l); 149 mutex_exit(&p->p_smutex); 150 151 return 0; 152 } 153 154 /* 155 * Exit of Asynchronous I/O subsystem of process. 156 */ 157 void 158 aio_exit(struct proc *p, struct aioproc *aio) 159 { 160 struct aio_job *a_job; 161 162 if (aio == NULL) 163 return; 164 165 /* Free AIO queue */ 166 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 167 a_job = TAILQ_FIRST(&aio->jobs_queue); 168 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 169 pool_put(&aio_job_pool, a_job); 170 aio_jobs_count--; /* XXXSMP */ 171 } 172 173 /* Destroy and free the entire AIO data structure */ 174 cv_destroy(&aio->aio_worker_cv); 175 cv_destroy(&aio->done_cv); 176 mutex_destroy(&aio->aio_mtx); 177 kmem_free(aio, sizeof(struct aioproc)); 178 } 179 180 /* 181 * AIO worker thread and processor. 182 */ 183 void 184 aio_worker(void *arg) 185 { 186 struct proc *p = curlwp->l_proc; 187 struct aioproc *aio = p->p_aio; 188 struct aio_job *a_job; 189 struct lio_req *lio; 190 sigset_t oss, nss; 191 int error, refcnt; 192 193 /* 194 * Make an empty signal mask, so it 195 * handles only SIGKILL and SIGSTOP. 196 */ 197 sigfillset(&nss); 198 mutex_enter(&p->p_smutex); 199 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 200 mutex_exit(&p->p_smutex); 201 KASSERT(error == 0); 202 203 for (;;) { 204 /* 205 * Loop for each job in the queue. If there 206 * are no jobs then sleep. 207 */ 208 mutex_enter(&aio->aio_mtx); 209 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 210 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 211 /* 212 * Thread was interrupted - check for 213 * pending exit or suspend. 214 */ 215 mutex_exit(&aio->aio_mtx); 216 lwp_userret(curlwp); 217 mutex_enter(&aio->aio_mtx); 218 } 219 } 220 221 /* Take the job from the queue */ 222 aio->curjob = a_job; 223 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 224 225 aio_jobs_count--; /* XXXSMP */ 226 aio->jobs_count--; 227 228 mutex_exit(&aio->aio_mtx); 229 230 /* Process an AIO operation */ 231 aio_process(a_job); 232 233 /* Copy data structure back to the user-space */ 234 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 235 sizeof(struct aiocb)); 236 237 mutex_enter(&aio->aio_mtx); 238 aio->curjob = NULL; 239 240 /* Decrease a reference counter, if there is a LIO structure */ 241 lio = a_job->lio; 242 refcnt = (lio != NULL ? --lio->refcnt : -1); 243 244 /* Notify all suspenders */ 245 cv_broadcast(&aio->done_cv); 246 mutex_exit(&aio->aio_mtx); 247 248 /* Send a signal, if any */ 249 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 250 251 /* Destroy the LIO structure */ 252 if (refcnt == 0) { 253 aio_sendsig(p, &lio->sig); 254 pool_put(&aio_lio_pool, lio); 255 } 256 257 /* Destroy the the job */ 258 pool_put(&aio_job_pool, a_job); 259 } 260 261 /* NOTREACHED */ 262 } 263 264 static void 265 aio_process(struct aio_job *a_job) 266 { 267 struct proc *p = curlwp->l_proc; 268 struct aiocb *aiocbp = &a_job->aiocbp; 269 struct file *fp; 270 struct filedesc *fdp = p->p_fd; 271 int fd = aiocbp->aio_fildes; 272 int error = 0; 273 274 KASSERT(fdp != NULL); 275 KASSERT(a_job->aio_op != 0); 276 277 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 278 struct iovec aiov; 279 struct uio auio; 280 281 if (aiocbp->aio_nbytes > SSIZE_MAX) { 282 error = EINVAL; 283 goto done; 284 } 285 286 fp = fd_getfile(fdp, fd); 287 if (fp == NULL) { 288 error = EBADF; 289 goto done; 290 } 291 292 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 293 aiov.iov_len = aiocbp->aio_nbytes; 294 auio.uio_iov = &aiov; 295 auio.uio_iovcnt = 1; 296 auio.uio_resid = aiocbp->aio_nbytes; 297 auio.uio_vmspace = p->p_vmspace; 298 299 FILE_USE(fp); 300 if (a_job->aio_op & AIO_READ) { 301 /* 302 * Perform a Read operation 303 */ 304 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 305 306 if ((fp->f_flag & FREAD) == 0) { 307 FILE_UNUSE(fp, curlwp); 308 error = EBADF; 309 goto done; 310 } 311 auio.uio_rw = UIO_READ; 312 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 313 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 314 } else { 315 /* 316 * Perform a Write operation 317 */ 318 KASSERT(a_job->aio_op & AIO_WRITE); 319 320 if ((fp->f_flag & FWRITE) == 0) { 321 FILE_UNUSE(fp, curlwp); 322 error = EBADF; 323 goto done; 324 } 325 auio.uio_rw = UIO_WRITE; 326 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 327 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 328 } 329 FILE_UNUSE(fp, curlwp); 330 331 /* Store the result value */ 332 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 333 a_job->aiocbp._retval = (error == 0) ? 334 a_job->aiocbp.aio_nbytes : -1; 335 336 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 337 /* 338 * Perform a file Sync operation 339 */ 340 struct vnode *vp; 341 342 if ((error = getvnode(fdp, fd, &fp)) != 0) 343 goto done; 344 345 if ((fp->f_flag & FWRITE) == 0) { 346 FILE_UNUSE(fp, curlwp); 347 error = EBADF; 348 goto done; 349 } 350 351 vp = (struct vnode *)fp->f_data; 352 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 353 if (a_job->aio_op & AIO_DSYNC) { 354 error = VOP_FSYNC(vp, fp->f_cred, 355 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0, curlwp); 356 } else if (a_job->aio_op & AIO_SYNC) { 357 error = VOP_FSYNC(vp, fp->f_cred, 358 FSYNC_WAIT, 0, 0, curlwp); 359 if (error == 0 && bioopsp != NULL && 360 vp->v_mount && 361 (vp->v_mount->mnt_flag & MNT_SOFTDEP)) 362 bioopsp->io_fsync(vp, 0); 363 } 364 VOP_UNLOCK(vp, 0); 365 FILE_UNUSE(fp, curlwp); 366 367 /* Store the result value */ 368 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 369 370 } else 371 panic("aio_process: invalid operation code\n"); 372 373 done: 374 /* Job is done, set the error, if any */ 375 a_job->aiocbp._errno = error; 376 a_job->aiocbp._state = JOB_DONE; 377 } 378 379 /* 380 * Send AIO signal. 381 */ 382 static void 383 aio_sendsig(struct proc *p, struct sigevent *sig) 384 { 385 ksiginfo_t ksi; 386 387 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 388 return; 389 390 KSI_INIT(&ksi); 391 ksi.ksi_signo = sig->sigev_signo; 392 ksi.ksi_code = SI_ASYNCIO; 393 ksi.ksi_value = sig->sigev_value; 394 mutex_enter(&proclist_mutex); 395 kpsignal(p, &ksi, NULL); 396 mutex_exit(&proclist_mutex); 397 } 398 399 /* 400 * Enqueue the job. 401 */ 402 static int 403 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 404 { 405 struct proc *p = curlwp->l_proc; 406 struct aioproc *aio; 407 struct aio_job *a_job; 408 struct aiocb aiocbp; 409 struct sigevent *sig; 410 int error; 411 412 /* Check for the limit */ 413 if (aio_jobs_count + 1 > aio_max) /* XXXSMP */ 414 return EAGAIN; 415 416 /* Get the data structure from user-space */ 417 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 418 if (error) 419 return error; 420 421 /* Check if signal is set, and validate it */ 422 sig = &aiocbp.aio_sigevent; 423 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 424 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 425 return EINVAL; 426 427 /* Buffer and byte count */ 428 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 429 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 430 return EINVAL; 431 432 /* Check the opcode, if LIO_NOP - simply ignore */ 433 if (op == AIO_LIO) { 434 KASSERT(lio != NULL); 435 if (aiocbp.aio_lio_opcode == LIO_WRITE) 436 op = AIO_WRITE; 437 else if (aiocbp.aio_lio_opcode == LIO_READ) 438 op = AIO_READ; 439 else 440 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 441 } else { 442 KASSERT(lio == NULL); 443 } 444 445 /* 446 * Look for already existing job. If found - the job is in-progress. 447 * According to POSIX this is invalid, so return the error. 448 */ 449 aio = p->p_aio; 450 if (aio) { 451 mutex_enter(&aio->aio_mtx); 452 if (aio->curjob) { 453 a_job = aio->curjob; 454 if (a_job->aiocb_uptr == aiocb_uptr) { 455 mutex_exit(&aio->aio_mtx); 456 return EINVAL; 457 } 458 } 459 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 460 if (a_job->aiocb_uptr != aiocb_uptr) 461 continue; 462 mutex_exit(&aio->aio_mtx); 463 return EINVAL; 464 } 465 mutex_exit(&aio->aio_mtx); 466 } 467 468 /* 469 * Check if AIO structure is initialized, if not - initialize it. 470 * In LIO case, we did that already. We will recheck this with 471 * the lock in aio_init(). 472 */ 473 if (lio == NULL && p->p_aio == NULL) 474 if (aio_init(p)) 475 return EAGAIN; 476 aio = p->p_aio; 477 478 /* 479 * Set the state with errno, and copy data 480 * structure back to the user-space. 481 */ 482 aiocbp._state = JOB_WIP; 483 aiocbp._errno = EINPROGRESS; 484 aiocbp._retval = -1; 485 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 486 if (error) 487 return error; 488 489 /* Allocate and initialize a new AIO job */ 490 a_job = pool_get(&aio_job_pool, PR_WAITOK); 491 memset(a_job, 0, sizeof(struct aio_job)); 492 493 /* 494 * Set the data. 495 * Store the user-space pointer for searching. Since we 496 * are storing only per proc pointers - it is safe. 497 */ 498 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 499 a_job->aiocb_uptr = aiocb_uptr; 500 a_job->aio_op |= op; 501 a_job->lio = lio; 502 503 /* 504 * Add the job to the queue, update the counters, and 505 * notify the AIO worker thread to handle the job. 506 */ 507 mutex_enter(&aio->aio_mtx); 508 509 /* Fail, if the limit was reached */ 510 if (aio->jobs_count >= aio_listio_max) { 511 mutex_exit(&aio->aio_mtx); 512 pool_put(&aio_job_pool, a_job); 513 return EAGAIN; 514 } 515 516 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 517 aio_jobs_count++; /* XXXSMP */ 518 aio->jobs_count++; 519 if (lio) 520 lio->refcnt++; 521 cv_signal(&aio->aio_worker_cv); 522 523 mutex_exit(&aio->aio_mtx); 524 525 /* 526 * One would handle the errors only with aio_error() function. 527 * This way is appropriate according to POSIX. 528 */ 529 return 0; 530 } 531 532 /* 533 * Syscall functions. 534 */ 535 536 int 537 sys_aio_cancel(struct lwp *l, void *v, register_t *retval) 538 { 539 struct sys_aio_cancel_args /* { 540 syscallarg(int) fildes; 541 syscallarg(struct aiocb *) aiocbp; 542 } */ *uap = v; 543 struct proc *p = l->l_proc; 544 struct aioproc *aio; 545 struct aio_job *a_job; 546 struct aiocb *aiocbp_ptr; 547 struct lio_req *lio; 548 struct filedesc *fdp = p->p_fd; 549 unsigned int cn, errcnt, fildes; 550 551 TAILQ_HEAD(, aio_job) tmp_jobs_list; 552 553 /* Check for invalid file descriptor */ 554 fildes = (unsigned int)SCARG(uap, fildes); 555 if (fildes >= fdp->fd_nfiles || fdp->fd_ofiles[fildes] == NULL) 556 return EBADF; 557 558 /* Check if AIO structure is initialized */ 559 if (p->p_aio == NULL) { 560 *retval = AIO_NOTCANCELED; 561 return 0; 562 } 563 564 aio = p->p_aio; 565 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 566 567 mutex_enter(&aio->aio_mtx); 568 569 /* Cancel the jobs, and remove them from the queue */ 570 cn = 0; 571 TAILQ_INIT(&tmp_jobs_list); 572 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 573 if (aiocbp_ptr) { 574 if (aiocbp_ptr != a_job->aiocb_uptr) 575 continue; 576 if (fildes != a_job->aiocbp.aio_fildes) { 577 mutex_exit(&aio->aio_mtx); 578 return EBADF; 579 } 580 } else if (a_job->aiocbp.aio_fildes != fildes) 581 continue; 582 583 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 584 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 585 586 /* Decrease the counters */ 587 aio_jobs_count--; /* XXXSMP */ 588 aio->jobs_count--; 589 lio = a_job->lio; 590 if (lio != NULL && --lio->refcnt != 0) 591 a_job->lio = NULL; 592 593 cn++; 594 if (aiocbp_ptr) 595 break; 596 } 597 598 /* There are canceled jobs */ 599 if (cn) 600 *retval = AIO_CANCELED; 601 602 /* We cannot cancel current job */ 603 a_job = aio->curjob; 604 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 605 (a_job->aiocb_uptr == aiocbp_ptr))) 606 *retval = AIO_NOTCANCELED; 607 608 mutex_exit(&aio->aio_mtx); 609 610 /* Free the jobs after the lock */ 611 errcnt = 0; 612 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 613 a_job = TAILQ_FIRST(&tmp_jobs_list); 614 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 615 /* Set the errno and copy structures back to the user-space */ 616 a_job->aiocbp._errno = ECANCELED; 617 a_job->aiocbp._state = JOB_DONE; 618 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 619 sizeof(struct aiocb))) 620 errcnt++; 621 /* Send a signal if any */ 622 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 623 if (a_job->lio) { 624 lio = a_job->lio; 625 aio_sendsig(p, &lio->sig); 626 pool_put(&aio_lio_pool, lio); 627 } 628 pool_put(&aio_job_pool, a_job); 629 } 630 631 if (errcnt) 632 return EFAULT; 633 634 /* Set a correct return value */ 635 if (*retval == 0) 636 *retval = AIO_ALLDONE; 637 638 return 0; 639 } 640 641 int 642 sys_aio_error(struct lwp *l, void *v, register_t *retval) 643 { 644 struct sys_aio_error_args /* { 645 syscallarg(const struct aiocb *) aiocbp; 646 } */ *uap = v; 647 struct proc *p = l->l_proc; 648 struct aioproc *aio = p->p_aio; 649 struct aiocb aiocbp; 650 int error; 651 652 if (aio == NULL) 653 return EINVAL; 654 655 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 656 if (error) 657 return error; 658 659 if (aiocbp._state == JOB_NONE) 660 return EINVAL; 661 662 *retval = aiocbp._errno; 663 664 return 0; 665 } 666 667 int 668 sys_aio_fsync(struct lwp *l, void *v, register_t *retval) 669 { 670 struct sys_aio_fsync_args /* { 671 syscallarg(int) op; 672 syscallarg(struct aiocb *) aiocbp; 673 } */ *uap = v; 674 int op = SCARG(uap, op); 675 676 if ((op != O_DSYNC) && (op != O_SYNC)) 677 return EINVAL; 678 679 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 680 681 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 682 } 683 684 int 685 sys_aio_read(struct lwp *l, void *v, register_t *retval) 686 { 687 struct sys_aio_read_args /* { 688 syscallarg(struct aiocb *) aiocbp; 689 } */ *uap = v; 690 691 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 692 } 693 694 int 695 sys_aio_return(struct lwp *l, void *v, register_t *retval) 696 { 697 struct sys_aio_return_args /* { 698 syscallarg(struct aiocb *) aiocbp; 699 } */ *uap = v; 700 struct proc *p = l->l_proc; 701 struct aioproc *aio = p->p_aio; 702 struct aiocb aiocbp; 703 int error; 704 705 if (aio == NULL) 706 return EINVAL; 707 708 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 709 if (error) 710 return error; 711 712 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 713 return EINVAL; 714 715 *retval = aiocbp._retval; 716 717 /* Reset the internal variables */ 718 aiocbp._errno = 0; 719 aiocbp._retval = -1; 720 aiocbp._state = JOB_NONE; 721 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 722 723 return error; 724 } 725 726 int 727 sys_aio_suspend(struct lwp *l, void *v, register_t *retval) 728 { 729 struct sys_aio_suspend_args /* { 730 syscallarg(const struct aiocb *const[]) list; 731 syscallarg(int) nent; 732 syscallarg(const struct timespec *) timeout; 733 } */ *uap = v; 734 struct proc *p = l->l_proc; 735 struct aioproc *aio; 736 struct aio_job *a_job; 737 struct aiocb **aiocbp_list; 738 struct timespec ts; 739 int i, error, nent, timo; 740 741 if (p->p_aio == NULL) 742 return EAGAIN; 743 aio = p->p_aio; 744 745 nent = SCARG(uap, nent); 746 if (nent <= 0 || nent > aio_listio_max) 747 return EAGAIN; 748 749 if (SCARG(uap, timeout)) { 750 /* Convert timespec to ticks */ 751 error = copyin(SCARG(uap, timeout), &ts, 752 sizeof(struct timespec)); 753 if (error) 754 return error; 755 timo = mstohz((ts.tv_sec * 1000) + (ts.tv_nsec / 1000000)); 756 if (timo == 0 && ts.tv_sec == 0 && ts.tv_nsec > 0) 757 timo = 1; 758 if (timo <= 0) 759 return EAGAIN; 760 } else 761 timo = 0; 762 763 /* Get the list from user-space */ 764 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 765 error = copyin(SCARG(uap, list), aiocbp_list, 766 nent * sizeof(struct aiocb)); 767 if (error) { 768 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 769 return error; 770 } 771 772 mutex_enter(&aio->aio_mtx); 773 for (;;) { 774 775 for (i = 0; i < nent; i++) { 776 777 /* Skip NULL entries */ 778 if (aiocbp_list[i] == NULL) 779 continue; 780 781 /* Skip current job */ 782 if (aio->curjob) { 783 a_job = aio->curjob; 784 if (a_job->aiocb_uptr == aiocbp_list[i]) 785 continue; 786 } 787 788 /* Look for a job in the queue */ 789 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 790 if (a_job->aiocb_uptr == aiocbp_list[i]) 791 break; 792 793 if (a_job == NULL) { 794 struct aiocb aiocbp; 795 796 mutex_exit(&aio->aio_mtx); 797 798 error = copyin(aiocbp_list[i], &aiocbp, 799 sizeof(struct aiocb)); 800 if (error == 0 && aiocbp._state != JOB_DONE) { 801 mutex_enter(&aio->aio_mtx); 802 continue; 803 } 804 805 kmem_free(aiocbp_list, 806 nent * sizeof(struct aio_job)); 807 return error; 808 } 809 } 810 811 /* Wait for a signal or when timeout occurs */ 812 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 813 if (error) { 814 if (error == EWOULDBLOCK) 815 error = EAGAIN; 816 break; 817 } 818 } 819 mutex_exit(&aio->aio_mtx); 820 821 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 822 return error; 823 } 824 825 int 826 sys_aio_write(struct lwp *l, void *v, register_t *retval) 827 { 828 struct sys_aio_write_args /* { 829 syscallarg(struct aiocb *) aiocbp; 830 } */ *uap = v; 831 832 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 833 } 834 835 int 836 sys_lio_listio(struct lwp *l, void *v, register_t *retval) 837 { 838 struct sys_lio_listio_args /* { 839 syscallarg(int) mode; 840 syscallarg(struct aiocb *const[]) list; 841 syscallarg(int) nent; 842 syscallarg(struct sigevent *) sig; 843 } */ *uap = v; 844 struct proc *p = l->l_proc; 845 struct aioproc *aio; 846 struct aiocb **aiocbp_list; 847 struct lio_req *lio; 848 int i, error, errcnt, mode, nent; 849 850 mode = SCARG(uap, mode); 851 nent = SCARG(uap, nent); 852 853 /* Check for the limits, and invalid values */ 854 if (nent < 1 || nent > aio_listio_max) 855 return EINVAL; 856 if (aio_jobs_count + nent > aio_max) /* XXXSMP */ 857 return EAGAIN; 858 859 /* Check if AIO structure is initialized, if not - initialize it */ 860 if (p->p_aio == NULL) 861 if (aio_init(p)) 862 return EAGAIN; 863 aio = p->p_aio; 864 865 /* Create a LIO structure */ 866 lio = pool_get(&aio_lio_pool, PR_WAITOK); 867 lio->refcnt = 1; 868 error = 0; 869 870 switch (mode) { 871 case LIO_WAIT: 872 memset(&lio->sig, 0, sizeof(struct sigevent)); 873 break; 874 case LIO_NOWAIT: 875 /* Check for signal, validate it */ 876 if (SCARG(uap, sig)) { 877 struct sigevent *sig = &lio->sig; 878 879 error = copyin(SCARG(uap, sig), &lio->sig, 880 sizeof(struct sigevent)); 881 if (error == 0 && 882 (sig->sigev_signo < 0 || 883 sig->sigev_signo >= NSIG || 884 sig->sigev_notify < SIGEV_NONE || 885 sig->sigev_notify > SIGEV_SA)) 886 error = EINVAL; 887 } else 888 memset(&lio->sig, 0, sizeof(struct sigevent)); 889 break; 890 default: 891 error = EINVAL; 892 break; 893 } 894 895 if (error != 0) { 896 pool_put(&aio_lio_pool, lio); 897 return error; 898 } 899 900 /* Get the list from user-space */ 901 aiocbp_list = kmem_zalloc(nent * sizeof(struct aio_job), KM_SLEEP); 902 error = copyin(SCARG(uap, list), aiocbp_list, 903 nent * sizeof(struct aiocb)); 904 if (error) { 905 mutex_enter(&aio->aio_mtx); 906 goto err; 907 } 908 909 /* Enqueue all jobs */ 910 errcnt = 0; 911 for (i = 0; i < nent; i++) { 912 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 913 /* 914 * According to POSIX, in such error case it may 915 * fail with other I/O operations initiated. 916 */ 917 if (error) 918 errcnt++; 919 } 920 921 mutex_enter(&aio->aio_mtx); 922 923 /* Return an error, if any */ 924 if (errcnt) { 925 error = EIO; 926 goto err; 927 } 928 929 if (mode == LIO_WAIT) { 930 /* 931 * Wait for AIO completion. In such case, 932 * the LIO structure will be freed here. 933 */ 934 while (lio->refcnt > 1 && error == 0) 935 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 936 if (error) 937 error = EINTR; 938 } 939 940 err: 941 if (--lio->refcnt != 0) 942 lio = NULL; 943 mutex_exit(&aio->aio_mtx); 944 if (lio != NULL) { 945 aio_sendsig(p, &lio->sig); 946 pool_put(&aio_lio_pool, lio); 947 } 948 kmem_free(aiocbp_list, nent * sizeof(struct aio_job)); 949 return error; 950 } 951 952 /* 953 * SysCtl 954 */ 955 956 static int 957 sysctl_aio_listio_max(SYSCTLFN_ARGS) 958 { 959 struct sysctlnode node; 960 int error, newsize; 961 962 node = *rnode; 963 node.sysctl_data = &newsize; 964 965 newsize = aio_listio_max; 966 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 967 if (error || newp == NULL) 968 return error; 969 970 /* XXXSMP */ 971 if (newsize < 1 || newsize > aio_max) 972 return EINVAL; 973 aio_listio_max = newsize; 974 975 return 0; 976 } 977 978 static int 979 sysctl_aio_max(SYSCTLFN_ARGS) 980 { 981 struct sysctlnode node; 982 int error, newsize; 983 984 node = *rnode; 985 node.sysctl_data = &newsize; 986 987 newsize = aio_max; 988 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 989 if (error || newp == NULL) 990 return error; 991 992 /* XXXSMP */ 993 if (newsize < 1 || newsize < aio_listio_max) 994 return EINVAL; 995 aio_max = newsize; 996 997 return 0; 998 } 999 1000 SYSCTL_SETUP(sysctl_aio_setup, "sysctl aio setup") 1001 { 1002 1003 sysctl_createv(clog, 0, NULL, NULL, 1004 CTLFLAG_PERMANENT, 1005 CTLTYPE_NODE, "kern", NULL, 1006 NULL, 0, NULL, 0, 1007 CTL_KERN, CTL_EOL); 1008 sysctl_createv(clog, 0, NULL, NULL, 1009 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1010 CTLTYPE_INT, "posix_aio", 1011 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1012 "Asynchronous I/O option to which the " 1013 "system attempts to conform"), 1014 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1015 CTL_KERN, CTL_CREATE, CTL_EOL); 1016 sysctl_createv(clog, 0, NULL, NULL, 1017 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1018 CTLTYPE_INT, "aio_listio_max", 1019 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1020 "operations in a single list I/O call"), 1021 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1022 CTL_KERN, CTL_CREATE, CTL_EOL); 1023 sysctl_createv(clog, 0, NULL, NULL, 1024 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1025 CTLTYPE_INT, "aio_max", 1026 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1027 "operations"), 1028 sysctl_aio_max, 0, &aio_max, 0, 1029 CTL_KERN, CTL_CREATE, CTL_EOL); 1030 } 1031 1032 /* 1033 * Debugging 1034 */ 1035 #if defined(DDB) 1036 void 1037 aio_print_jobs(void (*pr)(const char *, ...)) 1038 { 1039 struct proc *p = (curlwp == NULL ? NULL : curlwp->l_proc); 1040 struct aioproc *aio; 1041 struct aio_job *a_job; 1042 struct aiocb *aiocbp; 1043 1044 if (p == NULL) { 1045 (*pr)("AIO: We are not in the processes right now.\n"); 1046 return; 1047 } 1048 1049 aio = p->p_aio; 1050 if (aio == NULL) { 1051 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1052 return; 1053 } 1054 1055 (*pr)("AIO: PID = %d\n", p->p_pid); 1056 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1057 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1058 1059 if (aio->curjob) { 1060 a_job = aio->curjob; 1061 (*pr)("\nAIO current job:\n"); 1062 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1063 a_job->aio_op, a_job->aiocbp._errno, 1064 a_job->aiocbp._state, a_job->aiocb_uptr); 1065 aiocbp = &a_job->aiocbp; 1066 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1067 aiocbp->aio_fildes, aiocbp->aio_offset, 1068 aiocbp->aio_buf, aiocbp->aio_nbytes); 1069 } 1070 1071 (*pr)("\nAIO queue:\n"); 1072 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1073 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1074 a_job->aio_op, a_job->aiocbp._errno, 1075 a_job->aiocbp._state, a_job->aiocb_uptr); 1076 aiocbp = &a_job->aiocbp; 1077 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1078 aiocbp->aio_fildes, aiocbp->aio_offset, 1079 aiocbp->aio_buf, aiocbp->aio_nbytes); 1080 } 1081 } 1082 #endif /* defined(DDB) */ 1083
Indexes created Tue Sep 30 20:09:53 GMT 2025