sys_aio.c revision 1.44.4.1
1 /* $NetBSD: sys_aio.c,v 1.44.4.1 2024/11/20 14:01:59 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2007 Mindaugas Rasiukevicius <rmind at NetBSD org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Implementation of POSIX asynchronous I/O. 31 * Defined in the Base Definitions volume of IEEE Std 1003.1-2001. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.44.4.1 2024/11/20 14:01:59 martin Exp $"); 36 37 #ifdef _KERNEL_OPT 38 #include "opt_ddb.h" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/condvar.h> 43 #include <sys/file.h> 44 #include <sys/filedesc.h> 45 #include <sys/kernel.h> 46 #include <sys/kmem.h> 47 #include <sys/lwp.h> 48 #include <sys/mutex.h> 49 #include <sys/pool.h> 50 #include <sys/proc.h> 51 #include <sys/queue.h> 52 #include <sys/signal.h> 53 #include <sys/signalvar.h> 54 #include <sys/syscall.h> 55 #include <sys/syscallargs.h> 56 #include <sys/syscallvar.h> 57 #include <sys/sysctl.h> 58 #include <sys/systm.h> 59 #include <sys/types.h> 60 #include <sys/vnode.h> 61 #include <sys/atomic.h> 62 #include <sys/module.h> 63 #include <sys/buf.h> 64 65 #include <uvm/uvm_extern.h> 66 67 MODULE(MODULE_CLASS_MISC, aio, NULL); 68 69 /* 70 * System-wide limits and counter of AIO operations. 71 */ 72 u_int aio_listio_max = AIO_LISTIO_MAX; 73 static u_int aio_max = AIO_MAX; 74 static u_int aio_jobs_count; 75 76 static struct sysctllog *aio_sysctl; 77 static struct pool aio_job_pool; 78 static struct pool aio_lio_pool; 79 static void * aio_ehook; 80 81 static void aio_worker(void *); 82 static void aio_process(struct aio_job *); 83 static void aio_sendsig(struct proc *, struct sigevent *); 84 static int aio_enqueue_job(int, void *, struct lio_req *); 85 static void aio_exit(proc_t *, void *); 86 87 static int sysctl_aio_listio_max(SYSCTLFN_PROTO); 88 static int sysctl_aio_max(SYSCTLFN_PROTO); 89 static int sysctl_aio_init(void); 90 91 static const struct syscall_package aio_syscalls[] = { 92 { SYS_aio_cancel, 0, (sy_call_t *)sys_aio_cancel }, 93 { SYS_aio_error, 0, (sy_call_t *)sys_aio_error }, 94 { SYS_aio_fsync, 0, (sy_call_t *)sys_aio_fsync }, 95 { SYS_aio_read, 0, (sy_call_t *)sys_aio_read }, 96 { SYS_aio_return, 0, (sy_call_t *)sys_aio_return }, 97 { SYS___aio_suspend50, 0, (sy_call_t *)sys___aio_suspend50 }, 98 { SYS_aio_write, 0, (sy_call_t *)sys_aio_write }, 99 { SYS_lio_listio, 0, (sy_call_t *)sys_lio_listio }, 100 { 0, 0, NULL }, 101 }; 102 103 /* 104 * Tear down all AIO state. 105 */ 106 static int 107 aio_fini(bool interface) 108 { 109 int error; 110 proc_t *p; 111 112 if (interface) { 113 /* Stop syscall activity. */ 114 error = syscall_disestablish(NULL, aio_syscalls); 115 if (error != 0) 116 return error; 117 /* Abort if any processes are using AIO. */ 118 mutex_enter(proc_lock); 119 PROCLIST_FOREACH(p, &allproc) { 120 if (p->p_aio != NULL) 121 break; 122 } 123 mutex_exit(proc_lock); 124 if (p != NULL) { 125 error = syscall_establish(NULL, aio_syscalls); 126 KASSERT(error == 0); 127 return EBUSY; 128 } 129 } 130 if (aio_sysctl != NULL) 131 sysctl_teardown(&aio_sysctl); 132 133 KASSERT(aio_jobs_count == 0); 134 exithook_disestablish(aio_ehook); 135 pool_destroy(&aio_job_pool); 136 pool_destroy(&aio_lio_pool); 137 return 0; 138 } 139 140 /* 141 * Initialize global AIO state. 142 */ 143 static int 144 aio_init(void) 145 { 146 int error; 147 148 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0, 149 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE); 150 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0, 151 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE); 152 aio_ehook = exithook_establish(aio_exit, NULL); 153 154 error = sysctl_aio_init(); 155 if (error != 0) { 156 (void)aio_fini(false); 157 return error; 158 } 159 error = syscall_establish(NULL, aio_syscalls); 160 if (error != 0) 161 (void)aio_fini(false); 162 return error; 163 } 164 165 /* 166 * Module interface. 167 */ 168 static int 169 aio_modcmd(modcmd_t cmd, void *arg) 170 { 171 172 switch (cmd) { 173 case MODULE_CMD_INIT: 174 return aio_init(); 175 case MODULE_CMD_FINI: 176 return aio_fini(true); 177 default: 178 return ENOTTY; 179 } 180 } 181 182 /* 183 * Initialize Asynchronous I/O data structures for the process. 184 */ 185 static int 186 aio_procinit(struct proc *p) 187 { 188 struct aioproc *aio; 189 struct lwp *l; 190 int error; 191 vaddr_t uaddr; 192 193 /* Allocate and initialize AIO structure */ 194 aio = kmem_zalloc(sizeof(struct aioproc), KM_SLEEP); 195 196 /* Initialize queue and their synchronization structures */ 197 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE); 198 cv_init(&aio->aio_worker_cv, "aiowork"); 199 cv_init(&aio->done_cv, "aiodone"); 200 TAILQ_INIT(&aio->jobs_queue); 201 202 /* 203 * Create an AIO worker thread. 204 * XXX: Currently, AIO thread is not protected against user's actions. 205 */ 206 uaddr = uvm_uarea_alloc(); 207 if (uaddr == 0) { 208 aio_exit(p, aio); 209 return EAGAIN; 210 } 211 error = lwp_create(curlwp, p, uaddr, 0, NULL, 0, aio_worker, 212 NULL, &l, curlwp->l_class, &curlwp->l_sigmask, &curlwp->l_sigstk); 213 if (error != 0) { 214 uvm_uarea_free(uaddr); 215 aio_exit(p, aio); 216 return error; 217 } 218 219 /* Recheck if we are really first */ 220 mutex_enter(p->p_lock); 221 if (p->p_aio) { 222 mutex_exit(p->p_lock); 223 aio_exit(p, aio); 224 lwp_exit(l); 225 return 0; 226 } 227 p->p_aio = aio; 228 229 /* Complete the initialization of thread, and run it */ 230 aio->aio_worker = l; 231 lwp_lock(l); 232 l->l_stat = LSRUN; 233 l->l_priority = MAXPRI_USER; 234 sched_enqueue(l, false); 235 lwp_unlock(l); 236 mutex_exit(p->p_lock); 237 238 return 0; 239 } 240 241 /* 242 * Exit of Asynchronous I/O subsystem of process. 243 */ 244 static void 245 aio_exit(struct proc *p, void *cookie) 246 { 247 struct aio_job *a_job; 248 struct aioproc *aio; 249 250 if (cookie != NULL) 251 aio = cookie; 252 else if ((aio = p->p_aio) == NULL) 253 return; 254 255 /* Free AIO queue */ 256 while (!TAILQ_EMPTY(&aio->jobs_queue)) { 257 a_job = TAILQ_FIRST(&aio->jobs_queue); 258 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 259 pool_put(&aio_job_pool, a_job); 260 atomic_dec_uint(&aio_jobs_count); 261 } 262 263 /* Destroy and free the entire AIO data structure */ 264 cv_destroy(&aio->aio_worker_cv); 265 cv_destroy(&aio->done_cv); 266 mutex_destroy(&aio->aio_mtx); 267 kmem_free(aio, sizeof(struct aioproc)); 268 } 269 270 /* 271 * AIO worker thread and processor. 272 */ 273 static void 274 aio_worker(void *arg) 275 { 276 struct proc *p = curlwp->l_proc; 277 struct aioproc *aio = p->p_aio; 278 struct aio_job *a_job; 279 struct lio_req *lio; 280 sigset_t oss, nss; 281 int error __diagused, refcnt; 282 283 /* 284 * Make an empty signal mask, so it 285 * handles only SIGKILL and SIGSTOP. 286 */ 287 sigfillset(&nss); 288 mutex_enter(p->p_lock); 289 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss); 290 mutex_exit(p->p_lock); 291 KASSERT(error == 0); 292 293 for (;;) { 294 /* 295 * Loop for each job in the queue. If there 296 * are no jobs then sleep. 297 */ 298 mutex_enter(&aio->aio_mtx); 299 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) { 300 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) { 301 /* 302 * Thread was interrupted - check for 303 * pending exit or suspend. 304 */ 305 mutex_exit(&aio->aio_mtx); 306 lwp_userret(curlwp); 307 mutex_enter(&aio->aio_mtx); 308 } 309 } 310 311 /* Take the job from the queue */ 312 aio->curjob = a_job; 313 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 314 315 atomic_dec_uint(&aio_jobs_count); 316 aio->jobs_count--; 317 318 mutex_exit(&aio->aio_mtx); 319 320 /* Process an AIO operation */ 321 aio_process(a_job); 322 323 /* Copy data structure back to the user-space */ 324 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr, 325 sizeof(struct aiocb)); 326 327 mutex_enter(&aio->aio_mtx); 328 KASSERT(aio->curjob == a_job); 329 aio->curjob = NULL; 330 331 /* Decrease a reference counter, if there is a LIO structure */ 332 lio = a_job->lio; 333 refcnt = (lio != NULL ? --lio->refcnt : -1); 334 335 /* Notify all suspenders */ 336 cv_broadcast(&aio->done_cv); 337 mutex_exit(&aio->aio_mtx); 338 339 /* Send a signal, if any */ 340 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 341 342 /* Destroy the LIO structure */ 343 if (refcnt == 0) { 344 aio_sendsig(p, &lio->sig); 345 pool_put(&aio_lio_pool, lio); 346 } 347 348 /* Destroy the job */ 349 pool_put(&aio_job_pool, a_job); 350 } 351 352 /* NOTREACHED */ 353 } 354 355 static void 356 aio_process(struct aio_job *a_job) 357 { 358 struct proc *p = curlwp->l_proc; 359 struct aiocb *aiocbp = &a_job->aiocbp; 360 struct file *fp; 361 int fd = aiocbp->aio_fildes; 362 int error = 0; 363 364 KASSERT(a_job->aio_op != 0); 365 366 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) { 367 struct iovec aiov; 368 struct uio auio; 369 370 if (aiocbp->aio_nbytes > SSIZE_MAX) { 371 error = EINVAL; 372 goto done; 373 } 374 375 fp = fd_getfile(fd); 376 if (fp == NULL) { 377 error = EBADF; 378 goto done; 379 } 380 381 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf; 382 aiov.iov_len = aiocbp->aio_nbytes; 383 auio.uio_iov = &aiov; 384 auio.uio_iovcnt = 1; 385 auio.uio_resid = aiocbp->aio_nbytes; 386 auio.uio_vmspace = p->p_vmspace; 387 388 if (a_job->aio_op & AIO_READ) { 389 /* 390 * Perform a Read operation 391 */ 392 KASSERT((a_job->aio_op & AIO_WRITE) == 0); 393 394 if ((fp->f_flag & FREAD) == 0) { 395 fd_putfile(fd); 396 error = EBADF; 397 goto done; 398 } 399 auio.uio_rw = UIO_READ; 400 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset, 401 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 402 } else { 403 /* 404 * Perform a Write operation 405 */ 406 KASSERT(a_job->aio_op & AIO_WRITE); 407 408 if ((fp->f_flag & FWRITE) == 0) { 409 fd_putfile(fd); 410 error = EBADF; 411 goto done; 412 } 413 auio.uio_rw = UIO_WRITE; 414 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset, 415 &auio, fp->f_cred, FOF_UPDATE_OFFSET); 416 } 417 fd_putfile(fd); 418 419 /* Store the result value */ 420 a_job->aiocbp.aio_nbytes -= auio.uio_resid; 421 a_job->aiocbp._retval = (error == 0) ? 422 a_job->aiocbp.aio_nbytes : -1; 423 424 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) { 425 /* 426 * Perform a file Sync operation 427 */ 428 struct vnode *vp; 429 430 if ((error = fd_getvnode(fd, &fp)) != 0) 431 goto done; 432 433 if ((fp->f_flag & FWRITE) == 0) { 434 fd_putfile(fd); 435 error = EBADF; 436 goto done; 437 } 438 439 vp = fp->f_vnode; 440 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 441 if (a_job->aio_op & AIO_DSYNC) { 442 error = VOP_FSYNC(vp, fp->f_cred, 443 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0); 444 } else if (a_job->aio_op & AIO_SYNC) { 445 error = VOP_FSYNC(vp, fp->f_cred, 446 FSYNC_WAIT, 0, 0); 447 } 448 VOP_UNLOCK(vp); 449 fd_putfile(fd); 450 451 /* Store the result value */ 452 a_job->aiocbp._retval = (error == 0) ? 0 : -1; 453 454 } else 455 panic("aio_process: invalid operation code\n"); 456 457 done: 458 /* Job is done, set the error, if any */ 459 a_job->aiocbp._errno = error; 460 a_job->aiocbp._state = JOB_DONE; 461 } 462 463 /* 464 * Send AIO signal. 465 */ 466 static void 467 aio_sendsig(struct proc *p, struct sigevent *sig) 468 { 469 ksiginfo_t ksi; 470 471 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE) 472 return; 473 474 KSI_INIT(&ksi); 475 ksi.ksi_signo = sig->sigev_signo; 476 ksi.ksi_code = SI_ASYNCIO; 477 ksi.ksi_value = sig->sigev_value; 478 mutex_enter(proc_lock); 479 kpsignal(p, &ksi, NULL); 480 mutex_exit(proc_lock); 481 } 482 483 /* 484 * Enqueue the job. 485 */ 486 static int 487 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio) 488 { 489 struct proc *p = curlwp->l_proc; 490 struct aioproc *aio; 491 struct aio_job *a_job; 492 struct aiocb aiocbp; 493 struct sigevent *sig; 494 int error; 495 496 /* Non-accurate check for the limit */ 497 if (aio_jobs_count + 1 > aio_max) 498 return EAGAIN; 499 500 /* Get the data structure from user-space */ 501 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb)); 502 if (error) 503 return error; 504 505 /* Check if signal is set, and validate it */ 506 sig = &aiocbp.aio_sigevent; 507 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG || 508 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA) 509 return EINVAL; 510 511 /* Buffer and byte count */ 512 if (((AIO_SYNC | AIO_DSYNC) & op) == 0) 513 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX) 514 return EINVAL; 515 516 /* Check the opcode, if LIO_NOP - simply ignore */ 517 if (op == AIO_LIO) { 518 KASSERT(lio != NULL); 519 if (aiocbp.aio_lio_opcode == LIO_WRITE) 520 op = AIO_WRITE; 521 else if (aiocbp.aio_lio_opcode == LIO_READ) 522 op = AIO_READ; 523 else 524 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL; 525 } else { 526 KASSERT(lio == NULL); 527 } 528 529 /* 530 * Look for already existing job. If found - the job is in-progress. 531 * According to POSIX this is invalid, so return the error. 532 */ 533 aio = p->p_aio; 534 if (aio) { 535 mutex_enter(&aio->aio_mtx); 536 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 537 if (a_job->aiocb_uptr != aiocb_uptr) 538 continue; 539 mutex_exit(&aio->aio_mtx); 540 return EINVAL; 541 } 542 mutex_exit(&aio->aio_mtx); 543 } 544 545 /* 546 * Check if AIO structure is initialized, if not - initialize it. 547 * In LIO case, we did that already. We will recheck this with 548 * the lock in aio_procinit(). 549 */ 550 if (lio == NULL && p->p_aio == NULL) 551 if (aio_procinit(p)) 552 return EAGAIN; 553 aio = p->p_aio; 554 555 /* 556 * Set the state with errno, and copy data 557 * structure back to the user-space. 558 */ 559 aiocbp._state = JOB_WIP; 560 aiocbp._errno = EINPROGRESS; 561 aiocbp._retval = -1; 562 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb)); 563 if (error) 564 return error; 565 566 /* Allocate and initialize a new AIO job */ 567 a_job = pool_get(&aio_job_pool, PR_WAITOK | PR_ZERO); 568 569 /* 570 * Set the data. 571 * Store the user-space pointer for searching. Since we 572 * are storing only per proc pointers - it is safe. 573 */ 574 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb)); 575 a_job->aiocb_uptr = aiocb_uptr; 576 a_job->aio_op |= op; 577 a_job->lio = lio; 578 579 /* 580 * Add the job to the queue, update the counters, and 581 * notify the AIO worker thread to handle the job. 582 */ 583 mutex_enter(&aio->aio_mtx); 584 585 /* Fail, if the limit was reached */ 586 if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max || 587 aio->jobs_count >= aio_listio_max) { 588 atomic_dec_uint(&aio_jobs_count); 589 mutex_exit(&aio->aio_mtx); 590 pool_put(&aio_job_pool, a_job); 591 return EAGAIN; 592 } 593 594 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list); 595 aio->jobs_count++; 596 if (lio) 597 lio->refcnt++; 598 cv_signal(&aio->aio_worker_cv); 599 600 mutex_exit(&aio->aio_mtx); 601 602 /* 603 * One would handle the errors only with aio_error() function. 604 * This way is appropriate according to POSIX. 605 */ 606 return 0; 607 } 608 609 /* 610 * Syscall functions. 611 */ 612 613 int 614 sys_aio_cancel(struct lwp *l, const struct sys_aio_cancel_args *uap, 615 register_t *retval) 616 { 617 /* { 618 syscallarg(int) fildes; 619 syscallarg(struct aiocb *) aiocbp; 620 } */ 621 struct proc *p = l->l_proc; 622 struct aioproc *aio; 623 struct aio_job *a_job; 624 struct aiocb *aiocbp_ptr; 625 struct lio_req *lio; 626 struct filedesc *fdp = p->p_fd; 627 unsigned int cn, errcnt, fildes; 628 fdtab_t *dt; 629 630 TAILQ_HEAD(, aio_job) tmp_jobs_list; 631 632 /* Check for invalid file descriptor */ 633 fildes = (unsigned int)SCARG(uap, fildes); 634 dt = atomic_load_consume(&fdp->fd_dt); 635 if (fildes >= dt->dt_nfiles) 636 return EBADF; 637 if (dt->dt_ff[fildes] == NULL || dt->dt_ff[fildes]->ff_file == NULL) 638 return EBADF; 639 640 /* Check if AIO structure is initialized */ 641 if (p->p_aio == NULL) { 642 *retval = AIO_NOTCANCELED; 643 return 0; 644 } 645 646 aio = p->p_aio; 647 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp); 648 649 mutex_enter(&aio->aio_mtx); 650 651 /* Cancel the jobs, and remove them from the queue */ 652 cn = 0; 653 TAILQ_INIT(&tmp_jobs_list); 654 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 655 if (aiocbp_ptr) { 656 if (aiocbp_ptr != a_job->aiocb_uptr) 657 continue; 658 if (fildes != a_job->aiocbp.aio_fildes) { 659 mutex_exit(&aio->aio_mtx); 660 return EBADF; 661 } 662 } else if (a_job->aiocbp.aio_fildes != fildes) 663 continue; 664 665 TAILQ_REMOVE(&aio->jobs_queue, a_job, list); 666 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list); 667 668 /* Decrease the counters */ 669 atomic_dec_uint(&aio_jobs_count); 670 aio->jobs_count--; 671 lio = a_job->lio; 672 if (lio != NULL && --lio->refcnt != 0) 673 a_job->lio = NULL; 674 675 cn++; 676 if (aiocbp_ptr) 677 break; 678 } 679 680 /* There are canceled jobs */ 681 if (cn) 682 *retval = AIO_CANCELED; 683 684 /* We cannot cancel current job */ 685 a_job = aio->curjob; 686 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) || 687 (a_job->aiocb_uptr == aiocbp_ptr))) 688 *retval = AIO_NOTCANCELED; 689 690 mutex_exit(&aio->aio_mtx); 691 692 /* Free the jobs after the lock */ 693 errcnt = 0; 694 while (!TAILQ_EMPTY(&tmp_jobs_list)) { 695 a_job = TAILQ_FIRST(&tmp_jobs_list); 696 TAILQ_REMOVE(&tmp_jobs_list, a_job, list); 697 /* Set the errno and copy structures back to the user-space */ 698 a_job->aiocbp._errno = ECANCELED; 699 a_job->aiocbp._state = JOB_DONE; 700 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr, 701 sizeof(struct aiocb))) 702 errcnt++; 703 /* Send a signal if any */ 704 aio_sendsig(p, &a_job->aiocbp.aio_sigevent); 705 if (a_job->lio) { 706 lio = a_job->lio; 707 aio_sendsig(p, &lio->sig); 708 pool_put(&aio_lio_pool, lio); 709 } 710 pool_put(&aio_job_pool, a_job); 711 } 712 713 if (errcnt) 714 return EFAULT; 715 716 /* Set a correct return value */ 717 if (*retval == 0) 718 *retval = AIO_ALLDONE; 719 720 return 0; 721 } 722 723 int 724 sys_aio_error(struct lwp *l, const struct sys_aio_error_args *uap, 725 register_t *retval) 726 { 727 /* { 728 syscallarg(const struct aiocb *) aiocbp; 729 } */ 730 struct proc *p = l->l_proc; 731 struct aioproc *aio = p->p_aio; 732 struct aiocb aiocbp; 733 int error; 734 735 if (aio == NULL) 736 return EINVAL; 737 738 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 739 if (error) 740 return error; 741 742 if (aiocbp._state == JOB_NONE) 743 return EINVAL; 744 745 *retval = aiocbp._errno; 746 747 return 0; 748 } 749 750 int 751 sys_aio_fsync(struct lwp *l, const struct sys_aio_fsync_args *uap, 752 register_t *retval) 753 { 754 /* { 755 syscallarg(int) op; 756 syscallarg(struct aiocb *) aiocbp; 757 } */ 758 int op = SCARG(uap, op); 759 760 if ((op != O_DSYNC) && (op != O_SYNC)) 761 return EINVAL; 762 763 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC; 764 765 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL); 766 } 767 768 int 769 sys_aio_read(struct lwp *l, const struct sys_aio_read_args *uap, 770 register_t *retval) 771 { 772 /* { 773 syscallarg(struct aiocb *) aiocbp; 774 } */ 775 776 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL); 777 } 778 779 int 780 sys_aio_return(struct lwp *l, const struct sys_aio_return_args *uap, 781 register_t *retval) 782 { 783 /* { 784 syscallarg(struct aiocb *) aiocbp; 785 } */ 786 struct proc *p = l->l_proc; 787 struct aioproc *aio = p->p_aio; 788 struct aiocb aiocbp; 789 int error; 790 791 if (aio == NULL) 792 return EINVAL; 793 794 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb)); 795 if (error) 796 return error; 797 798 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE) 799 return EINVAL; 800 801 *retval = aiocbp._retval; 802 803 /* Reset the internal variables */ 804 aiocbp._errno = 0; 805 aiocbp._retval = -1; 806 aiocbp._state = JOB_NONE; 807 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb)); 808 809 return error; 810 } 811 812 int 813 sys___aio_suspend50(struct lwp *l, const struct sys___aio_suspend50_args *uap, 814 register_t *retval) 815 { 816 /* { 817 syscallarg(const struct aiocb *const[]) list; 818 syscallarg(int) nent; 819 syscallarg(const struct timespec *) timeout; 820 } */ 821 struct aiocb **list; 822 struct timespec ts; 823 int error, nent; 824 825 nent = SCARG(uap, nent); 826 if (nent <= 0 || nent > aio_listio_max) 827 return EAGAIN; 828 829 if (SCARG(uap, timeout)) { 830 /* Convert timespec to ticks */ 831 error = copyin(SCARG(uap, timeout), &ts, 832 sizeof(struct timespec)); 833 if (error) 834 return error; 835 } 836 837 list = kmem_alloc(nent * sizeof(*list), KM_SLEEP); 838 error = copyin(SCARG(uap, list), list, nent * sizeof(*list)); 839 if (error) 840 goto out; 841 error = aio_suspend1(l, list, nent, SCARG(uap, timeout) ? &ts : NULL); 842 out: 843 kmem_free(list, nent * sizeof(*list)); 844 return error; 845 } 846 847 int 848 aio_suspend1(struct lwp *l, struct aiocb **aiocbp_list, int nent, 849 struct timespec *ts) 850 { 851 struct proc *p = l->l_proc; 852 struct aioproc *aio; 853 struct aio_job *a_job; 854 int i, error, timo; 855 856 if (p->p_aio == NULL) 857 return EAGAIN; 858 aio = p->p_aio; 859 860 if (ts) { 861 timo = mstohz((ts->tv_sec * 1000) + (ts->tv_nsec / 1000000)); 862 if (timo == 0 && ts->tv_sec == 0 && ts->tv_nsec > 0) 863 timo = 1; 864 if (timo <= 0) 865 return EAGAIN; 866 } else 867 timo = 0; 868 869 mutex_enter(&aio->aio_mtx); 870 for (;;) { 871 for (i = 0; i < nent; i++) { 872 873 /* Skip NULL entries */ 874 if (aiocbp_list[i] == NULL) 875 continue; 876 877 /* Skip current job */ 878 if (aio->curjob) { 879 a_job = aio->curjob; 880 if (a_job->aiocb_uptr == aiocbp_list[i]) 881 continue; 882 } 883 884 /* Look for a job in the queue */ 885 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) 886 if (a_job->aiocb_uptr == aiocbp_list[i]) 887 break; 888 889 if (a_job == NULL) { 890 struct aiocb aiocbp; 891 892 mutex_exit(&aio->aio_mtx); 893 894 /* Check if the job is done. */ 895 error = copyin(aiocbp_list[i], &aiocbp, 896 sizeof(struct aiocb)); 897 if (error == 0 && aiocbp._state != JOB_DONE) { 898 mutex_enter(&aio->aio_mtx); 899 continue; 900 } 901 return error; 902 } 903 } 904 905 /* Wait for a signal or when timeout occurs */ 906 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo); 907 if (error) { 908 if (error == EWOULDBLOCK) 909 error = EAGAIN; 910 break; 911 } 912 } 913 mutex_exit(&aio->aio_mtx); 914 return error; 915 } 916 917 int 918 sys_aio_write(struct lwp *l, const struct sys_aio_write_args *uap, 919 register_t *retval) 920 { 921 /* { 922 syscallarg(struct aiocb *) aiocbp; 923 } */ 924 925 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL); 926 } 927 928 int 929 sys_lio_listio(struct lwp *l, const struct sys_lio_listio_args *uap, 930 register_t *retval) 931 { 932 /* { 933 syscallarg(int) mode; 934 syscallarg(struct aiocb *const[]) list; 935 syscallarg(int) nent; 936 syscallarg(struct sigevent *) sig; 937 } */ 938 struct proc *p = l->l_proc; 939 struct aioproc *aio; 940 struct aiocb **aiocbp_list; 941 struct lio_req *lio; 942 int i, error, errcnt, mode, nent; 943 944 mode = SCARG(uap, mode); 945 nent = SCARG(uap, nent); 946 947 /* Non-accurate checks for the limit and invalid values */ 948 if (nent < 1 || nent > aio_listio_max) 949 return EINVAL; 950 if (aio_jobs_count + nent > aio_max) 951 return EAGAIN; 952 953 /* Check if AIO structure is initialized, if not - initialize it */ 954 if (p->p_aio == NULL) 955 if (aio_procinit(p)) 956 return EAGAIN; 957 aio = p->p_aio; 958 959 /* Create a LIO structure */ 960 lio = pool_get(&aio_lio_pool, PR_WAITOK); 961 lio->refcnt = 1; 962 error = 0; 963 964 switch (mode) { 965 case LIO_WAIT: 966 memset(&lio->sig, 0, sizeof(struct sigevent)); 967 break; 968 case LIO_NOWAIT: 969 /* Check for signal, validate it */ 970 if (SCARG(uap, sig)) { 971 struct sigevent *sig = &lio->sig; 972 973 error = copyin(SCARG(uap, sig), &lio->sig, 974 sizeof(struct sigevent)); 975 if (error == 0 && 976 (sig->sigev_signo < 0 || 977 sig->sigev_signo >= NSIG || 978 sig->sigev_notify < SIGEV_NONE || 979 sig->sigev_notify > SIGEV_SA)) 980 error = EINVAL; 981 } else 982 memset(&lio->sig, 0, sizeof(struct sigevent)); 983 break; 984 default: 985 error = EINVAL; 986 break; 987 } 988 989 if (error != 0) { 990 pool_put(&aio_lio_pool, lio); 991 return error; 992 } 993 994 /* Get the list from user-space */ 995 aiocbp_list = kmem_alloc(nent * sizeof(*aiocbp_list), KM_SLEEP); 996 error = copyin(SCARG(uap, list), aiocbp_list, 997 nent * sizeof(*aiocbp_list)); 998 if (error) { 999 mutex_enter(&aio->aio_mtx); 1000 goto err; 1001 } 1002 1003 /* Enqueue all jobs */ 1004 errcnt = 0; 1005 for (i = 0; i < nent; i++) { 1006 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio); 1007 /* 1008 * According to POSIX, in such error case it may 1009 * fail with other I/O operations initiated. 1010 */ 1011 if (error) 1012 errcnt++; 1013 } 1014 1015 mutex_enter(&aio->aio_mtx); 1016 1017 /* Return an error, if any */ 1018 if (errcnt) { 1019 error = EIO; 1020 goto err; 1021 } 1022 1023 if (mode == LIO_WAIT) { 1024 /* 1025 * Wait for AIO completion. In such case, 1026 * the LIO structure will be freed here. 1027 */ 1028 while (lio->refcnt > 1 && error == 0) 1029 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx); 1030 if (error) 1031 error = EINTR; 1032 } 1033 1034 err: 1035 if (--lio->refcnt != 0) 1036 lio = NULL; 1037 mutex_exit(&aio->aio_mtx); 1038 if (lio != NULL) { 1039 aio_sendsig(p, &lio->sig); 1040 pool_put(&aio_lio_pool, lio); 1041 } 1042 kmem_free(aiocbp_list, nent * sizeof(*aiocbp_list)); 1043 return error; 1044 } 1045 1046 /* 1047 * SysCtl 1048 */ 1049 1050 static int 1051 sysctl_aio_listio_max(SYSCTLFN_ARGS) 1052 { 1053 struct sysctlnode node; 1054 int error, newsize; 1055 1056 node = *rnode; 1057 node.sysctl_data = &newsize; 1058 1059 newsize = aio_listio_max; 1060 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1061 if (error || newp == NULL) 1062 return error; 1063 1064 if (newsize < 1 || newsize > aio_max) 1065 return EINVAL; 1066 aio_listio_max = newsize; 1067 1068 return 0; 1069 } 1070 1071 static int 1072 sysctl_aio_max(SYSCTLFN_ARGS) 1073 { 1074 struct sysctlnode node; 1075 int error, newsize; 1076 1077 node = *rnode; 1078 node.sysctl_data = &newsize; 1079 1080 newsize = aio_max; 1081 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1082 if (error || newp == NULL) 1083 return error; 1084 1085 if (newsize < 1 || newsize < aio_listio_max) 1086 return EINVAL; 1087 aio_max = newsize; 1088 1089 return 0; 1090 } 1091 1092 static int 1093 sysctl_aio_init(void) 1094 { 1095 int rv; 1096 1097 aio_sysctl = NULL; 1098 1099 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1100 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE, 1101 CTLTYPE_INT, "posix_aio", 1102 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its " 1103 "Asynchronous I/O option to which the " 1104 "system attempts to conform"), 1105 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0, 1106 CTL_KERN, CTL_CREATE, CTL_EOL); 1107 1108 if (rv != 0) 1109 return rv; 1110 1111 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1112 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1113 CTLTYPE_INT, "aio_listio_max", 1114 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1115 "operations in a single list I/O call"), 1116 sysctl_aio_listio_max, 0, &aio_listio_max, 0, 1117 CTL_KERN, CTL_CREATE, CTL_EOL); 1118 1119 if (rv != 0) 1120 return rv; 1121 1122 rv = sysctl_createv(&aio_sysctl, 0, NULL, NULL, 1123 CTLFLAG_PERMANENT | CTLFLAG_READWRITE, 1124 CTLTYPE_INT, "aio_max", 1125 SYSCTL_DESCR("Maximum number of asynchronous I/O " 1126 "operations"), 1127 sysctl_aio_max, 0, &aio_max, 0, 1128 CTL_KERN, CTL_CREATE, CTL_EOL); 1129 1130 return rv; 1131 } 1132 1133 /* 1134 * Debugging 1135 */ 1136 #if defined(DDB) 1137 void 1138 aio_print_jobs(void (*pr)(const char *, ...)) 1139 { 1140 struct proc *p = curlwp->l_proc; 1141 struct aioproc *aio; 1142 struct aio_job *a_job; 1143 struct aiocb *aiocbp; 1144 1145 if (p == NULL) { 1146 (*pr)("AIO: We are not in the processes right now.\n"); 1147 return; 1148 } 1149 1150 aio = p->p_aio; 1151 if (aio == NULL) { 1152 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid); 1153 return; 1154 } 1155 1156 (*pr)("AIO: PID = %d\n", p->p_pid); 1157 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count); 1158 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count); 1159 1160 if (aio->curjob) { 1161 a_job = aio->curjob; 1162 (*pr)("\nAIO current job:\n"); 1163 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1164 a_job->aio_op, a_job->aiocbp._errno, 1165 a_job->aiocbp._state, a_job->aiocb_uptr); 1166 aiocbp = &a_job->aiocbp; 1167 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1168 aiocbp->aio_fildes, aiocbp->aio_offset, 1169 aiocbp->aio_buf, aiocbp->aio_nbytes); 1170 } 1171 1172 (*pr)("\nAIO queue:\n"); 1173 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) { 1174 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n", 1175 a_job->aio_op, a_job->aiocbp._errno, 1176 a_job->aiocbp._state, a_job->aiocb_uptr); 1177 aiocbp = &a_job->aiocbp; 1178 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n", 1179 aiocbp->aio_fildes, aiocbp->aio_offset, 1180 aiocbp->aio_buf, aiocbp->aio_nbytes); 1181 } 1182 } 1183 #endif /* defined(DDB) */ 1184
Indexes created Tue Sep 30 20:09:53 GMT 2025