1 /* $NetBSD: usb_mem.c,v 1.84 2021/12/21 09:51:22 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 1998 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Lennart Augustsson (lennart (at) augustsson.net) at 9 * Carlstedt Research & Technology. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * USB DMA memory allocation. 35 * We need to allocate a lot of small (many 8 byte, some larger) 36 * memory blocks that can be used for DMA. Using the bus_dma 37 * routines directly would incur large overheads in space and time. 38 */ 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: usb_mem.c,v 1.84 2021/12/21 09:51:22 skrll Exp $"); 42 43 #ifdef _KERNEL_OPT 44 #include "opt_usb.h" 45 #endif 46 47 #include <sys/param.h> 48 #include <sys/bus.h> 49 #include <sys/cpu.h> 50 #include <sys/device.h> /* for usbdivar.h */ 51 #include <sys/kernel.h> 52 #include <sys/kmem.h> 53 #include <sys/once.h> 54 #include <sys/queue.h> 55 #include <sys/systm.h> 56 57 #include <dev/usb/usb.h> 58 #include <dev/usb/usbdi.h> 59 #include <dev/usb/usbdivar.h> /* just for usb_dma_t */ 60 #include <dev/usb/usbhist.h> 61 #include <dev/usb/usb_mem.h> 62 63 #define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(usbdebug,FMT,A,B,C,D) 64 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(usbdebug,N,FMT,A,B,C,D) 65 66 #define USB_MEM_SMALL roundup(64, CACHE_LINE_SIZE) 67 #define USB_MEM_CHUNKS 64 68 #define USB_MEM_BLOCK (USB_MEM_SMALL * USB_MEM_CHUNKS) 69 70 /* This struct is overlayed on free fragments. */ 71 struct usb_frag_dma { 72 usb_dma_block_t *ufd_block; 73 u_int ufd_offs; 74 LIST_ENTRY(usb_frag_dma) ufd_next; 75 }; 76 77 Static int usb_block_allocmem(bus_dma_tag_t, size_t, size_t, 78 u_int, usb_dma_block_t **); 79 Static void usb_block_freemem(usb_dma_block_t *); 80 81 LIST_HEAD(usb_dma_block_qh, usb_dma_block); 82 Static struct usb_dma_block_qh usb_blk_freelist = 83 LIST_HEAD_INITIALIZER(usb_blk_freelist); 84 kmutex_t usb_blk_lock; 85 86 #ifdef DEBUG 87 Static struct usb_dma_block_qh usb_blk_fraglist = 88 LIST_HEAD_INITIALIZER(usb_blk_fraglist); 89 Static struct usb_dma_block_qh usb_blk_fulllist = 90 LIST_HEAD_INITIALIZER(usb_blk_fulllist); 91 #endif 92 Static u_int usb_blk_nfree = 0; 93 /* XXX should have different free list for different tags (for speed) */ 94 Static LIST_HEAD(, usb_frag_dma) usb_frag_freelist = 95 LIST_HEAD_INITIALIZER(usb_frag_freelist); 96 97 Static int usb_mem_init(void); 98 99 Static int 100 usb_mem_init(void) 101 { 102 103 mutex_init(&usb_blk_lock, MUTEX_DEFAULT, IPL_NONE); 104 return 0; 105 } 106 107 Static int 108 usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align, 109 u_int flags, usb_dma_block_t **dmap) 110 { 111 usb_dma_block_t *b; 112 int error; 113 114 USBHIST_FUNC(); 115 USBHIST_CALLARGS(usbdebug, "size=%ju align=%ju flags=%#jx", size, align, flags, 0); 116 117 ASSERT_SLEEPABLE(); 118 KASSERT(size != 0); 119 KASSERT(mutex_owned(&usb_blk_lock)); 120 121 #ifdef USB_FRAG_DMA_WORKAROUND 122 flags |= USBMALLOC_ZERO; 123 #endif 124 125 bool multiseg = (flags & USBMALLOC_MULTISEG) != 0; 126 bool coherent = (flags & USBMALLOC_COHERENT) != 0; 127 bool zero = (flags & USBMALLOC_ZERO) != 0; 128 u_int dmaflags = coherent ? USB_DMA_COHERENT : 0; 129 130 /* First check the free list. */ 131 LIST_FOREACH(b, &usb_blk_freelist, next) { 132 /* Don't allocate multiple segments to unwilling callers */ 133 if (b->nsegs != 1 && !multiseg) 134 continue; 135 if (b->tag == tag && 136 b->size >= size && 137 b->align >= align && 138 (b->flags & USB_DMA_COHERENT) == dmaflags) { 139 LIST_REMOVE(b, next); 140 usb_blk_nfree--; 141 *dmap = b; 142 if (zero) { 143 memset(b->kaddr, 0, b->size); 144 bus_dmamap_sync(b->tag, b->map, 0, b->size, 145 BUS_DMASYNC_PREWRITE); 146 } 147 DPRINTFN(6, "free list size=%ju", b->size, 0, 0, 0); 148 return 0; 149 } 150 } 151 152 DPRINTFN(6, "no freelist entry", 0, 0, 0, 0); 153 mutex_exit(&usb_blk_lock); 154 155 b = kmem_zalloc(sizeof(*b), KM_SLEEP); 156 b->tag = tag; 157 b->size = size; 158 b->align = align; 159 b->flags = dmaflags; 160 161 if (!multiseg) 162 /* Caller wants one segment */ 163 b->nsegs = 1; 164 else 165 b->nsegs = howmany(size, PAGE_SIZE); 166 167 b->segs = kmem_alloc(b->nsegs * sizeof(*b->segs), KM_SLEEP); 168 b->nsegs_alloc = b->nsegs; 169 170 error = bus_dmamem_alloc(tag, b->size, align, 0, b->segs, b->nsegs, 171 &b->nsegs, BUS_DMA_WAITOK); 172 if (error) 173 goto free0; 174 175 error = bus_dmamem_map(tag, b->segs, b->nsegs, b->size, &b->kaddr, 176 BUS_DMA_WAITOK | (coherent ? BUS_DMA_COHERENT : 0)); 177 if (error) 178 goto free1; 179 180 error = bus_dmamap_create(tag, b->size, b->nsegs, b->size, 0, 181 BUS_DMA_WAITOK, &b->map); 182 if (error) 183 goto unmap; 184 185 error = bus_dmamap_load(tag, b->map, b->kaddr, b->size, NULL, 186 BUS_DMA_WAITOK); 187 if (error) 188 goto destroy; 189 190 *dmap = b; 191 192 if (zero) { 193 memset(b->kaddr, 0, b->size); 194 bus_dmamap_sync(b->tag, b->map, 0, b->size, 195 BUS_DMASYNC_PREWRITE); 196 } 197 198 mutex_enter(&usb_blk_lock); 199 200 return 0; 201 202 destroy: 203 bus_dmamap_destroy(tag, b->map); 204 unmap: 205 bus_dmamem_unmap(tag, b->kaddr, b->size); 206 free1: 207 bus_dmamem_free(tag, b->segs, b->nsegs); 208 free0: 209 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs)); 210 kmem_free(b, sizeof(*b)); 211 mutex_enter(&usb_blk_lock); 212 213 return error; 214 } 215 216 #if 0 217 void 218 usb_block_real_freemem(usb_dma_block_t *b) 219 { 220 ASSERT_SLEEPABLE(); 221 222 bus_dmamap_unload(b->tag, b->map); 223 bus_dmamap_destroy(b->tag, b->map); 224 bus_dmamem_unmap(b->tag, b->kaddr, b->size); 225 bus_dmamem_free(b->tag, b->segs, b->nsegs); 226 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs)); 227 kmem_free(b, sizeof(*b)); 228 } 229 #endif 230 231 #ifdef DEBUG 232 static bool 233 usb_valid_block_p(usb_dma_block_t *b, struct usb_dma_block_qh *qh) 234 { 235 usb_dma_block_t *xb; 236 LIST_FOREACH(xb, qh, next) { 237 if (xb == b) 238 return true; 239 } 240 return false; 241 } 242 #endif 243 244 /* 245 * Do not free the memory unconditionally since we might be called 246 * from an interrupt context and that is BAD. 247 * XXX when should we really free? 248 */ 249 Static void 250 usb_block_freemem(usb_dma_block_t *b) 251 { 252 USBHIST_FUNC(); 253 USBHIST_CALLARGS(usbdebug, "size=%ju", b->size, 0, 0, 0); 254 255 KASSERT(mutex_owned(&usb_blk_lock)); 256 257 #ifdef DEBUG 258 LIST_REMOVE(b, next); 259 #endif 260 LIST_INSERT_HEAD(&usb_blk_freelist, b, next); 261 usb_blk_nfree++; 262 } 263 264 int 265 usb_allocmem(bus_dma_tag_t tag, size_t size, size_t align, u_int flags, 266 usb_dma_t *p) 267 { 268 usbd_status err; 269 struct usb_frag_dma *f; 270 usb_dma_block_t *b; 271 int i; 272 static ONCE_DECL(init_control); 273 274 USBHIST_FUNC(); USBHIST_CALLED(usbdebug); 275 276 ASSERT_SLEEPABLE(); 277 278 RUN_ONCE(&init_control, usb_mem_init); 279 280 u_int dmaflags = (flags & USBMALLOC_COHERENT) ? USB_DMA_COHERENT : 0; 281 282 /* If the request is large then just use a full block. */ 283 if (size > USB_MEM_SMALL || align > USB_MEM_SMALL) { 284 DPRINTFN(1, "large alloc %jd", size, 0, 0, 0); 285 size = (size + USB_MEM_BLOCK - 1) & ~(USB_MEM_BLOCK - 1); 286 mutex_enter(&usb_blk_lock); 287 err = usb_block_allocmem(tag, size, align, flags, 288 &p->udma_block); 289 if (!err) { 290 #ifdef DEBUG 291 LIST_INSERT_HEAD(&usb_blk_fulllist, p->udma_block, next); 292 #endif 293 p->udma_block->flags = USB_DMA_FULLBLOCK | dmaflags; 294 p->udma_offs = 0; 295 } 296 mutex_exit(&usb_blk_lock); 297 return err; 298 } 299 300 mutex_enter(&usb_blk_lock); 301 /* Check for free fragments. */ 302 LIST_FOREACH(f, &usb_frag_freelist, ufd_next) { 303 KDASSERTMSG(usb_valid_block_p(f->ufd_block, &usb_blk_fraglist), 304 "%s: usb frag %p: unknown block pointer %p", 305 __func__, f, f->ufd_block); 306 if (f->ufd_block->tag == tag && 307 (f->ufd_block->flags & USB_DMA_COHERENT) == dmaflags) 308 break; 309 } 310 if (f == NULL) { 311 DPRINTFN(1, "adding fragments", 0, 0, 0, 0); 312 313 err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL, 314 flags, &b); 315 if (err) { 316 mutex_exit(&usb_blk_lock); 317 return err; 318 } 319 #ifdef DEBUG 320 LIST_INSERT_HEAD(&usb_blk_fraglist, b, next); 321 #endif 322 b->flags = 0; 323 for (i = 0; i < USB_MEM_BLOCK; i += USB_MEM_SMALL) { 324 f = (struct usb_frag_dma *)((char *)b->kaddr + i); 325 f->ufd_block = b; 326 f->ufd_offs = i; 327 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next); 328 #ifdef USB_FRAG_DMA_WORKAROUND 329 i += 1 * USB_MEM_SMALL; 330 #endif 331 } 332 f = LIST_FIRST(&usb_frag_freelist); 333 } 334 p->udma_block = f->ufd_block; 335 p->udma_offs = f->ufd_offs; 336 #ifdef USB_FRAG_DMA_WORKAROUND 337 p->udma_offs += USB_MEM_SMALL; 338 #endif 339 LIST_REMOVE(f, ufd_next); 340 mutex_exit(&usb_blk_lock); 341 DPRINTFN(5, "use frag=%#jx size=%jd", (uintptr_t)f, size, 0, 0); 342 343 return 0; 344 } 345 346 void 347 usb_freemem(usb_dma_t *p) 348 { 349 struct usb_frag_dma *f; 350 351 USBHIST_FUNC(); USBHIST_CALLED(usbdebug); 352 353 mutex_enter(&usb_blk_lock); 354 if (p->udma_block->flags & USB_DMA_FULLBLOCK) { 355 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fulllist), 356 "%s: dma %p: invalid block pointer %p", 357 __func__, p, p->udma_block); 358 DPRINTFN(1, "large free", 0, 0, 0, 0); 359 usb_block_freemem(p->udma_block); 360 mutex_exit(&usb_blk_lock); 361 return; 362 } 363 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fraglist), 364 "%s: dma %p: invalid block pointer %p", 365 __func__, p, p->udma_block); 366 //usb_syncmem(p, 0, USB_MEM_SMALL, BUS_DMASYNC_POSTREAD); 367 f = KERNADDR(p, 0); 368 #ifdef USB_FRAG_DMA_WORKAROUND 369 f = (void *)((uintptr_t)f - USB_MEM_SMALL); 370 #endif 371 f->ufd_block = p->udma_block; 372 f->ufd_offs = p->udma_offs; 373 #ifdef USB_FRAG_DMA_WORKAROUND 374 f->ufd_offs -= USB_MEM_SMALL; 375 #endif 376 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next); 377 mutex_exit(&usb_blk_lock); 378 DPRINTFN(5, "frag=%#jx", (uintptr_t)f, 0, 0, 0); 379 } 380 381 bus_addr_t 382 usb_dmaaddr(usb_dma_t *dma, unsigned int offset) 383 { 384 unsigned int i; 385 bus_size_t seg_offs; 386 387 offset += dma->udma_offs; 388 389 KASSERTMSG(offset < dma->udma_block->size, "offset %d vs %zu", offset, 390 dma->udma_block->size); 391 392 if (dma->udma_block->nsegs == 1) { 393 KASSERT(dma->udma_block->map->dm_segs[0].ds_len > offset); 394 return dma->udma_block->map->dm_segs[0].ds_addr + offset; 395 } 396 397 /* 398 * Search for a bus_segment_t corresponding to this offset. With no 399 * record of the offset in the map to a particular dma_segment_t, we 400 * have to iterate from the start of the list each time. Could be 401 * improved 402 */ 403 seg_offs = 0; 404 for (i = 0; i < dma->udma_block->nsegs; i++) { 405 if (seg_offs + dma->udma_block->map->dm_segs[i].ds_len > offset) 406 break; 407 408 seg_offs += dma->udma_block->map->dm_segs[i].ds_len; 409 } 410 411 KASSERT(i != dma->udma_block->nsegs); 412 offset -= seg_offs; 413 return dma->udma_block->map->dm_segs[i].ds_addr + offset; 414 } 415 416 void 417 usb_syncmem(usb_dma_t *p, bus_addr_t offset, bus_size_t len, int ops) 418 { 419 420 bus_dmamap_sync(p->udma_block->tag, p->udma_block->map, 421 p->udma_offs + offset, len, ops); 422 } 423
Indexes created Sun Sep 21 14:09:52 GMT 2025