1 /* $NetBSD: vmwgfx_blit.c,v 1.2 2021/12/18 23:45:45 riastradh Exp $ */ 2 3 // SPDX-License-Identifier: GPL-2.0 OR MIT 4 /************************************************************************** 5 * 6 * Copyright 2017 VMware, Inc., Palo Alto, CA., USA 7 * All Rights Reserved. 8 * 9 * Permission is hereby granted, free of charge, to any person obtaining a 10 * copy of this software and associated documentation files (the 11 * "Software"), to deal in the Software without restriction, including 12 * without limitation the rights to use, copy, modify, merge, publish, 13 * distribute, sub license, and/or sell copies of the Software, and to 14 * permit persons to whom the Software is furnished to do so, subject to 15 * the following conditions: 16 * 17 * The above copyright notice and this permission notice (including the 18 * next paragraph) shall be included in all copies or substantial portions 19 * of the Software. 20 * 21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 23 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 24 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 25 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 26 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 27 * USE OR OTHER DEALINGS IN THE SOFTWARE. 28 * 29 **************************************************************************/ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: vmwgfx_blit.c,v 1.2 2021/12/18 23:45:45 riastradh Exp $"); 33 34 #include "vmwgfx_drv.h" 35 36 /* 37 * Template that implements find_first_diff() for a generic 38 * unsigned integer type. @size and return value are in bytes. 39 */ 40 #define VMW_FIND_FIRST_DIFF(_type) \ 41 static size_t vmw_find_first_diff_ ## _type \ 42 (const _type * dst, const _type * src, size_t size)\ 43 { \ 44 size_t i; \ 45 \ 46 for (i = 0; i < size; i += sizeof(_type)) { \ 47 if (*dst++ != *src++) \ 48 break; \ 49 } \ 50 \ 51 return i; \ 52 } 53 54 55 /* 56 * Template that implements find_last_diff() for a generic 57 * unsigned integer type. Pointers point to the item following the 58 * *end* of the area to be examined. @size and return value are in 59 * bytes. 60 */ 61 #define VMW_FIND_LAST_DIFF(_type) \ 62 static ssize_t vmw_find_last_diff_ ## _type( \ 63 const _type * dst, const _type * src, size_t size) \ 64 { \ 65 while (size) { \ 66 if (*--dst != *--src) \ 67 break; \ 68 \ 69 size -= sizeof(_type); \ 70 } \ 71 return size; \ 72 } 73 74 75 /* 76 * Instantiate find diff functions for relevant unsigned integer sizes, 77 * assuming that wider integers are faster (including aligning) up to the 78 * architecture native width, which is assumed to be 32 bit unless 79 * CONFIG_64BIT is defined. 80 */ 81 VMW_FIND_FIRST_DIFF(u8); 82 VMW_FIND_LAST_DIFF(u8); 83 84 VMW_FIND_FIRST_DIFF(u16); 85 VMW_FIND_LAST_DIFF(u16); 86 87 VMW_FIND_FIRST_DIFF(u32); 88 VMW_FIND_LAST_DIFF(u32); 89 90 #ifdef CONFIG_64BIT 91 VMW_FIND_FIRST_DIFF(u64); 92 VMW_FIND_LAST_DIFF(u64); 93 #endif 94 95 96 /* We use size aligned copies. This computes (addr - align(addr)) */ 97 #define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1)) 98 99 100 /* 101 * Template to compute find_first_diff() for a certain integer type 102 * including a head copy for alignment, and adjustment of parameters 103 * for tail find or increased resolution find using an unsigned integer find 104 * of smaller width. If finding is complete, and resolution is sufficient, 105 * the macro executes a return statement. Otherwise it falls through. 106 */ 107 #define VMW_TRY_FIND_FIRST_DIFF(_type) \ 108 do { \ 109 unsigned int spill = SPILL(dst, _type); \ 110 size_t diff_offs; \ 111 \ 112 if (spill && spill == SPILL(src, _type) && \ 113 sizeof(_type) - spill <= size) { \ 114 spill = sizeof(_type) - spill; \ 115 diff_offs = vmw_find_first_diff_u8(dst, src, spill); \ 116 if (diff_offs < spill) \ 117 return round_down(offset + diff_offs, granularity); \ 118 \ 119 dst += spill; \ 120 src += spill; \ 121 size -= spill; \ 122 offset += spill; \ 123 spill = 0; \ 124 } \ 125 if (!spill && !SPILL(src, _type)) { \ 126 size_t to_copy = size & ~(sizeof(_type) - 1); \ 127 \ 128 diff_offs = vmw_find_first_diff_ ## _type \ 129 ((_type *) dst, (_type *) src, to_copy); \ 130 if (diff_offs >= size || granularity == sizeof(_type)) \ 131 return (offset + diff_offs); \ 132 \ 133 dst += diff_offs; \ 134 src += diff_offs; \ 135 size -= diff_offs; \ 136 offset += diff_offs; \ 137 } \ 138 } while (0) \ 139 140 141 /** 142 * vmw_find_first_diff - find the first difference between dst and src 143 * 144 * @dst: The destination address 145 * @src: The source address 146 * @size: Number of bytes to compare 147 * @granularity: The granularity needed for the return value in bytes. 148 * return: The offset from find start where the first difference was 149 * encountered in bytes. If no difference was found, the function returns 150 * a value >= @size. 151 */ 152 static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size, 153 size_t granularity) 154 { 155 size_t offset = 0; 156 157 /* 158 * Try finding with large integers if alignment allows, or we can 159 * fix it. Fall through if we need better resolution or alignment 160 * was bad. 161 */ 162 #ifdef CONFIG_64BIT 163 VMW_TRY_FIND_FIRST_DIFF(u64); 164 #endif 165 VMW_TRY_FIND_FIRST_DIFF(u32); 166 VMW_TRY_FIND_FIRST_DIFF(u16); 167 168 return round_down(offset + vmw_find_first_diff_u8(dst, src, size), 169 granularity); 170 } 171 172 173 /* 174 * Template to compute find_last_diff() for a certain integer type 175 * including a tail copy for alignment, and adjustment of parameters 176 * for head find or increased resolution find using an unsigned integer find 177 * of smaller width. If finding is complete, and resolution is sufficient, 178 * the macro executes a return statement. Otherwise it falls through. 179 */ 180 #define VMW_TRY_FIND_LAST_DIFF(_type) \ 181 do { \ 182 unsigned int spill = SPILL(dst, _type); \ 183 ssize_t location; \ 184 ssize_t diff_offs; \ 185 \ 186 if (spill && spill <= size && spill == SPILL(src, _type)) { \ 187 diff_offs = vmw_find_last_diff_u8(dst, src, spill); \ 188 if (diff_offs) { \ 189 location = size - spill + diff_offs - 1; \ 190 return round_down(location, granularity); \ 191 } \ 192 \ 193 dst -= spill; \ 194 src -= spill; \ 195 size -= spill; \ 196 spill = 0; \ 197 } \ 198 if (!spill && !SPILL(src, _type)) { \ 199 size_t to_copy = round_down(size, sizeof(_type)); \ 200 \ 201 diff_offs = vmw_find_last_diff_ ## _type \ 202 ((_type *) dst, (_type *) src, to_copy); \ 203 location = size - to_copy + diff_offs - sizeof(_type); \ 204 if (location < 0 || granularity == sizeof(_type)) \ 205 return location; \ 206 \ 207 dst -= to_copy - diff_offs; \ 208 src -= to_copy - diff_offs; \ 209 size -= to_copy - diff_offs; \ 210 } \ 211 } while (0) 212 213 214 /** 215 * vmw_find_last_diff - find the last difference between dst and src 216 * 217 * @dst: The destination address 218 * @src: The source address 219 * @size: Number of bytes to compare 220 * @granularity: The granularity needed for the return value in bytes. 221 * return: The offset from find start where the last difference was 222 * encountered in bytes, or a negative value if no difference was found. 223 */ 224 static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size, 225 size_t granularity) 226 { 227 dst += size; 228 src += size; 229 230 #ifdef CONFIG_64BIT 231 VMW_TRY_FIND_LAST_DIFF(u64); 232 #endif 233 VMW_TRY_FIND_LAST_DIFF(u32); 234 VMW_TRY_FIND_LAST_DIFF(u16); 235 236 return round_down(vmw_find_last_diff_u8(dst, src, size) - 1, 237 granularity); 238 } 239 240 241 /** 242 * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a 243 * struct vmw_diff_cpy. 244 * 245 * @diff: The struct vmw_diff_cpy closure argument (unused). 246 * @dest: The copy destination. 247 * @src: The copy source. 248 * @n: Number of bytes to copy. 249 */ 250 void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n) 251 { 252 memcpy(dest, src, n); 253 } 254 255 256 /** 257 * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference 258 * 259 * @diff: The struct vmw_diff_cpy used to track the modified bounding box. 260 * @diff_offs: The offset from @diff->line_offset where the difference was 261 * found. 262 */ 263 static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs) 264 { 265 size_t offs = (diff_offs + diff->line_offset) / diff->cpp; 266 struct drm_rect *rect = &diff->rect; 267 268 rect->x1 = min_t(int, rect->x1, offs); 269 rect->x2 = max_t(int, rect->x2, offs + 1); 270 rect->y1 = min_t(int, rect->y1, diff->line); 271 rect->y2 = max_t(int, rect->y2, diff->line + 1); 272 } 273 274 /** 275 * vmw_diff_memcpy - memcpy that creates a bounding box of modified content. 276 * 277 * @diff: The struct vmw_diff_cpy used to track the modified bounding box. 278 * @dest: The copy destination. 279 * @src: The copy source. 280 * @n: Number of bytes to copy. 281 * 282 * In order to correctly track the modified content, the field @diff->line must 283 * be pre-loaded with the current line number, the field @diff->line_offset must 284 * be pre-loaded with the line offset in bytes where the copy starts, and 285 * finally the field @diff->cpp need to be preloaded with the number of bytes 286 * per unit in the horizontal direction of the area we're examining. 287 * Typically bytes per pixel. 288 * This is needed to know the needed granularity of the difference computing 289 * operations. A higher cpp generally leads to faster execution at the cost of 290 * bounding box width precision. 291 */ 292 void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, 293 size_t n) 294 { 295 ssize_t csize, byte_len; 296 297 if (WARN_ON_ONCE(round_down(n, diff->cpp) != n)) 298 return; 299 300 /* TODO: Possibly use a single vmw_find_first_diff per line? */ 301 csize = vmw_find_first_diff(dest, src, n, diff->cpp); 302 if (csize < n) { 303 vmw_adjust_rect(diff, csize); 304 byte_len = diff->cpp; 305 306 /* 307 * Starting from where first difference was found, find 308 * location of last difference, and then copy. 309 */ 310 diff->line_offset += csize; 311 dest += csize; 312 src += csize; 313 n -= csize; 314 csize = vmw_find_last_diff(dest, src, n, diff->cpp); 315 if (csize >= 0) { 316 byte_len += csize; 317 vmw_adjust_rect(diff, csize); 318 } 319 memcpy(dest, src, byte_len); 320 } 321 diff->line_offset += n; 322 } 323 324 /** 325 * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line 326 * 327 * @mapped_dst: Already mapped destination page index in @dst_pages. 328 * @dst_addr: Kernel virtual address of mapped destination page. 329 * @dst_pages: Array of destination bo pages. 330 * @dst_num_pages: Number of destination bo pages. 331 * @dst_prot: Destination bo page protection. 332 * @mapped_src: Already mapped source page index in @dst_pages. 333 * @src_addr: Kernel virtual address of mapped source page. 334 * @src_pages: Array of source bo pages. 335 * @src_num_pages: Number of source bo pages. 336 * @src_prot: Source bo page protection. 337 * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine. 338 */ 339 struct vmw_bo_blit_line_data { 340 u32 mapped_dst; 341 u8 *dst_addr; 342 struct page **dst_pages; 343 u32 dst_num_pages; 344 pgprot_t dst_prot; 345 u32 mapped_src; 346 u8 *src_addr; 347 struct page **src_pages; 348 u32 src_num_pages; 349 pgprot_t src_prot; 350 struct vmw_diff_cpy *diff; 351 }; 352 353 /** 354 * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another. 355 * 356 * @d: Blit data as described above. 357 * @dst_offset: Destination copy start offset from start of bo. 358 * @src_offset: Source copy start offset from start of bo. 359 * @bytes_to_copy: Number of bytes to copy in this line. 360 */ 361 static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d, 362 u32 dst_offset, 363 u32 src_offset, 364 u32 bytes_to_copy) 365 { 366 struct vmw_diff_cpy *diff = d->diff; 367 368 while (bytes_to_copy) { 369 u32 copy_size = bytes_to_copy; 370 u32 dst_page = dst_offset >> PAGE_SHIFT; 371 u32 src_page = src_offset >> PAGE_SHIFT; 372 u32 dst_page_offset = dst_offset & ~PAGE_MASK; 373 u32 src_page_offset = src_offset & ~PAGE_MASK; 374 bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst; 375 bool unmap_src = d->src_addr && (src_page != d->mapped_src || 376 unmap_dst); 377 378 copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset); 379 copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset); 380 381 if (unmap_src) { 382 ttm_kunmap_atomic_prot(d->src_addr, d->src_prot); 383 d->src_addr = NULL; 384 } 385 386 if (unmap_dst) { 387 ttm_kunmap_atomic_prot(d->dst_addr, d->dst_prot); 388 d->dst_addr = NULL; 389 } 390 391 if (!d->dst_addr) { 392 if (WARN_ON_ONCE(dst_page >= d->dst_num_pages)) 393 return -EINVAL; 394 395 d->dst_addr = 396 ttm_kmap_atomic_prot(d->dst_pages[dst_page], 397 d->dst_prot); 398 if (!d->dst_addr) 399 return -ENOMEM; 400 401 d->mapped_dst = dst_page; 402 } 403 404 if (!d->src_addr) { 405 if (WARN_ON_ONCE(src_page >= d->src_num_pages)) 406 return -EINVAL; 407 408 d->src_addr = 409 ttm_kmap_atomic_prot(d->src_pages[src_page], 410 d->src_prot); 411 if (!d->src_addr) 412 return -ENOMEM; 413 414 d->mapped_src = src_page; 415 } 416 diff->do_cpy(diff, d->dst_addr + dst_page_offset, 417 d->src_addr + src_page_offset, copy_size); 418 419 bytes_to_copy -= copy_size; 420 dst_offset += copy_size; 421 src_offset += copy_size; 422 } 423 424 return 0; 425 } 426 427 /** 428 * ttm_bo_cpu_blit - in-kernel cpu blit. 429 * 430 * @dst: Destination buffer object. 431 * @dst_offset: Destination offset of blit start in bytes. 432 * @dst_stride: Destination stride in bytes. 433 * @src: Source buffer object. 434 * @src_offset: Source offset of blit start in bytes. 435 * @src_stride: Source stride in bytes. 436 * @w: Width of blit. 437 * @h: Height of blit. 438 * return: Zero on success. Negative error value on failure. Will print out 439 * kernel warnings on caller bugs. 440 * 441 * Performs a CPU blit from one buffer object to another avoiding a full 442 * bo vmap which may exhaust- or fragment vmalloc space. 443 * On supported architectures (x86), we're using kmap_atomic which avoids 444 * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems 445 * reference already set-up mappings. 446 * 447 * Neither of the buffer objects may be placed in PCI memory 448 * (Fixed memory in TTM terminology) when using this function. 449 */ 450 int vmw_bo_cpu_blit(struct ttm_buffer_object *dst, 451 u32 dst_offset, u32 dst_stride, 452 struct ttm_buffer_object *src, 453 u32 src_offset, u32 src_stride, 454 u32 w, u32 h, 455 struct vmw_diff_cpy *diff) 456 { 457 struct ttm_operation_ctx ctx = { 458 .interruptible = false, 459 .no_wait_gpu = false 460 }; 461 u32 j, initial_line = dst_offset / dst_stride; 462 struct vmw_bo_blit_line_data d; 463 int ret = 0; 464 465 /* Buffer objects need to be either pinned or reserved: */ 466 if (!(dst->mem.placement & TTM_PL_FLAG_NO_EVICT)) 467 dma_resv_assert_held(dst->base.resv); 468 if (!(src->mem.placement & TTM_PL_FLAG_NO_EVICT)) 469 dma_resv_assert_held(src->base.resv); 470 471 if (dst->ttm->state == tt_unpopulated) { 472 ret = dst->ttm->bdev->driver->ttm_tt_populate(dst->ttm, &ctx); 473 if (ret) 474 return ret; 475 } 476 477 if (src->ttm->state == tt_unpopulated) { 478 ret = src->ttm->bdev->driver->ttm_tt_populate(src->ttm, &ctx); 479 if (ret) 480 return ret; 481 } 482 483 d.mapped_dst = 0; 484 d.mapped_src = 0; 485 d.dst_addr = NULL; 486 d.src_addr = NULL; 487 d.dst_pages = dst->ttm->pages; 488 d.src_pages = src->ttm->pages; 489 d.dst_num_pages = dst->num_pages; 490 d.src_num_pages = src->num_pages; 491 d.dst_prot = ttm_io_prot(dst->mem.placement, PAGE_KERNEL); 492 d.src_prot = ttm_io_prot(src->mem.placement, PAGE_KERNEL); 493 d.diff = diff; 494 495 for (j = 0; j < h; ++j) { 496 diff->line = j + initial_line; 497 diff->line_offset = dst_offset % dst_stride; 498 ret = vmw_bo_cpu_blit_line(&d, dst_offset, src_offset, w); 499 if (ret) 500 goto out; 501 502 dst_offset += dst_stride; 503 src_offset += src_stride; 504 } 505 out: 506 if (d.src_addr) 507 ttm_kunmap_atomic_prot(d.src_addr, d.src_prot); 508 if (d.dst_addr) 509 ttm_kunmap_atomic_prot(d.dst_addr, d.dst_prot); 510 511 return ret; 512 } 513
Indexes created Sat Sep 20 22:09:52 GMT 2025