1/* 2 * Copyright © Microsoft Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#include "d3d12_resource.h" 25 26#include "d3d12_blit.h" 27#include "d3d12_context.h" 28#include "d3d12_format.h" 29#include "d3d12_screen.h" 30#include "d3d12_debug.h" 31 32#include "pipebuffer/pb_bufmgr.h" 33#include "util/slab.h" 34#include "util/format/u_format.h" 35#include "util/u_inlines.h" 36#include "util/u_memory.h" 37#include "util/format/u_format_zs.h" 38 39#include "frontend/sw_winsys.h" 40 41#include <directx/d3d12.h> 42#include <dxguids/dxguids.h> 43#include <memory> 44 45static bool 46can_map_directly(struct pipe_resource *pres) 47{ 48 return pres->target == PIPE_BUFFER && 49 pres->usage != PIPE_USAGE_DEFAULT && 50 pres->usage != PIPE_USAGE_IMMUTABLE; 51} 52 53static void 54init_valid_range(struct d3d12_resource *res) 55{ 56 if (can_map_directly(&res->base)) 57 util_range_init(&res->valid_buffer_range); 58} 59 60static void 61d3d12_resource_destroy(struct pipe_screen *pscreen, 62 struct pipe_resource *presource) 63{ 64 struct d3d12_resource *resource = d3d12_resource(presource); 65 if (can_map_directly(presource)) 66 util_range_destroy(&resource->valid_buffer_range); 67 if (resource->bo) 68 d3d12_bo_unreference(resource->bo); 69 FREE(resource); 70} 71 72static bool 73resource_is_busy(struct d3d12_context *ctx, 74 struct d3d12_resource *res) 75{ 76 bool busy = false; 77 78 for (unsigned i = 0; i < ARRAY_SIZE(ctx->batches); i++) 79 busy |= d3d12_batch_has_references(&ctx->batches[i], res->bo); 80 81 return busy; 82} 83 84void 85d3d12_resource_wait_idle(struct d3d12_context *ctx, 86 struct d3d12_resource *res) 87{ 88 if (d3d12_batch_has_references(d3d12_current_batch(ctx), res->bo)) { 89 d3d12_flush_cmdlist_and_wait(ctx); 90 } else { 91 d3d12_foreach_submitted_batch(ctx, batch) { 92 d3d12_reset_batch(ctx, batch, PIPE_TIMEOUT_INFINITE); 93 if (!resource_is_busy(ctx, res)) 94 break; 95 } 96 } 97} 98 99void 100d3d12_resource_release(struct d3d12_resource *resource) 101{ 102 if (!resource->bo) 103 return; 104 d3d12_bo_unreference(resource->bo); 105 resource->bo = NULL; 106} 107 108static bool 109init_buffer(struct d3d12_screen *screen, 110 struct d3d12_resource *res, 111 const struct pipe_resource *templ) 112{ 113 struct pb_desc buf_desc; 114 struct pb_manager *bufmgr; 115 struct pb_buffer *buf; 116 117 /* Assert that we don't want to create a buffer with one of the emulated 118 * formats, these are (currently) only supported when passing the vertex 119 * element state */ 120 assert(templ->format == d3d12_emulated_vtx_format(templ->format)); 121 122 switch (templ->usage) { 123 case PIPE_USAGE_DEFAULT: 124 case PIPE_USAGE_IMMUTABLE: 125 bufmgr = screen->cache_bufmgr; 126 buf_desc.usage = (pb_usage_flags)PB_USAGE_GPU_READ_WRITE; 127 break; 128 case PIPE_USAGE_DYNAMIC: 129 case PIPE_USAGE_STREAM: 130 bufmgr = screen->slab_bufmgr; 131 buf_desc.usage = (pb_usage_flags)(PB_USAGE_CPU_WRITE | PB_USAGE_GPU_READ); 132 break; 133 case PIPE_USAGE_STAGING: 134 bufmgr = screen->readback_slab_bufmgr; 135 buf_desc.usage = (pb_usage_flags)(PB_USAGE_GPU_WRITE | PB_USAGE_CPU_READ_WRITE); 136 break; 137 default: 138 unreachable("Invalid pipe usage"); 139 } 140 buf_desc.alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; 141 res->dxgi_format = DXGI_FORMAT_UNKNOWN; 142 buf = bufmgr->create_buffer(bufmgr, templ->width0, &buf_desc); 143 if (!buf) 144 return false; 145 res->bo = d3d12_bo_wrap_buffer(buf); 146 147 return true; 148} 149 150static bool 151init_texture(struct d3d12_screen *screen, 152 struct d3d12_resource *res, 153 const struct pipe_resource *templ) 154{ 155 ID3D12Resource *d3d12_res; 156 157 res->mip_levels = templ->last_level + 1; 158 res->dxgi_format = d3d12_get_format(templ->format); 159 160 D3D12_RESOURCE_DESC desc; 161 desc.Format = res->dxgi_format; 162 desc.Alignment = D3D12_DEFAULT_RESOURCE_PLACEMENT_ALIGNMENT; 163 desc.Width = templ->width0; 164 desc.Height = templ->height0; 165 desc.DepthOrArraySize = templ->array_size; 166 desc.MipLevels = templ->last_level + 1; 167 168 desc.SampleDesc.Count = MAX2(templ->nr_samples, 1); 169 desc.SampleDesc.Quality = 0; /* TODO: figure this one out */ 170 171 switch (templ->target) { 172 case PIPE_TEXTURE_1D: 173 case PIPE_TEXTURE_1D_ARRAY: 174 desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE1D; 175 break; 176 177 case PIPE_TEXTURE_CUBE: 178 case PIPE_TEXTURE_CUBE_ARRAY: 179 desc.DepthOrArraySize *= 6; 180 FALLTHROUGH; 181 case PIPE_TEXTURE_2D: 182 case PIPE_TEXTURE_2D_ARRAY: 183 case PIPE_TEXTURE_RECT: 184 desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D; 185 break; 186 187 case PIPE_TEXTURE_3D: 188 desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE3D; 189 desc.DepthOrArraySize = templ->depth0; 190 break; 191 192 default: 193 unreachable("Invalid texture type"); 194 } 195 196 desc.Flags = D3D12_RESOURCE_FLAG_NONE; 197 198 if (templ->bind & PIPE_BIND_SHADER_BUFFER) 199 desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; 200 201 if (templ->bind & PIPE_BIND_RENDER_TARGET) 202 desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET; 203 204 if (templ->bind & PIPE_BIND_DEPTH_STENCIL) { 205 desc.Flags |= D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL; 206 207 /* Sadly, we can't set D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE in the 208 * case where PIPE_BIND_SAMPLER_VIEW isn't set, because that would 209 * prevent us from using the resource with u_blitter, which requires 210 * sneaking in sampler-usage throught the back-door. 211 */ 212 } 213 214 desc.Layout = D3D12_TEXTURE_LAYOUT_UNKNOWN; 215 if (templ->bind & (PIPE_BIND_SCANOUT | 216 PIPE_BIND_SHARED | PIPE_BIND_LINEAR)) 217 desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; 218 219 D3D12_HEAP_PROPERTIES heap_pris = screen->dev->GetCustomHeapProperties(0, D3D12_HEAP_TYPE_DEFAULT); 220 221 HRESULT hres = screen->dev->CreateCommittedResource(&heap_pris, 222 D3D12_HEAP_FLAG_NONE, 223 &desc, 224 D3D12_RESOURCE_STATE_COMMON, 225 NULL, 226 IID_PPV_ARGS(&d3d12_res)); 227 if (FAILED(hres)) 228 return false; 229 230 if (screen->winsys && (templ->bind & PIPE_BIND_DISPLAY_TARGET)) { 231 struct sw_winsys *winsys = screen->winsys; 232 res->dt = winsys->displaytarget_create(screen->winsys, 233 res->base.bind, 234 res->base.format, 235 templ->width0, 236 templ->height0, 237 64, NULL, 238 &res->dt_stride); 239 } 240 241 res->bo = d3d12_bo_wrap_res(d3d12_res, templ->format); 242 243 return true; 244} 245 246static struct pipe_resource * 247d3d12_resource_create(struct pipe_screen *pscreen, 248 const struct pipe_resource *templ) 249{ 250 struct d3d12_screen *screen = d3d12_screen(pscreen); 251 struct d3d12_resource *res = CALLOC_STRUCT(d3d12_resource); 252 bool ret; 253 254 res->base = *templ; 255 256 if (D3D12_DEBUG_RESOURCE & d3d12_debug) { 257 debug_printf("D3D12: Create %sresource %s@%d %dx%dx%d as:%d mip:%d\n", 258 templ->usage == PIPE_USAGE_STAGING ? "STAGING " :"", 259 util_format_name(templ->format), templ->nr_samples, 260 templ->width0, templ->height0, templ->depth0, 261 templ->array_size, templ->last_level); 262 } 263 264 pipe_reference_init(&res->base.reference, 1); 265 res->base.screen = pscreen; 266 267 if (templ->target == PIPE_BUFFER) { 268 ret = init_buffer(screen, res, templ); 269 } else { 270 ret = init_texture(screen, res, templ); 271 } 272 273 if (!ret) { 274 FREE(res); 275 return NULL; 276 } 277 278 init_valid_range(res); 279 280 memset(&res->bind_counts, 0, sizeof(d3d12_resource::bind_counts)); 281 282 return &res->base; 283} 284 285static struct pipe_resource * 286d3d12_resource_from_handle(struct pipe_screen *pscreen, 287 const struct pipe_resource *templ, 288 struct winsys_handle *handle, unsigned usage) 289{ 290 if (handle->type != WINSYS_HANDLE_TYPE_D3D12_RES) 291 return NULL; 292 293 struct d3d12_resource *res = CALLOC_STRUCT(d3d12_resource); 294 if (!res) 295 return NULL; 296 297 res->base = *templ; 298 pipe_reference_init(&res->base.reference, 1); 299 res->base.screen = pscreen; 300 res->dxgi_format = templ->target == PIPE_BUFFER ? DXGI_FORMAT_UNKNOWN : 301 d3d12_get_format(templ->format); 302 res->bo = d3d12_bo_wrap_res((ID3D12Resource *)handle->com_obj, templ->format); 303 init_valid_range(res); 304 return &res->base; 305} 306 307static bool 308d3d12_resource_get_handle(struct pipe_screen *pscreen, 309 struct pipe_context *pcontext, 310 struct pipe_resource *pres, 311 struct winsys_handle *handle, 312 unsigned usage) 313{ 314 struct d3d12_resource *res = d3d12_resource(pres); 315 316 if (handle->type != WINSYS_HANDLE_TYPE_D3D12_RES) 317 return false; 318 319 handle->com_obj = d3d12_resource_resource(res); 320 return true; 321} 322 323void 324d3d12_screen_resource_init(struct pipe_screen *pscreen) 325{ 326 pscreen->resource_create = d3d12_resource_create; 327 pscreen->resource_from_handle = d3d12_resource_from_handle; 328 pscreen->resource_get_handle = d3d12_resource_get_handle; 329 pscreen->resource_destroy = d3d12_resource_destroy; 330} 331 332unsigned int 333get_subresource_id(struct d3d12_resource *res, unsigned resid, 334 unsigned z, unsigned base_level) 335{ 336 unsigned resource_stride = res->base.last_level + 1; 337 if (res->base.target == PIPE_TEXTURE_1D_ARRAY || 338 res->base.target == PIPE_TEXTURE_2D_ARRAY) 339 resource_stride *= res->base.array_size; 340 341 if (res->base.target == PIPE_TEXTURE_CUBE) 342 resource_stride *= 6; 343 344 if (res->base.target == PIPE_TEXTURE_CUBE_ARRAY) 345 resource_stride *= 6 * res->base.array_size; 346 347 unsigned layer_stride = res->base.last_level + 1; 348 349 return resid * resource_stride + z * layer_stride + 350 base_level; 351} 352 353static D3D12_TEXTURE_COPY_LOCATION 354fill_texture_location(struct d3d12_resource *res, 355 struct d3d12_transfer *trans, unsigned resid, unsigned z) 356{ 357 D3D12_TEXTURE_COPY_LOCATION tex_loc = {0}; 358 int subres = get_subresource_id(res, resid, z, trans->base.level); 359 360 tex_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; 361 tex_loc.SubresourceIndex = subres; 362 tex_loc.pResource = d3d12_resource_resource(res); 363 return tex_loc; 364} 365 366static D3D12_TEXTURE_COPY_LOCATION 367fill_buffer_location(struct d3d12_context *ctx, 368 struct d3d12_resource *res, 369 struct d3d12_resource *staging_res, 370 struct d3d12_transfer *trans, 371 unsigned depth, 372 unsigned resid, unsigned z) 373{ 374 D3D12_TEXTURE_COPY_LOCATION buf_loc = {0}; 375 D3D12_PLACED_SUBRESOURCE_FOOTPRINT footprint; 376 uint64_t offset = 0; 377 auto descr = d3d12_resource_underlying(res, &offset)->GetDesc(); 378 ID3D12Device* dev = d3d12_screen(ctx->base.screen)->dev; 379 380 unsigned sub_resid = get_subresource_id(res, resid, z, trans->base.level); 381 dev->GetCopyableFootprints(&descr, sub_resid, 1, 0, &footprint, nullptr, nullptr, nullptr); 382 383 buf_loc.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; 384 buf_loc.pResource = d3d12_resource_underlying(staging_res, &offset); 385 buf_loc.PlacedFootprint = footprint; 386 buf_loc.PlacedFootprint.Offset += offset; 387 388 buf_loc.PlacedFootprint.Footprint.Width = ALIGN(trans->base.box.width, 389 util_format_get_blockwidth(res->base.format)); 390 buf_loc.PlacedFootprint.Footprint.Height = ALIGN(trans->base.box.height, 391 util_format_get_blockheight(res->base.format)); 392 buf_loc.PlacedFootprint.Footprint.Depth = ALIGN(depth, 393 util_format_get_blockdepth(res->base.format)); 394 395 buf_loc.PlacedFootprint.Footprint.RowPitch = trans->base.stride; 396 397 return buf_loc; 398} 399 400struct copy_info { 401 struct d3d12_resource *dst; 402 D3D12_TEXTURE_COPY_LOCATION dst_loc; 403 UINT dst_x, dst_y, dst_z; 404 struct d3d12_resource *src; 405 D3D12_TEXTURE_COPY_LOCATION src_loc; 406 D3D12_BOX *src_box; 407}; 408 409 410static void 411copy_texture_region(struct d3d12_context *ctx, 412 struct copy_info& info) 413{ 414 auto batch = d3d12_current_batch(ctx); 415 416 d3d12_batch_reference_resource(batch, info.src); 417 d3d12_batch_reference_resource(batch, info.dst); 418 d3d12_transition_resource_state(ctx, info.src, D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_BIND_INVALIDATE_FULL); 419 d3d12_transition_resource_state(ctx, info.dst, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_BIND_INVALIDATE_FULL); 420 d3d12_apply_resource_states(ctx); 421 ctx->cmdlist->CopyTextureRegion(&info.dst_loc, info.dst_x, info.dst_y, info.dst_z, 422 &info.src_loc, info.src_box); 423} 424 425static void 426transfer_buf_to_image_part(struct d3d12_context *ctx, 427 struct d3d12_resource *res, 428 struct d3d12_resource *staging_res, 429 struct d3d12_transfer *trans, 430 int z, int depth, int start_z, int dest_z, 431 int resid) 432{ 433 if (D3D12_DEBUG_RESOURCE & d3d12_debug) { 434 debug_printf("D3D12: Copy %dx%dx%d + %dx%dx%d from buffer %s to image %s\n", 435 trans->base.box.x, trans->base.box.y, trans->base.box.z, 436 trans->base.box.width, trans->base.box.height, trans->base.box.depth, 437 util_format_name(staging_res->base.format), 438 util_format_name(res->base.format)); 439 } 440 441 struct copy_info copy_info; 442 copy_info.src = staging_res; 443 copy_info.src_loc = fill_buffer_location(ctx, res, staging_res, trans, depth, resid, z); 444 copy_info.src_loc.PlacedFootprint.Offset = (z - start_z) * trans->base.layer_stride; 445 copy_info.src_box = nullptr; 446 copy_info.dst = res; 447 copy_info.dst_loc = fill_texture_location(res, trans, resid, z); 448 copy_info.dst_x = trans->base.box.x; 449 copy_info.dst_y = trans->base.box.y; 450 copy_info.dst_z = res->base.target == PIPE_TEXTURE_CUBE ? 0 : dest_z; 451 copy_info.src_box = nullptr; 452 453 copy_texture_region(ctx, copy_info); 454} 455 456static bool 457transfer_buf_to_image(struct d3d12_context *ctx, 458 struct d3d12_resource *res, 459 struct d3d12_resource *staging_res, 460 struct d3d12_transfer *trans, int resid) 461{ 462 if (res->base.target == PIPE_TEXTURE_3D) { 463 assert(resid == 0); 464 transfer_buf_to_image_part(ctx, res, staging_res, trans, 465 0, trans->base.box.depth, 0, 466 trans->base.box.z, 0); 467 } else { 468 int num_layers = trans->base.box.depth; 469 int start_z = trans->base.box.z; 470 471 for (int z = start_z; z < start_z + num_layers; ++z) { 472 transfer_buf_to_image_part(ctx, res, staging_res, trans, 473 z, 1, start_z, 0, resid); 474 } 475 } 476 return true; 477} 478 479static void 480transfer_image_part_to_buf(struct d3d12_context *ctx, 481 struct d3d12_resource *res, 482 struct d3d12_resource *staging_res, 483 struct d3d12_transfer *trans, 484 unsigned resid, int z, int start_layer, 485 int start_box_z, int depth) 486{ 487 struct pipe_box *box = &trans->base.box; 488 D3D12_BOX src_box = {}; 489 490 struct copy_info copy_info; 491 copy_info.src_box = nullptr; 492 copy_info.src = res; 493 copy_info.src_loc = fill_texture_location(res, trans, resid, z); 494 copy_info.dst = staging_res; 495 copy_info.dst_loc = fill_buffer_location(ctx, res, staging_res, trans, 496 depth, resid, z); 497 copy_info.dst_loc.PlacedFootprint.Offset = (z - start_layer) * trans->base.layer_stride; 498 copy_info.dst_x = copy_info.dst_y = copy_info.dst_z = 0; 499 500 if (!util_texrange_covers_whole_level(&res->base, trans->base.level, 501 box->x, box->y, start_box_z, 502 box->width, box->height, depth)) { 503 src_box.left = box->x; 504 src_box.right = box->x + box->width; 505 src_box.top = box->y; 506 src_box.bottom = box->y + box->height; 507 src_box.front = start_box_z; 508 src_box.back = start_box_z + depth; 509 copy_info.src_box = &src_box; 510 } 511 512 copy_texture_region(ctx, copy_info); 513} 514 515static bool 516transfer_image_to_buf(struct d3d12_context *ctx, 517 struct d3d12_resource *res, 518 struct d3d12_resource *staging_res, 519 struct d3d12_transfer *trans, 520 unsigned resid) 521{ 522 523 /* We only suppport loading from either an texture array 524 * or a ZS texture, so either resid is zero, or num_layers == 1) 525 */ 526 assert(resid == 0 || trans->base.box.depth == 1); 527 528 if (D3D12_DEBUG_RESOURCE & d3d12_debug) { 529 debug_printf("D3D12: Copy %dx%dx%d + %dx%dx%d from %s@%d to %s\n", 530 trans->base.box.x, trans->base.box.y, trans->base.box.z, 531 trans->base.box.width, trans->base.box.height, trans->base.box.depth, 532 util_format_name(res->base.format), resid, 533 util_format_name(staging_res->base.format)); 534 } 535 536 struct pipe_resource *resolved_resource = nullptr; 537 if (res->base.nr_samples > 1) { 538 struct pipe_resource tmpl = res->base; 539 tmpl.nr_samples = 0; 540 resolved_resource = d3d12_resource_create(ctx->base.screen, &tmpl); 541 struct pipe_blit_info resolve_info = {}; 542 struct pipe_box box = {0,0,0, (int)res->base.width0, (int16_t)res->base.height0, (int16_t)res->base.depth0}; 543 resolve_info.dst.resource = resolved_resource; 544 resolve_info.dst.box = box; 545 resolve_info.dst.format = res->base.format; 546 resolve_info.src.resource = &res->base; 547 resolve_info.src.box = box; 548 resolve_info.src.format = res->base.format; 549 resolve_info.filter = PIPE_TEX_FILTER_NEAREST; 550 resolve_info.mask = util_format_get_mask(tmpl.format); 551 552 553 554 d3d12_blit(&ctx->base, &resolve_info); 555 res = (struct d3d12_resource *)resolved_resource; 556 } 557 558 559 if (res->base.target == PIPE_TEXTURE_3D) { 560 transfer_image_part_to_buf(ctx, res, staging_res, trans, resid, 561 0, 0, trans->base.box.z, trans->base.box.depth); 562 } else { 563 int start_layer = trans->base.box.z; 564 for (int z = start_layer; z < start_layer + trans->base.box.depth; ++z) { 565 transfer_image_part_to_buf(ctx, res, staging_res, trans, resid, 566 z, start_layer, 0, 1); 567 } 568 } 569 570 pipe_resource_reference(&resolved_resource, NULL); 571 572 return true; 573} 574 575static void 576transfer_buf_to_buf(struct d3d12_context *ctx, 577 struct d3d12_resource *src, 578 struct d3d12_resource *dst, 579 uint64_t src_offset, 580 uint64_t dst_offset, 581 uint64_t width) 582{ 583 auto batch = d3d12_current_batch(ctx); 584 585 d3d12_batch_reference_resource(batch, src); 586 d3d12_batch_reference_resource(batch, dst); 587 588 uint64_t src_offset_suballoc = 0; 589 uint64_t dst_offset_suballoc = 0; 590 auto src_d3d12 = d3d12_resource_underlying(src, &src_offset_suballoc); 591 auto dst_d3d12 = d3d12_resource_underlying(dst, &dst_offset_suballoc); 592 src_offset += src_offset_suballoc; 593 dst_offset += dst_offset_suballoc; 594 595 // Same-resource copies not supported, since the resource would need to be in both states 596 assert(src_d3d12 != dst_d3d12); 597 d3d12_transition_resource_state(ctx, src, D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_BIND_INVALIDATE_FULL); 598 d3d12_transition_resource_state(ctx, dst, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_BIND_INVALIDATE_FULL); 599 d3d12_apply_resource_states(ctx); 600 ctx->cmdlist->CopyBufferRegion(dst_d3d12, dst_offset, 601 src_d3d12, src_offset, 602 width); 603} 604 605static unsigned 606linear_offset(int x, int y, int z, unsigned stride, unsigned layer_stride) 607{ 608 return x + 609 y * stride + 610 z * layer_stride; 611} 612 613static D3D12_RANGE 614linear_range(const struct pipe_box *box, unsigned stride, unsigned layer_stride) 615{ 616 D3D12_RANGE range; 617 618 range.Begin = linear_offset(box->x, box->y, box->z, 619 stride, layer_stride); 620 range.End = linear_offset(box->x + box->width, 621 box->y + box->height - 1, 622 box->z + box->depth - 1, 623 stride, layer_stride); 624 625 return range; 626} 627 628static bool 629synchronize(struct d3d12_context *ctx, 630 struct d3d12_resource *res, 631 unsigned usage, 632 D3D12_RANGE *range) 633{ 634 assert(can_map_directly(&res->base)); 635 636 /* Check whether that range contains valid data; if not, we might not need to sync */ 637 if (!(usage & PIPE_MAP_UNSYNCHRONIZED) && 638 usage & PIPE_MAP_WRITE && 639 !util_ranges_intersect(&res->valid_buffer_range, range->Begin, range->End)) { 640 usage |= PIPE_MAP_UNSYNCHRONIZED; 641 } 642 643 if (!(usage & PIPE_MAP_UNSYNCHRONIZED) && resource_is_busy(ctx, res)) { 644 if (usage & PIPE_MAP_DONTBLOCK) 645 return false; 646 647 d3d12_resource_wait_idle(ctx, res); 648 } 649 650 if (usage & PIPE_MAP_WRITE) 651 util_range_add(&res->base, &res->valid_buffer_range, 652 range->Begin, range->End); 653 654 return true; 655} 656 657/* A wrapper to make sure local resources are freed and unmapped with 658 * any exit path */ 659struct local_resource { 660 local_resource(pipe_screen *s, struct pipe_resource *tmpl) : 661 mapped(false) 662 { 663 res = d3d12_resource(d3d12_resource_create(s, tmpl)); 664 } 665 666 ~local_resource() { 667 if (res) { 668 if (mapped) 669 d3d12_bo_unmap(res->bo, nullptr); 670 pipe_resource_reference((struct pipe_resource **)&res, NULL); 671 } 672 } 673 674 void * 675 map() { 676 void *ptr; 677 ptr = d3d12_bo_map(res->bo, nullptr); 678 if (ptr) 679 mapped = true; 680 return ptr; 681 } 682 683 void unmap() 684 { 685 if (mapped) 686 d3d12_bo_unmap(res->bo, nullptr); 687 mapped = false; 688 } 689 690 operator struct d3d12_resource *() { 691 return res; 692 } 693 694 bool operator !() { 695 return !res; 696 } 697private: 698 struct d3d12_resource *res; 699 bool mapped; 700}; 701 702/* Combined depth-stencil needs a special handling for reading back: DX handled 703 * depth and stencil parts as separate resources and handles copying them only 704 * by using seperate texture copy calls with different formats. So create two 705 * buffers, read back both resources and interleave the data. 706 */ 707static void 708prepare_zs_layer_strides(struct d3d12_resource *res, 709 const struct pipe_box *box, 710 struct d3d12_transfer *trans) 711{ 712 trans->base.stride = align(util_format_get_stride(res->base.format, box->width), 713 D3D12_TEXTURE_DATA_PITCH_ALIGNMENT); 714 trans->base.layer_stride = util_format_get_2d_size(res->base.format, 715 trans->base.stride, 716 box->height); 717} 718 719static void * 720read_zs_surface(struct d3d12_context *ctx, struct d3d12_resource *res, 721 const struct pipe_box *box, 722 struct d3d12_transfer *trans) 723{ 724 pipe_screen *pscreen = ctx->base.screen; 725 726 prepare_zs_layer_strides(res, box, trans); 727 728 struct pipe_resource tmpl; 729 memset(&tmpl, 0, sizeof tmpl); 730 tmpl.target = PIPE_BUFFER; 731 tmpl.format = PIPE_FORMAT_R32_UNORM; 732 tmpl.bind = 0; 733 tmpl.usage = PIPE_USAGE_STAGING; 734 tmpl.flags = 0; 735 tmpl.width0 = trans->base.layer_stride; 736 tmpl.height0 = 1; 737 tmpl.depth0 = 1; 738 tmpl.array_size = 1; 739 740 local_resource depth_buffer(pscreen, &tmpl); 741 if (!depth_buffer) { 742 debug_printf("Allocating staging buffer for depth failed\n"); 743 return NULL; 744 } 745 746 if (!transfer_image_to_buf(ctx, res, depth_buffer, trans, 0)) 747 return NULL; 748 749 tmpl.format = PIPE_FORMAT_R8_UINT; 750 751 local_resource stencil_buffer(pscreen, &tmpl); 752 if (!stencil_buffer) { 753 debug_printf("Allocating staging buffer for stencilfailed\n"); 754 return NULL; 755 } 756 757 if (!transfer_image_to_buf(ctx, res, stencil_buffer, trans, 1)) 758 return NULL; 759 760 d3d12_flush_cmdlist_and_wait(ctx); 761 762 void *depth_ptr = depth_buffer.map(); 763 if (!depth_ptr) { 764 debug_printf("Mapping staging depth buffer failed\n"); 765 return NULL; 766 } 767 768 uint8_t *stencil_ptr = (uint8_t *)stencil_buffer.map(); 769 if (!stencil_ptr) { 770 debug_printf("Mapping staging stencil buffer failed\n"); 771 return NULL; 772 } 773 774 uint8_t *buf = (uint8_t *)malloc(trans->base.layer_stride); 775 if (!buf) 776 return NULL; 777 778 trans->data = buf; 779 780 switch (res->base.format) { 781 case PIPE_FORMAT_Z24_UNORM_S8_UINT: 782 util_format_z24_unorm_s8_uint_pack_separate(buf, trans->base.stride, 783 (uint32_t *)depth_ptr, trans->base.stride, 784 stencil_ptr, trans->base.stride, 785 trans->base.box.width, trans->base.box.height); 786 break; 787 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 788 util_format_z32_float_s8x24_uint_pack_z_float(buf, trans->base.stride, 789 (float *)depth_ptr, trans->base.stride, 790 trans->base.box.width, trans->base.box.height); 791 util_format_z32_float_s8x24_uint_pack_s_8uint(buf, trans->base.stride, 792 stencil_ptr, trans->base.stride, 793 trans->base.box.width, trans->base.box.height); 794 break; 795 default: 796 unreachable("Unsupported depth steancil format"); 797 }; 798 799 return trans->data; 800} 801 802static void * 803prepare_write_zs_surface(struct d3d12_resource *res, 804 const struct pipe_box *box, 805 struct d3d12_transfer *trans) 806{ 807 prepare_zs_layer_strides(res, box, trans); 808 uint32_t *buf = (uint32_t *)malloc(trans->base.layer_stride); 809 if (!buf) 810 return NULL; 811 812 trans->data = buf; 813 return trans->data; 814} 815 816static void 817write_zs_surface(struct pipe_context *pctx, struct d3d12_resource *res, 818 struct d3d12_transfer *trans) 819{ 820 struct pipe_resource tmpl; 821 memset(&tmpl, 0, sizeof tmpl); 822 tmpl.target = PIPE_BUFFER; 823 tmpl.format = PIPE_FORMAT_R32_UNORM; 824 tmpl.bind = 0; 825 tmpl.usage = PIPE_USAGE_STAGING; 826 tmpl.flags = 0; 827 tmpl.width0 = trans->base.layer_stride; 828 tmpl.height0 = 1; 829 tmpl.depth0 = 1; 830 tmpl.array_size = 1; 831 832 local_resource depth_buffer(pctx->screen, &tmpl); 833 if (!depth_buffer) { 834 debug_printf("Allocating staging buffer for depth failed\n"); 835 return; 836 } 837 838 local_resource stencil_buffer(pctx->screen, &tmpl); 839 if (!stencil_buffer) { 840 debug_printf("Allocating staging buffer for depth failed\n"); 841 return; 842 } 843 844 void *depth_ptr = depth_buffer.map(); 845 if (!depth_ptr) { 846 debug_printf("Mapping staging depth buffer failed\n"); 847 return; 848 } 849 850 uint8_t *stencil_ptr = (uint8_t *)stencil_buffer.map(); 851 if (!stencil_ptr) { 852 debug_printf("Mapping staging stencil buffer failed\n"); 853 return; 854 } 855 856 switch (res->base.format) { 857 case PIPE_FORMAT_Z24_UNORM_S8_UINT: 858 util_format_z32_unorm_unpack_z_32unorm((uint32_t *)depth_ptr, trans->base.stride, (uint8_t*)trans->data, 859 trans->base.stride, trans->base.box.width, 860 trans->base.box.height); 861 util_format_z24_unorm_s8_uint_unpack_s_8uint(stencil_ptr, trans->base.stride, (uint8_t*)trans->data, 862 trans->base.stride, trans->base.box.width, 863 trans->base.box.height); 864 break; 865 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 866 util_format_z32_float_s8x24_uint_unpack_z_float((float *)depth_ptr, trans->base.stride, (uint8_t*)trans->data, 867 trans->base.stride, trans->base.box.width, 868 trans->base.box.height); 869 util_format_z32_float_s8x24_uint_unpack_s_8uint(stencil_ptr, trans->base.stride, (uint8_t*)trans->data, 870 trans->base.stride, trans->base.box.width, 871 trans->base.box.height); 872 break; 873 default: 874 unreachable("Unsupported depth steancil format"); 875 }; 876 877 stencil_buffer.unmap(); 878 depth_buffer.unmap(); 879 880 transfer_buf_to_image(d3d12_context(pctx), res, depth_buffer, trans, 0); 881 transfer_buf_to_image(d3d12_context(pctx), res, stencil_buffer, trans, 1); 882} 883 884#define BUFFER_MAP_ALIGNMENT 64 885 886static void * 887d3d12_transfer_map(struct pipe_context *pctx, 888 struct pipe_resource *pres, 889 unsigned level, 890 unsigned usage, 891 const struct pipe_box *box, 892 struct pipe_transfer **transfer) 893{ 894 struct d3d12_context *ctx = d3d12_context(pctx); 895 struct d3d12_resource *res = d3d12_resource(pres); 896 897 if (usage & PIPE_MAP_DIRECTLY || !res->bo) 898 return NULL; 899 900 struct d3d12_transfer *trans = (struct d3d12_transfer *)slab_alloc(&ctx->transfer_pool); 901 struct pipe_transfer *ptrans = &trans->base; 902 if (!trans) 903 return NULL; 904 905 memset(trans, 0, sizeof(*trans)); 906 pipe_resource_reference(&ptrans->resource, pres); 907 908 ptrans->resource = pres; 909 ptrans->level = level; 910 ptrans->usage = (enum pipe_map_flags)usage; 911 ptrans->box = *box; 912 913 D3D12_RANGE range; 914 range.Begin = 0; 915 916 void *ptr; 917 if (can_map_directly(&res->base)) { 918 if (pres->target == PIPE_BUFFER) { 919 ptrans->stride = 0; 920 ptrans->layer_stride = 0; 921 } else { 922 ptrans->stride = util_format_get_stride(pres->format, box->width); 923 ptrans->layer_stride = util_format_get_2d_size(pres->format, 924 ptrans->stride, 925 box->height); 926 } 927 928 range = linear_range(box, ptrans->stride, ptrans->layer_stride); 929 if (!synchronize(ctx, res, usage, &range)) 930 return NULL; 931 ptr = d3d12_bo_map(res->bo, &range); 932 } else if (unlikely(pres->format == PIPE_FORMAT_Z24_UNORM_S8_UINT || 933 pres->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)) { 934 if (usage & PIPE_MAP_READ) { 935 ptr = read_zs_surface(ctx, res, box, trans); 936 } else if (usage & PIPE_MAP_WRITE){ 937 ptr = prepare_write_zs_surface(res, box, trans); 938 } else { 939 ptr = nullptr; 940 } 941 } else { 942 ptrans->stride = align(util_format_get_stride(pres->format, box->width), 943 D3D12_TEXTURE_DATA_PITCH_ALIGNMENT); 944 ptrans->layer_stride = util_format_get_2d_size(pres->format, 945 ptrans->stride, 946 box->height); 947 948 if (res->base.target != PIPE_TEXTURE_3D) 949 ptrans->layer_stride = align(ptrans->layer_stride, 950 D3D12_TEXTURE_DATA_PLACEMENT_ALIGNMENT); 951 952 unsigned staging_res_size = ptrans->layer_stride * box->depth; 953 if (res->base.target == PIPE_BUFFER) { 954 /* To properly support ARB_map_buffer_alignment, we need to return a pointer 955 * that's appropriately offset from a 64-byte-aligned base address. 956 */ 957 assert(box->x >= 0); 958 unsigned aligned_x = (unsigned)box->x % BUFFER_MAP_ALIGNMENT; 959 staging_res_size = align(box->width + aligned_x, 960 D3D12_TEXTURE_DATA_PITCH_ALIGNMENT); 961 range.Begin = aligned_x; 962 } 963 964 pipe_resource_usage staging_usage = (usage & (PIPE_MAP_READ | PIPE_MAP_READ_WRITE)) ? 965 PIPE_USAGE_STAGING : PIPE_USAGE_STREAM; 966 967 trans->staging_res = pipe_buffer_create(pctx->screen, 0, 968 staging_usage, 969 staging_res_size); 970 if (!trans->staging_res) 971 return NULL; 972 973 struct d3d12_resource *staging_res = d3d12_resource(trans->staging_res); 974 975 if (usage & PIPE_MAP_READ) { 976 bool ret = true; 977 if (pres->target == PIPE_BUFFER) { 978 uint64_t src_offset = box->x; 979 uint64_t dst_offset = src_offset % BUFFER_MAP_ALIGNMENT; 980 transfer_buf_to_buf(ctx, res, staging_res, src_offset, dst_offset, box->width); 981 } else 982 ret = transfer_image_to_buf(ctx, res, staging_res, trans, 0); 983 if (!ret) 984 return NULL; 985 d3d12_flush_cmdlist_and_wait(ctx); 986 } 987 988 range.End = staging_res_size - range.Begin; 989 990 ptr = d3d12_bo_map(staging_res->bo, &range); 991 } 992 993 *transfer = ptrans; 994 return ptr; 995} 996 997static void 998d3d12_transfer_unmap(struct pipe_context *pctx, 999 struct pipe_transfer *ptrans) 1000{ 1001 struct d3d12_resource *res = d3d12_resource(ptrans->resource); 1002 struct d3d12_transfer *trans = (struct d3d12_transfer *)ptrans; 1003 D3D12_RANGE range = { 0, 0 }; 1004 1005 if (trans->data != nullptr) { 1006 if (trans->base.usage & PIPE_MAP_WRITE) 1007 write_zs_surface(pctx, res, trans); 1008 free(trans->data); 1009 } else if (trans->staging_res) { 1010 struct d3d12_resource *staging_res = d3d12_resource(trans->staging_res); 1011 1012 if (trans->base.usage & PIPE_MAP_WRITE) { 1013 assert(ptrans->box.x >= 0); 1014 range.Begin = res->base.target == PIPE_BUFFER ? 1015 (unsigned)ptrans->box.x % BUFFER_MAP_ALIGNMENT : 0; 1016 range.End = staging_res->base.width0 - range.Begin; 1017 } 1018 d3d12_bo_unmap(staging_res->bo, &range); 1019 1020 if (trans->base.usage & PIPE_MAP_WRITE) { 1021 struct d3d12_context *ctx = d3d12_context(pctx); 1022 if (res->base.target == PIPE_BUFFER) { 1023 uint64_t dst_offset = trans->base.box.x; 1024 uint64_t src_offset = dst_offset % BUFFER_MAP_ALIGNMENT; 1025 transfer_buf_to_buf(ctx, staging_res, res, src_offset, dst_offset, ptrans->box.width); 1026 } else 1027 transfer_buf_to_image(ctx, res, staging_res, trans, 0); 1028 } 1029 1030 pipe_resource_reference(&trans->staging_res, NULL); 1031 } else { 1032 if (trans->base.usage & PIPE_MAP_WRITE) { 1033 range.Begin = ptrans->box.x; 1034 range.End = ptrans->box.x + ptrans->box.width; 1035 } 1036 d3d12_bo_unmap(res->bo, &range); 1037 } 1038 1039 pipe_resource_reference(&ptrans->resource, NULL); 1040 slab_free(&d3d12_context(pctx)->transfer_pool, ptrans); 1041} 1042 1043void 1044d3d12_resource_make_writeable(struct pipe_context *pctx, 1045 struct pipe_resource *pres) 1046{ 1047 struct d3d12_context *ctx = d3d12_context(pctx); 1048 struct d3d12_resource *res = d3d12_resource(pres); 1049 struct d3d12_resource *dup_res; 1050 1051 if (!res->bo || !d3d12_bo_is_suballocated(res->bo)) 1052 return; 1053 1054 dup_res = d3d12_resource(pipe_buffer_create(pres->screen, 1055 pres->bind & PIPE_BIND_STREAM_OUTPUT, 1056 (pipe_resource_usage) pres->usage, 1057 pres->width0)); 1058 1059 if (res->valid_buffer_range.end > res->valid_buffer_range.start) { 1060 struct pipe_box box; 1061 1062 box.x = res->valid_buffer_range.start; 1063 box.y = 0; 1064 box.z = 0; 1065 box.width = res->valid_buffer_range.end - res->valid_buffer_range.start; 1066 box.height = 1; 1067 box.depth = 1; 1068 1069 d3d12_direct_copy(ctx, dup_res, 0, &box, res, 0, &box, PIPE_MASK_RGBAZS); 1070 } 1071 1072 /* Move new BO to old resource */ 1073 d3d12_bo_unreference(res->bo); 1074 res->bo = dup_res->bo; 1075 d3d12_bo_reference(res->bo); 1076 1077 d3d12_resource_destroy(dup_res->base.screen, &dup_res->base); 1078} 1079 1080void 1081d3d12_context_resource_init(struct pipe_context *pctx) 1082{ 1083 pctx->buffer_map = d3d12_transfer_map; 1084 pctx->buffer_unmap = d3d12_transfer_unmap; 1085 pctx->texture_map = d3d12_transfer_map; 1086 pctx->texture_unmap = d3d12_transfer_unmap; 1087 1088 pctx->transfer_flush_region = u_default_transfer_flush_region; 1089 pctx->buffer_subdata = u_default_buffer_subdata; 1090 pctx->texture_subdata = u_default_texture_subdata; 1091} 1092