radv_meta_resolve.c revision 01e04c3f 
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include <assert.h>
25#include <stdbool.h>
26
27#include "radv_meta.h"
28#include "radv_private.h"
29#include "vk_format.h"
30#include "nir/nir_builder.h"
31#include "sid.h"
32
33/* emit 0, 0, 0, 1 */
34static nir_shader *
35build_nir_fs(void)
36{
37	const struct glsl_type *vec4 = glsl_vec4_type();
38	nir_builder b;
39	nir_variable *f_color; /* vec4, fragment output color */
40
41	nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
42	b.shader->info.name = ralloc_asprintf(b.shader,
43					       "meta_resolve_fs");
44
45	f_color = nir_variable_create(b.shader, nir_var_shader_out, vec4,
46				      "f_color");
47	f_color->data.location = FRAG_RESULT_DATA0;
48	nir_store_var(&b, f_color, nir_imm_vec4(&b, 0.0, 0.0, 0.0, 1.0), 0xf);
49
50	return b.shader;
51}
52
53static VkResult
54create_pass(struct radv_device *device, VkFormat vk_format, VkRenderPass *pass)
55{
56	VkResult result;
57	VkDevice device_h = radv_device_to_handle(device);
58	const VkAllocationCallbacks *alloc = &device->meta_state.alloc;
59	VkAttachmentDescription attachments[2];
60	int i;
61
62	for (i = 0; i < 2; i++) {
63		attachments[i].format = vk_format;
64		attachments[i].samples = 1;
65		attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
66		attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE;
67	}
68	attachments[0].initialLayout = VK_IMAGE_LAYOUT_GENERAL;
69	attachments[0].finalLayout = VK_IMAGE_LAYOUT_GENERAL;
70	attachments[1].initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
71	attachments[1].finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
72
73	result = radv_CreateRenderPass(device_h,
74				       &(VkRenderPassCreateInfo) {
75					       .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
76						       .attachmentCount = 2,
77						       .pAttachments = attachments,
78						       .subpassCount = 1,
79								.pSubpasses = &(VkSubpassDescription) {
80						       .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
81						       .inputAttachmentCount = 0,
82						       .colorAttachmentCount = 2,
83						       .pColorAttachments = (VkAttachmentReference[]) {
84							       {
85								       .attachment = 0,
86								       .layout = VK_IMAGE_LAYOUT_GENERAL,
87							       },
88							       {
89								       .attachment = 1,
90								       .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
91							       },
92						       },
93						       .pResolveAttachments = NULL,
94						       .pDepthStencilAttachment = &(VkAttachmentReference) {
95							       .attachment = VK_ATTACHMENT_UNUSED,
96						       },
97						       .preserveAttachmentCount = 0,
98						       .pPreserveAttachments = NULL,
99					       },
100								.dependencyCount = 0,
101									 },
102				       alloc,
103				       pass);
104
105	return result;
106}
107
108static VkResult
109create_pipeline(struct radv_device *device,
110		 VkShaderModule vs_module_h,
111		 VkPipeline *pipeline,
112		 VkRenderPass pass)
113{
114	VkResult result;
115	VkDevice device_h = radv_device_to_handle(device);
116
117	struct radv_shader_module fs_module = {
118		.nir = build_nir_fs(),
119	};
120
121	if (!fs_module.nir) {
122		/* XXX: Need more accurate error */
123		result = VK_ERROR_OUT_OF_HOST_MEMORY;
124		goto cleanup;
125	}
126
127	VkPipelineLayoutCreateInfo pl_create_info = {
128		.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
129		.setLayoutCount = 0,
130		.pSetLayouts = NULL,
131		.pushConstantRangeCount = 0,
132		.pPushConstantRanges = NULL,
133	};
134
135	if (!device->meta_state.resolve.p_layout) {
136		result = radv_CreatePipelineLayout(radv_device_to_handle(device),
137						   &pl_create_info,
138						   &device->meta_state.alloc,
139						   &device->meta_state.resolve.p_layout);
140		if (result != VK_SUCCESS)
141			goto cleanup;
142	}
143
144	result = radv_graphics_pipeline_create(device_h,
145					       radv_pipeline_cache_to_handle(&device->meta_state.cache),
146					       &(VkGraphicsPipelineCreateInfo) {
147						       .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
148						       .stageCount = 2,
149						       .pStages = (VkPipelineShaderStageCreateInfo[]) {
150						       {
151							       .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
152							       .stage = VK_SHADER_STAGE_VERTEX_BIT,
153							       .module = vs_module_h,
154							       .pName = "main",
155						       },
156						       {
157							       .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
158							       .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
159							       .module = radv_shader_module_to_handle(&fs_module),
160							       .pName = "main",
161						       },
162					       },
163					       .pVertexInputState = &(VkPipelineVertexInputStateCreateInfo) {
164						       .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
165						       .vertexBindingDescriptionCount = 0,
166						       .vertexAttributeDescriptionCount = 0,
167					       },
168					       .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
169						       .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
170						       .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
171						       .primitiveRestartEnable = false,
172					       },
173					       .pViewportState = &(VkPipelineViewportStateCreateInfo) {
174						       .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
175						       .viewportCount = 1,
176						       .scissorCount = 1,
177					       },
178					       .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
179						       .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
180						       .depthClampEnable = false,
181						       .rasterizerDiscardEnable = false,
182						       .polygonMode = VK_POLYGON_MODE_FILL,
183						       .cullMode = VK_CULL_MODE_NONE,
184						       .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
185					       },
186					       .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
187						       .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
188						       .rasterizationSamples = 1,
189						       .sampleShadingEnable = false,
190						       .pSampleMask = NULL,
191						       .alphaToCoverageEnable = false,
192						       .alphaToOneEnable = false,
193					       },
194					       .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
195						       .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
196						       .logicOpEnable = false,
197						       .attachmentCount = 2,
198						       .pAttachments = (VkPipelineColorBlendAttachmentState []) {
199							       {
200							       .colorWriteMask = VK_COLOR_COMPONENT_R_BIT |
201									       VK_COLOR_COMPONENT_G_BIT |
202									       VK_COLOR_COMPONENT_B_BIT |
203									       VK_COLOR_COMPONENT_A_BIT,
204							       },
205							       {
206							       .colorWriteMask = 0,
207
208							       }
209						       },
210						},
211						.pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
212							.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
213							.dynamicStateCount = 2,
214							.pDynamicStates = (VkDynamicState[]) {
215								VK_DYNAMIC_STATE_VIEWPORT,
216								VK_DYNAMIC_STATE_SCISSOR,
217							},
218						},
219						.layout = device->meta_state.resolve.p_layout,
220						.renderPass = pass,
221																       .subpass = 0,
222																       },
223					       &(struct radv_graphics_pipeline_create_info) {
224						       .use_rectlist = true,
225						       .custom_blend_mode = V_028808_CB_RESOLVE,
226							       },
227					       &device->meta_state.alloc, pipeline);
228	if (result != VK_SUCCESS)
229		goto cleanup;
230
231	goto cleanup;
232
233cleanup:
234	ralloc_free(fs_module.nir);
235	return result;
236}
237
238void
239radv_device_finish_meta_resolve_state(struct radv_device *device)
240{
241	struct radv_meta_state *state = &device->meta_state;
242
243	for (uint32_t j = 0; j < NUM_META_FS_KEYS; j++) {
244		radv_DestroyRenderPass(radv_device_to_handle(device),
245				       state->resolve.pass[j], &state->alloc);
246		radv_DestroyPipeline(radv_device_to_handle(device),
247				     state->resolve.pipeline[j], &state->alloc);
248	}
249	radv_DestroyPipelineLayout(radv_device_to_handle(device),
250				   state->resolve.p_layout, &state->alloc);
251
252}
253
254VkResult
255radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand)
256{
257	if (on_demand)
258		return VK_SUCCESS;
259
260	VkResult res = VK_SUCCESS;
261	struct radv_meta_state *state = &device->meta_state;
262	struct radv_shader_module vs_module = { .nir = radv_meta_build_nir_vs_generate_vertices() };
263	if (!vs_module.nir) {
264		/* XXX: Need more accurate error */
265		res = VK_ERROR_OUT_OF_HOST_MEMORY;
266		goto fail;
267	}
268
269	for (uint32_t i = 0; i < NUM_META_FS_KEYS; ++i) {
270		VkFormat format = radv_fs_key_format_exemplars[i];
271		unsigned fs_key = radv_format_meta_fs_key(format);
272		res = create_pass(device, format, &state->resolve.pass[fs_key]);
273		if (res != VK_SUCCESS)
274			goto fail;
275
276		VkShaderModule vs_module_h = radv_shader_module_to_handle(&vs_module);
277		res = create_pipeline(device, vs_module_h,
278				      &state->resolve.pipeline[fs_key], state->resolve.pass[fs_key]);
279		if (res != VK_SUCCESS)
280			goto fail;
281	}
282
283	goto cleanup;
284
285fail:
286	radv_device_finish_meta_resolve_state(device);
287
288cleanup:
289	ralloc_free(vs_module.nir);
290
291	return res;
292}
293
294static void
295emit_resolve(struct radv_cmd_buffer *cmd_buffer,
296	     VkFormat vk_format,
297             const VkOffset2D *dest_offset,
298             const VkExtent2D *resolve_extent)
299{
300	struct radv_device *device = cmd_buffer->device;
301	VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
302	unsigned fs_key = radv_format_meta_fs_key(vk_format);
303
304	cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
305
306	radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS,
307			     device->meta_state.resolve.pipeline[fs_key]);
308
309	radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkViewport) {
310		.x = dest_offset->x,
311		.y = dest_offset->y,
312		.width = resolve_extent->width,
313		.height = resolve_extent->height,
314		.minDepth = 0.0f,
315		.maxDepth = 1.0f
316	});
317
318	radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, &(VkRect2D) {
319		.offset = *dest_offset,
320		.extent = *resolve_extent,
321	});
322
323	radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
324	cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_CB;
325}
326
327enum radv_resolve_method {
328	RESOLVE_HW,
329	RESOLVE_COMPUTE,
330	RESOLVE_FRAGMENT,
331};
332
333static void radv_pick_resolve_method_images(struct radv_image *src_image,
334					    struct radv_image *dest_image,
335					    VkImageLayout dest_image_layout,
336					    struct radv_cmd_buffer *cmd_buffer,
337					    enum radv_resolve_method *method)
338
339{
340	uint32_t queue_mask = radv_image_queue_family_mask(dest_image,
341	                                                   cmd_buffer->queue_family_index,
342	                                                   cmd_buffer->queue_family_index);
343
344	if (src_image->vk_format == VK_FORMAT_R16G16_UNORM ||
345	    src_image->vk_format == VK_FORMAT_R16G16_SNORM)
346		*method = RESOLVE_COMPUTE;
347	else if (vk_format_is_int(src_image->vk_format))
348		*method = RESOLVE_COMPUTE;
349	else if (src_image->info.array_size > 1 ||
350		 dest_image->info.array_size > 1)
351		*method = RESOLVE_COMPUTE;
352
353	if (radv_layout_dcc_compressed(dest_image, dest_image_layout, queue_mask)) {
354		*method = RESOLVE_FRAGMENT;
355	} else if (dest_image->surface.micro_tile_mode != src_image->surface.micro_tile_mode) {
356		*method = RESOLVE_COMPUTE;
357	}
358}
359
360static VkResult
361build_resolve_pipeline(struct radv_device *device,
362                       unsigned fs_key)
363{
364	VkResult result = VK_SUCCESS;
365
366	if (device->meta_state.resolve.pipeline[fs_key])
367		return result;
368
369	mtx_lock(&device->meta_state.mtx);
370	if (device->meta_state.resolve.pipeline[fs_key]) {
371		mtx_unlock(&device->meta_state.mtx);
372		return result;
373	}
374
375	struct radv_shader_module vs_module = { .nir = radv_meta_build_nir_vs_generate_vertices() };
376
377	result = create_pass(device, radv_fs_key_format_exemplars[fs_key], &device->meta_state.resolve.pass[fs_key]);
378	if (result != VK_SUCCESS)
379		goto fail;
380
381	VkShaderModule vs_module_h = radv_shader_module_to_handle(&vs_module);
382	result = create_pipeline(device, vs_module_h, &device->meta_state.resolve.pipeline[fs_key], device->meta_state.resolve.pass[fs_key]);
383
384fail:
385	ralloc_free(vs_module.nir);
386	mtx_unlock(&device->meta_state.mtx);
387	return result;
388}
389
390void radv_CmdResolveImage(
391	VkCommandBuffer                             cmd_buffer_h,
392	VkImage                                     src_image_h,
393	VkImageLayout                               src_image_layout,
394	VkImage                                     dest_image_h,
395	VkImageLayout                               dest_image_layout,
396	uint32_t                                    region_count,
397	const VkImageResolve*                       regions)
398{
399	RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, cmd_buffer_h);
400	RADV_FROM_HANDLE(radv_image, src_image, src_image_h);
401	RADV_FROM_HANDLE(radv_image, dest_image, dest_image_h);
402	struct radv_device *device = cmd_buffer->device;
403	struct radv_meta_saved_state saved_state;
404	VkDevice device_h = radv_device_to_handle(device);
405	enum radv_resolve_method resolve_method = RESOLVE_HW;
406	/* we can use the hw resolve only for single full resolves */
407	if (region_count == 1) {
408		if (regions[0].srcOffset.x ||
409		    regions[0].srcOffset.y ||
410		    regions[0].srcOffset.z)
411			resolve_method = RESOLVE_COMPUTE;
412		if (regions[0].dstOffset.x ||
413		    regions[0].dstOffset.y ||
414		    regions[0].dstOffset.z)
415			resolve_method = RESOLVE_COMPUTE;
416
417		if (regions[0].extent.width != src_image->info.width ||
418		    regions[0].extent.height != src_image->info.height ||
419		    regions[0].extent.depth != src_image->info.depth)
420			resolve_method = RESOLVE_COMPUTE;
421	} else
422		resolve_method = RESOLVE_COMPUTE;
423
424	radv_pick_resolve_method_images(src_image, dest_image,
425					dest_image_layout, cmd_buffer,
426					&resolve_method);
427
428	if (resolve_method == RESOLVE_FRAGMENT) {
429		radv_meta_resolve_fragment_image(cmd_buffer,
430						 src_image,
431						 src_image_layout,
432						 dest_image,
433						 dest_image_layout,
434						 region_count, regions);
435		return;
436	}
437
438	if (resolve_method == RESOLVE_COMPUTE) {
439		radv_meta_resolve_compute_image(cmd_buffer,
440						src_image,
441						src_image_layout,
442						dest_image,
443						dest_image_layout,
444						region_count, regions);
445		return;
446	}
447
448	radv_meta_save(&saved_state, cmd_buffer,
449		       RADV_META_SAVE_GRAPHICS_PIPELINE);
450
451	assert(src_image->info.samples > 1);
452	if (src_image->info.samples <= 1) {
453		/* this causes GPU hangs if we get past here */
454		fprintf(stderr, "radv: Illegal resolve operation (src not multisampled), will hang GPU.");
455		return;
456	}
457	assert(dest_image->info.samples == 1);
458
459	if (src_image->info.samples >= 16) {
460		/* See commit aa3f9aaf31e9056a255f9e0472ebdfdaa60abe54 for the
461		 * glBlitFramebuffer workaround for samples >= 16.
462		 */
463		radv_finishme("vkCmdResolveImage: need interpolation workaround when "
464			      "samples >= 16");
465	}
466
467	if (src_image->info.array_size > 1)
468		radv_finishme("vkCmdResolveImage: multisample array images");
469
470	if (radv_image_has_dcc(dest_image)) {
471		radv_initialize_dcc(cmd_buffer, dest_image, 0xffffffff);
472	}
473	unsigned fs_key = radv_format_meta_fs_key(dest_image->vk_format);
474	for (uint32_t r = 0; r < region_count; ++r) {
475		const VkImageResolve *region = &regions[r];
476
477		/* From the Vulkan 1.0 spec:
478		 *
479		 *    - The aspectMask member of srcSubresource and dstSubresource must
480		 *      only contain VK_IMAGE_ASPECT_COLOR_BIT
481		 *
482		 *    - The layerCount member of srcSubresource and dstSubresource must
483		 *      match
484		 */
485		assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
486		assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
487		assert(region->srcSubresource.layerCount ==
488		       region->dstSubresource.layerCount);
489
490		const uint32_t src_base_layer =
491			radv_meta_get_iview_layer(src_image, &region->srcSubresource,
492						  &region->srcOffset);
493
494		const uint32_t dest_base_layer =
495			radv_meta_get_iview_layer(dest_image, &region->dstSubresource,
496						  &region->dstOffset);
497
498		/**
499		 * From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images
500		 *
501		 *    extent is the size in texels of the source image to resolve in width,
502		 *    height and depth. 1D images use only x and width. 2D images use x, y,
503		 *    width and height. 3D images use x, y, z, width, height and depth.
504		 *
505		 *    srcOffset and dstOffset select the initial x, y, and z offsets in
506		 *    texels of the sub-regions of the source and destination image data.
507		 *    extent is the size in texels of the source image to resolve in width,
508		 *    height and depth. 1D images use only x and width. 2D images use x, y,
509		 *    width and height. 3D images use x, y, z, width, height and depth.
510		 */
511		const struct VkExtent3D extent =
512			radv_sanitize_image_extent(src_image->type, region->extent);
513		const struct VkOffset3D dstOffset =
514			radv_sanitize_image_offset(dest_image->type, region->dstOffset);
515
516
517		for (uint32_t layer = 0; layer < region->srcSubresource.layerCount;
518		     ++layer) {
519
520			VkResult ret = build_resolve_pipeline(device, fs_key);
521			if (ret != VK_SUCCESS) {
522				cmd_buffer->record_result = ret;
523				break;
524			}
525
526			struct radv_image_view src_iview;
527			radv_image_view_init(&src_iview, cmd_buffer->device,
528					     &(VkImageViewCreateInfo) {
529						     .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
530							     .image = src_image_h,
531							     .viewType = radv_meta_get_view_type(src_image),
532							     .format = src_image->vk_format,
533							     .subresourceRange = {
534							     .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
535							     .baseMipLevel = region->srcSubresource.mipLevel,
536							     .levelCount = 1,
537							     .baseArrayLayer = src_base_layer + layer,
538							     .layerCount = 1,
539						     },
540					     });
541
542			struct radv_image_view dest_iview;
543			radv_image_view_init(&dest_iview, cmd_buffer->device,
544					     &(VkImageViewCreateInfo) {
545						     .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
546							     .image = dest_image_h,
547							     .viewType = radv_meta_get_view_type(dest_image),
548							     .format = dest_image->vk_format,
549							     .subresourceRange = {
550							     .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
551							     .baseMipLevel = region->dstSubresource.mipLevel,
552							     .levelCount = 1,
553							     .baseArrayLayer = dest_base_layer + layer,
554							     .layerCount = 1,
555						     },
556					      });
557
558			VkFramebuffer fb_h;
559			radv_CreateFramebuffer(device_h,
560					       &(VkFramebufferCreateInfo) {
561						       .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
562							       .attachmentCount = 2,
563							       .pAttachments = (VkImageView[]) {
564							       radv_image_view_to_handle(&src_iview),
565							       radv_image_view_to_handle(&dest_iview),
566						       },
567						       .width = radv_minify(dest_image->info.width,
568									    region->dstSubresource.mipLevel),
569						       .height = radv_minify(dest_image->info.height,
570									      region->dstSubresource.mipLevel),
571						       .layers = 1
572					       },
573					       &cmd_buffer->pool->alloc,
574					       &fb_h);
575
576			radv_CmdBeginRenderPass(cmd_buffer_h,
577						      &(VkRenderPassBeginInfo) {
578							      .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
579								      .renderPass = device->meta_state.resolve.pass[fs_key],
580								      .framebuffer = fb_h,
581								      .renderArea = {
582								      .offset = {
583									      dstOffset.x,
584									      dstOffset.y,
585								      },
586								      .extent = {
587									      extent.width,
588									      extent.height,
589								      }
590							      },
591							      .clearValueCount = 0,
592							      .pClearValues = NULL,
593						      },
594						      VK_SUBPASS_CONTENTS_INLINE);
595
596			emit_resolve(cmd_buffer,
597				     dest_iview.vk_format,
598				     &(VkOffset2D) {
599					     .x = dstOffset.x,
600					     .y = dstOffset.y,
601				     },
602				     &(VkExtent2D) {
603					     .width = extent.width,
604					     .height = extent.height,
605				     });
606
607			radv_CmdEndRenderPass(cmd_buffer_h);
608
609			radv_DestroyFramebuffer(device_h, fb_h,
610						&cmd_buffer->pool->alloc);
611		}
612	}
613
614	radv_meta_restore(&saved_state, cmd_buffer);
615}
616
617/**
618 * Emit any needed resolves for the current subpass.
619 */
620void
621radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer)
622{
623	struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
624	const struct radv_subpass *subpass = cmd_buffer->state.subpass;
625	struct radv_meta_saved_state saved_state;
626	enum radv_resolve_method resolve_method = RESOLVE_HW;
627
628	/* FINISHME(perf): Skip clears for resolve attachments.
629	 *
630	 * From the Vulkan 1.0 spec:
631	 *
632	 *    If the first use of an attachment in a render pass is as a resolve
633	 *    attachment, then the loadOp is effectively ignored as the resolve is
634	 *    guaranteed to overwrite all pixels in the render area.
635	 */
636
637	if (!subpass->has_resolve)
638		return;
639
640	for (uint32_t i = 0; i < subpass->color_count; ++i) {
641		struct radv_subpass_attachment src_att = subpass->color_attachments[i];
642		struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
643
644		if (src_att.attachment == VK_ATTACHMENT_UNUSED ||
645		    dest_att.attachment == VK_ATTACHMENT_UNUSED)
646			continue;
647
648		struct radv_image *dst_img = cmd_buffer->state.framebuffer->attachments[dest_att.attachment].attachment->image;
649		struct radv_image *src_img = cmd_buffer->state.framebuffer->attachments[src_att.attachment].attachment->image;
650
651		radv_pick_resolve_method_images(src_img, dst_img, dest_att.layout, cmd_buffer, &resolve_method);
652		if (resolve_method == RESOLVE_FRAGMENT) {
653			break;
654		}
655	}
656
657	if (resolve_method == RESOLVE_COMPUTE) {
658		radv_cmd_buffer_resolve_subpass_cs(cmd_buffer);
659		return;
660	} else if (resolve_method == RESOLVE_FRAGMENT) {
661		radv_cmd_buffer_resolve_subpass_fs(cmd_buffer);
662		return;
663	}
664
665	radv_meta_save(&saved_state, cmd_buffer,
666		       RADV_META_SAVE_GRAPHICS_PIPELINE);
667
668	for (uint32_t i = 0; i < subpass->color_count; ++i) {
669		struct radv_subpass_attachment src_att = subpass->color_attachments[i];
670		struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
671
672		if (src_att.attachment == VK_ATTACHMENT_UNUSED ||
673		    dest_att.attachment == VK_ATTACHMENT_UNUSED)
674			continue;
675
676		struct radv_image *dst_img = cmd_buffer->state.framebuffer->attachments[dest_att.attachment].attachment->image;
677
678		if (radv_image_has_dcc(dst_img)) {
679			radv_initialize_dcc(cmd_buffer, dst_img, 0xffffffff);
680			cmd_buffer->state.attachments[dest_att.attachment].current_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
681		}
682
683		struct radv_subpass resolve_subpass = {
684			.color_count = 2,
685			.color_attachments = (struct radv_subpass_attachment[]) { src_att, dest_att },
686			.depth_stencil_attachment = { .attachment = VK_ATTACHMENT_UNUSED },
687		};
688
689		radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass, false);
690
691		VkResult ret = build_resolve_pipeline(cmd_buffer->device, radv_format_meta_fs_key(dst_img->vk_format));
692		if (ret != VK_SUCCESS) {
693			cmd_buffer->record_result = ret;
694			continue;
695		}
696
697		emit_resolve(cmd_buffer,
698			     dst_img->vk_format,
699			     &(VkOffset2D) { 0, 0 },
700			     &(VkExtent2D) { fb->width, fb->height });
701	}
702
703	cmd_buffer->state.subpass = subpass;
704	radv_meta_restore(&saved_state, cmd_buffer);
705}
706
707/**
708 * Decompress CMask/FMask before resolving a multisampled source image inside a
709 * subpass.
710 */
711void
712radv_decompress_resolve_subpass_src(struct radv_cmd_buffer *cmd_buffer)
713{
714	const struct radv_subpass *subpass = cmd_buffer->state.subpass;
715	struct radv_framebuffer *fb = cmd_buffer->state.framebuffer;
716
717	for (uint32_t i = 0; i < subpass->color_count; ++i) {
718		struct radv_subpass_attachment src_att = subpass->color_attachments[i];
719		struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i];
720
721		if (src_att.attachment == VK_ATTACHMENT_UNUSED ||
722		    dest_att.attachment == VK_ATTACHMENT_UNUSED)
723			continue;
724
725		struct radv_image *src_image =
726			fb->attachments[src_att.attachment].attachment->image;
727
728		VkImageResolve region = {};
729		region.srcSubresource.baseArrayLayer = 0;
730		region.srcSubresource.mipLevel = 0;
731		region.srcSubresource.layerCount = src_image->info.array_size;
732
733		radv_decompress_resolve_src(cmd_buffer, src_image,
734					    src_att.layout, 1, &region);
735	}
736}
737
738/**
739 * Decompress CMask/FMask before resolving a multisampled source image.
740 */
741void
742radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer,
743			    struct radv_image *src_image,
744			    VkImageLayout src_image_layout,
745			    uint32_t region_count,
746			    const VkImageResolve *regions)
747{
748	for (uint32_t r = 0; r < region_count; ++r) {
749		const VkImageResolve *region = &regions[r];
750		const uint32_t src_base_layer =
751			radv_meta_get_iview_layer(src_image, &region->srcSubresource,
752						  &region->srcOffset);
753		VkImageSubresourceRange range;
754		range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
755		range.baseMipLevel = region->srcSubresource.mipLevel;
756		range.levelCount = 1;
757		range.baseArrayLayer = src_base_layer;
758		range.layerCount = region->srcSubresource.layerCount;
759
760		uint32_t queue_mask =
761			radv_image_queue_family_mask(src_image,
762						     cmd_buffer->queue_family_index,
763						     cmd_buffer->queue_family_index);
764
765		if (radv_layout_dcc_compressed(src_image, src_image_layout,
766					       queue_mask)) {
767			radv_decompress_dcc(cmd_buffer, src_image, &range);
768		} else {
769			radv_fast_clear_flush_image_inplace(cmd_buffer,
770							    src_image, &range);
771		}
772	}
773}
774