1/*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23/**
24 * @file iris_pipe_control.c
25 *
26 * PIPE_CONTROL is the main flushing and synchronization primitive on Intel
27 * GPUs.  It can invalidate caches, stall until rendering reaches various
28 * stages of completion, write to memory, and other things.  In a way, it's
29 * a swiss army knife command - it has all kinds of capabilities, but some
30 * significant limitations as well.
31 *
32 * Unfortunately, it's notoriously complicated and difficult to use.  Many
33 * sub-commands can't be used together.  Some are meant to be used at the
34 * top of the pipeline (invalidating caches before drawing), while some are
35 * meant to be used at the end (stalling or flushing after drawing).
36 *
37 * Also, there's a list of restrictions a mile long, which vary by generation.
38 * Do this before doing that, or suffer the consequences (usually a GPU hang).
39 *
40 * This file contains helpers for emitting them safely.  You can simply call
41 * iris_emit_pipe_control_flush() with the desired operations (as logical
42 * PIPE_CONTROL_* bits), and it will take care of splitting it into multiple
43 * PIPE_CONTROL commands as necessary.  The per-generation workarounds are
44 * applied in iris_emit_raw_pipe_control() in iris_state.c.
45 */
46
47#include "iris_context.h"
48#include "util/hash_table.h"
49#include "util/set.h"
50
51/**
52 * Emit a PIPE_CONTROL with various flushing flags.
53 *
54 * The caller is responsible for deciding what flags are appropriate for the
55 * given generation.
56 */
57void
58iris_emit_pipe_control_flush(struct iris_batch *batch, uint32_t flags)
59{
60   if ((flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
61       (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
62      /* A pipe control command with flush and invalidate bits set
63       * simultaneously is an inherently racy operation on Gen6+ if the
64       * contents of the flushed caches were intended to become visible from
65       * any of the invalidated caches.  Split it in two PIPE_CONTROLs, the
66       * first one should stall the pipeline to make sure that the flushed R/W
67       * caches are coherent with memory once the specified R/O caches are
68       * invalidated.  On pre-Gen6 hardware the (implicit) R/O cache
69       * invalidation seems to happen at the bottom of the pipeline together
70       * with any write cache flush, so this shouldn't be a concern.  In order
71       * to ensure a full stall, we do an end-of-pipe sync.
72       */
73      iris_emit_end_of_pipe_sync(batch, flags & PIPE_CONTROL_CACHE_FLUSH_BITS);
74      flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
75   }
76
77   batch->vtbl->emit_raw_pipe_control(batch, flags, NULL, 0, 0);
78}
79
80/**
81 * Emit a PIPE_CONTROL that writes to a buffer object.
82 *
83 * \p flags should contain one of the following items:
84 *  - PIPE_CONTROL_WRITE_IMMEDIATE
85 *  - PIPE_CONTROL_WRITE_TIMESTAMP
86 *  - PIPE_CONTROL_WRITE_DEPTH_COUNT
87 */
88void
89iris_emit_pipe_control_write(struct iris_batch *batch, uint32_t flags,
90                             struct iris_bo *bo, uint32_t offset,
91                             uint64_t imm)
92{
93   batch->vtbl->emit_raw_pipe_control(batch, flags, bo, offset, imm);
94}
95
96/*
97 * From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
98 *
99 *  Write synchronization is a special case of end-of-pipe
100 *  synchronization that requires that the render cache and/or depth
101 *  related caches are flushed to memory, where the data will become
102 *  globally visible. This type of synchronization is required prior to
103 *  SW (CPU) actually reading the result data from memory, or initiating
104 *  an operation that will use as a read surface (such as a texture
105 *  surface) a previous render target and/or depth/stencil buffer
106 *
107 * From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
108 *
109 *  Exercising the write cache flush bits (Render Target Cache Flush
110 *  Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
111 *  ensures the write caches are flushed and doesn't guarantee the data
112 *  is globally visible.
113 *
114 *  SW can track the completion of the end-of-pipe-synchronization by
115 *  using "Notify Enable" and "PostSync Operation - Write Immediate
116 *  Data" in the PIPE_CONTROL command.
117 */
118void
119iris_emit_end_of_pipe_sync(struct iris_batch *batch, uint32_t flags)
120{
121   /* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
122    *
123    *    "The most common action to perform upon reaching a synchronization
124    *    point is to write a value out to memory. An immediate value
125    *    (included with the synchronization command) may be written."
126    *
127    * From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
128    *
129    *    "In case the data flushed out by the render engine is to be read
130    *    back in to the render engine in coherent manner, then the render
131    *    engine has to wait for the fence completion before accessing the
132    *    flushed data. This can be achieved by following means on various
133    *    products: PIPE_CONTROL command with CS Stall and the required
134    *    write caches flushed with Post-Sync-Operation as Write Immediate
135    *    Data.
136    *
137    *    Example:
138    *       - Workload-1 (3D/GPGPU/MEDIA)
139    *       - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
140    *         Data, Required Write Cache Flush bits set)
141    *       - Workload-2 (Can use the data produce or output by Workload-1)
142    */
143   iris_emit_pipe_control_write(batch, flags | PIPE_CONTROL_CS_STALL |
144                                PIPE_CONTROL_WRITE_IMMEDIATE,
145                                batch->screen->workaround_bo, 0, 0);
146}
147
148static void
149iris_texture_barrier(struct pipe_context *ctx, unsigned flags)
150{
151   struct iris_context *ice = (void *) ctx;
152   struct iris_batch *render_batch = &ice->batches[IRIS_BATCH_RENDER];
153   struct iris_batch *compute_batch = &ice->batches[IRIS_BATCH_COMPUTE];
154
155   if (render_batch->contains_draw ||
156       render_batch->cache.render->entries ||
157       render_batch->cache.depth->entries) {
158      iris_emit_pipe_control_flush(&ice->batches[IRIS_BATCH_RENDER],
159                                   PIPE_CONTROL_DEPTH_CACHE_FLUSH |
160                                   PIPE_CONTROL_RENDER_TARGET_FLUSH |
161                                   PIPE_CONTROL_CS_STALL);
162      iris_emit_pipe_control_flush(&ice->batches[IRIS_BATCH_RENDER],
163                                   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
164   }
165
166   if (compute_batch->contains_draw) {
167      iris_emit_pipe_control_flush(&ice->batches[IRIS_BATCH_COMPUTE],
168                                   PIPE_CONTROL_CS_STALL);
169      iris_emit_pipe_control_flush(&ice->batches[IRIS_BATCH_COMPUTE],
170                                   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
171   }
172}
173
174static void
175iris_memory_barrier(struct pipe_context *ctx, unsigned flags)
176{
177   struct iris_context *ice = (void *) ctx;
178   unsigned bits = PIPE_CONTROL_DATA_CACHE_FLUSH | PIPE_CONTROL_CS_STALL;
179
180   if (flags & (PIPE_BARRIER_VERTEX_BUFFER |
181                PIPE_BARRIER_INDEX_BUFFER |
182                PIPE_BARRIER_INDIRECT_BUFFER)) {
183      bits |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
184   }
185
186   if (flags & PIPE_BARRIER_CONSTANT_BUFFER) {
187      bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
188              PIPE_CONTROL_CONST_CACHE_INVALIDATE;
189   }
190
191   if (flags & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_FRAMEBUFFER)) {
192      bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
193              PIPE_CONTROL_RENDER_TARGET_FLUSH;
194   }
195
196   for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
197      if (ice->batches[i].contains_draw ||
198          ice->batches[i].cache.render->entries)
199         iris_emit_pipe_control_flush(&ice->batches[i], bits);
200   }
201}
202
203void
204iris_init_flush_functions(struct pipe_context *ctx)
205{
206   ctx->memory_barrier = iris_memory_barrier;
207   ctx->texture_barrier = iris_texture_barrier;
208}
209