1/*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include "aco_ir.h"
26
27#include "util/memstream.h"
28
29#include <array>
30#include <map>
31#include <set>
32#include <vector>
33
34namespace aco {
35
36static void
37aco_log(Program* program, enum radv_compiler_debug_level level, const char* prefix,
38        const char* file, unsigned line, const char* fmt, va_list args)
39{
40   char* msg;
41
42   if (program->debug.shorten_messages) {
43      msg = ralloc_vasprintf(NULL, fmt, args);
44   } else {
45      msg = ralloc_strdup(NULL, prefix);
46      ralloc_asprintf_append(&msg, "    In file %s:%u\n", file, line);
47      ralloc_asprintf_append(&msg, "    ");
48      ralloc_vasprintf_append(&msg, fmt, args);
49   }
50
51   if (program->debug.func)
52      program->debug.func(program->debug.private_data, level, msg);
53
54   fprintf(program->debug.output, "%s\n", msg);
55
56   ralloc_free(msg);
57}
58
59void
60_aco_perfwarn(Program* program, const char* file, unsigned line, const char* fmt, ...)
61{
62   va_list args;
63
64   va_start(args, fmt);
65   aco_log(program, RADV_COMPILER_DEBUG_LEVEL_PERFWARN, "ACO PERFWARN:\n", file, line, fmt, args);
66   va_end(args);
67}
68
69void
70_aco_err(Program* program, const char* file, unsigned line, const char* fmt, ...)
71{
72   va_list args;
73
74   va_start(args, fmt);
75   aco_log(program, RADV_COMPILER_DEBUG_LEVEL_ERROR, "ACO ERROR:\n", file, line, fmt, args);
76   va_end(args);
77}
78
79bool
80validate_ir(Program* program)
81{
82   bool is_valid = true;
83   auto check = [&program, &is_valid](bool success, const char* msg,
84                                      aco::Instruction* instr) -> void
85   {
86      if (!success) {
87         char* out;
88         size_t outsize;
89         struct u_memstream mem;
90         u_memstream_open(&mem, &out, &outsize);
91         FILE* const memf = u_memstream_get(&mem);
92
93         fprintf(memf, "%s: ", msg);
94         aco_print_instr(instr, memf);
95         u_memstream_close(&mem);
96
97         aco_err(program, "%s", out);
98         free(out);
99
100         is_valid = false;
101      }
102   };
103
104   auto check_block = [&program, &is_valid](bool success, const char* msg,
105                                            aco::Block* block) -> void
106   {
107      if (!success) {
108         aco_err(program, "%s: BB%u", msg, block->index);
109         is_valid = false;
110      }
111   };
112
113   for (Block& block : program->blocks) {
114      for (aco_ptr<Instruction>& instr : block.instructions) {
115
116         /* check base format */
117         Format base_format = instr->format;
118         base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
119         base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
120         if ((uint32_t)base_format & (uint32_t)Format::VOP1)
121            base_format = Format::VOP1;
122         else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
123            base_format = Format::VOP2;
124         else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
125            base_format = Format::VOPC;
126         else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) {
127            if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
128                instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
129                instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
130                instr->opcode == aco_opcode::v_interp_p2_f16) {
131               /* v_interp_*_fp16 are considered VINTRP by the compiler but
132                * they are emitted as VOP3.
133                */
134               base_format = Format::VOP3;
135            } else {
136               base_format = Format::VINTRP;
137            }
138         }
139         check(base_format == instr_info.format[(int)instr->opcode],
140               "Wrong base format for instruction", instr.get());
141
142         /* check VOP3 modifiers */
143         if (instr->isVOP3() && instr->format != Format::VOP3) {
144            check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
145                     base_format == Format::VOPC || base_format == Format::VINTRP,
146                  "Format cannot have VOP3/VOP3B applied", instr.get());
147         }
148
149         /* check SDWA */
150         if (instr->isSDWA()) {
151            check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
152                     base_format == Format::VOPC,
153                  "Format cannot have SDWA applied", instr.get());
154
155            check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get());
156
157            SDWA_instruction& sdwa = instr->sdwa();
158            check(sdwa.omod == 0 || program->chip_class >= GFX9,
159                  "SDWA omod only supported on GFX9+", instr.get());
160            if (base_format == Format::VOPC) {
161               check(sdwa.clamp == false || program->chip_class == GFX8,
162                     "SDWA VOPC clamp only supported on GFX8", instr.get());
163               check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
164                        program->chip_class >= GFX9,
165                     "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
166            } else {
167               const Definition& def = instr->definitions[0];
168               check(def.bytes() <= 4, "SDWA definitions must not be larger than 4 bytes",
169                     instr.get());
170               check(def.bytes() >= sdwa.dst_sel.size() + sdwa.dst_sel.offset(),
171                     "SDWA definition selection size must be at most definition size", instr.get());
172               check(
173                  sdwa.dst_sel.size() == 1 || sdwa.dst_sel.size() == 2 || sdwa.dst_sel.size() == 4,
174                  "SDWA definition selection size must be 1, 2 or 4 bytes", instr.get());
175               check(sdwa.dst_sel.offset() % sdwa.dst_sel.size() == 0, "Invalid selection offset",
176                     instr.get());
177               check(def.bytes() == 4 || def.bytes() == sdwa.dst_sel.size(),
178                     "SDWA dst_sel size must be definition size for subdword definitions",
179                     instr.get());
180               check(def.bytes() == 4 || sdwa.dst_sel.offset() == 0,
181                     "SDWA dst_sel offset must be 0 for subdword definitions", instr.get());
182            }
183
184            for (unsigned i = 0; i < std::min<unsigned>(2, instr->operands.size()); i++) {
185               const Operand& op = instr->operands[i];
186               check(op.bytes() <= 4, "SDWA operands must not be larger than 4 bytes", instr.get());
187               check(op.bytes() >= sdwa.sel[i].size() + sdwa.sel[i].offset(),
188                     "SDWA operand selection size must be at most operand size", instr.get());
189               check(sdwa.sel[i].size() == 1 || sdwa.sel[i].size() == 2 || sdwa.sel[i].size() == 4,
190                     "SDWA operand selection size must be 1, 2 or 4 bytes", instr.get());
191               check(sdwa.sel[i].offset() % sdwa.sel[i].size() == 0, "Invalid selection offset",
192                     instr.get());
193            }
194            if (instr->operands.size() >= 3) {
195               check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
196                     "3rd operand must be fixed to vcc with SDWA", instr.get());
197            }
198            if (instr->definitions.size() >= 2) {
199               check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
200                     "2nd definition must be fixed to vcc with SDWA", instr.get());
201            }
202
203            const bool sdwa_opcodes =
204               instr->opcode != aco_opcode::v_fmac_f32 && instr->opcode != aco_opcode::v_fmac_f16 &&
205               instr->opcode != aco_opcode::v_fmamk_f32 &&
206               instr->opcode != aco_opcode::v_fmaak_f32 &&
207               instr->opcode != aco_opcode::v_fmamk_f16 &&
208               instr->opcode != aco_opcode::v_fmaak_f16 &&
209               instr->opcode != aco_opcode::v_madmk_f32 &&
210               instr->opcode != aco_opcode::v_madak_f32 &&
211               instr->opcode != aco_opcode::v_madmk_f16 &&
212               instr->opcode != aco_opcode::v_madak_f16 &&
213               instr->opcode != aco_opcode::v_readfirstlane_b32 &&
214               instr->opcode != aco_opcode::v_clrexcp && instr->opcode != aco_opcode::v_swap_b32;
215
216            const bool feature_mac =
217               program->chip_class == GFX8 &&
218               (instr->opcode == aco_opcode::v_mac_f32 && instr->opcode == aco_opcode::v_mac_f16);
219
220            check(sdwa_opcodes || feature_mac, "SDWA can't be used with this opcode", instr.get());
221         }
222
223         /* check opsel */
224         if (instr->isVOP3()) {
225            VOP3_instruction& vop3 = instr->vop3();
226            check(vop3.opsel == 0 || program->chip_class >= GFX9,
227                  "Opsel is only supported on GFX9+", instr.get());
228
229            for (unsigned i = 0; i < 3; i++) {
230               if (i >= instr->operands.size() ||
231                   (instr->operands[i].hasRegClass() &&
232                    instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed()))
233                  check((vop3.opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get());
234            }
235            if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed())
236               check((vop3.opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition",
237                     instr.get());
238         } else if (instr->isVOP3P()) {
239            VOP3P_instruction& vop3p = instr->vop3p();
240            for (unsigned i = 0; i < instr->operands.size(); i++) {
241               if (instr->operands[i].hasRegClass() &&
242                   instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed())
243                  check((vop3p.opsel_lo & (1 << i)) == 0 && (vop3p.opsel_hi & (1 << i)) == 0,
244                        "Unexpected opsel for subdword operand", instr.get());
245            }
246            check(instr->definitions[0].regClass() == v1, "VOP3P must have v1 definition",
247                  instr.get());
248         }
249
250         /* check for undefs */
251         for (unsigned i = 0; i < instr->operands.size(); i++) {
252            if (instr->operands[i].isUndefined()) {
253               bool flat = instr->isFlatLike();
254               bool can_be_undef = is_phi(instr) || instr->isEXP() || instr->isReduction() ||
255                                   instr->opcode == aco_opcode::p_create_vector ||
256                                   (flat && i == 1) || (instr->isMIMG() && (i == 1 || i == 2)) ||
257                                   ((instr->isMUBUF() || instr->isMTBUF()) && i == 1);
258               check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
259            } else {
260               check(instr->operands[i].isFixed() || instr->operands[i].isTemp() ||
261                        instr->operands[i].isConstant(),
262                     "Uninitialized Operand", instr.get());
263            }
264         }
265
266         /* check subdword definitions */
267         for (unsigned i = 0; i < instr->definitions.size(); i++) {
268            if (instr->definitions[i].regClass().is_subdword())
269               check(instr->isPseudo() || instr->definitions[i].bytes() <= 4,
270                     "Only Pseudo instructions can write subdword registers larger than 4 bytes",
271                     instr.get());
272         }
273
274         if (instr->isSALU() || instr->isVALU()) {
275            /* check literals */
276            Operand literal(s1);
277            for (unsigned i = 0; i < instr->operands.size(); i++) {
278               Operand op = instr->operands[i];
279               if (!op.isLiteral())
280                  continue;
281
282               check(!instr->isDPP() && !instr->isSDWA() &&
283                        (!instr->isVOP3() || program->chip_class >= GFX10) &&
284                        (!instr->isVOP3P() || program->chip_class >= GFX10),
285                     "Literal applied on wrong instruction format", instr.get());
286
287               check(literal.isUndefined() || (literal.size() == op.size() &&
288                                               literal.constantValue() == op.constantValue()),
289                     "Only 1 Literal allowed", instr.get());
290               literal = op;
291               check(instr->isSALU() || instr->isVOP3() || instr->isVOP3P() || i == 0 || i == 2,
292                     "Wrong source position for Literal argument", instr.get());
293            }
294
295            /* check num sgprs for VALU */
296            if (instr->isVALU()) {
297               bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
298                                 instr->opcode == aco_opcode::v_lshrrev_b64 ||
299                                 instr->opcode == aco_opcode::v_ashrrev_i64;
300               unsigned const_bus_limit = 1;
301               if (program->chip_class >= GFX10 && !is_shift64)
302                  const_bus_limit = 2;
303
304               uint32_t scalar_mask = instr->isVOP3() || instr->isVOP3P() ? 0x7 : 0x5;
305               if (instr->isSDWA())
306                  scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4;
307               else if (instr->isDPP())
308                  scalar_mask = 0x4;
309
310               if (instr->isVOPC() || instr->opcode == aco_opcode::v_readfirstlane_b32 ||
311                   instr->opcode == aco_opcode::v_readlane_b32 ||
312                   instr->opcode == aco_opcode::v_readlane_b32_e64) {
313                  check(instr->definitions[0].getTemp().type() == RegType::sgpr,
314                        "Wrong Definition type for VALU instruction", instr.get());
315               } else {
316                  check(instr->definitions[0].getTemp().type() == RegType::vgpr,
317                        "Wrong Definition type for VALU instruction", instr.get());
318               }
319
320               unsigned num_sgprs = 0;
321               unsigned sgpr[] = {0, 0};
322               for (unsigned i = 0; i < instr->operands.size(); i++) {
323                  Operand op = instr->operands[i];
324                  if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
325                      instr->opcode == aco_opcode::v_readlane_b32 ||
326                      instr->opcode == aco_opcode::v_readlane_b32_e64) {
327                     check(i != 1 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
328                              op.isConstant(),
329                           "Must be a SGPR or a constant", instr.get());
330                     check(i == 1 || (op.isTemp() && op.regClass().type() == RegType::vgpr &&
331                                      op.bytes() <= 4),
332                           "Wrong Operand type for VALU instruction", instr.get());
333                     continue;
334                  }
335                  if (instr->opcode == aco_opcode::v_permlane16_b32 ||
336                      instr->opcode == aco_opcode::v_permlanex16_b32) {
337                     check(i != 0 || (op.isTemp() && op.regClass().type() == RegType::vgpr),
338                           "Operand 0 of v_permlane must be VGPR", instr.get());
339                     check(i == 0 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
340                              op.isConstant(),
341                           "Lane select operands of v_permlane must be SGPR or constant",
342                           instr.get());
343                  }
344
345                  if (instr->opcode == aco_opcode::v_writelane_b32 ||
346                      instr->opcode == aco_opcode::v_writelane_b32_e64) {
347                     check(i != 2 || (op.isTemp() && op.regClass().type() == RegType::vgpr &&
348                                      op.bytes() <= 4),
349                           "Wrong Operand type for VALU instruction", instr.get());
350                     check(i == 2 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
351                              op.isConstant(),
352                           "Must be a SGPR or a constant", instr.get());
353                     continue;
354                  }
355                  if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
356                     check(scalar_mask & (1 << i), "Wrong source position for SGPR argument",
357                           instr.get());
358
359                     if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
360                        if (num_sgprs < 2)
361                           sgpr[num_sgprs++] = op.tempId();
362                     }
363                  }
364
365                  if (op.isConstant() && !op.isLiteral())
366                     check(scalar_mask & (1 << i), "Wrong source position for constant argument",
367                           instr.get());
368               }
369               check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit,
370                     "Too many SGPRs/literals", instr.get());
371            }
372
373            if (instr->isSOP1() || instr->isSOP2()) {
374               check(instr->definitions[0].getTemp().type() == RegType::sgpr,
375                     "Wrong Definition type for SALU instruction", instr.get());
376               for (const Operand& op : instr->operands) {
377                  check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
378                        "Wrong Operand type for SALU instruction", instr.get());
379               }
380            }
381         }
382
383         switch (instr->format) {
384         case Format::PSEUDO: {
385            if (instr->opcode == aco_opcode::p_create_vector) {
386               unsigned size = 0;
387               for (const Operand& op : instr->operands) {
388                  check(op.bytes() < 4 || size % 4 == 0, "Operand is not aligned", instr.get());
389                  size += op.bytes();
390               }
391               check(size == instr->definitions[0].bytes(),
392                     "Definition size does not match operand sizes", instr.get());
393               if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
394                  for (const Operand& op : instr->operands) {
395                     check(op.isConstant() || op.regClass().type() == RegType::sgpr,
396                           "Wrong Operand type for scalar vector", instr.get());
397                  }
398               }
399            } else if (instr->opcode == aco_opcode::p_extract_vector) {
400               check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(),
401                     "Wrong Operand types", instr.get());
402               check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <=
403                        instr->operands[0].bytes(),
404                     "Index out of range", instr.get());
405               check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
406                        instr->operands[0].regClass().type() == RegType::sgpr,
407                     "Cannot extract SGPR value from VGPR vector", instr.get());
408               check(program->chip_class >= GFX9 ||
409                        !instr->definitions[0].regClass().is_subdword() ||
410                        instr->operands[0].regClass().type() == RegType::vgpr,
411                     "Cannot extract subdword from SGPR before GFX9+", instr.get());
412            } else if (instr->opcode == aco_opcode::p_split_vector) {
413               check(instr->operands[0].isTemp(), "Operand must be a temporary", instr.get());
414               unsigned size = 0;
415               for (const Definition& def : instr->definitions) {
416                  size += def.bytes();
417               }
418               check(size == instr->operands[0].bytes(),
419                     "Operand size does not match definition sizes", instr.get());
420               if (instr->operands[0].getTemp().type() == RegType::vgpr) {
421                  for (const Definition& def : instr->definitions)
422                     check(def.regClass().type() == RegType::vgpr,
423                           "Wrong Definition type for VGPR split_vector", instr.get());
424               } else {
425                  for (const Definition& def : instr->definitions)
426                     check(program->chip_class >= GFX9 || !def.regClass().is_subdword(),
427                           "Cannot split SGPR into subdword VGPRs before GFX9+", instr.get());
428               }
429            } else if (instr->opcode == aco_opcode::p_parallelcopy) {
430               check(instr->definitions.size() == instr->operands.size(),
431                     "Number of Operands does not match number of Definitions", instr.get());
432               for (unsigned i = 0; i < instr->operands.size(); i++) {
433                  check(instr->definitions[i].bytes() == instr->operands[i].bytes(),
434                        "Operand and Definition size must match", instr.get());
435                  if (instr->operands[i].isTemp()) {
436                     check((instr->definitions[i].getTemp().type() ==
437                            instr->operands[i].regClass().type()) ||
438                              (instr->definitions[i].getTemp().type() == RegType::vgpr &&
439                               instr->operands[i].regClass().type() == RegType::sgpr),
440                           "Operand and Definition types do not match", instr.get());
441                     check(instr->definitions[i].regClass().is_linear_vgpr() ==
442                              instr->operands[i].regClass().is_linear_vgpr(),
443                           "Operand and Definition types do not match", instr.get());
444                  } else {
445                     check(!instr->definitions[i].regClass().is_linear_vgpr(),
446                           "Can only copy linear VGPRs into linear VGPRs, not constant/undef",
447                           instr.get());
448                  }
449               }
450            } else if (instr->opcode == aco_opcode::p_phi) {
451               check(instr->operands.size() == block.logical_preds.size(),
452                     "Number of Operands does not match number of predecessors", instr.get());
453               check(instr->definitions[0].getTemp().type() == RegType::vgpr,
454                     "Logical Phi Definition must be vgpr", instr.get());
455               for (const Operand& op : instr->operands)
456                  check(instr->definitions[0].size() == op.size(),
457                        "Operand sizes must match Definition size", instr.get());
458            } else if (instr->opcode == aco_opcode::p_linear_phi) {
459               for (const Operand& op : instr->operands) {
460                  check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type",
461                        instr.get());
462                  check(instr->definitions[0].size() == op.size(),
463                        "Operand sizes must match Definition size", instr.get());
464               }
465               check(instr->operands.size() == block.linear_preds.size(),
466                     "Number of Operands does not match number of predecessors", instr.get());
467            } else if (instr->opcode == aco_opcode::p_extract ||
468                       instr->opcode == aco_opcode::p_insert) {
469               check(instr->operands[0].isTemp(), "Data operand must be temporary", instr.get());
470               check(instr->operands[1].isConstant(), "Index must be constant", instr.get());
471               if (instr->opcode == aco_opcode::p_extract)
472                  check(instr->operands[3].isConstant(), "Sign-extend flag must be constant",
473                        instr.get());
474
475               check(instr->definitions[0].getTemp().type() != RegType::sgpr ||
476                        instr->operands[0].getTemp().type() == RegType::sgpr,
477                     "Can't extract/insert VGPR to SGPR", instr.get());
478
479               if (instr->opcode == aco_opcode::p_insert)
480                  check(instr->operands[0].bytes() == instr->definitions[0].bytes(),
481                        "Sizes of p_insert data operand and definition must match", instr.get());
482
483               if (instr->definitions[0].getTemp().type() == RegType::sgpr)
484                  check(instr->definitions.size() >= 2 && instr->definitions[1].isFixed() &&
485                           instr->definitions[1].physReg() == scc,
486                        "SGPR extract/insert needs an SCC definition", instr.get());
487
488               unsigned data_bits = instr->operands[0].getTemp().bytes() * 8u;
489               unsigned op_bits = instr->operands[2].constantValue();
490
491               if (instr->opcode == aco_opcode::p_insert) {
492                  check(op_bits == 8 || op_bits == 16, "Size must be 8 or 16", instr.get());
493                  check(op_bits < data_bits, "Size must be smaller than source", instr.get());
494               } else if (instr->opcode == aco_opcode::p_extract) {
495                  check(op_bits == 8 || op_bits == 16 || op_bits == 32,
496                        "Size must be 8 or 16 or 32", instr.get());
497                  check(data_bits >= op_bits, "Can't extract more bits than what the data has.",
498                        instr.get());
499               }
500
501               unsigned comp = data_bits / MAX2(op_bits, 1);
502               check(instr->operands[1].constantValue() < comp, "Index must be in-bounds",
503                     instr.get());
504            }
505            break;
506         }
507         case Format::PSEUDO_REDUCTION: {
508            for (const Operand& op : instr->operands)
509               check(op.regClass().type() == RegType::vgpr,
510                     "All operands of PSEUDO_REDUCTION instructions must be in VGPRs.",
511                     instr.get());
512
513            if (instr->opcode == aco_opcode::p_reduce &&
514                instr->reduction().cluster_size == program->wave_size)
515               check(instr->definitions[0].regClass().type() == RegType::sgpr ||
516                        program->wave_size == 32,
517                     "The result of unclustered reductions must go into an SGPR.", instr.get());
518            else
519               check(instr->definitions[0].regClass().type() == RegType::vgpr,
520                     "The result of scans and clustered reductions must go into a VGPR.",
521                     instr.get());
522
523            break;
524         }
525         case Format::SMEM: {
526            if (instr->operands.size() >= 1)
527               check((instr->operands[0].isFixed() && !instr->operands[0].isConstant()) ||
528                        (instr->operands[0].isTemp() &&
529                         instr->operands[0].regClass().type() == RegType::sgpr),
530                     "SMEM operands must be sgpr", instr.get());
531            if (instr->operands.size() >= 2)
532               check(instr->operands[1].isConstant() ||
533                        (instr->operands[1].isTemp() &&
534                         instr->operands[1].regClass().type() == RegType::sgpr),
535                     "SMEM offset must be constant or sgpr", instr.get());
536            if (!instr->definitions.empty())
537               check(instr->definitions[0].getTemp().type() == RegType::sgpr,
538                     "SMEM result must be sgpr", instr.get());
539            break;
540         }
541         case Format::MTBUF:
542         case Format::MUBUF: {
543            check(instr->operands.size() > 1, "VMEM instructions must have at least one operand",
544                  instr.get());
545            check(instr->operands[1].hasRegClass() &&
546                     instr->operands[1].regClass().type() == RegType::vgpr,
547                  "VADDR must be in vgpr for VMEM instructions", instr.get());
548            check(
549               instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr,
550               "VMEM resource constant must be sgpr", instr.get());
551            check(instr->operands.size() < 4 ||
552                     (instr->operands[3].isTemp() &&
553                      instr->operands[3].regClass().type() == RegType::vgpr),
554                  "VMEM write data must be vgpr", instr.get());
555            break;
556         }
557         case Format::MIMG: {
558            check(instr->operands.size() >= 4, "MIMG instructions must have at least 4 operands",
559                  instr.get());
560            check(instr->operands[0].hasRegClass() &&
561                     (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
562                  "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
563            if (instr->operands[1].hasRegClass())
564               check(instr->operands[1].regClass() == s4,
565                     "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
566            if (!instr->operands[2].isUndefined()) {
567               bool is_cmpswap = instr->opcode == aco_opcode::image_atomic_cmpswap ||
568                                 instr->opcode == aco_opcode::image_atomic_fcmpswap;
569               check(instr->definitions.empty() ||
570                        (instr->definitions[0].regClass() == instr->operands[2].regClass() ||
571                         is_cmpswap),
572                     "MIMG operands[2] (VDATA) must be the same as definitions[0] for atomics and "
573                     "TFE/LWE loads",
574                     instr.get());
575            }
576            check(instr->operands.size() == 4 || program->chip_class >= GFX10,
577                  "NSA is only supported on GFX10+", instr.get());
578            for (unsigned i = 3; i < instr->operands.size(); i++) {
579               if (instr->operands.size() == 4) {
580                  check(instr->operands[i].hasRegClass() &&
581                           instr->operands[i].regClass().type() == RegType::vgpr,
582                        "MIMG operands[3] (VADDR) must be VGPR", instr.get());
583               } else {
584                  check(instr->operands[i].regClass() == v1, "MIMG VADDR must be v1 if NSA is used",
585                        instr.get());
586               }
587            }
588            check(instr->definitions.empty() ||
589                     (instr->definitions[0].isTemp() &&
590                      instr->definitions[0].regClass().type() == RegType::vgpr),
591                  "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
592            break;
593         }
594         case Format::DS: {
595            for (const Operand& op : instr->operands) {
596               check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
597                     "Only VGPRs are valid DS instruction operands", instr.get());
598            }
599            if (!instr->definitions.empty())
600               check(instr->definitions[0].getTemp().type() == RegType::vgpr,
601                     "DS instruction must return VGPR", instr.get());
602            break;
603         }
604         case Format::EXP: {
605            for (unsigned i = 0; i < 4; i++)
606               check(instr->operands[i].hasRegClass() &&
607                        instr->operands[i].regClass().type() == RegType::vgpr,
608                     "Only VGPRs are valid Export arguments", instr.get());
609            break;
610         }
611         case Format::FLAT:
612            check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR",
613                  instr.get());
614            FALLTHROUGH;
615         case Format::GLOBAL:
616         case Format::SCRATCH: {
617            check(
618               instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr,
619               "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
620            check(instr->operands[1].hasRegClass() &&
621                     instr->operands[1].regClass().type() == RegType::sgpr,
622                  "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
623            if (!instr->definitions.empty())
624               check(instr->definitions[0].getTemp().type() == RegType::vgpr,
625                     "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
626            else
627               check(instr->operands[2].regClass().type() == RegType::vgpr,
628                     "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
629            break;
630         }
631         default: break;
632         }
633      }
634   }
635
636   /* validate CFG */
637   for (unsigned i = 0; i < program->blocks.size(); i++) {
638      Block& block = program->blocks[i];
639      check_block(block.index == i, "block.index must match actual index", &block);
640
641      /* predecessors/successors should be sorted */
642      for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
643         check_block(block.linear_preds[j] < block.linear_preds[j + 1],
644                     "linear predecessors must be sorted", &block);
645      for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
646         check_block(block.logical_preds[j] < block.logical_preds[j + 1],
647                     "logical predecessors must be sorted", &block);
648      for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
649         check_block(block.linear_succs[j] < block.linear_succs[j + 1],
650                     "linear successors must be sorted", &block);
651      for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
652         check_block(block.logical_succs[j] < block.logical_succs[j + 1],
653                     "logical successors must be sorted", &block);
654
655      /* critical edges are not allowed */
656      if (block.linear_preds.size() > 1) {
657         for (unsigned pred : block.linear_preds)
658            check_block(program->blocks[pred].linear_succs.size() == 1,
659                        "linear critical edges are not allowed", &program->blocks[pred]);
660         for (unsigned pred : block.logical_preds)
661            check_block(program->blocks[pred].logical_succs.size() == 1,
662                        "logical critical edges are not allowed", &program->blocks[pred]);
663      }
664   }
665
666   return is_valid;
667}
668
669/* RA validation */
670namespace {
671
672struct Location {
673   Location() : block(NULL), instr(NULL) {}
674
675   Block* block;
676   Instruction* instr; // NULL if it's the block's live-in
677};
678
679struct Assignment {
680   Location defloc;
681   Location firstloc;
682   PhysReg reg;
683};
684
685bool
686ra_fail(Program* program, Location loc, Location loc2, const char* fmt, ...)
687{
688   va_list args;
689   va_start(args, fmt);
690   char msg[1024];
691   vsprintf(msg, fmt, args);
692   va_end(args);
693
694   char* out;
695   size_t outsize;
696   struct u_memstream mem;
697   u_memstream_open(&mem, &out, &outsize);
698   FILE* const memf = u_memstream_get(&mem);
699
700   fprintf(memf, "RA error found at instruction in BB%d:\n", loc.block->index);
701   if (loc.instr) {
702      aco_print_instr(loc.instr, memf);
703      fprintf(memf, "\n%s", msg);
704   } else {
705      fprintf(memf, "%s", msg);
706   }
707   if (loc2.block) {
708      fprintf(memf, " in BB%d:\n", loc2.block->index);
709      aco_print_instr(loc2.instr, memf);
710   }
711   fprintf(memf, "\n\n");
712   u_memstream_close(&mem);
713
714   aco_err(program, "%s", out);
715   free(out);
716
717   return true;
718}
719
720bool
721validate_subdword_operand(chip_class chip, const aco_ptr<Instruction>& instr, unsigned index)
722{
723   Operand op = instr->operands[index];
724   unsigned byte = op.physReg().byte();
725
726   if (instr->opcode == aco_opcode::p_as_uniform)
727      return byte == 0;
728   if (instr->isPseudo() && chip >= GFX8)
729      return true;
730   if (instr->isSDWA())
731      return byte + instr->sdwa().sel[index].offset() + instr->sdwa().sel[index].size() <= 4 &&
732             byte % instr->sdwa().sel[index].size() == 0;
733   if (instr->isVOP3P())
734      return ((instr->vop3p().opsel_lo >> index) & 1) == (byte >> 1) &&
735             ((instr->vop3p().opsel_hi >> index) & 1) == (byte >> 1);
736   if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1))
737      return true;
738
739   switch (instr->opcode) {
740   case aco_opcode::v_cvt_f32_ubyte1:
741      if (byte == 1)
742         return true;
743      break;
744   case aco_opcode::v_cvt_f32_ubyte2:
745      if (byte == 2)
746         return true;
747      break;
748   case aco_opcode::v_cvt_f32_ubyte3:
749      if (byte == 3)
750         return true;
751      break;
752   case aco_opcode::ds_write_b8_d16_hi:
753   case aco_opcode::ds_write_b16_d16_hi:
754      if (byte == 2 && index == 1)
755         return true;
756      break;
757   case aco_opcode::buffer_store_byte_d16_hi:
758   case aco_opcode::buffer_store_short_d16_hi:
759      if (byte == 2 && index == 3)
760         return true;
761      break;
762   case aco_opcode::flat_store_byte_d16_hi:
763   case aco_opcode::flat_store_short_d16_hi:
764   case aco_opcode::scratch_store_byte_d16_hi:
765   case aco_opcode::scratch_store_short_d16_hi:
766   case aco_opcode::global_store_byte_d16_hi:
767   case aco_opcode::global_store_short_d16_hi:
768      if (byte == 2 && index == 2)
769         return true;
770      break;
771   default: break;
772   }
773
774   return byte == 0;
775}
776
777bool
778validate_subdword_definition(chip_class chip, const aco_ptr<Instruction>& instr)
779{
780   Definition def = instr->definitions[0];
781   unsigned byte = def.physReg().byte();
782
783   if (instr->isPseudo() && chip >= GFX8)
784      return true;
785   if (instr->isSDWA())
786      return byte + instr->sdwa().dst_sel.offset() + instr->sdwa().dst_sel.size() <= 4 &&
787             byte % instr->sdwa().dst_sel.size() == 0;
788   if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1))
789      return true;
790
791   switch (instr->opcode) {
792   case aco_opcode::buffer_load_ubyte_d16_hi:
793   case aco_opcode::buffer_load_short_d16_hi:
794   case aco_opcode::flat_load_ubyte_d16_hi:
795   case aco_opcode::flat_load_short_d16_hi:
796   case aco_opcode::scratch_load_ubyte_d16_hi:
797   case aco_opcode::scratch_load_short_d16_hi:
798   case aco_opcode::global_load_ubyte_d16_hi:
799   case aco_opcode::global_load_short_d16_hi:
800   case aco_opcode::ds_read_u8_d16_hi:
801   case aco_opcode::ds_read_u16_d16_hi: return byte == 2;
802   default: break;
803   }
804
805   return byte == 0;
806}
807
808unsigned
809get_subdword_bytes_written(Program* program, const aco_ptr<Instruction>& instr, unsigned index)
810{
811   chip_class chip = program->chip_class;
812   Definition def = instr->definitions[index];
813
814   if (instr->isPseudo())
815      return chip >= GFX8 ? def.bytes() : def.size() * 4u;
816   if (instr->isVALU()) {
817      assert(def.bytes() <= 2);
818      if (instr->isSDWA())
819         return instr->sdwa().dst_sel.size();
820
821      if (instr_is_16bit(chip, instr->opcode))
822         return 2;
823
824      return 4;
825   }
826
827   switch (instr->opcode) {
828   case aco_opcode::buffer_load_ubyte_d16:
829   case aco_opcode::buffer_load_short_d16:
830   case aco_opcode::flat_load_ubyte_d16:
831   case aco_opcode::flat_load_short_d16:
832   case aco_opcode::scratch_load_ubyte_d16:
833   case aco_opcode::scratch_load_short_d16:
834   case aco_opcode::global_load_ubyte_d16:
835   case aco_opcode::global_load_short_d16:
836   case aco_opcode::ds_read_u8_d16:
837   case aco_opcode::ds_read_u16_d16:
838   case aco_opcode::buffer_load_ubyte_d16_hi:
839   case aco_opcode::buffer_load_short_d16_hi:
840   case aco_opcode::flat_load_ubyte_d16_hi:
841   case aco_opcode::flat_load_short_d16_hi:
842   case aco_opcode::scratch_load_ubyte_d16_hi:
843   case aco_opcode::scratch_load_short_d16_hi:
844   case aco_opcode::global_load_ubyte_d16_hi:
845   case aco_opcode::global_load_short_d16_hi:
846   case aco_opcode::ds_read_u8_d16_hi:
847   case aco_opcode::ds_read_u16_d16_hi: return program->dev.sram_ecc_enabled ? 4 : 2;
848   default: return def.size() * 4;
849   }
850}
851
852} /* end namespace */
853
854bool
855validate_ra(Program* program)
856{
857   if (!(debug_flags & DEBUG_VALIDATE_RA))
858      return false;
859
860   bool err = false;
861   aco::live live_vars = aco::live_var_analysis(program);
862   std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
863   uint16_t sgpr_limit = get_addr_sgpr_from_waves(program, program->num_waves);
864
865   std::map<unsigned, Assignment> assignments;
866   for (Block& block : program->blocks) {
867      Location loc;
868      loc.block = &block;
869      for (aco_ptr<Instruction>& instr : block.instructions) {
870         if (instr->opcode == aco_opcode::p_phi) {
871            for (unsigned i = 0; i < instr->operands.size(); i++) {
872               if (instr->operands[i].isTemp() &&
873                   instr->operands[i].getTemp().type() == RegType::sgpr &&
874                   instr->operands[i].isFirstKill())
875                  phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
876            }
877         }
878
879         loc.instr = instr.get();
880         for (unsigned i = 0; i < instr->operands.size(); i++) {
881            Operand& op = instr->operands[i];
882            if (!op.isTemp())
883               continue;
884            if (!op.isFixed())
885               err |= ra_fail(program, loc, Location(), "Operand %d is not assigned a register", i);
886            if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
887               err |=
888                  ra_fail(program, loc, assignments.at(op.tempId()).firstloc,
889                          "Operand %d has an inconsistent register assignment with instruction", i);
890            if ((op.getTemp().type() == RegType::vgpr &&
891                 op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) ||
892                (op.getTemp().type() == RegType::sgpr &&
893                 op.physReg() + op.size() > program->config->num_sgprs &&
894                 op.physReg() < sgpr_limit))
895               err |= ra_fail(program, loc, assignments.at(op.tempId()).firstloc,
896                              "Operand %d has an out-of-bounds register assignment", i);
897            if (op.physReg() == vcc && !program->needs_vcc)
898               err |= ra_fail(program, loc, Location(),
899                              "Operand %d fixed to vcc but needs_vcc=false", i);
900            if (op.regClass().is_subdword() &&
901                !validate_subdword_operand(program->chip_class, instr, i))
902               err |= ra_fail(program, loc, Location(), "Operand %d not aligned correctly", i);
903            if (!assignments[op.tempId()].firstloc.block)
904               assignments[op.tempId()].firstloc = loc;
905            if (!assignments[op.tempId()].defloc.block)
906               assignments[op.tempId()].reg = op.physReg();
907         }
908
909         for (unsigned i = 0; i < instr->definitions.size(); i++) {
910            Definition& def = instr->definitions[i];
911            if (!def.isTemp())
912               continue;
913            if (!def.isFixed())
914               err |=
915                  ra_fail(program, loc, Location(), "Definition %d is not assigned a register", i);
916            if (assignments[def.tempId()].defloc.block)
917               err |= ra_fail(program, loc, assignments.at(def.tempId()).defloc,
918                              "Temporary %%%d also defined by instruction", def.tempId());
919            if ((def.getTemp().type() == RegType::vgpr &&
920                 def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) ||
921                (def.getTemp().type() == RegType::sgpr &&
922                 def.physReg() + def.size() > program->config->num_sgprs &&
923                 def.physReg() < sgpr_limit))
924               err |= ra_fail(program, loc, assignments.at(def.tempId()).firstloc,
925                              "Definition %d has an out-of-bounds register assignment", i);
926            if (def.physReg() == vcc && !program->needs_vcc)
927               err |= ra_fail(program, loc, Location(),
928                              "Definition %d fixed to vcc but needs_vcc=false", i);
929            if (def.regClass().is_subdword() &&
930                !validate_subdword_definition(program->chip_class, instr))
931               err |= ra_fail(program, loc, Location(), "Definition %d not aligned correctly", i);
932            if (!assignments[def.tempId()].firstloc.block)
933               assignments[def.tempId()].firstloc = loc;
934            assignments[def.tempId()].defloc = loc;
935            assignments[def.tempId()].reg = def.physReg();
936         }
937      }
938   }
939
940   for (Block& block : program->blocks) {
941      Location loc;
942      loc.block = &block;
943
944      std::array<unsigned, 2048> regs; /* register file in bytes */
945      regs.fill(0);
946
947      std::set<Temp> live;
948      for (unsigned id : live_vars.live_out[block.index])
949         live.insert(Temp(id, program->temp_rc[id]));
950      /* remove killed p_phi sgpr operands */
951      for (Temp tmp : phi_sgpr_ops[block.index])
952         live.erase(tmp);
953
954      /* check live out */
955      for (Temp tmp : live) {
956         PhysReg reg = assignments.at(tmp.id()).reg;
957         for (unsigned i = 0; i < tmp.bytes(); i++) {
958            if (regs[reg.reg_b + i]) {
959               err |= ra_fail(program, loc, Location(),
960                              "Assignment of element %d of %%%d already taken by %%%d in live-out",
961                              i, tmp.id(), regs[reg.reg_b + i]);
962            }
963            regs[reg.reg_b + i] = tmp.id();
964         }
965      }
966      regs.fill(0);
967
968      for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
969         aco_ptr<Instruction>& instr = *it;
970
971         /* check killed p_phi sgpr operands */
972         if (instr->opcode == aco_opcode::p_logical_end) {
973            for (Temp tmp : phi_sgpr_ops[block.index]) {
974               PhysReg reg = assignments.at(tmp.id()).reg;
975               for (unsigned i = 0; i < tmp.bytes(); i++) {
976                  if (regs[reg.reg_b + i])
977                     err |= ra_fail(
978                        program, loc, Location(),
979                        "Assignment of element %d of %%%d already taken by %%%d in live-out", i,
980                        tmp.id(), regs[reg.reg_b + i]);
981               }
982               live.emplace(tmp);
983            }
984         }
985
986         for (const Definition& def : instr->definitions) {
987            if (!def.isTemp())
988               continue;
989            live.erase(def.getTemp());
990         }
991
992         /* don't count phi operands as live-in, since they are actually
993          * killed when they are copied at the predecessor */
994         if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
995            for (const Operand& op : instr->operands) {
996               if (!op.isTemp())
997                  continue;
998               live.insert(op.getTemp());
999            }
1000         }
1001      }
1002
1003      for (Temp tmp : live) {
1004         PhysReg reg = assignments.at(tmp.id()).reg;
1005         for (unsigned i = 0; i < tmp.bytes(); i++)
1006            regs[reg.reg_b + i] = tmp.id();
1007      }
1008
1009      for (aco_ptr<Instruction>& instr : block.instructions) {
1010         loc.instr = instr.get();
1011
1012         /* remove killed p_phi operands from regs */
1013         if (instr->opcode == aco_opcode::p_logical_end) {
1014            for (Temp tmp : phi_sgpr_ops[block.index]) {
1015               PhysReg reg = assignments.at(tmp.id()).reg;
1016               for (unsigned i = 0; i < tmp.bytes(); i++)
1017                  regs[reg.reg_b + i] = 0;
1018            }
1019         }
1020
1021         if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1022            for (const Operand& op : instr->operands) {
1023               if (!op.isTemp())
1024                  continue;
1025               if (op.isFirstKillBeforeDef()) {
1026                  for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1027                     regs[op.physReg().reg_b + j] = 0;
1028               }
1029            }
1030         }
1031
1032         for (unsigned i = 0; i < instr->definitions.size(); i++) {
1033            Definition& def = instr->definitions[i];
1034            if (!def.isTemp())
1035               continue;
1036            Temp tmp = def.getTemp();
1037            PhysReg reg = assignments.at(tmp.id()).reg;
1038            for (unsigned j = 0; j < tmp.bytes(); j++) {
1039               if (regs[reg.reg_b + j])
1040                  err |= ra_fail(
1041                     program, loc, assignments.at(regs[reg.reg_b + j]).defloc,
1042                     "Assignment of element %d of %%%d already taken by %%%d from instruction", i,
1043                     tmp.id(), regs[reg.reg_b + j]);
1044               regs[reg.reg_b + j] = tmp.id();
1045            }
1046            if (def.regClass().is_subdword() && def.bytes() < 4) {
1047               unsigned written = get_subdword_bytes_written(program, instr, i);
1048               /* If written=4, the instruction still might write the upper half. In that case, it's
1049                * the lower half that isn't preserved */
1050               for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) {
1051                  unsigned written_reg = reg.reg() * 4u + j;
1052                  if (regs[written_reg] && regs[written_reg] != def.tempId())
1053                     err |= ra_fail(program, loc, assignments.at(regs[written_reg]).defloc,
1054                                    "Assignment of element %d of %%%d overwrites the full register "
1055                                    "taken by %%%d from instruction",
1056                                    i, tmp.id(), regs[written_reg]);
1057               }
1058            }
1059         }
1060
1061         for (const Definition& def : instr->definitions) {
1062            if (!def.isTemp())
1063               continue;
1064            if (def.isKill()) {
1065               for (unsigned j = 0; j < def.getTemp().bytes(); j++)
1066                  regs[def.physReg().reg_b + j] = 0;
1067            }
1068         }
1069
1070         if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1071            for (const Operand& op : instr->operands) {
1072               if (!op.isTemp())
1073                  continue;
1074               if (op.isLateKill() && op.isFirstKill()) {
1075                  for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1076                     regs[op.physReg().reg_b + j] = 0;
1077               }
1078            }
1079         }
1080      }
1081   }
1082
1083   return err;
1084}
1085} // namespace aco
1086