1/* -*- mesa-c++  -*-
2 *
3 * Copyright (c) 2018-2019 Collabora LTD
4 *
5 * Author: Gert Wollny <gert.wollny@collabora.com>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * on the rights to use, copy, modify, merge, publish, distribute, sub
11 * license, and/or sell copies of the Software, and to permit persons to whom
12 * the Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 */
26
27#include "sfn_debug.h"
28#include "sfn_value_gpr.h"
29#include "sfn_valuepool.h"
30
31#include <iostream>
32#include <queue>
33
34namespace r600 {
35
36using std::vector;
37using std::pair;
38using std::make_pair;
39using std::queue;
40
41ValuePool::ValuePool():
42   m_next_register_index(0),
43   current_temp_reg_index(0),
44   next_temp_reg_comp(4)
45{
46}
47
48PValue ValuePool::m_undef = Value::zero;
49
50GPRVector ValuePool::vec_from_nir(const nir_dest& dst, int num_components)
51{
52   std::array<PValue, 4> result;
53   for (int i = 0; i < 4; ++i)
54      result[i] = from_nir(dst, i < num_components ? i : 7);
55   return GPRVector(result);
56}
57
58std::vector<PValue> ValuePool::varvec_from_nir(const nir_dest& dst, int num_components)
59{
60   std::vector<PValue> result(num_components);
61   for (int i = 0; i < num_components; ++i)
62      result[i] = from_nir(dst, i);
63   return result;
64}
65
66
67std::vector<PValue> ValuePool::varvec_from_nir(const nir_src& src, int num_components)
68{
69   std::vector<PValue> result(num_components);
70   int i;
71   for (i = 0; i < num_components; ++i)
72      result[i] = from_nir(src, i);
73
74   return result;
75}
76
77
78PValue ValuePool::from_nir(const nir_src& v, unsigned component, unsigned swizzled)
79{
80   sfn_log << SfnLog::reg << "Search " << (v.is_ssa ? "ssa_reg " : "reg ")
81           << (v.is_ssa ? v.ssa->index : v.reg.reg->index);
82
83   if (!v.is_ssa) {
84      int idx = lookup_register_index(v);
85      sfn_log << SfnLog::reg << "  -> got index " <<  idx << "\n";
86      if (idx >= 0) {
87         auto reg = lookup_register(idx, swizzled, false);
88         if (reg) {
89            if (reg->type() == Value::gpr_vector) {
90               auto& array = static_cast<GPRArray&>(*reg);
91               reg = array.get_indirect(v.reg.base_offset,
92                                        v.reg.indirect ?
93                                           from_nir(*v.reg.indirect, 0, 0) : nullptr,
94                                        component);
95            }
96            return reg;
97         }
98      }
99      assert(0 && "local registers should always be found");
100   }
101
102   unsigned index = v.ssa->index;
103   /* For undefs we use zero and let ()yet to be implemeneted dce deal with it */
104   if (m_ssa_undef.find(index) != m_ssa_undef.end())
105      return Value::zero;
106
107
108   int idx = lookup_register_index(v);
109   sfn_log << SfnLog::reg << "  -> got index " <<  idx << "\n";
110   if (idx >= 0) {
111      auto reg = lookup_register(idx, swizzled, false);
112      if (reg)
113         return reg;
114   }
115
116   auto literal_val = nir_src_as_const_value(v);
117   if (literal_val) {
118      assert(v.is_ssa);
119      switch (v.ssa->bit_size) {
120      case 1:
121         return PValue(new LiteralValue(literal_val[swizzled].b ? 0xffffffff : 0, component));
122      case 32:
123         return literal(literal_val[swizzled].u32);
124      default:
125         sfn_log << SfnLog::reg << "Unsupported bit size " << v.ssa->bit_size
126                 << " fall back to 32\n";
127         return PValue(new LiteralValue(literal_val[swizzled].u32, component));
128      }
129   }
130
131   return PValue();
132}
133
134PValue ValuePool::from_nir(const nir_src& v, unsigned component)
135{
136   return from_nir(v, component, component);
137}
138
139PValue ValuePool::from_nir(const nir_tex_src &v, unsigned component)
140{
141   return from_nir(v.src, component, component);
142}
143
144PValue ValuePool::from_nir(const nir_alu_src &v, unsigned component)
145{
146   return from_nir(v.src, component, v.swizzle[component]);
147}
148
149PGPRValue ValuePool::get_temp_register(int channel)
150{
151   /* Skip to next register to get the channel we want */
152   if (channel >= 0) {
153      if (next_temp_reg_comp <= channel)
154         next_temp_reg_comp = channel;
155      else
156         next_temp_reg_comp = 4;
157   }
158
159   if (next_temp_reg_comp > 3) {
160      current_temp_reg_index = allocate_temp_register();
161      next_temp_reg_comp = 0;
162   }
163   return std::make_shared<GPRValue>(current_temp_reg_index, next_temp_reg_comp++);
164}
165
166GPRVector ValuePool::get_temp_vec4(const GPRVector::Swizzle& swizzle)
167{
168   int sel = allocate_temp_register();
169   return GPRVector(sel, swizzle);
170}
171
172PValue ValuePool::create_register_from_nir_src(const nir_src& src, int comp)
173{
174   int idx = src.is_ssa ? get_dst_ssa_register_index(*src.ssa):
175                          get_local_register_index(*src.reg.reg);
176
177   auto retval = lookup_register(idx, comp, false);
178   if (!retval || retval->type() != Value::gpr || retval->type() != Value::gpr_array_value)
179      retval = create_register(idx, comp);
180   return retval;
181}
182
183PValue ValuePool::from_nir(const nir_alu_dest &v, unsigned component)
184{
185   //assert(v->write_mask & (1 << component));
186   return from_nir(v.dest, component);
187}
188
189int ValuePool::lookup_register_index(const nir_dest& dst)
190{
191   return dst.is_ssa ? get_dst_ssa_register_index(dst.ssa):
192                       get_local_register_index(*dst.reg.reg);
193}
194
195int ValuePool::lookup_register_index(const nir_src& src) const
196{
197   int index = 0;
198
199   index = src.is_ssa ?
200              get_ssa_register_index(*src.ssa) :
201              get_local_register_index(*src.reg.reg);
202
203   sfn_log << SfnLog::reg << " LIDX:" << index;
204
205   auto r = m_register_map.find(index);
206   if (r == m_register_map.end()) {
207      return -1;
208   }
209   return static_cast<int>(r->second.index);
210}
211
212
213int ValuePool::allocate_temp_register()
214{
215   return m_next_register_index++;
216}
217
218
219PValue ValuePool::from_nir(const nir_dest& v, unsigned component)
220{
221   int idx = lookup_register_index(v);
222   sfn_log << SfnLog::reg << __func__  << ": ";
223   if (v.is_ssa)
224      sfn_log << "ssa_" << v.ssa.index;
225   else
226      sfn_log << "r" << v.reg.reg->index;
227   sfn_log << " -> " << idx << "\n";
228
229   auto retval = lookup_register(idx, component, false);
230   if (!retval)
231      retval = create_register(idx, component);
232
233   if (retval->type() == Value::gpr_vector) {
234      assert(!v.is_ssa);
235      auto& array = static_cast<GPRArray&>(*retval);
236      retval = array.get_indirect(v.reg.base_offset,
237                                  v.reg.indirect ?
238                                  from_nir(*v.reg.indirect, 0, 0) : nullptr,
239                                  component);
240   }
241
242   return retval;
243}
244
245ValueMap ValuePool::get_temp_registers() const
246{
247   ValueMap result;
248
249   for (auto& v : m_registers) {
250      if (v.second->type() == Value::gpr)
251         result.insert(v.second);
252      else if (v.second->type() == Value::gpr_vector) {
253         auto& array = static_cast<GPRArray&>(*v.second);
254         array.collect_registers(result);
255      }
256   }
257   return result;
258}
259
260static const char swz[] = "xyzw01?_";
261
262PValue ValuePool::create_register(unsigned sel, unsigned swizzle)
263{
264   sfn_log << SfnLog::reg
265           <<"Create register " << sel  << '.' << swz[swizzle] << "\n";
266   auto retval = PValue(new GPRValue(sel, swizzle));
267   m_registers[(sel << 3) + swizzle] = retval;
268   return retval;
269}
270
271bool ValuePool::inject_register(unsigned sel, unsigned swizzle,
272                                const PValue& reg, bool map)
273{
274   uint32_t ssa_index = sel;
275
276   if (map) {
277      auto pos = m_ssa_register_map.find(sel);
278      if (pos == m_ssa_register_map.end())
279         ssa_index = m_next_register_index++;
280      else
281         ssa_index = pos->second;
282   }
283
284   sfn_log << SfnLog::reg
285           << "Inject register " << sel  << '.' << swz[swizzle]
286           << " at index " <<  ssa_index << " ...";
287
288   if (map)
289      m_ssa_register_map[sel] = ssa_index;
290
291   allocate_with_mask(ssa_index, swizzle, true);
292
293   unsigned idx = (ssa_index << 3) + swizzle;
294   auto p = m_registers.find(idx);
295   if ( (p != m_registers.end()) && *p->second != *reg) {
296      std::cerr << "Register location (" << ssa_index << ", " << swizzle << ") was already reserved\n";
297      assert(0);
298      return false;
299   }
300   sfn_log << SfnLog::reg << " at idx:" << idx << " to " << *reg << "\n";
301   m_registers[idx] = reg;
302
303   if (m_next_register_index <= ssa_index)
304      m_next_register_index = ssa_index + 1;
305   return true;
306}
307
308
309PValue ValuePool::lookup_register(unsigned sel, unsigned swizzle,
310                                  bool required)
311{
312
313   PValue retval;
314   sfn_log << SfnLog::reg
315           << "lookup register " << sel  << '.' << swz[swizzle] << "("
316           << ((sel << 3) + swizzle) << ")...";
317
318
319   auto reg = m_registers.find((sel << 3) + swizzle);
320   if (reg != m_registers.end()) {
321      sfn_log << SfnLog::reg << " -> Found " << *reg->second << "\n";
322      retval = reg->second;
323   } else if (swizzle == 7) {
324      PValue retval = create_register(sel, swizzle);
325      sfn_log << SfnLog::reg << " -> Created " << *retval << "\n";
326   } else if (required) {
327      sfn_log << SfnLog::reg << "Register (" << sel << ", "
328              << swizzle << ") not found but required\n";
329      assert(0 && "Unallocated register value requested\n");
330   }
331   sfn_log << SfnLog::reg << " -> Not required and not  allocated\n";
332   return retval;
333}
334
335unsigned ValuePool::get_dst_ssa_register_index(const nir_ssa_def& ssa)
336{
337   sfn_log << SfnLog::reg << __func__ << ": search dst ssa "
338           << ssa.index;
339
340   auto pos = m_ssa_register_map.find(ssa.index);
341   if (pos == m_ssa_register_map.end()) {
342      sfn_log << SfnLog::reg << " Need to allocate ...";
343      allocate_ssa_register(ssa);
344      pos = m_ssa_register_map.find(ssa.index);
345      assert(pos != m_ssa_register_map.end());
346   }
347   sfn_log << SfnLog::reg << "... got " << pos->second << "\n";
348   return pos->second;
349}
350
351unsigned ValuePool::get_ssa_register_index(const nir_ssa_def& ssa) const
352{
353   sfn_log << SfnLog::reg << __func__ << ": search ssa "
354           << ssa.index;
355
356   auto pos = m_ssa_register_map.find(ssa.index);
357   sfn_log << SfnLog::reg << " got " << pos->second<< "\n";
358   if (pos == m_ssa_register_map.end()) {
359      sfn_log << SfnLog::reg << __func__ << ": ssa register "
360              << ssa.index << " lookup failed\n";
361      return -1;
362   }
363   return pos->second;
364}
365
366unsigned ValuePool::get_local_register_index(const nir_register& reg)
367{
368   unsigned index = reg.index | 0x80000000;
369
370   auto pos = m_ssa_register_map.find(index);
371   if (pos == m_ssa_register_map.end()) {
372      allocate_local_register(reg);
373      pos = m_ssa_register_map.find(index);
374      assert(pos != m_ssa_register_map.end());
375   }
376   return pos->second;
377}
378
379unsigned ValuePool::get_local_register_index(const nir_register& reg) const
380{
381   unsigned index = reg.index | 0x80000000;
382   auto pos = m_ssa_register_map.find(index);
383   if (pos == m_ssa_register_map.end()) {
384      sfn_log << SfnLog::err << __func__ << ": local register "
385              << reg.index << " lookup failed";
386      return -1;
387   }
388   return pos->second;
389}
390
391void ValuePool::allocate_ssa_register(const nir_ssa_def& ssa)
392{
393   sfn_log << SfnLog::reg << "ValuePool: Allocate ssa register " << ssa.index
394           << " as " << m_next_register_index << "\n";
395   int index = m_next_register_index++;
396   m_ssa_register_map[ssa.index] = index;
397   allocate_with_mask(index, 0xf, true);
398}
399
400void ValuePool::allocate_arrays(array_list& arrays)
401{
402   int ncomponents = 0;
403   int current_index = m_next_register_index;
404   unsigned instance = 0;
405
406   while (!arrays.empty()) {
407      auto a = arrays.top();
408      arrays.pop();
409
410      /* This is a bit hackish, return an id that encodes the array merge. To make sure
411       * that the mapping doesn't go wrong we have to make sure the arrays is longer than
412       * the number of instances in this arrays slot */
413      if (a.ncomponents + ncomponents > 4 ||
414          a.length < instance) {
415         current_index = m_next_register_index;
416         ncomponents = 0;
417         instance = 0;
418      }
419
420      if (ncomponents == 0)
421         m_next_register_index += a.length;
422
423      uint32_t mask = ((1 << a.ncomponents) - 1) << ncomponents;
424
425      PGPRArray array = PGPRArray(new GPRArray(current_index, a.length, mask, ncomponents));
426
427      m_reg_arrays.push_back(array);
428
429      sfn_log << SfnLog::reg << "Add array at "<< current_index
430              << " of size " << a.length << " with " << a.ncomponents
431              << " components, mask " << mask << "\n";
432
433      m_ssa_register_map[a.index | 0x80000000] = current_index + instance;
434
435      for (unsigned  i = 0; i < a.ncomponents; ++i)
436         m_registers[((current_index  + instance) << 3) + i] = array;
437
438      VRec next_reg = {current_index + instance, mask, mask};
439      m_register_map[current_index + instance] = next_reg;
440
441      ncomponents += a.ncomponents;
442      ++instance;
443   }
444}
445
446void ValuePool::allocate_local_register(const nir_register& reg)
447{
448   int index = m_next_register_index++;
449   m_ssa_register_map[reg.index | 0x80000000] = index;
450   allocate_with_mask(index, 0xf, true);
451
452   /* Create actual register and map it */;
453   for (int i = 0; i < 4; ++i) {
454      int k = (index << 3) + i;
455      m_registers[k] = std::make_shared<GPRValue>(index, i);
456   }
457}
458
459void ValuePool::allocate_local_register(const nir_register& reg, array_list& arrays)
460{
461   sfn_log << SfnLog::reg << "ValuePool: Allocate local register " << reg.index
462           << " as " << m_next_register_index << "\n";
463
464   if (reg.num_array_elems) {
465      array_entry ae = {reg.index, reg.num_array_elems, reg.num_components};
466      arrays.push(ae);
467   }
468   else
469      allocate_local_register(reg);
470}
471
472bool ValuePool::create_undef(nir_ssa_undef_instr* instr)
473{
474   m_ssa_undef.insert(instr->def.index);
475   return true;
476}
477
478int ValuePool::allocate_with_mask(unsigned index, unsigned mask, bool pre_alloc)
479{
480   int retval;
481   VRec next_register = { index, mask };
482
483   sfn_log << SfnLog::reg << (pre_alloc ? "Pre-alloc" : "Allocate")
484           << " register (" << index << ", " << mask << ")\n";
485   retval = index;
486   auto r = m_register_map.find(index);
487
488   if (r != m_register_map.end()) {
489      if ((r->second.mask & next_register.mask) &&
490          !(r->second.pre_alloc_mask & next_register.mask)) {
491         std::cerr << "r600 ERR: register ("
492                   << index << ", " << mask
493                   << ") already allocated as (" << r->second.index << ", "
494                   << r->second.mask << ", " << r->second.pre_alloc_mask
495                   << ") \n";
496         retval = -1;
497      } else {
498         r->second.mask |= next_register.mask;
499         if (pre_alloc)
500            r->second.pre_alloc_mask |= next_register.mask;
501         retval = r->second.index;
502      }
503   } else  {
504      if (pre_alloc)
505         next_register.pre_alloc_mask = mask;
506      m_register_map[index] = next_register;
507      retval = next_register.index;
508   }
509
510   sfn_log << SfnLog::reg << "Allocate register (" << index << "," << mask << ") in R"
511           << retval << "\n";
512
513   return retval;
514}
515
516PValue ValuePool::literal(uint32_t value)
517{
518   auto l = m_literals.find(value);
519   if (l != m_literals.end())
520      return l->second;
521
522   m_literals[value] = PValue(new LiteralValue(value));
523   return m_literals[value];
524}
525
526}
527