1/*
2 * Copyright (c) 2017 Lima Project
3 * Copyright (c) 2013 Connor Abbott
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a copy
6 * of this software and associated documentation files (the "Software"), to deal
7 * in the Software without restriction, including without limitation the rights
8 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 * copies of the Software, and to permit persons to whom the Software is
10 * furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 * THE SOFTWARE.
22 *
23 */
24
25#ifndef LIMA_IR_PP_PPIR_H
26#define LIMA_IR_PP_PPIR_H
27
28#include "util/u_math.h"
29#include "util/list.h"
30#include "util/set.h"
31
32#include "ir/lima_ir.h"
33
34typedef enum {
35   ppir_op_mov,
36   ppir_op_abs,
37   ppir_op_neg,
38   ppir_op_sat,
39   ppir_op_add,
40
41   ppir_op_ddx,
42   ppir_op_ddy,
43
44   ppir_op_mul,
45   ppir_op_rcp,
46
47   ppir_op_sin_lut,
48   ppir_op_cos_lut,
49
50   ppir_op_sum3,
51   ppir_op_sum4,
52
53   ppir_op_normalize2,
54   ppir_op_normalize3,
55   ppir_op_normalize4,
56
57   ppir_op_select,
58
59   ppir_op_sin,
60   ppir_op_cos,
61   ppir_op_tan,
62   ppir_op_asin,
63   ppir_op_acos,
64
65   ppir_op_atan,
66   ppir_op_atan2,
67   ppir_op_atan_pt1,
68   ppir_op_atan2_pt1,
69   ppir_op_atan_pt2,
70
71   ppir_op_exp,
72   ppir_op_log,
73   ppir_op_exp2,
74   ppir_op_log2,
75   ppir_op_sqrt,
76   ppir_op_rsqrt,
77
78   ppir_op_sign,
79   ppir_op_floor,
80   ppir_op_ceil,
81   ppir_op_fract,
82   ppir_op_mod,
83   ppir_op_min,
84   ppir_op_max,
85   ppir_op_trunc,
86
87   ppir_op_and,
88   ppir_op_or,
89   ppir_op_xor,
90
91   ppir_op_lt,
92   ppir_op_gt,
93   ppir_op_le,
94   ppir_op_ge,
95   ppir_op_eq,
96   ppir_op_ne,
97   ppir_op_not,
98
99   ppir_op_load_uniform,
100   ppir_op_load_varying,
101   ppir_op_load_coords,
102   ppir_op_load_coords_reg,
103   ppir_op_load_fragcoord,
104   ppir_op_load_pointcoord,
105   ppir_op_load_frontface,
106   ppir_op_load_texture,
107   ppir_op_load_temp,
108
109   ppir_op_store_temp,
110
111   ppir_op_const,
112
113   ppir_op_discard,
114   ppir_op_branch,
115
116   ppir_op_undef,
117   ppir_op_dummy,
118
119   ppir_op_num,
120} ppir_op;
121
122typedef enum {
123   ppir_node_type_alu,
124   ppir_node_type_const,
125   ppir_node_type_load,
126   ppir_node_type_store,
127   ppir_node_type_load_texture,
128   ppir_node_type_discard,
129   ppir_node_type_branch,
130} ppir_node_type;
131
132typedef struct {
133   char *name;
134   ppir_node_type type;
135   int *slots;
136} ppir_op_info;
137
138extern const ppir_op_info ppir_op_infos[];
139
140typedef enum {
141   ppir_dep_src,
142   ppir_dep_write_after_read,
143   ppir_dep_sequence,
144} ppir_dep_type;
145
146typedef struct {
147   void *pred, *succ;
148   ppir_dep_type type;
149   struct list_head pred_link;
150   struct list_head succ_link;
151} ppir_dep;
152
153typedef struct ppir_node {
154   struct list_head list;
155   struct list_head sched_list;
156   ppir_op op;
157   ppir_node_type type;
158   int index;
159   char name[16];
160   bool printed;
161   struct ppir_instr *instr;
162   int instr_pos;
163   struct ppir_block *block;
164   bool is_end;
165   bool succ_different_block;
166
167   /* for scheduler */
168   struct list_head succ_list;
169   struct list_head pred_list;
170} ppir_node;
171
172typedef enum {
173   ppir_pipeline_reg_const0,
174   ppir_pipeline_reg_const1,
175   ppir_pipeline_reg_sampler,
176   ppir_pipeline_reg_uniform,
177   ppir_pipeline_reg_vmul,
178   ppir_pipeline_reg_fmul,
179   ppir_pipeline_reg_discard, /* varying load */
180} ppir_pipeline;
181
182typedef struct ppir_reg {
183   struct list_head list;
184   int index;
185   int regalloc_index;
186   int num_components;
187
188   /* whether this reg has to start from the x component
189    * of a full physical reg, this is true for reg used
190    * in load/store instr which has no swizzle field */
191   bool is_head;
192   bool spilled;
193   bool undef;
194} ppir_reg;
195
196typedef enum {
197   ppir_target_ssa,
198   ppir_target_pipeline,
199   ppir_target_register,
200} ppir_target;
201
202typedef struct ppir_src {
203   ppir_target type;
204   ppir_node *node;
205
206   union {
207      ppir_reg *ssa;
208      ppir_reg *reg;
209      ppir_pipeline pipeline;
210   };
211
212   uint8_t swizzle[4];
213   bool absolute, negate;
214} ppir_src;
215
216typedef enum {
217   ppir_outmod_none,
218   ppir_outmod_clamp_fraction,
219   ppir_outmod_clamp_positive,
220   ppir_outmod_round,
221} ppir_outmod;
222
223typedef struct ppir_dest {
224   ppir_target type;
225
226   union {
227      ppir_reg ssa;
228      ppir_reg *reg;
229      ppir_pipeline pipeline;
230   };
231
232   ppir_outmod modifier;
233   unsigned write_mask : 4;
234} ppir_dest;
235
236typedef struct {
237   ppir_node node;
238   ppir_dest dest;
239   ppir_src src[3];
240   int num_src;
241   int shift : 3; /* Only used for ppir_op_mul */
242} ppir_alu_node;
243
244typedef struct ppir_const {
245   union fi value[4];
246   int num;
247} ppir_const;
248
249typedef struct {
250   ppir_node node;
251   ppir_const constant;
252   ppir_dest dest;
253} ppir_const_node;
254
255typedef struct {
256   ppir_node node;
257   int index;
258   int num_components;
259   ppir_dest dest;
260   ppir_src src;
261   int num_src;
262} ppir_load_node;
263
264typedef struct {
265   ppir_node node;
266   int index;
267   int num_components;
268   ppir_src src;
269} ppir_store_node;
270
271typedef struct {
272   ppir_node node;
273   ppir_dest dest;
274   ppir_src src[2];
275   int num_src;
276   int sampler;
277   int sampler_dim;
278   bool lod_bias_en;
279   bool explicit_lod;
280} ppir_load_texture_node;
281
282typedef struct {
283   ppir_node node;
284} ppir_discard_node;
285
286enum ppir_instr_slot {
287   PPIR_INSTR_SLOT_VARYING,
288   PPIR_INSTR_SLOT_TEXLD,
289   PPIR_INSTR_SLOT_UNIFORM,
290   PPIR_INSTR_SLOT_ALU_VEC_MUL,
291   PPIR_INSTR_SLOT_ALU_SCL_MUL,
292   PPIR_INSTR_SLOT_ALU_VEC_ADD,
293   PPIR_INSTR_SLOT_ALU_SCL_ADD,
294   PPIR_INSTR_SLOT_ALU_COMBINE,
295   PPIR_INSTR_SLOT_STORE_TEMP,
296   PPIR_INSTR_SLOT_BRANCH,
297   PPIR_INSTR_SLOT_NUM,
298   PPIR_INSTR_SLOT_END,
299   PPIR_INSTR_SLOT_ALU_START = PPIR_INSTR_SLOT_ALU_VEC_MUL,
300   PPIR_INSTR_SLOT_ALU_END = PPIR_INSTR_SLOT_ALU_COMBINE,
301};
302
303typedef struct ppir_instr {
304   struct list_head list;
305   int index;
306   bool printed;
307   int seq; /* command sequence after schedule */
308
309   ppir_node *slots[PPIR_INSTR_SLOT_NUM];
310   ppir_const constant[2];
311   bool is_end;
312
313   /* for scheduler */
314   struct list_head succ_list;
315   struct list_head pred_list;
316   float reg_pressure;
317   int est; /* earliest start time */
318   int parent_index;
319   bool scheduled;
320   int offset;
321   int encode_size;
322
323   /* for liveness analysis */
324   BITSET_WORD *live_set;
325   uint8_t *live_mask; /* mask for non-ssa registers */
326   /* live_internal is to mark registers only live within an
327    * instruction, without propagation */
328   BITSET_WORD *live_internal;
329} ppir_instr;
330
331typedef struct ppir_block {
332   struct list_head list;
333   struct list_head node_list;
334   struct list_head instr_list;
335
336   struct ppir_block *successors[2];
337
338   struct ppir_compiler *comp;
339
340   /* for scheduler */
341   int sched_instr_index;
342   int sched_instr_base;
343   int index;
344} ppir_block;
345
346typedef struct {
347   ppir_node node;
348   ppir_src src[2];
349   int num_src;
350   bool cond_gt;
351   bool cond_eq;
352   bool cond_lt;
353   bool negate;
354   ppir_block *target;
355} ppir_branch_node;
356
357struct ra_regs;
358struct lima_fs_compiled_shader;
359
360typedef struct ppir_compiler {
361   struct list_head block_list;
362   struct hash_table_u64 *blocks;
363   int cur_index;
364   int cur_instr_index;
365
366   struct list_head reg_list;
367   int reg_num;
368
369   /* array for searching ssa/reg node */
370   ppir_node **var_nodes;
371   unsigned reg_base;
372
373   struct ra_regs *ra;
374   struct lima_fs_compiled_shader *prog;
375   bool uses_discard;
376
377   /* for scheduler */
378   int sched_instr_base;
379
380   /* for regalloc spilling debug */
381   int force_spilling;
382
383   /* shaderdb */
384   int num_loops;
385   int num_spills;
386   int num_fills;
387
388   ppir_block *discard_block;
389   ppir_block *current_block;
390   ppir_block *loop_break_block;
391   ppir_block *loop_cont_block;
392} ppir_compiler;
393
394void *ppir_node_create(ppir_block *block, ppir_op op, int index, unsigned mask);
395void ppir_node_add_dep(ppir_node *succ, ppir_node *pred, ppir_dep_type type);
396void ppir_node_remove_dep(ppir_dep *dep);
397void ppir_node_delete(ppir_node *node);
398void ppir_node_print_prog(ppir_compiler *comp);
399void ppir_node_replace_child(ppir_node *parent, ppir_node *old_child, ppir_node *new_child);
400void ppir_node_replace_all_succ(ppir_node *dst, ppir_node *src);
401void ppir_node_replace_pred(ppir_dep *dep, ppir_node *new_pred);
402ppir_dep *ppir_dep_for_pred(ppir_node *node, ppir_node *pred);
403/* Assumes that node successors are in the same block */
404ppir_node *ppir_node_insert_mov(ppir_node *node);
405
406static inline bool ppir_node_is_root(ppir_node *node)
407{
408   return list_is_empty(&node->succ_list);
409}
410
411static inline bool ppir_node_is_leaf(ppir_node *node)
412{
413   return list_is_empty(&node->pred_list);
414}
415
416static inline bool ppir_node_has_single_succ(ppir_node *node)
417{
418   return list_is_singular(&node->succ_list)
419      && !node->succ_different_block;
420}
421
422bool ppir_node_has_single_src_succ(ppir_node *node);
423
424static inline ppir_node *ppir_node_first_succ(ppir_node *node)
425{
426   return list_first_entry(&node->succ_list, ppir_dep, succ_link)->succ;
427}
428
429static inline bool ppir_node_has_single_pred(ppir_node *node)
430{
431   return list_is_singular(&node->pred_list);
432}
433
434static inline ppir_node *ppir_node_first_pred(ppir_node *node)
435{
436   return list_first_entry(&node->pred_list, ppir_dep, pred_link)->pred;
437}
438
439#define ppir_node_foreach_succ(node, dep) \
440   list_for_each_entry(ppir_dep, dep, &node->succ_list, succ_link)
441#define ppir_node_foreach_succ_safe(node, dep) \
442   list_for_each_entry_safe(ppir_dep, dep, &node->succ_list, succ_link)
443#define ppir_node_foreach_pred(node, dep) \
444   list_for_each_entry(ppir_dep, dep, &node->pred_list, pred_link)
445#define ppir_node_foreach_pred_safe(node, dep) \
446   list_for_each_entry_safe(ppir_dep, dep, &node->pred_list, pred_link)
447
448#define ppir_node_to_alu(node) ((ppir_alu_node *)(node))
449#define ppir_node_to_const(node) ((ppir_const_node *)(node))
450#define ppir_node_to_load(node) ((ppir_load_node *)(node))
451#define ppir_node_to_store(node) ((ppir_store_node *)(node))
452#define ppir_node_to_load_texture(node) ((ppir_load_texture_node *)(node))
453#define ppir_node_to_discard(node) ((ppir_discard_node *)(node))
454#define ppir_node_to_branch(node) ((ppir_branch_node *)(node))
455
456static inline ppir_dest *ppir_node_get_dest(ppir_node *node)
457{
458   switch (node->type) {
459   case ppir_node_type_alu:
460      return &ppir_node_to_alu(node)->dest;
461   case ppir_node_type_load:
462      return &ppir_node_to_load(node)->dest;
463   case ppir_node_type_const:
464      return &ppir_node_to_const(node)->dest;
465   case ppir_node_type_load_texture:
466      return &ppir_node_to_load_texture(node)->dest;
467   default:
468      return NULL;
469   }
470}
471
472static inline int ppir_node_get_src_num(ppir_node *node)
473{
474   switch (node->type) {
475   case ppir_node_type_alu:
476      return ppir_node_to_alu(node)->num_src;
477   case ppir_node_type_branch:
478      return ppir_node_to_branch(node)->num_src;
479   case ppir_node_type_load:
480      return ppir_node_to_load(node)->num_src;
481   case ppir_node_type_load_texture:
482      return ppir_node_to_load_texture(node)->num_src;
483   case ppir_node_type_store:
484      return 1;
485   default:
486      return 0;
487   }
488
489   return 0;
490}
491
492static inline ppir_src *ppir_node_get_src(ppir_node *node, int idx)
493{
494   if (idx < 0 || idx >= ppir_node_get_src_num(node))
495      return NULL;
496
497   switch (node->type) {
498   case ppir_node_type_alu:
499      return &ppir_node_to_alu(node)->src[idx];
500   case ppir_node_type_branch:
501      return &ppir_node_to_branch(node)->src[idx];
502   case ppir_node_type_load_texture:
503      return &ppir_node_to_load_texture(node)->src[idx];
504   case ppir_node_type_load:
505      return &ppir_node_to_load(node)->src;
506   case ppir_node_type_store:
507      return &ppir_node_to_store(node)->src;
508   default:
509      break;
510   }
511
512   return NULL;
513}
514
515static inline ppir_reg *ppir_src_get_reg(ppir_src *src)
516{
517   switch (src->type) {
518   case ppir_target_ssa:
519      return src->ssa;
520   case ppir_target_register:
521      return src->reg;
522   default:
523      return NULL;
524   }
525}
526
527static inline ppir_reg *ppir_dest_get_reg(ppir_dest *dest)
528{
529   switch (dest->type) {
530   case ppir_target_ssa:
531      return &dest->ssa;
532   case ppir_target_register:
533      return dest->reg;
534   default:
535      return NULL;
536   }
537}
538
539static inline void ppir_node_target_assign(ppir_src *src, ppir_node *node)
540{
541   ppir_dest *dest = ppir_node_get_dest(node);
542   src->type = dest->type;
543   switch (src->type) {
544   case ppir_target_ssa:
545      src->ssa = &dest->ssa;
546      src->node = node;
547      break;
548   case ppir_target_register:
549      src->reg = dest->reg;
550      /* Registers can be assigned from multiple nodes, so don't keep
551       * pointer to the node here
552       */
553      src->node = NULL;
554      break;
555   case ppir_target_pipeline:
556      src->pipeline = dest->pipeline;
557      src->node = node;
558      break;
559   }
560}
561
562static inline bool ppir_node_target_equal(ppir_src *src, ppir_dest *dest)
563{
564   if (src->type != dest->type ||
565       (src->type == ppir_target_ssa && src->ssa != &dest->ssa) ||
566       (src->type == ppir_target_register && src->reg != dest->reg) ||
567       (src->type == ppir_target_pipeline && src->pipeline != dest->pipeline))
568      return false;
569
570   return true;
571}
572
573static inline int ppir_target_get_src_reg_index(ppir_src *src)
574{
575   switch (src->type) {
576   case ppir_target_ssa:
577      if (src->ssa)
578         return src->ssa->index;
579      break;
580   case ppir_target_register:
581      if (src->reg)
582         return src->reg->index;
583      break;
584   case ppir_target_pipeline:
585      if (src->pipeline == ppir_pipeline_reg_discard)
586         return 15 * 4;
587      return (src->pipeline + 12) * 4;
588   }
589
590   return -1;
591}
592
593static inline int ppir_target_get_dest_reg_index(ppir_dest *dest)
594{
595   switch (dest->type) {
596   case ppir_target_ssa:
597      return dest->ssa.index;
598   case ppir_target_register:
599      return dest->reg->index;
600   case ppir_target_pipeline:
601      if (dest->pipeline == ppir_pipeline_reg_discard)
602         return 15 * 4;
603      return (dest->pipeline + 12) * 4;
604   }
605
606   return -1;
607}
608
609static inline int ppir_src_get_mask(ppir_src *src)
610{
611   ppir_reg *reg = ppir_src_get_reg(src);
612   int mask = 0;
613
614   for (int i = 0; i < reg->num_components; i++)
615      mask |= (1 << src->swizzle[i]);
616
617   return mask;
618}
619
620static inline bool ppir_target_is_scalar(ppir_dest *dest)
621{
622   switch (dest->type) {
623   case ppir_target_ssa:
624      return dest->ssa.num_components == 1;
625   case ppir_target_register:
626      /* only one bit in mask is set */
627      if ((dest->write_mask & 0x3) == 0x3 ||
628          (dest->write_mask & 0x5) == 0x5 ||
629          (dest->write_mask & 0x9) == 0x9 ||
630          (dest->write_mask & 0x6) == 0x6 ||
631          (dest->write_mask & 0xa) == 0xa ||
632          (dest->write_mask & 0xc) == 0xc)
633         return false;
634      else
635         return true;
636   case ppir_target_pipeline:
637      if (dest->pipeline == ppir_pipeline_reg_fmul)
638         return true;
639      else
640         return false;
641   default:
642      return false;
643   }
644}
645
646static inline bool ppir_node_schedulable_slot(ppir_node *node,
647                                              enum ppir_instr_slot slot)
648{
649   int *slots = ppir_op_infos[node->op].slots;
650   for (int i = 0; slots[i] != PPIR_INSTR_SLOT_END; i++)
651      if (slots[i] == slot)
652         return true;
653
654   return false;
655}
656
657ppir_instr *ppir_instr_create(ppir_block *block);
658bool ppir_instr_insert_node(ppir_instr *instr, ppir_node *node);
659void ppir_instr_add_dep(ppir_instr *succ, ppir_instr *pred);
660void ppir_instr_print_list(ppir_compiler *comp);
661void ppir_instr_print_dep(ppir_compiler *comp);
662void ppir_instr_insert_mul_node(ppir_node *add, ppir_node *mul);
663
664#define ppir_instr_foreach_succ(instr, dep) \
665   list_for_each_entry(ppir_dep, dep, &instr->succ_list, succ_link)
666#define ppir_instr_foreach_succ_safe(instr, dep) \
667   list_for_each_entry_safe(ppir_dep, dep, &instr->succ_list, succ_link)
668#define ppir_instr_foreach_pred(instr, dep) \
669   list_for_each_entry(ppir_dep, dep, &instr->pred_list, pred_link)
670#define ppir_instr_foreach_pred_safe(instr, dep) \
671   list_for_each_entry_safe(ppir_dep, dep, &instr->pred_list, pred_link)
672
673static inline bool ppir_instr_is_root(ppir_instr *instr)
674{
675   return list_is_empty(&instr->succ_list);
676}
677
678static inline bool ppir_instr_is_leaf(ppir_instr *instr)
679{
680   return list_is_empty(&instr->pred_list);
681}
682
683bool ppir_lower_prog(ppir_compiler *comp);
684bool ppir_node_to_instr(ppir_compiler *comp);
685bool ppir_schedule_prog(ppir_compiler *comp);
686bool ppir_regalloc_prog(ppir_compiler *comp);
687bool ppir_codegen_prog(ppir_compiler *comp);
688void ppir_liveness_analysis(ppir_compiler *comp);
689
690static inline unsigned int reg_mask_size(unsigned int num_reg)
691{
692   return (num_reg + 1) / 2;
693}
694
695static inline uint8_t get_reg_mask(uint8_t *set, unsigned index)
696{
697   unsigned int i = index / 2;
698   unsigned int shift = index % 2 ? 4 : 0;
699   uint8_t mask = 0x0f << shift;
700   return (set[i] & mask) >> shift;
701}
702
703static inline void set_reg_mask(uint8_t *set, unsigned int index, uint8_t bits)
704{
705   unsigned int i = index / 2;
706   unsigned int shift = index % 2 ? 4 : 0;
707   uint8_t mask = 0x0f << shift;
708   set[i] &= ~mask;
709   set[i] |= (bits << shift);
710}
711
712#endif
713