Home | History | Annotate | Line # | Download | only in nvptx 
      1 /* Copyright (C) 2023-2024 Free Software Foundation, Inc.
      2 
      3    This file is part of the GNU Offloading and Multi Processing Library
      4    (libgomp).
      5 
      6    Libgomp is free software; you can redistribute it and/or modify it
      7    under the terms of the GNU General Public License as published by
      8    the Free Software Foundation; either version 3, or (at your option)
      9    any later version.
     10 
     11    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
     12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
     14    more details.
     15 
     16    Under Section 7 of GPL version 3, you are granted additional
     17    permissions described in the GCC Runtime Library Exception, version
     18    3.1, as published by the Free Software Foundation.
     19 
     20    You should have received a copy of the GNU General Public License and
     21    a copy of the GCC Runtime Library Exception along with this program;
     22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
     23    <http://www.gnu.org/licenses/>.  */
     24 
     25 /* The low-latency allocators use space reserved in .shared memory when the
     26    kernel is launched.  The heap is initialized in gomp_nvptx_main and all
     27    allocations are forgotten when the kernel exits.  Allocations to other
     28    memory spaces all use the system malloc syscall.
     29 
     30    The root heap descriptor is stored elsewhere in shared memory, and each
     31    free chunk contains a similar descriptor for the next free chunk in the
     32    chain.
     33 
     34    The descriptor is two 16-bit values: offset and size, which describe the
     35    location of a chunk of memory available for allocation. The offset is
     36    relative to the base of the heap.  The special value 0xffff, 0xffff
     37    indicates that the heap is locked.  The descriptor is encoded into a
     38    single 32-bit integer so that it may be easily accessed atomically.
     39 
     40    Memory is allocated to the first free chunk that fits.  The free chain
     41    is always stored in order of the offset to assist coalescing adjacent
     42    chunks.  */
     43 
     44 #include "libgomp.h"
     45 #include <stdlib.h>
     46 
     47 #define BASIC_ALLOC_PREFIX __nvptx_lowlat
     48 #include "../../basic-allocator.c"
     49 
     50 /* There should be some .shared space reserved for us.  There's no way to
     51    express this magic extern sizeless array in C so use asm.  */
     52 asm (".extern .shared .u8 __nvptx_lowlat_pool[];\n");
     53 
     54 static void *
     55 nvptx_memspace_alloc (omp_memspace_handle_t memspace, size_t size)
     56 {
     57   if (memspace == omp_low_lat_mem_space)
     58     {
     59       char *shared_pool;
     60       asm ("cvta.shared.u64\t%0, __nvptx_lowlat_pool;" : "=r" (shared_pool));
     61 
     62       return __nvptx_lowlat_alloc (shared_pool, size);
     63     }
     64   else
     65     return malloc (size);
     66 }
     67 
     68 static void *
     69 nvptx_memspace_calloc (omp_memspace_handle_t memspace, size_t size)
     70 {
     71   if (memspace == omp_low_lat_mem_space)
     72     {
     73       char *shared_pool;
     74       asm ("cvta.shared.u64\t%0, __nvptx_lowlat_pool;" : "=r" (shared_pool));
     75 
     76       return __nvptx_lowlat_calloc (shared_pool, size);
     77     }
     78   else
     79     return calloc (1, size);
     80 }
     81 
     82 static void
     83 nvptx_memspace_free (omp_memspace_handle_t memspace, void *addr, size_t size)
     84 {
     85   if (memspace == omp_low_lat_mem_space)
     86     {
     87       char *shared_pool;
     88       asm ("cvta.shared.u64\t%0, __nvptx_lowlat_pool;" : "=r" (shared_pool));
     89 
     90       __nvptx_lowlat_free (shared_pool, addr, size);
     91     }
     92   else
     93     free (addr);
     94 }
     95 
     96 static void *
     97 nvptx_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
     98 			size_t oldsize, size_t size)
     99 {
    100   if (memspace == omp_low_lat_mem_space)
    101     {
    102       char *shared_pool;
    103       asm ("cvta.shared.u64\t%0, __nvptx_lowlat_pool;" : "=r" (shared_pool));
    104 
    105       return __nvptx_lowlat_realloc (shared_pool, addr, oldsize, size);
    106     }
    107   else
    108     return realloc (addr, size);
    109 }
    110 
    111 static inline int
    112 nvptx_memspace_validate (omp_memspace_handle_t memspace, unsigned access)
    113 {
    114 #if __PTX_ISA_VERSION_MAJOR__ > 4 \
    115     || (__PTX_ISA_VERSION_MAJOR__ == 4 && __PTX_ISA_VERSION_MINOR >= 1)
    116   /* Disallow use of low-latency memory when it must be accessible by
    117      all threads.  */
    118   return (memspace != omp_low_lat_mem_space
    119 	  || access != omp_atv_all);
    120 #else
    121   /* Low-latency memory is not available before PTX 4.1.  */
    122   return (memspace != omp_low_lat_mem_space);
    123 #endif
    124 }
    125 
    126 #define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
    127   nvptx_memspace_alloc (MEMSPACE, ((void)(PIN), (SIZE)))
    128 #define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
    129   nvptx_memspace_calloc (MEMSPACE, ((void)(PIN), (SIZE)))
    130 #define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
    131   nvptx_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, \
    132 			  ((void)(OLDPIN), (void)(PIN), (SIZE)))
    133 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
    134   nvptx_memspace_free (MEMSPACE, ADDR, ((void)(PIN), (SIZE)))
    135 #define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \
    136   nvptx_memspace_validate (MEMSPACE, ((void)(PIN), (ACCESS)))
    137 
    138 /* The default low-latency memspace implies omp_atv_all, which is incompatible
    139    with the .shared memory space.  */
    140 #define OMP_LOW_LAT_MEM_ALLOC_INVALID 1
    141 
    142 #include "../../allocator.c"
    143