Home | History | Annotate | Line # | Download | only in linux 
      1 /* Copyright (C) 2022-2024 Free Software Foundation, Inc.
      2    Contributed by Jakub Jelinek <jakub (at) redhat.com>.
      3 
      4    This file is part of the GNU Offloading and Multi Processing Library
      5    (libgomp).
      6 
      7    Libgomp is free software; you can redistribute it and/or modify it
      8    under the terms of the GNU General Public License as published by
      9    the Free Software Foundation; either version 3, or (at your option)
     10    any later version.
     11 
     12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
     13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
     15    more details.
     16 
     17    Under Section 7 of GPL version 3, you are granted additional
     18    permissions described in the GCC Runtime Library Exception, version
     19    3.1, as published by the Free Software Foundation.
     20 
     21    You should have received a copy of the GNU General Public License and
     22    a copy of the GCC Runtime Library Exception along with this program;
     23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
     24    <http://www.gnu.org/licenses/>.  */
     25 
     26 /* This file contains wrappers for the system allocation routines.  Most
     27    places in the OpenMP API do not make any provision for failure, so in
     28    general we cannot allow memory allocation to fail.  */
     29 
     30 #define _GNU_SOURCE
     31 #include "libgomp.h"
     32 #if defined(PLUGIN_SUPPORT) && defined(LIBGOMP_USE_PTHREADS)
     33 #define LIBGOMP_USE_MEMKIND
     34 #define LIBGOMP_USE_LIBNUMA
     35 #endif
     36 
     37 /* Implement malloc routines that can handle pinned memory on Linux.
     38 
     39    It's possible to use mlock on any heap memory, but using munlock is
     40    problematic if there are multiple pinned allocations on the same page.
     41    Tracking all that manually would be possible, but adds overhead. This may
     42    be worth it if there are a lot of small allocations getting pinned, but
     43    this seems less likely in a HPC application.
     44 
     45    Instead we optimize for large pinned allocations, and use mmap to ensure
     46    that two pinned allocations don't share the same page.  This also means
     47    that large allocations don't pin extra pages by being poorly aligned.  */
     48 
     49 #define _GNU_SOURCE
     50 #include <sys/mman.h>
     51 #include <string.h>
     52 #include "libgomp.h"
     53 #ifdef HAVE_INTTYPES_H
     54 # include <inttypes.h>  /* For PRIu64.  */
     55 #endif
     56 
     57 static void *
     58 linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin)
     59 {
     60   (void)memspace;
     61 
     62   if (pin)
     63     {
     64       /* Note that mmap always returns zeroed memory and is therefore also a
     65 	 suitable implementation of calloc.  */
     66       void *addr = mmap (NULL, size, PROT_READ | PROT_WRITE,
     67 			 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
     68       if (addr == MAP_FAILED)
     69 	return NULL;
     70 
     71       if (mlock (addr, size))
     72 	{
     73 #ifdef HAVE_INTTYPES_H
     74 	  gomp_debug (0, "libgomp: failed to pin %"PRIu64" bytes of"
     75 		      " memory (ulimit too low?)\n", (uint64_t) size);
     76 #else
     77 	  gomp_debug (0, "libgomp: failed to pin %lu bytes of"
     78 		      " memory (ulimit too low?)\n", (unsigned long) size);
     79 #endif
     80 	  munmap (addr, size);
     81 	  return NULL;
     82 	}
     83 
     84       return addr;
     85     }
     86   else
     87     return malloc (size);
     88 }
     89 
     90 static void *
     91 linux_memspace_calloc (omp_memspace_handle_t memspace, size_t size, int pin)
     92 {
     93   if (pin)
     94     return linux_memspace_alloc (memspace, size, pin);
     95   else
     96     return calloc (1, size);
     97 }
     98 
     99 static void
    100 linux_memspace_free (omp_memspace_handle_t memspace, void *addr, size_t size,
    101 		     int pin)
    102 {
    103   (void)memspace;
    104 
    105   if (pin)
    106     munmap (addr, size);
    107   else
    108     free (addr);
    109 }
    110 
    111 static void *
    112 linux_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
    113 			size_t oldsize, size_t size, int oldpin, int pin)
    114 {
    115   if (oldpin && pin)
    116     {
    117       void *newaddr = mremap (addr, oldsize, size, MREMAP_MAYMOVE);
    118       if (newaddr == MAP_FAILED)
    119 	return NULL;
    120 
    121       return newaddr;
    122     }
    123   else if (oldpin || pin)
    124     {
    125       void *newaddr = linux_memspace_alloc (memspace, size, pin);
    126       if (newaddr)
    127 	{
    128 	  memcpy (newaddr, addr, oldsize < size ? oldsize : size);
    129 	  linux_memspace_free (memspace, addr, oldsize, oldpin);
    130 	}
    131 
    132       return newaddr;
    133     }
    134   else
    135     return realloc (addr, size);
    136 }
    137 
    138 static int
    139 linux_memspace_validate (omp_memspace_handle_t, unsigned, int)
    140 {
    141   /* Everything should be accepted on Linux, including pinning.  */
    142   return 1;
    143 }
    144 
    145 #define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
    146   linux_memspace_alloc (MEMSPACE, SIZE, PIN)
    147 #define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
    148   linux_memspace_calloc (MEMSPACE, SIZE, PIN)
    149 #define MEMSPACE_REALLOC(MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN) \
    150   linux_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, SIZE, OLDPIN, PIN)
    151 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
    152   linux_memspace_free (MEMSPACE, ADDR, SIZE, PIN)
    153 #define MEMSPACE_VALIDATE(MEMSPACE, ACCESS, PIN) \
    154   linux_memspace_validate (MEMSPACE, ACCESS, PIN)
    155 
    156 #include "../../allocator.c"
    157