Home | History | Annotate | Line # | Download | only in internal 
      1 #ifndef JEMALLOC_INTERNAL_HPA_H
      2 #define JEMALLOC_INTERNAL_HPA_H
      3 
      4 #include "jemalloc/internal/exp_grow.h"
      5 #include "jemalloc/internal/hpa_hooks.h"
      6 #include "jemalloc/internal/hpa_opts.h"
      7 #include "jemalloc/internal/pai.h"
      8 #include "jemalloc/internal/psset.h"
      9 
     10 typedef struct hpa_central_s hpa_central_t;
     11 struct hpa_central_s {
     12 	/*
     13 	 * The mutex guarding most of the operations on the central data
     14 	 * structure.
     15 	 */
     16 	malloc_mutex_t mtx;
     17 	/*
     18 	 * Guards expansion of eden.  We separate this from the regular mutex so
     19 	 * that cheaper operations can still continue while we're doing the OS
     20 	 * call.
     21 	 */
     22 	malloc_mutex_t grow_mtx;
     23 	/*
     24 	 * Either NULL (if empty), or some integer multiple of a
     25 	 * hugepage-aligned number of hugepages.  We carve them off one at a
     26 	 * time to satisfy new pageslab requests.
     27 	 *
     28 	 * Guarded by grow_mtx.
     29 	 */
     30 	void *eden;
     31 	size_t eden_len;
     32 	/* Source for metadata. */
     33 	base_t *base;
     34 	/* Number of grow operations done on this hpa_central_t. */
     35 	uint64_t age_counter;
     36 
     37 	/* The HPA hooks. */
     38 	hpa_hooks_t hooks;
     39 };
     40 
     41 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
     42 struct hpa_shard_nonderived_stats_s {
     43 	/*
     44 	 * The number of times we've purged within a hugepage.
     45 	 *
     46 	 * Guarded by mtx.
     47 	 */
     48 	uint64_t npurge_passes;
     49 	/*
     50 	 * The number of individual purge calls we perform (which should always
     51 	 * be bigger than npurge_passes, since each pass purges at least one
     52 	 * extent within a hugepage.
     53 	 *
     54 	 * Guarded by mtx.
     55 	 */
     56 	uint64_t npurges;
     57 
     58 	/*
     59 	 * The number of times we've hugified a pageslab.
     60 	 *
     61 	 * Guarded by mtx.
     62 	 */
     63 	uint64_t nhugifies;
     64 	/*
     65 	 * The number of times we've dehugified a pageslab.
     66 	 *
     67 	 * Guarded by mtx.
     68 	 */
     69 	uint64_t ndehugifies;
     70 };
     71 
     72 /* Completely derived; only used by CTL. */
     73 typedef struct hpa_shard_stats_s hpa_shard_stats_t;
     74 struct hpa_shard_stats_s {
     75 	psset_stats_t psset_stats;
     76 	hpa_shard_nonderived_stats_t nonderived_stats;
     77 };
     78 
     79 typedef struct hpa_shard_s hpa_shard_t;
     80 struct hpa_shard_s {
     81 	/*
     82 	 * pai must be the first member; we cast from a pointer to it to a
     83 	 * pointer to the hpa_shard_t.
     84 	 */
     85 	pai_t pai;
     86 
     87 	/* The central allocator we get our hugepages from. */
     88 	hpa_central_t *central;
     89 	/* Protects most of this shard's state. */
     90 	malloc_mutex_t mtx;
     91 	/*
     92 	 * Guards the shard's access to the central allocator (preventing
     93 	 * multiple threads operating on this shard from accessing the central
     94 	 * allocator).
     95 	 */
     96 	malloc_mutex_t grow_mtx;
     97 	/* The base metadata allocator. */
     98 	base_t *base;
     99 
    100 	/*
    101 	 * This edata cache is the one we use when allocating a small extent
    102 	 * from a pageslab.  The pageslab itself comes from the centralized
    103 	 * allocator, and so will use its edata_cache.
    104 	 */
    105 	edata_cache_fast_t ecf;
    106 
    107 	psset_t psset;
    108 
    109 	/*
    110 	 * How many grow operations have occurred.
    111 	 *
    112 	 * Guarded by grow_mtx.
    113 	 */
    114 	uint64_t age_counter;
    115 
    116 	/* The arena ind we're associated with. */
    117 	unsigned ind;
    118 
    119 	/*
    120 	 * Our emap.  This is just a cache of the emap pointer in the associated
    121 	 * hpa_central.
    122 	 */
    123 	emap_t *emap;
    124 
    125 	/* The configuration choices for this hpa shard. */
    126 	hpa_shard_opts_t opts;
    127 
    128 	/*
    129 	 * How many pages have we started but not yet finished purging in this
    130 	 * hpa shard.
    131 	 */
    132 	size_t npending_purge;
    133 
    134 	/*
    135 	 * Those stats which are copied directly into the CTL-centric hpa shard
    136 	 * stats.
    137 	 */
    138 	hpa_shard_nonderived_stats_t stats;
    139 
    140 	/*
    141 	 * Last time we performed purge on this shard.
    142 	 */
    143 	nstime_t last_purge;
    144 };
    145 
    146 /*
    147  * Whether or not the HPA can be used given the current configuration.  This is
    148  * is not necessarily a guarantee that it backs its allocations by hugepages,
    149  * just that it can function properly given the system it's running on.
    150  */
    151 bool hpa_supported(void);
    152 bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
    153 bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
    154     base_t *base, edata_cache_t *edata_cache, unsigned ind,
    155     const hpa_shard_opts_t *opts);
    156 
    157 void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
    158 void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
    159     hpa_shard_stats_t *dst);
    160 
    161 /*
    162  * Notify the shard that we won't use it for allocations much longer.  Due to
    163  * the possibility of races, we don't actually prevent allocations; just flush
    164  * and disable the embedded edata_cache_small.
    165  */
    166 void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard);
    167 void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard);
    168 
    169 void hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard,
    170     bool deferral_allowed);
    171 void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
    172 
    173 /*
    174  * We share the fork ordering with the PA and arena prefork handling; that's why
    175  * these are 3 and 4 rather than 0 and 1.
    176  */
    177 void hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard);
    178 void hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard);
    179 void hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard);
    180 void hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard);
    181 
    182 #endif /* JEMALLOC_INTERNAL_HPA_H */
    183