Home | History | Annotate | Line # | Download | only in internal 
      1 #ifndef JEMALLOC_INTERNAL_PSSET_H
      2 #define JEMALLOC_INTERNAL_PSSET_H
      3 
      4 #include "jemalloc/internal/hpdata.h"
      5 
      6 /*
      7  * A page-slab set.  What the eset is to PAC, the psset is to HPA.  It maintains
      8  * a collection of page-slabs (the intent being that they are backed by
      9  * hugepages, or at least could be), and handles allocation and deallocation
     10  * requests.
     11  */
     12 
     13 /*
     14  * One more than the maximum pszind_t we will serve out of the HPA.
     15  * Practically, we expect only the first few to be actually used.  This
     16  * corresponds to a maximum size of of 512MB on systems with 4k pages and
     17  * SC_NGROUP == 4, which is already an unreasonably large maximum.  Morally, you
     18  * can think of this as being SC_NPSIZES, but there's no sense in wasting that
     19  * much space in the arena, making bitmaps that much larger, etc.
     20  */
     21 #define PSSET_NPSIZES 64
     22 
     23 /*
     24  * We keep two purge lists per page size class; one for hugified hpdatas (at
     25  * index 2*pszind), and one for the non-hugified hpdatas (at index 2*pszind +
     26  * 1).  This lets us implement a preference for purging non-hugified hpdatas
     27  * among similarly-dirty ones.
     28  * We reserve the last two indices for empty slabs, in that case purging
     29  * hugified ones (which are definitionally all waste) before non-hugified ones
     30  * (i.e. reversing the order).
     31  */
     32 #define PSSET_NPURGE_LISTS (2 * PSSET_NPSIZES)
     33 
     34 typedef struct psset_bin_stats_s psset_bin_stats_t;
     35 struct psset_bin_stats_s {
     36 	/* How many pageslabs are in this bin? */
     37 	size_t npageslabs;
     38 	/* Of them, how many pages are active? */
     39 	size_t nactive;
     40 	/* And how many are dirty? */
     41 	size_t ndirty;
     42 };
     43 
     44 typedef struct psset_stats_s psset_stats_t;
     45 struct psset_stats_s {
     46 	/*
     47 	 * The second index is huge stats; nonfull_slabs[pszind][0] contains
     48 	 * stats for the non-huge slabs in bucket pszind, while
     49 	 * nonfull_slabs[pszind][1] contains stats for the huge slabs.
     50 	 */
     51 	psset_bin_stats_t nonfull_slabs[PSSET_NPSIZES][2];
     52 
     53 	/*
     54 	 * Full slabs don't live in any edata heap, but we still track their
     55 	 * stats.
     56 	 */
     57 	psset_bin_stats_t full_slabs[2];
     58 
     59 	/* Empty slabs are similar. */
     60 	psset_bin_stats_t empty_slabs[2];
     61 };
     62 
     63 typedef struct psset_s psset_t;
     64 struct psset_s {
     65 	/*
     66 	 * The pageslabs, quantized by the size class of the largest contiguous
     67 	 * free run of pages in a pageslab.
     68 	 */
     69 	hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
     70 	/* Bitmap for which set bits correspond to non-empty heaps. */
     71 	fb_group_t pageslab_bitmap[FB_NGROUPS(PSSET_NPSIZES)];
     72 	/*
     73 	 * The sum of all bin stats in stats.  This lets us quickly answer
     74 	 * queries for the number of dirty, active, and retained pages in the
     75 	 * entire set.
     76 	 */
     77 	psset_bin_stats_t merged_stats;
     78 	psset_stats_t stats;
     79 	/*
     80 	 * Slabs with no active allocations, but which are allowed to serve new
     81 	 * allocations.
     82 	 */
     83 	hpdata_empty_list_t empty;
     84 	/*
     85 	 * Slabs which are available to be purged, ordered by how much we want
     86 	 * to purge them (with later indices indicating slabs we want to purge
     87 	 * more).
     88 	 */
     89 	hpdata_purge_list_t to_purge[PSSET_NPURGE_LISTS];
     90 	/* Bitmap for which set bits correspond to non-empty purge lists. */
     91 	fb_group_t purge_bitmap[FB_NGROUPS(PSSET_NPURGE_LISTS)];
     92 	/* Slabs which are available to be hugified. */
     93 	hpdata_hugify_list_t to_hugify;
     94 };
     95 
     96 void psset_init(psset_t *psset);
     97 void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src);
     98 
     99 /*
    100  * Begin or end updating the given pageslab's metadata.  While the pageslab is
    101  * being updated, it won't be returned from psset_fit calls.
    102  */
    103 void psset_update_begin(psset_t *psset, hpdata_t *ps);
    104 void psset_update_end(psset_t *psset, hpdata_t *ps);
    105 
    106 /* Analogous to the eset_fit; pick a hpdata to serve the request. */
    107 hpdata_t *psset_pick_alloc(psset_t *psset, size_t size);
    108 /* Pick one to purge. */
    109 hpdata_t *psset_pick_purge(psset_t *psset);
    110 /* Pick one to hugify. */
    111 hpdata_t *psset_pick_hugify(psset_t *psset);
    112 
    113 void psset_insert(psset_t *psset, hpdata_t *ps);
    114 void psset_remove(psset_t *psset, hpdata_t *ps);
    115 
    116 static inline size_t
    117 psset_npageslabs(psset_t *psset) {
    118 	return psset->merged_stats.npageslabs;
    119 }
    120 
    121 static inline size_t
    122 psset_nactive(psset_t *psset) {
    123 	return psset->merged_stats.nactive;
    124 }
    125 
    126 static inline size_t
    127 psset_ndirty(psset_t *psset) {
    128 	return psset->merged_stats.ndirty;
    129 }
    130 
    131 #endif /* JEMALLOC_INTERNAL_PSSET_H */
    132