Home | History | Annotate | Line # | Download | only in internal 
      1 #ifndef JEMALLOC_INTERNAL_BIN_INLINES_H
      2 #define JEMALLOC_INTERNAL_BIN_INLINES_H
      3 
      4 #include "jemalloc/internal/jemalloc_preamble.h"
      5 #include "jemalloc/internal/bin.h"
      6 #include "jemalloc/internal/bin_info.h"
      7 #include "jemalloc/internal/bitmap.h"
      8 #include "jemalloc/internal/div.h"
      9 #include "jemalloc/internal/edata.h"
     10 #include "jemalloc/internal/sc.h"
     11 
     12 /*
     13  * The dalloc bin info contains just the information that the common paths need
     14  * during tcache flushes.  By force-inlining these paths, and using local copies
     15  * of data (so that the compiler knows it's constant), we avoid a whole bunch of
     16  * redundant loads and stores by leaving this information in registers.
     17  */
     18 typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
     19 struct bin_dalloc_locked_info_s {
     20 	div_info_t div_info;
     21 	uint32_t   nregs;
     22 	uint64_t   ndalloc;
     23 };
     24 
     25 /* Find the region index of a pointer within a slab. */
     26 JEMALLOC_ALWAYS_INLINE size_t
     27 bin_slab_regind_impl(
     28     div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) {
     29 	size_t diff, regind;
     30 
     31 	/* Freeing a pointer outside the slab can cause assertion failure. */
     32 	assert((uintptr_t)ptr >= (uintptr_t)edata_addr_get(slab));
     33 	assert((uintptr_t)ptr < (uintptr_t)edata_past_get(slab));
     34 	/* Freeing an interior pointer can cause assertion failure. */
     35 	assert(((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab))
     36 	        % (uintptr_t)bin_infos[binind].reg_size
     37 	    == 0);
     38 
     39 	diff = (size_t)((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab));
     40 
     41 	/* Avoid doing division with a variable divisor. */
     42 	regind = div_compute(div_info, diff);
     43 	assert(regind < bin_infos[binind].nregs);
     44 	return regind;
     45 }
     46 
     47 JEMALLOC_ALWAYS_INLINE size_t
     48 bin_slab_regind(bin_dalloc_locked_info_t *info, szind_t binind,
     49     edata_t *slab, const void *ptr) {
     50 	size_t regind = bin_slab_regind_impl(
     51 	    &info->div_info, binind, slab, ptr);
     52 	return regind;
     53 }
     54 
     55 JEMALLOC_ALWAYS_INLINE void
     56 bin_dalloc_locked_begin(
     57     bin_dalloc_locked_info_t *info, szind_t binind) {
     58 	info->div_info = arena_binind_div_info[binind];
     59 	info->nregs = bin_infos[binind].nregs;
     60 	info->ndalloc = 0;
     61 }
     62 
     63 /*
     64  * Does the deallocation work associated with freeing a single pointer (a
     65  * "step") in between a bin_dalloc_locked begin and end call.
     66  *
     67  * Returns true if arena_slab_dalloc must be called on slab.  Doesn't do
     68  * stats updates, which happen during finish (this lets running counts get left
     69  * in a register).
     70  */
     71 JEMALLOC_ALWAYS_INLINE bool
     72 bin_dalloc_locked_step(tsdn_t *tsdn, bool is_auto, bin_t *bin,
     73     bin_dalloc_locked_info_t *info, szind_t binind, edata_t *slab,
     74     void *ptr) {
     75 	const bin_info_t *bin_info = &bin_infos[binind];
     76 	size_t            regind = bin_slab_regind(info, binind, slab, ptr);
     77 	slab_data_t      *slab_data = edata_slab_data_get(slab);
     78 
     79 	assert(edata_nfree_get(slab) < bin_info->nregs);
     80 	/* Freeing an unallocated pointer can cause assertion failure. */
     81 	assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind));
     82 
     83 	bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind);
     84 	edata_nfree_inc(slab);
     85 
     86 	if (config_stats) {
     87 		info->ndalloc++;
     88 	}
     89 
     90 	unsigned nfree = edata_nfree_get(slab);
     91 	if (nfree == bin_info->nregs) {
     92 		bin_dalloc_locked_handle_newly_empty(
     93 		    tsdn, is_auto, slab, bin);
     94 		return true;
     95 	} else if (nfree == 1 && slab != bin->slabcur) {
     96 		bin_dalloc_locked_handle_newly_nonempty(
     97 		    tsdn, is_auto, slab, bin);
     98 	}
     99 	return false;
    100 }
    101 
    102 JEMALLOC_ALWAYS_INLINE void
    103 bin_dalloc_locked_finish(tsdn_t *tsdn, bin_t *bin,
    104     bin_dalloc_locked_info_t *info) {
    105 	if (config_stats) {
    106 		bin->stats.ndalloc += info->ndalloc;
    107 		assert(bin->stats.curregs >= (size_t)info->ndalloc);
    108 		bin->stats.curregs -= (size_t)info->ndalloc;
    109 	}
    110 }
    111 
    112 #endif /* JEMALLOC_INTERNAL_BIN_INLINES_H */
    113