Home | History | Annotate | Line # | Download | only in isc
mem.c revision 1.17
      1  1.17  christos /*	$NetBSD: mem.c,v 1.17 2025/01/26 16:25:37 christos Exp $	*/
      2   1.1  christos 
      3   1.1  christos /*
      4   1.1  christos  * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
      5   1.1  christos  *
      6  1.12  christos  * SPDX-License-Identifier: MPL-2.0
      7  1.12  christos  *
      8   1.1  christos  * This Source Code Form is subject to the terms of the Mozilla Public
      9   1.1  christos  * License, v. 2.0. If a copy of the MPL was not distributed with this
     10   1.8  christos  * file, you can obtain one at https://mozilla.org/MPL/2.0/.
     11   1.1  christos  *
     12   1.1  christos  * See the COPYRIGHT file distributed with this work for additional
     13   1.1  christos  * information regarding copyright ownership.
     14   1.1  christos  */
     15   1.1  christos 
     16   1.1  christos /*! \file */
     17   1.1  christos 
     18   1.3  christos #include <inttypes.h>
     19   1.6  christos #include <limits.h>
     20   1.3  christos #include <stdbool.h>
     21   1.6  christos #include <stddef.h>
     22   1.1  christos #include <stdio.h>
     23   1.1  christos #include <stdlib.h>
     24   1.1  christos 
     25   1.3  christos #include <isc/hash.h>
     26   1.1  christos #include <isc/magic.h>
     27   1.1  christos #include <isc/mem.h>
     28   1.3  christos #include <isc/mutex.h>
     29   1.1  christos #include <isc/once.h>
     30  1.15  christos #include <isc/os.h>
     31  1.17  christos #include <isc/overflow.h>
     32   1.3  christos #include <isc/refcount.h>
     33  1.17  christos #include <isc/strerr.h>
     34   1.1  christos #include <isc/string.h>
     35  1.15  christos #include <isc/types.h>
     36  1.17  christos #include <isc/urcu.h>
     37   1.1  christos #include <isc/util.h>
     38   1.6  christos 
     39   1.6  christos #ifdef HAVE_LIBXML2
     40   1.6  christos #include <libxml/xmlwriter.h>
     41   1.6  christos #define ISC_XMLCHAR (const xmlChar *)
     42   1.6  christos #endif /* HAVE_LIBXML2 */
     43   1.6  christos 
     44   1.6  christos #ifdef HAVE_JSON_C
     45   1.6  christos #include <json_object.h>
     46   1.6  christos #endif /* HAVE_JSON_C */
     47   1.1  christos 
     48  1.15  christos /* On DragonFly BSD the header does not provide jemalloc API */
     49  1.15  christos #if defined(HAVE_MALLOC_NP_H) && !defined(__DragonFly__)
     50  1.15  christos #include <malloc_np.h>
     51  1.15  christos #define JEMALLOC_API_SUPPORTED 1
     52  1.15  christos #elif defined(HAVE_JEMALLOC)
     53  1.15  christos #include <jemalloc/jemalloc.h>
     54  1.15  christos #define JEMALLOC_API_SUPPORTED 1
     55  1.15  christos #else
     56  1.15  christos #include "jemalloc_shim.h"
     57  1.15  christos #endif
     58  1.15  christos 
     59   1.3  christos #include "mem_p.h"
     60   1.3  christos 
     61   1.6  christos #define MCTXLOCK(m)   LOCK(&m->lock)
     62   1.6  christos #define MCTXUNLOCK(m) UNLOCK(&m->lock)
     63   1.1  christos 
     64   1.1  christos #ifndef ISC_MEM_DEBUGGING
     65   1.1  christos #define ISC_MEM_DEBUGGING 0
     66   1.6  christos #endif /* ifndef ISC_MEM_DEBUGGING */
     67  1.15  christos unsigned int isc_mem_debugging = ISC_MEM_DEBUGGING;
     68  1.15  christos unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT;
     69  1.15  christos 
     70  1.15  christos #define ISC_MEM_ILLEGAL_ARENA (UINT_MAX)
     71   1.1  christos 
     72  1.16  christos volatile void *isc__mem_malloc = mallocx;
     73  1.16  christos 
     74   1.1  christos /*
     75   1.1  christos  * Constants.
     76   1.1  christos  */
     77   1.1  christos 
     78  1.15  christos #define ZERO_ALLOCATION_SIZE sizeof(void *)
     79  1.15  christos #define ALIGNMENT	     8U /*%< must be a power of 2 */
     80  1.15  christos #define ALIGNMENT_SIZE	     sizeof(size_info)
     81  1.15  christos #define DEBUG_TABLE_COUNT    512U
     82   1.1  christos 
     83   1.1  christos /*
     84   1.1  christos  * Types.
     85   1.1  christos  */
     86   1.1  christos #if ISC_MEM_TRACKLINES
     87   1.1  christos typedef struct debuglink debuglink_t;
     88   1.1  christos struct debuglink {
     89   1.6  christos 	ISC_LINK(debuglink_t) link;
     90   1.6  christos 	const void *ptr;
     91   1.6  christos 	size_t size;
     92   1.6  christos 	const char *file;
     93   1.6  christos 	unsigned int line;
     94   1.1  christos };
     95   1.1  christos 
     96   1.6  christos typedef ISC_LIST(debuglink_t) debuglist_t;
     97   1.1  christos 
     98   1.6  christos #define FLARG_PASS , file, line
     99   1.6  christos #define FLARG	   , const char *file, unsigned int line
    100   1.6  christos #else /* if ISC_MEM_TRACKLINES */
    101   1.1  christos #define FLARG_PASS
    102   1.1  christos #define FLARG
    103   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
    104   1.1  christos 
    105   1.1  christos typedef struct element element;
    106   1.1  christos struct element {
    107   1.6  christos 	element *next;
    108   1.1  christos };
    109   1.1  christos 
    110   1.6  christos #define MEM_MAGIC	 ISC_MAGIC('M', 'e', 'm', 'C')
    111   1.6  christos #define VALID_CONTEXT(c) ISC_MAGIC_VALID(c, MEM_MAGIC)
    112   1.1  christos 
    113   1.1  christos /* List of all active memory contexts. */
    114   1.1  christos 
    115  1.15  christos static ISC_LIST(isc_mem_t) contexts;
    116   1.1  christos 
    117  1.10  christos static isc_once_t init_once = ISC_ONCE_INIT;
    118  1.10  christos static isc_once_t shut_once = ISC_ONCE_INIT;
    119   1.6  christos static isc_mutex_t contextslock;
    120   1.1  christos 
    121  1.15  christos struct isc_mem {
    122  1.15  christos 	unsigned int magic;
    123   1.6  christos 	unsigned int flags;
    124  1.15  christos 	unsigned int jemalloc_flags;
    125  1.15  christos 	unsigned int jemalloc_arena;
    126  1.17  christos 	unsigned int debugging;
    127   1.6  christos 	isc_mutex_t lock;
    128   1.6  christos 	bool checkfree;
    129   1.6  christos 	isc_refcount_t references;
    130   1.6  christos 	char name[16];
    131  1.15  christos 	atomic_size_t inuse;
    132  1.15  christos 	atomic_bool hi_called;
    133  1.15  christos 	atomic_bool is_overmem;
    134  1.15  christos 	atomic_size_t hi_water;
    135  1.15  christos 	atomic_size_t lo_water;
    136  1.15  christos 	ISC_LIST(isc_mempool_t) pools;
    137   1.6  christos 	unsigned int poolcnt;
    138   1.1  christos 
    139   1.1  christos #if ISC_MEM_TRACKLINES
    140   1.6  christos 	debuglist_t *debuglist;
    141   1.6  christos 	size_t debuglistcnt;
    142   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
    143   1.1  christos 
    144  1.15  christos 	ISC_LINK(isc_mem_t) link;
    145   1.1  christos };
    146   1.1  christos 
    147   1.6  christos #define MEMPOOL_MAGIC	 ISC_MAGIC('M', 'E', 'M', 'p')
    148   1.6  christos #define VALID_MEMPOOL(c) ISC_MAGIC_VALID(c, MEMPOOL_MAGIC)
    149   1.1  christos 
    150  1.15  christos struct isc_mempool {
    151   1.1  christos 	/* always unlocked */
    152  1.15  christos 	unsigned int magic;
    153  1.15  christos 	isc_mem_t *mctx;	      /*%< our memory context */
    154  1.15  christos 	ISC_LINK(isc_mempool_t) link; /*%< next pool in this mem context */
    155  1.15  christos 	element *items;		      /*%< low water item list */
    156  1.15  christos 	size_t size;		      /*%< size of each item on this pool */
    157  1.15  christos 	size_t allocated;	      /*%< # of items currently given out */
    158  1.15  christos 	size_t freecount;	      /*%< # of items on reserved list */
    159  1.15  christos 	size_t freemax;		      /*%< # of items allowed on free list */
    160  1.15  christos 	size_t fillcount;	      /*%< # of items to fetch on each fill */
    161   1.1  christos 	/*%< Stats only. */
    162  1.15  christos 	size_t gets; /*%< # of requests to this pool */
    163  1.15  christos 	/*%< Debugging only. */
    164   1.6  christos 	char name[16]; /*%< printed name in stats reports */
    165   1.1  christos };
    166   1.1  christos 
    167   1.1  christos /*
    168   1.1  christos  * Private Inline-able.
    169   1.1  christos  */
    170   1.1  christos 
    171   1.6  christos #if !ISC_MEM_TRACKLINES
    172  1.17  christos #define ADD_TRACE(mctx, ptr, size, file, line)
    173  1.17  christos #define DELETE_TRACE(mctx, ptr, size, file, line)
    174   1.1  christos #define ISC_MEMFUNC_SCOPE
    175   1.6  christos #else /* if !ISC_MEM_TRACKLINES */
    176   1.6  christos #define TRACE_OR_RECORD (ISC_MEM_DEBUGTRACE | ISC_MEM_DEBUGRECORD)
    177  1.15  christos 
    178  1.17  christos #define SHOULD_TRACE_OR_RECORD(mctx, ptr) \
    179  1.17  christos 	(((mctx)->debugging & TRACE_OR_RECORD) != 0 && ptr != NULL)
    180  1.15  christos 
    181  1.17  christos #define ADD_TRACE(mctx, ptr, size, file, line)                \
    182  1.17  christos 	if (SHOULD_TRACE_OR_RECORD(mctx, ptr)) {              \
    183  1.17  christos 		add_trace_entry(mctx, ptr, size, file, line); \
    184  1.15  christos 	}
    185  1.15  christos 
    186  1.17  christos #define DELETE_TRACE(mctx, ptr, size, file, line)                \
    187  1.17  christos 	if (SHOULD_TRACE_OR_RECORD(mctx, ptr)) {                 \
    188  1.17  christos 		delete_trace_entry(mctx, ptr, size, file, line); \
    189  1.15  christos 	}
    190   1.1  christos 
    191   1.1  christos static void
    192  1.15  christos print_active(isc_mem_t *ctx, FILE *out);
    193  1.15  christos #endif /* ISC_MEM_TRACKLINES */
    194  1.15  christos 
    195   1.1  christos #if ISC_MEM_TRACKLINES
    196   1.1  christos /*!
    197  1.15  christos  * mctx must not be locked.
    198   1.1  christos  */
    199   1.1  christos static void
    200  1.15  christos add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) {
    201  1.15  christos 	debuglink_t *dl = NULL;
    202   1.3  christos 	uint32_t hash;
    203   1.3  christos 	uint32_t idx;
    204   1.1  christos 
    205  1.15  christos 	MCTXLOCK(mctx);
    206  1.15  christos 
    207  1.17  christos 	if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
    208   1.4  christos 		fprintf(stderr, "add %p size %zu file %s line %u mctx %p\n",
    209   1.1  christos 			ptr, size, file, line, mctx);
    210   1.4  christos 	}
    211   1.1  christos 
    212   1.6  christos 	if (mctx->debuglist == NULL) {
    213  1.15  christos 		goto unlock;
    214   1.6  christos 	}
    215   1.1  christos 
    216   1.6  christos #ifdef __COVERITY__
    217   1.6  christos 	/*
    218   1.6  christos 	 * Use simple conversion from pointer to hash to avoid
    219  1.17  christos 	 * tainting 'ptr' due to byte swap in isc_hash32.
    220   1.6  christos 	 */
    221   1.6  christos 	hash = (uintptr_t)ptr >> 3;
    222   1.6  christos #else
    223  1.17  christos 	hash = isc_hash32(&ptr, sizeof(ptr), true);
    224   1.6  christos #endif
    225   1.1  christos 	idx = hash % DEBUG_TABLE_COUNT;
    226   1.1  christos 
    227  1.17  christos 	dl = mallocx(sizeof(debuglink_t), mctx->jemalloc_flags);
    228   1.1  christos 	INSIST(dl != NULL);
    229   1.1  christos 
    230   1.1  christos 	ISC_LINK_INIT(dl, link);
    231   1.1  christos 	dl->ptr = ptr;
    232   1.1  christos 	dl->size = size;
    233   1.1  christos 	dl->file = file;
    234   1.1  christos 	dl->line = line;
    235   1.1  christos 
    236   1.1  christos 	ISC_LIST_PREPEND(mctx->debuglist[idx], dl, link);
    237   1.1  christos 	mctx->debuglistcnt++;
    238  1.15  christos unlock:
    239  1.15  christos 	MCTXUNLOCK(mctx);
    240   1.1  christos }
    241   1.1  christos 
    242   1.1  christos static void
    243  1.15  christos delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size,
    244   1.6  christos 		   const char *file, unsigned int line) {
    245  1.15  christos 	debuglink_t *dl = NULL;
    246   1.3  christos 	uint32_t hash;
    247   1.3  christos 	uint32_t idx;
    248   1.1  christos 
    249  1.15  christos 	MCTXLOCK(mctx);
    250  1.15  christos 
    251  1.17  christos 	if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
    252   1.4  christos 		fprintf(stderr, "del %p size %zu file %s line %u mctx %p\n",
    253   1.1  christos 			ptr, size, file, line, mctx);
    254   1.4  christos 	}
    255   1.1  christos 
    256   1.6  christos 	if (mctx->debuglist == NULL) {
    257  1.15  christos 		goto unlock;
    258   1.6  christos 	}
    259   1.1  christos 
    260   1.6  christos #ifdef __COVERITY__
    261   1.6  christos 	/*
    262   1.6  christos 	 * Use simple conversion from pointer to hash to avoid
    263  1.17  christos 	 * tainting 'ptr' due to byte swap in isc_hash32.
    264   1.6  christos 	 */
    265   1.6  christos 	hash = (uintptr_t)ptr >> 3;
    266   1.6  christos #else
    267  1.17  christos 	hash = isc_hash32(&ptr, sizeof(ptr), true);
    268   1.6  christos #endif
    269   1.1  christos 	idx = hash % DEBUG_TABLE_COUNT;
    270   1.1  christos 
    271   1.1  christos 	dl = ISC_LIST_HEAD(mctx->debuglist[idx]);
    272  1.15  christos 	while (dl != NULL) {
    273  1.15  christos 		if (dl->ptr == ptr) {
    274   1.1  christos 			ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link);
    275  1.15  christos 			sdallocx(dl, sizeof(*dl), mctx->jemalloc_flags);
    276  1.15  christos 			goto unlock;
    277   1.1  christos 		}
    278   1.1  christos 		dl = ISC_LIST_NEXT(dl, link);
    279   1.1  christos 	}
    280   1.1  christos 
    281   1.1  christos 	/*
    282   1.1  christos 	 * If we get here, we didn't find the item on the list.  We're
    283   1.1  christos 	 * screwed.
    284   1.1  christos 	 */
    285  1.12  christos 	UNREACHABLE();
    286  1.15  christos unlock:
    287  1.15  christos 	MCTXUNLOCK(mctx);
    288   1.1  christos }
    289   1.1  christos #endif /* ISC_MEM_TRACKLINES */
    290   1.1  christos 
    291  1.15  christos #define ADJUST_ZERO_ALLOCATION_SIZE(s)    \
    292  1.15  christos 	if (s == 0) {                     \
    293  1.15  christos 		s = ZERO_ALLOCATION_SIZE; \
    294   1.6  christos 	}
    295   1.1  christos 
    296  1.15  christos /*!
    297  1.15  christos  * Perform a malloc, doing memory filling and overrun detection as necessary.
    298  1.15  christos  */
    299  1.12  christos static void *
    300  1.15  christos mem_get(isc_mem_t *ctx, size_t size, int flags) {
    301  1.15  christos 	char *ret = NULL;
    302   1.1  christos 
    303  1.15  christos 	ADJUST_ZERO_ALLOCATION_SIZE(size);
    304   1.1  christos 
    305  1.15  christos 	ret = mallocx(size, flags | ctx->jemalloc_flags);
    306  1.15  christos 	INSIST(ret != NULL);
    307   1.1  christos 
    308  1.17  christos 	if ((flags & ISC__MEM_ZERO) == 0 &&
    309  1.17  christos 	    (ctx->flags & ISC_MEMFLAG_FILL) != 0)
    310  1.17  christos 	{
    311  1.15  christos 		memset(ret, 0xbe, size); /* Mnemonic for "beef". */
    312   1.6  christos 	}
    313   1.1  christos 
    314  1.17  christos 	return ret;
    315   1.1  christos }
    316   1.1  christos 
    317  1.15  christos /*!
    318  1.15  christos  * Perform a free, doing memory filling and overrun detection as necessary.
    319  1.15  christos  */
    320  1.15  christos /* coverity[+free : arg-1] */
    321  1.12  christos static void
    322  1.15  christos mem_put(isc_mem_t *ctx, void *mem, size_t size, int flags) {
    323  1.15  christos 	ADJUST_ZERO_ALLOCATION_SIZE(size);
    324   1.1  christos 
    325  1.15  christos 	if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) {
    326  1.15  christos 		memset(mem, 0xde, size); /* Mnemonic for "dead". */
    327   1.1  christos 	}
    328  1.15  christos 	sdallocx(mem, size, flags | ctx->jemalloc_flags);
    329   1.1  christos }
    330   1.1  christos 
    331  1.15  christos static void *
    332  1.15  christos mem_realloc(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size,
    333  1.15  christos 	    int flags) {
    334  1.15  christos 	void *new_ptr = NULL;
    335   1.1  christos 
    336  1.15  christos 	ADJUST_ZERO_ALLOCATION_SIZE(new_size);
    337   1.1  christos 
    338  1.15  christos 	new_ptr = rallocx(old_ptr, new_size, flags | ctx->jemalloc_flags);
    339  1.15  christos 	INSIST(new_ptr != NULL);
    340   1.1  christos 
    341  1.17  christos 	if ((flags & ISC__MEM_ZERO) == 0 &&
    342  1.17  christos 	    (ctx->flags & ISC_MEMFLAG_FILL) != 0)
    343  1.17  christos 	{
    344  1.15  christos 		ssize_t diff_size = new_size - old_size;
    345  1.15  christos 		void *diff_ptr = (uint8_t *)new_ptr + old_size;
    346  1.15  christos 		if (diff_size > 0) {
    347  1.15  christos 			/* Mnemonic for "beef". */
    348  1.15  christos 			memset(diff_ptr, 0xbe, diff_size);
    349   1.6  christos 		}
    350   1.1  christos 	}
    351   1.1  christos 
    352  1.17  christos 	return new_ptr;
    353   1.1  christos }
    354   1.1  christos 
    355   1.1  christos /*!
    356   1.1  christos  * Update internal counters after a memory get.
    357   1.1  christos  */
    358  1.12  christos static void
    359  1.15  christos mem_getstats(isc_mem_t *ctx, size_t size) {
    360  1.17  christos 	atomic_fetch_add_relaxed(&ctx->inuse, size);
    361   1.1  christos }
    362   1.1  christos 
    363   1.1  christos /*!
    364   1.1  christos  * Update internal counters after a memory put.
    365   1.1  christos  */
    366  1.12  christos static void
    367  1.17  christos mem_putstats(isc_mem_t *ctx, size_t size) {
    368  1.17  christos 	atomic_size_t s = atomic_fetch_sub_relaxed(&ctx->inuse, size);
    369  1.15  christos 	INSIST(s >= size);
    370   1.1  christos }
    371   1.1  christos 
    372   1.1  christos /*
    373   1.1  christos  * Private.
    374   1.1  christos  */
    375   1.1  christos 
    376  1.15  christos static bool
    377  1.15  christos mem_jemalloc_arena_create(unsigned int *pnew_arenano) {
    378  1.15  christos 	REQUIRE(pnew_arenano != NULL);
    379   1.3  christos 
    380  1.17  christos #ifdef JEMALLOC_API_SUPPORTED
    381  1.15  christos 	unsigned int arenano = 0;
    382  1.15  christos 	size_t len = sizeof(arenano);
    383  1.15  christos 	int res = 0;
    384   1.3  christos 
    385  1.15  christos 	res = mallctl("arenas.create", &arenano, &len, NULL, 0);
    386  1.15  christos 	if (res != 0) {
    387  1.17  christos 		return false;
    388  1.15  christos 	}
    389   1.3  christos 
    390  1.15  christos 	*pnew_arenano = arenano;
    391   1.3  christos 
    392  1.17  christos 	return true;
    393  1.15  christos #else
    394  1.15  christos 	*pnew_arenano = ISC_MEM_ILLEGAL_ARENA;
    395  1.17  christos 	return true;
    396  1.17  christos #endif /* JEMALLOC_API_SUPPORTED */
    397   1.1  christos }
    398   1.1  christos 
    399  1.15  christos static bool
    400  1.15  christos mem_jemalloc_arena_destroy(unsigned int arenano) {
    401  1.17  christos #ifdef JEMALLOC_API_SUPPORTED
    402  1.15  christos 	int res = 0;
    403  1.15  christos 	char buf[256] = { 0 };
    404  1.15  christos 
    405  1.15  christos 	(void)snprintf(buf, sizeof(buf), "arena.%u.destroy", arenano);
    406  1.15  christos 	res = mallctl(buf, NULL, NULL, NULL, 0);
    407  1.15  christos 	if (res != 0) {
    408  1.17  christos 		return false;
    409  1.15  christos 	}
    410  1.15  christos 
    411  1.17  christos 	return true;
    412  1.15  christos #else
    413  1.15  christos 	UNUSED(arenano);
    414  1.17  christos 	return true;
    415  1.17  christos #endif /* JEMALLOC_API_SUPPORTED */
    416   1.1  christos }
    417   1.1  christos 
    418   1.1  christos static void
    419  1.10  christos mem_initialize(void) {
    420  1.17  christos /*
    421  1.17  christos  * Check if the values copied from jemalloc still match
    422  1.17  christos  */
    423  1.17  christos #ifdef JEMALLOC_API_SUPPORTED
    424  1.17  christos 	RUNTIME_CHECK(ISC__MEM_ZERO == MALLOCX_ZERO);
    425  1.17  christos #endif /* JEMALLOC_API_SUPPORTED */
    426  1.17  christos 
    427   1.3  christos 	isc_mutex_init(&contextslock);
    428   1.1  christos 	ISC_LIST_INIT(contexts);
    429   1.1  christos }
    430   1.1  christos 
    431  1.10  christos void
    432  1.10  christos isc__mem_initialize(void) {
    433  1.17  christos 	isc_once_do(&init_once, mem_initialize);
    434  1.10  christos }
    435  1.10  christos 
    436  1.10  christos static void
    437  1.10  christos mem_shutdown(void) {
    438  1.17  christos 	bool empty;
    439  1.17  christos 
    440  1.10  christos 	isc__mem_checkdestroyed();
    441  1.10  christos 
    442  1.17  christos 	LOCK(&contextslock);
    443  1.17  christos 	empty = ISC_LIST_EMPTY(contexts);
    444  1.17  christos 	UNLOCK(&contextslock);
    445  1.17  christos 
    446  1.17  christos 	if (empty) {
    447  1.17  christos 		isc_mutex_destroy(&contextslock);
    448  1.17  christos 	}
    449  1.10  christos }
    450  1.10  christos 
    451  1.10  christos void
    452  1.10  christos isc__mem_shutdown(void) {
    453  1.17  christos 	isc_once_do(&shut_once, mem_shutdown);
    454  1.10  christos }
    455  1.10  christos 
    456   1.6  christos static void
    457  1.17  christos mem_create(isc_mem_t **ctxp, unsigned int debugging, unsigned int flags,
    458  1.17  christos 	   unsigned int jemalloc_flags) {
    459  1.15  christos 	isc_mem_t *ctx = NULL;
    460  1.15  christos 
    461   1.6  christos 	REQUIRE(ctxp != NULL && *ctxp == NULL);
    462  1.10  christos 
    463  1.17  christos 	ctx = mallocx(sizeof(*ctx), jemalloc_flags);
    464  1.15  christos 	INSIST(ctx != NULL);
    465  1.15  christos 
    466  1.15  christos 	*ctx = (isc_mem_t){
    467  1.15  christos 		.magic = MEM_MAGIC,
    468  1.17  christos 		.debugging = debugging,
    469  1.15  christos 		.flags = flags,
    470  1.15  christos 		.jemalloc_flags = jemalloc_flags,
    471  1.15  christos 		.jemalloc_arena = ISC_MEM_ILLEGAL_ARENA,
    472  1.15  christos 		.checkfree = true,
    473  1.15  christos 	};
    474   1.1  christos 
    475   1.6  christos 	isc_mutex_init(&ctx->lock);
    476  1.15  christos 	isc_refcount_init(&ctx->references, 1);
    477   1.1  christos 
    478  1.15  christos 	atomic_init(&ctx->inuse, 0);
    479  1.15  christos 	atomic_init(&ctx->hi_water, 0);
    480  1.15  christos 	atomic_init(&ctx->lo_water, 0);
    481  1.15  christos 	atomic_init(&ctx->hi_called, false);
    482  1.15  christos 	atomic_init(&ctx->is_overmem, false);
    483  1.15  christos 
    484   1.1  christos 	ISC_LIST_INIT(ctx->pools);
    485   1.1  christos 
    486   1.1  christos #if ISC_MEM_TRACKLINES
    487  1.17  christos 	if ((ctx->debugging & ISC_MEM_DEBUGRECORD) != 0) {
    488   1.1  christos 		unsigned int i;
    489   1.1  christos 
    490  1.17  christos 		ctx->debuglist = mallocx(
    491  1.17  christos 			ISC_CHECKED_MUL(DEBUG_TABLE_COUNT, sizeof(debuglist_t)),
    492  1.17  christos 			jemalloc_flags);
    493  1.15  christos 		INSIST(ctx->debuglist != NULL);
    494  1.15  christos 
    495   1.6  christos 		for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
    496   1.1  christos 			ISC_LIST_INIT(ctx->debuglist[i]);
    497   1.6  christos 		}
    498   1.1  christos 	}
    499   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
    500   1.1  christos 
    501   1.1  christos 	LOCK(&contextslock);
    502   1.1  christos 	ISC_LIST_INITANDAPPEND(contexts, ctx, link);
    503   1.1  christos 	UNLOCK(&contextslock);
    504   1.1  christos 
    505  1.15  christos 	*ctxp = ctx;
    506   1.6  christos }
    507   1.3  christos 
    508   1.6  christos /*
    509   1.6  christos  * Public.
    510   1.6  christos  */
    511   1.1  christos 
    512   1.1  christos static void
    513  1.15  christos destroy(isc_mem_t *ctx) {
    514  1.15  christos 	unsigned int arena_no;
    515   1.1  christos 	LOCK(&contextslock);
    516   1.1  christos 	ISC_LIST_UNLINK(contexts, ctx, link);
    517   1.1  christos 	UNLOCK(&contextslock);
    518   1.1  christos 
    519  1.15  christos 	ctx->magic = 0;
    520  1.15  christos 
    521  1.15  christos 	arena_no = ctx->jemalloc_arena;
    522   1.1  christos 
    523   1.1  christos 	INSIST(ISC_LIST_EMPTY(ctx->pools));
    524   1.1  christos 
    525   1.1  christos #if ISC_MEM_TRACKLINES
    526  1.15  christos 	if (ctx->debuglist != NULL) {
    527   1.1  christos 		debuglink_t *dl;
    528  1.17  christos 		for (size_t i = 0; i < DEBUG_TABLE_COUNT; i++) {
    529   1.6  christos 			for (dl = ISC_LIST_HEAD(ctx->debuglist[i]); dl != NULL;
    530   1.6  christos 			     dl = ISC_LIST_HEAD(ctx->debuglist[i]))
    531   1.6  christos 			{
    532   1.6  christos 				if (ctx->checkfree && dl->ptr != NULL) {
    533   1.1  christos 					print_active(ctx, stderr);
    534   1.6  christos 				}
    535   1.6  christos 				INSIST(!ctx->checkfree || dl->ptr == NULL);
    536   1.1  christos 
    537   1.6  christos 				ISC_LIST_UNLINK(ctx->debuglist[i], dl, link);
    538  1.15  christos 				sdallocx(dl, sizeof(*dl), ctx->jemalloc_flags);
    539   1.1  christos 			}
    540   1.6  christos 		}
    541   1.1  christos 
    542  1.17  christos 		sdallocx(
    543  1.17  christos 			ctx->debuglist,
    544  1.17  christos 			ISC_CHECKED_MUL(DEBUG_TABLE_COUNT, sizeof(debuglist_t)),
    545  1.17  christos 			ctx->jemalloc_flags);
    546   1.1  christos 	}
    547   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
    548   1.1  christos 
    549  1.15  christos 	isc_mutex_destroy(&ctx->lock);
    550  1.15  christos 
    551  1.15  christos 	if (ctx->checkfree) {
    552  1.17  christos 		INSIST(atomic_load(&ctx->inuse) == 0);
    553   1.1  christos 	}
    554  1.17  christos 	sdallocx(ctx, sizeof(*ctx), ctx->jemalloc_flags);
    555   1.1  christos 
    556  1.15  christos 	if (arena_no != ISC_MEM_ILLEGAL_ARENA) {
    557  1.15  christos 		RUNTIME_CHECK(mem_jemalloc_arena_destroy(arena_no) == true);
    558   1.6  christos 	}
    559   1.1  christos }
    560   1.1  christos 
    561   1.1  christos void
    562  1.15  christos isc_mem_attach(isc_mem_t *source, isc_mem_t **targetp) {
    563  1.15  christos 	REQUIRE(VALID_CONTEXT(source));
    564   1.6  christos 	REQUIRE(targetp != NULL && *targetp == NULL);
    565   1.6  christos 
    566   1.3  christos 	isc_refcount_increment(&source->references);
    567   1.1  christos 
    568  1.15  christos 	*targetp = source;
    569   1.1  christos }
    570   1.1  christos 
    571   1.1  christos void
    572  1.15  christos isc__mem_detach(isc_mem_t **ctxp FLARG) {
    573  1.15  christos 	isc_mem_t *ctx = NULL;
    574  1.15  christos 
    575   1.3  christos 	REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
    576   1.6  christos 
    577  1.15  christos 	ctx = *ctxp;
    578   1.3  christos 	*ctxp = NULL;
    579   1.1  christos 
    580   1.3  christos 	if (isc_refcount_decrement(&ctx->references) == 1) {
    581   1.3  christos 		isc_refcount_destroy(&ctx->references);
    582  1.15  christos #if ISC_MEM_TRACKLINES
    583  1.17  christos 		if ((ctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
    584  1.15  christos 			fprintf(stderr, "destroy mctx %p file %s line %u\n",
    585  1.15  christos 				ctx, file, line);
    586  1.15  christos 		}
    587  1.15  christos #endif
    588   1.1  christos 		destroy(ctx);
    589   1.3  christos 	}
    590   1.1  christos }
    591   1.1  christos 
    592   1.1  christos /*
    593   1.1  christos  * isc_mem_putanddetach() is the equivalent of:
    594   1.1  christos  *
    595   1.1  christos  * mctx = NULL;
    596   1.1  christos  * isc_mem_attach(ptr->mctx, &mctx);
    597   1.1  christos  * isc_mem_detach(&ptr->mctx);
    598   1.1  christos  * isc_mem_put(mctx, ptr, sizeof(*ptr);
    599   1.1  christos  * isc_mem_detach(&mctx);
    600   1.1  christos  */
    601   1.1  christos 
    602   1.1  christos void
    603  1.15  christos isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size,
    604  1.17  christos 		      int flags FLARG) {
    605   1.3  christos 	REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
    606   1.1  christos 	REQUIRE(ptr != NULL);
    607  1.15  christos 	REQUIRE(size != 0);
    608   1.6  christos 
    609  1.17  christos 	isc_mem_t *ctx = *ctxp;
    610   1.1  christos 	*ctxp = NULL;
    611   1.1  christos 
    612  1.17  christos 	isc__mem_put(ctx, ptr, size, flags FLARG_PASS);
    613  1.17  christos 	isc__mem_detach(&ctx FLARG_PASS);
    614   1.1  christos }
    615   1.1  christos 
    616   1.1  christos void
    617  1.15  christos isc__mem_destroy(isc_mem_t **ctxp FLARG) {
    618  1.15  christos 	isc_mem_t *ctx = NULL;
    619  1.15  christos 
    620   1.1  christos 	/*
    621   1.1  christos 	 * This routine provides legacy support for callers who use mctxs
    622   1.1  christos 	 * without attaching/detaching.
    623   1.1  christos 	 */
    624   1.1  christos 
    625   1.6  christos 	REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp));
    626   1.6  christos 
    627  1.15  christos 	ctx = *ctxp;
    628  1.15  christos 	*ctxp = NULL;
    629   1.1  christos 
    630  1.17  christos 	rcu_barrier();
    631  1.17  christos 
    632   1.1  christos #if ISC_MEM_TRACKLINES
    633  1.17  christos 	if ((ctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
    634  1.15  christos 		fprintf(stderr, "destroy mctx %p file %s line %u\n", ctx, file,
    635  1.15  christos 			line);
    636  1.15  christos 	}
    637  1.15  christos 
    638   1.6  christos 	if (isc_refcount_decrement(&ctx->references) > 1) {
    639   1.1  christos 		print_active(ctx, stderr);
    640   1.3  christos 	}
    641   1.6  christos #else  /* if ISC_MEM_TRACKLINES */
    642   1.8  christos 	isc_refcount_decrementz(&ctx->references);
    643   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
    644   1.3  christos 	isc_refcount_destroy(&ctx->references);
    645   1.1  christos 	destroy(ctx);
    646   1.1  christos 
    647   1.1  christos 	*ctxp = NULL;
    648   1.1  christos }
    649   1.1  christos 
    650  1.15  christos void *
    651  1.17  christos isc__mem_get(isc_mem_t *ctx, size_t size, int flags FLARG) {
    652  1.15  christos 	void *ptr = NULL;
    653   1.1  christos 
    654  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    655   1.1  christos 
    656  1.17  christos 	ptr = mem_get(ctx, size, flags);
    657   1.1  christos 
    658  1.15  christos 	mem_getstats(ctx, size);
    659  1.15  christos 	ADD_TRACE(ctx, ptr, size, file, line);
    660   1.1  christos 
    661  1.17  christos 	return ptr;
    662   1.1  christos }
    663   1.1  christos 
    664   1.1  christos void
    665  1.17  christos isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size, int flags FLARG) {
    666  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    667  1.15  christos 
    668  1.15  christos 	DELETE_TRACE(ctx, ptr, size, file, line);
    669   1.6  christos 
    670  1.17  christos 	mem_putstats(ctx, size);
    671  1.17  christos 	mem_put(ctx, ptr, size, flags);
    672   1.1  christos }
    673   1.1  christos 
    674   1.1  christos #if ISC_MEM_TRACKLINES
    675   1.1  christos static void
    676  1.15  christos print_active(isc_mem_t *mctx, FILE *out) {
    677   1.1  christos 	if (mctx->debuglist != NULL) {
    678   1.1  christos 		debuglink_t *dl;
    679   1.1  christos 		unsigned int i;
    680   1.3  christos 		bool found;
    681   1.1  christos 
    682  1.15  christos 		fprintf(out, "Dump of all outstanding memory "
    683  1.15  christos 			     "allocations:\n");
    684   1.3  christos 		found = false;
    685   1.1  christos 		for (i = 0; i < DEBUG_TABLE_COUNT; i++) {
    686   1.1  christos 			dl = ISC_LIST_HEAD(mctx->debuglist[i]);
    687   1.1  christos 
    688   1.4  christos 			if (dl != NULL) {
    689   1.3  christos 				found = true;
    690   1.4  christos 			}
    691   1.1  christos 
    692   1.1  christos 			while (dl != NULL) {
    693   1.4  christos 				if (dl->ptr != NULL) {
    694   1.4  christos 					fprintf(out,
    695  1.15  christos 						"\tptr %p size %zu "
    696  1.15  christos 						"file %s "
    697   1.6  christos 						"line %u\n",
    698   1.6  christos 						dl->ptr, dl->size, dl->file,
    699   1.6  christos 						dl->line);
    700   1.4  christos 				}
    701   1.1  christos 				dl = ISC_LIST_NEXT(dl, link);
    702   1.1  christos 			}
    703   1.1  christos 		}
    704   1.1  christos 
    705   1.4  christos 		if (!found) {
    706  1.15  christos 			fprintf(out, "\tNone.\n");
    707   1.4  christos 		}
    708   1.1  christos 	}
    709   1.1  christos }
    710   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
    711   1.1  christos 
    712   1.1  christos /*
    713   1.1  christos  * Print the stats[] on the stream "out" with suitable formatting.
    714   1.1  christos  */
    715   1.1  christos void
    716  1.15  christos isc_mem_stats(isc_mem_t *ctx, FILE *out) {
    717  1.15  christos 	isc_mempool_t *pool = NULL;
    718   1.6  christos 
    719  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    720   1.1  christos 
    721   1.6  christos 	MCTXLOCK(ctx);
    722   1.1  christos 
    723   1.1  christos 	/*
    724  1.15  christos 	 * Note that since a pool can be locked now, these stats might
    725  1.15  christos 	 * be somewhat off if the pool is in active use at the time the
    726  1.15  christos 	 * stats are dumped.  The link fields are protected by the
    727  1.15  christos 	 * isc_mem_t's lock, however, so walking this list and
    728  1.15  christos 	 * extracting integers from stats fields is always safe.
    729   1.1  christos 	 */
    730   1.1  christos 	pool = ISC_LIST_HEAD(ctx->pools);
    731   1.1  christos 	if (pool != NULL) {
    732  1.15  christos 		fprintf(out, "[Pool statistics]\n");
    733  1.15  christos 		fprintf(out, "%15s %10s %10s %10s %10s %10s %10s %1s\n", "name",
    734  1.15  christos 			"size", "allocated", "freecount", "freemax",
    735  1.15  christos 			"fillcount", "gets", "L");
    736   1.1  christos 	}
    737   1.1  christos 	while (pool != NULL) {
    738  1.15  christos 		fprintf(out,
    739  1.15  christos 			"%15s %10zu %10zu %10zu %10zu %10zu %10zu %10zu %s\n",
    740  1.15  christos 			pool->name, pool->size, (size_t)0, pool->allocated,
    741  1.15  christos 			pool->freecount, pool->freemax, pool->fillcount,
    742  1.15  christos 			pool->gets, "N");
    743   1.1  christos 		pool = ISC_LIST_NEXT(pool, link);
    744   1.1  christos 	}
    745   1.1  christos 
    746   1.1  christos #if ISC_MEM_TRACKLINES
    747   1.1  christos 	print_active(ctx, out);
    748   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
    749   1.1  christos 
    750   1.6  christos 	MCTXUNLOCK(ctx);
    751   1.1  christos }
    752   1.1  christos 
    753  1.15  christos void *
    754  1.17  christos isc__mem_allocate(isc_mem_t *ctx, size_t size, int flags FLARG) {
    755  1.15  christos 	void *ptr = NULL;
    756  1.15  christos 
    757  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    758  1.15  christos 
    759  1.17  christos 	ptr = mem_get(ctx, size, flags);
    760   1.1  christos 
    761  1.15  christos 	/* Recalculate the real allocated size */
    762  1.17  christos 	size = sallocx(ptr, flags | ctx->jemalloc_flags);
    763   1.1  christos 
    764  1.15  christos 	mem_getstats(ctx, size);
    765  1.15  christos 	ADD_TRACE(ctx, ptr, size, file, line);
    766   1.1  christos 
    767  1.17  christos 	return ptr;
    768   1.1  christos }
    769   1.1  christos 
    770   1.1  christos void *
    771  1.15  christos isc__mem_reget(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size,
    772  1.17  christos 	       int flags FLARG) {
    773  1.15  christos 	void *new_ptr = NULL;
    774   1.6  christos 
    775  1.15  christos 	if (old_ptr == NULL) {
    776  1.15  christos 		REQUIRE(old_size == 0);
    777  1.17  christos 		new_ptr = isc__mem_get(ctx, new_size, flags FLARG_PASS);
    778  1.15  christos 	} else if (new_size == 0) {
    779  1.17  christos 		isc__mem_put(ctx, old_ptr, old_size, flags FLARG_PASS);
    780  1.15  christos 	} else {
    781  1.15  christos 		DELETE_TRACE(ctx, old_ptr, old_size, file, line);
    782  1.17  christos 		mem_putstats(ctx, old_size);
    783   1.1  christos 
    784  1.17  christos 		new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, flags);
    785   1.1  christos 
    786  1.15  christos 		mem_getstats(ctx, new_size);
    787  1.15  christos 		ADD_TRACE(ctx, new_ptr, new_size, file, line);
    788   1.1  christos 
    789  1.15  christos 		/*
    790  1.15  christos 		 * We want to postpone the call to water in edge case
    791  1.15  christos 		 * where the realloc will exactly hit on the boundary of
    792  1.15  christos 		 * the water and we would call water twice.
    793  1.15  christos 		 */
    794   1.1  christos 	}
    795   1.1  christos 
    796  1.17  christos 	return new_ptr;
    797   1.1  christos }
    798   1.1  christos 
    799   1.1  christos void *
    800  1.17  christos isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size,
    801  1.17  christos 		    int flags FLARG) {
    802  1.15  christos 	void *new_ptr = NULL;
    803   1.6  christos 
    804  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    805  1.15  christos 
    806  1.15  christos 	if (old_ptr == NULL) {
    807  1.17  christos 		new_ptr = isc__mem_allocate(ctx, new_size, flags FLARG_PASS);
    808  1.15  christos 	} else if (new_size == 0) {
    809  1.17  christos 		isc__mem_free(ctx, old_ptr, flags FLARG_PASS);
    810  1.15  christos 	} else {
    811  1.17  christos 		size_t old_size = sallocx(old_ptr, flags | ctx->jemalloc_flags);
    812   1.1  christos 
    813  1.15  christos 		DELETE_TRACE(ctx, old_ptr, old_size, file, line);
    814  1.17  christos 		mem_putstats(ctx, old_size);
    815   1.1  christos 
    816  1.17  christos 		new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, flags);
    817   1.1  christos 
    818  1.15  christos 		/* Recalculate the real allocated size */
    819  1.17  christos 		new_size = sallocx(new_ptr, flags | ctx->jemalloc_flags);
    820   1.6  christos 
    821  1.15  christos 		mem_getstats(ctx, new_size);
    822  1.15  christos 		ADD_TRACE(ctx, new_ptr, new_size, file, line);
    823   1.1  christos 
    824  1.15  christos 		/*
    825  1.15  christos 		 * We want to postpone the call to water in edge case
    826  1.15  christos 		 * where the realloc will exactly hit on the boundary of
    827  1.15  christos 		 * the water and we would call water twice.
    828  1.15  christos 		 */
    829   1.1  christos 	}
    830   1.1  christos 
    831  1.17  christos 	return new_ptr;
    832  1.15  christos }
    833   1.1  christos 
    834  1.15  christos void
    835  1.17  christos isc__mem_free(isc_mem_t *ctx, void *ptr, int flags FLARG) {
    836  1.15  christos 	size_t size = 0;
    837   1.1  christos 
    838  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    839  1.17  christos 	REQUIRE(ptr != NULL);
    840   1.1  christos 
    841  1.17  christos 	size = sallocx(ptr, flags | ctx->jemalloc_flags);
    842   1.1  christos 
    843  1.15  christos 	DELETE_TRACE(ctx, ptr, size, file, line);
    844   1.1  christos 
    845  1.17  christos 	mem_putstats(ctx, size);
    846  1.17  christos 	mem_put(ctx, ptr, size, flags);
    847   1.1  christos }
    848   1.1  christos 
    849   1.1  christos /*
    850   1.1  christos  * Other useful things.
    851   1.1  christos  */
    852   1.1  christos 
    853   1.1  christos char *
    854  1.15  christos isc__mem_strdup(isc_mem_t *mctx, const char *s FLARG) {
    855  1.15  christos 	size_t len;
    856  1.15  christos 	char *ns = NULL;
    857  1.15  christos 
    858  1.15  christos 	REQUIRE(VALID_CONTEXT(mctx));
    859   1.6  christos 	REQUIRE(s != NULL);
    860   1.6  christos 
    861   1.1  christos 	len = strlen(s) + 1;
    862   1.1  christos 
    863  1.17  christos 	ns = isc__mem_allocate(mctx, len, 0 FLARG_PASS);
    864   1.1  christos 
    865  1.15  christos 	strlcpy(ns, s, len);
    866   1.1  christos 
    867  1.17  christos 	return ns;
    868   1.1  christos }
    869   1.1  christos 
    870  1.11  christos char *
    871  1.15  christos isc__mem_strndup(isc_mem_t *mctx, const char *s, size_t size FLARG) {
    872  1.15  christos 	size_t len;
    873  1.15  christos 	char *ns = NULL;
    874  1.15  christos 
    875  1.15  christos 	REQUIRE(VALID_CONTEXT(mctx));
    876  1.11  christos 	REQUIRE(s != NULL);
    877  1.15  christos 	REQUIRE(size != 0);
    878  1.11  christos 
    879  1.11  christos 	len = strlen(s) + 1;
    880  1.11  christos 	if (len > size) {
    881  1.11  christos 		len = size;
    882  1.11  christos 	}
    883  1.11  christos 
    884  1.17  christos 	ns = isc__mem_allocate(mctx, len, 0 FLARG_PASS);
    885  1.11  christos 
    886  1.15  christos 	strlcpy(ns, s, len);
    887  1.11  christos 
    888  1.17  christos 	return ns;
    889  1.11  christos }
    890  1.11  christos 
    891   1.1  christos void
    892  1.15  christos isc_mem_setdestroycheck(isc_mem_t *ctx, bool flag) {
    893  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    894   1.1  christos 
    895   1.6  christos 	MCTXLOCK(ctx);
    896   1.1  christos 
    897   1.1  christos 	ctx->checkfree = flag;
    898   1.1  christos 
    899   1.6  christos 	MCTXUNLOCK(ctx);
    900   1.1  christos }
    901   1.1  christos 
    902   1.1  christos size_t
    903  1.15  christos isc_mem_inuse(isc_mem_t *ctx) {
    904  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    905   1.6  christos 
    906  1.17  christos 	return atomic_load_relaxed(&ctx->inuse);
    907  1.15  christos }
    908   1.1  christos 
    909  1.17  christos void
    910  1.17  christos isc_mem_clearwater(isc_mem_t *mctx) {
    911  1.17  christos 	isc_mem_setwater(mctx, 0, 0);
    912  1.15  christos }
    913   1.1  christos 
    914  1.17  christos void
    915  1.17  christos isc_mem_setwater(isc_mem_t *ctx, size_t hiwater, size_t lowater) {
    916  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    917  1.17  christos 	REQUIRE(hiwater >= lowater);
    918   1.1  christos 
    919  1.17  christos 	atomic_store_release(&ctx->hi_water, hiwater);
    920  1.17  christos 	atomic_store_release(&ctx->lo_water, lowater);
    921   1.1  christos 
    922  1.17  christos 	return;
    923   1.1  christos }
    924   1.1  christos 
    925  1.17  christos bool
    926  1.17  christos isc_mem_isovermem(isc_mem_t *ctx) {
    927  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    928   1.6  christos 
    929  1.17  christos 	bool is_overmem = atomic_load_relaxed(&ctx->is_overmem);
    930   1.1  christos 
    931  1.17  christos 	if (!is_overmem) {
    932  1.17  christos 		/* We are not overmem, check whether we should be? */
    933  1.17  christos 		size_t hiwater = atomic_load_relaxed(&ctx->hi_water);
    934  1.17  christos 		if (hiwater == 0) {
    935  1.17  christos 			return false;
    936  1.17  christos 		}
    937   1.1  christos 
    938  1.17  christos 		size_t inuse = atomic_load_relaxed(&ctx->inuse);
    939  1.17  christos 		if (inuse <= hiwater) {
    940  1.17  christos 			return false;
    941  1.17  christos 		}
    942   1.1  christos 
    943  1.17  christos 		if ((isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) {
    944  1.17  christos 			fprintf(stderr,
    945  1.17  christos 				"overmem mctx %p inuse %zu hi_water %zu\n", ctx,
    946  1.17  christos 				inuse, hiwater);
    947  1.17  christos 		}
    948  1.15  christos 
    949  1.17  christos 		atomic_store_relaxed(&ctx->is_overmem, true);
    950  1.17  christos 		return true;
    951  1.17  christos 	} else {
    952  1.17  christos 		/* We are overmem, check whether we should not be? */
    953  1.17  christos 		size_t lowater = atomic_load_relaxed(&ctx->lo_water);
    954  1.17  christos 		if (lowater == 0) {
    955  1.17  christos 			return false;
    956  1.17  christos 		}
    957  1.15  christos 
    958  1.17  christos 		size_t inuse = atomic_load_relaxed(&ctx->inuse);
    959  1.17  christos 		if (inuse >= lowater) {
    960  1.17  christos 			return true;
    961  1.17  christos 		}
    962  1.15  christos 
    963  1.17  christos 		if ((isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) {
    964  1.17  christos 			fprintf(stderr,
    965  1.17  christos 				"overmem mctx %p inuse %zu lo_water %zu\n", ctx,
    966  1.17  christos 				inuse, lowater);
    967  1.17  christos 		}
    968  1.17  christos 		atomic_store_relaxed(&ctx->is_overmem, false);
    969  1.17  christos 		return false;
    970   1.1  christos 	}
    971   1.1  christos }
    972   1.1  christos 
    973   1.1  christos void
    974  1.15  christos isc_mem_setname(isc_mem_t *ctx, const char *name) {
    975  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    976   1.1  christos 
    977   1.1  christos 	LOCK(&ctx->lock);
    978   1.1  christos 	strlcpy(ctx->name, name, sizeof(ctx->name));
    979   1.1  christos 	UNLOCK(&ctx->lock);
    980   1.1  christos }
    981   1.1  christos 
    982   1.1  christos const char *
    983  1.15  christos isc_mem_getname(isc_mem_t *ctx) {
    984  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
    985   1.1  christos 
    986   1.6  christos 	if (ctx->name[0] == 0) {
    987  1.17  christos 		return "";
    988   1.6  christos 	}
    989   1.1  christos 
    990  1.17  christos 	return ctx->name;
    991   1.1  christos }
    992   1.1  christos 
    993   1.1  christos /*
    994   1.1  christos  * Memory pool stuff
    995   1.1  christos  */
    996   1.1  christos 
    997   1.6  christos void
    998  1.15  christos isc__mempool_create(isc_mem_t *restrict mctx, const size_t element_size,
    999  1.15  christos 		    isc_mempool_t **restrict mpctxp FLARG) {
   1000  1.15  christos 	isc_mempool_t *restrict mpctx = NULL;
   1001  1.15  christos 	size_t size = element_size;
   1002  1.15  christos 
   1003  1.15  christos 	REQUIRE(VALID_CONTEXT(mctx));
   1004   1.6  christos 	REQUIRE(size > 0U);
   1005   1.6  christos 	REQUIRE(mpctxp != NULL && *mpctxp == NULL);
   1006   1.6  christos 
   1007   1.1  christos 	/*
   1008  1.15  christos 	 * Mempools are stored as a linked list of element.
   1009   1.1  christos 	 */
   1010  1.15  christos 	if (size < sizeof(element)) {
   1011  1.15  christos 		size = sizeof(element);
   1012  1.15  christos 	}
   1013   1.1  christos 
   1014   1.3  christos 	/*
   1015  1.15  christos 	 * Allocate space for this pool, initialize values, and if all
   1016  1.15  christos 	 * works well, attach to the memory context.
   1017   1.3  christos 	 */
   1018  1.15  christos 	mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t));
   1019  1.15  christos 
   1020  1.15  christos 	*mpctx = (isc_mempool_t){
   1021  1.15  christos 		.size = size,
   1022  1.15  christos 		.freemax = 1,
   1023  1.15  christos 		.fillcount = 1,
   1024  1.15  christos 	};
   1025  1.15  christos 
   1026  1.15  christos #if ISC_MEM_TRACKLINES
   1027  1.17  christos 	if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
   1028  1.15  christos 		fprintf(stderr, "create pool %p file %s line %u mctx %p\n",
   1029  1.15  christos 			mpctx, file, line, mctx);
   1030   1.3  christos 	}
   1031  1.15  christos #endif /* ISC_MEM_TRACKLINES */
   1032  1.15  christos 
   1033  1.15  christos 	isc_mem_attach(mctx, &mpctx->mctx);
   1034  1.15  christos 	mpctx->magic = MEMPOOL_MAGIC;
   1035   1.1  christos 
   1036   1.1  christos 	*mpctxp = (isc_mempool_t *)mpctx;
   1037   1.1  christos 
   1038   1.6  christos 	MCTXLOCK(mctx);
   1039   1.1  christos 	ISC_LIST_INITANDAPPEND(mctx->pools, mpctx, link);
   1040   1.1  christos 	mctx->poolcnt++;
   1041   1.6  christos 	MCTXUNLOCK(mctx);
   1042   1.1  christos }
   1043   1.1  christos 
   1044   1.1  christos void
   1045  1.15  christos isc_mempool_setname(isc_mempool_t *restrict mpctx, const char *name) {
   1046  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1047   1.6  christos 	REQUIRE(name != NULL);
   1048   1.6  christos 
   1049   1.1  christos 	strlcpy(mpctx->name, name, sizeof(mpctx->name));
   1050   1.1  christos }
   1051   1.1  christos 
   1052   1.1  christos void
   1053  1.15  christos isc__mempool_destroy(isc_mempool_t **restrict mpctxp FLARG) {
   1054  1.15  christos 	isc_mempool_t *restrict mpctx = NULL;
   1055  1.15  christos 	isc_mem_t *mctx = NULL;
   1056  1.15  christos 	element *restrict item = NULL;
   1057  1.15  christos 
   1058   1.6  christos 	REQUIRE(mpctxp != NULL);
   1059   1.6  christos 	REQUIRE(VALID_MEMPOOL(*mpctxp));
   1060   1.6  christos 
   1061  1.15  christos 	mpctx = *mpctxp;
   1062  1.15  christos 	*mpctxp = NULL;
   1063  1.15  christos 
   1064  1.15  christos 	mctx = mpctx->mctx;
   1065  1.15  christos 
   1066  1.15  christos #if ISC_MEM_TRACKLINES
   1067  1.17  christos 	if ((mctx->debugging & ISC_MEM_DEBUGTRACE) != 0) {
   1068  1.15  christos 		fprintf(stderr, "destroy pool %p file %s line %u mctx %p\n",
   1069  1.15  christos 			mpctx, file, line, mctx);
   1070  1.15  christos 	}
   1071  1.15  christos #endif
   1072   1.1  christos 
   1073   1.6  christos 	if (mpctx->allocated > 0) {
   1074  1.15  christos 		UNEXPECTED_ERROR("mempool %s leaked memory", mpctx->name);
   1075   1.6  christos 	}
   1076   1.1  christos 	REQUIRE(mpctx->allocated == 0);
   1077   1.1  christos 
   1078   1.1  christos 	/*
   1079   1.1  christos 	 * Return any items on the free list
   1080   1.1  christos 	 */
   1081   1.1  christos 	while (mpctx->items != NULL) {
   1082   1.1  christos 		INSIST(mpctx->freecount > 0);
   1083   1.1  christos 		mpctx->freecount--;
   1084  1.15  christos 
   1085   1.1  christos 		item = mpctx->items;
   1086   1.1  christos 		mpctx->items = item->next;
   1087  1.15  christos 
   1088  1.17  christos 		mem_putstats(mctx, mpctx->size);
   1089  1.15  christos 		mem_put(mctx, item, mpctx->size, 0);
   1090   1.1  christos 	}
   1091   1.1  christos 
   1092   1.1  christos 	/*
   1093   1.1  christos 	 * Remove our linked list entry from the memory context.
   1094   1.1  christos 	 */
   1095   1.6  christos 	MCTXLOCK(mctx);
   1096   1.1  christos 	ISC_LIST_UNLINK(mctx->pools, mpctx, link);
   1097   1.1  christos 	mctx->poolcnt--;
   1098   1.6  christos 	MCTXUNLOCK(mctx);
   1099   1.1  christos 
   1100  1.15  christos 	mpctx->magic = 0;
   1101   1.1  christos 
   1102  1.15  christos 	isc_mem_putanddetach(&mpctx->mctx, mpctx, sizeof(isc_mempool_t));
   1103   1.1  christos }
   1104   1.1  christos 
   1105  1.10  christos void *
   1106  1.15  christos isc__mempool_get(isc_mempool_t *restrict mpctx FLARG) {
   1107  1.15  christos 	element *restrict item = NULL;
   1108  1.10  christos 
   1109  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1110  1.10  christos 
   1111  1.10  christos 	mpctx->allocated++;
   1112  1.10  christos 
   1113  1.15  christos 	if (mpctx->items == NULL) {
   1114  1.15  christos 		isc_mem_t *mctx = mpctx->mctx;
   1115  1.15  christos #if !__SANITIZE_ADDRESS__
   1116  1.15  christos 		const size_t fillcount = mpctx->fillcount;
   1117  1.15  christos #else
   1118  1.15  christos 		const size_t fillcount = 1;
   1119  1.15  christos #endif
   1120   1.1  christos 		/*
   1121  1.15  christos 		 * We need to dip into the well.  Fill up our free list.
   1122   1.1  christos 		 */
   1123  1.15  christos 		for (size_t i = 0; i < fillcount; i++) {
   1124  1.15  christos 			item = mem_get(mctx, mpctx->size, 0);
   1125  1.12  christos 			mem_getstats(mctx, mpctx->size);
   1126   1.1  christos 			item->next = mpctx->items;
   1127   1.1  christos 			mpctx->items = item;
   1128   1.1  christos 			mpctx->freecount++;
   1129   1.1  christos 		}
   1130   1.1  christos 	}
   1131   1.1  christos 
   1132  1.17  christos 	INSIST(mpctx->items != NULL);
   1133   1.1  christos 	item = mpctx->items;
   1134   1.1  christos 
   1135   1.1  christos 	mpctx->items = item->next;
   1136  1.15  christos 
   1137   1.1  christos 	INSIST(mpctx->freecount > 0);
   1138   1.1  christos 	mpctx->freecount--;
   1139   1.1  christos 	mpctx->gets++;
   1140   1.1  christos 
   1141  1.15  christos 	ADD_TRACE(mpctx->mctx, item, mpctx->size, file, line);
   1142   1.1  christos 
   1143  1.17  christos 	return item;
   1144   1.1  christos }
   1145   1.1  christos 
   1146   1.1  christos /* coverity[+free : arg-1] */
   1147   1.1  christos void
   1148  1.15  christos isc__mempool_put(isc_mempool_t *restrict mpctx, void *mem FLARG) {
   1149  1.15  christos 	element *restrict item = NULL;
   1150  1.15  christos 
   1151  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1152   1.6  christos 	REQUIRE(mem != NULL);
   1153   1.6  christos 
   1154  1.15  christos 	isc_mem_t *mctx = mpctx->mctx;
   1155  1.15  christos 	const size_t freecount = mpctx->freecount;
   1156  1.15  christos #if !__SANITIZE_ADDRESS__
   1157  1.15  christos 	const size_t freemax = mpctx->freemax;
   1158  1.15  christos #else
   1159  1.15  christos 	const size_t freemax = 0;
   1160  1.15  christos #endif
   1161   1.1  christos 
   1162   1.1  christos 	INSIST(mpctx->allocated > 0);
   1163   1.1  christos 	mpctx->allocated--;
   1164   1.1  christos 
   1165  1.15  christos 	DELETE_TRACE(mctx, mem, mpctx->size, file, line);
   1166   1.1  christos 
   1167   1.1  christos 	/*
   1168   1.1  christos 	 * If our free list is full, return this to the mctx directly.
   1169   1.1  christos 	 */
   1170  1.15  christos 	if (freecount >= freemax) {
   1171  1.17  christos 		mem_putstats(mctx, mpctx->size);
   1172  1.15  christos 		mem_put(mctx, mem, mpctx->size, 0);
   1173   1.1  christos 		return;
   1174   1.1  christos 	}
   1175   1.1  christos 
   1176   1.1  christos 	/*
   1177   1.1  christos 	 * Otherwise, attach it to our free list and bump the counter.
   1178   1.1  christos 	 */
   1179   1.1  christos 	item = (element *)mem;
   1180   1.1  christos 	item->next = mpctx->items;
   1181   1.1  christos 	mpctx->items = item;
   1182  1.15  christos 	mpctx->freecount++;
   1183   1.1  christos }
   1184   1.1  christos 
   1185   1.1  christos /*
   1186   1.1  christos  * Quotas
   1187   1.1  christos  */
   1188   1.1  christos 
   1189   1.1  christos void
   1190  1.15  christos isc_mempool_setfreemax(isc_mempool_t *restrict mpctx,
   1191  1.15  christos 		       const unsigned int limit) {
   1192  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1193   1.1  christos 	mpctx->freemax = limit;
   1194   1.1  christos }
   1195   1.1  christos 
   1196   1.1  christos unsigned int
   1197  1.15  christos isc_mempool_getfreemax(isc_mempool_t *restrict mpctx) {
   1198  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1199   1.1  christos 
   1200  1.17  christos 	return mpctx->freemax;
   1201   1.1  christos }
   1202   1.1  christos 
   1203   1.1  christos unsigned int
   1204  1.15  christos isc_mempool_getfreecount(isc_mempool_t *restrict mpctx) {
   1205  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1206   1.1  christos 
   1207  1.17  christos 	return mpctx->freecount;
   1208   1.1  christos }
   1209   1.1  christos 
   1210   1.1  christos unsigned int
   1211  1.15  christos isc_mempool_getallocated(isc_mempool_t *restrict mpctx) {
   1212  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1213   1.1  christos 
   1214  1.17  christos 	return mpctx->allocated;
   1215   1.1  christos }
   1216   1.1  christos 
   1217   1.1  christos void
   1218  1.15  christos isc_mempool_setfillcount(isc_mempool_t *restrict mpctx,
   1219  1.15  christos 			 unsigned int const limit) {
   1220  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1221   1.6  christos 	REQUIRE(limit > 0);
   1222   1.6  christos 
   1223   1.1  christos 	mpctx->fillcount = limit;
   1224   1.1  christos }
   1225   1.1  christos 
   1226   1.1  christos unsigned int
   1227  1.15  christos isc_mempool_getfillcount(isc_mempool_t *restrict mpctx) {
   1228  1.15  christos 	REQUIRE(VALID_MEMPOOL(mpctx));
   1229   1.1  christos 
   1230  1.17  christos 	return mpctx->fillcount;
   1231   1.1  christos }
   1232   1.1  christos 
   1233   1.1  christos /*
   1234   1.1  christos  * Requires contextslock to be held by caller.
   1235   1.1  christos  */
   1236  1.15  christos #if ISC_MEM_TRACKLINES
   1237   1.1  christos static void
   1238   1.1  christos print_contexts(FILE *file) {
   1239  1.15  christos 	isc_mem_t *ctx;
   1240   1.1  christos 
   1241   1.6  christos 	for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
   1242  1.13  christos 	     ctx = ISC_LIST_NEXT(ctx, link))
   1243  1.13  christos 	{
   1244   1.3  christos 		fprintf(file, "context: %p (%s): %" PRIuFAST32 " references\n",
   1245   1.6  christos 			ctx, ctx->name[0] == 0 ? "<unknown>" : ctx->name,
   1246   1.3  christos 			isc_refcount_current(&ctx->references));
   1247   1.1  christos 		print_active(ctx, file);
   1248   1.1  christos 	}
   1249   1.1  christos 	fflush(file);
   1250   1.1  christos }
   1251  1.15  christos #endif
   1252   1.1  christos 
   1253  1.12  christos static atomic_uintptr_t checkdestroyed = 0;
   1254  1.10  christos 
   1255   1.1  christos void
   1256   1.1  christos isc_mem_checkdestroyed(FILE *file) {
   1257  1.10  christos 	atomic_store_release(&checkdestroyed, (uintptr_t)file);
   1258  1.10  christos }
   1259  1.10  christos 
   1260  1.10  christos void
   1261  1.10  christos isc__mem_checkdestroyed(void) {
   1262  1.10  christos 	FILE *file = (FILE *)atomic_load_acquire(&checkdestroyed);
   1263   1.1  christos 
   1264  1.10  christos 	if (file == NULL) {
   1265  1.10  christos 		return;
   1266  1.10  christos 	}
   1267   1.1  christos 
   1268   1.1  christos 	LOCK(&contextslock);
   1269   1.1  christos 	if (!ISC_LIST_EMPTY(contexts)) {
   1270   1.1  christos #if ISC_MEM_TRACKLINES
   1271  1.15  christos 		if ((isc_mem_debugging & TRACE_OR_RECORD) != 0) {
   1272   1.1  christos 			print_contexts(file);
   1273   1.1  christos 		}
   1274   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
   1275  1.12  christos 		UNREACHABLE();
   1276   1.1  christos 	}
   1277   1.1  christos 	UNLOCK(&contextslock);
   1278   1.1  christos }
   1279   1.1  christos 
   1280   1.1  christos unsigned int
   1281  1.15  christos isc_mem_references(isc_mem_t *ctx) {
   1282  1.17  christos 	return isc_refcount_current(&ctx->references);
   1283   1.1  christos }
   1284   1.1  christos 
   1285   1.1  christos #ifdef HAVE_LIBXML2
   1286   1.6  christos #define TRY0(a)                     \
   1287   1.6  christos 	do {                        \
   1288   1.6  christos 		xmlrc = (a);        \
   1289   1.6  christos 		if (xmlrc < 0)      \
   1290   1.6  christos 			goto error; \
   1291   1.9    rillig 	} while (0)
   1292   1.1  christos static int
   1293  1.17  christos xml_renderctx(isc_mem_t *ctx, size_t *inuse, xmlTextWriterPtr writer) {
   1294   1.6  christos 	REQUIRE(VALID_CONTEXT(ctx));
   1295   1.6  christos 
   1296   1.1  christos 	int xmlrc;
   1297   1.1  christos 
   1298   1.6  christos 	MCTXLOCK(ctx);
   1299   1.1  christos 
   1300   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "context"));
   1301   1.1  christos 
   1302   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "id"));
   1303   1.1  christos 	TRY0(xmlTextWriterWriteFormatString(writer, "%p", ctx));
   1304   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* id */
   1305   1.1  christos 
   1306   1.1  christos 	if (ctx->name[0] != 0) {
   1307   1.1  christos 		TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "name"));
   1308   1.1  christos 		TRY0(xmlTextWriterWriteFormatString(writer, "%s", ctx->name));
   1309   1.1  christos 		TRY0(xmlTextWriterEndElement(writer)); /* name */
   1310   1.1  christos 	}
   1311   1.1  christos 
   1312   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "references"));
   1313   1.6  christos 	TRY0(xmlTextWriterWriteFormatString(
   1314   1.6  christos 		writer, "%" PRIuFAST32,
   1315   1.6  christos 		isc_refcount_current(&ctx->references)));
   1316   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* references */
   1317   1.1  christos 
   1318  1.17  christos 	*inuse += isc_mem_inuse(ctx);
   1319   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "inuse"));
   1320   1.6  christos 	TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
   1321  1.15  christos 					    (uint64_t)isc_mem_inuse(ctx)));
   1322   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* inuse */
   1323   1.1  christos 
   1324   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "malloced"));
   1325   1.6  christos 	TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
   1326  1.17  christos 					    (uint64_t)isc_mem_inuse(ctx)));
   1327   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* malloced */
   1328   1.1  christos 
   1329   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "pools"));
   1330   1.1  christos 	TRY0(xmlTextWriterWriteFormatString(writer, "%u", ctx->poolcnt));
   1331   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* pools */
   1332   1.1  christos 
   1333   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "hiwater"));
   1334  1.15  christos 	TRY0(xmlTextWriterWriteFormatString(
   1335  1.15  christos 		writer, "%" PRIu64 "",
   1336  1.15  christos 		(uint64_t)atomic_load_relaxed(&ctx->hi_water)));
   1337   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* hiwater */
   1338   1.1  christos 
   1339   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "lowater"));
   1340  1.15  christos 	TRY0(xmlTextWriterWriteFormatString(
   1341  1.15  christos 		writer, "%" PRIu64 "",
   1342  1.15  christos 		(uint64_t)atomic_load_relaxed(&ctx->lo_water)));
   1343   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* lowater */
   1344   1.1  christos 
   1345   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* context */
   1346   1.1  christos 
   1347   1.6  christos error:
   1348   1.6  christos 	MCTXUNLOCK(ctx);
   1349   1.1  christos 
   1350  1.17  christos 	return xmlrc;
   1351   1.1  christos }
   1352   1.1  christos 
   1353   1.1  christos int
   1354   1.6  christos isc_mem_renderxml(void *writer0) {
   1355  1.15  christos 	isc_mem_t *ctx;
   1356  1.17  christos 	size_t inuse = 0;
   1357   1.1  christos 	int xmlrc;
   1358   1.6  christos 	xmlTextWriterPtr writer = (xmlTextWriterPtr)writer0;
   1359   1.1  christos 
   1360   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "contexts"));
   1361   1.1  christos 
   1362   1.1  christos 	LOCK(&contextslock);
   1363   1.6  christos 	for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
   1364  1.13  christos 	     ctx = ISC_LIST_NEXT(ctx, link))
   1365  1.13  christos 	{
   1366  1.17  christos 		xmlrc = xml_renderctx(ctx, &inuse, writer);
   1367   1.1  christos 		if (xmlrc < 0) {
   1368   1.1  christos 			UNLOCK(&contextslock);
   1369   1.1  christos 			goto error;
   1370   1.1  christos 		}
   1371   1.1  christos 	}
   1372   1.1  christos 	UNLOCK(&contextslock);
   1373   1.1  christos 
   1374   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* contexts */
   1375   1.1  christos 
   1376   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "summary"));
   1377   1.1  christos 
   1378  1.17  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "Malloced"));
   1379   1.6  christos 	TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
   1380  1.17  christos 					    (uint64_t)inuse));
   1381  1.17  christos 	TRY0(xmlTextWriterEndElement(writer)); /* malloced */
   1382   1.1  christos 
   1383   1.1  christos 	TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "InUse"));
   1384   1.6  christos 	TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "",
   1385  1.17  christos 					    (uint64_t)inuse));
   1386   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* InUse */
   1387   1.1  christos 
   1388   1.1  christos 	TRY0(xmlTextWriterEndElement(writer)); /* summary */
   1389   1.6  christos error:
   1390  1.17  christos 	return xmlrc;
   1391   1.1  christos }
   1392   1.1  christos 
   1393   1.1  christos #endif /* HAVE_LIBXML2 */
   1394   1.1  christos 
   1395   1.6  christos #ifdef HAVE_JSON_C
   1396   1.3  christos #define CHECKMEM(m) RUNTIME_CHECK(m != NULL)
   1397   1.1  christos 
   1398   1.1  christos static isc_result_t
   1399  1.17  christos json_renderctx(isc_mem_t *ctx, size_t *inuse, json_object *array) {
   1400   1.1  christos 	REQUIRE(VALID_CONTEXT(ctx));
   1401   1.1  christos 	REQUIRE(array != NULL);
   1402   1.1  christos 
   1403   1.6  christos 	json_object *ctxobj, *obj;
   1404   1.6  christos 	char buf[1024];
   1405   1.6  christos 
   1406   1.6  christos 	MCTXLOCK(ctx);
   1407   1.1  christos 
   1408  1.17  christos 	*inuse += isc_mem_inuse(ctx);
   1409   1.1  christos 
   1410   1.1  christos 	ctxobj = json_object_new_object();
   1411   1.1  christos 	CHECKMEM(ctxobj);
   1412   1.1  christos 
   1413   1.1  christos 	snprintf(buf, sizeof(buf), "%p", ctx);
   1414   1.1  christos 	obj = json_object_new_string(buf);
   1415   1.1  christos 	CHECKMEM(obj);
   1416   1.1  christos 	json_object_object_add(ctxobj, "id", obj);
   1417   1.1  christos 
   1418   1.1  christos 	if (ctx->name[0] != 0) {
   1419   1.1  christos 		obj = json_object_new_string(ctx->name);
   1420   1.1  christos 		CHECKMEM(obj);
   1421   1.1  christos 		json_object_object_add(ctxobj, "name", obj);
   1422   1.1  christos 	}
   1423   1.1  christos 
   1424   1.3  christos 	obj = json_object_new_int64(isc_refcount_current(&ctx->references));
   1425   1.1  christos 	CHECKMEM(obj);
   1426   1.1  christos 	json_object_object_add(ctxobj, "references", obj);
   1427   1.1  christos 
   1428  1.17  christos 	obj = json_object_new_int64(isc_mem_inuse(ctx));
   1429   1.1  christos 	CHECKMEM(obj);
   1430  1.17  christos 	json_object_object_add(ctxobj, "malloced", obj);
   1431   1.1  christos 
   1432  1.15  christos 	obj = json_object_new_int64(isc_mem_inuse(ctx));
   1433   1.1  christos 	CHECKMEM(obj);
   1434   1.1  christos 	json_object_object_add(ctxobj, "inuse", obj);
   1435   1.1  christos 
   1436   1.1  christos 	obj = json_object_new_int64(ctx->poolcnt);
   1437   1.1  christos 	CHECKMEM(obj);
   1438   1.1  christos 	json_object_object_add(ctxobj, "pools", obj);
   1439   1.1  christos 
   1440  1.15  christos 	obj = json_object_new_int64(atomic_load_relaxed(&ctx->hi_water));
   1441   1.1  christos 	CHECKMEM(obj);
   1442   1.1  christos 	json_object_object_add(ctxobj, "hiwater", obj);
   1443   1.1  christos 
   1444  1.15  christos 	obj = json_object_new_int64(atomic_load_relaxed(&ctx->lo_water));
   1445   1.1  christos 	CHECKMEM(obj);
   1446   1.1  christos 	json_object_object_add(ctxobj, "lowater", obj);
   1447   1.1  christos 
   1448   1.6  christos 	MCTXUNLOCK(ctx);
   1449   1.1  christos 	json_object_array_add(array, ctxobj);
   1450  1.17  christos 	return ISC_R_SUCCESS;
   1451   1.1  christos }
   1452   1.1  christos 
   1453   1.1  christos isc_result_t
   1454   1.6  christos isc_mem_renderjson(void *memobj0) {
   1455   1.1  christos 	isc_result_t result = ISC_R_SUCCESS;
   1456  1.15  christos 	isc_mem_t *ctx;
   1457  1.17  christos 	size_t inuse = 0;
   1458   1.1  christos 	json_object *ctxarray, *obj;
   1459   1.6  christos 	json_object *memobj = (json_object *)memobj0;
   1460   1.1  christos 
   1461   1.1  christos 	ctxarray = json_object_new_array();
   1462   1.1  christos 	CHECKMEM(ctxarray);
   1463   1.1  christos 
   1464   1.1  christos 	LOCK(&contextslock);
   1465   1.6  christos 	for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL;
   1466  1.13  christos 	     ctx = ISC_LIST_NEXT(ctx, link))
   1467  1.13  christos 	{
   1468  1.17  christos 		result = json_renderctx(ctx, &inuse, ctxarray);
   1469   1.1  christos 		if (result != ISC_R_SUCCESS) {
   1470   1.1  christos 			UNLOCK(&contextslock);
   1471   1.1  christos 			goto error;
   1472   1.1  christos 		}
   1473   1.1  christos 	}
   1474   1.1  christos 	UNLOCK(&contextslock);
   1475   1.1  christos 
   1476  1.17  christos 	obj = json_object_new_int64(inuse);
   1477   1.1  christos 	CHECKMEM(obj);
   1478   1.1  christos 	json_object_object_add(memobj, "InUse", obj);
   1479   1.1  christos 
   1480  1.17  christos 	obj = json_object_new_int64(inuse);
   1481   1.1  christos 	CHECKMEM(obj);
   1482   1.1  christos 	json_object_object_add(memobj, "Malloced", obj);
   1483   1.1  christos 
   1484   1.1  christos 	json_object_object_add(memobj, "contexts", ctxarray);
   1485  1.17  christos 	return ISC_R_SUCCESS;
   1486   1.1  christos 
   1487   1.6  christos error:
   1488   1.6  christos 	if (ctxarray != NULL) {
   1489   1.1  christos 		json_object_put(ctxarray);
   1490   1.6  christos 	}
   1491  1.17  christos 	return result;
   1492   1.1  christos }
   1493   1.6  christos #endif /* HAVE_JSON_C */
   1494   1.1  christos 
   1495   1.6  christos void
   1496  1.15  christos isc__mem_create(isc_mem_t **mctxp FLARG) {
   1497  1.17  christos 	mem_create(mctxp, isc_mem_debugging, isc_mem_defaultflags, 0);
   1498  1.15  christos #if ISC_MEM_TRACKLINES
   1499  1.15  christos 	if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
   1500  1.15  christos 		fprintf(stderr, "create mctx %p file %s line %u\n", *mctxp,
   1501  1.15  christos 			file, line);
   1502  1.15  christos 	}
   1503  1.15  christos #endif /* ISC_MEM_TRACKLINES */
   1504   1.1  christos }
   1505   1.1  christos 
   1506  1.15  christos void
   1507  1.15  christos isc__mem_create_arena(isc_mem_t **mctxp FLARG) {
   1508  1.15  christos 	unsigned int arena_no = ISC_MEM_ILLEGAL_ARENA;
   1509   1.1  christos 
   1510  1.15  christos 	RUNTIME_CHECK(mem_jemalloc_arena_create(&arena_no));
   1511   1.1  christos 
   1512  1.15  christos 	/*
   1513  1.15  christos 	 * We use MALLOCX_TCACHE_NONE to bypass the tcache and route
   1514  1.15  christos 	 * allocations directly to the arena. That is a recommendation
   1515  1.15  christos 	 * from jemalloc developers:
   1516  1.15  christos 	 *
   1517  1.15  christos 	 * https://github.com/jemalloc/jemalloc/issues/2483#issuecomment-1698173849
   1518  1.15  christos 	 */
   1519  1.17  christos 	mem_create(mctxp, isc_mem_debugging, isc_mem_defaultflags,
   1520  1.15  christos 		   arena_no == ISC_MEM_ILLEGAL_ARENA
   1521  1.15  christos 			   ? 0
   1522  1.15  christos 			   : MALLOCX_ARENA(arena_no) | MALLOCX_TCACHE_NONE);
   1523  1.15  christos 	(*mctxp)->jemalloc_arena = arena_no;
   1524  1.15  christos #if ISC_MEM_TRACKLINES
   1525  1.15  christos 	if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) {
   1526  1.15  christos 		fprintf(stderr,
   1527  1.15  christos 			"create mctx %p file %s line %u for jemalloc arena "
   1528  1.15  christos 			"%u\n",
   1529  1.15  christos 			*mctxp, file, line, arena_no);
   1530  1.15  christos 	}
   1531  1.15  christos #endif /* ISC_MEM_TRACKLINES */
   1532   1.1  christos }
   1533   1.1  christos 
   1534  1.17  christos #ifdef JEMALLOC_API_SUPPORTED
   1535  1.15  christos static bool
   1536  1.15  christos jemalloc_set_ssize_value(const char *valname, ssize_t newval) {
   1537  1.15  christos 	int ret;
   1538   1.1  christos 
   1539  1.15  christos 	ret = mallctl(valname, NULL, NULL, &newval, sizeof(newval));
   1540  1.17  christos 	return ret == 0;
   1541   1.1  christos }
   1542  1.17  christos #endif /* JEMALLOC_API_SUPPORTED */
   1543   1.1  christos 
   1544  1.15  christos static isc_result_t
   1545  1.15  christos mem_set_arena_ssize_value(isc_mem_t *mctx, const char *arena_valname,
   1546  1.15  christos 			  const ssize_t newval) {
   1547  1.15  christos 	REQUIRE(VALID_CONTEXT(mctx));
   1548  1.17  christos #ifdef JEMALLOC_API_SUPPORTED
   1549  1.15  christos 	bool ret;
   1550  1.15  christos 	char buf[256] = { 0 };
   1551   1.1  christos 
   1552  1.15  christos 	if (mctx->jemalloc_arena == ISC_MEM_ILLEGAL_ARENA) {
   1553  1.17  christos 		return ISC_R_UNEXPECTED;
   1554  1.15  christos 	}
   1555   1.1  christos 
   1556  1.15  christos 	(void)snprintf(buf, sizeof(buf), "arena.%u.%s", mctx->jemalloc_arena,
   1557  1.15  christos 		       arena_valname);
   1558   1.1  christos 
   1559  1.15  christos 	ret = jemalloc_set_ssize_value(buf, newval);
   1560   1.1  christos 
   1561  1.15  christos 	if (!ret) {
   1562  1.17  christos 		return ISC_R_FAILURE;
   1563  1.15  christos 	}
   1564   1.1  christos 
   1565  1.17  christos 	return ISC_R_SUCCESS;
   1566  1.15  christos #else
   1567  1.15  christos 	UNUSED(arena_valname);
   1568  1.15  christos 	UNUSED(newval);
   1569  1.17  christos 	return ISC_R_NOTIMPLEMENTED;
   1570  1.17  christos #endif
   1571   1.1  christos }
   1572   1.1  christos 
   1573  1.15  christos isc_result_t
   1574  1.15  christos isc_mem_arena_set_muzzy_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) {
   1575  1.17  christos 	return mem_set_arena_ssize_value(mctx, "muzzy_decay_ms", decay_ms);
   1576  1.11  christos }
   1577  1.11  christos 
   1578  1.15  christos isc_result_t
   1579  1.15  christos isc_mem_arena_set_dirty_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) {
   1580  1.17  christos 	return mem_set_arena_ssize_value(mctx, "dirty_decay_ms", decay_ms);
   1581   1.1  christos }
   1582   1.1  christos 
   1583   1.3  christos void
   1584  1.15  christos isc__mem_printactive(isc_mem_t *ctx, FILE *file) {
   1585   1.3  christos #if ISC_MEM_TRACKLINES
   1586  1.15  christos 	REQUIRE(VALID_CONTEXT(ctx));
   1587   1.6  christos 	REQUIRE(file != NULL);
   1588   1.6  christos 
   1589   1.3  christos 	print_active(ctx, file);
   1590   1.6  christos #else  /* if ISC_MEM_TRACKLINES */
   1591  1.15  christos 	UNUSED(ctx);
   1592   1.3  christos 	UNUSED(file);
   1593   1.6  christos #endif /* if ISC_MEM_TRACKLINES */
   1594   1.1  christos }
   1595