diff --git a/src/zmalloc.c b/src/zmalloc.c index 0117d8d91..e740a4167 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -52,6 +52,7 @@ void zlibc_free(void *ptr) { #include #include "zmalloc.h" #include +#include "serverassert.h" #define UNUSED(x) ((void)(x)) @@ -87,10 +88,31 @@ void zlibc_free(void *ptr) { #define dallocx(ptr, flags) je_dallocx(ptr, flags) #endif -#define update_zmalloc_stat_alloc(__n) atomic_fetch_add_explicit(&used_memory, (__n), memory_order_relaxed) -#define update_zmalloc_stat_free(__n) atomic_fetch_sub_explicit(&used_memory, (__n), memory_order_relaxed) +#define update_zmalloc_stat_alloc(__n) \ + do { \ + if (unlikely(thread_index == -1)) zmalloc_register_thread_index(); \ + used_memory_thread[thread_index] += (__n); \ + } while (0) + +#define update_zmalloc_stat_free(__n) \ + do { \ + if (unlikely(thread_index == -1)) zmalloc_register_thread_index(); \ + used_memory_thread[thread_index] -= (__n); \ + } while (0) + +/* A thread-local storage which keep the current thread's index in the used_memory_thread array. */ +static __thread int thread_index = -1; +/* MAX_THREADS_NUM = IO_THREADS_MAX_NUM(128) + BIO threads(3) + main thread(1). */ +#define MAX_THREADS_NUM 132 +static unsigned long long used_memory_thread[MAX_THREADS_NUM]; + +static atomic_int total_active_threads; -static _Atomic size_t used_memory = 0; +/* Register the thread index in start_routine. */ +void zmalloc_register_thread_index(void) { + thread_index = atomic_fetch_add_explicit(&total_active_threads, 1, memory_order_relaxed); + assert(total_active_threads < MAX_THREADS_NUM); +} static void zmalloc_default_oom(size_t size) { fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n", size); @@ -388,7 +410,11 @@ char *zstrdup(const char *s) { } size_t zmalloc_used_memory(void) { - size_t um = atomic_load_explicit(&used_memory, memory_order_relaxed); + assert(total_active_threads < MAX_THREADS_NUM); + size_t um = 0; + for (int i = 0; i < total_active_threads; i++) { + um += used_memory_thread[i]; + } return um; } @@ -604,8 +630,6 @@ size_t zmalloc_get_rss(void) { #if defined(USE_JEMALLOC) -#include "serverassert.h" - #define STRINGIFY_(x) #x #define STRINGIFY(x) STRINGIFY_(x) diff --git a/src/zmalloc.h b/src/zmalloc.h index a909366c1..a068c671c 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -142,6 +142,7 @@ size_t zmalloc_get_memory_size(void); void zlibc_free(void *ptr); void zlibc_trim(void); void zmadvise_dontneed(void *ptr); +void zmalloc_register_thread_index(void); #ifdef HAVE_DEFRAG void zfree_no_tcache(void *ptr);