Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/zabbix/zabbix.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndris Zeila <andris.zeila@zabbix.com>2013-05-22 14:02:50 +0400
committerAndris Zeila <andris.zeila@zabbix.com>2013-05-22 14:02:50 +0400
commit2943dde551ae5e083017c362a3925ffa5dcf6a15 (patch)
tree2fb5ea62897e5aec5e9cb107954b179819bdc908
parent6207475f64c301bbdaa4408aa87fbbc99b9cae3e (diff)
.......PS. [ZBXNEXT-322] added 64 bit support to memory allocator
This allows shared memory allocation of 4GB and more (current hard limit is set to 64GB). Also added 'allow out of memory' mode. If shared memory is created with allow_oom flag, then out of memory situation is not treated as critical failure and memory allocators return NULL instead of aborting execution.
-rw-r--r--include/memalloc.h17
-rw-r--r--include/zbxtypes.h6
-rw-r--r--src/libs/zbxalgo/hashset.c19
-rw-r--r--src/libs/zbxdbcache/dbcache.c6
-rw-r--r--src/libs/zbxdbcache/dbconfig.c2
-rw-r--r--src/libs/zbxmemory/memalloc.c128
-rw-r--r--src/libs/zbxmemory/strpool.c2
7 files changed, 101 insertions, 79 deletions
diff --git a/include/memalloc.h b/include/memalloc.h
index 432ab365166..19aab382ab0 100644
--- a/include/memalloc.h
+++ b/include/memalloc.h
@@ -28,19 +28,26 @@ typedef struct
void **buckets;
void *lo_bound;
void *hi_bound;
- uint32_t free_size;
- uint32_t used_size;
- uint32_t orig_size;
- uint32_t total_size;
+ zbx_uint64_t free_size;
+ zbx_uint64_t used_size;
+ zbx_uint64_t orig_size;
+ zbx_uint64_t total_size;
int shm_id;
char use_lock;
+
+ /* Continue execution in out of memory situation. */
+ /* Normally allocator forces exit when it runs out of allocatable memory. */
+ /* Set this flag to 1 to allow execution in out of memory situations. */
+ char allow_oom;
+
ZBX_MUTEX mem_lock;
const char *mem_descr;
const char *mem_param;
}
zbx_mem_info_t;
-void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, size_t size, const char *descr, const char *param);
+void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, zbx_uint64_t size,
+ const char *descr, const char *param, int allow_oom);
void zbx_mem_destroy(zbx_mem_info_t *info);
#define zbx_mem_malloc(info, old, size) __zbx_mem_malloc(__FILE__, __LINE__, info, old, size)
diff --git a/include/zbxtypes.h b/include/zbxtypes.h
index b0a1b45767c..556d3e2feba 100644
--- a/include/zbxtypes.h
+++ b/include/zbxtypes.h
@@ -23,9 +23,6 @@
#define ZBX_FS_DBL "%lf"
#define ZBX_FS_DBL_EXT(p) "%." #p "lf"
-#define ZBX_FS_SIZE_T "%u"
-#define zbx_fs_size_t unsigned int /* use this type only in calls to printf() for formatting size_t */
-
#define ZBX_PTR_SIZE sizeof(void *)
#if defined(_WINDOWS)
@@ -115,6 +112,9 @@
#endif /* _WINDOWS */
+#define ZBX_FS_SIZE_T ZBX_FS_UI64
+#define zbx_fs_size_t zbx_uint64_t /* use this type only in calls to printf() for formatting size_t */
+
#ifndef S_ISREG
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
diff --git a/src/libs/zbxalgo/hashset.c b/src/libs/zbxalgo/hashset.c
index b7f0ecd1587..0145375b830 100644
--- a/src/libs/zbxalgo/hashset.c
+++ b/src/libs/zbxalgo/hashset.c
@@ -54,10 +54,14 @@ void zbx_hashset_create_ext(zbx_hashset_t *hs, size_t init_size,
zbx_mem_realloc_func_t mem_realloc_func,
zbx_mem_free_func_t mem_free_func)
{
+ int nslots = next_prime(init_size);
+
+ if (NULL == (hs->slots = mem_malloc_func(NULL, nslots * sizeof(ZBX_HASHSET_ENTRY_T *))))
+ return;
+
hs->num_data = 0;
- hs->num_slots = next_prime(init_size);
+ hs->num_slots = nslots;
- hs->slots = mem_malloc_func(NULL, hs->num_slots * sizeof(ZBX_HASHSET_ENTRY_T *));
memset(hs->slots, 0, hs->num_slots * sizeof(ZBX_HASHSET_ENTRY_T *));
hs->hash_func = hash_func;
@@ -122,8 +126,15 @@ void *zbx_hashset_insert_ext(zbx_hashset_t *hs, const void *data, size_t size, s
if (NULL == entry)
{
- entry = hs->mem_malloc_func(NULL, sizeof(ZBX_HASHSET_ENTRY_T));
- entry->data = hs->mem_malloc_func(NULL, size);
+ if (NULL == (entry = hs->mem_malloc_func(NULL, sizeof(ZBX_HASHSET_ENTRY_T))))
+ return NULL;
+
+ if (NULL == (entry->data = hs->mem_malloc_func(NULL, size)))
+ {
+ zbx_free(entry);
+ return NULL;
+ }
+
memcpy((char *)entry->data + offset, (const char *)data + offset, size - offset);
entry->hash = hash;
entry->next = hs->slots[slot];
diff --git a/src/libs/zbxdbcache/dbcache.c b/src/libs/zbxdbcache/dbcache.c
index f25ba71cbc1..8bc3b4669ed 100644
--- a/src/libs/zbxdbcache/dbcache.c
+++ b/src/libs/zbxdbcache/dbcache.c
@@ -3118,7 +3118,7 @@ static void init_trend_cache()
sz = zbx_mem_required_size(1, "trend cache", "TrendCacheSize");
zbx_mem_create(&trend_mem, trend_shm_key, ZBX_NO_MUTEX, CONFIG_TRENDS_CACHE_SIZE,
- "trend cache", "TrendCacheSize");
+ "trend cache", "TrendCacheSize", 0);
CONFIG_TRENDS_CACHE_SIZE -= sz;
cache->trends_num = 0;
@@ -3194,7 +3194,7 @@ void init_database_cache()
sz += sz_history;
- zbx_mem_create(&history_mem, history_shm_key, ZBX_NO_MUTEX, sz, "history cache", "HistoryCacheSize");
+ zbx_mem_create(&history_mem, history_shm_key, ZBX_NO_MUTEX, sz, "history cache", "HistoryCacheSize", 0);
cache = (ZBX_DC_CACHE *)__history_mem_malloc_func(NULL, sizeof(ZBX_DC_CACHE));
@@ -3214,7 +3214,7 @@ void init_database_cache()
sz = zbx_mem_required_size(1, "history text cache", "HistoryTextCacheSize");
zbx_mem_create(&history_text_mem, history_text_shm_key, ZBX_NO_MUTEX, CONFIG_TEXT_CACHE_SIZE,
- "history text cache", "HistoryTextCacheSize");
+ "history text cache", "HistoryTextCacheSize", 0);
CONFIG_TEXT_CACHE_SIZE -= sz;
cache->text = (char *)__history_text_mem_malloc_func(NULL, CONFIG_TEXT_CACHE_SIZE);
diff --git a/src/libs/zbxdbcache/dbconfig.c b/src/libs/zbxdbcache/dbconfig.c
index 30b8a3ae6c9..bb5512d18a3 100644
--- a/src/libs/zbxdbcache/dbconfig.c
+++ b/src/libs/zbxdbcache/dbconfig.c
@@ -2890,7 +2890,7 @@ void init_configuration_cache()
exit(FAIL);
}
- zbx_mem_create(&config_mem, shm_key, ZBX_NO_MUTEX, config_size, "configuration cache", "CacheSize");
+ zbx_mem_create(&config_mem, shm_key, ZBX_NO_MUTEX, config_size, "configuration cache", "CacheSize", 0);
config = __config_mem_malloc_func(NULL, sizeof(ZBX_DC_CONFIG) +
CONFIG_TIMER_FORKS * sizeof(zbx_vector_ptr_t));
diff --git a/src/libs/zbxmemory/memalloc.c b/src/libs/zbxmemory/memalloc.c
index 36670b642cc..5794a15a3e1 100644
--- a/src/libs/zbxmemory/memalloc.c
+++ b/src/libs/zbxmemory/memalloc.c
@@ -34,7 +34,7 @@
* *
* *
* +-------- size of + --------------+ *
- * | (4 bytes) | | *
+ * | (8 bytes) | | *
* | v | *
* | | *
* | +- allocatable memory --+ | *
@@ -44,7 +44,7 @@
* |--------|----------------...----|--------| *
* *
* ^ ^ ^ ^ *
- * 4-aligned | | 4-aligned *
+ * 8-aligned | | 8-aligned *
* *
* 8-aligned 8-aligned *
* *
@@ -98,11 +98,11 @@ static void *ALIGN4(void *ptr);
static void *ALIGN8(void *ptr);
static void *ALIGNPTR(void *ptr);
-static uint32_t mem_proper_alloc_size(uint32_t size);
-static int mem_bucket_by_size(uint32_t size);
+static zbx_uint64_t mem_proper_alloc_size(zbx_uint64_t size);
+static int mem_bucket_by_size(zbx_uint64_t size);
-static void mem_set_chunk_size(void *chunk, uint32_t size);
-static void mem_set_used_chunk_size(void *chunk, uint32_t size);
+static void mem_set_chunk_size(void *chunk, zbx_uint64_t size);
+static void mem_set_used_chunk_size(void *chunk, zbx_uint64_t size);
static void *mem_get_prev_chunk(void *chunk);
static void mem_set_prev_chunk(void *chunk, void *prev);
@@ -114,25 +114,26 @@ static void **mem_ptr_to_next_field(void *chunk, void **first_chunk);
static void mem_link_chunk(zbx_mem_info_t *info, void *chunk);
static void mem_unlink_chunk(zbx_mem_info_t *info, void *chunk);
-static void *__mem_malloc(zbx_mem_info_t *info, uint32_t size);
-static void *__mem_realloc(zbx_mem_info_t *info, void *old, uint32_t size);
+static void *__mem_malloc(zbx_mem_info_t *info, zbx_uint64_t size);
+static void *__mem_realloc(zbx_mem_info_t *info, void *old, zbx_uint64_t size);
static void __mem_free(zbx_mem_info_t *info, void *ptr);
-#define MEM_SIZE_FIELD sizeof(uint32_t)
+#define MEM_SIZE_FIELD sizeof(zbx_uint64_t)
-#define MEM_FLG_USED (((uint32_t)1)<<31)
+#define MEM_FLG_USED ((__UINT64_C(1))<<63)
-#define FREE_CHUNK(ptr) (((*(uint32_t *)(ptr)) & MEM_FLG_USED) == 0)
-#define CHUNK_SIZE(ptr) ((*(uint32_t *)(ptr)) & ~MEM_FLG_USED)
+#define FREE_CHUNK(ptr) (((*(zbx_uint64_t *)(ptr)) & MEM_FLG_USED) == 0)
+#define CHUNK_SIZE(ptr) ((*(zbx_uint64_t *)(ptr)) & ~MEM_FLG_USED)
-#define MEM_MIN_SIZE 128
-#define MEM_MAX_SIZE 0x7fffffff /* just below 2 GB */
+
+#define MEM_MIN_SIZE __UINT64_C(128)
+#define MEM_MAX_SIZE __UINT64_C(0x1000000000) /* 64 GB */
#define MEM_MIN_ALLOC 24 /* should be a multiple of 8 and at least (2 * ZBX_PTR_SIZE) */
-#define MEM_MIN_BUCKET_SIZE MEM_MIN_ALLOC
-#define MEM_MAX_BUCKET_SIZE 256 /* starting from this size all free chunks are put into the same bucket */
-#define MEM_BUCKET_COUNT ((MEM_MAX_BUCKET_SIZE - MEM_MIN_BUCKET_SIZE) / 8 + 1)
+#define MEM_MIN_BUCKET_SIZE MEM_MIN_ALLOC
+#define MEM_MAX_BUCKET_SIZE 256 /* starting from this size all free chunks are put into the same bucket */
+#define MEM_BUCKET_COUNT ((MEM_MAX_BUCKET_SIZE - MEM_MIN_BUCKET_SIZE) / 8 + 1)
/* helper functions */
@@ -155,15 +156,15 @@ static void *ALIGNPTR(void *ptr)
assert(0);
}
-static uint32_t mem_proper_alloc_size(uint32_t size)
+static zbx_uint64_t mem_proper_alloc_size(zbx_uint64_t size)
{
if (size >= MEM_MIN_ALLOC)
- return size + (8 - size % 8) % 8; /* allocate in multiples of 8... */
+ return size + ((8 - (size & 7)) & 7); /* allocate in multiples of 8... */
else
return MEM_MIN_ALLOC; /* ...and at least MEM_MIN_ALLOC */
}
-static int mem_bucket_by_size(uint32_t size)
+static int mem_bucket_by_size(zbx_uint64_t size)
{
if (size < MEM_MIN_BUCKET_SIZE)
return 0;
@@ -172,16 +173,16 @@ static int mem_bucket_by_size(uint32_t size)
return MEM_BUCKET_COUNT - 1;
}
-static void mem_set_chunk_size(void *chunk, uint32_t size)
+static void mem_set_chunk_size(void *chunk, zbx_uint64_t size)
{
- *(uint32_t *)chunk = size;
- *(uint32_t *)(chunk + MEM_SIZE_FIELD + size) = size;
+ *(zbx_uint64_t *)chunk = size;
+ *(zbx_uint64_t *)(chunk + MEM_SIZE_FIELD + size) = size;
}
-static void mem_set_used_chunk_size(void *chunk, uint32_t size)
+static void mem_set_used_chunk_size(void *chunk, zbx_uint64_t size)
{
- *(uint32_t *)chunk = MEM_FLG_USED | size;
- *(uint32_t *)(chunk + MEM_SIZE_FIELD + size) = MEM_FLG_USED | size;
+ *(zbx_uint64_t *)chunk = MEM_FLG_USED | size;
+ *(zbx_uint64_t *)(chunk + MEM_SIZE_FIELD + size) = MEM_FLG_USED | size;
}
static void *mem_get_prev_chunk(void *chunk)
@@ -250,11 +251,11 @@ static void mem_unlink_chunk(zbx_mem_info_t *info, void *chunk)
/* private memory functions */
-static void *__mem_malloc(zbx_mem_info_t *info, uint32_t size)
+static void *__mem_malloc(zbx_mem_info_t *info, zbx_uint64_t size)
{
int index;
void *chunk;
- uint32_t chunk_size;
+ zbx_uint64_t chunk_size;
size = mem_proper_alloc_size(size);
@@ -272,8 +273,7 @@ static void *__mem_malloc(zbx_mem_info_t *info, uint32_t size)
/* otherwise, find a chunk big enough according to first-fit strategy */
int counter = 0;
- uint32_t skip_min = 0xffffffff, skip_max = 0;
-
+ zbx_uint64_t skip_min = __UINT64_C(0xffffffffffffffff), skip_max = __UINT64_C(0);
while (NULL != chunk && CHUNK_SIZE(chunk) < size)
{
counter++;
@@ -282,12 +282,16 @@ static void *__mem_malloc(zbx_mem_info_t *info, uint32_t size)
chunk = mem_get_next_chunk(chunk);
}
- if (NULL == chunk)
- zabbix_log(LOG_LEVEL_CRIT, "__mem_malloc: skipped %d asked %u skip_min %u skip_max %u",
- counter, size, skip_min, skip_max);
- else if (counter >= 100)
- zabbix_log(LOG_LEVEL_DEBUG, "__mem_malloc: skipped %d asked %u skip_min %u skip_max %u size %u",
- counter, size, skip_min, skip_max, CHUNK_SIZE(chunk));
+ /* don't log errors if malloc can return null in low memory situations */
+ if (0 == info->allow_oom)
+ {
+ if (NULL == chunk)
+ zabbix_log(LOG_LEVEL_CRIT, "__mem_malloc: skipped %d asked %u skip_min %u skip_max %u",
+ counter, size, skip_min, skip_max);
+ else if (counter >= 100)
+ zabbix_log(LOG_LEVEL_DEBUG, "__mem_malloc: skipped %d asked %u skip_min %u skip_max %u size %u",
+ counter, size, skip_min, skip_max, CHUNK_SIZE(chunk));
+ }
}
if (NULL == chunk)
@@ -308,7 +312,7 @@ static void *__mem_malloc(zbx_mem_info_t *info, uint32_t size)
else
{
void *new_chunk;
- uint32_t new_chunk_size;
+ zbx_uint64_t new_chunk_size;
new_chunk = chunk + MEM_SIZE_FIELD + size + MEM_SIZE_FIELD;
new_chunk_size = chunk_size - size - 2 * MEM_SIZE_FIELD;
@@ -325,10 +329,10 @@ static void *__mem_malloc(zbx_mem_info_t *info, uint32_t size)
return chunk;
}
-static void *__mem_realloc(zbx_mem_info_t *info, void *old, uint32_t size)
+static void *__mem_realloc(zbx_mem_info_t *info, void *old, zbx_uint64_t size)
{
void *chunk, *new_chunk, *next_chunk;
- uint32_t chunk_size, new_chunk_size;
+ zbx_uint64_t chunk_size, new_chunk_size;
int next_free;
size = mem_proper_alloc_size(size);
@@ -453,7 +457,7 @@ static void __mem_free(zbx_mem_info_t *info, void *ptr)
{
void *chunk;
void *prev_chunk, *next_chunk;
- uint32_t chunk_size;
+ zbx_uint64_t chunk_size;
int prev_free, next_free;
chunk = ptr - MEM_SIZE_FIELD;
@@ -518,12 +522,13 @@ static void __mem_free(zbx_mem_info_t *info, void *ptr)
/* public memory interface */
-void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, size_t size, const char *descr, const char *param)
+void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, zbx_uint64_t size,
+ const char *descr, const char *param, int allow_oom)
{
const char *__function_name = "zbx_mem_create";
int shm_id, index;
- void *base, *chunk_lsize, *chunk_rsize;
+ void *base;
descr = (NULL == descr ? "(null)" : descr);
param = (NULL == param ? "(null)" : param);
@@ -542,7 +547,7 @@ void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, size_t
if (!(MEM_MIN_SIZE <= size && size <= MEM_MAX_SIZE))
{
- zabbix_log(LOG_LEVEL_CRIT, "requested size " ZBX_FS_SIZE_T " not within bounds [%d <= size <= %d]",
+ zabbix_log(LOG_LEVEL_CRIT, "requested size " ZBX_FS_SIZE_T " not within bounds [%llu <= size <= %llu]",
(zbx_fs_size_t)size, MEM_MIN_SIZE, MEM_MAX_SIZE);
exit(FAIL);
}
@@ -563,7 +568,7 @@ void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, size_t
*info = ALIGN8(base);
(*info)->shm_id = shm_id;
- (*info)->orig_size = (uint32_t)size;
+ (*info)->orig_size = size;
size -= (void *)(*info + 1) - base;
base = (void *)(*info + 1);
@@ -582,6 +587,8 @@ void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, size_t
size -= strlen(param) + 1;
base += strlen(param) + 1;
+ (*info)->allow_oom = allow_oom;
+
/* allocate mutex */
if (ZBX_NO_MUTEX != lock_name)
@@ -599,20 +606,10 @@ void zbx_mem_create(zbx_mem_info_t **info, key_t shm_key, int lock_name, size_t
(*info)->use_lock = 0;
/* prepare shared memory for further allocation by creating one big chunk */
+ (*info)->lo_bound = ALIGN8(base);
+ (*info)->hi_bound = ALIGN8(base + size - 8);
- chunk_lsize = ALIGN8(base);
- if (chunk_lsize - MEM_SIZE_FIELD < base)
- chunk_lsize += 8;
- chunk_lsize -= MEM_SIZE_FIELD;
-
- chunk_rsize = ALIGN8(base + size - 8);
- if (chunk_rsize + MEM_SIZE_FIELD > base + size)
- chunk_rsize -= 8;
-
- (*info)->lo_bound = chunk_lsize;
- (*info)->hi_bound = chunk_rsize + MEM_SIZE_FIELD;
-
- (*info)->total_size = (uint32_t)(chunk_rsize - chunk_lsize - MEM_SIZE_FIELD);
+ (*info)->total_size = (zbx_uint64_t)((*info)->hi_bound - (*info)->lo_bound - 2 * MEM_SIZE_FIELD);
index = mem_bucket_by_size((*info)->total_size);
(*info)->buckets[index] = (*info)->lo_bound;
@@ -671,12 +668,15 @@ void *__zbx_mem_malloc(const char *file, int line, zbx_mem_info_t *info, const v
LOCK_INFO;
- chunk = __mem_malloc(info, (uint32_t)size);
+ chunk = __mem_malloc(info, size);
UNLOCK_INFO;
if (NULL == chunk)
{
+ if (1 == info->allow_oom)
+ return NULL;
+
zabbix_log(LOG_LEVEL_CRIT, "[file:%s,line:%d] %s(): out of memory (requested " ZBX_FS_SIZE_T " bytes)",
file, line, __function_name, (zbx_fs_size_t)size);
zabbix_log(LOG_LEVEL_CRIT, "[file:%s,line:%d] %s(): please increase %s configuration parameter",
@@ -703,14 +703,17 @@ void *__zbx_mem_realloc(const char *file, int line, zbx_mem_info_t *info, void *
LOCK_INFO;
if (NULL == old)
- chunk = __mem_malloc(info, (uint32_t)size);
+ chunk = __mem_malloc(info, size);
else
- chunk = __mem_realloc(info, old, (uint32_t)size);
+ chunk = __mem_realloc(info, old, size);
UNLOCK_INFO;
if (NULL == chunk)
{
+ if (1 == info->allow_oom)
+ return NULL;
+
zabbix_log(LOG_LEVEL_CRIT, "[file:%s,line:%d] %s(): out of memory (requested " ZBX_FS_SIZE_T " bytes)",
file, line, __function_name, (zbx_fs_size_t)size);
zabbix_log(LOG_LEVEL_CRIT, "[file:%s,line:%d] %s(): please increase %s configuration parameter",
@@ -766,8 +769,9 @@ void zbx_mem_clear(zbx_mem_info_t *info)
void zbx_mem_dump_stats(zbx_mem_info_t *info)
{
void *chunk;
- int index, counter, total, total_free = 0;
- uint32_t min_size = 0xffffffff, max_size = 0;
+ int index;
+ zbx_uint64_t counter, total, total_free = 0;
+ zbx_uint64_t min_size = __UINT64_C(0xffffffffffffffff), max_size = __UINT64_C(0);
LOCK_INFO;
diff --git a/src/libs/zbxmemory/strpool.c b/src/libs/zbxmemory/strpool.c
index 3f2f51964ea..ce43c47146e 100644
--- a/src/libs/zbxmemory/strpool.c
+++ b/src/libs/zbxmemory/strpool.c
@@ -72,7 +72,7 @@ void zbx_strpool_create(size_t size)
exit(FAIL);
}
- zbx_mem_create(&strpool.mem_info, shm_key, ZBX_NO_MUTEX, size, "string pool", "CacheSize");
+ zbx_mem_create(&strpool.mem_info, shm_key, ZBX_NO_MUTEX, size, "string pool", "CacheSize", 0);
if (ZBX_MUTEX_ERROR == zbx_mutex_create_force(&strpool.pool_lock, ZBX_MUTEX_STRPOOL))
{