diff options
author | Vladislavs Sokurenko <vladislavs.sokurenko@zabbix.com> | 2022-11-09 13:51:41 +0300 |
---|---|---|
committer | Vladislavs Sokurenko <vladislavs.sokurenko@zabbix.com> | 2022-11-09 13:51:41 +0300 |
commit | 81f13e3c4387bb7180daaa09906c78d381c5aec3 (patch) | |
tree | 2a6f1b381917169c8d6185ba4d4413076572a5a8 | |
parent | a4afd0a56adc59628728b210b8faeb1258ed888a (diff) | |
parent | f9c6a0f86d5ddd60304abbad07fc163df546e741 (diff) |
........S. [ZBX-21825] improved history syncer performance
Merge in ZBX/zabbix from feature/ZBX-21825-6.0 to release/6.0
* commit 'f9c6a0f86d5ddd60304abbad07fc163df546e741':
........S. [ZBX-21825] improved history syncer performance
-rw-r--r-- | ChangeLog.d/bugfix/ZBX-21825 | 1 | ||||
-rw-r--r-- | src/libs/zbxcommon/str.c | 18 | ||||
-rw-r--r-- | src/libs/zbxdbcache/dbcache.c | 148 | ||||
-rw-r--r-- | src/libs/zbxdbcache/dbconfig.c | 12 | ||||
-rw-r--r-- | src/libs/zbxdbhigh/proxy.c | 2 | ||||
-rw-r--r-- | src/libs/zbxserver/expression.c | 3 |
6 files changed, 90 insertions, 94 deletions
diff --git a/ChangeLog.d/bugfix/ZBX-21825 b/ChangeLog.d/bugfix/ZBX-21825 new file mode 100644 index 00000000000..f1e98ad97d0 --- /dev/null +++ b/ChangeLog.d/bugfix/ZBX-21825 @@ -0,0 +1 @@ +........S. [ZBX-21825] improved history syncer performance (vso) diff --git a/src/libs/zbxcommon/str.c b/src/libs/zbxcommon/str.c index 44c85e9e1b8..ff2e25a8389 100644 --- a/src/libs/zbxcommon/str.c +++ b/src/libs/zbxcommon/str.c @@ -647,17 +647,21 @@ char *zbx_str_printable_dyn(const char *text) ******************************************************************************/ size_t zbx_strlcpy(char *dst, const char *src, size_t siz) { - const char *s = src; + size_t len = strlen(src); - if (0 != siz) + if (len + 1 <= siz) { - while (0 != --siz && '\0' != *s) - *dst++ = *s++; - - *dst = '\0'; + memcpy(dst, src, len + 1); + return len; } - return s - src; /* count does not include null */ + if (0 == siz) + return 0; + + memcpy(dst, src, siz - 1); + dst[siz - 1] = '\0'; + + return siz - 1; } /****************************************************************************** diff --git a/src/libs/zbxdbcache/dbcache.c b/src/libs/zbxdbcache/dbcache.c index f22e062a30d..e4337477579 100644 --- a/src/libs/zbxdbcache/dbcache.c +++ b/src/libs/zbxdbcache/dbcache.c @@ -1548,22 +1548,15 @@ static void DCsync_trends(void) ******************************************************************************/ static void recalculate_triggers(const ZBX_DC_HISTORY *history, int history_num, const zbx_vector_uint64_t *history_itemids, const DC_ITEM *history_items, const int *history_errcodes, - const zbx_vector_ptr_t *timers, zbx_vector_ptr_t *trigger_diff) + const zbx_vector_ptr_t *timers, zbx_vector_ptr_t *trigger_diff, zbx_uint64_t *itemids, + zbx_timespec_t *timespecs, zbx_hashset_t *trigger_info, zbx_vector_ptr_t *trigger_order) { int i, item_num = 0, timers_num = 0; - zbx_uint64_t *itemids = NULL; - zbx_timespec_t *timespecs = NULL; - zbx_hashset_t trigger_info; - zbx_vector_ptr_t trigger_order; - zbx_vector_ptr_t trigger_items; zabbix_log(LOG_LEVEL_DEBUG, "In %s()", __func__); if (0 != history_num) { - itemids = (zbx_uint64_t *)zbx_malloc(itemids, sizeof(zbx_uint64_t) * (size_t)history_num); - timespecs = (zbx_timespec_t *)zbx_malloc(timespecs, sizeof(zbx_timespec_t) * (size_t)history_num); - for (i = 0; i < history_num; i++) { const ZBX_DC_HISTORY *h = &history[i]; @@ -1588,48 +1581,39 @@ static void recalculate_triggers(const ZBX_DC_HISTORY *history, int history_num, if (0 == item_num && 0 == timers_num) goto out; - zbx_hashset_create(&trigger_info, MAX(100, 2 * item_num + timers_num), - ZBX_DEFAULT_UINT64_HASH_FUNC, ZBX_DEFAULT_UINT64_COMPARE_FUNC); - - zbx_vector_ptr_create(&trigger_order); - zbx_vector_ptr_reserve(&trigger_order, trigger_info.num_slots); + zbx_hashset_reserve(trigger_info, MAX(100, 2 * item_num + timers_num)); - zbx_vector_ptr_create(&trigger_items); + zbx_vector_ptr_reserve(trigger_order, trigger_info->num_slots); if (0 != item_num) { - DCconfig_get_triggers_by_itemids(&trigger_info, &trigger_order, itemids, timespecs, item_num); - prepare_triggers((DC_TRIGGER **)trigger_order.values, trigger_order.values_num); - zbx_determine_items_in_expressions(&trigger_order, itemids, item_num); + DCconfig_get_triggers_by_itemids(trigger_info, trigger_order, itemids, timespecs, item_num); + prepare_triggers((DC_TRIGGER **)trigger_order->values, trigger_order->values_num); + zbx_determine_items_in_expressions(trigger_order, itemids, item_num); } if (0 != timers_num) { - int offset = trigger_order.values_num; + int offset = trigger_order->values_num; - zbx_dc_get_triggers_by_timers(&trigger_info, &trigger_order, timers); + zbx_dc_get_triggers_by_timers(trigger_info, trigger_order, timers); - if (offset != trigger_order.values_num) + if (offset != trigger_order->values_num) { - prepare_triggers((DC_TRIGGER **)trigger_order.values + offset, - trigger_order.values_num - offset); + prepare_triggers((DC_TRIGGER **)trigger_order->values + offset, + trigger_order->values_num - offset); } } - zbx_vector_ptr_sort(&trigger_order, ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC); - evaluate_expressions(&trigger_order, history_itemids, history_items, history_errcodes); - zbx_process_triggers(&trigger_order, trigger_diff); + zbx_vector_ptr_sort(trigger_order, ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC); + evaluate_expressions(trigger_order, history_itemids, history_items, history_errcodes); + zbx_process_triggers(trigger_order, trigger_diff); - DCfree_triggers(&trigger_order); + DCfree_triggers(trigger_order); - zbx_vector_ptr_destroy(&trigger_items); - - zbx_hashset_destroy(&trigger_info); - zbx_vector_ptr_destroy(&trigger_order); + zbx_hashset_clear(trigger_info); + zbx_vector_ptr_clear(trigger_order); out: - zbx_free(timespecs); - zbx_free(itemids); - zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __func__); } @@ -1667,7 +1651,7 @@ static void DCinventory_value_add(zbx_vector_ptr_t *inventory_values, const DC_I return; } - zbx_format_value(value, sizeof(value), item->valuemapid, item->units, h->value_type); + zbx_format_value(value, sizeof(value), item->valuemapid, ZBX_NULL2EMPTY_STR(item->units), h->value_type); inventory_value = (zbx_inventory_value_t *)zbx_malloc(NULL, sizeof(zbx_inventory_value_t)); @@ -1942,7 +1926,7 @@ static zbx_item_diff_t *calculate_item_update(DC_ITEM *item, const ZBX_DC_HISTOR zbx_add_event(EVENT_SOURCE_INTERNAL, EVENT_OBJECT_ITEM, item->itemid, &h->ts, h->state, NULL, NULL, NULL, 0, 0, NULL, 0, NULL, 0, NULL, NULL, h->value.err); - if (0 != strcmp(item->error, h->value.err)) + if (0 != strcmp(ZBX_NULL2EMPTY_STR(item->error), h->value.err)) item_error = h->value.err; } else @@ -1958,7 +1942,7 @@ static zbx_item_diff_t *calculate_item_update(DC_ITEM *item, const ZBX_DC_HISTOR item_error = ""; } } - else if (ITEM_STATE_NOTSUPPORTED == h->state && 0 != strcmp(item->error, h->value.err)) + else if (ITEM_STATE_NOTSUPPORTED == h->state && 0 != strcmp(ZBX_NULL2EMPTY_STR(item->error), h->value.err)) { zabbix_log(LOG_LEVEL_WARNING, "error reason for \"%s:%s\" changed: %s", item->host.host, item->key_orig, h->value.err); @@ -2617,9 +2601,9 @@ static void DBmass_proxy_add_history(ZBX_DC_HISTORY *history, int history_num) * proxy_subscribtions - [IN] history compression age * * * ******************************************************************************/ -static void DCmass_prepare_history(ZBX_DC_HISTORY *history, const zbx_vector_uint64_t *itemids, - DC_ITEM *items, const int *errcodes, int history_num, zbx_vector_ptr_t *item_diff, - zbx_vector_ptr_t *inventory_values, int compression_age, zbx_vector_uint64_pair_t *proxy_subscribtions) +static void DCmass_prepare_history(ZBX_DC_HISTORY *history, DC_ITEM *items, const int *errcodes, int history_num, + zbx_vector_ptr_t *item_diff, zbx_vector_ptr_t *inventory_values, int compression_age, + zbx_vector_uint64_pair_t *proxy_subscribtions) { static time_t last_history_discard = 0; time_t now; @@ -2634,7 +2618,6 @@ static void DCmass_prepare_history(ZBX_DC_HISTORY *history, const zbx_vector_uin ZBX_DC_HISTORY *h = &history[i]; DC_ITEM *item; zbx_item_diff_t *diff; - int index; /* discard history items that are older than compression age */ if (0 != compression_age && h->ts.sec < compression_age) @@ -2650,20 +2633,13 @@ static void DCmass_prepare_history(ZBX_DC_HISTORY *history, const zbx_vector_uin continue; } - if (FAIL == (index = zbx_vector_uint64_bsearch(itemids, h->itemid, ZBX_DEFAULT_UINT64_COMPARE_FUNC))) - { - THIS_SHOULD_NEVER_HAPPEN; - h->flags |= ZBX_DC_FLAG_UNDEF; - continue; - } - - if (SUCCEED != errcodes[index]) + if (SUCCEED != errcodes[i]) { h->flags |= ZBX_DC_FLAG_UNDEF; continue; } - item = &items[index]; + item = &items[i]; if (ITEM_STATUS_ACTIVE != item->status || HOST_STATUS_MONITORED != item->host.status) { @@ -3079,43 +3055,56 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) static ZBX_HISTORY_STRING *history_string; static ZBX_HISTORY_TEXT *history_text; static ZBX_HISTORY_LOG *history_log; + static int module_enabled = FAIL; int i, history_num, history_float_num, history_integer_num, history_string_num, history_text_num, history_log_num, txn_error, compression_age; unsigned int item_retrieve_mode; time_t sync_start; zbx_vector_uint64_t triggerids ; - zbx_vector_ptr_t history_items, trigger_diff, item_diff, inventory_values, trigger_timers; + zbx_vector_ptr_t history_items, trigger_diff, item_diff, inventory_values, trigger_timers, + trigger_order; zbx_vector_uint64_pair_t trends_diff, proxy_subscribtions; ZBX_DC_HISTORY history[ZBX_HC_SYNC_MAX]; + zbx_uint64_t trigger_itemids[ZBX_HC_SYNC_MAX]; + zbx_timespec_t trigger_timespecs[ZBX_HC_SYNC_MAX]; + DC_ITEM *items = NULL; + int *errcodes = NULL; + zbx_vector_uint64_t itemids; + zbx_hashset_t trigger_info; item_retrieve_mode = NULL == CONFIG_EXPORT_DIR ? ZBX_ITEM_GET_SYNC : ZBX_ITEM_GET_SYNC_EXPORT; if (NULL == history_float && NULL != history_float_cbs) { + module_enabled = SUCCEED; history_float = (ZBX_HISTORY_FLOAT *)zbx_malloc(history_float, ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_FLOAT)); } if (NULL == history_integer && NULL != history_integer_cbs) { + module_enabled = SUCCEED; history_integer = (ZBX_HISTORY_INTEGER *)zbx_malloc(history_integer, ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_INTEGER)); } if (NULL == history_string && NULL != history_string_cbs) { + module_enabled = SUCCEED; history_string = (ZBX_HISTORY_STRING *)zbx_malloc(history_string, ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_STRING)); } if (NULL == history_text && NULL != history_text_cbs) { + module_enabled = SUCCEED; history_text = (ZBX_HISTORY_TEXT *)zbx_malloc(history_text, ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_TEXT)); } if (NULL == history_log && NULL != history_log_cbs) { + module_enabled = SUCCEED; history_log = (ZBX_HISTORY_LOG *)zbx_malloc(history_log, ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_LOG)); } @@ -3137,17 +3126,18 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) zbx_vector_ptr_create(&history_items); zbx_vector_ptr_reserve(&history_items, ZBX_HC_SYNC_MAX); + zbx_vector_ptr_create(&trigger_order); + zbx_hashset_create(&trigger_info, 100, ZBX_DEFAULT_UINT64_HASH_FUNC, ZBX_DEFAULT_UINT64_COMPARE_FUNC); + + zbx_vector_uint64_create(&itemids); + sync_start = time(NULL); do { - DC_ITEM *items; - int *errcodes, trends_num = 0, timers_num = 0, ret = SUCCEED; - zbx_vector_uint64_t itemids; + int trends_num = 0, timers_num = 0, ret = SUCCEED; ZBX_DC_TREND *trends = NULL; - zbx_vector_uint64_create(&itemids); - *more = ZBX_SYNC_DONE; LOCK_CACHE; @@ -3169,22 +3159,25 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) if (0 != history_num) { + zbx_vector_ptr_sort(&history_items, ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC); + hc_get_item_values(history, &history_items); /* copy item data from history cache */ - items = (DC_ITEM *)zbx_malloc(NULL, sizeof(DC_ITEM) * (size_t)history_num); - errcodes = (int *)zbx_malloc(NULL, sizeof(int) * (size_t)history_num); + if (NULL == items) + items = (DC_ITEM *)zbx_calloc(NULL, 1, sizeof(DC_ITEM) * (size_t)ZBX_HC_SYNC_MAX); + + if (NULL == errcodes) + errcodes = (int *)zbx_malloc(NULL, sizeof(int) * (size_t)ZBX_HC_SYNC_MAX); zbx_vector_uint64_reserve(&itemids, history_num); for (i = 0; i < history_num; i++) zbx_vector_uint64_append(&itemids, history[i].itemid); - zbx_vector_uint64_sort(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); - DCconfig_get_items_by_itemids_partial(items, itemids.values, errcodes, history_num, item_retrieve_mode); - DCmass_prepare_history(history, &itemids, items, errcodes, history_num, &item_diff, + DCmass_prepare_history(history, items, errcodes, history_num, &item_diff, &inventory_values, compression_age, &proxy_subscribtions); if (FAIL != (ret = DBmass_add_history(history, history_num))) @@ -3250,7 +3243,8 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) DBbegin(); recalculate_triggers(history, history_num, &itemids, items, errcodes, - &trigger_timers, &trigger_diff); + &trigger_timers, &trigger_diff, trigger_itemids, + trigger_timespecs, &trigger_info, &trigger_order); /* process trigger events generated by recalculate_triggers() */ zbx_process_events(&trigger_diff, &triggerids); @@ -3321,14 +3315,17 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) const ZBX_DC_TREND *ptrends = NULL; int history_num_loc = 0, trends_num_loc = 0; - DCmodule_prepare_history(history, history_num, history_float, &history_float_num, - history_integer, &history_integer_num, history_string, - &history_string_num, history_text, &history_text_num, history_log, - &history_log_num); - - DCmodule_sync_history(history_float_num, history_integer_num, history_string_num, - history_text_num, history_log_num, history_float, history_integer, - history_string, history_text, history_log); + if (SUCCEED == module_enabled) + { + DCmodule_prepare_history(history, history_num, history_float, &history_float_num, + history_integer, &history_integer_num, history_string, + &history_string_num, history_text, &history_text_num, history_log, + &history_log_num); + + DCmodule_sync_history(history_float_num, history_integer_num, history_string_num, + history_text_num, history_log_num, history_float, history_integer, + history_string, history_text, history_log); + } if (SUCCEED == zbx_is_export_enabled(ZBX_FLAG_EXPTYPE_HISTORY)) { @@ -3360,20 +3357,25 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) { zbx_free(trends); DCconfig_clean_items(items, errcodes, history_num); - zbx_free(errcodes); - zbx_free(items); zbx_vector_ptr_clear(&history_items); hc_free_item_values(history, history_num); } - zbx_vector_uint64_destroy(&itemids); + zbx_vector_uint64_clear(&itemids); /* Exit from sync loop if we have spent too much time here. */ /* This is done to allow syncer process to update its statistics. */ } while (ZBX_SYNC_MORE == *more && ZBX_HC_SYNC_TIME_MAX >= time(NULL) - sync_start); + zbx_free(items); + zbx_free(errcodes); + + zbx_vector_ptr_destroy(&trigger_order); + zbx_hashset_destroy(&trigger_info); + + zbx_vector_uint64_destroy(&itemids); zbx_vector_ptr_destroy(&history_items); zbx_vector_ptr_destroy(&inventory_values); zbx_vector_ptr_destroy(&item_diff); diff --git a/src/libs/zbxdbcache/dbconfig.c b/src/libs/zbxdbcache/dbconfig.c index 254cd2ae136..72153b05dd9 100644 --- a/src/libs/zbxdbcache/dbconfig.c +++ b/src/libs/zbxdbcache/dbconfig.c @@ -8233,8 +8233,7 @@ void DCconfig_get_items_by_itemids_partial(DC_ITEM *items, const zbx_uint64_t *i const ZBX_DC_ITEM *dc_item; const ZBX_DC_HOST *dc_host = NULL; - memset(items, 0, sizeof(DC_ITEM) * (size_t)num); - memset(errcodes, 0, sizeof(int) * (size_t)num); + memset(errcodes, 0, sizeof(int) * num); RDLOCK_CACHE; @@ -8268,15 +8267,6 @@ void DCconfig_get_items_by_itemids_partial(DC_ITEM *items, const zbx_uint64_t *i continue; items[i].itemid = itemids[i]; - - if (NULL == items[i].error) - items[i].error = zbx_strdup(NULL, ""); - - if (ITEM_VALUE_TYPE_FLOAT == items[i].value_type || ITEM_VALUE_TYPE_UINT64 == items[i].value_type) - { - if (NULL == items[i].units) - items[i].units = zbx_strdup(NULL, ""); - } } } diff --git a/src/libs/zbxdbhigh/proxy.c b/src/libs/zbxdbhigh/proxy.c index c33c29b668f..b63b6a0e718 100644 --- a/src/libs/zbxdbhigh/proxy.c +++ b/src/libs/zbxdbhigh/proxy.c @@ -3478,7 +3478,7 @@ static int process_history_data_by_itemids(zbx_socket_t *sock, zbx_client_item_v zabbix_log(LOG_LEVEL_DEBUG, "In %s()", __func__); - items = (DC_ITEM *)zbx_malloc(NULL, sizeof(DC_ITEM) * ZBX_HISTORY_VALUES_MAX); + items = (DC_ITEM *)zbx_calloc(NULL, 1, sizeof(DC_ITEM) * ZBX_HISTORY_VALUES_MAX); errcodes = (int *)zbx_malloc(NULL, sizeof(int) * ZBX_HISTORY_VALUES_MAX); sec = zbx_time(); diff --git a/src/libs/zbxserver/expression.c b/src/libs/zbxserver/expression.c index 1f09db2c799..263ac61398d 100644 --- a/src/libs/zbxserver/expression.c +++ b/src/libs/zbxserver/expression.c @@ -4856,7 +4856,6 @@ void zbx_determine_items_in_expressions(zbx_vector_ptr_t *trigger_order, const z zbx_vector_uint64_create(&itemids_sorted); zbx_vector_uint64_append_array(&itemids_sorted, itemids, item_num); - zbx_vector_uint64_sort(&itemids_sorted, ZBX_DEFAULT_UINT64_COMPARE_FUNC); zbx_vector_ptr_create(&triggers_func_pos); zbx_vector_ptr_reserve(&triggers_func_pos, trigger_order->values_num); @@ -5069,7 +5068,7 @@ static void zbx_evaluate_item_functions(zbx_hashset_t *funcs, const zbx_vector_u zbx_vector_uint64_sort(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); zbx_vector_uint64_uniq(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); - items = (DC_ITEM *)zbx_malloc(items, sizeof(DC_ITEM) * (size_t)itemids.values_num); + items = (DC_ITEM *)zbx_calloc(items, 1, sizeof(DC_ITEM) * (size_t)itemids.values_num); errcodes = (int *)zbx_malloc(errcodes, sizeof(int) * (size_t)itemids.values_num); DCconfig_get_items_by_itemids_partial(items, itemids.values, errcodes, itemids.values_num, |