........S. [ZBX-21825] improved history syncer performance

Merge in ZBX/zabbix from feature/ZBX-21825-6.3 to master

* commit 'db421a90914101fc3da46b8041b65c7170d1b2ec':
  ........S. [ZBX-21825] improved history syncer performance
  ........S. [ZBX-21825] improved history syncer performance

6 files changed, 90 insertions, 95 deletions

diff --git a/ChangeLog.d/bugfix/ZBX-21825 b/ChangeLog.d/bugfix/ZBX-21825
new file mode 100644
index 00000000000..f1e98ad97d0
--- /dev/null
+++ b/ChangeLog.d/bugfix/ZBX-21825
@@ -0,0 +1 @@
+........S. [ZBX-21825] improved history syncer performance (vso)
diff --git a/src/libs/zbxcacheconfig/dbconfig.c b/src/libs/zbxcacheconfig/dbconfig.c
index 0785566d4e6..01be09648e6 100644
--- a/src/libs/zbxcacheconfig/dbconfig.c
+++ b/src/libs/zbxcacheconfig/dbconfig.c
@@ -9219,8 +9219,7 @@ void	DCconfig_get_items_by_itemids_partial(DC_ITEM *items, const zbx_uint64_t *i
 	zbx_config_hk_t		config_hk;
 	zbx_dc_um_handle_t	*um_handle;
 
-	memset(items, 0, sizeof(DC_ITEM) * (size_t)num);
-	memset(errcodes, 0, sizeof(int) * (size_t)num);
+	memset(errcodes, 0, sizeof(int) * num);
 
 	RDLOCK_CACHE;
 
@@ -9260,15 +9259,6 @@ void	DCconfig_get_items_by_itemids_partial(DC_ITEM *items, const zbx_uint64_t *i
 
 		items[i].itemid = itemids[i];
 
-		if (NULL == items[i].error)
-			items[i].error = zbx_strdup(NULL, "");
-
-		if (ITEM_VALUE_TYPE_FLOAT == items[i].value_type || ITEM_VALUE_TYPE_UINT64 == items[i].value_type)
-		{
-			if (NULL == items[i].units)
-				items[i].units = zbx_strdup(NULL, "");
-		}
-
 		if (0 != (mode & ZBX_ITEM_GET_HOUSEKEEPING))
 			dc_items_convert_hk_periods(&config_hk, &items[i]);
 	}
diff --git a/src/libs/zbxcachehistory/dbcache.c b/src/libs/zbxcachehistory/dbcache.c
index a402f530547..d90d0cb7a50 100644
--- a/src/libs/zbxcachehistory/dbcache.c
+++ b/src/libs/zbxcachehistory/dbcache.c
@@ -1710,22 +1710,15 @@ out:
  ******************************************************************************/
 static void	recalculate_triggers(const ZBX_DC_HISTORY *history, int history_num,
 		const zbx_vector_uint64_t *history_itemids, const DC_ITEM *history_items, const int *history_errcodes,
-		const zbx_vector_ptr_t *timers, zbx_vector_ptr_t *trigger_diff)
+		const zbx_vector_ptr_t *timers, zbx_vector_ptr_t *trigger_diff, zbx_uint64_t *itemids,
+		zbx_timespec_t *timespecs, zbx_hashset_t *trigger_info, zbx_vector_ptr_t *trigger_order)
 {
 	int			i, item_num = 0, timers_num = 0;
-	zbx_uint64_t		*itemids = NULL;
-	zbx_timespec_t		*timespecs = NULL;
-	zbx_hashset_t		trigger_info;
-	zbx_vector_ptr_t	trigger_order;
-	zbx_vector_ptr_t	trigger_items;
 
 	zabbix_log(LOG_LEVEL_DEBUG, "In %s()", __func__);
 
 	if (0 != history_num)
 	{
-		itemids = (zbx_uint64_t *)zbx_malloc(itemids, sizeof(zbx_uint64_t) * (size_t)history_num);
-		timespecs = (zbx_timespec_t *)zbx_malloc(timespecs, sizeof(zbx_timespec_t) * (size_t)history_num);
-
 		for (i = 0; i < history_num; i++)
 		{
 			const ZBX_DC_HISTORY	*h = &history[i];
@@ -1750,48 +1743,39 @@ static void	recalculate_triggers(const ZBX_DC_HISTORY *history, int history_num,
 	if (0 == item_num && 0 == timers_num)
 		goto out;
 
-	zbx_hashset_create(&trigger_info, MAX(100, 2 * item_num + timers_num),
-			ZBX_DEFAULT_UINT64_HASH_FUNC, ZBX_DEFAULT_UINT64_COMPARE_FUNC);
-
-	zbx_vector_ptr_create(&trigger_order);
-	zbx_vector_ptr_reserve(&trigger_order, trigger_info.num_slots);
+	zbx_hashset_reserve(trigger_info, MAX(100, 2 * item_num + timers_num));
 
-	zbx_vector_ptr_create(&trigger_items);
+	zbx_vector_ptr_reserve(trigger_order, trigger_info->num_slots);
 
 	if (0 != item_num)
 	{
-		DCconfig_get_triggers_by_itemids(&trigger_info, &trigger_order, itemids, timespecs, item_num);
-		zbx_prepare_triggers((DC_TRIGGER **)trigger_order.values, trigger_order.values_num);
-		zbx_determine_items_in_expressions(&trigger_order, itemids, item_num);
+		DCconfig_get_triggers_by_itemids(trigger_info, trigger_order, itemids, timespecs, item_num);
+		zbx_prepare_triggers((DC_TRIGGER **)trigger_order->values, trigger_order->values_num);
+		zbx_determine_items_in_expressions(trigger_order, itemids, item_num);
 	}
 
 	if (0 != timers_num)
 	{
-		int	offset = trigger_order.values_num;
+		int	offset = trigger_order->values_num;
 
-		zbx_dc_get_triggers_by_timers(&trigger_info, &trigger_order, timers);
+		zbx_dc_get_triggers_by_timers(trigger_info, trigger_order, timers);
 
-		if (offset != trigger_order.values_num)
+		if (offset != trigger_order->values_num)
 		{
-			zbx_prepare_triggers((DC_TRIGGER **)trigger_order.values + offset,
-					trigger_order.values_num - offset);
+			zbx_prepare_triggers((DC_TRIGGER **)trigger_order->values + offset,
+					trigger_order->values_num - offset);
 		}
 	}
 
-	zbx_vector_ptr_sort(&trigger_order, ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC);
-	zbx_evaluate_expressions(&trigger_order, history_itemids, history_items, history_errcodes);
-	zbx_process_triggers(&trigger_order, trigger_diff);
+	zbx_vector_ptr_sort(trigger_order, ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC);
+	zbx_evaluate_expressions(trigger_order, history_itemids, history_items, history_errcodes);
+	zbx_process_triggers(trigger_order, trigger_diff);
 
-	DCfree_triggers(&trigger_order);
+	DCfree_triggers(trigger_order);
 
-	zbx_vector_ptr_destroy(&trigger_items);
-
-	zbx_hashset_destroy(&trigger_info);
-	zbx_vector_ptr_destroy(&trigger_order);
+	zbx_hashset_clear(trigger_info);
+	zbx_vector_ptr_clear(trigger_order);
 out:
-	zbx_free(timespecs);
-	zbx_free(itemids);
-
 	zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __func__);
 }
 
@@ -1829,7 +1813,7 @@ static void	DCinventory_value_add(zbx_vector_ptr_t *inventory_values, const DC_I
 			return;
 	}
 
-	zbx_format_value(value, sizeof(value), item->valuemapid, item->units, h->value_type);
+	zbx_format_value(value, sizeof(value), item->valuemapid, ZBX_NULL2EMPTY_STR(item->units), h->value_type);
 
 	inventory_value = (zbx_inventory_value_t *)zbx_malloc(NULL, sizeof(zbx_inventory_value_t));
 
@@ -2107,7 +2091,7 @@ static zbx_item_diff_t	*calculate_item_update(DC_ITEM *item, const ZBX_DC_HISTOR
 			zbx_add_event(EVENT_SOURCE_INTERNAL, EVENT_OBJECT_ITEM, item->itemid, &h->ts, h->state, NULL,
 					NULL, NULL, 0, 0, NULL, 0, NULL, 0, NULL, NULL, h->value.err);
 
-			if (0 != strcmp(item->error, h->value.err))
+			if (0 != strcmp(ZBX_NULL2EMPTY_STR(item->error), h->value.err))
 				item_error = h->value.err;
 		}
 		else
@@ -2123,7 +2107,7 @@ static zbx_item_diff_t	*calculate_item_update(DC_ITEM *item, const ZBX_DC_HISTOR
 			item_error = "";
 		}
 	}
-	else if (ITEM_STATE_NOTSUPPORTED == h->state && 0 != strcmp(item->error, h->value.err))
+	else if (ITEM_STATE_NOTSUPPORTED == h->state && 0 != strcmp(ZBX_NULL2EMPTY_STR(item->error), h->value.err))
 	{
 		zabbix_log(LOG_LEVEL_WARNING, "error reason for \"%s:%s\" changed: %s", item->host.host,
 				item->key_orig, h->value.err);
@@ -2790,9 +2774,9 @@ static void	DBmass_proxy_add_history(ZBX_DC_HISTORY *history, int history_num)
  *             proxy_subscribtions - [IN] history compression age             *
  *                                                                            *
  ******************************************************************************/
-static void	DCmass_prepare_history(ZBX_DC_HISTORY *history, const zbx_vector_uint64_t *itemids,
-		DC_ITEM *items, const int *errcodes, int history_num, zbx_vector_ptr_t *item_diff,
-		zbx_vector_ptr_t *inventory_values, int compression_age, zbx_vector_uint64_pair_t *proxy_subscribtions)
+static void	DCmass_prepare_history(ZBX_DC_HISTORY *history, DC_ITEM *items, const int *errcodes, int history_num,
+		zbx_vector_ptr_t *item_diff, zbx_vector_ptr_t *inventory_values, int compression_age,
+		zbx_vector_uint64_pair_t *proxy_subscribtions)
 {
 	static time_t	last_history_discard = 0;
 	time_t		now;
@@ -2807,7 +2791,6 @@ static void	DCmass_prepare_history(ZBX_DC_HISTORY *history, const zbx_vector_uin
 		ZBX_DC_HISTORY	*h = &history[i];
 		DC_ITEM		*item;
 		zbx_item_diff_t	*diff;
-		int		index;
 
 		/* discard history items that are older than compression age */
 		if (0 != compression_age && h->ts.sec < compression_age)
@@ -2823,20 +2806,13 @@ static void	DCmass_prepare_history(ZBX_DC_HISTORY *history, const zbx_vector_uin
 			continue;
 		}
 
-		if (FAIL == (index = zbx_vector_uint64_bsearch(itemids, h->itemid, ZBX_DEFAULT_UINT64_COMPARE_FUNC)))
-		{
-			THIS_SHOULD_NEVER_HAPPEN;
-			h->flags |= ZBX_DC_FLAG_UNDEF;
-			continue;
-		}
-
-		if (SUCCEED != errcodes[index])
+		if (SUCCEED != errcodes[i])
 		{
 			h->flags |= ZBX_DC_FLAG_UNDEF;
 			continue;
 		}
 
-		item = &items[index];
+		item = &items[i];
 
 		if (ITEM_STATUS_ACTIVE != item->status || HOST_STATUS_MONITORED != item->host.status)
 		{
@@ -3100,7 +3076,7 @@ static void	proxy_prepare_history(ZBX_DC_HISTORY *history, int history_num)
 	for (i = 0; i < history_num; i++)
 		zbx_vector_uint64_append(&itemids, history[i].itemid);
 
-	items = (DC_ITEM *)zbx_malloc(NULL, sizeof(DC_ITEM) * (size_t)history_num);
+	items = (DC_ITEM *)zbx_calloc(NULL, 1, sizeof(DC_ITEM) * (size_t)history_num);
 	errcodes = (int *)zbx_malloc(NULL, sizeof(int) * (size_t)history_num);
 
 	DCconfig_get_items_by_itemids_partial(items, itemids.values, errcodes, itemids.values_num,
@@ -3253,43 +3229,56 @@ static void	sync_server_history(int *values_num, int *triggers_num, int *more)
 	static ZBX_HISTORY_STRING	*history_string;
 	static ZBX_HISTORY_TEXT		*history_text;
 	static ZBX_HISTORY_LOG		*history_log;
+	static int			module_enabled = FAIL;
 	int				i, history_num, history_float_num, history_integer_num, history_string_num,
 					history_text_num, history_log_num, txn_error, compression_age;
 	unsigned int			item_retrieve_mode;
 	time_t				sync_start;
 	zbx_vector_uint64_t		triggerids ;
-	zbx_vector_ptr_t		history_items, trigger_diff, item_diff, inventory_values, trigger_timers;
+	zbx_vector_ptr_t		history_items, trigger_diff, item_diff, inventory_values, trigger_timers,
+					trigger_order;
 	zbx_vector_uint64_pair_t	trends_diff, proxy_subscribtions;
 	ZBX_DC_HISTORY			history[ZBX_HC_SYNC_MAX];
+	zbx_uint64_t			trigger_itemids[ZBX_HC_SYNC_MAX];
+	zbx_timespec_t			trigger_timespecs[ZBX_HC_SYNC_MAX];
+	DC_ITEM				*items = NULL;
+	int				*errcodes = NULL;
+	zbx_vector_uint64_t		itemids;
+	zbx_hashset_t			trigger_info;
 
 	item_retrieve_mode = NULL == CONFIG_EXPORT_DIR ? ZBX_ITEM_GET_SYNC : ZBX_ITEM_GET_SYNC_EXPORT;
 
 	if (NULL == history_float && NULL != history_float_cbs)
 	{
+		module_enabled = SUCCEED;
 		history_float = (ZBX_HISTORY_FLOAT *)zbx_malloc(history_float,
 				ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_FLOAT));
 	}
 
 	if (NULL == history_integer && NULL != history_integer_cbs)
 	{
+		module_enabled = SUCCEED;
 		history_integer = (ZBX_HISTORY_INTEGER *)zbx_malloc(history_integer,
 				ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_INTEGER));
 	}
 
 	if (NULL == history_string && NULL != history_string_cbs)
 	{
+		module_enabled = SUCCEED;
 		history_string = (ZBX_HISTORY_STRING *)zbx_malloc(history_string,
 				ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_STRING));
 	}
 
 	if (NULL == history_text && NULL != history_text_cbs)
 	{
+		module_enabled = SUCCEED;
 		history_text = (ZBX_HISTORY_TEXT *)zbx_malloc(history_text,
 				ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_TEXT));
 	}
 
 	if (NULL == history_log && NULL != history_log_cbs)
 	{
+		module_enabled = SUCCEED;
 		history_log = (ZBX_HISTORY_LOG *)zbx_malloc(history_log,
 				ZBX_HC_SYNC_MAX * sizeof(ZBX_HISTORY_LOG));
 	}
@@ -3311,17 +3300,18 @@ static void	sync_server_history(int *values_num, int *triggers_num, int *more)
 	zbx_vector_ptr_create(&history_items);
 	zbx_vector_ptr_reserve(&history_items, ZBX_HC_SYNC_MAX);
 
+	zbx_vector_ptr_create(&trigger_order);
+	zbx_hashset_create(&trigger_info, 100, ZBX_DEFAULT_UINT64_HASH_FUNC, ZBX_DEFAULT_UINT64_COMPARE_FUNC);
+
+	zbx_vector_uint64_create(&itemids);
+
 	sync_start = time(NULL);
 
 	do
 	{
-		DC_ITEM			*items;
-		int			*errcodes, trends_num = 0, timers_num = 0, ret = SUCCEED;
-		zbx_vector_uint64_t	itemids;
+		int			trends_num = 0, timers_num = 0, ret = SUCCEED;
 		ZBX_DC_TREND		*trends = NULL;
 
-		zbx_vector_uint64_create(&itemids);
-
 		*more = ZBX_SYNC_DONE;
 
 		LOCK_CACHE;
@@ -3345,24 +3335,26 @@ static void	sync_server_history(int *values_num, int *triggers_num, int *more)
 		{
 			zbx_dc_um_handle_t	*um_handle;
 
+			zbx_vector_ptr_sort(&history_items, ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC);
 			hc_get_item_values(history, &history_items);	/* copy item data from history cache */
 
-			items = (DC_ITEM *)zbx_malloc(NULL, sizeof(DC_ITEM) * (size_t)history_num);
-			errcodes = (int *)zbx_malloc(NULL, sizeof(int) * (size_t)history_num);
+			if (NULL == items)
+				items = (DC_ITEM *)zbx_calloc(NULL, 1, sizeof(DC_ITEM) * (size_t)ZBX_HC_SYNC_MAX);
+
+			if (NULL == errcodes)
+				errcodes = (int *)zbx_malloc(NULL, sizeof(int) * (size_t)ZBX_HC_SYNC_MAX);
 
 			zbx_vector_uint64_reserve(&itemids, history_num);
 
 			for (i = 0; i < history_num; i++)
 				zbx_vector_uint64_append(&itemids, history[i].itemid);
 
-			zbx_vector_uint64_sort(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC);
-
 			DCconfig_get_items_by_itemids_partial(items, itemids.values, errcodes, history_num,
 					item_retrieve_mode);
 
 			um_handle = zbx_dc_open_user_macros();
 
-			DCmass_prepare_history(history, &itemids, items, errcodes, history_num, &item_diff,
+			DCmass_prepare_history(history, items, errcodes, history_num, &item_diff,
 					&inventory_values, compression_age, &proxy_subscribtions);
 
 			if (FAIL != (ret = DBmass_add_history(history, history_num)))
@@ -3430,7 +3422,8 @@ static void	sync_server_history(int *values_num, int *triggers_num, int *more)
 					DBbegin();
 
 					recalculate_triggers(history, history_num, &itemids, items, errcodes,
-							&trigger_timers, &trigger_diff);
+							&trigger_timers, &trigger_diff, trigger_itemids,
+							trigger_timespecs, &trigger_info, &trigger_order);
 
 					/* process trigger events generated by recalculate_triggers() */
 					zbx_process_events(&trigger_diff, &triggerids);
@@ -3501,14 +3494,17 @@ static void	sync_server_history(int *values_num, int *triggers_num, int *more)
 				const ZBX_DC_TREND	*ptrends = NULL;
 				int			history_num_loc = 0, trends_num_loc = 0;
 
-				DCmodule_prepare_history(history, history_num, history_float, &history_float_num,
-						history_integer, &history_integer_num, history_string,
-						&history_string_num, history_text, &history_text_num, history_log,
-						&history_log_num);
-
-				DCmodule_sync_history(history_float_num, history_integer_num, history_string_num,
-						history_text_num, history_log_num, history_float, history_integer,
-						history_string, history_text, history_log);
+				if (SUCCEED == module_enabled)
+				{
+					DCmodule_prepare_history(history, history_num, history_float, &history_float_num,
+							history_integer, &history_integer_num, history_string,
+							&history_string_num, history_text, &history_text_num, history_log,
+							&history_log_num);
+
+					DCmodule_sync_history(history_float_num, history_integer_num, history_string_num,
+							history_text_num, history_log_num, history_float, history_integer,
+							history_string, history_text, history_log);
+				}
 
 				if (SUCCEED == zbx_is_export_enabled(ZBX_FLAG_EXPTYPE_HISTORY))
 				{
@@ -3540,20 +3536,25 @@ static void	sync_server_history(int *values_num, int *triggers_num, int *more)
 		{
 			zbx_free(trends);
 			DCconfig_clean_items(items, errcodes, history_num);
-			zbx_free(errcodes);
-			zbx_free(items);
 
 			zbx_vector_ptr_clear(&history_items);
 			hc_free_item_values(history, history_num);
 		}
 
-		zbx_vector_uint64_destroy(&itemids);
+		zbx_vector_uint64_clear(&itemids);
 
 		/* Exit from sync loop if we have spent too much time here.       */
 		/* This is done to allow syncer process to update its statistics. */
 	}
 	while (ZBX_SYNC_MORE == *more && ZBX_HC_SYNC_TIME_MAX >= time(NULL) - sync_start);
 
+	zbx_free(items);
+	zbx_free(errcodes);
+
+	zbx_vector_ptr_destroy(&trigger_order);
+	zbx_hashset_destroy(&trigger_info);
+
+	zbx_vector_uint64_destroy(&itemids);
 	zbx_vector_ptr_destroy(&history_items);
 	zbx_vector_ptr_destroy(&inventory_values);
 	zbx_vector_ptr_destroy(&item_diff);
diff --git a/src/libs/zbxcommon/common_str.c b/src/libs/zbxcommon/common_str.c
index 7d59e6faa33..df64d6fe9b2 100644
--- a/src/libs/zbxcommon/common_str.c
+++ b/src/libs/zbxcommon/common_str.c
@@ -125,17 +125,21 @@ char	*zbx_dsprintf(char *dest, const char *f, ...)
  ******************************************************************************/
 size_t	zbx_strlcpy(char *dst, const char *src, size_t siz)
 {
-	const char	*s = src;
+	size_t	len = strlen(src);
 
-	if (0 != siz)
+	if (len + 1 <= siz)
 	{
-		while (0 != --siz && '\0' != *s)
-			*dst++ = *s++;
-
-		*dst = '\0';
+		memcpy(dst, src, len + 1);
+		return len;
 	}
 
-	return s - src;	/* count does not include null */
+	if (0 == siz)
+		return 0;
+
+	memcpy(dst, src, siz - 1);
+	dst[siz - 1] = '\0';
+
+	return siz - 1;
 }
 
 /******************************************************************************
diff --git a/src/libs/zbxdbwrap/proxy.c b/src/libs/zbxdbwrap/proxy.c
index 6a731bad5d4..ec0111fac3d 100644
--- a/src/libs/zbxdbwrap/proxy.c
+++ b/src/libs/zbxdbwrap/proxy.c
@@ -1789,7 +1789,7 @@ static int	process_history_data_by_itemids(zbx_socket_t *sock, zbx_client_item_v
 
 	zabbix_log(LOG_LEVEL_DEBUG, "In %s()", __func__);
 
-	items = (DC_ITEM *)zbx_malloc(NULL, sizeof(DC_ITEM) * ZBX_HISTORY_VALUES_MAX);
+	items = (DC_ITEM *)zbx_calloc(NULL, 1, sizeof(DC_ITEM) * ZBX_HISTORY_VALUES_MAX);
 	errcodes = (int *)zbx_malloc(NULL, sizeof(int) * ZBX_HISTORY_VALUES_MAX);
 
 	sec = zbx_time();
diff --git a/src/libs/zbxserver/expression.c b/src/libs/zbxserver/expression.c
index a6cba156ae6..a1a61bb9628 100644
--- a/src/libs/zbxserver/expression.c
+++ b/src/libs/zbxserver/expression.c
@@ -5091,7 +5091,6 @@ void	zbx_determine_items_in_expressions(zbx_vector_ptr_t *trigger_order, const z
 
 	zbx_vector_uint64_create(&itemids_sorted);
 	zbx_vector_uint64_append_array(&itemids_sorted, itemids, item_num);
-	zbx_vector_uint64_sort(&itemids_sorted, ZBX_DEFAULT_UINT64_COMPARE_FUNC);
 
 	zbx_vector_ptr_create(&triggers_func_pos);
 	zbx_vector_ptr_reserve(&triggers_func_pos, trigger_order->values_num);
@@ -5304,7 +5303,7 @@ static void	zbx_evaluate_item_functions(zbx_hashset_t *funcs, const zbx_vector_u
 		zbx_vector_uint64_uniq(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC);
 
 		*items_num = itemids.values_num;
-		*items = (DC_ITEM *)zbx_malloc(NULL, sizeof(DC_ITEM) * (size_t)itemids.values_num);
+		*items = (DC_ITEM *)zbx_calloc(NULL, 1, sizeof(DC_ITEM) * (size_t)itemids.values_num);
 		*items_err = (int *)zbx_malloc(NULL, sizeof(int) * (size_t)itemids.values_num);
 
 		DCconfig_get_items_by_itemids_partial(*items, itemids.values, *items_err, itemids.values_num,