diff options
| -rw-r--r-- | ChangeLog.d/bugfix/ZBX-19307 | 1 | ||||
| -rw-r--r-- | include/dbcache.h | 22 | ||||
| -rw-r--r-- | include/zbxserver.h | 3 | ||||
| -rw-r--r-- | src/libs/zbxdbcache/dbcache.c | 39 | ||||
| -rw-r--r-- | src/libs/zbxdbcache/dbconfig.c | 271 | ||||
| -rw-r--r-- | src/libs/zbxdbhigh/proxy.c | 9 | ||||
| -rw-r--r-- | src/libs/zbxserver/expression.c | 76 |
7 files changed, 303 insertions, 118 deletions
diff --git a/ChangeLog.d/bugfix/ZBX-19307 b/ChangeLog.d/bugfix/ZBX-19307 new file mode 100644 index 00000000000..149d799d35a --- /dev/null +++ b/ChangeLog.d/bugfix/ZBX-19307 @@ -0,0 +1 @@ +........S. [ZBX-19307] improved performance of history syncers by reducing time spent in locks when syncing history and receiving new history from proxy (vso) diff --git a/include/dbcache.h b/include/dbcache.h index d5af39b9fe8..3a01dfca9fe 100644 --- a/include/dbcache.h +++ b/include/dbcache.h @@ -650,6 +650,26 @@ zbx_uint64_t DCget_nextid(const char *table_name, int num); #define ZBX_DBSYNC_UPDATE 1 #define ZBX_SYNC_SECRETS 2 +#define ZBX_ITEM_GET_MISC 0x001 +#define ZBX_ITEM_GET_DELAY 0x002 +#define ZBX_ITEM_GET_EMPTY_ERROR 0x004 +#define ZBX_ITEM_GET_NUM 0x008 +#define ZBX_ITEM_GET_EMPTY_UNITS 0x010 +#define ZBX_ITEM_GET_LOGTIMEFMT 0x020 +#define ZBX_ITEM_GET_POLLINFO 0x040 +#define ZBX_ITEM_GET_INTERFACE 0x080 +#define ZBX_ITEM_GET_HOSTNAME 0x100 +#define ZBX_ITEM_GET_HOSTINFO 0x200 +#define ZBX_ITEM_GET_MAINTENANCE 0x400 +#define ZBX_ITEM_GET_INVENTORY 0x800 + +#define ZBX_ITEM_GET_ALL (~(unsigned int)0) + +#define ZBX_ITEM_GET_SYNC (ZBX_ITEM_GET_INVENTORY|ZBX_ITEM_GET_NUM) +#define ZBX_ITEM_GET_SYNC_EXPORT (ZBX_ITEM_GET_INVENTORY|ZBX_ITEM_GET_NUM|ZBX_ITEM_GET_HOSTNAME) + +#define ZBX_ITEM_GET_PROCESS (ZBX_ITEM_GET_MAINTENANCE|ZBX_ITEM_GET_MISC|ZBX_ITEM_GET_LOGTIMEFMT) + void DCsync_configuration(unsigned char mode, const struct zbx_json_parse *jp_kvs_paths); int init_configuration_cache(char **error); void free_configuration_cache(void); @@ -662,6 +682,8 @@ int DCconfig_get_hostid_by_name(const char *host, zbx_uint64_t *hostid); void DCconfig_get_hosts_by_itemids(DC_HOST *hosts, const zbx_uint64_t *itemids, int *errcodes, size_t num); void DCconfig_get_items_by_keys(DC_ITEM *items, zbx_host_key_t *keys, int *errcodes, size_t num); void DCconfig_get_items_by_itemids(DC_ITEM *items, const zbx_uint64_t *itemids, int *errcodes, size_t num); +void DCconfig_get_items_by_itemids_partial(DC_ITEM *items, const zbx_uint64_t *itemids, int *errcodes, size_t num, + unsigned int mode); void DCconfig_get_preprocessable_items(zbx_hashset_t *items, int *timestamp); void DCconfig_get_functions_by_functionids(DC_FUNCTION *functions, zbx_uint64_t *functionids, int *errcodes, size_t num); diff --git a/include/zbxserver.h b/include/zbxserver.h index 77e534feb41..f41f19121f5 100644 --- a/include/zbxserver.h +++ b/include/zbxserver.h @@ -78,7 +78,8 @@ int substitute_simple_macros_unmasked(const zbx_uint64_t *actionid, const DB_EVE const DB_ALERT *alert, const DB_ACKNOWLEDGE *ack, const char *tz, char **data, int macro_type, char *error, int maxerrlen); -void evaluate_expressions(zbx_vector_ptr_t *triggers); +void evaluate_expressions(zbx_vector_ptr_t *triggers, const zbx_vector_uint64_t *history_itemids, + const DC_ITEM *history_items, const int *history_errcodes); void prepare_triggers(DC_TRIGGER **triggers, int triggers_num); void zbx_format_value(char *value, size_t max_len, zbx_uint64_t valuemapid, diff --git a/src/libs/zbxdbcache/dbcache.c b/src/libs/zbxdbcache/dbcache.c index b453671a966..b3a2bfcb4bd 100644 --- a/src/libs/zbxdbcache/dbcache.c +++ b/src/libs/zbxdbcache/dbcache.c @@ -62,6 +62,7 @@ static size_t sql_alloc = 4 * ZBX_KIBIBYTE; extern unsigned char program_type; extern int CONFIG_DOUBLE_PRECISION; +extern char *CONFIG_EXPORT_DIR; #define ZBX_IDS_SIZE 10 @@ -1597,8 +1598,9 @@ static void DCsync_trends(void) * timers_num - [OUT] processed timer triggers * * * ******************************************************************************/ -static void recalculate_triggers(const ZBX_DC_HISTORY *history, int history_num, const zbx_vector_ptr_t *timers, - zbx_vector_ptr_t *trigger_diff) +static void recalculate_triggers(const ZBX_DC_HISTORY *history, int history_num, + const zbx_vector_uint64_t *history_itemids, const DC_ITEM *history_items, const int *history_errcodes, + const zbx_vector_ptr_t *timers, zbx_vector_ptr_t *trigger_diff) { int i, item_num = 0, timers_num = 0; zbx_uint64_t *itemids = NULL; @@ -1667,7 +1669,7 @@ static void recalculate_triggers(const ZBX_DC_HISTORY *history, int history_num, } zbx_vector_ptr_sort(&trigger_order, ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC); - evaluate_expressions(&trigger_order); + evaluate_expressions(&trigger_order, history_itemids, history_items, history_errcodes); zbx_process_triggers(&trigger_order, trigger_diff); DCfree_triggers(&trigger_order); @@ -1979,10 +1981,12 @@ static void normalize_item_value(const DC_ITEM *item, ZBX_DC_HISTORY *hdata) ******************************************************************************/ static zbx_item_diff_t *calculate_item_update(const DC_ITEM *item, const ZBX_DC_HISTORY *h) { - zbx_uint64_t flags = ZBX_FLAGS_ITEM_DIFF_UPDATE_LASTCLOCK; + zbx_uint64_t flags; const char *item_error = NULL; zbx_item_diff_t *diff; + flags = item->host.proxy_hostid == 0 ? ZBX_FLAGS_ITEM_DIFF_UPDATE_LASTCLOCK : 0; + if (0 != (ZBX_DC_FLAG_META & h->flags)) { if (item->lastlogsize != h->lastlogsize) @@ -2031,6 +2035,9 @@ static zbx_item_diff_t *calculate_item_update(const DC_ITEM *item, const ZBX_DC_ if (NULL != item_error) flags |= ZBX_FLAGS_ITEM_DIFF_UPDATE_ERROR; + if (0 == flags) + return NULL; + diff = (zbx_item_diff_t *)zbx_malloc(NULL, sizeof(zbx_item_diff_t)); diff->itemid = item->itemid; diff->lastclock = h->ts.sec; @@ -2622,8 +2629,9 @@ static void DCmass_prepare_history(ZBX_DC_HISTORY *history, const zbx_vector_uin normalize_item_value(item, h); - diff = calculate_item_update(item, h); - zbx_vector_ptr_append(item_diff, diff); + if (NULL != (diff = calculate_item_update(item, h))) + zbx_vector_ptr_append(item_diff, diff); + DCinventory_value_add(inventory_values, item, h); if (0 != item->host.proxy_hostid && FAIL == is_item_processed_by_server(item->type, item->key_orig)) @@ -2991,6 +2999,9 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) zbx_vector_ptr_t history_items, trigger_diff, item_diff, inventory_values, trigger_timers; zbx_vector_uint64_pair_t trends_diff, proxy_subscribtions; ZBX_DC_HISTORY history[ZBX_HC_SYNC_MAX]; + int item_retrieve_mode; + + item_retrieve_mode = NULL == CONFIG_EXPORT_DIR ? ZBX_ITEM_GET_SYNC : ZBX_ITEM_GET_SYNC_EXPORT; if (NULL == history_float && NULL != history_float_cbs) { @@ -3048,6 +3059,8 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) zbx_vector_uint64_t itemids; ZBX_DC_TREND *trends = NULL; + zbx_vector_uint64_create(&itemids); + *more = ZBX_SYNC_DONE; LOCK_CACHE; @@ -3074,7 +3087,6 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) items = (DC_ITEM *)zbx_malloc(NULL, sizeof(DC_ITEM) * (size_t)history_num); errcodes = (int *)zbx_malloc(NULL, sizeof(int) * (size_t)history_num); - zbx_vector_uint64_create(&itemids); zbx_vector_uint64_reserve(&itemids, history_num); for (i = 0; i < history_num; i++) @@ -3082,7 +3094,8 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) zbx_vector_uint64_sort(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); - DCconfig_get_items_by_itemids(items, itemids.values, errcodes, history_num); + DCconfig_get_items_by_itemids_partial(items, itemids.values, errcodes, history_num, + item_retrieve_mode); DCmass_prepare_history(history, &itemids, items, errcodes, history_num, &item_diff, &inventory_values, compression_age, &proxy_subscribtions); @@ -3149,7 +3162,8 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) { DBbegin(); - recalculate_triggers(history, history_num, &trigger_timers, &trigger_diff); + recalculate_triggers(history, history_num, &itemids, items, errcodes, + &trigger_timers, &trigger_diff); /* process trigger events generated by recalculate_triggers() */ zbx_process_events(&trigger_diff, &triggerids); @@ -3261,7 +3275,6 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) if (0 != history_num) { zbx_free(trends); - zbx_vector_uint64_destroy(&itemids); DCconfig_clean_items(items, errcodes, history_num); zbx_free(errcodes); zbx_free(items); @@ -3270,6 +3283,8 @@ static void sync_server_history(int *values_num, int *triggers_num, int *more) hc_free_item_values(history, history_num); } + zbx_vector_uint64_destroy(&itemids); + /* Exit from sync loop if we have spent too much time here. */ /* This is done to allow syncer process to update its statistics. */ } @@ -4364,6 +4379,7 @@ int hc_queue_get_size(void) int hc_get_history_compression_age(void) { +#if defined(HAVE_POSTGRESQL) zbx_config_t cfg; int compression_age = 0; @@ -4377,6 +4393,9 @@ int hc_get_history_compression_age(void) zbx_config_clean(&cfg); return compression_age; +#else + return 0; +#endif } /****************************************************************************** diff --git a/src/libs/zbxdbcache/dbconfig.c b/src/libs/zbxdbcache/dbconfig.c index 1fece7d4813..f9363db236a 100644 --- a/src/libs/zbxdbcache/dbconfig.c +++ b/src/libs/zbxdbcache/dbconfig.c @@ -7234,55 +7234,69 @@ int in_maintenance_without_data_collection(unsigned char maintenance_status, uns return SUCCEED; } -static void DCget_host(DC_HOST *dst_host, const ZBX_DC_HOST *src_host) +static void DCget_host(DC_HOST *dst_host, const ZBX_DC_HOST *src_host, unsigned int mode) { const ZBX_DC_IPMIHOST *ipmihost; const ZBX_DC_HOST_INVENTORY *host_inventory; dst_host->hostid = src_host->hostid; dst_host->proxy_hostid = src_host->proxy_hostid; - strscpy(dst_host->host, src_host->host); - zbx_strlcpy_utf8(dst_host->name, src_host->name, sizeof(dst_host->name)); - dst_host->maintenance_status = src_host->maintenance_status; - dst_host->maintenance_type = src_host->maintenance_type; - dst_host->maintenance_from = src_host->maintenance_from; dst_host->status = src_host->status; - dst_host->tls_connect = src_host->tls_connect; - dst_host->tls_accept = src_host->tls_accept; -#if defined(HAVE_GNUTLS) || defined(HAVE_OPENSSL) - strscpy(dst_host->tls_issuer, src_host->tls_issuer); - strscpy(dst_host->tls_subject, src_host->tls_subject); - if (NULL == src_host->tls_dc_psk) - { - *dst_host->tls_psk_identity = '\0'; - *dst_host->tls_psk = '\0'; - } - else + strscpy(dst_host->host, src_host->host); + + if (ZBX_ITEM_GET_HOSTNAME & mode) + zbx_strlcpy_utf8(dst_host->name, src_host->name, sizeof(dst_host->name)); + + if (ZBX_ITEM_GET_MAINTENANCE & mode) { - strscpy(dst_host->tls_psk_identity, src_host->tls_dc_psk->tls_psk_identity); - strscpy(dst_host->tls_psk, src_host->tls_dc_psk->tls_psk); + dst_host->maintenance_status = src_host->maintenance_status; + dst_host->maintenance_type = src_host->maintenance_type; + dst_host->maintenance_from = src_host->maintenance_from; } -#endif - if (NULL != (ipmihost = (ZBX_DC_IPMIHOST *)zbx_hashset_search(&config->ipmihosts, &src_host->hostid))) + + if (ZBX_ITEM_GET_HOSTINFO & mode) { - dst_host->ipmi_authtype = ipmihost->ipmi_authtype; - dst_host->ipmi_privilege = ipmihost->ipmi_privilege; - strscpy(dst_host->ipmi_username, ipmihost->ipmi_username); - strscpy(dst_host->ipmi_password, ipmihost->ipmi_password); + dst_host->tls_connect = src_host->tls_connect; + dst_host->tls_accept = src_host->tls_accept; + #if defined(HAVE_GNUTLS) || defined(HAVE_OPENSSL) + strscpy(dst_host->tls_issuer, src_host->tls_issuer); + strscpy(dst_host->tls_subject, src_host->tls_subject); + + if (NULL == src_host->tls_dc_psk) + { + *dst_host->tls_psk_identity = '\0'; + *dst_host->tls_psk = '\0'; + } + else + { + strscpy(dst_host->tls_psk_identity, src_host->tls_dc_psk->tls_psk_identity); + strscpy(dst_host->tls_psk, src_host->tls_dc_psk->tls_psk); + } + #endif + if (NULL != (ipmihost = (ZBX_DC_IPMIHOST *)zbx_hashset_search(&config->ipmihosts, &src_host->hostid))) + { + dst_host->ipmi_authtype = ipmihost->ipmi_authtype; + dst_host->ipmi_privilege = ipmihost->ipmi_privilege; + strscpy(dst_host->ipmi_username, ipmihost->ipmi_username); + strscpy(dst_host->ipmi_password, ipmihost->ipmi_password); + } + else + { + dst_host->ipmi_authtype = ZBX_IPMI_DEFAULT_AUTHTYPE; + dst_host->ipmi_privilege = ZBX_IPMI_DEFAULT_PRIVILEGE; + *dst_host->ipmi_username = '\0'; + *dst_host->ipmi_password = '\0'; + } } - else + + if (ZBX_ITEM_GET_INVENTORY & mode) { - dst_host->ipmi_authtype = ZBX_IPMI_DEFAULT_AUTHTYPE; - dst_host->ipmi_privilege = ZBX_IPMI_DEFAULT_PRIVILEGE; - *dst_host->ipmi_username = '\0'; - *dst_host->ipmi_password = '\0'; + if (NULL != (host_inventory = (ZBX_DC_HOST_INVENTORY *)zbx_hashset_search(&config->host_inventories, &src_host->hostid))) + dst_host->inventory_mode = (char)host_inventory->inventory_mode; + else + dst_host->inventory_mode = HOST_INVENTORY_DISABLED; } - - if (NULL != (host_inventory = (ZBX_DC_HOST_INVENTORY *)zbx_hashset_search(&config->host_inventories, &src_host->hostid))) - dst_host->inventory_mode = (char)host_inventory->inventory_mode; - else - dst_host->inventory_mode = HOST_INVENTORY_DISABLED; } /****************************************************************************** @@ -7308,7 +7322,7 @@ int DCget_host_by_hostid(DC_HOST *host, zbx_uint64_t hostid) if (NULL != (dc_host = (ZBX_DC_HOST *)zbx_hashset_search(&config->hosts, &hostid))) { - DCget_host(host, dc_host); + DCget_host(host, dc_host, ZBX_ITEM_GET_ALL); ret = SUCCEED; } @@ -7588,7 +7602,7 @@ static void DCget_interface(DC_INTERFACE *dst_interface, const ZBX_DC_INTERFACE dst_interface->port = 0; } -static void DCget_item(DC_ITEM *dst_item, const ZBX_DC_ITEM *src_item) +static void DCget_item(DC_ITEM *dst_item, const ZBX_DC_ITEM *src_item, unsigned int mode) { const ZBX_DC_NUMITEM *numitem; const ZBX_DC_LOGITEM *logitem; @@ -7606,44 +7620,77 @@ static void DCget_item(DC_ITEM *dst_item, const ZBX_DC_ITEM *src_item) const ZBX_DC_HTTPITEM *httpitem; const ZBX_DC_SCRIPTITEM *scriptitem; - dst_item->itemid = src_item->itemid; dst_item->type = src_item->type; dst_item->value_type = src_item->value_type; - strscpy(dst_item->key_orig, src_item->key); - dst_item->key = NULL; - dst_item->delay = zbx_strdup(NULL, src_item->delay); - dst_item->nextcheck = src_item->nextcheck; + dst_item->state = src_item->state; - dst_item->lastclock = src_item->lastclock; - dst_item->flags = src_item->flags; dst_item->lastlogsize = src_item->lastlogsize; dst_item->mtime = src_item->mtime; + dst_item->history = src_item->history; + dst_item->inventory_link = src_item->inventory_link; dst_item->valuemapid = src_item->valuemapid; dst_item->status = src_item->status; + dst_item->history_sec = src_item->history_sec; + strscpy(dst_item->key_orig, src_item->key); + + if (ZBX_ITEM_GET_MISC & mode) + { + dst_item->itemid = src_item->itemid; /* set after lock */ + dst_item->flags = src_item->flags; + dst_item->nextcheck = src_item->nextcheck; + dst_item->lastclock = src_item->lastclock; + dst_item->key = NULL; /* set during initialization */ + } - dst_item->error = zbx_strdup(NULL, src_item->error); + if (ZBX_ITEM_GET_DELAY & mode) + dst_item->delay = zbx_strdup(NULL, src_item->delay); /* not used, should be initialized */ + + if ((ZBX_ITEM_GET_EMPTY_ERROR & mode) || '\0' != *src_item->error) /* allocate after lock */ + dst_item->error = zbx_strdup(NULL, src_item->error); switch (src_item->value_type) { case ITEM_VALUE_TYPE_FLOAT: case ITEM_VALUE_TYPE_UINT64: - numitem = (ZBX_DC_NUMITEM *)zbx_hashset_search(&config->numitems, &src_item->itemid); + if (0 != (ZBX_ITEM_GET_NUM & mode)) + { + numitem = (ZBX_DC_NUMITEM *)zbx_hashset_search(&config->numitems, &src_item->itemid); - dst_item->trends = numitem->trends; - dst_item->trends_sec = numitem->trends_sec; - dst_item->units = zbx_strdup(NULL, numitem->units); + dst_item->trends = numitem->trends; + dst_item->trends_sec = numitem->trends_sec; + + /* allocate after lock */ + if (0 != (ZBX_ITEM_GET_EMPTY_UNITS & mode) || '\0' != *numitem->units) + dst_item->units = zbx_strdup(NULL, numitem->units); + } break; case ITEM_VALUE_TYPE_LOG: - if (NULL != (logitem = (ZBX_DC_LOGITEM *)zbx_hashset_search(&config->logitems, &src_item->itemid))) - strscpy(dst_item->logtimefmt, logitem->logtimefmt); - else - *dst_item->logtimefmt = '\0'; + if (ZBX_ITEM_GET_LOGTIMEFMT & mode) + { + if (NULL != (logitem = (ZBX_DC_LOGITEM *)zbx_hashset_search(&config->logitems, + &src_item->itemid))) + { + strscpy(dst_item->logtimefmt, logitem->logtimefmt); + } + else + *dst_item->logtimefmt = '\0'; + } break; } + if (ZBX_ITEM_GET_INTERFACE & mode) /* not used by history syncer */ + { + dc_interface = (ZBX_DC_INTERFACE *)zbx_hashset_search(&config->interfaces, &src_item->interfaceid); + + DCget_interface(&dst_item->interface, dc_interface); + } + + if (0 == (ZBX_ITEM_GET_POLLINFO & mode)) /* not used by history syncer */ + return; + switch (src_item->type) { case ITEM_TYPE_SNMP: @@ -7895,10 +7942,6 @@ static void DCget_item(DC_ITEM *dst_item, const ZBX_DC_ITEM *src_item) default: /* nothing to do */; } - - dc_interface = (ZBX_DC_INTERFACE *)zbx_hashset_search(&config->interfaces, &src_item->interfaceid); - - DCget_interface(&dst_item->interface, dc_interface); } void DCconfig_clean_items(DC_ITEM *items, int *errcodes, size_t num) @@ -8085,8 +8128,8 @@ void DCconfig_get_items_by_keys(DC_ITEM *items, zbx_host_key_t *keys, int *errco continue; } - DCget_host(&items[i].host, dc_host); - DCget_item(&items[i], dc_item); + DCget_host(&items[i].host, dc_host, ZBX_ITEM_GET_ALL); + DCget_item(&items[i], dc_item, ZBX_ITEM_GET_ALL); errcodes[i] = SUCCEED; } @@ -8144,14 +8187,68 @@ void DCconfig_get_items_by_itemids(DC_ITEM *items, const zbx_uint64_t *itemids, continue; } - DCget_host(&items[i].host, dc_host); - DCget_item(&items[i], dc_item); + DCget_host(&items[i].host, dc_host, ZBX_ITEM_GET_ALL); + DCget_item(&items[i], dc_item, ZBX_ITEM_GET_ALL); errcodes[i] = SUCCEED; } UNLOCK_CACHE; } +void DCconfig_get_items_by_itemids_partial(DC_ITEM *items, const zbx_uint64_t *itemids, int *errcodes, size_t num, + unsigned int mode) +{ + size_t i; + const ZBX_DC_ITEM *dc_item; + const ZBX_DC_HOST *dc_host = NULL; + + memset(items, 0, sizeof(DC_ITEM) * (size_t)num); + memset(errcodes, 0, sizeof(int) * (size_t)num); + + RDLOCK_CACHE; + + for (i = 0; i < num; i++) + { + if (NULL == (dc_item = (ZBX_DC_ITEM *)zbx_hashset_search(&config->items, &itemids[i]))) + { + errcodes[i] = FAIL; + continue; + } + + if (NULL == dc_host || dc_host->hostid != dc_item->hostid) + { + if (NULL == (dc_host = (ZBX_DC_HOST *)zbx_hashset_search(&config->hosts, &dc_item->hostid))) + { + errcodes[i] = FAIL; + continue; + } + } + + DCget_host(&items[i].host, dc_host, mode); + DCget_item(&items[i], dc_item, mode); + } + + UNLOCK_CACHE; + + /* avoid unnecessary allocations inside lock if there are no error or units */ + for (i = 0; i < num; i++) + { + if (FAIL == errcodes[i]) + continue; + + items[i].itemid = itemids[i]; + + if (NULL == items[i].error) + items[i].error = zbx_strdup(NULL, ""); + + if (ITEM_VALUE_TYPE_FLOAT == items[i].value_type || ITEM_VALUE_TYPE_UINT64 == items[i].value_type) + { + if (NULL == items[i].units) + items[i].units = zbx_strdup(NULL, ""); + } + } +} + /****************************************************************************** * * * Function: dc_preproc_item_init * @@ -8319,7 +8416,7 @@ void DCconfig_get_hosts_by_itemids(DC_HOST *hosts, const zbx_uint64_t *itemids, continue; } - DCget_host(&hosts[i], dc_host); + DCget_host(&hosts[i], dc_host, ZBX_ITEM_GET_ALL); errcodes[i] = SUCCEED; } @@ -8554,7 +8651,8 @@ void DCconfig_unlock_triggers(const zbx_vector_uint64_t *triggerids) int i; ZBX_DC_TRIGGER *dc_trigger; - WRLOCK_CACHE; + /* no other process can modify already locked triggers without write lock */ + RDLOCK_CACHE; for (i = 0; i < triggerids->values_num; i++) { @@ -8844,14 +8942,30 @@ static int trigger_timer_validate(zbx_trigger_timer_t *timer, ZBX_DC_TRIGGER **d ******************************************************************************/ void zbx_dc_get_trigger_timers(zbx_vector_ptr_t *timers, int now, int soft_limit, int hard_limit) { - zbx_trigger_timer_t *first_timer = NULL; + zbx_trigger_timer_t *first_timer = NULL, *timer; + int found = 0; + zbx_binary_heap_elem_t *elem; + + RDLOCK_CACHE; + + if (SUCCEED != zbx_binary_heap_empty(&config->trigger_queue)) + { + elem = zbx_binary_heap_find_min(&config->trigger_queue); + timer = (zbx_trigger_timer_t *)elem->data; + + if (timer->exec_ts.sec <= now) + found = 1; + } + + UNLOCK_CACHE; + + if (0 == found) + return; WRLOCK_CACHE; while (SUCCEED != zbx_binary_heap_empty(&config->trigger_queue) && timers->values_num < hard_limit) { - zbx_binary_heap_elem_t *elem; - zbx_trigger_timer_t *timer; ZBX_DC_TRIGGER *dc_trigger; elem = zbx_binary_heap_find_min(&config->trigger_queue); @@ -9474,8 +9588,8 @@ int DCconfig_get_poller_items(unsigned char poller_type, DC_ITEM **items) dc_item_prev = dc_item; dc_item->location = ZBX_LOC_POLLER; - DCget_host(&(*items)[num].host, dc_host); - DCget_item(&(*items)[num], dc_item); + DCget_host(&(*items)[num].host, dc_host, ZBX_ITEM_GET_ALL); + DCget_item(&(*items)[num], dc_item, ZBX_ITEM_GET_ALL); num++; } @@ -9567,8 +9681,8 @@ int DCconfig_get_ipmi_poller_items(int now, DC_ITEM *items, int items_num, int * } dc_item->location = ZBX_LOC_POLLER; - DCget_host(&items[num].host, dc_host); - DCget_item(&items[num], dc_item); + DCget_host(&items[num].host, dc_host, ZBX_ITEM_GET_ALL); + DCget_item(&items[num], dc_item, ZBX_ITEM_GET_ALL); num++; } @@ -9677,8 +9791,8 @@ size_t DCconfig_get_snmp_items_by_interfaceid(zbx_uint64_t interfaceid, DC_ITEM *items = (DC_ITEM *)zbx_realloc(*items, items_alloc * sizeof(DC_ITEM)); } - DCget_host(&(*items)[items_num].host, dc_host); - DCget_item(&(*items)[items_num], dc_item); + DCget_host(&(*items)[items_num].host, dc_host, ZBX_ITEM_GET_ALL); + DCget_item(&(*items)[items_num], dc_item, ZBX_ITEM_GET_ALL); items_num++; } unlock: @@ -11965,7 +12079,7 @@ static void dc_get_hosts_by_functionids(const zbx_uint64_t *functionids, int fun if (NULL == (dc_host = (const ZBX_DC_HOST *)zbx_hashset_search(&config->hosts, &dc_item->hostid))) continue; - DCget_host(&host, dc_host); + DCget_host(&host, dc_host, ZBX_ITEM_GET_ALL); zbx_hashset_insert(hosts, &host, sizeof(host)); } } @@ -12984,13 +13098,17 @@ void zbx_dc_items_update_nextcheck(DC_ITEM *items, zbx_agent_value_t *values, in ZBX_DC_HOST *dc_host; ZBX_DC_INTERFACE *dc_interface; - WRLOCK_CACHE; + RDLOCK_CACHE; for (i = 0; i < values_num; i++) { if (FAIL == errcodes[i]) continue; + /* update nextcheck for items that are counted in queue for monitoring purposes */ + if (FAIL == zbx_is_counted_in_item_queue(items[i].type, items[i].key_orig)) + continue; + if (NULL == (dc_item = (ZBX_DC_ITEM *)zbx_hashset_search(&config->items, &items[i].itemid))) continue; @@ -13009,9 +13127,8 @@ void zbx_dc_items_update_nextcheck(DC_ITEM *items, zbx_agent_value_t *values, in dc_interface = (ZBX_DC_INTERFACE *)zbx_hashset_search(&config->interfaces, &dc_item->interfaceid); /* update nextcheck for items that are counted in queue for monitoring purposes */ - if (SUCCEED == zbx_is_counted_in_item_queue(dc_item->type, dc_item->key)) - DCitem_nextcheck_update(dc_item, dc_interface, ZBX_ITEM_COLLECTED, values[i].ts.sec, - NULL); + DCitem_nextcheck_update(dc_item, dc_interface, ZBX_ITEM_COLLECTED, values[i].ts.sec, + NULL); } UNLOCK_CACHE; diff --git a/src/libs/zbxdbhigh/proxy.c b/src/libs/zbxdbhigh/proxy.c index 62b0177d28f..76c0e7acb27 100644 --- a/src/libs/zbxdbhigh/proxy.c +++ b/src/libs/zbxdbhigh/proxy.c @@ -3540,7 +3540,7 @@ static int proxy_item_validator(DC_ITEM *item, zbx_socket_t *sock, void *args, c ******************************************************************************/ static int process_history_data_by_itemids(zbx_socket_t *sock, zbx_client_item_validator_t validator_func, void *validator_args, struct zbx_json_parse *jp_data, zbx_data_session_t *session, - zbx_proxy_suppress_t *nodata_win, char **info) + zbx_proxy_suppress_t *nodata_win, char **info, int mode) { const char *pnext = NULL; int ret = SUCCEED, processed_num = 0, total_num = 0, values_num, read_num, i, *errcodes; @@ -3561,7 +3561,7 @@ static int process_history_data_by_itemids(zbx_socket_t *sock, zbx_client_item_v while (SUCCEED == parse_history_data_by_itemids(jp_data, &pnext, values, itemids, &values_num, &read_num, &unique_shift, &error) && 0 != values_num) { - DCconfig_get_items_by_itemids(items, itemids, errcodes, values_num); + DCconfig_get_items_by_itemids_partial(items, itemids, errcodes, (size_t)values_num, mode); for (i = 0; i < values_num; i++) { @@ -3903,7 +3903,7 @@ static int process_client_history_data(zbx_socket_t *sock, struct zbx_json_parse session = zbx_dc_get_or_create_data_session(hostid, token); if (SUCCEED != (ret = process_history_data_by_itemids(sock, validator_func, validator_args, &jp_data, - session, NULL, info))) + session, NULL, info, ZBX_ITEM_GET_ALL))) { goto out; } @@ -4758,7 +4758,8 @@ int process_proxy_data(const DC_PROXY *proxy, struct zbx_json_parse *jp, zbx_tim } if (SUCCEED != (ret = process_history_data_by_itemids(NULL, proxy_item_validator, - (void *)&proxy->hostid, &jp_data, session, &proxy_diff.nodata_win, &error_step))) + (void *)&proxy->hostid, &jp_data, session, &proxy_diff.nodata_win, &error_step, + ZBX_ITEM_GET_PROCESS))) { zbx_strcatnl_alloc(error, &error_alloc, &error_offset, error_step); } diff --git a/src/libs/zbxserver/expression.c b/src/libs/zbxserver/expression.c index 8df3d2425c0..4a2f5195feb 100644 --- a/src/libs/zbxserver/expression.c +++ b/src/libs/zbxserver/expression.c @@ -4954,7 +4954,8 @@ static void zbx_populate_function_items(const zbx_vector_uint64_t *functionids, zabbix_log(LOG_LEVEL_DEBUG, "End of %s() ifuncs_num:%d", __func__, ifuncs->num_data); } -static void zbx_evaluate_item_functions(zbx_hashset_t *funcs) +static void zbx_evaluate_item_functions(zbx_hashset_t *funcs, const zbx_vector_uint64_t *history_itemids, + const DC_ITEM *history_items, const int *history_errcodes) { DC_ITEM *items = NULL; char *error = NULL; @@ -4967,26 +4968,47 @@ static void zbx_evaluate_item_functions(zbx_hashset_t *funcs) zabbix_log(LOG_LEVEL_DEBUG, "In %s() funcs_num:%d", __func__, funcs->num_data); zbx_vector_uint64_create(&itemids); - zbx_vector_uint64_reserve(&itemids, funcs->num_data); zbx_hashset_iter_reset(funcs, &iter); while (NULL != (func = (zbx_func_t *)zbx_hashset_iter_next(&iter))) - zbx_vector_uint64_append(&itemids, func->itemid); + { + if (FAIL == zbx_vector_uint64_bsearch(history_itemids, func->itemid, ZBX_DEFAULT_UINT64_COMPARE_FUNC)) + zbx_vector_uint64_append(&itemids, func->itemid); + } - zbx_vector_uint64_sort(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); - zbx_vector_uint64_uniq(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); + if (0 != itemids.values_num) + { + zbx_vector_uint64_sort(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); + zbx_vector_uint64_uniq(&itemids, ZBX_DEFAULT_UINT64_COMPARE_FUNC); - items = (DC_ITEM *)zbx_malloc(items, sizeof(DC_ITEM) * (size_t)itemids.values_num); - errcodes = (int *)zbx_malloc(errcodes, sizeof(int) * (size_t)itemids.values_num); + items = (DC_ITEM *)zbx_malloc(items, sizeof(DC_ITEM) * (size_t)itemids.values_num); + errcodes = (int *)zbx_malloc(errcodes, sizeof(int) * (size_t)itemids.values_num); - DCconfig_get_items_by_itemids(items, itemids.values, errcodes, itemids.values_num); + DCconfig_get_items_by_itemids_partial(items, itemids.values, errcodes, itemids.values_num, + ZBX_ITEM_GET_SYNC); + } zbx_hashset_iter_reset(funcs, &iter); while (NULL != (func = (zbx_func_t *)zbx_hashset_iter_next(&iter))) { - i = zbx_vector_uint64_bsearch(&itemids, func->itemid, ZBX_DEFAULT_UINT64_COMPARE_FUNC); + int errcode; + const DC_ITEM *item; - if (SUCCEED != errcodes[i]) + /* avoid double copying from configuration cache if already retrieved when saving history */ + if (FAIL != (i = zbx_vector_uint64_bsearch(history_itemids, func->itemid, + ZBX_DEFAULT_UINT64_COMPARE_FUNC))) + { + item = history_items + i; + errcode = history_errcodes[i]; + } + else + { + i = zbx_vector_uint64_bsearch(&itemids, func->itemid, ZBX_DEFAULT_UINT64_COMPARE_FUNC); + item = items + i; + errcode = errcodes[i]; + } + + if (SUCCEED != errcode) { zbx_free(func->error); func->error = zbx_eval_format_function_error(func->function, NULL, NULL, func->parameter, @@ -4996,39 +5018,39 @@ static void zbx_evaluate_item_functions(zbx_hashset_t *funcs) /* do not evaluate if the item is disabled or belongs to a disabled host */ - if (ITEM_STATUS_ACTIVE != items[i].status) + if (ITEM_STATUS_ACTIVE != item->status) { zbx_free(func->error); - func->error = zbx_eval_format_function_error(func->function, items[i].host.host, - items[i].key_orig, func->parameter, "item is disabled"); + func->error = zbx_eval_format_function_error(func->function, item->host.host, + item->key_orig, func->parameter, "item is disabled"); continue; } - if (HOST_STATUS_MONITORED != items[i].host.status) + if (HOST_STATUS_MONITORED != item->host.status) { zbx_free(func->error); - func->error = zbx_eval_format_function_error(func->function, items[i].host.host, - items[i].key_orig, func->parameter, "item belongs to a disabled host"); + func->error = zbx_eval_format_function_error(func->function, item->host.host, + item->key_orig, func->parameter, "item belongs to a disabled host"); continue; } - if (ITEM_STATE_NOTSUPPORTED == items[i].state && + if (ITEM_STATE_NOTSUPPORTED == item->state && FAIL == zbx_evaluatable_for_notsupported(func->function)) { /* set 'unknown' error value */ zbx_variant_set_error(&func->value, - zbx_eval_format_function_error(func->function, items[i].host.host, - items[i].key_orig, func->parameter, "item is not supported")); + zbx_eval_format_function_error(func->function, item->host.host, + item->key_orig, func->parameter, "item is not supported")); continue; } - if (SUCCEED != evaluate_function2(&func->value, &items[i], func->function, func->parameter, + if (SUCCEED != evaluate_function2(&func->value, item, func->function, func->parameter, &func->timespec, &error)) { /* compose and store error message for future use */ zbx_variant_set_error(&func->value, - zbx_eval_format_function_error(func->function, items[i].host.host, - items[i].key_orig, func->parameter, error)); + zbx_eval_format_function_error(func->function, item->host.host, + item->key_orig, func->parameter, error)); zbx_free(error); continue; } @@ -5163,7 +5185,8 @@ static void zbx_substitute_functions_results(zbx_hashset_t *ifuncs, zbx_vector_p * Comments: example: "({15}>10) or ({123}=1)" => "(26.416>10) or (0=1)" * * * ******************************************************************************/ -static void substitute_functions(zbx_vector_ptr_t *triggers) +static void substitute_functions(zbx_vector_ptr_t *triggers, const zbx_vector_uint64_t *history_itemids, + const DC_ITEM *history_items, const int *history_errcodes) { zbx_vector_uint64_t functionids; zbx_hashset_t ifuncs, funcs; @@ -5186,7 +5209,7 @@ static void substitute_functions(zbx_vector_ptr_t *triggers) if (0 != ifuncs.num_data) { - zbx_evaluate_item_functions(&funcs); + zbx_evaluate_item_functions(&funcs, history_itemids, history_items, history_errcodes); zbx_substitute_functions_results(&ifuncs, triggers); } @@ -5269,7 +5292,8 @@ static int evaluate_expression(zbx_eval_context_t *ctx, const zbx_timespec_t *ts * Author: Alexei Vladishev * * * ******************************************************************************/ -void evaluate_expressions(zbx_vector_ptr_t *triggers) +void evaluate_expressions(zbx_vector_ptr_t *triggers, const zbx_vector_uint64_t *history_itemids, + const DC_ITEM *history_items, const int *history_errcodes) { DB_EVENT event; DC_TRIGGER *tr; @@ -5301,7 +5325,7 @@ void evaluate_expressions(zbx_vector_ptr_t *triggers) } } - substitute_functions(triggers); + substitute_functions(triggers, history_itemids, history_items, history_errcodes); /* calculate new trigger values based on their recovery modes and expression evaluations */ for (i = 0; i < triggers->values_num; i++) |