diff options
author | Alex Kalimulin <aleksandrs.kalimulins@zabbix.com> | 2019-02-18 14:15:43 +0300 |
---|---|---|
committer | Alex Kalimulin <aleksandrs.kalimulins@zabbix.com> | 2019-02-18 14:15:43 +0300 |
commit | 39f16b600cd85aadbe43cb418c9260ff15eace30 (patch) | |
tree | d32df8c76399a0e8af5b9affe97ab5bc1487c09b | |
parent | e24843c70479ee194a5a7cc88e1715ae24a70763 (diff) |
.......PS. [ZBXNEXT-4941] changed overflow statements to execute one by one for Oracle for better performance4.0.5rc1
-rw-r--r-- | ChangeLog | 1 | ||||
-rw-r--r-- | include/db.h | 44 | ||||
-rw-r--r-- | include/zbxdb.h | 13 | ||||
-rw-r--r-- | src/libs/zbxdbhigh/db.c | 18 | ||||
-rw-r--r-- | src/libs/zbxdbhigh/lld_item.c | 134 | ||||
-rw-r--r-- | src/libs/zbxdbhigh/proxy.c | 2 | ||||
-rw-r--r-- | src/zabbix_server/alerter/alert_manager.c | 20 |
7 files changed, 174 insertions, 58 deletions
diff --git a/ChangeLog b/ChangeLog index 61a9b33c08d..06dd1a1e6bb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ New features: ..FGI..PST [ZBXNEXT-401,ZBXNEXT-4907,ZBXNEXT-4955] added monitoring of internal metrics from an external Zabbix instance (vasilijs, viktors) Bug fixes: +.......PS. [ZBXNEXT-4941] changed overflow statements to execute one by one for Oracle for better performance (kalimulin) ..F....... [ZBX-15664] fixed "Undefined index" errors on maps with enabled automatic icon mapping (Sasha) ....I..... [ZBX-15667] fixed wrong default Zabbix server status for new installation (kalimulin) ..F....... [ZBX-15512] fixed performance issues and significant memory consumption on several pages which contains host popup menus (Sasha) diff --git a/include/db.h b/include/db.h index 1355131beb1..159c0730417 100644 --- a/include/db.h +++ b/include/db.h @@ -209,23 +209,37 @@ struct _DC_TRIGGER; #define ZBX_SQL_ITEM_SELECT ZBX_SQL_ITEM_FIELDS " from " ZBX_SQL_ITEM_TABLES #ifdef HAVE_ORACLE -#define DBbegin_multiple_update(sql, sql_alloc, sql_offset) zbx_strcpy_alloc(sql, sql_alloc, sql_offset, "begin\n") -#define DBend_multiple_update(sql, sql_alloc, sql_offset) zbx_strcpy_alloc(sql, sql_alloc, sql_offset, "end;") - -#define ZBX_SQL_STRCMP "%s%s%s" -#define ZBX_SQL_STRVAL_EQ(str) '\0' != *str ? "='" : "", \ - '\0' != *str ? str : " is null", \ - '\0' != *str ? "'" : "" -#define ZBX_SQL_STRVAL_NE(str) '\0' != *str ? "<>'" : "", \ - '\0' != *str ? str : " is not null", \ - '\0' != *str ? "'" : "" +# define ZBX_PLSQL_BEGIN "begin\n" +# define ZBX_PLSQL_END "end;" +# define DBbegin_multiple_update(sql, sql_alloc, sql_offset) \ + zbx_strcpy_alloc(sql, sql_alloc, sql_offset, ZBX_PLSQL_BEGIN) +# define DBend_multiple_update(sql, sql_alloc, sql_offset) \ + zbx_strcpy_alloc(sql, sql_alloc, sql_offset, ZBX_PLSQL_END) +# if 0 == ZBX_MAX_OVERFLOW_SQL_SIZE +# define ZBX_SQL_EXEC_FROM ZBX_CONST_STRLEN(ZBX_PLSQL_BEGIN) +# else +# define ZBX_SQL_EXEC_FROM 0 +# endif + +# define ZBX_SQL_STRCMP "%s%s%s" +# define ZBX_SQL_STRVAL_EQ(str) \ + '\0' != *str ? "='" : "", \ + '\0' != *str ? str : " is null", \ + '\0' != *str ? "'" : "" +# define ZBX_SQL_STRVAL_NE(str) \ + '\0' != *str ? "<>'" : "", \ + '\0' != *str ? str : " is not null", \ + '\0' != *str ? "'" : "" + #else -#define DBbegin_multiple_update(sql, sql_alloc, sql_offset) -#define DBend_multiple_update(sql, sql_alloc, sql_offset) +# define DBbegin_multiple_update(sql, sql_alloc, sql_offset) do {} while (0) +# define DBend_multiple_update(sql, sql_alloc, sql_offset) do {} while (0) + +# define ZBX_SQL_EXEC_FROM 0 -#define ZBX_SQL_STRCMP "%s'%s'" -#define ZBX_SQL_STRVAL_EQ(str) "=", str -#define ZBX_SQL_STRVAL_NE(str) "<>", str +# define ZBX_SQL_STRCMP "%s'%s'" +# define ZBX_SQL_STRVAL_EQ(str) "=", str +# define ZBX_SQL_STRVAL_NE(str) "<>", str #endif #define ZBX_SQL_NULLCMP(f1, f2) "((" f1 " is null and " f2 " is null) or " f1 "=" f2 ")" diff --git a/include/zbxdb.h b/include/zbxdb.h index 41b92714c87..5dcffa32d04 100644 --- a/include/zbxdb.h +++ b/include/zbxdb.h @@ -29,6 +29,19 @@ #define ZBX_DB_WAIT_DOWN 10 #define ZBX_MAX_SQL_SIZE 262144 /* 256KB */ +#ifndef ZBX_MAX_OVERFLOW_SQL_SIZE +# ifdef HAVE_ORACLE + /* Do not use "overflowing" (multi-statement) queries for Oracle. */ + /* Zabbix benefits from cursor_sharing=force Oracle parameter */ + /* which doesn't apply to PL/SQL blocks. */ +# define ZBX_MAX_OVERFLOW_SQL_SIZE 0 +# else +# define ZBX_MAX_OVERFLOW_SQL_SIZE ZBX_MAX_SQL_SIZE +# endif +#elif 0 != ZBX_MAX_OVERFLOW_SQL_SIZE && \ + (1024 > ZBX_MAX_OVERFLOW_SQL_SIZE || ZBX_MAX_OVERFLOW_SQL_SIZE > ZBX_MAX_SQL_SIZE) +#error ZBX_MAX_OVERFLOW_SQL_SIZE is out of range +#endif typedef char **DB_ROW; typedef struct zbx_db_result *DB_RESULT; diff --git a/src/libs/zbxdbhigh/db.c b/src/libs/zbxdbhigh/db.c index 0a632223bf9..23728843f90 100644 --- a/src/libs/zbxdbhigh/db.c +++ b/src/libs/zbxdbhigh/db.c @@ -1552,7 +1552,7 @@ int DBexecute_overflowed_sql(char **sql, size_t *sql_alloc, size_t *sql_offset) { int ret = SUCCEED; - if (ZBX_MAX_SQL_SIZE < *sql_offset) + if (ZBX_MAX_OVERFLOW_SQL_SIZE < *sql_offset) { #ifdef HAVE_MULTIROW_INSERT if (',' == (*sql)[*sql_offset - 1]) @@ -1561,10 +1561,22 @@ int DBexecute_overflowed_sql(char **sql, size_t *sql_alloc, size_t *sql_offset) zbx_strcpy_alloc(sql, sql_alloc, sql_offset, ";\n"); } #endif - DBend_multiple_update(sql, sql_alloc, sql_offset); +#if defined(HAVE_ORACLE) && 0 == ZBX_MAX_OVERFLOW_SQL_SIZE + /* make sure we are not called twice without */ + /* putting a new sql into the buffer first */ + if (*sql_offset <= ZBX_SQL_EXEC_FROM) + THIS_SHOULD_NEVER_HAPPEN; - if (ZBX_DB_OK > DBexecute("%s", *sql)) + /* Oracle fails with ORA-00911 if it encounters ';' w/o PL/SQL block */ + zbx_rtrim(*sql, ZBX_WHITESPACE ";"); +#else + DBend_multiple_update(sql, sql_alloc, sql_offset); +#endif + /* For Oracle with max_overflow_sql_size == 0, jump over "begin\n" */ + /* before execution. ZBX_SQL_EXEC_FROM is 0 for all other cases. */ + if (ZBX_DB_OK > DBexecute("%s", *sql + ZBX_SQL_EXEC_FROM)) ret = FAIL; + *sql_offset = 0; DBbegin_multiple_update(sql, sql_alloc, sql_offset); diff --git a/src/libs/zbxdbhigh/lld_item.c b/src/libs/zbxdbhigh/lld_item.c index b9e7e2fb466..3b955c9f62d 100644 --- a/src/libs/zbxdbhigh/lld_item.c +++ b/src/libs/zbxdbhigh/lld_item.c @@ -2682,8 +2682,7 @@ static void lld_item_save(zbx_uint64_t hostid, const zbx_vector_ptr_t *item_prot * * * Purpose: prepare sql to update LLD item * * * - * Parameters: hostid - [IN] parent host id * - * item_prototypes - [IN] item prototypes * + * Parameters: item_prototype - [IN] item prototype * * item - [IN] item to be updated * * sql - [IN/OUT] sql buffer pointer used for * * update operations * @@ -2693,25 +2692,11 @@ static void lld_item_save(zbx_uint64_t hostid, const zbx_vector_ptr_t *item_prot * buffer * * * ******************************************************************************/ -static void lld_item_prepare_update(const zbx_vector_ptr_t *item_prototypes, const zbx_lld_item_t *item, char **sql, - size_t *sql_alloc, size_t *sql_offset) +static void lld_item_prepare_update(const zbx_lld_item_prototype_t *item_prototype, const zbx_lld_item_t *item, + char **sql, size_t *sql_alloc, size_t *sql_offset) { - const zbx_lld_item_prototype_t *item_prototype; char *value_esc; const char *d = ""; - int index; - - if (0 == (item->flags & ZBX_FLAG_LLD_ITEM_DISCOVERED) || 0 == (item->flags & ZBX_FLAG_LLD_ITEM_UPDATE)) - return; - - if (FAIL == (index = zbx_vector_ptr_bsearch(item_prototypes, &item->parent_itemid, - ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC))) - { - THIS_SHOULD_NEVER_HAPPEN; - return; - } - - item_prototype = item_prototypes->values[index]; zbx_strcpy_alloc(sql, sql_alloc, sql_offset, "update items set "); if (0 != (item->flags & ZBX_FLAG_LLD_ITEM_UPDATE_NAME)) @@ -3050,6 +3035,30 @@ static void lld_item_prepare_update(const zbx_vector_ptr_t *item_prototypes, con zbx_snprintf_alloc(sql, sql_alloc, sql_offset, " where itemid=" ZBX_FS_UI64 ";\n", item->itemid); + DBexecute_overflowed_sql(sql, sql_alloc, sql_offset); +} + +/****************************************************************************** + * * + * Function: lld_item_discovery_prepare_update * + * * + * Purpose: prepare sql to update key in LLD item discovery * + * * + * Parameters: item_prototype - [IN] item prototype * + * item - [IN] item to be updated * + * sql - [IN/OUT] sql buffer pointer used for * + * update operations * + * sql_alloc - [IN/OUT] sql buffer already allocated * + * memory * + * sql_offset - [IN/OUT] offset for writing within sql * + * buffer * + * * + ******************************************************************************/ +static void lld_item_discovery_prepare_update(const zbx_lld_item_prototype_t *item_prototype, + const zbx_lld_item_t *item, char **sql, size_t *sql_alloc, size_t *sql_offset) +{ + char *value_esc; + if (0 != (item->flags & ZBX_FLAG_LLD_ITEM_UPDATE_KEY)) { value_esc = DBdyn_escape_string(item_prototype->key); @@ -3059,9 +3068,12 @@ static void lld_item_prepare_update(const zbx_vector_ptr_t *item_prototypes, con " where itemid=" ZBX_FS_UI64 ";\n", value_esc, item->itemid); zbx_free(value_esc); + + DBexecute_overflowed_sql(sql, sql_alloc, sql_offset); } } + /****************************************************************************** * * * Function: lld_items_save * @@ -3181,8 +3193,10 @@ static int lld_items_save(zbx_uint64_t hostid, const zbx_vector_ptr_t *item_prot if (0 != upd_items) { - char *sql = NULL; - size_t sql_alloc = 8 * ZBX_KIBIBYTE, sql_offset = 0; + char *sql = NULL; + size_t sql_alloc = 8 * ZBX_KIBIBYTE, sql_offset = 0; + zbx_lld_item_prototype_t *item_prototype; + int index; sql = (char*)zbx_malloc(NULL, sql_alloc); DBbegin_multiple_update(&sql, &sql_alloc, &sql_offset); @@ -3191,8 +3205,23 @@ static int lld_items_save(zbx_uint64_t hostid, const zbx_vector_ptr_t *item_prot { item = (zbx_lld_item_t *)items->values[i]; - lld_item_prepare_update(item_prototypes, item, &sql, &sql_alloc, &sql_offset); - DBexecute_overflowed_sql(&sql, &sql_alloc, &sql_offset); + if (0 == (item->flags & ZBX_FLAG_LLD_ITEM_DISCOVERED) || + 0 == (item->flags & ZBX_FLAG_LLD_ITEM_UPDATE)) + { + continue; + } + + if (FAIL == (index = zbx_vector_ptr_bsearch(item_prototypes, &item->parent_itemid, + ZBX_DEFAULT_UINT64_PTR_COMPARE_FUNC))) + { + THIS_SHOULD_NEVER_HAPPEN; + continue; + } + + item_prototype = item_prototypes->values[index]; + + lld_item_prepare_update(item_prototype, item, &sql, &sql_alloc, &sql_offset); + lld_item_discovery_prepare_update(item_prototype, item, &sql, &sql_alloc, &sql_offset); } DBend_multiple_update(&sql, &sql_alloc, &sql_offset); @@ -3453,9 +3482,6 @@ static int lld_applications_save(zbx_uint64_t hostid, zbx_vector_ptr_t *applicat for (i = 0; i < applications->values_num; i++) { - DBexecute_overflowed_sql(&sql_a, &sql_a_alloc, &sql_a_offset); - DBexecute_overflowed_sql(&sql_ad, &sql_ad_alloc, &sql_ad_offset); - application = (zbx_lld_application_t *)applications->values[i]; if (0 != (application->flags & ZBX_FLAG_LLD_APPLICATION_REMOVE_DISCOVERY)) @@ -3485,6 +3511,11 @@ static int lld_applications_save(zbx_uint64_t hostid, zbx_vector_ptr_t *applicat if (0 != (application->flags & ZBX_FLAG_LLD_APPLICATION_UPDATE_NAME)) { + if (NULL == sql_a) + DBbegin_multiple_update(&sql_a, &sql_a_alloc, &sql_a_offset); + if (NULL == sql_ad) + DBbegin_multiple_update(&sql_ad, &sql_ad_alloc, &sql_ad_offset); + name = DBdyn_escape_string(application->name); zbx_snprintf_alloc(&sql_a, &sql_a_alloc, &sql_a_offset, "update applications set name='%s'" @@ -3498,6 +3529,10 @@ static int lld_applications_save(zbx_uint64_t hostid, zbx_vector_ptr_t *applicat " where application_discoveryid=" ZBX_FS_UI64 ";\n", name, application->application_discoveryid); zbx_free(name); + + DBexecute_overflowed_sql(&sql_a, &sql_a_alloc, &sql_a_offset); + DBexecute_overflowed_sql(&sql_ad, &sql_ad_alloc, &sql_ad_offset); + continue; } @@ -3510,34 +3545,49 @@ static int lld_applications_save(zbx_uint64_t hostid, zbx_vector_ptr_t *applicat application_prototype->name); } + if (NULL != sql_a) + { + DBend_multiple_update(&sql_a, &sql_a_alloc, &sql_a_offset); + + if (16 < sql_a_offset) /* in ORACLE always present begin..end; */ + DBexecute("%s", sql_a); + } + + if (NULL != sql_ad) + { + DBend_multiple_update(&sql_ad, &sql_ad_alloc, &sql_ad_offset); + + if (16 < sql_ad_offset) + DBexecute("%s", sql_ad); + } + if (0 != del_applicationids.values_num) { + sql_a_offset = 0; + zbx_strcpy_alloc(&sql_a, &sql_a_alloc, &sql_a_offset, "delete from applications where"); DBadd_condition_alloc(&sql_a, &sql_a_alloc, &sql_a_offset, "applicationid", del_applicationids.values, del_applicationids.values_num); zbx_strcpy_alloc(&sql_a, &sql_a_alloc, &sql_a_offset, ";\n"); + + DBexecute("%s", sql_a); } if (0 != del_discoveryids.values_num) { + sql_ad_offset = 0; + zbx_strcpy_alloc(&sql_a, &sql_a_alloc, &sql_a_offset, "delete from application_discovery where"); DBadd_condition_alloc(&sql_a, &sql_a_alloc, &sql_a_offset, "application_discoveryid", del_discoveryids.values, del_discoveryids.values_num); zbx_strcpy_alloc(&sql_a, &sql_a_alloc, &sql_a_offset, ";\n"); - } - - if (NULL != sql_a) - { - DBexecute("%s", sql_a); - zbx_free(sql_a); - } - if (NULL != sql_ad) - { DBexecute("%s", sql_ad); - zbx_free(sql_ad); } + zbx_free(sql_a); + zbx_free(sql_ad); + if (0 != new_applications) { zbx_db_insert_execute(&db_insert); @@ -3813,6 +3863,8 @@ static void lld_remove_lost_items(const zbx_vector_ptr_t *items, int lifetime, i DBadd_condition_alloc(&sql, &sql_alloc, &sql_offset, "itemid", ts_itemids.values, ts_itemids.values_num); zbx_strcpy_alloc(&sql, &sql_alloc, &sql_offset, ";\n"); + + DBexecute_overflowed_sql(&sql, &sql_alloc, &sql_offset); } DBend_multiple_update(&sql, &sql_alloc, &sql_offset); @@ -4003,6 +4055,8 @@ static void lld_remove_lost_applications(zbx_uint64_t lld_ruleid, const zbx_vect DBadd_condition_alloc(&sql, &sql_alloc, &sql_offset, "applicationid", del_applicationids.values, del_applicationids.values_num); zbx_strcpy_alloc(&sql, &sql_alloc, &sql_offset, ";\n"); + + DBexecute_overflowed_sql(&sql, &sql_alloc, &sql_offset); } DBend_multiple_update(&sql, &sql_alloc, &sql_offset); @@ -4289,11 +4343,14 @@ static void lld_applications_get(zbx_uint64_t lld_ruleid, zbx_vector_ptr_t *appl static void lld_application_make(const zbx_lld_application_prototype_t *application_prototype, const zbx_lld_row_t *lld_row, zbx_vector_ptr_t *applications, zbx_hashset_t *applications_index) { + const char *__function_name = "lld_application_make"; zbx_lld_application_t *application; zbx_lld_application_index_t *application_index, application_index_local; struct zbx_json_parse *jp_row = (struct zbx_json_parse *)&lld_row->jp_row; char *buffer = NULL; + zabbix_log(LOG_LEVEL_DEBUG, "In %s(), proto %s", __function_name, application_prototype->name); + application_index_local.application_prototypeid = application_prototype->application_prototypeid; application_index_local.lld_row = lld_row; @@ -4318,6 +4375,9 @@ static void lld_application_make(const zbx_lld_application_prototype_t *applicat application_index_local.application = application; zbx_hashset_insert(applications_index, &application_index_local, sizeof(zbx_lld_application_index_t)); + + zabbix_log(LOG_LEVEL_TRACE, "%s(): created new application, proto %s, name %s", __function_name, + application_prototype->name, application->name); } else { @@ -4334,6 +4394,8 @@ static void lld_application_make(const zbx_lld_application_prototype_t *applicat application->name_orig = application->name; application->name = buffer; application->flags |= ZBX_FLAG_LLD_APPLICATION_UPDATE_NAME; + zabbix_log(LOG_LEVEL_TRACE, "%s(): updated application name to %s", __function_name, + application->name); } else zbx_free(buffer); @@ -4341,6 +4403,8 @@ static void lld_application_make(const zbx_lld_application_prototype_t *applicat } application->flags |= ZBX_FLAG_LLD_APPLICATION_DISCOVERED; + + zabbix_log(LOG_LEVEL_DEBUG, "End of %s()", __function_name); } /****************************************************************************** diff --git a/src/libs/zbxdbhigh/proxy.c b/src/libs/zbxdbhigh/proxy.c index 5b207164f3d..6576328b940 100644 --- a/src/libs/zbxdbhigh/proxy.c +++ b/src/libs/zbxdbhigh/proxy.c @@ -1466,7 +1466,7 @@ static int process_proxyconfig_table(const ZBX_TABLE *table, struct zbx_json_par } } - if (sql_offset > 16) /* in ORACLE always present begin..end; */ + if (16 < sql_offset) /* in ORACLE always present begin..end; */ { DBend_multiple_update(&sql, &sql_alloc, &sql_offset); diff --git a/src/zabbix_server/alerter/alert_manager.c b/src/zabbix_server/alerter/alert_manager.c index c6e939890ae..dc0fbbbdd90 100644 --- a/src/zabbix_server/alerter/alert_manager.c +++ b/src/zabbix_server/alerter/alert_manager.c @@ -1460,13 +1460,23 @@ static int am_db_flush_alert_updates(zbx_am_t *manager) if (ZBX_DB_DOWN == zbx_db_begin()) goto cleanup; - for (i = 0; i < updates.values_num; i += 100) +#if defined(HAVE_ORACLE) && 0 == ZBX_MAX_OVERFLOW_SQL_SIZE +# define ZBX_SQL_UPDATE_BATCH_SIZE 1 +# define ZBX_SQL_DELIMITER +#else +# define ZBX_SQL_UPDATE_BATCH_SIZE 100 +# define ZBX_SQL_DELIMITER ";\n" +#endif + + for (i = 0; i < updates.values_num; i += ZBX_SQL_UPDATE_BATCH_SIZE) { sql_offset = 0; - limit = MIN(i + 100, updates.values_num); + limit = MIN(i + ZBX_SQL_UPDATE_BATCH_SIZE, updates.values_num); +#if !defined(HAVE_ORACLE) || 0 != ZBX_MAX_OVERFLOW_SQL_SIZE DBbegin_multiple_update(&sql, &sql_alloc, &sql_offset); +#endif for (j = i; j < limit; j++) { @@ -1479,15 +1489,17 @@ static int am_db_flush_alert_updates(zbx_am_t *manager) " set status=%d," "retries=%d," "error='%s'" - " where alertid=" ZBX_FS_UI64 ";\n", + " where alertid=" ZBX_FS_UI64 ZBX_SQL_DELIMITER, update->status, update->retries, error_esc, update->alertid); zbx_free(error_esc); } +#if !defined(HAVE_ORACLE) || 0 != ZBX_MAX_OVERFLOW_SQL_SIZE DBend_multiple_update(&sql, &sql_alloc, &sql_offset); +#endif - if (ZBX_DB_DOWN == DBexecute_once("%s", sql)) + if (16 < sql_offset && ZBX_DB_DOWN == DBexecute_once("%s", sql)) goto cleanup; } |