zabbix_export: version: '6.2' date: '2022-04-06T19:54:13Z' groups: - uuid: 748ad4d098d447d492bb935c907f652f name: Templates/Databases templates: - uuid: 95f2053c21094ad4968440b562cc7ea2 template: 'ClickHouse by HTTP' name: 'ClickHouse by HTTP' description: | Get node metrics from ClickHouse HTTP interface using HTTP agent. Template tooling version used: 0.41 groups: - name: Templates/Databases items: - uuid: 595446e408b844519e8db1ea91543134 name: 'ClickHouse: Current distribute connections' type: DEPENDENT key: clickhouse.connections.distribute delay: '0' history: 7d description: 'Number of connections to remote servers sending data that was INSERTed into Distributed tables.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "DistributedSend")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: connections - uuid: d2d72a607672442b957228f0bea8dcca name: 'ClickHouse: Current HTTP connections' type: DEPENDENT key: clickhouse.connections.http delay: '0' history: 7d description: 'Number of connections to HTTP server.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "HTTPConnection")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: connections - uuid: 0829591332244c75a219f3b811ebf9ce name: 'ClickHouse: Current Interserver connections' type: DEPENDENT key: clickhouse.connections.interserver delay: '0' history: 7d description: 'Number of connections from other replicas to fetch parts.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "InterserverConnection")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: connections - uuid: 88d4855d22604d79927d40c0d3b106c5 name: 'ClickHouse: Current MySQL connections' type: DEPENDENT key: clickhouse.connections.mysql delay: '0' history: 7d description: 'Number of connections to MySQL server.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "MySQLConnection")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: clickhouse.system.metrics tags: - tag: component value: connections - uuid: 11786fff830d4d79b914a22c5f85da7f name: 'ClickHouse: Current TCP connections' type: DEPENDENT key: clickhouse.connections.tcp delay: '0' history: 7d description: 'Number of connections to TCP server (clients with native interface).' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "TCPConnection")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: connections - uuid: 46d8ed7374274d71a06119f50463f26c name: 'ClickHouse: Get dictionaries info' type: HTTP_AGENT key: clickhouse.dictionaries history: '0' trends: '0' value_type: TEXT preprocessing: - type: JSONPATH parameters: - $.data url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'SELECT * FROM system.dictionaries format JSON' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: dictionaries - tag: component value: raw - uuid: b7bbca72bbd14b20930e60c3c1cdcb8e name: 'ClickHouse: Current distributed files to insert' type: DEPENDENT key: clickhouse.distributed.files delay: '0' history: 7d description: 'Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "DistributedFilesToInsert")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: system triggers: - uuid: 4ecadb993aa74126a807df3af4b892ed expression: 'min(/ClickHouse by HTTP/clickhouse.distributed.files,5m)>{$CLICKHOUSE.DELAYED.FILES.DISTRIBUTED.COUNT.MAX.WARN}' name: 'ClickHouse: Too many distributed files to insert' event_name: 'ClickHouse: Too many distributed files to insert (over {$CLICKHOUSE.DELAYED.FILES.DISTRIBUTED.COUNT.MAX.WARN} for 5 min)' priority: WARNING description: | "Clickhouse servers and in config.xml https://clickhouse.tech/docs/en/operations/table_engines/distributed/" manual_close: 'YES' tags: - tag: scope value: capacity - uuid: e056f8519dec4c8dace036bf993a6d47 name: 'ClickHouse: Distributed connection fail with retry per second' type: DEPENDENT key: clickhouse.distributed.files.fail.rate delay: '0' history: 7d value_type: FLOAT description: '"Connection failures after all retries in replicated DB connection pool"' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "DistributedConnectionFailAtAll")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: connections - uuid: 3859c5ee6f25419f9da7e7899c9a71af name: 'ClickHouse: Distributed connection fail with retry per second' type: DEPENDENT key: clickhouse.distributed.files.retry.rate delay: '0' history: 7d value_type: FLOAT description: 'Connection retries in replicated DB connection pool' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "DistributedConnectionFailTry")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: connections - uuid: 81130b88e0b04b9ea677a80b6fd6ce22 name: 'ClickHouse: Delayed insert queries' type: DEPENDENT key: clickhouse.insert.delay delay: '0' history: 7d value_type: FLOAT description: '"Number of INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree table."' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "DelayedInserts")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: queries triggers: - uuid: 164dc9841cdd4b77a650792cf1bac701 expression: 'min(/ClickHouse by HTTP/clickhouse.insert.delay,5m)>{$CLICKHOUSE.DELAYED.INSERTS.MAX.WARN}' name: 'ClickHouse: Too many throttled insert queries' event_name: 'ClickHouse: Too many throttled insert queries (over {$CLICKHOUSE.DELAYED.INSERTS.MAX.WARN) for 5 min)' priority: WARNING description: 'Clickhouse have INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree, please decrease INSERT frequency' manual_close: 'YES' tags: - tag: scope value: performance - uuid: e2477dd38510489c8ea7509e9a37cb50 name: 'ClickHouse: Inserted bytes per second' type: DEPENDENT key: clickhouse.inserted_bytes.rate delay: '0' history: 7d value_type: FLOAT units: Bps description: 'The number of uncompressed bytes inserted in all tables.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "InsertedBytes")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: network - uuid: e09cfd7db8be48a485bda8c5f033d210 name: 'ClickHouse: Inserted rows per second' type: DEPENDENT key: clickhouse.inserted_rows.rate delay: '0' history: 7d value_type: FLOAT description: 'The number of rows inserted in all tables.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "InsertedRows")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: system - uuid: 410f7ee66ee84e3eb27940312ddf23d2 name: 'ClickHouse: New INSERT queries per second' type: DEPENDENT key: clickhouse.insert_query.rate delay: '0' history: 7d value_type: FLOAT description: 'Number of INSERT queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "InsertQuery")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: queries - uuid: 04c970694a6f40c89e4d3b53f27b7530 name: 'ClickHouse: Allocated bytes' type: DEPENDENT key: clickhouse.jemalloc.allocated delay: '0' history: 7d units: B description: '"Total number of bytes allocated by the application."' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "jemalloc.allocated")].value.first()' master_item: key: clickhouse.system.asynchronous_metrics tags: - tag: component value: memory - uuid: 7aa71031aec74fbb9a2ec126076705a2 name: 'ClickHouse: Mapped memory' type: DEPENDENT key: clickhouse.jemalloc.mapped delay: '0' history: 7d units: B description: '"Total number of bytes in active extents mapped by the allocator."' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "jemalloc.mapped")].value.first()' master_item: key: clickhouse.system.asynchronous_metrics tags: - tag: component value: memory - uuid: 32cc450854e94c6a9adbdde3c76b94d7 name: 'ClickHouse: Resident memory' type: DEPENDENT key: clickhouse.jemalloc.resident delay: '0' history: 7d units: B description: | Maximum number of bytes in physically resident data pages mapped by the allocator, comprising all pages dedicated to allocator metadata, pages backing active allocations, and unused dirty pages. preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "jemalloc.resident")].value.first()' master_item: key: clickhouse.system.asynchronous_metrics tags: - tag: component value: memory - uuid: c99265c296f14189aaf65bf68b7d5037 name: 'ClickHouse: Max count of parts per partition across all tables' type: DEPENDENT key: clickhouse.max.part.count.for.partition delay: '0' history: 7d description: | Clickhouse MergeTree table engine split each INSERT query to partitions (PARTITION BY expression) and add one or more PARTS per INSERT inside each partition, after that background merge process run. preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "MaxPartCountForPartition")].value.first()' master_item: key: clickhouse.system.asynchronous_metrics tags: - tag: component value: system triggers: - uuid: 21e6d3c6047145c59cbd00032699ee99 expression: 'min(/ClickHouse by HTTP/clickhouse.max.part.count.for.partition,5m)>{$CLICKHOUSE.PARTS.PER.PARTITION.WARN} * 0.9' name: 'ClickHouse: Too many MergeTree parts' event_name: 'ClickHouse: Too many MergeTree parts (over 90% of {$CLICKHOUSE.PARTS.PER.PARTITION.WARN})' priority: WARNING description: | Descease INSERT queries frequency. Clickhouse MergeTree table engine split each INSERT query to partitions (PARTITION BY expression) and add one or more PARTS per INSERT inside each partition, after that background merge process run, and when you have too much unmerged parts inside partition, SELECT queries performance can significate degrade, so clickhouse try delay insert, or abort it. manual_close: 'YES' tags: - tag: scope value: performance - uuid: a4344162643142c9be3b09439b466a5d name: 'ClickHouse: Memory used for queries' type: DEPENDENT key: clickhouse.memory.tracking delay: '0' history: 7d value_type: FLOAT units: B description: '"Total amount of memory (bytes) allocated in currently executing queries."' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "MemoryTracking")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: memory - uuid: 921670c46d9a4e5b85f2049d64298ab4 name: 'ClickHouse: Memory used for background merges' type: DEPENDENT key: clickhouse.memory.tracking.background delay: '0' history: 7d value_type: FLOAT units: B description: | "Total amount of memory (bytes) allocated in background processing pool (that is dedicated for background merges, mutations and fetches). Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks." preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "MemoryTrackingInBackgroundProcessingPool")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: memory - uuid: f881ea3cb2a2434d9cfac0d0896279f5 name: 'ClickHouse: Memory used for background moves' type: DEPENDENT key: clickhouse.memory.tracking.background.moves delay: '0' history: 7d value_type: FLOAT units: B description: | "Total amount of memory (bytes) allocated in background processing pool (that is dedicated for background moves). Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks." preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "MemoryTrackingInBackgroundMoveProcessingPool")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: clickhouse.system.metrics tags: - tag: component value: memory - uuid: 3cd23448136c4ea8892483b1fbacec82 name: 'ClickHouse: Memory used for merges' type: DEPENDENT key: clickhouse.memory.tracking.merges delay: '0' history: 7d value_type: FLOAT units: B description: | Total amount of memory (bytes) allocated for background merges. Included in MemoryTrackingInBackgroundProcessingPool. Note that this value may include a drift when the memory was allocated in a context of background processing pool and freed in other context or vice-versa. This happens naturally due to caches for tables indexes and doesn't indicate memory leaks. preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "MemoryTrackingForMerges")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: memory - uuid: 1f4641ef040546e0afcccc818f4d3471 name: 'ClickHouse: Memory used for background schedule pool' type: DEPENDENT key: clickhouse.memory.tracking.schedule.pool delay: '0' history: 7d value_type: FLOAT units: B description: '"Total amount of memory (bytes) allocated in background schedule pool (that is dedicated for bookkeeping tasks of Replicated tables)."' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "MemoryTrackingInBackgroundSchedulePool")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: memory - uuid: 5e98b9c4f1ee42238f363d76354ef537 name: 'ClickHouse: Current running merges' type: DEPENDENT key: clickhouse.merge.current delay: '0' history: 7d description: 'Number of executing background merges' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "Merge")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: queries - uuid: 50e33e8fc1904870bdc6d7f13d16943d name: 'ClickHouse: Uncompressed bytes merged per second' type: DEPENDENT key: clickhouse.merge_bytes.rate delay: '0' history: 7d value_type: FLOAT units: Bps description: 'Uncompressed bytes that were read for background merges' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "MergedUncompressedBytes")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: system - uuid: a2afbf66601344bdb5ab72a7374bbe8b name: 'ClickHouse: Merged rows per second' type: DEPENDENT key: clickhouse.merge_rows.rate delay: '0' history: 7d value_type: FLOAT description: 'Rows read for background merges.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "MergedRows")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: system - uuid: 83061f56dce24364906cae761419b148 name: 'ClickHouse: Network errors per second' type: DEPENDENT key: clickhouse.network.error.rate delay: '0' history: 7d value_type: FLOAT description: 'Network errors (timeouts and connection failures) during query execution, background pool tasks and DNS cache update.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "NetworkErrors")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: network triggers: - uuid: d49b9e8978bd416f81ea01b935879d81 expression: 'min(/ClickHouse by HTTP/clickhouse.network.error.rate,5m)>{$CLICKHOUSE.NETWORK.ERRORS.MAX.WARN}' name: 'ClickHouse: Too many network errors' event_name: 'ClickHouse: Too many network errors (over {$CLICKHOUSE.NETWORK.ERRORS.MAX.WARN} in 5m)' priority: WARNING description: 'Number of errors (timeouts and connection failures) during query execution, background pool tasks and DNS cache update is too high.' tags: - tag: scope value: availability - uuid: cb40b0cf1296436d9a4e41062d1b3fa9 name: 'ClickHouse: Ping' type: HTTP_AGENT key: clickhouse.ping history: 7d valuemap: name: 'Service state' preprocessing: - type: REGEX parameters: - Ok\. - '1' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 10m url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/ping' tags: - tag: component value: health - tag: component value: network - uuid: 4fb14efc757542c1b50605bdfb2e23cb name: 'ClickHouse: Longest currently running query time' type: HTTP_AGENT key: clickhouse.process.elapsed history: 7d value_type: FLOAT units: s description: 'Get longest running query.' url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'SELECT max(elapsed) FROM system.processes' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: queries triggers: - uuid: 2410e7b63c8340b6b08ec729d7093be6 expression: 'last(/ClickHouse by HTTP/clickhouse.process.elapsed)>{$CLICKHOUSE.QUERY_TIME.MAX.WARN}' name: 'ClickHouse: There are queries running is long' event_name: 'ClickHouse: There are queries running more than {$CLICKHOUSE.QUERY_TIME.MAX.WARN} seconds' priority: AVERAGE manual_close: 'YES' tags: - tag: scope value: performance - uuid: 071ae233708b417b97cfeced5021c139 name: 'ClickHouse: Current running queries' type: DEPENDENT key: clickhouse.query.current delay: '0' history: 7d description: 'Number of executing queries' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "Query")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: queries - uuid: 0fb342f6a494443c8389f79439e94791 name: 'ClickHouse: New queries per second' type: DEPENDENT key: clickhouse.query.rate delay: '0' history: 7d value_type: FLOAT description: 'Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.' preprocessing: - type: JSONPATH parameters: - '$[?(@.data.event == "Query")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: queries - uuid: fb671accbbaa4bd58a107dc3661419af name: 'ClickHouse: Read syscalls in fly' type: DEPENDENT key: clickhouse.read delay: '0' history: 7d description: 'Number of read (read, pread, io_getevents, etc.) syscalls in fly' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "Read")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: system - uuid: eb4c47501d9c4cdf9e55347ccc9a38bd name: 'ClickHouse: Read bytes per second' type: DEPENDENT key: clickhouse.read_bytes.rate delay: '0' history: 7d value_type: FLOAT units: Bps description: '"Number of bytes (the number of bytes before decompression) read from compressed sources (files, network)."' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "ReadCompressedBytes")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: network - uuid: 3793962be3494772832796c932947e1d name: 'ClickHouse: Get replicas info' type: HTTP_AGENT key: clickhouse.replicas history: '0' trends: '0' value_type: TEXT preprocessing: - type: JSONPATH parameters: - $.data url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'SELECT database, table, is_readonly, is_session_expired, future_parts, parts_to_check, queue_size, inserts_in_queue, merges_in_queue, log_max_index, log_pointer, total_replicas, active_replicas, log_max_index - log_pointer as replica_lag FROM system.replicas format JSON' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: raw - tag: component value: replicas - uuid: 78ed9728eb634df1807c64637c862f9a name: 'ClickHouse: Replication lag across all tables' type: DEPENDENT key: clickhouse.replicas.max.absolute.delay delay: '0' history: 7d units: s description: 'Maximum replica queue delay relative to current time' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "ReplicasMaxAbsoluteDelay")].value.first()' master_item: key: clickhouse.system.asynchronous_metrics tags: - tag: component value: replication triggers: - uuid: fc47a407b15744e79b70088ec249951a expression: 'min(/ClickHouse by HTTP/clickhouse.replicas.max.absolute.delay,5m)>{$CLICKHOUSE.REPLICA.MAX.WARN}' name: 'ClickHouse: Replication lag is too high' event_name: 'ClickHouse: Replication lag is too high (over {$CLICKHOUSE.REPLICA.MAX.WARN} sec for 5min)' priority: WARNING description: | When replica have too much lag, it can be skipped from Distributed SELECT Queries without errors and you will have wrong query results. manual_close: 'YES' tags: - tag: scope value: performance - uuid: dd61675348374d2ea06cdde144fe4e26 name: 'ClickHouse: Total number read-only Replicas' type: DEPENDENT key: clickhouse.replicas.readonly.total delay: '0' history: 7d description: | Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured. preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "ReadonlyReplica")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: replication - uuid: 5d2a35daf81a49888e3dfab916fab8be name: 'ClickHouse: Total replication tasks in queue' type: DEPENDENT key: clickhouse.replicas.sum.queue.size delay: '0' history: 7d preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "ReplicasSumQueueSize")].value.first()' master_item: key: clickhouse.system.asynchronous_metrics tags: - tag: component value: replication - uuid: 572036e5b2c74197bd61afc753f6e759 name: 'ClickHouse: Revision' type: DEPENDENT key: clickhouse.revision delay: '0' history: 7d description: 'Revision of the server.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "Revision")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: application - uuid: f89f3e27ca4f4e109ae6fe6b74dcc2ca name: 'ClickHouse: New SELECT queries per second' type: DEPENDENT key: clickhouse.select_query.rate delay: '0' history: 7d value_type: FLOAT description: 'Number of SELECT queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "SelectQuery")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: queries - uuid: 6c84e0abd186467882d2d3842749f937 name: 'ClickHouse: Get system.asynchronous_metrics' type: HTTP_AGENT key: clickhouse.system.asynchronous_metrics history: '0' trends: '0' value_type: TEXT description: 'Get metrics that are calculated periodically in the background' preprocessing: - type: JSONPATH parameters: - $.data url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'select metric, value from system.asynchronous_metrics format JSON' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: asynchronous-metrics - tag: component value: raw - uuid: 8aa612d27547446e9457aeaf6e94e320 name: 'ClickHouse: Get system.events' type: HTTP_AGENT key: clickhouse.system.events history: '0' trends: '0' value_type: TEXT description: 'Get information about the number of events that have occurred in the system.' preprocessing: - type: JSONPATH parameters: - $.data url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'select event, value from system.events format JSON' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: raw - tag: component value: system-events - uuid: 2193464b67724392905deef21f2f7c20 name: 'ClickHouse: Get system.metrics' type: HTTP_AGENT key: clickhouse.system.metrics history: '0' trends: '0' value_type: TEXT description: 'Get metrics which can be calculated instantly, or have a current value format JSONEachRow' preprocessing: - type: JSONPATH parameters: - $.data url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'select metric, value from system.metrics format JSON' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: raw - tag: component value: system-metrics - uuid: 88c843c6d4fd43c794d1447109146312 name: 'ClickHouse: Get system.settings' type: HTTP_AGENT key: clickhouse.system.settings history: '0' trends: '0' value_type: TEXT description: 'Get information about settings that are currently in use.' preprocessing: - type: JSONPATH parameters: - $.data - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'select name, value from system.settings format JSON' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: raw - tag: component value: system-settings triggers: - uuid: dfd9500636824ebf9d51d93650d62613 expression: 'last(/ClickHouse by HTTP/clickhouse.system.settings,#1)<>last(/ClickHouse by HTTP/clickhouse.system.settings,#2) and length(last(/ClickHouse by HTTP/clickhouse.system.settings))>0' name: 'ClickHouse: Configuration has been changed' event_name: 'ClickHouse: Version has changed (new version: {ITEM.VALUE})' priority: INFO description: 'ClickHouse configuration has been changed. Ack to close.' manual_close: 'YES' tags: - tag: scope value: notice - uuid: 042f4b688cfe488fbd0e80f40f51ce78 name: 'ClickHouse: Get tables info' type: HTTP_AGENT key: clickhouse.tables history: '0' trends: '0' value_type: TEXT preprocessing: - type: JSONPATH parameters: - $.data url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'select database, table, sum(bytes) as bytes, count() as parts, sum(rows) as rows from system.parts where active = 1 group by database, table format JSON' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: raw - tag: component value: tables - uuid: 99b894b181b44a4f8ef10280b881b273 name: 'ClickHouse: Uptime' type: DEPENDENT key: clickhouse.uptime delay: '0' history: 7d units: s description: 'Number of seconds since ClickHouse server start' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "Uptime")].value.first()' master_item: key: clickhouse.system.asynchronous_metrics tags: - tag: component value: application triggers: - uuid: 975cebc1b4324f83b5a8853fd666a179 expression: 'nodata(/ClickHouse by HTTP/clickhouse.uptime,30m)=1' name: 'ClickHouse: Failed to fetch info data' event_name: 'ClickHouse: Failed to fetch info data (or no data for 30m)' priority: WARNING description: 'Zabbix has not received data for items for the last 30 minutes' manual_close: 'YES' dependencies: - name: 'ClickHouse: Service is down' expression: 'last(/ClickHouse by HTTP/clickhouse.ping)=0 or last(/ClickHouse by HTTP/net.tcp.service[{$CLICKHOUSE.SCHEME},"{HOST.CONN}","{$CLICKHOUSE.PORT}"]) = 0' tags: - tag: scope value: notice - uuid: e1316d3a83174cc49ab96ad90c46150b expression: 'last(/ClickHouse by HTTP/clickhouse.uptime)<10m' name: 'ClickHouse: has been restarted' event_name: 'ClickHouse: has been restarted (uptime < 10m)' priority: INFO description: 'Uptime is less than 10 minutes' manual_close: 'YES' tags: - tag: scope value: notice - uuid: d09b77c25dfd45249df05fc3f6d45b7b name: 'ClickHouse: Version' type: HTTP_AGENT key: clickhouse.version history: 7d trends: '0' value_type: CHAR description: 'Version of the server' preprocessing: - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1d url: '{$CLICKHOUSE.SCHEME}://{HOST.CONN}:{$CLICKHOUSE.PORT}/' query_fields: - name: query value: 'SELECT version()' headers: - name: X-ClickHouse-User value: '{$CLICKHOUSE.USER}' - name: X-ClickHouse-Key value: '{$CLICKHOUSE.PASSWORD}' tags: - tag: component value: application triggers: - uuid: 244dcf8dc8584e1686585160c261ef85 expression: 'last(/ClickHouse by HTTP/clickhouse.version,#1)<>last(/ClickHouse by HTTP/clickhouse.version,#2) and length(last(/ClickHouse by HTTP/clickhouse.version))>0' name: 'ClickHouse: Version has changed' event_name: 'ClickHouse: Version has changed (new version: {ITEM.VALUE})' priority: INFO description: 'ClickHouse version has changed. Ack to close.' manual_close: 'YES' tags: - tag: scope value: notice - uuid: 5bac794c8c7e438c924cd16b708dc281 name: 'ClickHouse: Write syscalls in fly' type: DEPENDENT key: clickhouse.write delay: '0' history: 7d description: 'Number of write (write, pwrite, io_getevents, etc.) syscalls in fly' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "Write")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: system - uuid: 381805a8145a48c2a1f2586996b66a94 name: 'ClickHouse: ZooKeeper exceptions per second' type: DEPENDENT key: clickhouse.zookeper.exceptions.rate delay: '0' history: 7d value_type: FLOAT description: 'Count of ZooKeeper exceptions that does not belong to user/hardware exceptions.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "ZooKeeperOtherExceptions")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: zookeper - uuid: f8e6a866d8c745d39ed124f19afe8041 name: 'ClickHouse: ZooKeeper hardware exceptions per second' type: DEPENDENT key: clickhouse.zookeper.hw_exceptions.rate delay: '0' history: 7d value_type: FLOAT description: 'Count of ZooKeeper exceptions caused by session moved/expired, connection loss, marshalling error, operation timed out and invalid zhandle state.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "ZooKeeperHardwareExceptions")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: zookeper - uuid: 071fb0471b8e40c3a5bb79c694cecce0 name: 'ClickHouse: ZooKeeper requests' type: DEPENDENT key: clickhouse.zookeper.request delay: '0' history: 7d description: 'Number of requests to ZooKeeper in progress.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "ZooKeeperRequest")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: zookeper - uuid: 3e344ea728a54bc38b4098e7f6e2d087 name: 'ClickHouse: ZooKeeper sessions' type: DEPENDENT key: clickhouse.zookeper.session delay: '0' history: 7d description: 'Number of sessions (connections) to ZooKeeper. Should be no more than one.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "ZooKeeperSession")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: zookeper triggers: - uuid: 655622aa6c5c48c8b38d5a519602ef05 expression: 'min(/ClickHouse by HTTP/clickhouse.zookeper.session,5m)>1' name: 'ClickHouse: Too many ZooKeeper sessions opened' priority: WARNING description: | Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows. tags: - tag: scope value: performance - uuid: 4931797c1034488fb9a2fc01e22b599c name: 'ClickHouse: ZooKeeper user exceptions per second' type: DEPENDENT key: clickhouse.zookeper.user_exceptions.rate delay: '0' history: 7d value_type: FLOAT description: 'Count of ZooKeeper exceptions caused by no znodes, bad version, node exists, node empty and no children for ephemeral.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "ZooKeeperUserExceptions")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: zookeper - uuid: b9fbc2f76b9c4de3b196ffc82ec10209 name: 'ClickHouse: ZooKeeper wait time' type: DEPENDENT key: clickhouse.zookeper.wait.time delay: '0' history: 7d value_type: FLOAT units: s description: 'Time spent in waiting for ZooKeeper operations.' preprocessing: - type: JSONPATH parameters: - '$[?(@.event == "ZooKeeperWaitMicroseconds")].value.first()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: MULTIPLIER parameters: - '0.000001' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: clickhouse.system.events tags: - tag: component value: zookeper - uuid: 696aa071a9b24b29acbce2f1b5c49da3 name: 'ClickHouse: ZooKeeper watches' type: DEPENDENT key: clickhouse.zookeper.watch delay: '0' history: 7d description: 'Number of watches (e.g., event subscriptions) in ZooKeeper.' preprocessing: - type: JSONPATH parameters: - '$[?(@.metric == "ZooKeeperWatch")].value.first()' master_item: key: clickhouse.system.metrics tags: - tag: component value: zookeper - uuid: 68d1e19a6ec2401ba0dd509de7df2812 name: 'ClickHouse: Check port availability' type: SIMPLE key: 'net.tcp.service[{$CLICKHOUSE.SCHEME},"{HOST.CONN}","{$CLICKHOUSE.PORT}"]' history: 7d valuemap: name: 'Service state' preprocessing: - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 10m tags: - tag: component value: health - tag: component value: network triggers: - uuid: 1d4b9842b66b4da9ab3750944b2fa050 expression: 'last(/ClickHouse by HTTP/net.tcp.service[{$CLICKHOUSE.SCHEME},"{HOST.CONN}","{$CLICKHOUSE.PORT}"])=0' name: 'ClickHouse: Port {$CLICKHOUSE.PORT} is unavailable' priority: AVERAGE manual_close: 'YES' tags: - tag: scope value: availability discovery_rules: - uuid: add428c918e34dc9aeb58f946f16cbd2 name: Dictionaries type: DEPENDENT key: clickhouse.dictionaries.discovery delay: '0' filter: evaltype: AND conditions: - macro: '{#NAME}' value: '{$CLICKHOUSE.LLD.FILTER.DICT.MATCHES}' formulaid: A - macro: '{#NAME}' value: '{$CLICKHOUSE.LLD.FILTER.DICT.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: B description: 'Info about dictionaries' item_prototypes: - uuid: 18cc050ff9c84d01ab61df1283fcce97 name: 'ClickHouse: Dictionary {#NAME}: Bytes allocated' type: DEPENDENT key: 'clickhouse.dictionary.bytes_allocated["{#NAME}"]' delay: '0' history: 7d units: B description: 'The amount of RAM the dictionary uses.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "{#NAME}")].bytes_allocated.first()' master_item: key: clickhouse.dictionaries tags: - tag: component value: dictionaries - tag: dictionary value: '{#NAME}' - uuid: fa4efa26e3f64fc89c9d7dad39a610ca name: 'ClickHouse: Dictionary {#NAME}: Element count' type: DEPENDENT key: 'clickhouse.dictionary.element_count["{#NAME}"]' delay: '0' history: 7d description: 'Number of items stored in the dictionary.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "{#NAME}")].element_count.first()' master_item: key: clickhouse.dictionaries tags: - tag: component value: dictionaries - tag: dictionary value: '{#NAME}' - uuid: d9e333b7c1694ef482683679275dcd8d name: 'ClickHouse: Dictionary {#NAME}: Load factor' type: DEPENDENT key: 'clickhouse.dictionary.load_factor["{#NAME}"]' delay: '0' history: 7d units: '%' description: 'The percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table).' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "{#NAME}")].bytes_allocated.first()' - type: MULTIPLIER parameters: - '100' master_item: key: clickhouse.dictionaries tags: - tag: component value: dictionaries - tag: dictionary value: '{#NAME}' master_item: key: clickhouse.dictionaries lld_macro_paths: - lld_macro: '{#NAME}' path: $.name - uuid: ddb7c8c688264a5a8907fbdd6974d395 name: Replicas type: DEPENDENT key: clickhouse.replicas.discovery delay: '0' filter: evaltype: AND conditions: - macro: '{#DB}' value: '{$CLICKHOUSE.LLD.FILTER.DB.MATCHES}' formulaid: A - macro: '{#DB}' value: '{$CLICKHOUSE.LLD.FILTER.DB.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: B description: 'Info about replicas' item_prototypes: - uuid: be577e21420244cb9d47c9fb9e23904a name: 'ClickHouse: {#DB}.{#TABLE}: Active replicas' type: DEPENDENT key: 'clickhouse.replica.active_replicas["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Number of replicas of this table that have a session in ZooKeeper (i.e., the number of functioning replicas). (Have a non-zero value only where there is an active session with ZooKeeper).' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].active_replicas.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 0a84f6788b3f447d958e031df0e2301c name: 'ClickHouse: {#DB}.{#TABLE}: Replica future parts' type: DEPENDENT key: 'clickhouse.replica.future_parts["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Number of data parts that will appear as the result of INSERTs or merges that haven''t been done yet.' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].future_parts.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 6b49a71fcf244b8cb88e21423ac92ae2 name: 'ClickHouse: {#DB}.{#TABLE}: Replica queue inserts size' type: DEPENDENT key: 'clickhouse.replica.inserts_in_queue["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Number of inserts of blocks of data that need to be made.' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].inserts_in_queue.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 29977fdd52c64c4ca0b3565187365a0c name: 'ClickHouse: {#DB}.{#TABLE}: Replica readonly' type: DEPENDENT key: 'clickhouse.replica.is_readonly["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: | Whether the replica is in read-only mode. This mode is turned on if the config doesn't have sections with ZooKeeper, if an unknown error occurred when re-initializing sessions in ZooKeeper, and during session re-initialization in ZooKeeper. valuemap: name: 'Replicas state' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].is_readonly.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' trigger_prototypes: - uuid: 7cfd2342d3654407a941cb56fde5c6bf expression: 'min(/ClickHouse by HTTP/clickhouse.replica.is_readonly["{#DB}.{#TABLE}"],5m)=1' name: 'ClickHouse: {#DB}.{#TABLE} Replica is readonly' priority: WARNING description: 'This mode is turned on if the config doesn''t have sections with ZooKeeper, if an unknown error occurred when re-initializing sessions in ZooKeeper, and during session re-initialization in ZooKeeper.' tags: - tag: scope value: availability - uuid: a889475b8ddf48109af3dd2b7ed58627 name: 'ClickHouse: {#DB}.{#TABLE}: Replica session expired' type: DEPENDENT key: 'clickhouse.replica.is_session_expired["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'True if the ZooKeeper session expired' valuemap: name: 'Replicas state' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].is_session_expired.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' trigger_prototypes: - uuid: 765475cc0be84ce1b85d1f026d08c326 expression: 'min(/ClickHouse by HTTP/clickhouse.replica.is_session_expired["{#DB}.{#TABLE}"],5m)=1' name: 'ClickHouse: {#DB}.{#TABLE} Replica session is expired' priority: WARNING description: 'This mode is turned on if the config doesn''t have sections with ZooKeeper, if an unknown error occurred when re-initializing sessions in ZooKeeper, and during session re-initialization in ZooKeeper.' tags: - tag: scope value: performance - uuid: 8acd6e2195064708886d9944af264c32 name: 'ClickHouse: {#DB}.{#TABLE}: Replica lag' type: DEPENDENT key: 'clickhouse.replica.lag["{#DB}.{#TABLE}"]' delay: '0' history: 7d value_type: FLOAT description: 'Difference between log_max_index and log_pointer' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].replica_lag.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' trigger_prototypes: - uuid: 0b06b448c0ad433c94c2a3788ca53e1a expression: 'min(/ClickHouse by HTTP/clickhouse.replica.lag["{#DB}.{#TABLE}"],5m) > {$CLICKHOUSE.LOG_POSITION.DIFF.MAX.WARN}' name: 'ClickHouse: {#DB}.{#TABLE}: Difference between log_max_index and log_pointer is too high' event_name: 'ClickHouse: {#DB}.{#TABLE}: Difference between log_max_index and log_pointer is too high (More than {$CLICKHOUSE.LOG_POSITION.DIFF.MAX.WARN} for 5m)' priority: WARNING tags: - tag: scope value: availability - uuid: 267603d761864e4cbf764558520ed83c name: 'ClickHouse: {#DB}.{#TABLE}: Replica log max index' type: DEPENDENT key: 'clickhouse.replica.log_max_index["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Maximum entry number in the log of general activity. (Have a non-zero value only where there is an active session with ZooKeeper).' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].log_max_index.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 93bf89b0befa4c238f2143c1841ef629 name: 'ClickHouse: {#DB}.{#TABLE}: Replica log pointer' type: DEPENDENT key: 'clickhouse.replica.log_pointer["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Maximum entry number in the log of general activity that the replica copied to its execution queue, plus one. (Have a non-zero value only where there is an active session with ZooKeeper).' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].log_pointer.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 7b0da84f1e3f4705a209c6be025fc738 name: 'ClickHouse: {#DB}.{#TABLE}: Replica queue merges size' type: DEPENDENT key: 'clickhouse.replica.merges_in_queue["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Number of merges waiting to be made.' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].merges_in_queue.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 51a83465b90e4c82835030fec4f79b6f name: 'ClickHouse: {#DB}.{#TABLE}: Replica parts to check' type: DEPENDENT key: 'clickhouse.replica.parts_to_check["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Number of data parts in the queue for verification. A part is put in the verification queue if there is suspicion that it might be damaged.' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].parts_to_check.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 52ef1c26f14842868424f0c5bf81b35b name: 'ClickHouse: {#DB}.{#TABLE}: Replica queue size' type: DEPENDENT key: 'clickhouse.replica.queue_size["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Size of the queue for operations waiting to be performed.' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].queue_size.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' trigger_prototypes: - uuid: 6fe0a15168db4abfa1cd2a2ee45d62ad expression: 'min(/ClickHouse by HTTP/clickhouse.replica.queue_size["{#DB}.{#TABLE}"],5m)>{$CLICKHOUSE.QUEUE.SIZE.MAX.WARN:"{#TABLE}"}' name: 'ClickHouse: {#DB}.{#TABLE}: Too many operations in queue' event_name: 'ClickHouse: {#DB}.{#TABLE}: Too many operations in queue (over {$CLICKHOUSE.QUEUE.SIZE.MAX.WARN} for 5m)' priority: WARNING tags: - tag: scope value: performance - uuid: 0fc2781f4be44f0e85176e5ed05dc05e name: 'ClickHouse: {#DB}.{#TABLE}: Total replicas' type: DEPENDENT key: 'clickhouse.replica.total_replicas["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Total number of known replicas of this table. (Have a non-zero value only where there is an active session with ZooKeeper).' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].total_replicas.first()' master_item: key: clickhouse.replicas tags: - tag: component value: replication - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' trigger_prototypes: - uuid: f46f1b8d75c54129b6e2ff9ede2501b5 expression: 'max(/ClickHouse by HTTP/clickhouse.replica.active_replicas["{#DB}.{#TABLE}"],5m) < last(/ClickHouse by HTTP/clickhouse.replica.total_replicas["{#DB}.{#TABLE}"])' name: 'ClickHouse: {#DB}.{#TABLE}: Number of active replicas less than number of total replicas' priority: WARNING tags: - tag: scope value: availability graph_prototypes: - uuid: 131e0d19c35f4d87aa62b7766e6e1db3 name: 'ClickHouse: {#DB}.{#TABLE}: Replica log activity' graph_items: - color: 1A7C11 item: host: 'ClickHouse by HTTP' key: 'clickhouse.replica.log_max_index["{#DB}.{#TABLE}"]' - sortorder: '1' color: 2774A4 item: host: 'ClickHouse by HTTP' key: 'clickhouse.replica.log_pointer["{#DB}.{#TABLE}"]' - uuid: 195f5d50a1f0488b9db5d2f1e6a6e8c3 name: 'ClickHouse: {#DB}.{#TABLE}: Replica queues' graph_items: - color: 1A7C11 item: host: 'ClickHouse by HTTP' key: 'clickhouse.replica.merges_in_queue["{#DB}.{#TABLE}"]' - sortorder: '1' color: 2774A4 item: host: 'ClickHouse by HTTP' key: 'clickhouse.replica.inserts_in_queue["{#DB}.{#TABLE}"]' - sortorder: '2' color: F63100 item: host: 'ClickHouse by HTTP' key: 'clickhouse.replica.queue_size["{#DB}.{#TABLE}"]' - sortorder: '3' color: A54F10 item: host: 'ClickHouse by HTTP' key: 'clickhouse.replica.future_parts["{#DB}.{#TABLE}"]' - sortorder: '4' color: FC6EA3 item: host: 'ClickHouse by HTTP' key: 'clickhouse.replica.parts_to_check["{#DB}.{#TABLE}"]' master_item: key: clickhouse.replicas lld_macro_paths: - lld_macro: '{#DB}' path: $.database - lld_macro: '{#TABLE}' path: $.table - uuid: a82f8d0f0a4b493dab0c2133b90113cc name: Tables type: DEPENDENT key: clickhouse.tables.discovery delay: '0' filter: evaltype: AND conditions: - macro: '{#DB}' value: '{$CLICKHOUSE.LLD.FILTER.DB.MATCHES}' formulaid: A - macro: '{#DB}' value: '{$CLICKHOUSE.LLD.FILTER.DB.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: B description: 'Info about tables' item_prototypes: - uuid: f7bb82b047a84c0185b5ca040a75e287 name: 'ClickHouse: {#DB}: Bytes' type: DEPENDENT key: 'clickhouse.db.bytes["{#DB}"]' delay: '0' history: 7d units: B description: 'Database size in bytes.' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}")].bytes.sum()' master_item: key: clickhouse.tables tags: - tag: component value: tables - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 3075954290af4f35b1620a5f15d2947d name: 'ClickHouse: {#DB}.{#TABLE}: Bytes' type: DEPENDENT key: 'clickhouse.table.bytes["{#DB}.{#TABLE}"]' delay: '0' history: 7d units: B description: 'Table size in bytes. Database: {#DB}, table: {#TABLE}' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].bytes.first()' master_item: key: clickhouse.tables tags: - tag: component value: tables - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: e50e12adb7ef47e6ba40db01e7ae01e4 name: 'ClickHouse: {#DB}.{#TABLE}: Parts' type: DEPENDENT key: 'clickhouse.table.parts["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Number of parts of the table. Database: {#DB}, table: {#TABLE}' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].parts.first()' master_item: key: clickhouse.tables tags: - tag: component value: tables - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' - uuid: 0ba557c544854f47bcd9cc65d33f14e7 name: 'ClickHouse: {#DB}.{#TABLE}: Rows' type: DEPENDENT key: 'clickhouse.table.rows["{#DB}.{#TABLE}"]' delay: '0' history: 7d description: 'Number of rows in the table. Database: {#DB}, table: {#TABLE}' preprocessing: - type: JSONPATH parameters: - '$[?(@.database == "{#DB}" && @.table == "{#TABLE}")].rows.first()' master_item: key: clickhouse.tables tags: - tag: component value: tables - tag: database value: '{#DB}' - tag: table value: '{#TABLE}' master_item: key: clickhouse.tables lld_macro_paths: - lld_macro: '{#DB}' path: $.database - lld_macro: '{#TABLE}' path: $.table tags: - tag: class value: database - tag: target value: clickhouse macros: - macro: '{$CLICKHOUSE.DELAYED.FILES.DISTRIBUTED.COUNT.MAX.WARN}' value: '600' description: 'Maximum size of distributed files queue to insert for trigger expression.' - macro: '{$CLICKHOUSE.DELAYED.INSERTS.MAX.WARN}' value: '0' description: 'Maximum number of delayed inserts for trigger expression.' - macro: '{$CLICKHOUSE.LLD.FILTER.DB.MATCHES}' value: '.*' description: 'Filter of discoverable databases' - macro: '{$CLICKHOUSE.LLD.FILTER.DB.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'Filter to exclude discovered databases' - macro: '{$CLICKHOUSE.LLD.FILTER.DICT.MATCHES}' value: '.*' description: 'Filter of discoverable dictionaries' - macro: '{$CLICKHOUSE.LLD.FILTER.DICT.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'Filter to exclude discovered dictionaries' - macro: '{$CLICKHOUSE.LOG_POSITION.DIFF.MAX.WARN}' value: '30' description: 'Maximum diff between log_pointer and log_max_index.' - macro: '{$CLICKHOUSE.NETWORK.ERRORS.MAX.WARN}' value: '5' description: 'Maximum number of smth for trigger expression' - macro: '{$CLICKHOUSE.PARTS.PER.PARTITION.WARN}' value: '300' description: 'Maximum number of parts per partition for trigger expression.' - macro: '{$CLICKHOUSE.PASSWORD}' value: zabbix_pass - macro: '{$CLICKHOUSE.PORT}' value: '8123' description: 'The port of ClickHouse HTTP endpoint' - macro: '{$CLICKHOUSE.QUERY_TIME.MAX.WARN}' value: '600' description: 'Maximum ClickHouse query time in seconds for trigger expression' - macro: '{$CLICKHOUSE.QUEUE.SIZE.MAX.WARN}' value: '20' description: 'Maximum size of the queue for operations waiting to be performed for trigger expression.' - macro: '{$CLICKHOUSE.REPLICA.MAX.WARN}' value: '600' description: 'Replication lag across all tables for trigger expression.' - macro: '{$CLICKHOUSE.SCHEME}' value: http description: 'Request scheme which may be http or https' - macro: '{$CLICKHOUSE.USER}' value: zabbix valuemaps: - uuid: 4d926c30529447b3816cec3f1440b47a name: 'Replicas state' mappings: - value: '0' newvalue: 'False' - value: '1' newvalue: 'True' - uuid: 94147888c7ad4f5bae864eb2698f0237 name: 'Service state' mappings: - value: '0' newvalue: Down - value: '1' newvalue: Up triggers: - uuid: 803947a6de9340ed8596510dcbbeacac expression: 'last(/ClickHouse by HTTP/clickhouse.ping)=0 or last(/ClickHouse by HTTP/net.tcp.service[{$CLICKHOUSE.SCHEME},"{HOST.CONN}","{$CLICKHOUSE.PORT}"]) = 0' name: 'ClickHouse: Service is down' priority: AVERAGE manual_close: 'YES' dependencies: - name: 'ClickHouse: Port {$CLICKHOUSE.PORT} is unavailable' expression: 'last(/ClickHouse by HTTP/net.tcp.service[{$CLICKHOUSE.SCHEME},"{HOST.CONN}","{$CLICKHOUSE.PORT}"])=0' tags: - tag: scope value: availability graphs: - uuid: fc2f8de59c874876addb08ec633338d4 name: 'ClickHouse: Current activity' graph_items: - drawtype: BOLD_LINE color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.query.current - sortorder: '1' drawtype: BOLD_LINE color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.merge.current - uuid: b1b66d510e424965b7e82b902e0f2e58 name: 'ClickHouse: Current connections' graph_items: - drawtype: BOLD_LINE color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.connections.tcp - sortorder: '1' drawtype: BOLD_LINE color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.connections.http - sortorder: '2' drawtype: BOLD_LINE color: F63100 item: host: 'ClickHouse by HTTP' key: clickhouse.connections.mysql - sortorder: '3' drawtype: BOLD_LINE color: A54F10 item: host: 'ClickHouse by HTTP' key: clickhouse.connections.interserver - sortorder: '4' drawtype: BOLD_LINE color: FC6EA3 item: host: 'ClickHouse by HTTP' key: clickhouse.connections.distribute - uuid: 59935d9f612f468b97288d9a0dc62b26 name: 'ClickHouse: Distributed' graph_items: - color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.distributed.files.retry.rate - sortorder: '1' color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.distributed.files.fail.rate - sortorder: '2' drawtype: BOLD_LINE color: F63100 yaxisside: RIGHT item: host: 'ClickHouse by HTTP' key: clickhouse.distributed.files - uuid: edd18c769edd442cba00adf97ae4192a name: 'ClickHouse: Insert/Merge bytes rate' graph_items: - drawtype: BOLD_LINE color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.inserted_bytes.rate - sortorder: '1' drawtype: BOLD_LINE color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.merge_bytes.rate - uuid: 0fd813049e7449b69366021ffa0d3ec7 name: 'ClickHouse: Insert/Merge rows rate' graph_items: - drawtype: BOLD_LINE color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.inserted_rows.rate - sortorder: '1' drawtype: BOLD_LINE color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.merge_rows.rate - uuid: 5a688818d4fa44f687aa8415abddc7c0 name: 'ClickHouse: Memory Usage' graph_items: - color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.memory.tracking.merges - sortorder: '1' color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.memory.tracking.schedule.pool - sortorder: '2' color: F63100 item: host: 'ClickHouse by HTTP' key: clickhouse.memory.tracking.background.moves - sortorder: '3' color: A54F10 item: host: 'ClickHouse by HTTP' key: clickhouse.memory.tracking.background - sortorder: '4' color: FC6EA3 item: host: 'ClickHouse by HTTP' key: clickhouse.memory.tracking - uuid: ba22cf13ef6f46299651d15fb860016d name: 'ClickHouse: Memory Utilization' graph_items: - drawtype: GRADIENT_LINE color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.jemalloc.allocated - sortorder: '1' drawtype: GRADIENT_LINE color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.jemalloc.resident - sortorder: '2' drawtype: GRADIENT_LINE color: F63100 item: host: 'ClickHouse by HTTP' key: clickhouse.jemalloc.mapped - uuid: 9593c61ecdfa4c83ab9421ed85a82f36 name: 'ClickHouse: Queries rate' graph_items: - drawtype: BOLD_LINE color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.query.rate - sortorder: '1' drawtype: BOLD_LINE color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.select_query.rate - sortorder: '2' drawtype: BOLD_LINE color: F63100 item: host: 'ClickHouse by HTTP' key: clickhouse.insert_query.rate - uuid: eaa80cf006e84d26903cf1004a268a31 name: 'ClickHouse: Syscalls in fly' graph_items: - drawtype: BOLD_LINE color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.read - sortorder: '1' drawtype: BOLD_LINE color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.write - uuid: eda00404a372434ea4f511c219bf3295 name: 'ClickHouse: Uptime' graph_items: - color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.uptime - uuid: c90d0b21105e42bd9f4399d77ba0a81a name: 'ClickHouse: Zookeeper exceptions rate' graph_items: - color: 1A7C11 item: host: 'ClickHouse by HTTP' key: clickhouse.zookeper.exceptions.rate - sortorder: '1' color: 2774A4 item: host: 'ClickHouse by HTTP' key: clickhouse.zookeper.hw_exceptions.rate - sortorder: '2' color: F63100 item: host: 'ClickHouse by HTTP' key: clickhouse.zookeper.user_exceptions.rate - sortorder: '3' drawtype: BOLD_LINE color: A54F10 yaxisside: RIGHT item: host: 'ClickHouse by HTTP' key: clickhouse.zookeper.watch