diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-20 02:18:09 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-20 02:18:09 +0300 |
commit | 6ed4ec3e0b1340f96b7c043ef51d1b33bbe85fde (patch) | |
tree | dc4d20fe6064752c0bd323187252c77e0a89144b /lib/gitlab/memory | |
parent | 9868dae7fc0655bd7ce4a6887d4e6d487690eeed (diff) |
Add latest changes from gitlab-org/gitlab@15-4-stable-eev15.4.0-rc42
Diffstat (limited to 'lib/gitlab/memory')
-rw-r--r-- | lib/gitlab/memory/jemalloc.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/memory/reports/jemalloc_stats.rb | 9 | ||||
-rw-r--r-- | lib/gitlab/memory/watchdog.rb | 139 |
3 files changed, 119 insertions, 39 deletions
diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb index 7163a70a5cb..e20e186cab9 100644 --- a/lib/gitlab/memory/jemalloc.rb +++ b/lib/gitlab/memory/jemalloc.rb @@ -27,21 +27,27 @@ module Gitlab # Write jemalloc stats to the given directory # @param [String] path Directory path the dump will be put into + # @param [String] tmp_dir Directory path the dump will be streaming to. It is moved to `path` when finished. # @param [String] format `json` or `txt` # @param [String] filename_label Optional custom string that will be injected into the file name, e.g. `worker_0` # @return [String] Full path to the resulting dump file - def dump_stats(path:, format: STATS_DEFAULT_FORMAT, filename_label: nil) + def dump_stats(path:, tmp_dir: Dir.tmpdir, format: STATS_DEFAULT_FORMAT, filename_label: nil) verify_format!(format) format_settings = STATS_FORMATS[format] + tmp_file_path = File.join(tmp_dir, file_name(format_settings[:extension], filename_label)) file_path = File.join(path, file_name(format_settings[:extension], filename_label)) with_malloc_stats_print do |stats_print| - File.open(file_path, 'wb') do |io| + File.open(tmp_file_path, 'wb') do |io| write_stats(stats_print, io, format_settings) end end + # On OSX, `with_malloc_stats_print` is no-op, and, as result, no file will be written + return unless File.exist?(tmp_file_path) + + FileUtils.mv(tmp_file_path, file_path) file_path end diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb index b99bec4ac3e..05f0717d7c3 100644 --- a/lib/gitlab/memory/reports/jemalloc_stats.rb +++ b/lib/gitlab/memory/reports/jemalloc_stats.rb @@ -18,12 +18,19 @@ module Gitlab def initialize(reports_path:) @reports_path = reports_path + + # Store report in tmp subdir while it is still streaming. + # This will clearly separate finished reports from the files we are still writing to. + @tmp_dir = File.join(@reports_path, 'tmp') + FileUtils.mkdir_p(@tmp_dir) end def run return unless active? - Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, filename_label: worker_id).tap { cleanup } + Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, filename_label: worker_id).tap do + cleanup + end end def active? diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 91edb68ad66..38231fa933b 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -16,8 +16,9 @@ module Gitlab # The duration for which a process may be above a given fragmentation # threshold is computed as `max_strikes * sleep_time_seconds`. class Watchdog - DEFAULT_SLEEP_TIME_SECONDS = 60 - DEFAULT_HEAP_FRAG_THRESHOLD = 0.5 + DEFAULT_SLEEP_TIME_SECONDS = 60 * 5 + DEFAULT_MAX_HEAP_FRAG = 0.5 + DEFAULT_MAX_MEM_GROWTH = 3.0 DEFAULT_MAX_STRIKES = 5 # This handler does nothing. It returns `false` to indicate to the @@ -29,7 +30,7 @@ module Gitlab class NullHandler include Singleton - def on_high_heap_fragmentation(value) + def call # NOP false end @@ -41,7 +42,7 @@ module Gitlab @pid = pid end - def on_high_heap_fragmentation(value) + def call Process.kill(:TERM, @pid) true end @@ -55,7 +56,7 @@ module Gitlab @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) end - def on_high_heap_fragmentation(value) + def call @worker.term true end @@ -63,6 +64,9 @@ module Gitlab # max_heap_fragmentation: # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. + # max_mem_growth: + # A multiplier for how much excess private memory a worker can map compared to a reference process + # (itself or the primary in a pre-fork server.) # max_strikes: # How many times the process is allowed to be above max_heap_fragmentation before # a handler is invoked. @@ -71,7 +75,8 @@ module Gitlab def initialize( handler: NullHandler.instance, logger: Logger.new($stdout), - max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_HEAP_FRAG_THRESHOLD, + max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_MAX_HEAP_FRAG, + max_mem_growth: ENV['GITLAB_MEMWD_MAX_MEM_GROWTH']&.to_f || DEFAULT_MAX_MEM_GROWTH, max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES, sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS, **options) @@ -79,17 +84,37 @@ module Gitlab @handler = handler @logger = logger - @max_heap_fragmentation = max_heap_fragmentation @sleep_time_seconds = sleep_time_seconds @max_strikes = max_strikes + @stats = { + heap_frag: { + max: max_heap_fragmentation, + strikes: 0 + }, + mem_growth: { + max: max_mem_growth, + strikes: 0 + } + } @alive = true - @strikes = 0 init_prometheus_metrics(max_heap_fragmentation) end - attr_reader :strikes, :max_heap_fragmentation, :max_strikes, :sleep_time_seconds + attr_reader :max_strikes, :sleep_time_seconds + + def max_heap_fragmentation + @stats[:heap_frag][:max] + end + + def max_mem_growth + @stats[:mem_growth][:max] + end + + def strikes(stat) + @stats[stat][:strikes] + end def call @logger.info(log_labels.merge(message: 'started')) @@ -97,7 +122,10 @@ module Gitlab while @alive sleep(@sleep_time_seconds) - monitor_heap_fragmentation if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + next unless Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + + monitor_heap_fragmentation + monitor_memory_growth end @logger.info(log_labels.merge(message: 'stopped')) @@ -109,32 +137,73 @@ module Gitlab private - def monitor_heap_fragmentation - heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + def monitor_memory_condition(stat_key) + return unless @alive + + stat = @stats[stat_key] + + ok, labels = yield(stat) - if heap_fragmentation > @max_heap_fragmentation - @strikes += 1 - @heap_frag_violations.increment + if ok + stat[:strikes] = 0 else - @strikes = 0 + stat[:strikes] += 1 + @counter_violations.increment(reason: stat_key.to_s) end - if @strikes > @max_strikes - # If the handler returns true, it means the event is handled and we can shut down. - @alive = !handle_heap_fragmentation_limit_exceeded(heap_fragmentation) - @strikes = 0 + if stat[:strikes] > @max_strikes + @alive = !memory_limit_exceeded_callback(stat_key, labels) + stat[:strikes] = 0 end end - def handle_heap_fragmentation_limit_exceeded(value) - @logger.warn( - log_labels.merge( - message: 'heap fragmentation limit exceeded', - memwd_cur_heap_frag: value - )) - @heap_frag_violations_handled.increment + def monitor_heap_fragmentation + monitor_memory_condition(:heap_frag) do |stat| + heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + [ + heap_fragmentation <= stat[:max], + { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: heap_fragmentation, + memwd_max_heap_frag: stat[:max] + } + ] + end + end + + def monitor_memory_growth + monitor_memory_condition(:mem_growth) do |stat| + worker_uss = Gitlab::Metrics::System.memory_usage_uss_pss[:uss] + reference_uss = reference_mem[:uss] + memory_limit = stat[:max] * reference_uss + [ + worker_uss <= memory_limit, + { + message: 'memory limit exceeded', + memwd_uss_bytes: worker_uss, + memwd_ref_uss_bytes: reference_uss, + memwd_max_uss_bytes: memory_limit + } + ] + end + end + + # On pre-fork systems this would be the primary process memory from which workers fork. + # Otherwise it is the current process' memory. + # + # We initialize this lazily because in the initializer the application may not have + # finished booting yet, which would yield an incorrect baseline. + def reference_mem + @reference_mem ||= Gitlab::Metrics::System.memory_usage_uss_pss(pid: Gitlab::Cluster::PRIMARY_PID) + end + + def memory_limit_exceeded_callback(stat_key, handler_labels) + all_labels = log_labels.merge(handler_labels) + .merge(memwd_cur_strikes: strikes(stat_key)) + @logger.warn(all_labels) + @counter_violations_handled.increment(reason: stat_key.to_s) - handler.on_high_heap_fragmentation(value) + handler.call end def handler @@ -151,9 +220,7 @@ module Gitlab worker_id: worker_id, memwd_handler_class: handler.class.name, memwd_sleep_time_s: @sleep_time_seconds, - memwd_max_heap_frag: @max_heap_fragmentation, memwd_max_strikes: @max_strikes, - memwd_cur_strikes: @strikes, memwd_rss_bytes: process_rss_bytes } end @@ -174,14 +241,14 @@ module Gitlab @heap_frag_limit.set({}, max_heap_fragmentation) default_labels = { pid: worker_id } - @heap_frag_violations = Gitlab::Metrics.counter( - :gitlab_memwd_heap_frag_violations_total, - 'Total number of times heap fragmentation in a Ruby process exceeded its allowed maximum', + @counter_violations = Gitlab::Metrics.counter( + :gitlab_memwd_violations_total, + 'Total number of times a Ruby process violated a memory threshold', default_labels ) - @heap_frag_violations_handled = Gitlab::Metrics.counter( - :gitlab_memwd_heap_frag_violations_handled_total, - 'Total number of times heap fragmentation violations in a Ruby process were handled', + @counter_violations_handled = Gitlab::Metrics.counter( + :gitlab_memwd_violations_handled_total, + 'Total number of times Ruby process memory violations were handled', default_labels ) end |