diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-22 15:10:30 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-22 15:10:30 +0300 |
commit | 49203bfa3c7eb607a7561ae7da9b5c52aa49fd77 (patch) | |
tree | f33cd54ec9a45d69a3e58fe93735070d3b718913 /gems | |
parent | 3c9a2dd62025043448c9ea9a6df86422874ee4be (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'gems')
-rw-r--r-- | gems/gitlab-rspec/lib/gitlab/rspec/all.rb | 2 | ||||
-rw-r--r-- | gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb | 66 | ||||
-rw-r--r-- | gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb | 39 | ||||
-rw-r--r-- | gems/gitlab-utils/lib/gitlab/utils/all.rb | 1 | ||||
-rw-r--r-- | gems/gitlab-utils/lib/gitlab/utils/system.rb | 172 | ||||
-rw-r--r-- | gems/gitlab-utils/spec/gitlab/utils/system_spec.rb | 364 |
6 files changed, 644 insertions, 0 deletions
diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/all.rb b/gems/gitlab-rspec/lib/gitlab/rspec/all.rb index 091d2ba0287..518161fa549 100644 --- a/gems/gitlab-rspec/lib/gitlab/rspec/all.rb +++ b/gems/gitlab-rspec/lib/gitlab/rspec/all.rb @@ -2,6 +2,8 @@ require_relative "../rspec" require_relative "stub_env" +require_relative "next_instance_of" +require_relative "next_found_instance_of" require_relative "configurations/time_travel" diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb b/gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb new file mode 100644 index 00000000000..ed402014df4 --- /dev/null +++ b/gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module NextFoundInstanceOf + ERROR_MESSAGE = 'NextFoundInstanceOf mock helpers can only be used with ActiveRecord targets' + HELPER_METHOD_PATTERN = /(?:allow|expect)_next_found_(?<number>\d+)_instances_of/ + + def method_missing(method_name, ...) + match_data = method_name.match(HELPER_METHOD_PATTERN) + return super unless match_data + + helper_method = method_name.to_s.sub("_#{match_data[:number]}", '') + + public_send(helper_method, *args, match_data[:number].to_i, &block) # rubocop:disable GitlabSecurity/PublicSend -- it is safe + end + + def respond_to_missing?(method_name, ...) + match_data = method_name.match(HELPER_METHOD_PATTERN) + return super unless match_data + + helper_method = method_name.to_s.sub("_#{match_data[:number]}", '') + helper_method.respond_to_missing?(helper_method, *args, &block) + end + + def expect_next_found_instance_of(klass, &block) + expect_next_found_instances_of(klass, nil, &block) + end + + def expect_next_found_instances_of(klass, number, &block) + check_if_active_record!(klass) + + stub_allocate(expect(klass), klass, number, &block) + end + + def allow_next_found_instance_of(klass, &block) + allow_next_found_instances_of(klass, nil, &block) + end + + def allow_next_found_instances_of(klass, number, &block) + check_if_active_record!(klass) + + stub_allocate(allow(klass), klass, number, &block) + end + + private + + def check_if_active_record!(klass) + raise ArgumentError, ERROR_MESSAGE unless klass < ActiveRecord::Base + end + + def stub_allocate(target, klass, number, &_block) + stub = receive(:allocate) + stub.exactly(number).times if number + + target.to stub.and_wrap_original do |method| + method.call.tap do |allocation| + # ActiveRecord::Core.allocate returns a frozen object: + # https://github.com/rails/rails/blob/291a3d2ef29a3842d1156ada7526f4ee60dd2b59/activerecord/lib/active_record/core.rb#L620 + # It's unexpected behavior and probably a bug in Rails + # Let's work it around by setting the attributes to default to unfreeze the object for now + allocation.instance_variable_set(:@attributes, klass._default_attributes) + + yield(allocation) + end + end + end +end diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb b/gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb new file mode 100644 index 00000000000..5cc63fe5c6e --- /dev/null +++ b/gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module NextInstanceOf + def expect_next_instance_of(klass, *new_args, &blk) + stub_new(expect(klass), nil, false, *new_args, &blk) + end + + def expect_next_instances_of(klass, number, ordered = false, *new_args, &blk) + stub_new(expect(klass), number, ordered, *new_args, &blk) + end + + def allow_next_instance_of(klass, *new_args, &blk) + stub_new(allow(klass), nil, false, *new_args, &blk) + end + + def allow_next_instances_of(klass, number, ordered = false, *new_args, &blk) + stub_new(allow(klass), number, ordered, *new_args, &blk) + end + + private + + def stub_new(target, number, ordered = false, *new_args, &blk) + receive_new = receive(:new) + receive_new.ordered if ordered + receive_new.with(*new_args) if new_args.present? + + if number.is_a?(Range) + receive_new.at_least(number.begin).times if number.begin + receive_new.at_most(number.end).times if number.end + elsif number + receive_new.exactly(number).times + end + + target.to receive_new.and_wrap_original do |*original_args, **original_kwargs| + method, *original_args = original_args + method.call(*original_args, **original_kwargs).tap(&blk) + end + end +end diff --git a/gems/gitlab-utils/lib/gitlab/utils/all.rb b/gems/gitlab-utils/lib/gitlab/utils/all.rb index 200a21aad88..2e306e7c763 100644 --- a/gems/gitlab-utils/lib/gitlab/utils/all.rb +++ b/gems/gitlab-utils/lib/gitlab/utils/all.rb @@ -4,3 +4,4 @@ require_relative "../utils" require_relative "../version_info" require_relative "version" require_relative "strong_memoize" +require_relative "system" diff --git a/gems/gitlab-utils/lib/gitlab/utils/system.rb b/gems/gitlab-utils/lib/gitlab/utils/system.rb new file mode 100644 index 00000000000..ada9da005b3 --- /dev/null +++ b/gems/gitlab-utils/lib/gitlab/utils/system.rb @@ -0,0 +1,172 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + # Module for gathering system/process statistics such as the memory usage. + # + # This module relies on the /proc filesystem being available. If /proc is + # not available the methods of this module will be stubbed. + module System + extend self + + PROC_STAT_PATH = '/proc/self/stat' + PROC_STATUS_PATH = '/proc/%s/status' + PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' + PROC_LIMITS_PATH = '/proc/self/limits' + PROC_FD_GLOB = '/proc/self/fd/*' + PROC_MEM_INFO = '/proc/meminfo' + + PRIVATE_PAGES_PATTERN = /^(?<type>Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/ + PSS_PATTERN = /^Pss:\s+(?<value>\d+)/ + RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/ + RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/ + RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/ + MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/ + MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (?<unit>.+)/ + + def summary + proportional_mem = memory_usage_uss_pss + { + version: RUBY_DESCRIPTION, + gc_stat: GC.stat, + memory_rss: memory_usage_rss[:total], + memory_uss: proportional_mem[:uss], + memory_pss: proportional_mem[:pss], + time_cputime: cpu_time, + time_realtime: real_time, + time_monotonic: monotonic_time + } + end + + # Returns the given process' RSS (resident set size) in bytes. + def memory_usage_rss(pid: 'self') + results = { total: 0, anon: 0, file: 0 } + + safe_yield_procfile(PROC_STATUS_PATH % pid) do |io| + io.each_line do |line| + if (value = parse_metric_value(line, RSS_TOTAL_PATTERN)) > 0 + results[:total] = value.kilobytes + elsif (value = parse_metric_value(line, RSS_ANON_PATTERN)) > 0 + results[:anon] = value.kilobytes + elsif (value = parse_metric_value(line, RSS_FILE_PATTERN)) > 0 + results[:file] = value.kilobytes + end + end + end + + results + end + + # Returns the given process' USS/PSS (unique/proportional set size) in bytes. + def memory_usage_uss_pss(pid: 'self') + sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) + .transform_values(&:kilobytes) + end + + def memory_total + sum_matches(PROC_MEM_INFO, memory_total: MEM_TOTAL_PATTERN)[:memory_total].kilobytes + end + + def file_descriptor_count + Dir.glob(PROC_FD_GLOB).length + end + + def max_open_file_descriptors + sum_matches(PROC_LIMITS_PATH, max_fds: MAX_OPEN_FILES_PATTERN)[:max_fds] + end + + def cpu_time + Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second) + end + + # Returns the current real time in a given precision. + # + # Returns the time as a Float for precision = :float_second. + def real_time(precision = :float_second) + Process.clock_gettime(Process::CLOCK_REALTIME, precision) + end + + # Returns the current monotonic clock time as seconds with microseconds precision. + # + # Returns the time as a Float. + def monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) + end + + def thread_cpu_time + # Not all OS kernels are supporting `Process::CLOCK_THREAD_CPUTIME_ID` + # Refer: https://gitlab.com/gitlab-org/gitlab/issues/30567#note_221765627 + return unless defined?(Process::CLOCK_THREAD_CPUTIME_ID) + + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) + end + + def thread_cpu_duration(start_time) + end_time = thread_cpu_time + return unless start_time && end_time + + end_time - start_time + end + + # Returns the total time the current process has been running in seconds. + def process_runtime_elapsed_seconds + # Entry 22 (1-indexed) contains the process `starttime`, see: + # https://man7.org/linux/man-pages/man5/proc.5.html + # + # This value is a fixed timestamp in clock ticks. + # To obtain an elapsed time in seconds, we divide by the number + # of ticks per second and subtract from the system uptime. + start_time_ticks = proc_stat_entries[21].to_f + clock_ticks_per_second = Etc.sysconf(Etc::SC_CLK_TCK) + uptime - (start_time_ticks / clock_ticks_per_second) + end + + private + + # Given a path to a file in /proc and a hash of (metric, pattern) pairs, + # sums up all values found for those patterns under the respective metric. + def sum_matches(proc_file, **patterns) + results = patterns.transform_values { 0 } + + safe_yield_procfile(proc_file) do |io| + io.each_line do |line| + patterns.each do |metric, pattern| + results[metric] += parse_metric_value(line, pattern) + end + end + end + + results + end + + def parse_metric_value(line, pattern) + match = line.match(pattern) + return 0 unless match + + match.named_captures.fetch('value', 0).to_i + end + + def proc_stat_entries + safe_yield_procfile(PROC_STAT_PATH) do |io| + io.read.split(' ') + end || [] + end + + def safe_yield_procfile(path, &block) + File.open(path, &block) + rescue Errno::ENOENT + # This means the procfile we're reading from did not exist; + # most likely we're on Darwin. + end + + # Equivalent to reading /proc/uptime on Linux 2.6+. + # + # Returns 0 if not supported, e.g. on Darwin. + def uptime + Process.clock_gettime(Process::CLOCK_BOOTTIME) + rescue NameError + 0 + end + end + end +end diff --git a/gems/gitlab-utils/spec/gitlab/utils/system_spec.rb b/gems/gitlab-utils/spec/gitlab/utils/system_spec.rb new file mode 100644 index 00000000000..6d4f3bf039c --- /dev/null +++ b/gems/gitlab-utils/spec/gitlab/utils/system_spec.rb @@ -0,0 +1,364 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Utils::System do + context 'when /proc files exist' do + # Modified column 22 to be 1000 (starttime ticks) + let(:proc_stat) do + <<~SNIP + 2095 (ruby) R 0 2095 2095 34818 2095 4194560 211267 7897 2 0 287 51 10 1 20 0 5 0 1000 566210560 80885 18446744073709551615 94736211292160 94736211292813 140720919612064 0 0 0 0 0 1107394127 0 0 0 17 3 0 0 0 0 0 94736211303768 94736211304544 94736226689024 140720919619473 140720919619513 140720919619513 140720919621604 0 + SNIP + end + + # Fixtures pulled from: + # Linux carbon 5.3.0-7648-generic #41~1586789791~19.10~9593806-Ubuntu SMP + # Mon Apr 13 17:50:40 UTC x86_64 x86_64 x86_64 GNU/Linux + let(:proc_status) do + # most rows omitted for brevity + <<~SNIP + Name: less + VmHWM: 2468 kB + VmRSS: 2468 kB + RssAnon: 260 kB + RssFile: 1024 kB + SNIP + end + + let(:proc_smaps_rollup) do + # full snapshot + <<~SNIP + Rss: 2564 kB + Pss: 503 kB + Pss_Anon: 312 kB + Pss_File: 191 kB + Pss_Shmem: 0 kB + Shared_Clean: 2100 kB + Shared_Dirty: 0 kB + Private_Clean: 152 kB + Private_Dirty: 312 kB + Referenced: 2564 kB + Anonymous: 312 kB + LazyFree: 0 kB + AnonHugePages: 0 kB + ShmemPmdMapped: 0 kB + Shared_Hugetlb: 0 kB + Private_Hugetlb: 0 kB + Swap: 0 kB + SwapPss: 0 kB + Locked: 0 kB + SNIP + end + + let(:proc_limits) do + # full snapshot + <<~SNIP + Limit Soft Limit Hard Limit Units + Max cpu time unlimited unlimited seconds + Max file size unlimited unlimited bytes + Max data size unlimited unlimited bytes + Max stack size 8388608 unlimited bytes + Max core file size 0 unlimited bytes + Max resident set unlimited unlimited bytes + Max processes 126519 126519 processes + Max open files 1024 1048576 files + Max locked memory 67108864 67108864 bytes + Max address space unlimited unlimited bytes + Max file locks unlimited unlimited locks + Max pending signals 126519 126519 signals + Max msgqueue size 819200 819200 bytes + Max nice priority 0 0 + Max realtime priority 0 0 + Max realtime timeout unlimited unlimited us + SNIP + end + + let(:mem_info) do + # full snapshot + <<~SNIP + MemTotal: 15362536 kB + MemFree: 3403136 kB + MemAvailable: 13044528 kB + Buffers: 272188 kB + Cached: 8171312 kB + SwapCached: 0 kB + Active: 3332084 kB + Inactive: 6981076 kB + Active(anon): 1603868 kB + Inactive(anon): 9044 kB + Active(file): 1728216 kB + Inactive(file): 6972032 kB + Unevictable: 18676 kB + Mlocked: 18676 kB + SwapTotal: 0 kB + SwapFree: 0 kB + Dirty: 6808 kB + Writeback: 0 kB + AnonPages: 1888300 kB + Mapped: 166164 kB + Shmem: 12932 kB + KReclaimable: 1275120 kB + Slab: 1495480 kB + SReclaimable: 1275120 kB + SUnreclaim: 220360 kB + KernelStack: 7072 kB + PageTables: 11936 kB + NFS_Unstable: 0 kB + Bounce: 0 kB + WritebackTmp: 0 kB + CommitLimit: 7681268 kB + Committed_AS: 4976100 kB + VmallocTotal: 34359738367 kB + VmallocUsed: 25532 kB + VmallocChunk: 0 kB + Percpu: 23200 kB + HardwareCorrupted: 0 kB + AnonHugePages: 202752 kB + ShmemHugePages: 0 kB + ShmemPmdMapped: 0 kB + FileHugePages: 0 kB + FilePmdMapped: 0 kB + CmaTotal: 0 kB + CmaFree: 0 kB + HugePages_Total: 0 + HugePages_Free: 0 + HugePages_Rsvd: 0 + HugePages_Surp: 0 + Hugepagesize: 2048 kB + Hugetlb: 0 kB + DirectMap4k: 4637504 kB + DirectMap2M: 11087872 kB + DirectMap1G: 2097152 kB + SNIP + end + + describe '.memory_usage_rss' do + context 'without PID' do + it "returns a hash containing RSS metrics in bytes for current process" do + mock_existing_proc_file('/proc/self/status', proc_status) + + expect(described_class.memory_usage_rss).to eq( + total: 2527232, + anon: 266240, + file: 1048576 + ) + end + end + + context 'with PID' do + it "returns a hash containing RSS metrics in bytes for given process" do + mock_existing_proc_file('/proc/7/status', proc_status) + + expect(described_class.memory_usage_rss(pid: 7)).to eq( + total: 2527232, + anon: 266240, + file: 1048576 + ) + end + end + end + + describe '.file_descriptor_count' do + it 'returns the amount of open file descriptors' do + expect(Dir).to receive(:glob).and_return(['/some/path', '/some/other/path']) + + expect(described_class.file_descriptor_count).to eq(2) + end + end + + describe '.max_open_file_descriptors' do + it 'returns the max allowed open file descriptors' do + mock_existing_proc_file('/proc/self/limits', proc_limits) + + expect(described_class.max_open_file_descriptors).to eq(1024) + end + end + + describe '.memory_usage_uss_pss' do + context 'without PID' do + it "returns the current process' unique and porportional set size (USS/PSS) in bytes" do + mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup) + + # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 + expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072) + end + end + + context 'with PID' do + it "returns the given process' unique and porportional set size (USS/PSS) in bytes" do + mock_existing_proc_file('/proc/7/smaps_rollup', proc_smaps_rollup) + + # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024 + expect(described_class.memory_usage_uss_pss(pid: 7)).to eq(uss: 475136, pss: 515072) + end + end + end + + describe '.memory_total' do + it "returns the current process' resident set size (RSS) in bytes" do + mock_existing_proc_file('/proc/meminfo', mem_info) + + expect(described_class.memory_total).to eq(15731236864) + end + end + + describe '.process_runtime_elapsed_seconds' do + it 'returns the seconds elapsed since the process was started' do + # sets process starttime ticks to 1000 + mock_existing_proc_file('/proc/self/stat', proc_stat) + # system clock ticks/sec + expect(Etc).to receive(:sysconf).with(Etc::SC_CLK_TCK).and_return(100) + # system uptime in seconds + expect(::Process).to receive(:clock_gettime).and_return(15) + + # uptime - (starttime_ticks / ticks_per_sec) + expect(described_class.process_runtime_elapsed_seconds).to eq(5) + end + + context 'when inputs are not available' do + it 'returns 0' do + mock_missing_proc_file + expect(::Process).to receive(:clock_gettime).and_raise(NameError) + + expect(described_class.process_runtime_elapsed_seconds).to eq(0) + end + end + end + + describe '.summary' do + it 'contains a selection of the available fields' do + stub_const('RUBY_DESCRIPTION', 'ruby-3.0-patch1') + mock_existing_proc_file('/proc/self/status', proc_status) + mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup) + + summary = described_class.summary + + expect(summary[:version]).to eq('ruby-3.0-patch1') + expect(summary[:gc_stat].keys).to eq(GC.stat.keys) + expect(summary[:memory_rss]).to eq(2527232) + expect(summary[:memory_uss]).to eq(475136) + expect(summary[:memory_pss]).to eq(515072) + expect(summary[:time_cputime]).to be_a(Float) + expect(summary[:time_realtime]).to be_a(Float) + expect(summary[:time_monotonic]).to be_a(Float) + end + end + end + + context 'when /proc files do not exist' do + before do + mock_missing_proc_file + end + + describe '.memory_usage_rss' do + it 'returns 0 for all components' do + expect(described_class.memory_usage_rss).to eq( + total: 0, + anon: 0, + file: 0 + ) + end + end + + describe '.memory_usage_uss_pss' do + it "returns 0 for all components" do + expect(described_class.memory_usage_uss_pss).to eq(uss: 0, pss: 0) + end + end + + describe '.file_descriptor_count' do + it 'returns 0' do + expect(Dir).to receive(:glob).and_return([]) + + expect(described_class.file_descriptor_count).to eq(0) + end + end + + describe '.max_open_file_descriptors' do + it 'returns 0' do + expect(described_class.max_open_file_descriptors).to eq(0) + end + end + + describe '.summary' do + it 'returns only available fields' do + summary = described_class.summary + + expect(summary[:version]).to be_a(String) + expect(summary[:gc_stat].keys).to eq(GC.stat.keys) + expect(summary[:memory_rss]).to eq(0) + expect(summary[:memory_uss]).to eq(0) + expect(summary[:memory_pss]).to eq(0) + expect(summary[:time_cputime]).to be_a(Float) + expect(summary[:time_realtime]).to be_a(Float) + expect(summary[:time_monotonic]).to be_a(Float) + end + end + end + + describe '.cpu_time' do + it 'returns a Float' do + expect(described_class.cpu_time).to be_an(Float) + end + end + + describe '.real_time' do + it 'returns a Float' do + expect(described_class.real_time).to be_an(Float) + end + end + + describe '.monotonic_time' do + it 'returns a Float' do + expect(described_class.monotonic_time).to be_an(Float) + end + end + + describe '.thread_cpu_time' do + it 'returns cpu_time on supported platform' do + stub_const("Process::CLOCK_THREAD_CPUTIME_ID", 16) + + expect(Process).to receive(:clock_gettime) + .with(16, kind_of(Symbol)).and_return(0.111222333) + + expect(described_class.thread_cpu_time).to eq(0.111222333) + end + + it 'returns nil on unsupported platform' do + hide_const("Process::CLOCK_THREAD_CPUTIME_ID") + + expect(described_class.thread_cpu_time).to be_nil + end + end + + describe '.thread_cpu_duration' do + let(:start_time) { described_class.thread_cpu_time } + + it 'returns difference between start and current time' do + stub_const("Process::CLOCK_THREAD_CPUTIME_ID", 16) + + expect(Process).to receive(:clock_gettime) + .with(16, kind_of(Symbol)) + .and_return( + 0.111222333, + 0.222333833 + ) + + expect(described_class.thread_cpu_duration(start_time)).to eq(0.1111115) + end + + it 'returns nil on unsupported platform' do + hide_const("Process::CLOCK_THREAD_CPUTIME_ID") + + expect(described_class.thread_cpu_duration(start_time)).to be_nil + end + end + + def mock_existing_proc_file(path, content) + allow(File).to receive(:open).with(path) { |_path, &block| block.call(StringIO.new(content)) } + end + + def mock_missing_proc_file + allow(File).to receive(:open).and_raise(Errno::ENOENT) + end +end |