diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-03-09 15:15:54 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-03-09 15:15:54 +0300 |
commit | 0c1344a7c19635e387e6f7af20591ad73f46ddff (patch) | |
tree | 2b92f62ea6e4e901127f6247a910d8f7b6f2c7c1 /vendor | |
parent | a74ca2457e7c8a26ff5e12211d741b473c86c0b8 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'vendor')
18 files changed, 1079 insertions, 0 deletions
diff --git a/vendor/gems/cloud_profiler_agent/.gitlab-ci.yml b/vendor/gems/cloud_profiler_agent/.gitlab-ci.yml new file mode 100644 index 00000000000..9e508d920e8 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/.gitlab-ci.yml @@ -0,0 +1,32 @@ +workflow: + rules: + - if: $CI_MERGE_REQUEST_ID + +.rspec: + cache: + key: cloud_profiler_agent-ruby-${RUBY_VERSION} + paths: + - vendor/gems/cloud_profiler_agent/vendor/ruby + before_script: + - cd vendor/gems/cloud_profiler_agent + - ruby -v # Print out ruby version for debugging + - gem install bundler --no-document # Bundler is not installed with the image + - bundle config set --local path 'vendor' # Install dependencies into ./vendor/ruby + - bundle config set with 'development' + - bundle config set --local frozen 'true' # Disallow Gemfile.lock changes on CI + - bundle config # Show bundler configuration + - bundle install -j $(nproc) + script: + - bundle exec rspec + +rspec-3.0: + image: "ruby:3.0" + extends: .rspec + +rspec-3.1: + image: "ruby:3.1" + extends: .rspec + +rspec-3.2: + image: "ruby:3.2" + extends: .rspec diff --git a/vendor/gems/cloud_profiler_agent/Gemfile b/vendor/gems/cloud_profiler_agent/Gemfile new file mode 100644 index 00000000000..5f10ba8c95a --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/Gemfile @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' +gemspec diff --git a/vendor/gems/cloud_profiler_agent/Gemfile.lock b/vendor/gems/cloud_profiler_agent/Gemfile.lock new file mode 100644 index 00000000000..47087362534 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/Gemfile.lock @@ -0,0 +1,126 @@ +PATH + remote: . + specs: + cloud_profiler_agent (0.0.1.pre) + google-cloud-profiler-v2 (~> 0.3) + google-protobuf (~> 3.13) + googleauth (>= 0.14) + stackprof (~> 0.2) + +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) + ast (2.4.2) + diff-lcs (1.5.0) + faraday (1.10.2) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.4) + multipart-post (~> 2) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + gapic-common (0.17.1) + faraday (>= 1.9, < 3.a) + faraday-retry (>= 1.0, < 3.a) + google-protobuf (~> 3.14) + googleapis-common-protos (>= 1.3.12, < 2.a) + googleapis-common-protos-types (>= 1.3.1, < 2.a) + googleauth (~> 1.0) + grpc (~> 1.36) + google-cloud-errors (1.3.0) + google-cloud-profiler-v2 (0.3.0) + gapic-common (>= 0.10, < 2.a) + google-cloud-errors (~> 1.0) + google-protobuf (3.22.0) + googleapis-common-protos (1.4.0) + google-protobuf (~> 3.14) + googleapis-common-protos-types (~> 1.2) + grpc (~> 1.27) + googleapis-common-protos-types (1.5.0) + google-protobuf (~> 3.14) + googleauth (1.3.0) + faraday (>= 0.17.3, < 3.a) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (>= 0.16, < 2.a) + grpc (1.52.0) + google-protobuf (~> 3.21) + googleapis-common-protos-types (~> 1.0) + json (2.6.2) + jwt (2.1.0) + memoist (0.16.2) + multi_json (1.14.1) + multipart-post (2.2.3) + os (1.1.1) + parallel (1.22.1) + parser (3.1.3.0) + ast (~> 2.4.1) + public_suffix (5.0.0) + rainbow (3.1.1) + regexp_parser (2.6.0) + rexml (3.2.5) + rspec (3.11.0) + rspec-core (~> 3.11.0) + rspec-expectations (~> 3.11.0) + rspec-mocks (~> 3.11.0) + rspec-core (3.11.0) + rspec-support (~> 3.11.0) + rspec-expectations (3.11.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.11.0) + rspec-mocks (3.11.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.11.0) + rspec-support (3.11.1) + rubocop (1.38.0) + json (~> 2.3) + parallel (~> 1.10) + parser (>= 3.1.2.1) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.8, < 3.0) + rexml (>= 3.2.5, < 4.0) + rubocop-ast (>= 1.23.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 1.4.0, < 3.0) + rubocop-ast (1.23.0) + parser (>= 3.1.1.0) + ruby-progressbar (1.11.0) + ruby2_keywords (0.0.5) + signet (0.17.0) + addressable (~> 2.8) + faraday (>= 0.17.5, < 3.a) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) + stackprof (0.2.21) + unicode-display_width (2.3.0) + +PLATFORMS + ruby + +DEPENDENCIES + cloud_profiler_agent! + rspec (>= 3.10) + rubocop (>= 1.2) + +BUNDLED WITH + 2.3.26 diff --git a/vendor/gems/cloud_profiler_agent/LICENSE b/vendor/gems/cloud_profiler_agent/LICENSE new file mode 100644 index 00000000000..4a8062eea52 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2020, Remind101, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/gems/cloud_profiler_agent/README.md b/vendor/gems/cloud_profiler_agent/README.md new file mode 100644 index 00000000000..5ae2ec7c15c --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/README.md @@ -0,0 +1,30 @@ +# ruby-cloud-profiler + +An implementation of [Google Cloud Profiler](https://cloud.google.com/profiler/docs) +for Ruby. + +This project is not officially supported or endorsed by Google in any way. + +Under the hood, the agent uses [Stackprof](https://github.com/tmm1/stackprof) +to collect the profiling data, and then converts it to +[the pprof format](https://github.com/google/pprof/blob/master/proto/profile.proto) +expected by Cloud Profiler. The Cloud Profiler API doesn't have pretty HTML +documentation, but is described +[in the googleapis specification](https://github.com/googleapis/googleapis/blob/master/google/devtools/cloudprofiler/v2/profiler.proto) +which creates +[generated code in google-api-ruby-client](https://github.com/googleapis/google-api-ruby-client/tree/master/generated/google/apis/cloudprofiler_v2). + +To use, you need to decide what to name your service and you need a Google +Cloud project ID: + + require 'cloud_profiler_agent' + agent = CloudProfilerAgent::Agent.new(service: 'my-service', project_id: 'my-project-id') + agent.start + +This will start a background thread that will merrily poll the Cloud Profiler +API to see what kinds of profiles it should collect, and when. Then it will run +stackprof, and upload the profiles. + +Note: the agent can only profile its own process. If your Ruby application is +running from a webserver that forks subprocesses, then you'll need to somehow +arrange to start the agent in the subprocess. diff --git a/vendor/gems/cloud_profiler_agent/cloud_profiler_agent.gemspec b/vendor/gems/cloud_profiler_agent/cloud_profiler_agent.gemspec new file mode 100644 index 00000000000..cf84c100fc3 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/cloud_profiler_agent.gemspec @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'cloud_profiler_agent' + +Gem::Specification.new do |spec| + spec.name = 'cloud_profiler_agent' + spec.version = CloudProfilerAgent::VERSION + spec.authors = ['Remind'] + + spec.summary = 'Profiling agent for Google Cloud Profiler' + spec.homepage = 'https://github.com/remind101/ruby-cloud-profiler' + spec.license = 'BSD-2-Clause' + + spec.required_ruby_version = '>= 3.0.0' + + spec.files = ['lib/profile_pb.rb', + 'lib/cloud_profiler_agent.rb', + 'lib/cloud_profiler_agent/agent.rb', + 'lib/cloud_profiler_agent/looper.rb', + 'lib/cloud_profiler_agent/pprof_builder.rb'] + + spec.add_runtime_dependency 'googleauth', '>= 0.14' + spec.add_runtime_dependency 'google-cloud-profiler-v2', '~> 0.3' + spec.add_runtime_dependency 'google-protobuf', '~> 3.13' + spec.add_runtime_dependency 'stackprof', '~> 0.2' + + spec.add_development_dependency 'rspec', '>= 3.10' + spec.add_development_dependency 'rubocop', '>= 1.2' +end diff --git a/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent.rb b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent.rb new file mode 100644 index 00000000000..60350e2e54a --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module CloudProfilerAgent + VERSION = '0.0.1.pre' + autoload :Agent, 'cloud_profiler_agent/agent' + autoload :PprofBuilder, 'cloud_profiler_agent/pprof_builder' + autoload :Looper, 'cloud_profiler_agent/looper' +end + +module Perftools + autoload :Profiles, 'profile_pb' +end diff --git a/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/agent.rb b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/agent.rb new file mode 100644 index 00000000000..d4231d12f72 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/agent.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +require "google/cloud/profiler/v2" +require 'googleauth' +require 'stackprof' +require 'logger' + +module CloudProfilerAgent + GoogleCloudProfiler = ::Google::Cloud::Profiler::V2 + + PROFILE_TYPES = { + CPU: :cpu, + WALL: :wall + }.freeze + # This regexp will ensure the service name is valid. + # See https://cloud.google.com/ruby/docs/reference/google-cloud-profiler-v2/latest/Google-Cloud-Profiler-V2-Deployment#Google__Cloud__Profiler__V2__Deployment_target_instance_ + SERVICE_REGEXP = /^[a-z]([-a-z0-9_.]{0,253}[a-z0-9])?$/ + + # Agent interfaces with the CloudProfiler API. + class Agent + def initialize( + service:, project_id:, service_version: nil, instance: nil, zone: nil, + logger: nil, log_labels: {}) + raise ArgumentError, "service must match #{SERVICE_REGEXP}" unless SERVICE_REGEXP =~ service + + @service = service + @project_id = project_id + + @labels = { language: 'ruby' } + @labels[:version] = service_version unless service_version.nil? + @labels[:zone] = zone unless zone.nil? + + @deployment = GoogleCloudProfiler::Deployment.new(project_id: project_id, target: service, labels: @labels) + + @profile_labels = {} + @profile_labels[:instance] = instance unless instance.nil? + + @google_profiler = GoogleCloudProfiler::ProfilerService::Client.new + + @logger = logger || ::Logger.new($stdout) + @log_labels = log_labels + end + + attr_reader :service, :project_id, :labels, :deployment, :profile_labels, :logger, :log_labels + + def create_google_profile + google_profile_request = GoogleCloudProfiler::CreateProfileRequest.new( + deployment: deployment, + profile_type: PROFILE_TYPES.keys) + + google_profile, wall_time, cpu_time = time { @google_profiler.create_profile(google_profile_request) } + + logger.info( + gcp_ruby_status: "google profile resource created", + duration_s: wall_time, + cpu_s: cpu_time, + **log_labels + ) + + google_profile + end + + # start will begin creating profiles in a background thread, looping + # forever. Exceptions are rescued and logged, and retries are made with + # exponential backoff. + def start + return if @thread&.alive? + + @thread = Thread.new do + Looper.new(logger: logger, log_labels: log_labels).run do + google_profile = create_google_profile + google_profile = profile_app(google_profile) + upload_profile_to_google(google_profile) + end + end + end + + private + + def time + start_monotonic_time = Gitlab::Metrics::System.monotonic_time + start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + + result = yield + + finish_monotonic_time = Gitlab::Metrics::System.monotonic_time + finish_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + [ + result, + finish_monotonic_time - start_monotonic_time, + finish_thread_cpu_time - start_thread_cpu_time + ] + end + + def stackprof_profile_as_pprof(duration, mode) + start_time = Time.now + + stackprof, wall_time, cpu_time = time do + StackProf.run(mode: mode, raw: true, interval: ::Gitlab::StackProf.interval(mode)) do + sleep(duration) + end + end + + logger.info( + gcp_ruby_status: "stackprof run finished", + duration_s: wall_time, + cpu_s: cpu_time, + **log_labels + ) + + pprof_result, wall_time, cpu_time = time do + CloudProfilerAgent::PprofBuilder.convert_stackprof(stackprof, start_time, Time.now) + end + + logger.info( + gcp_ruby_status: "stackprof to pprof converted", + duration_s: wall_time, + cpu_s: cpu_time, + **log_labels + ) + + pprof_result + end + + def profile_app(google_profile) + google_profile.profile_bytes = stackprof_profile_as_pprof(google_profile.duration.seconds, + PROFILE_TYPES.fetch(google_profile.profile_type)) + google_profile + end + + def upload_profile_to_google(google_profile) + start_monotonic_time = Gitlab::Metrics::System.monotonic_time + start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + + @google_profiler.update_profile(profile: google_profile) + + finish_monotonic_time = Gitlab::Metrics::System.monotonic_time + finish_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + logger.info( + gcp_ruby_status: "profile resource updated", + duration_s: finish_monotonic_time - start_monotonic_time, + cpu_s: finish_thread_cpu_time - start_thread_cpu_time, + **log_labels + ) + end + end +end diff --git a/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/looper.rb b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/looper.rb new file mode 100644 index 00000000000..9f0a5ef2abd --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/looper.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require "google/cloud/profiler/v2" +require "logger" + +module CloudProfilerAgent + # Looper is responsible for the main loop of the agent. It calls a + # block repeatedly, handling errors, backing off, and retrying as + # appropriate. + + class Looper + LOG_MESSAGE = "Google Cloud Profiler Ruby" + + def initialize( + min_iteration_sec: 10, + max_iteration_sec: 60 * 60, + backoff_factor: 1.5, + sleeper: ->(sec) { sleep(sec) }, + clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }, + rander: -> { rand }, + logger: nil, + log_labels: {} + ) + + # the minimum and maximum time between iterations of the profiler loop, + # in seconds. Normally the Cloud Profiler API tells us how fast to go, + # but we back off in case of error. + @min_iteration_sec = min_iteration_sec + @max_iteration_sec = max_iteration_sec + @backoff_factor = backoff_factor + + # stubbable for testing + @sleeper = sleeper + @clock = clock + @rander = rander + + @logger = logger || ::Logger.new($stdout) + @log_labels = log_labels + end + + attr_reader :min_iteration_sec, :max_iteration_sec, :backoff_factor, :logger, :log_labels + + def run(max_iterations = 0) + iterations = 0 + iteration_time = @min_iteration_sec + loop do + start_time = @clock.call + iterations += 1 + begin + yield + rescue ::Google::Cloud::Error => e + backoff = backoff_duration(e) + if backoff.nil? + iteration_time = @max_iteration_sec + else + # This might be longer than max_iteration_sec and that's OK: with + # a very large number of agents it might be necessary to achieve + # the objective of 1 profile per minute. + @sleeper.call(backoff) + iteration_time = @min_iteration_sec + end + rescue StandardError => e + iteration_time *= @backoff_factor + (@rander.call / 2) + elapsed = @clock.call - start_time + logger.error( + gcp_ruby_status: "error", + error: e.inspect, + duration_s: elapsed, + **log_labels + ) + else + iteration_time = @min_iteration_sec + end + + return unless iterations < max_iterations || max_iterations == 0 + + iteration_time = [@max_iteration_sec, iteration_time].min + next_time = start_time + iteration_time + delay = next_time - @clock.call + @sleeper.call(delay) if delay > 0 + end + end + + private + + def backoff_duration(error) + # It's unclear how this should work, so this is based on a guess. + # + # https://github.com/googleapis/google-api-ruby-client/issues/1498 + match = /backoff for (?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?/.match(error.message) + return if match.nil? + + hours = Integer(match[1] || 0) + minutes = Integer(match[2] || 0) + seconds = Integer(match[3] || 0) + + seconds + (minutes * 60) + (hours * 60 * 60) + end + end +end diff --git a/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/pprof_builder.rb b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/pprof_builder.rb new file mode 100644 index 00000000000..0f2ff71c791 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/lib/cloud_profiler_agent/pprof_builder.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +require 'zlib' +require 'stackprof' + +module CloudProfilerAgent + # PprofBuilder converts from stackprof to pprof formats. + # + # Typical usage: + # + # start_time = Time.now + # stackprof = StackProf.run(mode: :cpu, raw: true) do + # # ... + # end + # pprof = PprofBuilder.convert_stackprof(stackprof, start_time, Time.now) + # IO::binwrite('profile.pprof', pprof) + # + # StackProf must be invoked with raw: true for the conversion to work. + # + # The pprof format is a gzip-compressed protobuf, documented here: + # https://github.com/google/pprof/blob/master/proto/profile.proto + # + # The conversion is not quite isomorphic. In particular, each sample in pprof + # consists of a stack with line numbers and even instruction addresses. + # StackProf on the other hand records stack frames with only function-level + # granularity. Thus, the line numbers in the pprof output only reflect the + # first line of the function, regardless of which line was actually in the + # stack frame. + class PprofBuilder + def self.convert_stackprof(stackprof, start_time, end_time) + converter = PprofBuilder.new(stackprof, start_time, end_time) + converter.pprof_bytes + end + + def initialize(profile, start_time, end_time) + @profile = profile + @start_time = start_time + @duration = end_time - start_time + + @string_map = StringMap.new + end + + # message returns a Perftools::Profiles::Profile, the deserialized version + # of a pprof profile. + def message + main_mapping = Perftools::Profiles::Mapping.new( + id: 1, + filename: @string_map.add('TODO') + ) + + message = Perftools::Profiles::Profile.new( + sample_type: [sample_type], + mapping: [main_mapping], + time_nanos: @start_time.to_i * 1_000_000_000, + duration_nanos: @duration * 1_000_000_000, + period_type: sample_type, + period: period, + default_sample_type: 0 + ) + process_raw(message) + process_frames(message) + message.string_table += @string_map.strings + message + end + + # pprof_bytes returns a gzip'd protobuf object, like would be written to + # disk, returned by a profiling endpoint, or otherwise consumed by the + # `pprof` tool. + def pprof_bytes + Zlib.gzip(Perftools::Profiles::Profile.encode(message)) + end + + private + + def to_pprof_unit(value) + case @profile.fetch(:mode) + when :cpu, :wall + value * 1000 # stackprof uses microseconds, pprof nanoseconds + when :object + value # both stackprof and pprof are counting allocations + else + raise "unknown profile mode #{@profile.fetch(:mode)}" + end + end + + def period + to_pprof_unit @profile.fetch(:interval) + end + + def sample_type + case @profile.fetch(:mode) + when :cpu + type = 'cpu' + unit = 'nanoseconds' + when :wall + type = 'wall' + unit = 'nanoseconds' + when :object + type = 'alloc_objects' + unit = 'count' + else + raise "unknown profile mode #{@profile.fetch(:mode)}" + end + + Perftools::Profiles::ValueType.new( + type: @string_map.add(type), + unit: @string_map.add(unit) + ) + end + + # process_raw reads the :raw section of the stackprof profile and adds to + # the given protobuf message as appropriate + def process_raw(message) + i = 0 + raw = @profile.fetch(:raw, []) + + # It would be cleaner to use raw.shift here, but since that changes the + # size of the array each time it is much slower. So we use an + # incrementing pointer instead. + while i < raw.length + len = raw.fetch(i) + i += 1 + + frames = raw.slice(i, len) + i += len + frames.reverse! + + # "weight" is how many times stackprof has seen this stack. It's + # usually 1, but can be 2 or more if stackprof sees the same stack in + # sequential samples. + weight = raw.fetch(i) + i += 1 + + sample = Perftools::Profiles::Sample.new( + value: [weight * period], + location_id: frames + ) + message.sample.push(sample) + end + end + + # process_frames reads the :frames section of the stackprof profile and adds to + # the given protobuf message as appropriate + def process_frames(message) + @profile.fetch(:frames, []).each do |location_id, location| + message.function.push(Perftools::Profiles::Function.new( + id: location_id, + name: @string_map.add(location.fetch(:name)), + filename: @string_map.add(location.fetch(:file)), + start_line: location.fetch(:line, nil) + )) + + line = Perftools::Profiles::Line.new( + function_id: location_id, + line: location.fetch(:line, nil) + ) + + message.location.push(Perftools::Profiles::Location.new( + id: location_id, + line: [line] + )) + end + end + end + + # The pprof format has one table of strings, and objects that need strings + # (like filenames, function names, etc) are indexes into this table, + # achieving a cheap kind of compression for commonly repeated strings. + # StringMap is a helper for building this table. + class StringMap + def initialize + @strings = [] + @string_hash = {} + add('') # spec says string_table[0] must always be "". + end + + # strings an array of all the strings which will go into the pprof message. + attr_reader :strings + + # add will return an index for the given string, either returning the + # existing index or adding a new string to the table as appropriate. + def add(str) + i = @string_hash.fetch(str, nil) + return i unless i.nil? + + i = strings.push(str).length - 1 + @string_hash[str] = i + i + end + end +end diff --git a/vendor/gems/cloud_profiler_agent/lib/profile_pb.rb b/vendor/gems/cloud_profiler_agent/lib/profile_pb.rb new file mode 100644 index 00000000000..01ef0d99141 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/lib/profile_pb.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: profile.proto + +require 'google/protobuf' + +Google::Protobuf::DescriptorPool.generated_pool.build do + add_file("profile.proto", syntax: :proto3) do + add_message "perftools.profiles.Profile" do + repeated :sample_type, :message, 1, "perftools.profiles.ValueType" + repeated :sample, :message, 2, "perftools.profiles.Sample" + repeated :mapping, :message, 3, "perftools.profiles.Mapping" + repeated :location, :message, 4, "perftools.profiles.Location" + repeated :function, :message, 5, "perftools.profiles.Function" + repeated :string_table, :string, 6 + optional :drop_frames, :int64, 7 + optional :keep_frames, :int64, 8 + optional :time_nanos, :int64, 9 + optional :duration_nanos, :int64, 10 + optional :period_type, :message, 11, "perftools.profiles.ValueType" + optional :period, :int64, 12 + repeated :comment, :int64, 13 + optional :default_sample_type, :int64, 14 + end + add_message "perftools.profiles.ValueType" do + optional :type, :int64, 1 + optional :unit, :int64, 2 + end + add_message "perftools.profiles.Sample" do + repeated :location_id, :uint64, 1 + repeated :value, :int64, 2 + repeated :label, :message, 3, "perftools.profiles.Label" + end + add_message "perftools.profiles.Label" do + optional :key, :int64, 1 + optional :str, :int64, 2 + optional :num, :int64, 3 + optional :num_unit, :int64, 4 + end + add_message "perftools.profiles.Mapping" do + optional :id, :uint64, 1 + optional :memory_start, :uint64, 2 + optional :memory_limit, :uint64, 3 + optional :file_offset, :uint64, 4 + optional :filename, :int64, 5 + optional :build_id, :int64, 6 + optional :has_functions, :bool, 7 + optional :has_filenames, :bool, 8 + optional :has_line_numbers, :bool, 9 + optional :has_inline_frames, :bool, 10 + end + add_message "perftools.profiles.Location" do + optional :id, :uint64, 1 + optional :mapping_id, :uint64, 2 + optional :address, :uint64, 3 + repeated :line, :message, 4, "perftools.profiles.Line" + optional :is_folded, :bool, 5 + end + add_message "perftools.profiles.Line" do + optional :function_id, :uint64, 1 + optional :line, :int64, 2 + end + add_message "perftools.profiles.Function" do + optional :id, :uint64, 1 + optional :name, :int64, 2 + optional :system_name, :int64, 3 + optional :filename, :int64, 4 + optional :start_line, :int64, 5 + end + end +end + +module Perftools + module Profiles + Profile = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.Profile").msgclass + ValueType = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.ValueType").msgclass + Sample = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.Sample").msgclass + Label = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.Label").msgclass + Mapping = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.Mapping").msgclass + Location = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.Location").msgclass + Line = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.Line").msgclass + Function = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("perftools.profiles.Function").msgclass + end +end diff --git a/vendor/gems/cloud_profiler_agent/script/generate_profile.rb b/vendor/gems/cloud_profiler_agent/script/generate_profile.rb new file mode 100755 index 00000000000..d31c8415186 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/script/generate_profile.rb @@ -0,0 +1,17 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'prime' +require 'stackprof' + +StackProf.run(mode: :cpu, raw: true, interval: 100, out: 'spec/cloud_profiler_agent/cpu.stackprof') do + (1..1000).each { |i| Prime.prime_division(i) } +end + +StackProf.run(mode: :wall, raw: true, interval: 100, out: 'spec/cloud_profiler_agent/wall.stackprof') do + sleep(1) +end + +StackProf.run(mode: :object, raw: true, interval: 100, out: 'spec/cloud_profiler_agent/object.stackprof') do + (1..1000).each { |i| Prime.prime_division(i) } +end diff --git a/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/cpu.stackprof b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/cpu.stackprof Binary files differnew file mode 100644 index 00000000000..ccd82272fbb --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/cpu.stackprof diff --git a/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/looper_spec.rb b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/looper_spec.rb new file mode 100644 index 00000000000..407d185c18f --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/looper_spec.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +require 'cloud_profiler_agent' + +RSpec.describe CloudProfilerAgent::Looper, feature_category: :application_performance do + # rubocop:disable RSpec/InstanceVariable + before do + @now = 0.0 + end + + subject do + described_class.new( + clock: -> { @now }, + sleeper: ->(secs) { + sleeps.push(secs) + @now += secs + }, + rander: -> { rand }, + logger: Logger.new('/dev/null') + ) + end + + let!(:sleeps) { [] } + let(:rand) { 0.4 } # chosen by fair dice roll. guaranteed to be random. + let(:min_time) { subject.min_iteration_sec } + let(:max_time) { subject.max_iteration_sec } + let(:backoff) { subject.backoff_factor } + + describe '#run' do + it 'runs the block the specified number of times' do + runs = [] + subject.run(3) do + runs.push(true) + end + expect(runs.length).to eq(3) + end + + it 'runs the block forever when max_iterations is not given' do + # but we don't test this, because how do you test an infinite loop? + end + + it 'will not run faster than min_iteration_sec' do + subject.run(3) { nil } + expect(sleeps).to eq([min_time, min_time]) + end + + context 'when the block takes some time' do + it 'accounts for time taken by the block' do + subject.run(3) { @now += 1 } + expect(sleeps).to eq([min_time - 1, min_time - 1]) + end + end + + context 'when the block takes longer than min_iteration_sec' do + it 'does not sleep between iterations' do + subject.run(3) { @now += 11 } + expect(sleeps).to eq([]) + end + end + + context 'when the block raises a StandardError' do + it 'exponentially backs off' do + subject.run(3) do + @now += 1 + raise StandardError, 'bam' + end + + factor = backoff + (rand / 2) + expect(sleeps).to eq([(min_time * factor) - 1, (min_time * (factor**2)) - 1]) + end + + it 'respects max_iteration_sec' do + subject.run(15) do + @now += 1 + raise StandardError, 'bam' + end + + expect(sleeps.last).to eq(max_time - 1) + end + end + + context 'when Google asks for backoff' do + it 'slows down' do + subject.run(2) do + @now += 1 + raise backoff_exception('44m0s') + end + + expect(sleeps.first).to eq(60 * 44) + end + end + + context 'when the block raises some other ClientError' do + it 'goes to the maximum iteration time' do + subject.run(2) do + @now += 1 + raise ::Google::Cloud::InvalidArgumentError, 'you are a bad client' + end + + expect(sleeps).to eq([max_time - 1]) + end + end + + context 'when the block fails then works' do + it 'backs off then returns to normal' do + i = 0 + subject.run(4) do + @now += 1 + i += 1 + raise 'whoops' if i == 1 + end + + factor = backoff + (rand / 2) + expect(sleeps).to eq([(min_time * factor) - 1, min_time - 1, min_time - 1]) + end + end + end + + describe '#max_iteration_sec' do + it 'is 1 hour by default' do + expect(subject.max_iteration_sec).to eq(60 * 60) + end + end + + describe '#min_iteration_sec' do + it 'is 10 seconds by default' do + expect(subject.min_iteration_sec).to eq(10) + end + end + + describe '#backoff_factor' do + it 'is 1.5 by default' do + expect(subject.backoff_factor).to eq(1.5) + end + end + + def backoff_exception(duration) + body = "{ + \"error\": { + \"code\": 409, + \"message\": \"generic::aborted: action throttled, backoff for #{duration}\", + \"errors\": [ + { + \"message\": \"generic::aborted: action throttled, backoff for #{duration}\", + \"domain\": \"global\", + \"reason\": \"aborted\" + } + ], + \"status\": \"ABORTED\" + } + } + " + ::Google::Cloud::AlreadyExistsError.new(body) # AbortedError + end + + # rubocop:enable RSpec/InstanceVariable +end diff --git a/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/object.stackprof b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/object.stackprof Binary files differnew file mode 100644 index 00000000000..781e38dcb05 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/object.stackprof diff --git a/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/pprof_builder_spec.rb b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/pprof_builder_spec.rb new file mode 100644 index 00000000000..5c94a8e1e44 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/pprof_builder_spec.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +require 'cloud_profiler_agent' + +RSpec.describe CloudProfilerAgent::PprofBuilder, feature_category: :application_performance do + subject { described_class.new(profile, start_time, end_time) } + + # load_profile loads one of the example profiles created by + # script/generate_profile.rb + def load_profile(name) + # We are disabling this security check since we are loading static files that we generated ourselves, and it is + # only used in specs. This is the same method StackProf uses: + # https://github.com/tmm1/stackprof/blob/master/lib/stackprof/report.rb#L14 + Marshal.load(File.binread(File.expand_path("#{name}.stackprof", __dir__))) # rubocop:disable Security/MarshalLoad + end + + def get_str(index) + message.string_table.fetch(index) + end + + let(:start_time) { Time.new(2020, 10, 31, 17, 12, 0) } + let(:end_time) { Time.new(2020, 10, 31, 17, 12, 30) } + + # message is the protobuf object, which typically gets serialized and gzip'd + # before being sent to the Profiler API or written to disk. Rather than + # unzipping and deserializing all the time, we will be making a lot of + # assertions about the message directly. + let(:message) { subject.message } + + context 'with :cpu profile' do + let(:profile) { load_profile(:cpu) } + + it 'has a sample type of [["cpu", "nanoseconds"]]' do + expect(message.sample_type.length).to eq(1) + + sample_type = message.sample_type.first + expect(get_str(sample_type.type)).to eq('cpu') + expect(get_str(sample_type.unit)).to eq('nanoseconds') + end + + it 'has a period of 100,000 cpu nanoseconds' do + expect(message.period).to eq(100_000) + period_type = message.period_type + expect(get_str(period_type.type)).to eq('cpu') + expect(get_str(period_type.unit)).to eq('nanoseconds') + end + + it 'has a duration of 30 seconds' do + expect(message.duration_nanos).to eq(30 * 1_000_000_000) + end + + it 'has the start time' do + expect(message.time_nanos).to eq(start_time.to_i * 1_000_000_000) + end + end + + context 'with :wall profile' do + let(:profile) { load_profile(:wall) } + + it 'has a sample type of [["wall", "nanoseconds"]]' do + expect(message.sample_type.length).to eq(1) + + sample_type = message.sample_type.first + expect(get_str(sample_type.type)).to eq('wall') + expect(get_str(sample_type.unit)).to eq('nanoseconds') + end + + it 'has a period of 100,000 wall nanoseconds' do + expect(message.period).to eq(100_000) + period_type = message.period_type + expect(get_str(period_type.type)).to eq('wall') + expect(get_str(period_type.unit)).to eq('nanoseconds') + end + + it 'has a sum time of about 1 second' do + sum_nanos = 0 + message.sample.each do |sample| + sum_nanos += sample.value.first + end + + expect(sum_nanos).to be_within(1_000_000).of(1_000_000_000) + end + end + + context 'with :object profile' do + let(:profile) { load_profile(:object) } + + it 'has a sample type of [["alloc_objects", "count"]]' do + expect(message.sample_type.length).to eq(1) + + sample_type = message.sample_type.first + expect(get_str(sample_type.type)).to eq('alloc_objects') + expect(get_str(sample_type.unit)).to eq('count') + end + + it 'has a period of 100 alloc_objects count' do + expect(message.period).to eq(100) + period_type = message.period_type + expect(get_str(period_type.type)).to eq('alloc_objects') + expect(get_str(period_type.unit)).to eq('count') + end + end +end diff --git a/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/wall.stackprof b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/wall.stackprof Binary files differnew file mode 100644 index 00000000000..90db8c3721e --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/spec/cloud_profiler_agent/wall.stackprof diff --git a/vendor/gems/cloud_profiler_agent/spec/spec_helper.rb b/vendor/gems/cloud_profiler_agent/spec/spec_helper.rb new file mode 100644 index 00000000000..e02fd27e9c8 --- /dev/null +++ b/vendor/gems/cloud_profiler_agent/spec/spec_helper.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + config.expect_with :rspec do |expectations| + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + config.mock_with :rspec do |mocks| + mocks.verify_partial_doubles = true + end + + config.shared_context_metadata_behavior = :apply_to_host_groups + config.filter_run_when_matching :focus + config.example_status_persistence_file_path = 'spec/examples.txt' + config.disable_monkey_patching! + config.warnings = true + config.default_formatter = 'doc' if config.files_to_run.one? + config.order = :random + Kernel.srand config.seed +end |