diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-23 21:07:55 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-23 21:07:55 +0300 |
commit | 5421d61b1d5ffe11a9c7afbe2259b4e4d0e7c993 (patch) | |
tree | 093cfd5a5f80119f6e9c7ccd2c646ca2482cc3d2 /lib | |
parent | bc0f141f2f073a971aad1eb5349bb718747df028 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/ci/runner.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/auth/current_user_mode.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/ci/lint.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/command.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/process.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/create.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/seed.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/logger.rb | 42 | ||||
-rw-r--r-- | lib/gitlab/gfm/uploads_rewriter.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/git.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/git/repository.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/ref_service.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/memory/reporter.rb | 66 | ||||
-rw-r--r-- | lib/gitlab/memory/reports/jemalloc_stats.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/memory/reports_daemon.rb | 69 |
16 files changed, 157 insertions, 120 deletions
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index e64db375421..b073eb49bf1 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -229,15 +229,17 @@ module API params do requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) + optional :debug_trace, type: Boolean, desc: %q(Enable or Disable the debug trace) end patch '/:id/trace', urgency: :low, feature_category: :continuous_integration do job = authenticate_job!(heartbeat_runner: true) error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') content_range = request.headers['Content-Range'] + debug_trace = Gitlab::Utils.to_boolean(params[:debug_trace]) result = ::Ci::AppendBuildTraceService - .new(job, content_range: content_range) + .new(job, content_range: content_range, debug_trace: debug_trace) .execute(request.body.read) if result.status == 403 diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb index fc391543f4d..9bd4711c4bb 100644 --- a/lib/gitlab/auth/current_user_mode.rb +++ b/lib/gitlab/auth/current_user_mode.rb @@ -106,8 +106,8 @@ module Gitlab end def enable_admin_mode!(password: nil, skip_password_validation: false) - return unless user&.admin? - return unless skip_password_validation || user&.valid_password?(password) + return false unless user&.admin? + return false unless skip_password_validation || user&.valid_password?(password) raise NotRequestedError unless admin_mode_requested? @@ -115,6 +115,10 @@ module Gitlab current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY] = nil current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now + + audit_user_enable_admin_mode + + true end def disable_admin_mode! @@ -175,6 +179,10 @@ module Gitlab def privileged_runtime? Gitlab::Runtime.rake? || Gitlab::Runtime.rails_runner? || Gitlab::Runtime.console? end + + def audit_user_enable_admin_mode; end end end end + +Gitlab::Auth::CurrentUserMode.prepend_mod_with('Gitlab::Auth::CurrentUserMode') diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 51743a1f273..e0112a1b1c2 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -73,7 +73,7 @@ module Gitlab end def yaml_processor_result(content, logger) - logger.instrument(:yaml_process) do + logger.instrument(:yaml_process, once: true) do Gitlab::Ci::YamlProcessor.new(content, project: @project, user: @current_user, sha: @sha, @@ -119,7 +119,7 @@ module Gitlab environment: job[:environment], when: job[:when], allow_failure: job[:allow_failure], - needs: job.dig(:needs_attributes) + needs: job[:needs_attributes] } end end @@ -130,10 +130,10 @@ module Gitlab def build_logger Gitlab::Ci::Pipeline::Logger.new(project: @project) do |l| l.log_when do |observations| - values = observations['yaml_process_duration_s'] - next false if values.empty? + duration = observations['yaml_process_duration_s'] + next false unless duration - values.max >= LOG_MAX_DURATION_THRESHOLD + duration >= LOG_MAX_DURATION_THRESHOLD end end end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 07a3aff1862..53c8a7ac122 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -11,11 +11,10 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def perform! - ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project) - return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline + return if pipeline.parent_pipeline? # skip if child pipeline return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch| auto_cancel_interruptible_pipelines(cancelables_batch.ids) end @@ -29,19 +28,14 @@ module Gitlab private - def auto_cancelable_pipelines(ff_enabled) - relation = project.all_pipelines + def auto_cancelable_pipelines + project.all_pipelines .created_after(1.week.ago) .ci_and_parent_sources .for_ref(pipeline.ref) .where_not_sha(project.commit(pipeline.ref).try(:id)) .alive_or_scheduled - - if ff_enabled - relation.id_not_in(pipeline.id) - else - relation.id_not_in(pipeline.same_family_pipeline_ids) - end + .id_not_in(pipeline.id) end def auto_cancel_interruptible_pipelines(pipeline_ids) diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 5ec04b4889e..837c09e8045 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -98,7 +98,7 @@ module Gitlab def observe_step_duration(step_class, duration) step = step_class.name.underscore.parameterize(separator: '_') - logger.observe("pipeline_step_#{step}_duration_s", duration) + logger.observe("pipeline_step_#{step}_duration_s", duration, once: true) if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops) metrics.pipeline_creation_step_duration_histogram @@ -107,14 +107,14 @@ module Gitlab end def observe_creation_duration(duration) - logger.observe(:pipeline_creation_duration_s, duration) + logger.observe(:pipeline_creation_duration_s, duration, once: true) metrics.pipeline_creation_duration_histogram .observe({}, duration.seconds) end def observe_pipeline_size(pipeline) - logger.observe(:pipeline_size_count, pipeline.total_size) + logger.observe(:pipeline_size_count, pipeline.total_size, once: true) metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s, plan: project.actual_plan_name }, pipeline.total_size) diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 5548fca320f..ad6b2fd3411 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,7 +11,7 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - result = logger.instrument(:pipeline_config_process) do + result = logger.instrument(:pipeline_config_process, once: true) do processor = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 207b4b5ff8b..d4c4f94c7d3 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - logger.instrument_with_sql(:pipeline_save) do + logger.instrument_once_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do ::Ci::BulkInsertableTags.with_bulk_insert_tags do pipeline.transaction do diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index feae123f216..3f5df5ce71c 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,7 +13,7 @@ module Gitlab raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. - logger.instrument(:pipeline_allocate_seed_attributes) do + logger.instrument(:pipeline_allocate_seed_attributes, once: true) do pipeline.ensure_project_iid! pipeline.ensure_ci_ref! end @@ -25,7 +25,7 @@ module Gitlab ## # Gather all runtime build/stage errors # - seed_errors = logger.instrument(:pipeline_seed_evaluation) do + seed_errors = logger.instrument(:pipeline_seed_evaluation, once: true) do pipeline_seed.errors end @@ -44,7 +44,7 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do - logger.instrument(:pipeline_seed_initialization) do + logger.instrument(:pipeline_seed_initialization, once: true) do stages_attributes = @command.yaml_processor_result.stages_attributes Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) @@ -61,11 +61,13 @@ module Gitlab end def root_variables - logger.instrument(:pipeline_seed_merge_variables) do - ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, - @command.workflow_rules_result.variables - ) + strong_memoize(:root_variables) do + logger.instrument(:pipeline_seed_merge_variables, once: true) do + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, + @command.workflow_rules_result.variables + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 5a1743dcfca..9659cec4889 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -23,7 +23,7 @@ module Gitlab log_conditions.push(block) end - def instrument(operation) + def instrument(operation, once: false) return yield unless enabled? raise ArgumentError, 'block not given' unless block_given? @@ -32,25 +32,29 @@ module Gitlab result = yield - observe("#{operation}_duration_s", current_monotonic_time - op_started_at) + observe("#{operation}_duration_s", current_monotonic_time - op_started_at, once: once) result end - def instrument_with_sql(operation, &block) + def instrument_once_with_sql(operation, &block) op_start_db_counters = current_db_counter_payload - result = instrument(operation, &block) + result = instrument(operation, once: true, &block) - observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload) + observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload, once: true) result end - def observe(operation, value) + def observe(operation, value, once: false) return unless enabled? - observations[operation.to_s].push(value) + if once + observations[operation.to_s] = value + else + observations[operation.to_s].push(value) + end end def commit(pipeline:, caller:) @@ -79,14 +83,18 @@ module Gitlab end def observations_hash - observations.transform_values do |values| - next if values.empty? - - { - 'count' => values.size, - 'max' => values.max, - 'sum' => values.sum - } + observations.transform_values do |observation| + next if observation.blank? + + if observation.is_a?(Array) + { + 'count' => observation.size, + 'max' => observation.max, + 'sum' => observation.sum + } + else + observation + end end.compact end @@ -117,12 +125,12 @@ module Gitlab @observations ||= Hash.new { |hash, key| hash[key] = [] } end - def observe_sql_counters(operation, start_db_counters, end_db_counters) + def observe_sql_counters(operation, start_db_counters, end_db_counters, once: false) end_db_counters.each do |key, value| result = value - start_db_counters.fetch(key, 0) next if result == 0 - observe("#{operation}_#{key}", result) + observe("#{operation}_#{key}", result, once: once) end end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 58b46a85aae..6b5f10ed78e 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -54,8 +54,7 @@ module Gitlab file = find_file(match[:secret], match[:file]) # No file will be returned for a path traversal - return '' if file.nil? - + return markdown if file.nil? return markdown unless file.try(:exists?) klass = @target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4b877bf44da..8e1b51fcec5 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -16,6 +16,7 @@ module Gitlab CommitError = Class.new(BaseError) OSError = Class.new(BaseError) UnknownRef = Class.new(BaseError) + AmbiguousRef = Class.new(BaseError) CommandTimedOut = Class.new(CommandError) InvalidPageToken = Class.new(BaseError) InvalidRefFormatError = Class.new(BaseError) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 2d13d3378c2..1bbbc16fe67 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -134,6 +134,10 @@ module Gitlab wrapped_gitaly_errors do gitaly_ref_client.find_branch(name) end + rescue Gitlab::Git::AmbiguousRef + # Gitaly returns "reference is ambiguous" error in case when users request + # branch "my-branch", when another branch "my-branch/branch" exists. + # We handle this error here and return nil for this case. end def find_tag(name) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index de76ade76cb..3ab596da529 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -17,6 +17,8 @@ module Gitlab 'desc' => Gitaly::SortDirection::DESCENDING }.freeze + AMBIGUOUS_REFERENCE = 'reference is ambiguous' + # 'repository' is a Gitlab::Git::Repository def initialize(repository) @repository = repository @@ -109,6 +111,10 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name.dup, branch.target_commit.id, target_commit) + rescue GRPC::BadStatus => e + raise e unless e.message.include?(AMBIGUOUS_REFERENCE) + + raise Gitlab::Git::AmbiguousRef, "branch is ambiguous: #{branch_name}" end def find_tag(tag_name) diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb new file mode 100644 index 00000000000..b4f3f950b95 --- /dev/null +++ b/lib/gitlab/memory/reporter.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Reporter + def initialize + @worker_uuid = SecureRandom.uuid + + init_prometheus_metrics + end + + def run_report(report) + start_monotonic_time = Gitlab::Metrics::System.monotonic_time + start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + + file_path = report.run(report_id) + + cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) + duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + + log_report(name: report.name, cpu_s: cpu_s, duration_s: duration_s, size: file_size(file_path)) + + @report_duration_counter.increment({ report: report.name }, duration_s) + end + + private + + def log_report(name:, duration_s:, cpu_s:, size:) + Gitlab::AppLogger.info( + message: 'finished', + pid: $$, + worker_id: worker_id, + perf_report: name, + duration_s: duration_s.round(2), + cpu_s: cpu_s.round(2), + perf_report_size_bytes: size, + perf_report_worker_uuid: @worker_uuid + ) + end + + def report_id + [worker_id, @worker_uuid].join(".") + end + + def worker_id + ::Prometheus::PidProvider.worker_id + end + + def file_size(file_path) + File.size(file_path.to_s) + rescue Errno::ENOENT + 0 + end + + def init_prometheus_metrics + default_labels = { pid: worker_id } + + @report_duration_counter = Gitlab::Metrics.counter( + :gitlab_diag_report_duration_seconds_total, + 'Total time elapsed for running diagnostic report', + default_labels + ) + end + end + end +end diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb index 720f22ddbe4..2e5a053031e 100644 --- a/lib/gitlab/memory/reports/jemalloc_stats.rb +++ b/lib/gitlab/memory/reports/jemalloc_stats.rb @@ -16,9 +16,8 @@ module Gitlab # The cleanup logic will be redundant after we'll implement the uploads, which would perform the cleanup. DEFAULT_MAX_REPORTS_STORED = 250 - def initialize(reports_path:, filename_label:) + def initialize(reports_path:) @reports_path = reports_path - @filename_label = filename_label # Store report in tmp subdir while it is still streaming. # This will clearly separate finished reports from the files we are still writing to. @@ -26,11 +25,16 @@ module Gitlab FileUtils.mkdir_p(@tmp_dir) end - def run + def name + 'jemalloc_stats' + end + + def run(report_id) return unless active? - Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, - filename_label: filename_label).tap do + Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, + tmp_dir: @tmp_dir, + filename_label: report_id).tap do cleanup end end @@ -41,7 +45,7 @@ module Gitlab private - attr_reader :reports_path, :filename_label + attr_reader :reports_path def cleanup reports_files_modified_order[0...-max_reports_stored].each do |f| diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb index 7070c65c705..9a044dbd06c 100644 --- a/lib/gitlab/memory/reports_daemon.rb +++ b/lib/gitlab/memory/reports_daemon.rb @@ -9,7 +9,7 @@ module Gitlab DEFAULT_REPORTS_PATH = Dir.tmpdir - def initialize(**options) + def initialize(reporter: Reporter.new, reports: nil, **options) super @alive = true @@ -24,17 +24,10 @@ module Gitlab @reports_path = ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || DEFAULT_REPORTS_PATH - # Set unique uuid for every ReportsDaemon instance. - # Because we spawn a single instance of it per process, it will also uniquely identify the worker. - # Unlike `::Prometheus::PidProvider.worker_id`, this uuid will remain unique across all Puma clusters. - # This way, we can identify reports that were produced from the same worker process during its lifetime. - @worker_uuid = SecureRandom.uuid - - @reports = [ - Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path, filename_label: filename_label) + @reporter = reporter + @reports = reports || [ + Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path) ] - - init_prometheus_metrics end attr_reader :sleep_s, :sleep_max_delta_s, :sleep_between_reports_s, :reports_path @@ -44,16 +37,7 @@ module Gitlab sleep interval_with_jitter reports.select(&:active?).each do |report| - start_monotonic_time = Gitlab::Metrics::System.monotonic_time - start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time - - file_path = report.run - - cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) - duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time - - log_report(label: report_label(report), cpu_s: cpu_s, duration_s: duration_s, size: file_size(file_path)) - @report_duration_counter.increment({ report: report_label(report) }, duration_s) + @reporter.run_report(report) sleep sleep_between_reports_s end @@ -62,11 +46,7 @@ module Gitlab private - attr_reader :alive, :reports, :worker_uuid - - def filename_label - [worker_id, worker_uuid].join(".") - end + attr_reader :alive, :reports # Returns the sleep interval with a random adjustment. # The random adjustment is put in place to ensure continued availability. @@ -74,46 +54,9 @@ module Gitlab sleep_s + rand(sleep_max_delta_s) end - def log_report(label:, duration_s:, cpu_s:, size:) - Gitlab::AppLogger.info( - message: 'finished', - pid: $$, - worker_id: worker_id, - perf_report: label, - duration_s: duration_s.round(2), - cpu_s: cpu_s.round(2), - perf_report_size_bytes: size, - perf_report_worker_uuid: worker_uuid - ) - end - - def worker_id - ::Prometheus::PidProvider.worker_id - end - - def report_label(report) - report.class.to_s.demodulize.underscore - end - def stop_working @alive = false end - - def init_prometheus_metrics - default_labels = { pid: worker_id } - - @report_duration_counter = Gitlab::Metrics.counter( - :gitlab_diag_report_duration_seconds_total, - 'Total time elapsed for running diagnostic report', - default_labels - ) - end - - def file_size(file_path) - File.size(file_path.to_s) - rescue Errno::ENOENT - 0 - end end end end |