Add latest changes from gitlab-org/gitlab@master

16 files changed, 157 insertions, 120 deletions

diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index e64db375421..b073eb49bf1 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -229,15 +229,17 @@ module API
         params do
           requires :id, type: Integer, desc: %q(Job's ID)
           optional :token, type: String, desc: %q(Job's authentication token)
+          optional :debug_trace, type: Boolean, desc: %q(Enable or Disable the debug trace)
         end
         patch '/:id/trace', urgency: :low, feature_category: :continuous_integration do
           job = authenticate_job!(heartbeat_runner: true)
 
           error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
           content_range = request.headers['Content-Range']
+          debug_trace = Gitlab::Utils.to_boolean(params[:debug_trace])
 
           result = ::Ci::AppendBuildTraceService
-            .new(job, content_range: content_range)
+            .new(job, content_range: content_range, debug_trace: debug_trace)
             .execute(request.body.read)
 
           if result.status == 403
diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb
index fc391543f4d..9bd4711c4bb 100644
--- a/lib/gitlab/auth/current_user_mode.rb
+++ b/lib/gitlab/auth/current_user_mode.rb
@@ -106,8 +106,8 @@ module Gitlab
       end
 
       def enable_admin_mode!(password: nil, skip_password_validation: false)
-        return unless user&.admin?
-        return unless skip_password_validation || user&.valid_password?(password)
+        return false unless user&.admin?
+        return false unless skip_password_validation || user&.valid_password?(password)
 
         raise NotRequestedError unless admin_mode_requested?
 
@@ -115,6 +115,10 @@ module Gitlab
 
         current_session_data[ADMIN_MODE_REQUESTED_TIME_KEY] = nil
         current_session_data[ADMIN_MODE_START_TIME_KEY] = Time.now
+
+        audit_user_enable_admin_mode
+
+        true
       end
 
       def disable_admin_mode!
@@ -175,6 +179,10 @@ module Gitlab
       def privileged_runtime?
         Gitlab::Runtime.rake? || Gitlab::Runtime.rails_runner? || Gitlab::Runtime.console?
       end
+
+      def audit_user_enable_admin_mode; end
     end
   end
 end
+
+Gitlab::Auth::CurrentUserMode.prepend_mod_with('Gitlab::Auth::CurrentUserMode')
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index 51743a1f273..e0112a1b1c2 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -73,7 +73,7 @@ module Gitlab
       end
 
       def yaml_processor_result(content, logger)
-        logger.instrument(:yaml_process) do
+        logger.instrument(:yaml_process, once: true) do
           Gitlab::Ci::YamlProcessor.new(content, project: @project,
                                                  user: @current_user,
                                                  sha: @sha,
@@ -119,7 +119,7 @@ module Gitlab
               environment: job[:environment],
               when: job[:when],
               allow_failure: job[:allow_failure],
-              needs: job.dig(:needs_attributes)
+              needs: job[:needs_attributes]
             }
           end
         end
@@ -130,10 +130,10 @@ module Gitlab
       def build_logger
         Gitlab::Ci::Pipeline::Logger.new(project: @project) do |l|
           l.log_when do |observations|
-            values = observations['yaml_process_duration_s']
-            next false if values.empty?
+            duration = observations['yaml_process_duration_s']
+            next false unless duration
 
-            values.max >= LOG_MAX_DURATION_THRESHOLD
+            duration >= LOG_MAX_DURATION_THRESHOLD
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
index 07a3aff1862..53c8a7ac122 100644
--- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
+++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
@@ -11,11 +11,10 @@ module Gitlab
 
           # rubocop: disable CodeReuse/ActiveRecord
           def perform!
-            ff_enabled = Feature.enabled?(:ci_skip_auto_cancelation_on_child_pipelines, project)
-            return if ff_enabled && pipeline.parent_pipeline? # skip if child pipeline
+            return if pipeline.parent_pipeline? # skip if child pipeline
             return unless project.auto_cancel_pending_pipelines?
 
-            Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines(ff_enabled), name: 'cancel_pending_pipelines') do |cancelables|
+            Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables|
               cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch|
                 auto_cancel_interruptible_pipelines(cancelables_batch.ids)
               end
@@ -29,19 +28,14 @@ module Gitlab
 
           private
 
-          def auto_cancelable_pipelines(ff_enabled)
-            relation = project.all_pipelines
+          def auto_cancelable_pipelines
+            project.all_pipelines
               .created_after(1.week.ago)
               .ci_and_parent_sources
               .for_ref(pipeline.ref)
               .where_not_sha(project.commit(pipeline.ref).try(:id))
               .alive_or_scheduled
-
-            if ff_enabled
-              relation.id_not_in(pipeline.id)
-            else
-              relation.id_not_in(pipeline.same_family_pipeline_ids)
-            end
+              .id_not_in(pipeline.id)
           end
 
           def auto_cancel_interruptible_pipelines(pipeline_ids)
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 5ec04b4889e..837c09e8045 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -98,7 +98,7 @@ module Gitlab
 
           def observe_step_duration(step_class, duration)
             step = step_class.name.underscore.parameterize(separator: '_')
-            logger.observe("pipeline_step_#{step}_duration_s", duration)
+            logger.observe("pipeline_step_#{step}_duration_s", duration, once: true)
 
             if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops)
               metrics.pipeline_creation_step_duration_histogram
@@ -107,14 +107,14 @@ module Gitlab
           end
 
           def observe_creation_duration(duration)
-            logger.observe(:pipeline_creation_duration_s, duration)
+            logger.observe(:pipeline_creation_duration_s, duration, once: true)
 
             metrics.pipeline_creation_duration_histogram
               .observe({}, duration.seconds)
           end
 
           def observe_pipeline_size(pipeline)
-            logger.observe(:pipeline_size_count, pipeline.total_size)
+            logger.observe(:pipeline_size_count, pipeline.total_size, once: true)
 
             metrics.pipeline_size_histogram
               .observe({ source: pipeline.source.to_s, plan: project.actual_plan_name }, pipeline.total_size)
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index 5548fca320f..ad6b2fd3411 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -11,7 +11,7 @@ module Gitlab
             def perform!
               raise ArgumentError, 'missing config content' unless @command.config_content
 
-              result = logger.instrument(:pipeline_config_process) do
+              result = logger.instrument(:pipeline_config_process, once: true) do
                 processor = ::Gitlab::Ci::YamlProcessor.new(
                   @command.config_content, {
                     project: project,
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index 207b4b5ff8b..d4c4f94c7d3 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -9,7 +9,7 @@ module Gitlab
           include Gitlab::Utils::StrongMemoize
 
           def perform!
-            logger.instrument_with_sql(:pipeline_save) do
+            logger.instrument_once_with_sql(:pipeline_save) do
               BulkInsertableAssociations.with_bulk_insert do
                 ::Ci::BulkInsertableTags.with_bulk_insert_tags do
                   pipeline.transaction do
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index feae123f216..3f5df5ce71c 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -13,7 +13,7 @@ module Gitlab
             raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result
 
             # Allocate next IID. This operation must be outside of transactions of pipeline creations.
-            logger.instrument(:pipeline_allocate_seed_attributes) do
+            logger.instrument(:pipeline_allocate_seed_attributes, once: true) do
               pipeline.ensure_project_iid!
               pipeline.ensure_ci_ref!
             end
@@ -25,7 +25,7 @@ module Gitlab
             ##
             # Gather all runtime build/stage errors
             #
-            seed_errors = logger.instrument(:pipeline_seed_evaluation) do
+            seed_errors = logger.instrument(:pipeline_seed_evaluation, once: true) do
               pipeline_seed.errors
             end
 
@@ -44,7 +44,7 @@ module Gitlab
 
           def pipeline_seed
             strong_memoize(:pipeline_seed) do
-              logger.instrument(:pipeline_seed_initialization) do
+              logger.instrument(:pipeline_seed_initialization, once: true) do
                 stages_attributes = @command.yaml_processor_result.stages_attributes
 
                 Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes)
@@ -61,11 +61,13 @@ module Gitlab
           end
 
           def root_variables
-            logger.instrument(:pipeline_seed_merge_variables) do
-              ::Gitlab::Ci::Variables::Helpers.merge_variables(
-                @command.yaml_processor_result.root_variables,
-                @command.workflow_rules_result.variables
-              )
+            strong_memoize(:root_variables) do
+              logger.instrument(:pipeline_seed_merge_variables, once: true) do
+                ::Gitlab::Ci::Variables::Helpers.merge_variables(
+                  @command.yaml_processor_result.root_variables,
+                  @command.workflow_rules_result.variables
+                )
+              end
             end
           end
         end
diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb
index 5a1743dcfca..9659cec4889 100644
--- a/lib/gitlab/ci/pipeline/logger.rb
+++ b/lib/gitlab/ci/pipeline/logger.rb
@@ -23,7 +23,7 @@ module Gitlab
           log_conditions.push(block)
         end
 
-        def instrument(operation)
+        def instrument(operation, once: false)
           return yield unless enabled?
 
           raise ArgumentError, 'block not given' unless block_given?
@@ -32,25 +32,29 @@ module Gitlab
 
           result = yield
 
-          observe("#{operation}_duration_s", current_monotonic_time - op_started_at)
+          observe("#{operation}_duration_s", current_monotonic_time - op_started_at, once: once)
 
           result
         end
 
-        def instrument_with_sql(operation, &block)
+        def instrument_once_with_sql(operation, &block)
           op_start_db_counters = current_db_counter_payload
 
-          result = instrument(operation, &block)
+          result = instrument(operation, once: true, &block)
 
-          observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload)
+          observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload, once: true)
 
           result
         end
 
-        def observe(operation, value)
+        def observe(operation, value, once: false)
           return unless enabled?
 
-          observations[operation.to_s].push(value)
+          if once
+            observations[operation.to_s] = value
+          else
+            observations[operation.to_s].push(value)
+          end
         end
 
         def commit(pipeline:, caller:)
@@ -79,14 +83,18 @@ module Gitlab
         end
 
         def observations_hash
-          observations.transform_values do |values|
-            next if values.empty?
-
-            {
-              'count' => values.size,
-              'max' => values.max,
-              'sum' => values.sum
-            }
+          observations.transform_values do |observation|
+            next if observation.blank?
+
+            if observation.is_a?(Array)
+              {
+                'count' => observation.size,
+                'max' => observation.max,
+                'sum' => observation.sum
+              }
+            else
+              observation
+            end
           end.compact
         end
 
@@ -117,12 +125,12 @@ module Gitlab
           @observations ||= Hash.new { |hash, key| hash[key] = [] }
         end
 
-        def observe_sql_counters(operation, start_db_counters, end_db_counters)
+        def observe_sql_counters(operation, start_db_counters, end_db_counters, once: false)
           end_db_counters.each do |key, value|
             result = value - start_db_counters.fetch(key, 0)
             next if result == 0
 
-            observe("#{operation}_#{key}", result)
+            observe("#{operation}_#{key}", result, once: once)
           end
         end
 
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index 58b46a85aae..6b5f10ed78e 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -54,8 +54,7 @@ module Gitlab
         file = find_file(match[:secret], match[:file])
 
         # No file will be returned for a path traversal
-        return '' if file.nil?
-
+        return markdown if file.nil?
         return markdown unless file.try(:exists?)
 
         klass = @target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader
diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb
index 4b877bf44da..8e1b51fcec5 100644
--- a/lib/gitlab/git.rb
+++ b/lib/gitlab/git.rb
@@ -16,6 +16,7 @@ module Gitlab
     CommitError = Class.new(BaseError)
     OSError = Class.new(BaseError)
     UnknownRef = Class.new(BaseError)
+    AmbiguousRef = Class.new(BaseError)
     CommandTimedOut = Class.new(CommandError)
     InvalidPageToken = Class.new(BaseError)
     InvalidRefFormatError = Class.new(BaseError)
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 2d13d3378c2..1bbbc16fe67 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -134,6 +134,10 @@ module Gitlab
         wrapped_gitaly_errors do
           gitaly_ref_client.find_branch(name)
         end
+      rescue Gitlab::Git::AmbiguousRef
+        # Gitaly returns "reference is ambiguous" error in case when users request
+        # branch "my-branch", when another branch "my-branch/branch" exists.
+        # We handle this error here and return nil for this case.
       end
 
       def find_tag(name)
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index de76ade76cb..3ab596da529 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -17,6 +17,8 @@ module Gitlab
         'desc' => Gitaly::SortDirection::DESCENDING
       }.freeze
 
+      AMBIGUOUS_REFERENCE = 'reference is ambiguous'
+
       # 'repository' is a Gitlab::Git::Repository
       def initialize(repository)
         @repository = repository
@@ -109,6 +111,10 @@ module Gitlab
 
         target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit)
         Gitlab::Git::Branch.new(@repository, branch.name.dup, branch.target_commit.id, target_commit)
+      rescue GRPC::BadStatus => e
+        raise e unless e.message.include?(AMBIGUOUS_REFERENCE)
+
+        raise Gitlab::Git::AmbiguousRef, "branch is ambiguous: #{branch_name}"
       end
 
       def find_tag(tag_name)
diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb
new file mode 100644
index 00000000000..b4f3f950b95
--- /dev/null
+++ b/lib/gitlab/memory/reporter.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Memory
+    class Reporter
+      def initialize
+        @worker_uuid = SecureRandom.uuid
+
+        init_prometheus_metrics
+      end
+
+      def run_report(report)
+        start_monotonic_time = Gitlab::Metrics::System.monotonic_time
+        start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time
+
+        file_path = report.run(report_id)
+
+        cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time)
+        duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time
+
+        log_report(name: report.name, cpu_s: cpu_s, duration_s: duration_s, size: file_size(file_path))
+
+        @report_duration_counter.increment({ report: report.name }, duration_s)
+      end
+
+      private
+
+      def log_report(name:, duration_s:, cpu_s:, size:)
+        Gitlab::AppLogger.info(
+          message: 'finished',
+          pid: $$,
+          worker_id: worker_id,
+          perf_report: name,
+          duration_s: duration_s.round(2),
+          cpu_s: cpu_s.round(2),
+          perf_report_size_bytes: size,
+          perf_report_worker_uuid: @worker_uuid
+        )
+      end
+
+      def report_id
+        [worker_id, @worker_uuid].join(".")
+      end
+
+      def worker_id
+        ::Prometheus::PidProvider.worker_id
+      end
+
+      def file_size(file_path)
+        File.size(file_path.to_s)
+      rescue Errno::ENOENT
+        0
+      end
+
+      def init_prometheus_metrics
+        default_labels = { pid: worker_id }
+
+        @report_duration_counter = Gitlab::Metrics.counter(
+          :gitlab_diag_report_duration_seconds_total,
+          'Total time elapsed for running diagnostic report',
+          default_labels
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb
index 720f22ddbe4..2e5a053031e 100644
--- a/lib/gitlab/memory/reports/jemalloc_stats.rb
+++ b/lib/gitlab/memory/reports/jemalloc_stats.rb
@@ -16,9 +16,8 @@ module Gitlab
         # The cleanup logic will be redundant after we'll implement the uploads, which would perform the cleanup.
         DEFAULT_MAX_REPORTS_STORED = 250
 
-        def initialize(reports_path:, filename_label:)
+        def initialize(reports_path:)
           @reports_path = reports_path
-          @filename_label = filename_label
 
           # Store report in tmp subdir while it is still streaming.
           # This will clearly separate finished reports from the files we are still writing to.
@@ -26,11 +25,16 @@ module Gitlab
           FileUtils.mkdir_p(@tmp_dir)
         end
 
-        def run
+        def name
+          'jemalloc_stats'
+        end
+
+        def run(report_id)
           return unless active?
 
-          Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir,
-                                              filename_label: filename_label).tap do
+          Gitlab::Memory::Jemalloc.dump_stats(path: reports_path,
+                                              tmp_dir: @tmp_dir,
+                                              filename_label: report_id).tap do
             cleanup
           end
         end
@@ -41,7 +45,7 @@ module Gitlab
 
         private
 
-        attr_reader :reports_path, :filename_label
+        attr_reader :reports_path
 
         def cleanup
           reports_files_modified_order[0...-max_reports_stored].each do |f|
diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb
index 7070c65c705..9a044dbd06c 100644
--- a/lib/gitlab/memory/reports_daemon.rb
+++ b/lib/gitlab/memory/reports_daemon.rb
@@ -9,7 +9,7 @@ module Gitlab
 
       DEFAULT_REPORTS_PATH = Dir.tmpdir
 
-      def initialize(**options)
+      def initialize(reporter: Reporter.new, reports: nil, **options)
         super
 
         @alive = true
@@ -24,17 +24,10 @@ module Gitlab
         @reports_path =
           ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || DEFAULT_REPORTS_PATH
 
-        # Set unique uuid for every ReportsDaemon instance.
-        # Because we spawn a single instance of it per process, it will also uniquely identify the worker.
-        # Unlike `::Prometheus::PidProvider.worker_id`, this uuid will remain unique across all Puma clusters.
-        # This way, we can identify reports that were produced from the same worker process during its lifetime.
-        @worker_uuid = SecureRandom.uuid
-
-        @reports = [
-          Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path, filename_label: filename_label)
+        @reporter = reporter
+        @reports = reports || [
+          Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path)
         ]
-
-        init_prometheus_metrics
       end
 
       attr_reader :sleep_s, :sleep_max_delta_s, :sleep_between_reports_s, :reports_path
@@ -44,16 +37,7 @@ module Gitlab
           sleep interval_with_jitter
 
           reports.select(&:active?).each do |report|
-            start_monotonic_time = Gitlab::Metrics::System.monotonic_time
-            start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time
-
-            file_path = report.run
-
-            cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time)
-            duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time
-
-            log_report(label: report_label(report), cpu_s: cpu_s, duration_s: duration_s, size: file_size(file_path))
-            @report_duration_counter.increment({ report: report_label(report) }, duration_s)
+            @reporter.run_report(report)
 
             sleep sleep_between_reports_s
           end
@@ -62,11 +46,7 @@ module Gitlab
 
       private
 
-      attr_reader :alive, :reports, :worker_uuid
-
-      def filename_label
-        [worker_id, worker_uuid].join(".")
-      end
+      attr_reader :alive, :reports
 
       # Returns the sleep interval with a random adjustment.
       # The random adjustment is put in place to ensure continued availability.
@@ -74,46 +54,9 @@ module Gitlab
         sleep_s + rand(sleep_max_delta_s)
       end
 
-      def log_report(label:, duration_s:, cpu_s:, size:)
-        Gitlab::AppLogger.info(
-          message: 'finished',
-          pid: $$,
-          worker_id: worker_id,
-          perf_report: label,
-          duration_s: duration_s.round(2),
-          cpu_s: cpu_s.round(2),
-          perf_report_size_bytes: size,
-          perf_report_worker_uuid: worker_uuid
-        )
-      end
-
-      def worker_id
-        ::Prometheus::PidProvider.worker_id
-      end
-
-      def report_label(report)
-        report.class.to_s.demodulize.underscore
-      end
-
       def stop_working
         @alive = false
       end
-
-      def init_prometheus_metrics
-        default_labels = { pid: worker_id }
-
-        @report_duration_counter = Gitlab::Metrics.counter(
-          :gitlab_diag_report_duration_seconds_total,
-          'Total time elapsed for running diagnostic report',
-          default_labels
-        )
-      end
-
-      def file_size(file_path)
-        File.size(file_path.to_s)
-      rescue Errno::ENOENT
-        0
-      end
     end
   end
 end