Add latest changes from gitlab-org/gitlab@master

15 files changed, 79 insertions, 31 deletions

diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb
index 23efbc33581..2d246a217a7 100644
--- a/lib/bulk_imports/groups/graphql/get_labels_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_labels_query.rb
@@ -31,7 +31,7 @@ module BulkImports
         def variables(context)
           {
             full_path: context.entity.source_full_path,
-            cursor: context.entity.next_page_for(:labels)
+            cursor: context.tracker.next_page
           }
         end
 
diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/groups/graphql/get_members_query.rb
index e3a78124a47..f3a0d77acc2 100644
--- a/lib/bulk_imports/groups/graphql/get_members_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_members_query.rb
@@ -34,7 +34,7 @@ module BulkImports
         def variables(context)
           {
             full_path: context.entity.source_full_path,
-            cursor: context.entity.next_page_for(:group_members)
+            cursor: context.tracker.next_page
           }
         end
 
diff --git a/lib/bulk_imports/groups/graphql/get_milestones_query.rb b/lib/bulk_imports/groups/graphql/get_milestones_query.rb
index 2ade87e6fa0..c254b35054a 100644
--- a/lib/bulk_imports/groups/graphql/get_milestones_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_milestones_query.rb
@@ -33,7 +33,7 @@ module BulkImports
         def variables(context)
           {
             full_path: context.entity.source_full_path,
-            cursor: context.entity.next_page_for(:milestones)
+            cursor: context.tracker.next_page
           }
         end
 
diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
index 9f8b8682751..61d3e6c700e 100644
--- a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
@@ -16,8 +16,7 @@ module BulkImports
         end
 
         def after_run(extracted_data)
-          context.entity.update_tracker_for(
-            relation: :labels,
+          tracker.update(
             has_next_page: extracted_data.has_next_page?,
             next_page: extracted_data.next_page
           )
diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
index 32fc931e8c3..d29bd74c5ae 100644
--- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
@@ -19,8 +19,7 @@ module BulkImports
         end
 
         def after_run(extracted_data)
-          context.entity.update_tracker_for(
-            relation: :group_members,
+          tracker.update(
             has_next_page: extracted_data.has_next_page?,
             next_page: extracted_data.next_page
           )
diff --git a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
index 8497162e0e7..eb51424c14a 100644
--- a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
@@ -20,8 +20,7 @@ module BulkImports
         end
 
         def after_run(extracted_data)
-          context.entity.update_tracker_for(
-            relation: :milestones,
+          tracker.update(
             has_next_page: extracted_data.has_next_page?,
             next_page: extracted_data.next_page
           )
diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb
index f016b552fd4..ccc61ee787d 100644
--- a/lib/bulk_imports/importers/group_importer.rb
+++ b/lib/bulk_imports/importers/group_importer.rb
@@ -8,9 +8,18 @@ module BulkImports
       end
 
       def execute
-        context = BulkImports::Pipeline::Context.new(entity)
+        pipelines.each.with_index do |pipeline, stage|
+          pipeline_tracker = entity.trackers.create!(
+            pipeline_name: pipeline,
+            stage: stage
+          )
 
-        pipelines.each { |pipeline| pipeline.new(context).run }
+          context = BulkImports::Pipeline::Context.new(pipeline_tracker)
+
+          pipeline.new(context).run
+
+          pipeline_tracker.finish!
+        end
 
         entity.finish!
       end
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
index 14445162737..df4f020d6b2 100644
--- a/lib/bulk_imports/pipeline.rb
+++ b/lib/bulk_imports/pipeline.rb
@@ -15,6 +15,10 @@ module BulkImports
       @context = context
     end
 
+    def tracker
+      @tracker ||= context.tracker
+    end
+
     included do
       private
 
diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb
index dd121b2dbed..3c69c729f36 100644
--- a/lib/bulk_imports/pipeline/context.rb
+++ b/lib/bulk_imports/pipeline/context.rb
@@ -3,25 +3,33 @@
 module BulkImports
   module Pipeline
     class Context
-      attr_reader :entity, :bulk_import
       attr_accessor :extra
 
-      def initialize(entity, extra = {})
-        @entity = entity
-        @bulk_import = entity.bulk_import
+      attr_reader :tracker
+
+      def initialize(tracker, extra = {})
+        @tracker = tracker
         @extra = extra
       end
 
+      def entity
+        @entity ||= tracker.entity
+      end
+
       def group
-        entity.group
+        @group ||= entity.group
+      end
+
+      def bulk_import
+        @bulk_import ||= entity.bulk_import
       end
 
       def current_user
-        bulk_import.user
+        @current_user ||= bulk_import.user
       end
 
       def configuration
-        bulk_import.configuration
+        @configuration ||= bulk_import.configuration
       end
     end
   end
diff --git a/lib/bulk_imports/pipeline/extracted_data.rb b/lib/bulk_imports/pipeline/extracted_data.rb
index 685a91a4afe..15b5caa1a47 100644
--- a/lib/bulk_imports/pipeline/extracted_data.rb
+++ b/lib/bulk_imports/pipeline/extracted_data.rb
@@ -11,11 +11,14 @@ module BulkImports
       end
 
       def has_next_page?
-        @page_info['has_next_page']
+        Gitlab::Utils.to_boolean(
+          @page_info&.dig('has_next_page'),
+          default: false
+        )
       end
 
       def next_page
-        @page_info['end_cursor']
+        @page_info&.dig('end_cursor')
       end
 
       def each(&block)
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
index e3535e585cc..588d2c87209 100644
--- a/lib/bulk_imports/pipeline/runner.rb
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -26,7 +26,7 @@ module BulkImports
           end
         end
 
-        if respond_to?(:after_run)
+        if extracted_data && respond_to?(:after_run)
           run_pipeline_step(:after_run) do
             after_run(extracted_data)
           end
@@ -34,7 +34,7 @@ module BulkImports
 
         info(message: 'Pipeline finished')
       rescue MarkedAsFailedError
-        log_skip
+        skip!('Skipping pipeline due to failed entity')
       end
 
       private # rubocop:disable Lint/UselessAccessModifier
@@ -46,7 +46,11 @@ module BulkImports
 
         yield
       rescue MarkedAsFailedError
-        log_skip(step => class_name)
+        skip!(
+          'Skipping pipeline due to failed entity',
+          pipeline_step: step,
+          step_class: class_name
+        )
       rescue => e
         log_import_failure(e, step)
 
@@ -65,10 +69,13 @@ module BulkImports
         warn(message: 'Pipeline failed')
 
         context.entity.fail_op!
+        tracker.fail_op!
       end
 
-      def log_skip(extra = {})
-        info({ message: 'Skipping due to failed pipeline status' }.merge(extra))
+      def skip!(message, extra = {})
+        warn({ message: message }.merge(extra))
+
+        tracker.skip!
       end
 
       def log_import_failure(exception, step)
diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb
index 5398c19e536..b25e715bc00 100644
--- a/lib/gitlab/ci/queue/metrics.rb
+++ b/lib/gitlab/ci/queue/metrics.rb
@@ -10,7 +10,7 @@ module Gitlab
         QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
         QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
         QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
-        QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
+        QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
 
         METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
         DEFAULT_METRICS_SHARD = 'default'
@@ -100,7 +100,7 @@ module Gitlab
           self.class.queue_size_total.observe({}, size_proc.call.to_f)
         end
 
-        def observe_queue_time
+        def observe_queue_time(metric)
           start_time = ::Gitlab::Metrics::System.monotonic_time
 
           result = yield
@@ -108,7 +108,15 @@ module Gitlab
           return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
           seconds = ::Gitlab::Metrics::System.monotonic_time - start_time
-          self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)
+
+          case metric
+          when :process
+            self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)
+          when :retrieve
+            self.class.queue_retrieval_duration_seconds.observe({}, seconds.to_f)
+          else
+            raise ArgumentError unless Rails.env.production?
+          end
 
           result
         end
@@ -187,7 +195,18 @@ module Gitlab
           strong_memoize(:queue_iteration_duration_seconds) do
             name = :gitlab_ci_queue_iteration_duration_seconds
             comment = 'Time it takes to find a build in CI/CD queue'
-            buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
+            labels = {}
+
+            Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
+
+        def self.queue_retrieval_duration_seconds
+          strong_memoize(:queue_retrieval_duration_seconds) do
+            name = :gitlab_ci_queue_retrieval_duration_seconds
+            comment = 'Time it takes to execute a SQL query to retrieve builds queue'
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
             labels = {}
 
             Gitlab::Metrics.histogram(name, comment, labels, buckets)
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 3459b69bebc..f2c75227002 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -101,7 +101,7 @@ module Gitlab
         end
 
         def merged_yaml
-          @ci_config&.to_hash&.to_yaml
+          @ci_config&.to_hash&.deep_stringify_keys&.to_yaml
         end
 
         def variables_with_data
diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb
index c4af5e6608e..8ce71558ab3 100644
--- a/lib/gitlab/data_builder/build.rb
+++ b/lib/gitlab/data_builder/build.rb
@@ -26,6 +26,7 @@ module Gitlab
           build_name: build.name,
           build_stage: build.stage,
           build_status: build.status,
+          build_created_at: build.created_at,
           build_started_at: build.started_at,
           build_finished_at: build.finished_at,
           build_duration: build.duration,
diff --git a/lib/gitlab/updated_notes_paginator.rb b/lib/gitlab/updated_notes_paginator.rb
index 511d6dccf7c..d5c01bde6b3 100644
--- a/lib/gitlab/updated_notes_paginator.rb
+++ b/lib/gitlab/updated_notes_paginator.rb
@@ -37,7 +37,7 @@ module Gitlab
     end
 
     def fetch_page(relation)
-      relation = relation.by_updated_at
+      relation = relation.order_updated_asc.with_order_id_asc
       notes = relation.limit(LIMIT + 1).to_a
 
       return [notes, false] unless notes.size > LIMIT