Add latest changes from gitlab-org/gitlab@master

11 files changed, 257 insertions, 54 deletions

diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb
new file mode 100644
index 00000000000..b7a912da060
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # migrates pages from legacy storage to zip format
+    # we intentionally use application code here because
+    # it has a lot of dependencies including models, carrierwave uploaders and service objects
+    # and copying all or part of this code in the background migration doesn't add much value
+    # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion
+    class MigratePagesToZipStorage
+      def perform(start_id, stop_id)
+        ::Pages::MigrateFromLegacyStorageService.new(Gitlab::AppLogger,
+                                                     ignore_invalid_entries: false,
+                                                     mark_projects_as_not_deployed: false)
+          .execute_for_batch(start_id..stop_id)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 46ecb10ea2b..c3c1728602c 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -84,7 +84,7 @@ module Gitlab
           end
 
           def metrics
-            @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new
+            @metrics ||= ::Gitlab::Ci::Pipeline::Metrics
           end
 
           def observe_creation_duration(duration)
@@ -97,6 +97,11 @@ module Gitlab
               .observe({ source: pipeline.source.to_s }, pipeline.total_size)
           end
 
+          def increment_pipeline_failure_reason_counter(reason)
+            metrics.pipeline_failure_reason_counter
+              .increment(reason: (reason || :unknown_failure).to_s)
+          end
+
           def dangling_build?
             %i[ondemand_dast_scan webide].include?(source)
           end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index f995f62f87b..22a7dbe61aa 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -13,16 +13,7 @@ module Gitlab
 
             pipeline.add_error_message(message)
 
-            if drop_reason && persist_pipeline?
-              if Feature.enabled?(:ci_pipeline_ensure_iid_on_drop, pipeline.project, default_enabled: :yaml)
-                # Project iid must be called outside a transaction, so we ensure it is set here
-                # otherwise it may be set within the state transition transaction of the drop! call
-                # which it will lock the InternalId row for the whole transaction
-                pipeline.ensure_project_iid!
-              end
-
-              pipeline.drop!(drop_reason)
-            end
+            drop_pipeline!(drop_reason)
 
             # TODO: consider not to rely on AR errors directly as they can be
             # polluted with other unrelated errors (e.g. state machine)
@@ -34,8 +25,23 @@ module Gitlab
             pipeline.add_warning_message(message)
           end
 
-          def persist_pipeline?
-            command.save_incompleted && !pipeline.readonly?
+          private
+
+          def drop_pipeline!(drop_reason)
+            return if pipeline.readonly?
+
+            if drop_reason && command.save_incompleted
+              if Feature.enabled?(:ci_pipeline_ensure_iid_on_drop, pipeline.project, default_enabled: :yaml)
+                # Project iid must be called outside a transaction, so we ensure it is set here
+                # otherwise it may be set within the state transition transaction of the drop! call
+                # which it will lock the InternalId row for the whole transaction
+                pipeline.ensure_project_iid!
+              end
+
+              pipeline.drop!(drop_reason)
+            else
+              command.increment_pipeline_failure_reason_counter(drop_reason)
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb
index 0d7449813b4..b17ae77d445 100644
--- a/lib/gitlab/ci/pipeline/chain/metrics.rb
+++ b/lib/gitlab/ci/pipeline/chain/metrics.rb
@@ -14,7 +14,7 @@ module Gitlab
           end
 
           def counter
-            ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter
+            ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index c77f4dcca5a..6cb6fd3920d 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -4,55 +4,57 @@ module Gitlab
   module Ci
     module Pipeline
       class Metrics
-        include Gitlab::Utils::StrongMemoize
+        def self.pipeline_creation_duration_histogram
+          name = :gitlab_ci_pipeline_creation_duration_seconds
+          comment = 'Pipeline creation duration'
+          labels = {}
+          buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
 
-        def pipeline_creation_duration_histogram
-          strong_memoize(:pipeline_creation_duration_histogram) do
-            name = :gitlab_ci_pipeline_creation_duration_seconds
-            comment = 'Pipeline creation duration'
-            labels = {}
-            buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
+
+        def self.pipeline_size_histogram
+          name = :gitlab_ci_pipeline_size_builds
+          comment = 'Pipeline size'
+          labels = { source: nil }
+          buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
+
+        def self.pipeline_processing_events_counter
+          name = :gitlab_ci_pipeline_processing_events_total
+          comment = 'Total amount of pipeline processing events'
 
-            ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def pipeline_size_histogram
-          strong_memoize(:pipeline_size_histogram) do
-            name = :gitlab_ci_pipeline_size_builds
-            comment = 'Pipeline size'
-            labels = { source: nil }
-            buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+        def self.pipelines_created_counter
+          name = :pipelines_created_total
+          comment = 'Counter of pipelines created'
 
-            ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def pipeline_processing_events_counter
-          strong_memoize(:pipeline_processing_events_counter) do
-            name = :gitlab_ci_pipeline_processing_events_total
-            comment = 'Total amount of pipeline processing events'
+        def self.legacy_update_jobs_counter
+          name = :ci_legacy_update_jobs_as_retried_total
+          comment = 'Counter of occurrences when jobs were not being set as retried before update_retried'
 
-            Gitlab::Metrics.counter(name, comment)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def pipelines_created_counter
-          strong_memoize(:pipelines_created_count) do
-            name = :pipelines_created_total
-            comment = 'Counter of pipelines created'
+        def self.pipeline_failure_reason_counter
+          name = :gitlab_ci_pipeline_failure_reasons
+          comment = 'Counter of pipeline failure reasons'
 
-            Gitlab::Metrics.counter(name, comment)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
 
-        def legacy_update_jobs_counter
-          strong_memoize(:legacy_update_jobs_counter) do
-            name = :ci_legacy_update_jobs_as_retried_total
-            comment = 'Counter of occurrences when jobs were not being set as retried before update_retried'
+        def self.job_failure_reason_counter
+          name = :gitlab_ci_job_failure_reasons
+          comment = 'Counter of job failure reasons'
 
-            Gitlab::Metrics.counter(name, comment)
-          end
+          Gitlab::Metrics.counter(name, comment)
         end
       end
     end
diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb
index 324695adb1f..4aa33ed7946 100644
--- a/lib/gitlab/database/background_migration/batched_migration.rb
+++ b/lib/gitlab/database/background_migration/batched_migration.rb
@@ -57,6 +57,13 @@ module Gitlab
         def batch_class_name=(class_name)
           write_attribute(:batch_class_name, class_name.demodulize)
         end
+
+        def prometheus_labels
+          @prometheus_labels ||= {
+            migration_id: id,
+            migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name]
+          }
+        end
       end
     end
   end
diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
index bbaa8040203..c276f8ce75b 100644
--- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Database
     module BackgroundMigration
       class BatchedMigrationWrapper
+        extend Gitlab::Utils::StrongMemoize
+
         # Wraps the execution of a batched_background_migration.
         #
         # Updates the job's tracking records with the status of the migration
@@ -23,6 +25,7 @@ module Gitlab
           raise e
         ensure
           finish_tracking_execution(batch_tracking_record)
+          track_prometheus_metrics(batch_tracking_record)
         end
 
         private
@@ -51,6 +54,65 @@ module Gitlab
           tracking_record.finished_at = Time.current
           tracking_record.save!
         end
+
+        def track_prometheus_metrics(tracking_record)
+          migration = tracking_record.batched_migration
+          base_labels = migration.prometheus_labels
+
+          metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size)
+          metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size)
+          metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size)
+
+          # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1)
+          efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f
+          metric_for(:histogram_time_efficiency).observe(base_labels, efficiency)
+
+          if metrics = tracking_record.metrics
+            metrics['timings']&.each do |key, timings|
+              summary = metric_for(:histogram_timings)
+              labels = base_labels.merge(operation: key)
+
+              timings.each do |timing|
+                summary.observe(labels, timing)
+              end
+            end
+          end
+        end
+
+        def metric_for(name)
+          self.class.metrics[name]
+        end
+
+        def self.metrics
+          strong_memoize(:metrics) do
+            {
+              gauge_batch_size: Gitlab::Metrics.gauge(
+                :batched_migration_job_batch_size,
+                'Batch size for a batched migration job'
+              ),
+              gauge_sub_batch_size: Gitlab::Metrics.gauge(
+                :batched_migration_job_sub_batch_size,
+                'Sub-batch size for a batched migration job'
+              ),
+              counter_updated_tuples: Gitlab::Metrics.counter(
+                :batched_migration_job_updated_tuples_total,
+                'Number of tuples updated by batched migration job'
+              ),
+              histogram_timings: Gitlab::Metrics.histogram(
+                :batched_migration_job_duration_seconds,
+                'Timings for a batched migration job',
+                {},
+                [0.1, 0.25, 0.5, 1, 5].freeze
+              ),
+              histogram_time_efficiency: Gitlab::Metrics.histogram(
+                :batched_migration_job_time_efficiency,
+                'Ratio of job duration to interval',
+                {},
+                [0.5, 0.9, 1, 1.5, 2].freeze
+              )
+            }
+          end
+        end
       end
     end
   end
diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb
index 3dda68bf93f..a1fabe75a97 100644
--- a/lib/gitlab/metrics/background_transaction.rb
+++ b/lib/gitlab/metrics/background_transaction.rb
@@ -34,8 +34,9 @@ module Gitlab
 
       def labels
         @labels ||= {
-          endpoint_id: current_context&.get_attribute(:caller_id),
-          feature_category: current_context&.get_attribute(:feature_category)
+          endpoint_id: endpoint_id,
+          feature_category: feature_category,
+          queue: queue
         }
       end
 
@@ -44,6 +45,21 @@ module Gitlab
       def current_context
         Labkit::Context.current
       end
+
+      def feature_category
+        current_context&.get_attribute(:feature_category)
+      end
+
+      def endpoint_id
+        current_context&.get_attribute(:caller_id)
+      end
+
+      def queue
+        worker_class = endpoint_id.to_s.safe_constantize
+        return if worker_class.blank? || !worker_class.respond_to?(:queue)
+
+        worker_class.queue.to_s
+      end
     end
   end
 end
diff --git a/lib/gitlab/pages/migration_helper.rb b/lib/gitlab/pages/migration_helper.rb
new file mode 100644
index 00000000000..8f8667fafd9
--- /dev/null
+++ b/lib/gitlab/pages/migration_helper.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Pages
+    class MigrationHelper
+      def initialize(logger = nil)
+        @logger = logger
+      end
+
+      def migrate_to_remote_storage
+        deployments = ::PagesDeployment.with_files_stored_locally
+        migrate(deployments, ObjectStorage::Store::REMOTE)
+      end
+
+      def migrate_to_local_storage
+        deployments = ::PagesDeployment.with_files_stored_remotely
+        migrate(deployments, ObjectStorage::Store::LOCAL)
+      end
+
+      private
+
+      def batch_size
+        ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+      end
+
+      def migrate(deployments, store)
+        deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord
+          deployment.file.migrate!(store)
+
+          log_success(deployment, store)
+        rescue => e
+          log_error(e, deployment)
+        end
+      end
+
+      def log_success(deployment, store)
+        logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage")
+      end
+
+      def log_error(err, deployment)
+        logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}")
+      end
+
+      def storage_label(store)
+        if store == ObjectStorage::Store::LOCAL
+          'local'
+        else
+          'object'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb
index 6bd74a2a993..1f72bf4ce26 100644
--- a/lib/gitlab/usage_data_non_sql_metrics.rb
+++ b/lib/gitlab/usage_data_non_sql_metrics.rb
@@ -24,6 +24,12 @@ module Gitlab
       def histogram(relation, column, buckets:, bucket_size: buckets.size)
         SQL_METRIC_DEFAULT
       end
+
+      def maximum_id(model)
+      end
+
+      def minimum_id(model)
+      end
     end
   end
 end
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
index 606d1369e18..ee2931f0c4f 100644
--- a/lib/tasks/gitlab/pages.rake
+++ b/lib/tasks/gitlab/pages.rake
@@ -9,10 +9,9 @@ namespace :gitlab do
       logger.info('Starting to migrate legacy pages storage to zip deployments')
 
       result = ::Pages::MigrateFromLegacyStorageService.new(logger,
-                                                            migration_threads: migration_threads,
-                                                            batch_size: batch_size,
                                                             ignore_invalid_entries: ignore_invalid_entries,
-                                                            mark_projects_as_not_deployed: mark_projects_as_not_deployed).execute
+                                                            mark_projects_as_not_deployed: mark_projects_as_not_deployed)
+                 .execute_with_threads(threads: migration_threads, batch_size: batch_size)
 
       logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.")
       logger.info("- The #{result[:migrated]} projects migrated successfully")
@@ -58,5 +57,33 @@ namespace :gitlab do
         ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false')
       )
     end
+
+    namespace :deployments do
+      task migrate_to_object_storage: :gitlab_environment do
+        logger = Logger.new(STDOUT)
+        logger.info('Starting transfer of pages deployments to remote storage')
+
+        helper = Gitlab::Pages::MigrationHelper.new(logger)
+
+        begin
+          helper.migrate_to_remote_storage
+        rescue => e
+          logger.error(e.message)
+        end
+      end
+
+      task migrate_to_local: :gitlab_environment do
+        logger = Logger.new(STDOUT)
+        logger.info('Starting transfer of Pages deployments to local storage')
+
+        helper = Gitlab::Pages::MigrationHelper.new(logger)
+
+        begin
+          helper.migrate_to_local_storage
+        rescue => e
+          logger.error(e.message)
+        end
+      end
+    end
   end
 end