Add latest changes from gitlab-org/gitlab@master

4 files changed, 208 insertions, 2 deletions

diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb
new file mode 100644
index 00000000000..68fd0c17d29
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_pages_metadata.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will insert record into project_pages_metadata
+    # for each existing project
+    class MigratePagesMetadata
+      def perform(start_id, stop_id)
+        perform_on_relation(Project.where(id: start_id..stop_id))
+      end
+
+      def perform_on_relation(relation)
+        successful_pages_deploy = <<~SQL
+          SELECT TRUE
+          FROM ci_builds
+          WHERE ci_builds.type = 'GenericCommitStatus'
+            AND ci_builds.status = 'success'
+            AND ci_builds.stage = 'deploy'
+            AND ci_builds.name = 'pages:deploy'
+            AND ci_builds.project_id = projects.id
+          LIMIT 1
+        SQL
+
+        select_from = relation
+          .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)")
+          .to_sql
+
+        ActiveRecord::Base.connection_pool.with_connection do |connection|
+          connection.execute <<~SQL
+            INSERT INTO project_pages_metadata (project_id, deployed)
+          #{select_from}
+            ON CONFLICT (project_id) DO NOTHING
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb
new file mode 100644
index 00000000000..3c00b67911f
--- /dev/null
+++ b/lib/gitlab/ci/status/composite.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      class Composite
+        include Gitlab::Utils::StrongMemoize
+
+        # This class accepts an array of arrays/hashes/or objects
+        def initialize(all_statuses, with_allow_failure: true)
+          unless all_statuses.respond_to?(:pluck)
+            raise ArgumentError, "all_statuses needs to respond to `.pluck`"
+          end
+
+          @status_set = Set.new
+          @status_key = 0
+          @allow_failure_key = 1 if with_allow_failure
+
+          consume_all_statuses(all_statuses)
+        end
+
+        # The status calculation is order dependent,
+        # 1. In some cases we assume that that status is exact
+        #    if the we only have given statues,
+        # 2. In other cases we assume that status is of that type
+        #    based on what statuses are no longer valid based on the
+        #    data set that we have
+        def status
+          return if none?
+
+          strong_memoize(:status) do
+            if only_of?(:skipped, :ignored)
+              'skipped'
+            elsif only_of?(:success, :skipped, :success_with_warnings, :ignored)
+              'success'
+            elsif only_of?(:created, :success_with_warnings, :ignored)
+              'created'
+            elsif only_of?(:preparing, :success_with_warnings, :ignored)
+              'preparing'
+            elsif only_of?(:canceled, :success, :skipped, :success_with_warnings, :ignored)
+              'canceled'
+            elsif only_of?(:pending, :created, :skipped, :success_with_warnings, :ignored)
+              'pending'
+            elsif any_of?(:running, :pending)
+              'running'
+            elsif any_of?(:manual)
+              'manual'
+            elsif any_of?(:scheduled)
+              'scheduled'
+            elsif any_of?(:preparing)
+              'preparing'
+            elsif any_of?(:created)
+              'running'
+            else
+              'failed'
+            end
+          end
+        end
+
+        def warnings?
+          @status_set.include?(:success_with_warnings)
+        end
+
+        private
+
+        def none?
+          @status_set.empty?
+        end
+
+        def any_of?(*names)
+          names.any? { |name| @status_set.include?(name) }
+        end
+
+        def only_of?(*names)
+          matching = names.count { |name| @status_set.include?(name) }
+          matching > 0 &&
+            matching == @status_set.size
+        end
+
+        def consume_all_statuses(all_statuses)
+          columns = []
+          columns[@status_key] = :status
+          columns[@allow_failure_key] = :allow_failure if @allow_failure_key
+
+          all_statuses
+            .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord
+            .each(&method(:consume_status))
+        end
+
+        def consume_status(description)
+          # convert `"status"` into `["status"]`
+          description = Array(description)
+
+          status =
+            if success_with_warnings?(description)
+              :success_with_warnings
+            elsif ignored_status?(description)
+              :ignored
+            else
+              description[@status_key].to_sym
+            end
+
+          @status_set.add(status)
+        end
+
+        def success_with_warnings?(status)
+          @allow_failure_key &&
+            status[@allow_failure_key] &&
+            HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key])
+        end
+
+        def ignored_status?(status)
+          @allow_failure_key &&
+            status[@allow_failure_key] &&
+            HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 39a243bd433..017e536c3e7 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -52,6 +52,11 @@ module Gitlab
                                                                     project: restored_project)
       end
 
+      # A Hash of the imported merge request ID -> imported ID.
+      def merge_requests_mapping
+        @merge_requests_mapping ||= {}
+      end
+
       # Loops through the tree of models defined in import_export.yml and
       # finds them in the imported JSON so they can be instantiated and saved
       # in the DB. The structure and relationships between models are guessed from
@@ -80,10 +85,26 @@ module Gitlab
 
         @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
 
+        save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+
         # Restore the project again, extra query that skips holding the AR objects in memory
         @restored_project = Project.find(@project_id)
       end
 
+      # Older, serialized CI pipeline exports may only have a
+      # merge_request_id and not the full hash of the merge request. To
+      # import these pipelines, we need to preserve the mapping between
+      # the old and new the merge request ID.
+      def save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+        return unless relation_key == 'merge_requests'
+
+        relation_hash = Array(relation_hash)
+
+        Array(relation_hash_batch).each_with_index do |raw_data, index|
+          merge_requests_mapping[raw_data['id']] = relation_hash[index]['id']
+        end
+      end
+
       # Remove project models that became group models as we found them at group level.
       # This no longer required saving them at the root project level.
       # For example, in the case of an existing group label that matched the title.
@@ -222,6 +243,7 @@ module Gitlab
             relation_sym: relation_key.to_sym,
             relation_hash: relation_hash,
             members_mapper: members_mapper,
+            merge_requests_mapping: merge_requests_mapping,
             user: @user,
             project: @restored_project,
             excluded_keys: excluded_keys_for_relation(relation_key))
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 37f625288a9..9ec244b0960 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -55,10 +55,11 @@ module Gitlab
         relation_name.to_s.constantize
       end
 
-      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
+      def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: [])
         @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
         @relation_hash = relation_hash.except('noteable_id')
         @members_mapper = members_mapper
+        @merge_requests_mapping = merge_requests_mapping
         @user = user
         @project = project
         @imported_object_retries = 0
@@ -109,7 +110,10 @@ module Gitlab
         update_group_references
         remove_duplicate_assignees
 
-        setup_pipeline if @relation_name == :'Ci::Pipeline'
+        if @relation_name == :'Ci::Pipeline'
+          update_merge_request_references
+          setup_pipeline
+        end
 
         reset_tokens!
         remove_encrypted_attributes!
@@ -194,6 +198,28 @@ module Gitlab
         @relation_hash['group_id'] = @project.namespace_id
       end
 
+      # This code is a workaround for broken project exports that don't
+      # export merge requests with CI pipelines (i.e. exports that were
+      # generated from
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844).
+      # This method can be removed in GitLab 12.6.
+      def update_merge_request_references
+        # If a merge request was properly created, we don't need to fix
+        # up this export.
+        return if @relation_hash['merge_request']
+
+        merge_request_id = @relation_hash['merge_request_id']
+
+        return unless merge_request_id
+
+        new_merge_request_id = @merge_requests_mapping[merge_request_id]
+
+        return unless new_merge_request_id
+
+        @relation_hash['merge_request_id'] = new_merge_request_id
+        parsed_relation_hash['merge_request_id'] = new_merge_request_id
+      end
+
       def reset_tokens!
         return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name)