Add latest changes from gitlab-org/gitlab@master

7 files changed, 228 insertions, 11 deletions

diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb
index bd60470fbd6..3ddef0c16b3 100644
--- a/lib/api/helpers/graphql_helpers.rb
+++ b/lib/api/helpers/graphql_helpers.rb
@@ -6,7 +6,7 @@ module API
     # against the graphql API. Helper code for the graphql server implementation
     # should be in app/graphql/ or lib/gitlab/graphql/
     module GraphqlHelpers
-      def conditionally_graphql!(fallback:, query:, context: {}, transform: nil)
+      def run_graphql!(query:, context: {}, transform: nil)
         result = GitlabSchema.execute(query, context: context)
 
         if transform
diff --git a/lib/api/version.rb b/lib/api/version.rb
index eca1b529094..f79bb3428f2 100644
--- a/lib/api/version.rb
+++ b/lib/api/version.rb
@@ -19,11 +19,10 @@ module API
       detail 'This feature was introduced in GitLab 8.13.'
     end
     get '/version' do
-      conditionally_graphql!(
+      run_graphql!(
         query: METADATA_QUERY,
         context: { current_user: current_user },
-        transform: ->(result) { result.dig('data', 'metadata') },
-        fallback: -> { { version: Gitlab::VERSION, revision: Gitlab.revision } }
+        transform: ->(result) { result.dig('data', 'metadata') }
       )
     end
   end
diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb
index b3d5d2c95d7..a35b0d7a0b5 100644
--- a/lib/banzai/filter/video_link_filter.rb
+++ b/lib/banzai/filter/video_link_filter.rb
@@ -8,8 +8,8 @@ module Banzai
     # a "Download" link in the case the video cannot be played.
     class VideoLinkFilter < HTML::Pipeline::Filter
       def call
-        doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |el|
-          el.replace(video_node(doc, el)) if has_video_extension?(el)
+        doc.xpath(query).each do |el|
+          el.replace(video_node(doc, el))
         end
 
         doc
@@ -17,10 +17,22 @@ module Banzai
 
       private
 
-      def has_video_extension?(element)
-        src_attr = context[:asset_proxy_enabled] ? 'data-canonical-src' : 'src'
+      def query
+        @query ||= begin
+          src_query = UploaderHelper::SAFE_VIDEO_EXT.map do |ext|
+            "'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})"
+          end
 
-        element.attr(src_attr).downcase.end_with?(*UploaderHelper::SAFE_VIDEO_EXT)
+          if context[:asset_proxy_enabled].present?
+            src_query.concat(
+              UploaderHelper::SAFE_VIDEO_EXT.map do |ext|
+                "'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})"
+              end
+            )
+          end
+
+          "descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]"
+        end
       end
 
       def video_node(doc, element)
diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb
new file mode 100644
index 00000000000..68fd0c17d29
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_pages_metadata.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class that will insert record into project_pages_metadata
+    # for each existing project
+    class MigratePagesMetadata
+      def perform(start_id, stop_id)
+        perform_on_relation(Project.where(id: start_id..stop_id))
+      end
+
+      def perform_on_relation(relation)
+        successful_pages_deploy = <<~SQL
+          SELECT TRUE
+          FROM ci_builds
+          WHERE ci_builds.type = 'GenericCommitStatus'
+            AND ci_builds.status = 'success'
+            AND ci_builds.stage = 'deploy'
+            AND ci_builds.name = 'pages:deploy'
+            AND ci_builds.project_id = projects.id
+          LIMIT 1
+        SQL
+
+        select_from = relation
+          .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)")
+          .to_sql
+
+        ActiveRecord::Base.connection_pool.with_connection do |connection|
+          connection.execute <<~SQL
+            INSERT INTO project_pages_metadata (project_id, deployed)
+          #{select_from}
+            ON CONFLICT (project_id) DO NOTHING
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb
new file mode 100644
index 00000000000..3c00b67911f
--- /dev/null
+++ b/lib/gitlab/ci/status/composite.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      class Composite
+        include Gitlab::Utils::StrongMemoize
+
+        # This class accepts an array of arrays/hashes/or objects
+        def initialize(all_statuses, with_allow_failure: true)
+          unless all_statuses.respond_to?(:pluck)
+            raise ArgumentError, "all_statuses needs to respond to `.pluck`"
+          end
+
+          @status_set = Set.new
+          @status_key = 0
+          @allow_failure_key = 1 if with_allow_failure
+
+          consume_all_statuses(all_statuses)
+        end
+
+        # The status calculation is order dependent,
+        # 1. In some cases we assume that that status is exact
+        #    if the we only have given statues,
+        # 2. In other cases we assume that status is of that type
+        #    based on what statuses are no longer valid based on the
+        #    data set that we have
+        def status
+          return if none?
+
+          strong_memoize(:status) do
+            if only_of?(:skipped, :ignored)
+              'skipped'
+            elsif only_of?(:success, :skipped, :success_with_warnings, :ignored)
+              'success'
+            elsif only_of?(:created, :success_with_warnings, :ignored)
+              'created'
+            elsif only_of?(:preparing, :success_with_warnings, :ignored)
+              'preparing'
+            elsif only_of?(:canceled, :success, :skipped, :success_with_warnings, :ignored)
+              'canceled'
+            elsif only_of?(:pending, :created, :skipped, :success_with_warnings, :ignored)
+              'pending'
+            elsif any_of?(:running, :pending)
+              'running'
+            elsif any_of?(:manual)
+              'manual'
+            elsif any_of?(:scheduled)
+              'scheduled'
+            elsif any_of?(:preparing)
+              'preparing'
+            elsif any_of?(:created)
+              'running'
+            else
+              'failed'
+            end
+          end
+        end
+
+        def warnings?
+          @status_set.include?(:success_with_warnings)
+        end
+
+        private
+
+        def none?
+          @status_set.empty?
+        end
+
+        def any_of?(*names)
+          names.any? { |name| @status_set.include?(name) }
+        end
+
+        def only_of?(*names)
+          matching = names.count { |name| @status_set.include?(name) }
+          matching > 0 &&
+            matching == @status_set.size
+        end
+
+        def consume_all_statuses(all_statuses)
+          columns = []
+          columns[@status_key] = :status
+          columns[@allow_failure_key] = :allow_failure if @allow_failure_key
+
+          all_statuses
+            .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord
+            .each(&method(:consume_status))
+        end
+
+        def consume_status(description)
+          # convert `"status"` into `["status"]`
+          description = Array(description)
+
+          status =
+            if success_with_warnings?(description)
+              :success_with_warnings
+            elsif ignored_status?(description)
+              :ignored
+            else
+              description[@status_key].to_sym
+            end
+
+          @status_set.add(status)
+        end
+
+        def success_with_warnings?(status)
+          @allow_failure_key &&
+            status[@allow_failure_key] &&
+            HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key])
+        end
+
+        def ignored_status?(status)
+          @allow_failure_key &&
+            status[@allow_failure_key] &&
+            HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key])
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 39a243bd433..017e536c3e7 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -52,6 +52,11 @@ module Gitlab
                                                                     project: restored_project)
       end
 
+      # A Hash of the imported merge request ID -> imported ID.
+      def merge_requests_mapping
+        @merge_requests_mapping ||= {}
+      end
+
       # Loops through the tree of models defined in import_export.yml and
       # finds them in the imported JSON so they can be instantiated and saved
       # in the DB. The structure and relationships between models are guessed from
@@ -80,10 +85,26 @@ module Gitlab
 
         @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
 
+        save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+
         # Restore the project again, extra query that skips holding the AR objects in memory
         @restored_project = Project.find(@project_id)
       end
 
+      # Older, serialized CI pipeline exports may only have a
+      # merge_request_id and not the full hash of the merge request. To
+      # import these pipelines, we need to preserve the mapping between
+      # the old and new the merge request ID.
+      def save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+        return unless relation_key == 'merge_requests'
+
+        relation_hash = Array(relation_hash)
+
+        Array(relation_hash_batch).each_with_index do |raw_data, index|
+          merge_requests_mapping[raw_data['id']] = relation_hash[index]['id']
+        end
+      end
+
       # Remove project models that became group models as we found them at group level.
       # This no longer required saving them at the root project level.
       # For example, in the case of an existing group label that matched the title.
@@ -222,6 +243,7 @@ module Gitlab
             relation_sym: relation_key.to_sym,
             relation_hash: relation_hash,
             members_mapper: members_mapper,
+            merge_requests_mapping: merge_requests_mapping,
             user: @user,
             project: @restored_project,
             excluded_keys: excluded_keys_for_relation(relation_key))
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 37f625288a9..9ec244b0960 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -55,10 +55,11 @@ module Gitlab
         relation_name.to_s.constantize
       end
 
-      def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
+      def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: [])
         @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
         @relation_hash = relation_hash.except('noteable_id')
         @members_mapper = members_mapper
+        @merge_requests_mapping = merge_requests_mapping
         @user = user
         @project = project
         @imported_object_retries = 0
@@ -109,7 +110,10 @@ module Gitlab
         update_group_references
         remove_duplicate_assignees
 
-        setup_pipeline if @relation_name == :'Ci::Pipeline'
+        if @relation_name == :'Ci::Pipeline'
+          update_merge_request_references
+          setup_pipeline
+        end
 
         reset_tokens!
         remove_encrypted_attributes!
@@ -194,6 +198,28 @@ module Gitlab
         @relation_hash['group_id'] = @project.namespace_id
       end
 
+      # This code is a workaround for broken project exports that don't
+      # export merge requests with CI pipelines (i.e. exports that were
+      # generated from
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844).
+      # This method can be removed in GitLab 12.6.
+      def update_merge_request_references
+        # If a merge request was properly created, we don't need to fix
+        # up this export.
+        return if @relation_hash['merge_request']
+
+        merge_request_id = @relation_hash['merge_request_id']
+
+        return unless merge_request_id
+
+        new_merge_request_id = @merge_requests_mapping[merge_request_id]
+
+        return unless new_merge_request_id
+
+        @relation_hash['merge_request_id'] = new_merge_request_id
+        parsed_relation_hash['merge_request_id'] = new_merge_request_id
+      end
+
       def reset_tokens!
         return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name)