Add latest changes from gitlab-org/gitlab@master

13 files changed, 230 insertions, 13 deletions

diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb
index 9797c86478e..ad1d904cde7 100644
--- a/lib/gitlab/background_migration/user_mentions/models/epic.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb
@@ -18,7 +18,6 @@ module Gitlab
           self.table_name = 'epics'
 
           belongs_to :author, class_name: "User"
-          belongs_to :project
           belongs_to :group
 
           def self.user_mention_model
diff --git a/lib/gitlab/checks/snippet_check.rb b/lib/gitlab/checks/snippet_check.rb
index 26dd772764a..be25fe3e7c4 100644
--- a/lib/gitlab/checks/snippet_check.rb
+++ b/lib/gitlab/checks/snippet_check.rb
@@ -21,7 +21,7 @@ module Gitlab
 
       def exec
         if creation? || deletion?
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_delete_branch]
+          raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_delete_branch]
         end
 
         # TODO: https://gitlab.com/gitlab-org/gitlab/issues/205628
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index f028102da9b..b7b9288517d 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -34,8 +34,7 @@ module Gitlab
 
         ignore_auto_reply!(mail)
 
-        mail_key = extract_mail_key(mail)
-        handler = Handler.for(mail, mail_key)
+        handler = find_handler(mail)
 
         raise UnknownIncomingEmail unless handler
 
@@ -46,6 +45,11 @@ module Gitlab
 
       private
 
+      def find_handler(mail)
+        mail_key = extract_mail_key(mail)
+        Handler.for(mail, mail_key)
+      end
+
       def build_mail
         Mail::Message.new(@raw)
       rescue Encoding::UndefinedConversionError,
diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb
index ff1af9bede4..3956fc8a483 100644
--- a/lib/gitlab/git_access_snippet.rb
+++ b/lib/gitlab/git_access_snippet.rb
@@ -28,7 +28,7 @@ module Gitlab
       # TODO: Investigate if expanding actor/authentication types are needed.
       # https://gitlab.com/gitlab-org/gitlab/issues/202190
       if actor && !actor.is_a?(User) && !actor.instance_of?(Key)
-        raise UnauthorizedError, ERROR_MESSAGES[:authentication_mechanism]
+        raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism]
       end
 
       unless Feature.enabled?(:version_snippets, user)
@@ -53,7 +53,7 @@ module Gitlab
 
     override :check_push_access!
     def check_push_access!
-      raise UnauthorizedError, ERROR_MESSAGES[:update_snippet] unless user
+      raise ForbiddenError, ERROR_MESSAGES[:update_snippet] unless user
 
       check_change_access!
     end
@@ -74,7 +74,7 @@ module Gitlab
       passed = guest_can_download_code? || user_can_download_code?
 
       unless passed
-        raise UnauthorizedError, ERROR_MESSAGES[:read_snippet]
+        raise ForbiddenError, ERROR_MESSAGES[:read_snippet]
       end
     end
 
@@ -91,7 +91,7 @@ module Gitlab
     override :check_change_access!
     def check_change_access!
       unless user_access.can_do_action?(:update_snippet)
-        raise UnauthorizedError, ERROR_MESSAGES[:update_snippet]
+        raise ForbiddenError, ERROR_MESSAGES[:update_snippet]
       end
 
       changes_list.each do |change|
diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb
index 56524120ffd..0dd28b32511 100644
--- a/lib/gitlab/graphql/docs/helper.rb
+++ b/lib/gitlab/graphql/docs/helper.rb
@@ -25,6 +25,28 @@ module Gitlab
           fields.sort_by { |field| field[:name] }
         end
 
+        def render_field(field)
+          '| %s | %s | %s |' % [
+            render_field_name(field),
+            render_field_type(field[:type][:info]),
+            render_field_description(field)
+            ]
+        end
+
+        def render_field_name(field)
+          rendered_name = "`#{field[:name]}`"
+          rendered_name += ' **{warning-solid}**' if field[:is_deprecated]
+          rendered_name
+        end
+
+        # Returns the field description. If the field has been deprecated,
+        # the deprecation reason will be returned in place of the description.
+        def render_field_description(field)
+          return field[:description] unless field[:is_deprecated]
+
+          "**Deprecated:** #{field[:deprecation_reason]}"
+        end
+
         # Some fields types are arrays of other types and are displayed
         # on docs wrapped in square brackets, for example: [String!].
         # This makes GitLab docs renderer thinks they are links so here
diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml
index b126a22c301..8c033526557 100644
--- a/lib/gitlab/graphql/docs/templates/default.md.haml
+++ b/lib/gitlab/graphql/docs/templates/default.md.haml
@@ -11,6 +11,9 @@
 
   Each table below documents a GraphQL type. Types match loosely to models, but not all
   fields and methods on a model are available via GraphQL.
+
+  CAUTION: **Caution:**
+  Fields that are deprecated are marked with **{warning-solid}**.
 \
 - objects.each do |type|
   - unless type[:fields].empty?
@@ -22,5 +25,5 @@
     ~ "| Name  | Type  | Description |"
     ~ "| ---   |  ---- | ----------  |"
     - sorted_fields(type[:fields]).each do |field|
-      = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |"
+      = render_field(field)
     \
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
index 4547a9b0a01..2889dbc68cc 100644
--- a/lib/gitlab/incoming_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -28,8 +28,9 @@ module Gitlab
         config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}")
       end
 
-      def key_from_address(address)
-        regex = address_regex
+      def key_from_address(address, wildcard_address: nil)
+        wildcard_address ||= config.address
+        regex = address_regex(wildcard_address)
         return unless regex
 
         match = address.match(regex)
@@ -55,8 +56,7 @@ module Gitlab
 
       private
 
-      def address_regex
-        wildcard_address = config.address
+      def address_regex(wildcard_address)
         return unless wildcard_address
 
         regex = Regexp.escape(wildcard_address)
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index 6c27213df49..c3a52a1986d 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -20,6 +20,7 @@ module Gitlab
         chain.add Gitlab::SidekiqMiddleware::AdminMode::Server
         chain.add Gitlab::SidekiqStatus::ServerMiddleware
         chain.add Gitlab::SidekiqMiddleware::WorkerContext::Server
+        chain.add Gitlab::SidekiqMiddleware::DuplicateJobs::Server
       end
     end
 
@@ -33,6 +34,7 @@ module Gitlab
         chain.add Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware
         chain.add Labkit::Middleware::Sidekiq::Client
         chain.add Gitlab::SidekiqMiddleware::AdminMode::Client
+        chain.add Gitlab::SidekiqMiddleware::DuplicateJobs::Client
       end
     end
   end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
new file mode 100644
index 00000000000..bb0c18735bb
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    module DuplicateJobs
+      class Client
+        def call(worker_class, job, queue, _redis_pool, &block)
+          DuplicateJob.new(job, queue).schedule(&block)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
new file mode 100644
index 00000000000..b84673c4cee
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Gitlab
+  module SidekiqMiddleware
+    module DuplicateJobs
+      # This class defines an identifier of a job in a queue
+      # The identifier based on a job's class and arguments.
+      #
+      # As strategy decides when to keep track of the job in redis and when to
+      # remove it.
+      #
+      # Storing the deduplication key in redis can be done by calling `check!`
+      # check returns the `jid` of the job if it was scheduled, or the `jid` of
+      # the duplicate job if it was already scheduled
+      #
+      # When new jobs can be scheduled again, the strategy calls `#delete`.
+      class DuplicateJob
+        DUPLICATE_KEY_TTL = 6.hours
+
+        attr_reader :existing_jid
+
+        def initialize(job, queue_name, strategy: :until_executing)
+          @job = job
+          @queue_name = queue_name
+          @strategy = strategy
+        end
+
+        # This will continue the middleware chain if the job should be scheduled
+        # It will return false if the job needs to be cancelled
+        def schedule(&block)
+          Strategies.for(strategy).new(self).schedule(job, &block)
+        end
+
+        # This will continue the server middleware chain if the job should be
+        # executed.
+        # It will return false if the job should not be executed.
+        def perform(&block)
+          Strategies.for(strategy).new(self).perform(job, &block)
+        end
+
+        # This method will return the jid that was set in redis
+        def check!
+          read_jid = nil
+
+          Sidekiq.redis do |redis|
+            redis.multi do |multi|
+              redis.set(idempotency_key, jid, ex: DUPLICATE_KEY_TTL, nx: true)
+              read_jid = redis.get(idempotency_key)
+            end
+          end
+
+          self.existing_jid = read_jid.value
+        end
+
+        def delete!
+          Sidekiq.redis do |redis|
+            redis.del(idempotency_key)
+          end
+        end
+
+        def duplicate?
+          raise "Call `#check!` first to check for existing duplicates" unless existing_jid
+
+          jid != existing_jid
+        end
+
+        private
+
+        attr_reader :queue_name, :strategy, :job
+        attr_writer :existing_jid
+
+        def worker_class_name
+          job['class']
+        end
+
+        def arguments
+          job['args']
+        end
+
+        def jid
+          job['jid']
+        end
+
+        def idempotency_key
+          @idempotency_key ||= "#{namespace}:#{idempotency_hash}"
+        end
+
+        def idempotency_hash
+          Digest::SHA256.hexdigest(idempotency_string)
+        end
+
+        def namespace
+          "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:duplicate:#{queue_name}"
+        end
+
+        def idempotency_string
+          "#{worker_class_name}:#{arguments.join('-')}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb
new file mode 100644
index 00000000000..a35edc5774e
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/server.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    module DuplicateJobs
+      class Server
+        def call(worker, job, queue, &block)
+          DuplicateJob.new(job, queue).perform(&block)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb
new file mode 100644
index 00000000000..a08310a58ff
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    module DuplicateJobs
+      module Strategies
+        UnknownStrategyError = Class.new(StandardError)
+
+        STRATEGIES = {
+          until_executing: UntilExecuting
+        }.freeze
+
+        def self.for(name)
+          STRATEGIES.fetch(name)
+        rescue KeyError
+          raise UnknownStrategyError, "Unknown deduplication strategy #{name}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
new file mode 100644
index 00000000000..b8f49b67a59
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module SidekiqMiddleware
+    module DuplicateJobs
+      module Strategies
+        # This strategy takes a lock before scheduling the job in a queue and
+        # removes the lock before the job starts allowing a new job to be queued
+        # while a job is still executing.
+        class UntilExecuting
+          def initialize(duplicate_job)
+            @duplicate_job = duplicate_job
+          end
+
+          def schedule(job)
+            if duplicate_job.check! && duplicate_job.duplicate?
+              job['duplicate-of'] = duplicate_job.existing_jid
+            end
+
+            yield
+          end
+
+          def perform(_job)
+            duplicate_job.delete!
+
+            yield
+          end
+
+          private
+
+          attr_reader :duplicate_job
+        end
+      end
+    end
+  end
+end