Merge branch 'last-green-master' into 18471-restrict-tag-pushes-protected-tags

4 files changed, 68 insertions, 17 deletions

diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index fc445ab9483..525aa920328 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -26,6 +26,30 @@ module Gitlab
         add_index(table_name, column_name, options)
       end
 
+      # Removes an existed index, concurrently when supported
+      #
+      # On PostgreSQL this method removes an index concurrently.
+      #
+      # Example:
+      #
+      #     remove_concurrent_index :users, :some_column
+      #
+      # See Rails' `remove_index` for more info on the available arguments.
+      def remove_concurrent_index(table_name, column_name, options = {})
+        if transaction_open?
+          raise 'remove_concurrent_index can not be run inside a transaction, ' \
+            'you can disable transactions by calling disable_ddl_transaction! ' \
+            'in the body of your migration class'
+        end
+
+        if Database.postgresql?
+          options = options.merge({ algorithm: :concurrently })
+          disable_statement_timeout
+        end
+
+        remove_index(table_name, options.merge({ column: column_name }))
+      end
+
       # Adds a foreign key with only minimal locking on the tables involved.
       #
       # This method only requires minimal locking when using PostgreSQL. When
diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
index cd4e318033d..630fe4fa849 100644
--- a/lib/gitlab/etag_caching/middleware.rb
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -2,26 +2,34 @@ module Gitlab
   module EtagCaching
     class Middleware
       RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|')
-      ROUTE_REGEXP = Regexp.union(
-        %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z),
-        %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z)
-      )
+      ROUTES = [
+        {
+          regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z),
+          name: 'issue_notes'
+        },
+        {
+          regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z),
+          name: 'issue_title'
+        }
+      ].freeze
 
       def initialize(app)
         @app = app
       end
 
       def call(env)
-        return @app.call(env) unless enabled_for_current_route?(env)
-        Gitlab::Metrics.add_event(:etag_caching_middleware_used)
+        route = match_current_route(env)
+        return @app.call(env) unless route
+
+        track_event(:etag_caching_middleware_used, route)
 
         etag, cached_value_present = get_etag(env)
         if_none_match = env['HTTP_IF_NONE_MATCH']
 
         if if_none_match == etag
-          handle_cache_hit(etag)
+          handle_cache_hit(etag, route)
         else
-          track_cache_miss(if_none_match, cached_value_present)
+          track_cache_miss(if_none_match, cached_value_present, route)
 
           status, headers, body = @app.call(env)
           headers['ETag'] = etag
@@ -31,8 +39,8 @@ module Gitlab
 
       private
 
-      def enabled_for_current_route?(env)
-        ROUTE_REGEXP.match(env['PATH_INFO'])
+      def match_current_route(env)
+        ROUTES.find { |route| route[:regexp].match(env['PATH_INFO']) }
       end
 
       def get_etag(env)
@@ -52,23 +60,27 @@ module Gitlab
         %Q{W/"#{value}"}
       end
 
-      def handle_cache_hit(etag)
-        Gitlab::Metrics.add_event(:etag_caching_cache_hit)
+      def handle_cache_hit(etag, route)
+        track_event(:etag_caching_cache_hit, route)
 
         status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429
 
         [status_code, { 'ETag' => etag }, ['']]
       end
 
-      def track_cache_miss(if_none_match, cached_value_present)
+      def track_cache_miss(if_none_match, cached_value_present, route)
         if if_none_match.blank?
-          Gitlab::Metrics.add_event(:etag_caching_header_missing)
+          track_event(:etag_caching_header_missing, route)
         elsif !cached_value_present
-          Gitlab::Metrics.add_event(:etag_caching_key_not_found)
+          track_event(:etag_caching_key_not_found, route)
         else
-          Gitlab::Metrics.add_event(:etag_caching_resource_changed)
+          track_event(:etag_caching_resource_changed, route)
         end
       end
+
+      def track_event(name, route)
+        Gitlab::Metrics.add_event(name, endpoint: route[:name])
+      end
     end
   end
 end
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index 11e5f1b645c..ca8d3271541 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -72,6 +72,8 @@ module Gitlab
     # job_ids - The Sidekiq job IDs to check.
     #
     # Returns an array of true or false indicating job completion.
+    # true = job is still running
+    # false = job completed
     def self.job_status(job_ids)
       keys = job_ids.map { |jid| key_for(jid) }
 
@@ -82,6 +84,17 @@ module Gitlab
       end
     end
 
+    # Returns the JIDs that are completed
+    #
+    # job_ids - The Sidekiq job IDs to check.
+    #
+    # Returns an array of completed JIDs
+    def self.completed_jids(job_ids)
+      Sidekiq.redis do |redis|
+        job_ids.reject { |jid| redis.exists(key_for(jid)) }
+      end
+    end
+
     def self.key_for(jid)
       STATUS_KEY % jid
     end
diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb
index d47609f490d..00983b3284a 100644
--- a/lib/gitlab/sidekiq_status/client_middleware.rb
+++ b/lib/gitlab/sidekiq_status/client_middleware.rb
@@ -2,7 +2,9 @@ module Gitlab
   module SidekiqStatus
     class ClientMiddleware
       def call(_, job, _, _)
-        Gitlab::SidekiqStatus.set(job['jid'])
+        status_expiration = job['status_expiration'] || Gitlab::SidekiqStatus::DEFAULT_EXPIRATION
+
+        Gitlab::SidekiqStatus.set(job['jid'], status_expiration)
         yield
       end
     end