Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40

5 files changed, 194 insertions, 95 deletions

diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb
index fc3c05c57b2..8c916375a98 100644
--- a/lib/gitlab/etag_caching/middleware.rb
+++ b/lib/gitlab/etag_caching/middleware.rb
@@ -17,12 +17,12 @@ module Gitlab
 
       def call(env)
         request = ActionDispatch::Request.new(env)
-        route = Gitlab::EtagCaching::Router.match(request.path_info)
+        route = Gitlab::EtagCaching::Router.match(request)
         return @app.call(env) unless route
 
         track_event(:etag_caching_middleware_used, route)
 
-        etag, cached_value_present = get_etag(request)
+        etag, cached_value_present = get_etag(request, route)
         if_none_match = env['HTTP_IF_NONE_MATCH']
 
         if if_none_match == etag
@@ -36,8 +36,8 @@ module Gitlab
 
       private
 
-      def get_etag(request)
-        cache_key = request.path
+      def get_etag(request, route)
+        cache_key = route.cache_key(request)
         store = Gitlab::EtagCaching::Store.new
         current_value = store.get(cache_key)
         cached_value_present = current_value.present?
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index 769ac2784d1..742b72ecde9 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -2,99 +2,24 @@
 
 module Gitlab
   module EtagCaching
-    class Router
-      Route = Struct.new(:regexp, :name, :feature_category)
-      # We enable an ETag for every request matching the regex.
-      # To match a regex the path needs to match the following:
-      #   - Don't contain a reserved word (expect for the words used in the
-      #     regex itself)
-      #   - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route
-      #   - Ending in `issues/id`/realtime_changes` for the `issue_title` route
-      USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
-                          commit pipelines merge_requests builds
-                          new environments].freeze
-      RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
-      RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
-      RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*)
+    module Router
+      Route = Struct.new(:regexp, :name, :feature_category, :router) do
+        delegate :match, to: :regexp
+        delegate :cache_key, to: :router
+      end
 
-      ROUTES = [
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z),
-          'issue_notes',
-          'issue_tracking'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z),
-          'merge_request_notes',
-          'code_review'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z),
-          'issue_title',
-          'issue_tracking'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z),
-          'commit_pipelines',
-          'continuous_integration'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z),
-          'new_merge_request_pipelines',
-          'continuous_integration'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z),
-          'merge_request_pipelines',
-          'continuous_integration'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z),
-          'project_pipelines',
-          'continuous_integration'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z),
-          'project_pipeline',
-          'continuous_integration'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z),
-          'project_build',
-          'continuous_integration'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z),
-          'cluster_environments',
-          'continuous_delivery'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z),
-          'environments',
-          'continuous_delivery'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z),
-          'realtime_changes_import_github',
-          'importers'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z),
-          'realtime_changes_import_gitea',
-          'importers'
-        ),
-        Gitlab::EtagCaching::Router::Route.new(
-          %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z),
-          'merge_request_widget',
-          'code_review'
-        )
-      ].freeze
+      module Helpers
+        def build_route(attrs)
+          EtagCaching::Router::Route.new(*attrs, self)
+        end
+      end
 
-      def self.match(path)
-        ROUTES.find { |route| route.regexp.match(path) }
+      # Performing RESTful routing match before GraphQL would be more expensive
+      # for the GraphQL requests because we need to traverse all of the RESTful
+      # route definitions before falling back to GraphQL.
+      def self.match(request)
+        Router::Graphql.match(request) || Router::Restful.match(request)
       end
     end
   end
 end
-
-Gitlab::EtagCaching::Router.prepend_if_ee('EE::Gitlab::EtagCaching::Router')
diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb
new file mode 100644
index 00000000000..f1737f0ce5a
--- /dev/null
+++ b/lib/gitlab/etag_caching/router/graphql.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module EtagCaching
+    module Router
+      class Graphql
+        extend EtagCaching::Router::Helpers
+        GRAPHQL_ETAG_RESOURCE_HEADER = 'X-GITLAB-GRAPHQL-RESOURCE-ETAG'
+
+        ROUTES = [
+          [
+            %r(\Apipelines/id/\d+\z),
+            'pipelines_graph',
+            'continuous_integration'
+          ]
+        ].map(&method(:build_route)).freeze
+
+        def self.match(request)
+          return unless request.path_info == graphql_api_path
+
+          graphql_resource = request.headers[GRAPHQL_ETAG_RESOURCE_HEADER]
+          return unless graphql_resource
+
+          ROUTES.find { |route| route.match(graphql_resource) }
+        end
+
+        def self.cache_key(request)
+          [
+            request.path,
+            request.headers[GRAPHQL_ETAG_RESOURCE_HEADER]
+          ].compact.join(':')
+        end
+
+        def self.graphql_api_path
+          @graphql_api_path ||= Gitlab::Routing.url_helpers.api_graphql_path
+        end
+        private_class_method :graphql_api_path
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb
new file mode 100644
index 00000000000..08c20e30a48
--- /dev/null
+++ b/lib/gitlab/etag_caching/router/restful.rb
@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module EtagCaching
+    module Router
+      class Restful
+        extend EtagCaching::Router::Helpers
+
+        # We enable an ETag for every request matching the regex.
+        # To match a regex the path needs to match the following:
+        #   - Don't contain a reserved word (expect for the words used in the
+        #     regex itself)
+        #   - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route
+        #   - Ending in `issues/id`/realtime_changes` for the `issue_title` route
+        USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
+                            commit pipelines merge_requests builds
+                            new environments].freeze
+        RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
+        RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
+        RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*)
+
+        ROUTES = [
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z),
+            'issue_notes',
+            'issue_tracking'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z),
+            'merge_request_notes',
+            'code_review'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z),
+            'issue_title',
+            'issue_tracking'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z),
+            'commit_pipelines',
+            'continuous_integration'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z),
+            'new_merge_request_pipelines',
+            'continuous_integration'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z),
+            'merge_request_pipelines',
+            'continuous_integration'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z),
+            'project_pipelines',
+            'continuous_integration'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z),
+            'project_pipeline',
+            'continuous_integration'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z),
+            'project_build',
+            'continuous_integration'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z),
+            'cluster_environments',
+            'continuous_delivery'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z),
+            'environments',
+            'continuous_delivery'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z),
+            'realtime_changes_import_github',
+            'importers'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z),
+            'realtime_changes_import_gitea',
+            'importers'
+          ],
+          [
+            %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z),
+            'merge_request_widget',
+            'code_review'
+          ]
+        ].map(&method(:build_route)).freeze
+
+        # Overridden in EE to add more routes
+        def self.all_routes
+          ROUTES
+        end
+
+        def self.match(request)
+          all_routes.find { |route| route.match(request.path_info) }
+        end
+
+        def self.cache_key(request)
+          request.path
+        end
+      end
+    end
+  end
+end
+
+Gitlab::EtagCaching::Router::Restful.prepend_if_ee('EE::Gitlab::EtagCaching::Router::Restful')
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
index 1d2f0d7bbf4..d0d790a7c72 100644
--- a/lib/gitlab/etag_caching/store.rb
+++ b/lib/gitlab/etag_caching/store.rb
@@ -3,6 +3,8 @@
 module Gitlab
   module EtagCaching
     class Store
+      InvalidKeyError = Class.new(StandardError)
+
       EXPIRY_TIME = 20.minutes
       SHARED_STATE_NAMESPACE = 'etag:'
 
@@ -27,9 +29,28 @@ module Gitlab
       end
 
       def redis_shared_state_key(key)
-        raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key)
+        raise InvalidKeyError, "#{key} is invalid" unless valid_key?(key)
 
         "#{SHARED_STATE_NAMESPACE}#{key}"
+      rescue InvalidKeyError => e
+        Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+      end
+
+      def valid_key?(key)
+        return true if skip_validation?
+
+        path, header = key.split(':', 2)
+        env = {
+          'PATH_INFO' => path,
+          'HTTP_X_GITLAB_GRAPHQL_RESOURCE_ETAG' => header
+        }
+
+        fake_request = ActionDispatch::Request.new(env)
+        !!Gitlab::EtagCaching::Router.match(fake_request)
+      end
+
+      def skip_validation?
+        Rails.env.production?
       end
     end
   end