Add latest changes from gitlab-org/gitlab@master

8 files changed, 183 insertions, 33 deletions

diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index 5cfb65e1fbb..eae0ac96026 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -34,12 +34,12 @@ module API
             if runner_registration_token_valid?
               # Create shared runner. Requires admin access
               attributes.merge(runner_type: :instance_type)
-            elsif project = Project.find_by_runners_token(params[:token])
+            elsif @project = Project.find_by_runners_token(params[:token])
               # Create a specific runner for the project
-              attributes.merge(runner_type: :project_type, projects: [project])
-            elsif group = Group.find_by_runners_token(params[:token])
+              attributes.merge(runner_type: :project_type, projects: [@project])
+            elsif @group = Group.find_by_runners_token(params[:token])
               # Create a specific runner for the group
-              attributes.merge(runner_type: :group_type, groups: [group])
+              attributes.merge(runner_type: :group_type, groups: [@group])
             else
               forbidden!
             end
@@ -81,12 +81,7 @@ module API
       end
 
       resource :jobs do
-        before do
-          Gitlab::ApplicationContext.push(
-            user: -> { current_job&.user },
-            project: -> { current_job&.project }
-          )
-        end
+        before { set_application_context }
 
         desc 'Request a job' do
           success Entities::JobRequest::Response
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 1c85669a626..43d841b4d82 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -71,6 +71,26 @@ module API
         header 'Job-Status', job.status
         forbidden!(reason)
       end
+
+      def set_application_context
+        if current_job
+          Gitlab::ApplicationContext.push(
+            user: -> { current_job.user },
+            project: -> { current_job.project }
+          )
+        elsif current_runner&.project_type?
+          Gitlab::ApplicationContext.push(
+            project: -> do
+              projects = current_runner.projects.limit(2) # rubocop: disable CodeReuse/ActiveRecord
+              projects.first if projects.length == 1
+            end
+          )
+        elsif current_runner&.group_type?
+          Gitlab::ApplicationContext.push(
+            namespace: -> { current_runner.groups.first }
+          )
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
index d981f263c5e..207f03abb26 100644
--- a/lib/gitlab/cache/ci/project_pipeline_status.rb
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -9,6 +9,8 @@ module Gitlab
       class ProjectPipelineStatus
         include Gitlab::Utils::StrongMemoize
 
+        ALL_PIPELINES_STATUS_PATTERN = 'projects/*/pipeline_status'
+
         attr_accessor :sha, :status, :ref, :project, :loaded
 
         def self.load_for_project(project)
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index c089eed2ea2..2333e8b9d5b 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -52,7 +52,7 @@ module Gitlab
             return unless included?
 
             strong_memoize(:errors) do
-              needs_errors
+              [needs_errors, variable_expansion_errors].compact.flatten
             end
           end
 
@@ -153,6 +153,12 @@ module Gitlab
             @pipeline.project.actual_limits.ci_needs_size_limit
           end
 
+          def variable_expansion_errors
+            sorted_collection = evaluate_context.variables.sorted_collection(@pipeline.project)
+            errors = sorted_collection.errors
+            ["#{name}: #{errors}"] if errors
+          end
+
           def pipeline_attributes
             {
               pipeline: @pipeline,
diff --git a/lib/gitlab/cleanup/redis/batch_delete_by_pattern.rb b/lib/gitlab/cleanup/redis/batch_delete_by_pattern.rb
new file mode 100644
index 00000000000..8f3b73c7b0e
--- /dev/null
+++ b/lib/gitlab/cleanup/redis/batch_delete_by_pattern.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cleanup
+    module Redis
+      class BatchDeleteByPattern
+        REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
+        REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan
+
+        attr_reader :patterns
+
+        def initialize(patterns)
+          raise ArgumentError.new('Argument should be an Array of patterns') unless patterns.is_a?(Array)
+
+          @patterns = patterns
+        end
+
+        def execute
+          return if patterns.blank?
+
+          batch_delete_cache_keys
+        end
+
+        private
+
+        def batch_delete_cache_keys
+          Gitlab::Redis::Cache.with do |redis|
+            patterns.each do |match|
+              cursor = REDIS_SCAN_START_STOP
+              loop do
+                cursor, keys = redis.scan(
+                  cursor,
+                  match: match,
+                  count: REDIS_CLEAR_BATCH_SIZE
+                )
+
+                Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
+                  redis.del(*keys) if keys.any?
+                end
+
+                break if cursor == REDIS_SCAN_START_STOP
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/cleanup/redis/description_templates_cache_keys_pattern_builder.rb b/lib/gitlab/cleanup/redis/description_templates_cache_keys_pattern_builder.rb
new file mode 100644
index 00000000000..6d16f6c1e00
--- /dev/null
+++ b/lib/gitlab/cleanup/redis/description_templates_cache_keys_pattern_builder.rb
@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Cleanup
+    module Redis
+      class DescriptionTemplatesCacheKeysPatternBuilder
+        # project_ids - a list of project_ids for which to compute description templates cache keys or `:all` to compute
+        # a pattern that cover all description templates cache keys.
+        #
+        # Example
+        # * ::Gitlab::Cleanup::Redis::BatchDeleteDescriptionTemplates.new(:all).execute - to get 2
+        # patterns for all issue and merge request description templates cache keys.
+        #
+        # * ::Gitlab::Cleanup::Redis::BatchDeleteDescriptionTemplates.new([1,2,3,4]).execute - to get an array of
+        # patterns for each project's issue and merge request description templates cache keys.
+        def initialize(project_ids)
+          raise ArgumentError.new('project_ids can either be an array of project IDs or :all') if project_ids != :all && !project_ids.is_a?(Array)
+
+          @project_ids = parse_project_ids(project_ids)
+        end
+
+        def execute
+          case project_ids
+          when :all
+            all_instance_patterns
+          else
+            project_patterns
+          end
+        end
+
+        private
+
+        attr_reader :project_ids
+
+        def parse_project_ids(project_ids)
+          return project_ids if project_ids == :all
+
+          project_ids.map { |id| Integer(id) }
+        rescue ArgumentError
+          raise ArgumentError.new('Invalid Project ID. Please ensure all passed in project ids values are valid integer project ids.')
+        end
+
+        def project_patterns
+          cache_key_patterns = []
+          Project.id_in(project_ids).each_batch do |batch|
+            cache_key_patterns << batch.map do |pr|
+              next unless pr.repository.exists?
+
+              cache = Gitlab::RepositoryCache.new(pr.repository)
+
+              [repo_issue_templates_cache_key(cache), repo_merge_request_templates_cache_key(cache)]
+            end
+          end
+
+          cache_key_patterns.flatten.compact
+        end
+
+        def all_instance_patterns
+          [all_issue_templates_cache_key, all_merge_request_templates_cache_key]
+        end
+
+        def issue_templates_cache_key
+          Repository::METHOD_CACHES_FOR_FILE_TYPES[:issue_template]
+        end
+
+        def merge_request_templates_cache_key
+          Repository::METHOD_CACHES_FOR_FILE_TYPES[:merge_request_template]
+        end
+
+        def all_issue_templates_cache_key
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{issue_templates_cache_key}:*"
+        end
+
+        def all_merge_request_templates_cache_key
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{merge_request_templates_cache_key}:*"
+        end
+
+        def repo_issue_templates_cache_key(cache)
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{cache.cache_key(issue_templates_cache_key)}"
+        end
+
+        def repo_merge_request_templates_cache_key(cache)
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{cache.cache_key(merge_request_templates_cache_key)}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 48f204e0b86..7561e36cc33 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -115,7 +115,7 @@ module Gitlab
 
           config[:storage] = storages
 
-          internal_socket_dir = File.join(gitaly_dir, 'internal_sockets')
+          internal_socket_dir = options[:internal_socket_dir] || File.join(gitaly_dir, 'internal_sockets')
           FileUtils.mkdir(internal_socket_dir) unless File.exist?(internal_socket_dir)
           config[:internal_socket_dir] = internal_socket_dir
 
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 4d698e56444..46d940255de 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -2,32 +2,22 @@
 
 namespace :cache do
   namespace :clear do
-    REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
-    REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan
-
     desc "GitLab | Cache | Clear redis cache"
     task redis: :environment do
-      Gitlab::Redis::Cache.with do |redis|
-        cache_key_pattern = %W[#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*
-                               projects/*/pipeline_status]
+      cache_key_patterns = %W[
+        #{Gitlab::Redis::Cache::CACHE_NAMESPACE}*
+        #{Gitlab::Cache::Ci::ProjectPipelineStatus::ALL_PIPELINES_STATUS_PATTERN}
+      ]
 
-        cache_key_pattern.each do |match|
-          cursor = REDIS_SCAN_START_STOP
-          loop do
-            cursor, keys = redis.scan(
-              cursor,
-              match: match,
-              count: REDIS_CLEAR_BATCH_SIZE
-            )
+      ::Gitlab::Cleanup::Redis::BatchDeleteByPattern.new(cache_key_patterns).execute
+    end
 
-            Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
-              redis.del(*keys) if keys.any?
-            end
+    desc "GitLab | Cache | Clear description templates redis cache"
+    task description_templates: :environment do
+      project_ids = Array(ENV['project_ids']&.split(',')).map!(&:squish)
 
-            break if cursor == REDIS_SCAN_START_STOP
-          end
-        end
-      end
+      cache_key_patterns = ::Gitlab::Cleanup::Redis::DescriptionTemplatesCacheKeysPatternBuilder.new(project_ids).execute
+      ::Gitlab::Cleanup::Redis::BatchDeleteByPattern.new(cache_key_patterns).execute
     end
 
     task all: [:redis]