Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42

5 files changed, 144 insertions, 16 deletions

diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake
index 8a53b51d4fe..abb15f29328 100644
--- a/lib/tasks/gitlab/git.rake
+++ b/lib/tasks/gitlab/git.rake
@@ -21,5 +21,39 @@ namespace :gitlab do
         failures.each { |f| puts "- #{f}" }
       end
     end
+
+    # Example for all projects:
+    #
+    #   $ bin/rake gitlab:git:checksum_projects
+    #   1,cfa3f06ba235c13df0bb28e079bcea62c5848af2
+    #   2,
+    #   3,3f3fb58a8106230e3a6c6b48adc2712fb3b6ef87
+    #   4,0000000000000000000000000000000000000000
+    #
+    # Example with a list of project IDs:
+    #
+    #   $ CHECKSUM_PROJECT_IDS="1,3" bin/rake gitlab:git:checksum_projects
+    #   1,cfa3f06ba235c13df0bb28e079bcea62c5848af2
+    #   3,3f3fb58a8106230e3a6c6b48adc2712fb3b6ef87
+    #
+    # - If a repository does not exist, the project ID is output with a blank checksum
+    # - If a repository exists but is empty, the output checksum is `0000000000000000000000000000000000000000`
+    # - If given specific IDs, projects which do not exist are skipped
+    desc 'GitLab | Git | Generate checksum of project repository refs'
+    task checksum_projects: :environment do
+      project_ids = ENV['CHECKSUM_PROJECT_IDS']&.split(',')
+      relation = Project
+      relation = relation.where(id: project_ids) if project_ids.present?
+
+      relation.find_each(batch_size: 100) do |project|
+        next unless project.repo_exists?
+
+        result = project.repository.checksum
+      rescue => e
+        result = "Ignored error: #{e.message}".squish.truncate(255)
+      ensure
+        puts "#{project.id},#{result}"
+      end
+    end
   end
 end
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index 5a583183924..f708114c226 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -33,6 +33,44 @@ namespace :gitlab do
   )
 
   namespace :graphql do
+    desc 'Gitlab | GraphQL | Validate queries'
+    task validate: [:environment, :enable_feature_flags] do |t, args|
+      queries = if args.to_a.present?
+                  args.to_a.flat_map { |path| Gitlab::Graphql::Queries.find(path) }
+                else
+                  Gitlab::Graphql::Queries.all
+                end
+
+      failed = queries.flat_map do |defn|
+        summary, errs = defn.validate(GitlabSchema)
+
+        case summary
+        when :client_query
+          warn("SKIP  #{defn.file}: client query")
+        else
+          warn("OK    #{defn.file}") if errs.empty?
+          errs.each do |err|
+            warn(<<~MSG)
+            ERROR #{defn.file}: #{err.message} (at #{err.path.join('.')})
+            MSG
+          end
+        end
+
+        errs.empty? ? [] : [defn.file]
+      end
+
+      if failed.present?
+        format_output(
+          "#{failed.count} GraphQL #{'query'.pluralize(failed.count)} out of #{queries.count} failed validation:",
+          *failed.map do |name|
+            known_failure = Gitlab::Graphql::Queries.known_failure?(name)
+            "- #{name}" + (known_failure ? ' (known failure)' : '')
+          end
+        )
+        abort unless failed.all? { |name| Gitlab::Graphql::Queries.known_failure?(name) }
+      end
+    end
+
     desc 'GitLab | GraphQL | Generate GraphQL docs'
     task compile_docs: [:environment, :enable_feature_flags] do
       renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
@@ -78,11 +116,11 @@ def render_options
   }
 end
 
-def format_output(str)
+def format_output(*strs)
   heading = '#' * 10
   puts heading
   puts '#'
-  puts "# #{str}"
+  strs.each { |str| puts "# #{str}" }
   puts '#'
   puts heading
 end
diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake
index ca507fb5320..cfe97984dda 100644
--- a/lib/tasks/gitlab/packages/events.rake
+++ b/lib/tasks/gitlab/packages/events.rake
@@ -5,18 +5,18 @@ namespace :gitlab do
   namespace :packages do
     namespace :events do
       task generate: :environment do
-        Rake::Task["gitlab:packages:events:generate_guest"].invoke
+        Rake::Task["gitlab:packages:events:generate_counts"].invoke
         Rake::Task["gitlab:packages:events:generate_unique"].invoke
       rescue => e
         logger.error("Error building events list: #{e}")
       end
 
-      task generate_guest: :environment do
+      task generate_counts: :environment do
         logger = Logger.new(STDOUT)
         logger.info('Building list of package events...')
 
-        path = Gitlab::UsageDataCounters::GuestPackageEventCounter::KNOWN_EVENTS_PATH
-        File.open(path, "w") { |file| file << guest_events_list.to_yaml }
+        path = Gitlab::UsageDataCounters::PackageEventCounter::KNOWN_EVENTS_PATH
+        File.open(path, "w") { |file| file << counter_events_list.to_yaml }
 
         logger.info("Events file `#{path}` generated successfully")
       rescue => e
@@ -43,26 +43,32 @@ namespace :gitlab do
 
       def generate_unique_events_list
         events = event_pairs.each_with_object([]) do |(event_type, event_scope), events|
-          Packages::Event.originator_types.keys.excluding('guest').each do |originator|
-            if name = Packages::Event.allowed_event_name(event_scope, event_type, originator)
-              events << {
-                "name" => name,
-                "category" => "#{event_scope}_packages",
+          Packages::Event.originator_types.keys.excluding('guest').each do |originator_type|
+            events_definition = Packages::Event.unique_counters_for(event_scope, event_type, originator_type).map do |event_name|
+              {
+                "name" => event_name,
+                "category" => "#{originator_type}_packages",
                 "aggregation" => "weekly",
                 "redis_slot" => "package",
                 "feature_flag" => "collect_package_events_redis"
               }
             end
+
+            events.concat(events_definition)
           end
         end
 
-        events.sort_by { |event| event["name"] }
+        events.sort_by { |event| event["name"] }.uniq
       end
 
-      def guest_events_list
-        event_pairs.map do |event_type, event_scope|
-          Packages::Event.allowed_event_name(event_scope, event_type, "guest")
-        end.compact.sort
+      def counter_events_list
+        counters = event_pairs.flat_map do |event_type, event_scope|
+          Packages::Event.originator_types.keys.flat_map do |originator_type|
+            Packages::Event.counters_for(event_scope, event_type, originator_type)
+          end
+        end
+
+        counters.compact.sort.uniq
       end
     end
   end
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
new file mode 100644
index 00000000000..e15cbb4e32e
--- /dev/null
+++ b/lib/tasks/gitlab/pages.rake
@@ -0,0 +1,35 @@
+require 'logger'
+
+namespace :gitlab do
+  namespace :pages do
+    desc "GitLab | Pages | Migrate legacy storage to zip format"
+    task migrate_legacy_storage: :gitlab_environment do
+      logger = Logger.new(STDOUT)
+      logger.info('Starting to migrate legacy pages storage to zip deployments')
+      processed_projects = 0
+
+      ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch|
+        batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum|
+          project = metadatum.project
+
+          result = nil
+          time = Benchmark.realtime do
+            result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute
+          end
+          processed_projects += 1
+
+          if result[:status] == :success
+            logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds")
+          else
+            logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time} seconds: #{result[:message]}")
+          end
+        rescue => e
+          logger.error("#{e.message} project_id: #{project&.id}")
+          Gitlab::ErrorTracking.track_exception(e, project_id: project&.id)
+        end
+
+        logger.info("#{processed_projects} pages projects are processed")
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index ccc96b7edfb..f7819fd974b 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -116,6 +116,21 @@ namespace :gitlab do
       helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository))
     end
 
+    desc 'Gitlab | Storage | Prune projects using Hashed Storage. Remove all hashed directories that do not have a project associated'
+    task prune_hashed_projects: [:environment, :gitlab_environment] do
+      if Rails.env.production?
+        abort('This destructive action may only be run in development')
+      end
+
+      helper = Gitlab::HashedStorage::RakeHelper
+      name = 'projects using Hashed Storage'
+      relation = Project.with_storage_feature(:repository)
+      root = Gitlab.config.repositories.storages['default'].legacy_disk_path
+      dry_run = !ENV['FORCE'].present?
+
+      helper.prune(name, relation, dry_run: dry_run, root: root)
+    end
+
     desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage'
     task legacy_attachments: :environment do
       helper = Gitlab::HashedStorage::RakeHelper