Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

10 files changed, 118 insertions, 77 deletions

diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index ed74dd472ff..cc10d73f76a 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -91,6 +91,8 @@ namespace :gitlab do
         backup.cleanup
       end
 
+      backup.remove_tmp
+
       puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
            "and are not included in this backup. You will need to restore these files manually.".color(:red)
       puts "Restore task is done."
@@ -297,7 +299,7 @@ namespace :gitlab do
     end
 
     def repository_backup_strategy
-      if Feature.enabled?(:gitaly_backup)
+      if Feature.enabled?(:gitaly_backup, default_enabled: :yaml)
         max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
         max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
         Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency)
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 2b508b341dd..51f15f5a56a 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -161,7 +161,7 @@ namespace :gitlab do
         exit
       end
 
-      indexes = Gitlab::Database::Reindexing.candidate_indexes
+      indexes = Gitlab::Database::PostgresIndex.reindexing_support
 
       if identifier = args[:index_name]
         raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
@@ -173,6 +173,12 @@ namespace :gitlab do
 
       ActiveRecord::Base.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false)
 
+      # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any)
+      Gitlab::Database::Reindexing.cleanup_leftovers!
+
+      # Hack: Before we do actual reindexing work, create async indexes
+      Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
+
       Gitlab::Database::Reindexing.perform(indexes)
     rescue StandardError => e
       Gitlab::AppLogger.error(e)
@@ -217,7 +223,7 @@ namespace :gitlab do
       instrumentation = Gitlab::Database::Migrations::Instrumentation.new
 
       pending_migrations.each do |migration|
-        instrumentation.observe(migration.version) do
+        instrumentation.observe(version: migration.version, name: migration.name) do
           ActiveRecord::Migrator.new(:up, ctx.migrations, ctx.schema_migration, migration.version).run
         end
       end
diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake
index 990ff723eeb..123a4775605 100644
--- a/lib/tasks/gitlab/docs/redirect.rake
+++ b/lib/tasks/gitlab/docs/redirect.rake
@@ -57,68 +57,5 @@ namespace :gitlab do
         post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->"
       end
     end
-
-    desc 'GitLab | Docs | Clean up old redirects'
-    task :clean_redirects do
-      #
-      # Calculate new path from the redirect URL.
-      #
-      # If the redirect is not a full URL:
-      #   1. Create a new Pathname of the file
-      #   2. Use dirname to get all but the last component of the path
-      #   3. Join with the redirect_to entry
-      #   4. Substitute:
-      #      - '.md' => '.html'
-      #      - 'doc/' => '/ee/'
-      #
-      # If the redirect URL is a full URL pointing to the Docs site
-      # (cross-linking among the 4 products), remove the FQDN prefix:
-      #
-      #   From : https://docs.gitlab.com/ee/install/requirements.html
-      #   To   : /ee/install/requirements.html
-      #
-      def new_path(redirect, filename)
-        if !redirect.start_with?('http')
-          Pathname.new(filename).dirname.join(redirect).to_s.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/')
-        elsif redirect.start_with?('https://docs.gitlab.com')
-          redirect.gsub('https://docs.gitlab.com', '')
-        else
-          redirect
-        end
-      end
-
-      today = Time.now.utc.to_date
-
-      #
-      # Find the files to be deleted.
-      # Exclude 'doc/development/documentation/index.md' because it
-      # contains an example of the YAML front matter.
-      #
-      files_to_be_deleted = `grep -Ir 'remove_date:' doc | grep -v doc/development/documentation/index.md | cut -d ":" -f 1`.split("\n")
-
-      #
-      # Iterate over the files to be deleted and print the needed
-      # YAML entries for the Docs site redirects.
-      #
-      files_to_be_deleted.each do |filename|
-        frontmatter = YAML.safe_load(File.read(filename))
-        remove_date = Date.parse(frontmatter['remove_date'])
-        old_path = filename.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/')
-
-        #
-        # Check if the removal date is before today, and delete the file and
-        # print the content to be pasted in
-        # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/content/_data/redirects.yaml.
-        # The remove_date of redirects.yaml should be nine months in the future.
-        # To not be confused with the remove_date of the Markdown page.
-        #
-        next unless remove_date < today
-
-        File.delete(filename) if File.exist?(filename)
-        puts "  - from: #{old_path}"
-        puts "    to: #{new_path(frontmatter['redirect_to'], filename)}"
-        puts "    remove_date: #{remove_date >> 9}"
-      end
-    end
   end
 end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index df75b3cf716..6675439e430 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -2,6 +2,42 @@
 
 namespace :gitlab do
   namespace :gitaly do
+    desc 'Installs gitaly for running tests within gitlab-development-kit'
+    task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
+      inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT'))
+
+      if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk
+        Rake::Task["gitlab:gitaly:install"].invoke(*args)
+
+        next
+      end
+
+      gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly'))
+
+      # Our test setup expects a git repo, so clone rather than copy
+      version = Gitlab::GitalyClient.expected_server_version
+      checkout_or_clone_version(version: version, repo: gdk_gitaly_dir, target_dir: args.dir, clone_opts: %w[--depth 1])
+
+      # We assume the GDK gitaly already compiled binaries
+      build_dir = File.join(gdk_gitaly_dir, '_build')
+      FileUtils.cp_r(build_dir, args.dir)
+
+      # We assume the GDK gitaly already ran bundle install
+      bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle')
+      FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby'))
+
+      # For completeness we copy this for gitaly's make target
+      ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle')
+      FileUtils.cp_r(ruby_bundle_file, args.dir)
+
+      gitaly_binary = File.join(build_dir, 'bin', 'gitaly')
+      warn_gitaly_out_of_date!(gitaly_binary, version)
+    rescue Errno::ENOENT => e
+      puts "Could not copy files, did you run `gdk update`? Error: #{e.message}"
+
+      raise
+    end
+
     desc 'GitLab | Gitaly | Install or upgrade gitaly'
     task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
       warn_user_is_not_gitlab
@@ -41,5 +77,24 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
       _, status = Gitlab::Popen.popen(%w[which gmake])
       status == 0 ? 'gmake' : 'make'
     end
+
+    def warn_gitaly_out_of_date!(gitaly_binary, expected_version)
+      binary_version, exit_status = Gitlab::Popen.popen(%W[#{gitaly_binary} -version])
+
+      raise "Failed to run `#{gitaly_binary} -version`" unless exit_status == 0
+
+      binary_version = binary_version.strip
+
+      # See help for `git describe` for format
+      git_describe_sha = /g([a-f0-9]{5,40})\z/
+      match = binary_version.match(git_describe_sha)
+
+      # Just skip if the version does not have a sha component
+      return unless match
+
+      return if expected_version.start_with?(match[1])
+
+      puts "WARNING: #{binary_version.strip} does not exactly match repository version #{expected_version}"
+    end
   end
 end
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index b405cbd3f68..52c5c680292 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -56,9 +56,9 @@ namespace :gitlab do
           color = case complexity
                   when 0..GitlabSchema::DEFAULT_MAX_COMPLEXITY
                     :green
-                  when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_COMPLEXITY
+                  when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY
                     :yellow
-                  when GitlabSchema::AUTHENTICATED_COMPLEXITY..GitlabSchema::ADMIN_COMPLEXITY
+                  when GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY..GitlabSchema::ADMIN_MAX_COMPLEXITY
                     :orange
                   else
                     :red
diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake
index 2826002bdc2..68395d10d24 100644
--- a/lib/tasks/gitlab/info.rake
+++ b/lib/tasks/gitlab/info.rake
@@ -68,8 +68,8 @@ namespace :gitlab do
       puts "Version:\t#{Gitlab::VERSION}"
       puts "Revision:\t#{Gitlab.revision}"
       puts "Directory:\t#{Rails.root}"
-      puts "DB Adapter:\t#{Gitlab::Database.human_adapter_name}"
-      puts "DB Version:\t#{Gitlab::Database.version}"
+      puts "DB Adapter:\t#{Gitlab::Database.main.human_adapter_name}"
+      puts "DB Version:\t#{Gitlab::Database.main.version}"
       puts "URL:\t\t#{Gitlab.config.gitlab.url}"
       puts "HTTP Clone URL:\t#{http_clone_url}"
       puts "SSH Clone URL:\t#{ssh_clone_url}"
diff --git a/lib/tasks/gitlab/product_intelligence.rake b/lib/tasks/gitlab/product_intelligence.rake
new file mode 100644
index 00000000000..329cd9c8c2a
--- /dev/null
+++ b/lib/tasks/gitlab/product_intelligence.rake
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :product_intelligence do
+    # @example
+    #   bundle exec rake gitlab:product_intelligence:activate_metrics MILESTONE=14.0
+
+    desc 'GitLab | Product Intelligence | Update milestone metrics status to data_available'
+    task activate_metrics: :environment do
+      milestone = ENV['MILESTONE']
+      raise "Please supply the MILESTONE env var".color(:red) unless milestone.present?
+
+      Gitlab::Usage::MetricDefinition.definitions.values.each do |metric|
+        next if metric.attributes[:milestone] != milestone || metric.attributes[:status] != 'implemented'
+
+        metric.attributes[:status] = 'data_available'
+        path = metric.path
+        File.open(path, "w") { |file| file << metric.to_h.deep_stringify_keys.to_yaml }
+      end
+
+      puts "Task completed successfully"
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/smtp.rake b/lib/tasks/gitlab/smtp.rake
new file mode 100644
index 00000000000..23ad7577e3c
--- /dev/null
+++ b/lib/tasks/gitlab/smtp.rake
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :smtp do
+    namespace :secret do
+      desc 'GitLab | SMTP | Secret | Write SMTP secrets'
+      task write: [:environment] do
+        content = $stdin.tty? ? $stdin.gets : $stdin.read
+        Gitlab::EncryptedSmtpCommand.write(content)
+      end
+
+      desc 'GitLab | SMTP | Secret | Edit SMTP secrets'
+      task edit: [:environment] do
+        Gitlab::EncryptedSmtpCommand.edit
+      end
+
+      desc 'GitLab | SMTP | Secret | Show SMTP secrets'
+      task show: [:environment] do
+        Gitlab::EncryptedSmtpCommand.show
+      end
+    end
+  end
+end
diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake
index 6fa39a26488..fb9f9b9fe67 100644
--- a/lib/tasks/gitlab/storage.rake
+++ b/lib/tasks/gitlab/storage.rake
@@ -170,7 +170,7 @@ namespace :gitlab do
       inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f
 
       attempts.to_i.times do
-        unless Gitlab::Database.exists?
+        unless Gitlab::Database.main.exists?
           puts "Waiting until database is ready before continuing...".color(:yellow)
           sleep inverval
         end
diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake
index 166f08ef16a..ddd3424acda 100644
--- a/lib/tasks/gitlab/usage_data.rake
+++ b/lib/tasks/gitlab/usage_data.rake
@@ -24,12 +24,6 @@ namespace :gitlab do
       puts Gitlab::Json.pretty_generate(result.attributes)
     end
 
-    desc 'GitLab | UsageData | Generate metrics dictionary'
-    task generate_metrics_dictionary: :environment do
-      items = Gitlab::Usage::MetricDefinition.definitions
-      Gitlab::Usage::Docs::Renderer.new(items).write
-    end
-
     desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON'
     task generate_from_yaml: :environment do
       puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data)