Add latest changes from gitlab-org/gitlab@master

5 files changed, 2 insertions, 485 deletions

diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
deleted file mode 100644
index 43698b7955f..00000000000
--- a/lib/gitlab/background_migration/populate_untracked_uploads.rb
+++ /dev/null
@@ -1,111 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # This class processes a batch of rows in `untracked_files_for_uploads` by
-    # adding each file to the `uploads` table if it does not exist.
-    class PopulateUntrackedUploads
-      def perform(start_id, end_id)
-        return unless migrate?
-
-        files = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.where(id: start_id..end_id)
-        processed_files = insert_uploads_if_needed(files)
-        processed_files.delete_all
-
-        drop_temp_table_if_finished
-      end
-
-      private
-
-      def migrate?
-        Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.table_exists? &&
-          Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.table_exists?
-      end
-
-      def insert_uploads_if_needed(files)
-        filtered_files, error_files = filter_error_files(files)
-        filtered_files = filter_existing_uploads(filtered_files)
-        filtered_files = filter_deleted_models(filtered_files)
-        insert(filtered_files)
-
-        processed_files = files.where.not(id: error_files.map(&:id))
-        processed_files
-      end
-
-      def filter_error_files(files)
-        files.partition do |file|
-          file.to_h
-          true
-        rescue => e
-          msg = <<~MSG
-              Error parsing path "#{file.path}":
-                #{e.message}
-                #{e.backtrace.join("\n  ")}
-            MSG
-          Rails.logger.error(msg) # rubocop:disable Gitlab/RailsLogger
-          false
-        end
-      end
-
-      def filter_existing_uploads(files)
-        paths = files.map(&:upload_path)
-        existing_paths = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.where(path: paths).pluck(:path).to_set
-
-        files.reject do |file|
-          existing_paths.include?(file.upload_path)
-        end
-      end
-
-      # There are files on disk that are not in the uploads table because their
-      # model was deleted, and we don't delete the files on disk.
-      def filter_deleted_models(files)
-        ids = deleted_model_ids(files)
-
-        files.reject do |file|
-          ids[file.model_type].include?(file.model_id)
-        end
-      end
-
-      def deleted_model_ids(files)
-        ids = {
-          'Appearance' => [],
-          'Namespace' => [],
-          'Note' => [],
-          'Project' => [],
-          'User' => []
-        }
-
-        # group model IDs by model type
-        files.each do |file|
-          ids[file.model_type] << file.model_id
-        end
-
-        ids.each do |model_type, model_ids|
-          model_class = "Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::#{model_type}".constantize
-          found_ids = model_class.where(id: model_ids.uniq).pluck(:id)
-          deleted_ids = ids[model_type] - found_ids
-          ids[model_type] = deleted_ids
-        end
-
-        ids
-      end
-
-      def insert(files)
-        rows = files.map do |file|
-          file.to_h.merge(created_at: 'NOW()')
-        end
-
-        Gitlab::Database.bulk_insert('uploads', # rubocop:disable Gitlab/BulkInsert
-                                     rows,
-                                     disable_quote: :created_at)
-      end
-
-      def drop_temp_table_if_finished
-        if Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup
-          Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
-                                              if_exists: true)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb
deleted file mode 100644
index 23e8be4a9ab..00000000000
--- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb
+++ /dev/null
@@ -1,190 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
-  module BackgroundMigration
-    module PopulateUntrackedUploadsDependencies
-      # This class is responsible for producing the attributes necessary to
-      # track an uploaded file in the `uploads` table.
-      class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength
-        self.table_name = 'untracked_files_for_uploads'
-
-        # Ends with /:random_hex/:filename
-        FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}.freeze
-        FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/.freeze
-
-        # These regex patterns are tested against a relative path, relative to
-        # the upload directory.
-        # For convenience, if there exists a capture group in the pattern, then
-        # it indicates the model_id.
-        PATH_PATTERNS = [
-          {
-            pattern: %r{\A-/system/appearance/logo/(\d+)/},
-            uploader: 'AttachmentUploader',
-            model_type: 'Appearance'
-          },
-          {
-            pattern: %r{\A-/system/appearance/header_logo/(\d+)/},
-            uploader: 'AttachmentUploader',
-            model_type: 'Appearance'
-          },
-          {
-            pattern: %r{\A-/system/note/attachment/(\d+)/},
-            uploader: 'AttachmentUploader',
-            model_type: 'Note'
-          },
-          {
-            pattern: %r{\A-/system/user/avatar/(\d+)/},
-            uploader: 'AvatarUploader',
-            model_type: 'User'
-          },
-          {
-            pattern: %r{\A-/system/group/avatar/(\d+)/},
-            uploader: 'AvatarUploader',
-            model_type: 'Namespace'
-          },
-          {
-            pattern: %r{\A-/system/project/avatar/(\d+)/},
-            uploader: 'AvatarUploader',
-            model_type: 'Project'
-          },
-          {
-            pattern: FILE_UPLOADER_PATH,
-            uploader: 'FileUploader',
-            model_type: 'Project'
-          }
-        ].freeze
-
-        def to_h
-          @upload_hash ||= {
-            path: upload_path,
-            uploader: uploader,
-            model_type: model_type,
-            model_id: model_id,
-            size: file_size,
-            checksum: checksum
-          }
-        end
-
-        def upload_path
-          # UntrackedFile#path is absolute, but Upload#path depends on uploader
-          @upload_path ||=
-            if uploader == 'FileUploader'
-              # Path relative to project directory in uploads
-              matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH)
-              matchd[0].sub(%r{\A/}, '') # remove leading slash
-            else
-              path
-            end
-        end
-
-        def uploader
-          matching_pattern_map[:uploader]
-        end
-
-        def model_type
-          matching_pattern_map[:model_type]
-        end
-
-        def model_id
-          return @model_id if defined?(@model_id)
-
-          pattern = matching_pattern_map[:pattern]
-          matchd = path_relative_to_upload_dir.match(pattern)
-
-          # If something is captured (matchd[1] is not nil), it is a model_id
-          # Only the FileUploader pattern will not match an ID
-          @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id
-        end
-
-        def file_size
-          File.size(absolute_path)
-        end
-
-        def checksum
-          Digest::SHA256.file(absolute_path).hexdigest
-        end
-
-        private
-
-        def matching_pattern_map
-          @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map|
-            path_relative_to_upload_dir.match(path_pattern_map[:pattern])
-          end
-
-          unless @matching_pattern_map
-            raise "Unknown upload path pattern \"#{path}\""
-          end
-
-          @matching_pattern_map
-        end
-
-        def file_uploader_model_id
-          matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE)
-          not_found_msg = <<~MSG
-            Could not capture project full_path from a FileUploader path:
-              "#{path_relative_to_upload_dir}"
-          MSG
-          raise not_found_msg unless matchd
-
-          full_path = matchd[1]
-          project = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Project.find_by_full_path(full_path)
-          return unless project
-
-          project.id
-        end
-
-        # Not including a leading slash
-        def path_relative_to_upload_dir
-          upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR
-          base = %r{\A#{Regexp.escape(upload_dir)}/}
-          @path_relative_to_upload_dir ||= path.sub(base, '')
-        end
-
-        def absolute_path
-          File.join(Gitlab.config.uploads.storage_path, path)
-        end
-      end
-
-      # Avoid using application code
-      class Upload < ActiveRecord::Base
-        self.table_name = 'uploads'
-      end
-
-      # Avoid using application code
-      class Appearance < ActiveRecord::Base
-        self.table_name = 'appearances'
-      end
-
-      # Avoid using application code
-      class Namespace < ActiveRecord::Base
-        self.table_name = 'namespaces'
-      end
-
-      # Avoid using application code
-      class Note < ActiveRecord::Base
-        self.table_name = 'notes'
-      end
-
-      # Avoid using application code
-      class User < ActiveRecord::Base
-        self.table_name = 'users'
-      end
-
-      # Since project Markdown upload paths don't contain the project ID, we have to find the
-      # project by its full_path. Due to MySQL/PostgreSQL differences, and historical reasons,
-      # the logic is somewhat complex, so I've mostly copied it in here.
-      class Project < ActiveRecord::Base
-        self.table_name = 'projects'
-
-        def self.find_by_full_path(path)
-          order_sql = Arel.sql("(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)")
-          where_full_path_in(path).reorder(order_sql).take
-        end
-
-        def self.where_full_path_in(path)
-          where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))"
-          joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
deleted file mode 100644
index 3d943205783..00000000000
--- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb
+++ /dev/null
@@ -1,173 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module BackgroundMigration
-    # This class finds all non-hashed uploaded file paths and saves them to a
-    # `untracked_files_for_uploads` table.
-    class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength
-      # For bulk_queue_background_migration_jobs_by_range
-      include Database::MigrationHelpers
-      include ::Gitlab::Utils::StrongMemoize
-
-      FIND_BATCH_SIZE = 500
-      RELATIVE_UPLOAD_DIR = "uploads"
-      ABSOLUTE_UPLOAD_DIR = File.join(
-        Gitlab.config.uploads.storage_path,
-        RELATIVE_UPLOAD_DIR
-      )
-      FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'
-      START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}.freeze
-      EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*"
-      EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*"
-
-      # This class is used to iterate over batches of
-      # `untracked_files_for_uploads` rows.
-      class UntrackedFile < ActiveRecord::Base
-        include EachBatch
-
-        self.table_name = 'untracked_files_for_uploads'
-      end
-
-      def perform
-        ensure_temporary_tracking_table_exists
-
-        # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since
-        # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be
-        # slow, start with an empty table for Postgres < 9.5.
-        # That way we can do bulk inserts at ~30x the speed of individual
-        # inserts (~20 minutes worth of inserts at GitLab.com scale instead of
-        # ~10 hours).
-        # In all other cases, installations will get both bulk inserts and the
-        # ability for these jobs to retry without having to clear and reinsert.
-        clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates?
-
-        store_untracked_file_paths
-
-        if UntrackedFile.all.empty?
-          drop_temp_table
-        else
-          schedule_populate_untracked_uploads_jobs
-        end
-      end
-
-      private
-
-      def ensure_temporary_tracking_table_exists
-        table_name = :untracked_files_for_uploads
-
-        unless ActiveRecord::Base.connection.table_exists?(table_name)
-          UntrackedFile.connection.create_table table_name do |t|
-            t.string :path, limit: 600, null: false
-            t.index :path, unique: true
-          end
-        end
-      end
-
-      def clear_untracked_file_paths
-        UntrackedFile.delete_all
-      end
-
-      def store_untracked_file_paths
-        return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR)
-
-        each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths|
-          insert_file_paths(file_paths)
-        end
-      end
-
-      def each_file_batch(search_dir, batch_size, &block)
-        cmd = build_find_command(search_dir)
-
-        Open3.popen2(*cmd) do |stdin, stdout, status_thread|
-          yield_paths_in_batches(stdout, batch_size, &block)
-
-          raise "Find command failed" unless status_thread.value.success?
-        end
-      end
-
-      def yield_paths_in_batches(stdout, batch_size, &block)
-        paths = []
-
-        stdout.each_line("\0") do |line|
-          paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '')
-
-          if paths.size >= batch_size
-            yield(paths)
-            paths = []
-          end
-        end
-
-        yield(paths) if paths.any?
-      end
-
-      def build_find_command(search_dir)
-        cmd = %W[find -L #{search_dir}
-                 -type f
-                 ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune )
-                 ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune )
-                 -print0]
-
-        ionice = which_ionice
-        cmd = %W[#{ionice} -c Idle] + cmd if ionice
-
-        log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\""
-        Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger
-
-        cmd
-      end
-
-      def which_ionice
-        Gitlab::Utils.which('ionice')
-      rescue StandardError
-        # In this case, returning false is relatively safe,
-        # even though it isn't very nice
-        false
-      end
-
-      def insert_file_paths(file_paths)
-        sql = insert_sql(file_paths)
-
-        ActiveRecord::Base.connection.execute(sql)
-      end
-
-      def insert_sql(file_paths)
-        if postgresql_pre_9_5?
-          "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};"
-        else
-          "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\
-            " ON CONFLICT DO NOTHING;"
-        end
-      end
-
-      def table_columns_and_values_for_insert(file_paths)
-        values = file_paths.map do |file_path|
-          ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend
-        end.join(', ')
-
-        "#{UntrackedFile.table_name} (path) VALUES #{values}"
-      end
-
-      def can_bulk_insert_and_ignore_duplicates?
-        !postgresql_pre_9_5?
-      end
-
-      def postgresql_pre_9_5?
-        strong_memoize(:postgresql_pre_9_5) do
-          Gitlab::Database.version.to_f < 9.5
-        end
-      end
-
-      def schedule_populate_untracked_uploads_jobs
-        bulk_queue_background_migration_jobs_by_range(
-          UntrackedFile, FOLLOW_UP_MIGRATION)
-      end
-
-      def drop_temp_table
-        unless Rails.env.test? # Dropping a table intermittently breaks test cleanup
-          UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
-                                              if_exists: true)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb
index abb05ba56a7..f9a635734a3 100644
--- a/lib/gitlab/suggestions/suggestion_set.rb
+++ b/lib/gitlab/suggestions/suggestion_set.rb
@@ -83,7 +83,7 @@ module Gitlab
         end
 
         unless suggestion.appliable?(cached: false)
-          return _('A suggestion is not applicable.')
+          return suggestion.inapplicable_reason(cached: false)
         end
 
         unless latest_source_head?(suggestion)
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 6e703498abf..e3055ebc7bc 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -261,19 +261,10 @@ module Gitlab
           database: {
             adapter: alt_usage_data { Gitlab::Database.adapter_name },
             version: alt_usage_data { Gitlab::Database.version }
-          },
-          app_server: { type: app_server_type }
+          }
         }
       end
 
-      def app_server_type
-        Gitlab::Runtime.identify.to_s
-      rescue Gitlab::Runtime::IdentificationError => e
-        Gitlab::AppLogger.error(e.message)
-        Gitlab::ErrorTracking.track_exception(e)
-        'unknown_app_server_type'
-      end
-
       def object_store_config(component)
         config = alt_usage_data(fallback: nil) do
           Settings[component]['object_store']