Add latest changes from gitlab-org/gitlab@master

6 files changed, 79 insertions, 7 deletions

diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 65800e40d6c..f8047ed9b78 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -273,6 +273,18 @@ class ApplicationSetting < ApplicationRecord
                             greater_than_or_equal_to: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES,
                             less_than_or_equal_to: Gitlab::Git::Diff::MAX_PATCH_BYTES_UPPER_BOUND }
 
+  validates :diff_max_files,
+            presence: true,
+            numericality: { only_integer: true,
+                            greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_FILES_SETTING,
+                            less_than_or_equal_to: Commit::MAX_DIFF_FILES_SETTING_UPPER_BOUND }
+
+  validates :diff_max_lines,
+            presence: true,
+            numericality: { only_integer: true,
+                            greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_LINES_SETTING,
+                            less_than_or_equal_to: Commit::MAX_DIFF_LINES_SETTING_UPPER_BOUND }
+
   validates :user_default_internal_regex, js_regex: true, allow_nil: true
 
   validates :personal_access_token_prefix,
diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb
index bf9df3b9efc..b613e698471 100644
--- a/app/models/application_setting_implementation.rb
+++ b/app/models/application_setting_implementation.rb
@@ -60,6 +60,8 @@ module ApplicationSettingImplementation
         default_projects_limit: Settings.gitlab['default_projects_limit'],
         default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES,
+        diff_max_files: Commit::DEFAULT_MAX_DIFF_FILES_SETTING,
+        diff_max_lines: Commit::DEFAULT_MAX_DIFF_LINES_SETTING,
         disable_feed_token: false,
         disabled_oauth_sign_in_sources: [],
         dns_rebinding_protection_enabled: true,
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 23c1dffcc63..a1ed5eb9ab9 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -33,6 +33,12 @@ class Commit
   # Used by GFM to match and present link extensions on node texts and hrefs.
   LINK_EXTENSION_PATTERN = /(patch)/.freeze
 
+  DEFAULT_MAX_DIFF_LINES_SETTING = 50_000
+  DEFAULT_MAX_DIFF_FILES_SETTING = 1_000
+  MAX_DIFF_LINES_SETTING_UPPER_BOUND = 100_000
+  MAX_DIFF_FILES_SETTING_UPPER_BOUND = 3_000
+  DIFF_SAFE_LIMIT_FACTOR = 10
+
   cache_markdown_field :title, pipeline: :single_line
   cache_markdown_field :full_title, pipeline: :single_line, limit: 1.kilobyte
   cache_markdown_field :description, pipeline: :commit_description, limit: 1.megabyte
@@ -78,20 +84,24 @@ class Commit
     end
 
     def diff_safe_lines(project: nil)
-      Gitlab::Git::DiffCollection.default_limits(project: project)[:max_lines]
+      diff_safe_max_lines(project: project)
     end
 
-    def diff_hard_limit_files(project: nil)
+    def diff_max_files(project: nil)
       if Feature.enabled?(:increased_diff_limits, project)
         3000
+      elsif Feature.enabled?(:configurable_diff_limits, project)
+        Gitlab::CurrentSettings.diff_max_files
       else
         1000
       end
     end
 
-    def diff_hard_limit_lines(project: nil)
+    def diff_max_lines(project: nil)
       if Feature.enabled?(:increased_diff_limits, project)
         100000
+      elsif Feature.enabled?(:configurable_diff_limits, project)
+        Gitlab::CurrentSettings.diff_max_lines
       else
         50000
       end
@@ -99,11 +109,19 @@ class Commit
 
     def max_diff_options(project: nil)
       {
-        max_files: diff_hard_limit_files(project: project),
-        max_lines: diff_hard_limit_lines(project: project)
+        max_files: diff_max_files(project: project),
+        max_lines: diff_max_lines(project: project)
       }
     end
 
+    def diff_safe_max_files(project: nil)
+      diff_max_files(project: project) / DIFF_SAFE_LIMIT_FACTOR
+    end
+
+    def diff_safe_max_lines(project: nil)
+      diff_max_lines(project: project) / DIFF_SAFE_LIMIT_FACTOR
+    end
+
     def from_hash(hash, container)
       raw_commit = Gitlab::Git::Commit.new(container.repository.raw, hash)
       new(raw_commit, container)
diff --git a/app/models/group.rb b/app/models/group.rb
index 6c04bbdb032..1ce2fc18979 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -647,13 +647,17 @@ class Group < Namespace
   end
 
   def export_file_exists?
-    export_file&.file
+    import_export_upload&.export_file_exists?
   end
 
   def export_file
     import_export_upload&.export_file
   end
 
+  def export_archive_exists?
+    import_export_upload&.export_archive_exists?
+  end
+
   def adjourned_deletion?
     false
   end
diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb
index fce4ef40902..bc363cce8dd 100644
--- a/app/models/import_export_upload.rb
+++ b/app/models/import_export_upload.rb
@@ -11,10 +11,42 @@ class ImportExportUpload < ApplicationRecord
   mount_uploader :import_file, ImportExportUploader
   mount_uploader :export_file, ImportExportUploader
 
+  # This causes CarrierWave v1 and v3 (but not v2) to upload the file to
+  # object storage *after* the database entry has been committed to the
+  # database. This avoids idling in a transaction.
+  if Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_STORE_EXPORT_FILE_AFTER_COMMIT', true))
+    skip_callback :save, :after, :store_export_file!
+    set_callback :commit, :after, :store_export_file!
+  end
+
   scope :updated_before, ->(date) { where('updated_at < ?', date) }
   scope :with_export_file, -> { where.not(export_file: nil) }
 
   def retrieve_upload(_identifier, paths)
     Upload.find_by(model: self, path: paths)
   end
+
+  def export_file_exists?
+    !!carrierwave_export_file
+  end
+
+  # This checks if the export archive is actually stored on disk. It
+  # requires a HEAD request if object storage is used.
+  def export_archive_exists?
+    !!carrierwave_export_file&.exists?
+  # Handle any HTTP unexpected error
+  # https://github.com/excon/excon/blob/bbb5bd791d0bb2251593b80e3bce98dbec6e8f24/lib/excon/error.rb#L129-L169
+  rescue Excon::Error => e
+    # The HEAD request will fail with a 403 Forbidden if the file does not
+    # exist, and the user does not have permission to list the object
+    # storage bucket.
+    Gitlab::ErrorTracking.track_exception(e)
+    false
+  end
+
+  private
+
+  def carrierwave_export_file
+    export_file&.file
+  end
 end
diff --git a/app/models/project.rb b/app/models/project.rb
index 3a89a85d65d..a28389c359f 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1987,7 +1987,11 @@ class Project < ApplicationRecord
   end
 
   def export_file_exists?
-    export_file&.file
+    import_export_upload&.export_file_exists?
+  end
+
+  def export_archive_exists?
+    import_export_upload&.export_archive_exists?
   end
 
   def export_file