Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

15 files changed, 142 insertions, 63 deletions

diff --git a/app/services/projects/after_rename_service.rb b/app/services/projects/after_rename_service.rb
index 6d389035922..953b386b754 100644
--- a/app/services/projects/after_rename_service.rb
+++ b/app/services/projects/after_rename_service.rb
@@ -83,7 +83,7 @@ module Projects
 
     def update_repository_configuration
       project.reload_repository!
-      project.write_repository_config
+      project.set_full_path
       project.track_project_repository
     end
 
diff --git a/app/services/projects/cleanup_service.rb b/app/services/projects/cleanup_service.rb
index 5eafa5f9b29..75be3425029 100644
--- a/app/services/projects/cleanup_service.rb
+++ b/app/services/projects/cleanup_service.rb
@@ -65,7 +65,7 @@ module Projects
     def cleanup_diffs(response)
       old_commit_shas = extract_old_commit_shas(response.entries)
 
-      ActiveRecord::Base.transaction do
+      ApplicationRecord.transaction do
         cleanup_merge_request_diffs(old_commit_shas)
         cleanup_note_diff_files(old_commit_shas)
       end
diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb
index 9a5c260e488..302c047a65f 100644
--- a/app/services/projects/create_service.rb
+++ b/app/services/projects/create_service.rb
@@ -65,7 +65,7 @@ module Projects
 
       save_project_and_import_data
 
-      Gitlab::ApplicationContext.with_context(related_class: "Projects::CreateService", project: @project) do
+      Gitlab::ApplicationContext.with_context(project: @project) do
         after_create_actions if @project.persisted?
 
         import_schedule
@@ -92,7 +92,7 @@ module Projects
       # Skip writing the config for project imports/forks because it
       # will always fail since the Git directory doesn't exist until
       # a background job creates it (see Project#add_import_job).
-      @project.write_repository_config unless @project.import?
+      @project.set_full_path unless @project.import?
 
       unless @project.gitlab_project_import?
         @project.create_wiki unless skip_wiki?
@@ -101,6 +101,8 @@ module Projects
       @project.track_project_repository
       @project.create_project_setting unless @project.project_setting
 
+      yield if block_given?
+
       event_service.create_project(@project, current_user)
       system_hook_service.execute_hooks_for(@project, :create)
 
@@ -162,7 +164,7 @@ module Projects
         @project.create_or_update_import_data(data: @import_data[:data], credentials: @import_data[:credentials]) if @import_data
 
         if @project.save
-          Integration.create_from_active_default_integrations(@project, :project_id, with_templates: true)
+          Integration.create_from_active_default_integrations(@project, :project_id)
 
           @project.create_labels unless @project.gitlab_project_import?
 
diff --git a/app/services/projects/detect_repository_languages_service.rb b/app/services/projects/detect_repository_languages_service.rb
index c57773c3302..0356a6b0ccd 100644
--- a/app/services/projects/detect_repository_languages_service.rb
+++ b/app/services/projects/detect_repository_languages_service.rb
@@ -21,7 +21,7 @@ module Projects
             .update_all(share: update[:share])
         end
 
-        Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert
+        Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert
           RepositoryLanguage.table_name,
           detection.insertions(matching_programming_languages)
         )
diff --git a/app/services/projects/fetch_statistics_increment_service.rb b/app/services/projects/fetch_statistics_increment_service.rb
index b150fd2d9f1..3354a074d1e 100644
--- a/app/services/projects/fetch_statistics_increment_service.rb
+++ b/app/services/projects/fetch_statistics_increment_service.rb
@@ -15,7 +15,7 @@ module Projects
         ON CONFLICT (project_id, date) DO UPDATE SET fetch_count = #{table_name}.fetch_count + 1
       SQL
 
-      ActiveRecord::Base.connection.execute(increment_fetch_count_sql)
+      ProjectDailyStatistic.connection.execute(increment_fetch_count_sql)
     end
 
     private
diff --git a/app/services/projects/hashed_storage/migrate_repository_service.rb b/app/services/projects/hashed_storage/migrate_repository_service.rb
index c7989e04607..b65d0e63fd3 100644
--- a/app/services/projects/hashed_storage/migrate_repository_service.rb
+++ b/app/services/projects/hashed_storage/migrate_repository_service.rb
@@ -14,7 +14,7 @@ module Projects
         result = move_repositories
 
         if result
-          project.write_repository_config
+          project.set_full_path
           project.track_project_repository
         else
           rollback_folder_move
diff --git a/app/services/projects/hashed_storage/rollback_repository_service.rb b/app/services/projects/hashed_storage/rollback_repository_service.rb
index 6ab49630603..f4146ff9158 100644
--- a/app/services/projects/hashed_storage/rollback_repository_service.rb
+++ b/app/services/projects/hashed_storage/rollback_repository_service.rb
@@ -14,7 +14,7 @@ module Projects
         result = move_repositories
 
         if result
-          project.write_repository_config
+          project.set_full_path
           project.track_project_repository
         else
           rollback_folder_move
diff --git a/app/services/projects/lfs_pointers/lfs_link_service.rb b/app/services/projects/lfs_pointers/lfs_link_service.rb
index e86106f0a09..7c00b9e6105 100644
--- a/app/services/projects/lfs_pointers/lfs_link_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_link_service.rb
@@ -38,7 +38,7 @@ module Projects
           rows = existent_lfs_objects
                    .not_linked_to_project(project)
                    .map { |existing_lfs_object| { project_id: project.id, lfs_object_id: existing_lfs_object.id } }
-          Gitlab::Database.bulk_insert(:lfs_objects_projects, rows) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(:lfs_objects_projects, rows) # rubocop:disable Gitlab/BulkInsert
           iterations += 1
 
           linked_existing_objects += existent_lfs_objects.map(&:oid)
diff --git a/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb b/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb
index 75106297043..b4872cd9442 100644
--- a/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb
@@ -31,7 +31,7 @@ module Projects
         #
         LfsDownloadLinkListService
           .new(project, remote_uri: current_endpoint_uri)
-          .execute(lfs_pointers_in_repository)
+          .execute(missing_lfs_files)
       rescue LfsDownloadLinkListService::DownloadLinksError => e
         raise LfsObjectDownloadListError, "The LFS objects download list couldn't be imported. Error: #{e.message}"
       end
@@ -53,6 +53,22 @@ module Projects
         @lfs_pointers_in_repository ||= LfsListService.new(project).execute
       end
 
+      def existing_lfs_objects
+        project.lfs_objects
+      end
+
+      def existing_lfs_objects_hash
+        {}.tap do |hash|
+          existing_lfs_objects.find_each do |lfs_object|
+            hash[lfs_object.oid] = lfs_object.size
+          end
+        end
+      end
+
+      def missing_lfs_files
+        lfs_pointers_in_repository.except(*existing_lfs_objects_hash.keys)
+      end
+
       def lfsconfig_endpoint_uri
         strong_memoize(:lfsconfig_endpoint_uri) do
           # Retrieveing the blob data from the .lfsconfig file
diff --git a/app/services/projects/operations/update_service.rb b/app/services/projects/operations/update_service.rb
index 2cc6bcdf57c..51b8e3c6c54 100644
--- a/app/services/projects/operations/update_service.rb
+++ b/app/services/projects/operations/update_service.rb
@@ -63,8 +63,15 @@ module Projects
         # Make sure we're converting to symbols because
         # * ActionController::Parameters#keys returns a list of strings
         # * in specs we're using hashes with symbols as keys
+        update_keys = settings.keys.map(&:to_sym)
 
-        settings.keys.map(&:to_sym) == %i[enabled]
+        # Integrated error tracking works without Sentry integration,
+        # so we don't need to update all those values from error_tracking_params_for_update method.
+        # Instead we turn it on/off with partial update together with "enabled" attribute.
+        # But since its optional, we exclude it from the condition below.
+        update_keys.delete(:integrated)
+
+        update_keys == %i[enabled]
       end
 
       def error_tracking_params_for_partial_update(settings)
diff --git a/app/services/projects/overwrite_project_service.rb b/app/services/projects/overwrite_project_service.rb
index 6be3b1b5a6f..f35370c427f 100644
--- a/app/services/projects/overwrite_project_service.rb
+++ b/app/services/projects/overwrite_project_service.rb
@@ -5,6 +5,8 @@ module Projects
     def execute(source_project)
       return unless source_project && source_project.namespace == @project.namespace
 
+      start_time = ::Gitlab::Metrics::System.monotonic_time
+
       Project.transaction do
         move_before_destroy_relationships(source_project)
         # Reset is required in order to get the proper
@@ -25,10 +27,25 @@ module Projects
       else
         raise
       end
+
+    ensure
+      track_service(start_time, source_project, e)
     end
 
     private
 
+    def track_service(start_time, source_project, exception)
+      return if ::Feature.disabled?(:project_overwrite_service_tracking, source_project, default_enabled: :yaml)
+
+      duration = ::Gitlab::Metrics::System.monotonic_time - start_time
+
+      Gitlab::AppJsonLogger.info(class: self.class.name,
+                                 namespace_id: source_project.namespace.id,
+                                 project_id: source_project.id,
+                                 duration_s: duration.to_f,
+                                 error: exception.class.name)
+    end
+
     def move_before_destroy_relationships(source_project)
       options = { remove_remaining_elements: false }
 
diff --git a/app/services/projects/protect_default_branch_service.rb b/app/services/projects/protect_default_branch_service.rb
index 0111b9e377a..03d1c49657d 100644
--- a/app/services/projects/protect_default_branch_service.rb
+++ b/app/services/projects/protect_default_branch_service.rb
@@ -69,3 +69,5 @@ module Projects
     end
   end
 end
+
+Projects::ProtectDefaultBranchService.prepend_mod
diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb
index fb0fea756bc..074550e104d 100644
--- a/app/services/projects/transfer_service.rb
+++ b/app/services/projects/transfer_service.rb
@@ -89,6 +89,8 @@ module Projects
 
         update_integrations
 
+        remove_paid_features
+
         project.old_path_with_namespace = @old_path
 
         update_repository_configuration(@new_path)
@@ -109,6 +111,10 @@ module Projects
       move_pages(project)
     end
 
+    # Overridden in EE
+    def remove_paid_features
+    end
+
     def transfer_missing_group_resources(group)
       Labels::TransferService.new(current_user, group, project).execute
 
@@ -129,7 +135,7 @@ module Projects
     end
 
     def update_repository_configuration(full_path)
-      project.write_repository_config(gl_full_path: full_path)
+      project.set_full_path(gl_full_path: full_path)
       project.track_project_repository
     end
 
@@ -235,7 +241,7 @@ module Projects
     end
 
     def update_integrations
-      project.integrations.inherit.delete_all
+      project.integrations.with_default_settings.delete_all
       Integration.create_from_active_default_integrations(project, :project_id)
     end
   end
diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb
index a90c22c7de5..f96a6ee1255 100644
--- a/app/services/projects/update_pages_service.rb
+++ b/app/services/projects/update_pages_service.rb
@@ -37,14 +37,13 @@ module Projects
         job.run!
       end
 
-      raise InvalidStateError, 'missing pages artifacts' unless build.artifacts?
-      raise InvalidStateError, 'build SHA is outdated for this ref' unless latest?
+      validate_state!
+      validate_max_size!
+      validate_max_entries!
 
       build.artifacts_file.use_file do |artifacts_path|
         deploy_to_legacy_storage(artifacts_path)
-
         create_pages_deployment(artifacts_path, build)
-
         success
       end
     rescue InvalidStateError => e
@@ -92,8 +91,10 @@ module Projects
 
         # Check if we did extract public directory
         archive_public_path = File.join(tmp_path, PUBLIC_DIR)
+
         raise InvalidStateError, 'pages miss the public folder' unless Dir.exist?(archive_public_path)
-        raise InvalidStateError, 'build SHA is outdated for this ref' unless latest?
+
+        validate_outdated_sha!
 
         deploy_page!(archive_public_path)
       end
@@ -108,15 +109,6 @@ module Projects
     end
 
     def extract_zip_archive!(artifacts_path, temp_path)
-      raise InvalidStateError, 'missing artifacts metadata' unless build.artifacts_metadata?
-
-      # Calculate page size after extract
-      public_entry = build.artifacts_metadata_entry(PUBLIC_DIR + '/', recursive: true)
-
-      if public_entry.total_size > max_size
-        raise InvalidStateError, "artifacts for pages are too large: #{public_entry.total_size}"
-      end
-
       SafeZip::Extract.new(artifacts_path)
         .extract(directories: [PUBLIC_DIR], to: temp_path)
     rescue SafeZip::Extract::Error => e
@@ -151,19 +143,17 @@ module Projects
     end
 
     def create_pages_deployment(artifacts_path, build)
-      # we're using the full archive and pages daemon needs to read it
-      # so we want the total count from entries, not only "public/" directory
-      # because it better approximates work we need to do before we can serve the site
-      entries_count = build.artifacts_metadata_entry("", recursive: true).entries.count
       sha256 = build.job_artifacts_archive.file_sha256
 
       deployment = nil
       File.open(artifacts_path) do |file|
         deployment = project.pages_deployments.create!(file: file,
                                                        file_count: entries_count,
-                                                       file_sha256: sha256)
+                                                       file_sha256: sha256,
+                                                       ci_build_id: build.id
+                                                      )
 
-        raise InvalidStateError, 'build SHA is outdated for this ref' unless latest?
+        validate_outdated_sha!
 
         project.update_pages_deployment!(deployment)
       end
@@ -175,29 +165,6 @@ module Projects
       )
     end
 
-    def latest?
-      # check if sha for the ref is still the most recent one
-      # this helps in case when multiple deployments happens
-      sha == latest_sha
-    end
-
-    def blocks
-      # Calculate dd parameters: we limit the size of pages
-      1 + max_size / BLOCK_SIZE
-    end
-
-    def max_size_from_settings
-      Gitlab::CurrentSettings.max_pages_size.megabytes
-    end
-
-    def max_size
-      max_pages_size = max_size_from_settings
-
-      return ::Gitlab::Pages::MAX_SIZE if max_pages_size == 0
-
-      max_pages_size
-    end
-
     def tmp_path
       @tmp_path ||= File.join(::Settings.pages.path, TMP_EXTRACT_PATH)
     end
@@ -262,6 +229,73 @@ module Projects
     def tmp_dir_prefix
       "project-#{project.id}-build-#{build.id}-"
     end
+
+    def validate_state!
+      raise InvalidStateError, 'missing pages artifacts' unless build.artifacts?
+      raise InvalidStateError, 'missing artifacts metadata' unless build.artifacts_metadata?
+
+      validate_outdated_sha!
+    end
+
+    def validate_outdated_sha!
+      return if latest?
+
+      if Feature.enabled?(:pages_smart_check_outdated_sha, project, default_enabled: :yaml)
+        # use pipeline_id in case the build is retried
+        last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id
+
+        return unless last_deployed_pipeline_id
+        return if last_deployed_pipeline_id <= build.pipeline_id
+      end
+
+      raise InvalidStateError, 'build SHA is outdated for this ref'
+    end
+
+    def latest?
+      # check if sha for the ref is still the most recent one
+      # this helps in case when multiple deployments happens
+      sha == latest_sha
+    end
+
+    def validate_max_size!
+      if total_size > max_size
+        raise InvalidStateError, "artifacts for pages are too large: #{total_size}"
+      end
+    end
+
+    # Calculate page size after extract
+    def total_size
+      @total_size ||= build.artifacts_metadata_entry(PUBLIC_DIR + '/', recursive: true).total_size
+    end
+
+    def max_size_from_settings
+      Gitlab::CurrentSettings.max_pages_size.megabytes
+    end
+
+    def max_size
+      max_pages_size = max_size_from_settings
+
+      return ::Gitlab::Pages::MAX_SIZE if max_pages_size == 0
+
+      max_pages_size
+    end
+
+    def validate_max_entries!
+      if pages_file_entries_limit > 0 && entries_count > pages_file_entries_limit
+        raise InvalidStateError, "pages site contains #{entries_count} file entries, while limit is set to #{pages_file_entries_limit}"
+      end
+    end
+
+    def entries_count
+      # we're using the full archive and pages daemon needs to read it
+      # so we want the total count from entries, not only "public/" directory
+      # because it better approximates work we need to do before we can serve the site
+      @entries_count = build.artifacts_metadata_entry("", recursive: true).entries.count
+    end
+
+    def pages_file_entries_limit
+      project.actual_limits.pages_file_entries
+    end
   end
 end
 
diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb
index 6c29ba81910..898125c181c 100644
--- a/app/services/projects/update_remote_mirror_service.rb
+++ b/app/services/projects/update_remote_mirror_service.rb
@@ -43,12 +43,7 @@ module Projects
       # LFS objects must be sent first, or the push has dangling pointers
       send_lfs_objects!(remote_mirror)
 
-      response = if Feature.enabled?(:update_remote_mirror_inmemory, project, default_enabled: :yaml)
-                   remote_mirror.update_repository(inmemory_remote: true)
-                 else
-                   remote_mirror.ensure_remote!
-                   remote_mirror.update_repository(inmemory_remote: false)
-                 end
+      response = remote_mirror.update_repository
 
       if response.divergent_refs.any?
         message = "Some refs have diverged and have not been updated on the remote:"