diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 11:43:02 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 11:43:02 +0300 |
commit | d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb (patch) | |
tree | 2341ef426af70ad1e289c38036737e04b0aa5007 /app/services/projects | |
parent | d6e514dd13db8947884cd58fe2a9c2a063400a9b (diff) |
Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42
Diffstat (limited to 'app/services/projects')
-rw-r--r-- | app/services/projects/after_rename_service.rb | 8 | ||||
-rw-r--r-- | app/services/projects/container_repository/cache_tags_created_at_service.rb | 70 | ||||
-rw-r--r-- | app/services/projects/container_repository/cleanup_tags_service.rb | 176 | ||||
-rw-r--r-- | app/services/projects/create_service.rb | 6 | ||||
-rw-r--r-- | app/services/projects/destroy_service.rb | 24 | ||||
-rw-r--r-- | app/services/projects/group_links/update_service.rb | 22 | ||||
-rw-r--r-- | app/services/projects/import_service.rb | 23 | ||||
-rw-r--r-- | app/services/projects/overwrite_project_service.rb | 4 | ||||
-rw-r--r-- | app/services/projects/participants_service.rb | 13 | ||||
-rw-r--r-- | app/services/projects/transfer_service.rb | 45 | ||||
-rw-r--r-- | app/services/projects/update_pages_service.rb | 10 | ||||
-rw-r--r-- | app/services/projects/update_service.rb | 33 |
12 files changed, 314 insertions, 120 deletions
diff --git a/app/services/projects/after_rename_service.rb b/app/services/projects/after_rename_service.rb index 953b386b754..a3d54bc6b58 100644 --- a/app/services/projects/after_rename_service.rb +++ b/app/services/projects/after_rename_service.rb @@ -12,6 +12,8 @@ module Projects # # Projects::AfterRenameService.new(project).execute class AfterRenameService + include BaseServiceUtility + # @return [String] The Project being renamed. attr_reader :project @@ -78,7 +80,7 @@ module Projects def execute_system_hooks project.old_path_with_namespace = full_path_before - SystemHooksService.new.execute_hooks_for(project, :rename) + system_hook_service.execute_hooks_for(project, :rename) end def update_repository_configuration @@ -110,7 +112,7 @@ module Projects end def log_completion - Gitlab::AppLogger.info( + log_info( "Project #{project.id} has been renamed from " \ "#{full_path_before} to #{full_path_after}" ) @@ -140,7 +142,7 @@ module Projects def rename_failed! error = "Repository #{full_path_before} could not be renamed to #{full_path_after}" - Gitlab::AppLogger.error(error) + log_error(error) raise RenameFailedError, error end diff --git a/app/services/projects/container_repository/cache_tags_created_at_service.rb b/app/services/projects/container_repository/cache_tags_created_at_service.rb new file mode 100644 index 00000000000..3a5346d7a23 --- /dev/null +++ b/app/services/projects/container_repository/cache_tags_created_at_service.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Projects + module ContainerRepository + class CacheTagsCreatedAtService + def initialize(container_repository) + @container_repository = container_repository + @cached_tag_names = Set.new + end + + def populate(tags) + return if tags.empty? + + # This will load all tags in one Redis roundtrip + # the maximum number of tags is configurable and is set to 200 by default. + # https://gitlab.com/gitlab-org/gitlab/blob/master/doc/user/packages/container_registry/index.md#set-cleanup-limits-to-conserve-resources + keys = tags.map(&method(:cache_key)) + cached_tags_count = 0 + + ::Gitlab::Redis::Cache.with do |redis| + tags.zip(redis.mget(keys)).each do |tag, created_at| + next unless created_at + + tag.created_at = DateTime.rfc3339(created_at) + @cached_tag_names << tag.name + cached_tags_count += 1 + end + end + + cached_tags_count + end + + def insert(tags, max_ttl_in_seconds) + return unless max_ttl_in_seconds + return if tags.empty? + + # tags with nil created_at are not cacheable + # tags already cached don't need to be cached again + cacheable_tags = tags.select do |tag| + tag.created_at.present? && !tag.name.in?(@cached_tag_names) + end + + return if cacheable_tags.empty? + + now = Time.zone.now + + ::Gitlab::Redis::Cache.with do |redis| + # we use a pipeline instead of a MSET because each tag has + # a specific ttl + redis.pipelined do + cacheable_tags.each do |tag| + created_at = tag.created_at + # ttl is the max_ttl_in_seconds reduced by the number + # of seconds that the tag has already existed + ttl = max_ttl_in_seconds - (now - created_at).seconds + ttl = ttl.to_i + redis.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0 + end + end + end + end + + private + + def cache_key(tag) + "container_repository:{#{@container_repository.id}}:tag:#{tag.name}:created_at" + end + end + end +end diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index 793d2fec033..3a60de0f1ee 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -2,116 +2,152 @@ module Projects module ContainerRepository - class CleanupTagsService < BaseService - def execute(container_repository) + class CleanupTagsService + include BaseServiceUtility + include ::Gitlab::Utils::StrongMemoize + + def initialize(container_repository, user = nil, params = {}) + @container_repository = container_repository + @current_user = user + @params = params.dup + + @project = container_repository.project + @tags = container_repository.tags + tags_size = @tags.size + @counts = { + original_size: tags_size, + cached_tags_count: 0 + } + end + + def execute return error('access denied') unless can_destroy? return error('invalid regex') unless valid_regex? - tags = container_repository.tags - original_size = tags.size + filter_out_latest + filter_by_name - tags = without_latest(tags) - tags = filter_by_name(tags) + truncate + populate_from_cache - before_truncate_size = tags.size - tags = truncate(tags) - after_truncate_size = tags.size + filter_keep_n + filter_by_older_than - tags = filter_keep_n(tags) - tags = filter_by_older_than(tags) - - delete_tags(container_repository, tags).tap do |result| - result[:original_size] = original_size - result[:before_truncate_size] = before_truncate_size - result[:after_truncate_size] = after_truncate_size - result[:before_delete_size] = tags.size + delete_tags.merge(@counts).tap do |result| + result[:before_delete_size] = @tags.size result[:deleted_size] = result[:deleted]&.size - result[:status] = :error if before_truncate_size != after_truncate_size + result[:status] = :error if @counts[:before_truncate_size] != @counts[:after_truncate_size] end end private - def delete_tags(container_repository, tags) - return success(deleted: []) unless tags.any? - - tag_names = tags.map(&:name) + def delete_tags + return success(deleted: []) unless @tags.any? service = Projects::ContainerRepository::DeleteTagsService.new( - container_repository.project, - current_user, - tags: tag_names, - container_expiration_policy: params['container_expiration_policy'] + @project, + @current_user, + tags: @tags.map(&:name), + container_expiration_policy: container_expiration_policy ) - service.execute(container_repository) + service.execute(@container_repository) end - def without_latest(tags) - tags.reject(&:latest?) + def filter_out_latest + @tags.reject!(&:latest?) end - def order_by_date(tags) + def order_by_date now = DateTime.current - tags.sort_by { |tag| tag.created_at || now }.reverse + @tags.sort_by! { |tag| tag.created_at || now } + .reverse! end - def filter_by_name(tags) - regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_delete'] || params['name_regex']}\\z") - regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_keep']}\\z") + def filter_by_name + regex_delete = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_delete || name_regex}\\z") + regex_retain = ::Gitlab::UntrustedRegexp.new("\\A#{name_regex_keep}\\z") - tags.select do |tag| + @tags.select! do |tag| # regex_retain will override any overlapping matches by regex_delete regex_delete.match?(tag.name) && !regex_retain.match?(tag.name) end end - def filter_keep_n(tags) - return tags unless params['keep_n'] + def filter_keep_n + return unless keep_n - tags = order_by_date(tags) - tags.drop(keep_n) + order_by_date + cache_tags(@tags.first(keep_n_as_integer)) + @tags = @tags.drop(keep_n_as_integer) end - def filter_by_older_than(tags) - return tags unless params['older_than'] + def filter_by_older_than + return unless older_than - older_than = ChronicDuration.parse(params['older_than']).seconds.ago + older_than_timestamp = older_than_in_seconds.ago - tags.select do |tag| - tag.created_at && tag.created_at < older_than + @tags, tags_to_keep = @tags.partition do |tag| + tag.created_at && tag.created_at < older_than_timestamp end + + cache_tags(tags_to_keep) end def can_destroy? - return true if params['container_expiration_policy'] + return true if container_expiration_policy - can?(current_user, :destroy_container_image, project) + can?(@current_user, :destroy_container_image, @project) end def valid_regex? %w(name_regex_delete name_regex name_regex_keep).each do |param_name| - regex = params[param_name] + regex = @params[param_name] ::Gitlab::UntrustedRegexp.new(regex) unless regex.blank? end true rescue RegexpError => e - ::Gitlab::ErrorTracking.log_exception(e, project_id: project.id) + ::Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) false end - def truncate(tags) - return tags unless throttling_enabled? - return tags if max_list_size == 0 + def truncate + @counts[:before_truncate_size] = @tags.size + @counts[:after_truncate_size] = @tags.size + + return unless throttling_enabled? + return if max_list_size == 0 # truncate the list to make sure that after the #filter_keep_n # execution, the resulting list will be max_list_size - truncated_size = max_list_size + keep_n + truncated_size = max_list_size + keep_n_as_integer - return tags if tags.size <= truncated_size + return if @tags.size <= truncated_size + + @tags = @tags.sample(truncated_size) + @counts[:after_truncate_size] = @tags.size + end + + def populate_from_cache + @counts[:cached_tags_count] = cache.populate(@tags) if caching_enabled? + end + + def cache_tags(tags) + cache.insert(tags, older_than_in_seconds) if caching_enabled? + end + + def cache + strong_memoize(:cache) do + ::Projects::ContainerRepository::CacheTagsCreatedAtService.new(@container_repository) + end + end - tags.sample(truncated_size) + def caching_enabled? + container_expiration_policy && + older_than.present? && + Feature.enabled?(:container_registry_expiration_policies_caching, @project) end def throttling_enabled? @@ -123,7 +159,37 @@ module Projects end def keep_n - params['keep_n'].to_i + @params['keep_n'] + end + + def keep_n_as_integer + keep_n.to_i + end + + def older_than_in_seconds + strong_memoize(:older_than_in_seconds) do + ChronicDuration.parse(older_than).seconds + end + end + + def older_than + @params['older_than'] + end + + def name_regex_delete + @params['name_regex_delete'] + end + + def name_regex + @params['name_regex'] + end + + def name_regex_keep + @params['name_regex_keep'] + end + + def container_expiration_policy + @params['container_expiration_policy'] end end end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index e717491b19d..1536f0a22b8 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -8,6 +8,7 @@ module Projects @current_user = user @params = params.dup @skip_wiki = @params.delete(:skip_wiki) + @initialize_with_sast = Gitlab::Utils.to_boolean(@params.delete(:initialize_with_sast)) @initialize_with_readme = Gitlab::Utils.to_boolean(@params.delete(:initialize_with_readme)) @import_data = @params.delete(:import_data) @relations_block = @params.delete(:relations_block) @@ -118,6 +119,7 @@ module Projects Projects::PostCreationWorker.perform_async(@project.id) create_readme if @initialize_with_readme + create_sast_commit if @initialize_with_sast end # Add an authorization for the current user authorizations inline @@ -160,6 +162,10 @@ module Projects Files::CreateService.new(@project, current_user, commit_attrs).execute end + def create_sast_commit + ::Security::CiConfiguration::SastCreateService.new(@project, current_user, {}, commit_on_default: true).execute + end + def readme_content @readme_template.presence || experiment(:new_project_readme_content, namespace: @project.namespace).run_with(@project) end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index afa8de04fca..27f813f4661 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -5,6 +5,7 @@ module Projects include Gitlab::ShellAdapter DestroyError = Class.new(StandardError) + BATCH_SIZE = 100 def async_execute project.update_attribute(:pending_delete, true) @@ -119,6 +120,12 @@ module Projects destroy_web_hooks! destroy_project_bots! + if ::Feature.enabled?(:ci_optimize_project_records_destruction, project, default_enabled: :yaml) && + Feature.enabled?(:abort_deleted_project_pipelines, default_enabled: :yaml) + + destroy_ci_records! + end + # Rails attempts to load all related records into memory before # destroying: https://github.com/rails/rails/issues/22510 # This ensures we delete records in batches. @@ -133,6 +140,23 @@ module Projects log_info("Attempting to destroy #{project.full_path} (#{project.id})") end + def destroy_ci_records! + project.all_pipelines.find_each(batch_size: BATCH_SIZE) do |pipeline| # rubocop: disable CodeReuse/ActiveRecord + # Destroy artifacts, then builds, then pipelines + # All builds have already been dropped by Ci::AbortPipelinesService, + # so no Ci::Build-instantiating cancellations happen here. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/71342#note_691523196 + + ::Ci::DestroyPipelineService.new(project, current_user).execute(pipeline) + end + + deleted_count = project.commit_statuses.delete_all + + if deleted_count > 0 + Gitlab::AppLogger.info "Projects::DestroyService - Project #{project.id} - #{deleted_count} leftover commit statuses" + end + end + # The project can have multiple webhooks with hundreds of thousands of web_hook_logs. # By default, they are removed with "DELETE CASCADE" option defined via foreign_key. # But such queries can exceed the statement_timeout limit and fail to delete the project. diff --git a/app/services/projects/group_links/update_service.rb b/app/services/projects/group_links/update_service.rb index 475ab17f1a1..a836b96cac3 100644 --- a/app/services/projects/group_links/update_service.rb +++ b/app/services/projects/group_links/update_service.rb @@ -20,19 +20,15 @@ module Projects attr_reader :group_link def refresh_authorizations - if Feature.enabled?(:specialized_worker_for_project_share_update_auth_recalculation) - AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(project.id) - - # Until we compare the inconsistency rates of the new specialized worker and - # the old approach, we still run AuthorizedProjectsWorker - # but with some delay and lower urgency as a safety net. - group_link.group.refresh_members_authorized_projects( - blocking: false, - priority: UserProjectAccessChangedService::LOW_PRIORITY - ) - else - group_link.group.refresh_members_authorized_projects - end + AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(project.id) + + # Until we compare the inconsistency rates of the new specialized worker and + # the old approach, we still run AuthorizedProjectsWorker + # but with some delay and lower urgency as a safety net. + group_link.group.refresh_members_authorized_projects( + blocking: false, + priority: UserProjectAccessChangedService::LOW_PRIORITY + ) end def requires_authorization_refresh?(params) diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index b5288aad6f0..4979af6dfe1 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -16,6 +16,8 @@ module Projects end def execute + track_start_import + add_repository_to_project download_lfs_objects @@ -25,16 +27,17 @@ module Projects after_execute_hook success - rescue Gitlab::UrlBlocker::BlockedUrlError => e - Gitlab::ErrorTracking.track_exception(e, project_path: project.full_path, importer: project.import_type) + rescue Gitlab::UrlBlocker::BlockedUrlError, StandardError => e + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: e, + metrics: true + ) - error(s_("ImportProjects|Error importing repository %{project_safe_import_url} into %{project_full_path} - %{message}") % { project_safe_import_url: project.safe_import_url, project_full_path: project.full_path, message: e.message }) - rescue StandardError => e message = Projects::ImportErrorFilter.filter_message(e.message) - - Gitlab::ErrorTracking.track_exception(e, project_path: project.full_path, importer: project.import_type) - - error(s_("ImportProjects|Error importing repository %{project_safe_import_url} into %{project_full_path} - %{message}") % { project_safe_import_url: project.safe_import_url, project_full_path: project.full_path, message: message }) + error(s_("ImportProjects|Error importing repository %{project_safe_import_url} into %{project_full_path} - %{message}") % + { project_safe_import_url: project.safe_import_url, project_full_path: project.full_path, message: message }) end protected @@ -54,6 +57,10 @@ module Projects # Defined in EE::Projects::ImportService end + def track_start_import + has_importer? && importer_class.try(:track_start_import, project) + end + def add_repository_to_project if project.external_import? && !unknown_url? begin diff --git a/app/services/projects/overwrite_project_service.rb b/app/services/projects/overwrite_project_service.rb index f35370c427f..2612001eb95 100644 --- a/app/services/projects/overwrite_project_service.rb +++ b/app/services/projects/overwrite_project_service.rb @@ -3,7 +3,7 @@ module Projects class OverwriteProjectService < BaseService def execute(source_project) - return unless source_project && source_project.namespace == @project.namespace + return unless source_project && source_project.namespace_id == @project.namespace_id start_time = ::Gitlab::Metrics::System.monotonic_time @@ -40,7 +40,7 @@ module Projects duration = ::Gitlab::Metrics::System.monotonic_time - start_time Gitlab::AppJsonLogger.info(class: self.class.name, - namespace_id: source_project.namespace.id, + namespace_id: source_project.namespace_id, project_id: source_project.id, duration_s: duration.to_f, error: exception.class.name) diff --git a/app/services/projects/participants_service.rb b/app/services/projects/participants_service.rb index 228115d72b8..1616a8a4062 100644 --- a/app/services/projects/participants_service.rb +++ b/app/services/projects/participants_service.rb @@ -36,14 +36,17 @@ module Projects private def project_members_through_invited_groups - groups_with_ancestors_ids = Gitlab::ObjectHierarchy - .new(visible_groups) - .base_and_ancestors - .pluck_primary_key + groups_with_ancestors = if ::Feature.enabled?(:linear_participants_service_ancestor_scopes, current_user, default_enabled: :yaml) + visible_groups.self_and_ancestors + else + Gitlab::ObjectHierarchy + .new(visible_groups) + .base_and_ancestors + end GroupMember .active_without_invites_and_requests - .with_source_id(groups_with_ancestors_ids) + .with_source_id(groups_with_ancestors.pluck_primary_key) end def visible_groups diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 27376173f07..a69e6488ebc 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -81,7 +81,7 @@ module Projects # Apply changes to the project update_namespace_and_visibility(@new_namespace) - update_shared_runners_settings + project.reconcile_shared_runners_setting! project.save! # Notifications @@ -104,6 +104,8 @@ module Projects update_repository_configuration(@new_path) execute_system_hooks + + update_pending_builds! end post_update_hooks(project) @@ -154,19 +156,15 @@ module Projects user_ids = @old_namespace.user_ids_for_project_authorizations | @new_namespace.user_ids_for_project_authorizations - if Feature.enabled?(:specialized_worker_for_project_transfer_auth_recalculation) - AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(project.id) - - # Until we compare the inconsistency rates of the new specialized worker and - # the old approach, we still run AuthorizedProjectsWorker - # but with some delay and lower urgency as a safety net. - UserProjectAccessChangedService.new(user_ids).execute( - blocking: false, - priority: UserProjectAccessChangedService::LOW_PRIORITY - ) - else - UserProjectAccessChangedService.new(user_ids).execute - end + AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(project.id) + + # Until we compare the inconsistency rates of the new specialized worker and + # the old approach, we still run AuthorizedProjectsWorker + # but with some delay and lower urgency as a safety net. + UserProjectAccessChangedService.new(user_ids).execute( + blocking: false, + priority: UserProjectAccessChangedService::LOW_PRIORITY + ) end def rollback_side_effects @@ -189,7 +187,7 @@ module Projects end def execute_system_hooks - SystemHooksService.new.execute_hooks_for(project, :transfer) + system_hook_service.execute_hooks_for(project, :transfer) end def move_project_folders(project) @@ -241,18 +239,19 @@ module Projects "#{new_path}#{::Gitlab::GlRepository::DESIGN.path_suffix}" end - def update_shared_runners_settings - # If a project is being transferred to another group it means it can already - # have shared runners enabled but we need to check whether the new group allows that. - if project.group && project.group.shared_runners_setting == 'disabled_and_unoverridable' - project.shared_runners_enabled = false - end - end - def update_integrations project.integrations.with_default_settings.delete_all Integration.create_from_active_default_integrations(project, :project_id) end + + def update_pending_builds! + update_params = { + namespace_id: new_namespace.id, + namespace_traversal_ids: new_namespace.traversal_ids + } + + ::Ci::UpdatePendingBuildService.new(project, update_params).execute + end end end diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index dc75fe1014a..0000e713cb4 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -136,13 +136,11 @@ module Projects def validate_outdated_sha! return if latest? - if Feature.enabled?(:pages_smart_check_outdated_sha, project, default_enabled: :yaml) - # use pipeline_id in case the build is retried - last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id + # use pipeline_id in case the build is retried + last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id - return unless last_deployed_pipeline_id - return if last_deployed_pipeline_id <= build.pipeline_id - end + return unless last_deployed_pipeline_id + return if last_deployed_pipeline_id <= build.pipeline_id raise InvalidStateError, 'build SHA is outdated for this ref' end diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index b87564fcaef..a32e80af4b1 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -105,7 +105,7 @@ module Projects end update_pages_config if changing_pages_related_config? - update_pending_builds if shared_runners_toggled? + update_pending_builds if runners_settings_toggled? end def after_rename_service(project) @@ -181,13 +181,36 @@ module Projects end def update_pending_builds - update_params = { instance_runners_enabled: project.shared_runners_enabled } + update_params = { + instance_runners_enabled: project.shared_runners_enabled?, + namespace_traversal_ids: group_runner_traversal_ids + } - ::Ci::UpdatePendingBuildService.new(project, update_params).execute + ::Ci::UpdatePendingBuildService + .new(project, update_params) + .execute end - def shared_runners_toggled? - project.previous_changes.include?('shared_runners_enabled') + def shared_runners_settings_toggled? + project.previous_changes.include?(:shared_runners_enabled) + end + + def group_runners_settings_toggled? + return false unless project.ci_cd_settings.present? + + project.ci_cd_settings.previous_changes.include?(:group_runners_enabled) + end + + def runners_settings_toggled? + shared_runners_settings_toggled? || group_runners_settings_toggled? + end + + def group_runner_traversal_ids + if project.group_runners_enabled? + project.namespace.traversal_ids + else + [] + end end end end |