From 9f46488805e86b1bc341ea1620b866016c2ce5ed Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 20 May 2020 14:34:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-0-stable-ee --- app/services/projects/alerting/notify_service.rb | 17 +++++- .../container_repository/cleanup_tags_service.rb | 2 +- app/services/projects/create_service.rb | 32 +++++++++- .../projects/gitlab_projects_import_service.rb | 8 ++- .../hashed_storage/base_attachment_service.rb | 2 +- .../hashed_storage/base_repository_service.rb | 28 ++++++++- .../projects/import_export/export_service.rb | 33 +++++++++-- app/services/projects/import_service.rb | 25 ++++++++ .../lfs_pointers/lfs_download_link_list_service.rb | 2 +- app/services/projects/lsif_data_service.rb | 2 +- .../projects/prometheus/alerts/notify_service.rb | 9 +++ .../projects/propagate_service_template.rb | 54 ++++++++--------- app/services/projects/transfer_service.rb | 24 +++++++- .../projects/update_remote_mirror_service.rb | 14 +++-- .../projects/update_repository_storage_service.rb | 69 ++++++++++++++-------- 15 files changed, 242 insertions(+), 79 deletions(-) (limited to 'app/services/projects') diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb index 1ce1ef7a1cd..76c89e85f17 100644 --- a/app/services/projects/alerting/notify_service.rb +++ b/app/services/projects/alerting/notify_service.rb @@ -10,7 +10,10 @@ module Projects return forbidden unless alerts_service_activated? return unauthorized unless valid_token?(token) - process_incident_issues if process_issues? + alert = create_alert + return bad_request unless alert.persisted? + + process_incident_issues(alert) if process_issues? send_alert_email if send_email? ServiceResponse.success @@ -22,13 +25,21 @@ module Projects delegate :alerts_service, :alerts_service_activated?, to: :project + def am_alert_params + Gitlab::AlertManagement::AlertParams.from_generic_alert(project: project, payload: params.to_h) + end + + def create_alert + AlertManagement::Alert.create(am_alert_params) + end + def send_email? incident_management_setting.send_email? end - def process_incident_issues + def process_incident_issues(alert) IncidentManagement::ProcessAlertWorker - .perform_async(project.id, parsed_payload) + .perform_async(project.id, parsed_payload, alert.id) end def send_alert_email diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index fc09d14ba4d..b53a9c1561e 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -33,7 +33,7 @@ module Projects end def order_by_date(tags) - now = DateTime.now + now = DateTime.current tags.sort_by { |tag| tag.created_at || now }.reverse end diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index 429ae905e3d..3233d1799b8 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -108,8 +108,22 @@ module Projects # users in the background def setup_authorizations if @project.group - @project.group.refresh_members_authorized_projects(blocking: false) current_user.refresh_authorized_projects + + if Feature.enabled?(:specialized_project_authorization_workers) + AuthorizedProjectUpdate::ProjectCreateWorker.perform_async(@project.id) + # AuthorizedProjectsWorker uses an exclusive lease per user but + # specialized workers might have synchronization issues. Until we + # compare the inconsistency rates of both approaches, we still run + # AuthorizedProjectsWorker but with some delay and lower urgency as a + # safety net. + @project.group.refresh_members_authorized_projects( + blocking: false, + priority: UserProjectAccessChangedService::LOW_PRIORITY + ) + else + @project.group.refresh_members_authorized_projects(blocking: false) + end else @project.add_maintainer(@project.namespace.owner, current_user: current_user) end @@ -202,8 +216,19 @@ module Projects end end + def extra_attributes_for_measurement + { + current_user: current_user&.name, + project_full_path: "#{project_namespace&.full_path}/#{@params[:path]}" + } + end + private + def project_namespace + @project_namespace ||= Namespace.find_by_id(@params[:namespace_id]) || current_user.namespace + end + def create_from_template? @params[:template_name].present? || @params[:template_project_id].present? end @@ -224,4 +249,9 @@ module Projects end end +# rubocop: disable Cop/InjectEnterpriseEditionModule Projects::CreateService.prepend_if_ee('EE::Projects::CreateService') +# rubocop: enable Cop/InjectEnterpriseEditionModule + +# Measurable should be at the bottom of the ancestor chain, so it will measure execution of EE::Projects::CreateService as well +Projects::CreateService.prepend(Measurable) diff --git a/app/services/projects/gitlab_projects_import_service.rb b/app/services/projects/gitlab_projects_import_service.rb index 234ebbc6651..2e192942b9c 100644 --- a/app/services/projects/gitlab_projects_import_service.rb +++ b/app/services/projects/gitlab_projects_import_service.rb @@ -29,17 +29,21 @@ module Projects end def project_with_same_full_path? - Project.find_by_full_path("#{current_namespace.full_path}/#{params[:path]}").present? + Project.find_by_full_path(project_path).present? end # rubocop: disable CodeReuse/ActiveRecord def current_namespace strong_memoize(:current_namespace) do - Namespace.find_by(id: params[:namespace_id]) + Namespace.find_by(id: params[:namespace_id]) || current_user.namespace end end # rubocop: enable CodeReuse/ActiveRecord + def project_path + "#{current_namespace.full_path}/#{params[:path]}" + end + def overwrite? strong_memoize(:overwrite) do params.delete(:overwrite) diff --git a/app/services/projects/hashed_storage/base_attachment_service.rb b/app/services/projects/hashed_storage/base_attachment_service.rb index f8852c206e3..a2a7895ba17 100644 --- a/app/services/projects/hashed_storage/base_attachment_service.rb +++ b/app/services/projects/hashed_storage/base_attachment_service.rb @@ -70,7 +70,7 @@ module Projects # # @param [String] new_path def discard_path!(new_path) - discarded_path = "#{new_path}-#{Time.now.utc.to_i}" + discarded_path = "#{new_path}-#{Time.current.utc.to_i}" logger.info("Moving existing empty attachments folder from '#{new_path}' to '#{discarded_path}', (PROJECT_ID=#{project.id})") FileUtils.mv(new_path, discarded_path) diff --git a/app/services/projects/hashed_storage/base_repository_service.rb b/app/services/projects/hashed_storage/base_repository_service.rb index d81aa4de9f1..065bf8725be 100644 --- a/app/services/projects/hashed_storage/base_repository_service.rb +++ b/app/services/projects/hashed_storage/base_repository_service.rb @@ -8,13 +8,15 @@ module Projects class BaseRepositoryService < BaseService include Gitlab::ShellAdapter - attr_reader :old_disk_path, :new_disk_path, :old_storage_version, :logger, :move_wiki + attr_reader :old_disk_path, :new_disk_path, :old_storage_version, + :logger, :move_wiki, :move_design def initialize(project:, old_disk_path:, logger: nil) @project = project @logger = logger || Gitlab::AppLogger @old_disk_path = old_disk_path @move_wiki = has_wiki? + @move_design = has_design? end protected @@ -23,6 +25,10 @@ module Projects gitlab_shell.repository_exists?(project.repository_storage, "#{old_wiki_disk_path}.git") end + def has_design? + gitlab_shell.repository_exists?(project.repository_storage, "#{old_design_disk_path}.git") + end + def move_repository(from_name, to_name) from_exists = gitlab_shell.repository_exists?(project.repository_storage, "#{from_name}.git") to_exists = gitlab_shell.repository_exists?(project.repository_storage, "#{to_name}.git") @@ -58,12 +64,18 @@ module Projects project.clear_memoization(:wiki) end + if move_design + result &&= move_repository(old_design_disk_path, new_design_disk_path) + project.clear_memoization(:design_repository) + end + result end def rollback_folder_move move_repository(new_disk_path, old_disk_path) move_repository(new_wiki_disk_path, old_wiki_disk_path) + move_repository(new_design_disk_path, old_design_disk_path) if move_design end def try_to_set_repository_read_only! @@ -87,8 +99,18 @@ module Projects def new_wiki_disk_path @new_wiki_disk_path ||= "#{new_disk_path}#{wiki_path_suffix}" end + + def design_path_suffix + @design_path_suffix ||= ::Gitlab::GlRepository::DESIGN.path_suffix + end + + def old_design_disk_path + @old_design_disk_path ||= "#{old_disk_path}#{design_path_suffix}" + end + + def new_design_disk_path + @new_design_disk_path ||= "#{new_disk_path}#{design_path_suffix}" + end end end end - -Projects::HashedStorage::BaseRepositoryService.prepend_if_ee('EE::Projects::HashedStorage::BaseRepositoryService') diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index 8893bf18e1f..86cb4f35206 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -3,19 +3,35 @@ module Projects module ImportExport class ExportService < BaseService - def execute(after_export_strategy = nil, options = {}) + prepend Measurable + + def initialize(*args) + super + + @shared = project.import_export_shared + end + + def execute(after_export_strategy = nil) unless project.template_source? || can?(current_user, :admin_project, project) raise ::Gitlab::ImportExport::Error.permission_error(current_user, project) end - @shared = project.import_export_shared - save_all! execute_after_export_action(after_export_strategy) ensure cleanup end + protected + + def extra_attributes_for_measurement + { + current_user: current_user&.name, + project_full_path: project&.full_path, + file_path: shared.export_path + } + end + private attr_accessor :shared @@ -42,7 +58,10 @@ module Projects end def exporters - [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver, snippets_repo_saver] + [ + version_saver, avatar_saver, project_tree_saver, uploads_saver, + repo_saver, wiki_repo_saver, lfs_saver, snippets_repo_saver, design_repo_saver + ] end def version_saver @@ -81,6 +100,10 @@ module Projects Gitlab::ImportExport::SnippetsRepoSaver.new(current_user: current_user, project: project, shared: shared) end + def design_repo_saver + Gitlab::ImportExport::DesignRepoSaver.new(project: project, shared: shared) + end + def cleanup FileUtils.rm_rf(shared.archive_path) if shared&.archive_path end @@ -103,5 +126,3 @@ module Projects end end end - -Projects::ImportExport::ExportService.prepend_if_ee('EE::Projects::ImportExport::ExportService') diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index 4b294a97516..449c4c3de6b 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -3,6 +3,7 @@ module Projects class ImportService < BaseService Error = Class.new(StandardError) + PermissionError = Class.new(StandardError) # Returns true if this importer is supposed to perform its work in the # background. @@ -21,6 +22,8 @@ module Projects import_data + after_execute_hook + success rescue Gitlab::UrlBlocker::BlockedUrlError => e Gitlab::ErrorTracking.track_exception(e, project_path: project.full_path, importer: project.import_type) @@ -34,8 +37,23 @@ module Projects error(s_("ImportProjects|Error importing repository %{project_safe_import_url} into %{project_full_path} - %{message}") % { project_safe_import_url: project.safe_import_url, project_full_path: project.full_path, message: message }) end + protected + + def extra_attributes_for_measurement + { + current_user: current_user&.name, + project_full_path: project&.full_path, + import_type: project&.import_type, + file_path: project&.import_source + } + end + private + def after_execute_hook + # Defined in EE::Projects::ImportService + end + def add_repository_to_project if project.external_import? && !unknown_url? begin @@ -130,3 +148,10 @@ module Projects end end end + +# rubocop: disable Cop/InjectEnterpriseEditionModule +Projects::ImportService.prepend_if_ee('EE::Projects::ImportService') +# rubocop: enable Cop/InjectEnterpriseEditionModule + +# Measurable should be at the bottom of the ancestor chain, so it will measure execution of EE::Projects::ImportService as well +Projects::ImportService.prepend(Measurable) diff --git a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb index 48a21bf94ba..efd410088ab 100644 --- a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb +++ b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb @@ -69,7 +69,7 @@ module Projects # application/vnd.git-lfs+json # (https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md#requests), # HTTParty does not know this is actually JSON. - data = JSON.parse(response.body) + data = Gitlab::Json.parse(response.body) raise DownloadLinksError, "LFS Batch API did return any objects" unless data.is_a?(Hash) && data.key?('objects') diff --git a/app/services/projects/lsif_data_service.rb b/app/services/projects/lsif_data_service.rb index 142a5a910d4..5e7055b3309 100644 --- a/app/services/projects/lsif_data_service.rb +++ b/app/services/projects/lsif_data_service.rb @@ -42,7 +42,7 @@ module Projects file.open do |stream| Zlib::GzipReader.wrap(stream) do |gz_stream| - data = JSON.parse(gz_stream.read) + data = Gitlab::Json.parse(gz_stream.read) end end diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb index 6ebc061c2e3..2583a6cae9f 100644 --- a/app/services/projects/prometheus/alerts/notify_service.rb +++ b/app/services/projects/prometheus/alerts/notify_service.rb @@ -12,6 +12,7 @@ module Projects return unprocessable_entity unless valid_version? return unauthorized unless valid_alert_manager_token?(token) + process_prometheus_alerts persist_events send_alert_email if send_email? process_incident_issues if process_issues? @@ -115,6 +116,14 @@ module Projects end end + def process_prometheus_alerts + alerts.each do |alert| + AlertManagement::ProcessPrometheusAlertService + .new(project, nil, alert.to_h) + .execute + end + end + def persist_events CreateEventsService.new(project, nil, params).execute end diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb index 6013b00b8c6..0483c951f1e 100644 --- a/app/services/projects/propagate_service_template.rb +++ b/app/services/projects/propagate_service_template.rb @@ -4,8 +4,10 @@ module Projects class PropagateServiceTemplate BATCH_SIZE = 100 - def self.propagate(*args) - new(*args).propagate + delegate :data_fields_present?, to: :template + + def self.propagate(template) + new(template).propagate end def initialize(template) @@ -13,15 +15,15 @@ module Projects end def propagate - return unless @template.active? - - Rails.logger.info("Propagating services for template #{@template.id}") # rubocop:disable Gitlab/RailsLogger + return unless template.active? propagate_projects_with_template end private + attr_reader :template + def propagate_projects_with_template loop do batch = Project.uncached { project_ids_batch } @@ -38,7 +40,14 @@ module Projects end Project.transaction do - bulk_insert_services(service_hash.keys << 'project_id', service_list) + results = bulk_insert(Service, service_hash.keys << 'project_id', service_list) + + if data_fields_present? + data_list = results.map { |row| data_hash.values << row['id'] } + + bulk_insert(template.data_fields.class, data_hash.keys << 'service_id', data_list) + end + run_callbacks(batch) end end @@ -52,36 +61,27 @@ module Projects SELECT true FROM services WHERE services.project_id = projects.id - AND services.type = '#{@template.type}' + AND services.type = #{ActiveRecord::Base.connection.quote(template.type)} ) AND projects.pending_delete = false AND projects.archived = false LIMIT #{BATCH_SIZE} - SQL + SQL ) end - def bulk_insert_services(columns, values_array) - ActiveRecord::Base.connection.execute( - <<-SQL.strip_heredoc - INSERT INTO services (#{columns.join(', ')}) - VALUES #{values_array.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} - SQL - ) + def bulk_insert(klass, columns, values_array) + items_to_insert = values_array.map { |array| Hash[columns.zip(array)] } + + klass.insert_all(items_to_insert, returning: [:id]) end def service_hash - @service_hash ||= - begin - template_hash = @template.as_json(methods: :type).except('id', 'template', 'project_id') - - template_hash.each_with_object({}) do |(key, value), service_hash| - value = value.is_a?(Hash) ? value.to_json : value + @service_hash ||= template.as_json(methods: :type, except: %w[id template project_id]) + end - service_hash[ActiveRecord::Base.connection.quote_column_name(key)] = - ActiveRecord::Base.connection.quote(value) - end - end + def data_hash + @data_hash ||= template.data_fields.as_json(only: template.data_fields.class.column_names).except('id', 'service_id') end # rubocop: disable CodeReuse/ActiveRecord @@ -97,11 +97,11 @@ module Projects # rubocop: enable CodeReuse/ActiveRecord def active_external_issue_tracker? - @template.issue_tracker? && !@template.default + template.issue_tracker? && !template.default end def active_external_wiki? - @template.type == 'ExternalWikiService' + template.type == 'ExternalWikiService' end end end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 309eab59463..60e5b7e2639 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -135,7 +135,8 @@ module Projects return if project.hashed_storage?(:repository) move_repo_folder(@new_path, @old_path) - move_repo_folder("#{@new_path}.wiki", "#{@old_path}.wiki") + move_repo_folder(new_wiki_repo_path, old_wiki_repo_path) + move_repo_folder(new_design_repo_path, old_design_repo_path) end def move_repo_folder(from_name, to_name) @@ -157,8 +158,9 @@ module Projects # Disk path is changed; we need to ensure we reload it project.reload_repository! - # Move wiki repo also if present - move_repo_folder("#{@old_path}.wiki", "#{@new_path}.wiki") + # Move wiki and design repos also if present + move_repo_folder(old_wiki_repo_path, new_wiki_repo_path) + move_repo_folder(old_design_repo_path, new_design_repo_path) end def move_project_uploads(project) @@ -170,6 +172,22 @@ module Projects @new_namespace.full_path ) end + + def old_wiki_repo_path + "#{old_path}#{::Gitlab::GlRepository::WIKI.path_suffix}" + end + + def new_wiki_repo_path + "#{new_path}#{::Gitlab::GlRepository::WIKI.path_suffix}" + end + + def old_design_repo_path + "#{old_path}#{::Gitlab::GlRepository::DESIGN.path_suffix}" + end + + def new_design_repo_path + "#{new_path}#{::Gitlab::GlRepository::DESIGN.path_suffix}" + end end end diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb index 13a467a3ef9..e554bed6819 100644 --- a/app/services/projects/update_remote_mirror_service.rb +++ b/app/services/projects/update_remote_mirror_service.rb @@ -29,14 +29,16 @@ module Projects remote_mirror.ensure_remote! repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true) - opts = {} - if remote_mirror.only_protected_branches? - opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name) - end + response = remote_mirror.update_repository - remote_mirror.update_repository(opts) + if response.divergent_refs.any? + message = "Some refs have diverged and have not been updated on the remote:" + message += "\n\n#{response.divergent_refs.join("\n")}" - remote_mirror.update_finish! + remote_mirror.mark_as_failed!(message) + else + remote_mirror.update_finish! + end end def retry_or_fail(mirror, message, tries) diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb index 2e5de9411d1..0632df6f6d7 100644 --- a/app/services/projects/update_repository_storage_service.rb +++ b/app/services/projects/update_repository_storage_service.rb @@ -1,37 +1,49 @@ # frozen_string_literal: true module Projects - class UpdateRepositoryStorageService < BaseService - include Gitlab::ShellAdapter - + class UpdateRepositoryStorageService Error = Class.new(StandardError) SameFilesystemError = Class.new(Error) - def initialize(project) - @project = project + attr_reader :repository_storage_move + delegate :project, :destination_storage_name, to: :repository_storage_move + delegate :repository, to: :project + + def initialize(repository_storage_move) + @repository_storage_move = repository_storage_move end - def execute(new_repository_storage_key) - raise SameFilesystemError if same_filesystem?(project.repository.storage, new_repository_storage_key) + def execute + repository_storage_move.start! - mirror_repositories(new_repository_storage_key) + raise SameFilesystemError if same_filesystem?(repository.storage, destination_storage_name) - mark_old_paths_for_archive + mirror_repositories - project.update(repository_storage: new_repository_storage_key, repository_read_only: false) - project.leave_pool_repository - project.track_project_repository + project.transaction do + mark_old_paths_for_archive + + repository_storage_move.finish! + project.update!(repository_storage: destination_storage_name, repository_read_only: false) + project.leave_pool_repository + project.track_project_repository + end enqueue_housekeeping - success + ServiceResponse.success - rescue Error, ArgumentError, Gitlab::Git::BaseError => e - project.update(repository_read_only: false) + rescue StandardError => e + project.transaction do + repository_storage_move.do_fail! + project.update!(repository_read_only: false) + end Gitlab::ErrorTracking.track_exception(e, project_path: project.full_path) - error(s_("UpdateRepositoryStorage|Error moving repository storage for %{project_full_path} - %{message}") % { project_full_path: project.full_path, message: e.message }) + ServiceResponse.error( + message: s_("UpdateRepositoryStorage|Error moving repository storage for %{project_full_path} - %{message}") % { project_full_path: project.full_path, message: e.message } + ) end private @@ -40,15 +52,19 @@ module Projects Gitlab::GitalyClient.filesystem_id(old_storage) == Gitlab::GitalyClient.filesystem_id(new_storage) end - def mirror_repositories(new_repository_storage_key) - mirror_repository(new_repository_storage_key) + def mirror_repositories + mirror_repository if project.wiki.repository_exists? - mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI) + mirror_repository(type: Gitlab::GlRepository::WIKI) + end + + if project.design_repository.exists? + mirror_repository(type: ::Gitlab::GlRepository::DESIGN) end end - def mirror_repository(new_storage_key, type: Gitlab::GlRepository::PROJECT) + def mirror_repository(type: Gitlab::GlRepository::PROJECT) unless wait_for_pushes(type) raise Error, s_('UpdateRepositoryStorage|Timeout waiting for %{type} repository pushes') % { type: type.name } end @@ -60,7 +76,7 @@ module Projects # Initialize a git repository on the target path new_repository = Gitlab::Git::Repository.new( - new_storage_key, + destination_storage_name, raw_repository.relative_path, raw_repository.gl_repository, full_path @@ -94,11 +110,18 @@ module Projects wiki.disk_path, "#{new_project_path}.wiki") end + + if design_repository.exists? + GitlabShellWorker.perform_async(:mv_repository, + old_repository_storage, + design_repository.disk_path, + "#{new_project_path}.design") + end end end def moved_path(path) - "#{path}+#{project.id}+moved+#{Time.now.to_i}" + "#{path}+#{project.id}+moved+#{Time.current.to_i}" end # The underlying FetchInternalRemote call uses a `git fetch` to move data @@ -128,5 +151,3 @@ module Projects end end end - -Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService') -- cgit v1.2.3