diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-20 15:11:29 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-20 15:11:29 +0300 |
commit | 7b384a1f3d2608898318e67d11eea2914889ae81 (patch) | |
tree | af33e77d13b18a96156c69616cb09a3a4f7697c6 /lib | |
parent | d46287cc16ba244720c6d5c00491944336972988 (diff) |
Add latest changes from gitlab-org/gitlab@12-3-stable
Diffstat (limited to 'lib')
46 files changed, 206 insertions, 804 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb index ffff40141de..a2f3e87ebd2 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -37,7 +37,6 @@ module API optional :path, type: String, desc: 'The file path' optional :all, type: Boolean, desc: 'Every commit will be returned' optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response' - optional :first_parent, type: Boolean, desc: 'Only include the first parent of merges' use :pagination end get ':id/repository/commits' do @@ -48,7 +47,6 @@ module API offset = (params[:page] - 1) * params[:per_page] all = params[:all] with_stats = params[:with_stats] - first_parent = params[:first_parent] commits = user_project.repository.commits(ref, path: path, @@ -56,12 +54,11 @@ module API offset: offset, before: before, after: after, - all: all, - first_parent: first_parent) + all: all) commit_count = - if all || path || before || after || first_parent - user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all, first_parent: first_parent) + if all || path || before || after + user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all) else # Cacheable commit count. user_project.repository.commit_count_for_ref(ref) diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 2a05974509a..c10ef96922c 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -106,15 +106,9 @@ module API authorize_destroy_container_image! validate_tag! - result = ::Projects::ContainerRepository::DeleteTagsService - .new(repository.project, current_user, tags: [declared_params[:tag_name]]) - .execute(repository) - - if result[:status] == :success - status :ok - else - status :bad_request - end + tag.delete + + status :ok end end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index b3f17447ea0..7f1ae5ffbe6 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -29,7 +29,6 @@ module API requires :path, type: String, desc: 'The new project path and name' # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960 requires :file, type: File, desc: 'The project export file to be imported' # rubocop:disable Scalability/FileUploads - optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.' optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace." optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it' optional :override_params, @@ -56,7 +55,6 @@ module API project_params = { path: import_params[:path], namespace_id: namespace.id, - name: import_params[:name], file: import_params[:file]['tempfile'], overwrite: import_params[:overwrite] } diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 2bd8eb65306..15f40993ea3 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -2,7 +2,6 @@ require 'faraday' require 'faraday_middleware' -require 'digest' module ContainerRegistry class Client @@ -10,8 +9,6 @@ module ContainerRegistry DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' - CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json' - ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze # Taken from: FaradayMiddleware::FollowRedirects @@ -39,45 +36,6 @@ module ContainerRegistry faraday.delete("/v2/#{name}/manifests/#{reference}").success? end - def upload_raw_blob(path, blob) - digest = "sha256:#{Digest::SHA256.hexdigest(blob)}" - - if upload_blob(path, blob, digest).success? - [blob, digest] - end - end - - def upload_blob(name, content, digest) - upload = faraday.post("/v2/#{name}/blobs/uploads/") - return unless upload.success? - - location = URI(upload.headers['location']) - - faraday.put("#{location.path}?#{location.query}") do |req| - req.params['digest'] = digest - req.headers['Content-Type'] = 'application/octet-stream' - req.body = content - end - end - - def generate_empty_manifest(path) - image = { - config: {} - } - image, image_digest = upload_raw_blob(path, JSON.pretty_generate(image)) - return unless image - - { - schemaVersion: 2, - mediaType: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, - config: { - mediaType: CONTAINER_IMAGE_V1_TYPE, - size: image.size, - digest: image_digest - } - } - end - def blob(name, digest, type = nil) type ||= 'application/octet-stream' response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true @@ -87,15 +45,6 @@ module ContainerRegistry faraday.delete("/v2/#{name}/blobs/#{digest}").success? end - def put_tag(name, reference, manifest) - response = faraday.put("/v2/#{name}/manifests/#{reference}") do |req| - req.headers['Content-Type'] = DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE - req.body = JSON.pretty_generate(manifest) - end - - response.headers['docker-content-digest'] if response.success? - end - private def initialize_connection(conn, options) diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 2cc4c8d8b1c..ebea84fa1ca 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -98,10 +98,6 @@ module ContainerRegistry end end - def put(digests) - repository.client.put_tag(repository.path, name, digests) - end - # rubocop: disable CodeReuse/ActiveRecord def total_size return unless layers @@ -110,10 +106,7 @@ module ContainerRegistry end # rubocop: enable CodeReuse/ActiveRecord - # Deletes the image associated with this tag - # Note this will delete the image and all tags associated with it. - # Consider using DeleteTagsService instead. - def unsafe_delete + def delete return unless digest client.delete_repository_tag(repository.path, digest) diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 58572446de6..d21f344f483 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -18,8 +18,7 @@ module Gitlab StageEvents::MergeRequestMerged => 104, StageEvents::CodeStageStart => 1_000, StageEvents::IssueStageEnd => 1_001, - StageEvents::PlanStageStart => 1_002, - StageEvents::ProductionStageEnd => 1_003 + StageEvents::PlanStageStart => 1_002 }.freeze EVENTS = ENUM_MAPPING.keys.freeze @@ -33,8 +32,7 @@ module Gitlab StageEvents::MergeRequestCreated ], StageEvents::IssueCreated => [ - StageEvents::IssueStageEnd, - StageEvents::ProductionStageEnd + StageEvents::IssueStageEnd ], StageEvents::MergeRequestCreated => [ StageEvents::MergeRequestMerged diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index 6af1b90bccc..ff9c8a79225 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -16,21 +16,6 @@ module Gitlab def object_type MergeRequest end - - def timestamp_projection - issue_metrics_table[:first_mentioned_in_commit_at] - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - issue_metrics_join = mr_closing_issues_table - .join(issue_metrics_table) - .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id])) - .join_sources - - query.joins(:merge_requests_closing_issues).joins(issue_metrics_join) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb index 8c9a80740a9..a601c9797f8 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb @@ -16,10 +16,6 @@ module Gitlab def object_type Issue end - - def timestamp_projection - issue_table[:created_at] - end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb index fe7f2d85f8b..7424043ef7b 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb @@ -16,16 +16,6 @@ module Gitlab def object_type Issue end - - def timestamp_projection - issue_metrics_table[:first_mentioned_in_commit_at] - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb index 77e4092b9ab..ceb229c552f 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb @@ -16,19 +16,6 @@ module Gitlab def object_type Issue end - - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', [ - issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at] - ]) - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics).where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb index 7059c425b8f..8be00831b4f 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb @@ -16,10 +16,6 @@ module Gitlab def object_type MergeRequest end - - def timestamp_projection - mr_table[:created_at] - end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb index 3d7482eaaf0..6d7a2c023ff 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb @@ -16,16 +16,6 @@ module Gitlab def object_type MergeRequest end - - def timestamp_projection - mr_metrics_table[:first_deployed_to_production_at] - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics).where(timestamp_projection.gteq(mr_table[:created_at])) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb index 36bb4d6fc8d..12d82fe2c62 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb @@ -16,16 +16,6 @@ module Gitlab def object_type MergeRequest end - - def timestamp_projection - mr_metrics_table[:latest_build_finished_at] - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb index 468d9899cc7..9e749b0fdfa 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb @@ -16,16 +16,6 @@ module Gitlab def object_type MergeRequest end - - def timestamp_projection - mr_metrics_table[:latest_build_started_at] - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb index 82ecaf1cd6b..bbfb5d12992 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb @@ -16,16 +16,6 @@ module Gitlab def object_type MergeRequest end - - def timestamp_projection - mr_metrics_table[:merged_at] - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(:metrics) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb index 7ece7d62faa..803317d8b55 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -16,22 +16,6 @@ module Gitlab def object_type Issue end - - def timestamp_projection - Arel::Nodes::NamedFunction.new('COALESCE', [ - issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at] - ]) - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query - .joins(:metrics) - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb deleted file mode 100644 index 607371a32e8..00000000000 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Analytics - module CycleAnalytics - module StageEvents - class ProductionStageEnd < SimpleStageEvent - def self.name - PlanStageStart.name - end - - def self.identifier - :production_stage_end - end - - def object_type - Issue - end - - def timestamp_projection - mr_metrics_table[:first_deployed_to_production_at] - end - - # rubocop: disable CodeReuse/ActiveRecord - def apply_query_customization(query) - query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at])) - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end - end -end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index aa392140eb5..a55eee048c2 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -6,8 +6,6 @@ module Gitlab module StageEvents # Base class for expressing an event that can be used for a stage. class StageEvent - include Gitlab::CycleAnalytics::MetricsTables - def initialize(params) @params = params end @@ -23,21 +21,6 @@ module Gitlab def object_type raise NotImplementedError end - - # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query. - # Example: get me all the Issue records between start event end end event - def timestamp_projection - raise NotImplementedError - end - - # Optionally a StageEvent may apply additional filtering or join other tables on the base query. - def apply_query_customization(query) - query - end - - private - - attr_reader :params end end end diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb deleted file mode 100644 index 4f047ab3ea8..00000000000 --- a/lib/gitlab/artifacts/migration_helper.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Artifacts - class MigrationHelper - def migrate_to_remote_storage(&block) - artifacts = ::Ci::JobArtifact.with_files_stored_locally - migrate(artifacts, ObjectStorage::Store::REMOTE, &block) - end - - def migrate_to_local_storage(&block) - artifacts = ::Ci::JobArtifact.with_files_stored_remotely - migrate(artifacts, ObjectStorage::Store::LOCAL, &block) - end - - private - - def batch_size - ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i - end - - def migrate(artifacts, store, &block) - artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord - artifact.file.migrate!(store) - - yield artifact if block - rescue => e - raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}") - end - end - end - end -end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index c4eb3ad770a..2ac99b1ff02 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -142,13 +142,13 @@ module Gitlab # kwargs.merge(deadline: Time.now + 10) # end # - def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout) + def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil) start = Gitlab::Metrics::System.monotonic_time request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {} enforce_gitaly_request_limits(:call) - kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage) + kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend @@ -200,7 +200,7 @@ module Gitlab end private_class_method :authorization_token - def self.request_kwargs(storage, timeout:, remote_storage: nil) + def self.request_kwargs(storage, timeout, remote_storage: nil) metadata = { 'authorization' => "Bearer #{authorization_token(storage)}", 'client_name' => CLIENT_NAME @@ -216,7 +216,14 @@ module Gitlab result = { metadata: metadata } - result[:deadline] = real_time + timeout if timeout > 0 + # nil timeout indicates that we should use the default + timeout = default_timeout if timeout.nil? + + return result unless timeout > 0 + + deadline = real_time + timeout + result[:deadline] = deadline + result end @@ -350,6 +357,8 @@ module Gitlab # The default timeout on all Gitaly calls def self.default_timeout + return no_timeout if Sidekiq.server? + timeout(:gitaly_timeout_default) end @@ -361,12 +370,8 @@ module Gitlab timeout(:gitaly_timeout_medium) end - def self.long_timeout - if Sidekiq.server? - 6.hours - else - 55.seconds - end + def self.no_timeout + 0 end def self.storage_metadata_file_path(storage) diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb index e2293d3121a..a56bc35f6d7 100644 --- a/lib/gitlab/gitaly_client/cleanup_service.rb +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -18,7 +18,7 @@ module Gitlab :cleanup_service, :apply_bfg_object_map_stream, build_object_map_enum(io), - timeout: GitalyClient.long_timeout + timeout: GitalyClient.no_timeout ) responses.each(&blk) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 2eaf52355dd..a80ce462ab0 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -140,8 +140,7 @@ module Gitlab request = Gitaly::CountCommitsRequest.new( repository: @gitaly_repo, revision: encode_binary(ref), - all: !!options[:all], - first_parent: !!options[:first_parent] + all: !!options[:all] ) request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? @@ -255,7 +254,7 @@ module Gitlab def languages(ref = nil) request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') - response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request) response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } end @@ -326,7 +325,6 @@ module Gitlab follow: options[:follow], skip_merges: options[:skip_merges], all: !!options[:all], - first_parent: !!options[:first_parent], disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] @@ -362,7 +360,7 @@ module Gitlab def extract_signature(commit_id) request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id) - response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request, timeout: GitalyClient.fast_timeout) + response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request) signature = +''.b signed_text = +''.b diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index c497bd12738..d16e45c964d 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -20,7 +20,7 @@ module Gitlab our_commit_oid: @our_commit_oid, their_commit_oid: @their_commit_oid ) - response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request) GitalyClient::ConflictFilesStitcher.new(response) end diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index 0be214f3035..f0be3cbebd2 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -22,7 +22,7 @@ module Gitlab def remove(name) request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(:remove_namespace, request, timeout: nil) end def rename(from, to) diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index 786ef0ebebe..d7fac26bc13 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -15,15 +15,13 @@ module Gitlab object_pool: object_pool, origin: repository.gitaly_repository) - GitalyClient.call(storage, :object_pool_service, :create_object_pool, - request, timeout: GitalyClient.medium_timeout) + GitalyClient.call(storage, :object_pool_service, :create_object_pool, request) end def delete request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool) - GitalyClient.call(storage, :object_pool_service, :delete_object_pool, - request, timeout: GitalyClient.long_timeout) + GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request) end def link_repository(repository) @@ -42,8 +40,7 @@ module Gitlab origin: repository.gitaly_repository ) - GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, - request, timeout: GitalyClient.long_timeout) + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request) end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 6e486c763da..33ca428a942 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -19,7 +19,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -35,7 +35,7 @@ module Gitlab message: encode_binary(message.to_s) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error elsif response.exists @@ -55,7 +55,7 @@ module Gitlab start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :operation_service, - :user_create_branch, request, timeout: GitalyClient.long_timeout) + :user_create_branch, request) if response.pre_receive_error.present? raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error) @@ -79,8 +79,7 @@ module Gitlab oldrev: encode_binary(oldrev) ) - response = GitalyClient.call(@repository.storage, :operation_service, - :user_update_branch, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -94,8 +93,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, - :user_delete_branch, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -113,8 +111,7 @@ module Gitlab first_parent_ref: encode_binary(first_parent_ref) ) - response = GitalyClient.call(@repository.storage, :operation_service, - :user_merge_to_ref, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -129,8 +126,7 @@ module Gitlab @repository.storage, :operation_service, :user_merge_branch, - request_enum.each, - timeout: GitalyClient.long_timeout + request_enum.each ) request_enum.push( @@ -174,8 +170,7 @@ module Gitlab @repository.storage, :operation_service, :user_ff_branch, - request, - timeout: GitalyClient.long_timeout + request ) Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) @@ -220,7 +215,6 @@ module Gitlab :operation_service, :user_rebase, request, - timeout: GitalyClient.long_timeout, remote_storage: remote_repository.storage ) @@ -242,7 +236,6 @@ module Gitlab :operation_service, :user_rebase_confirmable, request_enum.each, - timeout: GitalyClient.long_timeout, remote_storage: remote_repository.storage ) @@ -293,8 +286,7 @@ module Gitlab @repository.storage, :operation_service, :user_squash, - request, - timeout: GitalyClient.long_timeout + request ) if response.git_error.presence @@ -318,8 +310,7 @@ module Gitlab @repository.storage, :operation_service, :user_update_submodule, - request, - timeout: GitalyClient.long_timeout + request ) if response.pre_receive_error.present? @@ -361,8 +352,7 @@ module Gitlab end response = GitalyClient.call(@repository.storage, :operation_service, - :user_commit_files, req_enum, timeout: GitalyClient.long_timeout, - remote_storage: start_repository.storage) + :user_commit_files, req_enum, remote_storage: start_repository.storage) if (pre_receive_error = response.pre_receive_error.presence) raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -394,8 +384,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :operation_service, - :user_apply_patch, chunks, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks) Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end @@ -435,7 +424,7 @@ module Gitlab :"user_#{rpc}", request, remote_storage: start_repository.storage, - timeout: GitalyClient.long_timeout + timeout: GitalyClient.medium_timeout ) handle_cherry_pick_or_revert_response(response) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index d1f848fae26..b7d509dfa48 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -21,7 +21,7 @@ module Gitlab def remote_branches(remote_name) request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name) - response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request) consume_find_all_remote_branches_response(remote_name, response) end @@ -158,7 +158,7 @@ module Gitlab start_point: encode_binary(start_point) ) - response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) case response.status when :OK @@ -182,7 +182,7 @@ module Gitlab name: encode_binary(branch_name) ) - GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request, timeout: GitalyClient.medium_timeout) + GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) end def delete_refs(refs: [], except_with_prefixes: []) @@ -192,7 +192,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end @@ -242,7 +242,7 @@ module Gitlab def pack_refs request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :ref_service, :pack_refs, request) end private diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index d01a29e1a05..f3589fea39f 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -38,7 +38,9 @@ module Gitlab def remove_remote(name) request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) - GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result + response = GitalyClient.call(@storage, :remote_service, :remove_remote, request) + + response.result end def fetch_internal_remote(repository) @@ -49,7 +51,6 @@ module Gitlab response = GitalyClient.call(@storage, :remote_service, :fetch_internal_remote, request, - timeout: GitalyClient.medium_timeout, remote_storage: repository.storage) response.result @@ -62,7 +63,7 @@ module Gitlab ) response = GitalyClient.call(@storage, :remote_service, - :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) + :find_remote_root_ref, request) encode_utf8(response.ref) end @@ -94,7 +95,7 @@ module Gitlab end end - GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum) end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index fc5bfa8e850..ca3e5b51ecc 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -28,17 +28,17 @@ module Gitlab def garbage_collect(create_bitmap) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :repository_service, :garbage_collect, request) end def repack_full(create_bitmap) request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - GitalyClient.call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :repository_service, :repack_full, request) end def repack_incremental request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :repository_service, :repack_incremental, request) end def repository_size @@ -86,12 +86,12 @@ module Gitlab end end - GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :repository_service, :fetch_remote, request) end def create_repository request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) + GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout) end def has_local_branches? @@ -189,7 +189,6 @@ module Gitlab :repository_service, :fetch_source_branch, request, - timeout: GitalyClient.default_timeout, remote_storage: source_repository.storage ) @@ -198,7 +197,7 @@ module Gitlab def fsck request = Gitaly::FsckRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout) + response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.no_timeout) if response.error.empty? return "", 0 @@ -212,7 +211,7 @@ module Gitlab save_path, :create_bundle, Gitaly::CreateBundleRequest, - GitalyClient.long_timeout + GitalyClient.no_timeout ) end @@ -230,7 +229,7 @@ module Gitlab bundle_path, :create_repository_from_bundle, Gitaly::CreateRepositoryFromBundleRequest, - GitalyClient.long_timeout + GitalyClient.no_timeout ) end @@ -255,7 +254,7 @@ module Gitlab :repository_service, :create_repository_from_snapshot, request, - timeout: GitalyClient.long_timeout + timeout: GitalyClient.no_timeout ) end @@ -334,7 +333,7 @@ module Gitlab def search_files_by_content(ref, query) request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) - response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) + response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request) search_results_from_response(response) end @@ -344,7 +343,7 @@ module Gitlab repository: @gitaly_repo ) - GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request) end private diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb index 94fb0d87364..4edcb0b8ba9 100644 --- a/lib/gitlab/gitaly_client/storage_service.rb +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -11,14 +11,14 @@ module Gitlab def list_directories(depth: 1) request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth) - GitalyClient.call(@storage, :storage_service, :list_directories, request, timeout: GitalyClient.medium_timeout) + GitalyClient.call(@storage, :storage_service, :list_directories, request) .flat_map(&:paths) end # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. def delete_all_repositories request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) - GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request, timeout: GitalyClient.long_timeout) + GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request) end end end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 15e0d7349dd..ce9faad825c 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -34,7 +34,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum) if error = response.duplicate_error.presence raise Gitlab::Git::Wiki::DuplicatePageError, error end @@ -61,7 +61,7 @@ module Gitlab end end - GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) + GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum) end def delete_page(page_path, commit_details) @@ -187,7 +187,7 @@ module Gitlab directory: encode_binary(dir) ) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request) response.reduce([]) { |memo, msg| memo << msg.data }.join end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index fe3a8d4aea5..9d925581441 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -32,11 +32,10 @@ module Gitlab def build(release) { - name: release.name, tag: release.tag_name, description: description_for(release), created_at: release.created_at, - updated_at: release.created_at, + updated_at: release.updated_at, released_at: release.published_at, project_id: project.id } diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index b30258123d4..c5fb39b7b52 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -10,9 +10,11 @@ module Gitlab StrategyError = Class.new(StandardError) + AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action' + private - attr_reader :project, :current_user, :lock_file + attr_reader :project, :current_user public @@ -27,9 +29,8 @@ module Gitlab def execute(current_user, project) @project = project - ensure_export_ready! - ensure_lock_files_path! - @lock_file = File.join(lock_files_path, SecureRandom.hex) + return unless @project.export_status == :finished + @current_user = current_user if invalid? @@ -47,32 +48,19 @@ module Gitlab false ensure delete_after_export_lock - delete_export_file - delete_archive_path end def to_json(options = {}) @options.to_h.merge!(klass: self.class.name).to_json end - def ensure_export_ready! - raise StrategyError unless project.export_file_exists? - end - - def ensure_lock_files_path! - FileUtils.mkdir_p(lock_files_path) unless Dir.exist?(lock_files_path) - end - - def lock_files_path - project.import_export_shared.lock_files_path - end + def self.lock_file_path(project) + return unless project.export_path || export_file_exists? - def archive_path - project.import_export_shared.archive_path - end + lock_path = project.import_export_shared.archive_path - def locks_present? - project.import_export_shared.locks_present? + mkdir_p(lock_path) + File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME) end protected @@ -81,33 +69,25 @@ module Gitlab raise NotImplementedError end - def delete_export? - true - end - private - def delete_export_file - return if locks_present? || !delete_export? - - project.remove_exports - end - - def delete_archive_path - FileUtils.rm_rf(archive_path) if File.directory?(archive_path) - end - def create_or_update_after_export_lock - FileUtils.touch(lock_file) + FileUtils.touch(self.class.lock_file_path(project)) end def delete_after_export_lock + lock_file = self.class.lock_file_path(project) + FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file) end def log_validation_errors errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end + + def export_file_exists? + project.export_file_exists? + end end end end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb index 39a6090ad87..1b391314a74 100644 --- a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -4,12 +4,6 @@ module Gitlab module ImportExport module AfterExportStrategies class DownloadNotificationStrategy < BaseAfterExportStrategy - protected - - def delete_export? - false - end - private def strategy_execute diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index fd98bc2caad..aaa70f0b36d 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -24,6 +24,8 @@ module Gitlab def strategy_execute handle_response_error(send_file) + + project.remove_exports end def handle_response_error(response) diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 9144a4406cd..725c1101d70 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,32 +1,10 @@ # frozen_string_literal: true -# -# This class encapsulates the directories used by project import/export: -# -# 1. The project export job first generates the project metadata tree -# (e.g. `project.json) and repository bundle (e.g. `project.bundle`) -# inside a temporary `export_path` -# (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA/:randomB). -# -# 2. The job then creates a tarball (e.g. `project.tar.gz`) in -# `archive_path` (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA). -# CarrierWave moves this tarball files into its permanent location. -# -# 3. Lock files are used to indicate whether a project is in the -# `after_export` state. These are stored in a directory -# (e.g. /path/to/shared/tmp/project_exports/namespace/project/locks. The -# number of lock files present signifies how many concurrent project -# exports are running. Note that this assumes the temporary directory -# is a shared mount: -# https://gitlab.com/gitlab-org/gitlab/issues/32203 -# -# NOTE: Stale files should be cleaned up via ImportExportCleanupService. + module Gitlab module ImportExport class Shared attr_reader :errors, :project - LOCKS_DIRECTORY = 'locks' - def initialize(project) @project = project @errors = [] @@ -34,27 +12,17 @@ module Gitlab end def active_export_count - Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) } + Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) } end - # The path where the project metadata and repository bundle is saved def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end - # The path where the tarball is saved def archive_path @archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path) end - def base_path - @base_path ||= Gitlab::ImportExport.export_path(relative_path: relative_base_path) - end - - def lock_files_path - @locks_files_path ||= File.join(base_path, LOCKS_DIRECTORY) - end - def error(error) log_error(message: error.message, caller: caller[0].dup) log_debug(backtrace: error.backtrace&.join("\n")) @@ -69,24 +37,16 @@ module Gitlab end def after_export_in_progress? - locks_present? - end - - def locks_present? - Dir.exist?(lock_files_path) && !Dir.empty?(lock_files_path) + File.exist?(after_export_lock_file) end private def relative_path - @relative_path ||= File.join(relative_archive_path, SecureRandom.hex) + File.join(relative_archive_path, SecureRandom.hex) end def relative_archive_path - @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex) - end - - def relative_base_path @project.disk_path end @@ -110,6 +70,10 @@ module Gitlab def filtered_error_message(message) Projects::ImportErrorFilter.filter_message(message) end + + def after_export_lock_file + AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project) + end end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 404e0c31871..7e64fe2a1f4 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -135,8 +135,7 @@ module Gitlab end types Issue condition do - !quick_action_target.confidential? && - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end command :confidential do @updates[:confidential] = true diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb deleted file mode 100644 index 25b91318dfe..00000000000 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ /dev/null @@ -1,221 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqDaemon - class MemoryKiller < Daemon - include ::Gitlab::Utils::StrongMemoize - - # Today 64-bit CPU support max 256T memory. It is big enough. - MAX_MEMORY_KB = 256 * 1024 * 1024 * 1024 - # RSS below `soft_limit_rss` is considered safe - SOFT_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_MAX_RSS', 2000000).to_i - # RSS above `hard_limit_rss` will be stopped - HARD_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', MAX_MEMORY_KB).to_i - # RSS in range (soft_limit_rss, hard_limit_rss) is allowed for GRACE_BALLOON_SECONDS - GRACE_BALLOON_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', 15 * 60).to_i - # Check RSS every CHECK_INTERVAL_SECONDS, minimum 2 seconds - CHECK_INTERVAL_SECONDS = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max - # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit - SHUTDOWN_TIMEOUT_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i - # Developer/admin should always set `memory_killer_max_memory_growth_kb` explicitly - # In case not set, default to 300M. This is for extra-safe. - DEFAULT_MAX_MEMORY_GROWTH_KB = 300_000 - - def initialize - super - - @enabled = true - end - - private - - def start_working - Sidekiq.logger.info( - class: self.class.to_s, - action: 'start', - pid: pid, - message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon' - ) - - while enabled? - begin - restart_sidekiq unless rss_within_range? - sleep(CHECK_INTERVAL_SECONDS) - rescue => e - log_exception(e, __method__) - rescue Exception => e # rubocop:disable Lint/RescueException - log_exception(e, __method__ ) - raise e - end - end - ensure - Sidekiq.logger.warn( - class: self.class.to_s, - action: 'stop', - pid: pid, - message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon' - ) - end - - def log_exception(exception, method) - Sidekiq.logger.warn( - class: self.class.to_s, - pid: pid, - message: "Exception from #{method}: #{exception.message}" - ) - end - - def stop_working - @enabled = false - end - - def enabled? - @enabled - end - - def restart_sidekiq - # Tell Sidekiq to stop fetching new jobs - # We first SIGNAL and then wait given time - # We also monitor a number of running jobs and allow to restart early - signal_and_wait(SHUTDOWN_TIMEOUT_SECONDS, 'SIGTSTP', 'stop fetching new jobs') - return unless enabled? - - # Tell sidekiq to restart itself - # Keep extra safe to wait `Sidekiq.options[:timeout] + 2` seconds before SIGKILL - signal_and_wait(Sidekiq.options[:timeout] + 2, 'SIGTERM', 'gracefully shut down') - return unless enabled? - - # Ideally we should never reach this condition - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't - # Kill the whole pgroup, so we can be sure no children are left behind - signal_pgroup('SIGKILL', 'die') - end - - def rss_within_range? - current_rss = nil - deadline = Gitlab::Metrics::System.monotonic_time + GRACE_BALLOON_SECONDS.seconds - loop do - return true unless enabled? - - current_rss = get_rss - - # RSS go above hard limit should trigger forcible shutdown right away - break if current_rss > hard_limit_rss - - # RSS go below the soft limit - return true if current_rss < soft_limit_rss - - # RSS did not go below the soft limit within deadline, restart - break if Gitlab::Metrics::System.monotonic_time > deadline - - sleep(CHECK_INTERVAL_SECONDS) - end - - log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss) - - false - end - - def log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss) - Sidekiq.logger.warn( - class: self.class.to_s, - pid: pid, - message: 'Sidekiq worker RSS out of range', - current_rss: current_rss, - hard_limit_rss: hard_limit_rss, - soft_limit_rss: soft_limit_rss, - reason: out_of_range_description(current_rss, hard_limit_rss, soft_limit_rss) - ) - end - - def out_of_range_description(rss, hard_limit, soft_limit) - if rss > hard_limit - "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})" - else - "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})" - end - end - - def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) - return 0 unless status&.zero? - - output.to_i - end - - def soft_limit_rss - SOFT_LIMIT_RSS_KB + rss_increase_by_jobs - end - - def hard_limit_rss - HARD_LIMIT_RSS_KB - end - - def signal_and_wait(time, signal, explanation) - Sidekiq.logger.warn( - class: self.class.to_s, - pid: pid, - signal: signal, - explanation: explanation, - wait_time: time, - message: "Sending signal and waiting" - ) - Process.kill(signal, pid) - - deadline = Gitlab::Metrics::System.monotonic_time + time - - # we try to finish as early as all jobs finished - # so we retest that in loop - sleep(CHECK_INTERVAL_SECONDS) while enabled? && any_jobs? && Gitlab::Metrics::System.monotonic_time < deadline - end - - def signal_pgroup(signal, explanation) - if Process.getpgrp == pid - pid_or_pgrp_str = 'PGRP' - pid_to_signal = 0 - else - pid_or_pgrp_str = 'PID' - pid_to_signal = pid - end - - Sidekiq.logger.warn( - class: self.class.to_s, - signal: signal, - pid: pid, - message: "sending Sidekiq worker #{pid_or_pgrp_str}-#{pid} #{signal} (#{explanation})" - ) - Process.kill(signal, pid_to_signal) - end - - def rss_increase_by_jobs - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord - rss_increase_by_job(job) - end - end - - def rss_increase_by_job(job) - memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i - max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i - - return 0 if memory_growth_kb.zero? - - time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max - [memory_growth_kb * time_elapsed, max_memory_growth_kb].min - end - - def get_job_options(job, key, default) - job[:worker_class].sidekiq_options.fetch(key, default) - rescue - default - end - - def pid - Process.pid - end - - def any_jobs? - Gitlab::SidekiqDaemon::Monitor.instance.jobs.any? - end - end - end -end diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb index 09f30837cd2..bbfca130425 100644 --- a/lib/gitlab/sidekiq_daemon/monitor.rb +++ b/lib/gitlab/sidekiq_daemon/monitor.rb @@ -14,19 +14,19 @@ module Gitlab # that should not be caught by application CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException - attr_reader :jobs + attr_reader :jobs_thread attr_reader :jobs_mutex def initialize super - @jobs = {} + @jobs_thread = {} @jobs_mutex = Mutex.new end - def within_job(worker_class, jid, queue) + def within_job(jid, queue) jobs_mutex.synchronize do - jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time } + jobs_thread[jid] = Thread.current end if cancelled?(jid) @@ -43,7 +43,7 @@ module Gitlab yield ensure jobs_mutex.synchronize do - jobs.delete(jid) + jobs_thread.delete(jid) end end @@ -62,27 +62,23 @@ module Gitlab private def start_working - return unless notification_channel_enabled? - - begin - Sidekiq.logger.info( - class: self.class.to_s, - action: 'start', - message: 'Starting Monitor Daemon' - ) - - while enabled? - process_messages - sleep(RECONNECT_TIME) - end + Sidekiq.logger.info( + class: self.class.to_s, + action: 'start', + message: 'Starting Monitor Daemon' + ) - ensure - Sidekiq.logger.warn( - class: self.class.to_s, - action: 'stop', - message: 'Stopping Monitor Daemon' - ) + while enabled? + process_messages + sleep(RECONNECT_TIME) end + + ensure + Sidekiq.logger.warn( + class: self.class.to_s, + action: 'stop', + message: 'Stopping Monitor Daemon' + ) end def stop_working @@ -160,7 +156,7 @@ module Gitlab # This is why it passes thread in block, # to ensure that we do process this thread def find_thread_unsafe(jid) - jobs.dig(jid, :thread) + jobs_thread[jid] end def find_thread_with_lock(jid) @@ -183,10 +179,6 @@ module Gitlab def self.cancel_job_key(jid) "sidekiq:cancel:#{jid}" end - - def notification_channel_enabled? - ENV.fetch("SIDEKIQ_MONITOR_WORKER", 0).to_i.nonzero? - end end end end diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb index ed825dbfd60..00965bf5506 100644 --- a/lib/gitlab/sidekiq_middleware/monitor.rb +++ b/lib/gitlab/sidekiq_middleware/monitor.rb @@ -4,7 +4,7 @@ module Gitlab module SidekiqMiddleware class Monitor def call(worker, job, queue) - Gitlab::SidekiqDaemon::Monitor.instance.within_job(worker.class, job['jid'], queue) do + Gitlab::SidekiqDaemon::Monitor.instance.within_job(job['jid'], queue) do yield end rescue Gitlab::SidekiqDaemon::Monitor::CancelledError diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb deleted file mode 100644 index 4ff064007f1..00000000000 --- a/lib/gitlab/uploads/migration_helper.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Uploads - class MigrationHelper - attr_reader :logger - - CATEGORIES = [%w(AvatarUploader Project :avatar), - %w(AvatarUploader Group :avatar), - %w(AvatarUploader User :avatar), - %w(AttachmentUploader Note :attachment), - %w(AttachmentUploader Appearance :logo), - %w(AttachmentUploader Appearance :header_logo), - %w(FaviconUploader Appearance :favicon), - %w(FileUploader Project), - %w(PersonalFileUploader Snippet), - %w(NamespaceFileUploader Snippet), - %w(FileUploader MergeRequest)].freeze - - def initialize(args, logger) - prepare_variables(args, logger) - end - - def migrate_to_remote_storage - @to_store = ObjectStorage::Store::REMOTE - - uploads.each_batch(of: batch_size, &method(:enqueue_batch)) - end - - def migrate_to_local_storage - @to_store = ObjectStorage::Store::LOCAL - - uploads(ObjectStorage::Store::REMOTE).each_batch(of: batch_size, &method(:enqueue_batch)) - end - - private - - def batch_size - ENV.fetch('MIGRATION_BATCH_SIZE', 200).to_i - end - - def prepare_variables(args, logger) - @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym - @uploader_class = args.uploader_class.constantize - @model_class = args.model_class.constantize - @logger = logger - end - - def enqueue_batch(batch, index) - job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, - @model_class, - @mounted_as, - @to_store) - logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job) - rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e - # continue for the next batch - logger.warn(message: "[Uploads migration] Could not enqueue batch", ids: batch.ids, reason: e.message) # rubocop:disable CodeReuse/ActiveRecord - end - - # rubocop:disable CodeReuse/ActiveRecord - def uploads(store_type = [nil, ObjectStorage::Store::LOCAL]) - Upload.class_eval { include EachBatch } unless Upload < EachBatch - - Upload - .where(store: store_type, - uploader: @uploader_class.to_s, - model_type: @model_class.base_class.sti_name) - end - # rubocop:enable CodeReuse/ActiveRecord - end - end -end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index ed2693aaedf..c6c2876033d 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -116,7 +116,6 @@ module Gitlab def features_usage_data_ce { container_registry_enabled: Gitlab.config.registry.enabled, - dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled, gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled, gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?, influxdb_metrics_enabled: Gitlab::Metrics.influx_metrics_enabled?, diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 0d09fd0a4e3..9012e55a70c 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -6,31 +6,18 @@ namespace :gitlab do namespace :artifacts do task migrate: :environment do logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts to remote storage') + logger.info('Starting transfer of artifacts') - helper = Gitlab::Artifacts::MigrationHelper.new + Ci::Build.joins(:project) + .with_artifacts_stored_locally + .find_each(batch_size: 10) do |build| - begin - helper.migrate_to_remote_storage do |artifact| - logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage") - end - rescue => e - logger.error(e.message) - end - end - - task migrate_to_local: :environment do - logger = Logger.new(STDOUT) - logger.info('Starting transfer of artifacts to local storage') - - helper = Gitlab::Artifacts::MigrationHelper.new + build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE) + build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE) - begin - helper.migrate_to_local_storage do |artifact| - logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage") - end + logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage") rescue => e - logger.error(e.message) + logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") end end end diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 4142903d9c3..97c15175a23 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -17,20 +17,5 @@ namespace :gitlab do logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") end end - - task migrate_to_local: :environment do - logger = Logger.new(STDOUT) - logger.info('Starting transfer of LFS files to local storage') - - LfsObject.with_files_stored_remotely - .find_each(batch_size: 10) do |lfs_object| - - lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL) - - logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage") - rescue => e - logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}") - end - end end end diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake new file mode 100644 index 00000000000..5e1ec481ece --- /dev/null +++ b/lib/tasks/gitlab/traces.rake @@ -0,0 +1,38 @@ +require 'logger' +require 'resolv-replace' + +desc "GitLab | Archive legacy traces to trace artifacts" +namespace :gitlab do + namespace :traces do + task archive: :environment do + logger = Logger.new(STDOUT) + logger.info('Archiving legacy traces') + + Ci::Build.finished.without_archived_trace + .order(id: :asc) + .find_in_batches(batch_size: 1000) do |jobs| + job_ids = jobs.map { |job| [job.id] } + + ArchiveTraceWorker.bulk_perform_async(job_ids) + + logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}") + end + end + + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of job traces') + + Ci::Build.joins(:project) + .with_archived_trace_stored_locally + .find_each(batch_size: 10) do |build| + + build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE) + + logger.info("Transferred job trace of #{build.id} to object storage") + rescue => e + logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}") + end + end + end +end diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 44536a447c7..1c93609a006 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -3,7 +3,19 @@ namespace :gitlab do namespace :migrate do desc "GitLab | Uploads | Migrate all uploaded files to object storage" task all: :environment do - Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + categories = [%w(AvatarUploader Project :avatar), + %w(AvatarUploader Group :avatar), + %w(AvatarUploader User :avatar), + %w(AttachmentUploader Note :attachment), + %w(AttachmentUploader Appearance :logo), + %w(AttachmentUploader Appearance :header_logo), + %w(FaviconUploader Appearance :favicon), + %w(FileUploader Project), + %w(PersonalFileUploader Snippet), + %w(NamespaceFileUploader Snippet), + %w(FileUploader MergeRequest)] + + categories.each do |args| Rake::Task["gitlab:uploads:migrate"].invoke(*args) Rake::Task["gitlab:uploads:migrate"].reenable end @@ -13,23 +25,34 @@ namespace :gitlab do # The following is the actual rake task that migrates uploads of specified # category to object storage desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage' - task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| - Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage + task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args| + batch_size = ENV.fetch('BATCH', 200).to_i + @to_store = ObjectStorage::Store::REMOTE + @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym + @uploader_class = args.uploader_class.constantize + @model_class = args.model_class.constantize + + uploads.each_batch(of: batch_size, &method(:enqueue_batch)) end - namespace :migrate_to_local do - desc "GitLab | Uploads | Migrate all uploaded files to local storage" - task all: :environment do - Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| - Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args) - Rake::Task["gitlab:uploads:migrate_to_local"].reenable - end - end + def enqueue_batch(batch, index) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, + @model_class, + @mounted_as, + @to_store) + puts "Enqueued job ##{index}: #{job}" + rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e + # continue for the next batch + puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red) end - desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage' - task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args| - Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage + def uploads + Upload.class_eval { include EachBatch } unless Upload < EachBatch + + Upload + .where(store: [nil, ObjectStorage::Store::LOCAL], + uploader: @uploader_class.to_s, + model_type: @model_class.base_class.sti_name) end end end |