Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2019-09-20 15:11:29 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2019-09-20 15:11:29 +0300
commit7b384a1f3d2608898318e67d11eea2914889ae81 (patch)
treeaf33e77d13b18a96156c69616cb09a3a4f7697c6 /lib
parentd46287cc16ba244720c6d5c00491944336972988 (diff)
Add latest changes from gitlab-org/gitlab@12-3-stable
Diffstat (limited to 'lib')
-rw-r--r--lib/api/commits.rb9
-rw-r--r--lib/api/project_container_repositories.rb12
-rw-r--r--lib/api/project_import.rb2
-rw-r--r--lib/container_registry/client.rb51
-rw-r--r--lib/container_registry/tag.rb9
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events.rb6
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb15
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb4
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb10
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb13
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb4
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb10
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb10
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb10
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb10
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb16
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb33
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb17
-rw-r--r--lib/gitlab/artifacts/migration_helper.rb33
-rw-r--r--lib/gitlab/gitaly_client.rb25
-rw-r--r--lib/gitlab/gitaly_client/cleanup_service.rb2
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb8
-rw-r--r--lib/gitlab/gitaly_client/conflicts_service.rb2
-rw-r--r--lib/gitlab/gitaly_client/namespace_service.rb2
-rw-r--r--lib/gitlab/gitaly_client/object_pool_service.rb9
-rw-r--r--lib/gitlab/gitaly_client/operation_service.rb37
-rw-r--r--lib/gitlab/gitaly_client/ref_service.rb10
-rw-r--r--lib/gitlab/gitaly_client/remote_service.rb9
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb23
-rw-r--r--lib/gitlab/gitaly_client/storage_service.rb4
-rw-r--r--lib/gitlab/gitaly_client/wiki_service.rb6
-rw-r--r--lib/gitlab/github_import/importer/releases_importer.rb3
-rw-r--r--lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb54
-rw-r--r--lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb6
-rw-r--r--lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb2
-rw-r--r--lib/gitlab/import_export/shared.rb52
-rw-r--r--lib/gitlab/quick_actions/issue_actions.rb3
-rw-r--r--lib/gitlab/sidekiq_daemon/memory_killer.rb221
-rw-r--r--lib/gitlab/sidekiq_daemon/monitor.rb50
-rw-r--r--lib/gitlab/sidekiq_middleware/monitor.rb2
-rw-r--r--lib/gitlab/uploads/migration_helper.rb72
-rw-r--r--lib/gitlab/usage_data.rb1
-rw-r--r--lib/tasks/gitlab/artifacts/migrate.rake29
-rw-r--r--lib/tasks/gitlab/lfs/migrate.rake15
-rw-r--r--lib/tasks/gitlab/traces.rake38
-rw-r--r--lib/tasks/gitlab/uploads/migrate.rake51
46 files changed, 206 insertions, 804 deletions
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index ffff40141de..a2f3e87ebd2 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -37,7 +37,6 @@ module API
optional :path, type: String, desc: 'The file path'
optional :all, type: Boolean, desc: 'Every commit will be returned'
optional :with_stats, type: Boolean, desc: 'Stats about each commit will be added to the response'
- optional :first_parent, type: Boolean, desc: 'Only include the first parent of merges'
use :pagination
end
get ':id/repository/commits' do
@@ -48,7 +47,6 @@ module API
offset = (params[:page] - 1) * params[:per_page]
all = params[:all]
with_stats = params[:with_stats]
- first_parent = params[:first_parent]
commits = user_project.repository.commits(ref,
path: path,
@@ -56,12 +54,11 @@ module API
offset: offset,
before: before,
after: after,
- all: all,
- first_parent: first_parent)
+ all: all)
commit_count =
- if all || path || before || after || first_parent
- user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all, first_parent: first_parent)
+ if all || path || before || after
+ user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all)
else
# Cacheable commit count.
user_project.repository.commit_count_for_ref(ref)
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index 2a05974509a..c10ef96922c 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -106,15 +106,9 @@ module API
authorize_destroy_container_image!
validate_tag!
- result = ::Projects::ContainerRepository::DeleteTagsService
- .new(repository.project, current_user, tags: [declared_params[:tag_name]])
- .execute(repository)
-
- if result[:status] == :success
- status :ok
- else
- status :bad_request
- end
+ tag.delete
+
+ status :ok
end
end
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index b3f17447ea0..7f1ae5ffbe6 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -29,7 +29,6 @@ module API
requires :path, type: String, desc: 'The new project path and name'
# TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
requires :file, type: File, desc: 'The project export file to be imported' # rubocop:disable Scalability/FileUploads
- optional :name, type: String, desc: 'The name of the project to be imported. Defaults to the path of the project if not provided.'
optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it'
optional :override_params,
@@ -56,7 +55,6 @@ module API
project_params = {
path: import_params[:path],
namespace_id: namespace.id,
- name: import_params[:name],
file: import_params[:file]['tempfile'],
overwrite: import_params[:overwrite]
}
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 2bd8eb65306..15f40993ea3 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -2,7 +2,6 @@
require 'faraday'
require 'faraday_middleware'
-require 'digest'
module ContainerRegistry
class Client
@@ -10,8 +9,6 @@ module ContainerRegistry
DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json'
- CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json'
-
ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze
# Taken from: FaradayMiddleware::FollowRedirects
@@ -39,45 +36,6 @@ module ContainerRegistry
faraday.delete("/v2/#{name}/manifests/#{reference}").success?
end
- def upload_raw_blob(path, blob)
- digest = "sha256:#{Digest::SHA256.hexdigest(blob)}"
-
- if upload_blob(path, blob, digest).success?
- [blob, digest]
- end
- end
-
- def upload_blob(name, content, digest)
- upload = faraday.post("/v2/#{name}/blobs/uploads/")
- return unless upload.success?
-
- location = URI(upload.headers['location'])
-
- faraday.put("#{location.path}?#{location.query}") do |req|
- req.params['digest'] = digest
- req.headers['Content-Type'] = 'application/octet-stream'
- req.body = content
- end
- end
-
- def generate_empty_manifest(path)
- image = {
- config: {}
- }
- image, image_digest = upload_raw_blob(path, JSON.pretty_generate(image))
- return unless image
-
- {
- schemaVersion: 2,
- mediaType: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE,
- config: {
- mediaType: CONTAINER_IMAGE_V1_TYPE,
- size: image.size,
- digest: image_digest
- }
- }
- end
-
def blob(name, digest, type = nil)
type ||= 'application/octet-stream'
response_body faraday_blob.get("/v2/#{name}/blobs/#{digest}", nil, 'Accept' => type), allow_redirect: true
@@ -87,15 +45,6 @@ module ContainerRegistry
faraday.delete("/v2/#{name}/blobs/#{digest}").success?
end
- def put_tag(name, reference, manifest)
- response = faraday.put("/v2/#{name}/manifests/#{reference}") do |req|
- req.headers['Content-Type'] = DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE
- req.body = JSON.pretty_generate(manifest)
- end
-
- response.headers['docker-content-digest'] if response.success?
- end
-
private
def initialize_connection(conn, options)
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 2cc4c8d8b1c..ebea84fa1ca 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -98,10 +98,6 @@ module ContainerRegistry
end
end
- def put(digests)
- repository.client.put_tag(repository.path, name, digests)
- end
-
# rubocop: disable CodeReuse/ActiveRecord
def total_size
return unless layers
@@ -110,10 +106,7 @@ module ContainerRegistry
end
# rubocop: enable CodeReuse/ActiveRecord
- # Deletes the image associated with this tag
- # Note this will delete the image and all tags associated with it.
- # Consider using DeleteTagsService instead.
- def unsafe_delete
+ def delete
return unless digest
client.delete_repository_tag(repository.path, digest)
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
index 58572446de6..d21f344f483 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb
@@ -18,8 +18,7 @@ module Gitlab
StageEvents::MergeRequestMerged => 104,
StageEvents::CodeStageStart => 1_000,
StageEvents::IssueStageEnd => 1_001,
- StageEvents::PlanStageStart => 1_002,
- StageEvents::ProductionStageEnd => 1_003
+ StageEvents::PlanStageStart => 1_002
}.freeze
EVENTS = ENUM_MAPPING.keys.freeze
@@ -33,8 +32,7 @@ module Gitlab
StageEvents::MergeRequestCreated
],
StageEvents::IssueCreated => [
- StageEvents::IssueStageEnd,
- StageEvents::ProductionStageEnd
+ StageEvents::IssueStageEnd
],
StageEvents::MergeRequestCreated => [
StageEvents::MergeRequestMerged
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
index 6af1b90bccc..ff9c8a79225 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb
@@ -16,21 +16,6 @@ module Gitlab
def object_type
MergeRequest
end
-
- def timestamp_projection
- issue_metrics_table[:first_mentioned_in_commit_at]
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- issue_metrics_join = mr_closing_issues_table
- .join(issue_metrics_table)
- .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id]))
- .join_sources
-
- query.joins(:merge_requests_closing_issues).joins(issue_metrics_join)
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
index 8c9a80740a9..a601c9797f8 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb
@@ -16,10 +16,6 @@ module Gitlab
def object_type
Issue
end
-
- def timestamp_projection
- issue_table[:created_at]
- end
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
index fe7f2d85f8b..7424043ef7b 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb
@@ -16,16 +16,6 @@ module Gitlab
def object_type
Issue
end
-
- def timestamp_projection
- issue_metrics_table[:first_mentioned_in_commit_at]
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query.joins(:metrics)
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
index 77e4092b9ab..ceb229c552f 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb
@@ -16,19 +16,6 @@ module Gitlab
def object_type
Issue
end
-
- def timestamp_projection
- Arel::Nodes::NamedFunction.new('COALESCE', [
- issue_metrics_table[:first_associated_with_milestone_at],
- issue_metrics_table[:first_added_to_board_at]
- ])
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query.joins(:metrics).where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
index 7059c425b8f..8be00831b4f 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb
@@ -16,10 +16,6 @@ module Gitlab
def object_type
MergeRequest
end
-
- def timestamp_projection
- mr_table[:created_at]
- end
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
index 3d7482eaaf0..6d7a2c023ff 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb
@@ -16,16 +16,6 @@ module Gitlab
def object_type
MergeRequest
end
-
- def timestamp_projection
- mr_metrics_table[:first_deployed_to_production_at]
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query.joins(:metrics).where(timestamp_projection.gteq(mr_table[:created_at]))
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
index 36bb4d6fc8d..12d82fe2c62 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb
@@ -16,16 +16,6 @@ module Gitlab
def object_type
MergeRequest
end
-
- def timestamp_projection
- mr_metrics_table[:latest_build_finished_at]
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query.joins(:metrics)
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
index 468d9899cc7..9e749b0fdfa 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb
@@ -16,16 +16,6 @@ module Gitlab
def object_type
MergeRequest
end
-
- def timestamp_projection
- mr_metrics_table[:latest_build_started_at]
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query.joins(:metrics)
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
index 82ecaf1cd6b..bbfb5d12992 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb
@@ -16,16 +16,6 @@ module Gitlab
def object_type
MergeRequest
end
-
- def timestamp_projection
- mr_metrics_table[:merged_at]
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query.joins(:metrics)
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
index 7ece7d62faa..803317d8b55 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb
@@ -16,22 +16,6 @@ module Gitlab
def object_type
Issue
end
-
- def timestamp_projection
- Arel::Nodes::NamedFunction.new('COALESCE', [
- issue_metrics_table[:first_associated_with_milestone_at],
- issue_metrics_table[:first_added_to_board_at]
- ])
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query
- .joins(:metrics)
- .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
- .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil))
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
deleted file mode 100644
index 607371a32e8..00000000000
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Analytics
- module CycleAnalytics
- module StageEvents
- class ProductionStageEnd < SimpleStageEvent
- def self.name
- PlanStageStart.name
- end
-
- def self.identifier
- :production_stage_end
- end
-
- def object_type
- Issue
- end
-
- def timestamp_projection
- mr_metrics_table[:first_deployed_to_production_at]
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def apply_query_customization(query)
- query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at]))
- end
- # rubocop: enable CodeReuse/ActiveRecord
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
index aa392140eb5..a55eee048c2 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb
@@ -6,8 +6,6 @@ module Gitlab
module StageEvents
# Base class for expressing an event that can be used for a stage.
class StageEvent
- include Gitlab::CycleAnalytics::MetricsTables
-
def initialize(params)
@params = params
end
@@ -23,21 +21,6 @@ module Gitlab
def object_type
raise NotImplementedError
end
-
- # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query.
- # Example: get me all the Issue records between start event end end event
- def timestamp_projection
- raise NotImplementedError
- end
-
- # Optionally a StageEvent may apply additional filtering or join other tables on the base query.
- def apply_query_customization(query)
- query
- end
-
- private
-
- attr_reader :params
end
end
end
diff --git a/lib/gitlab/artifacts/migration_helper.rb b/lib/gitlab/artifacts/migration_helper.rb
deleted file mode 100644
index 4f047ab3ea8..00000000000
--- a/lib/gitlab/artifacts/migration_helper.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Artifacts
- class MigrationHelper
- def migrate_to_remote_storage(&block)
- artifacts = ::Ci::JobArtifact.with_files_stored_locally
- migrate(artifacts, ObjectStorage::Store::REMOTE, &block)
- end
-
- def migrate_to_local_storage(&block)
- artifacts = ::Ci::JobArtifact.with_files_stored_remotely
- migrate(artifacts, ObjectStorage::Store::LOCAL, &block)
- end
-
- private
-
- def batch_size
- ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
- end
-
- def migrate(artifacts, store, &block)
- artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord
- artifact.file.migrate!(store)
-
- yield artifact if block
- rescue => e
- raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}")
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index c4eb3ad770a..2ac99b1ff02 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -142,13 +142,13 @@ module Gitlab
# kwargs.merge(deadline: Time.now + 10)
# end
#
- def self.call(storage, service, rpc, request, remote_storage: nil, timeout: default_timeout)
+ def self.call(storage, service, rpc, request, remote_storage: nil, timeout: nil)
start = Gitlab::Metrics::System.monotonic_time
request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {}
enforce_gitaly_request_limits(:call)
- kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage)
+ kwargs = request_kwargs(storage, timeout, remote_storage: remote_storage)
kwargs = yield(kwargs) if block_given?
stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend
@@ -200,7 +200,7 @@ module Gitlab
end
private_class_method :authorization_token
- def self.request_kwargs(storage, timeout:, remote_storage: nil)
+ def self.request_kwargs(storage, timeout, remote_storage: nil)
metadata = {
'authorization' => "Bearer #{authorization_token(storage)}",
'client_name' => CLIENT_NAME
@@ -216,7 +216,14 @@ module Gitlab
result = { metadata: metadata }
- result[:deadline] = real_time + timeout if timeout > 0
+ # nil timeout indicates that we should use the default
+ timeout = default_timeout if timeout.nil?
+
+ return result unless timeout > 0
+
+ deadline = real_time + timeout
+ result[:deadline] = deadline
+
result
end
@@ -350,6 +357,8 @@ module Gitlab
# The default timeout on all Gitaly calls
def self.default_timeout
+ return no_timeout if Sidekiq.server?
+
timeout(:gitaly_timeout_default)
end
@@ -361,12 +370,8 @@ module Gitlab
timeout(:gitaly_timeout_medium)
end
- def self.long_timeout
- if Sidekiq.server?
- 6.hours
- else
- 55.seconds
- end
+ def self.no_timeout
+ 0
end
def self.storage_metadata_file_path(storage)
diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb
index e2293d3121a..a56bc35f6d7 100644
--- a/lib/gitlab/gitaly_client/cleanup_service.rb
+++ b/lib/gitlab/gitaly_client/cleanup_service.rb
@@ -18,7 +18,7 @@ module Gitlab
:cleanup_service,
:apply_bfg_object_map_stream,
build_object_map_enum(io),
- timeout: GitalyClient.long_timeout
+ timeout: GitalyClient.no_timeout
)
responses.each(&blk)
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 2eaf52355dd..a80ce462ab0 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -140,8 +140,7 @@ module Gitlab
request = Gitaly::CountCommitsRequest.new(
repository: @gitaly_repo,
revision: encode_binary(ref),
- all: !!options[:all],
- first_parent: !!options[:first_parent]
+ all: !!options[:all]
)
request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present?
request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present?
@@ -255,7 +254,7 @@ module Gitlab
def languages(ref = nil)
request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '')
- response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request)
response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } }
end
@@ -326,7 +325,6 @@ module Gitlab
follow: options[:follow],
skip_merges: options[:skip_merges],
all: !!options[:all],
- first_parent: !!options[:first_parent],
disable_walk: true # This option is deprecated. The 'walk' implementation is being removed.
)
request.after = GitalyClient.timestamp(options[:after]) if options[:after]
@@ -362,7 +360,7 @@ module Gitlab
def extract_signature(commit_id)
request = Gitaly::ExtractCommitSignatureRequest.new(repository: @gitaly_repo, commit_id: commit_id)
- response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request, timeout: GitalyClient.fast_timeout)
+ response = GitalyClient.call(@repository.storage, :commit_service, :extract_commit_signature, request)
signature = +''.b
signed_text = +''.b
diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb
index c497bd12738..d16e45c964d 100644
--- a/lib/gitlab/gitaly_client/conflicts_service.rb
+++ b/lib/gitlab/gitaly_client/conflicts_service.rb
@@ -20,7 +20,7 @@ module Gitlab
our_commit_oid: @our_commit_oid,
their_commit_oid: @their_commit_oid
)
- response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request)
GitalyClient::ConflictFilesStitcher.new(response)
end
diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb
index 0be214f3035..f0be3cbebd2 100644
--- a/lib/gitlab/gitaly_client/namespace_service.rb
+++ b/lib/gitlab/gitaly_client/namespace_service.rb
@@ -22,7 +22,7 @@ module Gitlab
def remove(name)
request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name)
- gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout)
+ gitaly_client_call(:remove_namespace, request, timeout: nil)
end
def rename(from, to)
diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb
index 786ef0ebebe..d7fac26bc13 100644
--- a/lib/gitlab/gitaly_client/object_pool_service.rb
+++ b/lib/gitlab/gitaly_client/object_pool_service.rb
@@ -15,15 +15,13 @@ module Gitlab
object_pool: object_pool,
origin: repository.gitaly_repository)
- GitalyClient.call(storage, :object_pool_service, :create_object_pool,
- request, timeout: GitalyClient.medium_timeout)
+ GitalyClient.call(storage, :object_pool_service, :create_object_pool, request)
end
def delete
request = Gitaly::DeleteObjectPoolRequest.new(object_pool: object_pool)
- GitalyClient.call(storage, :object_pool_service, :delete_object_pool,
- request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(storage, :object_pool_service, :delete_object_pool, request)
end
def link_repository(repository)
@@ -42,8 +40,7 @@ module Gitlab
origin: repository.gitaly_repository
)
- GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool,
- request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
end
end
end
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 6e486c763da..33ca428a942 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -19,7 +19,7 @@ module Gitlab
user: Gitlab::Git::User.from_gitlab(user).to_gitaly
)
- response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout)
if pre_receive_error = response.pre_receive_error.presence
raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -35,7 +35,7 @@ module Gitlab
message: encode_binary(message.to_s)
)
- response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout)
if pre_receive_error = response.pre_receive_error.presence
raise Gitlab::Git::PreReceiveError, pre_receive_error
elsif response.exists
@@ -55,7 +55,7 @@ module Gitlab
start_point: encode_binary(start_point)
)
response = GitalyClient.call(@repository.storage, :operation_service,
- :user_create_branch, request, timeout: GitalyClient.long_timeout)
+ :user_create_branch, request)
if response.pre_receive_error.present?
raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error)
@@ -79,8 +79,7 @@ module Gitlab
oldrev: encode_binary(oldrev)
)
- response = GitalyClient.call(@repository.storage, :operation_service,
- :user_update_branch, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :operation_service, :user_update_branch, request)
if pre_receive_error = response.pre_receive_error.presence
raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -94,8 +93,7 @@ module Gitlab
user: Gitlab::Git::User.from_gitlab(user).to_gitaly
)
- response = GitalyClient.call(@repository.storage, :operation_service,
- :user_delete_branch, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_branch, request)
if pre_receive_error = response.pre_receive_error.presence
raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -113,8 +111,7 @@ module Gitlab
first_parent_ref: encode_binary(first_parent_ref)
)
- response = GitalyClient.call(@repository.storage, :operation_service,
- :user_merge_to_ref, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request)
if pre_receive_error = response.pre_receive_error.presence
raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -129,8 +126,7 @@ module Gitlab
@repository.storage,
:operation_service,
:user_merge_branch,
- request_enum.each,
- timeout: GitalyClient.long_timeout
+ request_enum.each
)
request_enum.push(
@@ -174,8 +170,7 @@ module Gitlab
@repository.storage,
:operation_service,
:user_ff_branch,
- request,
- timeout: GitalyClient.long_timeout
+ request
)
Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
@@ -220,7 +215,6 @@ module Gitlab
:operation_service,
:user_rebase,
request,
- timeout: GitalyClient.long_timeout,
remote_storage: remote_repository.storage
)
@@ -242,7 +236,6 @@ module Gitlab
:operation_service,
:user_rebase_confirmable,
request_enum.each,
- timeout: GitalyClient.long_timeout,
remote_storage: remote_repository.storage
)
@@ -293,8 +286,7 @@ module Gitlab
@repository.storage,
:operation_service,
:user_squash,
- request,
- timeout: GitalyClient.long_timeout
+ request
)
if response.git_error.presence
@@ -318,8 +310,7 @@ module Gitlab
@repository.storage,
:operation_service,
:user_update_submodule,
- request,
- timeout: GitalyClient.long_timeout
+ request
)
if response.pre_receive_error.present?
@@ -361,8 +352,7 @@ module Gitlab
end
response = GitalyClient.call(@repository.storage, :operation_service,
- :user_commit_files, req_enum, timeout: GitalyClient.long_timeout,
- remote_storage: start_repository.storage)
+ :user_commit_files, req_enum, remote_storage: start_repository.storage)
if (pre_receive_error = response.pre_receive_error.presence)
raise Gitlab::Git::PreReceiveError, pre_receive_error
@@ -394,8 +384,7 @@ module Gitlab
end
end
- response = GitalyClient.call(@repository.storage, :operation_service,
- :user_apply_patch, chunks, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks)
Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
end
@@ -435,7 +424,7 @@ module Gitlab
:"user_#{rpc}",
request,
remote_storage: start_repository.storage,
- timeout: GitalyClient.long_timeout
+ timeout: GitalyClient.medium_timeout
)
handle_cherry_pick_or_revert_response(response)
diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb
index d1f848fae26..b7d509dfa48 100644
--- a/lib/gitlab/gitaly_client/ref_service.rb
+++ b/lib/gitlab/gitaly_client/ref_service.rb
@@ -21,7 +21,7 @@ module Gitlab
def remote_branches(remote_name)
request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name)
- response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout)
+ response = GitalyClient.call(@repository.storage, :ref_service, :find_all_remote_branches, request)
consume_find_all_remote_branches_response(remote_name, response)
end
@@ -158,7 +158,7 @@ module Gitlab
start_point: encode_binary(start_point)
)
- response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request, timeout: GitalyClient.medium_timeout)
+ response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request)
case response.status
when :OK
@@ -182,7 +182,7 @@ module Gitlab
name: encode_binary(branch_name)
)
- GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request, timeout: GitalyClient.medium_timeout)
+ GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request)
end
def delete_refs(refs: [], except_with_prefixes: [])
@@ -192,7 +192,7 @@ module Gitlab
except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) }
)
- response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout)
+ response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.default_timeout)
raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present?
end
@@ -242,7 +242,7 @@ module Gitlab
def pack_refs
request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo)
- GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :ref_service, :pack_refs, request)
end
private
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index d01a29e1a05..f3589fea39f 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -38,7 +38,9 @@ module Gitlab
def remove_remote(name)
request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name)
- GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result
+ response = GitalyClient.call(@storage, :remote_service, :remove_remote, request)
+
+ response.result
end
def fetch_internal_remote(repository)
@@ -49,7 +51,6 @@ module Gitlab
response = GitalyClient.call(@storage, :remote_service,
:fetch_internal_remote, request,
- timeout: GitalyClient.medium_timeout,
remote_storage: repository.storage)
response.result
@@ -62,7 +63,7 @@ module Gitlab
)
response = GitalyClient.call(@storage, :remote_service,
- :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout)
+ :find_remote_root_ref, request)
encode_utf8(response.ref)
end
@@ -94,7 +95,7 @@ module Gitlab
end
end
- GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :remote_service, :update_remote_mirror, req_enum)
end
end
end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index fc5bfa8e850..ca3e5b51ecc 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -28,17 +28,17 @@ module Gitlab
def garbage_collect(create_bitmap)
request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
- GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :repository_service, :garbage_collect, request)
end
def repack_full(create_bitmap)
request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
- GitalyClient.call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :repository_service, :repack_full, request)
end
def repack_incremental
request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo)
- GitalyClient.call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :repository_service, :repack_incremental, request)
end
def repository_size
@@ -86,12 +86,12 @@ module Gitlab
end
end
- GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :repository_service, :fetch_remote, request)
end
def create_repository
request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo)
- GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout)
+ GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout)
end
def has_local_branches?
@@ -189,7 +189,6 @@ module Gitlab
:repository_service,
:fetch_source_branch,
request,
- timeout: GitalyClient.default_timeout,
remote_storage: source_repository.storage
)
@@ -198,7 +197,7 @@ module Gitlab
def fsck
request = Gitaly::FsckRequest.new(repository: @gitaly_repo)
- response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout)
+ response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.no_timeout)
if response.error.empty?
return "", 0
@@ -212,7 +211,7 @@ module Gitlab
save_path,
:create_bundle,
Gitaly::CreateBundleRequest,
- GitalyClient.long_timeout
+ GitalyClient.no_timeout
)
end
@@ -230,7 +229,7 @@ module Gitlab
bundle_path,
:create_repository_from_bundle,
Gitaly::CreateRepositoryFromBundleRequest,
- GitalyClient.long_timeout
+ GitalyClient.no_timeout
)
end
@@ -255,7 +254,7 @@ module Gitlab
:repository_service,
:create_repository_from_snapshot,
request,
- timeout: GitalyClient.long_timeout
+ timeout: GitalyClient.no_timeout
)
end
@@ -334,7 +333,7 @@ module Gitlab
def search_files_by_content(ref, query)
request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query)
- response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout)
+ response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request)
search_results_from_response(response)
end
@@ -344,7 +343,7 @@ module Gitlab
repository: @gitaly_repo
)
- GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request)
end
private
diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb
index 94fb0d87364..4edcb0b8ba9 100644
--- a/lib/gitlab/gitaly_client/storage_service.rb
+++ b/lib/gitlab/gitaly_client/storage_service.rb
@@ -11,14 +11,14 @@ module Gitlab
def list_directories(depth: 1)
request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth)
- GitalyClient.call(@storage, :storage_service, :list_directories, request, timeout: GitalyClient.medium_timeout)
+ GitalyClient.call(@storage, :storage_service, :list_directories, request)
.flat_map(&:paths)
end
# Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation.
def delete_all_repositories
request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage)
- GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request, timeout: GitalyClient.long_timeout)
+ GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request)
end
end
end
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index 15e0d7349dd..ce9faad825c 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -34,7 +34,7 @@ module Gitlab
end
end
- response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum, timeout: GitalyClient.medium_timeout)
+ response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_write_page, enum)
if error = response.duplicate_error.presence
raise Gitlab::Git::Wiki::DuplicatePageError, error
end
@@ -61,7 +61,7 @@ module Gitlab
end
end
- GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout)
+ GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum)
end
def delete_page(page_path, commit_details)
@@ -187,7 +187,7 @@ module Gitlab
directory: encode_binary(dir)
)
- response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request, timeout: GitalyClient.medium_timeout)
+ response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request)
response.reduce([]) { |memo, msg| memo << msg.data }.join
end
diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb
index fe3a8d4aea5..9d925581441 100644
--- a/lib/gitlab/github_import/importer/releases_importer.rb
+++ b/lib/gitlab/github_import/importer/releases_importer.rb
@@ -32,11 +32,10 @@ module Gitlab
def build(release)
{
- name: release.name,
tag: release.tag_name,
description: description_for(release),
created_at: release.created_at,
- updated_at: release.created_at,
+ updated_at: release.updated_at,
released_at: release.published_at,
project_id: project.id
}
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index b30258123d4..c5fb39b7b52 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -10,9 +10,11 @@ module Gitlab
StrategyError = Class.new(StandardError)
+ AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'
+
private
- attr_reader :project, :current_user, :lock_file
+ attr_reader :project, :current_user
public
@@ -27,9 +29,8 @@ module Gitlab
def execute(current_user, project)
@project = project
- ensure_export_ready!
- ensure_lock_files_path!
- @lock_file = File.join(lock_files_path, SecureRandom.hex)
+ return unless @project.export_status == :finished
+
@current_user = current_user
if invalid?
@@ -47,32 +48,19 @@ module Gitlab
false
ensure
delete_after_export_lock
- delete_export_file
- delete_archive_path
end
def to_json(options = {})
@options.to_h.merge!(klass: self.class.name).to_json
end
- def ensure_export_ready!
- raise StrategyError unless project.export_file_exists?
- end
-
- def ensure_lock_files_path!
- FileUtils.mkdir_p(lock_files_path) unless Dir.exist?(lock_files_path)
- end
-
- def lock_files_path
- project.import_export_shared.lock_files_path
- end
+ def self.lock_file_path(project)
+ return unless project.export_path || export_file_exists?
- def archive_path
- project.import_export_shared.archive_path
- end
+ lock_path = project.import_export_shared.archive_path
- def locks_present?
- project.import_export_shared.locks_present?
+ mkdir_p(lock_path)
+ File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME)
end
protected
@@ -81,33 +69,25 @@ module Gitlab
raise NotImplementedError
end
- def delete_export?
- true
- end
-
private
- def delete_export_file
- return if locks_present? || !delete_export?
-
- project.remove_exports
- end
-
- def delete_archive_path
- FileUtils.rm_rf(archive_path) if File.directory?(archive_path)
- end
-
def create_or_update_after_export_lock
- FileUtils.touch(lock_file)
+ FileUtils.touch(self.class.lock_file_path(project))
end
def delete_after_export_lock
+ lock_file = self.class.lock_file_path(project)
+
FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file)
end
def log_validation_errors
errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
end
+
+ def export_file_exists?
+ project.export_file_exists?
+ end
end
end
end
diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
index 39a6090ad87..1b391314a74 100644
--- a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
@@ -4,12 +4,6 @@ module Gitlab
module ImportExport
module AfterExportStrategies
class DownloadNotificationStrategy < BaseAfterExportStrategy
- protected
-
- def delete_export?
- false
- end
-
private
def strategy_execute
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index fd98bc2caad..aaa70f0b36d 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -24,6 +24,8 @@ module Gitlab
def strategy_execute
handle_response_error(send_file)
+
+ project.remove_exports
end
def handle_response_error(response)
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 9144a4406cd..725c1101d70 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -1,32 +1,10 @@
# frozen_string_literal: true
-#
-# This class encapsulates the directories used by project import/export:
-#
-# 1. The project export job first generates the project metadata tree
-# (e.g. `project.json) and repository bundle (e.g. `project.bundle`)
-# inside a temporary `export_path`
-# (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA/:randomB).
-#
-# 2. The job then creates a tarball (e.g. `project.tar.gz`) in
-# `archive_path` (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA).
-# CarrierWave moves this tarball files into its permanent location.
-#
-# 3. Lock files are used to indicate whether a project is in the
-# `after_export` state. These are stored in a directory
-# (e.g. /path/to/shared/tmp/project_exports/namespace/project/locks. The
-# number of lock files present signifies how many concurrent project
-# exports are running. Note that this assumes the temporary directory
-# is a shared mount:
-# https://gitlab.com/gitlab-org/gitlab/issues/32203
-#
-# NOTE: Stale files should be cleaned up via ImportExportCleanupService.
+
module Gitlab
module ImportExport
class Shared
attr_reader :errors, :project
- LOCKS_DIRECTORY = 'locks'
-
def initialize(project)
@project = project
@errors = []
@@ -34,27 +12,17 @@ module Gitlab
end
def active_export_count
- Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
+ Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) }
end
- # The path where the project metadata and repository bundle is saved
def export_path
@export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
end
- # The path where the tarball is saved
def archive_path
@archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path)
end
- def base_path
- @base_path ||= Gitlab::ImportExport.export_path(relative_path: relative_base_path)
- end
-
- def lock_files_path
- @locks_files_path ||= File.join(base_path, LOCKS_DIRECTORY)
- end
-
def error(error)
log_error(message: error.message, caller: caller[0].dup)
log_debug(backtrace: error.backtrace&.join("\n"))
@@ -69,24 +37,16 @@ module Gitlab
end
def after_export_in_progress?
- locks_present?
- end
-
- def locks_present?
- Dir.exist?(lock_files_path) && !Dir.empty?(lock_files_path)
+ File.exist?(after_export_lock_file)
end
private
def relative_path
- @relative_path ||= File.join(relative_archive_path, SecureRandom.hex)
+ File.join(relative_archive_path, SecureRandom.hex)
end
def relative_archive_path
- @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex)
- end
-
- def relative_base_path
@project.disk_path
end
@@ -110,6 +70,10 @@ module Gitlab
def filtered_error_message(message)
Projects::ImportErrorFilter.filter_message(message)
end
+
+ def after_export_lock_file
+ AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project)
+ end
end
end
end
diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb
index 404e0c31871..7e64fe2a1f4 100644
--- a/lib/gitlab/quick_actions/issue_actions.rb
+++ b/lib/gitlab/quick_actions/issue_actions.rb
@@ -135,8 +135,7 @@ module Gitlab
end
types Issue
condition do
- !quick_action_target.confidential? &&
- current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
+ current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target)
end
command :confidential do
@updates[:confidential] = true
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb
deleted file mode 100644
index 25b91318dfe..00000000000
--- a/lib/gitlab/sidekiq_daemon/memory_killer.rb
+++ /dev/null
@@ -1,221 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module SidekiqDaemon
- class MemoryKiller < Daemon
- include ::Gitlab::Utils::StrongMemoize
-
- # Today 64-bit CPU support max 256T memory. It is big enough.
- MAX_MEMORY_KB = 256 * 1024 * 1024 * 1024
- # RSS below `soft_limit_rss` is considered safe
- SOFT_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_MAX_RSS', 2000000).to_i
- # RSS above `hard_limit_rss` will be stopped
- HARD_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', MAX_MEMORY_KB).to_i
- # RSS in range (soft_limit_rss, hard_limit_rss) is allowed for GRACE_BALLOON_SECONDS
- GRACE_BALLOON_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', 15 * 60).to_i
- # Check RSS every CHECK_INTERVAL_SECONDS, minimum 2 seconds
- CHECK_INTERVAL_SECONDS = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max
- # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit
- SHUTDOWN_TIMEOUT_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i
- # Developer/admin should always set `memory_killer_max_memory_growth_kb` explicitly
- # In case not set, default to 300M. This is for extra-safe.
- DEFAULT_MAX_MEMORY_GROWTH_KB = 300_000
-
- def initialize
- super
-
- @enabled = true
- end
-
- private
-
- def start_working
- Sidekiq.logger.info(
- class: self.class.to_s,
- action: 'start',
- pid: pid,
- message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon'
- )
-
- while enabled?
- begin
- restart_sidekiq unless rss_within_range?
- sleep(CHECK_INTERVAL_SECONDS)
- rescue => e
- log_exception(e, __method__)
- rescue Exception => e # rubocop:disable Lint/RescueException
- log_exception(e, __method__ )
- raise e
- end
- end
- ensure
- Sidekiq.logger.warn(
- class: self.class.to_s,
- action: 'stop',
- pid: pid,
- message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon'
- )
- end
-
- def log_exception(exception, method)
- Sidekiq.logger.warn(
- class: self.class.to_s,
- pid: pid,
- message: "Exception from #{method}: #{exception.message}"
- )
- end
-
- def stop_working
- @enabled = false
- end
-
- def enabled?
- @enabled
- end
-
- def restart_sidekiq
- # Tell Sidekiq to stop fetching new jobs
- # We first SIGNAL and then wait given time
- # We also monitor a number of running jobs and allow to restart early
- signal_and_wait(SHUTDOWN_TIMEOUT_SECONDS, 'SIGTSTP', 'stop fetching new jobs')
- return unless enabled?
-
- # Tell sidekiq to restart itself
- # Keep extra safe to wait `Sidekiq.options[:timeout] + 2` seconds before SIGKILL
- signal_and_wait(Sidekiq.options[:timeout] + 2, 'SIGTERM', 'gracefully shut down')
- return unless enabled?
-
- # Ideally we should never reach this condition
- # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't
- # Kill the whole pgroup, so we can be sure no children are left behind
- signal_pgroup('SIGKILL', 'die')
- end
-
- def rss_within_range?
- current_rss = nil
- deadline = Gitlab::Metrics::System.monotonic_time + GRACE_BALLOON_SECONDS.seconds
- loop do
- return true unless enabled?
-
- current_rss = get_rss
-
- # RSS go above hard limit should trigger forcible shutdown right away
- break if current_rss > hard_limit_rss
-
- # RSS go below the soft limit
- return true if current_rss < soft_limit_rss
-
- # RSS did not go below the soft limit within deadline, restart
- break if Gitlab::Metrics::System.monotonic_time > deadline
-
- sleep(CHECK_INTERVAL_SECONDS)
- end
-
- log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss)
-
- false
- end
-
- def log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss)
- Sidekiq.logger.warn(
- class: self.class.to_s,
- pid: pid,
- message: 'Sidekiq worker RSS out of range',
- current_rss: current_rss,
- hard_limit_rss: hard_limit_rss,
- soft_limit_rss: soft_limit_rss,
- reason: out_of_range_description(current_rss, hard_limit_rss, soft_limit_rss)
- )
- end
-
- def out_of_range_description(rss, hard_limit, soft_limit)
- if rss > hard_limit
- "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})"
- else
- "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})"
- end
- end
-
- def get_rss
- output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
- return 0 unless status&.zero?
-
- output.to_i
- end
-
- def soft_limit_rss
- SOFT_LIMIT_RSS_KB + rss_increase_by_jobs
- end
-
- def hard_limit_rss
- HARD_LIMIT_RSS_KB
- end
-
- def signal_and_wait(time, signal, explanation)
- Sidekiq.logger.warn(
- class: self.class.to_s,
- pid: pid,
- signal: signal,
- explanation: explanation,
- wait_time: time,
- message: "Sending signal and waiting"
- )
- Process.kill(signal, pid)
-
- deadline = Gitlab::Metrics::System.monotonic_time + time
-
- # we try to finish as early as all jobs finished
- # so we retest that in loop
- sleep(CHECK_INTERVAL_SECONDS) while enabled? && any_jobs? && Gitlab::Metrics::System.monotonic_time < deadline
- end
-
- def signal_pgroup(signal, explanation)
- if Process.getpgrp == pid
- pid_or_pgrp_str = 'PGRP'
- pid_to_signal = 0
- else
- pid_or_pgrp_str = 'PID'
- pid_to_signal = pid
- end
-
- Sidekiq.logger.warn(
- class: self.class.to_s,
- signal: signal,
- pid: pid,
- message: "sending Sidekiq worker #{pid_or_pgrp_str}-#{pid} #{signal} (#{explanation})"
- )
- Process.kill(signal, pid_to_signal)
- end
-
- def rss_increase_by_jobs
- Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord
- rss_increase_by_job(job)
- end
- end
-
- def rss_increase_by_job(job)
- memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i
- max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i
-
- return 0 if memory_growth_kb.zero?
-
- time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max
- [memory_growth_kb * time_elapsed, max_memory_growth_kb].min
- end
-
- def get_job_options(job, key, default)
- job[:worker_class].sidekiq_options.fetch(key, default)
- rescue
- default
- end
-
- def pid
- Process.pid
- end
-
- def any_jobs?
- Gitlab::SidekiqDaemon::Monitor.instance.jobs.any?
- end
- end
- end
-end
diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb
index 09f30837cd2..bbfca130425 100644
--- a/lib/gitlab/sidekiq_daemon/monitor.rb
+++ b/lib/gitlab/sidekiq_daemon/monitor.rb
@@ -14,19 +14,19 @@ module Gitlab
# that should not be caught by application
CancelledError = Class.new(Exception) # rubocop:disable Lint/InheritException
- attr_reader :jobs
+ attr_reader :jobs_thread
attr_reader :jobs_mutex
def initialize
super
- @jobs = {}
+ @jobs_thread = {}
@jobs_mutex = Mutex.new
end
- def within_job(worker_class, jid, queue)
+ def within_job(jid, queue)
jobs_mutex.synchronize do
- jobs[jid] = { worker_class: worker_class, thread: Thread.current, started_at: Gitlab::Metrics::System.monotonic_time }
+ jobs_thread[jid] = Thread.current
end
if cancelled?(jid)
@@ -43,7 +43,7 @@ module Gitlab
yield
ensure
jobs_mutex.synchronize do
- jobs.delete(jid)
+ jobs_thread.delete(jid)
end
end
@@ -62,27 +62,23 @@ module Gitlab
private
def start_working
- return unless notification_channel_enabled?
-
- begin
- Sidekiq.logger.info(
- class: self.class.to_s,
- action: 'start',
- message: 'Starting Monitor Daemon'
- )
-
- while enabled?
- process_messages
- sleep(RECONNECT_TIME)
- end
+ Sidekiq.logger.info(
+ class: self.class.to_s,
+ action: 'start',
+ message: 'Starting Monitor Daemon'
+ )
- ensure
- Sidekiq.logger.warn(
- class: self.class.to_s,
- action: 'stop',
- message: 'Stopping Monitor Daemon'
- )
+ while enabled?
+ process_messages
+ sleep(RECONNECT_TIME)
end
+
+ ensure
+ Sidekiq.logger.warn(
+ class: self.class.to_s,
+ action: 'stop',
+ message: 'Stopping Monitor Daemon'
+ )
end
def stop_working
@@ -160,7 +156,7 @@ module Gitlab
# This is why it passes thread in block,
# to ensure that we do process this thread
def find_thread_unsafe(jid)
- jobs.dig(jid, :thread)
+ jobs_thread[jid]
end
def find_thread_with_lock(jid)
@@ -183,10 +179,6 @@ module Gitlab
def self.cancel_job_key(jid)
"sidekiq:cancel:#{jid}"
end
-
- def notification_channel_enabled?
- ENV.fetch("SIDEKIQ_MONITOR_WORKER", 0).to_i.nonzero?
- end
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/monitor.rb b/lib/gitlab/sidekiq_middleware/monitor.rb
index ed825dbfd60..00965bf5506 100644
--- a/lib/gitlab/sidekiq_middleware/monitor.rb
+++ b/lib/gitlab/sidekiq_middleware/monitor.rb
@@ -4,7 +4,7 @@ module Gitlab
module SidekiqMiddleware
class Monitor
def call(worker, job, queue)
- Gitlab::SidekiqDaemon::Monitor.instance.within_job(worker.class, job['jid'], queue) do
+ Gitlab::SidekiqDaemon::Monitor.instance.within_job(job['jid'], queue) do
yield
end
rescue Gitlab::SidekiqDaemon::Monitor::CancelledError
diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb
deleted file mode 100644
index 4ff064007f1..00000000000
--- a/lib/gitlab/uploads/migration_helper.rb
+++ /dev/null
@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Uploads
- class MigrationHelper
- attr_reader :logger
-
- CATEGORIES = [%w(AvatarUploader Project :avatar),
- %w(AvatarUploader Group :avatar),
- %w(AvatarUploader User :avatar),
- %w(AttachmentUploader Note :attachment),
- %w(AttachmentUploader Appearance :logo),
- %w(AttachmentUploader Appearance :header_logo),
- %w(FaviconUploader Appearance :favicon),
- %w(FileUploader Project),
- %w(PersonalFileUploader Snippet),
- %w(NamespaceFileUploader Snippet),
- %w(FileUploader MergeRequest)].freeze
-
- def initialize(args, logger)
- prepare_variables(args, logger)
- end
-
- def migrate_to_remote_storage
- @to_store = ObjectStorage::Store::REMOTE
-
- uploads.each_batch(of: batch_size, &method(:enqueue_batch))
- end
-
- def migrate_to_local_storage
- @to_store = ObjectStorage::Store::LOCAL
-
- uploads(ObjectStorage::Store::REMOTE).each_batch(of: batch_size, &method(:enqueue_batch))
- end
-
- private
-
- def batch_size
- ENV.fetch('MIGRATION_BATCH_SIZE', 200).to_i
- end
-
- def prepare_variables(args, logger)
- @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym
- @uploader_class = args.uploader_class.constantize
- @model_class = args.model_class.constantize
- @logger = logger
- end
-
- def enqueue_batch(batch, index)
- job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
- @model_class,
- @mounted_as,
- @to_store)
- logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job)
- rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
- # continue for the next batch
- logger.warn(message: "[Uploads migration] Could not enqueue batch", ids: batch.ids, reason: e.message) # rubocop:disable CodeReuse/ActiveRecord
- end
-
- # rubocop:disable CodeReuse/ActiveRecord
- def uploads(store_type = [nil, ObjectStorage::Store::LOCAL])
- Upload.class_eval { include EachBatch } unless Upload < EachBatch
-
- Upload
- .where(store: store_type,
- uploader: @uploader_class.to_s,
- model_type: @model_class.base_class.sti_name)
- end
- # rubocop:enable CodeReuse/ActiveRecord
- end
- end
-end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index ed2693aaedf..c6c2876033d 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -116,7 +116,6 @@ module Gitlab
def features_usage_data_ce
{
container_registry_enabled: Gitlab.config.registry.enabled,
- dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled,
gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled,
gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?,
influxdb_metrics_enabled: Gitlab::Metrics.influx_metrics_enabled?,
diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake
index 0d09fd0a4e3..9012e55a70c 100644
--- a/lib/tasks/gitlab/artifacts/migrate.rake
+++ b/lib/tasks/gitlab/artifacts/migrate.rake
@@ -6,31 +6,18 @@ namespace :gitlab do
namespace :artifacts do
task migrate: :environment do
logger = Logger.new(STDOUT)
- logger.info('Starting transfer of artifacts to remote storage')
+ logger.info('Starting transfer of artifacts')
- helper = Gitlab::Artifacts::MigrationHelper.new
+ Ci::Build.joins(:project)
+ .with_artifacts_stored_locally
+ .find_each(batch_size: 10) do |build|
- begin
- helper.migrate_to_remote_storage do |artifact|
- logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage")
- end
- rescue => e
- logger.error(e.message)
- end
- end
-
- task migrate_to_local: :environment do
- logger = Logger.new(STDOUT)
- logger.info('Starting transfer of artifacts to local storage')
-
- helper = Gitlab::Artifacts::MigrationHelper.new
+ build.artifacts_file.migrate!(ObjectStorage::Store::REMOTE)
+ build.artifacts_metadata.migrate!(ObjectStorage::Store::REMOTE)
- begin
- helper.migrate_to_local_storage do |artifact|
- logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage")
- end
+ logger.info("Transferred artifact ID #{build.id} with size #{build.artifacts_size} to object storage")
rescue => e
- logger.error(e.message)
+ logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
end
end
end
diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake
index 4142903d9c3..97c15175a23 100644
--- a/lib/tasks/gitlab/lfs/migrate.rake
+++ b/lib/tasks/gitlab/lfs/migrate.rake
@@ -17,20 +17,5 @@ namespace :gitlab do
logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
end
end
-
- task migrate_to_local: :environment do
- logger = Logger.new(STDOUT)
- logger.info('Starting transfer of LFS files to local storage')
-
- LfsObject.with_files_stored_remotely
- .find_each(batch_size: 10) do |lfs_object|
-
- lfs_object.file.migrate!(LfsObjectUploader::Store::LOCAL)
-
- logger.info("Transferred LFS object #{lfs_object.oid} of size #{lfs_object.size.to_i.bytes} to local storage")
- rescue => e
- logger.error("Failed to transfer LFS object #{lfs_object.oid} with error: #{e.message}")
- end
- end
end
end
diff --git a/lib/tasks/gitlab/traces.rake b/lib/tasks/gitlab/traces.rake
new file mode 100644
index 00000000000..5e1ec481ece
--- /dev/null
+++ b/lib/tasks/gitlab/traces.rake
@@ -0,0 +1,38 @@
+require 'logger'
+require 'resolv-replace'
+
+desc "GitLab | Archive legacy traces to trace artifacts"
+namespace :gitlab do
+ namespace :traces do
+ task archive: :environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Archiving legacy traces')
+
+ Ci::Build.finished.without_archived_trace
+ .order(id: :asc)
+ .find_in_batches(batch_size: 1000) do |jobs|
+ job_ids = jobs.map { |job| [job.id] }
+
+ ArchiveTraceWorker.bulk_perform_async(job_ids)
+
+ logger.info("Scheduled #{job_ids.count} jobs. From #{job_ids.min} to #{job_ids.max}")
+ end
+ end
+
+ task migrate: :environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting transfer of job traces')
+
+ Ci::Build.joins(:project)
+ .with_archived_trace_stored_locally
+ .find_each(batch_size: 10) do |build|
+
+ build.job_artifacts_trace.file.migrate!(ObjectStorage::Store::REMOTE)
+
+ logger.info("Transferred job trace of #{build.id} to object storage")
+ rescue => e
+ logger.error("Failed to transfer artifacts of #{build.id} with error: #{e.message}")
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake
index 44536a447c7..1c93609a006 100644
--- a/lib/tasks/gitlab/uploads/migrate.rake
+++ b/lib/tasks/gitlab/uploads/migrate.rake
@@ -3,7 +3,19 @@ namespace :gitlab do
namespace :migrate do
desc "GitLab | Uploads | Migrate all uploaded files to object storage"
task all: :environment do
- Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
+ categories = [%w(AvatarUploader Project :avatar),
+ %w(AvatarUploader Group :avatar),
+ %w(AvatarUploader User :avatar),
+ %w(AttachmentUploader Note :attachment),
+ %w(AttachmentUploader Appearance :logo),
+ %w(AttachmentUploader Appearance :header_logo),
+ %w(FaviconUploader Appearance :favicon),
+ %w(FileUploader Project),
+ %w(PersonalFileUploader Snippet),
+ %w(NamespaceFileUploader Snippet),
+ %w(FileUploader MergeRequest)]
+
+ categories.each do |args|
Rake::Task["gitlab:uploads:migrate"].invoke(*args)
Rake::Task["gitlab:uploads:migrate"].reenable
end
@@ -13,23 +25,34 @@ namespace :gitlab do
# The following is the actual rake task that migrates uploads of specified
# category to object storage
desc 'GitLab | Uploads | Migrate the uploaded files of specified type to object storage'
- task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
- Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_remote_storage
+ task :migrate, [:uploader_class, :model_class, :mounted_as] => :environment do |task, args|
+ batch_size = ENV.fetch('BATCH', 200).to_i
+ @to_store = ObjectStorage::Store::REMOTE
+ @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym
+ @uploader_class = args.uploader_class.constantize
+ @model_class = args.model_class.constantize
+
+ uploads.each_batch(of: batch_size, &method(:enqueue_batch))
end
- namespace :migrate_to_local do
- desc "GitLab | Uploads | Migrate all uploaded files to local storage"
- task all: :environment do
- Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args|
- Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args)
- Rake::Task["gitlab:uploads:migrate_to_local"].reenable
- end
- end
+ def enqueue_batch(batch, index)
+ job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch,
+ @model_class,
+ @mounted_as,
+ @to_store)
+ puts "Enqueued job ##{index}: #{job}"
+ rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e
+ # continue for the next batch
+ puts "Could not enqueue batch (#{batch.ids}) #{e.message}".color(:red)
end
- desc 'GitLab | Uploads | Migrate the uploaded files of specified type to local storage'
- task :migrate_to_local, [:uploader_class, :model_class, :mounted_as] => :environment do |_t, args|
- Gitlab::Uploads::MigrationHelper.new(args, Logger.new(STDOUT)).migrate_to_local_storage
+ def uploads
+ Upload.class_eval { include EachBatch } unless Upload < EachBatch
+
+ Upload
+ .where(store: [nil, ObjectStorage::Store::LOCAL],
+ uploader: @uploader_class.to_s,
+ model_type: @model_class.base_class.sti_name)
end
end
end