blob: 2539a6828c329a34772c69bfc7257abc59b87986 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
# frozen_string_literal: true
#
# This class encapsulates the directories used by project import/export:
#
# 1. The project export job first generates the project metadata tree
# (e.g. `project.json) and repository bundle (e.g. `project.bundle`)
# inside a temporary `export_path`
# (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA/:randomB).
#
# 2. The job then creates a tarball (e.g. `project.tar.gz`) in
# `archive_path` (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA).
# CarrierWave moves this tarball files into its permanent location.
#
# 3. Lock files are used to indicate whether a project is in the
# `after_export` state. These are stored in a directory
# (e.g. /path/to/shared/tmp/project_exports/namespace/project/locks. The
# number of lock files present signifies how many concurrent project
# exports are running. Note that this assumes the temporary directory
# is a shared mount:
# https://gitlab.com/gitlab-org/gitlab/issues/32203
#
# NOTE: Stale files should be cleaned up via ImportExportCleanupService.
module Gitlab
module ImportExport
class Shared
attr_reader :errors, :exportable, :logger
LOCKS_DIRECTORY = 'locks'
def initialize(exportable)
@exportable = exportable
@errors = []
@logger = Gitlab::Import::Logger.build
end
def active_export_count
Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
end
# The path where the exportable metadata and repository bundle (in case of project) is saved
def export_path
@export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
end
# The path where the tarball is saved
def archive_path
@archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path)
end
def base_path
@base_path ||= Gitlab::ImportExport.export_path(relative_path: relative_base_path)
end
def lock_files_path
@locks_files_path ||= File.join(base_path, LOCKS_DIRECTORY)
end
def error(error)
error_payload = { message: error.message }
error_payload[:error_backtrace] = Gitlab::Profiler.clean_backtrace(error.backtrace) if error.backtrace
log_error(error_payload)
Gitlab::Sentry.track_acceptable_exception(error, extra: log_base_data)
add_error_message(error.message)
end
def add_error_message(message)
@errors << filtered_error_message(message)
end
def after_export_in_progress?
locks_present?
end
def locks_present?
Dir.exist?(lock_files_path) && !Dir.empty?(lock_files_path)
end
private
def relative_path
@relative_path ||= File.join(relative_archive_path, SecureRandom.hex)
end
def relative_archive_path
@relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex)
end
def relative_base_path
case exportable_type
when 'Project'
@exportable.disk_path
when 'Group'
@exportable.full_path
else
raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}")
end
end
def log_error(details)
@logger.error(log_base_data.merge(details))
end
def log_debug(details)
@logger.debug(log_base_data.merge(details))
end
def log_base_data
log = {
importer: 'Import/Export',
exportable_id: @exportable&.id,
exportable_path: @exportable&.full_path
}
log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project'
log
end
def filtered_error_message(message)
Projects::ImportErrorFilter.filter_message(message)
end
def exportable_type
@exportable.class.name
end
end
end
end
|
