1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
# frozen_string_literal: true
module Gitlab
module HashedStorage
module RakeHelper
def self.batch_size
ENV.fetch('BATCH', 200).to_i
end
def self.listing_limit
ENV.fetch('LIMIT', 500).to_i
end
def self.range_from
ENV['ID_FROM']
end
def self.range_to
ENV['ID_TO']
end
def self.using_ranges?
!range_from.nil? && !range_to.nil?
end
def self.range_single_item?
using_ranges? && range_from == range_to
end
# rubocop: disable CodeReuse/ActiveRecord
def self.project_id_batches_migration(&block)
Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches
ids = relation.pluck(:id)
yield ids.min, ids.max
end
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def self.project_id_batches_rollback(&block)
Project.with_storage_feature(:repository).in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches
ids = relation.pluck(:id)
yield ids.min, ids.max
end
end
# rubocop: enable CodeReuse/ActiveRecord
def self.legacy_attachments_relation
Upload.inner_join_local_uploads_projects.merge(Project.without_storage_feature(:attachments))
end
def self.hashed_attachments_relation
Upload.inner_join_local_uploads_projects.merge(Project.with_storage_feature(:attachments))
end
def self.relation_summary(relation_name, relation)
relation_count = relation.count
$stdout.puts "* Found #{relation_count} #{relation_name}".color(:green)
relation_count
end
def self.projects_list(relation_name, relation)
listing(relation_name, relation.with_route) do |project|
$stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red)
$stdout.puts " #{project.repository.disk_path}"
end
end
def self.attachments_list(relation_name, relation)
listing(relation_name, relation) do |upload|
$stdout.puts " - #{upload.path} (id: #{upload.id})".color(:red)
end
end
# rubocop: disable CodeReuse/ActiveRecord
def self.listing(relation_name, relation)
relation_count = relation_summary(relation_name, relation)
return unless relation_count > 0
limit = listing_limit
if relation_count > limit
$stdout.puts " ! Displaying first #{limit} #{relation_name}..."
end
relation.find_each(batch_size: batch_size).with_index do |element, index|
yield element
break if index + 1 >= limit
end
end
# rubocop: enable CodeReuse/ActiveRecord
def self.prune(relation_name, relation, dry_run: true, root: nil)
root ||= '../repositories'
known_paths = Set.new
listing(relation_name, relation) { |p| known_paths << "#{root}/#{p.repository.disk_path}" }
marked_for_deletion = Set.new(Dir["#{root}/@hashed/*/*/*"])
marked_for_deletion.reject! do |path|
base = path.gsub(/\.(\w+\.)?git$/, '')
known_paths.include?(base)
end
if marked_for_deletion.empty?
$stdout.puts "No orphaned directories found. Nothing to do!"
else
n = marked_for_deletion.size
$stdout.puts "Found #{n} orphaned #{'directory'.pluralize(n)}"
$stdout.puts "Dry run. (Run again with FORCE=1 to delete). We would have deleted:" if dry_run
end
marked_for_deletion.each do |p|
p = Pathname.new(p)
if dry_run
$stdout.puts " - #{p}"
else
$stdout.puts "Removing #{p}"
p.rmtree
end
end
end
end
end
end
|
