1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
# frozen_string_literal: true
namespace :gitlab do
desc "GitLab | Update templates"
task :update_templates do
TEMPLATE_DATA.each { |template| update(template) }
end
desc "GitLab | Update project templates"
task :update_project_templates, [] => :environment do |_task, args|
# we need an instance method from Gitlab::ImportExport::CommandLineUtil and don't
# want to include it in the task, as this would affect subsequent tasks as well
downloader = Class.new do
extend Gitlab::ImportExport::CommandLineUtil
def self.call(uploader, upload_path)
download_or_copy_upload(uploader, upload_path)
end
end
template_names = args.extras.to_set
if Rails.env.production?
raise "This rake task is not meant for production instances"
end
# Find an admin user with an SSH key
admin = User.where(admin: true).joins(:keys).where.not(keys: { id: nil }).take
unless admin
raise "No admin user with SSH key could be found"
end
tmp_namespace_path = "tmp-project-import-#{Time.now.to_i}"
puts "Creating temporary namespace #{tmp_namespace_path}"
tmp_namespace = Namespace.create!(owner: admin, name: tmp_namespace_path, path: tmp_namespace_path, type: Namespaces::UserNamespace.sti_name)
templates = if template_names.empty?
Gitlab::ProjectTemplate.all
else
Gitlab::ProjectTemplate.all.select { |template| template_names.include?(template.name) }
end
templates.each do |template|
params = {
namespace_id: tmp_namespace.id,
path: template.name,
skip_wiki: true
}
puts "Creating project for #{template.title}"
project = Projects::CreateService.new(admin, params).execute
unless project.persisted?
raise "Failed to create project: #{project.errors.messages}"
end
uri_encoded_project_path = template.uri_encoded_project_path
# extract a concrete commit for signing off what we actually downloaded
# this way we do the right thing even if the repository gets updated in the meantime
get_commits_response = Gitlab::HTTP.get("#{template.project_host}/api/v4/projects/#{uri_encoded_project_path}/repository/commits",
query: { page: 1, per_page: 1 }
)
raise "Failed to retrieve latest commit for template '#{template.name}'" unless get_commits_response.success?
commit_sha = get_commits_response.parsed_response.dig(0, 'id')
project_archive_uri = "#{template.project_host}/api/v4/projects/#{uri_encoded_project_path}/repository/archive.tar.gz?sha=#{commit_sha}"
commit_message = <<~MSG
Initialized from '#{template.title}' project template
Template repository: #{template.preview}
Commit SHA: #{commit_sha}
MSG
local_remote = project.ssh_url_to_repo
Dir.mktmpdir do |tmpdir|
Dir.chdir(tmpdir) do
Gitlab::TaskHelpers.run_command!(['wget', project_archive_uri, '-O', 'archive.tar.gz'])
Gitlab::TaskHelpers.run_command!(['tar', 'xf', 'archive.tar.gz'])
extracted_project_basename = Dir['*/'].first
Dir.chdir(extracted_project_basename) do
Gitlab::TaskHelpers.run_command!(%w(git init --initial-branch=master))
Gitlab::TaskHelpers.run_command!(%W(git remote add origin #{local_remote}))
Gitlab::TaskHelpers.run_command!(%w(git add .))
Gitlab::TaskHelpers.run_command!(['git', 'commit', '--author', 'GitLab <root@localhost>', '--message', commit_message])
Gitlab::TaskHelpers.run_command!(['git', 'push', '-u', 'origin', 'master'])
end
end
end
project.reset
Projects::ImportExport::ExportService.new(project, admin).execute
downloader.call(project.export_file, template.archive_path)
unless Projects::DestroyService.new(project, admin).execute
puts "Failed to destroy project #{template.name} (but namespace will be cleaned up later)"
end
puts "Exported #{template.name}".green
end
success = true
ensure
if tmp_namespace
puts "Destroying temporary namespace #{tmp_namespace_path}"
tmp_namespace.destroy
end
puts "Done".green if success
end
def update(template)
sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1]
dir = File.join(vendor_directory, sub_dir)
unless clone_repository(template.repo_url, dir)
puts "Cloning the #{sub_dir} templates failed".red
return
end
remove_unneeded_files(dir, template.cleanup_regex)
puts "Done".green
end
def clone_repository(url, directory)
FileUtils.rm_rf(directory) if Dir.exist?(directory)
system("git clone #{url} --depth=1 --branch=master #{directory}")
end
# Retain only certain files:
# - The LICENSE, because we have to
# - The sub dirs so we can organise the file by category
# - The templates themself
# - Dir.entries returns also the entries '.' and '..'
def remove_unneeded_files(directory, regex)
Dir.foreach(directory) do |file|
FileUtils.rm_rf(File.join(directory, file)) unless file =~ regex
end
end
private
Template = Struct.new(:repo_url, :cleanup_regex)
TEMPLATE_DATA = [
Template.new(
"https://github.com/github/gitignore.git",
/(\.{1,2}|LICENSE|Global|\.gitignore)\z/
),
Template.new(
"https://gitlab.com/gitlab-org/Dockerfile.git",
/(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/
)
].freeze
def vendor_directory
Rails.root.join('vendor')
end
end
|