diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2016-05-18 21:19:01 +0300 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2016-05-18 21:19:01 +0300 |
commit | 08fddae7441ce9ac61bc80e5b93a2eafb8441430 (patch) | |
tree | 3921871fc9363f9ef869e6b6b36963e91446bd9e /lib | |
parent | b7d83acf5b03e08dc9e387e1abb83c5e3c80444c (diff) | |
parent | 2bea20b8ef4bb4c0577bfe68bb77f96958e164d2 (diff) |
Merge branch 'docker-registry-view' into 'master'
Add container registry support
Tasks:
- [x] Merge docker/distribution authentication service: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3787
- [x] Implement Docker Registry API
- [x] Show a list of docker images in GitLab
- [x] Remove registry repository on project deletion
- [x] Support project rename, move and namespace rename
- [x] Use token when connecting the registry
- [x] Allow to delete images from GitLab
- [x] Support pushing from GitLab CI (gitlab-ci-token / $CI_BUILD_TOKEN)
- [x] Support GitLab Runner pulling for public repositories
- [ ] Support GitLab Runner pulling for private repositories
- [x] Add tests for Docker Registry API
- [x] Add tests for a views
- [x] Make texts nicer
- [x] Implement a backup support
- [ ] Create administration documentation https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/4141
- [ ] Create user documentation
See merge request !4040
Diffstat (limited to 'lib')
-rw-r--r-- | lib/backup/manager.rb | 2 | ||||
-rw-r--r-- | lib/backup/registry.rb | 13 | ||||
-rw-r--r-- | lib/container_registry/blob.rb | 48 | ||||
-rw-r--r-- | lib/container_registry/client.rb | 61 | ||||
-rw-r--r-- | lib/container_registry/config.rb | 16 | ||||
-rw-r--r-- | lib/container_registry/registry.rb | 21 | ||||
-rw-r--r-- | lib/container_registry/repository.rb | 48 | ||||
-rw-r--r-- | lib/container_registry/tag.rb | 77 | ||||
-rw-r--r-- | lib/gitlab/regex.rb | 4 | ||||
-rw-r--r-- | lib/tasks/gitlab/backup.rake | 21 |
10 files changed, 310 insertions, 1 deletions
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 4962f5e53ce..7d0608f09da 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -157,7 +157,7 @@ module Backup end def archives_to_backup - %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact + %w{uploads builds artifacts lfs registry}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact end def folders_to_backup diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb new file mode 100644 index 00000000000..67fe0231087 --- /dev/null +++ b/lib/backup/registry.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Registry < Files + def initialize + super('registry', Settings.registry.path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb new file mode 100644 index 00000000000..4e20dc4f875 --- /dev/null +++ b/lib/container_registry/blob.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Blob + attr_reader :repository, :config + + delegate :registry, :client, to: :repository + + def initialize(repository, config) + @repository = repository + @config = config || {} + end + + def valid? + digest.present? + end + + def path + "#{repository.path}@#{digest}" + end + + def digest + config['digest'] + end + + def type + config['mediaType'] + end + + def size + config['size'] + end + + def revision + digest.split(':')[1] + end + + def short_revision + revision[0..8] + end + + def delete + client.delete_blob(repository.name, digest) + end + + def data + @data ||= client.blob(repository.name, digest, type) + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb new file mode 100644 index 00000000000..4d726692f45 --- /dev/null +++ b/lib/container_registry/client.rb @@ -0,0 +1,61 @@ +require 'faraday' +require 'faraday_middleware' + +module ContainerRegistry + class Client + attr_accessor :uri + + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + + def initialize(base_uri, options = {}) + @base_uri = base_uri + @faraday = Faraday.new(@base_uri) do |conn| + initialize_connection(conn, options) + end + end + + def repository_tags(name) + @faraday.get("/v2/#{name}/tags/list").body + end + + def repository_manifest(name, reference) + @faraday.get("/v2/#{name}/manifests/#{reference}").body + end + + def repository_tag_digest(name, reference) + response = @faraday.head("/v2/#{name}/manifests/#{reference}") + response.headers['docker-content-digest'] if response.success? + end + + def delete_repository_tag(name, reference) + @faraday.delete("/v2/#{name}/manifests/#{reference}").success? + end + + def blob(name, digest, type = nil) + headers = {} + headers['Accept'] = type if type + @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body + end + + def delete_blob(name, digest) + @faraday.delete("/v2/#{name}/blobs/#{digest}").success? + end + + private + + def initialize_connection(conn, options) + conn.request :json + conn.headers['Accept'] = MANIFEST_VERSION + + conn.response :json, content_type: /\bjson$/ + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + conn.adapter :net_http + end + end +end diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb new file mode 100644 index 00000000000..589f9f4380a --- /dev/null +++ b/lib/container_registry/config.rb @@ -0,0 +1,16 @@ +module ContainerRegistry + class Config + attr_reader :tag, :blob, :data + + def initialize(tag, blob) + @tag, @blob = tag, blob + @data = JSON.parse(blob.data) + end + + def [](key) + return unless data + + data[key] + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb new file mode 100644 index 00000000000..0e634f6b6ef --- /dev/null +++ b/lib/container_registry/registry.rb @@ -0,0 +1,21 @@ +module ContainerRegistry + class Registry + attr_reader :uri, :client, :path + + def initialize(uri, options = {}) + @uri = uri + @path = options[:path] || default_path + @client = ContainerRegistry::Client.new(uri, options) + end + + def repository(name) + ContainerRegistry::Repository.new(self, name) + end + + private + + def default_path + @uri.sub(/^https?:\/\//, '') + end + end +end diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb new file mode 100644 index 00000000000..0e4a7cb3cc9 --- /dev/null +++ b/lib/container_registry/repository.rb @@ -0,0 +1,48 @@ +module ContainerRegistry + class Repository + attr_reader :registry, :name + + delegate :client, to: :registry + + def initialize(registry, name) + @registry, @name = registry, name + end + + def path + [registry.path, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_tags(name) + end + + def valid? + manifest.present? + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + end +end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb new file mode 100644 index 00000000000..43f8d6dc8c2 --- /dev/null +++ b/lib/container_registry/tag.rb @@ -0,0 +1,77 @@ +module ContainerRegistry + class Tag + attr_reader :repository, :name + + delegate :registry, :client, to: :repository + + def initialize(repository, name) + @repository, @name = repository, name + end + + def valid? + manifest.present? + end + + def manifest + return @manifest if defined?(@manifest) + + @manifest = client.repository_manifest(repository.name, name) + end + + def path + "#{repository.path}:#{name}" + end + + def [](key) + return unless manifest + + manifest[key] + end + + def digest + return @digest if defined?(@digest) + + @digest = client.repository_tag_digest(repository.name, name) + end + + def config_blob + return @config_blob if defined?(@config_blob) + return unless manifest && manifest['config'] + + @config_blob = repository.blob(manifest['config']) + end + + def config + return unless config_blob + + @config ||= ContainerRegistry::Config.new(self, config_blob) + end + + def created_at + return unless config + + @created_at ||= DateTime.rfc3339(config['created']) + end + + def layers + return @layers if defined?(@layers) + return unless manifest + + @layers = manifest['layers'].map do |layer| + repository.blob(layer) + end + end + + def total_size + return unless layers + + layers.map(&:size).sum + end + + def delete + return unless digest + + client.delete_repository_tag(repository.name, digest) + end + end +end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ace906a6f59..1cbd6d945a0 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -96,5 +96,9 @@ module Gitlab (?<![\/.]) (?# rule #6-7) }x.freeze end + + def container_registry_reference_regex + git_reference_regex + end end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 402bb338f27..d97d974ec20 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -14,6 +14,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke + Rake::Task["gitlab:backup:registry:create"].invoke backup = Backup::Manager.new backup.pack @@ -54,6 +55,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') + Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke backup.cleanup @@ -173,6 +175,25 @@ namespace :gitlab do end end + namespace :registry do + task create: :environment do + $progress.puts "Dumping container registry images ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("registry") + $progress.puts "[SKIPPED]".cyan + else + Backup::Registry.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring container registry images ... ".blue + Backup::Registry.new.restore + $progress.puts "done".green + end + end + def configure_cron_mode if ENV['CRON'] # We need an object we can say 'puts' and 'print' to; let's use a |