Merge branch 'docker-registry-view' into 'master'

Add container registry support

Tasks:

- [x] Merge docker/distribution authentication service: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3787
- [x] Implement Docker Registry API
- [x] Show a list of docker images in GitLab
- [x] Remove registry repository on project deletion
- [x] Support project rename, move and namespace rename
- [x] Use token when connecting the registry
- [x] Allow to delete images from GitLab
- [x] Support pushing from GitLab CI (gitlab-ci-token / $CI_BUILD_TOKEN)
- [x] Support GitLab Runner pulling for public repositories
- [ ] Support GitLab Runner pulling for private repositories
- [x] Add tests for Docker Registry API
- [x] Add tests for a views
- [x] Make texts nicer
- [x] Implement a backup support

- [ ] Create administration documentation https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/4141
- [ ] Create user documentation

See merge request !4040

10 files changed, 310 insertions, 1 deletions

diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 4962f5e53ce..7d0608f09da 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -157,7 +157,7 @@ module Backup
     end
 
     def archives_to_backup
-      %w{uploads builds artifacts lfs}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact
+      %w{uploads builds artifacts lfs registry}.map{ |name| (name + ".tar.gz") unless skipped?(name) }.compact
     end
 
     def folders_to_backup
diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb
new file mode 100644
index 00000000000..67fe0231087
--- /dev/null
+++ b/lib/backup/registry.rb
@@ -0,0 +1,13 @@
+require 'backup/files'
+
+module Backup
+  class Registry < Files
+    def initialize
+      super('registry', Settings.registry.path)
+    end
+
+    def create_files_dir
+      Dir.mkdir(app_files_dir, 0700)
+    end
+  end
+end
diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb
new file mode 100644
index 00000000000..4e20dc4f875
--- /dev/null
+++ b/lib/container_registry/blob.rb
@@ -0,0 +1,48 @@
+module ContainerRegistry
+  class Blob
+    attr_reader :repository, :config
+
+    delegate :registry, :client, to: :repository
+
+    def initialize(repository, config)
+      @repository = repository
+      @config = config || {}
+    end
+
+    def valid?
+      digest.present?
+    end
+
+    def path
+      "#{repository.path}@#{digest}"
+    end
+
+    def digest
+      config['digest']
+    end
+
+    def type
+      config['mediaType']
+    end
+
+    def size
+      config['size']
+    end
+
+    def revision
+      digest.split(':')[1]
+    end
+
+    def short_revision
+      revision[0..8]
+    end
+
+    def delete
+      client.delete_blob(repository.name, digest)
+    end
+
+    def data
+      @data ||= client.blob(repository.name, digest, type)
+    end
+  end
+end
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
new file mode 100644
index 00000000000..4d726692f45
--- /dev/null
+++ b/lib/container_registry/client.rb
@@ -0,0 +1,61 @@
+require 'faraday'
+require 'faraday_middleware'
+
+module ContainerRegistry
+  class Client
+    attr_accessor :uri
+
+    MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'
+
+    def initialize(base_uri, options = {})
+      @base_uri = base_uri
+      @faraday = Faraday.new(@base_uri) do |conn|
+        initialize_connection(conn, options)
+      end
+    end
+
+    def repository_tags(name)
+      @faraday.get("/v2/#{name}/tags/list").body
+    end
+
+    def repository_manifest(name, reference)
+      @faraday.get("/v2/#{name}/manifests/#{reference}").body
+    end
+
+    def repository_tag_digest(name, reference)
+      response = @faraday.head("/v2/#{name}/manifests/#{reference}")
+      response.headers['docker-content-digest'] if response.success?
+    end
+
+    def delete_repository_tag(name, reference)
+      @faraday.delete("/v2/#{name}/manifests/#{reference}").success?
+    end
+
+    def blob(name, digest, type = nil)
+      headers = {}
+      headers['Accept'] = type if type
+      @faraday.get("/v2/#{name}/blobs/#{digest}", nil, headers).body
+    end
+
+    def delete_blob(name, digest)
+      @faraday.delete("/v2/#{name}/blobs/#{digest}").success?
+    end
+    
+    private
+    
+    def initialize_connection(conn, options)
+      conn.request :json
+      conn.headers['Accept'] = MANIFEST_VERSION
+
+      conn.response :json, content_type: /\bjson$/
+
+      if options[:user] && options[:password]
+        conn.request(:basic_auth, options[:user].to_s, options[:password].to_s)
+      elsif options[:token]
+        conn.request(:authorization, :bearer, options[:token].to_s)
+      end
+
+      conn.adapter :net_http
+    end
+  end
+end
diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb
new file mode 100644
index 00000000000..589f9f4380a
--- /dev/null
+++ b/lib/container_registry/config.rb
@@ -0,0 +1,16 @@
+module ContainerRegistry
+  class Config
+    attr_reader :tag, :blob, :data
+
+    def initialize(tag, blob)
+      @tag, @blob = tag, blob
+      @data = JSON.parse(blob.data)
+    end
+
+    def [](key)
+      return unless data
+
+      data[key]
+    end
+  end
+end
diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb
new file mode 100644
index 00000000000..0e634f6b6ef
--- /dev/null
+++ b/lib/container_registry/registry.rb
@@ -0,0 +1,21 @@
+module ContainerRegistry
+  class Registry
+    attr_reader :uri, :client, :path
+
+    def initialize(uri, options = {})
+      @uri = uri
+      @path = options[:path] || default_path
+      @client = ContainerRegistry::Client.new(uri, options)
+    end
+
+    def repository(name)
+      ContainerRegistry::Repository.new(self, name)
+    end
+
+    private
+
+    def default_path
+      @uri.sub(/^https?:\/\//, '')
+    end
+  end
+end
diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb
new file mode 100644
index 00000000000..0e4a7cb3cc9
--- /dev/null
+++ b/lib/container_registry/repository.rb
@@ -0,0 +1,48 @@
+module ContainerRegistry
+  class Repository
+    attr_reader :registry, :name
+
+    delegate :client, to: :registry
+
+    def initialize(registry, name)
+      @registry, @name = registry, name
+    end
+
+    def path
+      [registry.path, name].compact.join('/')
+    end
+
+    def tag(tag)
+      ContainerRegistry::Tag.new(self, tag)
+    end
+
+    def manifest
+      return @manifest if defined?(@manifest)
+
+      @manifest = client.repository_tags(name)
+    end
+
+    def valid?
+      manifest.present?
+    end
+
+    def tags
+      return @tags if defined?(@tags)
+      return [] unless manifest && manifest['tags']
+
+      @tags = manifest['tags'].map do |tag|
+        ContainerRegistry::Tag.new(self, tag)
+      end
+    end
+
+    def blob(config)
+      ContainerRegistry::Blob.new(self, config)
+    end
+
+    def delete_tags
+      return unless tags
+
+      tags.all?(&:delete)
+    end
+  end
+end
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
new file mode 100644
index 00000000000..43f8d6dc8c2
--- /dev/null
+++ b/lib/container_registry/tag.rb
@@ -0,0 +1,77 @@
+module ContainerRegistry
+  class Tag
+    attr_reader :repository, :name
+
+    delegate :registry, :client, to: :repository
+
+    def initialize(repository, name)
+      @repository, @name = repository, name
+    end
+
+    def valid?
+      manifest.present?
+    end
+
+    def manifest
+      return @manifest if defined?(@manifest)
+
+      @manifest = client.repository_manifest(repository.name, name)
+    end
+
+    def path
+      "#{repository.path}:#{name}"
+    end
+
+    def [](key)
+      return unless manifest
+
+      manifest[key]
+    end
+
+    def digest
+      return @digest if defined?(@digest)
+
+      @digest = client.repository_tag_digest(repository.name, name)
+    end
+
+    def config_blob
+      return @config_blob if defined?(@config_blob)
+      return unless manifest && manifest['config']
+
+      @config_blob = repository.blob(manifest['config'])
+    end
+
+    def config
+      return unless config_blob
+
+      @config ||= ContainerRegistry::Config.new(self, config_blob)
+    end
+
+    def created_at
+      return unless config
+
+      @created_at ||= DateTime.rfc3339(config['created'])
+    end
+
+    def layers
+      return @layers if defined?(@layers)
+      return unless manifest
+
+      @layers = manifest['layers'].map do |layer|
+        repository.blob(layer)
+      end
+    end
+
+    def total_size
+      return unless layers
+
+      layers.map(&:size).sum
+    end
+
+    def delete
+      return unless digest
+
+      client.delete_repository_tag(repository.name, digest)
+    end
+  end
+end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index ace906a6f59..1cbd6d945a0 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -96,5 +96,9 @@ module Gitlab
         (?<![\/.])                (?# rule #6-7)
       }x.freeze
     end
+
+    def container_registry_reference_regex
+      git_reference_regex
+    end
   end
 end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 402bb338f27..d97d974ec20 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -14,6 +14,7 @@ namespace :gitlab do
       Rake::Task["gitlab:backup:builds:create"].invoke
       Rake::Task["gitlab:backup:artifacts:create"].invoke
       Rake::Task["gitlab:backup:lfs:create"].invoke
+      Rake::Task["gitlab:backup:registry:create"].invoke
 
       backup = Backup::Manager.new
       backup.pack
@@ -54,6 +55,7 @@ namespace :gitlab do
       Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds')
       Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
       Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
+      Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
       Rake::Task['gitlab:shell:setup'].invoke
 
       backup.cleanup
@@ -173,6 +175,25 @@ namespace :gitlab do
       end
     end
 
+    namespace :registry do
+      task create: :environment do
+        $progress.puts "Dumping container registry images ... ".blue
+
+        if ENV["SKIP"] && ENV["SKIP"].include?("registry")
+          $progress.puts "[SKIPPED]".cyan
+        else
+          Backup::Registry.new.dump
+          $progress.puts "done".green
+        end
+      end
+
+      task restore: :environment do
+        $progress.puts "Restoring container registry images ... ".blue
+        Backup::Registry.new.restore
+        $progress.puts "done".green
+      end
+    end
+
     def configure_cron_mode
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a