Multi-level container image names backend implementation

  - Adds Registry events API endpoint
  - Adds container_images_repository and container_images models
  - Changes JWT authentication to allow multi-level scopes
  - Adds services for container image maintenance

14 files changed, 275 insertions, 7 deletions

diff --git a/app/models/container_image.rb b/app/models/container_image.rb
new file mode 100644
index 00000000000..dcc4a7af629
--- /dev/null
+++ b/app/models/container_image.rb
@@ -0,0 +1,58 @@
+class ContainerImage < ActiveRecord::Base
+  belongs_to :container_images_repository
+
+  delegate :registry, :registry_path_with_namespace, :client, to: :container_images_repository
+
+  validates :manifest, presence: true
+
+  before_validation :update_token, on: :create
+  def update_token
+    paths = container_images_repository.allowed_paths << name_with_namespace
+    token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths)
+    client.update_token(token)
+  end
+
+  def path
+    [registry.path, name_with_namespace].compact.join('/')
+  end
+
+  def name_with_namespace
+    [registry_path_with_namespace, name].compact.join('/')
+  end
+
+  def tag(tag)
+    ContainerRegistry::Tag.new(self, tag)
+  end
+
+  def manifest
+    @manifest ||= client.repository_tags(name_with_namespace)
+  end
+
+  def tags
+    return @tags if defined?(@tags)
+    return [] unless manifest && manifest['tags']
+
+    @tags = manifest['tags'].map do |tag|
+      ContainerRegistry::Tag.new(self, tag)
+    end
+  end
+
+  def blob(config)
+    ContainerRegistry::Blob.new(self, config)
+  end
+
+  def delete_tags
+    return unless tags
+
+    tags.all?(&:delete)
+  end
+
+  def self.split_namespace(full_path)
+    image_name = full_path.split('/').last
+    namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1')
+    if namespace.count('/') < 1
+      namespace, image_name = full_path, ""
+    end
+    return namespace, image_name
+  end
+end
diff --git a/app/models/container_images_repository.rb b/app/models/container_images_repository.rb
new file mode 100644
index 00000000000..99e94d2a6d0
--- /dev/null
+++ b/app/models/container_images_repository.rb
@@ -0,0 +1,26 @@
+class ContainerImagesRepository < ActiveRecord::Base
+
+  belongs_to :project
+
+  has_many :container_images, dependent: :destroy
+
+  delegate :client, to: :registry
+
+  def registry_path_with_namespace
+    project.path_with_namespace.downcase
+  end
+
+  def allowed_paths
+    @allowed_paths ||= [registry_path_with_namespace] +
+      container_images.map { |i| i.name_with_namespace }
+  end
+
+  def registry
+    @registry ||= begin
+      token = Auth::ContainerRegistryAuthenticationService.full_access_token(allowed_paths)
+      url = Gitlab.config.registry.api_url
+      host_port = Gitlab.config.registry.host_port
+      ContainerRegistry::Registry.new(url, token: token, path: host_port)
+    end
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index 411299eef63..703e24eb79a 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -157,6 +157,7 @@ class Project < ActiveRecord::Base
   has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
   has_one :project_feature, dependent: :destroy
   has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
+  has_one :container_images_repository, dependent: :destroy
 
   has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id
   has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id
diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb
index 5cb7a86a5ee..6b83b38fa4d 100644
--- a/app/services/auth/container_registry_authentication_service.rb
+++ b/app/services/auth/container_registry_authentication_service.rb
@@ -16,7 +16,7 @@ module Auth
       { token: authorized_token(scope).encoded }
     end
 
-    def self.full_access_token(*names)
+    def self.full_access_token(names)
       registry = Gitlab.config.registry
       token = JSONWebToken::RSAToken.new(registry.key)
       token.issuer = registry.issuer
@@ -61,7 +61,12 @@ module Auth
     end
 
     def process_repository_access(type, name, actions)
-      requested_project = Project.find_by_full_path(name)
+      # Strips image name due to lack of
+      # per image authentication.
+      # Removes only last occurence in light
+      # of future nested groups
+      namespace, _ = ContainerImage::split_namespace(name)
+      requested_project = Project.find_by_full_path(namespace)
       return unless requested_project
 
       actions = actions.select do |action|
diff --git a/app/services/container_images_repositories/container_images/create_service.rb b/app/services/container_images_repositories/container_images/create_service.rb
new file mode 100644
index 00000000000..0c2c69d5183
--- /dev/null
+++ b/app/services/container_images_repositories/container_images/create_service.rb
@@ -0,0 +1,16 @@
+module ContainerImagesRepositories
+  module ContainerImages
+    class CreateService < BaseService
+      def execute
+        @container_image = container_images_repository.container_images.create(params)
+        @container_image if @container_image.valid?
+      end
+
+      private
+
+      def container_images_repository
+        @container_images_repository ||= project.container_images_repository
+      end
+    end
+  end
+end
diff --git a/app/services/container_images_repositories/container_images/destroy_service.rb b/app/services/container_images_repositories/container_images/destroy_service.rb
new file mode 100644
index 00000000000..91b8cfeea47
--- /dev/null
+++ b/app/services/container_images_repositories/container_images/destroy_service.rb
@@ -0,0 +1,11 @@
+module ContainerImagesRepositories
+  module ContainerImages
+    class DestroyService < BaseService
+      def execute(container_image)
+        return false unless container_image
+
+        container_image.destroy
+      end
+    end
+  end
+end
diff --git a/app/services/container_images_repositories/container_images/push_service.rb b/app/services/container_images_repositories/container_images/push_service.rb
new file mode 100644
index 00000000000..2731cf1d52e
--- /dev/null
+++ b/app/services/container_images_repositories/container_images/push_service.rb
@@ -0,0 +1,26 @@
+module ContainerImagesRepositories
+  module ContainerImages
+    class PushService < BaseService
+      def execute(container_image_name, event)
+        find_or_create_container_image(container_image_name).valid?
+      end
+
+      private
+
+      def find_or_create_container_image(container_image_name)
+        options = {name: container_image_name}
+        container_images.find_by(options) ||
+          ::ContainerImagesRepositories::ContainerImages::CreateService.new(project,
+            current_user, options).execute
+      end
+
+      def container_images_repository
+        @container_images_repository ||= project.container_images_repository
+      end
+
+      def container_images
+        @container_images ||= container_images_repository.container_images
+      end
+    end
+  end
+end
diff --git a/app/services/container_images_repositories/create_service.rb b/app/services/container_images_repositories/create_service.rb
new file mode 100644
index 00000000000..7e9dd3abe5f
--- /dev/null
+++ b/app/services/container_images_repositories/create_service.rb
@@ -0,0 +1,7 @@
+module ContainerImagesRepositories
+  class CreateService < BaseService
+    def execute
+      project.container_images_repository || ::ContainerImagesRepository.create(project: project)
+    end
+  end
+end
diff --git a/db/migrate/20161029153736_create_container_images_repository.rb b/db/migrate/20161029153736_create_container_images_repository.rb
new file mode 100644
index 00000000000..d93180b1674
--- /dev/null
+++ b/db/migrate/20161029153736_create_container_images_repository.rb
@@ -0,0 +1,31 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateContainerImagesRepository < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  # When using the methods "add_concurrent_index" or "add_column_with_default"
+  # you must disable the use of transactions as these methods can not run in an
+  # existing transaction. When using "add_concurrent_index" make sure that this
+  # method is the _only_ method called in the migration, any other changes
+  # should go in a separate migration. This ensures that upon failure _only_ the
+  # index creation fails and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
+
+  def change
+    create_table :container_images_repositories do |t|
+      t.integer :project_id, null: false
+    end
+  end
+end
diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb
new file mode 100644
index 00000000000..94feae280a6
--- /dev/null
+++ b/db/migrate/20161031013926_create_container_image.rb
@@ -0,0 +1,32 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateContainerImage < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # When a migration requires downtime you **must** uncomment the following
+  # constant and define a short and easy to understand explanation as to why the
+  # migration requires downtime.
+  # DOWNTIME_REASON = ''
+
+  # When using the methods "add_concurrent_index" or "add_column_with_default"
+  # you must disable the use of transactions as these methods can not run in an
+  # existing transaction. When using "add_concurrent_index" make sure that this
+  # method is the _only_ method called in the migration, any other changes
+  # should go in a separate migration. This ensures that upon failure _only_ the
+  # index creation fails and can be retried or reverted easily.
+  #
+  # To disable transactions uncomment the following line and remove these
+  # comments:
+  # disable_ddl_transaction!
+
+  def change
+    create_table :container_images do |t|
+      t.integer :container_images_repository_id
+      t.string :name
+    end
+  end
+end
diff --git a/lib/api/api.rb b/lib/api/api.rb
index a0282ff8deb..ed775f898d2 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -84,6 +84,7 @@ module API
     mount ::API::Namespaces
     mount ::API::Notes
     mount ::API::NotificationSettings
+    mount ::API::RegistryEvents
     mount ::API::Pipelines
     mount ::API::ProjectHooks
     mount ::API::Projects
diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb
new file mode 100644
index 00000000000..c0473051424
--- /dev/null
+++ b/lib/api/registry_events.rb
@@ -0,0 +1,52 @@
+module API
+  # RegistryEvents API
+  class RegistryEvents < Grape::API
+    # before { authenticate! }
+
+    content_type :json, 'application/vnd.docker.distribution.events.v1+json'
+
+    params do
+      requires :events, type: Array, desc: 'The ID of a project' do
+        requires :id, type: String, desc: 'The ID of the event'
+        requires :timestamp, type: String, desc: 'Timestamp of the event'
+        requires :action, type: String, desc: 'Action performed by event'
+        requires :target, type: Hash, desc: 'Target of the event' do
+          optional :mediaType, type: String, desc: 'Media type of the target'
+          optional :size, type: Integer, desc: 'Size in bytes of the target'
+          requires :digest, type: String, desc: 'Digest of the target'
+          requires :repository, type: String, desc: 'Repository of target'
+          optional :url, type: String, desc: 'Url of the target'
+          optional :tag, type: String, desc: 'Tag of the target'
+        end
+        requires :request, type: Hash, desc: 'Request of the event' do
+          requires :id, type: String, desc: 'The ID of the request'
+          optional :addr, type: String, desc: 'IP Address of the request client'
+          optional :host, type: String, desc: 'Hostname of the registry instance'
+          requires :method, type: String, desc: 'Request method'
+          requires :useragent, type: String, desc: 'UserAgent header of the request'
+        end
+        requires :actor, type: Hash, desc: 'Actor that initiated the event' do
+          optional :name, type: String, desc: 'Actor name'
+        end
+        requires :source, type: Hash, desc: 'Source of the event' do
+          optional :addr, type: String, desc: 'Hostname of source registry node'
+          optional :instanceID, type: String, desc: 'Source registry node instanceID'
+        end
+      end
+    end
+    resource :registry_events do
+      post do
+        params['events'].each do |event|
+          repository = event['target']['repository']
+
+          if event['action'] == 'push' and !!event['target']['tag']
+            namespace, container_image_name = ContainerImage::split_namespace(repository)
+            ::ContainerImagesRepositories::ContainerImages::PushService.new(
+              Project::find_with_namespace(namespace), current_user
+                ).execute(container_image_name, event)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 2edddb84fc3..2cbb7bfb67d 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -15,6 +15,10 @@ module ContainerRegistry
       @options = options
     end
 
+    def update_token(token)
+      @options[:token] = token
+    end
+
     def repository_tags(name)
       response_body faraday.get("/v2/#{name}/tags/list")
     end
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 59040199920..68dd87c979d 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -22,9 +22,7 @@ module ContainerRegistry
     end
 
     def manifest
-      return @manifest if defined?(@manifest)
-
-      @manifest = client.repository_manifest(repository.name, name)
+      @manifest ||= client.repository_manifest(repository.name_with_namespace, name)
     end
 
     def path
@@ -40,7 +38,7 @@ module ContainerRegistry
     def digest
       return @digest if defined?(@digest)
 
-      @digest = client.repository_tag_digest(repository.name, name)
+      @digest = client.repository_tag_digest(repository.name_with_namespace, name)
     end
 
     def config_blob
@@ -82,7 +80,7 @@ module ContainerRegistry
     def delete
       return unless digest
 
-      client.delete_repository_tag(repository.name, digest)
+      client.delete_repository_tag(repository.name_with_namespace, digest)
     end
   end
 end