diff options
Diffstat (limited to 'lib/container_registry')
| -rw-r--r-- | lib/container_registry/base_client.rb | 136 | ||||
| -rw-r--r-- | lib/container_registry/client.rb | 135 | ||||
| -rw-r--r-- | lib/container_registry/gitlab_api_client.rb | 76 | ||||
| -rw-r--r-- | lib/container_registry/migration.rb | 53 | ||||
| -rw-r--r-- | lib/container_registry/registry.rb | 18 |
5 files changed, 288 insertions, 130 deletions
diff --git a/lib/container_registry/base_client.rb b/lib/container_registry/base_client.rb new file mode 100644 index 00000000000..22d4510fe71 --- /dev/null +++ b/lib/container_registry/base_client.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'faraday' +require 'faraday_middleware' +require 'digest' + +module ContainerRegistry + class BaseClient + DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' + DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE = 'application/vnd.docker.distribution.manifest.list.v2+json' + OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' + CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json' + + ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze + ACCEPTED_TYPES_RAW = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE, DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE].freeze + + RETRY_EXCEPTIONS = [Faraday::Request::Retry::DEFAULT_EXCEPTIONS, Faraday::ConnectionFailed].flatten.freeze + RETRY_OPTIONS = { + max: 1, + interval: 5, + exceptions: RETRY_EXCEPTIONS + }.freeze + + ERROR_CALLBACK_OPTIONS = { + callback: -> (env, exception) do + Gitlab::ErrorTracking.log_exception( + exception, + class: name, + url: env[:url] + ) + end + }.freeze + + # Taken from: FaradayMiddleware::FollowRedirects + REDIRECT_CODES = Set.new [301, 302, 303, 307] + + class << self + private + + def with_dummy_client(return_value_if_disabled: nil) + registry_config = Gitlab.config.registry + unless registry_config.enabled && registry_config.api_url.present? + return return_value_if_disabled + end + + token = Auth::ContainerRegistryAuthenticationService.access_token([], []) + yield new(registry_config.api_url, token: token) + end + end + + def initialize(base_uri, options = {}) + @base_uri = base_uri + @options = options + end + + private + + def faraday(timeout_enabled: true) + @faraday ||= faraday_base(timeout_enabled: timeout_enabled) do |conn| + initialize_connection(conn, @options, &method(:configure_connection)) + end + end + + def faraday_base(timeout_enabled: true, &block) + request_options = timeout_enabled ? Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS : nil + + Faraday.new( + @base_uri, + headers: { user_agent: "GitLab/#{Gitlab::VERSION}" }, + request: request_options, + &block + ) + end + + def initialize_connection(conn, options) + conn.request :json + + if options[:user] && options[:password] + conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) + elsif options[:token] + conn.request(:authorization, :bearer, options[:token].to_s) + end + + yield(conn) if block_given? + + conn.request(:retry, RETRY_OPTIONS) + conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS) + conn.adapter :net_http + end + + def response_body(response, allow_redirect: false) + if allow_redirect && REDIRECT_CODES.include?(response.status) + response = redirect_response(response.headers['location']) + end + + response.body if response && response.success? + end + + def redirect_response(location) + return unless location + + uri = URI(@base_uri).merge(location) + raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme) + + faraday_redirect.get(uri) + end + + def configure_connection(conn) + conn.headers['Accept'] = ACCEPTED_TYPES + + conn.response :json, content_type: 'application/json' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' + conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' + conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE + conn.response :json, content_type: OCI_MANIFEST_V1_TYPE + end + + # Create a new request to make sure the Authorization header is not inserted + # via the Faraday middleware + def faraday_redirect + @faraday_redirect ||= faraday_base do |conn| + conn.request :json + + conn.request(:retry, RETRY_OPTIONS) + conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS) + conn.adapter :net_http + end + end + + def delete_if_exists(path) + result = faraday.delete(path) + + result.success? || result.status == 404 + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index c2ad9e6ae89..add238350dd 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -1,68 +1,25 @@ # frozen_string_literal: true -require 'faraday' -require 'faraday_middleware' -require 'digest' - module ContainerRegistry - class Client + class Client < BaseClient include Gitlab::Utils::StrongMemoize attr_accessor :uri - DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json' - DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE = 'application/vnd.docker.distribution.manifest.list.v2+json' - OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json' - CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json' REGISTRY_VERSION_HEADER = 'gitlab-container-registry-version' REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features' REGISTRY_TAG_DELETE_FEATURE = 'tag_delete' - ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze - - ACCEPTED_TYPES_RAW = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE, DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE].freeze - - # Taken from: FaradayMiddleware::FollowRedirects - REDIRECT_CODES = Set.new [301, 302, 303, 307] - - RETRY_EXCEPTIONS = [Faraday::Request::Retry::DEFAULT_EXCEPTIONS, Faraday::ConnectionFailed].flatten.freeze - RETRY_OPTIONS = { - max: 1, - interval: 5, - exceptions: RETRY_EXCEPTIONS - }.freeze - - ERROR_CALLBACK_OPTIONS = { - callback: -> (env, exception) do - Gitlab::ErrorTracking.log_exception( - exception, - class: name, - url: env[:url] - ) - end - }.freeze - def self.supports_tag_delete? - registry_config = Gitlab.config.registry - return false unless registry_config.enabled && registry_config.api_url.present? - - token = Auth::ContainerRegistryAuthenticationService.access_token([], []) - client = new(registry_config.api_url, token: token) - client.supports_tag_delete? + with_dummy_client(return_value_if_disabled: false) do |client| + client.supports_tag_delete? + end end def self.registry_info - registry_config = Gitlab.config.registry - return unless registry_config.enabled && registry_config.api_url.present? - - token = Auth::ContainerRegistryAuthenticationService.access_token([], []) - client = new(registry_config.api_url, token: token) - client.registry_info - end - - def initialize(base_uri, options = {}) - @base_uri = base_uri - @options = options + with_dummy_client do |client| + client.registry_info + end end def registry_info @@ -176,89 +133,11 @@ module ContainerRegistry private - def initialize_connection(conn, options) - conn.request :json - - if options[:user] && options[:password] - conn.request(:basic_auth, options[:user].to_s, options[:password].to_s) - elsif options[:token] - conn.request(:authorization, :bearer, options[:token].to_s) - end - - yield(conn) if block_given? - - conn.request(:retry, RETRY_OPTIONS) - conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS) - conn.adapter :net_http - end - - def accept_manifest(conn) - conn.headers['Accept'] = ACCEPTED_TYPES - - conn.response :json, content_type: 'application/json' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+prettyjws' - conn.response :json, content_type: 'application/vnd.docker.distribution.manifest.v1+json' - conn.response :json, content_type: DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE - conn.response :json, content_type: OCI_MANIFEST_V1_TYPE - end - - def response_body(response, allow_redirect: false) - if allow_redirect && REDIRECT_CODES.include?(response.status) - response = redirect_response(response.headers['location']) - end - - response.body if response && response.success? - end - - def redirect_response(location) - return unless location - - uri = URI(@base_uri).merge(location) - raise ArgumentError, "Invalid scheme for #{location}" unless %w[http https].include?(uri.scheme) - - faraday_redirect.get(uri) - end - - def faraday(timeout_enabled: true) - @faraday ||= faraday_base(timeout_enabled: timeout_enabled) do |conn| - initialize_connection(conn, @options, &method(:accept_manifest)) - end - end - def faraday_blob @faraday_blob ||= faraday_base do |conn| initialize_connection(conn, @options) end end - - # Create a new request to make sure the Authorization header is not inserted - # via the Faraday middleware - def faraday_redirect - @faraday_redirect ||= faraday_base do |conn| - conn.request :json - - conn.request(:retry, RETRY_OPTIONS) - conn.request(:gitlab_error_callback, ERROR_CALLBACK_OPTIONS) - conn.adapter :net_http - end - end - - def faraday_base(timeout_enabled: true, &block) - request_options = timeout_enabled ? Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS : nil - - Faraday.new( - @base_uri, - headers: { user_agent: "GitLab/#{Gitlab::VERSION}" }, - request: request_options, - &block - ) - end - - def delete_if_exists(path) - result = faraday.delete(path) - - result.success? || result.status == 404 - end end end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb new file mode 100644 index 00000000000..20b8e1d419b --- /dev/null +++ b/lib/container_registry/gitlab_api_client.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module ContainerRegistry + class GitlabApiClient < BaseClient + include Gitlab::Utils::StrongMemoize + + JSON_TYPE = 'application/json' + + IMPORT_RESPONSES = { + 200 => :already_imported, + 202 => :ok, + 401 => :unauthorized, + 404 => :not_found, + 409 => :already_being_imported, + 424 => :pre_import_failed, + 425 => :already_being_imported, + 429 => :too_many_imports + }.freeze + + REGISTRY_GITLAB_V1_API_FEATURE = 'gitlab_v1_api' + + def self.supports_gitlab_api? + with_dummy_client(return_value_if_disabled: false) do |client| + client.supports_gitlab_api? + end + end + + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#compliance-check + def supports_gitlab_api? + strong_memoize(:supports_gitlab_api) do + registry_features = Gitlab::CurrentSettings.container_registry_features || [] + next true if ::Gitlab.com? && registry_features.include?(REGISTRY_GITLAB_V1_API_FEATURE) + + response = faraday.get('/gitlab/v1/') + response.success? || response.status == 401 + end + end + + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#import-repository + def pre_import_repository(path) + response = start_import_for(path, pre: true) + IMPORT_RESPONSES.fetch(response.status, :error) + end + + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#import-repository + def import_repository(path) + response = start_import_for(path, pre: false) + IMPORT_RESPONSES.fetch(response.status, :error) + end + + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-import-status + def import_status(path) + body_hash = response_body(faraday.get(import_url_for(path))) + body_hash['status'] || 'error' + end + + private + + def start_import_for(path, pre:) + faraday.put(import_url_for(path)) do |req| + req.params['pre'] = pre.to_s + end + end + + def import_url_for(path) + "/gitlab/v1/import/#{path}/" + end + + # overrides the default configuration + def configure_connection(conn) + conn.headers['Accept'] = [JSON_TYPE] + + conn.response :json, content_type: JSON_TYPE + end + end +end diff --git a/lib/container_registry/migration.rb b/lib/container_registry/migration.rb new file mode 100644 index 00000000000..b03c94e5ebf --- /dev/null +++ b/lib/container_registry/migration.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module ContainerRegistry + module Migration + class << self + delegate :container_registry_import_max_tags_count, to: ::Gitlab::CurrentSettings + delegate :container_registry_import_max_retries, to: ::Gitlab::CurrentSettings + delegate :container_registry_import_start_max_retries, to: ::Gitlab::CurrentSettings + delegate :container_registry_import_max_step_duration, to: ::Gitlab::CurrentSettings + delegate :container_registry_import_target_plan, to: ::Gitlab::CurrentSettings + delegate :container_registry_import_created_before, to: ::Gitlab::CurrentSettings + + alias_method :max_tags_count, :container_registry_import_max_tags_count + alias_method :max_retries, :container_registry_import_max_retries + alias_method :start_max_retries, :container_registry_import_start_max_retries + alias_method :max_step_duration, :container_registry_import_max_step_duration + alias_method :target_plan_name, :container_registry_import_target_plan + alias_method :created_before, :container_registry_import_created_before + end + + def self.enabled? + Feature.enabled?(:container_registry_migration_phase2_enabled) + end + + def self.limit_gitlab_org? + Feature.enabled?(:container_registry_migration_limit_gitlab_org) + end + + def self.enqueue_waiting_time + return 0 if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_fast) + return 6.hours if Feature.enabled?(:container_registry_migration_phase2_enqueue_speed_slow) + + 1.hour + end + + def self.capacity + # Increasing capacity numbers will increase the n+1 API calls we can have + # in ContainerRegistry::Migration::GuardWorker#external_migration_in_progress? + # + # TODO: See https://gitlab.com/gitlab-org/container-registry/-/issues/582 + # + return 25 if Feature.enabled?(:container_registry_migration_phase2_capacity_25) + return 10 if Feature.enabled?(:container_registry_migration_phase2_capacity_10) + return 1 if Feature.enabled?(:container_registry_migration_phase2_capacity_1) + + 0 + end + + def self.target_plan + Plan.find_by_name(target_plan_name) + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 523364ac7c7..710f8169a00 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -2,12 +2,26 @@ module ContainerRegistry class Registry + include Gitlab::Utils::StrongMemoize + attr_reader :uri, :client, :path def initialize(uri, options = {}) @uri = uri - @path = options[:path] || default_path - @client = ContainerRegistry::Client.new(uri, options) + @options = options + @path = @options[:path] || default_path + @client = ContainerRegistry::Client.new(@uri, @options) + end + + def gitlab_api_client + strong_memoize(:gitlab_api_client) do + token = Auth::ContainerRegistryAuthenticationService.import_access_token + + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + + ContainerRegistry::GitlabApiClient.new(url, token: token, path: host_port) + end end private |