diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/concerns/packages/debian_endpoints.rb | 133 | ||||
| -rw-r--r-- | lib/api/debian_group_packages.rb | 4 | ||||
| -rw-r--r-- | lib/api/debian_package_endpoints.rb | 127 | ||||
| -rw-r--r-- | lib/api/debian_project_packages.rb | 6 | ||||
| -rw-r--r-- | lib/banzai/filter/base_relative_link_filter.rb | 33 | ||||
| -rw-r--r-- | lib/banzai/filter/upload_link_filter.rb | 12 | ||||
| -rw-r--r-- | lib/file_size_validator.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/api_authentication/token_locator.rb | 11 | ||||
| -rw-r--r-- | lib/gitlab/api_authentication/token_resolver.rb | 48 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/metrics.rb | 7 | ||||
| -rw-r--r-- | lib/gitlab/jwt_token.rb | 70 | ||||
| -rw-r--r-- | lib/gitlab/terraform_registry_token.rb | 13 |
12 files changed, 317 insertions, 149 deletions
diff --git a/lib/api/concerns/packages/debian_endpoints.rb b/lib/api/concerns/packages/debian_endpoints.rb new file mode 100644 index 00000000000..6fc7c439464 --- /dev/null +++ b/lib/api/concerns/packages/debian_endpoints.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +module API + module Concerns + module Packages + module DebianEndpoints + extend ActiveSupport::Concern + + DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze + COMPONENT_REGEX = %r{[a-z-]+}.freeze + ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze + LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze + PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX + DISTRIBUTION_REQUIREMENTS = { + distribution: DISTRIBUTION_REGEX + }.freeze + COMPONENT_ARCHITECTURE_REQUIREMENTS = { + component: COMPONENT_REGEX, + architecture: ARCHITECTURE_REGEX + }.freeze + COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { + component: COMPONENT_REGEX, + letter: LETTER_REGEX, + source_package: PACKAGE_REGEX + }.freeze + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + included do + feature_category :package_registry + + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + + format :txt + content_type :txt, 'text/plain' + + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + before do + require_packages_enabled! + end + + namespace 'packages/debian' do + params do + requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex + end + + namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg + desc 'The Release file signature' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'Release.gpg' do + not_found! + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release + desc 'The unsigned Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'Release' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Release' + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease + desc 'The signed Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'InRelease' do + not_found! + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex + end + + namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + desc 'The binary files index' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get 'Packages' do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO Packages' + end + end + end + + params do + requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex + requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' + requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex + end + + namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name + params do + requires :file_name, type: String, desc: 'The Debian File Name' + end + desc 'The package' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true + get ':file_name', requirements: FILE_NAME_REQUIREMENTS do + # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 + 'TODO File' + end + end + end + end + end + end + end +end diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index cf9fa388e9e..06edab662bf 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -15,8 +15,8 @@ module API authorize_read_package!(user_group) end - namespace ':id/-/packages/debian' do - include DebianPackageEndpoints + namespace ':id/-' do + include ::API::Concerns::Packages::DebianEndpoints end end end diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb deleted file mode 100644 index e7689b3feff..00000000000 --- a/lib/api/debian_package_endpoints.rb +++ /dev/null @@ -1,127 +0,0 @@ -# frozen_string_literal: true - -module API - module DebianPackageEndpoints - extend ActiveSupport::Concern - - DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze - COMPONENT_REGEX = %r{[a-z-]+}.freeze - ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze - LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze - PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX - DISTRIBUTION_REQUIREMENTS = { - distribution: DISTRIBUTION_REGEX - }.freeze - COMPONENT_ARCHITECTURE_REQUIREMENTS = { - component: COMPONENT_REGEX, - architecture: ARCHITECTURE_REGEX - }.freeze - COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { - component: COMPONENT_REGEX, - letter: LETTER_REGEX, - source_package: PACKAGE_REGEX - }.freeze - FILE_NAME_REQUIREMENTS = { - file_name: API::NO_SLASH_URL_PART_REGEX - }.freeze - - included do - feature_category :package_registry - - helpers ::API::Helpers::PackagesHelpers - helpers ::API::Helpers::Packages::BasicAuthHelpers - - format :txt - content_type :txt, 'text/plain' - - rescue_from ArgumentError do |e| - render_api_error!(e.message, 400) - end - - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) - end - - before do - require_packages_enabled! - end - - params do - requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex - end - - namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg - desc 'The Release file signature' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'Release.gpg' do - not_found! - end - - # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release - desc 'The unsigned Release file' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'Release' do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO Release' - end - - # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease - desc 'The signed Release file' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'InRelease' do - not_found! - end - - params do - requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex - requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex - end - - namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages - desc 'The binary files index' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get 'Packages' do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO Packages' - end - end - end - - params do - requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex - requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' - requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex - end - - namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name - params do - requires :file_name, type: String, desc: 'The Debian File Name' - end - desc 'The package' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true - get ':file_name', requirements: FILE_NAME_REQUIREMENTS do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO File' - end - end - end - end -end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 8c0db42a448..0ed828fd639 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -15,14 +15,14 @@ module API authorize_read_package! end - namespace ':id/packages/debian' do - include DebianPackageEndpoints + namespace ':id' do + include ::API::Concerns::Packages::DebianEndpoints params do requires :file_name, type: String, desc: 'The file name' end - namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do + namespace 'packages/debian/:file_name', requirements: FILE_NAME_REQUIREMENTS do content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE # PUT {projects|groups}/:id/packages/debian/:file_name diff --git a/lib/banzai/filter/base_relative_link_filter.rb b/lib/banzai/filter/base_relative_link_filter.rb index fd526df4c48..84a6e18e77b 100644 --- a/lib/banzai/filter/base_relative_link_filter.rb +++ b/lib/banzai/filter/base_relative_link_filter.rb @@ -10,19 +10,16 @@ module Banzai protected def linkable_attributes - strong_memoize(:linkable_attributes) do - attrs = [] - - attrs += doc.search('a:not(.gfm)').map do |el| - el.attribute('href') - end - - attrs += doc.search('img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| - [el.attribute('src'), el.attribute('data-src')] - end - - attrs.reject do |attr| - attr.blank? || attr.value.start_with?('//') + if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) + # Nokorigi Nodeset#search performs badly for documents with many nodes + # + # Here we store fetched attributes in the shared variable "result" + # This variable is passed through the chain of filters and can be + # accessed by them + result[:linkable_attributes] ||= fetch_linkable_attributes + else + strong_memoize(:linkable_attributes) do + fetch_linkable_attributes end end end @@ -40,6 +37,16 @@ module Banzai def unescape_and_scrub_uri(uri) Addressable::URI.unescape(uri).scrub.delete("\0") end + + def fetch_linkable_attributes + attrs = [] + + attrs += doc.search('a:not(.gfm), img:not(.gfm), video:not(.gfm), audio:not(.gfm)').flat_map do |el| + [el.attribute('href'), el.attribute('src'), el.attribute('data-src')] + end + + attrs.reject { |attr| attr.blank? || attr.value.start_with?('//') } + end end end end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index 762371e1418..ceb7547a85d 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -15,8 +15,16 @@ module Banzai def call return doc if context[:system_note] - linkable_attributes.each do |attr| - process_link_to_upload_attr(attr) + if Feature.enabled?(:optimize_linkable_attributes, project, default_enabled: :yaml) + # We exclude processed upload links from the linkable attributes to + # prevent further modifications by RepositoryLinkFilter + linkable_attributes.reject! do |attr| + process_link_to_upload_attr(attr) + end + else + linkable_attributes.each do |attr| + process_link_to_upload_attr(attr) + end end doc diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index e9868732172..ac3210b4e98 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -62,7 +62,7 @@ class FileSizeValidator < ActiveModel::EachValidator default_message = options[MESSAGES[key]] errors_options[:message] ||= default_message if default_message - record.errors.add(attribute, MESSAGES[key], errors_options) + record.errors.add(attribute, MESSAGES[key], **errors_options) end end diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb index 09039f3fc43..6ab37487822 100644 --- a/lib/gitlab/api_authentication/token_locator.rb +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :location - validates :location, inclusion: { in: %i[http_basic_auth http_token] } + validates :location, inclusion: { in: %i[http_basic_auth http_token token_param] } def initialize(location) @location = location @@ -23,6 +23,8 @@ module Gitlab extract_from_http_basic_auth request when :http_token extract_from_http_token request + when :token_param + extract_from_token_param request end end @@ -41,6 +43,13 @@ module Gitlab UsernameAndPassword.new(nil, password) end + + def extract_from_token_param(request) + password = request.query_parameters['token'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end end end end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb index 9234837cdf7..dd9039e37f6 100644 --- a/lib/gitlab/api_authentication/token_resolver.rb +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -15,9 +15,14 @@ module Gitlab personal_access_token job_token deploy_token + personal_access_token_from_jwt + deploy_token_from_jwt + job_token_from_jwt ] } + UsernameAndPassword = ::Gitlab::APIAuthentication::TokenLocator::UsernameAndPassword + def initialize(token_type) @token_type = token_type validate! @@ -56,6 +61,15 @@ module Gitlab when :deploy_token_with_username resolve_deploy_token_with_username raw + + when :personal_access_token_from_jwt + resolve_personal_access_token_from_jwt raw + + when :deploy_token_from_jwt + resolve_deploy_token_from_jwt raw + + when :job_token_from_jwt + resolve_job_token_from_jwt raw end end @@ -116,6 +130,33 @@ module Gitlab end end + def resolve_personal_access_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(Integer) + + pat = ::PersonalAccessToken.find(jwt_token['token']) + break unless pat + + pat + end + end + + def resolve_deploy_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(String) + + resolve_deploy_token(UsernameAndPassword.new(nil, jwt_token['token'])) + end + end + + def resolve_job_token_from_jwt(raw) + with_jwt_token(raw) do |jwt_token| + break unless jwt_token['token'].is_a?(String) + + resolve_job_token(UsernameAndPassword.new(nil, jwt_token['token'])) + end + end + def with_personal_access_token(raw, &block) pat = ::PersonalAccessToken.find_by_token(raw.password) return unless pat @@ -136,6 +177,13 @@ module Gitlab yield(job) end + + def with_jwt_token(raw, &block) + jwt_token = ::Gitlab::JWTToken.decode(raw.password) + raise ::Gitlab::Auth::UnauthorizedError unless jwt_token + + yield(jwt_token) + end end end end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 6cb6fd3920d..35e88ded416 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -13,6 +13,13 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_security_orchestration_policy_processing_duration_histogram + name = :gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds + comment = 'Pipeline security orchestration policy processing duration' + + ::Gitlab::Metrics.histogram(name, comment) + end + def self.pipeline_size_histogram name = :gitlab_ci_pipeline_size_builds comment = 'Pipeline size' diff --git a/lib/gitlab/jwt_token.rb b/lib/gitlab/jwt_token.rb new file mode 100644 index 00000000000..11bc5479b6e --- /dev/null +++ b/lib/gitlab/jwt_token.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + class JWTToken < JSONWebToken::HMACToken + HMAC_ALGORITHM = 'SHA256' + HMAC_KEY = 'gitlab-jwt' + HMAC_EXPIRES_IN = 5.minutes.freeze + + class << self + def decode(jwt) + payload = super(jwt, secret).first + + new.tap do |jwt_token| + jwt_token.id = payload.delete('jti') + jwt_token.issued_at = payload.delete('iat') + jwt_token.not_before = payload.delete('nbf') + jwt_token.expire_time = payload.delete('exp') + + payload.each do |key, value| + jwt_token[key] = value + end + end + rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature => ex + # we want to log and return on expired and errored tokens + Gitlab::ErrorTracking.track_exception(ex) + nil + end + + def secret + OpenSSL::HMAC.hexdigest( + HMAC_ALGORITHM, + ::Settings.attr_encrypted_db_key_base, + HMAC_KEY + ) + end + end + + def initialize + super(self.class.secret) + self.expire_time = self.issued_at + HMAC_EXPIRES_IN.to_i + end + + def ==(other) + self.id == other.id && + self.payload == other.payload + end + + def issued_at=(value) + super(convert_time(value)) + end + + def not_before=(value) + super(convert_time(value)) + end + + def expire_time=(value) + super(convert_time(value)) + end + + private + + def convert_time(value) + # JSONWebToken::Token truncates subsecond precision causing comparisons to + # fail unless we truncate it here first + value = value.to_i if value.is_a?(Float) + value = Time.zone.at(value) if value.is_a?(Integer) + value + end + end +end diff --git a/lib/gitlab/terraform_registry_token.rb b/lib/gitlab/terraform_registry_token.rb new file mode 100644 index 00000000000..ae7df49835f --- /dev/null +++ b/lib/gitlab/terraform_registry_token.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + class TerraformRegistryToken < JWTToken + class << self + def from_token(token) + new.tap do |terraform_registry_token| + terraform_registry_token['token'] = token.try(:token).presence || token.try(:id).presence + end + end + end + end +end |