diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 18:11:26 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 18:11:26 +0300 |
commit | 4e752429e6173020567f9509f1fa993cc82a258a (patch) | |
tree | 49648f91db0d7849065d2d8897757f7de815c773 /lib | |
parent | 255831389a5080bb61242b3b50426918c4e1a5aa (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/concerns/packages/npm_endpoints.rb | 34 | ||||
-rw-r--r-- | lib/api/helpers/packages/dependency_proxy_helpers.rb | 6 | ||||
-rw-r--r-- | lib/api/nuget_project_packages.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/database.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/database/gitlab_schemas.yml | 4 | ||||
-rw-r--r-- | lib/gitlab/memory/jemalloc.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/memory/reports/jemalloc_stats.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/memory/reports_daemon.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/middleware/compressed_json.rb | 27 |
9 files changed, 103 insertions, 22 deletions
diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 4cc680068b6..2e22f65731d 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -26,6 +26,24 @@ module API authenticate_non_get! end + helpers do + def redirect_or_present_audit_report + redirect_registry_request( + forward_to_registry: true, + package_type: :npm, + path: options[:path][0], + body: Gitlab::Json.dump(request.POST), + target: project_or_nil, + method: route.request_method + ) do + authorize_read_package!(project) + + status :ok + present [] + end + end + end + params do requires :package_name, type: String, desc: 'Package name' end @@ -130,6 +148,22 @@ module API with: ::API::Entities::NpmPackage end end + + desc 'NPM registry bulk advisory endpoint' do + detail 'This feature was introduced in GitLab 15.6' + end + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + post '-/npm/v1/security/advisories/bulk' do + redirect_or_present_audit_report + end + + desc 'NPM registry quick audit endpoint' do + detail 'This feature was introduced in GitLab 15.6' + end + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + post '-/npm/v1/security/audits/quick' do + redirect_or_present_audit_report + end end end end diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index 1ae863a5a25..4b0e63c8f3b 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -19,7 +19,9 @@ module API def redirect_registry_request(forward_to_registry: false, package_type: nil, target: nil, **options) if forward_to_registry && redirect_registry_request_available?(package_type, target) && maven_forwarding_ff_enabled?(package_type, target) ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward") - redirect(registry_url(package_type, options)) + redirect(registry_url(package_type, options), body: options[:body]) + # For the requests with POST methods we need to set status 307 in order to keep request's method + status :temporary_redirect if options[:method] == 'POST' else yield end @@ -32,7 +34,7 @@ module API case package_type when :npm - "#{base_url}#{options[:package_name]}" + "#{base_url}#{[options[:path], options[:package_name]].compact.join('/')}" when :pypi "#{base_url}#{options[:package_name]}/" when :maven diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index d549a8be035..b4c00eef3b4 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -39,18 +39,19 @@ module API end def project_or_group - authorized_user_project + authorized_user_project(action: :read_package) end def snowplow_gitlab_standard_context - { project: authorized_user_project, namespace: authorized_user_project.namespace } + { project: project_or_group, namespace: project_or_group.namespace } end def authorize_nuget_upload + project = project_or_group authorize_workhorse!( - subject: project_or_group, + subject: project, has_length: false, - maximum_size: project_or_group.actual_limits.nuget_max_file_size + maximum_size: project.actual_limits.nuget_max_file_size ) end @@ -67,8 +68,9 @@ module API end def upload_nuget_package_file(symbol_package: false) - authorize_upload!(project_or_group) - bad_request!('File is too large') if project_or_group.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) + project = project_or_group + authorize_upload!(project) + bad_request!('File is too large') if project.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size) file_params = params.merge( file: params[:package], @@ -76,7 +78,7 @@ module API ) package = ::Packages::CreateTemporaryPackageService.new( - project_or_group, current_user, declared_params.merge(build: current_authenticated_job) + project, current_user, declared_params.merge(build: current_authenticated_job) ).execute(:nuget, name: temp_file_name(symbol_package)) package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 04cf056199c..51d5bfcee38 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -101,7 +101,8 @@ module Gitlab gitlab_main: [self.database_base_models.fetch(:main)], gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main gitlab_shared: database_base_models_with_gitlab_shared.values, # all models - gitlab_internal: database_base_models.values # all models + gitlab_internal: database_base_models.values, # all models + gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models }.with_indifferent_access.freeze end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 704bd929595..788d343dad9 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -409,6 +409,10 @@ path_locks: :gitlab_main personal_access_tokens: :gitlab_main plan_limits: :gitlab_main plans: :gitlab_main +pm_licenses: :gitlab_pm +pm_packages: :gitlab_pm +pm_package_versions: :gitlab_pm +pm_package_version_licenses: :gitlab_pm pool_repositories: :gitlab_main postgres_async_indexes: :gitlab_shared postgres_autovacuum_activity: :gitlab_shared diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb index e20e186cab9..81c4be0f7fc 100644 --- a/lib/gitlab/memory/jemalloc.rb +++ b/lib/gitlab/memory/jemalloc.rb @@ -97,7 +97,9 @@ module Gitlab end def file_name(extension, filename_label) - [FILENAME_PREFIX, $$, filename_label, Time.current.to_i, extension].reject(&:blank?).join('.') + timestamp = Time.current.strftime('%Y-%m-%d.%H:%M:%S:%L') + + [FILENAME_PREFIX, timestamp, filename_label, extension].reject(&:blank?).join('.') end end end diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb index 05f0717d7c3..720f22ddbe4 100644 --- a/lib/gitlab/memory/reports/jemalloc_stats.rb +++ b/lib/gitlab/memory/reports/jemalloc_stats.rb @@ -16,8 +16,9 @@ module Gitlab # The cleanup logic will be redundant after we'll implement the uploads, which would perform the cleanup. DEFAULT_MAX_REPORTS_STORED = 250 - def initialize(reports_path:) + def initialize(reports_path:, filename_label:) @reports_path = reports_path + @filename_label = filename_label # Store report in tmp subdir while it is still streaming. # This will clearly separate finished reports from the files we are still writing to. @@ -28,7 +29,8 @@ module Gitlab def run return unless active? - Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, filename_label: worker_id).tap do + Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, + filename_label: filename_label).tap do cleanup end end @@ -39,7 +41,7 @@ module Gitlab private - attr_reader :reports_path + attr_reader :reports_path, :filename_label def cleanup reports_files_modified_order[0...-max_reports_stored].each do |f| @@ -61,10 +63,6 @@ module Gitlab end end - def worker_id - ::Prometheus::PidProvider.worker_id - end - def max_reports_stored ENV["GITLAB_DIAGNOSTIC_REPORTS_JEMALLOC_MAX_REPORTS_STORED"] || DEFAULT_MAX_REPORTS_STORED end diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb index 0dfc31235e7..7070c65c705 100644 --- a/lib/gitlab/memory/reports_daemon.rb +++ b/lib/gitlab/memory/reports_daemon.rb @@ -24,7 +24,15 @@ module Gitlab @reports_path = ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || DEFAULT_REPORTS_PATH - @reports = [Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path)] + # Set unique uuid for every ReportsDaemon instance. + # Because we spawn a single instance of it per process, it will also uniquely identify the worker. + # Unlike `::Prometheus::PidProvider.worker_id`, this uuid will remain unique across all Puma clusters. + # This way, we can identify reports that were produced from the same worker process during its lifetime. + @worker_uuid = SecureRandom.uuid + + @reports = [ + Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path, filename_label: filename_label) + ] init_prometheus_metrics end @@ -54,7 +62,11 @@ module Gitlab private - attr_reader :alive, :reports + attr_reader :alive, :reports, :worker_uuid + + def filename_label + [worker_id, worker_uuid].join(".") + end # Returns the sleep interval with a random adjustment. # The random adjustment is put in place to ensure continued availability. @@ -70,7 +82,8 @@ module Gitlab perf_report: label, duration_s: duration_s.round(2), cpu_s: cpu_s.round(2), - perf_report_size_bytes: size + perf_report_size_bytes: size, + perf_report_worker_uuid: worker_uuid ) end diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index f66dfe44054..80916eab5ac 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -4,7 +4,18 @@ module Gitlab module Middleware class CompressedJson COLLECTOR_PATH = '/api/v4/error_tracking/collector' + PACKAGES_PATH = %r{ + \A/api/v4/ (?# prefix) + (?:projects/ + (?<project_id> + .+ (?# at least one character) + )/ + )? (?# projects segment) + packages/npm/-/npm/v1/security/ + (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) + }xi.freeze MAXIMUM_BODY_SIZE = 200.kilobytes.to_i + UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze def initialize(app) @app = app @@ -60,7 +71,21 @@ module Gitlab end def match_path?(env) - env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) + env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) || + match_packages_path?(env) + end + + def match_packages_path?(env) + match_data = env['PATH_INFO'].delete_prefix(relative_url).match(PACKAGES_PATH) + return false unless match_data + + return true unless match_data[:project_id] # instance level endpoint was matched + + url_encoded?(match_data[:project_id]) + end + + def url_encoded?(project_id) + project_id !~ UNSAFE_CHARACTERS end end end |