Add latest changes from gitlab-org/gitlab@master

9 files changed, 103 insertions, 22 deletions

diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb
index 4cc680068b6..2e22f65731d 100644
--- a/lib/api/concerns/packages/npm_endpoints.rb
+++ b/lib/api/concerns/packages/npm_endpoints.rb
@@ -26,6 +26,24 @@ module API
             authenticate_non_get!
           end
 
+          helpers do
+            def redirect_or_present_audit_report
+              redirect_registry_request(
+                forward_to_registry: true,
+                package_type: :npm,
+                path: options[:path][0],
+                body: Gitlab::Json.dump(request.POST),
+                target: project_or_nil,
+                method: route.request_method
+              ) do
+                authorize_read_package!(project)
+
+                status :ok
+                present []
+              end
+            end
+          end
+
           params do
             requires :package_name, type: String, desc: 'Package name'
           end
@@ -130,6 +148,22 @@ module API
                 with: ::API::Entities::NpmPackage
             end
           end
+
+          desc 'NPM registry bulk advisory endpoint' do
+            detail 'This feature was introduced in GitLab 15.6'
+          end
+          route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+          post '-/npm/v1/security/advisories/bulk' do
+            redirect_or_present_audit_report
+          end
+
+          desc 'NPM registry quick audit endpoint' do
+            detail 'This feature was introduced in GitLab 15.6'
+          end
+          route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
+          post '-/npm/v1/security/audits/quick' do
+            redirect_or_present_audit_report
+          end
         end
       end
     end
diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb
index 1ae863a5a25..4b0e63c8f3b 100644
--- a/lib/api/helpers/packages/dependency_proxy_helpers.rb
+++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb
@@ -19,7 +19,9 @@ module API
         def redirect_registry_request(forward_to_registry: false, package_type: nil, target: nil, **options)
           if forward_to_registry && redirect_registry_request_available?(package_type, target) && maven_forwarding_ff_enabled?(package_type, target)
             ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward")
-            redirect(registry_url(package_type, options))
+            redirect(registry_url(package_type, options), body: options[:body])
+            # For the requests with POST methods we need to set status 307 in order to keep request's method
+            status :temporary_redirect if options[:method] == 'POST'
           else
             yield
           end
@@ -32,7 +34,7 @@ module API
 
           case package_type
           when :npm
-            "#{base_url}#{options[:package_name]}"
+            "#{base_url}#{[options[:path], options[:package_name]].compact.join('/')}"
           when :pypi
             "#{base_url}#{options[:package_name]}/"
           when :maven
diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb
index d549a8be035..b4c00eef3b4 100644
--- a/lib/api/nuget_project_packages.rb
+++ b/lib/api/nuget_project_packages.rb
@@ -39,18 +39,19 @@ module API
       end
 
       def project_or_group
-        authorized_user_project
+        authorized_user_project(action: :read_package)
       end
 
       def snowplow_gitlab_standard_context
-        { project: authorized_user_project, namespace: authorized_user_project.namespace }
+        { project: project_or_group, namespace: project_or_group.namespace }
       end
 
       def authorize_nuget_upload
+        project = project_or_group
         authorize_workhorse!(
-          subject: project_or_group,
+          subject: project,
           has_length: false,
-          maximum_size: project_or_group.actual_limits.nuget_max_file_size
+          maximum_size: project.actual_limits.nuget_max_file_size
         )
       end
 
@@ -67,8 +68,9 @@ module API
       end
 
       def upload_nuget_package_file(symbol_package: false)
-        authorize_upload!(project_or_group)
-        bad_request!('File is too large') if project_or_group.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size)
+        project = project_or_group
+        authorize_upload!(project)
+        bad_request!('File is too large') if project.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size)
 
         file_params = params.merge(
           file: params[:package],
@@ -76,7 +78,7 @@ module API
         )
 
         package = ::Packages::CreateTemporaryPackageService.new(
-          project_or_group, current_user, declared_params.merge(build: current_authenticated_job)
+          project, current_user, declared_params.merge(build: current_authenticated_job)
         ).execute(:nuget, name: temp_file_name(symbol_package))
 
         package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job))
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index 04cf056199c..51d5bfcee38 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -101,7 +101,8 @@ module Gitlab
         gitlab_main: [self.database_base_models.fetch(:main)],
         gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main
         gitlab_shared: database_base_models_with_gitlab_shared.values, # all models
-        gitlab_internal: database_base_models.values # all models
+        gitlab_internal: database_base_models.values, # all models
+        gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models
       }.with_indifferent_access.freeze
     end
 
diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml
index 704bd929595..788d343dad9 100644
--- a/lib/gitlab/database/gitlab_schemas.yml
+++ b/lib/gitlab/database/gitlab_schemas.yml
@@ -409,6 +409,10 @@ path_locks: :gitlab_main
 personal_access_tokens: :gitlab_main
 plan_limits: :gitlab_main
 plans: :gitlab_main
+pm_licenses: :gitlab_pm
+pm_packages: :gitlab_pm
+pm_package_versions: :gitlab_pm
+pm_package_version_licenses: :gitlab_pm
 pool_repositories: :gitlab_main
 postgres_async_indexes: :gitlab_shared
 postgres_autovacuum_activity: :gitlab_shared
diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb
index e20e186cab9..81c4be0f7fc 100644
--- a/lib/gitlab/memory/jemalloc.rb
+++ b/lib/gitlab/memory/jemalloc.rb
@@ -97,7 +97,9 @@ module Gitlab
       end
 
       def file_name(extension, filename_label)
-        [FILENAME_PREFIX, $$, filename_label, Time.current.to_i, extension].reject(&:blank?).join('.')
+        timestamp = Time.current.strftime('%Y-%m-%d.%H:%M:%S:%L')
+
+        [FILENAME_PREFIX, timestamp, filename_label, extension].reject(&:blank?).join('.')
       end
     end
   end
diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb
index 05f0717d7c3..720f22ddbe4 100644
--- a/lib/gitlab/memory/reports/jemalloc_stats.rb
+++ b/lib/gitlab/memory/reports/jemalloc_stats.rb
@@ -16,8 +16,9 @@ module Gitlab
         # The cleanup logic will be redundant after we'll implement the uploads, which would perform the cleanup.
         DEFAULT_MAX_REPORTS_STORED = 250
 
-        def initialize(reports_path:)
+        def initialize(reports_path:, filename_label:)
           @reports_path = reports_path
+          @filename_label = filename_label
 
           # Store report in tmp subdir while it is still streaming.
           # This will clearly separate finished reports from the files we are still writing to.
@@ -28,7 +29,8 @@ module Gitlab
         def run
           return unless active?
 
-          Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, filename_label: worker_id).tap do
+          Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir,
+                                              filename_label: filename_label).tap do
             cleanup
           end
         end
@@ -39,7 +41,7 @@ module Gitlab
 
         private
 
-        attr_reader :reports_path
+        attr_reader :reports_path, :filename_label
 
         def cleanup
           reports_files_modified_order[0...-max_reports_stored].each do |f|
@@ -61,10 +63,6 @@ module Gitlab
           end
         end
 
-        def worker_id
-          ::Prometheus::PidProvider.worker_id
-        end
-
         def max_reports_stored
           ENV["GITLAB_DIAGNOSTIC_REPORTS_JEMALLOC_MAX_REPORTS_STORED"] || DEFAULT_MAX_REPORTS_STORED
         end
diff --git a/lib/gitlab/memory/reports_daemon.rb b/lib/gitlab/memory/reports_daemon.rb
index 0dfc31235e7..7070c65c705 100644
--- a/lib/gitlab/memory/reports_daemon.rb
+++ b/lib/gitlab/memory/reports_daemon.rb
@@ -24,7 +24,15 @@ module Gitlab
         @reports_path =
           ENV["GITLAB_DIAGNOSTIC_REPORTS_PATH"] || DEFAULT_REPORTS_PATH
 
-        @reports = [Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path)]
+        # Set unique uuid for every ReportsDaemon instance.
+        # Because we spawn a single instance of it per process, it will also uniquely identify the worker.
+        # Unlike `::Prometheus::PidProvider.worker_id`, this uuid will remain unique across all Puma clusters.
+        # This way, we can identify reports that were produced from the same worker process during its lifetime.
+        @worker_uuid = SecureRandom.uuid
+
+        @reports = [
+          Gitlab::Memory::Reports::JemallocStats.new(reports_path: reports_path, filename_label: filename_label)
+        ]
 
         init_prometheus_metrics
       end
@@ -54,7 +62,11 @@ module Gitlab
 
       private
 
-      attr_reader :alive, :reports
+      attr_reader :alive, :reports, :worker_uuid
+
+      def filename_label
+        [worker_id, worker_uuid].join(".")
+      end
 
       # Returns the sleep interval with a random adjustment.
       # The random adjustment is put in place to ensure continued availability.
@@ -70,7 +82,8 @@ module Gitlab
           perf_report: label,
           duration_s: duration_s.round(2),
           cpu_s: cpu_s.round(2),
-          perf_report_size_bytes: size
+          perf_report_size_bytes: size,
+          perf_report_worker_uuid: worker_uuid
         )
       end
 
diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb
index f66dfe44054..80916eab5ac 100644
--- a/lib/gitlab/middleware/compressed_json.rb
+++ b/lib/gitlab/middleware/compressed_json.rb
@@ -4,7 +4,18 @@ module Gitlab
   module Middleware
     class CompressedJson
       COLLECTOR_PATH = '/api/v4/error_tracking/collector'
+      PACKAGES_PATH = %r{
+        \A/api/v4/ (?# prefix)
+        (?:projects/
+          (?<project_id>
+            .+ (?# at least one character)
+          )/
+        )? (?# projects segment)
+       packages/npm/-/npm/v1/security/
+       (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end)
+      }xi.freeze
       MAXIMUM_BODY_SIZE = 200.kilobytes.to_i
+      UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze
 
       def initialize(app)
         @app = app
@@ -60,7 +71,21 @@ module Gitlab
       end
 
       def match_path?(env)
-        env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH)))
+        env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) ||
+          match_packages_path?(env)
+      end
+
+      def match_packages_path?(env)
+        match_data = env['PATH_INFO'].delete_prefix(relative_url).match(PACKAGES_PATH)
+        return false unless match_data
+
+        return true unless match_data[:project_id] # instance level endpoint was matched
+
+        url_encoded?(match_data[:project_id])
+      end
+
+      def url_encoded?(project_id)
+        project_id !~ UNSAFE_CHARACTERS
       end
     end
   end