diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-25 18:10:33 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-05-25 18:10:33 +0300 |
commit | a8c1bc6f757ecacbc3481e52a3f4cefb6c6db5fd (patch) | |
tree | 7ba85d57835274f11674c33155e68b6af33f2687 /lib | |
parent | 76ef00aac974a463243dcda6f692b17ff5d439bc (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
21 files changed, 151 insertions, 51 deletions
diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 0ed828fd639..feb83b52695 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -37,6 +37,21 @@ module API track_package_event('push_package', :debian) + file_params = { + file: params['file'], + file_name: params['file_name'], + file_sha1: params['file.sha1'], + file_md5: params['file.md5'] + } + + package = ::Packages::Debian::FindOrCreateIncomingService.new(authorized_user_project, current_user).execute + + package_file = ::Packages::Debian::CreatePackageFileService.new(package, file_params).execute + + if params['file_name'].end_with? '.changes' + ::Packages::Debian::ProcessChangesWorker.perform_async(package_file.id, current_user.id) # rubocop:disable CodeReuse/Worker + end + created! rescue ObjectStorage::RemoteStoreError => e Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) diff --git a/lib/api/helpers/packages/basic_auth_helpers.rb b/lib/api/helpers/packages/basic_auth_helpers.rb index c32ce199dd6..6c381d85cd8 100644 --- a/lib/api/helpers/packages/basic_auth_helpers.rb +++ b/lib/api/helpers/packages/basic_auth_helpers.rb @@ -22,6 +22,14 @@ module API unauthorized_user_project || not_found! end + def unauthorized_user_group + @unauthorized_user_group ||= find_group(params[:id]) + end + + def unauthorized_user_group! + unauthorized_user_group || not_found! + end + def authorized_user_project @authorized_user_project ||= authorized_project_find! end diff --git a/lib/api/lint.rb b/lib/api/lint.rb index e0806674c6a..3580a7b5e24 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -11,7 +11,11 @@ module API optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response' end post '/lint' do - unauthorized! if Gitlab::CurrentSettings.signup_disabled? && current_user.nil? + if Feature.enabled?(:security_ci_lint_authorization) + unauthorized! if (Gitlab::CurrentSettings.signup_disabled? || Gitlab::CurrentSettings.signup_limited?) && current_user.nil? + else + unauthorized! if Gitlab::CurrentSettings.signup_disabled? && current_user.nil? + end result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 73b2f658825..969b619c1cd 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -28,6 +28,73 @@ module API require_packages_enabled! end + helpers do + params :package_download do + requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true + requires :sha256, type: String, desc: 'The PyPi package sha256 check sum' + end + + params :package_name do + requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' + end + end + + params do + requires :id, type: Integer, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + after_validation do + unauthorized_user_group! + end + + namespace ':id/-/packages/pypi' do + params do + use :package_download + end + + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'files/:sha256/*file_identifier' do + group = unauthorized_user_group! + + filename = "#{params[:file_identifier]}.#{params[:format]}" + package = Packages::Pypi::PackageFinder.new(current_user, group, { filename: filename, sha256: params[:sha256] }).execute + package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute + + track_package_event('pull_package', :pypi) + + present_carrierwave_file!(package_file.file, supports_direct_download: true) + end + + desc 'The PyPi Simple Endpoint' do + detail 'This feature was introduced in GitLab 12.10' + end + + params do + use :package_name + end + + # An Api entry point but returns an HTML file instead of JSON. + # PyPi simple API returns the package descriptor as a simple HTML file. + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + get 'simple/*package_name', format: :txt do + group = find_authorized_group! + authorize_read_package!(group) + + track_package_event('list_package', :pypi) + + packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute! + presenter = ::Packages::Pypi::PackagePresenter.new(packages, group) + + # Adjusts grape output format + # to be HTML + content_type "text/html; charset=utf-8" + env['api.format'] = :binary + + body presenter.body + end + end + end + params do requires :id, type: Integer, desc: 'The ID of a project' end @@ -43,8 +110,7 @@ module API end params do - requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true - requires :sha256, type: String, desc: 'The PyPi package sha256 check sum' + use :package_download end route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth @@ -65,7 +131,7 @@ module API end params do - requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' + use :package_name end # An Api entry point but returns an HTML file instead of JSON. diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index c599b261d77..a6d400b6350 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -9,8 +9,8 @@ apply: script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: - refs: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: reports: cluster_applications: gl-cluster-applications.json diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 90fad1550ff..f3c9a93d9fb 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables stages: - build diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 8fa33026011..0c4c39cbcd6 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure API fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-cicd-variables variables: FUZZAPI_VERSION: "1" diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 695aaa37924..84d9a92663a 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -10,7 +10,8 @@ # - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the # DOCKERFILE_PATH variable. # -# For more information, see https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: # Setting this variable will affect all Security templates (e.g.: SAST, Dependency Scanning) diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index ab71049bac2..2dbfb80b419 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing -# Configure the fuzzing tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure coverage fuzzing with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing/#available-cicd-variables variables: # Which branch we want to run full fledged long running fuzzing jobs. diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index b40c4e982f7..9170e943e9d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -13,9 +13,8 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html -# Configure the scanning tool with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html) -# List of variables available to configure the DAST API scanning tool: -# https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables +# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables variables: # Setting this variable affects all Security templates diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 7abecfb7e49..b355b6e36a2 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables stages: - build diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index b6282da18a4..693cf1469c2 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -13,9 +13,8 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure DAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: DAST_VERSION: 1 diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 53d68c24d26..8df5ce79fe8 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/dependency-scanning#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 21e926ef275..870684c9f1d 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a8d45e80356..65e1046ad0d 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ # -# Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/sast#settings -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables variables: # Setting this variable will affect all Security templates diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index c255fb4707a..657ac43b78e 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -1,8 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection # -# Configure the scanning tool through the environment variables. -# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables -# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +# Configure secret detection with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 7f55734f796..e7ffeeb9849 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,6 +7,10 @@ module Gitlab !signup_enabled? end + def signup_limited? + domain_allowlist.present? || email_restrictions_enabled? || require_admin_approval_after_user_signup? + end + def current_application_settings Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index cab3538a447..05daa08530e 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -65,10 +65,9 @@ module Gitlab def project_from_key return unless match = service_desk_key.match(PROJECT_KEY_PATTERN) - project = Project.find_by_service_desk_project_key(match[:key]) - return unless valid_project_key?(project, match[:slug]) - - project + Project.with_service_desk_key(match[:key]).find do |project| + valid_project_key?(project, match[:slug]) + end end def valid_project_key?(project, slug) diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 8a432edbd78..e62e1172b65 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -11,12 +11,16 @@ module Gitlab end def cache_key(key) - "#{cache_type}:#{key}:set" + "#{cache_namespace}:#{key}:set" + end + + def new_cache_key(key) + super(key) end def clear_cache!(key) with do |redis| - keys = read(key).map { |value| "#{cache_type}:#{value}" } + keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) redis.pipelined do @@ -24,11 +28,5 @@ module Gitlab end end end - - private - - def cache_type - Gitlab::Redis::Cache::CACHE_NAMESPACE - end end end diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index f73ac628bce..a20e9845fe6 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -17,6 +17,11 @@ module Gitlab "#{type}:#{namespace}:set" end + # NOTE Remove as part of #331319 + def new_cache_key(type) + super("#{type}:#{namespace}") + end + def write(key, value) full_key = cache_key(key) diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 0f2b7b194c9..30cd63e80c0 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -14,15 +14,21 @@ module Gitlab "#{key}:set" end + # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 + def new_cache_key(key) + "#{cache_namespace}:#{key}:set" + end + # Returns the number of keys deleted by Redis def expire(*keys) return 0 if keys.empty? with do |redis| - keys = keys.map { |key| cache_key(key) } + keys_to_expire = keys.map { |key| cache_key(key) } + keys_to_expire += keys.map { |key| new_cache_key(key) } # NOTE Remove as part of #331319 Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.unlink(*keys) + redis.unlink(*keys_to_expire) end end end @@ -73,5 +79,9 @@ module Gitlab def with(&blk) Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end + + def cache_namespace + Gitlab::Redis::Cache::CACHE_NAMESPACE + end end end |