diff options
Diffstat (limited to 'lib')
81 files changed, 1476 insertions, 959 deletions
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb index 679e231b283..b724d3a38dc 100644 --- a/lib/api/admin/instance_clusters.rb +++ b/lib/api/admin/instance_clusters.rb @@ -76,6 +76,7 @@ module API optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :domain, type: String, desc: 'Cluster base domain' optional :management_project_id, type: Integer, desc: 'The ID of the management project' + optional :managed, type: Boolean, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/conan_instance_packages.rb b/lib/api/conan_instance_packages.rb index 08265201328..8c13b580092 100644 --- a/lib/api/conan_instance_packages.rb +++ b/lib/api/conan_instance_packages.rb @@ -4,7 +4,7 @@ module API class ConanInstancePackages < ::API::Base namespace 'packages/conan/v1' do - include ConanPackageEndpoints + include ::API::Concerns::Packages::ConanEndpoints end end end diff --git a/lib/api/conan_package_endpoints.rb b/lib/api/conan_package_endpoints.rb deleted file mode 100644 index 188a42f26f8..00000000000 --- a/lib/api/conan_package_endpoints.rb +++ /dev/null @@ -1,351 +0,0 @@ -# frozen_string_literal: true - -# Conan Package Manager Client API -# -# These API endpoints are not consumed directly by users, so there is no documentation for the -# individual endpoints. They are called by the Conan package manager client when users run commands -# like `conan install` or `conan upload`. The usage of the GitLab Conan repository is documented here: -# https://docs.gitlab.com/ee/user/packages/conan_repository/#installing-a-package -# -# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 -module API - module ConanPackageEndpoints - extend ActiveSupport::Concern - - PACKAGE_REQUIREMENTS = { - package_name: API::NO_SLASH_URL_PART_REGEX, - package_version: API::NO_SLASH_URL_PART_REGEX, - package_username: API::NO_SLASH_URL_PART_REGEX, - package_channel: API::NO_SLASH_URL_PART_REGEX - }.freeze - - FILE_NAME_REQUIREMENTS = { - file_name: API::NO_SLASH_URL_PART_REGEX - }.freeze - - PACKAGE_COMPONENT_REGEX = Gitlab::Regex.conan_recipe_component_regex - CONAN_REVISION_REGEX = Gitlab::Regex.conan_revision_regex - - CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze - - included do - feature_category :package_registry - - helpers ::API::Helpers::PackagesManagerClientsHelpers - helpers ::API::Helpers::Packages::Conan::ApiHelpers - helpers ::API::Helpers::RelatedResourcesHelpers - - before do - require_packages_enabled! - - # Personal access token will be extracted from Bearer or Basic authorization - # in the overridden find_personal_access_token or find_user_from_job_token helpers - authenticate! - end - - desc 'Ping the Conan API' do - detail 'This feature was introduced in GitLab 12.2' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'ping' do - header 'X-Conan-Server-Capabilities', [].join(',') - end - - desc 'Search for packages' do - detail 'This feature was introduced in GitLab 12.4' - end - - params do - requires :q, type: String, desc: 'Search query' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'conans/search' do - service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute - service.payload - end - - namespace 'users' do - format :txt - - desc 'Authenticate user against conan CLI' do - detail 'This feature was introduced in GitLab 12.2' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'authenticate' do - unauthorized! unless token - - token.to_jwt - end - - desc 'Check for valid user credentials per conan CLI' do - detail 'This feature was introduced in GitLab 12.4' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'check_credentials' do - authenticate! - :ok - end - end - - params do - requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' - requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' - end - namespace 'conans/:package_name/:package_version/:package_username/:package_channel', requirements: PACKAGE_REQUIREMENTS do - # Get the snapshot - # - # the snapshot is a hash of { filename: md5 hash } - # md5 hash is the has of that file. This hash is used to diff the files existing on the client - # to determine which client files need to be uploaded if no recipe exists the snapshot is empty - desc 'Package Snapshot' do - detail 'This feature was introduced in GitLab 12.5' - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'packages/:conan_package_reference' do - authorize!(:read_package, project) - - presenter = ::Packages::Conan::PackagePresenter.new( - package, - current_user, - project, - conan_package_reference: params[:conan_package_reference] - ) - - present presenter, with: ::API::Entities::ConanPackage::ConanPackageSnapshot - end - - desc 'Recipe Snapshot' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get do - authorize!(:read_package, project) - - presenter = ::Packages::Conan::PackagePresenter.new(package, current_user, project) - - present presenter, with: ::API::Entities::ConanPackage::ConanRecipeSnapshot - end - - # Get the manifest - # returns the download urls for the existing recipe in the registry - # - # the manifest is a hash of { filename: url } - # where the url is the download url for the file - desc 'Package Digest' do - detail 'This feature was introduced in GitLab 12.5' - end - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'packages/:conan_package_reference/digest' do - present_package_download_urls - end - - desc 'Recipe Digest' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'digest' do - present_recipe_download_urls - end - - # Get the download urls - # - # returns the download urls for the existing recipe or package in the registry - # - # the manifest is a hash of { filename: url } - # where the url is the download url for the file - desc 'Package Download Urls' do - detail 'This feature was introduced in GitLab 12.5' - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'packages/:conan_package_reference/download_urls' do - present_package_download_urls - end - - desc 'Recipe Download Urls' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get 'download_urls' do - present_recipe_download_urls - end - - # Get the upload urls - # - # request body contains { filename: filesize } where the filename is the - # name of the file the conan client is requesting to upload - # - # returns { filename: url } - # where the url is the upload url for the file that the conan client will use - desc 'Package Upload Urls' do - detail 'This feature was introduced in GitLab 12.4' - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan package ID' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - post 'packages/:conan_package_reference/upload_urls' do - authorize!(:read_package, project) - - status 200 - present package_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls - end - - desc 'Recipe Upload Urls' do - detail 'This feature was introduced in GitLab 12.4' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - post 'upload_urls' do - authorize!(:read_package, project) - - status 200 - present recipe_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls - end - - desc 'Delete Package' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - delete do - authorize!(:destroy_package, project) - - track_package_event('delete_package', :conan, category: 'API::ConanPackages') - - package.destroy - end - end - - params do - requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' - requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' - requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' - requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' - requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision' - end - namespace 'files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision', requirements: PACKAGE_REQUIREMENTS do - before do - authenticate_non_get! - end - - params do - requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES - end - namespace 'export/:file_name', requirements: FILE_NAME_REQUIREMENTS do - desc 'Download recipe files' do - detail 'This feature was introduced in GitLab 12.6' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get do - download_package_file(:recipe_file) - end - - desc 'Upload recipe package files' do - detail 'This feature was introduced in GitLab 12.6' - end - - params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put do - upload_package_file(:recipe_file) - end - - desc 'Workhorse authorize the conan recipe file' do - detail 'This feature was introduced in GitLab 12.6' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put 'authorize' do - authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) - end - end - - params do - requires :conan_package_reference, type: String, desc: 'Conan Package ID' - requires :package_revision, type: String, desc: 'Conan Package Revision' - requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES - end - namespace 'package/:conan_package_reference/:package_revision/:file_name', requirements: FILE_NAME_REQUIREMENTS do - desc 'Download package files' do - detail 'This feature was introduced in GitLab 12.5' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - get do - download_package_file(:package_file) - end - - desc 'Workhorse authorize the conan package file' do - detail 'This feature was introduced in GitLab 12.6' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put 'authorize' do - authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) - end - - desc 'Upload package files' do - detail 'This feature was introduced in GitLab 12.6' - end - - params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' - end - - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true - - put do - upload_package_file(:package_file) - end - end - end - end - end -end diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb index db8cd187811..636b5dca5ed 100644 --- a/lib/api/conan_project_packages.rb +++ b/lib/api/conan_project_packages.rb @@ -9,7 +9,7 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do namespace ':id/packages/conan/v1' do - include ConanPackageEndpoints + include ::API::Concerns::Packages::ConanEndpoints end end end diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb new file mode 100644 index 00000000000..6c8b3a1ba4a --- /dev/null +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -0,0 +1,355 @@ +# frozen_string_literal: true + +# Conan Package Manager Client API +# +# These API endpoints are not consumed directly by users, so there is no documentation for the +# individual endpoints. They are called by the Conan package manager client when users run commands +# like `conan install` or `conan upload`. The usage of the GitLab Conan repository is documented here: +# https://docs.gitlab.com/ee/user/packages/conan_repository/#installing-a-package +# +# Technical debt: https://gitlab.com/gitlab-org/gitlab/issues/35798 +module API + module Concerns + module Packages + module ConanEndpoints + extend ActiveSupport::Concern + + PACKAGE_REQUIREMENTS = { + package_name: API::NO_SLASH_URL_PART_REGEX, + package_version: API::NO_SLASH_URL_PART_REGEX, + package_username: API::NO_SLASH_URL_PART_REGEX, + package_channel: API::NO_SLASH_URL_PART_REGEX + }.freeze + + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + PACKAGE_COMPONENT_REGEX = Gitlab::Regex.conan_recipe_component_regex + CONAN_REVISION_REGEX = Gitlab::Regex.conan_revision_regex + + CONAN_FILES = (Gitlab::Regex::Packages::CONAN_RECIPE_FILES + Gitlab::Regex::Packages::CONAN_PACKAGE_FILES).freeze + + included do + feature_category :package_registry + + helpers ::API::Helpers::PackagesManagerClientsHelpers + helpers ::API::Helpers::Packages::Conan::ApiHelpers + helpers ::API::Helpers::RelatedResourcesHelpers + + before do + require_packages_enabled! + + # Personal access token will be extracted from Bearer or Basic authorization + # in the overridden find_personal_access_token or find_user_from_job_token helpers + authenticate! + end + + desc 'Ping the Conan API' do + detail 'This feature was introduced in GitLab 12.2' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'ping' do + header 'X-Conan-Server-Capabilities', [].join(',') + end + + desc 'Search for packages' do + detail 'This feature was introduced in GitLab 12.4' + end + + params do + requires :q, type: String, desc: 'Search query' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'conans/search' do + service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute + service.payload + end + + namespace 'users' do + format :txt + + desc 'Authenticate user against conan CLI' do + detail 'This feature was introduced in GitLab 12.2' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'authenticate' do + unauthorized! unless token + + token.to_jwt + end + + desc 'Check for valid user credentials per conan CLI' do + detail 'This feature was introduced in GitLab 12.4' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'check_credentials' do + authenticate! + :ok + end + end + + params do + requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' + requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' + requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' + requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' + end + namespace 'conans/:package_name/:package_version/:package_username/:package_channel', requirements: PACKAGE_REQUIREMENTS do + # Get the snapshot + # + # the snapshot is a hash of { filename: md5 hash } + # md5 hash is the has of that file. This hash is used to diff the files existing on the client + # to determine which client files need to be uploaded if no recipe exists the snapshot is empty + desc 'Package Snapshot' do + detail 'This feature was introduced in GitLab 12.5' + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'packages/:conan_package_reference' do + authorize!(:read_package, project) + + presenter = ::Packages::Conan::PackagePresenter.new( + package, + current_user, + project, + conan_package_reference: params[:conan_package_reference] + ) + + present presenter, with: ::API::Entities::ConanPackage::ConanPackageSnapshot + end + + desc 'Recipe Snapshot' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get do + authorize!(:read_package, project) + + presenter = ::Packages::Conan::PackagePresenter.new(package, current_user, project) + + present presenter, with: ::API::Entities::ConanPackage::ConanRecipeSnapshot + end + + # Get the manifest + # returns the download urls for the existing recipe in the registry + # + # the manifest is a hash of { filename: url } + # where the url is the download url for the file + desc 'Package Digest' do + detail 'This feature was introduced in GitLab 12.5' + end + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'packages/:conan_package_reference/digest' do + present_package_download_urls + end + + desc 'Recipe Digest' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'digest' do + present_recipe_download_urls + end + + # Get the download urls + # + # returns the download urls for the existing recipe or package in the registry + # + # the manifest is a hash of { filename: url } + # where the url is the download url for the file + desc 'Package Download Urls' do + detail 'This feature was introduced in GitLab 12.5' + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'packages/:conan_package_reference/download_urls' do + present_package_download_urls + end + + desc 'Recipe Download Urls' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get 'download_urls' do + present_recipe_download_urls + end + + # Get the upload urls + # + # request body contains { filename: filesize } where the filename is the + # name of the file the conan client is requesting to upload + # + # returns { filename: url } + # where the url is the upload url for the file that the conan client will use + desc 'Package Upload Urls' do + detail 'This feature was introduced in GitLab 12.4' + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan package ID' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + post 'packages/:conan_package_reference/upload_urls' do + authorize!(:read_package, project) + + status 200 + present package_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls + end + + desc 'Recipe Upload Urls' do + detail 'This feature was introduced in GitLab 12.4' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + post 'upload_urls' do + authorize!(:read_package, project) + + status 200 + present recipe_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls + end + + desc 'Delete Package' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + delete do + authorize!(:destroy_package, project) + + track_package_event('delete_package', :conan, category: 'API::ConanPackages') + + package.destroy + end + end + + params do + requires :package_name, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package name' + requires :package_version, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package version' + requires :package_username, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package username' + requires :package_channel, type: String, regexp: PACKAGE_COMPONENT_REGEX, desc: 'Package channel' + requires :recipe_revision, type: String, regexp: CONAN_REVISION_REGEX, desc: 'Conan Recipe Revision' + end + namespace 'files/:package_name/:package_version/:package_username/:package_channel/:recipe_revision', requirements: PACKAGE_REQUIREMENTS do + before do + authenticate_non_get! + end + + params do + requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES + end + namespace 'export/:file_name', requirements: FILE_NAME_REQUIREMENTS do + desc 'Download recipe files' do + detail 'This feature was introduced in GitLab 12.6' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get do + download_package_file(:recipe_file) + end + + desc 'Upload recipe package files' do + detail 'This feature was introduced in GitLab 12.6' + end + + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put do + upload_package_file(:recipe_file) + end + + desc 'Workhorse authorize the conan recipe file' do + detail 'This feature was introduced in GitLab 12.6' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put 'authorize' do + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) + end + end + + params do + requires :conan_package_reference, type: String, desc: 'Conan Package ID' + requires :package_revision, type: String, desc: 'Conan Package Revision' + requires :file_name, type: String, desc: 'Package file name', values: CONAN_FILES + end + namespace 'package/:conan_package_reference/:package_revision/:file_name', requirements: FILE_NAME_REQUIREMENTS do + desc 'Download package files' do + detail 'This feature was introduced in GitLab 12.5' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + get do + download_package_file(:package_file) + end + + desc 'Workhorse authorize the conan package file' do + detail 'This feature was introduced in GitLab 12.6' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put 'authorize' do + authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size) + end + + desc 'Upload package files' do + detail 'This feature was introduced in GitLab 12.6' + end + + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + + put do + upload_package_file(:package_file) + end + end + end + end + end + end + end +end diff --git a/lib/api/entities/cluster.rb b/lib/api/entities/cluster.rb index 67459092a33..b7e76e763f7 100644 --- a/lib/api/entities/cluster.rb +++ b/lib/api/entities/cluster.rb @@ -3,7 +3,7 @@ module API module Entities class Cluster < Grape::Entity - expose :id, :name, :created_at, :domain + expose :id, :name, :created_at, :domain, :enabled, :managed expose :provider_type, :platform_type, :environment_scope, :cluster_type, :namespace_per_environment expose :user, using: Entities::UserBasic expose :platform_kubernetes, using: Entities::Platform::Kubernetes diff --git a/lib/api/entities/feature.rb b/lib/api/entities/feature.rb index 618a7be9c7b..d1151849cd7 100644 --- a/lib/api/entities/feature.rb +++ b/lib/api/entities/feature.rb @@ -17,6 +17,16 @@ module API { key: gate.key, value: value } end.compact end + + class Definition < Grape::Entity + ::Feature::Definition::PARAMS.each do |param| + expose param + end + end + + expose :definition, using: Definition do |feature| + ::Feature::Definition.definitions[feature.name.to_sym] + end end end end diff --git a/lib/api/entities/related_issue.rb b/lib/api/entities/related_issue.rb index 491c606bd49..60793fed5e0 100644 --- a/lib/api/entities/related_issue.rb +++ b/lib/api/entities/related_issue.rb @@ -5,6 +5,8 @@ module API class RelatedIssue < ::API::Entities::Issue expose :issue_link_id expose :issue_link_type, as: :link_type + expose :issue_link_created_at, as: :link_created_at + expose :issue_link_updated_at, as: :link_updated_at end end end diff --git a/lib/api/features.rb b/lib/api/features.rb index 2c2e3e3d0c9..2ab60e2617d 100644 --- a/lib/api/features.rb +++ b/lib/api/features.rb @@ -46,6 +46,15 @@ module API present features, with: Entities::Feature, current_user: current_user end + desc 'Get a list of all feature definitions' do + success Entities::Feature::Definition + end + get :definitions do + definitions = ::Feature::Definition.definitions.values.map(&:to_h) + + present definitions, with: Entities::Feature::Definition, current_user: current_user + end + desc 'Set the gate value for the given feature' do success Entities::Feature end @@ -56,6 +65,7 @@ module API optional :user, type: String, desc: 'A GitLab username' optional :group, type: String, desc: "A GitLab group's path, such as 'gitlab-org'" optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce' + optional :force, type: Boolean, desc: 'Skip feature flag validation checks, ie. YAML definition' mutually_exclusive :key, :feature_group mutually_exclusive :key, :user @@ -63,7 +73,7 @@ module API mutually_exclusive :key, :project end post ':name' do - validate_feature_flag_name!(params[:name]) + validate_feature_flag_name!(params[:name]) unless params[:force] feature = Feature.get(params[:name]) # rubocop:disable Gitlab/AvoidFeatureGet targets = gate_targets(params) diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb index a435b050042..81944a653c8 100644 --- a/lib/api/group_clusters.rb +++ b/lib/api/group_clusters.rb @@ -75,10 +75,12 @@ module API params do requires :cluster_id, type: Integer, desc: 'The cluster ID' optional :name, type: String, desc: 'Cluster name' + optional :enabled, type: Boolean, desc: 'Determines if cluster is active or not' optional :domain, type: String, desc: 'Cluster base domain' optional :environment_scope, type: String, desc: 'The associated environment to the cluster' optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' + optional :managed, type: Boolean, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 147d8407142..3fa743d70fb 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -536,6 +536,15 @@ module API ) end + def increment_counter(event_name) + feature_name = "usage_data_#{event_name}" + return unless Feature.enabled?(feature_name) + + Gitlab::UsageDataCounters.count(event_name) + rescue => error + Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") + end + # @param event_name [String] the event name # @param values [Array|String] the values counted def increment_unique_values(event_name, values) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 6a6ee7a4e1c..73e2163248d 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -435,3 +435,5 @@ module API end end end + +API::Issues.prepend_if_ee('EE::API::Issues') diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb index cfb0c5fd705..6785b28ddef 100644 --- a/lib/api/project_clusters.rb +++ b/lib/api/project_clusters.rb @@ -83,6 +83,8 @@ module API optional :environment_scope, type: String, desc: 'The associated environment to the cluster' optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace' optional :management_project_id, type: Integer, desc: 'The ID of the management project' + optional :enabled, type: Boolean, desc: 'Determines if cluster is active or not' + optional :managed, type: Boolean, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster' optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do optional :api_url, type: String, desc: 'URL to access the Kubernetes API' optional :token, type: String, desc: 'Token to authenticate against Kubernetes' diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb index 1814e1a6782..6818c04fd2e 100644 --- a/lib/api/statistics.rb +++ b/lib/api/statistics.rb @@ -4,7 +4,7 @@ module API class Statistics < ::API::Base before { authenticated_as_admin! } - feature_category :instance_statistics + feature_category :devops_reports COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue, MergeRequest, Note, Snippet, Key, Milestone].freeze diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 7b038ec74bb..cad2f52e951 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -20,6 +20,18 @@ module API requires :event, type: String, desc: 'The event name that should be tracked' end + post 'increment_counter' do + event_name = params[:event] + + increment_counter(event_name) + + status :ok + end + + params do + requires :event, type: String, desc: 'The event name that should be tracked' + end + post 'increment_unique_users' do event_name = params[:event] diff --git a/lib/api/users.rb b/lib/api/users.rb index 501ed629c7e..8b9b82877f7 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -534,6 +534,24 @@ module API user.activate end + + desc 'Approve a pending user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/approve', feature_category: :authentication_and_authorization do + user = User.find_by(id: params[:id]) + not_found!('User') unless can?(current_user, :read_user, user) + + result = ::Users::ApproveService.new(current_user).execute(user) + + if result[:success] + result + else + render_api_error!(result[:message], result[:http_status]) + end + end + # rubocop: enable CodeReuse/ActiveRecord desc 'Deactivate an active user. Available only for admins.' params do diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb index 7d58032cfcc..c0cef61d2b2 100644 --- a/lib/bulk_imports/common/extractors/graphql_extractor.rb +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -6,15 +6,16 @@ module BulkImports class GraphqlExtractor def initialize(query) @query = query[:query] - @query_string = @query.to_s - @variables = @query.variables end def extract(context) - @context = context + client = graphql_client(context) Enumerator.new do |yielder| - result = graphql_client.execute(parsed_query, query_variables(context.entity)) + result = client.execute( + client.parse(query.to_s), + query.variables(context.entity) + ) yielder << result.original_hash.deep_dup end @@ -22,23 +23,17 @@ module BulkImports private - def graphql_client + attr_reader :query + + def graphql_client(context) @graphql_client ||= BulkImports::Clients::Graphql.new( - url: @context.configuration.url, - token: @context.configuration.access_token + url: context.configuration.url, + token: context.configuration.access_token ) end def parsed_query - @parsed_query ||= graphql_client.parse(@query.to_s) - end - - def query_variables(entity) - return unless @variables - - @variables.transform_values do |entity_attribute| - entity.public_send(entity_attribute) # rubocop:disable GitlabSecurity/PublicSend - end + @parsed_query ||= graphql_client.parse(query.to_s) end end end diff --git a/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb b/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb deleted file mode 100644 index dce0fac6999..00000000000 --- a/lib/bulk_imports/common/transformers/graphql_cleaner_transformer.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -# Cleanup GraphQL original response hash from unnecessary nesting -# 1. Remove ['data']['group'] or ['data']['project'] hash nesting -# 2. Remove ['edges'] & ['nodes'] array wrappings -# 3. Remove ['node'] hash wrapping -# -# @example -# data = {"data"=>{"group"=> { -# "name"=>"test", -# "fullName"=>"test", -# "description"=>"test", -# "labels"=>{"edges"=>[{"node"=>{"title"=>"label1"}}, {"node"=>{"title"=>"label2"}}, {"node"=>{"title"=>"label3"}}]}}}} -# -# BulkImports::Common::Transformers::GraphqlCleanerTransformer.new.transform(nil, data) -# -# {"name"=>"test", "fullName"=>"test", "description"=>"test", "labels"=>[{"title"=>"label1"}, {"title"=>"label2"}, {"title"=>"label3"}]} -module BulkImports - module Common - module Transformers - class GraphqlCleanerTransformer - EDGES = 'edges' - NODE = 'node' - - def initialize(options = {}) - @options = options - end - - def transform(_, data) - return data unless data.is_a?(Hash) - - data = data.dig('data', 'group') || data.dig('data', 'project') || data - - clean_edges_and_nodes(data) - end - - def clean_edges_and_nodes(data) - case data - when Array - data.map(&method(:clean_edges_and_nodes)) - when Hash - if data.key?(NODE) - clean_edges_and_nodes(data[NODE]) - else - data.transform_values { |value| clean_edges_and_nodes(value.try(:fetch, EDGES, value) || value) } - end - else - data - end - end - end - end - end -end diff --git a/lib/bulk_imports/common/transformers/hash_key_digger.rb b/lib/bulk_imports/common/transformers/hash_key_digger.rb new file mode 100644 index 00000000000..b4897b5b2bf --- /dev/null +++ b/lib/bulk_imports/common/transformers/hash_key_digger.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Transformers + class HashKeyDigger + def initialize(options = {}) + @key_path = options[:key_path] + end + + def transform(_, data) + raise ArgumentError, "Given data must be a Hash" unless data.is_a?(Hash) + + data.dig(*Array.wrap(key_path)) + end + + private + + attr_reader :key_path + end + end + end +end diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb index c50b99aae4e..2bc0f60baa2 100644 --- a/lib/bulk_imports/groups/graphql/get_group_query.rb +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -29,8 +29,8 @@ module BulkImports GRAPHQL end - def variables - { full_path: :source_full_path } + def variables(entity) + { full_path: entity.source_full_path } end end end diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb index 2b7d0ef7658..75af126ebf4 100644 --- a/lib/bulk_imports/groups/pipelines/group_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb @@ -8,7 +8,7 @@ module BulkImports extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetGroupQuery - transformer Common::Transformers::GraphqlCleanerTransformer + transformer Common::Transformers::HashKeyDigger, key_path: %w[data group] transformer Common::Transformers::UnderscorifyKeysTransformer transformer Groups::Transformers::GroupAttributesTransformer diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb index c7253590c87..82cb1ca03a2 100644 --- a/lib/bulk_imports/importers/group_importer.rb +++ b/lib/bulk_imports/importers/group_importer.rb @@ -19,6 +19,7 @@ module BulkImports ) BulkImports::Groups::Pipelines::GroupPipeline.new.run(context) + 'BulkImports::EE::Groups::Pipelines::EpicsPipeline'.constantize.new.run(context) if Gitlab.ee? BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline.new.run(context) entity.finish! diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 70e6030ea2c..4b4690a43e9 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -3,10 +3,77 @@ module BulkImports module Pipeline extend ActiveSupport::Concern + include Gitlab::ClassAttributes included do - include Attributes include Runner + + private + + def extractors + @extractors ||= self.class.extractors.map(&method(:instantiate)) + end + + def transformers + @transformers ||= self.class.transformers.map(&method(:instantiate)) + end + + def loaders + @loaders ||= self.class.loaders.map(&method(:instantiate)) + end + + def after_run + @after_run ||= self.class.after_run_callback + end + + def pipeline_name + @pipeline ||= self.class.name + end + + def instantiate(class_config) + class_config[:klass].new(class_config[:options]) + end + end + + class_methods do + def extractor(klass, options = nil) + add_attribute(:extractors, klass, options) + end + + def transformer(klass, options = nil) + add_attribute(:transformers, klass, options) + end + + def loader(klass, options = nil) + add_attribute(:loaders, klass, options) + end + + def after_run(&block) + class_attributes[:after_run] = block + end + + def extractors + class_attributes[:extractors] + end + + def transformers + class_attributes[:transformers] + end + + def loaders + class_attributes[:loaders] + end + + def after_run_callback + class_attributes[:after_run] + end + + private + + def add_attribute(sym, klass, options) + class_attributes[sym] ||= [] + class_attributes[sym] << { klass: klass, options: options } + end end end end diff --git a/lib/bulk_imports/pipeline/attributes.rb b/lib/bulk_imports/pipeline/attributes.rb deleted file mode 100644 index ebfbaf6f6ba..00000000000 --- a/lib/bulk_imports/pipeline/attributes.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Pipeline - module Attributes - extend ActiveSupport::Concern - include Gitlab::ClassAttributes - - class_methods do - def extractor(klass, options = nil) - add_attribute(:extractors, klass, options) - end - - def transformer(klass, options = nil) - add_attribute(:transformers, klass, options) - end - - def loader(klass, options = nil) - add_attribute(:loaders, klass, options) - end - - def add_attribute(sym, klass, options) - class_attributes[sym] ||= [] - class_attributes[sym] << { klass: klass, options: options } - end - - def extractors - class_attributes[:extractors] - end - - def transformers - class_attributes[:transformers] - end - - def loaders - class_attributes[:loaders] - end - end - end - end -end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 04038e50399..898540d42ed 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -5,30 +5,6 @@ module BulkImports module Runner extend ActiveSupport::Concern - included do - private - - def extractors - @extractors ||= self.class.extractors.map(&method(:instantiate)) - end - - def transformers - @transformers ||= self.class.transformers.map(&method(:instantiate)) - end - - def loaders - @loaders ||= self.class.loaders.map(&method(:instantiate)) - end - - def pipeline_name - @pipeline ||= self.class.name - end - - def instantiate(class_config) - class_config[:klass].new(class_config[:options]) - end - end - def run(context) info(context, message: "Pipeline started", pipeline: pipeline_name) @@ -47,6 +23,8 @@ module BulkImports end end end + + after_run.call(context) if after_run.present? end private # rubocop:disable Lint/UselessAccessModifier diff --git a/lib/feature.rb b/lib/feature.rb index 1f8c530bee5..c9871881dc9 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -136,8 +136,6 @@ class Feature end def register_definitions - return unless check_feature_flags_definition? - Feature::Definition.reload! end diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index 0ba1bdc4799..24f4cb3dbf0 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -13,6 +13,12 @@ class Feature end end + TYPES.each do |type, _| + define_method("#{type}?") do + attributes[:type].to_sym == type + end + end + def initialize(path, opts = {}) @path = path @attributes = {} @@ -94,6 +100,10 @@ class Feature @definitions = load_all! end + def has_definition?(key) + definitions.has_key?(key.to_sym) + end + def valid_usage!(key, type:, default_enabled:) if definition = definitions[key.to_sym] definition.valid_usage!(type_in_code: type, default_enabled_in_code: default_enabled) @@ -119,10 +129,6 @@ class Feature private def load_all! - # We currently do not load feature flag definitions - # in production environments - return [] unless Gitlab.dev_or_test_env? - paths.each_with_object({}) do |glob_path, definitions| load_all_from_path!(definitions, glob_path) end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index e92bbe4f529..fbba86d1253 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -34,7 +34,8 @@ module Gitlab group_testing_hook: { threshold: 5, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, - update_environment_canary_ingress: { threshold: 1, interval: 1.minute } + update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, + auto_rollback_deployment: { threshold: 1, interval: 3.minutes } }.freeze end diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb index a0c89cc4664..2e81b1615d8 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb @@ -5,7 +5,7 @@ module Gitlab # This class creates/updates those project historical vulnerability statistics # that haven't been created nor initialized. It should only be executed in EE. class PopulateVulnerabilityHistoricalStatistics - def perform(project_ids) + def perform(project_ids, retention_period = 90) end end end diff --git a/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb b/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb new file mode 100644 index 00000000000..d97765cd398 --- /dev/null +++ b/lib/gitlab/background_migration/update_existing_users_that_require_two_factor_auth.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class UpdateExistingUsersThatRequireTwoFactorAuth # rubocop:disable Metrics/ClassLength + def perform(start_id, stop_id) + ActiveRecord::Base.connection.execute <<~SQL + UPDATE + users + SET + require_two_factor_authentication_from_group = FALSE + WHERE + users.id BETWEEN #{start_id} + AND #{stop_id} + AND users.require_two_factor_authentication_from_group = TRUE + AND users.id NOT IN ( SELECT DISTINCT + users_groups_query.user_id + FROM ( + SELECT + users.id AS user_id, + members.source_id AS group_ids + FROM + users + LEFT JOIN members ON members.source_type = 'Namespace' + AND members.requested_at IS NULL + AND members.user_id = users.id + AND members.type = 'GroupMember' + WHERE + users.require_two_factor_authentication_from_group = TRUE + AND users.id BETWEEN #{start_id} + AND #{stop_id}) AS users_groups_query + INNER JOIN LATERAL ( WITH RECURSIVE "base_and_ancestors" AS ( + ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."id" = users_groups_query.group_ids) + UNION ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces", + "base_and_ancestors" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."id" = "base_and_ancestors"."parent_id")), + "base_and_descendants" AS ( + ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."id" = users_groups_query.group_ids) + UNION ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "namespaces", + "base_and_descendants" + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."parent_id" = "base_and_descendants"."id")) + SELECT + "namespaces".* + FROM (( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "base_and_ancestors" AS "namespaces" + WHERE + "namespaces"."type" = 'Group') + UNION ( + SELECT + "namespaces"."type", + "namespaces"."id", + "namespaces"."parent_id", + "namespaces"."require_two_factor_authentication" + FROM + "base_and_descendants" AS "namespaces" + WHERE + "namespaces"."type" = 'Group')) namespaces + WHERE + "namespaces"."type" = 'Group' + AND "namespaces"."require_two_factor_authentication" = TRUE) AS hierarchy_tree ON TRUE); + SQL + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index cbecce57163..9c2f6eea1dd 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -11,7 +11,7 @@ module Gitlab def satisfied_by?(pipeline, context) return true if pipeline.modified_paths.nil? - expanded_globs = expand_globs(pipeline, context) + expanded_globs = expand_globs(context) pipeline.modified_paths.any? do |path| expanded_globs.any? do |glob| File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB) @@ -19,8 +19,7 @@ module Gitlab end end - def expand_globs(pipeline, context) - return @globs unless ::Feature.enabled?(:ci_variable_expansion_in_rules_changes, pipeline.project, default_enabled: true) + def expand_globs(context) return @globs unless context @globs.map do |glob| diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 0e44475607b..57f73c265b2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -10,7 +10,8 @@ module Gitlab junit: ::Gitlab::Ci::Parsers::Test::Junit, cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura, terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan, - accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y + accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y, + codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate } end diff --git a/lib/gitlab/ci/parsers/codequality/code_climate.rb b/lib/gitlab/ci/parsers/codequality/code_climate.rb new file mode 100644 index 00000000000..628d50b84cb --- /dev/null +++ b/lib/gitlab/ci/parsers/codequality/code_climate.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Codequality + class CodeClimate + def parse!(json_data, codequality_report) + root = Gitlab::Json.parse(json_data) + + parse_all(root, codequality_report) + rescue JSON::ParserError => e + codequality_report.set_error_message("JSON parsing failed: #{e}") + end + + private + + def parse_all(root, codequality_report) + return unless root.present? + + root.each do |degradation| + break unless codequality_report.add_degradation(degradation) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index a864c843dd8..2ca51930c19 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -35,7 +35,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def pipelines - if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: false) + if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: true) project.all_pipelines.ci_and_parent_sources else project.ci_pipelines diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb new file mode 100644 index 00000000000..060a1e2399b --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityReports + attr_reader :degradations, :error_message + + CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s + + def initialize + @degradations = {}.with_indifferent_access + @error_message = nil + end + + def add_degradation(degradation) + valid_degradation?(degradation) && @degradations[degradation.dig('fingerprint')] = degradation + end + + def set_error_message(error) + @error_message = error + end + + def degradations_count + @degradations.size + end + + def all_degradations + @degradations.values + end + + private + + def valid_degradation?(degradation) + JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation) + rescue JSON::Schema::ValidationError => e + set_error_message("Invalid degradation format: #{e.message}") + false + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 385959389de..e5b40e5f49a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 65abee1f5eb..3faf07546de 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: ayufan/openshift-cli +image: openshift/origin-cli stages: - build # dummy stage to follow the template guidelines diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 3cbde9d30c8..5ea2363a0c5 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: container_scanning: stage: test - image: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + image: "$CS_ANALYZER_IMAGE" variables: # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes @@ -18,6 +18,7 @@ container_scanning: # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION allow_failure: true services: - name: $CLAIR_DB_IMAGE diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 3789f0edc1c..b534dad9593 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -28,11 +28,8 @@ dependency_scanning: .ds-analyzer: extends: dependency_scanning allow_failure: true - rules: - - if: $DEPENDENCY_SCANNING_DISABLED - when: never - - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: - /analyzer run diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index a51cb61da6d..671e2346fcb 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -30,10 +30,8 @@ sast: .sast-analyzer: extends: sast allow_failure: true - rules: - - if: $SAST_DISABLED - when: never - - if: $CI_COMMIT_BRANCH + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 script: - /analyzer run diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 6ebff102ccb..8ca1d2e08ba 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -14,6 +14,9 @@ variables: stage: test image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" services: [] + allow_failure: true + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 artifacts: reports: secret_detection: gl-secret-detection-report.json diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index e455bfac9de..910e711f046 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -56,5 +56,6 @@ cache: .destroy: &destroy stage: cleanup script: + - cd ${TF_ROOT} - gitlab-terraform destroy when: manual diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb index 607ca1200a0..92af6849b2f 100644 --- a/lib/gitlab/danger/changelog.rb +++ b/lib/gitlab/danger/changelog.rb @@ -39,6 +39,7 @@ module Gitlab def required? git.added_files.any? { |path| path =~ %r{\Adb/(migrate|post_migrate)/} } end + alias_method :db_changes?, :required? def optional? categories_need_changelog? && without_no_changelog_label? diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 6f79e965cd5..5a506da0d05 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -49,6 +49,8 @@ module Gitlab MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep ALLOWED_MODES = [:itself, :distinct].freeze + FALLBACK_FINISH = 0 + OFFSET_BY_ONE = 1 # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 DEFAULT_DISTINCT_BATCH_SIZE = 10_000 @@ -65,7 +67,7 @@ module Gitlab (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || - start > finish + start >= finish end def count(batch_size: nil, mode: :itself, start: nil, finish: nil) @@ -85,11 +87,13 @@ module Gitlab results = nil batch_start = start - while batch_start <= finish - batch_relation = build_relation_batch(batch_start, batch_start + batch_size, mode) + while batch_start < finish + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end, mode) + begin results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend - batch_start += batch_size + batch_start = batch_end rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE @@ -99,6 +103,7 @@ module Gitlab return FALLBACK end end + sleep(SLEEP_TIME_IN_SECONDS) end @@ -138,7 +143,7 @@ module Gitlab end def actual_finish(finish) - finish || @relation.unscope(:group, :having).maximum(@column) || 0 + (finish || @relation.unscope(:group, :having).maximum(@column) || FALLBACK_FINISH) + OFFSET_BY_ONE end def check_mode!(mode) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index a6cc03aa9eb..36073844765 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -55,7 +55,8 @@ module Gitlab bulk_migrate_async(jobs) unless jobs.empty? end - # Queues background migration jobs for an entire table, batched by ID range. + # Queues background migration jobs for an entire table in batches. + # The default batching column used is the standard primary key `id`. # Each job is scheduled with a `delay_interval` in between. # If you use a small interval, then some jobs may run at the same time. # @@ -68,6 +69,7 @@ module Gitlab # is scheduled to be run. These records can be used to trace execution of the background job, but there is no # builtin support to manage that automatically at this time. You should only set this flag if you are aware of # how it works, and intend to manually cleanup the database records in your background job. + # primary_column_name - The name of the primary key column if the primary key is not `id` # # *Returns the final migration delay* # @@ -87,8 +89,9 @@ module Gitlab # # do something # end # end - def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false) - raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) + raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) + raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer # To not overload the worker too much we enforce a minimum interval both # when scheduling and performing jobs. @@ -99,7 +102,7 @@ module Gitlab final_delay = 0 model_class.each_batch(of: batch_size) do |relation, index| - start_id, end_id = relation.pluck(Arel.sql('MIN(id), MAX(id)')).first + start_id, end_id = relation.pluck(Arel.sql("MIN(#{primary_column_name}), MAX(#{primary_column_name})")).first # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for # the same time, which is not helpful in most cases where we wish to diff --git a/lib/gitlab/database/postgres_hll_batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll_batch_distinct_counter.rb new file mode 100644 index 00000000000..4b2afcb128f --- /dev/null +++ b/lib/gitlab/database/postgres_hll_batch_distinct_counter.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # For large tables, PostgreSQL can take a long time to count rows due to MVCC. + # Implements a distinct batch counter based on HyperLogLog algorithm + # Needs indexes on the column below to calculate max, min and range queries + # For larger tables just set higher batch_size with index optimization + # + # In order to not use a possible complex time consuming query when calculating min and max values, + # the start and finish can be sent specifically, start and finish should contain max and min values for PRIMARY KEY of + # relation (most cases `id` column) rather than counted attribute eg: + # estimate_distinct_count(start: ::Project.with_active_services.minimum(:id), finish: ::Project.with_active_services.maximum(:id)) + # + # Grouped relations are NOT supported yet. + # + # @example Usage + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).estimate_distinct_count + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.with_active_services.service_desk_enabled.where(time_period)) + # .estimate_distinct_count( + # batch_size: 1_000, + # start: ::Project.with_active_services.service_desk_enabled.where(time_period).minimum(:id), + # finish: ::Project.with_active_services.service_desk_enabled.where(time_period).maximum(:id) + # ) + # + # @note HyperLogLog is an PROBABILISTIC algorithm that ESTIMATES distinct count of given attribute value for supplied relation + # Like all probabilistic algorithm is has ERROR RATE margin, that can affect values, + # for given implementation no higher value was reported (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45673#accuracy-estimation) than 5.3% + # for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used. + class PostgresHllBatchDistinctCounter + FALLBACK = -1 + MIN_REQUIRED_BATCH_SIZE = 1_250 + MAX_ALLOWED_LOOPS = 10_000 + SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep + + # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 + DEFAULT_BATCH_SIZE = 100_000 + + BIT_31_MASK = "B'0#{'1' * 31}'" + BIT_9_MASK = "B'#{'0' * 23}#{'1' * 9}'" + # @example source_query + # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits + # FROM %{relation} + # WHERE %{pkey} >= %{batch_start} + # AND %{pkey} < %{batch_end} + # AND %{column} IS NOT NULL + BUCKETED_DATA_SQL = <<~SQL + WITH hashed_attributes AS (%{source_query}) + SELECT (attr_hash_32_bits & #{BIT_9_MASK})::int AS bucket_num, + (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash + FROM hashed_attributes + GROUP BY 1 ORDER BY 1 + SQL + + TOTAL_BUCKETS_NUMBER = 512 + + def initialize(relation, column = nil) + @relation = relation + @column = column || relation.primary_key + end + + def unwanted_configuration?(finish, batch_size, start) + batch_size <= MIN_REQUIRED_BATCH_SIZE || + (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || + start > finish + end + + def estimate_distinct_count(batch_size: nil, start: nil, finish: nil) + raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? + + batch_size ||= DEFAULT_BATCH_SIZE + + start = actual_start(start) + finish = actual_finish(finish) + + raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 + return FALLBACK if unwanted_configuration?(finish, batch_size, start) + + batch_start = start + hll_blob = {} + + while batch_start <= finish + begin + hll_blob.merge!(hll_blob_for_batch(batch_start, batch_start + batch_size)) {|_key, old, new| new > old ? new : old } + batch_start += batch_size + end + sleep(SLEEP_TIME_IN_SECONDS) + end + + estimate_cardinality(hll_blob) + end + + private + + # arbitrary values that are present in #estimate_cardinality + # are sourced from https://www.sisense.com/blog/hyperloglog-in-pure-sql/ + # article, they are not representing any entity and serves as tune value + # for the whole equation + def estimate_cardinality(hll_blob) + num_zero_buckets = TOTAL_BUCKETS_NUMBER - hll_blob.size + + num_uniques = ( + ((TOTAL_BUCKETS_NUMBER**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER))) / + (num_zero_buckets + hll_blob.values.sum { |bucket_hash, _| 2**(-1 * bucket_hash)} ) + ).to_i + + if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS_NUMBER + ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER)) * (TOTAL_BUCKETS_NUMBER * + Math.log2(TOTAL_BUCKETS_NUMBER.to_f / num_zero_buckets))) + else + num_uniques + end + end + + def hll_blob_for_batch(start, finish) + @relation + .connection + .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) }) + .map(&:values) + .to_h + end + + # Generate the source query SQL snippet for the provided id range + # + # @example SQL query template + # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits + # FROM %{relation} + # WHERE %{pkey} >= %{batch_start} AND %{pkey} < %{batch_end} + # AND %{column} IS NOT NULL + # + # @param start initial id range + # @param finish final id range + # @return [String] SQL query fragment + def source_query(start, finish) + col_as_arel = @column.is_a?(Arel::Attributes::Attribute) ? @column : Arel.sql(@column.to_s) + col_as_text = Arel::Nodes::NamedFunction.new('CAST', [col_as_arel.as('text')]) + md5_of_col = Arel::Nodes::NamedFunction.new('md5', [col_as_text]) + md5_as_hex = Arel::Nodes::Concat.new(Arel.sql("'X'"), md5_of_col) + bits = Arel::Nodes::NamedFunction.new('CAST', [md5_as_hex.as('bit(32)')]) + + @relation + .where(@relation.primary_key => (start...finish)) + .where(col_as_arel.not_eq(nil)) + .select(bits.as('attr_hash_32_bits')).to_sql + end + + def actual_start(start) + start || @relation.unscope(:group, :having).minimum(@relation.primary_key) || 0 + end + + def actual_finish(finish) + finish || @relation.unscope(:group, :having).maximum(@relation.primary_key) || 0 + end + end + end +end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 6e39776bbd4..d511972757f 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -4,7 +4,6 @@ # # Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant. # Experiment options: -# - environment (optional, defaults to enabled for development and GitLab.com) # - tracking_category (optional, used to set the category when tracking an experiment event) # - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility) # @@ -55,10 +54,6 @@ module Gitlab tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA', use_backwards_compatible_subject_index: true }, - new_create_project_ui: { - tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi', - use_backwards_compatible_subject_index: true - }, contact_sales_btn_in_app: { tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', use_backwards_compatible_subject_index: true @@ -87,14 +82,13 @@ module Gitlab class << self def experiment(key) - Experiment.new(EXPERIMENTS[key].merge(key: key)) + Gitlab::Experimentation::Experiment.new(key, **EXPERIMENTS[key]) end def enabled?(experiment_key) return false unless EXPERIMENTS.key?(experiment_key) - experiment = experiment(experiment_key) - experiment.enabled_for_environment? && experiment.enabled? + experiment(experiment_key).enabled? end def enabled_for_attribute?(experiment_key, attribute) @@ -106,36 +100,5 @@ module Gitlab enabled?(experiment_key) && experiment(experiment_key).enabled_for_index?(value) end end - - Experiment = Struct.new( - :key, - :environment, - :tracking_category, - :use_backwards_compatible_subject_index, - keyword_init: true - ) do - def enabled? - experiment_percentage > 0 - end - - def enabled_for_environment? - return ::Gitlab.dev_env_or_com? if environment.nil? - - environment - end - - def enabled_for_index?(index) - return false if index.blank? - - index <= experiment_percentage - end - - private - - # When a feature does not exist, the `percentage_of_time_value` method will return 0 - def experiment_percentage - @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet - end - end end end diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb new file mode 100644 index 00000000000..bacbaa6949c --- /dev/null +++ b/lib/gitlab/experimentation/experiment.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Experimentation + class Experiment + attr_reader :tracking_category, :use_backwards_compatible_subject_index + + def initialize(key, **params) + @tracking_category = params[:tracking_category] + @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] + + @experiment_percentage = Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet + end + + def enabled? + ::Gitlab.dev_env_or_com? && experiment_percentage > 0 + end + + def enabled_for_index?(index) + return false if index.blank? + + index <= experiment_percentage + end + + private + + attr_reader :experiment_percentage + end + end +end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 96f3487fd6f..a2215366bdc 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -17,6 +17,7 @@ module Gitlab CommitError = Class.new(BaseError) OSError = Class.new(BaseError) UnknownRef = Class.new(BaseError) + CommandTimedOut = Class.new(CommandError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb index 9963bcfbf1c..2009683d32c 100644 --- a/lib/gitlab/git/wraps_gitaly_errors.rb +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -9,6 +9,8 @@ module Gitlab raise Gitlab::Git::Repository::NoRepository.new(e) rescue GRPC::InvalidArgument => e raise ArgumentError.new(e) + rescue GRPC::DeadlineExceeded => e + raise Gitlab::Git::CommandTimedOut.new(e) rescue GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2d41ad76618..7261589473a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -83,3 +83,5 @@ module Gitlab end end end + +Gitlab::GonHelper.prepend_if_ee('EE::Gitlab::GonHelper') diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index c70127553fd..6ee446011d4 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -62,8 +62,8 @@ module Gitlab end end - def raise_resource_not_available_error! - raise Gitlab::Graphql::Errors::ResourceNotAvailable, RESOURCE_ACCESS_ERROR + def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR) + raise Gitlab::Graphql::Errors::ResourceNotAvailable, msg end end end diff --git a/lib/gitlab/graphql/connection_collection_methods.rb b/lib/gitlab/graphql/connection_collection_methods.rb new file mode 100644 index 00000000000..0e2c4a98bb6 --- /dev/null +++ b/lib/gitlab/graphql/connection_collection_methods.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module ConnectionCollectionMethods + extend ActiveSupport::Concern + + included do + delegate :to_a, :size, :include?, :empty?, to: :nodes + end + end + end +end diff --git a/lib/gitlab/graphql/connection_redaction.rb b/lib/gitlab/graphql/connection_redaction.rb new file mode 100644 index 00000000000..5e037bb9f63 --- /dev/null +++ b/lib/gitlab/graphql/connection_redaction.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module ConnectionRedaction + class RedactionState + attr_reader :redactor + attr_reader :redacted_nodes + + def redactor=(redactor) + @redactor = redactor + @redacted_nodes = nil + end + + def redacted(&block) + @redacted_nodes ||= redactor.present? ? redactor.redact(yield) : yield + end + end + + delegate :redactor=, to: :redaction_state + + def nodes + redaction_state.redacted { super.to_a } + end + + private + + def redaction_state + @redaction_state ||= RedactionState.new + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/array_connection.rb b/lib/gitlab/graphql/pagination/array_connection.rb new file mode 100644 index 00000000000..efc912eaeca --- /dev/null +++ b/lib/gitlab/graphql/pagination/array_connection.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# We use the Keyset / Stable cursor connection by default for ActiveRecord::Relation. +# However, there are times when that may not be powerful enough (yet), and we +# want to use standard offset pagination. +module Gitlab + module Graphql + module Pagination + class ArrayConnection < ::GraphQL::Pagination::ArrayConnection + prepend ::Gitlab::Graphql::ConnectionRedaction + include ::Gitlab::Graphql::ConnectionCollectionMethods + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb index 8f37fa3f474..54a84be4274 100644 --- a/lib/gitlab/graphql/pagination/connections.rb +++ b/lib/gitlab/graphql/pagination/connections.rb @@ -12,6 +12,10 @@ module Gitlab schema.connections.add( Gitlab::Graphql::ExternallyPaginatedArray, Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection) + + schema.connections.add( + Array, + Gitlab::Graphql::Pagination::ArrayConnection) end end end diff --git a/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb b/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb index 12e047420bf..90a20861b0d 100644 --- a/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb +++ b/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb @@ -5,6 +5,9 @@ module Gitlab module Graphql module Pagination class ExternallyPaginatedArrayConnection < GraphQL::Pagination::ArrayConnection + include ::Gitlab::Graphql::ConnectionCollectionMethods + prepend ::Gitlab::Graphql::ConnectionRedaction + def start_cursor items.previous_cursor end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 252f6371765..2ad8d2f7ab7 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -31,6 +31,8 @@ module Gitlab module Keyset class Connection < GraphQL::Pagination::ActiveRecordRelationConnection include Gitlab::Utils::StrongMemoize + include ::Gitlab::Graphql::ConnectionCollectionMethods + prepend ::Gitlab::Graphql::ConnectionRedaction # rubocop: disable Naming/PredicateName # https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index f3ce3a10703..d37264c1343 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -127,3 +127,5 @@ module Gitlab end end end + +Gitlab::Graphql::Pagination::Keyset::OrderInfo.prepend_if_ee('EE::Gitlab::Graphql::Pagination::Keyset::OrderInfo') diff --git a/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb b/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb index 33f84701562..4a57b7aceca 100644 --- a/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb +++ b/lib/gitlab/graphql/pagination/offset_active_record_relation_connection.rb @@ -7,6 +7,8 @@ module Gitlab module Graphql module Pagination class OffsetActiveRecordRelationConnection < GraphQL::Pagination::ActiveRecordRelationConnection + prepend ::Gitlab::Graphql::ConnectionRedaction + include ::Gitlab::Graphql::ConnectionCollectionMethods end end end diff --git a/lib/gitlab/i18n/html_todo.yml b/lib/gitlab/i18n/html_todo.yml index 91e01f8a0b8..610381b9b48 100644 --- a/lib/gitlab/i18n/html_todo.yml +++ b/lib/gitlab/i18n/html_todo.yml @@ -55,260 +55,3 @@ - "La branĉo <strong>%{branch_name}</strong> estis kreita. Por agordi aŭtomatan disponigadon, bonvolu elekti Yaml-ŝablonon por GitLab CI kaj enmeti viajn ŝanĝojn. %{link_to_autodeploy_doc}" - "La branche <strong>%{branch_name}</strong> a été créée. Pour mettre en place le déploiement automatisé, sélectionnez un modèle de fichier YAML pour l’intégration continue (CI) de GitLab, et validez les modifications. %{link_to_autodeploy_doc}" - "La rama <strong>%{branch_name}</strong> fue creada. Para configurar el auto despliegue, escoge una plantilla Yaml para GitLab CI y envía tus cambios. %{link_to_autodeploy_doc}" -"GitLabPages|GitLab Pages are disabled for this project. You can enable them on your project's %{strong_start}Settings > General > Visibility%{strong_end} page.": - plural_id: - translations: - - "GitLab Pagesはこのプロジェクトでは無効になっています。 プロジェクトの%{strong_start} 設定> 全般> 可視性%{strong_end}ページで有効にできます。" - - "GitLab Pages отключены для этого проекта. Вы можете включить в поле %{strong_start}Настройки > Общие > Видимость%{strong_end} вашего проекта." - - "此项目禁用GitLab Pages。您可以在您的项目的%{strong_start}设置 > 常规 > 可见性%{strong_end} 页面启用。" - - "GitLab Pages вимкнено для цього проєкту. Ви можете їх увімкнути перейшовши на сторінку проєкту %{strong_start}Налаштування > Загальні > Видимість%{strong_end}." - - "Las páginas de GitLab están deshabilitadas para este proyecto. Puede habilitarlas en los ajustes %{strong_start} de su proyecto > General > Visibilidad%{strong_end}." -"You can invite a new member to <strong>%{project_name}</strong> or invite another group.": - plural_id: - translations: - - "新しいメンバーを<strong>%{project_name} </strong>に招待するか、別のグループを招待することができます。" - - "Podes convidar um novo para <strong>%{project_name}</strong> ou convidar outro grupo." - - "邀请新成员或另一个群组加入<strong>%{project_name}</strong>。" - - "Puede invitar a un nuevo miembro a <strong>%{project_name}</strong> o invitar a otro grupo." - - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilir veya başka bir grubu davet edebilirsiniz." - - "Вы можете пригласить нового участника в <strong>%{project_name}</strong> или пригласить другую группу." - - "Ви можете запросити нового учасника до <strong>%{project_name}</strong> або запросити іншу групу." -"You can invite a new member to <strong>%{project_name}</strong>.": - plural_id: - translations: - - "新しいメンバーを<strong>%{project_name} </strong>に招待できます。" - - "Podes convidar um novo membro para <strong>%{project_name}</strong>." - - "邀请新成员加入<strong>%{project_name}</strong>。" - - "Puedes invitar a un nuevo miembro a <strong>%{project_name}</strong>." - - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilirsiniz." - - "Вы можете пригласить нового участника в <strong>%{project_name}</strong>." - - "Ви можете запросити нового учасника до <strong>%{project_name}</strong>." -"You can invite another group to <strong>%{project_name}</strong>.": - plural_id: - translations: - - "他のグループを<strong>%{project_name} </strong>に招待できます。" - - "Podes convidar outro grupo para <strong>%{project_name}</strong>." - - "您可以邀请另一个群组加入<strong>%{project_name}</strong>。" - - "Ви можете запросити нову групу до <strong>%{project_name}</strong>." - - "Puedes invitar a otro grupo a <strong>%{project_name}</strong>." -"Example: <code>192.168.0.0/24</code>. %{read_more_link}.": - plural_id: - translations: -"Note that PostgreSQL %{pg_version_upcoming} will become the minimum required version in GitLab %{gl_version_upcoming} (%{gl_version_upcoming_date}). Please consider upgrading your environment to a supported PostgreSQL version soon, see <a href=\\\"%{pg_version_upcoming_url}\\\">the related epic</a> for details.": - plural_id: - translations: -"Authorize <strong>%{user}</strong> to use your account?": - plural_id: - translations: -"DeployFreeze|Specify times when deployments are not allowed for an environment. The <code>gitlab-ci.yml</code> file must be updated to make deployment jobs aware of the %{freeze_period_link_start}freeze period%{freeze_period_link_end}.": - plural_id: - translations: -"<project name>": - translations: - - "<название проекта>" - - "<project name>" - - "<proje adı>" - - "<naziv projekta>" - - "<ім’я проєкту>" - - "<프로젝트 이름>" -"<strong>Deletes</strong> source branch": - plural_id: - translations: - - "<strong>刪除</strong>來源分支" - - "<strong>Apagar</strong> branch de origem" - - "ソースブランチを<strong>削除</strong>" - - "<strong>刪除</strong>來源分支" - - "<strong>Apagar</strong> o ramo de origem" - - "<strong>Удаляет</strong> исходную ветку" - - "<strong>删除</strong>源分支" - - "<strong>Видаляє</strong> гілку-джерело" - - "<strong>Löscht</strong> den Quellbranch" - - "소스 브랜치 <strong>삭제</strong>" - - "<strong>Supprime</strong> la branche source" - - "<strong>elimina</strong> la rama origen" - - "Kaynak dalı <strong>siler</strong>" -"Badges|You are going to delete this badge. Deleted badges <strong>cannot</strong> be restored.": - plural_id: - translations: - - "Você está prestes a excluir este selo. Selos excluídos <strong>não podem</strong> ser restaurados." - - "このバッジを削除しようとしています。削除されたバッジは<strong>復元できません</strong>。" - - "Estás prestes a apagar este emblema. Emblemas apagados <strong>não podem</strong> ser restaurados." - - "Вы собираетесь удалить этот значок. Удаленные значки <strong>не могут</strong> быть восстановлены." - - "您即将删除此徽章。徽章被删除后 <strong>不能</strong> 恢复。" - - "Ви збираєтеся видалити цей значок. Вилучені значки <strong>не можуть</strong> бути відновлені." - - "Du bist gerade dabei dieses Badge zu entfernen. Entfernte Badges können <strong>nicht</strong> rückgängig gemacht werden." - - "이 배지를 삭제하려고합니다. 삭제 된 배지는 <strong>복원 할 수 없습니다</strong>." - - "Vous êtes sur le point de supprimer ce badge. Les badges supprimés <strong>ne peuvent pas</strong> être restaurés." - - "Va a eliminar esta insignia. Las insignias eliminadas <strong>no se pueden</strong> restaurar." - - "Bu rozeti sileceksiniz. Silinen rozetler geri <strong>yüklenemez</strong>." -"ClusterIntegration| This will permanently delete the following resources: <ul> <li>All installed applications and related resources</li> <li>The <code>gitlab-managed-apps</code> namespace</li> <li>Any project namespaces</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>": - plural_id: - translations: - - "これにより、次のリソースは完全に削除されます <ul> <li>インストールされているすべてのアプリケーションと関連したリソース</li> <li> <code>gitlab-managed-apps</code> 名前空間</li> <li>任意のプロジェクト名前空間</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>" - - "此操作将永久删除下列资源： <ul> <li>所有已安装的应用程序和相关资源</li> <li> <code>GitLab管理的应用</code> 命名空间</li> <li>任何项目命名空间</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>" - - "Esto eliminará permanentemente los siguientes recursos: <ul> <li>Todas las aplicaciones instaladas y sus recursos relacionados</li> <li>El espacio de nombres <code>gitlab-managed-apps</code></li> <li>Cualquier espacio de nombres de proyecto</li> <li><code> clusterroles </code></li> <li><code>clusterrolebindings</code></li> </ul>" -"Configure a <code>.gitlab-webide.yml</code> file in the <code>.gitlab</code> directory to start using the Web Terminal. %{helpStart}Learn more.%{helpEnd}": - plural_id: - translations: - - "Configure um arquivo <code>.gitlab-webide.yml</code> no diretório <code>.gitlab</code> para começar a usar o Terminal Web. %{helpStart}Saiba mais.%{helpEnd}" - - "Webターミナルの使用を開始するには、 <code>.gitlab</code> ディレクトリの <code>.gitlab-webide.yml</code> ファイルを設定します。 詳細は%{helpStart}こちら%{helpEnd}です。" - - "Сконфигурируйте файл <code>.gitlab-webide.yml</code> в каталоге <code>.gitlab</code> чтобы начать использовать веб-терминал. %{helpStart}Узнайте больше.%{helpEnd}" - - "在 <code>.gitlab</code> 目录中配置 <code>.gitlab-webide.yml</code> 文件以开始使用Web终端。 %{helpStart}了解更多。%{helpEnd}" - - "Налаштуйте файл <code>.gitlab-webide.yml</code> у директорії <code>.gitlab</code>, щоб почати використовувати Веб-термінал. %{helpStart}Докладніше.%{helpEnd}" - - "웹 터미널 사용을 시작하도록 <code>.gitlab</code> 디렉토리에서 <code>.gitlab-webide.yml</code> 파일을 구성하십시오. %{helpStart}자세히 알아보십시오.%{helpEnd}" - - "Configure un archivo <code>.gitlab-webide.yml</code> en el directorio <code>.gitlab</code> para comenzar a utilizar el Terminal Web. %{helpStart}Aprende más.%{helpEnd}" -"Depends on <strong>%d closed</strong> merge request.": - plural_id: "Depends on <strong>%d closed</strong> merge requests." - translations: - - "В зависимости от <strong>%d закрытого</strong> запроса на слияние." - - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." - - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." - - "В зависимости от <strong>%d закрытых</strong> запросов на слияние." - - "依赖于<strong>%d个已关闭的</strong>合并请求" - - "Залежить від %d <strong>закритого</strong> запиту на злиття." - - "Залежить від %d <strong>закритих</strong> запитів на злиття." - - "Залежить від %d <strong>закритих</strong> запитів на злиття." - - "Залежить від %d <strong>закритих</strong> запитів на злиття." - - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır." - - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır." -"Go to <strong>Issues</strong> > <strong>Boards</strong> to access your personalized learning issue board.": - plural_id: - translations: - - "转至<strong>议题</strong> > <strong>看板</strong>访问您的个性化学习议题看板。" -"Labels|<span>Promote label</span> %{labelTitle} <span>to Group Label?</span>": - plural_id: - translations: - - "<span>要讓標籤</span> %{labelTitle} <span>提升到群組標籤嗎？</span>" - - "<span>Promover a etiqueta</span> %{labelTitle} <span>para etiqueta do Grupo?</span>" - - "%{labelTitle} <span>ラベルをグループラベルに昇格しますか？</span>" - - "<span>Повысить метку</span> %{labelTitle} <span>до групповой метки?</span>" - - "<span>将标记</span> %{labelTitle} <span>升级为群组标记？</span>" - - "<span>Перенести мітку</span> %{labelTitle} <span>на рівень групи?</span>" - - "<span>Label</span> %{labelTitle} <span>zu Gruppenlabel hochstufen?</span>" - - "<span>라벨</span> %{labelTitle} <span>(을)를 그룹 라벨로 승격하시겠습니까?</span>" - - "<span>Promouvoir l’étiquette</span> %{labelTitle} <span>en étiquette de groupe ?</span>" - - "<span>¿Promocionar la etiqueta</span> %{labelTitle} <span>a etiqueta de grupo?</span>" -"Lock this %{issuableDisplayName}? Only <strong>project members</strong> will be able to comment.": - plural_id: - translations: - - "Travar este %{issuableDisplayName}? Apenas <strong>membros do projeto</strong> poderão comentar." - - "%{issuableDisplayName} をロックしますか？<strong>プロジェクトメンバー</strong> のみコメントできます。" - - "锁定此%{issuableDisplayName}吗？锁定后将只有<strong>项目成员</strong>可以发表评论。" - - "Заблокувати цю %{issuableDisplayName}? Лише <strong>учасники проекту</strong> зможуть коментувати." - - "%{issuableDisplayName} sperren? Es werden nur noch <strong>Projektmitglieder</strong> kommentieren können." - - "Verrouiller ce·t·te %{issuableDisplayName} ? Seuls les <strong>membres du projet</strong> seront en mesure de commenter." - - "¿Bloquear este %{issuableDisplayName}? Sólo los <strong>miembros del proyecto</strong> podrán comentar." -"PrometheusService|<p class=\\\"text-tertiary\\\">No <a href=\\\"%{docsUrl}\\\">common metrics</a> were found</p>": - plural_id: - translations: - - "<p class=\\\"text-tertiary\\\">Nenhuma <a href=\\\"%{docsUrl}\\\">métrica comum</a> foi encontrada</p>" - - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">共通メトリクス</a>は見つかりませんでした</p>" - - "<p class=\\\"text-tertiary\\\">Ни одной <a href=\\\"%{docsUrl}\\\">общей метрики</a> не найдено</p>" - - "<p class=\\\"text-tertiary\\\">无<a href=\\\"%{docsUrl}\\\">常用指标</a> </p>" - - "<p class=\\\"text-tertiary\\\">Ніяких <a href=\\\"%{docsUrl}\\\">загальних метрик</a> не знайдено</p>" - - "<p class=\\\"text-tertiary\\\">Es wurden keine <a href=\\\"%{docsUrl}\\\">allgemeinen Metriken</a> gefunden</p>" - - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">공통 메트릭스</a>가 발견되지 않았습니다.</p>" - - "<p class=\\\"text-tertiary\\\">Aucune <a href=\\\"%{docsUrl}\\\">métrique commune</a> trouvée</p>" - - "<p class=\\\"text-tertiary\\\">No se han encontrado<a href=\\\"%{docsUrl}\\\">métricas comunes</a> </p>" -"This project does not have billing enabled. To create a cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">enable billing <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> and try again.": - plural_id: - translations: - - "Este projeto não possui faturamento ativado. Para criar um cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">ative o faturamento <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e tente novamente." - - "このプロジェクトでは課金が有効になっていません。クラスターを作成するには、<a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\"> 課金を有効<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> にして再度お試しください。" - - "此项目未启用账单。要创建群集，请 <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">启用账单 <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> 并重试。" - - "Для цього проекту вимкнено білінг. Щоб створити кластер, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">увімкніть білінг <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> і спробуйте знову." - - "Für dieses Projekt ist keine Abrechnung aktiviert. Um ein Cluster zu erstellen, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">aktiviere die Abrechnung<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> und versuche es erneut." - - "Ce projet n’a pas de facturation activée. Afin de créer une grappe de serveurs, veuillez <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">activer la facturation<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> et réessayer." - - "Este proyecto no tiene la facturación habilitada. Para crear un clúster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">habilite la facturación <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e inténtelo de nuevo." -"Unlock this %{issuableDisplayName}? <strong>Everyone</strong> will be able to comment.": - plural_id: - translations: - - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> poderão comentar." - - "%{issuableDisplayName} のロックを解除しますか？ <strong>全員</strong>がコメントできるようになります。" - - "解锁此%{issuableDisplayName}吗？解锁后<strong>所有人</strong>都将可以发表评论。" - - "Розблокувати %{issuableDisplayName}? <strong>Будь-хто</strong> зможе залишати коментарі." - - "Dieses %{issuableDisplayName} entsperren? <strong>Jeder</strong> wird in der Lage sein zu kommentieren." - - "%{issuableDisplayName}(을)를 잠금해제 하시겠습니까? <strong>모두가</strong> 코멘트 할 수 있게 됩니다." - - "Déverrouiller %{issuableDisplayName} ? <strong>Tout le monde</strong> sera en mesure de commenter." - - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> podrán comentar." -"confidentiality|You are going to turn off the confidentiality. This means <strong>everyone</strong> will be able to see and leave a comment on this issue.": - plural_id: - translations: - - "Você está prestes a desligar a confidencialidade. Isso significa que <strong>todos</strong> serão capazes de ver e deixar comentários nesse issue." - - "あなたは公開設定に変更しようとしています。これは<strong>すべての人</strong> が閲覧可能になり、課題に対してコメントを残すことができるようになることを意味します。" - - "即将关闭私密性。这将使得 <strong>所有用户</strong>都可以查看并且评论当前议题。" - - "Ви вимикаєте конфіденційність. Це означає, що <strong>будь-хто</strong> зможе бачити і залишати коментарі для цієї задачі." - - "Du willst die Vertraulichkeit deaktivieren. Das bedeutet, dass <strong>alle</strong> das Ticket betrachten und kommentieren können." - - "Vous êtes sur le point de désactiver la confidentialité. Cela signifie que <strong>tout le monde</strong> sera en mesure de voir et de laisser un commentaire sur ce ticket." - - "Va a desactivar la confidencialidad. Esto significa que <strong>todos</strong> podrán ver y dejar un comentario sobre este tema." -"confidentiality|You are going to turn on the confidentiality. This means that only team members with <strong>at least Reporter access</strong> are able to see and leave comments on the issue.": - plural_id: - translations: - - "Você está prestes a ligar a confidencialidade. Isso significa que apenas membros da equipe com <strong>ao menos acesso de Relator</strong> serão capazes de ver e deixar comentários nesse issue." - - "あなたは公開設定に変更しようとしています。これはチームに限定していた<strong>最小限の報告権限</strong>をなくし、課題に対してコメントを残すことができるようになることを意味します。" - - "即将设置私密性。这将使得 <strong>至少有Reporter以上权限</strong>的团队成员才能查看并且评论当前议题。" - - "Ви вмикаєте конфіденційність. Це означає що лише учасники команди <strong>рівня репортер або вище</strong> матимуть змогу бачити та залишати коментарі для цієї задачі." - - "Du willst die Vertraulichkeit aktivieren. Das bedeutet, dass nur Teammitglieder mit <strong>mindestens Reporter-Zugriff</strong> das Ticket betrachten und kommentieren können." - - "Vous êtes sur le point de d’activer la confidentialité. Cela signifie que seuls les membres de l’équipe avec <strong>au moins un accès en tant que rapporteur</strong> seront en mesure de voir et de laisser des commentaires sur le ticket." - - "Va a activar la confidencialidad. Esto significa que solo los miembros del equipo con como mínimo,<strong>acceso como Reporter</strong> podrán ver y dejar comentarios sobre la incidencia." - - "あなたは非公開設定をオンにしようとしています。これは、最低でも<strong>報告権限</strong>を持ったチームメンバーのみが課題を表示したりコメントを残したりすることができるようになるということです。" -" or <!merge request id>": - translations: - - " ወይም <!merge request id>" - - " ou <!merge request id>" - - " または <!merge request id>" - - "或 <!合併請求 id>" - - " или <!merge request id>" - - "或<!merge request id>" - - " або <!merge request id>" - - " oder <!merge request id>" - - " o <!merge request id>" - - " 또는 <!merge request id>" - - " o <!merge request id>" - - " veya <!merge request id>" - - " neu <!merge request id>" - - " neu <#issue id>" -" or <#issue id>": - translations: - - "或 <#issue id>" - - " ወይም ‹#issue id›" - - " ou <identificación #issue>" - - " ou <#issue id>" - - " または <#課題 ID>" - - " o <#issue id>" - - "或 <#議題 id>" - - " ou <#issue id>" - - " или <#issue id>" - - "或 <#issue id>" - - " або <#issue id>" - - " oder <#issue id>" - - " o <#issue id>" - - " 또는 <#issue id>" - - " ou <#issue id>" - - " o <#issue id>" - - " veya <#issue id>" - - " neu <#issue id>" -" or <&epic id>": - translations: - - " ወይም <&epic id>" - - " または <&エピックID>" - - " 或 <#史詩 id>" - - " или <&epic id>" - - " 或<#epic id>" - - " або <&epic id>" - - " oder <&epic id>" - - " o <&epic id>" - - " veya <&epic id>" - - " neu <#epic id>" - - " 또는 <&epic id>" -"< 1 hour": - translations: - - "1 時間未満" - - "< 1 小時" - - "< 1 часа" - - "< 1小时" - - "< 1 години" - - "< 1 hora" - - "< 1 saat" - - "< 1 Stunde" - - "< 1시간" diff --git a/lib/gitlab/import_export/import_failure_service.rb b/lib/gitlab/import_export/import_failure_service.rb index d4eca551b49..bf7200726a1 100644 --- a/lib/gitlab/import_export/import_failure_service.rb +++ b/lib/gitlab/import_export/import_failure_service.rb @@ -28,23 +28,26 @@ module Gitlab end def log_import_failure(source:, relation_key: nil, relation_index: nil, exception:, retry_count: 0) - extra = { - source: source, - relation_key: relation_key, + attributes = { relation_index: relation_index, - retry_count: retry_count + source: source, + retry_count: retry_count, + importable_column_name => importable.id } - extra[importable_column_name] = importable.id - - Gitlab::ErrorTracking.track_exception(exception, extra) - - attributes = { - exception_class: exception.class.to_s, - exception_message: exception.message.truncate(255), - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id - }.merge(extra) - ImportFailure.create(attributes) + Gitlab::ErrorTracking.track_exception( + exception, + attributes.merge(relation_name: relation_key) + ) + + ImportFailure.create( + attributes.merge( + exception_class: exception.class.to_s, + exception_message: exception.message.truncate(255), + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + relation_key: relation_key + ) + ) end private diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index ae7ddbc5eba..8c094603d53 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -169,6 +169,7 @@ excluded_attributes: - :compliance_framework_setting - :show_default_award_emojis - :services + - :exported_protected_branches namespaces: - :runners_token - :runners_token_encrypted diff --git a/lib/gitlab/kubernetes/helm/v2/client_command.rb b/lib/gitlab/kubernetes/helm/v2/client_command.rb index 88693a28d6c..8b15af9aeea 100644 --- a/lib/gitlab/kubernetes/helm/v2/client_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/client_command.rb @@ -22,17 +22,6 @@ module Gitlab def repository_update_command 'helm repo update' end - - def optional_tls_flags - return [] unless files.key?(:'ca.pem') - - [ - '--tls', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tls-cert', "#{files_dir}/cert.pem", - '--tls-key', "#{files_dir}/key.pem" - ] - end end end end diff --git a/lib/gitlab/kubernetes/helm/v2/reset_command.rb b/lib/gitlab/kubernetes/helm/v2/reset_command.rb index 172a0884c49..00626501a9a 100644 --- a/lib/gitlab/kubernetes/helm/v2/reset_command.rb +++ b/lib/gitlab/kubernetes/helm/v2/reset_command.rb @@ -9,9 +9,8 @@ module Gitlab def generate_script super + [ - reset_helm_command, - delete_tiller_replicaset, - delete_tiller_clusterrolebinding + init_command, + reset_helm_command ].join("\n") end @@ -21,27 +20,8 @@ module Gitlab private - # This method can be delete once we upgrade Helm to > 12.13.0 - # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27096#note_159695900 - # - # Tracking this method to be removed here: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/52791#note_199374155 - def delete_tiller_replicaset - delete_args = %w[replicaset -n gitlab-managed-apps -l name=tiller] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - - def delete_tiller_clusterrolebinding - delete_args = %w[clusterrolebinding tiller-admin] - - Gitlab::Kubernetes::KubectlCmd.delete(*delete_args) - end - def reset_helm_command - command = %w[helm reset] + optional_tls_flags - - command.shelljoin + 'helm reset --force' end end end diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb new file mode 100644 index 00000000000..222f5d57adf --- /dev/null +++ b/lib/gitlab/rack_attack.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# Integration specs for throttling can be found in: +# spec/requests/rack_attack_global_spec.rb +module Gitlab + module RackAttack + def self.configure(rack_attack) + # This adds some methods used by our throttles to the `Rack::Request` + rack_attack::Request.include(Gitlab::RackAttack::Request) + # Configure the throttles + configure_throttles(rack_attack) + end + + def self.configure_throttles(rack_attack) + throttle_or_track(rack_attack, 'throttle_unauthenticated', Gitlab::Throttle.unauthenticated_options) do |req| + if !req.should_be_skipped? && + Gitlab::Throttle.settings.throttle_unauthenticated_enabled && + req.unauthenticated? + req.ip + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req| + if req.api_request? && + Gitlab::Throttle.settings.throttle_authenticated_api_enabled + req.authenticated_user_id([:api]) + end + end + + # Product analytics feature is in experimental stage. + # At this point we want to limit amount of events registered + # per application (aid stands for application id). + throttle_or_track(rack_attack, 'throttle_product_analytics_collector', limit: 100, period: 60) do |req| + if req.product_analytics_collector_request? + req.params['aid'] + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req| + if req.web_request? && + Gitlab::Throttle.settings.throttle_authenticated_web_enabled + req.authenticated_user_id([:api, :rss, :ics]) + end + end + + throttle_or_track(rack_attack, 'throttle_unauthenticated_protected_paths', Gitlab::Throttle.protected_paths_options) do |req| + if req.post? && + !req.should_be_skipped? && + req.protected_path? && + Gitlab::Throttle.protected_paths_enabled? && + req.unauthenticated? + req.ip + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_api', Gitlab::Throttle.protected_paths_options) do |req| + if req.post? && + req.api_request? && + req.protected_path? && + Gitlab::Throttle.protected_paths_enabled? + req.authenticated_user_id([:api]) + end + end + + throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_web', Gitlab::Throttle.protected_paths_options) do |req| + if req.post? && + req.web_request? && + req.protected_path? && + Gitlab::Throttle.protected_paths_enabled? + req.authenticated_user_id([:api, :rss, :ics]) + end + end + + rack_attack.safelist('throttle_bypass_header') do |req| + Gitlab::Throttle.bypass_header.present? && + req.get_header(Gitlab::Throttle.bypass_header) == '1' + end + end + + def self.throttle_or_track(rack_attack, throttle_name, *args, &block) + if track?(throttle_name) + rack_attack.track(throttle_name, *args, &block) + else + rack_attack.throttle(throttle_name, *args, &block) + end + end + + def self.track?(name) + dry_run_config = ENV['GITLAB_THROTTLE_DRY_RUN'].to_s.strip + + return false if dry_run_config.empty? + return true if dry_run_config == '*' + + dry_run_config.split(',').map(&:strip).include?(name) + end + end +end +::Gitlab::RackAttack.prepend_if_ee('::EE::Gitlab::RackAttack') diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb new file mode 100644 index 00000000000..a75f66af263 --- /dev/null +++ b/lib/gitlab/rack_attack/request.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module RackAttack + module Request + def unauthenticated? + !(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id) + end + + def authenticated_user_id(request_formats) + request_authenticator.user(request_formats)&.id + end + + def authenticated_runner_id + request_authenticator.runner&.id + end + + def api_request? + path.start_with?('/api') + end + + def api_internal_request? + path =~ %r{^/api/v\d+/internal/} + end + + def health_check_request? + path =~ %r{^/-/(health|liveness|readiness|metrics)} + end + + def product_analytics_collector_request? + path.start_with?('/-/collector/i') + end + + def should_be_skipped? + api_internal_request? || health_check_request? + end + + def web_request? + !api_request? && !health_check_request? + end + + def protected_path? + !protected_path_regex.nil? + end + + def protected_path_regex + path =~ protected_paths_regex + end + + private + + def request_authenticator + @request_authenticator ||= Gitlab::Auth::RequestAuthenticator.new(self) + end + + def protected_paths + Gitlab::CurrentSettings.current_application_settings.protected_paths + end + + def protected_paths_regex + Regexp.union(protected_paths.map { |path| /\A#{Regexp.escape(path)}/ }) + end + end + end +end +::Gitlab::RackAttack::Request.prepend_if_ee('::EE::Gitlab::RackAttack::Request') diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb new file mode 100644 index 00000000000..aebf8d92cb3 --- /dev/null +++ b/lib/gitlab/throttle.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + class Throttle + def self.settings + Gitlab::CurrentSettings.current_application_settings + end + + # Returns true if we should use the Admin Area protected paths throttle + def self.protected_paths_enabled? + self.settings.throttle_protected_paths_enabled? + end + + def self.omnibus_protected_paths_present? + Rack::Attack.throttles.key?('protected paths') + end + + def self.bypass_header + env_value = ENV['GITLAB_THROTTLE_BYPASS_HEADER'] + return unless env_value.present? + + "HTTP_#{env_value.upcase.tr('-', '_')}" + end + + def self.unauthenticated_options + limit_proc = proc { |req| settings.throttle_unauthenticated_requests_per_period } + period_proc = proc { |req| settings.throttle_unauthenticated_period_in_seconds.seconds } + { limit: limit_proc, period: period_proc } + end + + def self.authenticated_api_options + limit_proc = proc { |req| settings.throttle_authenticated_api_requests_per_period } + period_proc = proc { |req| settings.throttle_authenticated_api_period_in_seconds.seconds } + { limit: limit_proc, period: period_proc } + end + + def self.authenticated_web_options + limit_proc = proc { |req| settings.throttle_authenticated_web_requests_per_period } + period_proc = proc { |req| settings.throttle_authenticated_web_period_in_seconds.seconds } + { limit: limit_proc, period: period_proc } + end + + def self.protected_paths_options + limit_proc = proc { |req| settings.throttle_protected_paths_requests_per_period } + period_proc = proc { |req| settings.throttle_protected_paths_period_in_seconds.seconds } + + { limit: limit_proc, period: period_proc } + end + end +end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 19be468e3d5..461b5dc3afc 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -26,6 +26,7 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: nil) snowplow.event(category, action, label: label, property: property, value: value, context: context) + product_analytics.event(category, action, label: label, property: property, value: value, context: context) end def self_describing_event(schema_url, event_data_json, context: nil) @@ -49,6 +50,10 @@ module Gitlab def snowplow @snowplow ||= Gitlab::Tracking::Destinations::Snowplow.new end + + def product_analytics + @product_analytics ||= Gitlab::Tracking::Destinations::ProductAnalytics.new + end end end end diff --git a/lib/gitlab/tracking/destinations/product_analytics.rb b/lib/gitlab/tracking/destinations/product_analytics.rb new file mode 100644 index 00000000000..cacedbc5b83 --- /dev/null +++ b/lib/gitlab/tracking/destinations/product_analytics.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Destinations + class ProductAnalytics < Base + extend ::Gitlab::Utils::Override + include ::Gitlab::Utils::StrongMemoize + + override :event + def event(category, action, label: nil, property: nil, value: nil, context: nil) + return unless event_allowed?(category, action) + return unless enabled? + + tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) + end + + private + + def event_allowed?(category, action) + category == 'epics' && action == 'promote' + end + + def enabled? + Feature.enabled?(:product_analytics_tracking, type: :ops) && + Gitlab::CurrentSettings.usage_ping_enabled? && + Gitlab::CurrentSettings.self_monitoring_project_id.present? + end + + def tracker + @tracker ||= SnowplowTracker::Tracker.new( + SnowplowTracker::AsyncEmitter.new(::ProductAnalytics::Tracker::COLLECTOR_URL, protocol: Gitlab.config.gitlab.protocol), + SnowplowTracker::Subject.new, + Gitlab::Tracking::SNOWPLOW_NAMESPACE, + Gitlab::CurrentSettings.self_monitoring_project_id.to_s + ) + end + end + end + end +end diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index 9377ccfec1e..b610d2a10c6 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -75,3 +75,5 @@ module Gitlab end end end + +Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper') diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 4b0dd54683b..77a74c86c63 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -296,20 +296,7 @@ module Gitlab # @return [Array<#totals>] An array of objects that respond to `#totals` def usage_data_counters - [ - Gitlab::UsageDataCounters::WikiPageCounter, - Gitlab::UsageDataCounters::WebIdeCounter, - Gitlab::UsageDataCounters::NoteCounter, - Gitlab::UsageDataCounters::SnippetCounter, - Gitlab::UsageDataCounters::SearchCounter, - Gitlab::UsageDataCounters::CycleAnalyticsCounter, - Gitlab::UsageDataCounters::ProductivityAnalyticsCounter, - Gitlab::UsageDataCounters::SourceCodeCounter, - Gitlab::UsageDataCounters::MergeRequestCounter, - Gitlab::UsageDataCounters::DesignsCounter, - Gitlab::UsageDataCounters::KubernetesAgentCounter, - Gitlab::UsageDataCounters::StaticSiteEditorCounter - ] + Gitlab::UsageDataCounters.counters end def components_usage_data diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb new file mode 100644 index 00000000000..79f0cd117bc --- /dev/null +++ b/lib/gitlab/usage_data_counters.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + COUNTERS = [ + WikiPageCounter, + WebIdeCounter, + NoteCounter, + SnippetCounter, + SearchCounter, + CycleAnalyticsCounter, + ProductivityAnalyticsCounter, + SourceCodeCounter, + MergeRequestCounter, + DesignsCounter, + KubernetesAgentCounter, + StaticSiteEditorCounter + ].freeze + + UsageDataCounterError = Class.new(StandardError) + UnknownEvent = Class.new(UsageDataCounterError) + + class << self + def counters + self::COUNTERS + end + + def count(event_name) + counters.each do |counter| + event = counter.fetch_supported_event(event_name) + + return counter.count(event) if event + end + + raise UnknownEvent, "Cannot find counter for event #{event_name}" + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index 97ec8423b95..dd43b60bd45 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -15,3 +15,28 @@ - name: product_analytics_test_metrics_intersection operator: AND events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] +- name: incident_management_alerts_total_unique_counts + operator: OR + events: [ + 'incident_management_alert_status_changed', + 'incident_management_alert_assigned', + 'incident_management_alert_todo', + 'incident_management_alert_create_incident' + ] + feature_flag: usage_data_incident_management_alerts_total_unique_counts +- name: incident_management_incidents_total_unique_counts + operator: OR + events: [ + 'incident_management_incident_created', + 'incident_management_incident_reopened', + 'incident_management_incident_closed', + 'incident_management_incident_assigned', + 'incident_management_incident_todo', + 'incident_management_incident_comment', + 'incident_management_incident_zoom_meeting', + 'incident_management_incident_published', + 'incident_management_incident_relate', + 'incident_management_incident_unrelate', + 'incident_management_incident_change_confidential' + ] + feature_flag: usage_data_incident_management_incidents_total_unique_counts diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 44893645cc2..d28fd17a989 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -29,6 +29,12 @@ module Gitlab::UsageDataCounters known_events.map { |event| [counter_key(event), -1] }.to_h end + def fetch_supported_event(event_name) + return if prefix.present? && !event_name.start_with?(prefix) + + known_events.find { |event| counter_key(event) == event_name.to_sym } + end + private def require_known_event(event) diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 85f16ea807b..58b023d374c 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -229,6 +229,12 @@ category: incident_management aggregation: weekly feature_flag: usage_data_incident_management_incident_change_confidential +# Incident management alerts +- name: incident_management_alert_create_incident + redis_slot: incident_management + category: incident_management_alerts + aggregation: weekly + feature_flag: usage_data_incident_management_alert_create_incident # Testing category - name: i_testing_test_case_parsed category: testing diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb index 61f98887adc..46aec52b95a 100644 --- a/lib/gitlab/usage_data_counters/search_counter.rb +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -4,6 +4,7 @@ module Gitlab module UsageDataCounters class SearchCounter < BaseCounter KNOWN_EVENTS = %w[all_searches navbar_searches].freeze + PREFIX = nil class << self def redis_key(event) diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index 0b21c355a54..39005f56dcb 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -14,7 +14,7 @@ module MicrosoftTeams response = Gitlab::HTTP.post( @webhook.to_str, headers: @header, - body: body(options) + body: body(**options) ) result = true if response @@ -27,14 +27,13 @@ module MicrosoftTeams private - def body(options = {}) + def body(title: nil, summary: nil, attachments: nil, activity:) result = { 'sections' => [] } - result['title'] = options[:title] - result['summary'] = options[:summary] - result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare + result['title'] = title + result['summary'] = summary + result['sections'] << MicrosoftTeams::Activity.new(**activity).prepare - attachments = options[:attachments] unless attachments.blank? result['sections'] << { text: attachments } end diff --git a/lib/object_storage/config.rb b/lib/object_storage/config.rb index cc536ce9b46..f933d4e4866 100644 --- a/lib/object_storage/config.rb +++ b/lib/object_storage/config.rb @@ -93,6 +93,11 @@ module ObjectStorage private + # This returns a Hash of HTTP encryption headers to send along to S3. + # + # They can also be passed in as Fog::AWS::Storage::File attributes, since there + # are aliases defined for them: + # https://github.com/fog/fog-aws/blob/ab288f29a0974d64fd8290db41080e5578be9651/lib/fog/aws/models/storage/file.rb#L24-L25 def aws_server_side_encryption_headers { 'x-amz-server-side-encryption' => server_side_encryption, diff --git a/lib/product_analytics/tracker.rb b/lib/product_analytics/tracker.rb index 2dc5e1f53ce..d4a88b879f0 100644 --- a/lib/product_analytics/tracker.rb +++ b/lib/product_analytics/tracker.rb @@ -7,36 +7,5 @@ module ProductAnalytics # The collector URL minus protocol and /i COLLECTOR_URL = Gitlab.config.gitlab.url.sub(/\Ahttps?\:\/\//, '') + '/-/collector' - - class << self - include Gitlab::Utils::StrongMemoize - - def event(category, action, label: nil, property: nil, value: nil, context: nil) - return unless enabled? - - snowplow.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) - end - - private - - def enabled? - Gitlab::CurrentSettings.usage_ping_enabled? - end - - def project_id - Gitlab::CurrentSettings.self_monitoring_project_id - end - - def snowplow - strong_memoize(:snowplow) do - SnowplowTracker::Tracker.new( - SnowplowTracker::AsyncEmitter.new(COLLECTOR_URL, protocol: Gitlab.config.gitlab.protocol), - SnowplowTracker::Subject.new, - Gitlab::Tracking::SNOWPLOW_NAMESPACE, - project_id.to_s - ) - end - end - end end end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index e2c92054d62..0834d41521b 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -1,39 +1,38 @@ +# frozen_string_literal: true + require "gettext_i18n_rails/tasks" namespace :gettext do - # Customize list of translatable files - # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files - def files_to_translate - folders = %W(ee app lib config #{locale_path}).join(',') - exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') - - Dir.glob( - "{#{folders}}/**/*.{#{exts}}" - ) - end - - # Disallow HTML from translatable strings - # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html - def html_todolist - return @html_todolist if defined?(@html_todolist) - - @html_todolist = YAML.load_file(Rails.root.join('lib/gitlab/i18n/html_todo.yml')) - end - task :compile do # See: https://gitlab.com/gitlab-org/gitlab-foss/issues/33014#note_31218998 - FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) + FileUtils.touch(pot_file_path) Rake::Task['gettext:po_to_json'].invoke end desc 'Regenerate gitlab.pot file' task :regenerate do - pot_file = 'locale/gitlab.pot' - # Remove all translated files, this speeds up finding - FileUtils.rm Dir['locale/**/gitlab.*'] + ensure_locale_folder_presence! + + # Clean up folders that do not contain a gitlab.po file + Pathname.new(locale_path).children.each do |child| + next unless child.directory? + + folder_path = child.to_path + + if File.exist?("#{folder_path}/gitlab.po") + # remove all translated files to speed up finding + FileUtils.rm Dir["#{folder_path}/gitlab.*"] + else + # remove empty translation folders so we don't generate un-needed .po files + puts "Deleting #{folder_path} as it does not contain a 'gitlab.po' file." + + FileUtils.rm_r folder_path + end + end + # remove the `pot` file to ensure it's completely regenerated - FileUtils.rm_f pot_file + FileUtils.rm_f(pot_file_path) Rake::Task['gettext:find'].invoke @@ -42,10 +41,12 @@ namespace :gettext do raise 'failed to cleanup generated locale/*/gitlab.po files' end + raise 'gitlab.pot file not generated' unless File.exist?(pot_file_path) + # Remove timestamps from the pot file - pot_content = File.read pot_file + pot_content = File.read pot_file_path pot_content.gsub!(/^"POT?\-(?:Creation|Revision)\-Date\:.*\n/, '') - File.write pot_file, pot_content + File.write pot_file_path, pot_content puts <<~MSG All done. Please commit the changes to `locale/gitlab.pot`. @@ -67,8 +68,7 @@ namespace :gettext do Gitlab::I18n::PoLinter.new(po_path: file, html_todolist: html_todolist, locale: locale) end - pot_file = Rails.root.join('locale/gitlab.pot') - linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file, html_todolist: html_todolist)) + linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file_path, html_todolist: html_todolist)) failed_linters = linters.select { |linter| linter.errors.any? } @@ -84,12 +84,11 @@ namespace :gettext do end task :updated_check do - pot_file = 'locale/gitlab.pot' # Removing all pre-translated files speeds up `gettext:find` as the # files don't need to be merged. # Having `LC_MESSAGES/gitlab.mo files present also confuses the output. FileUtils.rm Dir['locale/**/gitlab.*'] - FileUtils.rm_f pot_file + FileUtils.rm_f pot_file_path # `gettext:find` writes touches to temp files to `stderr` which would cause # `static-analysis` to report failures. We can ignore these. @@ -97,18 +96,18 @@ namespace :gettext do Rake::Task['gettext:find'].invoke end - pot_diff = `git diff -- #{pot_file} | grep -E '^(\\+|-)msgid'`.strip + pot_diff = `git diff -- #{pot_file_path} | grep -E '^(\\+|-)msgid'`.strip # reset the locale folder for potential next tasks `git checkout -- locale` if pot_diff.present? raise <<~MSG - Changes in translated strings found, please update file `#{pot_file}` by running: + Changes in translated strings found, please update file `#{pot_file_path}` by running: bin/rake gettext:regenerate - Then commit and push the resulting changes to `#{pot_file}`. + Then commit and push the resulting changes to `#{pot_file_path}`. The diff was: @@ -117,6 +116,27 @@ namespace :gettext do end end + private + + # Customize list of translatable files + # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files + def files_to_translate + folders = %W(ee app lib config #{locale_path}).join(',') + exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') + + Dir.glob( + "{#{folders}}/**/*.{#{exts}}" + ) + end + + # Disallow HTML from translatable strings + # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html + def html_todolist + return @html_todolist if defined?(@html_todolist) + + @html_todolist = YAML.safe_load(File.read(Rails.root.join('lib/gitlab/i18n/html_todo.yml'))) + end + def report_errors_for_file(file, errors_for_file) puts "Errors in `#{file}`:" @@ -140,4 +160,21 @@ namespace :gettext do $stderr.reopen(old_stderr) old_stderr.close end + + def ensure_locale_folder_presence! + unless Dir.exist?(locale_path) + raise <<~MSG + Cannot find '#{locale_path}' folder. Please ensure you're running this task from the gitlab repo. + + MSG + end + end + + def locale_path + @locale_path ||= Rails.root.join('locale') + end + + def pot_file_path + @pot_file_path ||= File.join(locale_path, 'gitlab.pot') + end end |