diff options
Diffstat (limited to 'lib')
409 files changed, 6723 insertions, 2514 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index ada0da28749..0598f03c7ab 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -30,7 +30,7 @@ module API ] allow_access_with_scope :api - allow_access_with_scope :read_api, if: -> (request) { request.get? } + allow_access_with_scope :read_api, if: -> (request) { request.get? || request.head? } prefix :api version 'v3', using: :path do @@ -68,6 +68,10 @@ module API set_peek_enabled_for_current_request end + after do + Gitlab::UsageDataCounters::VSCodeExtensionActivityUniqueCounter.track_api_request_when_trackable(user_agent: request&.user_agent, user: @current_user) + end + # The locale is set to the current user's locale when `current_user` is loaded after { Gitlab::I18n.use_default_locale } @@ -123,13 +127,32 @@ module API format :json formatter :json, Gitlab::Json::GrapeFormatter + content_type :json, 'application/json' + # Remove the `text/plain+deprecated` with `api_always_use_application_json` feature flag # There is a small chance some users depend on the old behavior. # We this change under a feature flag to see if affects GitLab.com users. - if Gitlab::Database.cached_table_exists?('features') && Feature.enabled?(:api_json_content_type) - content_type :json, 'application/json' - else - content_type :txt, 'text/plain' + # The `+deprecated` is added to distinguish content type + # as defined by `API::API` vs ex. `API::Repositories` + content_type :txt, 'text/plain+deprecated' + + before do + # the feature flag workaround is only for `.txt` + api_format = env[Grape::Env::API_FORMAT] + next unless api_format == :txt + + # get all defined content-types for the endpoint + api_endpoint = env[Grape::Env::API_ENDPOINT] + content_types = api_endpoint&.namespace_stackable_with_hash(:content_types).to_h + + # Only overwrite `text/plain+deprecated` + if content_types[api_format] == 'text/plain+deprecated' + if Feature.enabled?(:api_always_use_application_json) + content_type 'application/json' + else + content_type 'text/plain' + end + end end # Ensure the namespace is right, otherwise we might load Grape::API::Helpers @@ -249,6 +272,8 @@ module API mount ::API::Release::Links mount ::API::RemoteMirrors mount ::API::Repositories + mount ::API::ResourceAccessTokens + mount ::API::RubygemPackages mount ::API::Search mount ::API::Services mount ::API::Settings @@ -294,4 +319,4 @@ module API end end -API::API.prepend_if_ee('::EE::API::API') +API::API.prepend_ee_mod diff --git a/lib/api/applications.rb b/lib/api/applications.rb index 8b14e16b495..b883f83cc19 100644 --- a/lib/api/applications.rb +++ b/lib/api/applications.rb @@ -8,15 +8,6 @@ module API feature_category :authentication_and_authorization resource :applications do - helpers do - def validate_redirect_uri(value) - uri = ::URI.parse(value) - !uri.is_a?(URI::HTTP) || uri.host - rescue URI::InvalidURIError - false - end - end - desc 'Create a new application' do detail 'This feature was introduced in GitLab 10.5' success Entities::ApplicationWithSecret @@ -30,13 +21,6 @@ module API desc: 'Application will be used where the client secret is confidential' end post do - # Validate that host in uri is specified - # Please remove it when https://github.com/doorkeeper-gem/doorkeeper/pull/1440 is merged - # and the doorkeeper gem version is bumped - unless validate_redirect_uri(declared_params[:redirect_uri]) - render_api_error!({ redirect_uri: ["must be an absolute URI."] }, :bad_request) - end - application = Doorkeeper::Application.new(declared_params) if application.save diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 1b36e27f6c9..fa75d012613 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -121,6 +121,7 @@ module API end params do requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + optional :include_retried, type: Boolean, default: false, desc: 'Includes retried jobs' use :optional_scope use :pagination end diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index 6c8b3a1ba4a..1796d51324f 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -72,6 +72,7 @@ module API namespace 'users' do format :txt + content_type :txt, 'text/plain' desc 'Authenticate user against conan CLI' do detail 'This feature was introduced in GitLab 12.2' diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index e3cacc4132f..f138f400601 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -8,12 +8,14 @@ module API resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do + require_packages_enabled! + not_found! unless ::Feature.enabled?(:debian_packages, user_group) authorize_read_package!(user_group) end - namespace ':id/-/packages/debian' do + namespace ':id/packages/debian' do include DebianPackageEndpoints end end diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb index c95c75b7e5c..e7689b3feff 100644 --- a/lib/api/debian_package_endpoints.rb +++ b/lib/api/debian_package_endpoints.rb @@ -32,6 +32,7 @@ module API helpers ::API::Helpers::Packages::BasicAuthHelpers format :txt + content_type :txt, 'text/plain' rescue_from ArgumentError do |e| render_api_error!(e.message, 400) @@ -50,33 +51,33 @@ module API end namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release.gpg + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg desc 'The Release file signature' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'Release.gpg' do not_found! end - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release desc 'The unsigned Release file' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'Release' do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO Release' end - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/InRelease + # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease desc 'The signed Release file' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'InRelease' do not_found! end @@ -87,12 +88,12 @@ module API end namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do - # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages desc 'The binary files index' do detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get 'Packages' do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO Packages' @@ -107,7 +108,7 @@ module API end namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do - # GET {projects|groups}/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name + # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name params do requires :file_name, type: String, desc: 'The Debian File Name' end @@ -115,7 +116,7 @@ module API detail 'This feature was introduced in GitLab 13.5' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true get ':file_name', requirements: FILE_NAME_REQUIREMENTS do # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 'TODO File' diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index f8129c18dff..8c0db42a448 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -8,27 +8,29 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do + require_packages_enabled! + not_found! unless ::Feature.enabled?(:debian_packages, user_project) authorize_read_package! end - namespace ':id/-/packages/debian' do + namespace ':id/packages/debian' do include DebianPackageEndpoints params do requires :file_name, type: String, desc: 'The file name' end - namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do + namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE - # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name + # PUT {projects|groups}/:id/packages/debian/:file_name params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' end - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true put do authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size) @@ -42,8 +44,8 @@ module API forbidden! end - # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize - route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth + # PUT {projects|groups}/:id/packages/debian/:file_name/authorize + route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth, authenticate_non_public: true put 'authorize' do authorize_workhorse!( subject: authorized_user_project, diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb index 5fab590eb4e..30ec4e52b2a 100644 --- a/lib/api/deploy_tokens.rb +++ b/lib/api/deploy_tokens.rb @@ -28,8 +28,6 @@ module API use :pagination end get 'deploy_tokens' do - service_unavailable! unless Feature.enabled?(:deploy_tokens_api, default_enabled: true) - authenticated_as_admin! present paginate(DeployToken.all), with: Entities::DeployToken @@ -39,10 +37,6 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do - service_unavailable! unless Feature.enabled?(:deploy_tokens_api, user_project, default_enabled: true) - end - params do use :pagination end @@ -102,10 +96,6 @@ module API requires :id, type: String, desc: 'The ID of a group' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - before do - service_unavailable! unless Feature.enabled?(:deploy_tokens_api, user_group, default_enabled: true) - end - params do use :pagination end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 5346fcf03c9..d0c842bb19d 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -36,7 +36,7 @@ module API get ':id/deployments' do authorize! :read_deployment, user_project - deployments = DeploymentsFinder.new(user_project, params).execute + deployments = DeploymentsFinder.new(params.merge(project: user_project)).execute present paginate(deployments), with: Entities::Deployment end diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb index e9572a8d430..2468c1f9b18 100644 --- a/lib/api/entities/application_setting.rb +++ b/lib/api/entities/application_setting.rb @@ -31,6 +31,7 @@ module API expose :password_authentication_enabled_for_web, as: :password_authentication_enabled expose :password_authentication_enabled_for_web, as: :signin_enabled expose :allow_local_requests_from_web_hooks_and_services, as: :allow_local_requests_from_hooks_and_services + expose :asset_proxy_allowlist, as: :asset_proxy_whitelist end end end diff --git a/lib/api/entities/ci/job.rb b/lib/api/entities/ci/job.rb index 7fe1a802e24..76487ed01dc 100644 --- a/lib/api/entities/ci/job.rb +++ b/lib/api/entities/ci/job.rb @@ -9,6 +9,9 @@ module API expose :job_artifacts, as: :artifacts, using: ::API::Entities::Ci::JobArtifact expose :runner, with: ::API::Entities::Runner expose :artifacts_expire_at + expose :tag_list do |job| + job.tags.map(&:name).sort + end end end end diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb index 7f1b5b87725..88c84c494e2 100644 --- a/lib/api/entities/merge_request_basic.rb +++ b/lib/api/entities/merge_request_basic.rb @@ -27,7 +27,7 @@ module API expose(:downvotes) { |merge_request, options| issuable_metadata.downvotes } expose :author, :assignees, :assignee, using: Entities::UserBasic - expose :reviewers, if: -> (merge_request, _) { merge_request.allows_reviewers? }, using: Entities::UserBasic + expose :reviewers, using: Entities::UserBasic expose :source_project_id, :target_project_id expose :labels do |merge_request, options| if options[:with_labels_details] diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb index 4aa5c9b7236..248a86751d2 100644 --- a/lib/api/entities/user.rb +++ b/lib/api/entities/user.rb @@ -6,9 +6,16 @@ module API include UsersHelper expose :created_at, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } expose :bio, :bio_html, :location, :public_email, :skype, :linkedin, :twitter, :website_url, :organization, :job_title + expose :bot?, as: :bot expose :work_information do |user| work_information(user) end + expose :followers, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user| + user.followers.count + end + expose :following, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user| + user.followees.count + end end end end diff --git a/lib/api/entities/user_status.rb b/lib/api/entities/user_status.rb index 1d5cc27e5ef..ef4772f60c6 100644 --- a/lib/api/entities/user_status.rb +++ b/lib/api/entities/user_status.rb @@ -9,6 +9,7 @@ module API expose :message_html do |entity| MarkupHelper.markdown_field(entity, :message) end + expose :clear_status_at end end end diff --git a/lib/api/events.rb b/lib/api/events.rb index 233c62b5389..db5ed7b7e6e 100644 --- a/lib/api/events.rb +++ b/lib/api/events.rb @@ -6,7 +6,7 @@ module API include APIGuard helpers ::API::Helpers::EventsHelpers - allow_access_with_scope :read_user, if: -> (request) { request.get? } + allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } feature_category :users diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 167531fdaec..24d726f4a41 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -7,6 +7,8 @@ module API file_name: API::NO_SLASH_URL_PART_REGEX }.freeze + ALLOWED_STATUSES = %w[default hidden].freeze + feature_category :package_registry before do @@ -35,6 +37,7 @@ module API requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + optional :status, type: String, values: ALLOWED_STATUSES, desc: 'Package status' end put 'authorize' do @@ -49,6 +52,7 @@ module API requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true + optional :status, type: String, values: ALLOWED_STATUSES, desc: 'Package status' requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' end diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb index 2d978019f2a..ea30f17522e 100755 --- a/lib/api/go_proxy.rb +++ b/lib/api/go_proxy.rb @@ -30,20 +30,6 @@ module API str.gsub(/![[:alpha:]]/) { |s| s[1..].upcase } end - def find_project!(id) - # based on API::Helpers::Packages::BasicAuthHelpers#authorized_project_find! - - project = find_project(id) - - return project if project && can?(current_user, :read_project, project) - - if current_user - not_found!('Project') - else - unauthorized! - end - end - def find_module not_found! unless Feature.enabled?(:go_proxy, user_project) @@ -74,7 +60,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :module_name, type: String, desc: 'Module name', coerce_with: ->(val) { CGI.unescape(val) } end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, authenticate_non_public: true resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do before do authorize_read_package! diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb index 7fbf4445116..bea538441ee 100644 --- a/lib/api/group_labels.rb +++ b/lib/api/group_labels.rb @@ -12,7 +12,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :groups, requirements: ::API::Labels::LABEL_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the group' do detail 'This feature was added in GitLab 11.8' success Entities::GroupLabel diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb index d482f4d0585..ab4e91ff925 100644 --- a/lib/api/group_packages.rb +++ b/lib/api/group_packages.rb @@ -33,12 +33,14 @@ module API desc: 'Return packages with this name' optional :include_versionless, type: Boolean, desc: 'Returns packages without a version' + optional :status, type: String, values: Packages::Package.statuses.keys, + desc: 'Return packages with specified status' end get ':id/packages' do packages = Packages::GroupPackagesFinder.new( current_user, user_group, - declared(params).slice(:exclude_subgroups, :order_by, :sort, :package_type, :package_name, :include_versionless) + declared(params).slice(:exclude_subgroups, :order_by, :sort, :package_type, :package_name, :include_versionless, :status) ).execute present paginate(packages), with: ::API::Entities::Package, user: current_user, group: true diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 79af9c37378..0abb21c9831 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -119,11 +119,10 @@ module API def find_project!(id) project = find_project(id) - if can?(current_user, :read_project, project) - project - else - not_found!('Project') - end + return project if can?(current_user, :read_project, project) + return unauthorized! if authenticate_non_public? + + not_found!('Project') end # rubocop: disable CodeReuse/ActiveRecord @@ -139,11 +138,10 @@ module API def find_group!(id) group = find_group(id) - if can?(current_user, :read_group, group) - group - else - not_found!('Group') - end + return group if can?(current_user, :read_group, group) + return unauthorized! if authenticate_non_public? + + not_found!('Group') end def check_namespace_access(namespace) @@ -657,6 +655,10 @@ module API Gitlab::Shell.secret_token end + def authenticate_non_public? + route_authentication_setting[:authenticate_non_public] && !current_user + end + def send_git_blob(repository, blob) env['api.format'] = :txt content_type 'text/plain' diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 12b0a053e79..9a1ff2ba8ce 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -124,7 +124,7 @@ module API repository: repository.gitaly_repository.to_h, address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags + features: Feature::Gitaly.server_feature_flags(repository.project) } end end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 8aed578905e..2de077b5a3b 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -# rubocop:disable GitlabSecurity/PublicSend - module API module Helpers module MembersHelpers diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb index 6798c4d284b..71a18524104 100644 --- a/lib/api/helpers/notes_helpers.rb +++ b/lib/api/helpers/notes_helpers.rb @@ -138,7 +138,7 @@ module API parent = noteable_parent(noteable) ::Discussions::ResolveService.new(parent, current_user, one_or_more_discussions: discussion).execute else - discussion.unresolve! + ::Discussions::UnresolveService.new(discussion, current_user).execute end present discussion, with: Entities::Discussion diff --git a/lib/api/helpers/rate_limiter.rb b/lib/api/helpers/rate_limiter.rb index 5a531b5324a..3a16aef6a74 100644 --- a/lib/api/helpers/rate_limiter.rb +++ b/lib/api/helpers/rate_limiter.rb @@ -3,8 +3,8 @@ module API module Helpers module RateLimiter - def check_rate_limit!(key, scope) - if rate_limiter.throttled?(key, scope: scope) + def check_rate_limit!(key, scope, users_allowlist = nil) + if rate_limiter.throttled?(key, scope: scope, users_allowlist: users_allowlist) log_request(key) render_exceeded_limit_error! end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 12bb6e77c3e..a3fee49cd8f 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -52,7 +52,9 @@ module API actor.update_last_used_at! check_result = begin - access_check!(actor, params) + Gitlab::Auth::CurrentUserMode.bypass_session!(actor.user&.id) do + access_check!(actor, params) + end rescue Gitlab::GitAccess::ForbiddenError => e # The return code needs to be 401. If we return 403 # the custom message we return won't be shown to the user @@ -114,6 +116,10 @@ module API 'Could not find a user for the given key' unless actor.user end + + def two_factor_otp_check + { success: false, message: 'Feature is not available' } + end end namespace 'internal' do @@ -276,6 +282,11 @@ module API present response, with: Entities::InternalPostReceive::Response end + # This endpoint was added in https://gitlab.com/gitlab-org/gitlab/-/issues/212308 + # It was added with the plan to be used by GitLab PAM module but we + # decided to pursue a different approach, so it's currently not used. + # We might revive the PAM module though as it provides better user + # flow. post '/two_factor_config', feature_category: :authentication_and_authorization do status 200 @@ -301,28 +312,7 @@ module API post '/two_factor_otp_check', feature_category: :authentication_and_authorization do status 200 - break { success: false, message: 'Feature flag is disabled' } unless Feature.enabled?(:two_factor_for_cli) - - actor.update_last_used_at! - user = actor.user - - error_message = validate_actor_key(actor, params[:key_id]) - - break { success: false, message: error_message } if error_message - - break { success: false, message: 'Deploy keys cannot be used for Two Factor' } if actor.key.is_a?(DeployKey) - - break { success: false, message: 'Two-factor authentication is not enabled for this user' } unless user.two_factor_enabled? - - otp_validation_result = ::Users::ValidateOtpService.new(user).execute(params.fetch(:otp_attempt)) - - if otp_validation_result[:status] == :success - ::Gitlab::Auth::Otp::SessionEnforcer.new(actor.key).update_session - - { success: true } - else - { success: false, message: 'Invalid OTP' } - end + two_factor_otp_check end end end diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index 73723a96401..87ad79d601f 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -52,6 +52,8 @@ module API def check_agent_token forbidden! unless agent_token + + forbidden! unless Gitlab::Kas.included_in_gitlab_com_rollout?(agent.project) end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 73e2163248d..ea09174f03a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -231,7 +231,7 @@ module API post ':id/issues' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42320') - check_rate_limit! :issues_create, [current_user, :issues_create] + check_rate_limit! :issues_create, [current_user] authorize! :create_issue, user_project diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index e14a4a5e680..390dbc892e2 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -45,7 +45,7 @@ module API builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, pipeline: :project) + builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project) present paginate(builds), with: Entities::Ci::Job end # rubocop: enable CodeReuse/ActiveRecord @@ -82,7 +82,8 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace.raw + # The trace can be nil bu body method expects a string as an argument. + trace = build.trace.raw || '' body trace end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c9f29865664..aa3746dae42 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -9,10 +9,14 @@ module API feature_category :issue_tracking + LABEL_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + name: API::NO_SLASH_URL_PART_REGEX, + label_id: API::NO_SLASH_URL_PART_REGEX) + params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource :projects, requirements: LABEL_ENDPOINT_REQUIREMENTS do desc 'Get all labels of the project' do success Entities::ProjectLabel end diff --git a/lib/api/members.rb b/lib/api/members.rb index 9bea74e2ce9..42f608102b3 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -137,12 +137,14 @@ module API authorize_admin_source!(source_type, source) member = source_members(source).find_by!(user_id: params[:user_id]) - updated_member = - ::Members::UpdateService - .new(current_user, declared_params(include_missing: false)) - .execute(member) - if updated_member.valid? + result = ::Members::UpdateService + .new(current_user, declared_params(include_missing: false)) + .execute(member) + + updated_member = result[:member] + + if result[:status] == :success present_members updated_member else render_validation_error!(updated_member) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 142ecd0dc1e..5051c1a5529 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -26,6 +26,7 @@ module API %i[ assignee_id assignee_ids + reviewer_ids description labels add_labels @@ -160,7 +161,8 @@ module API helpers do params :optional_params do optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' - optional :assignee_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The array of user IDs to assign issue' + optional :assignee_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Comma-separated list of assignee ids' + optional :reviewer_ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Comma-separated list of reviewer ids' optional :description, type: String, desc: 'The description of the merge request' optional :labels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' optional :add_labels, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' @@ -368,7 +370,7 @@ module API with: Entities::MergeRequestChanges, current_user: current_user, project: user_project, - access_raw_diffs: params.fetch(:access_raw_diffs, false) + access_raw_diffs: to_boolean(params.fetch(:access_raw_diffs, false)) end desc 'Get the merge request pipelines' do diff --git a/lib/api/notes.rb b/lib/api/notes.rb index d249431b2f8..418efe3d1a7 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -4,6 +4,7 @@ module API class Notes < ::API::Base include PaginationParams helpers ::API::Helpers::NotesHelpers + helpers Helpers::RateLimiter before { authenticate! } @@ -72,6 +73,9 @@ module API optional :created_at, type: String, desc: 'The creation date of the note' end post ":id/#{noteables_str}/:noteable_id/notes", feature_category: feature_category do + allowlist = + Gitlab::CurrentSettings.current_application_settings.notes_create_limit_allowlist + check_rate_limit! :notes_create, [current_user], allowlist noteable = find_noteable(noteable_type, params[:noteable_id]) opts = { diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index e373f051b24..a80de06d6b0 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -18,7 +18,7 @@ module API default_format :json authenticate_with do |accept| - accept.token_types(:personal_access_token, :deploy_token, :job_token) + accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) .sent_through(:http_basic_auth) end diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 2146f4d4b78..e071b6bd68f 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -20,7 +20,7 @@ module API default_format :json authenticate_with do |accept| - accept.token_types(:personal_access_token, :deploy_token, :job_token) + accept.token_types(:personal_access_token_with_username, :deploy_token_with_username, :job_token_with_username) .sent_through(:http_basic_auth) end diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 32636662987..0fdaa4b2656 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -32,11 +32,13 @@ module API desc: 'Return packages with this name' optional :include_versionless, type: Boolean, desc: 'Returns packages without a version' + optional :status, type: String, values: Packages::Package.statuses.keys, + desc: 'Return packages with specified status' end get ':id/packages' do packages = ::Packages::PackagesFinder.new( user_project, - declared_params.slice(:order_by, :sort, :package_type, :package_name, :include_versionless) + declared_params.slice(:order_by, :sort, :package_type, :package_name, :include_versionless, :status) ).execute present paginate(packages), with: ::API::Entities::Package, user: current_user diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index 19244ed697f..fdfdc244cbe 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -26,9 +26,7 @@ module API use :pagination end get ':id/templates/:type' do - templates = TemplateFinder - .build(params[:type], user_project) - .execute + templates = TemplateFinder.all_template_names_array(user_project, params[:type]) present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList end @@ -38,16 +36,22 @@ module API end params do requires :name, type: String, desc: 'The name of the template' - + optional :source_template_project_id, type: Integer, + desc: 'The project id where a given template is being stored. This is useful when multiple templates from different projects have the same name' optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses' optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses' end get ':id/templates/:type/:name', requirements: TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS do begin - template = TemplateFinder - .build(params[:type], user_project, name: params[:name]) - .execute + template = TemplateFinder.build( + params[:type], + user_project, + { + name: params[:name], + source_template_project_id: params[:source_template_project_id] + } + ).execute rescue ::Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError not_found!('Template') end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 2d09ad01757..fca68c3606b 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -295,6 +295,8 @@ module API optional :namespace_path, type: String, desc: 'The path of the namespace that the project will be forked into' optional :path, type: String, desc: 'The path that will be assigned to the fork' optional :name, type: String, desc: 'The name that will be assigned to the fork' + optional :description, type: String, desc: 'The description that will be assigned to the fork' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork' end post ':id/fork', feature_category: :source_code_management do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284') diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 8af8ffc3b63..eaedd53aedb 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -170,6 +170,65 @@ module API not_found!("Merge Base") end end + + desc 'Generates a changelog section for a release' do + detail 'This feature was introduced in GitLab 13.9' + end + params do + requires :version, + type: String, + regexp: Gitlab::Regex.unbounded_semver_regex, + desc: 'The version of the release, using the semantic versioning format' + + optional :from, + type: String, + desc: 'The first commit in the range of commits to use for the changelog' + + requires :to, + type: String, + desc: 'The last commit in the range of commits to use for the changelog' + + optional :date, + type: DateTime, + desc: 'The date and time of the release' + + optional :branch, + type: String, + desc: 'The branch to commit the changelog changes to' + + optional :trailer, + type: String, + desc: 'The Git trailer to use for determining if commits are to be included in the changelog', + default: ::Repositories::ChangelogService::DEFAULT_TRAILER + + optional :file, + type: String, + desc: 'The file to commit the changelog changes to', + default: ::Repositories::ChangelogService::DEFAULT_FILE + + optional :message, + type: String, + desc: 'The commit message to use when committing the changelog' + end + post ':id/repository/changelog' do + branch = params[:branch] || user_project.default_branch_or_master + access = Gitlab::UserAccess.new(current_user, container: user_project) + + unless access.can_push_to_branch?(branch) + forbidden!("You are not allowed to commit a changelog on this branch") + end + + service = ::Repositories::ChangelogService.new( + user_project, + current_user, + **declared_params(include_missing: false) + ) + + service.execute + status(200) + rescue Gitlab::Changelog::Error => ex + render_api_error!("Failed to generate the changelog: #{ex.message}", 422) + end end end end diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb new file mode 100644 index 00000000000..66948f9eaf3 --- /dev/null +++ b/lib/api/resource_access_tokens.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module API + class ResourceAccessTokens < ::API::Base + include PaginationParams + + before { authenticate! } + + feature_category :authentication_and_authorization + + %w[project].each do |source_type| + resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get list of all access tokens for the specified resource' do + detail 'This feature was introduced in GitLab 13.9.' + end + params do + requires :id, type: String, desc: "The #{source_type} ID" + end + get ":id/access_tokens" do + resource = find_source(source_type, params[:id]) + + next unauthorized! unless has_permission_to_read?(resource) + + tokens = PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).execute + + present paginate(tokens), with: Entities::PersonalAccessToken + end + + desc 'Revoke a resource access token' do + detail 'This feature was introduced in GitLab 13.9.' + end + params do + requires :id, type: String, desc: "The #{source_type} ID" + requires :token_id, type: String, desc: "The ID of the token" + end + delete ':id/access_tokens/:token_id' do + resource = find_source(source_type, params[:id]) + token = find_token(resource, params[:token_id]) + + if token.nil? + next not_found!("Could not find #{source_type} access token with token_id: #{params[:token_id]}") + end + + service = ::ResourceAccessTokens::RevokeService.new( + current_user, + resource, + token + ).execute + + service.success? ? no_content! : bad_request!(service.message) + end + + desc 'Create a resource access token' do + detail 'This feature was introduced in GitLab 13.9.' + end + params do + requires :id, type: String, desc: "The #{source_type} ID" + requires :name, type: String, desc: "Resource access token name" + requires :scopes, type: Array[String], desc: "The permissions of the token" + optional :expires_at, type: Date, desc: "The expiration date of the token" + end + post ':id/access_tokens' do + resource = find_source(source_type, params[:id]) + + token_response = ::ResourceAccessTokens::CreateService.new( + current_user, + resource, + declared_params + ).execute + + if token_response.success? + present token_response.payload[:access_token], with: Entities::PersonalAccessToken + else + bad_request!(token_response.message) + end + end + end + end + + helpers do + def find_source(source_type, id) + public_send("find_#{source_type}!", id) # rubocop:disable GitlabSecurity/PublicSend + end + + def find_token(resource, token_id) + PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).find_by_id(token_id) + end + + def has_permission_to_read?(resource) + can?(current_user, :project_bot_access, resource) || can?(current_user, :admin_resource_access_tokens, resource) + end + end + end +end diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb new file mode 100644 index 00000000000..7819aab879c --- /dev/null +++ b/lib/api/rubygem_packages.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +### +# API endpoints for the RubyGem package registry +module API + class RubygemPackages < ::API::Base + include ::API::Helpers::Authentication + helpers ::API::Helpers::PackagesHelpers + + feature_category :package_registry + + # The Marshal version can be found by "#{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}" + # Updating the version should require a GitLab API version change. + MARSHAL_VERSION = '4.8' + + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + content_type :binary, 'application/octet-stream' + + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_token) + end + + before do + require_packages_enabled! + authenticate! + not_found! unless Feature.enabled?(:rubygem_packages, user_project) + end + + params do + requires :id, type: String, desc: 'The ID or full path of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/rubygems' do + desc 'Download the spec index file' do + detail 'This feature was introduced in GitLab 13.9' + end + params do + requires :file_name, type: String, desc: 'Spec file name' + end + get ":file_name", requirements: FILE_NAME_REQUIREMENTS do + # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299267 + not_found! + end + + desc 'Download the gemspec file' do + detail 'This feature was introduced in GitLab 13.9' + end + params do + requires :file_name, type: String, desc: 'Gemspec file name' + end + get "quick/Marshal.#{MARSHAL_VERSION}/:file_name", requirements: FILE_NAME_REQUIREMENTS do + # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299284 + not_found! + end + + desc 'Download the .gem package' do + detail 'This feature was introduced in GitLab 13.9' + end + params do + requires :file_name, type: String, desc: 'Package file name' + end + get "gems/:file_name", requirements: FILE_NAME_REQUIREMENTS do + # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299283 + not_found! + end + + namespace 'api/v1' do + desc 'Authorize a gem upload from workhorse' do + detail 'This feature was introduced in GitLab 13.9' + end + post 'gems/authorize' do + # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263 + not_found! + end + + desc 'Upload a gem' do + detail 'This feature was introduced in GitLab 13.9' + end + post 'gems' do + # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263 + not_found! + end + + desc 'Fetch a list of dependencies' do + detail 'This feature was introduced in GitLab 13.9' + end + params do + optional :gems, type: String, desc: 'Comma delimited gem names' + end + get 'dependencies' do + # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299282 + not_found! + end + end + end + end + end +end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index f329a94adf2..e7ee8b08d87 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -10,8 +10,7 @@ module API helpers do def current_settings - @current_setting ||= - (ApplicationSetting.current_without_cache || ApplicationSetting.create_from_defaults) + @current_setting ||= ApplicationSetting.find_or_create_without_cache end def filter_attributes_using_license(attrs) @@ -42,7 +41,8 @@ module API optional :asset_proxy_enabled, type: Boolean, desc: 'Enable proxying of assets' optional :asset_proxy_url, type: String, desc: 'URL of the asset proxy server' optional :asset_proxy_secret_key, type: String, desc: 'Shared secret with the asset proxy server' - optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' + optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :asset_proxy_allowlist instead. Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' + optional :asset_proxy_allowlist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed.' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_ci_config_path, type: String, desc: 'The instance default CI configuration path for new projects' @@ -211,6 +211,11 @@ module API attrs[:abuse_notification_email] = attrs.delete(:admin_notification_email) end + # support legacy names, can be removed in v5 + if attrs.has_key?(:asset_proxy_whitelist) + attrs[:asset_proxy_allowlist] = attrs.delete(:asset_proxy_whitelist) + end + # since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0 attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled) diff --git a/lib/api/snippet_repository_storage_moves.rb b/lib/api/snippet_repository_storage_moves.rb index 1a5b41eb1ec..84dbc03ba33 100644 --- a/lib/api/snippet_repository_storage_moves.rb +++ b/lib/api/snippet_repository_storage_moves.rb @@ -58,9 +58,14 @@ module API resource :snippets do helpers do def user_snippet - Snippet.find_by(id: params[:id]) # rubocop: disable CodeReuse/ActiveRecord + @user_snippet ||= Snippet.find_by(id: params[:id]) # rubocop: disable CodeReuse/ActiveRecord end end + + before do + not_found!('Snippet') unless user_snippet + end + desc 'Get a list of all snippets repository storage moves' do detail 'This feature was introduced in GitLab 13.8.' success Entities::SnippetRepositoryStorageMove diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 914bab52929..87dc1358a51 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -6,6 +6,9 @@ module API before { authenticate! } + SUBSCRIBE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge( + subscribable_id: API::NO_SLASH_URL_PART_REGEX) + subscribables = [ { type: 'merge_requests', @@ -44,7 +47,7 @@ module API requires :id, type: String, desc: "The #{source_type} ID" requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource source_type.pluralize, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + resource source_type.pluralize, requirements: SUBSCRIBE_ENDPOINT_REQUIREMENTS do desc 'Subscribe to a resource' do success subscribable[:entity] end diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb index a024d6de874..7921700e365 100644 --- a/lib/api/suggestions.rb +++ b/lib/api/suggestions.rb @@ -12,12 +12,13 @@ module API end params do requires :id, type: String, desc: 'The suggestion ID' + optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end put ':id/apply' do suggestion = Suggestion.find_by_id(params[:id]) if suggestion - apply_suggestions(suggestion, current_user) + apply_suggestions(suggestion, current_user, params[:commit_message]) else render_api_error!(_('Suggestion is not applicable as the suggestion was not found.'), :not_found) end @@ -28,6 +29,7 @@ module API end params do requires :ids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: "An array of suggestion ID's" + optional :commit_message, type: String, desc: "A custom commit message to use instead of the default generated message or the project's default message" end put 'batch_apply' do ids = params[:ids] @@ -35,7 +37,7 @@ module API suggestions = Suggestion.id_in(ids) if suggestions.size == ids.length - apply_suggestions(suggestions, current_user) + apply_suggestions(suggestions, current_user, params[:commit_message]) else render_api_error!(_('Suggestions are not applicable as one or more suggestions were not found.'), :not_found) end @@ -43,10 +45,10 @@ module API end helpers do - def apply_suggestions(suggestions, current_user) + def apply_suggestions(suggestions, current_user, message) authorize_suggestions(*suggestions) - result = ::Suggestions::ApplyService.new(current_user, *suggestions).execute + result = ::Suggestions::ApplyService.new(current_user, *suggestions, message: message).execute if result[:status] == :success present suggestions, with: Entities::Suggestion, current_user: current_user diff --git a/lib/api/support/git_access_actor.rb b/lib/api/support/git_access_actor.rb index 39730da1251..71395086ac2 100644 --- a/lib/api/support/git_access_actor.rb +++ b/lib/api/support/git_access_actor.rb @@ -37,6 +37,8 @@ module API end def update_last_used_at! + return if Feature.enabled?(:disable_ssh_key_used_tracking) + key&.update_last_used_at end diff --git a/lib/api/users.rb b/lib/api/users.rb index cee09f60a2b..f91e3c34ef2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -6,7 +6,7 @@ module API include APIGuard include Helpers::CustomAttributes - allow_access_with_scope :read_user, if: -> (request) { request.get? } + allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } feature_category :users, ['/users/:id/custom_attributes', '/users/:id/custom_attributes/:key'] @@ -159,6 +159,68 @@ module API present user.status || {}, with: Entities::UserStatus end + desc 'Follow a user' do + success Entities::User + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/follow', feature_category: :users do + user = find_user(params[:id]) + not_found!('User') unless user + + if current_user.follow(user) + present user, with: Entities::UserBasic + else + not_modified! + end + end + + desc 'Unfollow a user' do + success Entities::User + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/unfollow', feature_category: :users do + user = find_user(params[:id]) + not_found!('User') unless user + + if current_user.unfollow(user) + present user, with: Entities::UserBasic + else + not_modified! + end + end + + desc 'Get the users who follow a user' do + success Entities::UserBasic + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/following', feature_category: :users do + user = find_user(params[:id]) + not_found!('User') unless user && can?(current_user, :read_user_profile, user) + + present paginate(user.followees), with: Entities::UserBasic + end + + desc 'Get the followers of a user' do + success Entities::UserBasic + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/followers', feature_category: :users do + user = find_user(params[:id]) + not_found!('User') unless user && can?(current_user, :read_user_profile, user) + + present paginate(user.followers), with: Entities::UserBasic + end + desc 'Create a user. Available only for admins.' do success Entities::UserWithAdmin end @@ -1004,11 +1066,15 @@ module API optional :emoji, type: String, desc: "The emoji to set on the status" optional :message, type: String, desc: "The status message to set" optional :availability, type: String, desc: "The availability of user to set" + optional :clear_status_after, type: String, desc: "Automatically clear emoji, message and availability fields after a certain time", values: UserStatus::CLEAR_STATUS_QUICK_OPTIONS.keys end put "status", feature_category: :users do forbidden! unless can?(current_user, :update_user_status, current_user) - if ::Users::SetStatusService.new(current_user, declared_params).execute + update_params = declared_params + update_params.delete(:clear_status_after) if Feature.disabled?(:clear_status_with_quick_options, current_user, default_enabled: :yaml) + + if ::Users::SetStatusService.new(current_user, update_params).execute present current_user.status, with: Entities::UserStatus else render_validation_error!(current_user.status) diff --git a/lib/api/version.rb b/lib/api/version.rb index f8072658cc6..86eb34ca589 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -5,7 +5,7 @@ module API helpers ::API::Helpers::GraphqlHelpers include APIGuard - allow_access_with_scope :read_user, if: -> (request) { request.get? } + allow_access_with_scope :read_user, if: -> (request) { request.get? || request.head? } before { authenticate! } diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index c67fe24d456..6f87b7b7d3c 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -4,7 +4,7 @@ module Atlassian module JiraConnect class Client < Gitlab::HTTP def self.generate_update_sequence_id - Gitlab::Metrics::System.monotonic_time.to_i + (Time.now.utc.to_f * 1000).round end def initialize(base_uri, shared_secret) @@ -33,8 +33,6 @@ module Atlassian private def store_ff_info(project:, feature_flags:, **opts) - return unless Feature.enabled?(:jira_sync_feature_flags, project) - items = feature_flags.map { |flag| ::Atlassian::JiraConnect::Serializers::FeatureFlagEntity.represent(flag, opts) } items.reject! { |item| item.issue_keys.empty? } @@ -57,8 +55,6 @@ module Atlassian end def store_deploy_info(project:, deployments:, **opts) - return unless Feature.enabled?(:jira_sync_deployments, project) - items = deployments.map { |d| ::Atlassian::JiraConnect::Serializers::DeploymentEntity.represent(d, opts) } items.reject! { |d| d.issue_keys.empty? } @@ -69,8 +65,6 @@ module Atlassian end def store_build_info(project:, pipelines:, update_sequence_id: nil) - return unless Feature.enabled?(:jira_sync_builds, project) - builds = pipelines.map do |pipeline| build = ::Atlassian::JiraConnect::Serializers::BuildEntity.represent( pipeline, diff --git a/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb b/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb index e17c150aacb..3193d5bbd1e 100644 --- a/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb +++ b/lib/atlassian/jira_connect/serializers/feature_flag_entity.rb @@ -50,7 +50,7 @@ module Atlassian # edit path as an interim solution. def summary(strategies = flag.strategies) { - url: project_url(flag.project) + "/-/feature_flags/#{flag.id}/edit", + url: edit_project_feature_flag_url(flag.project, flag), lastUpdated: flag.updated_at.iso8601, status: { enabled: flag.active, diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 0f6ed847dea..42cfff98239 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -137,7 +137,7 @@ module Backup if s == DEFAULT_EXCLUDE '--exclude=' + s elsif fmt == :rsync - '--exclude=/' + s + '--exclude=/' + File.join(File.basename(app_files_dir), s) elsif fmt == :tar '--exclude=./' + s end diff --git a/lib/banzai/filter/asset_proxy_filter.rb b/lib/banzai/filter/asset_proxy_filter.rb index 55dc426edaf..4c14ee7299b 100644 --- a/lib/banzai/filter/asset_proxy_filter.rb +++ b/lib/banzai/filter/asset_proxy_filter.rb @@ -59,7 +59,9 @@ module Banzai end def self.determine_allowlist(application_settings) - application_settings.asset_proxy_whitelist.presence || [Gitlab.config.gitlab.host] + application_settings.try(:asset_proxy_allowlist).presence || + application_settings.try(:asset_proxy_whitelist).presence || + [Gitlab.config.gitlab.host] end end end diff --git a/lib/banzai/filter/custom_emoji_filter.rb b/lib/banzai/filter/custom_emoji_filter.rb new file mode 100644 index 00000000000..1ee8f4e31e8 --- /dev/null +++ b/lib/banzai/filter/custom_emoji_filter.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class CustomEmojiFilter < HTML::Pipeline::Filter + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + + def call + return doc unless context[:project] + return doc unless Feature.enabled?(:custom_emoji, context[:project]) + + doc.search(".//text()").each do |node| + content = node.to_html + + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) + next unless content.include?(':') + next unless namespace && namespace.custom_emoji.any? + + html = custom_emoji_name_element_filter(content) + + node.replace(html) unless html == content + end + + doc + end + + def custom_emoji_pattern + @emoji_pattern ||= + /(?<=[^[:alnum:]:]|\n|^) + :(#{CustomEmoji::NAME_REGEXP}): + (?=[^[:alnum:]:]|$)/x + end + + def custom_emoji_name_element_filter(text) + text.gsub(custom_emoji_pattern) do |match| + name = Regexp.last_match[1] + custom_emoji = all_custom_emoji[name] + + if custom_emoji + Gitlab::Emoji.custom_emoji_tag(custom_emoji.name, custom_emoji.url) + else + match + end + end + end + + private + + def namespace + context[:project].namespace.root_ancestor + end + + def custom_emoji_candidates + doc.to_html.scan(/:(#{CustomEmoji::NAME_REGEXP}):/).flatten + end + + def all_custom_emoji + @all_custom_emoji ||= namespace.custom_emoji.by_name(custom_emoji_candidates).index_by(&:name) + end + end + end +end diff --git a/lib/banzai/filter/feature_flag_reference_filter.rb b/lib/banzai/filter/feature_flag_reference_filter.rb new file mode 100644 index 00000000000..c11576901ce --- /dev/null +++ b/lib/banzai/filter/feature_flag_reference_filter.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class FeatureFlagReferenceFilter < IssuableReferenceFilter + self.reference_type = :feature_flag + + def self.object_class + Operations::FeatureFlag + end + + def self.object_sym + :feature_flag + end + + def parent_records(parent, ids) + parent.operations_feature_flags.where(iid: ids.to_a) + end + + def url_for_object(feature_flag, project) + ::Gitlab::Routing.url_helpers.edit_project_feature_flag_url( + project, + feature_flag.iid, + only_path: context[:only_path] + ) + end + + def object_link_title(object, matches) + object.name + end + end + end +end diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb new file mode 100644 index 00000000000..ad32e9afbf5 --- /dev/null +++ b/lib/banzai/filter/markdown_post_escape_filter.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Banzai + module Filter + class MarkdownPostEscapeFilter < HTML::Pipeline::Filter + LITERAL_KEYWORD = MarkdownPreEscapeFilter::LITERAL_KEYWORD + LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-(.*?)-#{LITERAL_KEYWORD}}.freeze + NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze + SPAN_REGEX = %r{<span>(.*?)</span>}.freeze + + def call + return doc unless result[:escaped_literals] + + # For any literals that actually didn't get escape processed + # (for example in code blocks), remove the special sequence. + html.gsub!(NOT_LITERAL_REGEX, '\1') + + # Replace any left over literal sequences with `span` so that our + # reference processing is short-circuited + html.gsub!(LITERAL_REGEX, '<span>\1</span>') + + # Since literals are converted in links, we need to remove any surrounding `span`. + # Note: this could have been done in the renderer, + # Banzai::Renderer::CommonMark::HTML. However, we eventually want to use + # the built-in compiled renderer, rather than the ruby version, for speed. + # So let's do this work here. + doc.css('a').each do |node| + node.attributes['href'].value = node.attributes['href'].value.gsub(SPAN_REGEX, '\1') if node.attributes['href'] + node.attributes['title'].value = node.attributes['title'].value.gsub(SPAN_REGEX, '\1') if node.attributes['title'] + end + + doc.css('code').each do |node| + node.attributes['lang'].value = node.attributes['lang'].value.gsub(SPAN_REGEX, '\1') if node.attributes['lang'] + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb new file mode 100644 index 00000000000..9fd77c48659 --- /dev/null +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # In order to allow a user to short-circuit our reference shortcuts + # (such as # or !), the user should be able to escape them, like \#. + # CommonMark supports this, however it removes all information about + # what was actually a literal. In order to short-circuit the reference, + # we must surround backslash escaped ASCII punctuation with a custom sequence. + # This way CommonMark will properly handle the backslash escaped chars + # but we will maintain knowledge (the sequence) that it was a literal. + # + # We need to surround the character, not just prefix it. It could + # get converted into an entity by CommonMark and we wouldn't know how many + # characters there are. The entire literal needs to be surrounded with + # a `span` tag, which short-circuits our reference processing. + # + # We can't use a custom HTML tag since we could be initially surrounding + # text in an href, and then CommonMark will not be able to parse links + # properly. So we use `cmliteral-` and `-cmliteral` + # + # https://spec.commonmark.org/0.29/#backslash-escapes + # + # This filter does the initial surrounding, and MarkdownPostEscapeFilter + # does the conversion into span tags. + class MarkdownPreEscapeFilter < HTML::Pipeline::TextFilter + ASCII_PUNCTUATION = %r{([\\][!"#$%&'()*+,-./:;<=>?@\[\\\]^_`{|}~])}.freeze + LITERAL_KEYWORD = 'cmliteral' + + def call + return @text unless Feature.enabled?(:honor_escaped_markdown, context[:group] || context[:project]&.group) + + @text.gsub(ASCII_PUNCTUATION) do |match| + # The majority of markdown does not have literals. If none + # are found, we can bypass the post filter + result[:escaped_literals] = true + + "#{LITERAL_KEYWORD}-#{match}-#{LITERAL_KEYWORD}" + end + end + end + end +end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index 1a75cd14b11..37d4126c1ba 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -9,7 +9,7 @@ module Banzai # class PlantumlFilter < HTML::Pipeline::Filter def call - return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled + return doc unless settings.plantuml_enabled? && doc.at('pre > code[lang="plantuml"]') plantuml_setup diff --git a/lib/banzai/filter/truncate_source_filter.rb b/lib/banzai/filter/truncate_source_filter.rb index c903b83d868..44f88b253d9 100644 --- a/lib/banzai/filter/truncate_source_filter.rb +++ b/lib/banzai/filter/truncate_source_filter.rb @@ -6,7 +6,9 @@ module Banzai def call return text unless context.key?(:limit) - text.truncate_bytes(context[:limit]) + # Use three dots instead of the ellipsis Unicode character because + # some clients show the raw Unicode value in the merge commit. + text.truncate_bytes(context[:limit], omission: '...') end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 344afc9b33c..e5ec0a0a006 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -34,6 +34,7 @@ module Banzai Filter::FootnoteFilter, *reference_filters, Filter::EmojiFilter, + Filter::CustomEmojiFilter, Filter::TaskListFilter, Filter::InlineDiffFilter, Filter::SetDirectionFilter @@ -61,7 +62,8 @@ module Banzai Filter::CommitReferenceFilter, Filter::LabelReferenceFilter, Filter::MilestoneReferenceFilter, - Filter::AlertReferenceFilter + Filter::AlertReferenceFilter, + Filter::FeatureFlagReferenceFilter ] end diff --git a/lib/banzai/pipeline/plain_markdown_pipeline.rb b/lib/banzai/pipeline/plain_markdown_pipeline.rb index b64f13cde47..1da0f72996b 100644 --- a/lib/banzai/pipeline/plain_markdown_pipeline.rb +++ b/lib/banzai/pipeline/plain_markdown_pipeline.rb @@ -5,7 +5,9 @@ module Banzai class PlainMarkdownPipeline < BasePipeline def self.filters FilterArray[ - Filter::MarkdownFilter + Filter::MarkdownPreEscapeFilter, + Filter::MarkdownFilter, + Filter::MarkdownPostEscapeFilter ] end end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index a2fe6d52a90..4bf98099662 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -24,7 +24,8 @@ module Banzai Filter::SnippetReferenceFilter, Filter::CommitRangeReferenceFilter, Filter::CommitReferenceFilter, - Filter::AlertReferenceFilter + Filter::AlertReferenceFilter, + Filter::FeatureFlagReferenceFilter ] end diff --git a/lib/banzai/reference_parser/feature_flag_parser.rb b/lib/banzai/reference_parser/feature_flag_parser.rb new file mode 100644 index 00000000000..6092b0f7e66 --- /dev/null +++ b/lib/banzai/reference_parser/feature_flag_parser.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Banzai + module ReferenceParser + class FeatureFlagParser < BaseParser + self.reference_type = :feature_flag + + def references_relation + Operations::FeatureFlag + end + + private + + def can_read_reference?(user, feature_flag, node) + can?(user, :read_feature_flag, feature_flag) + end + end + end +end diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb index af274ee1299..cde3d1cad5b 100644 --- a/lib/bulk_imports/common/extractors/graphql_extractor.rb +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -4,17 +4,22 @@ module BulkImports module Common module Extractors class GraphqlExtractor - def initialize(query) - @query = query[:query] + def initialize(options = {}) + @query = options[:query] end def extract(context) client = graphql_client(context) - client.execute( + response = client.execute( client.parse(query.to_s), - query.variables(context.entity) + query.variables(context) ).original_hash.deep_dup + + BulkImports::Pipeline::ExtractedData.new( + data: response.dig(*query.data_path), + page_info: response.dig(*query.page_info_path) + ) end private @@ -27,10 +32,6 @@ module BulkImports token: context.configuration.access_token ) end - - def parsed_query - @parsed_query ||= graphql_client.parse(query.to_s) - end end end end diff --git a/lib/bulk_imports/common/loaders/entity_loader.rb b/lib/bulk_imports/common/loaders/entity_loader.rb index 4540b892c88..8644f3c9dcb 100644 --- a/lib/bulk_imports/common/loaders/entity_loader.rb +++ b/lib/bulk_imports/common/loaders/entity_loader.rb @@ -7,7 +7,7 @@ module BulkImports def initialize(*args); end def load(context, entity) - context.entity.bulk_import.entities.create!(entity) + context.bulk_import.entities.create!(entity) end end end diff --git a/lib/bulk_imports/common/transformers/award_emoji_transformer.rb b/lib/bulk_imports/common/transformers/award_emoji_transformer.rb new file mode 100644 index 00000000000..260b47ab917 --- /dev/null +++ b/lib/bulk_imports/common/transformers/award_emoji_transformer.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Transformers + class AwardEmojiTransformer + def initialize(*args); end + + def transform(context, data) + user = find_user(context, data&.dig('user', 'public_email')) || context.current_user + + data + .except('user') + .merge('user_id' => user.id) + end + + private + + def find_user(context, email) + return if email.blank? + + context.group.users.find_by_any_email(email, confirmed: true) # rubocop: disable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/bulk_imports/common/transformers/hash_key_digger.rb b/lib/bulk_imports/common/transformers/hash_key_digger.rb deleted file mode 100644 index b4897b5b2bf..00000000000 --- a/lib/bulk_imports/common/transformers/hash_key_digger.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Common - module Transformers - class HashKeyDigger - def initialize(options = {}) - @key_path = options[:key_path] - end - - def transform(_, data) - raise ArgumentError, "Given data must be a Hash" unless data.is_a?(Hash) - - data.dig(*Array.wrap(key_path)) - end - - private - - attr_reader :key_path - end - end - end -end diff --git a/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb b/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb deleted file mode 100644 index b32ab28fdbb..00000000000 --- a/lib/bulk_imports/common/transformers/underscorify_keys_transformer.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Common - module Transformers - class UnderscorifyKeysTransformer - def initialize(options = {}) - @options = options - end - - def transform(_, data) - data.deep_transform_keys do |key| - key.to_s.underscore - end - end - end - end - end -end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb index 5c5e686cec5..b01fb6f68ac 100644 --- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -9,9 +9,11 @@ module BulkImports def extract(context) encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path) - http_client(context.entity.bulk_import.configuration) + response = http_client(context.configuration) .each_page(:get, "groups/#{encoded_parent_path}/subgroups") .flat_map(&:itself) + + BulkImports::Pipeline::ExtractedData.new(data: response) end private diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb index 2bc0f60baa2..6852e25c87f 100644 --- a/lib/bulk_imports/groups/graphql/get_group_query.rb +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -12,25 +12,37 @@ module BulkImports group(fullPath: $full_path) { name path - fullPath + full_path: fullPath description visibility - emailsDisabled - lfsEnabled - mentionsDisabled - projectCreationLevel - requestAccessEnabled - requireTwoFactorAuthentication - shareWithGroupLock - subgroupCreationLevel - twoFactorGracePeriod + emails_disabled: emailsDisabled + lfs_enabled: lfsEnabled + mentions_disabled: mentionsDisabled + project_creation_level: projectCreationLevel + request_access_enabled: requestAccessEnabled + require_two_factor_authentication: requireTwoFactorAuthentication + share_with_group_lock: shareWithGroupLock + subgroup_creation_level: subgroupCreationLevel + two_factor_grace_period: twoFactorGracePeriod } } GRAPHQL end - def variables(entity) - { full_path: entity.source_full_path } + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data group] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' end end end diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb new file mode 100644 index 00000000000..d1fe791c2ce --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_labels_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetLabelsQuery + extend self + + def to_s + <<-'GRAPHQL' + query ($full_path: ID!, $cursor: String) { + group(fullPath: $full_path) { + labels(first: 100, after: $cursor) { + page_info: pageInfo { + end_cursor: endCursor + has_next_page: hasNextPage + } + nodes { + title + description + color + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.entity.next_page_for(:labels) + } + end + + def base_path + %w[data group labels] + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/groups/graphql/get_members_query.rb new file mode 100644 index 00000000000..e3a78124a47 --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_members_query.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetMembersQuery + extend self + def to_s + <<-'GRAPHQL' + query($full_path: ID!, $cursor: String) { + group(fullPath: $full_path) { + group_members: groupMembers(relations: DIRECT, first: 100, after: $cursor) { + page_info: pageInfo { + end_cursor: endCursor + has_next_page: hasNextPage + } + nodes { + created_at: createdAt + updated_at: updatedAt + expires_at: expiresAt + access_level: accessLevel { + integer_value: integerValue + } + user { + public_email: publicEmail + } + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.entity.next_page_for(:group_members) + } + end + + def base_path + %w[data group group_members] + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/loaders/labels_loader.rb b/lib/bulk_imports/groups/loaders/labels_loader.rb new file mode 100644 index 00000000000..b8c9ba9609c --- /dev/null +++ b/lib/bulk_imports/groups/loaders/labels_loader.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Loaders + class LabelsLoader + def initialize(*); end + + def load(context, data) + Labels::CreateService.new(data).execute(group: context.group) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/loaders/members_loader.rb b/lib/bulk_imports/groups/loaders/members_loader.rb new file mode 100644 index 00000000000..ccf44b31aee --- /dev/null +++ b/lib/bulk_imports/groups/loaders/members_loader.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Loaders + class MembersLoader + def initialize(*); end + + def load(context, data) + return unless data + + context.group.members.create!(data) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb index 5169e292180..8c6f089e8a4 100644 --- a/lib/bulk_imports/groups/pipelines/group_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb @@ -10,8 +10,6 @@ module BulkImports extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetGroupQuery - transformer Common::Transformers::HashKeyDigger, key_path: %w[data group] - transformer Common::Transformers::UnderscorifyKeysTransformer transformer Common::Transformers::ProhibitedAttributesTransformer transformer Groups::Transformers::GroupAttributesTransformer diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb new file mode 100644 index 00000000000..40dab9b444c --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class LabelsPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::GraphqlExtractor, + query: BulkImports::Groups::Graphql::GetLabelsQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + loader BulkImports::Groups::Loaders::LabelsLoader + + def after_run(extracted_data) + context.entity.update_tracker_for( + relation: :labels, + has_next_page: extracted_data.has_next_page?, + next_page: extracted_data.next_page + ) + + if extracted_data.has_next_page? + run + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb new file mode 100644 index 00000000000..b00c4c1a659 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class MembersPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::GraphqlExtractor, + query: BulkImports::Groups::Graphql::GetMembersQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + transformer BulkImports::Groups::Transformers::MemberAttributesTransformer + + loader BulkImports::Groups::Loaders::MembersLoader + + def after_run(extracted_data) + context.entity.update_tracker_for( + relation: :group_members, + has_next_page: extracted_data.has_next_page?, + next_page: extracted_data.next_page + ) + + if extracted_data.has_next_page? + run + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb new file mode 100644 index 00000000000..622f5b60ffe --- /dev/null +++ b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Transformers + class MemberAttributesTransformer + def initialize(*); end + + def transform(context, data) + data + .then { |data| add_user(data) } + .then { |data| add_access_level(data) } + .then { |data| add_author(data, context) } + end + + private + + def add_user(data) + user = find_user(data&.dig('user', 'public_email')) + + return unless user + + data + .except('user') + .merge('user_id' => user.id) + end + + def find_user(email) + return unless email + + User.find_by_any_email(email, confirmed: true) + end + + def add_access_level(data) + access_level = data&.dig('access_level', 'integer_value') + + return unless valid_access_level?(access_level) + + data.merge('access_level' => access_level) + end + + def valid_access_level?(access_level) + Gitlab::Access + .options_with_owner + .value?(access_level) + end + + def add_author(data, context) + return unless data + + data.merge('created_by_id' => context.current_user.id) + end + end + end + end +end diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb index 6e1b86e9515..f967b7ad7ab 100644 --- a/lib/bulk_imports/importers/group_importer.rb +++ b/lib/bulk_imports/importers/group_importer.rb @@ -8,16 +8,9 @@ module BulkImports end def execute - bulk_import = entity.bulk_import - configuration = bulk_import.configuration + context = BulkImports::Pipeline::Context.new(entity) - context = BulkImports::Pipeline::Context.new( - current_user: bulk_import.user, - entity: entity, - configuration: configuration - ) - - pipelines.each { |pipeline| pipeline.new.run(context) } + pipelines.each { |pipeline| pipeline.new(context).run } entity.finish! end @@ -29,7 +22,9 @@ module BulkImports def pipelines [ BulkImports::Groups::Pipelines::GroupPipeline, - BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline + BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + BulkImports::Groups::Pipelines::MembersPipeline, + BulkImports::Groups::Pipelines::LabelsPipeline ] end end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 06b81b5da14..1d55ad95887 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -4,12 +4,17 @@ module BulkImports module Pipeline extend ActiveSupport::Concern include Gitlab::ClassAttributes + include Runner - included do - include Runner + def initialize(context) + @context = context + end + included do private + attr_reader :context + def extractor @extractor ||= instantiate(self.class.get_extractor) end @@ -22,10 +27,6 @@ module BulkImports @loaders ||= instantiate(self.class.get_loader) end - def after_run - @after_run ||= self.class.after_run_callback - end - def pipeline @pipeline ||= self.class.name end @@ -52,10 +53,6 @@ module BulkImports class_attributes[:loader] = { klass: klass, options: options } end - def after_run(&block) - class_attributes[:after_run] = block - end - def get_extractor class_attributes[:extractor] end @@ -68,10 +65,6 @@ module BulkImports class_attributes[:loader] end - def after_run_callback - class_attributes[:after_run] - end - def abort_on_failure! class_attributes[:abort_on_failure] = true end diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb index ad19f5cad7d..dd121b2dbed 100644 --- a/lib/bulk_imports/pipeline/context.rb +++ b/lib/bulk_imports/pipeline/context.rb @@ -3,30 +3,25 @@ module BulkImports module Pipeline class Context - include Gitlab::Utils::LazyAttributes + attr_reader :entity, :bulk_import + attr_accessor :extra - Attribute = Struct.new(:name, :type) - - PIPELINE_ATTRIBUTES = [ - Attribute.new(:current_user, User), - Attribute.new(:entity, ::BulkImports::Entity), - Attribute.new(:configuration, ::BulkImports::Configuration) - ].freeze - - def initialize(args) - assign_attributes(args) + def initialize(entity, extra = {}) + @entity = entity + @bulk_import = entity.bulk_import + @extra = extra end - private + def group + entity.group + end - PIPELINE_ATTRIBUTES.each do |attr| - lazy_attr_reader attr.name, type: attr.type + def current_user + bulk_import.user end - def assign_attributes(values) - values.slice(*PIPELINE_ATTRIBUTES.map(&:name)).each do |name, value| - instance_variable_set("@#{name}", value) - end + def configuration + bulk_import.configuration end end end diff --git a/lib/bulk_imports/pipeline/extracted_data.rb b/lib/bulk_imports/pipeline/extracted_data.rb new file mode 100644 index 00000000000..685a91a4afe --- /dev/null +++ b/lib/bulk_imports/pipeline/extracted_data.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + class ExtractedData + attr_reader :data + + def initialize(data: nil, page_info: {}) + @data = Array.wrap(data) + @page_info = page_info + end + + def has_next_page? + @page_info['has_next_page'] + end + + def next_page + @page_info['end_cursor'] + end + + def each(&block) + data.each(&block) + end + end + end +end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 11fb9722173..d39f4121b51 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -7,75 +7,86 @@ module BulkImports MarkedAsFailedError = Class.new(StandardError) - def run(context) - raise MarkedAsFailedError if marked_as_failed?(context) + def run + raise MarkedAsFailedError if marked_as_failed? - info(context, message: 'Pipeline started', pipeline_class: pipeline) + info(message: 'Pipeline started') - Array.wrap(extracted_data_from(context)).each do |entry| + extracted_data = extracted_data_from + + extracted_data&.each do |entry| transformers.each do |transformer| - entry = run_pipeline_step(:transformer, transformer.class.name, context) do + entry = run_pipeline_step(:transformer, transformer.class.name) do transformer.transform(context, entry) end end - run_pipeline_step(:loader, loader.class.name, context) do + run_pipeline_step(:loader, loader.class.name) do loader.load(context, entry) end end - after_run.call(context) if after_run.present? + if respond_to?(:after_run) + run_pipeline_step(:after_run) do + after_run(extracted_data) + end + end + + info(message: 'Pipeline finished') rescue MarkedAsFailedError - log_skip(context) + log_skip end private # rubocop:disable Lint/UselessAccessModifier - def run_pipeline_step(type, class_name, context) - raise MarkedAsFailedError if marked_as_failed?(context) + def run_pipeline_step(step, class_name = nil) + raise MarkedAsFailedError if marked_as_failed? - info(context, type => class_name) + info(pipeline_step: step, step_class: class_name) yield rescue MarkedAsFailedError - log_skip(context, type => class_name) + log_skip(step => class_name) rescue => e - log_import_failure(e, context) + log_import_failure(e, step) - mark_as_failed(context) if abort_on_failure? + mark_as_failed if abort_on_failure? + + nil end - def extracted_data_from(context) - run_pipeline_step(:extractor, extractor.class.name, context) do + def extracted_data_from + run_pipeline_step(:extractor, extractor.class.name) do extractor.extract(context) end end - def mark_as_failed(context) - warn(context, message: 'Pipeline failed', pipeline_class: pipeline) + def mark_as_failed + warn(message: 'Pipeline failed', pipeline_class: pipeline) context.entity.fail_op! end - def marked_as_failed?(context) + def marked_as_failed? return true if context.entity.failed? false end - def log_skip(context, extra = {}) + def log_skip(extra = {}) log = { message: 'Skipping due to failed pipeline status', pipeline_class: pipeline }.merge(extra) - info(context, log) + info(log) end - def log_import_failure(exception, context) + def log_import_failure(exception, step) attributes = { bulk_import_entity_id: context.entity.id, pipeline_class: pipeline, + pipeline_step: step, exception_class: exception.class.to_s, exception_message: exception.message.truncate(255), correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id @@ -84,19 +95,22 @@ module BulkImports BulkImports::Failure.create(attributes) end - def warn(context, extra = {}) - logger.warn(log_base_params(context).merge(extra)) + def warn(extra = {}) + logger.warn(log_params(extra)) end - def info(context, extra = {}) - logger.info(log_base_params(context).merge(extra)) + def info(extra = {}) + logger.info(log_params(extra)) end - def log_base_params(context) - { + def log_params(extra) + defaults = { bulk_import_entity_id: context.entity.id, - bulk_import_entity_type: context.entity.source_type + bulk_import_entity_type: context.entity.source_type, + pipeline_class: pipeline } + + defaults.merge(extra).compact end def logger diff --git a/lib/feature.rb b/lib/feature.rb index 3d4a919b043..7c926b25587 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -133,7 +133,7 @@ class Feature # This method is called from config/initializers/flipper.rb and can be used # to register Flipper groups. - # See https://docs.gitlab.com/ee/development/feature_flags.html#feature-groups + # See https://docs.gitlab.com/ee/development/feature_flags/index.html def register_feature_groups end diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index 2d0fdf98e8c..e603a1dc8d2 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -5,25 +5,25 @@ class Feature PREFIX = "gitaly_" class << self - def enabled?(feature_flag) + def enabled?(feature_flag, project = nil) return false unless Feature::FlipperFeature.table_exists? - Feature.enabled?("#{PREFIX}#{feature_flag}") + Feature.enabled?("#{PREFIX}#{feature_flag}", project) rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad false end - def server_feature_flags + def server_feature_flags(project = nil) # We need to check that both the DB connection and table exists return {} unless ::Gitlab::Database.cached_table_exists?(FlipperFeature.table_name) Feature.persisted_names .select { |f| f.start_with?(PREFIX) } - .map do |f| + .to_h do |f| flag = f.delete_prefix(PREFIX) - ["gitaly-feature-#{flag.tr('_', '-')}", enabled?(flag).to_s] - end.to_h + ["gitaly-feature-#{flag.tr('_', '-')}", enabled?(flag, project).to_s] + end end end end diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb new file mode 100644 index 00000000000..d3fac4c74f3 --- /dev/null +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require 'rails/generators' + +module Gitlab + class UsageMetricDefinitionGenerator < Rails::Generators::Base + Directory = Struct.new(:name, :time_frame, :value_type) do + def match?(str) + (name == str || time_frame == str) && str != 'none' + end + end + + TIME_FRAME_DIRS = [ + Directory.new('counts_7d', '7d', 'number'), + Directory.new('counts_28d', '28d', 'number'), + Directory.new('counts_all', 'all', 'number'), + Directory.new('settings', 'none', 'boolean'), + Directory.new('license', 'none', 'string') + ].freeze + + VALID_INPUT_DIRS = (TIME_FRAME_DIRS.flat_map { |d| [d.name, d.time_frame] } - %w(none)).freeze + + source_root File.expand_path('../../../generator_templates/usage_metric_definition', __dir__) + + desc 'Generates a metric definition yml file' + + class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if metric is for ee' + class_option :dir, + type: :string, desc: "Indicates the metric location. It must be one of: #{VALID_INPUT_DIRS.join(', ')}" + + argument :key_path, type: :string, desc: 'Unique JSON key path for the metric' + + def create_metric_file + validate! + + template "metric_definition.yml", file_path + end + + def time_frame + directory&.time_frame + end + + def value_type + directory&.value_type + end + + def distribution + value = ['- ce'] + value << '- ee' if ee? + value.join("\n") + end + + def milestone + Gitlab::VERSION.match('(\d+\.\d+)').captures.first + end + + private + + def file_path + path = File.join('config', 'metrics', directory&.name, "#{file_name}.yml") + path = File.join('ee', path) if ee? + path + end + + def validate! + raise "--dir option is required" unless input_dir.present? + raise "Invalid dir #{input_dir}, allowed options are #{VALID_INPUT_DIRS.join(', ')}" unless directory.present? + end + + def ee? + options[:ee] + end + + def input_dir + options[:dir] + end + + # Example of file name + # + # 20210201124931_g_project_management_issue_title_changed_weekly.yml + def file_name + "#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_#{key_path.split('.').last}" + end + + def directory + @directory ||= TIME_FRAME_DIRS.find { |d| d.match?(input_dir) } + end + end +end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 0f2fd01e3c7..f943b532454 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -48,6 +48,10 @@ module Gitlab Gitlab.config.gitlab.url == COM_URL || gl_subdomain? end + def self.com + yield if com? + end + def self.staging? Gitlab.config.gitlab.url == STAGING_COM_URL end @@ -117,7 +121,7 @@ module Gitlab end def self.maintenance_mode? - return false unless ::Feature.enabled?(:maintenance_mode) + return false unless ::Gitlab::CurrentSettings.current_application_settings? ::Gitlab::CurrentSettings.maintenance_mode end diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index ce09ffd87ee..a1063001330 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -17,13 +17,14 @@ module Gitlab # @param project [Project] # @param payload [Hash] # @param monitoring_tool [String] - def parse(project, payload, monitoring_tool: nil) + # @param integration [AlertManagement::HttpIntegration] + def parse(project, payload, monitoring_tool: nil, integration: nil) payload_class = payload_class_for( monitoring_tool: monitoring_tool || payload&.dig('monitoring_tool'), payload: payload ) - payload_class.new(project: project, payload: payload) + payload_class.new(project: project, payload: payload, integration: integration) end private @@ -47,3 +48,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload.prepend_if_ee('EE::Gitlab::AlertManagement::Payload') diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 0fd593a3780..c8b8d6c259d 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -12,7 +12,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include Gitlab::Routing - attr_accessor :project, :payload + attr_accessor :project, :payload, :integration # Any attribute expected to be specifically read from # or derived from an alert payload should be defined. @@ -44,11 +44,25 @@ module Gitlab :title ].freeze + private_constant :EXPECTED_PAYLOAD_ATTRIBUTES + # Define expected API for a payload EXPECTED_PAYLOAD_ATTRIBUTES.each do |key| define_method(key) {} end + SEVERITY_MAPPING = { + 'critical' => :critical, + 'high' => :high, + 'medium' => :medium, + 'low' => :low, + 'info' => :info + }.freeze + + # Handle an unmapped severity value the same way we treat missing values + # so we can fallback to alert's default severity `critical`. + UNMAPPED_SEVERITY = nil + # Defines a method which allows access to a given # value within an alert payload # @@ -131,9 +145,21 @@ module Gitlab true end + def severity + severity_mapping.fetch(severity_raw.to_s.downcase, UNMAPPED_SEVERITY) + end + private - def plain_gitlab_fingerprint; end + def plain_gitlab_fingerprint + end + + def severity_raw + end + + def severity_mapping + SEVERITY_MAPPING + end def truncate_hosts(hosts) return hosts if hosts.join.length <= ::AlertManagement::Alert::HOSTS_MAX_LENGTH @@ -147,6 +173,7 @@ module Gitlab end end + # Overriden in EE::Gitlab::AlertManagement::Payload::Generic def value_for_paths(paths) target_path = paths.find { |path| payload&.dig(*path) } diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index e8e85155bef..0eb1bee8181 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -6,7 +6,6 @@ module Gitlab module Payload class Generic < Base DEFAULT_TITLE = 'New: Incident' - DEFAULT_SEVERITY = 'critical' attribute :description, paths: 'description' attribute :ends_at, paths: 'end_time', type: :time @@ -15,10 +14,12 @@ module Gitlab attribute :monitoring_tool, paths: 'monitoring_tool' attribute :runbook, paths: 'runbook' attribute :service, paths: 'service' - attribute :severity, paths: 'severity', fallback: -> { DEFAULT_SEVERITY } attribute :starts_at, paths: 'start_time', type: :time, fallback: -> { Time.current.utc } attribute :title, paths: 'title', fallback: -> { DEFAULT_TITLE } + attribute :severity_raw, paths: 'severity' + private :severity_raw + attribute :plain_gitlab_fingerprint, paths: 'fingerprint' private :plain_gitlab_fingerprint end diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb index 336e9b319e8..4c36ebbf3aa 100644 --- a/lib/gitlab/alert_management/payload/prometheus.rb +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true -# Attribute mapping for alerts via prometheus alerting integration. module Gitlab module AlertManagement module Payload + # Attribute mapping for alerts via prometheus alerting integration. class Prometheus < Base + extend Gitlab::Utils::Override + attribute :alert_markdown, paths: %w(annotations gitlab_incident_markdown) attribute :annotations, paths: 'annotations' attribute :description, paths: %w(annotations description) @@ -26,13 +28,40 @@ module Gitlab paths: [%w(annotations title), %w(annotations summary), %w(labels alertname)] - attribute :starts_at_raw, paths: [%w(startsAt)] private :starts_at_raw + attribute :severity_raw, paths: %w(labels severity) + private :severity_raw + METRIC_TIME_WINDOW = 30.minutes + ADDITIONAL_SEVERITY_MAPPING = { + 's1' => :critical, + 's2' => :high, + 's3' => :medium, + 's4' => :low, + 's5' => :info, + 'p1' => :critical, + 'p2' => :high, + 'p3' => :medium, + 'p4' => :low, + 'p5' => :info, + 'debug' => :info, + 'information' => :info, + 'notice' => :info, + 'warn' => :low, + 'warning' => :low, + 'minor' => :low, + 'error' => :medium, + 'major' => :high, + 'emergency' => :critical, + 'fatal' => :critical, + 'alert' => :medium, + 'page' => :high + }.freeze + def monitoring_tool Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus] end @@ -67,6 +96,11 @@ module Gitlab private + override :severity_mapping + def severity_mapping + super.merge(ADDITIONAL_SEVERITY_MAPPING) + end + def plain_gitlab_fingerprint [starts_at_raw, title, full_query].join('/') end diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb index 32a98908e5b..09039f3fc43 100644 --- a/lib/gitlab/api_authentication/token_locator.rb +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :location - validates :location, inclusion: { in: %i[http_basic_auth] } + validates :location, inclusion: { in: %i[http_basic_auth http_token] } def initialize(location) @location = location @@ -21,6 +21,8 @@ module Gitlab case @location when :http_basic_auth extract_from_http_basic_auth request + when :http_token + extract_from_http_token request end end @@ -32,6 +34,13 @@ module Gitlab UsernameAndPassword.new(username, password) end + + def extract_from_http_token(request) + password = request.headers['Authorization'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end end end end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb index 5b30777b6ec..9234837cdf7 100644 --- a/lib/gitlab/api_authentication/token_resolver.rb +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -7,7 +7,16 @@ module Gitlab attr_reader :token_type - validates :token_type, inclusion: { in: %i[personal_access_token job_token deploy_token] } + validates :token_type, inclusion: { + in: %i[ + personal_access_token_with_username + job_token_with_username + deploy_token_with_username + personal_access_token + job_token + deploy_token + ] + } def initialize(token_type) @token_type = token_type @@ -38,49 +47,94 @@ module Gitlab when :deploy_token resolve_deploy_token raw + + when :personal_access_token_with_username + resolve_personal_access_token_with_username raw + + when :job_token_with_username + resolve_job_token_with_username raw + + when :deploy_token_with_username + resolve_deploy_token_with_username raw end end private - def resolve_personal_access_token(raw) - # Check if the password is a personal access token - pat = ::PersonalAccessToken.find_by_token(raw.password) - return unless pat + def resolve_personal_access_token_with_username(raw) + raise ::Gitlab::Auth::UnauthorizedError unless raw.username + + with_personal_access_token(raw) do |pat| + break unless pat - # Ensure that the username matches the token. This check is a subtle - # departure from the existing behavior of #find_personal_access_token_from_http_basic_auth. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_435907856 - raise ::Gitlab::Auth::UnauthorizedError unless pat.user.username == raw.username + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #find_personal_access_token_from_http_basic_auth. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_435907856 + raise ::Gitlab::Auth::UnauthorizedError unless pat.user.username == raw.username - pat + pat + end end - def resolve_job_token(raw) + def resolve_job_token_with_username(raw) # Only look for a job if the username is correct return if ::Gitlab::Auth::CI_JOB_USER != raw.username - job = ::Ci::AuthJobFinder.new(token: raw.password).execute + with_job_token(raw) do |job| + job + end + end - # Actively reject credentials with the username `gitlab-ci-token` if - # the password is not a valid job token. This replicates existing - # behavior of #find_user_from_job_token. - raise ::Gitlab::Auth::UnauthorizedError unless job + def resolve_deploy_token_with_username(raw) + with_deploy_token(raw) do |token| + break unless token + + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #deploy_token_from_request. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_474826205 + raise ::Gitlab::Auth::UnauthorizedError unless token.username == raw.username - job + token + end + end + + def resolve_personal_access_token(raw) + with_personal_access_token(raw) do |pat| + pat + end + end + + def resolve_job_token(raw) + with_job_token(raw) do |job| + job + end end def resolve_deploy_token(raw) - # Check if the password is a deploy token + with_deploy_token(raw) do |token| + token + end + end + + def with_personal_access_token(raw, &block) + pat = ::PersonalAccessToken.find_by_token(raw.password) + return unless pat + + yield(pat) + end + + def with_deploy_token(raw, &block) token = ::DeployToken.active.find_by_token(raw.password) return unless token - # Ensure that the username matches the token. This check is a subtle - # departure from the existing behavior of #deploy_token_from_request. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_474826205 - raise ::Gitlab::Auth::UnauthorizedError unless token.username == raw.username + yield(token) + end + + def with_job_token(raw, &block) + job = ::Ci::AuthJobFinder.new(token: raw.password).execute + raise ::Gitlab::Auth::UnauthorizedError unless job - token + yield(job) end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index fbba86d1253..0a69a9c503d 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -20,6 +20,7 @@ module Gitlab def rate_limits { issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute }, + notes_create: { threshold: -> { application_settings.notes_create_limit }, interval: 1.minute }, project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute }, project_repositories_archive: { threshold: 5, interval: 1.minute }, @@ -46,15 +47,17 @@ module Gitlab # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits` + # @option users_allowlist [Array<String>] Optional list of usernames to excepted from the limit. This param will only be functional if Scope includes a current user. # # @return [Boolean] Whether or not a request should be throttled - def throttled?(key, scope: nil, interval: nil, threshold: nil) + def throttled?(key, **options) return unless rate_limits[key] - threshold_value = threshold || threshold(key) + return if scoped_user_in_allowlist?(options) + threshold_value = options[:threshold] || threshold(key) threshold_value > 0 && - increment(key, scope, interval) > threshold_value + increment(key, options[:scope], options[:interval]) > threshold_value end # Increments the given cache key and increments the value by 1 with the @@ -140,6 +143,15 @@ module Gitlab def application_settings Gitlab::CurrentSettings.current_application_settings end + + def scoped_user_in_allowlist?(options) + return unless options[:users_allowlist].present? + + scoped_user = [options[:scope]].flatten.find { |s| s.is_a?(User) } + return unless scoped_user + + scoped_user.username.downcase.in?(options[:users_allowlist]) + end end end end diff --git a/lib/gitlab/auth/otp/session_enforcer.rb b/lib/gitlab/auth/otp/session_enforcer.rb deleted file mode 100644 index 8cc280756cc..00000000000 --- a/lib/gitlab/auth/otp/session_enforcer.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Auth - module Otp - class SessionEnforcer - OTP_SESSIONS_NAMESPACE = 'session:otp' - DEFAULT_EXPIRATION = 15.minutes.to_i - - def initialize(key) - @key = key - end - - def update_session - Gitlab::Redis::SharedState.with do |redis| - redis.setex(key_name, DEFAULT_EXPIRATION, true) - end - end - - def access_restricted? - Gitlab::Redis::SharedState.with do |redis| - !redis.get(key_name) - end - end - - private - - attr_reader :key - - def key_name - @key_name ||= "#{OTP_SESSIONS_NAMESPACE}:#{key.id}" - end - end - end - end -end diff --git a/lib/gitlab/auth/u2f_webauthn_converter.rb b/lib/gitlab/auth/u2f_webauthn_converter.rb new file mode 100644 index 00000000000..f85b2248aeb --- /dev/null +++ b/lib/gitlab/auth/u2f_webauthn_converter.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + class U2fWebauthnConverter + def initialize(u2f_registration) + @u2f_registration = u2f_registration + end + + def convert + now = Time.current + + converted_credential = WebAuthn::U2fMigrator.new( + app_id: Gitlab.config.gitlab.url, + certificate: u2f_registration.certificate, + key_handle: u2f_registration.key_handle, + public_key: u2f_registration.public_key, + counter: u2f_registration.counter + ).credential + + { + credential_xid: Base64.strict_encode64(converted_credential.id), + public_key: Base64.strict_encode64(converted_credential.public_key), + counter: u2f_registration.counter || 0, + name: u2f_registration.name || '', + user_id: u2f_registration.user_id, + u2f_registration_id: u2f_registration.id, + created_at: now, + updated_at: now + } + end + + private + + attr_reader :u2f_registration + end + end +end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d1b9062a23c..9f4d6557023 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -33,7 +33,7 @@ module Gitlab next unless job.queue == self.queue next unless migration_class == steal_class - next if block_given? && !(yield migration_args) + next if block_given? && !(yield job) begin perform(migration_class, migration_args) if job.delete diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb new file mode 100644 index 00000000000..61eb3b332de --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Update existent project update_at column after their repository storage was moved + class BackfillProjectUpdatedAtAfterRepositoryStorageMove + def perform(*project_ids) + updated_repository_storages = ProjectRepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) + + Project.connection.execute <<-SQL + WITH repository_storage_cte as ( + #{updated_repository_storages.to_sql} + ) + UPDATE projects + SET updated_at = (repository_storage_cte.updated_at + interval '1 second') + FROM repository_storage_cte + WHERE projects.id = repository_storage_cte.project_id AND projects.updated_at <= repository_storage_cte.updated_at + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb new file mode 100644 index 00000000000..de2d9909961 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +module Gitlab + module BackgroundMigration + # EE-specific migration + class MigrateDevopsSegmentsToGroups + def perform + # no-op for CE + end + end + end +end + +Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb index b8c14aa2573..091e6660bac 100644 --- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb +++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb @@ -16,26 +16,9 @@ module Gitlab def perform(start_id, end_id) old_registrations = U2fRegistration.where(id: start_id..end_id) old_registrations.each_slice(100) do |slice| - now = Time.now values = slice.map do |u2f_registration| - converted_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - { - credential_xid: Base64.strict_encode64(converted_credential.id), - public_key: Base64.strict_encode64(converted_credential.public_key), - counter: u2f_registration.counter || 0, - name: u2f_registration.name || '', - user_id: u2f_registration.user_id, - u2f_registration_id: u2f_registration.id, - created_at: now, - updated_at: now - } + converter = Gitlab::Auth::U2fWebauthnConverter.new(u2f_registration) + converter.convert end WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false) diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb new file mode 100644 index 00000000000..d6795296fb7 --- /dev/null +++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to migrate service_desk_reply_to email addresses to issue_email_participants + class PopulateIssueEmailParticipants + # rubocop:disable Style/Documentation + class TmpIssue < ActiveRecord::Base + self.table_name = 'issues' + end + + def perform(start_id, stop_id) + issues = TmpIssue.select(:id, :service_desk_reply_to, :created_at).where(id: (start_id..stop_id)).where.not(service_desk_reply_to: nil) + + rows = issues.map do |issue| + { + issue_id: issue.id, + email: issue.service_desk_reply_to, + created_at: issue.created_at, + updated_at: issue.created_at + } + end + + Gitlab::Database.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb new file mode 100644 index 00000000000..3d3970f50e1 --- /dev/null +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PopulateUuidsForSecurityFindings + NOP_RELATION = Class.new { def each_batch(*); end } + + def self.security_findings + NOP_RELATION.new + end + + def perform(_scan_ids); end + end + end +end + +Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_if_ee('::EE::Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb new file mode 100644 index 00000000000..ca61118a06c --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings + DELETE_BATCH_SIZE = 100 + + # rubocop:disable Gitlab/NamespacedClass + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + end + # rubocop:enable Gitlab/NamespacedClass + + def perform(start_id, end_id) + batch = VulnerabilitiesFinding.where(id: start_id..end_id) + + cte = Gitlab::SQL::CTE.new(:batch, batch.select(:report_type, :location_fingerprint, :primary_identifier_id, :project_id)) + + query = VulnerabilitiesFinding + .select('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id', 'array_agg(id) as ids') + .distinct + .with(cte.to_arel) + .from(cte.alias_to(Arel.sql('batch'))) + .joins( + %( + INNER JOIN + vulnerability_occurrences ON + vulnerability_occurrences.report_type = batch.report_type AND + vulnerability_occurrences.location_fingerprint = batch.location_fingerprint AND + vulnerability_occurrences.primary_identifier_id = batch.primary_identifier_id AND + vulnerability_occurrences.project_id = batch.project_id + )).group('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id') + .having('COUNT(*) > 1') + + ids_to_delete = [] + + query.to_a.each do |record| + # We want to keep the latest finding since it might have recent metadata + duplicate_ids = record.ids.uniq.sort + duplicate_ids.pop + ids_to_delete.concat(duplicate_ids) + + if ids_to_delete.size == DELETE_BATCH_SIZE + VulnerabilitiesFinding.where(id: ids_to_delete).delete_all + ids_to_delete.clear + end + end + + VulnerabilitiesFinding.where(id: ids_to_delete).delete_all if ids_to_delete.any? + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index 6d7b9a86e69..8fa0db5fd4b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -6,6 +6,7 @@ module Gitlab module Models # isolated Namespace model class Namespace < ApplicationRecord + include FeatureGate include ::Gitlab::VisibilityLevel include ::Gitlab::Utils::StrongMemoize include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal diff --git a/lib/gitlab/changelog/ast.rb b/lib/gitlab/changelog/ast.rb new file mode 100644 index 00000000000..2c787d396f5 --- /dev/null +++ b/lib/gitlab/changelog/ast.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # AST nodes to evaluate when rendering a template. + # + # Evaluating an AST is done by walking over the nodes and calling + # `evaluate`. This method takes two arguments: + # + # 1. An instance of `EvalState`, used for tracking data such as the number + # of nested loops. + # 2. An object used as the data for the current scope. This can be an Array, + # Hash, String, or something else. It's up to the AST node to determine + # what to do with it. + # + # While tree walking interpreters (such as implemented here) aren't usually + # the fastest type of interpreter, they are: + # + # 1. Fast enough for our use case + # 2. Easy to implement and maintain + # + # In addition, our AST interpreter doesn't allow for arbitrary code + # execution, unlike existing template engines such as Mustache + # (https://github.com/mustache/mustache/issues/244) or ERB. + # + # Our interpreter also takes care of limiting the number of nested loops. + # And unlike Liquid, our interpreter is much smaller and thus has a smaller + # attack surface. Liquid isn't without its share of issues, such as + # https://github.com/Shopify/liquid/pull/1071. + # + # We also evaluated using Handlebars using the project + # https://github.com/SmartBear/ruby-handlebars. Sadly, this implementation + # of Handlebars doesn't support control of whitespace + # (https://github.com/SmartBear/ruby-handlebars/issues/37), and the project + # didn't appear to be maintained that much. + # + # This doesn't mean these template engines aren't good, instead it means + # they won't work for our use case. For more information, refer to the + # comment https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50063#note_469293322. + module AST + # An identifier in a selector. + Identifier = Struct.new(:name) do + def evaluate(state, data) + return data if name == 'it' + + data[name] if data.is_a?(Hash) + end + end + + # An integer used in a selector. + Integer = Struct.new(:value) do + def evaluate(state, data) + data[value] if data.is_a?(Array) + end + end + + # A selector used for loading a value. + Selector = Struct.new(:steps) do + def evaluate(state, data) + steps.reduce(data) do |current, step| + break if current.nil? + + step.evaluate(state, current) + end + end + end + + # A tag used for displaying a value in the output. + Variable = Struct.new(:selector) do + def evaluate(state, data) + selector.evaluate(state, data).to_s + end + end + + # A collection of zero or more expressions. + Expressions = Struct.new(:nodes) do + def evaluate(state, data) + nodes.map { |node| node.evaluate(state, data) }.join('') + end + end + + # A single text node. + Text = Struct.new(:text) do + def evaluate(*) + text + end + end + + # An `if` expression, with an optional `else` clause. + If = Struct.new(:condition, :true_body, :false_body) do + def evaluate(state, data) + result = + if truthy?(condition.evaluate(state, data)) + true_body.evaluate(state, data) + elsif false_body + false_body.evaluate(state, data) + end + + result.to_s + end + + def truthy?(value) + # We treat empty collections and such as false, removing the need for + # some sort of `if length(x) > 0` expression. + value.respond_to?(:empty?) ? !value.empty? : !!value + end + end + + # An `each` expression. + Each = Struct.new(:collection, :body) do + def evaluate(state, data) + values = collection.evaluate(state, data) + + return '' unless values.respond_to?(:each) + + # While unlikely to happen, it's possible users attempt to nest many + # loops in order to negatively impact the GitLab instance. To make + # this more difficult, we limit the number of nested loops a user can + # create. + state.enter_loop do + values.map { |value| body.evaluate(state, value) }.join('') + end + end + end + + # A class for transforming a raw Parslet AST into a more structured/easier + # to work with AST. + # + # For more information about Parslet transformations, refer to the + # documentation at http://kschiess.github.io/parslet/transform.html. + class Transformer < Parslet::Transform + rule(ident: simple(:name)) { Identifier.new(name.to_s) } + rule(int: simple(:name)) { Integer.new(name.to_i) } + rule(text: simple(:text)) { Text.new(text.to_s) } + rule(exprs: subtree(:nodes)) { Expressions.new(nodes) } + rule(selector: sequence(:steps)) { Selector.new(steps) } + rule(selector: simple(:step)) { Selector.new([step]) } + rule(variable: simple(:selector)) { Variable.new(selector) } + rule(each: simple(:values), body: simple(:body)) do + Each.new(values, body) + end + + rule(if: simple(:cond), true_body: simple(:true_body)) do + If.new(cond, true_body) + end + + rule( + if: simple(:cond), + true_body: simple(:true_body), + false_body: simple(:false_body) + ) do + If.new(cond, true_body, false_body) + end + end + end + end +end diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb new file mode 100644 index 00000000000..31661650eff --- /dev/null +++ b/lib/gitlab/changelog/committer.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A class used for committing a release's changelog to a Git repository. + class Committer + def initialize(project, user) + @project = project + @user = user + end + + # Commits a release's changelog to a file on a branch. + # + # The `release` argument is a `Gitlab::Changelog::Release` for which to + # update the changelog. + # + # The `file` argument specifies the path to commit the changes to. + # + # The `branch` argument specifies the branch to commit the changes on. + # + # The `message` argument specifies the commit message to use. + def commit(release:, file:, branch:, message:) + # When retrying, we need to reprocess the existing changelog from + # scratch, otherwise we may end up throwing away changes. As such, all + # the logic is contained within the retry block. + Retriable.retriable(on: Error) do + commit = Gitlab::Git::Commit.last_for_path( + @project.repository, + branch, + file, + literal_pathspec: true + ) + + content = blob_content(file, commit) + + # If the release has already been added (e.g. concurrently by another + # API call), we don't want to add it again. + break if content&.match?(release.header_start_pattern) + + service = Files::MultiService.new( + @project, + @user, + commit_message: message, + branch_name: branch, + start_branch: branch, + actions: [ + { + action: content ? 'update' : 'create', + content: Generator.new(content.to_s).add(release), + file_path: file, + last_commit_id: commit&.sha + } + ] + ) + + result = service.execute + + raise Error.new(result[:message]) if result[:status] != :success + end + end + + def blob_content(file, commit = nil) + return unless commit + + @project.repository.blob_at(commit.sha, file)&.data + end + end + end +end diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb new file mode 100644 index 00000000000..105050936ce --- /dev/null +++ b/lib/gitlab/changelog/config.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Configuration settings used when generating changelogs. + class Config + # When rendering changelog entries, authors are not included. + AUTHORS_NONE = 'none' + + # The path to the configuration file as stored in the project's Git + # repository. + FILE_PATH = '.gitlab/changelog_config.yml' + + # The default date format to use for formatting release dates. + DEFAULT_DATE_FORMAT = '%Y-%m-%d' + + # The default template to use for generating release sections. + DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl')) + + attr_accessor :date_format, :categories, :template + + def self.from_git(project) + if (yaml = project.repository.changelog_config) + from_hash(project, YAML.safe_load(yaml)) + else + new(project) + end + end + + def self.from_hash(project, hash) + config = new(project) + + if (date = hash['date_format']) + config.date_format = date + end + + if (template = hash['template']) + config.template = Parser.new.parse_and_transform(template) + end + + if (categories = hash['categories']) + if categories.is_a?(Hash) + config.categories = categories + else + raise Error, 'The "categories" configuration key must be a Hash' + end + end + + config + end + + def initialize(project) + @project = project + @date_format = DEFAULT_DATE_FORMAT + @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE) + @categories = {} + end + + def contributor?(user) + @project.team.contributor?(user) + end + + def category(name) + @categories[name] || name + end + + def format_date(date) + date.strftime(@date_format) + end + end + end +end diff --git a/lib/gitlab/changelog/error.rb b/lib/gitlab/changelog/error.rb new file mode 100644 index 00000000000..0bd886fbdb7 --- /dev/null +++ b/lib/gitlab/changelog/error.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # An error raised when a changelog couldn't be generated. + Error = Class.new(StandardError) + end +end diff --git a/lib/gitlab/changelog/eval_state.rb b/lib/gitlab/changelog/eval_state.rb new file mode 100644 index 00000000000..a0439df60cf --- /dev/null +++ b/lib/gitlab/changelog/eval_state.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A class for tracking state when evaluating a template + class EvalState + MAX_LOOPS = 4 + + def initialize + @loops = 0 + end + + def enter_loop + if @loops == MAX_LOOPS + raise Error, "You can only nest up to #{MAX_LOOPS} loops" + end + + @loops += 1 + retval = yield + @loops -= 1 + + retval + end + end + end +end diff --git a/lib/gitlab/changelog/generator.rb b/lib/gitlab/changelog/generator.rb new file mode 100644 index 00000000000..a80ca0728f9 --- /dev/null +++ b/lib/gitlab/changelog/generator.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Parsing and generating of Markdown changelogs. + class Generator + # The regex used to parse a release header. + RELEASE_REGEX = + /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/.freeze + + # The `input` argument must be a `String` containing the existing + # changelog Markdown. If no changelog exists, this should be an empty + # `String`. + def initialize(input = '') + @lines = input.lines + @locations = {} + + @lines.each_with_index do |line, index| + matches = line.match(RELEASE_REGEX) + + next if !matches || !matches[:version] + + @locations[matches[:version]] = index + end + end + + # Generates the Markdown for the given release and returns the new + # changelog Markdown content. + # + # The `release` argument must be an instance of + # `Gitlab::Changelog::Release`. + def add(release) + versions = [release.version, *@locations.keys] + + VersionSorter.rsort!(versions) + + new_index = versions.index(release.version) + new_lines = @lines.dup + markdown = release.to_markdown + + if (insert_after = versions[new_index + 1]) + line_index = @locations[insert_after] + + new_lines.insert(line_index, markdown) + else + # When adding to the end of the changelog, the previous section only + # has a single newline, resulting in the release section title + # following it immediately. When this is the case, we insert an extra + # empty line to keep the changelog readable in its raw form. + new_lines.push("\n") if versions.length > 1 + new_lines.push(markdown.rstrip) + new_lines.push("\n") + end + + new_lines.join + end + end + end +end diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/changelog/parser.rb new file mode 100644 index 00000000000..a4c8da283cd --- /dev/null +++ b/lib/gitlab/changelog/parser.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A parser for the template syntax used for generating changelogs. + # + # As a quick primer on the template syntax, a basic template looks like + # this: + # + # {% each users %} + # Name: {{name}} + # Age: {{age}} + # + # {% if birthday %} + # This user is celebrating their birthday today! Yay! + # {% end %} + # {% end %} + # + # For more information, refer to the Parslet documentation found at + # http://kschiess.github.io/parslet/. + class Parser < Parslet::Parser + root(:exprs) + + rule(:exprs) do + ( + variable | if_expr | each_expr | escaped | text | newline + ).repeat.as(:exprs) + end + + rule(:space) { match('[ \\t]') } + rule(:whitespace) { match('\s').repeat } + rule(:lf) { str("\n") } + rule(:newline) { lf.as(:text) } + + # Escaped newlines are ignored, allowing the user to control the + # whitespace in the output. All other escape sequences are treated as + # literal text. + # + # For example, this: + # + # foo \ + # bar + # + # Is parsed into this: + # + # foo bar + rule(:escaped) do + backslash = str('\\') + + (backslash >> lf).ignore | (backslash >> chars).as(:text) + end + + # A sequence of regular characters, with the exception of newlines and + # escaped newlines. + rule(:chars) do + char = match("[^{\\\\\n]") + + # The rules here are such that we do treat single curly braces or + # non-opening tags (e.g. `{foo}`) as text, but not opening tags + # themselves (e.g. `{{`). + ( + char.repeat(1) | curly_open >> (curly_open | percent).absent? + ).repeat(1) + end + + rule(:text) { chars.as(:text) } + + # An integer, limited to 10 digits (= a 32 bits integer). + # + # The size is limited to prevents users from creating integers that are + # too large, as this may result in runtime errors. + rule(:integer) { match('\d').repeat(1, 10).as(:int) } + + # An identifier to look up in a data structure. + # + # We only support simple ASCII identifiers as we simply don't have a need + # for more complex identifiers (e.g. those containing multibyte + # characters). + rule(:ident) { match('[a-zA-Z_]').repeat(1).as(:ident) } + + # A selector is used for reading a value, consisting of one or more + # "steps". + # + # Examples: + # + # name + # users.0.name + # 0 + # it + rule(:selector) do + step = ident | integer + + whitespace >> + (step >> (str('.') >> step).repeat).as(:selector) >> + whitespace + end + + rule(:curly_open) { str('{') } + rule(:curly_close) { str('}') } + rule(:percent) { str('%') } + + # A variable tag. + # + # Examples: + # + # {{name}} + # {{users.0.name}} + rule(:variable) do + curly_open.repeat(2) >> selector.as(:variable) >> curly_close.repeat(2) + end + + rule(:expr_open) { curly_open >> percent >> whitespace } + rule(:expr_close) do + # Since whitespace control is important (as Markdown is whitespace + # sensitive), we default to stripping a newline that follows a %} tag. + # This is less annoying compared to having to opt-in to this behaviour. + whitespace >> percent >> curly_close >> lf.maybe.ignore + end + + rule(:end_tag) { expr_open >> str('end') >> expr_close } + + # An `if` expression, with an optional `else` clause. + # + # Examples: + # + # {% if foo %} + # yes + # {% end %} + # + # {% if foo %} + # yes + # {% else %} + # no + # {% end %} + rule(:if_expr) do + else_tag = + expr_open >> str('else') >> expr_close >> exprs.as(:false_body) + + expr_open >> + str('if') >> + space.repeat(1) >> + selector.as(:if) >> + expr_close >> + exprs.as(:true_body) >> + else_tag.maybe >> + end_tag + end + + # An `each` expression, used for iterating over collections. + # + # Example: + # + # {% each users %} + # * {{name}} + # {% end %} + rule(:each_expr) do + expr_open >> + str('each') >> + space.repeat(1) >> + selector.as(:each) >> + expr_close >> + exprs.as(:body) >> + end_tag + end + + def parse_and_transform(input) + AST::Transformer.new.apply(parse(input)) + rescue Parslet::ParseFailed => ex + # We raise a custom error so it's easier to catch different changelog + # related errors. In addition, this ensures the caller of this method + # doesn't depend on a Parslet specific error class. + raise Error.new("Failed to parse the template: #{ex.message}") + end + end + end +end diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb new file mode 100644 index 00000000000..f2a01c2b0dc --- /dev/null +++ b/lib/gitlab/changelog/release.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A release to add to a changelog. + class Release + attr_reader :version + + def initialize(version:, date:, config:) + @version = version + @date = date + @config = config + @entries = Hash.new { |h, k| h[k] = [] } + + # This ensures that entries are presented in the same order as the + # categories Hash in the user's configuration. + @config.categories.values.each do |category| + @entries[category] = [] + end + end + + def add_entry( + title:, + commit:, + category:, + author: nil, + merge_request: nil + ) + # When changing these fields, keep in mind that this needs to be + # backwards compatible. For example, you can't just remove a field as + # this will break the changelog generation process for existing users. + entry = { + 'title' => title, + 'commit' => { + 'reference' => commit.to_reference(full: true), + 'trailers' => commit.trailers + } + } + + if author + entry['author'] = { + 'reference' => author.to_reference(full: true), + 'contributor' => @config.contributor?(author) + } + end + + if merge_request + entry['merge_request'] = { + 'reference' => merge_request.to_reference(full: true) + } + end + + @entries[@config.category(category)] << entry + end + + def to_markdown + state = EvalState.new + data = { 'categories' => entries_for_template } + + # While not critical, we would like release sections to be separated by + # an empty line in the changelog; ensuring it's readable even in its + # raw form. + # + # Since it can be a bit tricky to get this right in a template, we + # enforce an empty line separator ourselves. + markdown = @config.template.evaluate(state, data).strip + + # The release header can't be changed using the Liquid template, as we + # need this to be in a known format. Without this restriction, we won't + # know where to insert a new release section in an existing changelog. + "## #{@version} (#{release_date})\n\n#{markdown}\n\n" + end + + def header_start_pattern + /^##\s*#{Regexp.escape(@version)}/ + end + + private + + def release_date + @config.format_date(@date) + end + + def entries_for_template + rows = [] + + @entries.each do |category, entries| + next if entries.empty? + + rows << { + 'title' => category, + 'count' => entries.length, + 'single_change' => entries.length == 1, + 'entries' => entries + } + end + + rows + end + end + end +end diff --git a/lib/gitlab/changelog/template.tpl b/lib/gitlab/changelog/template.tpl new file mode 100644 index 00000000000..584939dff51 --- /dev/null +++ b/lib/gitlab/changelog/template.tpl @@ -0,0 +1,15 @@ +{% if categories %} +{% each categories %} +### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %}) + +{% each entries %} +- [{{ title }}]({{ commit.reference }})\ +{% if author.contributor %} by {{ author.reference }}{% end %}\ +{% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %} + +{% end %} + +{% end %} +{% else %} +No changes. +{% end %} diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 911f2993b8a..029a9210dc9 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -47,5 +47,13 @@ module Gitlab def self.kill Process.kill("KILL", Process.pid) end + + def self.run_gc + # Tenure any live objects from young-gen to old-gen + 4.times { GC.start(full_mark: false) } + # Run a full mark-and-sweep collection + GC.start + GC.stat + end end end diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/ci/badge/base.rb index fb55b9e2f1f..c65f120753d 100644 --- a/lib/gitlab/badge/base.rb +++ b/lib/gitlab/ci/badge/base.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge class Base def entity diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/ci/badge/coverage/metadata.rb index 9181ba2d4b0..7654b6d6fc5 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/ci/badge/coverage/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 390da014a5a..28863a0703b 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 1b985f83b22..7589fa5ff8b 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb index b9ae68134b0..eec9fedfaa9 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/ci/badge/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge ## # Abstract class for badge metadata diff --git a/lib/gitlab/badge/pipeline/metadata.rb b/lib/gitlab/ci/badge/pipeline/metadata.rb index d4d789558c9..2aa08476336 100644 --- a/lib/gitlab/badge/pipeline/metadata.rb +++ b/lib/gitlab/ci/badge/pipeline/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/ci/badge/pipeline/status.rb index f061ba22688..a2ee2642872 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/ci/badge/pipeline/status.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index af8e318395b..8430b01fc9a 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/ci/badge/template.rb index 9ac8f1c17f2..0580dad72ba 100644 --- a/lib/gitlab/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge ## # Abstract template class for badges diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb index 58adf6e506d..2aeb8453703 100644 --- a/lib/gitlab/ci/build/credentials/base.rb +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -6,7 +6,7 @@ module Gitlab module Credentials class Base def type - self.class.name.demodulize.underscore + raise NotImplementedError end end end diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb index fa805abb8bb..e8996cb9dc4 100644 --- a/lib/gitlab/ci/build/credentials/factory.rb +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -20,7 +20,7 @@ module Gitlab end def providers - [Registry] + [Registry::GitlabRegistry, Registry::DependencyProxy] end end end diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb deleted file mode 100644 index 1c8588d9913..00000000000 --- a/lib/gitlab/ci/build/credentials/registry.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Credentials - class Registry < Base - attr_reader :username, :password - - def initialize(build) - @username = 'gitlab-ci-token' - @password = build.token - end - - def url - Gitlab.config.registry.host_port - end - - def valid? - Gitlab.config.registry.enabled - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb new file mode 100644 index 00000000000..b6ac06cfb53 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class DependencyProxy < GitlabRegistry + def url + "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}" + end + + def valid? + Gitlab.config.dependency_proxy.enabled + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb new file mode 100644 index 00000000000..5bd30e677e9 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class GitlabRegistry < Credentials::Base + attr_reader :username, :password + + def initialize(build) + @username = Gitlab::Auth::CI_JOB_USER + @password = build.token + end + + def url + Gitlab.config.registry.host_port + end + + def valid? + Gitlab.config.registry.enabled + end + + def type + 'registry' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index a39afee194c..2d4f9cf635b 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -7,30 +7,17 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize Result = Struct.new(:when, :start_in, :allow_failure, :variables) do - def build_attributes(seed_attributes = {}) + def build_attributes { when: self.when, options: { start_in: start_in }.compact, - allow_failure: allow_failure, - yaml_variables: yaml_variables(seed_attributes[:yaml_variables]) + allow_failure: allow_failure }.compact end def pass? self.when != 'never' end - - private - - def yaml_variables(seed_variables) - return unless variables && seed_variables - - indexed_seed_variables = seed_variables.deep_dup.index_by { |var| var[:key] } - - variables.each_with_object(indexed_seed_variables) do |var, hash| - hash[var[0].to_s] = { key: var[0].to_s, value: var[1], public: true } - end.values - end end def initialize(rule_hashes, default_when:) diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 25fb9c0ca97..797193a6be5 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -31,9 +31,10 @@ module Gitlab current = @from while current <= @to - @labels << current.strftime(@format) - @total << (totals_count[current] || 0) - @success << (success_count[current] || 0) + label = current.strftime(@format) + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) current += interval_step end @@ -45,6 +46,7 @@ module Gitlab query .group("date_trunc('#{interval}', #{::Ci::Pipeline.table_name}.created_at)") .count(:created_at) + .transform_keys { |date| date.strftime(@format) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 8ed4dc61920..dbb48a81030 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -13,7 +13,8 @@ module Gitlab RESCUE_ERRORS = [ Gitlab::Config::Loader::FormatError, Extendable::ExtensionError, - External::Processor::IncludeError + External::Processor::IncludeError, + Config::Yaml::Tags::TagError ].freeze attr_reader :root @@ -89,9 +90,10 @@ module Gitlab end def build_config(config) - initial_config = Gitlab::Config::Loader::Yaml.new(config).load! + initial_config = Config::Yaml.load!(config) initial_config = Config::External::Processor.new(initial_config, @context).perform initial_config = Config::Extendable.new(initial_config).to_hash + initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash initial_config diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 7a86fca3056..341f87b44ab 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -10,12 +10,14 @@ module Gitlab class Commands < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + MAX_NESTING_LEVEL = 10 + validations do - validates :config, string_or_nested_array_of_strings: true + validates :config, string_or_nested_array_of_strings: { max_level: MAX_NESTING_LEVEL } end def value - Array(@config).flatten(1) + Array(@config).flatten(MAX_NESTING_LEVEL) end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 85e3514499c..a20b802be58 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script type image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout - resource_group release secrets].freeze + release secrets].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -30,7 +30,6 @@ module Gitlab } validates :dependencies, array_of_strings: true - validates :resource_group, type: String validates :allow_failure, hash_or_boolean: true end @@ -124,7 +123,7 @@ module Gitlab attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, :resource_group, + :interruptible, :timeout, :release, :allow_failure def self.matching?(name, config) @@ -174,7 +173,6 @@ module Gitlab ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, - resource_group: resource_group, scheduling_type: needs_defined? ? :dag : :stage ).compact end @@ -186,8 +184,6 @@ module Gitlab private def allow_failure_criteria - return unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? - if allow_failure_defined? && allow_failure_value.is_a?(Hash) allow_failure_value end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 5ef8cfbddb7..9584d19bdec 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,7 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs].freeze + inherit allow_failure when needs resource_group].freeze included do validations do @@ -32,6 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :resource_group, type: String end end @@ -64,7 +65,7 @@ module Gitlab inherit: false, default: {} - attributes :extends, :rules + attributes :extends, :rules, :resource_group end def compose!(deps = nil) @@ -125,7 +126,8 @@ module Gitlab rules: rules_value, variables: root_and_job_variables_value, only: only_value, - except: except_value }.compact + except: except_value, + resource_group: resource_group }.compact end def root_and_job_variables_value diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 4684a9eb981..7d3fddd850d 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -60,7 +60,7 @@ module Gitlab def content_hash strong_memoize(:content_yaml) do - Gitlab::Config::Loader::Yaml.new(content).load! + ::Gitlab::Ci::Config::Yaml.load!(content) end rescue Gitlab::Config::Loader::FormatError nil diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 4d91cfd4c57..b85b7a9edeb 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -99,8 +99,6 @@ module Gitlab end def expand_variables(data) - return data unless ::Feature.enabled?(:variables_in_include_section_ci) - if data.is_a?(String) expand(data) else diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb new file mode 100644 index 00000000000..de833619c8d --- /dev/null +++ b/lib/gitlab/ci/config/yaml.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + + class << self + def load!(content) + ensure_custom_tags + + Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + end + + private + + def ensure_custom_tags + @ensure_custom_tags ||= begin + AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + + true + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags.rb b/lib/gitlab/ci/config/yaml/tags.rb new file mode 100644 index 00000000000..1575edad3b0 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + TagError = Class.new(StandardError) + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/base.rb b/lib/gitlab/ci/config/yaml/tags/base.rb new file mode 100644 index 00000000000..13416a4afb6 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/base.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + class Base + CircularReferenceError = Class.new(Tags::TagError) + NotValidError = Class.new(Tags::TagError) + + extend ::Gitlab::Utils::Override + + attr_accessor :resolved_status, :resolved_value, :data + + def self.tag + raise NotImplementedError + end + + # Only one of the `seq`, `scalar`, `map` fields is available. + def init_with(coder) + @data = { + tag: coder.tag, # This is the custom YAML tag, like !reference or !flatten + style: coder.style, + seq: coder.seq, # This holds Array data + scalar: coder.scalar, # This holds data of basic types, like String. + map: coder.map # This holds Hash data. + } + end + + def valid? + raise NotImplementedError + end + + def resolve(resolver) + raise NotValidError, validation_error_message unless valid? + raise CircularReferenceError, circular_error_message if resolving? + return resolved_value if resolved? + + self.resolved_status = :in_progress + self.resolved_value = _resolve(resolver) + self.resolved_status = :done + resolved_value + end + + private + + def _resolve(resolver) + raise NotImplementedError + end + + def resolved? + resolved_status == :done + end + + def resolving? + resolved_status == :in_progress + end + + def circular_error_message + "#{data[:tag]} #{data[:seq].inspect} is part of a circular chain" + end + + def validation_error_message + "#{data[:tag]} #{(data[:scalar].presence || data[:map].presence || data[:seq]).inspect} is not valid" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb new file mode 100644 index 00000000000..22822614b67 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + class Reference < Base + MissingReferenceError = Class.new(Tags::TagError) + + def self.tag + '!reference' + end + + override :valid? + def valid? + data[:seq].is_a?(Array) && + !data[:seq].empty? && + data[:seq].all? { |identifier| identifier.is_a?(String) } + end + + private + + def location + data[:seq].to_a.map(&:to_sym) + end + + override :_resolve + def _resolve(resolver) + object = resolver.config.dig(*location) + value = resolver.deep_resolve(object) + + raise MissingReferenceError, missing_ref_error_message unless value + + value + end + + def missing_ref_error_message + "#{data[:tag]} #{data[:seq].inspect} could not be found" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/resolver.rb b/lib/gitlab/ci/config/yaml/tags/resolver.rb new file mode 100644 index 00000000000..e207ec296b6 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/resolver.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + # This class is the entry point for transforming custom YAML tags back + # into primitive objects. + # Usage: `Resolver.new(a_hash_including_custom_tag_objects).to_hash` + # + class Resolver + attr_reader :config + + def initialize(config) + @config = config.deep_dup + end + + def to_hash + deep_resolve(config) + end + + def deep_resolve(object) + case object + when Array + object.map(&method(:resolve_wrapper)) + when Hash + object.deep_transform_values(&method(:resolve_wrapper)) + else + resolve_wrapper(object) + end + end + + def resolve_wrapper(object) + if object.respond_to?(:resolve) + object.resolve(self) + else + object + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 7956cf14203..d1a366125ef 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -55,21 +55,30 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.pipeline_open_merge_requests?(project) - ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true) - end - def self.ci_pipeline_editor_page_enabled?(project) ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) end - def self.allow_failure_with_exit_codes_enabled? - ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml) - end - def self.rules_variables_enabled?(project) ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) end + + def self.validate_build_dependencies?(project) + ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && + ::Feature.disabled?(:ci_validate_build_dependencies_override, project) + end + + def self.display_quality_on_mr_diff?(project) + ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) + end + + def self.display_codequality_backend_comparison?(project) + ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) + end + + def self.use_coverage_data_new_finder?(record) + ::Feature.enabled?(:coverage_data_new_finder, record, default_enabled: :yaml) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index a8943eadf4f..0870c74053a 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -45,7 +45,7 @@ module Gitlab end def custom_claims - { + fields = { namespace_id: namespace.id.to_s, namespace_path: namespace.full_path, project_id: project.id.to_s, @@ -59,6 +59,15 @@ module Gitlab ref_type: ref_type, ref_protected: build.protected.to_s } + + if include_environment_claims? + fields.merge!( + environment: environment.name, + environment_protected: environment_protected?.to_s + ) + end + + fields end def key @@ -102,6 +111,20 @@ module Gitlab def ref_type ::Ci::BuildRunnerPresenter.new(build).ref_type end + + def environment + build.persisted_environment + end + + def environment_protected? + false # Overridden in EE + end + + def include_environment_claims? + Feature.enabled?(:ci_jwt_include_environment) && environment.present? + end end end end + +Gitlab::Ci::Jwt.prepend_if_ee('::EE::Gitlab::Ci::Jwt') diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 985639982aa..2baa8faf849 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -20,6 +20,10 @@ module Gitlab rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end + + def self.instrument! + parsers.values.each { |parser_class| parser_class.prepend(Parsers::Instrumentation) } + end end end end diff --git a/lib/gitlab/ci/parsers/instrumentation.rb b/lib/gitlab/ci/parsers/instrumentation.rb new file mode 100644 index 00000000000..ab4a923d9aa --- /dev/null +++ b/lib/gitlab/ci/parsers/instrumentation.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Instrumentation + BUCKETS = [0.25, 1, 5, 10].freeze + + def parse!(*args) + parser_result = nil + + duration = Benchmark.realtime do + parser_result = super + end + + labels = {} + + histogram = Gitlab::Metrics.histogram( + :ci_report_parser_duration_seconds, + 'Duration of parsing a CI report artifact', + labels, + BUCKETS + ) + + histogram.observe({ parser: self.class.name }, duration) + + parser_result + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index f0548284001..d3bc3a38f1f 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -23,7 +23,7 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - locked: @command.project.latest_pipeline_locked, + locked: @command.project.default_pipeline_lock, variables_attributes: variables_attributes ) end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 2ca51930c19..f0214bb4e38 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -25,7 +25,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - pipelines + project.all_pipelines.ci_and_parent_sources .where(ref: pipeline.ref) .where.not(id: pipeline.same_family_pipeline_ids) .where.not(sha: project.commit(pipeline.ref).try(:id)) @@ -33,14 +33,6 @@ module Gitlab .with_only_interruptible_builds end # rubocop: enable CodeReuse/ActiveRecord - - def pipelines - if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: true) - project.all_pipelines.ci_and_parent_sources - else - project.ci_pipelines - end - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index 5314fd471c3..a7680f6e593 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -34,16 +34,22 @@ module Gitlab private def find_config - SOURCES.each do |source| + sources.each do |source| config = source.new(@pipeline, @command) return config if config.exists? end nil end + + def sources + SOURCES + end end end end end end end + +Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb index c1a7b4ed453..2fcf1740b5f 100644 --- a/lib/gitlab/ci/pipeline/chain/template_usage.rb +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -19,7 +19,7 @@ module Gitlab def track_event(template) Gitlab::UsageDataCounters::CiTemplateUniqueCounter - .track_unique_project_event(project_id: pipeline.project_id, template: template) + .track_unique_project_event(project_id: pipeline.project_id, template: template, config_source: pipeline.config_source) end def included_templates diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index db6cca27f1c..c77f4dcca5a 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -45,6 +45,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def legacy_update_jobs_counter + strong_memoize(:legacy_update_jobs_counter) do + name = :ci_legacy_update_jobs_as_retried_total + comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index fe3c2bca551..3770bb4b328 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -73,17 +73,28 @@ module Gitlab def to_resource strong_memoize(:resource) do - if bridge? - ::Ci::Bridge.new(attributes) - else - ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.environment_attributes_for(build)) - build.resource_group = Seed::Build::ResourceGroup.new(build, @resource_group_key).to_resource - end + processable = initialize_processable + assign_resource_group(processable) + processable + end + end + + def initialize_processable + if bridge? + ::Ci::Bridge.new(attributes) + else + ::Ci::Build.new(attributes).tap do |build| + build.assign_attributes(self.class.environment_attributes_for(build)) end end end + def assign_resource_group(processable) + processable.resource_group = + Seed::Processable::ResourceGroup.new(processable, @resource_group_key) + .to_resource + end + def self.environment_attributes_for(build) return {} unless build.has_environment? @@ -159,7 +170,11 @@ module Gitlab next {} unless @using_rules if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) - rules_result.build_attributes(@seed_attributes) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) + + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) else rules_result.build_attributes end @@ -188,7 +203,6 @@ module Gitlab # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. def allow_failure_criteria_attributes - return {} unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? return {} if rules_attributes[:allow_failure].nil? return {} unless @seed_attributes.dig(:options, :allow_failure_criteria) diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb index c0641d9ff0a..f8ea6d4184c 100644 --- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb @@ -4,21 +4,21 @@ module Gitlab module Ci module Pipeline module Seed - class Build + module Processable class ResourceGroup < Seed::Base include Gitlab::Utils::StrongMemoize - attr_reader :build, :resource_group_key + attr_reader :processable, :resource_group_key - def initialize(build, resource_group_key) - @build = build + def initialize(processable, resource_group_key) + @processable = processable @resource_group_key = resource_group_key end def to_resource return unless resource_group_key.present? - resource_group = build.project.resource_groups + resource_group = processable.project.resource_groups .safe_find_or_create_by(key: expanded_resource_group_key) resource_group if resource_group.persisted? @@ -28,7 +28,7 @@ module Gitlab def expanded_resource_group_key strong_memoize(:expanded_resource_group_key) do - ExpandVariables.expand(resource_group_key, -> { build.simple_variables }) + ExpandVariables.expand(resource_group_key, -> { processable.simple_variables }) end end end diff --git a/lib/gitlab/ci/reports/codequality_mr_diff.rb b/lib/gitlab/ci/reports/codequality_mr_diff.rb new file mode 100644 index 00000000000..e60a075e3f5 --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_mr_diff.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityMrDiff + attr_reader :files + + def initialize(raw_report) + @raw_report = raw_report + @files = {} + build_report! + end + + private + + def build_report! + codequality_files = @raw_report.all_degradations.each_with_object({}) do |degradation, codequality_files| + unless codequality_files[degradation.dig(:location, :path)].present? + codequality_files[degradation.dig(:location, :path)] = [] + end + + build_mr_diff_payload(codequality_files, degradation) + end + + @files = codequality_files + end + + def build_mr_diff_payload(codequality_files, degradation) + codequality_files[degradation.dig(:location, :path)] << { + line: degradation.dig(:location, :lines, :begin) || degradation.dig(:location, :positions, :begin, :line), + description: degradation[:description], + severity: degradation[:severity] + } + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index b9bd66cee71..4d5a94a3beb 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -8,6 +8,7 @@ module Gitlab def self.extended_statuses [[Status::Bridge::Failed], [Status::Bridge::Manual], + [Status::Bridge::WaitingForResource], [Status::Bridge::Play], [Status::Bridge::Action]] end diff --git a/lib/gitlab/ci/status/bridge/waiting_for_resource.rb b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb new file mode 100644 index 00000000000..d2c8f71a609 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class WaitingForResource < Status::Processable::WaitingForResource + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/waiting_for_resource.rb b/lib/gitlab/ci/status/build/waiting_for_resource.rb index 008e6a17bdd..5dcc060a990 100644 --- a/lib/gitlab/ci/status/build/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/build/waiting_for_resource.rb @@ -4,22 +4,7 @@ module Gitlab module Ci module Status module Build - class WaitingForResource < Status::Extended - ## - # TODO: image is shared with 'pending' - # until we get a dedicated one - # - def illustration - { - image: 'illustrations/pending_job_empty.svg', - size: 'svg-430', - title: _('This job is waiting for resource: ') + subject.resource_group.key - } - end - - def self.matches?(build, _) - build.waiting_for_resource? - end + class WaitingForResource < Status::Processable::WaitingForResource end end end diff --git a/lib/gitlab/ci/status/processable/waiting_for_resource.rb b/lib/gitlab/ci/status/processable/waiting_for_resource.rb new file mode 100644 index 00000000000..c9b1dd795d0 --- /dev/null +++ b/lib/gitlab/ci/status/processable/waiting_for_resource.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Processable + class WaitingForResource < Status::Extended + ## + # TODO: image is shared with 'pending' + # until we get a dedicated one + # + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job is waiting for resource: ') + subject.resource_group.key + } + end + + def self.matches?(processable, _) + processable.waiting_for_resource? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 501d8737acd..daed75a42ee 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.22" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 192b1509fdc..6f30fc2dcd5 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.40.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 84bb0ff3b33..8f64da24410 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -40,14 +40,14 @@ verify:jdk8: <<: *verify # To deploy packages from CI, create a ci_settings.xml file -# For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for more details. +# For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for more details. # Please note: The GitLab Maven Repository is currently only available in GitLab Premium / Ultimate. # For `master` branch run `mvn deploy` automatically. deploy:jdk8: stage: deploy script: - if [ ! -f ci_settings.xml ]; - then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for instructions."; + then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for instructions."; fi - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' only: diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 63237e41376..21e926ef275 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/ +# Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # # Configure the scanning tool through the environment variables. # List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings @@ -21,7 +21,6 @@ license_scanning: LM_REPORT_VERSION: '2.1' SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD allow_failure: true - needs: [] script: - /run.sh analyze . artifacts: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 56c6fbd96bc..828352743b4 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf, semgrep" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 @@ -66,7 +66,8 @@ brakeman-sast: - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: - - 'config/routes.rb' + - '**/*.rb' + - '**/Gemfile' eslint-sast: extends: .sast-analyzer @@ -243,6 +244,23 @@ security-code-scan-sast: - '**/*.csproj' - '**/*.vbproj' +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_DEFAULT_ANALYZERS =~ /semgrep/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.py' + sobelow-sast: extends: .sast-analyzer image: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 0222ca021b7..3258d965c93 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -182,7 +182,7 @@ module Gitlab if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| archive_stream!(stream) - stream.destroy! + destroy_stream(job) { stream.destroy! } end elsif current_path File.open(current_path) do |stream| @@ -268,7 +268,21 @@ module Gitlab end def trace_artifact - job.job_artifacts_trace + read_trace_artifact(job) { job.job_artifacts_trace } + end + + ## + # Overridden in EE + # + def destroy_stream(job) + yield + end + + ## + # Overriden in EE + # + def read_trace_artifact(job) + yield end def being_watched_cache_key @@ -277,3 +291,5 @@ module Gitlab end end end + +::Gitlab::Ci::Trace.prepend_if_ee('EE::Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb index 7cdb6a6c03c..92bed817875 100644 --- a/lib/gitlab/ci/trace/checksum.rb +++ b/lib/gitlab/ci/trace/checksum.rb @@ -30,7 +30,11 @@ module Gitlab end def state_crc32 - strong_memoize(:state_crc32) { build.pending_state&.crc32 } + strong_memoize(:state_crc32) do + ::Gitlab::Database::Consistency.with_read_consistency do + build.pending_state&.crc32 + end + end end def chunks_crc32 @@ -59,8 +63,10 @@ module Gitlab # def trace_chunks strong_memoize(:trace_chunks) do - build.trace_chunks.persisted - .select(::Ci::BuildTraceChunk.metadata_attributes) + ::Ci::BuildTraceChunk.with_read_consistency(build) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end end end diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 6f3e4ccf48d..7c2e39b1e53 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -227,12 +227,20 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def build_chunk - @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) + def next_chunk + @chunks_cache[chunk_index] = begin + if ::Ci::BuildTraceChunk.consistent_reads_enabled?(build) + ::Ci::BuildTraceChunk + .safe_find_or_create_by(build: build, chunk_index: chunk_index) + else + ::Ci::BuildTraceChunk + .new(build: build, chunk_index: chunk_index) + end + end end def ensure_chunk - current_chunk || build_chunk + current_chunk || next_chunk || current_chunk end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb index 6abc6a5644f..e641df10462 100644 --- a/lib/gitlab/ci/variables/collection/sorted.rb +++ b/lib/gitlab/ci/variables/collection/sorted.rb @@ -8,8 +8,9 @@ module Gitlab include TSort include Gitlab::Utils::StrongMemoize - def initialize(variables) + def initialize(variables, project) @variables = variables + @project = project end def valid? @@ -19,7 +20,7 @@ module Gitlab # errors sorts an array of variables, ignoring unknown variable references, # and returning an error string if a circular variable reference is found def errors - return if Feature.disabled?(:variable_inside_variable) + return if Feature.disabled?(:variable_inside_variable, @project) strong_memoize(:errors) do # Check for cyclic dependencies and build error message in that case @@ -34,7 +35,7 @@ module Gitlab # sort sorts an array of variables, ignoring unknown variable references. # If a circular variable reference is found, the original array is returned def sort - return @variables if Feature.disabled?(:variable_inside_variable) + return @variables if Feature.disabled?(:variable_inside_variable, @project) return @variables if errors tsort diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb new file mode 100644 index 00000000000..e2a54f90ecb --- /dev/null +++ b/lib/gitlab/ci/variables/helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Helpers + class << self + def merge_variables(current_vars, new_vars) + current_vars = transform_from_yaml_variables(current_vars) + new_vars = transform_from_yaml_variables(new_vars) + + transform_to_yaml_variables( + current_vars.merge(new_vars) + ) + end + + def transform_to_yaml_variables(vars) + vars.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def transform_from_yaml_variables(vars) + return vars.stringify_keys if vars.is_a?(Hash) + + vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 86749cda9c7..3459b69bebc 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -123,9 +123,7 @@ module Gitlab end def transform_to_yaml_variables(variables) - variables.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) end end end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 6d18f9070cc..48a1ab23fc2 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -12,10 +12,9 @@ module Gitlab VALID_NICENESS_LEVELS = %w{none realtime best-effort idle}.freeze attr_accessor :batch, :total_found, :total_cleaned - attr_reader :limit, :dry_run, :niceness, :logger + attr_reader :dry_run, :niceness, :logger - def initialize(limit: nil, dry_run: true, niceness: nil, logger: nil) - @limit = limit + def initialize(dry_run: true, niceness: nil, logger: nil) @dry_run = dry_run @niceness = (niceness || DEFAULT_NICENESS).downcase @logger = logger || Gitlab::AppLogger @@ -31,7 +30,11 @@ module Gitlab batch << artifact_file clean_batch! if batch.full? - break if limit_reached? + + if limit_reached? + log_info("Exiting due to reaching limit of #{limit}.") + break + end end clean_batch! @@ -128,6 +131,10 @@ module Gitlab def log_error(msg, params = {}) logger.error(msg) end + + def limit + ENV['LIMIT']&.to_i + end end end end diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index a6638b2cbc8..99e7550629a 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -5,15 +5,14 @@ module Gitlab class OrphanLfsFileReferences include Gitlab::Utils::StrongMemoize - attr_reader :project, :dry_run, :logger, :limit + attr_reader :project, :dry_run, :logger DEFAULT_REMOVAL_LIMIT = 1000 - def initialize(project, dry_run: true, logger: nil, limit: nil) + def initialize(project, dry_run: true, logger: nil) @project = project @dry_run = dry_run @logger = logger || Gitlab::AppLogger - @limit = limit end def run! @@ -67,6 +66,10 @@ module Gitlab def log_info(msg) logger.info("#{'[DRY RUN] ' if dry_run}#{msg}") end + + def limit + ENV['LIMIT']&.to_i + end end end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 4ae75e0db0a..3c71ca9fcf0 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../utils' # Gitlab::Utils + module Gitlab module Cluster # @@ -64,6 +66,10 @@ module Gitlab # Blocks will be executed in the order in which they are registered. # class LifecycleEvents + FatalError = Class.new(Exception) # rubocop:disable Lint/InheritException + + USE_FATAL_LIFECYCLE_EVENTS = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_FATAL_LIFECYCLE_EVENTS', 'true')) + class << self # # Hook registration methods (called from initializers) @@ -111,24 +117,24 @@ module Gitlab # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) # def do_worker_start - call(@worker_start_hooks) + call(:worker_start_hooks, @worker_start_hooks) end def do_before_fork - call(@before_fork_hooks) + call(:before_fork_hooks, @before_fork_hooks) end def do_before_graceful_shutdown - call(@master_blackout_period) + call(:master_blackout_period, @master_blackout_period) blackout_seconds = ::Settings.shutdown.blackout_seconds.to_i sleep(blackout_seconds) if blackout_seconds > 0 - call(@master_graceful_shutdown) + call(:master_graceful_shutdown, @master_graceful_shutdown) end def do_before_master_restart - call(@master_restart_hooks) + call(:master_restart_hooks, @master_restart_hooks) end # DEPRECATED @@ -143,8 +149,18 @@ module Gitlab private - def call(hooks) - hooks&.each(&:call) + def call(name, hooks) + return unless hooks + + hooks.each do |hook| + hook.call + rescue => e + Gitlab::ErrorTracking.track_exception(e, type: 'LifecycleEvents', hook: hook) + warn("ERROR: The hook #{name} failed with exception (#{e.class}) \"#{e.message}\".") + + # we consider lifecycle hooks to be fatal errors + raise FatalError, e if USE_FATAL_LIFECYCLE_EVENTS + end end def in_clustered_environment? diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 822012e0ed6..fd9f58a34f3 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -35,6 +35,10 @@ module Gitlab # regularly rather than rely on OOM behavior for periodic restarting. config.rolling_restart_frequency = 43200 # 12 hours in seconds. + # Spread the rolling restarts out over 1 hour to avoid too many simultaneous + # process startups. + config.rolling_restart_splay_seconds = 0.0..3600.0 # 0 to 1 hour in seconds. + observer = Gitlab::Cluster::PumaWorkerKillerObserver.new config.pre_term = observer.callback end diff --git a/lib/gitlab/composer/cache.rb b/lib/gitlab/composer/cache.rb new file mode 100644 index 00000000000..1f404d63047 --- /dev/null +++ b/lib/gitlab/composer/cache.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'tempfile' + +module Gitlab + module Composer + class Cache + def initialize(project:, name:, last_page_sha: nil) + @project = project + @name = name + @last_page_sha = last_page_sha + end + + def execute + Packages::Composer::Metadatum.transaction do # rubocop: disable CodeReuse/ActiveRecord + # make sure we lock these records at the start + locked_package_metadata + + if locked_package_metadata.any? + mark_pages_for_delete(shas_to_delete) + + create_cache_page! + + # assign the newest page SHA to the packages + locked_package_metadata.update_all(version_cache_sha: version_index.sha) + elsif @last_page_sha + mark_pages_for_delete([@last_page_sha]) + end + end + end + + private + + def mark_pages_for_delete(shas) + Packages::Composer::CacheFile + .with_namespace(@project.namespace) + .with_sha(shas) + .update_all(delete_at: 1.day.from_now) + end + + def create_cache_page! + Packages::Composer::CacheFile + .safe_find_or_create_by!(namespace_id: @project.namespace_id, file_sha256: version_index.sha) do |cache_file| + cache_file.file = CarrierWaveStringFile.new(version_index.to_json) + end + end + + def version_index + @version_index ||= ::Gitlab::Composer::VersionIndex.new(siblings) + end + + def siblings + @siblings ||= locked_package_metadata.map(&:package) + end + + # find all metadata of the package versions and lock it for update + def locked_package_metadata + @locked_package_metadata ||= Packages::Composer::Metadatum + .for_package(@name, @project.id) + .locked_for_update + end + + def shas_to_delete + locked_package_metadata + .map(&:version_cache_sha) + .reject { |sha| sha == version_index.sha } + .compact + end + end + end +end diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index de9a17a453f..ac0071cdc53 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -20,7 +20,7 @@ module Gitlab private def package_versions_map - @packages.each_with_object({}) do |package, map| + @packages.sort_by(&:version).each_with_object({}) do |package, map| map[package.version] = package_metadata(package) end end diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb index 7526c10b608..d03997b4158 100644 --- a/lib/gitlab/conan_token.rb +++ b/lib/gitlab/conan_token.rb @@ -35,7 +35,7 @@ module Gitlab def secret OpenSSL::HMAC.hexdigest( - OpenSSL::Digest::SHA256.new, + OpenSSL::Digest.new('SHA256'), ::Settings.attr_encrypted_db_key_base, HMAC_KEY ) diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 88786ed82ff..8120f2c1243 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -268,17 +268,16 @@ module Gitlab end end - class StringOrNestedArrayOfStringsValidator < NestedArrayOfStringsValidator - def validate_each(record, attribute, value) - unless validate_string_or_nested_array_of_strings(value) - record.errors.add(attribute, 'should be a string or an array containing strings and arrays of strings') - end - end + class StringOrNestedArrayOfStringsValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + include NestedArrayHelpers - private + def validate_each(record, attribute, value) + max_level = options.fetch(:max_level, 1) - def validate_string_or_nested_array_of_strings(values) - validate_string(values) || validate_nested_array_of_strings(values) + unless validate_string(value) || validate_nested_array(value, max_level, &method(:validate_string)) + record.errors.add(attribute, "should be a string or a nested array of strings up to #{max_level} levels deep") + end end end diff --git a/lib/gitlab/config/entry/validators/nested_array_helpers.rb b/lib/gitlab/config/entry/validators/nested_array_helpers.rb new file mode 100644 index 00000000000..9f5d17d74b0 --- /dev/null +++ b/lib/gitlab/config/entry/validators/nested_array_helpers.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Validators + # Include this module to validate deeply nested array of values + # + # class MyNestedValidator < ActiveModel::EachValidator + # include NestedArrayHelpers + # + # def validate_each(record, attribute, value) + # max_depth = options.fetch(:max_depth, 1) + # + # unless validate_nested_array(value, max_depth) { |v| v.is_a?(Integer) } + # record.errors.add(attribute, "is invalid") + # end + # end + # end + # + module NestedArrayHelpers + def validate_nested_array(value, max_depth = 1, &validator_proc) + return false unless value.is_a?(Array) + + validate_nested_array_recursively(value, max_depth, &validator_proc) + end + + private + + # rubocop: disable Performance/RedundantBlockCall + # Disables Rubocop rule for easier readability reasons. + def validate_nested_array_recursively(value, nesting_level, &validator_proc) + return true if validator_proc.call(value) + return false if nesting_level <= 0 + return false unless value.is_a?(Array) + + value.all? do |element| + validate_nested_array_recursively(element, nesting_level - 1, &validator_proc) + end + end + # rubocop: enable Performance/RedundantBlockCall + end + end + end + end +end diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index cb3fc49944c..80c9abecd8e 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -12,8 +12,12 @@ module Gitlab MAX_YAML_SIZE = 1.megabyte MAX_YAML_DEPTH = 100 - def initialize(config) - @config = YAML.safe_load(config, [Symbol], [], true) + def initialize(config, additional_permitted_classes: []) + @config = YAML.safe_load(config, + permitted_classes: [Symbol, *additional_permitted_classes], + permitted_symbols: [], + aliases: true + ) rescue Psych::Exception => e raise Loader::FormatError, e.message end diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 87a03d9c58f..4428354642d 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -6,25 +6,44 @@ module Gitlab AES256_GCM_OPTIONS = { algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32, - iv: Settings.attr_encrypted_db_key_base_12 + key: Settings.attr_encrypted_db_key_base_32 }.freeze + AES256_GCM_IV_STATIC = Settings.attr_encrypted_db_key_base_12 + def sha256(value) salt = Settings.attr_encrypted_db_key_base_truncated ::Digest::SHA256.base64digest("#{value}#{salt}") end - def aes256_gcm_encrypt(value) - encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) - Base64.strict_encode64(encrypted_token) + def aes256_gcm_encrypt(value, nonce: nil) + aes256_gcm_encrypt_using_static_nonce(value) end def aes256_gcm_decrypt(value) return unless value + nonce = Feature.enabled?(:dynamic_nonce_creation) ? dynamic_nonce(value) : AES256_GCM_IV_STATIC encrypted_token = Base64.decode64(value) - Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token)) + decrypted_token = Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce)) + decrypted_token + end + + def dynamic_nonce(value) + TokenWithIv.find_nonce_by_hashed_token(value) || AES256_GCM_IV_STATIC + end + + def aes256_gcm_encrypt_using_static_nonce(value) + create_encrypted_token(value, AES256_GCM_IV_STATIC) + end + + def read_only? + Gitlab::Database.read_only? + end + + def create_encrypted_token(value, iv) + encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: iv)) + Base64.strict_encode64(encrypted_token) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index d0579a44219..0bf41f9dc0d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,6 +7,10 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end + def current_application_settings? + Gitlab::SafeRequestStore.exist?(:current_application_settings) || ::ApplicationSetting.current.present? + end + def expire_current_application_settings ::ApplicationSetting.expire Gitlab::SafeRequestStore.delete(:current_application_settings) diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 5125c8e64ee..aaa2554dbfa 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -15,9 +15,16 @@ module Gitlab private def deployments_count - query = @project.deployments.success.where("created_at >= ?", @from) - query = query.where("created_at <= ?", @to) if @to - query.count + if Feature.enabled?(:query_deploymenys_via_finished_at_in_vsa, default_enabled: :yaml) + DeploymentsFinder + .new(project: @project, finished_after: @from, finished_before: @to, status: :success) + .execute + .count + else + query = @project.deployments.success.where("created_at >= ?", @from) + query = query.where("created_at <= ?", @to) if @to + query.count + end end end end diff --git a/lib/gitlab/danger/base_linter.rb b/lib/gitlab/danger/base_linter.rb deleted file mode 100644 index 898434724bd..00000000000 --- a/lib/gitlab/danger/base_linter.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -require_relative 'title_linting' - -module Gitlab - module Danger - class BaseLinter - MIN_SUBJECT_WORDS_COUNT = 3 - MAX_LINE_LENGTH = 72 - - attr_reader :commit, :problems - - def self.problems_mapping - { - subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words", - subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters", - subject_starts_with_lowercase: "The %s must start with a capital letter", - subject_ends_with_a_period: "The %s must not end with a period" - } - end - - def self.subject_description - 'commit subject' - end - - def initialize(commit) - @commit = commit - @problems = {} - end - - def failed? - problems.any? - end - - def add_problem(problem_key, *args) - @problems[problem_key] = sprintf(self.class.problems_mapping[problem_key], *args) - end - - def lint_subject - if subject_too_short? - add_problem(:subject_too_short, self.class.subject_description) - end - - if subject_too_long? - add_problem(:subject_too_long, self.class.subject_description) - end - - if subject_starts_with_lowercase? - add_problem(:subject_starts_with_lowercase, self.class.subject_description) - end - - if subject_ends_with_a_period? - add_problem(:subject_ends_with_a_period, self.class.subject_description) - end - - self - end - - private - - def subject - TitleLinting.remove_draft_flag(message_parts[0]) - end - - def subject_too_short? - subject.split(' ').length < MIN_SUBJECT_WORDS_COUNT - end - - def subject_too_long? - line_too_long?(subject) - end - - def line_too_long?(line) - line.length > MAX_LINE_LENGTH - end - - def subject_starts_with_lowercase? - return false if ('A'..'Z').cover?(subject[0]) - - first_char = subject.sub(/\A(\[.+\]|\w+:)\s/, '')[0] - first_char_downcased = first_char.downcase - return true unless ('a'..'z').cover?(first_char_downcased) - - first_char.downcase == first_char - end - - def subject_ends_with_a_period? - subject.end_with?('.') - end - - def message_parts - @message_parts ||= commit.message.split("\n", 3) - end - end - end -end diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb deleted file mode 100644 index 4b85775ed98..00000000000 --- a/lib/gitlab/danger/changelog.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -require_relative 'title_linting' - -module Gitlab - module Danger - module Changelog - NO_CHANGELOG_LABELS = [ - 'tooling', - 'tooling::pipelines', - 'tooling::workflow', - 'ci-build', - 'meta' - ].freeze - NO_CHANGELOG_CATEGORIES = %i[docs none].freeze - CREATE_CHANGELOG_COMMAND = 'bin/changelog -m %<mr_iid>s "%<mr_title>s"' - CREATE_EE_CHANGELOG_COMMAND = 'bin/changelog --ee -m %<mr_iid>s "%<mr_title>s"' - CHANGELOG_MODIFIED_URL_TEXT = "**CHANGELOG.md was edited.** Please remove the additions and create a CHANGELOG entry.\n\n" - CHANGELOG_MISSING_URL_TEXT = "**[CHANGELOG missing](https://docs.gitlab.com/ee/development/changelog.html)**:\n\n" - - OPTIONAL_CHANGELOG_MESSAGE = <<~MSG - If you want to create a changelog entry for GitLab FOSS, run the following: - - #{CREATE_CHANGELOG_COMMAND} - - If you want to create a changelog entry for GitLab EE, run the following instead: - - #{CREATE_EE_CHANGELOG_COMMAND} - - If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message. - MSG - - REQUIRED_CHANGELOG_MESSAGE = <<~MSG - To create a changelog entry, run the following: - - #{CREATE_CHANGELOG_COMMAND} - - This merge request requires a changelog entry because it [introduces a database migration](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry). - MSG - - def required? - git.added_files.any? { |path| path =~ %r{\Adb/(migrate|post_migrate)/} } - end - alias_method :db_changes?, :required? - - def optional? - categories_need_changelog? && without_no_changelog_label? - end - - def found - @found ||= git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} } - end - - def ee_changelog? - found.start_with?('ee/') - end - - def modified_text - CHANGELOG_MODIFIED_URL_TEXT + - format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - def required_text - CHANGELOG_MISSING_URL_TEXT + - format(REQUIRED_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - def optional_text - CHANGELOG_MISSING_URL_TEXT + - format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - private - - def mr_iid - gitlab.mr_json["iid"] - end - - def sanitized_mr_title - TitleLinting.sanitize_mr_title(gitlab.mr_json["title"]) - end - - def categories_need_changelog? - (helper.changes_by_category.keys - NO_CHANGELOG_CATEGORIES).any? - end - - def without_no_changelog_label? - (gitlab.mr_labels & NO_CHANGELOG_LABELS).empty? - end - end - end -end diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb deleted file mode 100644 index e23f5900433..00000000000 --- a/lib/gitlab/danger/commit_linter.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: true - -emoji_checker_path = File.expand_path('emoji_checker', __dir__) -base_linter_path = File.expand_path('base_linter', __dir__) - -if defined?(Rails) - require_dependency(base_linter_path) - require_dependency(emoji_checker_path) -else - require_relative(base_linter_path) - require_relative(emoji_checker_path) -end - -module Gitlab - module Danger - class CommitLinter < BaseLinter - MAX_CHANGED_FILES_IN_COMMIT = 3 - MAX_CHANGED_LINES_IN_COMMIT = 30 - SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(?<!`)(#|!|&|%)\d+(?<!`)}.freeze - - def self.problems_mapping - super.merge( - { - separator_missing: "The commit subject and body must be separated by a blank line", - details_too_many_changes: "Commits that change #{MAX_CHANGED_LINES_IN_COMMIT} or more lines across " \ - "at least #{MAX_CHANGED_FILES_IN_COMMIT} files must describe these changes in the commit body", - details_line_too_long: "The commit body should not contain more than #{MAX_LINE_LENGTH} characters per line", - message_contains_text_emoji: "Avoid the use of Markdown Emoji such as `:+1:`. These add limited value " \ - "to the commit message, and are displayed as plain text outside of GitLab", - message_contains_unicode_emoji: "Avoid the use of Unicode Emoji. These add no value to the commit " \ - "message, and may not be displayed properly everywhere", - message_contains_short_reference: "Use full URLs instead of short references (`gitlab-org/gitlab#123` or " \ - "`!123`), as short references are displayed as plain text outside of GitLab" - } - ) - end - - def initialize(commit) - super - - @linted = false - end - - def fixup? - commit.message.start_with?('fixup!', 'squash!') - end - - def suggestion? - commit.message.start_with?('Apply suggestion to') - end - - def merge? - commit.message.start_with?('Merge branch') - end - - def revert? - commit.message.start_with?('Revert "') - end - - def multi_line? - !details.nil? && !details.empty? - end - - def lint - return self if @linted - - @linted = true - lint_subject - lint_separator - lint_details - lint_message - - self - end - - private - - def lint_separator - return self unless separator && !separator.empty? - - add_problem(:separator_missing) - - self - end - - def lint_details - if !multi_line? && many_changes? - add_problem(:details_too_many_changes) - end - - details&.each_line do |line| - line_without_urls = line.strip.gsub(%r{https?://\S+}, '') - - # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but - # only if the line _without_ the URL does not exceed this limit. - next unless line_too_long?(line_without_urls) - - add_problem(:details_line_too_long) - break - end - - self - end - - def lint_message - if message_contains_text_emoji? - add_problem(:message_contains_text_emoji) - end - - if message_contains_unicode_emoji? - add_problem(:message_contains_unicode_emoji) - end - - if message_contains_short_reference? - add_problem(:message_contains_short_reference) - end - - self - end - - def files_changed - commit.diff_parent.stats[:total][:files] - end - - def lines_changed - commit.diff_parent.stats[:total][:lines] - end - - def many_changes? - files_changed > MAX_CHANGED_FILES_IN_COMMIT && lines_changed > MAX_CHANGED_LINES_IN_COMMIT - end - - def separator - message_parts[1] - end - - def details - message_parts[2]&.gsub(/^Signed-off-by.*$/, '') - end - - def message_contains_text_emoji? - emoji_checker.includes_text_emoji?(commit.message) - end - - def message_contains_unicode_emoji? - emoji_checker.includes_unicode_emoji?(commit.message) - end - - def message_contains_short_reference? - commit.message.match?(SHORT_REFERENCE_REGEX) - end - - def emoji_checker - @emoji_checker ||= Gitlab::Danger::EmojiChecker.new - end - end - end -end diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb deleted file mode 100644 index e31a6ae5011..00000000000 --- a/lib/gitlab/danger/emoji_checker.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -module Gitlab - module Danger - class EmojiChecker - DIGESTS = File.expand_path('../../../fixtures/emojis/digests.json', __dir__) - ALIASES = File.expand_path('../../../fixtures/emojis/aliases.json', __dir__) - - # A regex that indicates a piece of text _might_ include an Emoji. The regex - # alone is not enough, as we'd match `:foo:bar:baz`. Instead, we use this - # regex to save us from having to check for all possible emoji names when we - # know one definitely is not included. - LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/.freeze - - UNICODE_EMOJI_REGEX = %r{( - [\u{1F300}-\u{1F5FF}] | - [\u{1F1E6}-\u{1F1FF}] | - [\u{2700}-\u{27BF}] | - [\u{1F900}-\u{1F9FF}] | - [\u{1F600}-\u{1F64F}] | - [\u{1F680}-\u{1F6FF}] | - [\u{2600}-\u{26FF}] - )}x.freeze - - def initialize - names = JSON.parse(File.read(DIGESTS)).keys + - JSON.parse(File.read(ALIASES)).keys - - @emoji = names.map { |name| ":#{name}:" } - end - - def includes_text_emoji?(text) - return false unless text.match?(LIKELY_EMOJI) - - @emoji.any? { |emoji| text.include?(emoji) } - end - - def includes_unicode_emoji?(text) - text.match?(UNICODE_EMOJI_REGEX) - end - end - end -end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb deleted file mode 100644 index 09e013e24b8..00000000000 --- a/lib/gitlab/danger/helper.rb +++ /dev/null @@ -1,273 +0,0 @@ -# frozen_string_literal: true - -require_relative 'teammate' -require_relative 'title_linting' - -module Gitlab - module Danger - module Helper - RELEASE_TOOLS_BOT = 'gitlab-release-tools-bot' - - # Returns a list of all files that have been added, modified or renamed. - # `git.modified_files` might contain paths that already have been renamed, - # so we need to remove them from the list. - # - # Considering these changes: - # - # - A new_file.rb - # - D deleted_file.rb - # - M modified_file.rb - # - R renamed_file_before.rb -> renamed_file_after.rb - # - # it will return - # ``` - # [ 'new_file.rb', 'modified_file.rb', 'renamed_file_after.rb' ] - # ``` - # - # @return [Array<String>] - def all_changed_files - Set.new - .merge(git.added_files.to_a) - .merge(git.modified_files.to_a) - .merge(git.renamed_files.map { |x| x[:after] }) - .subtract(git.renamed_files.map { |x| x[:before] }) - .to_a - .sort - end - - # Returns a string containing changed lines as git diff - # - # Considering changing a line in lib/gitlab/usage_data.rb it will return: - # - # [ "--- a/lib/gitlab/usage_data.rb", - # "+++ b/lib/gitlab/usage_data.rb", - # "+ # Test change", - # "- # Old change" ] - def changed_lines(changed_file) - diff = git.diff_for_file(changed_file) - return [] unless diff - - diff.patch.split("\n").select { |line| %r{^[+-]}.match?(line) } - end - - def all_ee_changes - all_changed_files.grep(%r{\Aee/}) - end - - def ee? - # Support former project name for `dev` and support local Danger run - %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__)) - end - - def gitlab_helper - # Unfortunately the following does not work: - # - respond_to?(:gitlab) - # - respond_to?(:gitlab, true) - gitlab - rescue NameError - nil - end - - def release_automation? - gitlab_helper&.mr_author == RELEASE_TOOLS_BOT - end - - def project_name - ee? ? 'gitlab' : 'gitlab-foss' - end - - def markdown_list(items) - list = items.map { |item| "* `#{item}`" }.join("\n") - - if items.size > 10 - "\n<details>\n\n#{list}\n\n</details>\n" - else - list - end - end - - # @return [Hash<String,Array<String>>] - def changes_by_category - all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| - categories_for_file(file).each { |category| hash[category] << file } - end - end - - # Determines the categories a file is in, e.g., `[:frontend]`, `[:backend]`, or `%i[frontend engineering_productivity]` - # using filename regex and specific change regex if given. - # - # @return Array<Symbol> - def categories_for_file(file) - _, categories = CATEGORIES.find do |key, _| - filename_regex, changes_regex = Array(key) - - found = filename_regex.match?(file) - found &&= changed_lines(file).any? { |changed_line| changes_regex.match?(changed_line) } if changes_regex - - found - end - - Array(categories || :unknown) - end - - # Returns the GFM for a category label, making its best guess if it's not - # a category we know about. - # - # @return[String] - def label_for_category(category) - CATEGORY_LABELS.fetch(category, "~#{category}") - end - - CATEGORY_LABELS = { - docs: "~documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. - none: "", - qa: "~QA", - test: "~test ~Quality for `spec/features/*`", - engineering_productivity: '~"Engineering Productivity" for CI, Danger', - ci_template: '~"ci::templates"' - }.freeze - # First-match win, so be sure to put more specific regex at the top... - CATEGORIES = { - [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend], - - %r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs, - %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, - - %r{\A(ee/)?app/(assets|views)/} => :frontend, - %r{\A(ee/)?public/} => :frontend, - %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend, - %r{\A(ee/)?vendor/assets/} => :frontend, - %r{\A(ee/)?scripts/frontend/} => :frontend, - %r{(\A|/)( - \.babelrc | - \.eslintignore | - \.eslintrc(\.yml)? | - \.nvmrc | - \.prettierignore | - \.prettierrc | - \.scss-lint.yml | - \.stylelintrc | - \.haml-lint.yml | - \.haml-lint_todo.yml | - babel\.config\.js | - jest\.config\.js | - package\.json | - yarn\.lock | - config/.+\.js - )\z}x => :frontend, - - %r{(\A|/)( - \.gitlab/ci/frontend\.gitlab-ci\.yml - )\z}x => %i[frontend engineering_productivity], - - %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, - %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, - %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, - %r{\A(ee/)?app/finders/} => :database, - %r{\Arubocop/cop/migration(/|\.rb)} => :database, - - %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, - %r{\A\.codeclimate\.yml\z} => :engineering_productivity, - %r{\Alefthook.yml\z} => :engineering_productivity, - %r{\A\.editorconfig\z} => :engineering_productivity, - %r{Dangerfile\z} => :engineering_productivity, - %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, - %r{\A(ee/)?scripts/} => :engineering_productivity, - %r{\Atooling/} => :engineering_productivity, - %r{(CODEOWNERS)} => :engineering_productivity, - %r{(tests.yml)} => :engineering_productivity, - - %r{\Alib/gitlab/ci/templates} => :ci_template, - - %r{\A(ee/)?spec/features/} => :test, - %r{\A(ee/)?spec/support/shared_examples/features/} => :test, - %r{\A(ee/)?spec/support/shared_contexts/features/} => :test, - %r{\A(ee/)?spec/support/helpers/features/} => :test, - - %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, - %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, - %r{\A(ee/)?spec/} => :backend, - %r{\A(ee/)?vendor/} => :backend, - %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, - %r{\A[A-Z_]+_VERSION\z} => :backend, - %r{\A\.rubocop((_manual)?_todo)?\.yml\z} => :backend, - %r{\Afile_hooks/} => :backend, - - %r{\A(ee/)?qa/} => :qa, - - # Files that don't fit into any category are marked with :none - %r{\A(ee/)?changelogs/} => :none, - %r{\Alocale/gitlab\.pot\z} => :none, - %r{\Adata/whats_new/} => :none, - - # GraphQL auto generated doc files and schema - %r{\Adoc/api/graphql/reference/} => :backend, - - # Fallbacks in case the above patterns miss anything - %r{\.rb\z} => :backend, - %r{( - \.(md|txt)\z | - \.markdownlint\.json - )}x => :none, # To reinstate roulette for documentation, set to `:docs`. - %r{\.js\z} => :frontend - }.freeze - - def new_teammates(usernames) - usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } - end - - def draft_mr? - return false unless gitlab_helper - - TitleLinting.has_draft_flag?(gitlab_helper.mr_json['title']) - end - - def security_mr? - return false unless gitlab_helper - - gitlab_helper.mr_json['web_url'].include?('/gitlab-org/security/') - end - - def cherry_pick_mr? - return false unless gitlab_helper - - /cherry[\s-]*pick/i.match?(gitlab_helper.mr_json['title']) - end - - def stable_branch? - return false unless gitlab_helper - - /\A\d+-\d+-stable-ee/i.match?(gitlab_helper.mr_json['target_branch']) - end - - def mr_has_labels?(*labels) - return false unless gitlab_helper - - labels = labels.flatten.uniq - (labels & gitlab_helper.mr_labels) == labels - end - - def labels_list(labels, sep: ', ') - labels.map { |label| %Q{~"#{label}"} }.join(sep) - end - - def prepare_labels_for_mr(labels) - return '' unless labels.any? - - "/label #{labels_list(labels, sep: ' ')}" - end - - def changed_files(regex) - all_changed_files.grep(regex) - end - - def has_database_scoped_labels?(current_mr_labels) - current_mr_labels.any? { |label| label.start_with?('database::') } - end - - def has_ci_changes? - changed_files(%r{\A(\.gitlab-ci\.yml|\.gitlab/ci/)}).any? - end - end - end -end diff --git a/lib/gitlab/danger/merge_request_linter.rb b/lib/gitlab/danger/merge_request_linter.rb deleted file mode 100644 index ed354bfc68d..00000000000 --- a/lib/gitlab/danger/merge_request_linter.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -base_linter_path = File.expand_path('base_linter', __dir__) - -if defined?(Rails) - require_dependency(base_linter_path) -else - require_relative(base_linter_path) -end - -module Gitlab - module Danger - class MergeRequestLinter < BaseLinter - alias_method :lint, :lint_subject - - def self.subject_description - 'merge request title' - end - - def self.mr_run_options_regex - [ - 'RUN AS-IF-FOSS', - 'UPDATE CACHE', - 'RUN ALL RSPEC', - 'SKIP RSPEC FAIL-FAST' - ].join('|') - end - - private - - def subject - super.gsub(/\[?(#{self.class.mr_run_options_regex})\]?/, '').strip - end - end - end -end diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb deleted file mode 100644 index 06da4ed9ad3..00000000000 --- a/lib/gitlab/danger/request_helper.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'net/http' -require 'json' - -module Gitlab - module Danger - module RequestHelper - HTTPError = Class.new(RuntimeError) - - # @param [String] url - def self.http_get_json(url) - rsp = Net::HTTP.get_response(URI.parse(url)) - - unless rsp.is_a?(Net::HTTPOK) - raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" - end - - JSON.parse(rsp.body) - end - end - end -end diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb deleted file mode 100644 index 21feda2cf20..00000000000 --- a/lib/gitlab/danger/roulette.rb +++ /dev/null @@ -1,169 +0,0 @@ -# frozen_string_literal: true - -require_relative 'teammate' -require_relative 'request_helper' unless defined?(Gitlab::Danger::RequestHelper) -require_relative 'weightage/reviewers' -require_relative 'weightage/maintainers' - -module Gitlab - module Danger - module Roulette - ROULETTE_DATA_URL = 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json' - HOURS_WHEN_PERSON_CAN_BE_PICKED = (6..14).freeze - - INCLUDE_TIMEZONE_FOR_CATEGORY = { - database: false - }.freeze - - Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role, :timezone_experiment) - - def team_mr_author - team.find { |person| person.username == mr_author_username } - end - - # Assigns GitLab team members to be reviewer and maintainer - # for each change category that a Merge Request contains. - # - # @return [Array<Spin>] - def spin(project, categories, timezone_experiment: false) - spins = categories.sort.map do |category| - including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(category, timezone_experiment) - - spin_for_category(project, category, timezone_experiment: including_timezone) - end - - backend_spin = spins.find { |spin| spin.category == :backend } - - spins.each do |spin| - including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(spin.category, timezone_experiment) - case spin.category - when :qa - # MR includes QA changes, but also other changes, and author isn't an SET - if categories.size > 1 && !team_mr_author&.reviewer?(project, spin.category, []) - spin.optional_role = :maintainer - end - when :test - spin.optional_role = :maintainer - - if spin.reviewer.nil? - # Fetch an already picked backend reviewer, or pick one otherwise - spin.reviewer = backend_spin&.reviewer || spin_for_category(project, :backend, timezone_experiment: including_timezone).reviewer - end - when :engineering_productivity - if spin.maintainer.nil? - # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer - end - when :ci_template - if spin.maintainer.nil? - # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer - end - end - end - - spins - end - - # Looks up the current list of GitLab team members and parses it into a - # useful form - # - # @return [Array<Teammate>] - def team - @team ||= - begin - data = Gitlab::Danger::RequestHelper.http_get_json(ROULETTE_DATA_URL) - data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } - rescue JSON::ParserError - raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" - end - end - - # Like +team+, but only returns teammates in the current project, based on - # project_name. - # - # @return [Array<Teammate>] - def project_team(project_name) - team.select { |member| member.in_project?(project_name) } - rescue => err - warn("Reviewer roulette failed to load team data: #{err.message}") - [] - end - - # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the - # selection will change on next spin - # @param [Array<Teammate>] people - def spin_for_person(people, random:, timezone_experiment: false) - shuffled_people = people.shuffle(random: random) - - if timezone_experiment - shuffled_people.find(&method(:valid_person_with_timezone?)) - else - shuffled_people.find(&method(:valid_person?)) - end - end - - private - - # @param [Teammate] person - # @return [Boolean] - def valid_person?(person) - !mr_author?(person) && person.available - end - - # @param [Teammate] person - # @return [Boolean] - def valid_person_with_timezone?(person) - valid_person?(person) && HOURS_WHEN_PERSON_CAN_BE_PICKED.cover?(person.local_hour) - end - - # @param [Teammate] person - # @return [Boolean] - def mr_author?(person) - person.username == mr_author_username - end - - def mr_author_username - helper.gitlab_helper&.mr_author || `whoami` - end - - def mr_source_branch - return `git rev-parse --abbrev-ref HEAD` unless helper.gitlab_helper&.mr_json - - helper.gitlab_helper.mr_json['source_branch'] - end - - def mr_labels - helper.gitlab_helper&.mr_labels || [] - end - - def new_random(seed) - Random.new(Digest::MD5.hexdigest(seed).to_i(16)) - end - - def spin_role_for_category(team, role, project, category) - team.select do |member| - member.public_send("#{role}?", project, category, mr_labels) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def spin_for_category(project, category, timezone_experiment: false) - team = project_team(project) - reviewers, traintainers, maintainers = - %i[reviewer traintainer maintainer].map do |role| - spin_role_for_category(team, role, project, category) - end - - random = new_random(mr_source_branch) - - weighted_reviewers = Weightage::Reviewers.new(reviewers, traintainers).execute - weighted_maintainers = Weightage::Maintainers.new(maintainers).execute - - reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) - maintainer = spin_for_person(weighted_maintainers, random: random, timezone_experiment: timezone_experiment) - - Spin.new(category, reviewer, maintainer, false, timezone_experiment) - end - end - end -end diff --git a/lib/gitlab/danger/sidekiq_queues.rb b/lib/gitlab/danger/sidekiq_queues.rb deleted file mode 100644 index 726b6134abf..00000000000 --- a/lib/gitlab/danger/sidekiq_queues.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module SidekiqQueues - def changed_queue_files - @changed_queue_files ||= git.modified_files.grep(%r{\A(ee/)?app/workers/all_queues\.yml}) - end - - def added_queue_names - @added_queue_names ||= new_queues.keys - old_queues.keys - end - - def changed_queue_names - @changed_queue_names ||= - (new_queues.values_at(*old_queues.keys) - old_queues.values) - .compact.map { |queue| queue[:name] } - end - - private - - def old_queues - @old_queues ||= queues_for(gitlab.base_commit) - end - - def new_queues - @new_queues ||= queues_for(gitlab.head_commit) - end - - def queues_for(branch) - changed_queue_files - .flat_map { |file| YAML.safe_load(`git show #{branch}:#{file}`, permitted_classes: [Symbol]) } - .to_h { |queue| [queue[:name], queue] } - end - end - end -end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb deleted file mode 100644 index 911b84d93ec..00000000000 --- a/lib/gitlab/danger/teammate.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :reduced_capacity, :tz_offset_hours - - # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb - def initialize(options = {}) - @options = options - @username = options['username'] - @name = options['name'] - @markdown_name = options['markdown_name'] - @role = options['role'] - @projects = options['projects'] - @available = options['available'] - @hungry = options['hungry'] - @reduced_capacity = options['reduced_capacity'] - @tz_offset_hours = options['tz_offset_hours'] - end - - def to_h - options - end - - def ==(other) - return false unless other.respond_to?(:username) - - other.username == username - end - - def in_project?(name) - projects&.has_key?(name) - end - - def reviewer?(project, category, labels) - has_capability?(project, category, :reviewer, labels) - end - - def traintainer?(project, category, labels) - has_capability?(project, category, :trainee_maintainer, labels) - end - - def maintainer?(project, category, labels) - has_capability?(project, category, :maintainer, labels) - end - - def markdown_name(author: nil) - "#{@markdown_name} (#{utc_offset_text(author)})" - end - - def local_hour - (Time.now.utc + tz_offset_hours * 3600).hour - end - - protected - - def floored_offset_hours - floored_offset = tz_offset_hours.floor(0) - - floored_offset == tz_offset_hours ? floored_offset : tz_offset_hours - end - - private - - def utc_offset_text(author = nil) - offset_text = - if floored_offset_hours >= 0 - "UTC+#{floored_offset_hours}" - else - "UTC#{floored_offset_hours}" - end - - return offset_text unless author - - "#{offset_text}, #{offset_diff_compared_to_author(author)}" - end - - def offset_diff_compared_to_author(author) - diff = floored_offset_hours - author.floored_offset_hours - return "same timezone as `@#{author.username}`" if diff == 0 - - ahead_or_behind = diff < 0 ? 'behind' : 'ahead of' - pluralized_hours = pluralize(diff.abs, 'hour', 'hours') - - "#{pluralized_hours} #{ahead_or_behind} `@#{author.username}`" - end - - def has_capability?(project, category, kind, labels) - case category - when :test - area = role[/Software Engineer in Test(?:.*?, (\w+))/, 1] - - area && labels.any?("devops::#{area.downcase}") if kind == :reviewer - when :engineering_productivity - return false unless role[/Engineering Productivity/] - return true if kind == :reviewer - return true if capabilities(project).include?("#{kind} engineering_productivity") - - capabilities(project).include?("#{kind} backend") - else - capabilities(project).include?("#{kind} #{category}") - end - end - - def capabilities(project) - Array(projects.fetch(project, [])) - end - - def pluralize(count, singular, plural) - word = count == 1 || count.to_s =~ /^1(\.0+)?$/ ? singular : plural - - "#{count || 0} #{word}" - end - end - end -end diff --git a/lib/gitlab/danger/title_linting.rb b/lib/gitlab/danger/title_linting.rb deleted file mode 100644 index db1ccaaf9a9..00000000000 --- a/lib/gitlab/danger/title_linting.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module TitleLinting - DRAFT_REGEX = /\A*#{Regexp.union(/(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/, /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/)}+\s*/i.freeze - - module_function - - def sanitize_mr_title(title) - remove_draft_flag(title).gsub(/`/, '\\\`') - end - - def remove_draft_flag(title) - title.gsub(DRAFT_REGEX, '') - end - - def has_draft_flag?(title) - DRAFT_REGEX.match?(title) - end - end - end -end diff --git a/lib/gitlab/danger/weightage.rb b/lib/gitlab/danger/weightage.rb deleted file mode 100644 index 67fade27573..00000000000 --- a/lib/gitlab/danger/weightage.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module Weightage - CAPACITY_MULTIPLIER = 2 # change this number to change what it means to be a reduced capacity reviewer 1/this number - BASE_REVIEWER_WEIGHT = 1 - end - end -end diff --git a/lib/gitlab/danger/weightage/maintainers.rb b/lib/gitlab/danger/weightage/maintainers.rb deleted file mode 100644 index cc0eb370e7a..00000000000 --- a/lib/gitlab/danger/weightage/maintainers.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require_relative '../weightage' - -module Gitlab - module Danger - module Weightage - class Maintainers - def initialize(maintainers) - @maintainers = maintainers - end - - def execute - maintainers.each_with_object([]) do |maintainer, weighted_maintainers| - add_weighted_reviewer(weighted_maintainers, maintainer, BASE_REVIEWER_WEIGHT) - end - end - - private - - attr_reader :maintainers - - def add_weighted_reviewer(reviewers, reviewer, weight) - if reviewer.reduced_capacity - reviewers.fill(reviewer, reviewers.size, weight) - else - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) - end - end - end - end - end -end diff --git a/lib/gitlab/danger/weightage/reviewers.rb b/lib/gitlab/danger/weightage/reviewers.rb deleted file mode 100644 index c8019be716e..00000000000 --- a/lib/gitlab/danger/weightage/reviewers.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -require_relative '../weightage' - -module Gitlab - module Danger - module Weightage - # Weights after (current multiplier of 2) - # - # +------------------------------+--------------------------------+ - # | reviewer type | weight(times in reviewer pool) | - # +------------------------------+--------------------------------+ - # | reduced capacity reviewer | 1 | - # | reviewer | 2 | - # | hungry reviewer | 4 | - # | reduced capacity traintainer | 3 | - # | traintainer | 6 | - # | hungry traintainer | 8 | - # +------------------------------+--------------------------------+ - # - class Reviewers - DEFAULT_REVIEWER_WEIGHT = CAPACITY_MULTIPLIER * BASE_REVIEWER_WEIGHT - TRAINTAINER_WEIGHT = 3 - - def initialize(reviewers, traintainers) - @reviewers = reviewers - @traintainers = traintainers - end - - def execute - # TODO: take CODEOWNERS into account? - # https://gitlab.com/gitlab-org/gitlab/issues/26723 - - weighted_reviewers + weighted_traintainers - end - - private - - attr_reader :reviewers, :traintainers - - def weighted_reviewers - reviewers.each_with_object([]) do |reviewer, total_reviewers| - add_weighted_reviewer(total_reviewers, reviewer, BASE_REVIEWER_WEIGHT) - end - end - - def weighted_traintainers - traintainers.each_with_object([]) do |reviewer, total_traintainers| - add_weighted_reviewer(total_traintainers, reviewer, TRAINTAINER_WEIGHT) - end - end - - def add_weighted_reviewer(reviewers, reviewer, weight) - if reviewer.reduced_capacity - reviewers.fill(reviewer, reviewers.size, weight) - elsif reviewer.hungry - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER + DEFAULT_REVIEWER_WEIGHT) - else - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) - end - end - end - end - end -end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index e6702c5a38b..e17bd25e57e 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -82,7 +82,8 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type? + is_shared: runner.instance_type?, + tags: runner.tags&.map(&:name) } end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 14facd6b1d4..3036bc57ca5 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -13,7 +13,7 @@ module Gitlab user: pipeline.user.try(:hook_attrs), project: pipeline.project.hook_attrs(backward: false), commit: pipeline.commit.try(:hook_attrs), - builds: pipeline.builds.map(&method(:build_hook_attrs)) + builds: pipeline.builds.latest.map(&method(:build_hook_attrs)) } end @@ -76,7 +76,8 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type? + is_shared: runner.instance_type?, + tags: runner.tags&.map(&:name) } end end diff --git a/lib/gitlab/database/consistency.rb b/lib/gitlab/database/consistency.rb new file mode 100644 index 00000000000..b7d06a26ddb --- /dev/null +++ b/lib/gitlab/database/consistency.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + ## + # This class is used to make it possible to ensure read consistency in + # GitLab EE without the need of overriding a lot of methods / classes / + # classs. + # + # This is a CE class that does nothing in CE, because database load + # balancing is EE-only feature, but you can still use it in CE. It will + # start ensuring read consistency once it is overridden in EE. + # + # Using this class in CE helps to avoid creeping discrepancy between CE / + # EE only to force usage of the primary database in EE. + # + class Consistency + ## + # In CE there is no database load balancing, so all reads are expected to + # be consistent by the ACID guarantees of a single PostgreSQL instance. + # + # This method is overridden in EE. + # + def self.with_read_consistency(&block) + yield + end + end + end +end + +::Gitlab::Database::Consistency.singleton_class.prepend_if_ee('EE::Gitlab::Database::Consistency') diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb new file mode 100644 index 00000000000..f20a9b30fa7 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module V2 + include Gitlab::Database::MigrationHelpers + + # Renames a column without requiring downtime. + # + # Concurrent renames work by using database triggers to ensure both the + # old and new column are in sync. However, this method will _not_ remove + # the triggers or the old column automatically; this needs to be done + # manually in a post-deployment migration. This can be done using the + # method `cleanup_concurrent_column_rename`. + # + # table - The name of the database table containing the column. + # old_column - The old column name. + # new_column - The new column name. + # type - The type of the new column. If no type is given the old column's + # type is used. + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id) + setup_renamed_column(__callee__, table, old_column, new_column, type, batch_column_name) + + with_lock_retries do + install_bidirectional_triggers(table, old_column, new_column) + end + end + + # Reverses operations performed by rename_column_concurrently. + # + # This method takes care of removing previously installed triggers as well + # as removing the new column. + # + # table - The name of the database table. + # old_column - The name of the old column. + # new_column - The name of the new column. + def undo_rename_column_concurrently(table, old_column, new_column) + teardown_rename_mechanism(table, old_column, new_column, column_to_remove: new_column) + end + + # Cleans up a concurrent column name. + # + # This method takes care of removing previously installed triggers as well + # as removing the old column. + # + # table - The name of the database table. + # old_column - The name of the old column. + # new_column - The name of the new column. + def cleanup_concurrent_column_rename(table, old_column, new_column) + teardown_rename_mechanism(table, old_column, new_column, column_to_remove: old_column) + end + + # Reverses the operations performed by cleanup_concurrent_column_rename. + # + # This method adds back the old_column removed + # by cleanup_concurrent_column_rename. + # It also adds back the triggers that are removed + # by cleanup_concurrent_column_rename. + # + # table - The name of the database table containing the column. + # old_column - The old column name. + # new_column - The new column name. + # type - The type of the old column. If no type is given the new column's + # type is used. + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + # + def undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id) + setup_renamed_column(__callee__, table, new_column, old_column, type, batch_column_name) + + with_lock_retries do + install_bidirectional_triggers(table, old_column, new_column) + end + end + + private + + def setup_renamed_column(calling_operation, table, old_column, new_column, type, batch_column_name) + if transaction_open? + raise "#{calling_operation} can not be run inside a transaction" + end + + column = columns(table).find { |column| column.name == old_column.to_s } + + unless column + raise "Column #{old_column} does not exist on #{table}" + end + + if column.default + raise "#{calling_operation} does not currently support columns with default values" + end + + unless column_exists?(table, batch_column_name) + raise "Column #{batch_column_name} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + unless column_exists?(table, new_column) + create_column_from(table, old_column, new_column, type: type, batch_column_name: batch_column_name) + end + end + + def teardown_rename_mechanism(table, old_column, new_column, column_to_remove:) + return unless column_exists?(table, column_to_remove) + + with_lock_retries do + check_trigger_permissions!(table) + + remove_bidirectional_triggers(table, old_column, new_column) + + remove_column(table, column_to_remove) + end + end + + def install_bidirectional_triggers(table, old_column, new_column) + insert_trigger_name, update_old_trigger_name, update_new_trigger_name = + bidirectional_trigger_names(table, old_column, new_column) + + quoted_table = quote_table_name(table) + quoted_old = quote_column_name(old_column) + quoted_new = quote_column_name(new_column) + + create_insert_trigger(insert_trigger_name, quoted_table, quoted_old, quoted_new) + create_update_trigger(update_old_trigger_name, quoted_table, quoted_new, quoted_old) + create_update_trigger(update_new_trigger_name, quoted_table, quoted_old, quoted_new) + end + + def remove_bidirectional_triggers(table, old_column, new_column) + insert_trigger_name, update_old_trigger_name, update_new_trigger_name = + bidirectional_trigger_names(table, old_column, new_column) + + quoted_table = quote_table_name(table) + + drop_trigger(insert_trigger_name, quoted_table) + drop_trigger(update_old_trigger_name, quoted_table) + drop_trigger(update_new_trigger_name, quoted_table) + end + + def bidirectional_trigger_names(table, old_column, new_column) + %w[insert update_old update_new].map do |operation| + 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old_column}_#{new_column}_#{operation}").first(12) + end + end + + def function_name_for_trigger(trigger_name) + "function_for_#{trigger_name}" + end + + def create_insert_trigger(trigger_name, quoted_table, quoted_old_column, quoted_new_column) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{function_name}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.#{quoted_old_column} IS NULL AND NEW.#{quoted_new_column} IS NOT NULL THEN + NEW.#{quoted_old_column} = NEW.#{quoted_new_column}; + END IF; + + IF NEW.#{quoted_new_column} IS NULL AND NEW.#{quoted_old_column} IS NOT NULL THEN + NEW.#{quoted_new_column} = NEW.#{quoted_old_column}; + END IF; + + RETURN NEW; + END + $$; + + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + CREATE TRIGGER #{trigger_name} + BEFORE INSERT ON #{quoted_table} + FOR EACH ROW EXECUTE FUNCTION #{function_name}(); + SQL + end + + def create_update_trigger(trigger_name, quoted_table, quoted_source_column, quoted_target_column) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{function_name}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + NEW.#{quoted_target_column} := NEW.#{quoted_source_column}; + RETURN NEW; + END + $$; + + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + CREATE TRIGGER #{trigger_name} + BEFORE UPDATE OF #{quoted_source_column} ON #{quoted_table} + FOR EACH ROW EXECUTE FUNCTION #{function_name}(); + SQL + end + + def drop_trigger(trigger_name, quoted_table) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + DROP FUNCTION IF EXISTS #{function_name}; + SQL + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb new file mode 100644 index 00000000000..959028ce00b --- /dev/null +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class Instrumentation + attr_reader :observations + + def initialize(observers = ::Gitlab::Database::Migrations::Observers.all_observers) + @observers = observers + @observations = [] + end + + def observe(migration, &block) + observation = Observation.new(migration) + observation.success = true + + exception = nil + + on_each_observer { |observer| observer.before } + + observation.walltime = Benchmark.realtime do + yield + rescue => e + exception = e + observation.success = false + end + + on_each_observer { |observer| observer.after } + on_each_observer { |observer| observer.record(observation) } + + record_observation(observation) + + raise exception if exception + + observation + end + + private + + attr_reader :observers + + def record_observation(observation) + @observations << observation + end + + def on_each_observer(&block) + observers.each do |observer| + yield observer + rescue => e + Gitlab::AppLogger.error("Migration observer #{observer.class} failed with: #{e}") + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb new file mode 100644 index 00000000000..518c2c560d2 --- /dev/null +++ b/lib/gitlab/database/migrations/observation.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + Observation = Struct.new( + :migration, + :walltime, + :success, + :total_database_size_change + ) + end + end +end diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb new file mode 100644 index 00000000000..4b931d3c19c --- /dev/null +++ b/lib/gitlab/database/migrations/observers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + def self.all_observers + [ + TotalDatabaseSizeChange.new + ] + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/observers/migration_observer.rb b/lib/gitlab/database/migrations/observers/migration_observer.rb new file mode 100644 index 00000000000..9bfbf35887d --- /dev/null +++ b/lib/gitlab/database/migrations/observers/migration_observer.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class MigrationObserver + attr_reader :connection + + def initialize + @connection = ActiveRecord::Base.connection + end + + def before + # implement in subclass + end + + def after + # implement in subclass + end + + def record(observation) + raise NotImplementedError, 'implement in subclass' + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/observers/total_database_size_change.rb b/lib/gitlab/database/migrations/observers/total_database_size_change.rb new file mode 100644 index 00000000000..0b76b0bef5e --- /dev/null +++ b/lib/gitlab/database/migrations/observers/total_database_size_change.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class TotalDatabaseSizeChange < MigrationObserver + def before + @size_before = get_total_database_size + end + + def after + @size_after = get_total_database_size + end + + def record(observation) + return unless @size_after && @size_before + + observation.total_database_size_change = @size_after - @size_before + end + + private + + def get_total_database_size + connection.execute("select pg_database_size(current_database())").first['pg_database_size'] + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 686dda80207..f4cf576dda7 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -164,8 +164,8 @@ module Gitlab "this could indicate the previous partitioning migration has been rolled back." end - Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |raw_arguments| - JobArguments.from_array(raw_arguments).source_table_name == table_name.to_s + Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |background_job| + JobArguments.from_array(background_job.args.second).source_table_name == table_name.to_s end primary_key = connection.primary_key(table_name) diff --git a/lib/gitlab/diff/char_diff.rb b/lib/gitlab/diff/char_diff.rb new file mode 100644 index 00000000000..c8bb39e9f5d --- /dev/null +++ b/lib/gitlab/diff/char_diff.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class CharDiff + include Gitlab::Utils::StrongMemoize + + def initialize(old_string, new_string) + @old_string = old_string.to_s + @new_string = new_string.to_s + @changes = [] + end + + def generate_diff + @changes = diff_match_patch.diff_main(@old_string, @new_string) + diff_match_patch.diff_cleanupSemantic(@changes) + + @changes + end + + def changed_ranges(offset: 0) + old_diffs = [] + new_diffs = [] + new_pointer = old_pointer = offset + + generate_diff.each do |(action, content)| + content_size = content.size + + if action == :equal + new_pointer += content_size + old_pointer += content_size + end + + if action == :delete + old_diffs << (old_pointer..(old_pointer + content_size - 1)) + old_pointer += content_size + end + + if action == :insert + new_diffs << (new_pointer..(new_pointer + content_size - 1)) + new_pointer += content_size + end + end + + [old_diffs, new_diffs] + end + + def to_html + @changes.map do |op, text| + %{<span class="#{html_class_names(op)}">#{ERB::Util.html_escape(text)}</span>} + end.join.html_safe + end + + private + + def diff_match_patch + strong_memoize(:diff_match_patch) { DiffMatchPatch.new } + end + + def html_class_names(operation) + class_names = ['idiff'] + + case operation + when :insert + class_names << 'addition' + when :delete + class_names << 'deletion' + end + + class_names.join(' ') + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 8f4f8febec0..627abfbfe7e 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -117,7 +117,7 @@ module Gitlab end def sort_diffs(diffs) - return diffs unless Feature.enabled?(:sort_diffs, project, default_enabled: false) + return diffs unless Feature.enabled?(:sort_diffs, project, default_enabled: :yaml) Gitlab::Diff::FileCollectionSorter.new(diffs).sort end diff --git a/lib/gitlab/diff/file_collection_sorter.rb b/lib/gitlab/diff/file_collection_sorter.rb index 94626875580..7b099543c83 100644 --- a/lib/gitlab/diff/file_collection_sorter.rb +++ b/lib/gitlab/diff/file_collection_sorter.rb @@ -3,6 +3,10 @@ module Gitlab module Diff class FileCollectionSorter + B_FOLLOWS_A = 1 + A_FOLLOWS_B = -1 + EQUIVALENT = 0 + attr_reader :diffs def initialize(diffs) @@ -29,14 +33,16 @@ module Gitlab a_part = a_parts.shift b_part = b_parts.shift - return 1 if a_parts.size < b_parts.size && a_parts.empty? - return -1 if a_parts.size > b_parts.size && b_parts.empty? + return B_FOLLOWS_A if a_parts.size < b_parts.size && a_parts.empty? + return A_FOLLOWS_B if a_parts.size > b_parts.size && b_parts.empty? comparison = a_part <=> b_part - return comparison unless comparison == 0 + return comparison unless comparison == EQUIVALENT + return compare_path_parts(a_parts, b_parts) if a_parts.any? && b_parts.any? - compare_path_parts(a_parts, b_parts) + # If A and B have the same name (e.g. symlink change), they are identical so return 0 + EQUIVALENT end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index a5259079345..035084d4861 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,12 +3,13 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines, :repository + attr_reader :diff_file, :diff_lines, :raw_lines, :repository, :project delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff def initialize(diff_lines, repository: nil) @repository = repository + @project = repository&.project if diff_lines.is_a?(Gitlab::Diff::File) @diff_file = diff_lines @@ -66,7 +67,7 @@ module Gitlab end def inline_diffs - @inline_diffs ||= InlineDiff.for_lines(@raw_lines) + @inline_diffs ||= InlineDiff.for_lines(@raw_lines, project: project) end def old_lines diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 90cb9c8638a..7932cd2a837 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -8,6 +8,7 @@ module Gitlab EXPIRATION = 1.week VERSION = 1 + NEXT_VERSION = 2 delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection @@ -69,12 +70,20 @@ module Gitlab def key strong_memoize(:redis_key) do - ['highlighted-diff-files', diffable.cache_key, VERSION, diff_options].join(":") + ['highlighted-diff-files', diffable.cache_key, version, diff_options].join(":") end end private + def version + if Feature.enabled?(:improved_merge_diff_highlighting, diffable.project, default_enabled: :yaml) + NEXT_VERSION + else + VERSION + end + end + def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 5815d1bae4a..cf769262958 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -27,28 +27,19 @@ module Gitlab @offset = offset end - def inline_diffs + def inline_diffs(project: nil) # Skip inline diff if empty line was replaced with content return if old_line == "" - lcp = longest_common_prefix(old_line, new_line) - lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) - - lcp += offset - old_length = old_line.length + offset - new_length = new_line.length + offset - - old_diff_range = lcp..(old_length - lcs - 1) - new_diff_range = lcp..(new_length - lcs - 1) - - old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end - new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end - - [old_diffs, new_diffs] + if Feature.enabled?(:improved_merge_diff_highlighting, project, default_enabled: :yaml) + CharDiff.new(old_line, new_line).changed_ranges(offset: offset) + else + deprecated_diff + end end class << self - def for_lines(lines) + def for_lines(lines, project: nil) changed_line_pairs = find_changed_line_pairs(lines) inline_diffs = [] @@ -57,7 +48,7 @@ module Gitlab old_line = lines[old_index] new_line = lines[new_index] - old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs(project: project) inline_diffs[old_index] = old_diffs inline_diffs[new_index] = new_diffs @@ -97,6 +88,24 @@ module Gitlab private + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/299884 + def deprecated_diff + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + + lcp += offset + old_length = old_line.length + offset + new_length = new_line.length + offset + + old_diff_range = lcp..(old_length - lcs - 1) + new_diff_range = lcp..(new_length - lcs - 1) + + old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + + [old_diffs, new_diffs] + end + def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName max_length = [a.length, b.length].max diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index f66e8a8794f..d1dd616385d 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -35,7 +35,11 @@ module Gitlab raise ProjectNotFound if project.nil? create_issue! - send_thank_you_email! if from_address + + if from_address + add_email_participant + send_thank_you_email! + end end def metrics_params @@ -146,6 +150,10 @@ module Gitlab def author User.support_bot end + + def add_email_participant + @issue.issue_email_participants.create(email: from_address) + end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index cab21d875ab..e6f71e3ad3c 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -63,6 +63,16 @@ module Gitlab ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], options) end + def custom_emoji_tag(name, image_source) + data = { + name: name + } + + ActionController::Base.helpers.content_tag('gl-emoji', title: name, data: data) do + emoji_image_tag(name, image_source).html_safe + end + end + private def emoji_unicode_versions_by_name diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 196203211ed..423f238a0a2 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -6,6 +6,7 @@ # Experiment options: # - tracking_category (optional, used to set the category when tracking an experiment event) # - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility -- you likely do not need this, see note in the next paragraph.) +# - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout # # Using the backwards-compatible subject index (use_backwards_compatible_subject_index option): # This option was added when [the calculation of experimentation_subject_index was changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45733/diffs#41af4a6fa5a10c7068559ce21c5188483751d934_157_173). It is not intended to be used by new experiments, it exists merely for the segmentation integrity of in-flight experiments at the time the change was deployed. That is, we want users who were assigned to the "experimental" group or the "control" group before the change to still be in those same groups after the change. See [the original issue](https://gitlab.com/gitlab-org/gitlab/-/issues/270858) and [this related comment](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48110#note_458223745) for more information. @@ -33,10 +34,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - onboarding_issues: { - tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues', - use_backwards_compatible_subject_index: true - }, ci_notification_dot: { tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot', use_backwards_compatible_subject_index: true @@ -69,13 +66,6 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', use_backwards_compatible_subject_index: true }, - default_to_issues_board: { - tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard', - use_backwards_compatible_subject_index: true - }, - jobs_empty_state: { - tracking_category: 'Growth::Activation::Experiment::JobsEmptyState' - }, remove_known_trial_form_fields: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' }, @@ -92,19 +82,27 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' }, ci_syntax_templates: { - tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates' - }, - pipelines_empty_state: { - tracking_category: 'Growth::Activation::Experiment::PipelinesEmptyState' + tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates', + rollout_strategy: :user }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' }, show_trial_status_in_sidebar: { - tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar' + tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar', + rollout_strategy: :group }, trial_onboarding_issues: { tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' + }, + learn_gitlab_a: { + tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA' + }, + learn_gitlab_b: { + tracking_category: 'Growth::Activation::Experiment::LearnGitLabB' + }, + in_product_marketing_emails: { + tracking_category: 'Growth::Activation::Experiment::InProductMarketingEmails' } }.freeze @@ -126,12 +124,44 @@ module Gitlab return false if subject.blank? return false unless active?(experiment_key) + log_invalid_rollout(experiment_key, subject) + experiment = get_experiment(experiment_key) return false unless experiment experiment.enabled_for_index?(index_for_subject(experiment, subject)) end + def rollout_strategy(experiment_key) + experiment = get_experiment(experiment_key) + return unless experiment + + experiment.rollout_strategy + end + + def log_invalid_rollout(experiment_key, subject) + return if valid_subject_for_rollout_strategy?(experiment_key, subject) + + logger = Gitlab::ExperimentationLogger.build + logger.warn message: 'Subject must conform to the rollout strategy', + experiment_key: experiment_key, + subject: subject.class.to_s, + rollout_strategy: rollout_strategy(experiment_key) + end + + def valid_subject_for_rollout_strategy?(experiment_key, subject) + case rollout_strategy(experiment_key) + when :user + subject.is_a?(User) + when :group + subject.is_a?(Group) + when :cookie + subject.nil? || subject.is_a?(String) + else + false + end + end + private def index_for_subject(experiment, subject) diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index e43f3c8c007..2b38b12c914 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -40,6 +40,8 @@ module Gitlab return true if forced_enabled?(experiment_key) return false if dnt_enabled? + Experimentation.log_invalid_rollout(experiment_key, subject) + subject ||= fallback_experimentation_subject_index(experiment_key) Experimentation.in_experiment_group?(experiment_key, subject: subject) @@ -65,7 +67,9 @@ module Gitlab return if dnt_enabled? return unless Experimentation.active?(experiment_key) && current_user - ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: current_user), current_user, context) + subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : current_user + + ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) end def record_experiment_conversion_event(experiment_key) @@ -136,7 +140,7 @@ module Gitlab cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } end - def tracking_label(subject) + def tracking_label(subject = nil) return experimentation_subject_id if subject.blank? if subject.respond_to?(:to_global_id) diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index 36cd673a38f..17dda45f5b7 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -5,12 +5,13 @@ module Gitlab class Experiment FEATURE_FLAG_SUFFIX = "_experiment_percentage" - attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index + attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index, :rollout_strategy def initialize(key, **params) @key = key @tracking_category = params[:tracking_category] @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] + @rollout_strategy = params[:rollout_strategy] || :cookie end def active? diff --git a/lib/gitlab/experimentation_logger.rb b/lib/gitlab/experimentation_logger.rb new file mode 100644 index 00000000000..ba1b60d6b4c --- /dev/null +++ b/lib/gitlab/experimentation_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class ExperimentationLogger < ::Gitlab::JsonLogger + def self.file_name_noext + 'experimentation_json' + end + end +end diff --git a/lib/gitlab/faraday.rb b/lib/gitlab/faraday.rb deleted file mode 100644 index f92392ec1a9..00000000000 --- a/lib/gitlab/faraday.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Faraday - ::Faraday::Request.register_middleware(gitlab_error_callback: -> { ::Gitlab::Faraday::ErrorCallback }) - end -end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index 38ccd2c38a9..ed25310b5cf 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -19,7 +19,7 @@ # `Content-Type` and `Content-Disposition` to the one we get from the detection. module Gitlab module FileTypeDetection - SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico webp].freeze SAFE_IMAGE_FOR_SCALING_EXT = %w[png jpg jpeg].freeze PDF_EXT = 'pdf' diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 0bc7ecccf5e..35c3dc5b0b3 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -16,7 +16,7 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email + :committed_date, :committer_name, :committer_email, :trailers ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -389,6 +389,7 @@ module Gitlab @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) + @trailers = Hash[commit.trailers.map { |t| [t.key, t.value] }] end # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 209917073c7..53df0b7b389 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -244,6 +244,8 @@ module Gitlab def prune_diff_if_eligible if too_large? + ::Gitlab::Metrics.add_event(:patch_hard_limit_bytes_hit) + too_large! elsif collapsed? collapse! diff --git a/lib/gitlab/git/push.rb b/lib/gitlab/git/push.rb index b6577ba17f1..3d533a5185f 100644 --- a/lib/gitlab/git/push.rb +++ b/lib/gitlab/git/push.rb @@ -33,7 +33,9 @@ module Gitlab end def force_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + strong_memoize(:force_push) do + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end end def branch_push? diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 0eff35ab1c4..0607b151de2 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -103,6 +103,7 @@ module Gitlab @committer_name = committer[:name] @committer_email = committer[:email] @parent_ids = commit.parents.map(&:oid) + @trailers = Hash[commit.trailers] end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 11919be594d..55ff3c6caf1 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -151,6 +151,8 @@ module Gitlab end def gitaly_find_page(title:, version: nil, dir: nil) + return unless title.present? + wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir) return unless wiki_page diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index e0b145f69aa..c5ca46827cb 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -77,7 +77,6 @@ module Gitlab check_authentication_abilities! check_command_disabled! check_command_existence! - check_otp_session! custom_action = check_custom_action return custom_action if custom_action @@ -255,31 +254,6 @@ module Gitlab end end - def check_otp_session! - return unless ssh? - return if !key? || deploy_key? - return unless Feature.enabled?(:two_factor_for_cli) - return unless user.two_factor_enabled? - - if ::Gitlab::Auth::Otp::SessionEnforcer.new(actor).access_restricted? - message = "OTP verification is required to access the repository.\n\n"\ - " Use: #{build_ssh_otp_verify_command}" - - raise ForbiddenError, message - end - end - - def build_ssh_otp_verify_command - user = "#{Gitlab.config.gitlab_shell.ssh_user}@" unless Gitlab.config.gitlab_shell.ssh_user.empty? - user_host = "#{user}#{Gitlab.config.gitlab_shell.ssh_host}" - - if Gitlab.config.gitlab_shell.ssh_port != 22 - "ssh #{user_host} -p #{Gitlab.config.gitlab_shell.ssh_port} 2fa_verify" - else - "ssh #{user_host} 2fa_verify" - end - end - def check_db_accessibility! return unless receive_pack? @@ -345,10 +319,8 @@ module Gitlab end def check_change_access! - return if deploy_key? && !deploy_keys_on_protected_branches_enabled? - if changes == ANY - can_push = (deploy_key? && deploy_keys_on_protected_branches_enabled?) || + can_push = deploy_key? || user_can_push? || project&.any_branch_allows_collaboration?(user_access.user) @@ -479,7 +451,7 @@ module Gitlab CiAccess.new elsif user && request_from_ci_build? BuildAccess.new(user, container: container) - elsif deploy_key? && deploy_keys_on_protected_branches_enabled? + elsif deploy_key? DeployKeyAccess.new(deploy_key, container: container) else UserAccess.new(user, container: container) @@ -558,10 +530,6 @@ module Gitlab def size_checker container.repository_size_checker end - - def deploy_keys_on_protected_branches_enabled? - Feature.enabled?(:deploy_keys_on_protected_branches, project) - end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 31734abe77f..3c7fa88977e 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -203,7 +203,7 @@ module Gitlab def self.authorization_token(storage) token = token(storage).to_s issued_at = real_time.to_i.to_s - hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at) + hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('SHA256'), token, issued_at) "v2.#{hmac}.#{issued_at}" end @@ -226,6 +226,7 @@ module Gitlab metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) metadata.merge!(Feature::Gitaly.server_feature_flags) + metadata.merge!(route_to_primary) deadline_info = request_deadline(timeout) metadata.merge!(deadline_info.slice(:deadline_type)) @@ -233,6 +234,26 @@ module Gitlab { metadata: metadata, deadline: deadline_info[:deadline] } end + # Gitlab::Git::HookEnv will set the :gitlab_git_env variable in case we're + # running in the context of a Gitaly hook call, which may make use of + # quarantined object directories. We thus need to pass along the path of + # the quarantined object directory to Gitaly, otherwise it won't be able to + # find these quarantined objects. Given that the quarantine directory is + # generated with a random name, they'll have different names when multiple + # Gitaly nodes take part in a single transaction. As a result, we are + # forced to route all requests to the primary node which has injected the + # quarantine object directory to us. + def self.route_to_primary + return {} unless Gitlab::SafeRequestStore.active? + + return {} unless Gitlab::SafeRequestStore[:gitlab_git_env] + + return {} if Gitlab::SafeRequestStore[:gitlab_git_env].empty? + + { 'gitaly-route-repository-accessor-policy' => 'primary-only' } + end + private_class_method :route_to_primary + def self.request_deadline(timeout) # timeout being 0 means the request is allowed to run indefinitely. # We can't allow that inside a request, but this won't count towards Gitaly diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ea940150941..ef5221a8042 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -335,7 +335,8 @@ module Gitlab all: !!options[:all], first_parent: !!options[:first_parent], global_options: parse_global_options!(options), - disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. + disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed. + trailers: options[:trailers] ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 6f08dcc69b6..fc40c23611a 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -67,7 +67,8 @@ module Gitlab source_branch: encode_binary(source_branch), target_branch: encode_binary(target_branch), commit_message: encode_binary(resolution.commit_message), - user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly + user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 4850d646de4..6f302b2c4e7 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -32,7 +32,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, tag_name: encode_binary(tag_name), target_revision: encode_binary(target), - message: encode_binary(message.to_s) + message: encode_binary(message.to_s), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) @@ -111,7 +112,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, message: encode_binary(message), first_parent_ref: encode_binary(first_parent_ref), - allow_conflicts: allow_conflicts + allow_conflicts: allow_conflicts, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) response = GitalyClient.call(@repository.storage, :operation_service, @@ -140,7 +142,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, branch: encode_binary(target_branch), - message: encode_binary(message) + message: encode_binary(message), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) ) @@ -234,7 +237,8 @@ module Gitlab branch_sha: branch_sha, remote_repository: remote_repository.gitaly_repository, remote_branch: encode_binary(remote_branch), - git_push_options: push_options + git_push_options: push_options, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) ) ) @@ -255,7 +259,7 @@ module Gitlab request_enum.close end - def user_squash(user, squash_id, start_sha, end_sha, author, message) + def user_squash(user, squash_id, start_sha, end_sha, author, message, time = Time.now.utc) request = Gitaly::UserSquashRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, @@ -263,7 +267,8 @@ module Gitlab start_sha: start_sha, end_sha: end_sha, author: Gitlab::Git::User.from_gitlab(author).to_gitaly, - commit_message: encode_binary(message) + commit_message: encode_binary(message), + timestamp: Google::Protobuf::Timestamp.new(seconds: time.to_i) ) response = GitalyClient.call( @@ -288,7 +293,8 @@ module Gitlab commit_sha: commit_sha, branch: encode_binary(branch), submodule: encode_binary(submodule), - commit_message: encode_binary(message) + commit_message: encode_binary(message), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) response = GitalyClient.call( @@ -357,7 +363,8 @@ module Gitlab header = Gitaly::UserApplyPatchRequest::Header.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - target_branch: encode_binary(branch_name) + target_branch: encode_binary(branch_name), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) reader = binary_io(patches) @@ -446,7 +453,8 @@ module Gitlab start_branch_name: encode_binary(start_branch_name), start_repository: start_repository.gitaly_repository, force: force, - start_sha: encode_binary(start_sha) + start_sha: encode_binary(start_sha), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 43848772947..5e50ac72965 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -23,7 +23,6 @@ module Gitlab MUTEX = Mutex.new - DISK_ACCESS_DENIED_FLAG = :deny_disk_access ALLOW_KEY = :allow_disk_access # If your code needs this method then your code needs to be fixed. @@ -34,7 +33,7 @@ module Gitlab def self.disk_access_denied? return false if rugged_enabled? - !temporarily_allowed?(ALLOW_KEY) && Feature::Gitaly.enabled?(DISK_ACCESS_DENIED_FLAG) + !temporarily_allowed?(ALLOW_KEY) rescue false # Err on the side of caution, don't break gitlab for people end @@ -62,7 +61,7 @@ module Gitlab def legacy_disk_path if self.class.disk_access_denied? - raise DirectPathAccessError, "git disk access denied via the gitaly_#{DISK_ACCESS_DENIED_FLAG} feature" + raise DirectPathAccessError, "git disk access denied" end @legacy_disk_path diff --git a/lib/gitlab/global_id.rb b/lib/gitlab/global_id.rb index e8a6006dce1..7e9412236cf 100644 --- a/lib/gitlab/global_id.rb +++ b/lib/gitlab/global_id.rb @@ -19,8 +19,8 @@ module Gitlab value when URI::GID GlobalID.new(value) - when Integer - raise CoerceError, 'Cannot coerce Integer' unless model_name.present? + when Integer, String + raise CoerceError, "Cannot coerce #{value.class}" unless model_name.present? GlobalID.new(::Gitlab::GlobalId.build(model_name: model_name, id: value)) else diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 0ba535b500e..3dd317c5a64 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -13,7 +13,6 @@ module Gitlab gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.shortcuts_path = Gitlab::Routing.url_helpers.help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class if Gitlab.config.sentry.enabled @@ -33,6 +32,7 @@ module Gitlab gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week gon.ee = Gitlab.ee? + gon.dot_com = Gitlab.com? if current_user gon.current_user_id = current_user.id @@ -43,11 +43,9 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, default_enabled: true) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - push_frontend_feature_flag(:gl_tooltips, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 8f5a1788fa5..9dfb9b090a8 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -16,6 +16,8 @@ Fields that are deprecated are marked with **{warning-solid}**. Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-process) can be found in [Removed Items](../removed_items.md). + + <!-- vale gitlab.Spelling = NO --> \ :plain diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb index 54a84be4274..965c01dd02f 100644 --- a/lib/gitlab/graphql/pagination/connections.rb +++ b/lib/gitlab/graphql/pagination/connections.rb @@ -6,6 +6,10 @@ module Gitlab module Connections def self.use(schema) schema.connections.add( + ::Gitlab::Graphql::Pagination::OffsetPaginatedRelation, + ::Gitlab::Graphql::Pagination::OffsetActiveRecordRelationConnection) + + schema.connections.add( ActiveRecord::Relation, Gitlab::Graphql::Pagination::Keyset::Connection) diff --git a/lib/gitlab/graphql/pagination/offset_paginated_relation.rb b/lib/gitlab/graphql/pagination/offset_paginated_relation.rb new file mode 100644 index 00000000000..8a8c6e5db50 --- /dev/null +++ b/lib/gitlab/graphql/pagination/offset_paginated_relation.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Marker class to enable us to choose the correct +# connection type during resolution +module Gitlab + module Graphql + module Pagination + class OffsetPaginatedRelation < SimpleDelegator + end + end + end +end diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index de971743490..fcf293fb13e 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -145,6 +145,20 @@ module Gitlab return redacted if printer.fields_printed > 0 end + def complexity(schema) + # See BaseResolver::resolver_complexity + # we want to see the max possible complexity. + fake_args = Struct + .new(:if, :keyword_arguments) + .new(nil, { sort: true, search: true }) + + query = GraphQL::Query.new(schema, text) + # We have no arguments, so fake them. + query.define_singleton_method(:arguments_for) { |_x, _y| fake_args } + + GraphQL::Analysis::AST.analyze_query(query, [GraphQL::Analysis::AST::QueryComplexity]).first + end + def query return @query if defined?(@query) diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 7e1c5331b07..2eef359cc6e 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -11,6 +11,10 @@ module Gitlab name.sub(/_check$/, '').capitalize end + def available? + true + end + def readiness raise NotImplementedError end diff --git a/lib/gitlab/health_checks/master_check.rb b/lib/gitlab/health_checks/master_check.rb index 057bce84ddd..b2c3695e6d9 100644 --- a/lib/gitlab/health_checks/master_check.rb +++ b/lib/gitlab/health_checks/master_check.rb @@ -8,7 +8,16 @@ module Gitlab extend SimpleAbstractCheck class << self + extend ::Gitlab::Utils::Override + + override :available? + def available? + Gitlab::Runtime.puma_in_clustered_mode? + end + def register_master + return unless available? + # when we fork, we pass the read pipe to child # child can then react on whether the other end # of pipe is still available @@ -16,11 +25,15 @@ module Gitlab end def finish_master + return unless available? + close_read close_write end def register_worker + return unless available? + # fork needs to close the pipe close_write end diff --git a/lib/gitlab/health_checks/probes/collection.rb b/lib/gitlab/health_checks/probes/collection.rb index 08b6d82291e..b34e4273d85 100644 --- a/lib/gitlab/health_checks/probes/collection.rb +++ b/lib/gitlab/health_checks/probes/collection.rb @@ -48,6 +48,7 @@ module Gitlab def probe_readiness checks + .select(&:available?) .flat_map(&:readiness) .compact .group_by(&:name) diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb index 434d30d9717..e5bae61ae4e 100644 --- a/lib/gitlab/hook_data/base_builder.rb +++ b/lib/gitlab/hook_data/base_builder.rb @@ -21,6 +21,12 @@ module Gitlab private + def event_data(event) + event_name = "#{object.class.name.downcase}_#{event}" + + { event_name: event_name } + end + def timestamps_data { created_at: object.created_at&.xmlschema, diff --git a/lib/gitlab/hook_data/group_builder.rb b/lib/gitlab/hook_data/group_builder.rb new file mode 100644 index 00000000000..5f76144eb83 --- /dev/null +++ b/lib/gitlab/hook_data/group_builder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class GroupBuilder < BaseBuilder + alias_method :group, :object + + # Sample data + # { + # :created_at=>"2021-01-20T09:40:12Z", + # :updated_at=>"2021-01-20T09:40:12Z", + # :event_name=>"group_rename", + # :name=>"group1", + # :path=>"group1", + # :full_path=>"group1", + # :group_id=>1, + # :old_path=>"old-path", + # :old_full_path=>"old-path" + # } + + def build(event) + [ + timestamps_data, + event_data(event), + group_data, + event_specific_group_data(event) + ].reduce(:merge) + end + + private + + def group_data + { + name: group.name, + path: group.path, + full_path: group.full_path, + group_id: group.id + } + end + + def event_specific_group_data(event) + return {} unless event == :rename + + { + old_path: group.path_before_last_save, + old_full_path: group.full_path_before_last_save + } + end + end + end +end diff --git a/lib/gitlab/hook_data/subgroup_builder.rb b/lib/gitlab/hook_data/subgroup_builder.rb new file mode 100644 index 00000000000..a620219675a --- /dev/null +++ b/lib/gitlab/hook_data/subgroup_builder.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class SubgroupBuilder < GroupBuilder + # Sample data + # { + # :created_at=>"2021-01-20T09:40:12Z", + # :updated_at=>"2021-01-20T09:40:12Z", + # :event_name=>"subgroup_create", + # :name=>"subgroup1", + # :path=>"subgroup1", + # :full_path=>"group1/subgroup1", + # :group_id=>10, + # :parent_group_id=>7, + # :parent_name=>group1, + # :parent_path=>group1, + # :parent_full_path=>group1 + # } + + private + + def event_data(event) + event_name = case event + when :create + 'subgroup_create' + when :destroy + 'subgroup_destroy' + end + + { event_name: event_name } + end + + def group_data + parent = group.parent + + super.merge( + parent_group_id: parent.id, + parent_name: parent.name, + parent_path: parent.path, + parent_full_path: parent.full_path + ) + end + + def event_specific_group_data(event) + {} + end + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 921072a4970..c4867746b0f 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -103,3 +103,7 @@ module Gitlab end Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport') + +# The methods in `Gitlab::ImportExport::GroupHelper` should be available as both +# instance and class methods. +Gitlab::ImportExport.extend_if_ee('Gitlab::ImportExport::GroupHelper') diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 219821a7150..37f1bdc3009 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -1,24 +1,16 @@ # frozen_string_literal: true -require 'zlib' - module Gitlab module ImportExport class DecompressedArchiveSizeValidator include Gitlab::Utils::StrongMemoize DEFAULT_MAX_BYTES = 10.gigabytes.freeze - CHUNK_SIZE = 4096.freeze - - attr_reader :error + TIMEOUT_LIMIT = 60.seconds def initialize(archive_path:, max_bytes: self.class.max_bytes) @archive_path = archive_path @max_bytes = max_bytes - @bytes_read = 0 - @total_reads = 0 - @denominator = 5 - @error = nil end def valid? @@ -31,59 +23,62 @@ module Gitlab DEFAULT_MAX_BYTES end - def archive_file - @archive_file ||= File.open(@archive_path) - end - private def validate - until archive_file.eof? - compressed_chunk = archive_file.read(CHUNK_SIZE) + pgrp = nil + valid_archive = true - inflate_stream.inflate(compressed_chunk) do |chunk| - @bytes_read += chunk.size - @total_reads += 1 - end + Timeout.timeout(TIMEOUT_LIMIT) do + stdin, stdout, stderr, wait_thr = Open3.popen3(command, pgroup: true) + stdin.close + pgrp = Process.getpgid(wait_thr[:pid]) + status = wait_thr.value - # Start garbage collection every 5 reads in order - # to prevent memory bloat during archive decompression - GC.start if gc_start? + if status.success? + result = stdout.readline - if @bytes_read > @max_bytes - @error = error_message + if result.to_i > @max_bytes + valid_archive = false - return false + log_error('Decompressed archive size limit reached') + end + else + valid_archive = false + + log_error(stderr.readline) end + + ensure + stdout.close + stderr.close end - true - rescue => e - @error = error_message + valid_archive + rescue Timeout::Error + log_error('Timeout reached during archive decompression') - Gitlab::ErrorTracking.track_exception(e) - - Gitlab::Import::Logger.info( - message: @error, - error: e.message - ) + Process.kill(-1, pgrp) if pgrp false - ensure - inflate_stream.close - archive_file.close - end + rescue => e + log_error(e.message) - def inflate_stream - @inflate_stream ||= Zlib::Inflate.new(Zlib::MAX_WBITS + 32) + Process.kill(-1, pgrp) if pgrp + + false end - def gc_start? - @total_reads % @denominator == 0 + def command + "gzip -dc #{@archive_path} | wc -c" end - def error_message - _('Decompressed archive size validation failed.') + def log_error(error) + Gitlab::Import::Logger.info( + message: error, + import_upload_archive_path: @archive_path, + import_upload_archive_size: File.size(@archive_path) + ) end end end diff --git a/lib/gitlab/import_export/design_repo_restorer.rb b/lib/gitlab/import_export/design_repo_restorer.rb index a702c58a7c2..e093b4b0697 100644 --- a/lib/gitlab/import_export/design_repo_restorer.rb +++ b/lib/gitlab/import_export/design_repo_restorer.rb @@ -3,10 +3,11 @@ module Gitlab module ImportExport class DesignRepoRestorer < RepoRestorer - def initialize(project:, shared:, path_to_bundle:) - super(project: project, shared: shared, path_to_bundle: path_to_bundle) + extend ::Gitlab::Utils::Override - @repository = project.design_repository + override :repository + def repository + @repository ||= importable.design_repository end # `restore` method is handled in super class diff --git a/lib/gitlab/import_export/design_repo_saver.rb b/lib/gitlab/import_export/design_repo_saver.rb index db9ebee6a13..b400aedc205 100644 --- a/lib/gitlab/import_export/design_repo_saver.rb +++ b/lib/gitlab/import_export/design_repo_saver.rb @@ -3,16 +3,18 @@ module Gitlab module ImportExport class DesignRepoSaver < RepoSaver - def save - @repository = project.design_repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.design_repository end private - def bundle_full_path - File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.design_repo_bundle_filename end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 5a6f6e017d2..51d58aae54f 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -87,7 +87,7 @@ module Gitlab end def validate_decompressed_archive_size - raise ImporterError.new(size_validator.error) unless size_validator.valid? + raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid? end def size_validator diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index dfe27118d66..925ab6680ba 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -6,7 +6,7 @@ module Gitlab class TreeRestorer include Gitlab::Utils::StrongMemoize - attr_reader :user, :shared + attr_reader :user, :shared, :groups_mapping def initialize(user:, shared:, group:) @user = user diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 789249c7d91..390909efe36 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -75,19 +75,19 @@ module Gitlab def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: shared, - project: project) + importable: project) end def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: shared, - project: ProjectWiki.new(project)) + importable: ProjectWiki.new(project)) end def design_repo_restorer Gitlab::ImportExport::DesignRepoRestorer.new(path_to_bundle: design_repo_path, shared: shared, - project: project) + importable: project) end def uploads_restorer diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index f808e30bd6e..8af7b68d78e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -5,10 +5,12 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:) - @repository = project.repository + attr_reader :importable + + def initialize(importable:, shared:, path_to_bundle:) @path_to_bundle = path_to_bundle @shared = shared + @importable = importable end def restore @@ -17,14 +19,25 @@ module Gitlab ensure_repository_does_not_exist! repository.create_from_bundle(path_to_bundle) + update_importable_repository_info + + true rescue => e shared.error(e) false end + def repository + @repository ||= importable.repository + end + private - attr_accessor :repository, :path_to_bundle, :shared + attr_accessor :path_to_bundle, :shared + + def update_importable_repository_info + # No-op. Overridden in EE + end def ensure_repository_does_not_exist! if repository.exists? @@ -38,3 +51,5 @@ module Gitlab end end end + +Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer') diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 898cd7898ba..0fdd0722b65 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -5,12 +5,11 @@ module Gitlab class RepoSaver include Gitlab::ImportExport::CommandLineUtil - attr_reader :project, :repository, :shared + attr_reader :exportable, :shared - def initialize(project:, shared:) - @project = project + def initialize(exportable:, shared:) + @exportable = exportable @shared = shared - @repository = @project.repository end def save @@ -19,6 +18,10 @@ module Gitlab bundle_to_disk end + def repository + @repository ||= @exportable.repository + end + private def repository_exists? @@ -26,11 +29,16 @@ module Gitlab end def bundle_full_path - File.join(shared.export_path, ImportExport.project_bundle_filename) + File.join(shared.export_path, bundle_filename) + end + + def bundle_filename + ::Gitlab::ImportExport.project_bundle_filename end def bundle_to_disk - mkdir_p(shared.export_path) + mkdir_p(File.dirname(bundle_full_path)) + repository.bundle_to_disk(bundle_full_path) rescue => e shared.error(e) diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 045ba2495bf..bb2bbda4bd6 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -31,7 +31,7 @@ module Gitlab @shared.error(e) false ensure - remove_base_tmp_dir + remove_archive_tmp_dir end private @@ -40,8 +40,8 @@ module Gitlab tar_czf(archive: archive_file, dir: @shared.export_path) end - def remove_base_tmp_dir - FileUtils.rm_rf(@shared.base_path) + def remove_archive_tmp_dir + FileUtils.rm_rf(@shared.archive_path) end def archive_file diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 93ae6f6b02a..4b1cf4915e4 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -3,18 +3,21 @@ module Gitlab module ImportExport class WikiRepoSaver < RepoSaver - def save - wiki = ProjectWiki.new(project) - @repository = wiki.repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.wiki.repository end private - def bundle_full_path - File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.wiki_repo_bundle_filename end end end end + +Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver') diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb index 56179eda22d..4bef043ecb0 100644 --- a/lib/gitlab/instrumentation/elasticsearch_transport.rb +++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb @@ -9,12 +9,17 @@ module Gitlab start = Time.now headers = (headers || {}) .reverse_merge({ 'X-Opaque-Id': Labkit::Correlation::CorrelationId.current_or_new_id }) - super + response = super ensure if ::Gitlab::SafeRequestStore.active? duration = (Time.now - start) ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count + + if response&.body && response.body.is_a?(Hash) && response.body['timed_out'] + ::Gitlab::Instrumentation::ElasticsearchTransport.increment_timed_out_count + end + ::Gitlab::Instrumentation::ElasticsearchTransport.add_duration(duration) ::Gitlab::Instrumentation::ElasticsearchTransport.add_call_details(duration, method, path, params, body) end @@ -25,6 +30,7 @@ module Gitlab ELASTICSEARCH_REQUEST_COUNT = :elasticsearch_request_count ELASTICSEARCH_CALL_DURATION = :elasticsearch_call_duration ELASTICSEARCH_CALL_DETAILS = :elasticsearch_call_details + ELASTICSEARCH_TIMED_OUT_COUNT = :elasticsearch_timed_out_count def self.get_request_count ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0 @@ -49,6 +55,15 @@ module Gitlab ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration end + def self.increment_timed_out_count + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 + end + + def self.get_timed_out_count + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 + end + def self.add_call_details(duration, method, path, params, body) return unless Gitlab::PerformanceBar.enabled_for_request? diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 6800e5667f6..644a5fc4fff 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -61,7 +61,7 @@ module Gitlab key_slot(args.first) end - unless key_slots.uniq.length == 1 + if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord raise CrossSlotError.new("Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") end end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 6b0f01757b7..61de6b02453 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -7,14 +7,33 @@ module Gitlab DURATION_PRECISION = 6 # microseconds def keys - @keys ||= [:gitaly_calls, - :gitaly_duration_s, - :rugged_calls, - :rugged_duration_s, - :elasticsearch_calls, - :elasticsearch_duration_s, - *::Gitlab::Instrumentation::Redis.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS] + @keys ||= [ + :cpu_s, + :gitaly_calls, + :gitaly_duration_s, + :rugged_calls, + :rugged_duration_s, + :elasticsearch_calls, + :elasticsearch_duration_s, + :elasticsearch_timed_out_count, + *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, + *::Gitlab::Instrumentation::Redis.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS, + *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, + *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS + ] + end + + def init_instrumentation_data(request_ip: nil) + # Set `request_start_time` only if this is request + # This is done, as `request_start_time` imply `request_deadline` + if request_ip + Gitlab::RequestContext.instance.client_ip = request_ip + Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time + end + + Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + Gitlab::RequestContext.instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations end def add_instrumentation_data(payload) @@ -24,6 +43,10 @@ module Gitlab instrument_elasticsearch(payload) instrument_throttle(payload) instrument_active_record(payload) + instrument_external_http(payload) + instrument_rack_attack(payload) + instrument_cpu(payload) + instrument_thread_memory_allocations(payload) end def instrument_gitaly(payload) @@ -57,6 +80,15 @@ module Gitlab payload[:elasticsearch_calls] = elasticsearch_calls payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time + payload[:elasticsearch_timed_out_count] = Gitlab::Instrumentation::ElasticsearchTransport.get_timed_out_count + end + + def instrument_external_http(payload) + external_http_count = Gitlab::Metrics::Subscribers::ExternalHttp.request_count + + return if external_http_count == 0 + + payload.merge! Gitlab::Metrics::Subscribers::ExternalHttp.payload end def instrument_throttle(payload) @@ -70,6 +102,26 @@ module Gitlab payload.merge!(db_counters) end + def instrument_rack_attack(payload) + rack_attack_redis_count = ::Gitlab::Metrics::Subscribers::RackAttack.payload[:rack_attack_redis_count] + return if rack_attack_redis_count == 0 + + payload.merge!(::Gitlab::Metrics::Subscribers::RackAttack.payload) + end + + def instrument_cpu(payload) + cpu_s = ::Gitlab::Metrics::System.thread_cpu_duration( + ::Gitlab::RequestContext.instance.start_thread_cpu_time) + + payload[:cpu_s] = cpu_s.round(DURATION_PRECISION) if cpu_s + end + + def instrument_thread_memory_allocations(payload) + counters = ::Gitlab::Memory::Instrumentation.measure_thread_memory_allocations( + ::Gitlab::RequestContext.instance.thread_memory_allocations) + payload.merge!(counters) if counters + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # @@ -96,7 +148,7 @@ module Gitlab # # @param [Time] start def self.elapsed_by_absolute_time(start) - (Time.now - start).to_f.round(6) + (Time.now - start).to_f.round(DURATION_PRECISION) end private_class_method :elapsed_by_absolute_time diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 08dde98e965..329c0f221b5 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -23,6 +23,12 @@ module Gitlab write_secret end + + def included_in_gitlab_com_rollout?(project) + return true unless ::Gitlab.com? + + Feature.enabled?(:kubernetes_agent_on_gitlab_com, project) + end end end end diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index 8c5652fb766..2948b6ebd5b 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -13,11 +13,8 @@ module Gitlab packetdiag rackdiag ].freeze - # Diagrams that require a companion container are disabled for now - DIAGRAMS_FORMATS = ::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - .reject { |diagram_type| diagram_type == 'mermaid' || diagram_type == 'bpmn' || BLOCKDIAG_FORMATS.include?(diagram_type) } - DIAGRAMS_FORMATS_WO_PLANTUML = DIAGRAMS_FORMATS - .reject { |diagram_type| diagram_type == 'plantuml' } + DIAGRAMS_FORMATS = (::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - %w(mermaid)).freeze + DIAGRAMS_FORMATS_WO_PLANTUML = (DIAGRAMS_FORMATS - %w(plantuml)).freeze # Get the list of diagram formats that are currently enabled # @@ -28,10 +25,18 @@ module Gitlab # If PlantUML is enabled, PlantUML diagrams will be processed by the PlantUML server. # In other words, the PlantUML server has precedence over Kroki since both can process PlantUML diagrams. - if current_settings.plantuml_enabled - DIAGRAMS_FORMATS_WO_PLANTUML - else - DIAGRAMS_FORMATS + diagram_formats = if current_settings.plantuml_enabled + DIAGRAMS_FORMATS_WO_PLANTUML + else + DIAGRAMS_FORMATS + end + + # No additional diagram formats + return diagram_formats unless current_settings.kroki_formats.present? + + # Diagrams that require a companion container must be explicitly enabled from the settings + diagram_formats.select do |diagram_type| + current_settings.kroki_format_supported?(diagram_type) end end end diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb index f603ff44ef3..17ea2eb5188 100644 --- a/lib/gitlab/kubernetes/helm/v2/certificate.rb +++ b/lib/gitlab/kubernetes/helm/v2/certificate.rb @@ -59,7 +59,7 @@ module Gitlab cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) end - cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) + cert.sign(signed_by&.key || key, OpenSSL::Digest.new('SHA256')) new(key, cert) end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index e6dd87a8bec..dce7cdb31a1 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -29,10 +29,6 @@ module Gitlab payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route] payload[Labkit::Correlation::CorrelationId::LOG_KEY] = event.payload[Labkit::Correlation::CorrelationId::LOG_KEY] || Labkit::Correlation::CorrelationId.current_id - if cpu_s = Gitlab::Metrics::System.thread_cpu_duration(::Gitlab::RequestContext.instance.start_thread_cpu_time) - payload[:cpu_s] = cpu_s.round(2) - end - CLOUDFLARE_CUSTOM_HEADERS.each do |_, value| payload[value] = event.payload[value] if event.payload[value] end diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb new file mode 100644 index 00000000000..76e84e54d3a --- /dev/null +++ b/lib/gitlab/memory/instrumentation.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +# This class uses a custom Ruby patch to allow +# a per-thread memory allocation tracking in a efficient manner +# +# This concept is currently tried to be upstreamed here: +# - https://github.com/ruby/ruby/pull/3978 +module Gitlab + module Memory + class Instrumentation + KEY_MAPPING = { + total_allocated_objects: :mem_objects, + total_malloc_bytes: :mem_bytes, + total_mallocs: :mem_mallocs + }.freeze + + MUTEX = Mutex.new + + def self.available? + Thread.respond_to?(:trace_memory_allocations=) && + Thread.current.respond_to?(:memory_allocations) + end + + # This method changes a global state + def self.ensure_feature_flag! + return unless available? + + enabled = Feature.enabled?(:trace_memory_allocations) + return if enabled == Thread.trace_memory_allocations + + MUTEX.synchronize do + # This enables or disables feature dynamically + # based on a feature flag + Thread.trace_memory_allocations = enabled + end + end + + def self.start_thread_memory_allocations + return unless available? + + ensure_feature_flag! + + # it will return `nil` if disabled + Thread.current.memory_allocations + end + + # This method returns a hash with the following keys: + # - mem_objects: a number of allocated heap slots (as reflected by GC) + # - mem_mallocs: a number of malloc calls + # - mem_bytes: a number of bytes allocated with a mallocs tied to heap slots + def self.measure_thread_memory_allocations(previous) + return unless available? + return unless previous + + current = Thread.current.memory_allocations + return unless current + + # calculate difference in a memory allocations + previous.to_h do |key, value| + [KEY_MAPPING.fetch(key), current[key].to_i - value] + end + end + + def self.with_memory_allocations + previous = self.start_thread_memory_allocations + yield + self.measure_thread_memory_allocations(previous) + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index b6a27d8556a..558454eaa1c 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -12,6 +12,10 @@ module Gitlab Gitlab::HealthChecks::Result.new( 'web_exporter', exporter.running) end + + def available? + true + end end attr_reader :running diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 3100450bc00..8ddd76ad7ae 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -39,7 +39,7 @@ module Gitlab options.evaluate(&block) if disabled_by_feature(options) - synchronized_cache_fill(name) { NullMetric.instance } + synchronized_cache_fill(name) { ::Gitlab::Metrics::NullMetric.instance } else synchronized_cache_fill(name) { build_metric!(type, name, options) } end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index a6884ea6983..f7e53bf545b 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -10,7 +10,7 @@ module Gitlab # env - A Hash containing Rack environment details. def call(env) - trans = WebTransaction.new(env) + trans = Gitlab::Metrics::WebTransaction.new(env) begin retval = trans.run { @app.call(env) } diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb new file mode 100644 index 00000000000..94c5d965200 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total time spent in external HTTP + # See more at https://gitlab.com/gitlab-org/labkit-ruby/-/blob/v0.14.0/lib/gitlab-labkit.rb#L18 + class ExternalHttp < ActiveSupport::Subscriber + attach_to :external_http + + DEFAULT_STATUS_CODE = 'undefined' + + DETAIL_STORE = :external_http_detail_store + COUNTER = :external_http_count + DURATION = :external_http_duration_s + + KNOWN_PAYLOAD_KEYS = [COUNTER, DURATION].freeze + + def self.detail_store + ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] + end + + def self.duration + Gitlab::SafeRequestStore[DURATION].to_f + end + + def self.request_count + Gitlab::SafeRequestStore[COUNTER].to_i + end + + def self.payload + { + COUNTER => request_count, + DURATION => duration + } + end + + def request(event) + payload = event.payload + add_to_detail_store(payload) + add_to_request_store(payload) + expose_metrics(payload) + end + + private + + def current_transaction + ::Gitlab::Metrics::Transaction.current + end + + def add_to_detail_store(payload) + return unless Gitlab::PerformanceBar.enabled_for_request? + + self.class.detail_store << { + duration: payload[:duration], + scheme: payload[:scheme], + method: payload[:method], + host: payload[:host], + port: payload[:port], + path: payload[:path], + query: payload[:query], + code: payload[:code], + exception_object: payload[:exception_object], + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) + } + end + + def add_to_request_store(payload) + return unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 + Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f + end + + def expose_metrics(payload) + return unless current_transaction + + labels = { method: payload[:method], code: payload[:code] || DEFAULT_STATUS_CODE } + + current_transaction.increment(:gitlab_external_http_total, 1, labels) do + docstring 'External HTTP calls' + label_keys labels.keys + end + + current_transaction.observe(:gitlab_external_http_duration_seconds, payload[:duration]) do + docstring 'External HTTP time' + buckets [0.001, 0.01, 0.1, 1.0, 2.0, 5.0] + end + + if payload[:exception_object].present? + current_transaction.increment(:gitlab_external_http_exception_total, 1) do + docstring 'External HTTP exceptions' + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb new file mode 100644 index 00000000000..2791a39fb16 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + # - Adds logging for all Rack Attack blocks and throttling events. + # - Instrument the cache operations of RackAttack to use in structured + # logs. Two fields are exposed: + # + rack_attack_redis_count: the number of redis calls triggered by + # RackAttack in a request. + # + rack_attack_redis_duration_s: the total duration of all redis calls + # triggered by RackAttack in a request. + class RackAttack < ActiveSupport::Subscriber + attach_to 'rack_attack' + + INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation + + THROTTLES_WITH_USER_INFORMATION = [ + :throttle_authenticated_api, + :throttle_authenticated_web, + :throttle_authenticated_protected_paths_api, + :throttle_authenticated_protected_paths_web + ].freeze + + PAYLOAD_KEYS = [ + :rack_attack_redis_count, + :rack_attack_redis_duration_s + ].freeze + + def self.payload + Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { + rack_attack_redis_count: 0, + rack_attack_redis_duration_s: 0.0 + } + end + + def redis(event) + self.class.payload[:rack_attack_redis_count] += 1 + self.class.payload[:rack_attack_redis_duration_s] += event.duration.to_f / 1000 + end + + def safelist(event) + req = event.payload[:request] + Gitlab::Instrumentation::Throttle.safelist = req.env['rack.attack.matched'] + end + + def throttle(event) + log_into_auth_logger(event) + end + + def blocklist(event) + log_into_auth_logger(event) + end + + def track(event) + log_into_auth_logger(event) + end + + private + + def log_into_auth_logger(event) + req = event.payload[:request] + rack_attack_info = { + message: 'Rack_Attack', + env: req.env['rack.attack.match_type'], + remote_ip: req.ip, + request_method: req.request_method, + path: req.fullpath, + matched: req.env['rack.attack.matched'] + } + + if THROTTLES_WITH_USER_INFORMATION.include? req.env['rack.attack.matched'].to_sym + user_id = req.env['rack.attack.match_discriminator'] + user = User.find_by(id: user_id) # rubocop:disable CodeReuse/ActiveRecord + + rack_attack_info[:user_id] = user_id + rack_attack_info['meta.user'] = user.username unless user.nil? + end + + Gitlab::InstrumentationHelper.add_instrumentation_data(rack_attack_info) + + logger.error(rack_attack_info) + end + + def logger + Gitlab::AuthLogger + end + end + end + end +end diff --git a/lib/gitlab/middleware/request_context.rb b/lib/gitlab/middleware/request_context.rb index 953423b371c..07f6f87a68c 100644 --- a/lib/gitlab/middleware/request_context.rb +++ b/lib/gitlab/middleware/request_context.rb @@ -16,9 +16,7 @@ module Gitlab # load balancer's IP. req = Rack::Request.new(env) - Gitlab::RequestContext.instance.client_ip = req.ip - Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time - Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time + ::Gitlab::InstrumentationHelper.init_instrumentation_data(request_ip: req.ip) @app.call(env) end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb index 8ae0ec5a78a..c1ccfae3e1f 100644 --- a/lib/gitlab/pages_transfer.rb +++ b/lib/gitlab/pages_transfer.rb @@ -7,16 +7,26 @@ # module Gitlab class PagesTransfer < ProjectTransfer - class Async - METHODS = %w[move_namespace move_project rename_project rename_namespace].freeze + METHODS = %w[move_namespace move_project rename_project rename_namespace].freeze + class Async METHODS.each do |meth| define_method meth do |*args| + next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + PagesTransferWorker.perform_async(meth, args) end end end + METHODS.each do |meth| + define_method meth do |*args| + next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + + super(*args) + end + end + def async @async ||= Async.new end diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index 22ece0a6a8b..dde78cd9178 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -39,9 +39,14 @@ module Gitlab def class_methods super + class_methods_module = const_get(:ClassMethods, false) + if instance_variable_defined?(:@_prepended_class_methods) - const_get(:ClassMethods, false).prepend @_prepended_class_methods + class_methods_module.prepend @_prepended_class_methods end + + # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 + extend class_methods_module if ENV['STATIC_VERIFICATION'] end def prepended(base = nil, &block) diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index d1504d88315..380340b80be 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -27,27 +27,40 @@ module Gitlab end def log_sql_queries(id, data) - return [] unless queries = data.dig('data', 'active-record', 'details') - - queries.each do |query| - next unless location = parse_backtrace(query['backtrace']) + queries_by_location(data).each do |location, queries| + next unless location - log_info = location.merge( + duration = queries.sum { |query| query['duration'].to_f } + log_info = { + method_path: "#{location[:filename]}:#{location[:method]}", + filename: location[:filename], type: :sql, request_id: id, - duration_ms: query['duration'].to_f - ) + count: queries.count, + duration_ms: duration + } logger.info(log_info) end end + def queries_by_location(data) + return [] unless queries = data.dig('data', 'active-record', 'details') + + queries.group_by do |query| + parse_backtrace(query['backtrace']) + end + end + def parse_backtrace(backtrace) return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace.first) { filename: match[:filename], - filenum: match[:filenum].to_i, + # filenum may change quite frequently with every change in the file, + # because the intention is to aggregate these queries, we group + # them rather by method name which should not change so frequently + # filenum: match[:filenum].to_i, method: match[:method] } end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index b56fd8278a1..6a404c34044 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -181,13 +181,12 @@ module Gitlab end types MergeRequest condition do - Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |reviewer_param| extract_users(reviewer_param) end - command :assign_reviewer, :reviewer do |users| + command :assign_reviewer, :reviewer, :request_review do |users| next if users.empty? if quick_action_target.allows_multiple_reviewers? @@ -221,7 +220,6 @@ module Gitlab types MergeRequest condition do quick_action_target.persisted? && - Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && quick_action_target.reviewers.any? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index 2a94fb91880..ae3c89c3565 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -12,13 +12,15 @@ module Gitlab rack_attack::Request.include(Gitlab::RackAttack::Request) # This is Rack::Attack::DEFAULT_THROTTLED_RESPONSE, modified to allow a custom response - Rack::Attack.throttled_response = lambda do |env| + rack_attack.throttled_response = lambda do |env| throttled_headers = Gitlab::RackAttack.throttled_response_headers( env['rack.attack.matched'], env['rack.attack.match_data'] ) [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] end + rack_attack.cache.store = Gitlab::RackAttack::InstrumentedCacheStore.new + # Configure the throttles configure_throttles(rack_attack) diff --git a/lib/gitlab/rack_attack/instrumented_cache_store.rb b/lib/gitlab/rack_attack/instrumented_cache_store.rb new file mode 100644 index 00000000000..8cf9082384f --- /dev/null +++ b/lib/gitlab/rack_attack/instrumented_cache_store.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module RackAttack + # This class is a proxy for all Redis calls made by RackAttack. All the + # calls are instrumented, then redirected to ::Rails.cache. This class + # instruments the standard interfaces of ActiveRecord::Cache defined in + # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache.rb#L315 + # + # For more information, please see + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/751 + class InstrumentedCacheStore + NOTIFICATION_CHANNEL = 'redis.rack_attack' + + delegate :silence!, :mute, to: :@upstream_store + + def initialize(upstream_store: ::Rails.cache, notifier: ActiveSupport::Notifications) + @upstream_store = upstream_store + @notifier = notifier + end + + [:fetch, :read, :read_multi, :write_multi, :fetch_multi, :write, :delete, + :exist?, :delete_matched, :increment, :decrement, :cleanup, :clear].each do |interface| + define_method interface do |*args, **k_args, &block| + @notifier.instrument(NOTIFICATION_CHANNEL, operation: interface) do + @upstream_store.public_send(interface, *args, **k_args, &block) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index f3cbe1db901..a08cea5a435 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -2,8 +2,10 @@ module Gitlab module Recaptcha + extend Gitlab::Utils::StrongMemoize + def self.load_configurations! - if Gitlab::CurrentSettings.recaptcha_enabled || enabled_on_login? + if enabled? || enabled_on_login? ::Recaptcha.configure do |config| config.site_key = Gitlab::CurrentSettings.recaptcha_site_key config.secret_key = Gitlab::CurrentSettings.recaptcha_private_key diff --git a/lib/gitlab/relative_positioning.rb b/lib/gitlab/relative_positioning.rb index b5a923f0824..e2cbe4b2de0 100644 --- a/lib/gitlab/relative_positioning.rb +++ b/lib/gitlab/relative_positioning.rb @@ -13,5 +13,18 @@ module Gitlab MIN_GAP = 2 NoSpaceLeft = Class.new(StandardError) + IllegalRange = Class.new(ArgumentError) + + def self.range(lhs, rhs) + if lhs && rhs + ClosedRange.new(lhs, rhs) + elsif lhs + StartingFrom.new(lhs) + elsif rhs + EndingAt.new(rhs) + else + raise IllegalRange, 'One of rhs or lhs must be provided' unless lhs && rhs + end + end end end diff --git a/lib/gitlab/relative_positioning/range.rb b/lib/gitlab/relative_positioning/range.rb index 174d5ef4b35..0b0ccdf5be4 100644 --- a/lib/gitlab/relative_positioning/range.rb +++ b/lib/gitlab/relative_positioning/range.rb @@ -2,8 +2,6 @@ module Gitlab module RelativePositioning - IllegalRange = Class.new(ArgumentError) - class Range attr_reader :lhs, :rhs @@ -34,18 +32,6 @@ module Gitlab end end - def self.range(lhs, rhs) - if lhs && rhs - ClosedRange.new(lhs, rhs) - elsif lhs - StartingFrom.new(lhs) - elsif rhs - EndingAt.new(rhs) - else - raise IllegalRange, 'One of rhs or lhs must be provided' unless lhs && rhs - end - end - class ClosedRange < RelativePositioning::Range def initialize(lhs, rhs) @lhs, @rhs = lhs, rhs diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index 952ae55d90a..c9eefe9a647 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -7,7 +7,7 @@ module Gitlab RequestDeadlineExceeded = Class.new(StandardError) - attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time + attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations class << self def instance diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index 79562a8223b..a84a6ac2d14 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -23,7 +23,9 @@ module Gitlab end def self.verified?(env) - call(env) + minimal_env = env.slice('REQUEST_METHOD', 'rack.session', 'HTTP_X_CSRF_TOKEN') + .merge('rack.input' => '') + call(minimal_env) true rescue ActionController::InvalidAuthenticityToken diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 8b40aaa101a..647ac169f05 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -81,6 +81,10 @@ module Gitlab puma? || sidekiq? || action_cable? end + def puma_in_clustered_mode? + puma? && Puma.cli_config.options[:workers].to_i > 0 + end + def max_threads threads = 1 # main thread diff --git a/lib/gitlab/sample_data_template.rb b/lib/gitlab/sample_data_template.rb index 06ea53e4018..5a2a7d46bdf 100644 --- a/lib/gitlab/sample_data_template.rb +++ b/lib/gitlab/sample_data_template.rb @@ -5,7 +5,7 @@ module Gitlab class << self def localized_templates_table [ - SampleDataTemplate.new('sample', 'Sample GitLab Project', _('Get started with a project that follows best practices for setting up GitLab for your own organization, including sample Issues, Merge Requests, and Milestones'), 'https://gitlab.com/gitlab-org/sample-data-templates/sample-gitlab-project') + SampleDataTemplate.new('sample', 'Sample GitLab Project', _('An example project that shows off the best practices for setting up GitLab for your own organization, including sample issues, merge requests, and milestones'), 'https://gitlab.com/gitlab-org/sample-data-templates/sample-gitlab-project') ].freeze end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index 5b1f9400bc7..c0420126ada 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -5,6 +5,9 @@ module Gitlab class Query < SimpleDelegator include EncodingHelper + QUOTES_REGEXP = %r{\A"|"\Z}.freeze + TOKEN_WITH_QUOTES_REGEXP = %r{\s(?=(?:[^"]|"[^"]*")*$)}.freeze + def initialize(query, filter_opts = {}, &block) @raw_query = query.dup @filters = [] @@ -35,22 +38,24 @@ module Gitlab def extract_filters fragments = [] + query_tokens = parse_raw_query filters = @filters.each_with_object([]) do |filter, parsed_filters| - match = @raw_query.split.find { |part| part =~ /\A-?#{filter[:name]}:/ } + match = query_tokens.find { |part| part =~ /\A-?#{filter[:name]}:/ } + next unless match input = match.split(':')[1..-1].join next if input.empty? filter[:negated] = match.start_with?("-") - filter[:value] = parse_filter(filter, input) + filter[:value] = parse_filter(filter, input.gsub(QUOTES_REGEXP, '')) filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') fragments << match parsed_filters << filter end - query = (@raw_query.split - fragments).join(' ') + query = (query_tokens - fragments).join(' ') query = '*' if query.empty? [query, filters] @@ -61,6 +66,13 @@ module Gitlab @filter_options[:encode_binary] ? encode_binary(result) : result end + + def parse_raw_query + # Positive lookahead for any non-quote char or even number of quotes + # for example '"search term" path:"foo bar.txt"' would break into + # ["search term", "path:\"foo bar.txt\""] + @raw_query.split(TOKEN_WITH_QUOTES_REGEXP).reject(&:empty?) + end end end end diff --git a/lib/gitlab/search/sort_options.rb b/lib/gitlab/search/sort_options.rb index 3395c34d171..2ab38147462 100644 --- a/lib/gitlab/search/sort_options.rb +++ b/lib/gitlab/search/sort_options.rb @@ -11,6 +11,10 @@ module Gitlab :created_at_asc when %w[created_at desc], [nil, 'created_desc'] :created_at_desc + when %w[updated_at asc], [nil, 'updated_asc'] + :updated_at_asc + when %w[updated_at desc], [nil, 'updated_desc'] + :updated_at_desc else :unknown end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 0091ae1e8ce..d0beb74c289 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -136,6 +136,10 @@ module Gitlab scope.reorder('created_at ASC') when :created_at_desc scope.reorder('created_at DESC') + when :updated_at_asc + scope.reorder('updated_at ASC') + when :updated_at_desc + scope.reorder('updated_at DESC') else scope.reorder('created_at DESC') end diff --git a/lib/gitlab/sidekiq_death_handler.rb b/lib/gitlab/sidekiq_death_handler.rb index f86d9f17b5f..91bfc7dca80 100644 --- a/lib/gitlab/sidekiq_death_handler.rb +++ b/lib/gitlab/sidekiq_death_handler.rb @@ -6,7 +6,7 @@ module Gitlab include ::Gitlab::SidekiqMiddleware::MetricsHelper def handler(job, _exception) - labels = create_labels(job['class'].constantize, job['queue']) + labels = create_labels(job['class'].constantize, job['queue'], job) counter.increment(labels) end diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb deleted file mode 100644 index 8ae6addc2c6..00000000000 --- a/lib/gitlab/sidekiq_logging/exception_handler.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqLogging - class ExceptionHandler - def call(job_exception, context) - data = { - error_class: job_exception.class.name, - error_message: job_exception.message - } - - if context.is_a?(Hash) - data.merge!(context) - # correlation_id, jid, and class are available inside the job - # Hash, so promote these arguments to the root tree so that - # can be searched alongside other Sidekiq log messages. - job_data = data.delete(:job) - data.merge!(job_data) if job_data.present? - end - - data[:error_backtrace] = Rails.backtrace_cleaner.clean(job_exception.backtrace) if job_exception.backtrace.present? - - Sidekiq.logger.warn(data) - end - end - end -end diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index dc81c34c4d0..6f8cc1c60e9 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -12,6 +12,7 @@ module Gitlab # Error information from the previous try is in the payload for # displaying in the Sidekiq UI, but is very confusing in logs! job = job.except('error_backtrace', 'error_class', 'error_message') + job['class'] = job.delete('wrapped') if job['wrapped'].present? # Add process id params job['pid'] = ::Process.pid diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index eb845c5ff8d..654b17c5740 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -11,13 +11,25 @@ module Gitlab def call(job, queue) started_time = get_time base_payload = parse_job(job) + ActiveRecord::LogSubscriber.reset_runtime - Sidekiq.logger.info log_job_start(base_payload) + Sidekiq.logger.info log_job_start(job, base_payload) yield Sidekiq.logger.info log_job_done(job, started_time, base_payload) + rescue Sidekiq::JobRetry::Handled => job_exception + # Sidekiq::JobRetry::Handled is raised by the internal Sidekiq + # processor. It is a wrapper around real exception indicating an + # exception is already handled by the Job retrier. The real exception + # should be unwrapped before being logged. + # + # For more information: + # https://github.com/mperham/sidekiq/blob/v5.2.7/lib/sidekiq/processor.rb#L173 + Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception.cause || job_exception) + + raise rescue => job_exception Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) @@ -27,7 +39,9 @@ module Gitlab private def add_instrumentation_keys!(job, output_payload) - output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper.keys)) + instrumentation_values = job.slice(*::Gitlab::InstrumentationHelper.keys).stringify_keys + + output_payload.merge!(instrumentation_values) end def add_logging_extras!(job, output_payload) @@ -36,17 +50,15 @@ module Gitlab ) end - def add_db_counters!(job, output_payload) - output_payload.merge!(job.slice(*::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS)) - end - - def log_job_start(payload) + def log_job_start(job, payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' scheduling_latency_s = ::Gitlab::InstrumentationHelper.queue_duration_for_job(payload) payload['scheduling_latency_s'] = scheduling_latency_s if scheduling_latency_s + payload['job_size_bytes'] = Sidekiq.dump_json(job).bytesize + payload end @@ -54,7 +66,6 @@ module Gitlab payload = payload.dup add_instrumentation_keys!(job, payload) add_logging_extras!(job, payload) - add_db_counters!(job, payload) elapsed_time = elapsed(started_time) add_time_keys!(elapsed_time, payload) @@ -66,6 +77,7 @@ module Gitlab payload['job_status'] = 'fail' payload['error_message'] = job_exception.message payload['error_class'] = job_exception.class.name + add_exception_backtrace!(job_exception, payload) else payload['message'] = "#{message}: done: #{payload['duration_s']} sec" payload['job_status'] = 'done' @@ -79,26 +91,22 @@ module Gitlab def add_time_keys!(time, payload) payload['duration_s'] = time[:duration].round(Gitlab::InstrumentationHelper::DURATION_PRECISION) - - # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) - # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime - payload['cpu_s'] = time[:cputime].round(Gitlab::InstrumentationHelper::DURATION_PRECISION) if time[:cputime] > 0 payload['completed_at'] = Time.now.utc.to_f end + def add_exception_backtrace!(job_exception, payload) + return if job_exception.backtrace.blank? + + payload['error_backtrace'] = Rails.backtrace_cleaner.clean(job_exception.backtrace) + end + def elapsed(t0) t1 = get_time - { - duration: t1[:now] - t0[:now], - cputime: t1[:thread_cputime] - t0[:thread_cputime] - } + { duration: t1[:now] - t0[:now] } end def get_time - { - now: current_time, - thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 - } + { now: current_time } end def current_time diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb index 7ee8a623d30..6bc08a97c07 100644 --- a/lib/gitlab/sidekiq_middleware/client_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -11,10 +11,10 @@ module Gitlab @metrics = init_metrics end - def call(worker_class, _job, queue, _redis_pool) + def call(worker_class, job, queue, _redis_pool) # worker_class can either be the string or class of the worker being enqueued. worker_class = worker_class.safe_constantize if worker_class.respond_to?(:safe_constantize) - labels = create_labels(worker_class, queue) + labels = create_labels(worker_class, queue, job) @metrics.fetch(ENQUEUED).increment(labels, 1) diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb index 979a3fce7e6..a66a4de4655 100644 --- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -4,8 +4,11 @@ module Gitlab module SidekiqMiddleware class InstrumentationLogger def call(worker, job, queue) + ::Gitlab::InstrumentationHelper.init_instrumentation_data + yield + ensure # The Sidekiq logger is called outside the middleware block, so # we need to modify the job hash to pass along this information # since RequestStore is only active in the Sidekiq middleware. diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb index 5c1ce2b98e8..60e79ee1188 100644 --- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb +++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb @@ -8,9 +8,11 @@ module Gitlab private - def create_labels(worker_class, queue) + def create_labels(worker_class, queue, job) + worker_name = (job['wrapped'].presence || worker_class).to_s + labels = { queue: queue.to_s, - worker: worker_class.to_s, + worker: worker_name, urgency: "", external_dependencies: FALSE_LABEL, feature_category: "", diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 7f3048f4c6e..4ab8d313ad8 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -20,7 +20,7 @@ module Gitlab # in metrics and can use them in the `ThreadsSampler` for setting a label Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME - labels = create_labels(worker.class, queue) + labels = create_labels(worker.class, queue, job) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration diff --git a/lib/gitlab/suggestions/commit_message.rb b/lib/gitlab/suggestions/commit_message.rb index d59a8fc3730..5bca3efe6e1 100644 --- a/lib/gitlab/suggestions/commit_message.rb +++ b/lib/gitlab/suggestions/commit_message.rb @@ -6,14 +6,15 @@ module Gitlab DEFAULT_SUGGESTION_COMMIT_MESSAGE = 'Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)' - def initialize(user, suggestion_set) + def initialize(user, suggestion_set, custom_message = nil) @user = user @suggestion_set = suggestion_set + @custom_message = custom_message end def message project = suggestion_set.project - user_defined_message = project.suggestion_commit_message.presence + user_defined_message = @custom_message.presence || project.suggestion_commit_message.presence message = user_defined_message || DEFAULT_SUGGESTION_COMMIT_MESSAGE Gitlab::StringPlaceholderReplacer diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index c702c6f1add..db3c058184c 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -66,6 +66,18 @@ module Gitlab answer end + # Prompt the user to input a password + # + # message - custom message to display before input + def prompt_for_password(message = 'Enter password: ') + unless STDIN.tty? + print(message) + return STDIN.gets.chomp + end + + STDIN.getpass(message) + end + # Runs the given command and matches the output against the given pattern # # Returns nil if nothing matched diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index b659bff52ad..0f933a61598 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -8,6 +8,7 @@ module Gitlab def initialize(path, project = nil, category: nil) @path = path @category = category + @project = project @finder = self.class.finder(project) end @@ -31,6 +32,10 @@ module Gitlab # override with a comment to be placed at the top of the blob. end + def project_id + @project&.id + end + # Present for compatibility with license templates, which can replace text # like `[fullname]` with a user-specified string. This is a no-op for # other templates @@ -76,7 +81,7 @@ module Gitlab end # Defines which strategy will be used to get templates files - # RepoTemplateFinder - Finds templates on project repository, templates are filtered perproject + # RepoTemplateFinder - Finds templates on project repository, templates are filtered per project # GlobalTemplateFinder - Finds templates on gitlab installation source, templates can be used in all projects def finder(project = nil) raise NotImplementedError @@ -95,19 +100,29 @@ module Gitlab File.join(base_dir, categories[category]) end - # If template is organized by category it returns { category_name: [{ name: template_name }, { name: template2_name }] } - # If no category is present returns [{ name: template_name }, { name: template2_name}] - def dropdown_names(project = nil) - return [] if project && !project.repository.exists? + # `repository_template_names` - reads through Gitaly the actual templates names within a + # given project's repository. This is only used by issue and merge request templates, + # that need to call this once and then cache the returned value. + # + # `template_names` - is an alias to `repository_template_names`. It would read through + # Gitaly the actual template names within a given project's repository for all file templates + # other than `issue` and `merge request` description templates, which would instead + # overwrite the `template_names` method to return a redis cached version, by reading cached values + # from `repository.issue_template_names_by_category` and `repository.merge_request_template_names_by_category` + # methods. + def repository_template_names(project) + template_names_by_category(self.all(project)) + end + alias_method :template_names, :repository_template_names - if categories.any? - categories.keys.map do |category| - files = self.by_category(category, project) - [category, files.map { |t| { name: t.name } }] - end.to_h - else - files = self.all(project) - files.map { |t| { name: t.name } } + def template_names_by_category(items) + grouped = items.group_by(&:category) + categories = grouped.keys + + categories.each_with_object({}) do |category, hash| + hash[category] = grouped[category].map do |item| + { name: item.name, id: item.key, key: item.key, project_id: item.try(:project_id) } + end end end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 9b39d386674..6d2677175e6 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -5,9 +5,10 @@ module Gitlab module Template module Finders class GlobalTemplateFinder < BaseTemplateFinder - def initialize(base_dir, extension, categories = {}, excluded_patterns: []) + def initialize(base_dir, extension, categories = {}, include_categories_for_file = {}, excluded_patterns: []) @categories = categories @extension = extension + @include_categories_for_file = include_categories_for_file @excluded_patterns = excluded_patterns super(base_dir) @@ -47,7 +48,9 @@ module Gitlab end def select_directory(file_name) - @categories.keys.find do |category| + categories = @categories + categories.merge!(@include_categories_for_file[file_name]) if @include_categories_for_file[file_name].present? + categories.keys.find do |category| File.exist?(File.join(category_directory(category), file_name)) end end diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 8e234148a63..9f0ba97bcdf 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -11,8 +11,8 @@ module Gitlab def initialize(project, base_dir, extension, categories = {}) @categories = categories @extension = extension - @repository = project.repository - @commit = @repository.head_commit if @repository.exists? + @repository = project&.repository + @commit = @repository.head_commit if @repository&.exists? super(base_dir) end @@ -51,7 +51,7 @@ module Gitlab private def select_directory(file_name) - return [] unless @commit + return unless @commit # Insert root as directory directories = ["", *@categories.keys] diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index c295cc75da5..01158cafc4f 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -25,6 +25,12 @@ module Gitlab } end + def include_categories_for_file + { + "SAST#{self.extension}" => { 'Security' => 'Security' } + } + end + def excluded_patterns strong_memoize(:excluded_patterns) do BASE_EXCLUDED_PATTERNS + additional_excluded_patterns @@ -41,7 +47,11 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new( - self.base_dir, self.extension, self.categories, excluded_patterns: self.excluded_patterns + self.base_dir, + self.extension, + self.categories, + self.include_categories_for_file, + excluded_patterns: self.excluded_patterns ) end end diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb index 01b191733d4..3049f43b322 100644 --- a/lib/gitlab/template/issue_template.rb +++ b/lib/gitlab/template/issue_template.rb @@ -15,6 +15,16 @@ module Gitlab def finder(project) Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) end + + def template_names(project) + return {} unless project&.repository&.exists? + + # here we rely on project.repository caching mechanism. Ideally we would want the template finder to have its + # own caching mechanism to avoid the back and forth call jumps between finder and model. + # + # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 + project.repository.issue_template_names_by_category + end end end end diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb index 357b31cd82e..9442f3b13fb 100644 --- a/lib/gitlab/template/merge_request_template.rb +++ b/lib/gitlab/template/merge_request_template.rb @@ -15,6 +15,16 @@ module Gitlab def finder(project) Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) end + + def template_names(project) + return {} unless project&.repository&.exists? + + # here we rely on project.repository caching mechanism. Ideally we would want the template finder to have its + # own caching mechanism to avoid the back and forth call jumps between finder and model. + # + # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 + project.repository.merge_request_template_names_by_category + end end end end diff --git a/lib/gitlab/terraform/state_migration_helper.rb b/lib/gitlab/terraform/state_migration_helper.rb new file mode 100644 index 00000000000..04c1cbd0373 --- /dev/null +++ b/lib/gitlab/terraform/state_migration_helper.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Terraform + class StateMigrationHelper + class << self + def migrate_to_remote_storage(&block) + migrate_in_batches( + ::Terraform::StateVersion.with_files_stored_locally.preload_state, + ::Terraform::StateUploader::Store::REMOTE, + &block + ) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate_in_batches(versions, store, &block) + versions.find_each(batch_size: batch_size) do |version| # rubocop:disable CodeReuse/ActiveRecord + version.file.migrate!(store) + + yield version if block_given? + end + end + end + end + end +end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index ca4afb4c19c..09697705361 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -24,8 +24,8 @@ module Gitlab Gitlab::CurrentSettings.snowplow_enabled? end - def event(category, action, label: nil, property: nil, value: nil, context: [], standard_context: nil) - context.push(standard_context.to_context) if standard_context + def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil) # rubocop:disable Metrics/ParameterLists + context += [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context] snowplow.event(category, action, label: label, property: property, value: value, context: context) product_analytics.event(category, action, label: label, property: property, value: value, context: context) diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 71dfe27dd5a..92fdd008249 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,38 +3,36 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-1'.freeze + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-3'.freeze + GITLAB_RAILS_SOURCE = 'gitlab-rails'.freeze - def initialize(namespace: nil, project: nil, **data) - @namespace = namespace - @project = project + def initialize(namespace: nil, project: nil, user: nil, **data) @data = data end - def namespace_id - namespace&.id + def to_context + SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) end - def project_id - @project&.id + def environment + return 'production' if Gitlab.com_and_canary? + + return 'staging' if Gitlab.staging? + + 'development' end - def to_context - SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) + def source + GITLAB_RAILS_SOURCE end private - def namespace - @namespace || @project&.namespace - end - def to_h - public_methods(false).each_with_object({}) do |method, hash| - next if method == :to_context - - hash[method] = public_send(method) # rubocop:disable GitlabSecurity/PublicSend - end.merge(@data) + { + environment: environment, + source: source + }.merge(@data) end end end diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb new file mode 100644 index 00000000000..8483334800b --- /dev/null +++ b/lib/gitlab/usage/docs/helper.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + # Helper with functions to be used by HAML templates + module Helper + HEADER = %w(field value).freeze + SKIP_KEYS = %i(description).freeze + + def auto_generated_comment + <<-MARKDOWN.strip_heredoc + --- + stage: Growth + group: Product Intelligence + info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers + --- + + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake. + ---> + + <!-- vale gitlab.Spelling = NO --> + MARKDOWN + end + + def render_name(name) + "## `#{name}`\n" + end + + def render_description(object) + object.description + end + + def render_attribute_row(key, value) + value = Gitlab::Usage::Docs::ValueFormatter.format(key, value) + table_row(["`#{key}`", value]) + end + + def render_attributes_table(object) + <<~MARKDOWN + + #{table_row(HEADER)} + #{table_row(HEADER.map { '---' })} + #{table_value_rows(object.attributes)} + MARKDOWN + end + + def table_value_rows(attributes) + attributes.reject { |k, _| k.in?(SKIP_KEYS) }.map do |key, value| + render_attribute_row(key, value) + end.join("\n") + end + + def table_row(array) + "| #{array.join(' | ')} |" + end + end + end + end +end diff --git a/lib/gitlab/usage/docs/renderer.rb b/lib/gitlab/usage/docs/renderer.rb new file mode 100644 index 00000000000..7a7c58005bb --- /dev/null +++ b/lib/gitlab/usage/docs/renderer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + class Renderer + include Gitlab::Usage::Docs::Helper + DICTIONARY_PATH = Rails.root.join('doc', 'development', 'usage_ping') + TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'usage', 'docs', 'templates', 'default.md.haml') + + def initialize(metrics_definitions) + @layout = Haml::Engine.new(File.read(TEMPLATE_PATH)) + @metrics_definitions = metrics_definitions.sort + end + + def contents + # Render and remove an extra trailing new line + @contents ||= @layout.render(self, metrics_definitions: @metrics_definitions).sub!(/\n(?=\Z)/, '') + end + + def write + filename = DICTIONARY_PATH.join('dictionary.md').to_s + + FileUtils.mkdir_p(DICTIONARY_PATH) + File.write(filename, contents) + + filename + end + end + end + end +end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml new file mode 100644 index 00000000000..86e93be66c7 --- /dev/null +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -0,0 +1,28 @@ += auto_generated_comment + +:plain + # Metrics Dictionary + + This file is autogenerated, please do not edit directly. + + To generate these files from the GitLab repository, run: + + ```shell + bundle exec rake gitlab:usage_data:generate_metrics_dictionary + ``` + + The Metrics Dictionary is based on the following metrics definition YAML files: + + - [`config/metrics`]('https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics') + - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics) + +Each table includes a `milestone`, which corresponds to the GitLab version when the metric +was released. +\ +- metrics_definitions.each do |name, object| + + = render_name(name) + + = render_description(object) + + = render_attributes_table(object) diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb new file mode 100644 index 00000000000..a2dc9b081f8 --- /dev/null +++ b/lib/gitlab/usage/docs/value_formatter.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + class ValueFormatter + def self.format(key, value) + case key + when :key_path + "**`#{value}`**" + when :data_source + value.to_s.capitalize + when :product_group + "`#{value}`" + when :introduced_by_url + "[Introduced by](#{value})" + when :distribution, :tier + Array(value).join(', ') + else + value + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index e1648c78168..f3469209f48 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -7,16 +7,16 @@ module Gitlab InvalidMetricError = Class.new(RuntimeError) - attr_accessor :default_generation_path, :value + attr_accessor :key_path, :value - validates :default_generation_path, presence: true + validates :key_path, presence: true def definition - self.class.definitions[default_generation_path] + self.class.definitions[key_path] end - def unflatten_default_path - unflatten(default_generation_path.split('.'), value) + def unflatten_key_path + unflatten(key_path.split('.'), value) end class << self diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 96e572bb3db..01d202e4d45 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -13,9 +13,8 @@ module Gitlab @attributes = opts end - # The key is defined by default_generation and full_path def key - full_path[default_generation.to_sym] + key_path end def to_h @@ -23,8 +22,10 @@ module Gitlab end def validate! - self.class.schemer.validate(attributes.stringify_keys).map do |error| - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + unless skip_validation? + self.class.schemer.validate(attributes.stringify_keys).each do |error| + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + end end end @@ -79,6 +80,10 @@ module Gitlab def method_missing(method, *args) attributes[method] || super end + + def skip_validation? + !!attributes[:skip_validation] + end end end end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb new file mode 100644 index 00000000000..1fc40798320 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + UNION_OF_AGGREGATED_METRICS = 'OR' + INTERSECTION_OF_AGGREGATED_METRICS = 'AND' + ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze + AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml') + AggregatedMetricError = Class.new(StandardError) + UnknownAggregationOperator = Class.new(AggregatedMetricError) + UnknownAggregationSource = Class.new(AggregatedMetricError) + + DATABASE_SOURCE = 'database' + REDIS_SOURCE = 'redis' + + SOURCES = { + DATABASE_SOURCE => Sources::PostgresHll, + REDIS_SOURCE => Sources::RedisHll + }.freeze + + class Aggregate + delegate :weekly_time_range, + :monthly_time_range, + to: Gitlab::UsageDataCounters::HLLRedisCounter + + def initialize(recorded_at) + @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH) + @recorded_at = recorded_at + end + + def monthly_data + aggregated_metrics_data(**monthly_time_range) + end + + def weekly_data + aggregated_metrics_data(**weekly_time_range) + end + + private + + attr_accessor :aggregated_metrics, :recorded_at + + def aggregated_metrics_data(start_date:, end_date:) + aggregated_metrics.each_with_object({}) do |aggregation, data| + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development) + + case aggregation[:source] + when REDIS_SOURCE + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + when DATABASE_SOURCE + next unless Feature.enabled?('database_sourced_aggregated_metrics', default_enabled: false, type: :development) + + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + else + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception(UnknownAggregationSource.new("Aggregation source: '#{aggregation[:source]}' must be included in #{SOURCES.keys}")) + + data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK + end + end + end + + def calculate_count_for_aggregation(aggregation:, start_date:, end_date:) + source = SOURCES[aggregation[:source]] + + case aggregation[:operator] + when UNION_OF_AGGREGATED_METRICS + source.calculate_metrics_union(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) + when INTERSECTION_OF_AGGREGATED_METRICS + calculate_metrics_intersections(source: source, metric_names: aggregation[:events], start_date: start_date, end_date: end_date) + else + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception(UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")) + Gitlab::Utils::UsageData::FALLBACK + end + rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::Utils::UsageData::FALLBACK + end + + # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle + # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 + def calculate_metrics_intersections(source:, metric_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_metrics = begin + subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ + source.calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_metrics : -power_of_union_of_all_metrics) + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + + def subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) + subset_sizes = (1...metric_names.size) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + metric_names.combination(subset_size).sum do |metrics_subset| + subset_powers_cache[subset_size][metrics_subset.join('_&_')] ||= + calculate_metrics_intersections(source: source, metric_names: metrics_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + metric_names.sum do |metric| + subset_powers_cache[subset_size][metric] ||= \ + source.calculate_metrics_union(metric_names: metric, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + end + end + end + + def load_metrics(wildcard) + Dir[wildcard].each_with_object([]) do |path, metrics| + metrics.push(*load_yaml_from_path(path)) + end + end + + def load_yaml_from_path(path) + YAML.safe_load(File.read(path), aliases: true)&.map(&:with_indifferent_access) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb new file mode 100644 index 00000000000..33678d2b813 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + module Sources + class PostgresHll + class << self + def calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at:) + time_period = start_date && end_date ? (start_date..end_date) : nil + + Array(metric_names).each_with_object(Gitlab::Database::PostgresHll::Buckets.new) do |event, buckets| + json = read_aggregated_metric(metric_name: event, time_period: time_period, recorded_at: recorded_at) + raise UnionNotAvailable, "Union data not available for #{metric_names}" unless json + + buckets.merge_hash!(Gitlab::Json.parse(json)) + end.estimated_distinct_count + end + + def save_aggregated_metrics(metric_name:, time_period:, recorded_at_timestamp:, data:) + unless data.is_a? ::Gitlab::Database::PostgresHll::Buckets + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(StandardError.new("Unsupported data type: #{data.class}")) + return + end + + # Usage Ping report generation for gitlab.com is very long running process + # to make sure that saved keys are available at the end of report generation process + # lets use triple max generation time + keys_expiration = ::Gitlab::UsageData::MAX_GENERATION_TIME_FOR_SAAS * 3 + + Gitlab::Redis::SharedState.with do |redis| + redis.set( + redis_key(metric_name: metric_name, time_period: time_period&.values&.first, recorded_at: recorded_at_timestamp), + data.to_json, + ex: keys_expiration + ) + end + rescue ::Redis::CommandError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + private + + def read_aggregated_metric(metric_name:, time_period:, recorded_at:) + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_key(metric_name: metric_name, time_period: time_period, recorded_at: recorded_at)) + end + end + + def redis_key(metric_name:, time_period:, recorded_at:) + # add timestamp at the end of the key to avoid stale keys if + # usage ping job is retried + "#{metric_name}_#{time_period_to_human_name(time_period)}-#{recorded_at.to_i}" + end + + def time_period_to_human_name(time_period) + return Gitlab::Utils::UsageData::ALL_TIME_PERIOD_HUMAN_NAME if time_period.blank? + + start_date = time_period.first.to_date + end_date = time_period.last.to_date + + if (end_date - start_date).to_i > 7 + Gitlab::Utils::UsageData::MONTHLY_PERIOD_HUMAN_NAME + else + Gitlab::Utils::UsageData::WEEKLY_PERIOD_HUMAN_NAME + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb new file mode 100644 index 00000000000..f3a4dcf1e31 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + module Sources + UnionNotAvailable = Class.new(AggregatedMetricError) + + class RedisHll + def self.calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at: nil) + union = Gitlab::UsageDataCounters::HLLRedisCounter + .calculate_events_union(event_names: metric_names, start_date: start_date, end_date: end_date) + + return union if union >= 0 + + raise UnionNotAvailable, "Union data not available for #{metric_names}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f935c677930..8e096a9f351 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -12,6 +12,9 @@ # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } module Gitlab class UsageData + DEPRECATED_VALUE = -1000 + MAX_GENERATION_TIME_FOR_SAAS = 40.hours + CE_MEMOIZED_VALUES = %i( issue_minimum_id issue_maximum_id @@ -23,6 +26,8 @@ module Gitlab deployment_minimum_id deployment_maximum_id auth_providers + aggregated_metrics + recorded_at ).freeze class << self @@ -75,7 +80,7 @@ module Gitlab end def recorded_at - Time.current + @recorded_at ||= Time.current end # rubocop: disable Metrics/AbcSize @@ -158,7 +163,7 @@ module Gitlab projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), - projects_with_alerts_service_enabled: count(AlertsService.active), + projects_with_alerts_service_enabled: count(Service.active.where(type: 'AlertsService')), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), @@ -580,27 +585,35 @@ module Gitlab users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), + unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { - gitlab: distinct_count(::BulkImport.where(time_period, source_type: :gitlab), :user_id) + gitlab: DEPRECATED_VALUE, + gitlab_v1: count(::BulkImport.where(time_period, source_type: :gitlab)) }, + project_imports: project_imports(time_period), + issue_imports: issue_imports(time_period), + group_imports: group_imports(time_period), + + # Deprecated data to be removed projects_imported: { - total: distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id), - gitlab_project: projects_imported_count('gitlab_project', time_period), - gitlab: projects_imported_count('gitlab', time_period), - github: projects_imported_count('github', time_period), - bitbucket: projects_imported_count('bitbucket', time_period), - bitbucket_server: projects_imported_count('bitbucket_server', time_period), - gitea: projects_imported_count('gitea', time_period), - git: projects_imported_count('git', time_period), - manifest: projects_imported_count('manifest', time_period) + total: DEPRECATED_VALUE, + gitlab_project: DEPRECATED_VALUE, + gitlab: DEPRECATED_VALUE, + github: DEPRECATED_VALUE, + bitbucket: DEPRECATED_VALUE, + bitbucket_server: DEPRECATED_VALUE, + gitea: DEPRECATED_VALUE, + git: DEPRECATED_VALUE, + manifest: DEPRECATED_VALUE }, issues_imported: { - jira: distinct_count(::JiraImportState.where(time_period), :user_id), - fogbugz: projects_imported_count('fogbugz', time_period), - phabricator: projects_imported_count('phabricator', time_period), - csv: distinct_count(Issues::CsvImport.where(time_period), :user_id) + jira: DEPRECATED_VALUE, + fogbugz: DEPRECATED_VALUE, + phabricator: DEPRECATED_VALUE, + csv: DEPRECATED_VALUE }, - groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) + groups_imported: DEPRECATED_VALUE + # End of deprecated keys } end # rubocop: enable CodeReuse/ActiveRecord @@ -690,13 +703,13 @@ module Gitlab def aggregated_metrics_monthly { - aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_monthly_data + aggregated_metrics: aggregated_metrics.monthly_data } end def aggregated_metrics_weekly { - aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_weekly_data + aggregated_metrics: aggregated_metrics.weekly_data } end @@ -741,6 +754,10 @@ module Gitlab private + def aggregated_metrics + @aggregated_metrics ||= ::Gitlab::Usage::Metrics::Aggregates::Aggregate.new(recorded_at) + end + def event_monthly_active_users(date_range) data = { action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, @@ -893,10 +910,52 @@ module Gitlab count relation, start: deployment_minimum_id, finish: deployment_maximum_id end + def project_imports(time_period) + { + gitlab_project: projects_imported_count('gitlab_project', time_period), + gitlab: projects_imported_count('gitlab', time_period), + github: projects_imported_count('github', time_period), + bitbucket: projects_imported_count('bitbucket', time_period), + bitbucket_server: projects_imported_count('bitbucket_server', time_period), + gitea: projects_imported_count('gitea', time_period), + git: projects_imported_count('git', time_period), + manifest: projects_imported_count('manifest', time_period), + gitlab_migration: count(::BulkImports::Entity.where(time_period).project_entity) # rubocop: disable CodeReuse/ActiveRecord + } + end + def projects_imported_count(from, time_period) - distinct_count(::Project.imported_from(from).where(time_period).where.not(import_type: nil), :creator_id) # rubocop: disable CodeReuse/ActiveRecord + count(::Project.imported_from(from).where(time_period).where.not(import_type: nil)) # rubocop: disable CodeReuse/ActiveRecord end + def issue_imports(time_period) + { + jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord + fogbugz: projects_imported_count('fogbugz', time_period), + phabricator: projects_imported_count('phabricator', time_period), + csv: count(Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord + } + end + + def group_imports(time_period) + { + group_import: count(::GroupImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord + gitlab_migration: count(::BulkImports::Entity.where(time_period).group_entity) # rubocop: disable CodeReuse/ActiveRecord + } + end + + # rubocop:disable CodeReuse/ActiveRecord + def unique_users_all_imports(time_period) + project_imports = distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id) + bulk_imports = distinct_count(::BulkImport.where(time_period), :user_id) + jira_issue_imports = distinct_count(::JiraImportState.where(time_period), :user_id) + csv_issue_imports = distinct_count(Issues::CsvImport.where(time_period), :user_id) + group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) + + project_imports + bulk_imports + jira_issue_imports + csv_issue_imports + group_imports + end + # rubocop:enable CodeReuse/ActiveRecord + # rubocop:disable CodeReuse/ActiveRecord def distinct_count_user_auth_by_provider(time_period) counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index 4966afd534a..4d92202e7fd 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -4,21 +4,28 @@ # - "AND": counts unique elements that were observed triggering all of following events # events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes # see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events. +# source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are: +# - database +# - redis # feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. # Corresponding feature flag should have `default_enabled` attribute set to `false`. # This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. --- - name: compliance_features_track_unique_visits_union operator: OR + source: redis events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] - name: product_analytics_test_metrics_union operator: OR + source: redis events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: product_analytics_test_metrics_intersection operator: AND + source: redis events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: incident_management_alerts_total_unique_counts operator: OR + source: redis events: [ 'incident_management_alert_status_changed', 'incident_management_alert_assigned', @@ -27,6 +34,7 @@ ] - name: incident_management_incidents_total_unique_counts operator: OR + source: redis events: [ 'incident_management_incident_created', 'incident_management_incident_reopened', @@ -40,3 +48,13 @@ 'incident_management_incident_unrelate', 'incident_management_incident_change_confidential' ] +- name: i_testing_paid_monthly_active_user_total + operator: OR + source: redis + events: [ + 'i_testing_web_performance_widget_total', + 'i_testing_full_code_quality_report_total', + 'i_testing_group_code_coverage_visit_total', + 'i_testing_load_performance_widget_total', + 'i_testing_metrics_report_widget_total' +] diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 572ad866895..772a4623280 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -4,7 +4,9 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter REDIS_SLOT = 'ci_templates'.freeze + # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_` TEMPLATE_TO_EVENT = { + '5-Minute-Production-App.gitlab-ci.yml' => '5_min_production_app', 'Auto-DevOps.gitlab-ci.yml' => 'auto_devops', 'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' => 'aws_cf_deploy_ec2', 'AWS/Deploy-ECS.gitlab-ci.yml' => 'aws_deploy_ecs', @@ -17,19 +19,21 @@ module Gitlab::UsageDataCounters }.freeze class << self - def track_unique_project_event(project_id:, template:) + def track_unique_project_event(project_id:, template:, config_source:) return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml) - if event = unique_project_event(template) + if event = unique_project_event(template, config_source) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id) end end private - def unique_project_event(template) + def unique_project_event(template, config_source) if name = TEMPLATE_TO_EVENT[template] - "p_#{REDIS_SLOT}_#{name}" + prefix = 'implicit_' if config_source.to_s == 'auto_devops_source' + + "p_#{REDIS_SLOT}_#{prefix}#{name}" end end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 47361d831b2..68ae239debb 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -13,15 +13,10 @@ module Gitlab AggregationMismatch = Class.new(EventError) SlotMismatch = Class.new(EventError) CategoryMismatch = Class.new(EventError) - UnknownAggregationOperator = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze - UNION_OF_AGGREGATED_METRICS = 'OR' - INTERSECTION_OF_AGGREGATED_METRICS = 'AND' - ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = File.expand_path('aggregated_metrics/*.yml', __dir__) # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -90,37 +85,40 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(event_names: [event], start_date: 7.days.ago.to_date, end_date: Date.current) - hash["#{event}_monthly"] = unique_events(event_names: [event], start_date: 4.weeks.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) + hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) end if eligible_for_totals?(events_names) - event_results["#{category}_total_unique_counts_weekly"] = unique_events(event_names: events_names, start_date: 7.days.ago.to_date, end_date: Date.current) - event_results["#{category}_total_unique_counts_monthly"] = unique_events(event_names: events_names, start_date: 4.weeks.ago.to_date, end_date: Date.current) + event_results["#{category}_total_unique_counts_weekly"] = unique_events(**weekly_time_range.merge(event_names: events_names)) + event_results["#{category}_total_unique_counts_monthly"] = unique_events(**monthly_time_range.merge(event_names: events_names)) end category_results["#{category}"] = event_results end end - def known_event?(event_name) - event_for(event_name).present? + def weekly_time_range + { start_date: 7.days.ago.to_date, end_date: Date.current } end - def aggregated_metrics_monthly_data - aggregated_metrics_data(4.weeks.ago.to_date) + def monthly_time_range + { start_date: 4.weeks.ago.to_date, end_date: Date.current } end - def aggregated_metrics_weekly_data - aggregated_metrics_data(7.days.ago.to_date) + def known_event?(event_name) + event_for(event_name).present? end def known_events @known_events ||= load_events(KNOWN_EVENTS_PATH) end - def aggregated_metrics - @aggregated_metrics ||= load_events(AGGREGATED_METRICS_PATH) + def calculate_events_union(event_names:, start_date:, end_date:) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + end end private @@ -131,6 +129,8 @@ module Gitlab event = event_for(event_name) raise UnknownEvent, "Unknown event #{event_name}" unless event.present? + return unless feature_enabled?(event) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) end @@ -139,93 +139,6 @@ module Gitlab Plan.all_plans end - def aggregated_metrics_data(start_date) - aggregated_metrics.each_with_object({}) do |aggregation, weekly_data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development) - - weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: Date.current) - end - end - - def calculate_count_for_aggregation(aggregation, start_date:, end_date:) - case aggregation[:operator] - when UNION_OF_AGGREGATED_METRICS - calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date) - when INTERSECTION_OF_AGGREGATED_METRICS - calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date) - else - raise UnknownAggregationOperator, "Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}" - end - end - - # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle - # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 - def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) - # calculate power of intersection of all given metrics from inclusion exclusion principle - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - - # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... - subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) - - # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| - power_of_union_of_all_events = begin - subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \ - calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date) - end - - # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, - # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - subset_powers_size_even = subset_powers_data.size.even? - - # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => - sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) - - # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| - sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events) - end - - def sum_subset_powers(subset_powers_data, subset_powers_size_even) - sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| - (index + 1).odd? ? value : -value - end - - (subset_powers_size_even ? -1 : 1) * sum_without_sign - end - - def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) - subset_sizes = (1..(event_names.size - 1)) - - subset_sizes.map do |subset_size| - if subset_size > 1 - # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) - event_names.combination(subset_size).sum do |events_subset| - subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \ - calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) - end - else - # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... - event_names.sum do |event| - subset_powers_cache[subset_size][event] ||= \ - unique_events(event_names: event, start_date: start_date, end_date: end_date) - end - end - end - end - - def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise SlotMismatch, events unless events_in_same_slot?(events) - raise AggregationMismatch, events unless events_same_aggregation?(events) - end - end - def count_unique_events(event_names:, start_date:, end_date:, context: '') events = events_for(Array(event_names).map(&:to_s)) @@ -237,6 +150,12 @@ module Gitlab redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end + def feature_enabled?(event) + return true if event[:feature_flag].blank? + + Feature.enabled?(event[:feature_flag], default_enabled: :yaml) + end + # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level # and if there are more than 1 event def eligible_for_totals?(events_names) @@ -340,12 +259,6 @@ module Gitlab end.flatten end - def validate_aggregation_operator!(operator) - return true if ALLOWED_METRICS_AGGREGATIONS.include?(operator) - - raise UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") - end - def weekly_redis_keys(events:, start_date:, end_date:, context: '') end_date = end_date.end_of_week - 1.week (start_date.to_date..end_date.to_date).map do |date| diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index f649e7f407d..c2662a74432 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -145,7 +145,6 @@ module Gitlab private def track_unique_action(action, author, time) - return unless Feature.enabled?(:track_issue_activity_actions, default_enabled: true) return unless author Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml new file mode 100644 index 00000000000..9c19c9e8b8c --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -0,0 +1,91 @@ +# Implicit Auto DevOps pipeline events +- name: p_ci_templates_implicit_auto_devops + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_auto_devops_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_auto_devops_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_security_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_security_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +# Explicit include:template pipeline events +- name: p_ci_templates_5_min_production_app + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_aws_cf_deploy_ec2 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_aws_deploy_ecs + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops_deploy_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_security_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_security_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_terraform_base_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml new file mode 100644 index 00000000000..d657c5487d7 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -0,0 +1,166 @@ +--- +- name: i_code_review_mr_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_diffs +- name: i_code_review_user_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_single_file_diffs +- name: i_code_review_mr_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_single_file_diffs +- name: i_code_review_user_toggled_task_item_status + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_toggled_task_item_status +- name: i_code_review_user_create_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr +- name: i_code_review_user_close_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_close_mr +- name: i_code_review_user_reopen_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_reopen_mr +- name: i_code_review_user_approve_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approve_mr +- name: i_code_review_user_unapprove_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_unapprove_mr +- name: i_code_review_user_resolve_thread + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_resolve_thread +- name: i_code_review_user_unresolve_thread + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_unresolve_thread +- name: i_code_review_edit_mr_title + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_edit_mr_title +- name: i_code_review_edit_mr_desc + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_edit_mr_desc +- name: i_code_review_user_merge_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_merge_mr +- name: i_code_review_user_create_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr_comment +- name: i_code_review_user_edit_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_mr_comment +- name: i_code_review_user_remove_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_mr_comment +- name: i_code_review_user_create_review_note + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_review_note +- name: i_code_review_user_publish_review + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_publish_review +- name: i_code_review_user_create_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment +- name: i_code_review_user_edit_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment +- name: i_code_review_user_remove_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment +- name: i_code_review_user_add_suggestion + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_add_suggestion +- name: i_code_review_user_apply_suggestion + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_apply_suggestion +- name: i_code_review_user_assigned + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_assigned +- name: i_code_review_user_marked_as_draft + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_marked_as_draft +- name: i_code_review_user_unmarked_as_draft + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_unmarked_as_draft +- name: i_code_review_user_review_requested + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_review_requested +- name: i_code_review_user_approval_rule_added + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approval_rule_added +- name: i_code_review_user_approval_rule_deleted + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approval_rule_deleted +- name: i_code_review_user_approval_rule_edited + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approval_rule_edited +- name: i_code_review_user_vs_code_api_request + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_vs_code_api_request +- name: i_code_review_user_create_mr_from_issue + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr_from_issue diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 4cbde0c0372..79f319b2d58 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -268,172 +268,154 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_web_performance_widget_total +- name: i_testing_group_code_coverage_project_click_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_group_code_coverage_project_click_total +- name: i_testing_load_performance_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_load_performance_widget_total +- name: i_testing_metrics_report_artifact_uploaders + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_metrics_report_artifact_uploaders # Project Management group - name: g_project_management_issue_title_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_description_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_assignee_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_confidential category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_visible category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_created category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_closed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_reopened category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_label_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_milestone_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_iteration_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_weight_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_cross_referenced category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_moved category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_related category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_unrelated category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_marked_as_duplicate category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_locked category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_unlocked category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_added_to_epic category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_removed_from_epic category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_changed_epic category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_added category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_modified category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_removed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_due_date_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_estimate_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_spent_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_comment_added category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_comment_edited category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_comment_removed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_health_status_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_cloned category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions # Secrets Management - name: i_ci_secrets_management_vault_build_created category: ci_secrets_management @@ -445,126 +427,15 @@ redis_slot: snippets aggregation: weekly feature_flag: usage_data_i_snippets_show -# Merge request counters -- name: i_code_review_mr_diffs - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_mr_diffs -- name: i_code_review_user_single_file_diffs - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_single_file_diffs -- name: i_code_review_mr_single_file_diffs - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_mr_single_file_diffs -- name: i_code_review_user_create_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr -- name: i_code_review_user_close_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_close_mr -- name: i_code_review_user_reopen_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_reopen_mr -- name: i_code_review_user_merge_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_merge_mr -- name: i_code_review_user_create_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr_comment -- name: i_code_review_user_edit_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_mr_comment -- name: i_code_review_user_remove_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_mr_comment -- name: i_code_review_user_create_review_note - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_review_note -- name: i_code_review_user_publish_review - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_publish_review -- name: i_code_review_user_create_multiline_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment -- name: i_code_review_user_edit_multiline_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment -- name: i_code_review_user_remove_multiline_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment # Terraform - name: p_terraform_state_api_unique_users category: terraform redis_slot: terraform aggregation: weekly feature_flag: usage_data_p_terraform_state_api_unique_users -# CI templates -- name: p_ci_templates_auto_devops - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_aws_cf_deploy_ec2 - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_auto_devops_build - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_auto_devops_deploy - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_auto_devops_deploy_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_security_sast - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_security_secret_detection - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_terraform_base_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects +# Pipeline Authoring +- name: o_pipeline_authoring_unique_users_committing_ciconfigfile + category: pipeline_authoring + redis_slot: pipeline_authoring + aggregation: weekly + feature_flag: usage_data_unique_users_committing_ciconfigfile diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml new file mode 100644 index 00000000000..3fd02164f74 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -0,0 +1,22 @@ +--- +# Ecosystem category +- name: i_ecosystem_jira_service_close_issue + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_jira_service_cross_reference + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_jira_service_list_issues + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_jira_service_create_issue + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml new file mode 100644 index 00000000000..bf292047da0 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -0,0 +1,326 @@ +--- +- name: i_quickactions_approve + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_single + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_multiple + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_self + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_award + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_board_move + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_child_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_clear_weight + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_clone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_close + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_confidential + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_copy_metadata_merge_request + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_copy_metadata_issue + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_create_merge_request + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_done + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_draft + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_due + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_duplicate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_estimate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_iteration + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_label + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_lock + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_merge + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_milestone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_move + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_parent_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_promote + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_publish + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reassign + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reassign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_rebase + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_relabel + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_relate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_child_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_due_date + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_estimate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_iteration + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_milestone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_parent_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_time_spent + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_zoom + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reopen + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_shrug + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_spend_subtract + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_spend_add + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_submit_review + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_subscribe + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_tableflip + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_tag + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_target_branch + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_title + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_todo + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_specific + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_all + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlabel_specific + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlabel_all + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlock + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unsubscribe + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_weight + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_wip + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_zoom + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 11d59257ed9..b9856e1f74a 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -10,6 +10,8 @@ module Gitlab MR_CLOSE_ACTION = 'i_code_review_user_close_mr' MR_REOPEN_ACTION = 'i_code_review_user_reopen_mr' MR_MERGE_ACTION = 'i_code_review_user_merge_mr' + MR_APPROVE_ACTION = 'i_code_review_user_approve_mr' + MR_UNAPPROVE_ACTION = 'i_code_review_user_unapprove_mr' MR_CREATE_COMMENT_ACTION = 'i_code_review_user_create_mr_comment' MR_EDIT_COMMENT_ACTION = 'i_code_review_user_edit_mr_comment' MR_REMOVE_COMMENT_ACTION = 'i_code_review_user_remove_mr_comment' @@ -18,6 +20,21 @@ module Gitlab MR_CREATE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_create_multiline_mr_comment' MR_EDIT_MULTILINE_COMMENT_ACTION = 'i_code_review_user_edit_multiline_mr_comment' MR_REMOVE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_remove_multiline_mr_comment' + MR_ADD_SUGGESTION_ACTION = 'i_code_review_user_add_suggestion' + MR_APPLY_SUGGESTION_ACTION = 'i_code_review_user_apply_suggestion' + MR_MARKED_AS_DRAFT_ACTION = 'i_code_review_user_marked_as_draft' + MR_UNMARKED_AS_DRAFT_ACTION = 'i_code_review_user_unmarked_as_draft' + MR_RESOLVE_THREAD_ACTION = 'i_code_review_user_resolve_thread' + MR_UNRESOLVE_THREAD_ACTION = 'i_code_review_user_unresolve_thread' + MR_ASSIGNED_USERS_ACTION = 'i_code_review_user_assigned' + MR_REVIEW_REQUESTED_USERS_ACTION = 'i_code_review_user_review_requested' + MR_TASK_ITEM_STATUS_CHANGED_ACTION = 'i_code_review_user_toggled_task_item_status' + MR_APPROVAL_RULE_ADDED_USERS_ACTION = 'i_code_review_user_approval_rule_added' + MR_APPROVAL_RULE_EDITED_USERS_ACTION = 'i_code_review_user_approval_rule_edited' + MR_APPROVAL_RULE_DELETED_USERS_ACTION = 'i_code_review_user_approval_rule_deleted' + MR_EDIT_MR_TITLE_ACTION = 'i_code_review_edit_mr_title' + MR_EDIT_MR_DESC_ACTION = 'i_code_review_edit_mr_desc' + MR_CREATE_FROM_ISSUE_ACTION = 'i_code_review_user_create_mr_from_issue' class << self def track_mr_diffs_action(merge_request:) @@ -45,6 +62,22 @@ module Gitlab track_unique_action_by_user(MR_REOPEN_ACTION, user) end + def track_approve_mr_action(user:) + track_unique_action_by_user(MR_APPROVE_ACTION, user) + end + + def track_unapprove_mr_action(user:) + track_unique_action_by_user(MR_UNAPPROVE_ACTION, user) + end + + def track_resolve_thread_action(user:) + track_unique_action_by_user(MR_RESOLVE_THREAD_ACTION, user) + end + + def track_unresolve_thread_action(user:) + track_unique_action_by_user(MR_UNRESOLVE_THREAD_ACTION, user) + end + def track_create_comment_action(note:) track_unique_action_by_user(MR_CREATE_COMMENT_ACTION, note.author) track_multiline_unique_action(MR_CREATE_MULTILINE_COMMENT_ACTION, note) @@ -68,6 +101,58 @@ module Gitlab track_unique_action_by_user(MR_PUBLISH_REVIEW_ACTION, user) end + def track_add_suggestion_action(user:) + track_unique_action_by_user(MR_ADD_SUGGESTION_ACTION, user) + end + + def track_marked_as_draft_action(user:) + track_unique_action_by_user(MR_MARKED_AS_DRAFT_ACTION, user) + end + + def track_unmarked_as_draft_action(user:) + track_unique_action_by_user(MR_UNMARKED_AS_DRAFT_ACTION, user) + end + + def track_apply_suggestion_action(user:) + track_unique_action_by_user(MR_APPLY_SUGGESTION_ACTION, user) + end + + def track_users_assigned_to_mr(users:) + track_unique_action_by_users(MR_ASSIGNED_USERS_ACTION, users) + end + + def track_users_review_requested(users:) + track_unique_action_by_users(MR_REVIEW_REQUESTED_USERS_ACTION, users) + end + + def track_title_edit_action(user:) + track_unique_action_by_user(MR_EDIT_MR_TITLE_ACTION, user) + end + + def track_description_edit_action(user:) + track_unique_action_by_user(MR_EDIT_MR_DESC_ACTION, user) + end + + def track_approval_rule_added_action(user:) + track_unique_action_by_user(MR_APPROVAL_RULE_ADDED_USERS_ACTION, user) + end + + def track_approval_rule_edited_action(user:) + track_unique_action_by_user(MR_APPROVAL_RULE_EDITED_USERS_ACTION, user) + end + + def track_approval_rule_deleted_action(user:) + track_unique_action_by_user(MR_APPROVAL_RULE_DELETED_USERS_ACTION, user) + end + + def track_task_item_status_changed(user:) + track_unique_action_by_user(MR_TASK_ITEM_STATUS_CHANGED_ACTION, user) + end + + def track_mr_create_from_issue(user:) + track_unique_action_by_user(MR_CREATE_FROM_ISSUE_ACTION, user) + end + private def track_unique_action_by_merge_request(action, merge_request) @@ -80,6 +165,12 @@ module Gitlab track_unique_action(action, user.id) end + def track_unique_action_by_users(action, users) + return if users.blank? + + track_unique_action(action, users.map(&:id)) + end + def track_unique_action(action, value) Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) end diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb new file mode 100644 index 00000000000..f757b51f73c --- /dev/null +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module QuickActionActivityUniqueCounter + class << self + # Tracks the quick action with name `name`. + # `args` is expected to be a single string, will be split internally when necessary. + def track_unique_action(name, args:, user:) + return unless Feature.enabled?(:usage_data_track_quickactions, default_enabled: :yaml) + return unless user + + args ||= '' + name = prepare_name(name, args) + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:"i_quickactions_#{name}", values: user.id) + end + + private + + def prepare_name(name, args) + case name + when 'assign' + event_name_for_assign(args) + when 'copy_metadata' + event_name_for_copy_metadata(args) + when 'remove_reviewer' + 'unassign_reviewer' + when 'request_review', 'reviewer' + 'assign_reviewer' + when 'spend' + event_name_for_spend(args) + when 'unassign' + event_name_for_unassign(args) + when 'unlabel', 'remove_label' + event_name_for_unlabel(args) + else + name + end + end + + def event_name_for_assign(args) + args = args.split + + if args.count == 1 && args.first == 'me' + 'assign_self' + elsif args.count == 1 + 'assign_single' + else + 'assign_multiple' + end + end + + def event_name_for_copy_metadata(args) + if args.start_with?('#') + 'copy_metadata_issue' + else + 'copy_metadata_merge_request' + end + end + + def event_name_for_spend(args) + if args.start_with?('-') + 'spend_subtract' + else + 'spend_add' + end + end + + def event_name_for_unassign(args) + if args.present? + 'unassign_specific' + else + 'unassign_all' + end + end + + def event_name_for_unlabel(args) + if args.present? + 'unlabel_specific' + else + 'unlabel_all' + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb new file mode 100644 index 00000000000..703c4885b04 --- /dev/null +++ b/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module VSCodeExtensionActivityUniqueCounter + VS_CODE_API_REQUEST_ACTION = 'i_code_review_user_vs_code_api_request' + VS_CODE_USER_AGENT_REGEX = /\Avs-code-gitlab-workflow/.freeze + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(VS_CODE_USER_AGENT_REGEX) && track_unique_action_by_user(VS_CODE_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index e783ac785cc..5087020affe 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -4,7 +4,7 @@ module Gitlab module Utils module Markdown PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze - PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate)(\s+only)?\)\**/.freeze + PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|sass))?\)\**/.freeze def string_to_anchor(string) string diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index febe489f1f8..ffd12c1b518 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -53,14 +53,10 @@ module Gitlab end def with_gc_stats - GC.start # perform a full mark-and-sweep - stats_before = GC.stat + stats = ::Gitlab::Memory::Instrumentation.start_thread_memory_allocations yield - stats_after = GC.stat - @gc_stats = stats_after.map do |key, after_value| - before_value = stats_before[key] - [key, before: before_value, after: after_value, diff: after_value - before_value] - end.to_h + ensure + @gc_stats = ::Gitlab::Memory::Instrumentation.measure_thread_memory_allocations(stats) end def with_measure_time diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 784a6686962..c92865636d0 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -153,7 +153,13 @@ module Gitlab def extended(mod = nil) super - queue_verification(mod.singleton_class) if mod + # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 + is_not_concern_hack = + (mod.is_a?(Class) || !name&.end_with?('::ClassMethods')) + + if mod && is_not_concern_hack + queue_verification(mod.singleton_class) + end end def queue_verification(base, verify: false) @@ -174,7 +180,7 @@ module Gitlab end def self.verify! - extensions.values.each(&:verify!) + extensions.each_value(&:verify!) end end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index baccadd9594..28dc66e19f8 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -39,6 +39,9 @@ module Gitlab FALLBACK = -1 DISTRIBUTED_HLL_FALLBACK = -2 + ALL_TIME_PERIOD_HUMAN_NAME = "all_time" + WEEKLY_PERIOD_HUMAN_NAME = "weekly" + MONTHLY_PERIOD_HUMAN_NAME = "monthly" def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch @@ -61,10 +64,13 @@ module Gitlab end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - Gitlab::Database::PostgresHll::BatchDistinctCounter + buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter .new(relation, column) .execute(batch_size: batch_size, start: start, finish: finish) - .estimated_distinct_count + + yield buckets if block_given? + + buckets.estimated_distinct_count rescue ActiveRecord::StatementInvalid FALLBACK # catch all rescue should be removed as a part of feature flag rollout issue diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 8e7af8876a4..e9905bae985 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -32,7 +32,7 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags + features: Feature::Gitaly.server_feature_flags(repository.project) } } @@ -231,7 +231,7 @@ module Gitlab { address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags + features: Feature::Gitaly.server_feature_flags(repository.project) } end diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb deleted file mode 100644 index b0974e02edd..00000000000 --- a/lib/gitlab_danger.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -class GitlabDanger - LOCAL_RULES ||= %w[ - changes_size - documentation - frozen_string - duplicate_yarn_dependencies - prettier - eslint - karma - database - commit_messages - product_intelligence - utility_css - pajamas - pipeline - ].freeze - - CI_ONLY_RULES ||= %w[ - metadata - changelog - specs - roulette - ce_ee_vue_templates - sidekiq_queues - specialization_labels - ci_templates - ].freeze - - MESSAGE_PREFIX = '==>'.freeze - - attr_reader :gitlab_danger_helper - - def initialize(gitlab_danger_helper) - @gitlab_danger_helper = gitlab_danger_helper - end - - def self.local_warning_message - "#{MESSAGE_PREFIX} Only the following Danger rules can be run locally: #{LOCAL_RULES.join(', ')}" - end - - def self.success_message - "#{MESSAGE_PREFIX} No Danger rule violations!" - end - - def rule_names - ci? ? LOCAL_RULES | CI_ONLY_RULES : LOCAL_RULES - end - - def html_link(str) - self.ci? ? gitlab_danger_helper.html_link(str) : str - end - - def ci? - !gitlab_danger_helper.nil? - end -end diff --git a/lib/object_storage/config.rb b/lib/object_storage/config.rb index f933d4e4866..0e6408b4917 100644 --- a/lib/object_storage/config.rb +++ b/lib/object_storage/config.rb @@ -2,6 +2,8 @@ module ObjectStorage class Config + include Gitlab::Utils::StrongMemoize + AWS_PROVIDER = 'AWS' AZURE_PROVIDER = 'AzureRM' GOOGLE_PROVIDER = 'Google' @@ -66,6 +68,36 @@ module ObjectStorage def provider credentials[:provider].to_s end + + # This method converts fog-aws parameters to an endpoint for the + # Workhorse S3 client. + def s3_endpoint + strong_memoize(:s3_endpoint) do + # We could omit this line and let the following code handle this, but + # this will ensure that working configurations that use `endpoint` + # will continue to work. + next credentials[:endpoint] if credentials[:endpoint].present? + + generate_s3_endpoint_from_credentials + end + end + + def generate_s3_endpoint_from_credentials + # fog-aws has special handling of the host, region, scheme, etc: + # https://github.com/fog/fog-aws/blob/c7a11ba377a76d147861d0e921eb1e245bc11b6c/lib/fog/aws/storage.rb#L440-L449 + # Rather than reimplement this, we derive it from a sample GET URL. + url = fog_connection.get_object_url(bucket, "tmp", nil) + uri = ::Addressable::URI.parse(url) + + return unless uri&.scheme && uri&.host + + endpoint = "#{uri.scheme}://#{uri.host}" + endpoint += ":#{uri.port}" if uri.port + endpoint + rescue ::URI::InvalidComponentError, ::Addressable::URI::InvalidURIError => e + Gitlab::ErrorTracking.track_exception(e) + nil + end # End AWS-specific options # Begin Azure-specific options @@ -91,6 +123,10 @@ module ObjectStorage end end + def fog_connection + @connection ||= ::Fog::Storage.new(credentials) + end + private # This returns a Hash of HTTP encryption headers to send along to S3. diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index 7f1c30e574d..9fb4b571e06 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -80,7 +80,7 @@ module ObjectStorage S3Config: { Bucket: bucket_name, Region: credentials[:region], - Endpoint: credentials[:endpoint], + Endpoint: config.s3_endpoint, PathStyle: config.use_path_style?, UseIamProfile: config.use_iam_profile?, ServerSideEncryption: config.server_side_encryption, @@ -229,7 +229,7 @@ module ObjectStorage end def connection - @connection ||= ::Fog::Storage.new(credentials) + config.fog_connection end end end diff --git a/lib/peek/views/external_http.rb b/lib/peek/views/external_http.rb new file mode 100644 index 00000000000..b925e3db7b6 --- /dev/null +++ b/lib/peek/views/external_http.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Peek + module Views + class ExternalHttp < DetailedView + DEFAULT_THRESHOLDS = { + calls: 10, + duration: 1000, + individual_call: 100 + }.freeze + + THRESHOLDS = { + production: { + calls: 10, + duration: 1000, + individual_call: 100 + } + }.freeze + + def key + 'external-http' + end + + def results + super.merge(calls: calls) + end + + def self.thresholds + @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS) + end + + def format_call_details(call) + full_path = generate_path(call) + super.merge( + label: "#{call[:method]} #{full_path}", + code: code(call), + proxy: proxy(call), + error: error(call) + ) + end + + private + + def duration + ::Gitlab::Metrics::Subscribers::ExternalHttp.duration * 1000 + end + + def calls + ::Gitlab::Metrics::Subscribers::ExternalHttp.request_count + end + + def call_details + ::Gitlab::Metrics::Subscribers::ExternalHttp.detail_store + end + + def proxy(call) + if call[:proxy_host].present? + "Proxied via #{call[:proxy_host]}:#{call[:proxy_port]}" + else + nil + end + end + + def code(call) + if call[:code].present? + "Response status: #{call[:code]}" + else + nil + end + end + + def error(call) + if call[:exception_object].present? + "Exception: #{call[:exception_object]}" + else + nil + end + end + + def generate_path(call) + uri = URI("") + uri.scheme = call[:scheme] + # The host can be a domain, IPv4 or IPv6. + # Ruby handle IPv6 for us at + # https://github.com/ruby/ruby/blob/v2_6_0/lib/uri/generic.rb#L662 + uri.hostname = call[:host] + uri.port = call[:port] + uri.path = call[:path] + uri.query = generate_query(call[:query]) + + uri.to_s + rescue StandardError + 'unknown' + end + + def generate_query(query_string) + query_string.is_a?(Hash) ? query_string.to_query : query_string.to_s + end + end + end +end diff --git a/lib/release_highlights/validator/entry.rb b/lib/release_highlights/validator/entry.rb index 0dbe0cdf882..3c83ca21123 100644 --- a/lib/release_highlights/validator/entry.rb +++ b/lib/release_highlights/validator/entry.rb @@ -5,7 +5,7 @@ module ReleaseHighlights include ActiveModel::Validations include ActiveModel::Validations::Callbacks - PACKAGES = %w(Core Starter Premium Ultimate).freeze + PACKAGES = %w(Free Premium Ultimate).freeze attr_reader :entry diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index e7e0d4e471f..8f18d6433e0 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -8,9 +8,9 @@ module Rouge # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance. # # [+tag+] The tag (language) of the lexer used to generate the formatted tokens - def initialize(tag: nil) + def initialize(options = {}) @line_number = 1 - @tag = tag + @tag = options[:tag] end def stream(tokens) diff --git a/lib/security/ci_configuration/sast_build_actions.rb b/lib/security/ci_configuration/sast_build_actions.rb new file mode 100644 index 00000000000..b2d684bc1e1 --- /dev/null +++ b/lib/security/ci_configuration/sast_build_actions.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +module Security + module CiConfiguration + class SastBuildActions + SAST_DEFAULT_ANALYZERS = 'bandit, brakeman, eslint, flawfinder, gosec, kubesec, nodejs-scan, phpcs-security-audit, pmd-apex, security-code-scan, sobelow, spotbugs' + + def initialize(auto_devops_enabled, params, existing_gitlab_ci_content) + @auto_devops_enabled = auto_devops_enabled + @variables = variables(params) + @existing_gitlab_ci_content = existing_gitlab_ci_content || {} + @default_sast_values = default_sast_values(params) + @default_values_overwritten = false + end + + def generate + action = @existing_gitlab_ci_content.present? ? 'update' : 'create' + + update_existing_content! + + [{ action: action, file_path: '.gitlab-ci.yml', content: prepare_existing_content, default_values_overwritten: @default_values_overwritten }] + end + + private + + def variables(params) + # This early return is necessary for supporting REST API. + # Will be removed during the implementation of + # https://gitlab.com/gitlab-org/gitlab/-/issues/246737 + return params unless params['global'].present? + + collect_values(params, 'value') + end + + def default_sast_values(params) + collect_values(params, 'defaultValue') + end + + def collect_values(config, key) + global_variables = config['global']&.to_h { |k| [k['field'], k[key]] } || {} + pipeline_variables = config['pipeline']&.to_h { |k| [k['field'], k[key]] } || {} + + analyzer_variables = collect_analyzer_values(config, key) + + global_variables.merge!(pipeline_variables).merge!(analyzer_variables) + end + + def collect_analyzer_values(config, key) + analyzer_variables = analyzer_variables_for(config, key) + analyzer_variables['SAST_EXCLUDED_ANALYZERS'] = if key == 'value' + config['analyzers'] + &.reject {|a| a['enabled'] } + &.collect {|a| a['name'] } + &.sort + &.join(', ') + else + '' + end + + analyzer_variables + end + + def analyzer_variables_for(config, key) + config['analyzers'] + &.select {|a| a['enabled'] && a['variables'] } + &.flat_map {|a| a['variables'] } + &.collect {|v| [v['field'], v[key]] }.to_h + end + + def update_existing_content! + @existing_gitlab_ci_content['stages'] = set_stages + @existing_gitlab_ci_content['variables'] = set_variables(global_variables, @existing_gitlab_ci_content) + @existing_gitlab_ci_content['sast'] = set_sast_block + @existing_gitlab_ci_content['include'] = set_includes + + @existing_gitlab_ci_content.select! { |k, v| v.present? } + @existing_gitlab_ci_content['sast'].select! { |k, v| v.present? } + end + + def set_includes + includes = @existing_gitlab_ci_content['include'] || [] + includes = includes.is_a?(Array) ? includes : [includes] + includes << { 'template' => template } + includes.uniq + end + + def set_stages + existing_stages = @existing_gitlab_ci_content['stages'] || [] + base_stages = @auto_devops_enabled ? auto_devops_stages : ['test'] + (existing_stages + base_stages + [sast_stage]).uniq + end + + def auto_devops_stages + auto_devops_template = YAML.safe_load( Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content ) + auto_devops_template['stages'] + end + + def sast_stage + @variables['stage'].presence ? @variables['stage'] : 'test' + end + + def set_variables(variables, hash_to_update = {}) + hash_to_update['variables'] ||= {} + + variables.each do |key| + if @variables[key].present? && @variables[key].to_s != @default_sast_values[key].to_s + hash_to_update['variables'][key] = @variables[key] + @default_values_overwritten = true + else + hash_to_update['variables'].delete(key) + end + end + + hash_to_update['variables'] + end + + def set_sast_block + sast_content = @existing_gitlab_ci_content['sast'] || {} + sast_content['variables'] = set_variables(sast_variables) + sast_content['stage'] = sast_stage + sast_content.select { |k, v| v.present? } + end + + def prepare_existing_content + content = @existing_gitlab_ci_content.to_yaml + content = remove_document_delimeter(content) + + content.prepend(sast_comment) + end + + def remove_document_delimeter(content) + content.gsub(/^---\n/, '') + end + + def sast_comment + <<~YAML + # You can override the included template(s) by including variable overrides + # See https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings + # Note that environment variables can be set in several places + # See https://docs.gitlab.com/ee/ci/variables/#priority-of-environment-variables + YAML + end + + def template + return 'Auto-DevOps.gitlab-ci.yml' if @auto_devops_enabled + + 'Security/SAST.gitlab-ci.yml' + end + + def global_variables + %w( + SECURE_ANALYZERS_PREFIX + ) + end + + def sast_variables + %w( + SAST_ANALYZER_IMAGE_TAG + SAST_EXCLUDED_PATHS + SEARCH_MAX_DEPTH + SAST_EXCLUDED_ANALYZERS + SAST_BRAKEMAN_LEVEL + SAST_BANDIT_EXCLUDED_PATHS + SAST_FLAWFINDER_LEVEL + SAST_GOSEC_LEVEL + ) + end + end + end +end diff --git a/lib/tasks/benchmark.rake b/lib/tasks/benchmark.rake new file mode 100644 index 00000000000..6deafb2c351 --- /dev/null +++ b/lib/tasks/benchmark.rake @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :benchmark do + desc 'Benchmark | Banzai pipeline/filters' + RSpec::Core::RakeTask.new(:banzai) do |t| + t.pattern = 'spec/benchmarks/banzai_benchmark.rb' + ENV['BENCHMARK'] = '1' + end +end diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 2301ec9b228..44d2071751f 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 6af91d473a6..4d698e56444 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :cache do namespace :clear do REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 diff --git a/lib/tasks/ci/cleanup.rake b/lib/tasks/ci/cleanup.rake index 978a42be638..31c3928e797 100644 --- a/lib/tasks/ci/cleanup.rake +++ b/lib/tasks/ci/cleanup.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :ci do namespace :cleanup do desc "GitLab | CI | Clean running builds" diff --git a/lib/tasks/cleanup.rake b/lib/tasks/cleanup.rake index 8574f26dbdc..c13da77217e 100644 --- a/lib/tasks/cleanup.rake +++ b/lib/tasks/cleanup.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :cleanup do desc "GitLab | Cleanup | Delete moved repositories" diff --git a/lib/tasks/config_lint.rake b/lib/tasks/config_lint.rake index ddbcf1e1eb8..9bbefaa2d3a 100644 --- a/lib/tasks/config_lint.rake +++ b/lib/tasks/config_lint.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ConfigLint def self.run(files) failures = files.reject do |file| diff --git a/lib/tasks/db_obsolete_ignored_columns.rake b/lib/tasks/db_obsolete_ignored_columns.rake index 00f60231f4f..cf35a355ce9 100644 --- a/lib/tasks/db_obsolete_ignored_columns.rake +++ b/lib/tasks/db_obsolete_ignored_columns.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + desc 'Show a list of obsolete `ignored_columns`' task 'db:obsolete_ignored_columns' => :environment do list = Gitlab::Database::ObsoleteIgnoredColumns.new.execute diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index b3ba2434855..cb01f229cd3 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + task dev: ["dev:setup"] namespace :dev do diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake index ce97ed50fc7..3428e3f8f53 100644 --- a/lib/tasks/downtime_check.rake +++ b/lib/tasks/downtime_check.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + desc 'Checks if migrations in a branch require downtime' task downtime_check: :environment do repo = if defined?(Gitlab::License) diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake index 51f5d768102..ad63de66c81 100644 --- a/lib/tasks/eslint.rake +++ b/lib/tasks/eslint.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + unless Rails.env.production? desc "GitLab | Run ESLint" task eslint: ['yarn:check'] do diff --git a/lib/tasks/file_hooks.rake b/lib/tasks/file_hooks.rake index f767d63fe0d..a892d36b48e 100644 --- a/lib/tasks/file_hooks.rake +++ b/lib/tasks/file_hooks.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :file_hooks do desc 'Validate existing file hooks' task validate: :environment do diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake index 6e90229830d..b2d2c4e3f2b 100644 --- a/lib/tasks/frontend.rake +++ b/lib/tasks/frontend.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + unless Rails.env.production? namespace :frontend do desc 'GitLab | Frontend | Generate fixtures for JavaScript tests' @@ -5,7 +7,7 @@ unless Rails.env.production? directories = %w[spec] directories << 'ee/spec' if Gitlab.ee? directory_glob = "{#{directories.join(',')}}" - args.with_defaults(pattern: "#{directory_glob}/frontend/fixtures/*.rb") + args.with_defaults(pattern: "#{directory_glob}/frontend/fixtures/**/*.rb") ENV['NO_KNAPSACK'] = 'true' t.pattern = args[:pattern] t.rspec_opts = '--format documentation' diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 85393bba9a6..a4600a0ed16 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gemojione do desc 'Generates Emoji SHA256 digests' diff --git a/lib/tasks/gitlab/artifacts/check.rake b/lib/tasks/gitlab/artifacts/check.rake index a105261ed51..057bb17dbda 100644 --- a/lib/tasks/gitlab/artifacts/check.rake +++ b/lib/tasks/gitlab/artifacts/check.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :artifacts do desc 'GitLab | Artifacts | Check integrity of uploaded job artifacts' diff --git a/lib/tasks/gitlab/artifacts/migrate.rake b/lib/tasks/gitlab/artifacts/migrate.rake index 871fdfb4fde..94867e1a16a 100644 --- a/lib/tasks/gitlab/artifacts/migrate.rake +++ b/lib/tasks/gitlab/artifacts/migrate.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'logger' require 'resolv-replace' diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index de2dfca8c1b..c53ef8382b8 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'active_record/fixtures' namespace :gitlab do diff --git a/lib/tasks/gitlab/bulk_add_permission.rake b/lib/tasks/gitlab/bulk_add_permission.rake index 56cbbae1f67..df0c6a260a2 100644 --- a/lib/tasks/gitlab/bulk_add_permission.rake +++ b/lib/tasks/gitlab/bulk_add_permission.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :import do desc "GitLab | Import | Add all users to all projects (admin users are added as maintainers)" diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 9e60a585330..d4e38100609 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do desc 'GitLab | Check the configuration of GitLab and its environment' task check: :gitlab_environment do diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index a56a0435673..6c3a7a77e0e 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -56,7 +56,7 @@ namespace :gitlab do task orphan_job_artifact_files: :gitlab_environment do warn_user_is_not_gitlab - cleaner = Gitlab::Cleanup::OrphanJobArtifactFiles.new(limit: limit, dry_run: dry_run?, niceness: niceness, logger: logger) + cleaner = Gitlab::Cleanup::OrphanJobArtifactFiles.new(dry_run: dry_run?, niceness: niceness, logger: logger) cleaner.run! if dry_run? @@ -78,8 +78,7 @@ namespace :gitlab do cleaner = Gitlab::Cleanup::OrphanLfsFileReferences.new( project, dry_run: dry_run?, - logger: logger, - limit: limit + logger: logger ) cleaner.run! @@ -162,10 +161,6 @@ namespace :gitlab do ENV['DEBUG'].present? end - def limit - ENV['LIMIT']&.to_i - end - def niceness ENV['NICENESS'].presence end diff --git a/lib/tasks/gitlab/container_registry.rake b/lib/tasks/gitlab/container_registry.rake index cd18c873a5a..086f99971bc 100644 --- a/lib/tasks/gitlab/container_registry.rake +++ b/lib/tasks/gitlab/container_registry.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :container_registry do desc "GitLab | Container Registry | Configure" diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 901e349ea31..541a4fc62af 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :db do desc 'GitLab | DB | Manually insert schema migration version' @@ -229,5 +231,37 @@ namespace :gitlab do puts "Found user created projects. Database active" exit 0 end + + desc 'Run migrations with instrumentation' + task :migration_testing, [:result_file] => :environment do |_, args| + result_file = args[:result_file] || raise("Please specify result_file argument") + raise "File exists already, won't overwrite: #{result_file}" if File.exist?(result_file) + + verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, true + + ctx = ActiveRecord::Base.connection.migration_context + existing_versions = ctx.get_all_versions.to_set + + pending_migrations = ctx.migrations.reject do |migration| + existing_versions.include?(migration.version) + end + + instrumentation = Gitlab::Database::Migrations::Instrumentation.new + + pending_migrations.each do |migration| + instrumentation.observe(migration.version) do + ActiveRecord::Migrator.new(:up, ctx.migrations, ctx.schema_migration, migration.version).run + end + end + ensure + if instrumentation + File.open(result_file, 'wb+') do |io| + io << instrumentation.observations.to_json + end + end + + ActiveRecord::Base.clear_cache! + ActiveRecord::Migration.verbose = verbose_was + end end end diff --git a/lib/tasks/gitlab/doctor/secrets.rake b/lib/tasks/gitlab/doctor/secrets.rake index 3fdef9dfc80..6e3f474312c 100644 --- a/lib/tasks/gitlab/doctor/secrets.rake +++ b/lib/tasks/gitlab/doctor/secrets.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :doctor do desc "GitLab | Check if the database encrypted values can be decrypted using current secrets" diff --git a/lib/tasks/gitlab/exclusive_lease.rake b/lib/tasks/gitlab/exclusive_lease.rake index 63b06d5251a..bee9782f373 100644 --- a/lib/tasks/gitlab/exclusive_lease.rake +++ b/lib/tasks/gitlab/exclusive_lease.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :exclusive_lease do desc 'GitLab | Exclusive Lease | Clear existing exclusive leases for specified scope (default: *)' diff --git a/lib/tasks/gitlab/external_diffs.rake b/lib/tasks/gitlab/external_diffs.rake index 08f25914007..7a5fa07aed1 100644 --- a/lib/tasks/gitlab/external_diffs.rake +++ b/lib/tasks/gitlab/external_diffs.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :external_diffs do desc "Override external diffs in file storage to be in object storage instead. This does not change the actual location of the data" diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake index 2309aa5d214..e44328e0de1 100644 --- a/lib/tasks/gitlab/features.rake +++ b/lib/tasks/gitlab/features.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :features do desc 'GitLab | Features | Enable direct Git access via Rugged for NFS' diff --git a/lib/tasks/gitlab/generate_sample_prometheus_data.rake b/lib/tasks/gitlab/generate_sample_prometheus_data.rake index 250eaaa5568..4cd75af9d00 100644 --- a/lib/tasks/gitlab/generate_sample_prometheus_data.rake +++ b/lib/tasks/gitlab/generate_sample_prometheus_data.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do desc "GitLab | Generate Sample Prometheus Data" task :generate_sample_prometheus_data, [:environment_id] => :gitlab_environment do |_, args| diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index abb15f29328..9c5549b4a54 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :git do desc 'GitLab | Git | Check all repos integrity' diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 49d2d3d62a2..9e474b00ba7 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :gitaly do desc 'GitLab | Gitaly | Install or upgrade gitaly' diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index f708114c226..e4eb4604138 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -33,7 +33,43 @@ namespace :gitlab do ) namespace :graphql do - desc 'Gitlab | GraphQL | Validate queries' + desc 'GitLab | GraphQL | Analyze queries' + task analyze: [:environment, :enable_feature_flags] do |t, args| + queries = if args.to_a.present? + args.to_a.flat_map { |path| Gitlab::Graphql::Queries.find(path) } + else + Gitlab::Graphql::Queries.all + end + + queries.each do |defn| + $stdout.puts defn.file + summary, errs = defn.validate(GitlabSchema) + + if summary == :client_query + $stdout.puts " - client query" + elsif errs.present? + $stdout.puts " - invalid query".color(:red) + else + complexity = defn.complexity(GitlabSchema) + color = case complexity + when 0..GitlabSchema::DEFAULT_MAX_COMPLEXITY + :green + when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_COMPLEXITY + :yellow + when GitlabSchema::AUTHENTICATED_COMPLEXITY..GitlabSchema::ADMIN_COMPLEXITY + :orange + else + :red + end + + $stdout.puts " - complexity: #{complexity}".color(color) + end + + $stdout.puts "" + end + end + + desc 'GitLab | GraphQL | Validate queries' task validate: [:environment, :enable_feature_flags] do |t, args| queries = if args.to_a.present? args.to_a.flat_map { |path| Gitlab::Graphql::Queries.find(path) } @@ -48,10 +84,10 @@ namespace :gitlab do when :client_query warn("SKIP #{defn.file}: client query") else - warn("OK #{defn.file}") if errs.empty? + warn("#{'OK'.color(:green)} #{defn.file}") if errs.empty? errs.each do |err| warn(<<~MSG) - ERROR #{defn.file}: #{err.message} (at #{err.path.join('.')}) + #{'ERROR'.color(:red)} #{defn.file}: #{err.message} (at #{err.path.join('.')}) MSG end end diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index 14d1125a03d..b61b1833c5a 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Prevent StateMachine warnings from outputting during a cron task StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 701d40b7929..bf0ba40fb31 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :import do # How to use: diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index adf696350d7..72598cdf863 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :import_export do desc 'GitLab | Import/Export | Show Import/Export version' diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index d85c8fc7949..2826002bdc2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :env do desc 'GitLab | Env | Show information about GitLab and its environment' diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake index fe7920c621f..3b2834c0008 100644 --- a/lib/tasks/gitlab/ldap.rake +++ b/lib/tasks/gitlab/ldap.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :ldap do desc 'GitLab | LDAP | Rename provider' diff --git a/lib/tasks/gitlab/lfs/check.rake b/lib/tasks/gitlab/lfs/check.rake index 869463d4e5d..582b87337ae 100644 --- a/lib/tasks/gitlab/lfs/check.rake +++ b/lib/tasks/gitlab/lfs/check.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :lfs do desc 'GitLab | LFS | Check integrity of uploaded LFS objects' diff --git a/lib/tasks/gitlab/lfs/migrate.rake b/lib/tasks/gitlab/lfs/migrate.rake index 3d4c847a0f0..05249a126bc 100644 --- a/lib/tasks/gitlab/lfs/migrate.rake +++ b/lib/tasks/gitlab/lfs/migrate.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'logger' desc "GitLab | LFS | Migrate LFS objects to remote storage" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index b854c34a8e5..56ec94d2aef 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do task list_repos: :environment do scope = Project diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index cfe97984dda..4bf4ce430f1 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'logger' desc "GitLab | Packages | Events | Generate hll counter events file for packages" diff --git a/lib/tasks/gitlab/packages/migrate.rake b/lib/tasks/gitlab/packages/migrate.rake index cd6dcf78da3..20a8c51db66 100644 --- a/lib/tasks/gitlab/packages/migrate.rake +++ b/lib/tasks/gitlab/packages/migrate.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'logger' desc "GitLab | Packages | Migrate packages files to remote storage" diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index e15cbb4e32e..b598dab901d 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -1,35 +1,55 @@ +# frozen_string_literal: true + require 'logger' namespace :gitlab do namespace :pages do desc "GitLab | Pages | Migrate legacy storage to zip format" task migrate_legacy_storage: :gitlab_environment do - logger = Logger.new(STDOUT) logger.info('Starting to migrate legacy pages storage to zip deployments') - processed_projects = 0 - - ProjectPagesMetadatum.only_on_legacy_storage.each_batch(of: 10) do |batch| - batch.preload(project: [:namespace, :route, pages_metadatum: :pages_deployment]).each do |metadatum| - project = metadatum.project - - result = nil - time = Benchmark.realtime do - result = ::Pages::MigrateLegacyStorageToDeploymentService.new(project).execute - end - processed_projects += 1 - - if result[:status] == :success - logger.info("project_id: #{project.id} #{project.pages_path} has been migrated in #{time} seconds") - else - logger.error("project_id: #{project.id} #{project.pages_path} failed to be migrated in #{time} seconds: #{result[:message]}") - end - rescue => e - logger.error("#{e.message} project_id: #{project&.id}") - Gitlab::ErrorTracking.track_exception(e, project_id: project&.id) - end - - logger.info("#{processed_projects} pages projects are processed") + + result = ::Pages::MigrateFromLegacyStorageService.new(logger, + migration_threads: migration_threads, + batch_size: batch_size, + ignore_invalid_entries: ignore_invalid_entries).execute + + logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.") + logger.info("- The #{result[:migrated]} projects migrated successfully") + logger.info("- The #{result[:errored]} projects failed to be migrated") + end + + desc "GitLab | Pages | DANGER: Removes data which was migrated from legacy storage on zip storage. Can be used if some bugs in migration are discovered and migration needs to be restarted from scratch." + task clean_migrated_zip_storage: :gitlab_environment do + destroyed_deployments = 0 + + logger.info("Starting to delete migrated pages deployments") + + ::PagesDeployment.migrated_from_legacy_storage.each_batch(of: batch_size) do |batch| + destroyed_deployments += batch.count + + # we need to destroy associated files, so can't use delete_all + batch.destroy_all # rubocop: disable Cop/DestroyAll + + logger.info("#{destroyed_deployments} deployments were deleted") end end + + def logger + @logger ||= Logger.new(STDOUT) + end + + def migration_threads + ENV.fetch('PAGES_MIGRATION_THREADS', '3').to_i + end + + def batch_size + ENV.fetch('PAGES_MIGRATION_BATCH_SIZE', '10').to_i + end + + def ignore_invalid_entries + Gitlab::Utils.to_boolean( + ENV.fetch('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'false') + ) + end end end diff --git a/lib/tasks/gitlab/password.rake b/lib/tasks/gitlab/password.rake new file mode 100644 index 00000000000..02c28578a2a --- /dev/null +++ b/lib/tasks/gitlab/password.rake @@ -0,0 +1,31 @@ +# frozen_string_literal: true +namespace :gitlab do + namespace :password do + desc "GitLab | Password | Reset a user's password" + task :reset, [:username] => :environment do |_, args| + username = args[:username] || Gitlab::TaskHelpers.prompt('Enter username: ') + abort('Username can not be empty.') if username.blank? + + user = User.find_by(username: username) + abort("Unable to find user with username #{username}.") unless user + + password = Gitlab::TaskHelpers.prompt_for_password + password_confirm = Gitlab::TaskHelpers.prompt_for_password('Confirm password: ') + + user.password = password + user.password_confirmation = password_confirm + user.send_only_admin_changed_your_password_notification! + + unless user.save + message = <<~EOF + Unable to change password of the user with username #{username}. + #{user.errors.full_messages.to_sentence} + EOF + + abort(message) + end + + puts "Password successfully updated for user with username #{username}." + end + end +end diff --git a/lib/tasks/gitlab/praefect.rake b/lib/tasks/gitlab/praefect.rake index a9b126ae379..346df3e0c75 100644 --- a/lib/tasks/gitlab/praefect.rake +++ b/lib/tasks/gitlab/praefect.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :praefect do def int?(string) diff --git a/lib/tasks/gitlab/seed.rake b/lib/tasks/gitlab/seed.rake index d758280ba69..36761165af5 100644 --- a/lib/tasks/gitlab/seed.rake +++ b/lib/tasks/gitlab/seed.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :seed do desc "GitLab | Seed | Seeds issues" diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 50774de77c9..31bd80e78d4 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do desc "GitLab | Setup production application" task setup: :gitlab_environment do diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index edbaec85bd9..b3abc48f8e0 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :shell do desc "GitLab | Shell | Install or upgrade gitlab-shell" diff --git a/lib/tasks/gitlab/snippets.rake b/lib/tasks/gitlab/snippets.rake index ed2e88692d5..b55f82480e1 100644 --- a/lib/tasks/gitlab/snippets.rake +++ b/lib/tasks/gitlab/snippets.rake @@ -13,7 +13,7 @@ namespace :gitlab do raise "Please supply the list of ids through the SNIPPET_IDS env var" end - raise "Invalid limit value" if limit == 0 + raise "Invalid limit value" if snippet_task_limit == 0 if migration_running? raise "There are already snippet migrations running. Please wait until they are finished." @@ -41,8 +41,8 @@ namespace :gitlab do end end - if ids.size > limit - raise "The number of ids provided is higher than #{limit}. You can update this limit by using the env var `LIMIT`" + if ids.size > snippet_task_limit + raise "The number of ids provided is higher than #{snippet_task_limit}. You can update this limit by using the env var `LIMIT`" end ids @@ -68,14 +68,14 @@ namespace :gitlab do # bundle exec rake gitlab:snippets:list_non_migrated LIMIT=50 desc 'GitLab | Show non migrated snippets' task list_non_migrated: :environment do - raise "Invalid limit value" if limit == 0 + raise "Invalid limit value" if snippet_task_limit == 0 non_migrated_count = non_migrated_snippets.count if non_migrated_count == 0 puts "All snippets have been successfully migrated" else puts "There are #{non_migrated_count} snippets that haven't been migrated. Showing a batch of ids of those snippets:\n" - puts non_migrated_snippets.limit(limit).pluck(:id).join(',') + puts non_migrated_snippets.limit(snippet_task_limit).pluck(:id).join(',') end end @@ -84,7 +84,7 @@ namespace :gitlab do end # There are problems with the specs if we memoize this value - def limit + def snippet_task_limit ENV['LIMIT'] ? ENV['LIMIT'].to_i : DEFAULT_LIMIT end end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index f7819fd974b..ede6b6af80b 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :storage do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' diff --git a/lib/tasks/gitlab/tcp_check.rake b/lib/tasks/gitlab/tcp_check.rake index 1400f57d6b9..4790d86832d 100644 --- a/lib/tasks/gitlab/tcp_check.rake +++ b/lib/tasks/gitlab/tcp_check.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do desc "GitLab | Check TCP connectivity to a specific host and port" task :tcp_check, [:host, :port] => :environment do |_t, args| diff --git a/lib/tasks/gitlab/terraform/migrate.rake b/lib/tasks/gitlab/terraform/migrate.rake new file mode 100644 index 00000000000..a9c16049240 --- /dev/null +++ b/lib/tasks/gitlab/terraform/migrate.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'logger' + +desc "GitLab | Terraform | Migrate Terraform states to remote storage" +namespace :gitlab do + namespace :terraform_states do + task migrate: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting transfer of Terraform states to object storage') + + begin + Gitlab::Terraform::StateMigrationHelper.migrate_to_remote_storage do |state_version| + message = "Transferred Terraform state version ID #{state_version.id} (#{state_version.terraform_state.name}/#{state_version.version}) to object storage" + + logger.info(message) + end + rescue => e + logger.error("Failed to migrate: #{e.message}") + end + end + end +end diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake index 2222807fe13..a83ba69bc75 100644 --- a/lib/tasks/gitlab/test.rake +++ b/lib/tasks/gitlab/test.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do desc "GitLab | Run all tests" task :test do diff --git a/lib/tasks/gitlab/two_factor.rake b/lib/tasks/gitlab/two_factor.rake index 6a9e87e1541..9d10976fbce 100644 --- a/lib/tasks/gitlab/two_factor.rake +++ b/lib/tasks/gitlab/two_factor.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :two_factor do desc "GitLab | 2FA | Disable Two-factor authentication (2FA) for all users" diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index fdcd34320b1..e3a4e7f50b8 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do desc "GitLab | Update templates" task :update_templates do diff --git a/lib/tasks/gitlab/uploads/check.rake b/lib/tasks/gitlab/uploads/check.rake index 2be2ec7f9c9..8ccc84d15f4 100644 --- a/lib/tasks/gitlab/uploads/check.rake +++ b/lib/tasks/gitlab/uploads/check.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :uploads do desc 'GitLab | Uploads | Check integrity of uploaded files' diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 879b07da1df..6052ff90341 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :uploads do namespace :migrate do diff --git a/lib/tasks/gitlab/uploads/sanitize.rake b/lib/tasks/gitlab/uploads/sanitize.rake index 4f23a0a5d82..eec423cbb8b 100644 --- a/lib/tasks/gitlab/uploads/sanitize.rake +++ b/lib/tasks/gitlab/uploads/sanitize.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :uploads do namespace :sanitize do diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index d6f5661d5eb..95072444fcf 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :usage_data do desc 'GitLab | UsageData | Generate raw SQLs for usage ping in YAML' @@ -21,5 +23,11 @@ namespace :gitlab do puts Gitlab::Json.pretty_generate(result.attributes) end + + desc 'GitLab | UsageData | Generate metrics dictionary' + task generate_metrics_dictionary: :environment do + items = Gitlab::Usage::MetricDefinition.definitions + Gitlab::Usage::Docs::Renderer.new(items).write + end end end diff --git a/lib/tasks/gitlab/user_management.rake b/lib/tasks/gitlab/user_management.rake index f47e549e795..29f2360f64a 100644 --- a/lib/tasks/gitlab/user_management.rake +++ b/lib/tasks/gitlab/user_management.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :user_management do desc "GitLab | User management | Update all users of a group with personal project limit to 0 and can_create_group to false" diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index b242329d720..091743485c9 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :web_hook do desc "GitLab | Webhook | Adds a webhook to the projects" diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index 2d72a01f66f..0d9f8efa64f 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :workhorse do desc "GitLab | Workhorse | Install or upgrade gitlab-workhorse" diff --git a/lib/tasks/gitlab/x509/update.rake b/lib/tasks/gitlab/x509/update.rake index eaba9196acf..de878a3d093 100644 --- a/lib/tasks/gitlab/x509/update.rake +++ b/lib/tasks/gitlab/x509/update.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'logger' desc "GitLab | X509 | Update signatures when certificate store has changed" diff --git a/lib/tasks/gitlab_danger.rake b/lib/tasks/gitlab_danger.rake index e75539f048c..5df4a8ce4f1 100644 --- a/lib/tasks/gitlab_danger.rake +++ b/lib/tasks/gitlab_danger.rake @@ -1,6 +1,8 @@ +# frozen_string_literal: true + desc 'Run local Danger rules' task :danger_local do - require 'gitlab_danger' + require_relative '../../tooling/gitlab_danger' require 'gitlab/popen' puts("#{GitlabDanger.local_warning_message}\n") diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake index ea2698da606..c72403a375a 100644 --- a/lib/tasks/grape.rake +++ b/lib/tasks/grape.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :grape do desc 'Print compiled grape routes' task routes: :environment do diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index 305e15d69d5..270793359e1 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + unless Rails.env.production? require 'haml_lint/rake_task' require Rails.root.join('haml_lint/inline_javascript') diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 500891df43d..633beb132d8 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'benchmark' require 'rainbow/ext/string' diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 36590010406..fa3f8805159 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + unless Rails.env.production? namespace :karma do # alias exists for legacy reasons diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 7a4d09bb6d4..5d60bc41f21 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + unless Rails.env.production? namespace :lint do task :static_verification_env do diff --git a/lib/tasks/migrate/composite_primary_keys.rake b/lib/tasks/migrate/composite_primary_keys.rake index 732dedf4d4f..68f7c4d6c4a 100644 --- a/lib/tasks/migrate/composite_primary_keys.rake +++ b/lib/tasks/migrate/composite_primary_keys.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + namespace :gitlab do namespace :db do desc 'GitLab | DB | Adds primary keys to tables that only have composite unique keys' diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index cb7c496c31c..e0666a87656 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + desc "GitLab | Build internal ids for issues and merge requests" task migrate_iids: :environment do puts 'Issues'.color(:yellow) diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 4c8f13b63a4..27dd9e842ff 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + desc 'GitLab | Generate PostgreSQL Password Hash' task :postgresql_md5_hash do require 'digest' diff --git a/lib/tasks/pngquant.rake b/lib/tasks/pngquant.rake index 63bc1c7c16e..45c0288cadf 100644 --- a/lib/tasks/pngquant.rake +++ b/lib/tasks/pngquant.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + return if Rails.env.production? require 'png_quantizator' diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index 78ffccc9d06..f5d16835347 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + unless Rails.env.production? require 'rubocop/rake_task' diff --git a/lib/tasks/scss-lint.rake b/lib/tasks/scss-lint.rake index 250fd8699e4..8a4809f80fd 100644 --- a/lib/tasks/scss-lint.rake +++ b/lib/tasks/scss-lint.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + unless Rails.env.production? require 'scss_lint/rake_task' diff --git a/lib/tasks/setup.rake b/lib/tasks/setup.rake index 4c79ffbfa6b..bae22f9a590 100644 --- a/lib/tasks/setup.rake +++ b/lib/tasks/setup.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + desc "GitLab | Setup gitlab db" task :setup do Rake::Task["gitlab:setup"].invoke diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index b52af81fc16..b24817468c6 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + Rake::Task["test"].clear desc "GitLab | Run all tests" diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 46635cd7c8f..b0ff2cce482 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_relative '../../app/models/concerns/token_authenticatable.rb' require_relative '../../app/models/concerns/token_authenticatable_strategies/base.rb' require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure.rb' |