diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-03-16 21:18:33 +0300 |
commit | f64a639bcfa1fc2bc89ca7db268f594306edfd7c (patch) | |
tree | a2c3c2ebcc3b45e596949db485d6ed18ffaacfa1 /lib | |
parent | bfbc3e0d6583ea1a91f627528bedc3d65ba4b10f (diff) |
Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40
Diffstat (limited to 'lib')
337 files changed, 5555 insertions, 1914 deletions
diff --git a/lib/api/admin/plan_limits.rb b/lib/api/admin/plan_limits.rb new file mode 100644 index 00000000000..92f7d3dce0d --- /dev/null +++ b/lib/api/admin/plan_limits.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module API + module Admin + class PlanLimits < ::API::Base + before { authenticated_as_admin! } + + feature_category :not_owned + + helpers do + def current_plan(name) + plan = ::Admin::PlansFinder.new({ name: name }).execute + + not_found!('Plan') unless plan + plan + end + end + + desc 'Get current plan limits' do + success Entities::PlanLimit + end + params do + optional :plan_name, type: String, values: Plan.all_plans, default: Plan::DEFAULT, desc: 'Name of the plan' + end + get "application/plan_limits" do + params = declared_params(include_missing: false) + plan = current_plan(params.delete(:plan_name)) + + present plan.actual_limits, with: Entities::PlanLimit + end + + desc 'Modify plan limits' do + success Entities::PlanLimit + end + params do + requires :plan_name, type: String, values: Plan.all_plans, desc: 'Name of the plan' + + optional :conan_max_file_size, type: Integer, desc: 'Maximum Conan package file size in bytes' + optional :generic_packages_max_file_size, type: Integer, desc: 'Maximum generic package file size in bytes' + optional :maven_max_file_size, type: Integer, desc: 'Maximum Maven package file size in bytes' + optional :npm_max_file_size, type: Integer, desc: 'Maximum NPM package file size in bytes' + optional :nuget_max_file_size, type: Integer, desc: 'Maximum NuGet package file size in bytes' + optional :pypi_max_file_size, type: Integer, desc: 'Maximum PyPI package file size in bytes' + end + put "application/plan_limits" do + params = declared_params(include_missing: false) + plan = current_plan(params.delete(:plan_name)) + + if plan.actual_limits.update(params) + present plan.actual_limits, with: Entities::PlanLimit + else + render_validation_error!(plan.actual_limits) + end + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index 0598f03c7ab..f83a36068dd 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -58,6 +58,7 @@ module API user: -> { @current_user }, project: -> { @project }, namespace: -> { @group }, + runner: -> { @current_runner || @runner }, caller_id: route.origin, remote_ip: request.ip, feature_category: feature_category @@ -147,7 +148,7 @@ module API # Only overwrite `text/plain+deprecated` if content_types[api_format] == 'text/plain+deprecated' - if Feature.enabled?(:api_always_use_application_json) + if Feature.enabled?(:api_always_use_application_json, default_enabled: :yaml) content_type 'application/json' else content_type 'text/plain' @@ -169,6 +170,7 @@ module API mount ::API::AccessRequests mount ::API::Admin::Ci::Variables mount ::API::Admin::InstanceClusters + mount ::API::Admin::PlanLimits mount ::API::Admin::Sidekiq mount ::API::Appearance mount ::API::Applications diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 5fd4ca3546c..79f4b02f26a 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -30,7 +30,7 @@ module API use :pagination end get '/' do - authorize!(:read_board, user_project) + authorize!(:read_issue_board, user_project) present paginate(board_parent.boards.with_associations), with: Entities::Board end @@ -39,7 +39,7 @@ module API success Entities::Board end get '/:board_id' do - authorize!(:read_board, user_project) + authorize!(:read_issue_board, user_project) present board, with: Entities::Board end @@ -51,7 +51,7 @@ module API requires :name, type: String, desc: 'The board name' end post '/' do - authorize!(:admin_board, board_parent) + authorize!(:admin_issue_board, board_parent) create_board end @@ -64,7 +64,7 @@ module API use :update_params end put '/:board_id' do - authorize!(:admin_board, board_parent) + authorize!(:admin_issue_board, board_parent) update_board end @@ -75,7 +75,7 @@ module API end delete '/:board_id' do - authorize!(:admin_board, board_parent) + authorize!(:admin_issue_board, board_parent) delete_board end @@ -93,7 +93,7 @@ module API use :pagination end get '/lists' do - authorize!(:read_board, user_project) + authorize!(:read_issue_board, user_project) present paginate(board_lists), with: Entities::List end @@ -105,7 +105,7 @@ module API requires :list_id, type: Integer, desc: 'The ID of a list' end get '/lists/:list_id' do - authorize!(:read_board, user_project) + authorize!(:read_issue_board, user_project) present board_lists.find(params[:list_id]), with: Entities::List end @@ -117,7 +117,7 @@ module API use :list_creation_params end post '/lists' do - authorize!(:admin_list, user_project) + authorize!(:admin_issue_board_list, user_project) create_list end @@ -133,7 +133,7 @@ module API put '/lists/:list_id' do list = board_lists.find(params[:list_id]) - authorize!(:admin_list, user_project) + authorize!(:admin_issue_board_list, user_project) move_list(list) end @@ -146,7 +146,7 @@ module API requires :list_id, type: Integer, desc: 'The ID of a board list' end delete "/lists/:list_id" do - authorize!(:admin_list, user_project) + authorize!(:admin_issue_board_list, user_project) list = board_lists.find(params[:list_id]) destroy_list(list) diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 5cfb65e1fbb..80d5e80e21e 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -34,22 +34,22 @@ module API if runner_registration_token_valid? # Create shared runner. Requires admin access attributes.merge(runner_type: :instance_type) - elsif project = Project.find_by_runners_token(params[:token]) + elsif @project = Project.find_by_runners_token(params[:token]) # Create a specific runner for the project - attributes.merge(runner_type: :project_type, projects: [project]) - elsif group = Group.find_by_runners_token(params[:token]) + attributes.merge(runner_type: :project_type, projects: [@project]) + elsif @group = Group.find_by_runners_token(params[:token]) # Create a specific runner for the group - attributes.merge(runner_type: :group_type, groups: [group]) + attributes.merge(runner_type: :group_type, groups: [@group]) else forbidden! end - runner = ::Ci::Runner.create(attributes) + @runner = ::Ci::Runner.create(attributes) - if runner.persisted? - present runner, with: Entities::RunnerRegistrationDetails + if @runner.persisted? + present @runner, with: Entities::RunnerRegistrationDetails else - render_validation_error!(runner) + render_validation_error!(@runner) end end @@ -62,9 +62,7 @@ module API delete '/' do authenticate_runner! - runner = ::Ci::Runner.find_by_token(params[:token]) - - destroy_conditionally!(runner) + destroy_conditionally!(current_runner) end desc 'Validates authentication credentials' do @@ -81,12 +79,7 @@ module API end resource :jobs do - before do - Gitlab::ApplicationContext.push( - user: -> { current_job&.user }, - project: -> { current_job&.project } - ) - end + before { set_application_context } desc 'Request a job' do success Entities::JobRequest::Response diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 26af921432c..e199111c975 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -100,7 +100,12 @@ module API attributes_for_keys(%w[target_url description coverage]) status.update(optional_attributes) if optional_attributes.any? - render_validation_error!(status) if status.invalid? + + if status.valid? + status.update_older_statuses_retried! if Feature.enabled?(:ci_fix_commit_status_retried, user_project, default_enabled: :yaml) + else + render_validation_error!(status) + end begin case params[:state] diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 1181650fe96..ec7585363e2 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -93,6 +93,20 @@ module API route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + get ':id/-/packages/composer/p2/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do + not_found! if packages.empty? + + presenter.package_versions + end + + desc 'Composer packages endpoint at group level for package versions metadata' + + params do + requires :package_name, type: String, file_path: true, desc: 'The Composer package name' + end + + route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true + get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do not_found! if packages.empty? not_found! if params[:sha].blank? diff --git a/lib/api/concerns/packages/conan_endpoints.rb b/lib/api/concerns/packages/conan_endpoints.rb index 1796d51324f..eb762be8285 100644 --- a/lib/api/concerns/packages/conan_endpoints.rb +++ b/lib/api/concerns/packages/conan_endpoints.rb @@ -42,7 +42,7 @@ module API # Personal access token will be extracted from Bearer or Basic authorization # in the overridden find_personal_access_token or find_user_from_job_token helpers - authenticate! + authenticate_non_get! end desc 'Ping the Conan API' do @@ -71,6 +71,10 @@ module API end namespace 'users' do + before do + authenticate! + end + format :txt content_type :txt, 'text/plain' diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 833288c6013..d6e006df976 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -41,8 +41,8 @@ module API authorize_read_package!(project) - packages = ::Packages::Npm::PackageFinder.new(project, package_name) - .execute + packages = ::Packages::Npm::PackageFinder.new(package_name, project: project) + .execute not_found! if packages.empty? @@ -68,9 +68,8 @@ module API authorize_create_package!(project) - package = ::Packages::Npm::PackageFinder - .new(project, package_name) - .find_by_version(version) + package = ::Packages::Npm::PackageFinder.new(package_name, project: project) + .find_by_version(version) not_found!('Package') unless package ::Packages::Npm::CreateTagService.new(package, tag).execute @@ -112,9 +111,8 @@ module API route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true get '*package_name', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do package_name = params[:package_name] - - packages = ::Packages::Npm::PackageFinder.new(project_or_nil, package_name) - .execute + packages = ::Packages::Npm::PackageFinder.new(package_name, project: project_or_nil) + .execute redirect_request = project_or_nil.blank? || packages.empty? diff --git a/lib/api/entities/ci/pipeline_basic.rb b/lib/api/entities/ci/pipeline_basic.rb index dbb9b828757..f4f2356c812 100644 --- a/lib/api/entities/ci/pipeline_basic.rb +++ b/lib/api/entities/ci/pipeline_basic.rb @@ -4,7 +4,7 @@ module API module Entities module Ci class PipelineBasic < Grape::Entity - expose :id, :sha, :ref, :status + expose :id, :project_id, :sha, :ref, :status expose :created_at, :updated_at expose :web_url do |pipeline, _options| diff --git a/lib/api/entities/job_request/response.rb b/lib/api/entities/job_request/response.rb index 8db9aff3dc9..bf22ea1e6e2 100644 --- a/lib/api/entities/job_request/response.rb +++ b/lib/api/entities/job_request/response.rb @@ -20,7 +20,7 @@ module API model end - expose :variables + expose :runner_variables, as: :variables expose :steps, using: Entities::JobRequest::Step expose :image, using: Entities::JobRequest::Image expose :services, using: Entities::JobRequest::Service diff --git a/lib/api/entities/plan_limit.rb b/lib/api/entities/plan_limit.rb new file mode 100644 index 00000000000..40e8b348c18 --- /dev/null +++ b/lib/api/entities/plan_limit.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + class PlanLimit < Grape::Entity + expose :conan_max_file_size + expose :generic_packages_max_file_size + expose :maven_max_file_size + expose :npm_max_file_size + expose :nuget_max_file_size + expose :pypi_max_file_size + end + end +end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 6ad6123a20e..e332e5e40fa 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -5,6 +5,8 @@ module API class Project < BasicProjectDetails include ::API::Helpers::RelatedResourcesHelpers + expose :container_registry_url, as: :container_registry_image_prefix, if: -> (_, _) { Gitlab.config.registry.enabled } + expose :_links do expose :self do |project| expose_url(api_v4_projects_path(id: project.id)) diff --git a/lib/api/entities/project_repository_storage_move.rb b/lib/api/entities/project_repository_storage_move.rb deleted file mode 100644 index 191bbaf19d7..00000000000 --- a/lib/api/entities/project_repository_storage_move.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - class ProjectRepositoryStorageMove < BasicRepositoryStorageMove - expose :project, using: Entities::ProjectIdentity - end - end -end diff --git a/lib/api/entities/projects/repository_storage_move.rb b/lib/api/entities/projects/repository_storage_move.rb new file mode 100644 index 00000000000..7844cd36e02 --- /dev/null +++ b/lib/api/entities/projects/repository_storage_move.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + module Projects + class RepositoryStorageMove < BasicRepositoryStorageMove + expose :project, using: Entities::ProjectIdentity + end + end + end +end diff --git a/lib/api/entities/protected_branch.rb b/lib/api/entities/protected_branch.rb index 80c8a791053..e5dbaffb591 100644 --- a/lib/api/entities/protected_branch.rb +++ b/lib/api/entities/protected_branch.rb @@ -7,6 +7,7 @@ module API expose :name expose :push_access_levels, using: Entities::ProtectedRefAccess expose :merge_access_levels, using: Entities::ProtectedRefAccess + expose :allow_force_push end end end diff --git a/lib/api/entities/public_group_details.rb b/lib/api/entities/public_group_details.rb new file mode 100644 index 00000000000..0dfe28d251a --- /dev/null +++ b/lib/api/entities/public_group_details.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module API + module Entities + class PublicGroupDetails < BasicGroupDetails + expose :avatar_url do |group, options| + group.avatar_url(only_path: false) + end + expose :full_name, :full_path + end + end +end diff --git a/lib/api/entities/snippets/repository_storage_move.rb b/lib/api/entities/snippets/repository_storage_move.rb new file mode 100644 index 00000000000..4e14d1dfba2 --- /dev/null +++ b/lib/api/entities/snippets/repository_storage_move.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module API + module Entities + module Snippets + class RepositoryStorageMove < BasicRepositoryStorageMove + expose :snippet, using: Entities::BasicSnippet + end + end + end +end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 5dd2fa22690..3e1e430c2f9 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -75,6 +75,33 @@ module API end end + desc "Delete multiple stopped review apps" do + detail "Remove multiple stopped review environments older than a specific age" + success Entities::Environment + end + params do + optional :before, type: Time, desc: "The timestamp before which environments can be deleted. Defaults to 30 days ago.", default: -> { 30.days.ago } + optional :limit, type: Integer, desc: "Maximum number of environments to delete. Defaults to 100.", default: 100, values: 1..1000 + optional :dry_run, type: Boolean, desc: "If set, perform a dry run where no actual deletions will be performed. Defaults to true.", default: true + end + delete ":id/environments/review_apps" do + authorize! :read_environment, user_project + + result = ::Environments::ScheduleToDeleteReviewAppsService.new(user_project, current_user, params).execute + + response = { + scheduled_entries: Entities::Environment.represent(result.scheduled_entries), + unprocessable_entries: Entities::Environment.represent(result.unprocessable_entries) + } + + if result.success? + status result.status + present response, current_user: current_user + else + render_api_error!(response.merge!(message: result.error_message), result.status) + end + end + desc 'Deletes an existing environment' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Environment diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb index 24d726f4a41..3d0ba97b51a 100644 --- a/lib/api/generic_packages.rb +++ b/lib/api/generic_packages.rb @@ -13,7 +13,7 @@ module API before do require_packages_enabled! - authenticate! + authenticate_non_get! require_generic_packages_available! end diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb index 7425e1bd145..90632048354 100644 --- a/lib/api/group_boards.rb +++ b/lib/api/group_boards.rb @@ -30,7 +30,7 @@ module API use :pagination end get '/' do - authorize!(:read_board, user_group) + authorize!(:read_issue_board, user_group) present paginate(board_parent.boards.with_associations), with: Entities::Board end @@ -39,7 +39,7 @@ module API success Entities::Board end get '/:board_id' do - authorize!(:read_board, user_group) + authorize!(:read_issue_board, user_group) present board, with: Entities::Board end @@ -51,7 +51,7 @@ module API use :update_params end put '/:board_id' do - authorize!(:admin_board, board_parent) + authorize!(:admin_issue_board, board_parent) update_board end @@ -69,7 +69,7 @@ module API use :pagination end get '/lists' do - authorize!(:read_board, user_group) + authorize!(:read_issue_board, user_group) present paginate(board_lists), with: Entities::List end @@ -81,7 +81,7 @@ module API requires :list_id, type: Integer, desc: 'The ID of a list' end get '/lists/:list_id' do - authorize!(:read_board, user_group) + authorize!(:read_issue_board, user_group) present board_lists.find(params[:list_id]), with: Entities::List end @@ -93,7 +93,7 @@ module API use :list_creation_params end post '/lists' do - authorize!(:admin_list, user_group) + authorize!(:admin_issue_board_list, user_group) create_list end @@ -109,7 +109,7 @@ module API put '/lists/:list_id' do list = board_lists.find(params[:list_id]) - authorize!(:admin_list, user_group) + authorize!(:admin_issue_board_list, user_group) move_list(list) end @@ -122,7 +122,7 @@ module API requires :list_id, type: Integer, desc: 'The ID of a board list' end delete "/lists/:list_id" do - authorize!(:admin_list, user_group) + authorize!(:admin_issue_board_list, user_group) list = board_lists.find(params[:list_id]) destroy_list(list) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index a8b1cdab021..26fa00d6186 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -137,6 +137,10 @@ module API end end # rubocop: enable CodeReuse/ActiveRecord + + def authorize_group_creation! + authorize! :create_group + end end resource :groups do @@ -169,7 +173,7 @@ module API if parent_group authorize! :create_subgroup, parent_group else - authorize! :create_group + authorize_group_creation! end group = create_group diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 0abb21c9831..9db4a03c5b9 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -467,7 +467,7 @@ module API def handle_api_exception(exception) if report_exception?(exception) define_params_for_grape_middleware - Gitlab::ErrorTracking.with_context(current_user) do + Gitlab::ApplicationContext.with_context(user: current_user) do Gitlab::ErrorTracking.track_exception(exception) end end diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb index b303f1f845d..2b1ed479692 100644 --- a/lib/api/helpers/issues_helpers.rb +++ b/lib/api/helpers/issues_helpers.rb @@ -11,10 +11,7 @@ module API params :optional_issue_params_ee do end - params :optional_issues_params_ee do - end - - params :optional_issue_not_params_ee do + params :issues_stats_params_ee do end def self.update_params_at_least_one_of diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index 39ecfc171a9..d5f5448fd42 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -221,7 +221,7 @@ module API def find_user_from_job_token return unless route_authentication_setting[:job_token_allowed] - job = find_job_from_token || raise(::Gitlab::Auth::UnauthorizedError) + job = find_job_from_token || return @current_authenticated_job = job # rubocop:disable Gitlab/ModuleWithInstanceVariables job.user diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index c1f6a001201..2d556f889bf 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -49,13 +49,28 @@ module API when :project params[:id] when :instance - ::Packages::Package.npm - .with_name(params[:package_name]) - .first - &.project_id + namespace_path = namespace_path_from_package_name + next unless namespace_path + + namespace = Namespace.top_most + .by_path(namespace_path) + next unless namespace + + finder = ::Packages::Npm::PackageFinder.new(params[:package_name], namespace: namespace) + + finder.last&.project_id end end end + + # from "@scope/package-name" return "scope" or nil + def namespace_path_from_package_name + package_name = params[:package_name] + return unless package_name.starts_with?('@') + return unless package_name.include?('/') + + package_name.match(Gitlab::Regex.npm_package_name_regex)&.captures&.first + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 1c85669a626..39586483990 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -71,6 +71,15 @@ module API header 'Job-Status', job.status forbidden!(reason) end + + def set_application_context + return unless current_job + + Gitlab::ApplicationContext.push( + user: -> { current_job.user }, + project: -> { current_job.project } + ) + end end end end diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb index 6101a8d307e..ed3d694f006 100644 --- a/lib/api/helpers/services_helpers.rb +++ b/lib/api/helpers/services_helpers.rb @@ -635,7 +635,7 @@ module API required: true, name: :google_iap_audience_client_id, type: String, - desc: 'Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)' + desc: 'Client ID of the IAP-secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)' }, { required: true, diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 2ab1f97afe6..52c32b4d1cf 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -49,6 +49,39 @@ module API present_member_invitations invitations end + desc 'Updates a group or project invitation.' do + success Entities::Member + end + params do + requires :email, type: String, desc: 'The email address of the invitation.' + optional :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level).' + optional :expires_at, type: DateTime, desc: 'Date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`).' + end + put ":id/invitations/:email", requirements: { email: /[^\/]+/ } do + source = find_source(source_type, params.delete(:id)) + invite_email = params[:email] + authorize_admin_source!(source_type, source) + + invite = retrieve_member_invitations(source, invite_email).first + not_found! unless invite + + update_params = declared_params(include_missing: false) + update_params.delete(:email) + bad_request! unless update_params.any? + + result = ::Members::UpdateService + .new(current_user, update_params) + .execute(invite) + + updated_member = result[:member] + + if result[:status] == :success + present_members updated_member + else + render_validation_error!(updated_member) + end + end + desc 'Removes an invitation from a group or project.' params do requires :email, type: String, desc: 'The email address of the invitation' diff --git a/lib/api/issues.rb b/lib/api/issues.rb index ea09174f03a..13dac1c174c 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -15,6 +15,24 @@ module API optional :labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' optional :milestone, type: String, desc: 'Milestone title' optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of issues' + + optional :author_id, type: Integer, desc: 'Return issues which are not authored by the user with the given ID' + optional :author_username, type: String, desc: 'Return issues which are not authored by the user with the given username' + mutually_exclusive :author_id, :author_username + + optional :assignee_id, type: Integer, desc: 'Return issues which are not assigned to the user with the given ID' + optional :assignee_username, type: Array[String], check_assignees_count: true, + coerce_with: Validations::Validators::CheckAssigneesCount.coerce, + desc: 'Return issues which are not assigned to the user with the given username' + mutually_exclusive :assignee_id, :assignee_username + + use :negatable_issue_filter_params_ee + end + + params :issues_stats_params do + optional :labels, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma-separated list of label names' + optional :milestone, type: String, desc: 'Milestone title' + optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of issues' optional :search, type: String, desc: 'Search issues for text present in the title, description, or any combination of these' optional :in, type: String, desc: '`title`, `description`, or a string joining them with comma' @@ -29,11 +47,6 @@ module API desc: 'Return issues which are assigned to the user with the given username' mutually_exclusive :assignee_id, :assignee_username - use :negatable_issue_filter_params_ee - end - - params :issues_stats_params do - use :negatable_issue_filter_params optional :created_after, type: DateTime, desc: 'Return issues created after the specified time' optional :created_before, type: DateTime, desc: 'Return issues created before the specified time' optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' @@ -48,7 +61,7 @@ module API optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' optional :confidential, type: Boolean, desc: 'Filter confidential or public issues' - use :optional_issues_params_ee + use :issues_stats_params_ee end params :issues_params do diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb index 28737f61f61..3dec0a29181 100644 --- a/lib/api/job_artifacts.rb +++ b/lib/api/job_artifacts.rb @@ -45,6 +45,7 @@ module API requires :job, type: String, desc: 'The name for the job' requires :artifact_path, type: String, desc: 'Artifact path' end + route_setting :authentication, job_token_allowed: true get ':id/jobs/artifacts/:ref_name/raw/*artifact_path', format: false, requirements: { ref_name: /.+/ } do @@ -84,13 +85,14 @@ module API requires :job_id, type: Integer, desc: 'The ID of a job' requires :artifact_path, type: String, desc: 'Artifact path' end + route_setting :authentication, job_token_allowed: true get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do authorize_download_artifacts! build = find_build!(params[:job_id]) authorize_read_job_artifacts!(build) - not_found! unless build.artifacts? + not_found! unless build.available_artifacts? path = Gitlab::Ci::Build::Artifacts::Path .new(params[:artifact_path]) diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index 390dbc892e2..7390219b60e 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -8,10 +8,11 @@ module API feature_category :continuous_integration - params do - requires :id, type: String, desc: 'The ID of a project' - end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :id, type: String, desc: 'The ID of a project' + end + helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', @@ -168,6 +169,20 @@ module API end end + resource :job do + desc 'Get current project using job token' do + success Entities::Ci::Job + end + route_setting :authentication, job_token_allowed: true + get do + # current_authenticated_job will be nil if user is using + # a valid authentication that is not CI_JOB_TOKEN + not_found!('Job') unless current_authenticated_job + + present current_authenticated_job, with: Entities::Ci::Job + end + end + helpers do # rubocop: disable CodeReuse/ActiveRecord def filter_builds(builds, scope) diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index e071b6bd68f..73ecc140959 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -62,8 +62,9 @@ module API file_name: PACKAGE_FILENAME ) - package = ::Packages::Nuget::CreatePackageService.new(project_or_group, current_user, declared_params.merge(build: current_authenticated_job)) - .execute + package = ::Packages::CreateTemporaryPackageService.new( + project_or_group, current_user, declared_params.merge(build: current_authenticated_job) + ).execute(:nuget, name: ::Packages::Nuget::TEMPORARY_PACKAGE_NAME) package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) .execute diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 0fdaa4b2656..babc7b9dd58 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -70,7 +70,11 @@ module API package = ::Packages::PackageFinder .new(user_project, params[:package_id]).execute - destroy_conditionally!(package) + destroy_conditionally!(package) do |package| + if package.destroy + package.sync_maven_metadata(current_user) + end + end end end end diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb index 196b7d88500..ab5d8b3a888 100644 --- a/lib/api/project_repository_storage_moves.rb +++ b/lib/api/project_repository_storage_moves.rb @@ -11,28 +11,28 @@ module API resource :project_repository_storage_moves do desc 'Get a list of all project repository storage moves' do detail 'This feature was introduced in GitLab 13.0.' - success Entities::ProjectRepositoryStorageMove + success Entities::Projects::RepositoryStorageMove end params do use :pagination end get do - storage_moves = ProjectRepositoryStorageMove.with_projects.order_created_at_desc + storage_moves = ::Projects::RepositoryStorageMove.with_projects.order_created_at_desc - present paginate(storage_moves), with: Entities::ProjectRepositoryStorageMove, current_user: current_user + present paginate(storage_moves), with: Entities::Projects::RepositoryStorageMove, current_user: current_user end desc 'Get a project repository storage move' do detail 'This feature was introduced in GitLab 13.0.' - success Entities::ProjectRepositoryStorageMove + success Entities::Projects::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a project repository storage move' end get ':repository_storage_move_id' do - storage_move = ProjectRepositoryStorageMove.find(params[:repository_storage_move_id]) + storage_move = ::Projects::RepositoryStorageMove.find(params[:repository_storage_move_id]) - present storage_move, with: Entities::ProjectRepositoryStorageMove, current_user: current_user + present storage_move, with: Entities::Projects::RepositoryStorageMove, current_user: current_user end desc 'Schedule bulk project repository storage moves' do @@ -58,7 +58,7 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Get a list of all project repository storage moves' do detail 'This feature was introduced in GitLab 13.1.' - success Entities::ProjectRepositoryStorageMove + success Entities::Projects::RepositoryStorageMove end params do use :pagination @@ -66,12 +66,12 @@ module API get ':id/repository_storage_moves' do storage_moves = user_project.repository_storage_moves.with_projects.order_created_at_desc - present paginate(storage_moves), with: Entities::ProjectRepositoryStorageMove, current_user: current_user + present paginate(storage_moves), with: Entities::Projects::RepositoryStorageMove, current_user: current_user end desc 'Get a project repository storage move' do detail 'This feature was introduced in GitLab 13.1.' - success Entities::ProjectRepositoryStorageMove + success Entities::Projects::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a project repository storage move' @@ -79,12 +79,12 @@ module API get ':id/repository_storage_moves/:repository_storage_move_id' do storage_move = user_project.repository_storage_moves.find(params[:repository_storage_move_id]) - present storage_move, with: Entities::ProjectRepositoryStorageMove, current_user: current_user + present storage_move, with: Entities::Projects::RepositoryStorageMove, current_user: current_user end desc 'Schedule a project repository storage move' do detail 'This feature was introduced in GitLab 13.1.' - success Entities::ProjectRepositoryStorageMove + success Entities::Projects::RepositoryStorageMove end params do optional :destination_storage_name, type: String, desc: 'The destination storage shard' @@ -95,7 +95,7 @@ module API ) if storage_move.schedule - present storage_move, with: Entities::ProjectRepositoryStorageMove, current_user: current_user + present storage_move, with: Entities::Projects::RepositoryStorageMove, current_user: current_user else render_validation_error!(storage_move) end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index fca68c3606b..19b63c28f89 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -136,6 +136,17 @@ module API present records, options end + def present_groups(groups) + options = { + with: Entities::PublicGroupDetails, + current_user: current_user + } + + groups, options = with_custom_attributes(groups, options) + + present paginate(groups), options + end + def translate_params_for_compatibility(params) params[:builds_enabled] = params.delete(:jobs_enabled) if params.key?(:jobs_enabled) params @@ -561,6 +572,25 @@ module API present paginate(users), with: Entities::UserBasic end + desc 'Get ancestor and shared groups for a project' do + success Entities::PublicGroupDetails + end + params do + optional :search, type: String, desc: 'Return list of groups matching the search criteria' + optional :skip_groups, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'Array of group ids to exclude from list' + optional :with_shared, type: Boolean, default: false, + desc: 'Include shared groups' + optional :shared_min_access_level, type: Integer, values: Gitlab::Access.all_values, + desc: 'Limit returned shared groups by minimum access level to the project' + use :pagination + end + get ':id/groups', feature_category: :source_code_management do + groups = ::Projects::GroupsFinder.new(project: user_project, current_user: current_user, params: declared_params(include_missing: false)).execute + groups = groups.search(params[:search]) if params[:search].present? + + present_groups groups + end + desc 'Start the housekeeping task for a project' do detail 'This feature was introduced in GitLab 9.0.' end diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb index 17574739a7c..802dfdec511 100644 --- a/lib/api/protected_branches.rb +++ b/lib/api/protected_branches.rb @@ -60,6 +60,9 @@ module API optional :merge_access_level, type: Integer, values: ProtectedBranch::MergeAccessLevel.allowed_access_levels, desc: 'Access levels allowed to merge (defaults: `40`, maintainer access level)' + optional :allow_force_push, type: Boolean, + default: false, + desc: 'Allow force push for all users with push access.' use :optional_params_ee end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index eaedd53aedb..f6ffeeea829 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -184,7 +184,7 @@ module API type: String, desc: 'The first commit in the range of commits to use for the changelog' - requires :to, + optional :to, type: String, desc: 'The last commit in the range of commits to use for the changelog' diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb index 66948f9eaf3..99c278be8e7 100644 --- a/lib/api/resource_access_tokens.rb +++ b/lib/api/resource_access_tokens.rb @@ -69,7 +69,7 @@ module API ).execute if token_response.success? - present token_response.payload[:access_token], with: Entities::PersonalAccessToken + present token_response.payload[:access_token], with: Entities::PersonalAccessTokenWithToken else bad_request!(token_response.message) end diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 7819aab879c..8d2d4586d8d 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -12,7 +12,7 @@ module API # The Marshal version can be found by "#{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}" # Updating the version should require a GitLab API version change. MARSHAL_VERSION = '4.8' - + PACKAGE_FILENAME = 'package.gem' FILE_NAME_REQUIREMENTS = { file_name: API::NO_SLASH_URL_PART_REGEX }.freeze @@ -26,7 +26,7 @@ module API before do require_packages_enabled! - authenticate! + authenticate_non_get! not_found! unless Feature.enabled?(:rubygem_packages, user_project) end @@ -64,8 +64,15 @@ module API requires :file_name, type: String, desc: 'Package file name' end get "gems/:file_name", requirements: FILE_NAME_REQUIREMENTS do - # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299283 - not_found! + authorize!(:read_package, user_project) + + package_file = ::Packages::PackageFile.for_rubygem_with_file_name( + user_project, params[:file_name] + ).last! + + track_package_event('pull_package', :rubygems) + + present_carrierwave_file!(package_file.file) end namespace 'api/v1' do @@ -73,27 +80,69 @@ module API detail 'This feature was introduced in GitLab 13.9' end post 'gems/authorize' do - # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263 - not_found! + authorize_workhorse!( + subject: user_project, + has_length: false, + maximum_size: user_project.actual_limits.rubygems_max_file_size + ) end desc 'Upload a gem' do detail 'This feature was introduced in GitLab 13.9' end + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' + end post 'gems' do - # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299263 - not_found! + authorize_upload!(user_project) + bad_request!('File is too large') if user_project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size) + + track_package_event('push_package', :rubygems) + + ActiveRecord::Base.transaction do + package = ::Packages::CreateTemporaryPackageService.new( + user_project, current_user, declared_params.merge(build: current_authenticated_job) + ).execute(:rubygems, name: ::Packages::Rubygems::TEMPORARY_PACKAGE_NAME) + + file_params = { + file: params[:file], + file_name: PACKAGE_FILENAME + } + + ::Packages::CreatePackageFileService.new( + package, file_params.merge(build: current_authenticated_job) + ).execute + end + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: user_project.id }) + + forbidden! end desc 'Fetch a list of dependencies' do detail 'This feature was introduced in GitLab 13.9' end params do - optional :gems, type: String, desc: 'Comma delimited gem names' + optional :gems, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'Comma delimited gem names' end get 'dependencies' do - # To be implemented in https://gitlab.com/gitlab-org/gitlab/-/issues/299282 - not_found! + authorize_read_package! + + if params[:gems].blank? + status :ok + else + results = params[:gems].map do |gem_name| + service_result = Packages::Rubygems::DependencyResolverService.new(user_project, current_user, gem_name: gem_name).execute + render_api_error!(service_result.message, service_result.http_status) if service_result.error? + + service_result.payload + end + + content_type 'application/octet-stream' + Marshal.dump(results.flatten) + end end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index e7ee8b08d87..64a72b4cb7f 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -91,6 +91,7 @@ module API optional :import_sources, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, values: %w[github bitbucket bitbucket_server gitlab google_code fogbugz git gitlab_project gitea manifest phabricator], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' + optional :in_product_marketing_emails_enabled, type: Boolean, desc: 'By default, in-product marketing emails are enabled. To disable these emails, disable this option.' optional :invisible_captcha_enabled, type: Boolean, desc: 'Enable Invisible Captcha spam detection during signup.' optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' diff --git a/lib/api/snippet_repository_storage_moves.rb b/lib/api/snippet_repository_storage_moves.rb index 84dbc03ba33..e3034191641 100644 --- a/lib/api/snippet_repository_storage_moves.rb +++ b/lib/api/snippet_repository_storage_moves.rb @@ -11,28 +11,28 @@ module API resource :snippet_repository_storage_moves do desc 'Get a list of all snippet repository storage moves' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::SnippetRepositoryStorageMove + success Entities::Snippets::RepositoryStorageMove end params do use :pagination end get do - storage_moves = SnippetRepositoryStorageMove.order_created_at_desc + storage_moves = ::Snippets::RepositoryStorageMove.order_created_at_desc - present paginate(storage_moves), with: Entities::SnippetRepositoryStorageMove, current_user: current_user + present paginate(storage_moves), with: Entities::Snippets::RepositoryStorageMove, current_user: current_user end desc 'Get a snippet repository storage move' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::SnippetRepositoryStorageMove + success Entities::Snippets::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a snippet repository storage move' end get ':repository_storage_move_id' do - storage_move = SnippetRepositoryStorageMove.find(params[:repository_storage_move_id]) + storage_move = ::Snippets::RepositoryStorageMove.find(params[:repository_storage_move_id]) - present storage_move, with: Entities::SnippetRepositoryStorageMove, current_user: current_user + present storage_move, with: Entities::Snippets::RepositoryStorageMove, current_user: current_user end desc 'Schedule bulk snippet repository storage moves' do @@ -68,7 +68,7 @@ module API desc 'Get a list of all snippets repository storage moves' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::SnippetRepositoryStorageMove + success Entities::Snippets::RepositoryStorageMove end params do use :pagination @@ -76,12 +76,12 @@ module API get ':id/repository_storage_moves' do storage_moves = user_snippet.repository_storage_moves.order_created_at_desc - present paginate(storage_moves), with: Entities::SnippetRepositoryStorageMove, current_user: current_user + present paginate(storage_moves), with: Entities::Snippets::RepositoryStorageMove, current_user: current_user end desc 'Get a snippet repository storage move' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::SnippetRepositoryStorageMove + success Entities::Snippets::RepositoryStorageMove end params do requires :repository_storage_move_id, type: Integer, desc: 'The ID of a snippet repository storage move' @@ -89,12 +89,12 @@ module API get ':id/repository_storage_moves/:repository_storage_move_id' do storage_move = user_snippet.repository_storage_moves.find(params[:repository_storage_move_id]) - present storage_move, with: Entities::SnippetRepositoryStorageMove, current_user: current_user + present storage_move, with: Entities::Snippets::RepositoryStorageMove, current_user: current_user end desc 'Schedule a snippet repository storage move' do detail 'This feature was introduced in GitLab 13.8.' - success Entities::SnippetRepositoryStorageMove + success Entities::Snippets::RepositoryStorageMove end params do optional :destination_storage_name, type: String, desc: 'The destination storage shard' @@ -105,7 +105,7 @@ module API ) if storage_move.schedule - present storage_move, with: Entities::SnippetRepositoryStorageMove, current_user: current_user + present storage_move, with: Entities::Snippets::RepositoryStorageMove, current_user: current_user else render_validation_error!(storage_move) end diff --git a/lib/api/users.rb b/lib/api/users.rb index f91e3c34ef2..b2f99bb18dc 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -58,6 +58,7 @@ module API optional :color_scheme_id, type: Integer, desc: 'The color scheme for the file viewer' optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile' optional :note, type: String, desc: 'Admin note for this user' + optional :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' all_or_none_of :extern_uid, :provider use :optional_params_ee @@ -82,6 +83,7 @@ module API optional :search, type: String, desc: 'Search for a username' optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' + optional :exclude_external, as: :non_external, type: Boolean, default: false, desc: 'Filters only non external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' optional :created_after, type: DateTime, desc: 'Return users created after the specified time' optional :created_before, type: DateTime, desc: 'Return users created before the specified time' @@ -97,7 +99,7 @@ module API end # rubocop: disable CodeReuse/ActiveRecord get feature_category: :users do - authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) + authenticated_as_admin! if params[:extern_uid].present? && params[:provider].present? unless current_user&.admin? params.except!(:created_after, :created_before, :order_by, :sort, :two_factor, :without_projects) @@ -1071,10 +1073,7 @@ module API put "status", feature_category: :users do forbidden! unless can?(current_user, :update_user_status, current_user) - update_params = declared_params - update_params.delete(:clear_status_after) if Feature.disabled?(:clear_status_with_quick_options, current_user, default_enabled: :yaml) - - if ::Users::SetStatusService.new(current_user, update_params).execute + if ::Users::SetStatusService.new(current_user, declared_params).execute present current_user.status, with: Entities::UserStatus else render_validation_error!(current_user.status) diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 327335aec2d..2d25e76626a 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -194,13 +194,16 @@ module API # Self-hosted Jira (tested on 7.11.1) requests this endpoint right # after fetching branches. + # rubocop: disable CodeReuse/ActiveRecord get ':namespace/:project/events' do user_project = find_project_with_access(params) merge_requests = authorized_merge_requests_for_project(user_project) + merge_requests = merge_requests.preload(:author, :assignees, :metrics, source_project: :namespace, target_project: :namespace) present paginate(merge_requests), with: ::API::Github::Entities::PullRequestEvent end + # rubocop: enable CodeReuse/ActiveRecord params do use :project_full_path diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 3fa42be47a9..8441aeb10ab 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -111,7 +111,7 @@ module API if response.success? no_content! else - render_api_error!(reponse.message) + unprocessable_entity!(response.message) end end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index d15114a72a3..79b7b2c61f2 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -201,7 +201,12 @@ module Backup PoolRepository.includes(:source_project).find_each do |pool| progress.puts " - Object pool #{pool.disk_path}..." - pool.source_project ||= pool.member_projects.first.root_of_fork_network + pool.source_project ||= pool.member_projects.first&.root_of_fork_network + unless pool.source_project + progress.puts " - Object pool #{pool.disk_path}... " + "[SKIPPED]".color(:cyan) + next + end + pool.state = 'none' pool.save diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 0aa1ee8f604..d569711431c 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -120,7 +120,7 @@ module Banzai end def autolink_filter(text) - Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:| + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:, mode:| autolink_match(link).html_safe end end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index e16de13725f..8a7d3c49ffb 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -100,7 +100,7 @@ module Banzai if url?(content) path = content - elsif file = wiki.find_file(content) + elsif file = wiki.find_file(content, load_content: false) path = ::File.join(wiki_base_path, file.path) end diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb index 9fd77c48659..bedc2d0fd04 100644 --- a/lib/banzai/filter/markdown_pre_escape_filter.rb +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -24,8 +24,10 @@ module Banzai # This filter does the initial surrounding, and MarkdownPostEscapeFilter # does the conversion into span tags. class MarkdownPreEscapeFilter < HTML::Pipeline::TextFilter - ASCII_PUNCTUATION = %r{([\\][!"#$%&'()*+,-./:;<=>?@\[\\\]^_`{|}~])}.freeze - LITERAL_KEYWORD = 'cmliteral' + # We just need to target those that are special GitLab references + REFERENCE_CHARACTERS = '@#!$&~%^' + ASCII_PUNCTUATION = %r{([\\][#{REFERENCE_CHARACTERS}])}.freeze + LITERAL_KEYWORD = 'cmliteral' def call return @text unless Feature.enabled?(:honor_escaped_markdown, context[:group] || context[:project]&.group) diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index f6314040f28..06dddc74eba 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -12,8 +12,8 @@ module Banzai def customize_allowlist(allowlist) # Allow table alignment; we allow specific text-align values in a # transformer below - allowlist[:attributes]['th'] = %w(style) - allowlist[:attributes]['td'] = %w(style) + allowlist[:attributes]['th'] = %w[style] + allowlist[:attributes]['td'] = %w[style] allowlist[:css] = { properties: ['text-align'] } # Allow the 'data-sourcepos' from CommonMark on all elements @@ -25,7 +25,7 @@ module Banzai # Allow `id` in a and li elements for footnotes # and remove any `id` properties not matching for footnotes allowlist[:attributes]['a'].push('id') - allowlist[:attributes]['li'] = %w(id) + allowlist[:attributes]['li'] = %w[id] allowlist[:transformers].push(self.class.remove_non_footnote_ids) allowlist @@ -64,3 +64,5 @@ module Banzai end end end + +Banzai::Filter::SanitizationFilter.prepend_if_ee('EE::Banzai::Filter::SanitizationFilter') diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index 101b55a49e4..f52ffe117d9 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -76,7 +76,7 @@ module Banzai end def spaced_link_filter(text) - Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_OR_IMAGE_PATTERN) do |link, left:, right:| + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_OR_IMAGE_PATTERN) do |link, left:, right:, mode:| spaced_link_match(link).html_safe end end diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index 98987ee2019..3789a215dd8 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -15,7 +15,7 @@ module Banzai end def extra_element_attrs - { width: "400" } + { width: "400", preload: "metadata" } end end end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb index 9c496daccaa..a6a31de3ce6 100644 --- a/lib/bitbucket/collection.rb +++ b/lib/bitbucket/collection.rb @@ -11,13 +11,5 @@ module Bitbucket lazy end - - def method_missing(method, *args) - return super unless self.respond_to?(method) - - self.__send__(method, *args) do |item| # rubocop:disable GitlabSecurity/PublicSend - block_given? ? yield(item) : item - end - end end end diff --git a/lib/bitbucket_server/collection.rb b/lib/bitbucket_server/collection.rb index f549acbd87f..2880f587202 100644 --- a/lib/bitbucket_server/collection.rb +++ b/lib/bitbucket_server/collection.rb @@ -35,13 +35,5 @@ module BitbucketServer current_page + 1 end - - def method_missing(method, *args) - return super unless self.respond_to?(method) - - self.__send__(method, *args) do |item| # rubocop:disable GitlabSecurity/PublicSend - block_given? ? yield(item) : item - end - end end end diff --git a/lib/bulk_imports/common/loaders/entity_loader.rb b/lib/bulk_imports/common/loaders/entity_loader.rb deleted file mode 100644 index 8644f3c9dcb..00000000000 --- a/lib/bulk_imports/common/loaders/entity_loader.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Common - module Loaders - class EntityLoader - def initialize(*args); end - - def load(context, entity) - context.bulk_import.entities.create!(entity) - end - end - end - end -end diff --git a/lib/bulk_imports/common/transformers/award_emoji_transformer.rb b/lib/bulk_imports/common/transformers/award_emoji_transformer.rb deleted file mode 100644 index 260b47ab917..00000000000 --- a/lib/bulk_imports/common/transformers/award_emoji_transformer.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Common - module Transformers - class AwardEmojiTransformer - def initialize(*args); end - - def transform(context, data) - user = find_user(context, data&.dig('user', 'public_email')) || context.current_user - - data - .except('user') - .merge('user_id' => user.id) - end - - private - - def find_user(context, email) - return if email.blank? - - context.group.users.find_by_any_email(email, confirmed: true) # rubocop: disable CodeReuse/ActiveRecord - end - end - end - end -end diff --git a/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb b/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb index 858c4c8976b..38e2fc0b1b9 100644 --- a/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb +++ b/lib/bulk_imports/common/transformers/prohibited_attributes_transformer.rb @@ -14,11 +14,9 @@ module BulkImports /\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads ).freeze - def initialize(options = {}) - @options = options - end - def transform(context, data) + return unless data + data.each_with_object({}) do |(key, value), result| prohibited = prohibited_key?(key) diff --git a/lib/bulk_imports/common/transformers/user_reference_transformer.rb b/lib/bulk_imports/common/transformers/user_reference_transformer.rb new file mode 100644 index 00000000000..ca077b4ef43 --- /dev/null +++ b/lib/bulk_imports/common/transformers/user_reference_transformer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# UserReferenceTransformer replaces specified user +# reference key with a user id being either: +# - A user id found by `public_email` in the group +# - Current user id +# under a new key `"#{@reference}_id"`. +module BulkImports + module Common + module Transformers + class UserReferenceTransformer + DEFAULT_REFERENCE = 'user' + + def initialize(options = {}) + @reference = options[:reference] || DEFAULT_REFERENCE + @suffixed_reference = "#{@reference}_id" + end + + def transform(context, data) + return unless data + + user = find_user(context, data&.dig(@reference, 'public_email')) || context.current_user + + data + .except(@reference) + .merge(@suffixed_reference => user.id) + end + + private + + def find_user(context, email) + return if email.blank? + + context.group.users.find_by_any_email(email, confirmed: true) # rubocop: disable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb index b01fb6f68ac..e5e2b9fdbd4 100644 --- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -4,8 +4,6 @@ module BulkImports module Groups module Extractors class SubgroupsExtractor - def initialize(*args); end - def extract(context) encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path) diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb index d1fe791c2ce..23efbc33581 100644 --- a/lib/bulk_imports/groups/graphql/get_labels_query.rb +++ b/lib/bulk_imports/groups/graphql/get_labels_query.rb @@ -10,7 +10,7 @@ module BulkImports <<-'GRAPHQL' query ($full_path: ID!, $cursor: String) { group(fullPath: $full_path) { - labels(first: 100, after: $cursor) { + labels(first: 100, after: $cursor, onlyGroupLabels: true) { page_info: pageInfo { end_cursor: endCursor has_next_page: hasNextPage @@ -19,6 +19,8 @@ module BulkImports title description color + created_at: createdAt + updated_at: updatedAt } } } diff --git a/lib/bulk_imports/groups/graphql/get_milestones_query.rb b/lib/bulk_imports/groups/graphql/get_milestones_query.rb new file mode 100644 index 00000000000..2ade87e6fa0 --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_milestones_query.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetMilestonesQuery + extend self + + def to_s + <<-'GRAPHQL' + query ($full_path: ID!, $cursor: String) { + group(fullPath: $full_path) { + milestones(first: 100, after: $cursor, includeDescendants: false) { + page_info: pageInfo { + end_cursor: endCursor + has_next_page: hasNextPage + } + nodes { + title + description + state + start_date: startDate + due_date: dueDate + created_at: createdAt + updated_at: updatedAt + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.entity.next_page_for(:milestones) + } + end + + def base_path + %w[data group milestones] + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb index 386fc695182..a631685c2ad 100644 --- a/lib/bulk_imports/groups/loaders/group_loader.rb +++ b/lib/bulk_imports/groups/loaders/group_loader.rb @@ -4,10 +4,6 @@ module BulkImports module Groups module Loaders class GroupLoader - def initialize(options = {}) - @options = options - end - def load(context, data) return unless user_can_create_group?(context.current_user, data) diff --git a/lib/bulk_imports/groups/loaders/labels_loader.rb b/lib/bulk_imports/groups/loaders/labels_loader.rb deleted file mode 100644 index b8c9ba9609c..00000000000 --- a/lib/bulk_imports/groups/loaders/labels_loader.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Loaders - class LabelsLoader - def initialize(*); end - - def load(context, data) - Labels::CreateService.new(data).execute(group: context.group) - end - end - end - end -end diff --git a/lib/bulk_imports/groups/loaders/members_loader.rb b/lib/bulk_imports/groups/loaders/members_loader.rb deleted file mode 100644 index ccf44b31aee..00000000000 --- a/lib/bulk_imports/groups/loaders/members_loader.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module BulkImports - module Groups - module Loaders - class MembersLoader - def initialize(*); end - - def load(context, data) - return unless data - - context.group.members.create!(data) - end - end - end - end -end diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb index 40dab9b444c..9f8b8682751 100644 --- a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb @@ -11,7 +11,9 @@ module BulkImports transformer Common::Transformers::ProhibitedAttributesTransformer - loader BulkImports::Groups::Loaders::LabelsLoader + def load(context, data) + Labels::CreateService.new(data).execute(group: context.group) + end def after_run(extracted_data) context.entity.update_tracker_for( diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb index b00c4c1a659..32fc931e8c3 100644 --- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb @@ -12,7 +12,11 @@ module BulkImports transformer Common::Transformers::ProhibitedAttributesTransformer transformer BulkImports::Groups::Transformers::MemberAttributesTransformer - loader BulkImports::Groups::Loaders::MembersLoader + def load(context, data) + return unless data + + context.group.members.create!(data) + end def after_run(extracted_data) context.entity.update_tracker_for( diff --git a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb new file mode 100644 index 00000000000..8497162e0e7 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class MilestonesPipeline + include Pipeline + + extractor BulkImports::Common::Extractors::GraphqlExtractor, + query: BulkImports::Groups::Graphql::GetMilestonesQuery + + transformer Common::Transformers::ProhibitedAttributesTransformer + + def load(context, data) + return unless data + + raise ::BulkImports::Pipeline::NotAllowedError unless authorized? + + context.group.milestones.create!(data) + end + + def after_run(extracted_data) + context.entity.update_tracker_for( + relation: :milestones, + has_next_page: extracted_data.has_next_page?, + next_page: extracted_data.next_page + ) + + if extracted_data.has_next_page? + run + end + end + + private + + def authorized? + context.current_user.can?(:admin_milestone, context.group) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb index d7e1a118d0b..c47a8bd1daa 100644 --- a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb @@ -9,7 +9,10 @@ module BulkImports extractor BulkImports::Groups::Extractors::SubgroupsExtractor transformer Common::Transformers::ProhibitedAttributesTransformer transformer BulkImports::Groups::Transformers::SubgroupToEntityTransformer - loader BulkImports::Common::Loaders::EntityLoader + + def load(context, data) + context.bulk_import.entities.create!(data) + end end end end diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb index 7de9a430421..23e898a7bb2 100644 --- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -4,10 +4,6 @@ module BulkImports module Groups module Transformers class GroupAttributesTransformer - def initialize(options = {}) - @options = options - end - def transform(context, data) import_entity = context.entity @@ -39,12 +35,11 @@ module BulkImports end def transform_parent(context, import_entity, data) - current_user = context.current_user - namespace = Namespace.find_by_full_path(import_entity.destination_namespace) - - return data if namespace == current_user.namespace + unless import_entity.destination_namespace.blank? + namespace = Namespace.find_by_full_path(import_entity.destination_namespace) + data['parent_id'] = namespace.id + end - data['parent_id'] = namespace.id data end diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb index 622f5b60ffe..e92c898171a 100644 --- a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb @@ -4,8 +4,6 @@ module BulkImports module Groups module Transformers class MemberAttributesTransformer - def initialize(*); end - def transform(context, data) data .then { |data| add_user(data) } diff --git a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb index 6c3c299c2d2..676a6ca8d2a 100644 --- a/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb +++ b/lib/bulk_imports/groups/transformers/subgroup_to_entity_transformer.rb @@ -4,8 +4,6 @@ module BulkImports module Groups module Transformers class SubgroupToEntityTransformer - def initialize(*args); end - def transform(context, entry) { source_type: :group_entity, diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb index f967b7ad7ab..f016b552fd4 100644 --- a/lib/bulk_imports/importers/group_importer.rb +++ b/lib/bulk_imports/importers/group_importer.rb @@ -24,7 +24,8 @@ module BulkImports BulkImports::Groups::Pipelines::GroupPipeline, BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, BulkImports::Groups::Pipelines::MembersPipeline, - BulkImports::Groups::Pipelines::LabelsPipeline + BulkImports::Groups::Pipelines::LabelsPipeline, + BulkImports::Groups::Pipelines::MilestonesPipeline ] end end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 1d55ad95887..14445162737 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -3,9 +3,14 @@ module BulkImports module Pipeline extend ActiveSupport::Concern + include Gitlab::Utils::StrongMemoize include Gitlab::ClassAttributes include Runner + NotAllowedError = Class.new(StandardError) + + CACHE_KEY_EXPIRATION = 2.hours + def initialize(context) @context = context end @@ -15,16 +20,83 @@ module BulkImports attr_reader :context + # Fetch pipeline extractor. + # An extractor is defined either by instance `#extract(context)` method + # or by using `extractor` DSL. + # + # @example + # class MyPipeline + # extractor MyExtractor, foo: :bar + # end + # + # class MyPipeline + # def extract(context) + # puts 'Fetch some data' + # end + # end + # + # If pipeline implements instance method `extract` - use it + # and ignore class `extractor` method implementation. def extractor - @extractor ||= instantiate(self.class.get_extractor) + @extractor ||= self.respond_to?(:extract) ? self : instantiate(self.class.get_extractor) end + # Fetch pipeline transformers. + # + # A transformer can be defined using: + # - `transformer` class method + # - `transform` instance method + # + # Multiple transformers can be defined within a single + # pipeline and run sequentially for each record in the + # following order: + # - Transformers defined using `transformer` class method + # - Instance method `transform` + # + # Instance method `transform` is always the last to run. + # + # @example + # class MyPipeline + # transformer MyTransformerOne, foo: :bar + # transformer MyTransformerTwo, foo: :bar + # + # def transform(context, data) + # # perform transformation here + # end + # end + # + # In the example above `#transform` is the first to run and + # `MyTransformerTwo` method is the last. def transformers - @transformers ||= self.class.transformers.map(&method(:instantiate)) + strong_memoize(:transformers) do + defined_transformers = self.class.transformers.map(&method(:instantiate)) + + transformers = [] + transformers << self if respond_to?(:transform) + transformers.concat(defined_transformers) + transformers + end end + # Fetch pipeline loader. + # A loader is defined either by instance method `#load(context, data)` + # or by using `loader` DSL. + # + # @example + # class MyPipeline + # loader MyLoader, foo: :bar + # end + # + # class MyPipeline + # def load(context, data) + # puts 'Load some data' + # end + # end + # + # If pipeline implements instance method `load` - use it + # and ignore class `loader` method implementation. def loader - @loaders ||= instantiate(self.class.get_loader) + @loader ||= self.respond_to?(:load) ? self : instantiate(self.class.get_loader) end def pipeline @@ -32,7 +104,13 @@ module BulkImports end def instantiate(class_config) - class_config[:klass].new(class_config[:options]) + options = class_config[:options] + + if options + class_config[:klass].new(class_config[:options]) + else + class_config[:klass].new + end end def abort_on_failure? @@ -58,7 +136,7 @@ module BulkImports end def transformers - class_attributes[:transformers] + class_attributes[:transformers] || [] end def get_loader diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index d39f4121b51..e3535e585cc 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -8,7 +8,7 @@ module BulkImports MarkedAsFailedError = Class.new(StandardError) def run - raise MarkedAsFailedError if marked_as_failed? + raise MarkedAsFailedError if context.entity.failed? info(message: 'Pipeline started') @@ -40,7 +40,7 @@ module BulkImports private # rubocop:disable Lint/UselessAccessModifier def run_pipeline_step(step, class_name = nil) - raise MarkedAsFailedError if marked_as_failed? + raise MarkedAsFailedError if context.entity.failed? info(pipeline_step: step, step_class: class_name) @@ -62,24 +62,13 @@ module BulkImports end def mark_as_failed - warn(message: 'Pipeline failed', pipeline_class: pipeline) + warn(message: 'Pipeline failed') context.entity.fail_op! end - def marked_as_failed? - return true if context.entity.failed? - - false - end - def log_skip(extra = {}) - log = { - message: 'Skipping due to failed pipeline status', - pipeline_class: pipeline - }.merge(extra) - - info(log) + info({ message: 'Skipping due to failed pipeline status' }.merge(extra)) end def log_import_failure(exception, step) @@ -92,25 +81,39 @@ module BulkImports correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id } + error( + pipeline_step: step, + exception_class: exception.class.to_s, + exception_message: exception.message + ) + BulkImports::Failure.create(attributes) end + def info(extra = {}) + logger.info(log_params(extra)) + end + def warn(extra = {}) logger.warn(log_params(extra)) end - def info(extra = {}) - logger.info(log_params(extra)) + def error(extra = {}) + logger.error(log_params(extra)) end def log_params(extra) defaults = { + bulk_import_id: context.bulk_import.id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, - pipeline_class: pipeline + pipeline_class: pipeline, + context_extra: context.extra } - defaults.merge(extra).compact + defaults + .merge(extra) + .reject { |_key, value| value.blank? } end def logger diff --git a/lib/csv_builder.rb b/lib/csv_builder.rb index a9ef5a83ae8..6116009f171 100644 --- a/lib/csv_builder.rb +++ b/lib/csv_builder.rb @@ -16,6 +16,7 @@ class CsvBuilder DEFAULT_ORDER_BY = 'id'.freeze DEFAULT_BATCH_SIZE = 1000 + PREFIX_REGEX = /^[=\+\-@;]/.freeze attr_reader :rows_written @@ -114,8 +115,8 @@ class CsvBuilder def excel_sanitize(line) return if line.nil? + return line unless line.is_a?(String) && line.match?(PREFIX_REGEX) - line = ["'", line].join if line =~ /^[=\+\-@;]/ - line + ["'", line].join end end diff --git a/lib/declarative_policy/policy_dsl.rb b/lib/declarative_policy/policy_dsl.rb index 96741c0478e..69a2bbcc79e 100644 --- a/lib/declarative_policy/policy_dsl.rb +++ b/lib/declarative_policy/policy_dsl.rb @@ -6,7 +6,7 @@ module DeclarativePolicy # Policy class (context_class here). See Base.rule # # Note that the #policy method just performs an #instance_eval, - # which is useful for multiple #enable or #prevent callse. + # which is useful for multiple #enable or #prevent calls. # # Also provides a #method_missing proxy to the context # class's class methods, so that helper methods can be diff --git a/lib/sentry/client.rb b/lib/error_tracking/sentry_client.rb index dbf54a65081..68e64fba093 100644 --- a/lib/sentry/client.rb +++ b/lib/error_tracking/sentry_client.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true -module Sentry - class Client - include Sentry::Client::Event - include Sentry::Client::Projects - include Sentry::Client::Issue - include Sentry::Client::Repo - include Sentry::Client::IssueLink +module ErrorTracking + class SentryClient + include SentryClient::Event + include SentryClient::Projects + include SentryClient::Issue + include SentryClient::Repo + include SentryClient::IssueLink Error = Class.new(StandardError) MissingKeysError = Class.new(StandardError) @@ -21,7 +21,7 @@ module Sentry private def api_urls - @api_urls ||= Sentry::ApiUrls.new(@url) + @api_urls ||= SentryClient::ApiUrls.new(@url) end def handle_mapping_exceptions(&block) @@ -94,7 +94,7 @@ module Sentry end def raise_error(message) - raise Client::Error, message + raise SentryClient::Error, message end end end diff --git a/lib/error_tracking/sentry_client/api_urls.rb b/lib/error_tracking/sentry_client/api_urls.rb new file mode 100644 index 00000000000..387309bfbdb --- /dev/null +++ b/lib/error_tracking/sentry_client/api_urls.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module ErrorTracking + class SentryClient + class ApiUrls + def initialize(url_base) + @uri = URI(url_base).freeze + end + + def issues_url + with_path(File.join(@uri.path, '/issues/')) + end + + def issue_url(issue_id) + with_path("/api/0/issues/#{escape(issue_id)}/") + end + + def projects_url + with_path('/api/0/projects/') + end + + def issue_latest_event_url(issue_id) + with_path("/api/0/issues/#{escape(issue_id)}/events/latest/") + end + + private + + def with_path(new_path) + new_uri = @uri.dup + # Sentry API returns 404 if there are extra slashes in the URL + new_uri.path = new_path.squeeze('/') + + new_uri + end + + def escape(param) + CGI.escape(param.to_s) + end + end + end +end diff --git a/lib/sentry/client/event.rb b/lib/error_tracking/sentry_client/event.rb index 01dfaa25969..93449344d6c 100644 --- a/lib/sentry/client/event.rb +++ b/lib/error_tracking/sentry_client/event.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module Sentry - class Client +module ErrorTracking + class SentryClient module Event def issue_latest_event(issue_id:) latest_event = http_get(api_urls.issue_latest_event_url(issue_id))[:body] diff --git a/lib/sentry/client/issue.rb b/lib/error_tracking/sentry_client/issue.rb index f714bda49fd..513fb3daabe 100644 --- a/lib/sentry/client/issue.rb +++ b/lib/error_tracking/sentry_client/issue.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module Sentry - class Client +module ErrorTracking + class SentryClient module Issue BadRequestError = Class.new(StandardError) ResponseInvalidSizeError = Class.new(StandardError) @@ -49,7 +49,7 @@ module Sentry { issues: response[:body], - pagination: Sentry::PaginationParser.parse(response[:headers]) + pagination: SentryClient::PaginationParser.parse(response[:headers]) } end @@ -113,7 +113,7 @@ module Sentry uri = URI(url) uri.path.squeeze!('/') # Remove trailing slash - uri = uri.to_s.gsub(/\/\z/, '') + uri = uri.to_s.delete_suffix('/') uri end diff --git a/lib/sentry/client/issue_link.rb b/lib/error_tracking/sentry_client/issue_link.rb index 91498c19f8b..1c2e8c4147a 100644 --- a/lib/sentry/client/issue_link.rb +++ b/lib/error_tracking/sentry_client/issue_link.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module Sentry - class Client +module ErrorTracking + class SentryClient module IssueLink # Creates a link in Sentry corresponding to the provided # Sentry issue and GitLab issue diff --git a/lib/error_tracking/sentry_client/pagination_parser.rb b/lib/error_tracking/sentry_client/pagination_parser.rb new file mode 100644 index 00000000000..362a5d098f7 --- /dev/null +++ b/lib/error_tracking/sentry_client/pagination_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module ErrorTracking + class SentryClient + module PaginationParser + PATTERN = /rel=\"(?<direction>\w+)\";\sresults=\"(?<results>\w+)\";\scursor=\"(?<cursor>.+)\"/.freeze + + def self.parse(headers) + links = headers['link'].to_s.split(',') + + links.map { |link| parse_link(link) }.compact.to_h + end + + def self.parse_link(link) + match = link.match(PATTERN) + + return unless match + return if match['results'] != "true" + + [match['direction'], { 'cursor' => match['cursor'] }] + end + private_class_method :parse_link + end + end +end diff --git a/lib/sentry/client/projects.rb b/lib/error_tracking/sentry_client/projects.rb index e686d4ff715..9b8daa226b0 100644 --- a/lib/sentry/client/projects.rb +++ b/lib/error_tracking/sentry_client/projects.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module Sentry - class Client +module ErrorTracking + class SentryClient module Projects def projects projects = get_projects diff --git a/lib/sentry/client/repo.rb b/lib/error_tracking/sentry_client/repo.rb index 9a0ed3c7342..3baa7e69be6 100644 --- a/lib/sentry/client/repo.rb +++ b/lib/error_tracking/sentry_client/repo.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -module Sentry - class Client +module ErrorTracking + class SentryClient module Repo def repos(organization_slug) repos_url = repos_api_url(organization_slug) diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index 06cebab8f0a..d172df4920f 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -45,6 +45,9 @@ module ExpandVariables # Lazily initialise variables variables = variables.call if variables.is_a?(Proc) + # Convert Collection to variables + variables = variables.to_hash if variables.is_a?(Gitlab::Ci::Variables::Collection) + # Convert hash array to variables if variables.is_a?(Array) variables = variables.reduce({}) do |hash, variable| diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 5ad9af6ff7d..70e5b523adf 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -37,18 +37,6 @@ class Feature push_frontend_feature_flag(:my_ops_flag, project, type: :ops) EOS }, - licensed: { - description: 'Permanent feature flags used to temporarily disable licensed features.', - deprecated: true, - optional: true, - rollout_issue: false, - ee_only: true, - default_enabled: true, - example: <<-EOS - project.feature_available?(:my_licensed_feature) - namespace.feature_available?(:my_licensed_feature) - EOS - }, experiment: { description: 'Short lived, used specifically to run A/B/n experiments.', optional: true, diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb index d3fac4c74f3..7a01050ed0c 100644 --- a/lib/generators/gitlab/usage_metric_definition_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -18,6 +18,9 @@ module Gitlab Directory.new('license', 'none', 'string') ].freeze + TOP_LEVEL_DIR = 'config' + TOP_LEVEL_DIR_EE = 'ee' + VALID_INPUT_DIRS = (TIME_FRAME_DIRS.flat_map { |d| [d.name, d.time_frame] } - %w(none)).freeze source_root File.expand_path('../../../generator_templates/usage_metric_definition', __dir__) @@ -56,15 +59,22 @@ module Gitlab private + def metric_name_suggestion + return unless Feature.enabled?(:product_intelligence_metrics_names_suggestions, default_enabled: :yaml) + + "\nname: #{Usage::Metrics::NamesSuggestions::Generator.generate(key_path)}" + end + def file_path - path = File.join('config', 'metrics', directory&.name, "#{file_name}.yml") - path = File.join('ee', path) if ee? + path = File.join(TOP_LEVEL_DIR, 'metrics', directory&.name, "#{file_name}.yml") + path = File.join(TOP_LEVEL_DIR_EE, path) if ee? path end def validate! raise "--dir option is required" unless input_dir.present? raise "Invalid dir #{input_dir}, allowed options are #{VALID_INPUT_DIRS.join(', ')}" unless directory.present? + raise "Metric definition with key path '#{key_path}' already exists" if metric_definition_exists? end def ee? @@ -79,11 +89,23 @@ module Gitlab # # 20210201124931_g_project_management_issue_title_changed_weekly.yml def file_name - "#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_#{key_path.split('.').last}" + "#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_#{metric_name}" end def directory @directory ||= TIME_FRAME_DIRS.find { |d| d.match?(input_dir) } end + + def metric_name + key_path.split('.').last + end + + def metric_definitions + @definitions ||= Gitlab::Usage::MetricDefinition.definitions + end + + def metric_definition_exists? + metric_definitions[key_path].present? + end end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index ebf2cdd875a..11ecfb951aa 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -19,6 +19,10 @@ module Gitlab Settings end + def self.host_with_port + "#{self.config.gitlab.host}:#{self.config.gitlab.port}" + end + def self.revision @_revision ||= begin if File.exist?(root.join("REVISION")) diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index 0eb1bee8181..e2db9b62dd5 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -5,7 +5,7 @@ module Gitlab module AlertManagement module Payload class Generic < Base - DEFAULT_TITLE = 'New: Incident' + DEFAULT_TITLE = 'New: Alert' attribute :description, paths: 'description' attribute :ends_at, paths: 'end_time', type: :time diff --git a/lib/gitlab/analytics/cycle_analytics/average.rb b/lib/gitlab/analytics/cycle_analytics/average.rb new file mode 100644 index 00000000000..a449b71b165 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/average.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class Average + include Gitlab::Utils::StrongMemoize + include StageQueryHelpers + + def initialize(stage:, query:) + @stage = stage + @query = query + end + + def seconds + select_average ? select_average['average'] : nil + end + + def days + seconds ? seconds.fdiv(1.day) : nil + end + + private + + attr_reader :stage + + # rubocop: disable CodeReuse/ActiveRecord + def select_average + strong_memoize(:select_average) do + execute_query(@query.select(average_in_seconds.as('average')).reorder(nil)).first + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def average + Arel::Nodes::NamedFunction.new( + 'AVG', + [duration] + ) + end + + def average_in_seconds + Arel::Nodes::Extract.new(average, :epoch) + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb index 5eca364a697..10a008a76d5 100644 --- a/lib/gitlab/analytics/cycle_analytics/data_collector.rb +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -31,6 +31,12 @@ module Gitlab end end + def average + strong_memoize(:average) do + Average.new(stage: stage, query: query) + end + end + private attr_reader :stage, :params diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index be5d9be3d64..178ebe0d4d4 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -29,6 +29,8 @@ module Gitlab @stage = stage @query = query @params = params + @sort = params[:sort] || :end_event + @direction = params[:direction] || :desc end def serialized_records @@ -52,7 +54,7 @@ module Gitlab private - attr_reader :stage, :query, :params + attr_reader :stage, :query, :params, :sort, :direction def columns MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| @@ -90,7 +92,7 @@ module Gitlab end def ordered_and_limited_query - order_by_end_event(query, columns).limit(MAX_RECORDS) + order_by(query, sort, direction, columns).limit(MAX_RECORDS) end def records diff --git a/lib/gitlab/analytics/cycle_analytics/sorting.rb b/lib/gitlab/analytics/cycle_analytics/sorting.rb new file mode 100644 index 00000000000..828879d466d --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/sorting.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class Sorting + # rubocop: disable CodeReuse/ActiveRecord + SORTING_OPTIONS = { + end_event: { + asc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.asc) }, + desc: -> (query, stage) { query.reorder(stage.end_event.timestamp_projection.desc) } + }.freeze, + duration: { + asc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).asc) }, + desc: -> (query, stage) { query.reorder(Arel::Nodes::Subtraction.new(stage.end_event.timestamp_projection, stage.start_event.timestamp_projection).desc) } + }.freeze + }.freeze + # rubocop: enable CodeReuse/ActiveRecord, + + def self.apply(query, stage, sort, direction) + sort_lambda = SORTING_OPTIONS.dig(sort, direction) || SORTING_OPTIONS.dig(:end_event, :desc) + sort_lambda.call(query, stage) + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 27fc8bd9a1a..02b1024b8b3 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -8,10 +8,12 @@ module Gitlab # Issue: < 100 # MergeRequest: >= 100 && < 1000 # Custom events for default stages: >= 1000 (legacy) + # + # To avoid duplications, verify that the value does not exist in ee/lib/ee/gitlab/analytics/cycle_analytics/stage_events.rb ENUM_MAPPING = { StageEvents::IssueCreated => 1, StageEvents::IssueFirstMentionedInCommit => 2, - StageEvents::IssueDeployedToProduction => 3, + StageEvents::IssueDeployedToProduction => 10, StageEvents::MergeRequestCreated => 100, StageEvents::MergeRequestFirstDeployedToProduction => 101, StageEvents::MergeRequestLastBuildFinished => 102, diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 79738747e71..cfc9300a710 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -19,6 +19,10 @@ module Gitlab raise NotImplementedError end + def markdown_description + self.class.name + end + def self.identifier raise NotImplementedError end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index 80e426e6e17..777a8278e6e 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -24,8 +24,8 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - def order_by_end_event(query, extra_columns_to_select = [:id]) - ordered_query = query.reorder(stage.end_event.timestamp_projection.desc) + def order_by(query, sort, direction, extra_columns_to_select = [:id]) + ordered_query = Gitlab::Analytics::CycleAnalytics::Sorting.apply(query, stage, sort, direction) # When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause if requires_grouping? diff --git a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb b/lib/gitlab/analytics/usage_trends/workers_argument_builder.rb index 54b3bbb3ce6..a502f46287d 100644 --- a/lib/gitlab/analytics/instance_statistics/workers_argument_builder.rb +++ b/lib/gitlab/analytics/usage_trends/workers_argument_builder.rb @@ -2,7 +2,7 @@ module Gitlab module Analytics - module InstanceStatistics + module UsageTrends class WorkersArgumentBuilder def initialize(measurement_identifiers: [], recorded_at: Time.zone.now) @measurement_identifiers = measurement_identifiers @@ -35,11 +35,11 @@ module Gitlab end def custom_min_max_queries - ::Analytics::InstanceStatistics::Measurement.identifier_min_max_queries + ::Analytics::UsageTrends::Measurement.identifier_min_max_queries end def query_mappings - ::Analytics::InstanceStatistics::Measurement.identifier_query_mapping + ::Analytics::UsageTrends::Measurement.identifier_query_mapping end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index cefe983848c..a75da3a682b 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -4,6 +4,7 @@ module Gitlab # A GitLab-rails specific accessor for `Labkit::Logging::ApplicationContext` class ApplicationContext include Gitlab::Utils::LazyAttributes + include Gitlab::Utils::StrongMemoize Attribute = Struct.new(:name, :type) @@ -11,6 +12,7 @@ module Gitlab Attribute.new(:project, Project), Attribute.new(:namespace, Namespace), Attribute.new(:user, User), + Attribute.new(:runner, ::Ci::Runner), Attribute.new(:caller_id, String), Attribute.new(:remote_ip, String), Attribute.new(:related_class, String), @@ -27,8 +29,12 @@ module Gitlab Labkit::Context.push(application_context.to_lazy_hash) end + def self.current + Labkit::Context.current.to_h + end + def self.current_context_include?(attribute_name) - Labkit::Context.current.to_h.include?(Labkit::Context.log_key(attribute_name)) + current.include?(Labkit::Context.log_key(attribute_name)) end def initialize(**args) @@ -43,8 +49,9 @@ module Gitlab def to_lazy_hash {}.tap do |hash| hash[:user] = -> { username } if set_values.include?(:user) - hash[:project] = -> { project_path } if set_values.include?(:project) + hash[:project] = -> { project_path } if set_values.include?(:project) || set_values.include?(:runner) hash[:root_namespace] = -> { root_namespace_path } if include_namespace? + hash[:client_id] = -> { client } if include_client? hash[:caller_id] = caller_id if set_values.include?(:caller_id) hash[:remote_ip] = remote_ip if set_values.include?(:remote_ip) hash[:related_class] = related_class if set_values.include?(:related_class) @@ -71,7 +78,8 @@ module Gitlab end def project_path - project&.full_path + associated_routable = project || runner_project + associated_routable&.full_path end def username @@ -79,15 +87,43 @@ module Gitlab end def root_namespace_path - if namespace - namespace.full_path_components.first + associated_routable = namespace || project || runner_project || runner_group + associated_routable&.full_path_components&.first + end + + def include_namespace? + set_values.include?(:namespace) || set_values.include?(:project) || set_values.include?(:runner) + end + + def include_client? + set_values.include?(:user) || set_values.include?(:runner) || set_values.include?(:remote_ip) + end + + def client + if user + "user/#{user.id}" + elsif runner + "runner/#{runner.id}" else - project&.full_path_components&.first + "ip/#{remote_ip}" end end - def include_namespace? - set_values.include?(:namespace) || set_values.include?(:project) + def runner_project + strong_memoize(:runner_project) do + next unless runner&.project_type? + + projects = runner.projects.take(2) # rubocop: disable CodeReuse/ActiveRecord + projects.first if projects.one? + end + end + + def runner_group + strong_memoize(:runner_group) do + next unless runner&.group_type? + + runner.groups.first + end end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 0a69a9c503d..f74edf2b767 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -47,7 +47,7 @@ module Gitlab # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits` - # @option users_allowlist [Array<String>] Optional list of usernames to excepted from the limit. This param will only be functional if Scope includes a current user. + # @option users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. # # @return [Boolean] Whether or not a request should be throttled def throttled?(key, **options) diff --git a/lib/gitlab/auth/current_user_mode.rb b/lib/gitlab/auth/current_user_mode.rb index 0f327a39f61..a6d706c2a49 100644 --- a/lib/gitlab/auth/current_user_mode.rb +++ b/lib/gitlab/auth/current_user_mode.rb @@ -77,7 +77,7 @@ module Gitlab return false unless user Gitlab::SafeRequestStore.fetch(admin_mode_rs_key) do - user.admin? && session_with_admin_mode? + user.admin? && (privileged_runtime? || session_with_admin_mode?) end end @@ -154,6 +154,11 @@ module Gitlab Gitlab::SafeRequestStore.delete(admin_mode_rs_key) Gitlab::SafeRequestStore.delete(admin_mode_requested_rs_key) end + + # Runtimes which imply shell access get admin mode automatically, see Gitlab::Runtime + def privileged_runtime? + Gitlab::Runtime.rake? || Gitlab::Runtime.rails_runner? || Gitlab::Runtime.console? + end end end end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 57ff3fcd1f0..ab6ac815601 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -5,11 +5,12 @@ module Gitlab module OAuth class Provider LABELS = { - "github" => "GitHub", - "gitlab" => "GitLab.com", - "google_oauth2" => "Google", - "azure_oauth2" => "Azure AD", - 'atlassian_oauth2' => 'Atlassian' + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google", + "azure_oauth2" => "Azure AD", + "azure_activedirectory_v2" => "Azure AD v2", + 'atlassian_oauth2' => 'Atlassian' }.freeze def self.authentication(user, provider) diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index f556a7f40e9..fe1bf730e76 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -239,8 +239,9 @@ module Gitlab end def update_profile - clear_user_synced_attributes_metadata + return unless gl_user + clear_user_synced_attributes_metadata return unless sync_profile_from_provider? || creating_linked_ldap_user? metadata = gl_user.build_user_synced_attributes_metadata diff --git a/lib/gitlab/avatar_cache.rb b/lib/gitlab/avatar_cache.rb new file mode 100644 index 00000000000..30c8e089061 --- /dev/null +++ b/lib/gitlab/avatar_cache.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + class AvatarCache + class << self + # Increment this if a breaking change requires + # immediate cache expiry of all avatar caches. + # + # @return [Integer] + VERSION = 1 + + # @return [Symbol] + BASE_KEY = :avatar_cache + + # @return [ActiveSupport::Duration] + DEFAULT_EXPIRY = 7.days + + # Look up cached avatar data by email address. + # This accepts a block to provide the value to be + # cached in the event nothing is found. + # + # Multiple calls in the same request will be served from the + # request store. + # + # @param email [String] + # @param additional_keys [*Object] all must respond to `#to_s` + # @param expires_in [ActiveSupport::Duration, Integer] + # @yield [email, *additional_keys] yields the supplied params back to the block + # @return [String] + def by_email(email, *additional_keys, expires_in: DEFAULT_EXPIRY) + key = email_key(email) + subkey = additional_keys.join(":") + + Gitlab::SafeRequestStore.fetch([key, subkey]) do + with do |redis| + # Look for existing cache value + cached = redis.hget(key, subkey) + + # Return the cached entry if set + break cached unless cached.nil? + + # Otherwise, call the block to get the value + to_cache = yield(email, *additional_keys).to_s + + # Set it in the cache + redis.hset(key, subkey, to_cache) + + # Update the expiry time + redis.expire(key, expires_in) + + # Return this new value + break to_cache + end + end + end + + # Remove one or more emails from the cache + # + # @param emails [String] one or more emails to delete + # @return [Integer] the number of keys deleted + def delete_by_email(*emails) + return 0 if emails.empty? + + with do |redis| + keys = emails.map { |email| email_key(email) } + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.unlink(*keys) + end + end + end + + private + + # @param email [String] + # @return [String] + def email_key(email) + "#{BASE_KEY}:v#{VERSION}:#{email}" + end + + def with(&blk) + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb index 61eb3b332de..7484027a0fa 100644 --- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -5,7 +5,7 @@ module Gitlab # Update existent project update_at column after their repository storage was moved class BackfillProjectUpdatedAtAfterRepositoryStorageMove def perform(*project_ids) - updated_repository_storages = ProjectRepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) + updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) Project.connection.execute <<-SQL WITH repository_storage_cte as ( diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb new file mode 100644 index 00000000000..80693728e86 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Generic batching class for use with a BatchedBackgroundMigration. + # Batches over the given table and column combination, returning the MIN() and MAX() + # values for the next batch as an array. + # + # If no more batches exist in the table, returns nil. + class PrimaryKeyBatchingStrategy + include Gitlab::Database::DynamicModelHelpers + + # Finds and returns the next batch in the table. + # + # table_name - The table to batch over + # column_name - The column to batch over + # batch_min_value - The minimum value which the next batch will start at + # batch_size - The size of the next batch + def next_batch(table_name, column_name, batch_min_value:, batch_size:) + model_class = define_batchable_model(table_name) + + quoted_column_name = model_class.connection.quote_column_name(column_name) + relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + next_batch_bounds = nil + + relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop + next_batch_bounds = batch.pluck(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")).first + + break + end + + next_batch_bounds + end + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 16c0de39a3b..60682bd2ec1 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -2,13 +2,11 @@ module Gitlab module BackgroundMigration - # Background migration that extends CopyColumn to update the value of a + # Background migration that updates the value of a # column using the value of another column in the same table. # # - The {start_id, end_id} arguments are at the start so that it can be used - # with `queue_background_migration_jobs_by_range_at_intervals` - # - Provides support for background job tracking through the use of - # Gitlab::Database::BackgroundMigrationJob + # with `queue_batched_background_migration` # - Uses sub-batching so that we can keep each update's execution time at # low 100s ms, while being able to update more records per 2 minutes # that we allow background migration jobs to be scheduled one after the other @@ -22,28 +20,24 @@ module Gitlab # start_id - The start ID of the range of rows to update. # end_id - The end ID of the range of rows to update. - # table - The name of the table that contains the columns. - # primary_key - The primary key column of the table. - # copy_from - The column containing the data to copy. - # copy_to - The column to copy the data to. + # batch_table - The name of the table that contains the columns. + # batch_column - The name of the column we use to batch over the table. # sub_batch_size - We don't want updates to take more than ~100ms # This allows us to run multiple smaller batches during # the minimum 2.minute interval that we can schedule jobs - def perform(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, copy_from, copy_to) quoted_copy_from = connection.quote_column_name(copy_from) quoted_copy_to = connection.quote_column_name(copy_to) - parent_batch_relation = relation_scoped_to_range(table, primary_key, start_id, end_id) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - parent_batch_relation.each_batch(column: primary_key, of: sub_batch_size) do |sub_batch| + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") sleep(PAUSE_SECONDS) end - - # We have to add all arguments when marking a job as succeeded as they - # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` - mark_job_as_succeeded(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) end private @@ -52,10 +46,6 @@ module Gitlab ActiveRecord::Base.connection end - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) - end - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) define_batchable_model(source_table).where(source_key_column => start_id..stop_id) end diff --git a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb b/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb deleted file mode 100644 index de0c357ab1c..00000000000 --- a/lib/gitlab/background_migration/merge_request_assignees_migration_progress_check.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class MergeRequestAssigneesMigrationProgressCheck - include Gitlab::Utils::StrongMemoize - - RESCHEDULE_DELAY = 3.hours - WORKER = 'PopulateMergeRequestAssigneesTable' - DeadJobsError = Class.new(StandardError) - - def perform - raise DeadJobsError, "Only dead background jobs in the queue for #{WORKER}" if !ongoing? && dead_jobs? - - if ongoing? - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name) - else - Feature.enable(:multiple_merge_request_assignees) - end - end - - private - - def dead_jobs? - strong_memoize(:dead_jobs) do - migration_klass.dead_jobs?(WORKER) - end - end - - def ongoing? - strong_memoize(:ongoing) do - migration_klass.exists?(WORKER) || migration_klass.retrying_jobs?(WORKER) - end - end - - def migration_klass - Gitlab::BackgroundMigration - end - end - # rubocop: enable Style/Documentation - end -end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb new file mode 100644 index 00000000000..4eaef26c9c6 --- /dev/null +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration moves projects.container_registry_enabled values to + # project_features.container_registry_access_level for the projects within + # the given range of ids. + class MoveContainerRegistryEnabledToProjectFeature + MAX_BATCH_SIZE = 1_000 + + module Migratable + # Migration model namespace isolated from application code. + class ProjectFeature < ActiveRecord::Base + ENABLED = 20 + DISABLED = 0 + end + end + + def perform(from_id, to_id) + (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch| + process_batch(batch.first, batch.last) + end + end + + private + + def process_batch(from_id, to_id) + ActiveRecord::Base.connection.execute(update_sql(from_id, to_id)) + + logger.info(message: "#{self.class}: Copied container_registry_enabled values for projects with IDs between #{from_id}..#{to_id}") + end + + # For projects that have a project_feature: + # Set project_features.container_registry_access_level to ENABLED (20) or DISABLED (0) + # depending if container_registry_enabled is true or false. + def update_sql(from_id, to_id) + <<~SQL + UPDATE project_features + SET container_registry_access_level = (CASE p.container_registry_enabled + WHEN true THEN #{ProjectFeature::ENABLED} + WHEN false THEN #{ProjectFeature::DISABLED} + ELSE #{ProjectFeature::DISABLED} + END) + FROM projects p + WHERE project_id = p.id AND + project_id BETWEEN #{from_id} AND #{to_id} + SQL + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb index 52b09e07fd5..dc31f995ae0 100644 --- a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb +++ b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb @@ -61,16 +61,12 @@ module Gitlab private def calculated_uuid - Gitlab::UUID.v5(uuid_components) - end - - def uuid_components - [ - category, - vulnerability_finding.primary_identifier.fingerprint, - vulnerability_finding.location_fingerprint, - project_id - ].join('-') + ::Security::VulnerabilityUUID.generate( + report_type: category, + primary_identifier_fingerprint: vulnerability_finding.primary_identifier.fingerprint, + location_fingerprint: vulnerability_finding.location_fingerprint, + project_id: project_id + ) end def finding_key diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb new file mode 100644 index 00000000000..e352ae71de6 --- /dev/null +++ b/lib/gitlab/background_migration/populate_namespace_statistics.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class creates/updates those namespace statistics + # that haven't been created nor initialized. + # It also updates the related namespace statistics + # This is only required in EE + class PopulateNamespaceStatistics + def perform(group_ids, statistics) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb index 3d3970f50e1..4aff9d1e2c1 100644 --- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -10,7 +10,7 @@ module Gitlab NOP_RELATION.new end - def perform(_scan_ids); end + def perform(*_scan_ids); end end end end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb new file mode 100644 index 00000000000..7b18e617c81 --- /dev/null +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid + # rubocop: disable Gitlab/NamespacedClass + class VulnerabilitiesIdentifier < ActiveRecord::Base + self.table_name = "vulnerability_identifiers" + has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id' + end + + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' + REPORT_TYPES = { + sast: 0, + dependency_scanning: 1, + container_scanning: 2, + dast: 3, + secret_detection: 4, + coverage_fuzzing: 5, + api_fuzzing: 6 + }.with_indifferent_access.freeze + enum report_type: REPORT_TYPES + end + + class CalculateFindingUUID + FINDING_NAMESPACES_IDS = { + development: "a143e9e2-41b3-47bc-9a19-081d089229f4", + test: "a143e9e2-41b3-47bc-9a19-081d089229f4", + staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", + production: "58dc0f06-936c-43b3-93bb-71693f1b6570" + }.freeze + + NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze + PACK_PATTERN = "NnnnnN".freeze + + def self.call(value) + Digest::UUID.uuid_v5(namespace_id, value) + end + + def self.namespace_id + namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym) + # Digest::UUID is broken when using an UUID in namespace_id + # https://github.com/rails/rails/issues/37681#issue-520718028 + namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) + end + end + # rubocop: enable Gitlab/NamespacedClass + + def perform(start_id, end_id) + findings = VulnerabilitiesFinding + .joins(:primary_identifier) + .select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id) + .where(id: start_id..end_id) + + mappings = findings.each_with_object({}) do |finding, hash| + hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) } + end + + ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + end + + private + + def calculate_uuid_v5_for_finding(vulnerability_finding) + return unless vulnerability_finding + + uuid_v5_name_components = { + report_type: vulnerability_finding.report_type, + primary_identifier_fingerprint: vulnerability_finding.fingerprint, + location_fingerprint: vulnerability_finding.location_fingerprint, + project_id: vulnerability_finding.project_id + } + + name = uuid_v5_name_components.values.join('-') + + CalculateFindingUUID.call(name) + end +end diff --git a/lib/gitlab/background_migration/set_default_iteration_cadences.rb b/lib/gitlab/background_migration/set_default_iteration_cadences.rb new file mode 100644 index 00000000000..42f9d33ab71 --- /dev/null +++ b/lib/gitlab/background_migration/set_default_iteration_cadences.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class SetDefaultIterationCadences + class Iteration < ApplicationRecord + self.table_name = 'sprints' + end + + class IterationCadence < ApplicationRecord + self.table_name = 'iterations_cadences' + + include BulkInsertSafe + end + + class Group < ApplicationRecord + self.table_name = 'namespaces' + + self.inheritance_column = :_type_disabled + end + + def perform(*group_ids) + create_iterations_cadences(group_ids) + assign_iterations_cadences(group_ids) + end + + private + + def create_iterations_cadences(group_ids) + groups_with_cadence = IterationCadence.select(:group_id) + + new_cadences = Group.where(id: group_ids).where.not(id: groups_with_cadence).map do |group| + last_iteration = Iteration.where(group_id: group.id).order(:start_date)&.last + + next unless last_iteration + + time = Time.now + IterationCadence.new( + group_id: group.id, + title: "#{group.name} Iterations", + start_date: last_iteration.start_date, + last_run_date: last_iteration.start_date, + automatic: false, + created_at: time, + updated_at: time + ) + end + + IterationCadence.bulk_insert!(new_cadences.compact, skip_duplicates: true) + end + + def assign_iterations_cadences(group_ids) + IterationCadence.where(group_id: group_ids).each do |cadence| + Iteration.where(iterations_cadence_id: nil).where(group_id: cadence.group_id).update_all(iterations_cadence_id: cadence.id) + end + end + end + end +end diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 5382bdab7eb..78a8f39e143 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -19,16 +19,14 @@ module Gitlab commit = Commit.new(commit, project) commit.lazy_author # preload author - sha = commit.sha - if prev_sha != sha + if prev_sha != commit.sha groups << current_group if current_group current_group = { commit: commit, lines: [] } end - line = highlighted_lines[i].html_safe if highlight - current_group[:lines] << line + current_group[:lines] << (highlight ? highlighted_lines[i].html_safe : line) - prev_sha = sha + prev_sha = commit.sha i += 1 end groups << current_group if current_group diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index ad2a718ef67..a8287a97cc3 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -51,7 +51,7 @@ module Gitlab logger.log_timed(LOG_MESSAGES[:protected_branch_checks]) do return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks - if forced_push? + if forced_push? && !ProtectedBranch.allow_force_push?(project, branch_name) raise GitAccess::ForbiddenError, ERROR_MESSAGES[:force_push_protected_branch] end end diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb index b70a6a69b93..38f0b82c8b4 100644 --- a/lib/gitlab/checks/lfs_check.rb +++ b/lib/gitlab/checks/lfs_check.rb @@ -13,6 +13,7 @@ module Gitlab return unless project.lfs_enabled? return if skip_lfs_integrity_check + return if deletion? logger.log_timed(LOG_MESSAGE) do lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) diff --git a/lib/gitlab/ci/artifacts/metrics.rb b/lib/gitlab/ci/artifacts/metrics.rb new file mode 100644 index 00000000000..656f4d2cc13 --- /dev/null +++ b/lib/gitlab/ci/artifacts/metrics.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Artifacts + class Metrics + include Gitlab::Utils::StrongMemoize + + def increment_destroyed_artifacts(size) + destroyed_artifacts_counter.increment({}, size.to_i) + end + + private + + def destroyed_artifacts_counter + strong_memoize(:destroyed_artifacts_counter) do + name = :destroyed_job_artifacts_count_total + comment = 'Counter of destroyed expired job artifacts' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb new file mode 100644 index 00000000000..4fcb5168847 --- /dev/null +++ b/lib/gitlab/ci/build/cache.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Cache + include ::Gitlab::Utils::StrongMemoize + + def initialize(cache, pipeline) + if multiple_cache_per_job? + cache = Array.wrap(cache) + @cache = cache.map do |cache| + Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache) + end + else + @cache = Gitlab::Ci::Pipeline::Seed::Build::Cache + .new(pipeline, cache) + end + end + + def cache_attributes + strong_memoize(:cache_attributes) do + if multiple_cache_per_job? + if @cache.empty? + {} + else + { options: { cache: @cache.map(&:attributes) } } + end + else + @cache.build_attributes + end + end + end + + private + + def multiple_cache_per_job? + strong_memoize(:multiple_cache_per_job) do + ::Gitlab::Ci::Features.multiple_cache_per_job? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index dfd86d3ad72..641aa71fb4e 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -21,7 +21,7 @@ module Gitlab # to the CI variables to evaluate rules before we persist a Build # with the result. We should refactor away the extra Build.new, # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables_hash + stub_build.scoped_variables end end diff --git a/lib/gitlab/ci/build/context/global.rb b/lib/gitlab/ci/build/context/global.rb index fdd3ac358d5..dd0bc54d8b2 100644 --- a/lib/gitlab/ci/build/context/global.rb +++ b/lib/gitlab/ci/build/context/global.rb @@ -19,8 +19,7 @@ module Gitlab # to the CI variables to evaluate workflow:rules # with the result. We should refactor away the extra Build.new, # but be able to get CI Variables directly from the Seed::Build. - stub_build.scoped_variables_hash - .reject { |key, _value| key =~ /\ACI_(JOB|BUILD)/ } + stub_build.scoped_variables.reject { |var| var[:key] =~ /\ACI_(JOB|BUILD)/ } end end diff --git a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb index b6ac06cfb53..76eec2172b1 100644 --- a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb +++ b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb @@ -7,7 +7,7 @@ module Gitlab module Registry class DependencyProxy < GitlabRegistry def url - "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}" + Gitlab.host_with_port end def valid? diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index dbb48a81030..d3f030c3b36 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -99,10 +99,18 @@ module Gitlab initial_config end + def find_sha(project) + branches = project&.repository&.branches || [] + + unless branches.empty? + project.repository.root_ref_sha + end + end + def build_context(project:, sha:, user:, parent_pipeline:) Config::External::Context.new( project: project, - sha: sha || project&.repository&.root_ref_sha, + sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, variables: project&.predefined_variables&.to_runner_variables) diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index e8e2eef281e..73742298628 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Processable ALLOWED_WHEN = %w[on_success on_failure always manual].freeze - ALLOWED_KEYS = %i[trigger].freeze + ALLOWED_KEYS = %i[trigger parallel].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS @@ -48,7 +48,12 @@ module Gitlab inherit: false, metadata: { allowed_needs: %i[job bridge] } - attributes :when, :allow_failure + entry :parallel, Entry::Product::Parallel, + description: 'Parallel configuration for this job.', + inherit: false, + metadata: { allowed_strategies: %i(matrix) } + + attributes :when, :allow_failure, :parallel def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -66,7 +71,8 @@ module Gitlab needs: (needs_value if needs_defined?), ignore: ignored?, when: self.when, - scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage + scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage, + parallel: has_parallel? ? parallel_value : nil ).compact end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index 6b036182706..cf599ce5294 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -7,52 +7,90 @@ module Gitlab ## # Entry that represents a cache configuration # - class Cache < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Configurable - include ::Gitlab::Config::Entry::Validatable - include ::Gitlab::Config::Entry::Attributable - - ALLOWED_KEYS = %i[key untracked paths when policy].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze - DEFAULT_POLICY = 'pull-push' - ALLOWED_WHEN = %w[on_success on_failure always].freeze - DEFAULT_WHEN = 'on_success' - - validations do - validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, - inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, - allow_blank: true - - with_options allow_nil: true do - validates :when, - inclusion: { - in: ALLOWED_WHEN, - message: 'should be on_success, on_failure or always' - } + class Cache < ::Gitlab::Config::Entry::Simplifiable + strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) } + strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) } + + class Caches < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + MULTIPLE_CACHE_LIMIT = 4 + + validations do + validates :config, presence: true + + validate do + unless config.is_a?(Hash) || config.is_a?(Array) + errors.add(:config, 'can only be a Hash or an Array') + end + + if config.is_a?(Array) && config.count > MULTIPLE_CACHE_LIMIT + errors.add(:config, "no more than #{MULTIPLE_CACHE_LIMIT} caches can be created") + end + end + end + + def initialize(*args) + super + + @key = nil + end + + def composable_class + Entry::Cache::Cache end end - entry :key, Entry::Key, - description: 'Cache key used to define a cache affinity.' + class Cache < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[key untracked paths when policy].freeze + ALLOWED_POLICY = %w[pull-push push pull].freeze + DEFAULT_POLICY = 'pull-push' + ALLOWED_WHEN = %w[on_success on_failure always].freeze + DEFAULT_WHEN = 'on_success' + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :policy, + inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' }, + allow_blank: true + + with_options allow_nil: true do + validates :when, + inclusion: { + in: ALLOWED_WHEN, + message: 'should be on_success, on_failure or always' + } + end + end - entry :untracked, ::Gitlab::Config::Entry::Boolean, - description: 'Cache all untracked files.' + entry :key, Entry::Key, + description: 'Cache key used to define a cache affinity.' - entry :paths, Entry::Paths, - description: 'Specify which paths should be cached across builds.' + entry :untracked, ::Gitlab::Config::Entry::Boolean, + description: 'Cache all untracked files.' - attributes :policy, :when + entry :paths, Entry::Paths, + description: 'Specify which paths should be cached across builds.' - def value - result = super + attributes :policy, :when - result[:key] = key_value - result[:policy] = policy || DEFAULT_POLICY - # Use self.when to avoid conflict with reserved word - result[:when] = self.when || DEFAULT_WHEN + def value + result = super + + result[:key] = key_value + result[:policy] = policy || DEFAULT_POLICY + # Use self.when to avoid conflict with reserved word + result[:when] = self.when || DEFAULT_WHEN + + result + end + end - result + class UnknownStrategy < ::Gitlab::Config::Entry::Node end end end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 64e6d48133f..2066e9be3b1 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -10,7 +10,7 @@ module Gitlab class Environment < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes].freeze + ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes deployment_tier].freeze entry :kubernetes, Entry::Kubernetes, description: 'Kubernetes deployment configuration.' @@ -47,6 +47,11 @@ module Gitlab inclusion: { in: %w[start stop prepare], message: 'should be start, stop or prepare' }, allow_nil: true + validates :deployment_tier, + type: String, + inclusion: { in: ::Environment.tiers.keys, message: "must be one of #{::Environment.tiers.keys.join(', ')}" }, + allow_nil: true + validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true validates :auto_stop_in, duration: true, allow_nil: true @@ -85,6 +90,10 @@ module Gitlab value[:auto_stop_in] end + def deployment_tier + value[:deployment_tier] + end + def value case @config when String then { name: @config, action: 'start' } diff --git a/lib/gitlab/ci/config/entry/need.rb b/lib/gitlab/ci/config/entry/need.rb index 46191eca842..b3cf0f9e0fd 100644 --- a/lib/gitlab/ci/config/entry/need.rb +++ b/lib/gitlab/ci/config/entry/need.rb @@ -35,7 +35,14 @@ module Gitlab end def value - { name: @config, artifacts: true } + if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) + { name: @config, + artifacts: true, + optional: false } + else + { name: @config, + artifacts: true } + end end end @@ -43,14 +50,15 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[job artifacts].freeze - attributes :job, :artifacts + ALLOWED_KEYS = %i[job artifacts optional].freeze + attributes :job, :artifacts, :optional validations do validates :config, presence: true validates :config, allowed_keys: ALLOWED_KEYS validates :job, type: String, presence: true validates :artifacts, boolean: true, allow_nil: true + validates :optional, boolean: true, allow_nil: true end def type @@ -58,7 +66,14 @@ module Gitlab end def value - { name: job, artifacts: artifacts || artifacts.nil? } + if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) + { name: job, + artifacts: artifacts || artifacts.nil?, + optional: !!optional } + else + { name: job, + artifacts: artifacts || artifacts.nil? } + end end end diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb index cd9eabbbc66..5c78a8f68c7 100644 --- a/lib/gitlab/ci/config/entry/product/parallel.rb +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -22,6 +22,13 @@ module Gitlab greater_than_or_equal_to: 2, less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT }, allow_nil: true + + validate do + next unless opt(:allowed_strategies) + next if opt(:allowed_strategies).include?(:numeric) + + errors.add(:config, 'cannot use "parallel: <number>".') + end end def value @@ -38,6 +45,13 @@ module Gitlab validations do validates :config, allowed_keys: PERMITTED_KEYS validates :config, required_keys: PERMITTED_KEYS + + validate do + next unless opt(:allowed_strategies) + next if opt(:allowed_strategies).include?(:matrix) + + errors.add(:config, 'cannot use "parallel: matrix".') + end end entry :matrix, Entry::Product::Matrix, diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index d1a366125ef..c811ef211d6 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -38,10 +38,6 @@ module Gitlab ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project) end - def self.project_transactionless_destroy?(project) - Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false) - end - def self.trace_overwrite? ::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false) end @@ -55,14 +51,6 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.ci_pipeline_editor_page_enabled?(project) - ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) - end - - def self.rules_variables_enabled?(project) - ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) - end - def self.validate_build_dependencies?(project) ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && ::Feature.disabled?(:ci_validate_build_dependencies_override, project) @@ -76,8 +64,12 @@ module Gitlab ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) end - def self.use_coverage_data_new_finder?(record) - ::Feature.enabled?(:coverage_data_new_finder, record, default_enabled: :yaml) + def self.multiple_cache_per_job? + ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) + end + + def self.ci_commit_pipeline_mini_graph_vue_enabled?(project) + ::Feature.enabled?(:ci_commit_pipeline_mini_graph_vue, project, default_enabled: :yaml) end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 0870c74053a..af06e124736 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -60,7 +60,7 @@ module Gitlab ref_protected: build.protected.to_s } - if include_environment_claims? + if environment.present? fields.merge!( environment: environment.name, environment_protected: environment_protected?.to_s @@ -119,10 +119,6 @@ module Gitlab def environment_protected? false # Overridden in EE end - - def include_environment_claims? - Feature.enabled?(:ci_jwt_include_environment) && environment.present? - end end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 364e67db02b..4a7c11ee26e 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -21,7 +21,7 @@ module Gitlab def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user - @sha = sha || project.repository.commit.sha + @sha = sha || project.repository.commit&.sha end def validate(content, dry_run: false) diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index f0214bb4e38..1c0dfbdbee3 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -10,7 +10,7 @@ module Gitlab def perform! return unless project.auto_cancel_pending_pipelines? - Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables| + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables| cancelables.find_each do |cancelable| cancelable.auto_cancel_running(pipeline) end diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 0e81e1bd34c..5f3310dd668 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -7,9 +7,9 @@ module Gitlab class Statement StatementError = Class.new(Expression::ExpressionError) - def initialize(statement, variables = {}) + def initialize(statement, variables = nil) @lexer = Expression::Lexer.new(statement) - @variables = variables.with_indifferent_access + @variables = variables&.to_hash end def parse_tree diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 3770bb4b328..11b01822e4b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -28,8 +28,8 @@ module Gitlab .fabricate(attributes.delete(:except)) @rules = Gitlab::Ci::Build::Rules .new(attributes.delete(:rules), default_when: 'on_success') - @cache = Seed::Build::Cache - .new(pipeline, attributes.delete(:cache)) + @cache = Gitlab::Ci::Build::Cache + .new(attributes.delete(:cache), pipeline) end def name @@ -52,7 +52,7 @@ module Gitlab return unless included? strong_memoize(:errors) do - needs_errors + [needs_errors, variable_expansion_errors].compact.flatten end end @@ -61,7 +61,7 @@ module Gitlab .deep_merge(pipeline_attributes) .deep_merge(rules_attributes) .deep_merge(allow_failure_criteria_attributes) - .deep_merge(cache_attributes) + .deep_merge(@cache.cache_attributes) end def bridge? @@ -141,6 +141,8 @@ module Gitlab end @needs_attributes.flat_map do |need| + next if ::Feature.enabled?(:ci_needs_optional, default_enabled: :yaml) && need[:optional] + result = @previous_stages.any? do |stage| stage.seeds_names.include?(need[:name]) end @@ -153,6 +155,12 @@ module Gitlab @pipeline.project.actual_limits.ci_needs_size_limit end + def variable_expansion_errors + expanded_collection = evaluate_context.variables.sort_and_expand_all(@pipeline.project) + errors = expanded_collection.errors + ["#{name}: #{errors}"] if errors + end + def pipeline_attributes { pipeline: @pipeline, @@ -169,15 +177,11 @@ module Gitlab strong_memoize(:rules_attributes) do next {} unless @using_rules - if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) - rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( - @seed_attributes[:yaml_variables], rules_result.variables - ) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) - rules_result.build_attributes.merge(yaml_variables: rules_variables_result) - else - rules_result.build_attributes - end + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) end end @@ -193,12 +197,6 @@ module Gitlab end end - def cache_attributes - strong_memoize(:cache_attributes) do - @cache.build_attributes - end - end - # If a job uses `allow_failure:exit_codes` and `rules:allow_failure` # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 8d6fe13c3b9..78ffaaa7e81 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -18,18 +18,18 @@ module Gitlab raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end - def build_attributes + def attributes { - options: { - cache: { - key: key_string, - paths: @paths, - policy: @policy, - untracked: @untracked, - when: @when - }.compact.presence - }.compact - } + key: key_string, + paths: @paths, + policy: @policy, + untracked: @untracked, + when: @when + }.compact + end + + def build_attributes + { options: { cache: attributes.presence }.compact } end private diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 5dff0788ec9..c8795840e5f 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -13,7 +13,9 @@ module Gitlab def to_resource environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| + # Initialize the attributes at creation environment.auto_stop_in = auto_stop_in + environment.tier = deployment_tier end end @@ -27,6 +29,10 @@ module Gitlab job.environment_auto_stop_in end + def deployment_tier + job.environment_deployment_tier + end + def expanded_environment_name job.expanded_environment_name end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb new file mode 100644 index 00000000000..5398c19e536 --- /dev/null +++ b/lib/gitlab/ci/queue/metrics.rb @@ -0,0 +1,210 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Queue + class Metrics + extend Gitlab::Utils::StrongMemoize + + QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze + QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze + QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze + QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze + QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + + METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' + DEFAULT_METRICS_SHARD = 'default' + JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze + + OPERATION_COUNTERS = [ + :build_can_pick, + :build_not_pick, + :build_not_pending, + :build_temporary_locked, + :build_conflict_lock, + :build_conflict_exception, + :build_conflict_transition, + :queue_attempt, + :queue_conflict, + :queue_iteration, + :queue_depth_limit, + :queue_replication_lag, + :runner_pre_assign_checks_failed, + :runner_pre_assign_checks_success, + :runner_queue_tick + ].to_set.freeze + + QUEUE_DEPTH_HISTOGRAMS = [ + :found, + :not_found, + :conflict + ].to_set.freeze + + attr_reader :runner + + def initialize(runner) + @runner = runner + end + + def register_failure + self.class.failed_attempt_counter.increment + self.class.attempt_counter.increment + end + + def register_success(job) + labels = { shared_runner: runner.instance_type?, + jobs_running_for_project: jobs_running_for_project(job), + shard: DEFAULT_METRICS_SHARD } + + if runner.instance_type? + shard = runner.tag_list.sort.find { |name| name.starts_with?(METRICS_SHARD_TAG_PREFIX) } + labels[:shard] = shard.gsub(METRICS_SHARD_TAG_PREFIX, '') if shard + end + + self.class.job_queue_duration_seconds.observe(labels, Time.current - job.queued_at) unless job.queued_at.nil? + self.class.attempt_counter.increment + end + + # rubocop: disable CodeReuse/ActiveRecord + def jobs_running_for_project(job) + return '+Inf' unless runner.instance_type? + + # excluding currently started job + running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type) + .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 + running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" + end + # rubocop: enable CodeReuse/ActiveRecord + + def increment_queue_operation(operation) + if !Rails.env.production? && !OPERATION_COUNTERS.include?(operation) + raise ArgumentError, "unknown queue operation: #{operation}" + end + + self.class.queue_operations_total.increment(operation: operation) + end + + def observe_queue_depth(queue, size) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue) + raise ArgumentError, "unknown queue depth label: #{queue}" + end + + self.class.queue_depth_total.observe({ queue: queue }, size.to_f) + end + + def observe_queue_size(size_proc) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + self.class.queue_size_total.observe({}, size_proc.call.to_f) + end + + def observe_queue_time + start_time = ::Gitlab::Metrics::System.monotonic_time + + result = yield + + return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + seconds = ::Gitlab::Metrics::System.monotonic_time - start_time + self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f) + + result + end + + def self.observe_active_runners(runners_proc) + return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) + + queue_active_runners_total.observe({}, runners_proc.call.to_f) + end + + def self.increment_runner_tick(runner) + self.new(runner).increment_queue_operation(:runner_queue_tick) + end + + def self.failed_attempt_counter + strong_memoize(:failed_attempt_counter) do + name = :job_register_attempts_failed_total + comment = 'Counts the times a runner tries to register a job' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.attempt_counter + strong_memoize(:attempt_counter) do + name = :job_register_attempts_total + comment = 'Counts the times a runner tries to register a job' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.job_queue_duration_seconds + strong_memoize(:job_queue_duration_seconds) do + name = :job_queue_duration_seconds + comment = 'Request handling execution time' + buckets = QUEUE_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_operations_total + strong_memoize(:queue_operations_total) do + name = :gitlab_ci_queue_operations_total + comment = 'Counts all the operations that are happening inside a queue' + + Gitlab::Metrics.counter(name, comment) + end + end + + def self.queue_depth_total + strong_memoize(:queue_depth_total) do + name = :gitlab_ci_queue_depth_total + comment = 'Size of a CI/CD builds queue in relation to the operation result' + buckets = QUEUE_DEPTH_TOTAL_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_size_total + strong_memoize(:queue_size_total) do + name = :gitlab_ci_queue_size_total + comment = 'Size of initialized CI/CD builds queue' + buckets = QUEUE_SIZE_TOTAL_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_iteration_duration_seconds + strong_memoize(:queue_iteration_duration_seconds) do + name = :gitlab_ci_queue_iteration_duration_seconds + comment = 'Time it takes to find a build in CI/CD queue' + buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_active_runners_total + strong_memoize(:queue_active_runners_total) do + name = :gitlab_ci_queue_active_runners_total + comment = 'The amount of active runners that can process queue in a project' + buckets = QUEUE_ACTIVE_RUNNERS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb index 88e02cd9004..10748b8ca02 100644 --- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb +++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb @@ -5,7 +5,7 @@ module Gitlab module Reports class CodequalityReportsComparer < ReportsComparer def initialize(base_report, head_report) - @base_report = base_report || CodequalityReports.new + @base_report = base_report @head_report = head_report end @@ -15,12 +15,16 @@ module Gitlab def existing_errors strong_memoize(:existing_errors) do + next [] if not_found? + base_report.all_degradations & head_report.all_degradations end end def new_errors strong_memoize(:new_errors) do + next [] if not_found? + fingerprints = head_report.degradations.keys - base_report.degradations.keys head_report.degradations.fetch_values(*fingerprints) end @@ -28,6 +32,8 @@ module Gitlab def resolved_errors strong_memoize(:resolved_errors) do + next [] if not_found? + fingerprints = base_report.degradations.keys - head_report.degradations.keys base_report.degradations.fetch_values(*fingerprints) end diff --git a/lib/gitlab/ci/reports/reports_comparer.rb b/lib/gitlab/ci/reports/reports_comparer.rb index d413d3a74f6..16a7f6478b7 100644 --- a/lib/gitlab/ci/reports/reports_comparer.rb +++ b/lib/gitlab/ci/reports/reports_comparer.rb @@ -8,6 +8,7 @@ module Gitlab STATUS_SUCCESS = 'success' STATUS_FAILED = 'failed' + STATUS_NOT_FOUND = 'not_found' attr_reader :base_report, :head_report @@ -17,7 +18,13 @@ module Gitlab end def status - success? ? STATUS_SUCCESS : STATUS_FAILED + if base_report.nil? || head_report.nil? + STATUS_NOT_FOUND + elsif success? + STATUS_SUCCESS + else + STATUS_FAILED + end end def success? @@ -47,6 +54,10 @@ module Gitlab def total_count existing_errors.size + new_errors.size end + + def not_found? + status == STATUS_NOT_FOUND + end end end end diff --git a/lib/gitlab/ci/reports/test_suite_summary.rb b/lib/gitlab/ci/reports/test_suite_summary.rb index 32b06d0ad49..461aefc6fa9 100644 --- a/lib/gitlab/ci/reports/test_suite_summary.rb +++ b/lib/gitlab/ci/reports/test_suite_summary.rb @@ -4,6 +4,8 @@ module Gitlab module Ci module Reports class TestSuiteSummary + include Gitlab::Utils::StrongMemoize + def initialize(build_report_results) @build_report_results = build_report_results end @@ -42,6 +44,12 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord + def suite_error + strong_memoize(:suite_error) do + @build_report_results.map(&:suite_error).compact.first + end + end + def to_h { time: total_time, @@ -49,7 +57,8 @@ module Gitlab success: success_count, failed: failed_count, skipped: skipped_count, - error: error_count + error: error_count, + suite_error: suite_error } end end diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 9a4f5644f7d..5368e020a50 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,7 +7,10 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - def initialize(all_statuses, with_allow_failure: true, dag: false) + # + # The parameter `project` is only used for the feature flag check, and will be removed with + # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 + def initialize(all_statuses, with_allow_failure: true, dag: false, project: nil) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -16,6 +19,7 @@ module Gitlab @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag + @project = project consume_all_statuses(all_statuses) end @@ -32,7 +36,7 @@ module Gitlab return if none? strong_memoize(:status) do - if @dag && any_of?(:skipped) + if @dag && any_skipped_or_ignored? # The DAG job is skipped if one of the needs does not run at all. 'skipped' elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings) @@ -90,6 +94,14 @@ module Gitlab matching == @status_set.size end + def any_skipped_or_ignored? + if ::Feature.enabled?(:ci_fix_pipeline_status_for_dag_needs_manual, @project, default_enabled: :yaml) + any_of?(:skipped) || any_of?(:ignored) + else + any_of?(:skipped) + end + end + def consume_all_statuses(all_statuses) columns = [] columns[@status_key] = :status diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index 5f17c93b853..d879e27dfcb 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -20,11 +20,6 @@ stages: - functional - deploy -foodcritic: - stage: lint - script: - - chef exec foodcritic . - cookstyle: stage: lint script: diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index daed75a42ee..fd6c51ea350 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.22" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.23" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index c4e194bd658..29edada4041 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -23,7 +23,7 @@ review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -44,7 +44,7 @@ stop_review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -73,7 +73,7 @@ staging: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$STAGING_ENABLED' @@ -98,7 +98,7 @@ canary: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CANARY_ENABLED' when: manual @@ -136,7 +136,7 @@ production: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' production_manual: <<: *production_template @@ -148,12 +148,12 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $STAGING_ENABLED' when: manual - - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CANARY_ENABLED' when: manual -# This job implements incremental rollout on for every push to `master`. +# This job implements incremental rollout on for every push to the default branch. .rollout: &rollout_template extends: .auto-deploy @@ -184,7 +184,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' @@ -197,7 +197,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: delayed diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e5b40e5f49a..530ab1d0f99 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" dependencies: [] review: @@ -23,7 +23,7 @@ review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -44,7 +44,7 @@ stop_review: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$REVIEW_DISABLED' when: never @@ -73,7 +73,7 @@ staging: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$STAGING_ENABLED' @@ -98,7 +98,7 @@ canary: rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CANARY_ENABLED' when: manual @@ -135,7 +135,7 @@ production: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' production_manual: <<: *production_template @@ -147,12 +147,12 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE' when: never - - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $STAGING_ENABLED' when: manual - - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CANARY_ENABLED' when: manual -# This job implements incremental rollout on for every push to `master`. +# This job implements incremental rollout on for every push to the default branch. .rollout: &rollout_template extends: .auto-deploy @@ -181,7 +181,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never # $INCREMENTAL_ROLLOUT_ENABLED is for compatibility with pre-GitLab 11.4 syntax - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED' @@ -194,7 +194,7 @@ production_manual: when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"' when: delayed diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml index ed2172ef7f5..7efbcab221b 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml @@ -20,7 +20,7 @@ review_ec2: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -34,6 +34,6 @@ production_ec2: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 0289ba1c473..332c58c8695 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -46,7 +46,7 @@ review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -62,7 +62,7 @@ stop_review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' when: manual @@ -81,7 +81,7 @@ review_fargate: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -97,7 +97,7 @@ stop_review_fargate: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' when: manual @@ -109,7 +109,7 @@ production_ecs: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' @@ -120,6 +120,6 @@ production_fargate: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH' when: never - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 32d4e07d398..be0efc9180b 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -10,11 +10,11 @@ # Below is the template to run your tests in Julia .test_template: &test_definition - # Uncomment below if you would like to run the tests on specific references - # only, such as the branches `master`, `development`, etc. - # only: - # - master - # - development + # Uncomment below (and adjust as needed) to run the tests for specific references + # only, such as the default branch, a `development` branch, and so on: + # rules: + # - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + # - if: $CI_COMMIT_BRANCH == "development" script: # Let's run the tests. Substitute `coverage = false` below, if you do not # want coverage results. @@ -63,8 +63,8 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # WARNING: This template is using the `julia` images from [Docker # Hub][3]. One can use custom Julia images and/or the official ones found diff --git a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml index a4aed36889e..e4ed7fadfaa 100644 --- a/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/LaTeX.gitlab-ci.yml @@ -1,11 +1,20 @@ -# use docker image with latex preinstalled -# since there is no official latex image, use https://github.com/blang/latex-docker -# possible alternative: https://github.com/natlownes/docker-latex -image: blang/latex +--- +variables: + # Feel free to choose the image that suits you best. + # blang/latex:latest ... Former image used in this template. No longer maintained by author. + # listx/texlive:2020 ... The default, referring to TexLive 2020. Current at least to 2021-02-02. + + # Additional alternatives with high Docker pull counts: + # thomasweise/docker-texlive-full + # thomasweise/texlive + # adnrv/texlive + LATEX_IMAGE: listx/texlive:2020 build: + image: $LATEX_IMAGE script: - latexmk -pdf + artifacts: paths: - "*.pdf" diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 8f64da24410..97d0f611f47 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -33,7 +33,8 @@ cache: script: - 'mvn $MAVEN_CLI_OPTS verify' except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Verify merge requests using JDK8 verify:jdk8: @@ -51,4 +52,5 @@ deploy:jdk8: fi - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 10fb6be6c39..36fe27f54c2 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -24,8 +24,8 @@ before_script: release: stage: deploy - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: paths: - build/release/MyProject.exe diff --git a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml index 0a3cf3dcf77..0b03ba6c3d8 100644 --- a/lib/gitlab/ci/templates/Packer.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Packer.gitlab-ci.yml @@ -23,6 +23,6 @@ build: environment: production script: - find . -maxdepth 1 -name '*.json' -print0 | xargs -t0n1 packer build - when: manual - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index d2dd3fbfb75..90cd8472916 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index ba422c08614..7435afef572 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -9,5 +9,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 3a6eac63892..708c5063cc6 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -13,5 +13,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index 92f25280c6e..694446dd6c9 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index 0e206423fa5..a2fd6620909 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index d91a8d7421f..fd75e47e899 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -13,5 +13,5 @@ pages: paths: - node_modules key: project - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index 975cb3b7698..a6a605e35f0 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -10,7 +10,8 @@ test: script: - hugo except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: script: @@ -19,4 +20,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index 7a441a2f70f..1be2f4bad76 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -11,7 +11,8 @@ test: - pip install hyde - hyde gen except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: stage: deploy @@ -22,4 +23,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index f2f92fe0704..01e063c50ad 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -18,7 +18,8 @@ test: paths: - test except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: stage: deploy @@ -28,4 +29,5 @@ pages: paths: - public only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index 2d26b86a328..e39aa8a2063 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -33,5 +33,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index 93ab8e0be0d..13d3089f4fa 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index 6524405133a..e65cf3928f2 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -12,5 +12,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index 462b4737c4e..377fd8c396e 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -12,7 +12,8 @@ test: - bundle install --path vendor - bundle exec middleman build except: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH pages: script: @@ -23,5 +24,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index b512f8d77e9..89281b41b66 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -8,5 +8,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index 4318aadcaa6..8fd4702b90d 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -11,5 +11,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 8fd08ea7995..9fa8b07f7cb 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -25,5 +25,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 00b8b94b574..abce887d45b 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -47,5 +47,5 @@ pages: artifacts: paths: - public - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 135f0df99fe..654a03ced5f 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -45,13 +45,10 @@ apifuzzer_fuzz: entrypoint: ["/bin/bash", "-l", "-c"] variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_API: http://localhost:80 FUZZAPI_NEW_REPORT: 1 + FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log TZ: America/Los_Angeles - services: - - name: $FUZZAPI_IMAGE - alias: apifuzzer - entrypoint: ["dotnet", "/peach/Peach.Web.dll"] allow_failure: true rules: - if: $FUZZAPI_D_TARGET_IMAGE @@ -80,17 +77,26 @@ apifuzzer_fuzz: # Make sure asset path exists - mkdir -p $FUZZAPI_REPORT_ASSET_PATH # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & + - APISEC_PID=$! + # # Start scanning - worker-entry # # Run user provided post-script - sh -c "$FUZZAPI_POST_SCRIPT" # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # artifacts: when: always paths: - $FUZZAPI_REPORT_ASSET_PATH - $FUZZAPI_REPORT + - $FUZZAPI_LOG_SCANNER reports: api_fuzzing: $FUZZAPI_REPORT @@ -172,6 +178,7 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ -e FUZZAPI_TARGET_URL \ -e FUZZAPI_OVERRIDES_FILE \ -e FUZZAPI_OVERRIDES_ENV \ @@ -214,6 +221,7 @@ apifuzzer_fuzz_dnd: -e FUZZAPI_HAR \ -e FUZZAPI_OPENAPI \ -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ -e FUZZAPI_TARGET_URL \ -e FUZZAPI_OVERRIDES_FILE \ -e FUZZAPI_OVERRIDES_ENV \ diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 5ea2363a0c5..64001c2828a 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -18,6 +18,9 @@ container_scanning: # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details GIT_STRATEGY: none + # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION allow_failure: true services: diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index b534dad9593..3039d64514b 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -38,6 +38,9 @@ gemnasium-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -61,6 +64,9 @@ gemnasium-maven-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -79,6 +85,9 @@ gemnasium-python-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -104,6 +113,9 @@ bundler-audit-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED @@ -119,6 +131,9 @@ retire-js-dependency_scanning: image: name: "$DS_ANALYZER_IMAGE" variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" rules: - if: $DEPENDENCY_SCANNING_DISABLED diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 828352743b4..9693a4fbca2 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -41,6 +41,9 @@ bandit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -57,6 +60,9 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -74,6 +80,9 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -94,6 +103,9 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -111,6 +123,9 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -126,6 +141,9 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -140,11 +158,16 @@ gosec-sast: mobsf-android-sast: extends: .sast-analyzer services: - - name: opensecurity/mobile-security-framework-mobsf:latest + # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile + # Unfortunately, we need to keep track of mobsf version in 2 different places for now. + - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" MOBSF_API_KEY: key rules: @@ -161,11 +184,16 @@ mobsf-android-sast: mobsf-ios-sast: extends: .sast-analyzer services: - - name: opensecurity/mobile-security-framework-mobsf:latest + # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile + # Unfortunately, we need to keep track of mobsf version in 2 different places for now. + - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" MOBSF_API_KEY: key rules: @@ -184,6 +212,9 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -200,6 +231,9 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -216,6 +250,9 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -232,6 +269,9 @@ security-code-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -249,6 +289,9 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" rules: - if: $SAST_DISABLED @@ -266,6 +309,9 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_DISABLED @@ -282,6 +328,9 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: + # SAST_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index d2a6fa06dd8..c255fb4707a 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection # # Configure the scanning tool through the environment variables. -# List of the variables: https://gitlab.com/gitlab-org/security-products/secret_detection#available-variables +# List of the variables: https://docs.gitlab.com/ee/user/application_security/secret_detection/#available-variables # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index ffed7a0fec2..cca0ba5d38e 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -21,8 +21,8 @@ archive_project: script: - xcodebuild clean archive -archivePath build/ProjectName -scheme SchemeName - xcodebuild -exportArchive -exportFormat ipa -archivePath "build/ProjectName.xcarchive" -exportPath "build/ProjectName.ipa" -exportProvisioningProfile "ProvisioningProfileName" - only: - - master + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH artifacts: paths: - build/ProjectName.ipa diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index c2db0fc44f1..200388a274c 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -52,7 +52,8 @@ cache: - gitlab-terraform apply when: manual only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH .destroy: &destroy stage: cleanup diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index f7bbb58df7e..e2a8af9c26b 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -6,14 +6,22 @@ module Gitlab class Collection include Enumerable - def initialize(variables = []) + attr_reader :errors + + def initialize(variables = [], errors = nil) @variables = [] + @variables_by_key = {} + @errors = errors variables.each { |variable| self.append(variable) } end def append(resource) - tap { @variables.append(Collection::Item.fabricate(resource)) } + item = Collection::Item.fabricate(resource) + @variables.append(item) + @variables_by_key[item[:key]] = item + + self end def concat(resources) @@ -33,15 +41,67 @@ module Gitlab end end + def [](key) + @variables_by_key[key] + end + + def size + @variables.size + end + def to_runner_variables self.map(&:to_runner_variable) end def to_hash self.to_runner_variables - .map { |env| [env.fetch(:key), env.fetch(:value)] } - .to_h.with_indifferent_access + .to_h { |env| [env.fetch(:key), env.fetch(:value)] } + .with_indifferent_access end + + def reject(&block) + Collection.new(@variables.reject(&block)) + end + + def expand_value(value, keep_undefined: false) + value.gsub(ExpandVariables::VARIABLES_REGEXP) do + match = Regexp.last_match + result = @variables_by_key[match[1] || match[2]]&.value + result ||= match[0] if keep_undefined + result + end + end + + def sort_and_expand_all(project, keep_undefined: false) + return self if Feature.disabled?(:variable_inside_variable, project) + + sorted = Sort.new(self) + return self.class.new(self, sorted.errors) unless sorted.valid? + + new_collection = self.class.new + + sorted.tsort.each do |item| + unless item.depends_on + new_collection.append(item) + next + end + + # expand variables as they are added + variable = item.to_runner_variable + variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined) + new_collection.append(variable) + end + + new_collection + end + + def to_s + "#{@variables_by_key.keys}, @errors='#{@errors}'" + end + + protected + + attr_reader :variables end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 84a9280e507..77da2c4cb91 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -5,13 +5,21 @@ module Gitlab module Variables class Collection class Item - def initialize(key:, value:, public: true, file: false, masked: false) + include Gitlab::Utils::StrongMemoize + + def initialize(key:, value:, public: true, file: false, masked: false, raw: false) raise ArgumentError, "`#{key}` must be of type String or nil value, while it was: #{value.class}" unless value.is_a?(String) || value.nil? - @variable = { - key: key, value: value, public: public, file: file, masked: masked - } + @variable = { key: key, value: value, public: public, file: file, masked: masked, raw: raw } + end + + def value + @variable.fetch(:value) + end + + def raw + @variable.fetch(:raw) end def [](key) @@ -22,6 +30,16 @@ module Gitlab to_runner_variable == self.class.fabricate(other).to_runner_variable end + def depends_on + strong_memoize(:depends_on) do + next if raw + + next unless ExpandVariables.possible_var_reference?(value) + + value.scan(ExpandVariables::VARIABLES_REGEXP).map(&:first) + end + end + ## # If `file: true` has been provided we expose it, otherwise we # don't expose `file` attribute at all (stems from what the runner @@ -29,7 +47,7 @@ module Gitlab # def to_runner_variable @variable.reject do |hash_key, hash_value| - hash_key == :file && hash_value == false + (hash_key == :file || hash_key == :raw) && hash_value == false end end @@ -45,6 +63,12 @@ module Gitlab raise ArgumentError, "Unknown `#{resource.class}` variable resource!" end end + + def to_s + return to_runner_variable.to_s unless depends_on + + "#{to_runner_variable}, depends_on=#{depends_on}" + end end end end diff --git a/lib/gitlab/ci/variables/collection/sort.rb b/lib/gitlab/ci/variables/collection/sort.rb new file mode 100644 index 00000000000..90a929b8a07 --- /dev/null +++ b/lib/gitlab/ci/variables/collection/sort.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Collection + class Sort + include TSort + include Gitlab::Utils::StrongMemoize + + def initialize(collection) + raise(ArgumentError, "A Gitlab::Ci::Variables::Collection object was expected") unless + collection.is_a?(Collection) + + @collection = collection + end + + def valid? + errors.nil? + end + + # errors sorts an array of variables, ignoring unknown variable references, + # and returning an error string if a circular variable reference is found + def errors + strong_memoize(:errors) do + # Check for cyclic dependencies and build error message in that case + cyclic_vars = each_strongly_connected_component.filter_map do |component| + component.map { |v| v[:key] }.inspect if component.size > 1 + end + + "circular variable reference detected: #{cyclic_vars.join(', ')}" if cyclic_vars.any? + end + end + + private + + def tsort_each_node(&block) + @collection.each(&block) + end + + def tsort_each_child(var_item, &block) + depends_on = var_item.depends_on + return unless depends_on + + depends_on.filter_map { |var_ref_name| @collection[var_ref_name] }.each(&block) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb deleted file mode 100644 index e641df10462..00000000000 --- a/lib/gitlab/ci/variables/collection/sorted.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Variables - class Collection - class Sorted - include TSort - include Gitlab::Utils::StrongMemoize - - def initialize(variables, project) - @variables = variables - @project = project - end - - def valid? - errors.nil? - end - - # errors sorts an array of variables, ignoring unknown variable references, - # and returning an error string if a circular variable reference is found - def errors - return if Feature.disabled?(:variable_inside_variable, @project) - - strong_memoize(:errors) do - # Check for cyclic dependencies and build error message in that case - errors = each_strongly_connected_component.filter_map do |component| - component.map { |v| v[:key] }.inspect if component.size > 1 - end - - "circular variable reference detected: #{errors.join(', ')}" if errors.any? - end - end - - # sort sorts an array of variables, ignoring unknown variable references. - # If a circular variable reference is found, the original array is returned - def sort - return @variables if Feature.disabled?(:variable_inside_variable, @project) - return @variables if errors - - tsort - end - - private - - def tsort_each_node(&block) - @variables.each(&block) - end - - def tsort_each_child(variable, &block) - each_variable_reference(variable[:value], &block) - end - - def input_vars - strong_memoize(:input_vars) do - @variables.index_by { |env| env.fetch(:key) } - end - end - - def walk_references(value) - return unless ExpandVariables.possible_var_reference?(value) - - value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref| - yield(input_vars, var_ref.first) - end - end - - def each_variable_reference(value) - walk_references(value) do |vars_hash, ref_var_name| - variable = vars_hash.dig(ref_var_name) - yield variable if variable - end - end - end - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index aaa2554dbfa..c247ef0d2a7 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -15,16 +15,10 @@ module Gitlab private def deployments_count - if Feature.enabled?(:query_deploymenys_via_finished_at_in_vsa, default_enabled: :yaml) - DeploymentsFinder - .new(project: @project, finished_after: @from, finished_before: @to, status: :success) - .execute - .count - else - query = @project.deployments.success.where("created_at >= ?", @from) - query = query.where("created_at <= ?", @to) if @to - query.count - end + DeploymentsFinder + .new(project: @project, finished_after: @from, finished_before: @to, status: :success) + .execute + .count end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index e17bd25e57e..c4af5e6608e 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -62,7 +62,9 @@ module Gitlab git_http_url: project.http_url_to_repo, git_ssh_url: project.ssh_url_to_repo, visibility_level: project.visibility_level - } + }, + + environment: build_environment(build) } data @@ -86,6 +88,15 @@ module Gitlab tags: runner.tags&.map(&:name) } end + + def build_environment(build) + return unless build.has_environment? + + { + name: build.expanded_environment_name, + action: build.environment_action + } + end end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 3036bc57ca5..7fd1b9cd228 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -67,7 +67,8 @@ module Gitlab artifacts_file: { filename: build.artifacts_file&.filename, size: build.artifacts_size - } + }, + environment: environment_hook_attrs(build) } end @@ -80,6 +81,15 @@ module Gitlab tags: runner.tags&.map(&:name) } end + + def environment_hook_attrs(build) + return unless build.has_environment? + + { + name: build.expanded_environment_name, + action: build.environment_action + } + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 45d271a2fd4..3dc8976d8c5 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -256,11 +256,28 @@ module Gitlab row['system_identifier'] end + # @param [ActiveRecord::Connection] ar_connection + # @return [String] def self.get_write_location(ar_connection) - row = ar_connection - .select_all("SELECT pg_current_wal_insert_lsn()::text AS location") - .first - + use_new_load_balancer_query = Gitlab::Utils.to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: false) + + sql = if use_new_load_balancer_query + <<~NEWSQL + SELECT CASE + WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders()) + THEN pg_last_wal_replay_lsn()::text + WHEN pg_is_in_recovery() = false + THEN pg_current_wal_insert_lsn()::text + ELSE NULL + END AS location; + NEWSQL + else + <<~SQL + SELECT pg_current_wal_insert_lsn()::text AS location + SQL + end + + row = ar_connection.select_all(sql).first row['location'] if row end @@ -313,28 +330,18 @@ module Gitlab ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics) end - # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to - # record transaction durations. - def self.observe_transaction_duration(duration_seconds) - if current_transaction = ::Gitlab::Metrics::Transaction.current - current_transaction.observe(:gitlab_database_transaction_seconds, duration_seconds) do - docstring "Time spent in database transactions, in seconds" - end - end - rescue Prometheus::Client::LabelSetValidator::LabelSetError => err - # Ensure that errors in recording these metrics don't affect the operation of the application - Gitlab::AppLogger.error("Unable to observe database transaction duration: #{err}") - end - # MonkeyPatch for ActiveRecord::Base for adding observability module ActiveRecordBaseTransactionMetrics - # A monkeypatch over ActiveRecord::Base.transaction. - # It provides observability into transactional methods. - def transaction(options = {}, &block) - start_time = Gitlab::Metrics::System.monotonic_time - super(options, &block) - ensure - Gitlab::Database.observe_transaction_duration(Gitlab::Metrics::System.monotonic_time - start_time) + extend ActiveSupport::Concern + + class_methods do + # A monkeypatch over ActiveRecord::Base.transaction. + # It provides observability into transactional methods. + def transaction(**options, &block) + ActiveSupport::Notifications.instrument('transaction.active_record', { connection: connection }) do + super(**options, &block) + end + end end end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb new file mode 100644 index 00000000000..3b624df2bfd --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + self.table_name = :batched_background_migration_jobs + + belongs_to :batched_migration, foreign_key: :batched_background_migration_id + + enum status: { + pending: 0, + running: 1, + failed: 2, + succeeded: 3 + } + + delegate :aborted?, :job_class, :table_name, :column_name, :job_arguments, + to: :batched_migration, prefix: :migration + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb new file mode 100644 index 00000000000..0c9add9b355 --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' + BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies".freeze + + self.table_name = :batched_background_migrations + + has_many :batched_jobs, foreign_key: :batched_background_migration_id + has_one :last_job, -> { order(id: :desc) }, + class_name: 'Gitlab::Database::BackgroundMigration::BatchedJob', + foreign_key: :batched_background_migration_id + + scope :queue_order, -> { order(id: :asc) } + + enum status: { + paused: 0, + active: 1, + aborted: 2, + finished: 3 + } + + def interval_elapsed? + last_job.nil? || last_job.created_at <= Time.current - interval + end + + def create_batched_job!(min, max) + batched_jobs.create!(min_value: min, max_value: max, batch_size: batch_size, sub_batch_size: sub_batch_size) + end + + def next_min_value + last_job&.max_value&.next || min_value + end + + def job_class + "#{JOB_CLASS_MODULE}::#{job_class_name}".constantize + end + + def batch_class + "#{BATCH_CLASS_MODULE}::#{batch_class_name}".constantize + end + + def job_class_name=(class_name) + write_attribute(:job_class_name, class_name.demodulize) + end + + def batch_class_name=(class_name) + write_attribute(:batch_class_name, class_name.demodulize) + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb new file mode 100644 index 00000000000..299bd992197 --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedMigrationWrapper + def perform(batch_tracking_record) + start_tracking_execution(batch_tracking_record) + + execute_batch(batch_tracking_record) + + batch_tracking_record.status = :succeeded + rescue => e + batch_tracking_record.status = :failed + + raise e + ensure + finish_tracking_execution(batch_tracking_record) + end + + private + + def start_tracking_execution(tracking_record) + tracking_record.update!(attempts: tracking_record.attempts + 1, status: :running, started_at: Time.current) + end + + def execute_batch(tracking_record) + job_instance = tracking_record.migration_job_class.new + + job_instance.perform( + tracking_record.min_value, + tracking_record.max_value, + tracking_record.migration_table_name, + tracking_record.migration_column_name, + tracking_record.sub_batch_size, + *tracking_record.migration_job_arguments) + end + + def finish_tracking_execution(tracking_record) + tracking_record.finished_at = Time.current + tracking_record.save! + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/scheduler.rb new file mode 100644 index 00000000000..5f8a5ec06a5 --- /dev/null +++ b/lib/gitlab/database/background_migration/scheduler.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class Scheduler + def perform(migration_wrapper: BatchedMigrationWrapper.new) + active_migration = BatchedMigration.active.queue_order.first + + return unless active_migration&.interval_elapsed? + + if next_batched_job = create_next_batched_job!(active_migration) + migration_wrapper.perform(next_batched_job) + else + finish_active_migration(active_migration) + end + end + + private + + def create_next_batched_job!(active_migration) + next_batch_range = find_next_batch_range(active_migration) + + return if next_batch_range.nil? + + active_migration.create_batched_job!(next_batch_range.min, next_batch_range.max) + end + + def find_next_batch_range(active_migration) + batching_strategy = active_migration.batch_class.new + batch_min_value = active_migration.next_min_value + + next_batch_bounds = batching_strategy.next_batch( + active_migration.table_name, + active_migration.column_name, + batch_min_value: batch_min_value, + batch_size: active_migration.batch_size) + + return if next_batch_bounds.nil? + + clamped_batch_range(active_migration, next_batch_bounds) + end + + def clamped_batch_range(active_migration, next_bounds) + min_value, max_value = next_bounds + + return if min_value > active_migration.max_value + + max_value = max_value.clamp(min_value, active_migration.max_value) + + (min_value..max_value) + end + + def finish_active_migration(active_migration) + active_migration.finished! + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 6b169a504f3..31e733050e1 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -87,9 +87,10 @@ module Gitlab # See Rails' `create_table` for more info on the available arguments. def create_table_with_constraints(table_name, **options, &block) helper_context = self - check_constraints = [] with_lock_retries do + check_constraints = [] + create_table(table_name, **options) do |t| t.define_singleton_method(:check_constraint) do |name, definition| helper_context.send(:validate_check_constraint_name!, name) # rubocop:disable GitlabSecurity/PublicSend @@ -1015,7 +1016,7 @@ module Gitlab 'CopyColumnUsingBackgroundMigrationJob', interval, batch_size: batch_size, - other_job_arguments: [table, primary_key, column, tmp_column, sub_batch_size], + other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column], track_jobs: true, primary_column_name: primary_key ) diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 12dcf68da2f..e8cbea72887 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -4,8 +4,12 @@ module Gitlab module Database module Migrations module BackgroundMigrationHelpers - BACKGROUND_MIGRATION_BATCH_SIZE = 1_000 # Number of rows to process per job - BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time + BATCH_SIZE = 1_000 # Number of rows to process per job + SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch + JOB_BUFFER_SIZE = 1_000 # Number of jobs to bulk queue at a time + BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations + BATCH_MIN_VALUE = 1 # Default minimum value for batched migrations + BATCH_MIN_DELAY = 2.minutes.freeze # Minimum delay between batched migrations # Bulk queues background migration jobs for an entire table, batched by ID range. # "Bulk" meaning many jobs will be pushed at a time for efficiency. @@ -31,7 +35,7 @@ module Gitlab # # do something # end # end - def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) + def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BATCH_SIZE) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') jobs = [] @@ -40,7 +44,7 @@ module Gitlab model_class.each_batch(of: batch_size) do |relation| start_id, end_id = relation.pluck("MIN(#{table_name}.id)", "MAX(#{table_name}.id)").first - if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE + if jobs.length >= JOB_BUFFER_SIZE # Note: This code path generally only helps with many millions of rows # We push multiple jobs at a time to reduce the time spent in # Sidekiq/Redis operations. We're using this buffer based approach so we @@ -89,7 +93,7 @@ module Gitlab # # do something # end # end - def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) + def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer @@ -127,6 +131,79 @@ module Gitlab final_delay end + # Creates a batched background migration for the given table. A batched migration runs one job + # at a time, computing the bounds of the next batch based on the current migration settings and the previous + # batch bounds. Each job's execution status is tracked in the database as the migration runs. The given job + # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be + # present in the Gitlab::BackgroundMigration::BatchingStrategies module. + # + # job_class_name - The background migration job class as a string + # batch_table_name - The name of the table the migration will batch over + # batch_column_name - The name of the column the migration will batch over + # job_arguments - Extra arguments to pass to the job instance when the migration runs + # job_interval - The pause interval between each job's execution, minimum of 2 minutes + # batch_min_value - The value in the column the batching will begin at + # batch_max_value - The value in the column the batching will end at, defaults to `SELECT MAX(batch_column)` + # batch_class_name - The name of the class that will be called to find the range of each next batch + # batch_size - The maximum number of rows per job + # sub_batch_size - The maximum number of rows processed per "iteration" within the job + # + # + # *Returns the created BatchedMigration record* + # + # Example: + # + # queue_batched_background_migration( + # 'CopyColumnUsingBackgroundMigrationJob', + # :events, + # :id, + # job_interval: 2.minutes, + # other_job_arguments: ['column1', 'column2']) + # + # Where the the background migration exists: + # + # class Gitlab::BackgroundMigration::CopyColumnUsingBackgroundMigrationJob + # def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, *other_args) + # # do something + # end + # end + def queue_batched_background_migration( # rubocop:disable Metrics/ParameterLists + job_class_name, + batch_table_name, + batch_column_name, + *job_arguments, + job_interval:, + batch_min_value: BATCH_MIN_VALUE, + batch_max_value: nil, + batch_class_name: BATCH_CLASS_NAME, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + + job_interval = BATCH_MIN_DELAY if job_interval < BATCH_MIN_DELAY + + batch_max_value ||= connection.select_value(<<~SQL) + SELECT MAX(#{connection.quote_column_name(batch_column_name)}) + FROM #{connection.quote_table_name(batch_table_name)} + SQL + + migration_status = batch_max_value.nil? ? :finished : :active + batch_max_value ||= batch_min_value + + Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + job_class_name: job_class_name, + table_name: batch_table_name, + column_name: batch_column_name, + interval: job_interval, + min_value: batch_min_value, + max_value: batch_max_value, + batch_class_name: batch_class_name, + batch_size: batch_size, + sub_batch_size: sub_batch_size, + job_arguments: job_arguments, + status: migration_status) + end + def perform_background_migration_inline? Rails.env.test? || Rails.env.development? end diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 518c2c560d2..046843824a4 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -7,7 +7,8 @@ module Gitlab :migration, :walltime, :success, - :total_database_size_change + :total_database_size_change, + :query_statistics ) end end diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb index 4b931d3c19c..592993aeac5 100644 --- a/lib/gitlab/database/migrations/observers.rb +++ b/lib/gitlab/database/migrations/observers.rb @@ -6,7 +6,8 @@ module Gitlab module Observers def self.all_observers [ - TotalDatabaseSizeChange.new + TotalDatabaseSizeChange.new, + QueryStatistics.new ] end end diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb new file mode 100644 index 00000000000..466f4724256 --- /dev/null +++ b/lib/gitlab/database/migrations/observers/query_statistics.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + # This observer gathers statistics from the pg_stat_statements extension. + # Notice that this extension is not installed by default. In case it cannot + # be found, the observer does nothing and doesn't throw an error. + class QueryStatistics < MigrationObserver + include Gitlab::Database::SchemaHelpers + + def before + return unless enabled? + + connection.execute('select pg_stat_statements_reset()') + end + + def record(observation) + return unless enabled? + + observation.query_statistics = connection.execute(<<~SQL) + SELECT query, calls, total_time, max_time, mean_time, rows + FROM pg_stat_statements + ORDER BY total_time DESC + SQL + end + + private + + def enabled? + function_exists?(:pg_stat_statements_reset) && connection.view_exists?(:pg_stat_statements) + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index f4cf576dda7..1c289391e21 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Database::MigrationHelpers include ::Gitlab::Database::Migrations::BackgroundMigrationHelpers - ALLOWED_TABLES = %w[audit_events].freeze + ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze ERROR_SCOPE = 'table partitioning' MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable" diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index ff78fd0218c..40845c0d5e0 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -74,9 +74,14 @@ module Gitlab end # (SIMILARITY ...) + (SIMILARITY ...) - expressions.inject(first_expression) do |expression1, expression2| + additions = expressions.inject(first_expression) do |expression1, expression2| Arel::Nodes::Addition.new(expression1, expression2) end + + score_as_numeric = Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Grouping.new(additions).as('numeric')]) + + # Rounding the score to two decimals + Arel::Nodes::NamedFunction.new('ROUND', [score_as_numeric, 2]) end def self.order_by_similarity?(arel_query) diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index d735fb55652..36a840372c5 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -80,7 +80,7 @@ module Gitlab highlighted_lines.map!.with_index do |rich_line, i| marker = StringRegexMarker.new((plain_lines[i].chomp! || plain_lines[i]), rich_line.html_safe) - marker.mark(regex, group: :name) do |text, left:, right:| + marker.mark(regex, group: :name) do |text, left:, right:, mode:| url = yield(text) url ? link_tag(text, url) : text end diff --git a/lib/gitlab/dependency_linker/go_mod_linker.rb b/lib/gitlab/dependency_linker/go_mod_linker.rb index 4d6fe366333..fae4ee23383 100644 --- a/lib/gitlab/dependency_linker/go_mod_linker.rb +++ b/lib/gitlab/dependency_linker/go_mod_linker.rb @@ -22,7 +22,7 @@ module Gitlab i, j = match.offset(:name) marker = StringRangeMarker.new(plain_line, rich_line.html_safe) - marker.mark([i..(j - 1)]) do |text, left:, right:| + marker.mark([i..(j - 1)]) do |text, left:, right:, mode:| url = package_url(text, match[:version]) url ? link_tag(text, url) : text end diff --git a/lib/gitlab/dependency_linker/go_sum_linker.rb b/lib/gitlab/dependency_linker/go_sum_linker.rb index 20dc82ede9f..44826332f66 100644 --- a/lib/gitlab/dependency_linker/go_sum_linker.rb +++ b/lib/gitlab/dependency_linker/go_sum_linker.rb @@ -21,7 +21,7 @@ module Gitlab i2, j2 = match.offset(:checksum) marker = StringRangeMarker.new(plain_line, rich_line.html_safe) - marker.mark([i0..(j0 - 1), i2..(j2 - 1)]) do |text, left:, right:| + marker.mark([i0..(j0 - 1), i2..(j2 - 1)]) do |text, left:, right:, mode:| if left url = package_url(text, match[:version]) url ? link_tag(text, url) : text diff --git a/lib/gitlab/diff/char_diff.rb b/lib/gitlab/diff/char_diff.rb index c8bb39e9f5d..1b3af8f75ca 100644 --- a/lib/gitlab/diff/char_diff.rb +++ b/lib/gitlab/diff/char_diff.rb @@ -32,12 +32,12 @@ module Gitlab end if action == :delete - old_diffs << (old_pointer..(old_pointer + content_size - 1)) + old_diffs << MarkerRange.new(old_pointer, old_pointer + content_size - 1, mode: MarkerRange::DELETION) old_pointer += content_size end if action == :insert - new_diffs << (new_pointer..(new_pointer + content_size - 1)) + new_diffs << MarkerRange.new(new_pointer, new_pointer + content_size - 1, mode: MarkerRange::ADDITION) new_pointer += content_size end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 035084d4861..baa46e7e306 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -31,6 +31,12 @@ module Gitlab if line_inline_diffs = inline_diffs[i] begin + # MarkerRange objects are converted to Ranges to keep the previous behavior + # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 + if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) + line_inline_diffs = line_inline_diffs.map { |marker_range| marker_range.to_range } + end + rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) # This should only happen when the encoding of the diff doesn't # match the blob, which is a bug. But we shouldn't fail to render @@ -67,7 +73,7 @@ module Gitlab end def inline_diffs - @inline_diffs ||= InlineDiff.for_lines(@raw_lines, project: project) + @inline_diffs ||= InlineDiff.for_lines(@raw_lines) end def old_lines diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 7932cd2a837..c5e9bfdc321 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -7,8 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize EXPIRATION = 1.week - VERSION = 1 - NEXT_VERSION = 2 + VERSION = 2 delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection @@ -70,20 +69,17 @@ module Gitlab def key strong_memoize(:redis_key) do - ['highlighted-diff-files', diffable.cache_key, version, diff_options].join(":") + [ + 'highlighted-diff-files', + diffable.cache_key, VERSION, + diff_options, + Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml) + ].join(":") end end private - def version - if Feature.enabled?(:improved_merge_diff_highlighting, diffable.project, default_enabled: :yaml) - NEXT_VERSION - else - VERSION - end - end - def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index cf769262958..dd73e4d6c15 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -3,22 +3,6 @@ module Gitlab module Diff class InlineDiff - # Regex to find a run of deleted lines followed by the same number of added lines - LINE_PAIRS_PATTERN = %r{ - # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) - (?:\A|\s) - - # This matches a number of `-`s followed by the same number of `+`s through recursion - (?<del_ins> - - - \g<del_ins>? - \+ - ) - - # Runs end at the end of the string (the last line) or before a space (for an unchanged line) - (?=\s|\z) - }x.freeze - attr_accessor :old_line, :new_line, :offset def initialize(old_line, new_line, offset: 0) @@ -27,28 +11,24 @@ module Gitlab @offset = offset end - def inline_diffs(project: nil) + def inline_diffs # Skip inline diff if empty line was replaced with content return if old_line == "" - if Feature.enabled?(:improved_merge_diff_highlighting, project, default_enabled: :yaml) - CharDiff.new(old_line, new_line).changed_ranges(offset: offset) - else - deprecated_diff - end + CharDiff.new(old_line, new_line).changed_ranges(offset: offset) end class << self - def for_lines(lines, project: nil) - changed_line_pairs = find_changed_line_pairs(lines) + def for_lines(lines) + pair_selector = Gitlab::Diff::PairSelector.new(lines) inline_diffs = [] - changed_line_pairs.each do |old_index, new_index| + pair_selector.each do |old_index, new_index| old_line = lines[old_index] new_line = lines[new_index] - old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs(project: project) + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs inline_diffs[old_index] = old_diffs inline_diffs[new_index] = new_diffs @@ -56,74 +36,6 @@ module Gitlab inline_diffs end - - private - - # Finds pairs of old/new line pairs that represent the same line that changed - # rubocop: disable CodeReuse/ActiveRecord - def find_changed_line_pairs(lines) - # Prefixes of all diff lines, indicating their types - # For example: `" - + -+ ---+++ --+ -++"` - line_prefixes = lines.each_with_object(+"") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') - - changed_line_pairs = [] - line_prefixes.scan(LINE_PAIRS_PATTERN) do - # For `"---+++"`, `begin_index == 0`, `end_index == 6` - begin_index, end_index = Regexp.last_match.offset(:del_ins) - - # For `"---+++"`, `changed_line_count == 3` - changed_line_count = (end_index - begin_index) / 2 - - halfway_index = begin_index + changed_line_count - (begin_index...halfway_index).each do |i| - # For `"---+++"`, index 1 maps to 1 + 3 = 4 - changed_line_pairs << [i, i + changed_line_count] - end - end - - changed_line_pairs - end - # rubocop: enable CodeReuse/ActiveRecord - end - - private - - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/299884 - def deprecated_diff - lcp = longest_common_prefix(old_line, new_line) - lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) - - lcp += offset - old_length = old_line.length + offset - new_length = new_line.length + offset - - old_diff_range = lcp..(old_length - lcs - 1) - new_diff_range = lcp..(new_length - lcs - 1) - - old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end - new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end - - [old_diffs, new_diffs] - end - - def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName - max_length = [a.length, b.length].max - - length = 0 - (0..max_length - 1).each do |pos| - old_char = a[pos] - new_char = b[pos] - - break if old_char != new_char - - length += 1 - end - - length - end - - def longest_common_suffix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName - longest_common_prefix(a.reverse, b.reverse) end end end diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb index 3c536c43a9e..d8d596ebce7 100644 --- a/lib/gitlab/diff/inline_diff_markdown_marker.rb +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -8,8 +8,8 @@ module Gitlab deletion: "-" }.freeze - def mark(line_inline_diffs, mode: nil) - super(line_inline_diffs) do |text, left:, right:| + def mark(line_inline_diffs) + super(line_inline_diffs) do |text, left:, right:, mode:| symbol = MARKDOWN_SYMBOLS[mode] "{#{symbol}#{text}#{symbol}}" end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 29dff699ba5..c8cc1c0e649 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -7,8 +7,8 @@ module Gitlab super(line, rich_line || line) end - def mark(line_inline_diffs, mode: nil) - super(line_inline_diffs) do |text, left:, right:| + def mark(line_inline_diffs) + super(line_inline_diffs) do |text, left:, right:, mode:| %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}.html_safe end end diff --git a/lib/gitlab/diff/pair_selector.rb b/lib/gitlab/diff/pair_selector.rb new file mode 100644 index 00000000000..2e5ee3a7363 --- /dev/null +++ b/lib/gitlab/diff/pair_selector.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PairSelector + include Enumerable + + # Regex to find a run of deleted lines followed by the same number of added lines + # rubocop: disable Lint/MixedRegexpCaptureTypes + LINE_PAIRS_PATTERN = %r{ + # Runs start at the beginning of the string (the first line) or after a space (for an unchanged line) + (?:\A|\s) + + # This matches a number of `-`s followed by the same number of `+`s through recursion + (?<del_ins> + - + \g<del_ins>? + \+ + ) + + # Runs end at the end of the string (the last line) or before a space (for an unchanged line) + (?=\s|\z) + }x.freeze + # rubocop: enable Lint/MixedRegexpCaptureTypes + + def initialize(lines) + @lines = lines + end + + # Finds pairs of old/new line pairs that represent the same line that changed + # rubocop: disable CodeReuse/ActiveRecord + def each + # Prefixes of all diff lines, indicating their types + # For example: `" - + -+ ---+++ --+ -++"` + line_prefixes = lines.each_with_object(+"") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') + + line_prefixes.scan(LINE_PAIRS_PATTERN) do + # For `"---+++"`, `begin_index == 0`, `end_index == 6` + begin_index, end_index = Regexp.last_match.offset(:del_ins) + + # For `"---+++"`, `changed_line_count == 3` + changed_line_count = (end_index - begin_index) / 2 + + halfway_index = begin_index + changed_line_count + (begin_index...halfway_index).each do |i| + # For `"---+++"`, index 1 maps to 1 + 3 = 4 + yield [i, i + changed_line_count] + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + attr_reader :lines + end + end +end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index d1dd616385d..80e8b726099 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -79,7 +79,7 @@ module Gitlab @issue = Issues::CreateService.new( project, User.support_bot, - title: issue_title, + title: mail.subject, description: message_including_template, confidential: true, external_author: from_address @@ -137,12 +137,6 @@ module Gitlab (mail.reply_to || []).first || mail.from.first || mail.sender end - def issue_title - from = "(from #{from_address})" if from_address - - "Service Desk #{from}: #{mail.subject}" - end - def can_handle_legacy_format? project_path && project_path.include?('/') && !mail_key.include?('+') end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 1a8e5aaf07a..dfed8db8df0 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -27,33 +27,16 @@ module Gitlab config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) config.processors << ::Gitlab::ErrorTracking::Processor::SidekiqProcessor config.processors << ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor + config.processors << ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] - config.tags = extra_tags_from_env.merge(program: Gitlab.process_name) config.before_send = method(:before_send) yield config if block_given? end end - def with_context(current_user = nil) - last_user_context = Raven.context.user - - user_context = { - id: current_user&.id, - email: current_user&.email, - username: current_user&.username - }.compact - - Raven.tags_context(default_tags) - Raven.user_context(user_context) - - yield - ensure - Raven.user_context(last_user_context) - end - # This should be used when you want to passthrough exception handling: # rescue and raise to be catched in upper layers of the application. # @@ -118,37 +101,20 @@ module Gitlab end def process_exception(exception, sentry: false, logging: true, extra:) - exception.try(:sentry_extra_data)&.tap do |data| - extra = extra.merge(data) if data.is_a?(Hash) - end - - extra = sanitize_request_parameters(extra) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra) if sentry && Raven.configuration.server - Raven.capture_exception(exception, tags: default_tags, extra: extra) + Raven.capture_exception(exception, **context_payload) end if logging - # TODO: this logic could migrate into `Gitlab::ExceptionLogFormatter` - # and we could also flatten deep nested hashes if required for search - # (e.g. if `extra` includes hash of hashes). - # In the current implementation, we don't flatten multi-level folded hashes. - log_hash = {} - Raven.context.tags.each { |name, value| log_hash["tags.#{name}"] = value } - Raven.context.user.each { |name, value| log_hash["user.#{name}"] = value } - Raven.context.extra.merge(extra).each { |name, value| log_hash["extra.#{name}"] = value } - - Gitlab::ExceptionLogFormatter.format!(exception, log_hash) + formatter = Gitlab::ErrorTracking::LogFormatter.new + log_hash = formatter.generate_log(exception, context_payload) Gitlab::ErrorTracking::Logger.error(log_hash) end end - def sanitize_request_parameters(parameters) - filter = ActiveSupport::ParameterFilter.new(::Rails.application.config.filter_parameters) - filter.filter(parameters) - end - def sentry_dsn return unless Rails.env.production? || Rails.env.development? return unless Gitlab.config.sentry.enabled @@ -160,22 +126,6 @@ module Gitlab Rails.env.development? || Rails.env.test? end - def default_tags - { - Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id, - locale: I18n.locale - } - end - - # Static tags that are set on application start - def extra_tags_from_env - Gitlab::Json.parse(ENV.fetch('GITLAB_SENTRY_EXTRA_TAGS', '{}')).to_hash - rescue => e - Gitlab::AppLogger.debug("GITLAB_SENTRY_EXTRA_TAGS could not be parsed as JSON: #{e.class.name}: #{e.message}") - - {} - end - # Group common, mostly non-actionable exceptions by type and message, # rather than cause def custom_fingerprinting(event, ex) diff --git a/lib/gitlab/error_tracking/context_payload_generator.rb b/lib/gitlab/error_tracking/context_payload_generator.rb new file mode 100644 index 00000000000..c99283b3d20 --- /dev/null +++ b/lib/gitlab/error_tracking/context_payload_generator.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class ContextPayloadGenerator + def self.generate(exception, extra = {}) + new.generate(exception, extra) + end + + def generate(exception, extra = {}) + { + extra: extra_payload(exception, extra), + tags: tags_payload, + user: user_payload + } + end + + private + + def extra_payload(exception, extra) + inline_extra = exception.try(:sentry_extra_data) + if inline_extra.present? && inline_extra.is_a?(Hash) + extra = extra.merge(inline_extra) + end + + sanitize_request_parameters(extra) + end + + def sanitize_request_parameters(parameters) + filter = ActiveSupport::ParameterFilter.new(::Rails.application.config.filter_parameters) + filter.filter(parameters) + end + + def tags_payload + extra_tags_from_env.merge!( + program: Gitlab.process_name, + locale: I18n.locale, + feature_category: current_context['meta.feature_category'], + Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id + ) + end + + def user_payload + { + username: current_context['meta.user'] + } + end + + # Static tags that are set on application start + def extra_tags_from_env + Gitlab::Json.parse(ENV.fetch('GITLAB_SENTRY_EXTRA_TAGS', '{}')).to_hash + rescue => e + Gitlab::AppLogger.debug("GITLAB_SENTRY_EXTRA_TAGS could not be parsed as JSON: #{e.class.name}: #{e.message}") + + {} + end + + def current_context + # In case Gitlab::ErrorTracking is used when the app starts + return {} unless defined?(::Gitlab::ApplicationContext) + + ::Gitlab::ApplicationContext.current.to_h + end + end + end +end diff --git a/lib/gitlab/error_tracking/detailed_error.rb b/lib/gitlab/error_tracking/detailed_error.rb index 5d272efa64a..d0b3fc176aa 100644 --- a/lib/gitlab/error_tracking/detailed_error.rb +++ b/lib/gitlab/error_tracking/detailed_error.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class DetailedError diff --git a/lib/gitlab/error_tracking/error.rb b/lib/gitlab/error_tracking/error.rb index 6bfb9dae610..a256f87ec3d 100644 --- a/lib/gitlab/error_tracking/error.rb +++ b/lib/gitlab/error_tracking/error.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class Error diff --git a/lib/gitlab/error_tracking/error_collection.rb b/lib/gitlab/error_tracking/error_collection.rb index 56bcb671363..d01064bb677 100644 --- a/lib/gitlab/error_tracking/error_collection.rb +++ b/lib/gitlab/error_tracking/error_collection.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class ErrorCollection diff --git a/lib/gitlab/error_tracking/error_event.rb b/lib/gitlab/error_tracking/error_event.rb index 015d2c0ead0..d80289f6bc9 100644 --- a/lib/gitlab/error_tracking/error_event.rb +++ b/lib/gitlab/error_tracking/error_event.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class ErrorEvent diff --git a/lib/gitlab/error_tracking/log_formatter.rb b/lib/gitlab/error_tracking/log_formatter.rb new file mode 100644 index 00000000000..d004c4e20bb --- /dev/null +++ b/lib/gitlab/error_tracking/log_formatter.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class LogFormatter + # Note: all the accesses to Raven's contexts here are to keep the + # backward-compatibility to Sentry's built-in integrations. In future, + # they can be removed. + def generate_log(exception, context_payload) + payload = {} + + Gitlab::ExceptionLogFormatter.format!(exception, payload) + append_user_to_log!(payload, context_payload) + append_tags_to_log!(payload, context_payload) + append_extra_to_log!(payload, context_payload) + + payload + end + + private + + def append_user_to_log!(payload, context_payload) + user_context = Raven.context.user.merge(context_payload[:user]) + user_context.each do |key, value| + payload["user.#{key}"] = value + end + end + + def append_tags_to_log!(payload, context_payload) + tags_context = Raven.context.tags.merge(context_payload[:tags]) + tags_context.each do |key, value| + payload["tags.#{key}"] = value + end + end + + def append_extra_to_log!(payload, context_payload) + extra = Raven.context.extra.merge(context_payload[:extra]) + extra = extra.except(:server) + + # The extra value for sidekiq is a hash whose keys are strings. + if extra[:sidekiq].is_a?(Hash) && extra[:sidekiq].key?('args') + sidekiq_extra = extra.delete(:sidekiq) + sidekiq_extra['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor.loggable_arguments( + sidekiq_extra['args'], sidekiq_extra['class'] + ) + payload["extra.sidekiq"] = sidekiq_extra + end + + extra.each do |key, value| + payload["extra.#{key}"] = value + end + end + end + end +end diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb new file mode 100644 index 00000000000..5185205e94e --- /dev/null +++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + module Processor + class ContextPayloadProcessor < ::Raven::Processor + # This processor is added to inject application context into Sentry + # events generated by Sentry built-in integrations. When the + # integrations are re-implemented and use Gitlab::ErrorTracking, this + # processor should be removed. + def process(payload) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}) + payload.deep_merge!(context_payload) + end + end + end + end +end diff --git a/lib/gitlab/error_tracking/project.rb b/lib/gitlab/error_tracking/project.rb index 93e81da5034..a4ed8831e38 100644 --- a/lib/gitlab/error_tracking/project.rb +++ b/lib/gitlab/error_tracking/project.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class Project diff --git a/lib/gitlab/error_tracking/repo.rb b/lib/gitlab/error_tracking/repo.rb index 50611943bac..e88ac58ff0f 100644 --- a/lib/gitlab/error_tracking/repo.rb +++ b/lib/gitlab/error_tracking/repo.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking class Repo diff --git a/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb b/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb index 1e490e52c43..24f4c2a2dcf 100644 --- a/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb +++ b/lib/gitlab/error_tracking/stack_trace_highlight_decorator.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# This should be in the ErrorTracking namespace. For more details, see: +# https://gitlab.com/gitlab-org/gitlab/-/issues/323342 module Gitlab module ErrorTracking module StackTraceHighlightDecorator diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index fc3c05c57b2..8c916375a98 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -17,12 +17,12 @@ module Gitlab def call(env) request = ActionDispatch::Request.new(env) - route = Gitlab::EtagCaching::Router.match(request.path_info) + route = Gitlab::EtagCaching::Router.match(request) return @app.call(env) unless route track_event(:etag_caching_middleware_used, route) - etag, cached_value_present = get_etag(request) + etag, cached_value_present = get_etag(request, route) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag @@ -36,8 +36,8 @@ module Gitlab private - def get_etag(request) - cache_key = request.path + def get_etag(request, route) + cache_key = route.cache_key(request) store = Gitlab::EtagCaching::Store.new current_value = store.get(cache_key) cached_value_present = current_value.present? diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 769ac2784d1..742b72ecde9 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -2,99 +2,24 @@ module Gitlab module EtagCaching - class Router - Route = Struct.new(:regexp, :name, :feature_category) - # We enable an ETag for every request matching the regex. - # To match a regex the path needs to match the following: - # - Don't contain a reserved word (expect for the words used in the - # regex itself) - # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route - # - Ending in `issues/id`/realtime_changes` for the `issue_title` route - USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes - commit pipelines merge_requests builds - new environments].freeze - RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES - RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) - RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) + module Router + Route = Struct.new(:regexp, :name, :feature_category, :router) do + delegate :match, to: :regexp + delegate :cache_key, to: :router + end - ROUTES = [ - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), - 'issue_notes', - 'issue_tracking' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), - 'merge_request_notes', - 'code_review' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), - 'issue_title', - 'issue_tracking' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), - 'commit_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), - 'new_merge_request_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), - 'merge_request_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), - 'project_pipelines', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), - 'project_pipeline', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), - 'project_build', - 'continuous_integration' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), - 'cluster_environments', - 'continuous_delivery' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), - 'environments', - 'continuous_delivery' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), - 'realtime_changes_import_github', - 'importers' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), - 'realtime_changes_import_gitea', - 'importers' - ), - Gitlab::EtagCaching::Router::Route.new( - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), - 'merge_request_widget', - 'code_review' - ) - ].freeze + module Helpers + def build_route(attrs) + EtagCaching::Router::Route.new(*attrs, self) + end + end - def self.match(path) - ROUTES.find { |route| route.regexp.match(path) } + # Performing RESTful routing match before GraphQL would be more expensive + # for the GraphQL requests because we need to traverse all of the RESTful + # route definitions before falling back to GraphQL. + def self.match(request) + Router::Graphql.match(request) || Router::Restful.match(request) end end end end - -Gitlab::EtagCaching::Router.prepend_if_ee('EE::Gitlab::EtagCaching::Router') diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb new file mode 100644 index 00000000000..f1737f0ce5a --- /dev/null +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module EtagCaching + module Router + class Graphql + extend EtagCaching::Router::Helpers + GRAPHQL_ETAG_RESOURCE_HEADER = 'X-GITLAB-GRAPHQL-RESOURCE-ETAG' + + ROUTES = [ + [ + %r(\Apipelines/id/\d+\z), + 'pipelines_graph', + 'continuous_integration' + ] + ].map(&method(:build_route)).freeze + + def self.match(request) + return unless request.path_info == graphql_api_path + + graphql_resource = request.headers[GRAPHQL_ETAG_RESOURCE_HEADER] + return unless graphql_resource + + ROUTES.find { |route| route.match(graphql_resource) } + end + + def self.cache_key(request) + [ + request.path, + request.headers[GRAPHQL_ETAG_RESOURCE_HEADER] + ].compact.join(':') + end + + def self.graphql_api_path + @graphql_api_path ||= Gitlab::Routing.url_helpers.api_graphql_path + end + private_class_method :graphql_api_path + end + end + end +end diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb new file mode 100644 index 00000000000..08c20e30a48 --- /dev/null +++ b/lib/gitlab/etag_caching/router/restful.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module EtagCaching + module Router + class Restful + extend EtagCaching::Router::Helpers + + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/realtime_changes` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes + commit pipelines merge_requests builds + new environments].freeze + RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) + RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) + + ROUTES = [ + [ + %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), + 'issue_notes', + 'issue_tracking' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), + 'merge_request_notes', + 'code_review' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), + 'issue_title', + 'issue_tracking' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), + 'commit_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), + 'new_merge_request_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), + 'merge_request_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), + 'project_pipelines', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), + 'project_pipeline', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), + 'project_build', + 'continuous_integration' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), + 'cluster_environments', + 'continuous_delivery' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), + 'environments', + 'continuous_delivery' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), + 'realtime_changes_import_github', + 'importers' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), + 'realtime_changes_import_gitea', + 'importers' + ], + [ + %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), + 'merge_request_widget', + 'code_review' + ] + ].map(&method(:build_route)).freeze + + # Overridden in EE to add more routes + def self.all_routes + ROUTES + end + + def self.match(request) + all_routes.find { |route| route.match(request.path_info) } + end + + def self.cache_key(request) + request.path + end + end + end + end +end + +Gitlab::EtagCaching::Router::Restful.prepend_if_ee('EE::Gitlab::EtagCaching::Router::Restful') diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 1d2f0d7bbf4..d0d790a7c72 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -3,6 +3,8 @@ module Gitlab module EtagCaching class Store + InvalidKeyError = Class.new(StandardError) + EXPIRY_TIME = 20.minutes SHARED_STATE_NAMESPACE = 'etag:' @@ -27,9 +29,28 @@ module Gitlab end def redis_shared_state_key(key) - raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) + raise InvalidKeyError, "#{key} is invalid" unless valid_key?(key) "#{SHARED_STATE_NAMESPACE}#{key}" + rescue InvalidKeyError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + def valid_key?(key) + return true if skip_validation? + + path, header = key.split(':', 2) + env = { + 'PATH_INFO' => path, + 'HTTP_X_GITLAB_GRAPHQL_RESOURCE_ETAG' => header + } + + fake_request = ActionDispatch::Request.new(env) + !!Gitlab::EtagCaching::Router.match(fake_request) + end + + def skip_validation? + Rails.env.production? end end end diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 6aff8f909f3..9898651c9e3 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -12,16 +12,6 @@ module Gitlab 'exception.message' => exception.message ) - payload.delete('extra.server') - - payload['extra.sidekiq'].tap do |value| - if value.is_a?(Hash) && value.key?('args') - value = value.dup - payload['extra.sidekiq']['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor - .loggable_arguments(value['args'], value['class']) - end - end - if exception.backtrace payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 423f238a0a2..1bb29ba3eac 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -34,18 +34,10 @@ module Gitlab module Experimentation EXPERIMENTS = { - ci_notification_dot: { - tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot', - use_backwards_compatible_subject_index: true - }, upgrade_link_in_user_menu_a: { tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA', use_backwards_compatible_subject_index: true }, - invite_members_version_a: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA', - use_backwards_compatible_subject_index: true - }, invite_members_version_b: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', use_backwards_compatible_subject_index: true @@ -58,30 +50,16 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', use_backwards_compatible_subject_index: true }, - customize_homepage: { - tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage', - use_backwards_compatible_subject_index: true - }, - group_only_trials: { - tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', - use_backwards_compatible_subject_index: true - }, remove_known_trial_form_fields: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' }, - trimmed_skip_trial_copy: { - tracking_category: 'Growth::Conversion::Experiment::TrimmedSkipTrialCopy' - }, - trial_registration_with_social_signin: { - tracking_category: 'Growth::Conversion::Experiment::TrialRegistrationWithSocialSigning' - }, invite_members_empty_project_version_a: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyProjectVersionA' }, trial_during_signup: { tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' }, - ci_syntax_templates: { + ci_syntax_templates_b: { tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates', rollout_strategy: :user }, diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 2b38b12c914..248abfeada5 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -15,7 +15,7 @@ module Gitlab included do before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :tracking_label + helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :record_experiment_group, :tracking_label end def set_experimentation_subject_id_cookie @@ -72,12 +72,22 @@ module Gitlab ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) end - def record_experiment_conversion_event(experiment_key) + def record_experiment_group(experiment_key, group) + return if dnt_enabled? + return unless Experimentation.active?(experiment_key) && group + + variant_subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : group + variant = tracking_group(experiment_key, nil, subject: variant_subject) + + ::Experiment.add_group(experiment_key, group: group, variant: variant) + end + + def record_experiment_conversion_event(experiment_key, context = {}) return if dnt_enabled? return unless current_user return unless Experimentation.active?(experiment_key) - ::Experiment.record_conversion_event(experiment_key, current_user) + ::Experiment.record_conversion_event(experiment_key, current_user, context) end def experiment_tracking_category_and_group(experiment_key, subject: nil) diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index b118eda37f8..9e24306c05e 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -38,9 +38,11 @@ module Gitlab if line[0, 1] == "\t" lines << line[1, line.size] elsif m = /^(\w{40}) (\d+) (\d+)/.match(line) - commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + # Removed these instantiations for performance but keeping them for reference: + # commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + commit_id = m[1] commits[commit_id] = nil unless commits.key?(commit_id) - info[lineno] = [commit_id, old_lineno] + info[m[3].to_i] = [commit_id, m[2].to_i] end end @@ -50,8 +52,7 @@ module Gitlab # get it together info.sort.each do |lineno, (commit_id, old_lineno)| - commit = commits[commit_id] - final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1]) + final << BlameLine.new(lineno, old_lineno, commits[commit_id], lines[lineno - 1]) end @lines = final diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 35c3dc5b0b3..ff99803d8de 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -20,6 +20,7 @@ module Gitlab ].freeze attr_accessor(*SERIALIZE_KEYS) + attr_reader :repository def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb index c05a5adc00c..7f09173f05c 100644 --- a/lib/gitlab/git/wiki_file.rb +++ b/lib/gitlab/git/wiki_file.rb @@ -12,6 +12,19 @@ module Gitlab @name = gitaly_file.name @path = gitaly_file.path end + + def self.from_blob(blob) + hash = { + name: File.basename(blob.name), + mime_type: blob.mime_type, + path: blob.path, + raw_data: blob.data + } + + gitaly_file = Gitlab::GitalyClient::WikiFile.new(hash) + + Gitlab::Git::WikiFile.new(gitaly_file) + end end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 3c7fa88977e..e3788814dd5 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -246,9 +246,7 @@ module Gitlab def self.route_to_primary return {} unless Gitlab::SafeRequestStore.active? - return {} unless Gitlab::SafeRequestStore[:gitlab_git_env] - - return {} if Gitlab::SafeRequestStore[:gitlab_git_env].empty? + return {} if Gitlab::SafeRequestStore[:gitlab_git_env].blank? { 'gitaly-route-repository-accessor-policy' => 'primary-only' } end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 5e50ac72965..7edd42f9ef7 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -60,7 +60,8 @@ module Gitlab end def legacy_disk_path - if self.class.disk_access_denied? + # Do not use self.class due to Spring reloading issues + if Gitlab::GitalyClient::StorageSettings.disk_access_denied? raise DirectPathAccessError, "git disk access denied" end diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb index 11181edf0e9..8173fdd5e3e 100644 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb @@ -12,25 +12,27 @@ module Gitlab def execute merge_request = project.merge_requests.find_by_iid(pull_request.iid) + timestamp = Time.new.utc + merged_at = pull_request.merged_at user_finder = GithubImport::UserFinder.new(project, client) gitlab_user_id = user_finder.user_id_for(pull_request.merged_by) - if gitlab_user_id - timestamp = Time.new.utc - MergeRequest::Metrics.upsert({ - target_project_id: project.id, - merge_request_id: merge_request.id, - merged_by_id: gitlab_user_id, - created_at: timestamp, - updated_at: timestamp - }, unique_by: :merge_request_id) - else + MergeRequest::Metrics.upsert({ + target_project_id: project.id, + merge_request_id: merge_request.id, + merged_by_id: gitlab_user_id, + merged_at: merged_at, + created_at: timestamp, + updated_at: timestamp + }, unique_by: :merge_request_id) + + unless gitlab_user_id merge_request.notes.create!( importing: true, - note: "*Merged by: #{pull_request.merged_by.login}*", + note: missing_author_note, author_id: project.creator_id, project: project, - created_at: pull_request.created_at + created_at: merged_at ) end end @@ -38,6 +40,13 @@ module Gitlab private attr_reader :project, :pull_request, :client + + def missing_author_note + s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { + author: pull_request.merged_by.login, + timestamp: pull_request.merged_at + } + end end end end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index 14ee69ba089..9f495913897 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -77,12 +77,22 @@ module Gitlab def add_approval!(user_id) return unless review.review_type == 'APPROVED' - add_approval_system_note!(user_id) - - merge_request.approvals.create!( + approval_attribues = { + merge_request_id: merge_request.id, user_id: user_id, - created_at: review.submitted_at + created_at: review.submitted_at, + updated_at: review.submitted_at + } + + result = ::Approval.insert( + approval_attribues, + returning: [:id], + unique_by: [:user_id, :merge_request_id] ) + + if result.rows.present? + add_approval_system_note!(user_id) + end end def add_approval_system_note!(user_id) diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 3dd317c5a64..c7e215c143f 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -7,13 +7,14 @@ module Gitlab include WebpackHelper def add_gon_variables - gon.api_version = 'v4' - gon.default_avatar_url = default_avatar_url - gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size - gon.asset_host = ActionController::Base.asset_host - gon.webpack_public_path = webpack_public_path - gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + gon.api_version = 'v4' + gon.default_avatar_url = default_avatar_url + gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size + gon.asset_host = ActionController::Base.asset_host + gon.webpack_public_path = webpack_public_path + gon.relative_url_root = Gitlab.config.gitlab.relative_url_root + gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + gon.markdown_surround_selection = current_user&.markdown_surround_selection if Gitlab.config.sentry.enabled gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn diff --git a/lib/gitlab/graphql/calls_gitaly.rb b/lib/gitlab/graphql/calls_gitaly.rb deleted file mode 100644 index 40cd74a34f2..00000000000 --- a/lib/gitlab/graphql/calls_gitaly.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - # Wraps the field resolution to count Gitaly calls before and after. - # Raises an error if the field calls Gitaly but hadn't declared such. - module CallsGitaly - extend ActiveSupport::Concern - - def self.use(schema_definition) - schema_definition.instrument(:field, Gitlab::Graphql::CallsGitaly::Instrumentation.new, after_built_ins: true) - end - end - end -end diff --git a/lib/gitlab/graphql/calls_gitaly/field_extension.rb b/lib/gitlab/graphql/calls_gitaly/field_extension.rb new file mode 100644 index 00000000000..32530b47ce3 --- /dev/null +++ b/lib/gitlab/graphql/calls_gitaly/field_extension.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module CallsGitaly + # Check if any `calls_gitaly: true` declarations need to be added + # + # See BaseField: this extension is not applied if the field does not + # need it (i.e. it has a constant complexity or knows that it calls + # gitaly) + class FieldExtension < ::GraphQL::Schema::FieldExtension + include Laziness + + def resolve(object:, arguments:, **rest) + yield(object, arguments, [current_gitaly_call_count, accounted_for]) + end + + def after_resolve(value:, memo:, **rest) + (value, count) = value_with_count(value, memo) + calls_gitaly_check(count) + accounted_for(count) + + value + end + + private + + # Resolutions are not nested nicely (due to laziness), so we have to + # know not just how many calls were made before resolution started, but + # also how many were accounted for by fields with the correct settings + # in between. + # + # e.g. the following is not just plausible, but common: + # + # enter A.user (lazy) + # enter A.x + # leave A.x + # enter A.calls_gitaly + # leave A.calls_gitaly (accounts for 1 call) + # leave A.user + # + # In this circumstance we need to mark the calls made by A.calls_gitaly + # as accounted for, even though they were made after we yielded + # in A.user + def value_with_count(value, (previous_count, previous_accounted_for)) + newly_accounted_for = accounted_for - previous_accounted_for + value = force(value) + count = [current_gitaly_call_count - (previous_count + newly_accounted_for), 0].max + + [value, count] + end + + def current_gitaly_call_count + Gitlab::GitalyClient.get_request_count || 0 + end + + def calls_gitaly_check(calls) + return if calls < 1 || field.may_call_gitaly? + + error = RuntimeError.new(<<~ERROR) + #{field_name} unexpectedly calls Gitaly! + + Please either specify a constant complexity or add `calls_gitaly: true` + to the field declaration + ERROR + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + end + + def accounted_for(count = nil) + return 0 unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] ||= 0 + + if count.nil? + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] + else + Gitlab::SafeRequestStore["graphql_gitaly_accounted_for"] += count + end + end + + def field_name + "#{field.owner.graphql_name}.#{field.graphql_name}" + end + end + end + end +end diff --git a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb b/lib/gitlab/graphql/calls_gitaly/instrumentation.rb deleted file mode 100644 index 11d3c50e093..00000000000 --- a/lib/gitlab/graphql/calls_gitaly/instrumentation.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module CallsGitaly - class Instrumentation - # Check if any `calls_gitaly: true` declarations need to be added - # Do nothing if a constant complexity was provided - def instrument(_type, field) - type_object = field.metadata[:type_class] - return field unless type_object.respond_to?(:calls_gitaly?) - return field if type_object.constant_complexity? || type_object.calls_gitaly? - - old_resolver_proc = field.resolve_proc - - gitaly_wrapped_resolve = -> (typed_object, args, ctx) do - previous_gitaly_call_count = Gitlab::GitalyClient.get_request_count - result = old_resolver_proc.call(typed_object, args, ctx) - current_gitaly_call_count = Gitlab::GitalyClient.get_request_count - calls_gitaly_check(type_object, current_gitaly_call_count - previous_gitaly_call_count) - result - end - - field.redefine do - resolve(gitaly_wrapped_resolve) - end - end - - def calls_gitaly_check(type_object, calls) - return if calls < 1 - - # Will inform you if there needs to be `calls_gitaly: true` as a kwarg in the field declaration - # if there is at least 1 Gitaly call involved with the field resolution. - error = RuntimeError.new("Gitaly is called for field '#{type_object.name}' on #{type_object.owner.try(:name)} - please either specify a constant complexity or add `calls_gitaly: true` to the field declaration") - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - end - end - end - end -end diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index ad9e08e189c..e9ff85d9ca9 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -27,33 +27,36 @@ module Gitlab MD end - def render_name_and_description(object) - content = "### #{object[:name]}\n" + def render_name_and_description(object, level = 3) + content = [] + + content << "#{'#' * level} `#{object[:name]}`" if object[:description].present? - content += "\n#{object[:description]}.\n" + desc = object[:description].strip + desc += '.' unless desc.ends_with?('.') + content << desc end - content + content.join("\n\n") end def sorted_by_name(objects) + return [] unless objects.present? + objects.sort_by { |o| o[:name] } end def render_field(field) - '| %s | %s | %s |' % [ - render_name(field), - render_field_type(field[:type][:info]), - render_description(field) - ] + row(render_name(field), render_field_type(field[:type]), render_description(field)) end def render_enum_value(value) - '| %s | %s |' % [ - render_name(value), - render_description(value) - ] + row(render_name(value), render_description(value)) + end + + def row(*values) + "| #{values.join(' | ')} |" end def render_name(object) @@ -70,27 +73,19 @@ module Gitlab "**Deprecated:** #{object[:deprecation_reason]}" end - # Some fields types are arrays of other types and are displayed - # on docs wrapped in square brackets, for example: [String!]. - # This makes GitLab docs renderer thinks they are links so here - # we change them to be rendered as: String! => Array. def render_field_type(type) - array_type = type[/\[(.+)\]/, 1] + "[`#{type[:info]}`](##{type[:name].downcase})" + end - if array_type - "#{array_type} => Array" - else - type - end + def render_return_type(query) + "Returns #{render_field_type(query[:type])}.\n" end # We are ignoring connections and built in types for now, # they should be added when queries are generated. def objects object_types = graphql_object_types.select do |object_type| - !object_type[:name]["Connection"] && - !object_type[:name]["Edge"] && - !object_type[:name]["__"] + !object_type[:name]["__"] end object_types.each do |type| @@ -98,10 +93,14 @@ module Gitlab end end + def queries + graphql_operation_types.find { |type| type[:name] == 'Query' }.to_h.values_at(:fields, :connections).flatten + end + # We ignore the built-in enum types. def enums graphql_enum_types.select do |enum_type| - !enum_type[:name].in?(%w(__DirectiveLocation __TypeKind)) + !enum_type[:name].in?(%w[__DirectiveLocation __TypeKind]) end end end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 9dfb9b090a8..847f1777b08 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -14,13 +14,31 @@ WARNING: Fields that are deprecated are marked with **{warning-solid}**. - Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-process) can be found + Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-and-removal-process) can be found in [Removed Items](../removed_items.md). <!-- vale gitlab.Spelling = NO --> \ :plain + ## `Query` type + + The `Query` type contains the API's top-level entry points for all executable queries. +\ + +- sorted_by_name(queries).each do |query| + = render_name_and_description(query) + \ + = render_return_type(query) + - unless query[:arguments].empty? + ~ "#### Arguments\n" + ~ "| Name | Type | Description |" + ~ "| ---- | ---- | ----------- |" + - sorted_by_name(query[:arguments]).each do |argument| + = render_field(argument) + \ + +:plain ## Object types Object types represent the resources that the GitLab GraphQL API can return. @@ -36,6 +54,7 @@ - objects.each do |type| - unless type[:fields].empty? = render_name_and_description(type) + \ ~ "| Field | Type | Description |" ~ "| ----- | ---- | ----------- |" - sorted_by_name(type[:fields]).each do |field| @@ -56,8 +75,74 @@ - enums.each do |enum| - unless enum[:values].empty? = render_name_and_description(enum) + \ ~ "| Value | Description |" ~ "| ----- | ----------- |" - sorted_by_name(enum[:values]).each do |value| = render_enum_value(value) \ + +:plain + ## Scalar types + + Scalar values are atomic values, and do not have fields of their own. + Basic scalars include strings, boolean values, and numbers. This schema also + defines various custom scalar values, such as types for times and dates. + + This schema includes custom scalar types for identifiers, with a specific type for + each kind of object. + + For more information, read about [Scalar Types](https://graphql.org/learn/schema/#scalar-types) on `graphql.org`. +\ + +- graphql_scalar_types.each do |type| + = render_name_and_description(type) + \ + +:plain + ## Abstract types + + Abstract types (unions and interfaces) are ways the schema can represent + values that may be one of several concrete types. + + - A [`Union`](https://graphql.org/learn/schema/#union-types) is a set of possible types. + The types might not have any fields in common. + - An [`Interface`](https://graphql.org/learn/schema/#interfaces) is a defined set of fields. + Types may `implement` an interface, which + guarantees that they have all the fields in the set. A type may implement more than + one interface. + + See the [GraphQL documentation](https://graphql.org/learn/) for more information on using + abstract types. +\ + +:plain + ### Unions +\ + +- graphql_union_types.each do |type| + = render_name_and_description(type, 4) + \ + One of: + \ + - type[:possible_types].each do |type_name| + ~ "- [`#{type_name}`](##{type_name.downcase})" + \ + +:plain + ### Interfaces +\ + +- graphql_interface_types.each do |type| + = render_name_and_description(type, 4) + \ + Implementations: + \ + - type[:implemented_by].each do |type_name| + ~ "- [`#{type_name}`](##{type_name.downcase})" + \ + ~ "| Field | Type | Description |" + ~ "| ----- | ---- | ----------- |" + - sorted_by_name(type[:fields] + type[:connections]).each do |field| + = render_field(field) + \ diff --git a/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb b/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb index 1adedb500e6..f787e7be94a 100644 --- a/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb +++ b/lib/gitlab/graphql/extensions/externally_paginated_array_extension.rb @@ -4,7 +4,7 @@ module Gitlab module Extensions class ExternallyPaginatedArrayExtension < GraphQL::Schema::Field::ConnectionExtension def resolve(object:, arguments:, context:) - yield(object, arguments) + yield(object, arguments, arguments) end end end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index f95c91c5706..e525996ec10 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -33,6 +33,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include ::Gitlab::Graphql::ConnectionCollectionMethods prepend ::Gitlab::Graphql::ConnectionRedaction + prepend GenericKeysetPagination # rubocop: disable Naming/PredicateName # https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb new file mode 100644 index 00000000000..318c6e1734f --- /dev/null +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Pagination + module Keyset + # Use the generic keyset implementation if the given ActiveRecord scope supports it. + # Note: this module is temporary, at some point it will be merged with Keyset::Connection + module GenericKeysetPagination + extend ActiveSupport::Concern + + def ordered_items + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + items + end + + def cursor_for(node) + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(items) + encode(order.cursor_attributes_for_node(node).to_json) + end + + def slice_nodes(sliced, encoded_cursor, before_or_after) + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(sliced) + + order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(sliced) + order = order.reversed_order if before_or_after == :before + + decoded_cursor = ordering_from_encoded_json(encoded_cursor) + order.apply_cursor_conditions(sliced, decoded_cursor) + end + + def sliced_nodes + return super unless Gitlab::Pagination::Keyset::Order.keyset_aware?(items) + + sliced = ordered_items + sliced = slice_nodes(sliced, before, :before) if before.present? + sliced = slice_nodes(sliced, after, :after) if after.present? + sliced + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/last_items.rb b/lib/gitlab/graphql/pagination/keyset/last_items.rb index 45bf15236c1..960567a6fbc 100644 --- a/lib/gitlab/graphql/pagination/keyset/last_items.rb +++ b/lib/gitlab/graphql/pagination/keyset/last_items.rb @@ -10,46 +10,14 @@ module Gitlab class LastItems # rubocop: disable CodeReuse/ActiveRecord def self.take_items(scope, count) - if custom_order = lookup_custom_reverse_order(scope.order_values) - items = scope.reorder(*custom_order).first(count) # returns a single record when count is nil + if Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) + order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) + items = scope.reorder(order.reversed_order).first(count) items.is_a?(Array) ? items.reverse : items else scope.last(count) end end - # rubocop: enable CodeReuse/ActiveRecord - - # Detect special ordering and provide the reversed order - def self.lookup_custom_reverse_order(order_values) - if ordering_by_merged_at_and_mr_id_desc?(order_values) - [ - Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'ASC'), # reversing the order - MergeRequest.arel_table[:id].asc - ] - elsif ordering_by_merged_at_and_mr_id_asc?(order_values) - [ - Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'DESC'), - MergeRequest.arel_table[:id].asc - ] - end - end - - def self.ordering_by_merged_at_and_mr_id_desc?(order_values) - order_values.size == 2 && - order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'DESC') && - order_values.last.is_a?(Arel::Nodes::Descending) && - order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql - end - - def self.ordering_by_merged_at_and_mr_id_asc?(order_values) - order_values.size == 2 && - order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'ASC') && - order_values.last.is_a?(Arel::Nodes::Descending) && - order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql - end - - private_class_method :ordering_by_merged_at_and_mr_id_desc? - private_class_method :ordering_by_merged_at_and_mr_id_asc? end end end diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb index d37264c1343..0494329bfd9 100644 --- a/lib/gitlab/graphql/pagination/keyset/order_info.rb +++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb @@ -92,8 +92,6 @@ module Gitlab def extract_attribute_values(order_value) if ordering_by_lower?(order_value) [order_value.expr.expressions[0].name.to_s, order_value.direction, order_value.expr] - elsif ordering_by_similarity?(order_value) - ['similarity', order_value.direction, order_value.expr] elsif ordering_by_case?(order_value) ['case_order_value', order_value.direction, order_value.expr] elsif ordering_by_array_position?(order_value) @@ -113,11 +111,6 @@ module Gitlab order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'array_position' end - # determine if ordering using SIMILARITY scoring based on Gitlab::Database::SimilarityScore - def ordering_by_similarity?(order_value) - Gitlab::Database::SimilarityScore.order_by_similarity?(order_value) - end - # determine if ordering using CASE def ordering_by_case?(order_value) order_value.expr.is_a?(Arel::Nodes::Case) diff --git a/lib/gitlab/graphql/present.rb b/lib/gitlab/graphql/present.rb index 6d86d632ab4..fdaf075eb25 100644 --- a/lib/gitlab/graphql/present.rb +++ b/lib/gitlab/graphql/present.rb @@ -12,11 +12,30 @@ module Gitlab def self.presenter_class @presenter_class end + + def self.present(object, attrs) + klass = @presenter_class + return object if !klass || object.is_a?(klass) + + @presenter_class.new(object, **attrs) + end + end + + def unpresented + unwrapped || object end - def self.use(schema_definition) - schema_definition.instrument(:field, ::Gitlab::Graphql::Present::Instrumentation.new) + def present(object_type, attrs) + return unless object_type.respond_to?(:present) + + self.unwrapped ||= object + # @object belongs to Schema::Object, which does not expose a writer. + @object = object_type.present(unwrapped, attrs) # rubocop: disable Gitlab/ModuleWithInstanceVariables end + + private + + attr_accessor :unwrapped end end end diff --git a/lib/gitlab/graphql/present/field_extension.rb b/lib/gitlab/graphql/present/field_extension.rb new file mode 100644 index 00000000000..2e211b70d35 --- /dev/null +++ b/lib/gitlab/graphql/present/field_extension.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Present + class FieldExtension < ::GraphQL::Schema::FieldExtension + SAFE_CONTEXT_KEYS = %i[current_user].freeze + + def resolve(object:, arguments:, context:) + attrs = safe_context_values(context) + + # We need to handle the object being either a Schema::Object or an + # inner Schema::Object#object. This depends on whether the field + # has a @resolver_proc or not. + if object.is_a?(::Types::BaseObject) + object.present(field.owner, attrs) + yield(object, arguments) + else + # This is the legacy code-path, hit if the field has a @resolver_proc + # TODO: remove this when resolve procs are removed from the + # graphql-ruby library, and all field instrumentation is removed. + # See: https://github.com/rmosolgo/graphql-ruby/issues/3385 + presented = field.owner.try(:present, object, attrs) || object + yield(presented, arguments) + end + end + + private + + def safe_context_values(context) + context.to_h.slice(*SAFE_CONTEXT_KEYS) + end + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb deleted file mode 100644 index b8535575da5..00000000000 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Present - class Instrumentation - SAFE_CONTEXT_KEYS = %i[current_user].freeze - - def instrument(type, field) - return field unless field.metadata[:type_class] - - presented_in = field.metadata[:type_class].owner - return field unless presented_in.respond_to?(:presenter_class) - return field unless presented_in.presenter_class - - old_resolver = field.resolve_proc - - resolve_with_presenter = -> (presented_type, args, context) do - # We need to wrap the original presentation type into a type that - # uses the presenter as an object. - object = presented_type.object - - if object.is_a?(presented_in.presenter_class) - next old_resolver.call(presented_type, args, context) - end - - attrs = safe_context_values(context) - presenter = presented_in.presenter_class.new(object, **attrs) - - # we have to use the new `authorized_new` method, as `new` is protected - wrapped = presented_type.class.authorized_new(presenter, context) - - old_resolver.call(wrapped, args, context) - end - - field.redefine do - resolve(resolve_with_presenter) - end - end - - private - - def safe_context_values(context) - context.to_h.slice(*SAFE_CONTEXT_KEYS) - end - end - end - end -end diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 0665ea8b6c9..8acd27869a9 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -9,10 +9,6 @@ module Gitlab FIELD_USAGE_ANALYZER = GraphQL::Analysis::FieldUsage.new { |query, used_fields, used_deprecated_fields| [used_fields, used_deprecated_fields] } ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze - def analyze?(query) - Feature.enabled?(:graphql_logging, default_enabled: true) - end - def initial_value(query) variables = process_variables(query.provided_variables) default_initial_values(query).merge({ diff --git a/lib/gitlab/hook_data/project_member_builder.rb b/lib/gitlab/hook_data/project_member_builder.rb new file mode 100644 index 00000000000..90fc83fdf21 --- /dev/null +++ b/lib/gitlab/hook_data/project_member_builder.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class ProjectMemberBuilder < BaseBuilder + alias_method :project_member, :object + + # Sample data + + # { + # :created_at=>"2021-03-02T10:43:17Z", + # :updated_at=>"2021-03-02T10:43:17Z", + # :project_name=>"gitlab", + # :project_path=>"gitlab", + # :project_path_with_namespace=>"namespace1/gitlab", + # :project_id=>1, + # :user_username=>"johndoe", + # :user_name=>"John Doe", + # :user_email=>"john@example.com", + # :user_id=>2, + # :access_level=>"Developer", + # :project_visibility=>"internal", + # :event_name=>"user_update_for_team" + # } + + def build(event) + [ + timestamps_data, + project_member_data, + event_data(event) + ].reduce(:merge) + end + + private + + def project_member_data + project = project_member.project || Project.unscoped.find(project_member.source_id) + + { + project_name: project.name, + project_path: project.path, + project_path_with_namespace: project.full_path, + project_id: project.id, + user_username: project_member.user.username, + user_name: project_member.user.name, + user_email: project_member.user.email, + user_id: project_member.user.id, + access_level: project_member.human_access, + project_visibility: project.visibility + } + end + + def event_data(event) + event_name = case event + when :create + 'user_add_to_team' + when :destroy + 'user_remove_from_team' + when :update + 'user_update_for_team' + end + { event_name: event_name } + end + end + end +end diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 84eb60f3a5d..37f618ae879 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -11,13 +11,18 @@ # This option will take precedence over the global setting. module Gitlab class HTTPConnectionAdapter < HTTParty::ConnectionAdapter + extend ::Gitlab::Utils::Override + + override :connection def connection - begin - @uri, hostname = Gitlab::UrlBlocker.validate!(uri, allow_local_network: allow_local_requests?, - allow_localhost: allow_local_requests?, - dns_rebind_protection: dns_rebind_protection?) - rescue Gitlab::UrlBlocker::BlockedUrlError => e - raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" + @uri, hostname = validate_url!(uri) + + if options.key?(:http_proxyaddr) + proxy_uri_with_port = uri_with_port(options[:http_proxyaddr], options[:http_proxyport]) + proxy_uri_validated = validate_url!(proxy_uri_with_port).first + + @options[:http_proxyaddr] = proxy_uri_validated.omit(:port).to_s + @options[:http_proxyport] = proxy_uri_validated.port end super.tap do |http| @@ -27,6 +32,14 @@ module Gitlab private + def validate_url!(url) + Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection?) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{url}' is blocked: #{e.message}" + end + def allow_local_requests? options.fetch(:allow_local_requests, allow_settings_local_requests?) end @@ -40,5 +53,11 @@ module Gitlab def allow_settings_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end + + def uri_with_port(address, port) + uri = Addressable::URI.parse(address) + uri.port = port if port.present? + uri + end end end diff --git a/lib/gitlab/marginalia.rb b/lib/gitlab/marginalia.rb index 325a8c5c325..c99cf113638 100644 --- a/lib/gitlab/marginalia.rb +++ b/lib/gitlab/marginalia.rb @@ -2,8 +2,6 @@ module Gitlab module Marginalia - cattr_accessor :enabled, default: false - def self.set_application_name ::Marginalia.application_name = Gitlab.process_name end @@ -13,12 +11,5 @@ module Gitlab ::Marginalia::SidekiqInstrumentation.enable! end end - - def self.set_enabled_from_feature_flag - # During db:create and db:bootstrap skip feature query as DB is not available yet. - return false unless Gitlab::Database.cached_table_exists?('features') - - self.enabled = Feature.enabled?(:marginalia, type: :ops) - end end end diff --git a/lib/gitlab/marginalia/active_record_instrumentation.rb b/lib/gitlab/marginalia/active_record_instrumentation.rb deleted file mode 100644 index 452f472bf6a..00000000000 --- a/lib/gitlab/marginalia/active_record_instrumentation.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -# Patch to annotate sql only when the feature is enabled. -module Gitlab - module Marginalia - module ActiveRecordInstrumentation - def annotate_sql(sql) - Gitlab::Marginalia.enabled ? super(sql) : sql - end - end - end -end diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index 7b4e4b06f00..ee15d3b1812 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -37,6 +37,10 @@ module Gitlab job end end + + def endpoint_id + Labkit::Context.current&.get_attribute(:caller_id) + end end end end diff --git a/lib/gitlab/marker_range.rb b/lib/gitlab/marker_range.rb new file mode 100644 index 00000000000..50a59adebdf --- /dev/null +++ b/lib/gitlab/marker_range.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# It is a Range object extended with `mode` attribute +# MarkerRange not only keeps information about changed characters, but also +# the type of changes +module Gitlab + class MarkerRange < Range + DELETION = :deletion + ADDITION = :addition + + # Converts Range object to MarkerRange class + def self.from_range(range) + return range if range.is_a?(self) + + new(range.begin, range.end, exclude_end: range.exclude_end?) + end + + def initialize(first, last, exclude_end: false, mode: nil) + super(first, last, exclude_end) + @mode = mode + end + + def to_range + Range.new(self.begin, self.end, self.exclude_end?) + end + + attr_reader :mode + end +end diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb index 76e84e54d3a..8f9f6d19ce8 100644 --- a/lib/gitlab/memory/instrumentation.rb +++ b/lib/gitlab/memory/instrumentation.rb @@ -25,7 +25,7 @@ module Gitlab def self.ensure_feature_flag! return unless available? - enabled = Feature.enabled?(:trace_memory_allocations) + enabled = Feature.enabled?(:trace_memory_allocations, default_enabled: true) return if enabled == Thread.trace_memory_allocations MUTEX.synchronize do diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb new file mode 100644 index 00000000000..3dda68bf93f --- /dev/null +++ b/lib/gitlab/metrics/background_transaction.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + class BackgroundTransaction < Transaction + # Separate web transaction instance and background transaction instance + BACKGROUND_THREAD_KEY = :_gitlab_metrics_background_transaction + BACKGROUND_BASE_LABEL_KEYS = %i(endpoint_id feature_category).freeze + + class << self + def current + Thread.current[BACKGROUND_THREAD_KEY] + end + + def prometheus_metric(name, type, &block) + fetch_metric(type, name) do + # set default metric options + docstring "#{name.to_s.humanize} #{type}" + + evaluate(&block) + # always filter sensitive labels and merge with base ones + label_keys BACKGROUND_BASE_LABEL_KEYS | (label_keys - ::Gitlab::Metrics::Transaction::FILTERED_LABEL_KEYS) + end + end + end + + def run + Thread.current[BACKGROUND_THREAD_KEY] = self + + yield + ensure + Thread.current[BACKGROUND_THREAD_KEY] = nil + end + + def labels + @labels ||= { + endpoint_id: current_context&.get_attribute(:caller_id), + feature_category: current_context&.get_attribute(:feature_category) + } + end + + private + + def current_context + Labkit::Context.current + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 76175b465e4..3d29d38fa1f 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -7,7 +7,7 @@ module Gitlab module Samplers class RubySampler < BaseSampler DEFAULT_SAMPLING_INTERVAL_SECONDS = 60 - GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze + GC_REPORT_BUCKETS = [0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 1].freeze def initialize(*) GC::Profiler.clear diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb new file mode 100644 index 00000000000..a9355eeae40 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/action_cable.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + class ActionCable < ActiveSupport::Subscriber + include Gitlab::Utils::StrongMemoize + + attach_to :action_cable + + SINGLE_CLIENT_TRANSMISSION = :action_cable_single_client_transmissions_total + TRANSMIT_SUBSCRIPTION_CONFIRMATION = :action_cable_subscription_confirmations_total + TRANSMIT_SUBSCRIPTION_REJECTION = :action_cable_subscription_rejections_total + BROADCAST = :action_cable_broadcasts_total + + def transmit_subscription_confirmation(event) + confirm_subscription_counter.increment + end + + def transmit_subscription_rejection(event) + reject_subscription_counter.increment + end + + def transmit(event) + transmit_counter.increment + end + + def broadcast(event) + broadcast_counter.increment + end + + private + + def transmit_counter + strong_memoize("transmission_counter") do + ::Gitlab::Metrics.counter( + SINGLE_CLIENT_TRANSMISSION, + 'The number of ActionCable messages transmitted to any client in any channel' + ) + end + end + + def broadcast_counter + strong_memoize("broadcast_counter") do + ::Gitlab::Metrics.counter( + BROADCAST, + 'The number of ActionCable broadcasts emitted' + ) + end + end + + def confirm_subscription_counter + strong_memoize("confirm_subscription_counter") do + ::Gitlab::Metrics.counter( + TRANSMIT_SUBSCRIPTION_CONFIRMATION, + 'The number of ActionCable subscriptions from clients confirmed' + ) + end + end + + def reject_subscription_counter + strong_memoize("reject_subscription_counter") do + ::Gitlab::Metrics.counter( + TRANSMIT_SUBSCRIPTION_REJECTION, + 'The number of ActionCable subscriptions from clients rejected' + ) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index d725d8d7b29..5eefef02507 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,6 +9,16 @@ module Gitlab IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze + SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze + + DURATION_BUCKET = [0.05, 0.1, 0.25].freeze + + # This event is published from ActiveRecordBaseTransactionMetrics and + # used to record a database transaction duration when calling + # ActiveRecord::Base.transaction {} block. + def transaction(event) + observe(:gitlab_database_transaction_seconds, event) + end def sql(event) # Mark this thread as requiring a database connection. This is used @@ -17,51 +27,57 @@ module Gitlab Thread.current[:uses_db_connection] = true payload = event.payload - return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) + return if ignored_query?(payload) - increment_db_counters(payload) + increment(:db_count) + increment(:db_cached_count) if cached_query?(payload) + increment(:db_write_count) unless select_sql_command?(payload) - current_transaction&.observe(:gitlab_sql_duration_seconds, event.duration / 1000.0) do - buckets [0.05, 0.1, 0.25] - end + observe(:gitlab_sql_duration_seconds, event) end def self.db_counter_payload return {} unless Gitlab::SafeRequestStore.active? - DB_COUNTERS.map do |counter| - [counter, Gitlab::SafeRequestStore[counter].to_i] - end.to_h + payload = {} + DB_COUNTERS.each do |counter| + payload[counter] = Gitlab::SafeRequestStore[counter].to_i + end + payload end private - def select_sql_command?(payload) - payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i) + def ignored_query?(payload) + payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) end - def increment_db_counters(payload) - increment(:db_count) - - if payload.fetch(:cached, payload[:name] == 'CACHE') - increment(:db_cached_count) - end + def cached_query?(payload) + payload.fetch(:cached, payload[:name] == 'CACHE') + end - increment(:db_write_count) unless select_sql_command?(payload) + def select_sql_command?(payload) + payload[:sql].match(SQL_COMMANDS_WITH_COMMENTS_REGEX) end def increment(counter) current_transaction&.increment("gitlab_transaction_#{counter}_total".to_sym, 1) - if Gitlab::SafeRequestStore.active? - Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + end + + def observe(histogram, event) + current_transaction&.observe(histogram, event.duration / 1000.0) do + buckets DURATION_BUCKET end end def current_transaction - ::Gitlab::Metrics::Transaction.current + ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current end end end end end + +Gitlab::Metrics::Subscribers::ActiveRecord.prepend_if_ee('EE::Gitlab::Metrics::Subscribers::ActiveRecord') diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index e18b6d003e0..b1a1045a1f0 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -60,12 +60,27 @@ module Gitlab # ancestor to most nested object respectively. This uses a `depth` column # where `1` is defined as the depth for the base and increment as we go up # each parent. + # + # Note: By default the order is breadth-first # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors(upto: nil, hierarchy_order: nil) - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) - recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order - - read_only(recursive_query) + if use_distinct? + expose_depth = hierarchy_order.present? + hierarchy_order ||= :asc + + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct + + # if hierarchy_order is given, the calculated `depth` should be present in SELECT + if expose_depth + read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) + else + read_only(remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order)) + end + else + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) + recursive_query = recursive_query.order(depth: hierarchy_order) if hierarchy_order + read_only(recursive_query) + end end # rubocop: enable CodeReuse/ActiveRecord @@ -74,9 +89,22 @@ module Gitlab # # When `with_depth` is `true`, a `depth` column is included where it starts with `1` for the base objects # and incremented as we go down the descendant tree + # rubocop: disable CodeReuse/ActiveRecord def base_and_descendants(with_depth: false) - read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) + if use_distinct? + # Always calculate `depth`, remove it later if with_depth is false + base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct + + if with_depth + read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: :asc)) + else + read_only(remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc)) + end + else + read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) + end end + # rubocop: enable CodeReuse/ActiveRecord # Returns a relation that includes the base objects, their ancestors, # and the descendants of the base objects. @@ -108,13 +136,21 @@ module Gitlab ancestors_table = ancestors.alias_to(objects_table) descendants_table = descendants.alias_to(objects_table) + ancestors_scope = model.unscoped.from(ancestors_table) + descendants_scope = model.unscoped.from(descendants_table) + + if use_distinct? + ancestors_scope = ancestors_scope.distinct + descendants_scope = descendants_scope.distinct + end + relation = model .unscoped .with .recursive(ancestors.to_arel, descendants.to_arel) .from_union([ - model.unscoped.from(ancestors_table), - model.unscoped.from(descendants_table) + ancestors_scope, + descendants_scope ]) read_only(relation) @@ -123,12 +159,28 @@ module Gitlab private + # Use distinct on the Namespace queries to avoid bad planner behavior in PG11. + def use_distinct? + (model <= Namespace) && options[:use_distinct] + end + + # Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries + # and ordering the rows based on the `depth` column to maintain the row order. + # + # rubocop: disable CodeReuse/ActiveRecord + def remove_depth_and_maintain_order(relation, hierarchy_order: :asc) + joined_relation = model.joins("INNER JOIN (#{relation.select(:id, :depth).to_sql}) namespaces_join_table on namespaces_join_table.id = #{model.table_name}.id").order("namespaces_join_table.depth" => hierarchy_order) + + model.from(Arel::Nodes::As.new(joined_relation.arel, objects_table)) + end + # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def base_and_ancestors_cte(stop_id = nil, hierarchy_order = nil) cte = SQL::RecursiveCTE.new(:base_and_ancestors) base_query = ancestors_base.except(:order) - base_query = base_query.select("1 as #{DEPTH_COLUMN}", "ARRAY[id] AS tree_path", "false AS tree_cycle", objects_table[Arel.star]) if hierarchy_order + base_query = base_query.select("1 as #{DEPTH_COLUMN}", "ARRAY[#{objects_table.name}.id] AS tree_path", "false AS tree_cycle", objects_table[Arel.star]) if hierarchy_order cte << base_query @@ -161,7 +213,7 @@ module Gitlab cte = SQL::RecursiveCTE.new(:base_and_descendants) base_query = descendants_base.except(:order) - base_query = base_query.select("1 AS #{DEPTH_COLUMN}", "ARRAY[id] AS tree_path", "false AS tree_cycle", objects_table[Arel.star]) if with_depth + base_query = base_query.select("1 AS #{DEPTH_COLUMN}", "ARRAY[#{objects_table.name}.id] AS tree_path", "false AS tree_cycle", objects_table[Arel.star]) if with_depth cte << base_query diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index d51d718c826..b29240985f1 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -2,22 +2,61 @@ module Gitlab module OptimisticLocking + MAX_RETRIES = 100 + module_function - def retry_lock(subject, retries = nil, &block) - retries ||= 100 - # TODO(Observability): We should be recording details of the number of retries and the duration of the total execution here - ActiveRecord::Base.transaction do - yield(subject) - end - rescue ActiveRecord::StaleObjectError - retries -= 1 - raise unless retries >= 0 + def retry_lock(subject, max_retries = MAX_RETRIES, name:, &block) + start_time = Gitlab::Metrics::System.monotonic_time + retry_attempts = 0 + + begin + ActiveRecord::Base.transaction do + yield(subject) + end + rescue ActiveRecord::StaleObjectError + raise unless retry_attempts < max_retries + + subject.reset - subject.reset - retry + retry_attempts += 1 + retry + ensure + retry_lock_histogram.observe({}, retry_attempts) + + log_optimistic_lock_retries( + name: name, + retry_attempts: retry_attempts, + start_time: start_time) + end end alias_method :retry_optimistic_lock, :retry_lock + + def log_optimistic_lock_retries(name:, retry_attempts:, start_time:) + return unless retry_attempts > 0 + + elapsed_time = Gitlab::Metrics::System.monotonic_time - start_time + + retry_lock_logger.info( + message: "Optimistic Lock released with retries", + name: name, + retries: retry_attempts, + time_s: elapsed_time) + end + + def retry_lock_logger + @retry_lock_logger ||= Gitlab::Services::Logger.build + end + + def retry_lock_histogram + @retry_lock_histogram ||= + Gitlab::Metrics.histogram( + :gitlab_optimistic_locking_retries, + 'Number of retry attempts to execute optimistic retry lock', + {}, + [0, 1, 2, 3, 5, 10, 50] + ) + end end end diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb new file mode 100644 index 00000000000..0c8ec02a56b --- /dev/null +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # This class stores information for one column (or SQL expression) which can be used in an + # ORDER BY SQL clasue. + # The goal of this class is to encapsulate all the metadata in one place which are needed to + # make keyset pagination work in a generalized way. + # + # == Arguments + # + # **order expression** (Arel::Nodes::Node | String) + # + # The actual SQL expression for the ORDER BY clause. + # + # Examples: + # # Arel column order definition + # Project.arel_table[:id].asc # ORDER BY projects.id ASC + # + # # Arel expression, calculated order definition + # Arel::Nodes::NamedFunction.new("COALESCE", [Project.arel_table[:issue_count].asc, 0]).asc # ORDER BY COALESCE(projects.issue_count, 0) + # + # # Another Arel expression + # Arel::Nodes::Multiplication(Issue.arel_table[:weight], Issue.arel_table[:time_spent]).desc + # + # # Raw string order definition + # 'issues.type DESC NULLS LAST' + # + # **column_expression** (Arel::Nodes::Node | String) + # + # Expression for the database column or an expression. This value will be used with logical operations (>, <, =, !=) + # when building the database query for the next page. + # + # Examples: + # # Arel column reference + # Issue.arel_table[:title] + # + # # Calculated value + # Arel::Nodes::Multiplication(Issue.arel_table[:weight], Issue.arel_table[:time_spent]) + # + # **attribute_name** (String | Symbol) + # + # An attribute on the loaded ActiveRecord model where the value can be obtained. + # + # Examples: + # # Simple attribute definition + # attribute_name = :title + # + # # Later on this attribute will be used like this: + # my_record = Issue.find(x) + # value = my_record[attribute_name] # reads data from the title column + # + # # Calculated value based on an Arel or raw SQL expression + # + # attribute_name = :lowercase_title + # + # # `lowercase_title` is not is not a table column therefore we need to make sure it's available in the `SELECT` clause + # + # my_record = Issue.select(:id, 'LOWER(title) as lowercase_title').last + # value = my_record[:lowercase_title] + # + # **distinct** + # + # Boolean value. + # + # Tells us whether the database column contains only distinct values. If the column is covered by + # a unique index then set to true. + # + # **nullable** (:not_nullable | :nulls_last | :nulls_first) + # + # Tells us whether the database column is nullable or not. This information can be + # obtained from the DB schema. + # + # If the column is not nullable, set this attribute to :not_nullable. + # + # If the column is nullable, then additional information is needed. Based on the ordering, the null values + # will show up at the top or at the bottom of the resultset. + # + # Examples: + # # Nulls are showing up at the top (for example: ORDER BY column ASC): + # nullable = :nulls_first + # + # # Nulls are showing up at the bottom (for example: ORDER BY column DESC): + # nullable = :nulls_last + # + # **order_direction** + # + # :asc or :desc + # + # Note: this is an optional attribute, the value will be inferred from the order_expression. + # Sometimes it's not possible to infer the order automatically. In this case an exception will be + # raised (when the query is executed). If the reverse order cannot be computed, it must be provided explicitly. + # + # **reversed_order_expression** + # + # The reversed version of the order_expression. + # + # A ColumnOrderDefinition object is able to reverse itself which is used when paginating backwards. + # When a complex order_expression is provided (raw string), then reversing the order automatically + # is not possible. In this case an exception will be raised. + # + # Example: + # + # order_expression = Project.arel_table[:id].asc + # reversed_order_expression = Project.arel_table[:id].desc + # + # **add_to_projections** + # + # Set to true if the column is not part of the queried table. (Not part of SELECT *) + # + # Example: + # + # - When the order is a calculated expression or the column is in another table (JOIN-ed) + # + # If the add_to_projections is true, the query builder will automatically add the column to the SELECT values + class ColumnOrderDefinition + REVERSED_ORDER_DIRECTIONS = { asc: :desc, desc: :asc }.freeze + REVERSED_NULL_POSITIONS = { nulls_first: :nulls_last, nulls_last: :nulls_first }.freeze + AREL_ORDER_CLASSES = { Arel::Nodes::Ascending => :asc, Arel::Nodes::Descending => :desc }.freeze + ALLOWED_NULLABLE_VALUES = [:not_nullable, :nulls_first, :nulls_last].freeze + + attr_reader :attribute_name, :column_expression, :order_expression, :add_to_projections + + def initialize(attribute_name:, order_expression:, column_expression: nil, reversed_order_expression: nil, nullable: :not_nullable, distinct: true, order_direction: nil, add_to_projections: false) + @attribute_name = attribute_name + @order_expression = order_expression + @column_expression = column_expression || calculate_column_expression(order_expression) + @distinct = distinct + @reversed_order_expression = reversed_order_expression || calculate_reversed_order(order_expression) + @nullable = parse_nullable(nullable, distinct) + @order_direction = parse_order_direction(order_expression, order_direction) + @add_to_projections = add_to_projections + end + + def reverse + self.class.new( + attribute_name: attribute_name, + column_expression: column_expression, + order_expression: reversed_order_expression, + reversed_order_expression: order_expression, + nullable: not_nullable? ? :not_nullable : REVERSED_NULL_POSITIONS[nullable], + distinct: distinct, + order_direction: REVERSED_ORDER_DIRECTIONS[order_direction] + ) + end + + def ascending_order? + order_direction == :asc + end + + def descending_order? + order_direction == :desc + end + + def nulls_first? + nullable == :nulls_first + end + + def nulls_last? + nullable == :nulls_last + end + + def not_nullable? + nullable == :not_nullable + end + + def nullable? + !not_nullable? + end + + def distinct? + distinct + end + + private + + attr_reader :reversed_order_expression, :nullable, :distinct, :order_direction + + def calculate_reversed_order(order_expression) + unless AREL_ORDER_CLASSES.has_key?(order_expression.class) # Arel can reverse simple orders + raise "Couldn't determine reversed order for `#{order_expression}`, please provide the `reversed_order_expression` parameter." + end + + order_expression.reverse + end + + def calculate_column_expression(order_expression) + if order_expression.respond_to?(:expr) + order_expression.expr + else + raise("Couldn't calculate the column expression. Please pass an ARel node as the order_expression, not a string.") + end + end + + def parse_order_direction(order_expression, order_direction) + transformed_order_direction = if order_direction.nil? && AREL_ORDER_CLASSES[order_expression.class] + AREL_ORDER_CLASSES[order_expression.class] + elsif order_direction.present? + order_direction.to_s.downcase.to_sym + end + + unless REVERSED_ORDER_DIRECTIONS.has_key?(transformed_order_direction) + raise "Invalid or missing `order_direction` (value: #{order_direction}) was given, the allowed values are: :asc or :desc" + end + + transformed_order_direction + end + + def parse_nullable(nullable, distinct) + if ALLOWED_NULLABLE_VALUES.exclude?(nullable) + raise "Invalid `nullable` is given (value: #{nullable}), the allowed values are: #{ALLOWED_NULLABLE_VALUES.join(', ')}" + end + + if nullable != :not_nullable && distinct + raise 'Invalid column definition, `distinct` and `nullable` columns are not allowed at the same time' + end + + nullable + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb new file mode 100644 index 00000000000..e8e68a5c4a5 --- /dev/null +++ b/lib/gitlab/pagination/keyset/order.rb @@ -0,0 +1,248 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + # This class is a special ORDER BY clause which is compatible with ActiveRecord. It helps + # building keyset paginated queries. + # + # In ActiveRecord we use the `order()` method which will generate the `ORDER BY X` SQL clause + # + # Project.where(active: true).order(id: :asc) + # + # # Or + # + # Project.where(active: true).order(created_at: :asc, id: desc) + # + # Gitlab::Pagination::Keyset::Order class encapsulates more information about the order columns + # in order to implement keyset pagination in a generic way + # + # - Extract values from a record (usually the last item of the previous query) + # - Build query conditions based on the column configuration + # + # Example 1: Order by primary key + # + # # Simple order definition for the primary key as an ActiveRecord scope + # scope :id_asc_ordered, -> { + # keyset_order = Gitlab::Pagination::Keyset::Order.build([ + # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + # attribute: :id, + # order_expression: Project.arel_table[:id].asc + # ) + # ]) + # + # reorder(keyset_order) + # } + # + # # ... Later in the application code: + # + # # Compatible with ActiveRecord's `order()` method + # page1 = Project.where(active: true).id_asc_ordered.limit(5) + # keyset_order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(page1) + # + # last_record = page1.last + # cursor_values = keyset_order.cursor_attributes_for_node(last_record) # { id: x } + # + # page2 = keyset_order.apply_cursor_conditions(Project.where(active: true).id_asc_ordered, cursor_values).limit(5) + # + # last_record = page2.last + # cursor_values = keyset_order.cursor_attributes_for_node(last_record) + # + # page3 = keyset_order.apply_cursor_conditions(Project.where(active: true).id_asc_ordered, cursor_values).limit(5) + # + # Example 2: Order by creation time and primary key (primary key is the tie breaker) + # + # scope :created_at_ordered, -> { + # keyset_order = Gitlab::Pagination::Keyset::Order.build([ + # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + # attribute: :created_at, + # column_expression: Project.arel_table[:created_at], + # order_expression: Project.arel_table[:created_at].asc, + # distinct: false, # values in the column are not unique + # nullable: :nulls_last # we might see NULL values (bottom) + # ), + # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + # attribute: :id, + # order_expression: Project.arel_table[:id].asc + # ) + # ]) + # + # reorder(keyset_order) + # } + # + class Order < Arel::Nodes::SqlLiteral + attr_reader :column_definitions + + def initialize(column_definitions:) + @column_definitions = column_definitions + + super(to_sql_literal(@column_definitions)) + end + + # Tells whether the given ActiveRecord::Relation has keyset ordering + def self.keyset_aware?(scope) + scope.order_values.first.is_a?(self) && scope.order_values.one? + end + + def self.extract_keyset_order_object(scope) + scope.order_values.first + end + + def self.build(column_definitions) + new(column_definitions: column_definitions) + end + + def cursor_attributes_for_node(node) + column_definitions.each_with_object({}) do |column_definition, hash| + field_value = node[column_definition.attribute_name] + hash[column_definition.attribute_name] = if field_value.is_a?(Time) + field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') + elsif field_value.nil? + nil + else + field_value.to_s + end + end + end + + # This methods builds the conditions for the keyset pagination + # + # Example: + # + # |created_at|id| + # |----------|--| + # |2020-01-01| 1| + # | null| 2| + # | null| 3| + # |2020-02-01| 4| + # + # Note: created_at is not distinct and nullable + # Order `ORDER BY created_at DESC, id DESC` + # + # We get the following cursor values from the previous page: + # { id: 4, created_at: '2020-02-01' } + # + # To get the next rows, we need to build the following conditions: + # + # (created_at = '2020-02-01' AND id < 4) OR (created_at < '2020-01-01') + # + # DESC ordering ensures that NULL values are on top so we don't need conditions for NULL values + # + # Another cursor example: + # { id: 3, created_at: nil } + # + # To get the next rows, we need to build the following conditions: + # + # (id < 3 AND created_at IS NULL) OR (created_at IS NOT NULL) + def build_where_values(values) + return if values.blank? + + verify_incoming_values!(values) + + where_values = [] + + reversed_column_definitions = column_definitions.reverse + reversed_column_definitions.each_with_index do |column_definition, i| + value = values[column_definition.attribute_name] + + conditions_for_column(column_definition, value).each do |condition| + column_definitions_after_index = reversed_column_definitions.last(column_definitions.reverse.size - i - 1) + + equal_conditon_for_rest = column_definitions_after_index.map do |definition| + definition.column_expression.eq(values[definition.attribute_name]) + end + + where_values << Arel::Nodes::Grouping.new(Arel::Nodes::And.new([condition, *equal_conditon_for_rest].compact)) + end + end + + build_or_query(where_values) + end + + # rubocop: disable CodeReuse/ActiveRecord + def apply_cursor_conditions(scope, values = {}) + scope = apply_custom_projections(scope) + scope.where(build_where_values(values)) + end + # rubocop: enable CodeReuse/ActiveRecord + + def reversed_order + self.class.build(column_definitions.map(&:reverse)) + end + + private + + # Adds extra columns to the SELECT clause + def apply_custom_projections(scope) + additional_projections = column_definitions.select(&:add_to_projections).map do |column_definition| + # avoid mutating the original column_expression + column_definition.column_expression.dup.as(column_definition.attribute_name).to_sql + end + + scope = scope.select(*scope.arel.projections, *additional_projections) if additional_projections + scope + end + + def conditions_for_column(column_definition, value) + conditions = [] + # Depending on the order, build a query condition fragment for taking the next rows + if column_definition.distinct? || (!column_definition.distinct? && value.present?) + conditions << compare_column_with_value(column_definition, value) + end + + # When the column is nullable, additional conditions for NULL a NOT NULL values are necessary. + # This depends on the position of the nulls (top or bottom of the resultset). + if column_definition.nulls_first? && value.blank? + conditions << column_definition.column_expression.not_eq(nil) + elsif column_definition.nulls_last? && value.present? + conditions << column_definition.column_expression.eq(nil) + end + + conditions + end + + def compare_column_with_value(column_definition, value) + if column_definition.descending_order? + column_definition.column_expression.lt(value) + else + column_definition.column_expression.gt(value) + end + end + + def build_or_query(expressions) + or_expression = expressions.reduce { |or_expression, expression| Arel::Nodes::Or.new(or_expression, expression) } + + Arel::Nodes::Grouping.new(or_expression) + end + + def to_sql_literal(column_definitions) + column_definitions.map do |column_definition| + if column_definition.order_expression.respond_to?(:to_sql) + column_definition.order_expression.to_sql + else + column_definition.order_expression.to_s + end + end.join(', ') + end + + def verify_incoming_values!(values) + value_keys = values.keys.map(&:to_s) + order_attrbute_names = column_definitions.map(&:attribute_name).map(&:to_s) + missing_items = order_attrbute_names - value_keys + extra_items = value_keys - order_attrbute_names + + if missing_items.any? || extra_items.any? + error_text = ['Incorrect cursor values were given'] + + error_text << "Extra items: #{extra_items.join(', ')}" if extra_items.any? + error_text << "Missing items: #{missing_items.join(', ')}" if missing_items.any? + + error_text.compact + + raise error_text.join('. ') + end + end + end + end + end +end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb index 133d777fc32..ac5c907465e 100644 --- a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -17,7 +17,7 @@ module Gitlab # to a structured log # rubocop:disable Gitlab/ModuleWithInstanceVariables def enqueue_stats_job(request_id) - return unless gather_stats? + return unless Feature.enabled?(:performance_bar_stats) @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) @@ -43,12 +43,6 @@ module Gitlab ) end # rubocop:enable Gitlab/ModuleWithInstanceVariables - - def gather_stats? - return unless Feature.enabled?(:performance_bar_stats) - - Gitlab.com? || Gitlab.staging? || !Rails.env.production? - end end end end diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb index 31e6b120e45..5e46e26e14e 100644 --- a/lib/gitlab/query_limiting.rb +++ b/lib/gitlab/query_limiting.rb @@ -4,9 +4,8 @@ module Gitlab module QueryLimiting # Returns true if we should enable tracking of query counts. # - # This is only enabled in production/staging if we're running on GitLab.com. - # This ensures we don't produce any errors that users can't do anything - # about themselves. + # This is only enabled in development and test to ensure we don't produce + # any errors that users of other environments can't do anything about themselves. def self.enable? Rails.env.development? || Rails.env.test? end @@ -19,7 +18,7 @@ module Gitlab # The issue URL is only meant to push developers into creating an issue # instead of blindly whitelisting offending blocks of code. def self.whitelist(issue_url) - return unless enable_whitelist? + return unless enable? unless issue_url.start_with?('https://') raise( @@ -30,9 +29,5 @@ module Gitlab Transaction&.current&.whitelisted = true end - - def self.enable_whitelist? - Rails.env.development? || Rails.env.test? - end end end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 065862174bb..138fae7b641 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -6,9 +6,10 @@ module Gitlab attach_to :active_record def sql(event) - unless event.payload.fetch(:cached, event.payload[:name] == 'CACHE') - Transaction.current&.increment - end + return if !Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE') + + Transaction.current.increment + Transaction.current.executed_sql(event.payload[:sql]) end end end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index e8fad067fa6..196072dddda 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -15,6 +15,7 @@ module Gitlab # the sake of keeping things simple we hardcode this value here, it's not # supposed to be changed very often anyway. THRESHOLD = 100 + LOG_THRESHOLD = THRESHOLD * 1.5 # Error that is raised whenever exceeding the maximum number of queries. ThresholdExceededError = Class.new(StandardError) @@ -45,6 +46,7 @@ module Gitlab @action = nil @count = 0 @whitelisted = false + @sql_executed = [] end # Sends a notification based on the number of executed SQL queries. @@ -60,6 +62,10 @@ module Gitlab @count += 1 unless whitelisted end + def executed_sql(sql) + @sql_executed << sql if @count <= LOG_THRESHOLD + end + def raise_error? Rails.env.test? end @@ -71,8 +77,11 @@ module Gitlab def error_message header = 'Too many SQL queries were executed' header = "#{header} in #{action}" if action + msg = "a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + log = @sql_executed.each_with_index.map { |sql, i| "#{i}: #{sql}" }.join("\n").presence + ellipsis = '...' if @count > LOG_THRESHOLD - "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + ["#{header}: #{msg}", log, ellipsis].compact.join("\n") end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index c162ee545c6..012e495502f 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -235,6 +235,35 @@ module Gitlab @execution_message[:remove_zoom] = result.message end + desc _('Add email participant(s)') + explanation _('Adds email participant(s)') + params 'email1@example.com email2@example.com (up to 6 emails)' + types Issue + condition do + Feature.enabled?(:issue_email_participants, parent) && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) + end + command :invite_email do |emails = ""| + MAX_NUMBER_OF_EMAILS = 6 + + existing_emails = quick_action_target.email_participants_emails_downcase + emails_to_add = emails.split(' ').index_by { |email| [email.downcase, email] }.except(*existing_emails).each_value.first(MAX_NUMBER_OF_EMAILS) + added_emails = [] + + emails_to_add.each do |email| + new_participant = quick_action_target.issue_email_participants.create(email: email) + added_emails << email if new_participant.persisted? + end + + if added_emails.any? + message = _("added %{emails}") % { emails: added_emails.to_sentence } + SystemNoteService.add_email_participants(quick_action_target, quick_action_target.project, current_user, message) + @execution_message[:invite_email] = message.upcase_first << "." + else + @execution_message[:invite_email] = _("No email participants were added. Either none were provided, or they already exist.") + end + end + private def zoom_link_service diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 96f2b7570b3..00739c05386 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -61,6 +61,10 @@ module Gitlab maven_app_name_regex end + def npm_package_name_regex + @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z} + end + def nuget_package_name_regex @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze end diff --git a/lib/gitlab/relative_positioning/closed_range.rb b/lib/gitlab/relative_positioning/closed_range.rb new file mode 100644 index 00000000000..8916d1face5 --- /dev/null +++ b/lib/gitlab/relative_positioning/closed_range.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class ClosedRange < RelativePositioning::Range + def initialize(lhs, rhs) + @lhs, @rhs = lhs, rhs + raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs + raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs + end + end + end +end diff --git a/lib/gitlab/relative_positioning/ending_at.rb b/lib/gitlab/relative_positioning/ending_at.rb new file mode 100644 index 00000000000..61060638ee6 --- /dev/null +++ b/lib/gitlab/relative_positioning/ending_at.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class EndingAt < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(rhs) + @rhs = rhs + raise IllegalRange, 'rhs is required' unless rhs + end + + def lhs + strong_memoize(:lhs) { rhs.lhs_neighbour } + end + end + end +end diff --git a/lib/gitlab/relative_positioning/range.rb b/lib/gitlab/relative_positioning/range.rb index 0b0ccdf5be4..3214c72eb8b 100644 --- a/lib/gitlab/relative_positioning/range.rb +++ b/lib/gitlab/relative_positioning/range.rb @@ -31,39 +31,5 @@ module Gitlab other.is_a?(RelativePositioning::Range) && lhs == other.lhs && rhs == other.rhs end end - - class ClosedRange < RelativePositioning::Range - def initialize(lhs, rhs) - @lhs, @rhs = lhs, rhs - raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs - raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs - end - end - - class StartingFrom < RelativePositioning::Range - include Gitlab::Utils::StrongMemoize - - def initialize(lhs) - @lhs = lhs - raise IllegalRange, 'lhs is required' unless lhs - end - - def rhs - strong_memoize(:rhs) { lhs.rhs_neighbour } - end - end - - class EndingAt < RelativePositioning::Range - include Gitlab::Utils::StrongMemoize - - def initialize(rhs) - @rhs = rhs - raise IllegalRange, 'rhs is required' unless rhs - end - - def lhs - strong_memoize(:lhs) { rhs.lhs_neighbour } - end - end end end diff --git a/lib/gitlab/relative_positioning/starting_from.rb b/lib/gitlab/relative_positioning/starting_from.rb new file mode 100644 index 00000000000..6ddd35a39ad --- /dev/null +++ b/lib/gitlab/relative_positioning/starting_from.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class StartingFrom < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(lhs) + @lhs = lhs + raise IllegalRange, 'lhs is required' unless lhs + end + + def rhs + strong_memoize(:rhs) { lhs.rhs_neighbour } + end + end + end +end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 647ac169f05..968ef06b085 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -82,7 +82,10 @@ module Gitlab end def puma_in_clustered_mode? - puma? && Puma.cli_config.options[:workers].to_i > 0 + return unless puma? + return unless Puma.respond_to?(:cli_config) + + Puma.cli_config.options[:workers].to_i > 0 end def max_threads diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 48f204e0b86..7561e36cc33 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -115,7 +115,7 @@ module Gitlab config[:storage] = storages - internal_socket_dir = File.join(gitaly_dir, 'internal_sockets') + internal_socket_dir = options[:internal_socket_dir] || File.join(gitaly_dir, 'internal_sockets') FileUtils.mkdir(internal_socket_dir) unless File.exist?(internal_socket_dir) config[:internal_socket_dir] = internal_socket_dir diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 5f912818605..a2696e17078 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -36,6 +36,8 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client chain.add ::Gitlab::SidekiqStatus::ClientMiddleware chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client + # Size limiter should be placed at the bottom, but before the metrics midleware + chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Client chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 4ab8d313ad8..cf768811ffd 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -34,7 +34,8 @@ module Gitlab monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime begin - yield + transaction = Gitlab::Metrics::BackgroundTransaction.new + transaction.run { yield } job_succeeded = true ensure monotonic_time_end = Gitlab::Metrics::System.monotonic_time diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/client.rb b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb new file mode 100644 index 00000000000..bc8b1989e78 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/client.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # This midleware is inserted into Sidekiq **client** middleware chain. It + # prevents the caller from dispatching a too-large job payload. The job + # payload should be small and simple. Read more at: + # https://github.com/mperham/sidekiq/wiki/Best-Practices#1-make-your-job-parameters-small-and-simple + class Client + def call(worker_class, job, queue, _redis_pool) + ::Gitlab::SidekiqMiddleware::SizeLimiter::Validator.validate!(worker_class, job) + + yield + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb new file mode 100644 index 00000000000..da6c903ccae --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/exceed_limit_error.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # A custom exception for size limiter. It contains worker class and its + # size to easier track later + class ExceedLimitError < StandardError + attr_reader :worker_class, :size, :size_limit + + def initialize(worker_class, size, size_limit) + @worker_class = worker_class + @size = size + @size_limit = size_limit + + super "#{@worker_class} job exceeds payload size limit (#{size}/#{size_limit})" + end + + def sentry_extra_data + { + worker_class: @worker_class.to_s, + size: @size.to_i, + size_limit: @size_limit.to_i + } + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb new file mode 100644 index 00000000000..2c50c4a2157 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module SizeLimiter + # Validate a Sidekiq job payload limit based on current configuration. + # This validator pulls the configuration from the environment variables: + # + # - GITLAB_SIDEKIQ_SIZE_LIMITER_MODE: the current mode of the size + # limiter. This must be either `track` or `raise`. + # + # - GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES: the size limit in bytes. + # + # If the size of job payload after serialization exceeds the limit, an + # error is tracked raised adhering to the mode. + class Validator + def self.validate!(worker_class, job) + new(worker_class, job).validate! + end + + DEFAULT_SIZE_LIMIT = 0 + + MODES = [ + TRACK_MODE = 'track', + RAISE_MODE = 'raise' + ].freeze + + attr_reader :mode, :size_limit + + def initialize( + worker_class, job, + mode: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_MODE'], + size_limit: ENV['GITLAB_SIDEKIQ_SIZE_LIMITER_LIMIT_BYTES'] + ) + @worker_class = worker_class + @job = job + + @mode = (mode || TRACK_MODE).to_s.strip + unless MODES.include?(@mode) + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter mode: #{@mode}. Fallback to #{TRACK_MODE} mode." + @mode = TRACK_MODE + end + + @size_limit = (size_limit || DEFAULT_SIZE_LIMIT).to_i + if @size_limit < 0 + ::Sidekiq.logger.warn "Invalid Sidekiq size limiter limit: #{@size_limit}" + end + end + + def validate! + return unless @size_limit > 0 + + return if allow_big_payload? + return if job_size <= @size_limit + + exception = ExceedLimitError.new(@worker_class, job_size, @size_limit) + # This should belong to Gitlab::ErrorTracking. We'll remove this + # after this epic is done: + # https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/396 + exception.set_backtrace(backtrace) + + if raise_mode? + raise exception + else + track(exception) + end + end + + private + + def job_size + # This maynot be the optimal solution, but can be acceptable solution + # for now. Internally, Sidekiq calls Sidekiq.dump_json everywhere. + # There is no clean way to intefere to prevent double serialization. + @job_size ||= ::Sidekiq.dump_json(@job).bytesize + end + + def allow_big_payload? + worker_class = @worker_class.to_s.safe_constantize + worker_class.respond_to?(:big_payload?) && worker_class.big_payload? + end + + def raise_mode? + @mode == RAISE_MODE + end + + def track(exception) + Gitlab::ErrorTracking.track_exception(exception) + end + + def backtrace + Gitlab::BacktraceCleaner.clean_backtrace(caller) + end + end + end + end +end diff --git a/lib/gitlab/string_range_marker.rb b/lib/gitlab/string_range_marker.rb index 780fe4c7725..5ddc88edf50 100644 --- a/lib/gitlab/string_range_marker.rb +++ b/lib/gitlab/string_range_marker.rb @@ -15,8 +15,10 @@ module Gitlab end end - def mark(marker_ranges) - return rich_line unless marker_ranges&.any? + def mark(ranges) + return rich_line unless ranges&.any? + + marker_ranges = ranges.map { |range| Gitlab::MarkerRange.from_range(range) } if html_escaped rich_marker_ranges = [] @@ -24,7 +26,7 @@ module Gitlab # Map the inline-diff range based on the raw line to character positions in the rich line rich_positions = position_mapping[range].flatten # Turn the array of character positions into ranges - rich_marker_ranges.concat(collapse_ranges(rich_positions)) + rich_marker_ranges.concat(collapse_ranges(rich_positions, range.mode)) end else rich_marker_ranges = marker_ranges @@ -36,7 +38,7 @@ module Gitlab offset_range = (range.begin + offset)..(range.end + offset) original_text = rich_line[offset_range] - text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1) + text = yield(original_text, left: i == 0, right: i == rich_marker_ranges.length - 1, mode: range.mode) rich_line[offset_range] = text @@ -90,21 +92,21 @@ module Gitlab end # Takes an array of integers, and returns an array of ranges covering the same integers - def collapse_ranges(positions) + def collapse_ranges(positions, mode) return [] if positions.empty? ranges = [] start = prev = positions[0] - range = start..prev + range = MarkerRange.new(start, prev, mode: mode) positions[1..-1].each do |pos| if pos == prev + 1 - range = start..pos + range = MarkerRange.new(start, pos, mode: mode) prev = pos else ranges << range start = prev = pos - range = start..prev + range = MarkerRange.new(start, prev, mode: mode) end end ranges << range diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 3be77aff07e..dc006877129 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -87,11 +87,11 @@ module Gitlab raise NotImplementedError end - def by_category(category, project = nil) + def by_category(category, project = nil, empty_category_title: nil) directory = category_directory(category) files = finder(project).list_files_for(directory) - files.map { |f| new(f, project, category: category) }.sort + files.map { |f| new(f, project, category: category.presence || empty_category_title) }.sort end def category_directory(category) diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb index 30fff9bdae9..6e579018e45 100644 --- a/lib/gitlab/template/issue_template.rb +++ b/lib/gitlab/template/issue_template.rb @@ -25,6 +25,10 @@ module Gitlab # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 project.repository.issue_template_names_hash end + + def by_category(category, project = nil, empty_category_title: nil) + super(category, project, empty_category_title: _('Project Templates')) + end end end end diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb index 7491c8f26c6..241a823d870 100644 --- a/lib/gitlab/template/merge_request_template.rb +++ b/lib/gitlab/template/merge_request_template.rb @@ -25,6 +25,10 @@ module Gitlab # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 project.repository.merge_request_template_names_hash end + + def by_category(category, project = nil, empty_category_title: nil) + super(category, project, empty_category_title: _('Project Templates')) + end end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 09697705361..9bb793a75cc 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -25,10 +25,10 @@ module Gitlab end def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil) # rubocop:disable Metrics/ParameterLists - context += [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context] + contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context, *context] - snowplow.event(category, action, label: label, property: property, value: value, context: context) - product_analytics.event(category, action, label: label, property: property, value: value, context: context) + snowplow.event(category, action, label: label, property: property, value: value, context: contexts) + product_analytics.event(category, action, label: label, property: property, value: value, context: contexts) end def self_describing_event(schema_url, data:, context: nil) diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 92fdd008249..8ce16c11267 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -15,10 +15,14 @@ module Gitlab end def environment - return 'production' if Gitlab.com_and_canary? - return 'staging' if Gitlab.staging? + return 'production' if Gitlab.com? + + return 'org' if Gitlab.org? + + return 'self-managed' if Rails.env.production? + 'development' end diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index bc7b8bd2b94..86cd91f0a32 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -104,12 +104,12 @@ module Gitlab end def fetch_last_cached_commits_list - cache_key = ['projects', project.id, 'last_commits_list', commit.id, ensured_path, offset, limit] + cache_key = ['projects', project.id, 'last_commits', commit.id, ensured_path, offset, limit] commits = Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do repository .list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit, literal_pathspec: true) - .transform_values!(&:to_hash) + .transform_values! { |commit| commit_to_hash(commit) } end commits.transform_values! { |value| Commit.from_hash(value, project) } @@ -121,6 +121,12 @@ module Gitlab resolved_commits[commit.id] ||= commit end + def commit_to_hash(commit) + commit.to_hash.tap do |hash| + hash[:message] = hash[:message].to_s.truncate_bytes(1.kilobyte, omission: '...') + end + end + def commit_path(commit) Gitlab::Routing.url_helpers.project_commit_path(project, commit) end diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index 8483334800b..1dc660e574b 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -5,9 +5,6 @@ module Gitlab module Docs # Helper with functions to be used by HAML templates module Helper - HEADER = %w(field value).freeze - SKIP_KEYS = %i(description).freeze - def auto_generated_comment <<-MARKDOWN.strip_heredoc --- @@ -27,35 +24,33 @@ module Gitlab end def render_name(name) - "## `#{name}`\n" + "### `#{name}`" end def render_description(object) - object.description + return 'Missing description' unless object[:description].present? + + object[:description] end - def render_attribute_row(key, value) - value = Gitlab::Usage::Docs::ValueFormatter.format(key, value) - table_row(["`#{key}`", value]) + def render_yaml_link(yaml_path) + "[YAML definition](#{yaml_path})" end - def render_attributes_table(object) - <<~MARKDOWN + def render_status(object) + "Status: #{format(:status, object[:status])}" + end - #{table_row(HEADER)} - #{table_row(HEADER.map { '---' })} - #{table_value_rows(object.attributes)} - MARKDOWN + def render_owner(object) + "Group: `#{object[:product_group]}`" end - def table_value_rows(attributes) - attributes.reject { |k, _| k.in?(SKIP_KEYS) }.map do |key, value| - render_attribute_row(key, value) - end.join("\n") + def render_tiers(object) + "Tiers:#{format(:tier, object[:tier])}" end - def table_row(array) - "| #{array.join(' | ')} |" + def format(key, value) + Gitlab::Usage::Docs::ValueFormatter.format(key, value) end end end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 86e93be66c7..19ad668019e 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -13,16 +13,26 @@ The Metrics Dictionary is based on the following metrics definition YAML files: - - [`config/metrics`]('https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics') + - [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics) - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics) -Each table includes a `milestone`, which corresponds to the GitLab version when the metric -was released. + Each table includes a `milestone`, which corresponds to the GitLab version when the metric + was released. + + ## Metrics Definitions + \ - metrics_definitions.each do |name, object| = render_name(name) - - = render_description(object) - - = render_attributes_table(object) + \ + = render_description(object.attributes) + \ + = render_yaml_link(object.yaml_path) + \ + = render_owner(object.attributes) + \ + = render_status(object.attributes) + \ + = render_tiers(object.attributes) + \ diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb index a2dc9b081f8..379e5df4d52 100644 --- a/lib/gitlab/usage/docs/value_formatter.rb +++ b/lib/gitlab/usage/docs/value_formatter.rb @@ -5,17 +5,19 @@ module Gitlab module Docs class ValueFormatter def self.format(key, value) + return '' unless value.present? + case key when :key_path "**`#{value}`**" when :data_source value.to_s.capitalize - when :product_group + when :product_group, :product_category, :status "`#{value}`" when :introduced_by_url "[Introduced by](#{value})" when :distribution, :tier - Array(value).join(', ') + Array(value).map { |tier| " `#{tier}`" }.join(',') else value end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 01d202e4d45..4cb83348478 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,6 +4,7 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') + BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' attr_reader :path attr_reader :attributes @@ -21,10 +22,14 @@ module Gitlab attributes end + def yaml_path + "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}" + end + def validate! unless skip_validation? self.class.schemer.validate(attributes.stringify_keys).each do |error| - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) end end end @@ -59,7 +64,7 @@ module Gitlab self.new(path, definition).tap(&:validate!) rescue => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new(e.message)) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(e.message)) end def load_all_from_path!(definitions, glob_path) @@ -67,7 +72,7 @@ module Gitlab definition = load_from_file(path) if previous = definitions[definition.key] - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'")) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'")) end definitions[definition.key] = definition diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 1fc40798320..1aeca87d849 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -11,6 +11,7 @@ module Gitlab AggregatedMetricError = Class.new(StandardError) UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) + DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) DATABASE_SOURCE = 'database' REDIS_SOURCE = 'redis' @@ -30,25 +31,38 @@ module Gitlab @recorded_at = recorded_at end + def all_time_data + aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME) + end + def monthly_data - aggregated_metrics_data(**monthly_time_range) + aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME)) end def weekly_data - aggregated_metrics_data(**weekly_time_range) + aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME)) end private attr_accessor :aggregated_metrics, :recorded_at - def aggregated_metrics_data(start_date:, end_date:) + def aggregated_metrics_data(start_date:, end_date:, time_frame:) aggregated_metrics.each_with_object({}) do |aggregation, data| next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development) + next unless aggregation[:time_frame].include?(time_frame) case aggregation[:source] when REDIS_SOURCE - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + if time_frame == Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME + data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception( + DisallowedAggregationTimeFrame.new("Aggregation time frame: 'all' is not allowed for aggregation with source: '#{REDIS_SOURCE}'") + ) + else + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + end when DATABASE_SOURCE next unless Feature.enabled?('database_sourced_aggregated_metrics', default_enabled: false, type: :development) @@ -155,3 +169,5 @@ module Gitlab end end end + +Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_if_ee('EE::Gitlab::Usage::Metrics::Aggregates::Aggregate') diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb index 33678d2b813..a01efbdb1a6 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -55,15 +55,15 @@ module Gitlab end def time_period_to_human_name(time_period) - return Gitlab::Utils::UsageData::ALL_TIME_PERIOD_HUMAN_NAME if time_period.blank? + return Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_period.blank? start_date = time_period.first.to_date end_date = time_period.last.to_date if (end_date - start_date).to_i > 7 - Gitlab::Utils::UsageData::MONTHLY_PERIOD_HUMAN_NAME + Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME else - Gitlab::Utils::UsageData::WEEKLY_PERIOD_HUMAN_NAME + Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME end end end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb new file mode 100644 index 00000000000..33f025770e0 --- /dev/null +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module NamesSuggestions + class Generator < ::Gitlab::UsageData + FREE_TEXT_METRIC_NAME = "<please fill metric name>" + + class << self + def generate(key_path) + uncached_data.deep_stringify_keys.dig(*key_path.split('.')) + end + + private + + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + name_suggestion(column: column, relation: relation, prefix: 'count') + end + + def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) + end + + def redis_usage_counter + FREE_TEXT_METRIC_NAME + end + + def alt_usage_data(*) + FREE_TEXT_METRIC_NAME + end + + def redis_usage_data_totals(counter) + counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME} + end + + def sum(relation, column, *rest) + name_suggestion(column: column, relation: relation, prefix: 'sum') + end + + def estimate_batch_distinct_count(relation, column = nil, *rest) + name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') + end + + def add(*args) + "add_#{args.join('_and_')}" + end + + def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) + parts = [prefix] + + if column + parts << parse_target(column) + parts << 'from' + end + + source = parse_source(relation) + constraints = parse_constraints(relation: relation, column: column, distinct: distinct) + + if constraints.include?(source) + parts << "<adjective describing: '#{constraints}'>" + end + + parts << source + parts.compact.join('_') + end + + def parse_constraints(relation:, column: nil, distinct: nil) + connection = relation.connection + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints + .new(connection) + .accept(arel(relation: relation, column: column, distinct: distinct), collector(connection)) + .value + end + + def parse_target(column) + if column.is_a?(Arel::Attribute) + "#{column.relation.name}.#{column.name}" + else + column + end + end + + def parse_source(relation) + relation.table_name + end + + def collector(connection) + Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) + end + + def arel(relation:, column: nil, distinct: nil) + column ||= relation.primary_key + + if column.is_a?(Arel::Attribute) + relation.select(column.count(distinct)).arel + else + relation.select(relation.all.table[column].count(distinct)).arel + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb new file mode 100644 index 00000000000..199395e4b20 --- /dev/null +++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module NamesSuggestions + module RelationParsers + class Constraints < ::Arel::Visitors::PostgreSQL + # rubocop:disable Naming/MethodName + def visit_Arel_Nodes_SelectCore(object, collector) + collect_nodes_for(object.wheres, collector, "") || collector + end + # rubocop:enable Naming/MethodName + + def quote(value) + "#{value}" + end + + def quote_table_name(name) + "#{name}" + end + + def quote_column_name(name) + "#{name}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 8e096a9f351..5dc3f71329d 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -60,6 +60,7 @@ module Gitlab .merge(compliance_unique_visits_data) .merge(search_unique_visits_data) .merge(redis_hll_counters) + .deep_merge(aggregated_metrics_data) end end @@ -192,7 +193,7 @@ module Gitlab container_expiration_policies_usage, service_desk_counts ).tap do |data| - data[:snippets] = data[:personal_snippets] + data[:project_snippets] + data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } end @@ -224,10 +225,9 @@ module Gitlab project_snippets: count(ProjectSnippet.where(last_28_days_time_period)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(last_28_days_time_period), :project_id) }.merge( - snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)), - aggregated_metrics_monthly + snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp)) ).tap do |data| - data[:snippets] = data[:personal_snippets] + data[:project_snippets] + data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } end @@ -242,17 +242,15 @@ module Gitlab def system_usage_data_settings { settings: { - ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? } + ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, + operating_system: alt_usage_data(fallback: nil) { operating_system } } } end def system_usage_data_weekly { - counts_weekly: { - }.merge( - aggregated_metrics_weekly - ) + counts_weekly: {} } end @@ -505,6 +503,17 @@ module Gitlab end end + def operating_system + ohai_data = Ohai::System.new.tap do |oh| + oh.all_plugins(['platform']) + end.data + + platform = ohai_data['platform'] + platform = 'raspbian' if ohai_data['platform'] == 'debian' && /armv/.match?(ohai_data['kernel']['machine']) + + "#{platform}-#{ohai_data['platform_version']}" + end + def last_28_days_time_period(column: :created_at) { column => 30.days.ago..2.days.ago } end @@ -620,6 +629,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_monitor(time_period) + # Calculate histogram only for overall as other time periods aren't available/useful here. + integrations_histogram = time_period.empty? ? histogram(::AlertManagement::HttpIntegration.active, :project_id, buckets: 1..100) : nil + { clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id), clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), @@ -629,8 +641,9 @@ module Gitlab projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id), projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), - projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id) - } + projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id), + projects_with_enabled_alert_integrations_histogram: integrations_histogram + }.compact end # rubocop: enable CodeReuse/ActiveRecord @@ -701,15 +714,13 @@ module Gitlab { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end - def aggregated_metrics_monthly - { - aggregated_metrics: aggregated_metrics.monthly_data - } - end - - def aggregated_metrics_weekly + def aggregated_metrics_data { - aggregated_metrics: aggregated_metrics.weekly_data + counts_weekly: { aggregated_metrics: aggregated_metrics.weekly_data }, + counts_monthly: { aggregated_metrics: aggregated_metrics.monthly_data }, + counts: aggregated_metrics + .all_time_data + .to_h { |key, value| ["aggregate_#{key}".to_sym, value.round] } } end @@ -821,11 +832,9 @@ module Gitlab def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. - [ - count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id), + add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id), count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id), count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id) - ].reduce(:+) end def user_minimum_id @@ -952,7 +961,7 @@ module Gitlab csv_issue_imports = distinct_count(Issues::CsvImport.where(time_period), :user_id) group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) - project_imports + bulk_imports + jira_issue_imports + csv_issue_imports + group_imports + add(project_imports, bulk_imports, jira_issue_imports, csv_issue_imports, group_imports) end # rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml new file mode 100644 index 00000000000..4c2355d526a --- /dev/null +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml @@ -0,0 +1,108 @@ +# code_review_extension_category_monthly_active_users +# This is only metrics related to the VS Code Extension for now. +# +# code_review_category_monthly_active_users +# This is the user based metrics. These should only be user based metrics and only be related to the Code Review things inside of GitLab. +# +# code_review_group_monthly_active_users +# This is an aggregation of both of the above aggregations. It's intended to represent all users who interact with our group across all of our categories. +--- +- name: code_review_group_monthly_active_users + operator: OR + feature_flag: usage_data_code_review_aggregation + source: redis + time_frame: [7d, 28d] + events: [ + 'i_code_review_user_single_file_diffs', + 'i_code_review_user_create_mr', + 'i_code_review_user_close_mr', + 'i_code_review_user_reopen_mr', + 'i_code_review_user_resolve_thread', + 'i_code_review_user_unresolve_thread', + 'i_code_review_edit_mr_title', + 'i_code_review_edit_mr_desc', + 'i_code_review_user_merge_mr', + 'i_code_review_user_create_mr_comment', + 'i_code_review_user_edit_mr_comment', + 'i_code_review_user_remove_mr_comment', + 'i_code_review_user_create_review_note', + 'i_code_review_user_publish_review', + 'i_code_review_user_create_multiline_mr_comment', + 'i_code_review_user_edit_multiline_mr_comment', + 'i_code_review_user_remove_multiline_mr_comment', + 'i_code_review_user_add_suggestion', + 'i_code_review_user_apply_suggestion', + 'i_code_review_user_assigned', + 'i_code_review_user_review_requested', + 'i_code_review_user_approve_mr', + 'i_code_review_user_unapprove_mr', + 'i_code_review_user_marked_as_draft', + 'i_code_review_user_unmarked_as_draft', + 'i_code_review_user_approval_rule_added', + 'i_code_review_user_approval_rule_deleted', + 'i_code_review_user_approval_rule_edited', + 'i_code_review_user_vs_code_api_request', + 'i_code_review_user_toggled_task_item_status', + 'i_code_review_user_create_mr_from_issue', + 'i_code_review_user_mr_discussion_locked', + 'i_code_review_user_mr_discussion_unlocked', + 'i_code_review_user_time_estimate_changed', + 'i_code_review_user_time_spent_changed', + 'i_code_review_user_assignees_changed', + 'i_code_review_user_reviewers_changed', + 'i_code_review_user_milestone_changed', + 'i_code_review_user_labels_changed' + ] +- name: code_review_category_monthly_active_users + operator: OR + feature_flag: usage_data_code_review_aggregation + source: redis + time_frame: [7d, 28d] + events: [ + 'i_code_review_user_single_file_diffs', + 'i_code_review_user_create_mr', + 'i_code_review_user_close_mr', + 'i_code_review_user_reopen_mr', + 'i_code_review_user_resolve_thread', + 'i_code_review_user_unresolve_thread', + 'i_code_review_edit_mr_title', + 'i_code_review_edit_mr_desc', + 'i_code_review_user_merge_mr', + 'i_code_review_user_create_mr_comment', + 'i_code_review_user_edit_mr_comment', + 'i_code_review_user_remove_mr_comment', + 'i_code_review_user_create_review_note', + 'i_code_review_user_publish_review', + 'i_code_review_user_create_multiline_mr_comment', + 'i_code_review_user_edit_multiline_mr_comment', + 'i_code_review_user_remove_multiline_mr_comment', + 'i_code_review_user_add_suggestion', + 'i_code_review_user_apply_suggestion', + 'i_code_review_user_assigned', + 'i_code_review_user_review_requested', + 'i_code_review_user_approve_mr', + 'i_code_review_user_unapprove_mr', + 'i_code_review_user_marked_as_draft', + 'i_code_review_user_unmarked_as_draft', + 'i_code_review_user_approval_rule_added', + 'i_code_review_user_approval_rule_deleted', + 'i_code_review_user_approval_rule_edited', + 'i_code_review_user_toggled_task_item_status', + 'i_code_review_user_create_mr_from_issue', + 'i_code_review_user_mr_discussion_locked', + 'i_code_review_user_mr_discussion_unlocked', + 'i_code_review_user_time_estimate_changed', + 'i_code_review_user_time_spent_changed', + 'i_code_review_user_assignees_changed', + 'i_code_review_user_reviewers_changed', + 'i_code_review_user_milestone_changed', + 'i_code_review_user_labels_changed' + ] +- name: code_review_extension_category_monthly_active_users + operator: OR + feature_flag: usage_data_code_review_aggregation + source: redis + time_frame: [7d, 28d] + events: [ + 'i_code_review_user_vs_code_api_request' + ] diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index 4d92202e7fd..73a55b5d5fa 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -1,3 +1,5 @@ +# Aggregated metrics that include EE only event names within `events:` attribute have to be defined at ee/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +# instead of this file. #- name: unique name of aggregated metric # operator: aggregation operator. Valid values are: # - "OR": counts unique elements that were observed triggering any of following events @@ -7,6 +9,10 @@ # source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are: # - database # - redis +# time_frame: defines time frames for aggregated metrics: +# - 7d - last 7 days +# - 28d - last 28 days +# - all - all historical available data, this time frame is not available for redis source # feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. # Corresponding feature flag should have `default_enabled` attribute set to `false`. # This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. @@ -14,18 +20,22 @@ - name: compliance_features_track_unique_visits_union operator: OR source: redis + time_frame: [7d, 28d] events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] - name: product_analytics_test_metrics_union operator: OR source: redis + time_frame: [7d, 28d] events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: product_analytics_test_metrics_intersection operator: AND source: redis + time_frame: [7d, 28d] events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: incident_management_alerts_total_unique_counts operator: OR source: redis + time_frame: [7d, 28d] events: [ 'incident_management_alert_status_changed', 'incident_management_alert_assigned', @@ -35,6 +45,7 @@ - name: incident_management_incidents_total_unique_counts operator: OR source: redis + time_frame: [7d, 28d] events: [ 'incident_management_incident_created', 'incident_management_incident_reopened', @@ -51,10 +62,11 @@ - name: i_testing_paid_monthly_active_user_total operator: OR source: redis + time_frame: [7d, 28d] events: [ - 'i_testing_web_performance_widget_total', - 'i_testing_full_code_quality_report_total', - 'i_testing_group_code_coverage_visit_total', - 'i_testing_load_performance_widget_total', - 'i_testing_metrics_report_widget_total' -] + 'i_testing_web_performance_widget_total', + 'i_testing_full_code_quality_report_total', + 'i_testing_group_code_coverage_visit_total', + 'i_testing_load_performance_widget_total', + 'i_testing_metrics_report_widget_total' + ] diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index f6bddabdd44..e1648245f3f 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -41,6 +41,9 @@ - i_package_pypi_delete_package - i_package_pypi_pull_package - i_package_pypi_push_package +- i_package_rubygems_delete_package +- i_package_rubygems_pull_package +- i_package_rubygems_push_package - i_package_tag_delete_package - i_package_tag_pull_package - i_package_tag_push_package diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 68ae239debb..336bef081a6 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -127,11 +127,15 @@ module Gitlab return unless Gitlab::CurrentSettings.usage_ping_enabled? event = event_for(event_name) - raise UnknownEvent, "Unknown event #{event_name}" unless event.present? + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? return unless feature_enabled?(event) Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) + rescue => e + # Ignore any exceptions unless is dev or test env + # The application flow should not be blocked by erros in tracking + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end # The array of valid context on which we allow tracking diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index d657c5487d7..18c5dc73de2 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -164,3 +164,43 @@ category: code_review aggregation: weekly feature_flag: usage_data_i_code_review_user_create_mr_from_issue +- name: i_code_review_user_mr_discussion_locked + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_mr_discussion_locked +- name: i_code_review_user_mr_discussion_unlocked + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_mr_discussion_unlocked +- name: i_code_review_user_time_estimate_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_time_estimate_changed +- name: i_code_review_user_time_spent_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_time_spent_changed +- name: i_code_review_user_assignees_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_assignees_changed +- name: i_code_review_user_reviewers_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_reviewers_changed +- name: i_code_review_user_milestone_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_milestone_changed +- name: i_code_review_user_labels_changed + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_labels_changed diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 79f319b2d58..80a79682338 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -439,3 +439,18 @@ redis_slot: pipeline_authoring aggregation: weekly feature_flag: usage_data_unique_users_committing_ciconfigfile +- name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile + category: pipeline_authoring + redis_slot: pipeline_authoring + aggregation: weekly + feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile +# Epic events +# +# We are using the same slot of issue events 'project_management' for +# epic events to allow data aggregation. +# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 +- name: g_project_management_epic_created + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index 3fd02164f74..1c765bb1830 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -20,3 +20,49 @@ redis_slot: ecosystem aggregation: weekly feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_slack_service_issue_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_push_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_deployment_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_wiki_page_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_merge_request_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_note_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_tag_push_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_confidential_note_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service +- name: i_ecosystem_slack_service_confidential_issue_notification + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_slack_service + diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index 78a2a587b34..b7e583003c8 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -3,109 +3,95 @@ category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_composer_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_conan_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_conan_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_container_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_container_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_debian_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_debian_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_generic_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_generic_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_golang_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_golang_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_maven_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_maven_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_npm_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_npm_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_nuget_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_nuget_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_pypi_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_pypi_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis +- name: i_package_rubygems_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package +- name: i_package_rubygems_user + category: user_packages + aggregation: weekly + redis_slot: package - name: i_package_tag_deploy_token category: deploy_token_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis - name: i_package_tag_user category: user_packages aggregation: weekly redis_slot: package - feature_flag: collect_package_events_redis diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index bf292047da0..0fe65afb237 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -324,3 +324,13 @@ redis_slot: quickactions aggregation: weekly feature_flag: usage_data_track_quickactions +- name: i_quickactions_invite_email_single + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_invite_email_multiple + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index b9856e1f74a..eb28a387a97 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -35,6 +35,15 @@ module Gitlab MR_EDIT_MR_TITLE_ACTION = 'i_code_review_edit_mr_title' MR_EDIT_MR_DESC_ACTION = 'i_code_review_edit_mr_desc' MR_CREATE_FROM_ISSUE_ACTION = 'i_code_review_user_create_mr_from_issue' + MR_DISCUSSION_LOCKED_ACTION = 'i_code_review_user_mr_discussion_locked' + MR_DISCUSSION_UNLOCKED_ACTION = 'i_code_review_user_mr_discussion_unlocked' + MR_TIME_ESTIMATE_CHANGED_ACTION = 'i_code_review_user_time_estimate_changed' + MR_TIME_SPENT_CHANGED_ACTION = 'i_code_review_user_time_spent_changed' + MR_ASSIGNEES_CHANGED_ACTION = 'i_code_review_user_assignees_changed' + MR_REVIEWERS_CHANGED_ACTION = 'i_code_review_user_reviewers_changed' + MR_INCLUDING_CI_CONFIG_ACTION = 'o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile' + MR_MILESTONE_CHANGED_ACTION = 'i_code_review_user_milestone_changed' + MR_LABELS_CHANGED_ACTION = 'i_code_review_user_labels_changed' class << self def track_mr_diffs_action(merge_request:) @@ -153,6 +162,45 @@ module Gitlab track_unique_action_by_user(MR_CREATE_FROM_ISSUE_ACTION, user) end + def track_discussion_locked_action(user:) + track_unique_action_by_user(MR_DISCUSSION_LOCKED_ACTION, user) + end + + def track_discussion_unlocked_action(user:) + track_unique_action_by_user(MR_DISCUSSION_UNLOCKED_ACTION, user) + end + + def track_time_estimate_changed_action(user:) + track_unique_action_by_user(MR_TIME_ESTIMATE_CHANGED_ACTION, user) + end + + def track_time_spent_changed_action(user:) + track_unique_action_by_user(MR_TIME_SPENT_CHANGED_ACTION, user) + end + + def track_assignees_changed_action(user:) + track_unique_action_by_user(MR_ASSIGNEES_CHANGED_ACTION, user) + end + + def track_reviewers_changed_action(user:) + track_unique_action_by_user(MR_REVIEWERS_CHANGED_ACTION, user) + end + + def track_mr_including_ci_config(user:, merge_request:) + return unless Feature.enabled?(:usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile, user, default_enabled: :yaml) + return unless merge_request.includes_ci_config? + + track_unique_action_by_user(MR_INCLUDING_CI_CONFIG_ACTION, user) + end + + def track_milestone_changed_action(user:) + track_unique_action_by_user(MR_MILESTONE_CHANGED_ACTION, user) + end + + def track_labels_changed_action(user:) + track_unique_action_by_user(MR_LABELS_CHANGED_ACTION, user) + end + private def track_unique_action_by_merge_request(action, merge_request) diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb index f757b51f73c..15c68fb3945 100644 --- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -34,6 +34,8 @@ module Gitlab event_name_for_unassign(args) when 'unlabel', 'remove_label' event_name_for_unlabel(args) + when 'invite_email' + 'invite_email' + event_name_quantifier(args.split) else name end @@ -44,10 +46,8 @@ module Gitlab if args.count == 1 && args.first == 'me' 'assign_self' - elsif args.count == 1 - 'assign_single' else - 'assign_multiple' + 'assign' + event_name_quantifier(args) end end @@ -82,6 +82,14 @@ module Gitlab 'unlabel_all' end end + + def event_name_quantifier(args) + if args.count == 1 + '_single' + else + '_multiple' + end + end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index b275bdbacde..c00e7a2aa13 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -32,6 +32,10 @@ module Gitlab raw_sql(relation, column, :distinct) end + def add(*args) + 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') + end + private def raw_sql(relation, column, distinct = nil) diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 28dc66e19f8..854fc5c917d 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -24,7 +24,8 @@ # alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled } # # * redis_usage_data method -# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent +# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, +# Gitlab::UsageDataCounters::HLLRedisCounter::EventError # returns -1 when a block is sent or hash with all values -1 when a counter is sent # different behaviour due to 2 different implementations of redis counter # @@ -38,10 +39,12 @@ module Gitlab extend self FALLBACK = -1 + HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze DISTRIBUTED_HLL_FALLBACK = -2 - ALL_TIME_PERIOD_HUMAN_NAME = "all_time" - WEEKLY_PERIOD_HUMAN_NAME = "weekly" - MONTHLY_PERIOD_HUMAN_NAME = "monthly" + ALL_TIME_TIME_FRAME_NAME = "all" + SEVEN_DAYS_TIME_FRAME_NAME = "7d" + TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d" + MAX_BUCKET_SIZE = 100 def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch @@ -86,6 +89,81 @@ module Gitlab FALLBACK end + # We don't support batching with histograms. + # Please avoid using this method on large tables. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/323949. + # + # rubocop: disable CodeReuse/ActiveRecord + def histogram(relation, column, buckets:, bucket_size: buckets.size) + # Using lambda to avoid exposing histogram specific methods + parameters_valid = lambda do + error_message = + if buckets.first == buckets.last + 'Lower bucket bound cannot equal to upper bucket bound' + elsif bucket_size == 0 + 'Bucket size cannot be zero' + elsif bucket_size > MAX_BUCKET_SIZE + "Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}" + end + + return true unless error_message + + exception = ArgumentError.new(error_message) + exception.set_backtrace(caller) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + + false + end + + return HISTOGRAM_FALLBACK unless parameters_valid.call + + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + # For example, 9 segements gives 10 buckets + bucket_segments = bucket_size - 1 + + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + # Return the histogram as a Hash because buckets are unique. + relation + .connection + .exec_query(query.to_sql) + .rows + .to_h + # Keys are converted to strings in Usage Ping JSON + .stringify_keys + rescue ActiveRecord::StatementInvalid => e + Gitlab::AppJsonLogger.error( + event: 'histogram', + relation: relation.table_name, + operation: 'histogram', + operation_args: [column, buckets.first, buckets.last, bucket_segments], + query: query.to_sql, + message: e.message + ) + + HISTOGRAM_FALLBACK + end + # rubocop: enable CodeReuse/ActiveRecord + + def add(*args) + return -1 if args.any?(&:negative?) + + args.sum + rescue StandardError + FALLBACK + end + def alt_usage_data(value = nil, fallback: FALLBACK, &block) if block_given? yield @@ -104,11 +182,13 @@ module Gitlab end end - def with_prometheus_client(fallback: nil, verify: true) + def with_prometheus_client(fallback: {}, verify: true) client = prometheus_client(verify: verify) return fallback unless client yield client + rescue + fallback end def measure_duration @@ -126,8 +206,6 @@ module Gitlab # @param event_name [String] the event name # @param values [Array|String] the values counted def track_usage_event(event_name, values) - return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end @@ -160,7 +238,7 @@ module Gitlab def redis_usage_counter yield - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError FALLBACK end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 76cf769d041..abfb7e2310e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -44,9 +44,9 @@ module Gitlab def options { - N_('VisibilityLevel|Private') => PRIVATE, - N_('VisibilityLevel|Internal') => INTERNAL, - N_('VisibilityLevel|Public') => PUBLIC + s_('VisibilityLevel|Private') => PRIVATE, + s_('VisibilityLevel|Internal') => INTERNAL, + s_('VisibilityLevel|Public') => PUBLIC } end @@ -104,12 +104,7 @@ module Gitlab end def level_name(level) - level_name = N_('VisibilityLevel|Unknown') - options.each do |name, lvl| - level_name = name if lvl == level.to_i - end - - s_(level_name) + options.key(level.to_i) || s_('VisibilityLevel|Unknown') end def level_value(level) diff --git a/lib/gitlab/word_diff/chunk_collection.rb b/lib/gitlab/word_diff/chunk_collection.rb new file mode 100644 index 00000000000..dd388f75302 --- /dev/null +++ b/lib/gitlab/word_diff/chunk_collection.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module WordDiff + class ChunkCollection + def initialize + @chunks = [] + end + + def add(chunk) + @chunks << chunk + end + + def content + @chunks.join('') + end + + def reset + @chunks = [] + end + end + end +end diff --git a/lib/gitlab/word_diff/line_processor.rb b/lib/gitlab/word_diff/line_processor.rb new file mode 100644 index 00000000000..49263962dd6 --- /dev/null +++ b/lib/gitlab/word_diff/line_processor.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Converts a line from `git diff --word-diff=porcelain` output into a segment +# +# Possible options: +# 1. Diff hunk +# 2. Chunk +# 3. Newline +module Gitlab + module WordDiff + class LineProcessor + def initialize(line) + @line = line + end + + def extract + return if empty_line? + return Segments::DiffHunk.new(full_line) if diff_hunk? + return Segments::Newline.new if newline_delimiter? + + Segments::Chunk.new(full_line) + end + + private + + attr_reader :line + + def diff_hunk? + line =~ /^@@ -/ + end + + def empty_line? + full_line == ' ' + end + + def newline_delimiter? + full_line == '~' + end + + def full_line + @full_line ||= line.delete("\n") + end + end + end +end diff --git a/lib/gitlab/word_diff/parser.rb b/lib/gitlab/word_diff/parser.rb new file mode 100644 index 00000000000..3b6d4d4d384 --- /dev/null +++ b/lib/gitlab/word_diff/parser.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +# Converts git diff --word-diff=porcelain output to Gitlab::Diff::Line objects +# see: https://git-scm.com/docs/git-diff#Documentation/git-diff.txt-porcelain +module Gitlab + module WordDiff + class Parser + include Enumerable + + def parse(lines, diff_file: nil) + return [] if lines.blank? + + # By returning an Enumerator we make it possible to search for a single line (with #find) + # without having to instantiate all the others that come after it. + Enumerator.new do |yielder| + @chunks = ChunkCollection.new + @counter = PositionsCounter.new + + lines.each do |line| + segment = LineProcessor.new(line).extract + + case segment + when Segments::DiffHunk + next if segment.first_line? + + counter.set_pos_num(old: segment.pos_old, new: segment.pos_new) + + yielder << build_line(segment.to_s, 'match', parent_file: diff_file) + + when Segments::Chunk + @chunks.add(segment) + + when Segments::Newline + yielder << build_line(@chunks.content, nil, parent_file: diff_file) + + @chunks.reset + counter.increase_pos_num + end + end + end + end + + private + + attr_reader :counter + + def build_line(content, type, options = {}) + Gitlab::Diff::Line.new( + content, type, + counter.line_obj_index, counter.pos_old, counter.pos_new, + **options).tap do + counter.increase_obj_index + end + end + end + end +end diff --git a/lib/gitlab/word_diff/positions_counter.rb b/lib/gitlab/word_diff/positions_counter.rb new file mode 100644 index 00000000000..ca66b43755f --- /dev/null +++ b/lib/gitlab/word_diff/positions_counter.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# Responsible for keeping track of line numbers and created Gitlab::Diff::Line objects +module Gitlab + module WordDiff + class PositionsCounter + def initialize + @pos_old = 1 + @pos_new = 1 + @line_obj_index = 0 + end + + attr_reader :pos_old, :pos_new, :line_obj_index + + def increase_pos_num + @pos_old += 1 + @pos_new += 1 + end + + def increase_obj_index + @line_obj_index += 1 + end + + def set_pos_num(old:, new:) + @pos_old = old + @pos_new = new + end + end + end +end diff --git a/lib/gitlab/word_diff/segments/chunk.rb b/lib/gitlab/word_diff/segments/chunk.rb new file mode 100644 index 00000000000..7c5850666f9 --- /dev/null +++ b/lib/gitlab/word_diff/segments/chunk.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Chunk is a part of the line that starts with ` `, `-`, `+` +# Consecutive chunks build a line. Line that starts with `~` is an identifier of +# end of the line. +module Gitlab + module WordDiff + module Segments + class Chunk + def initialize(line) + @line = line + end + + def removed? + line[0] == '-' + end + + def added? + line[0] == '+' + end + + def to_s + line[1..] || '' + end + + def length + to_s.length + end + + private + + attr_reader :line + end + end + end +end diff --git a/lib/gitlab/word_diff/segments/diff_hunk.rb b/lib/gitlab/word_diff/segments/diff_hunk.rb new file mode 100644 index 00000000000..88b6817676f --- /dev/null +++ b/lib/gitlab/word_diff/segments/diff_hunk.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Diff hunk is line that starts with @@ +# It contains information about start line numbers +# +# Example: +# @@ -1,4 +1,5 @@ +# +# See more: https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html +module Gitlab + module WordDiff + module Segments + class DiffHunk + def initialize(line) + @line = line + end + + def pos_old + line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 + end + + def pos_new + line.match(/\+[0-9]*/)[0].to_i.abs rescue 0 + end + + def first_line? + pos_old <= 1 && pos_new <= 1 + end + + def to_s + line + end + + private + + attr_reader :line + end + end + end +end diff --git a/lib/gitlab/word_diff/segments/newline.rb b/lib/gitlab/word_diff/segments/newline.rb new file mode 100644 index 00000000000..de8bbf252ff --- /dev/null +++ b/lib/gitlab/word_diff/segments/newline.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module WordDiff + module Segments + class Newline + def to_s + '' + end + end + end + end +end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index 7d4d4d9d13a..edff1540cb3 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -52,6 +52,12 @@ module Gitlab strong_memoize(:cert_store) do store = OpenSSL::X509::Store.new store.set_default_paths + + if Feature.enabled?(:x509_forced_cert_loading, type: :ops) + # Forcibly load the default cert file because the OpenSSL library seemingly ignores it + store.add_file(OpenSSL::X509::DEFAULT_CERT_FILE) if File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE) + end + # valid_signing_time? checks the time attributes already # this flag is required, otherwise expired certificates would become # unverified when notAfter within certificate attribute is reached diff --git a/lib/pager_duty/webhook_payload_parser.rb b/lib/pager_duty/webhook_payload_parser.rb index 11071926cf2..c17e3df1a72 100644 --- a/lib/pager_duty/webhook_payload_parser.rb +++ b/lib/pager_duty/webhook_payload_parser.rb @@ -2,7 +2,7 @@ module PagerDuty class WebhookPayloadParser - SCHEMA_PATH = File.join('lib', 'pager_duty', 'validator', 'schemas', 'message.json') + SCHEMA_PATH = Rails.root.join('lib', 'pager_duty', 'validator', 'schemas', 'message.json') def initialize(payload) @payload = payload @@ -66,7 +66,7 @@ module PagerDuty end def valid_message?(message) - ::JSONSchemer.schema(Pathname.new(SCHEMA_PATH)).valid?(message) + ::JSONSchemer.schema(SCHEMA_PATH).valid?(message) end end end diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 77108bb81ca..523e673e9e1 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -39,16 +39,20 @@ module Peek super subscribe('sql.active_record') do |_, start, finish, _, data| - if Gitlab::PerformanceBar.enabled_for_request? - detail_store << { - duration: finish - start, - sql: data[:sql].strip, - backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller), - cached: data[:cached] ? 'cached' : '' - } - end + detail_store << generate_detail(start, finish, data) if Gitlab::PerformanceBar.enabled_for_request? end end + + def generate_detail(start, finish, data) + { + duration: finish - start, + sql: data[:sql].strip, + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller), + cached: data[:cached] ? 'cached' : '' + } + end end end end + +Peek::Views::ActiveRecord.prepend_if_ee('EE::Peek::Views::ActiveRecord') diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb index 45cfa9b373d..ad9de067375 100644 --- a/lib/quality/test_level.rb +++ b/lib/quality/test_level.rb @@ -43,6 +43,7 @@ module Quality serializers services sidekiq + spam support_specs tasks uploaders diff --git a/lib/release_highlights/validator/entry.rb b/lib/release_highlights/validator/entry.rb index 3c83ca21123..133afcb52ae 100644 --- a/lib/release_highlights/validator/entry.rb +++ b/lib/release_highlights/validator/entry.rb @@ -11,7 +11,7 @@ module ReleaseHighlights validates :title, :body, :stage, presence: true validates :'self-managed', :'gitlab-com', inclusion: { in: [true, false], message: "must be a boolean" } - validates :url, :image_url, format: { with: URI::DEFAULT_PARSER.make_regexp, message: 'must be a URL' } + validates :url, :image_url, public_url: { dns_rebind_protection: true } validates :release, numericality: true validate :validate_published_at validate :validate_packages diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb deleted file mode 100644 index 55c1d4747b4..00000000000 --- a/lib/rspec_flaky/config.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module RspecFlaky - class Config - def self.generate_report? - !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/) - end - - def self.suite_flaky_examples_report_path - ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/suite-report.json") - end - - def self.flaky_examples_report_path - ENV['FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/report.json") - end - - def self.new_flaky_examples_report_path - ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/new-report.json") - end - end -end diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb deleted file mode 100644 index 3c1b05257a0..00000000000 --- a/lib/rspec_flaky/example.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module RspecFlaky - # This is a wrapper class for RSpec::Core::Example - class Example - delegate :status, :exception, to: :execution_result - - def initialize(rspec_example) - @rspec_example = rspec_example.try(:example) || rspec_example - end - - def uid - @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") - end - - def example_id - rspec_example.id - end - - def file - metadata[:file_path] - end - - def line - metadata[:line_number] - end - - def description - metadata[:full_description] - end - - def attempts - rspec_example.try(:attempts) || 1 - end - - private - - attr_reader :rspec_example - - def metadata - rspec_example.metadata - end - - def execution_result - rspec_example.execution_result - end - end -end diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb deleted file mode 100644 index da5dbf06bc9..00000000000 --- a/lib/rspec_flaky/flaky_example.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module RspecFlaky - # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file - class FlakyExample < OpenStruct - def initialize(example) - if example.respond_to?(:example_id) - super( - example_id: example.example_id, - file: example.file, - line: example.line, - description: example.description, - last_attempts_count: example.attempts, - flaky_reports: 0) - else - super - end - end - - def update_flakiness!(last_attempts_count: nil) - self.first_flaky_at ||= Time.now - self.last_flaky_at = Time.now - self.flaky_reports += 1 - self.last_attempts_count = last_attempts_count if last_attempts_count - - if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] - self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" - end - end - - def to_h - super.merge( - first_flaky_at: first_flaky_at, - last_flaky_at: last_flaky_at, - last_flaky_job: last_flaky_job) - end - end -end diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb deleted file mode 100644 index acbfb411873..00000000000 --- a/lib/rspec_flaky/flaky_examples_collection.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require 'active_support/hash_with_indifferent_access' -require 'delegate' - -require_relative 'flaky_example' - -module RspecFlaky - class FlakyExamplesCollection < SimpleDelegator - def initialize(collection = {}) - unless collection.is_a?(Hash) - raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" - end - - collection_of_flaky_examples = - collection.map do |uid, example| - [ - uid, - example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example) - ] - end - - super(Hash[collection_of_flaky_examples]) - end - - def to_h - transform_values { |example| example.to_h }.deep_symbolize_keys - end - - def -(other) - unless other.respond_to?(:key) - raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" - end - - self.class.new(reject { |uid, _| other.key?(uid) }) - end - end -end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb deleted file mode 100644 index 37e4e16e87e..00000000000 --- a/lib/rspec_flaky/listener.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -require_dependency 'rspec_flaky/config' -require_dependency 'rspec_flaky/example' -require_dependency 'rspec_flaky/flaky_example' -require_dependency 'rspec_flaky/flaky_examples_collection' -require_dependency 'rspec_flaky/report' - -module RspecFlaky - class Listener - # - suite_flaky_examples: contains all the currently tracked flacky example - # for the whole RSpec suite - # - flaky_examples: contains the examples detected as flaky during the - # current RSpec run - attr_reader :suite_flaky_examples, :flaky_examples - - def initialize(suite_flaky_examples_json = nil) - @flaky_examples = RspecFlaky::FlakyExamplesCollection.new - @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) - end - - def example_passed(notification) - current_example = RspecFlaky::Example.new(notification.example) - - return unless current_example.attempts > 1 - - flaky_example = suite_flaky_examples.fetch(current_example.uid) { RspecFlaky::FlakyExample.new(current_example) } - flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) - - flaky_examples[current_example.uid] = flaky_example - end - - # rubocop:disable Gitlab/RailsLogger - def dump_summary(_) - RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path) - # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) - - new_flaky_examples = flaky_examples - suite_flaky_examples - if new_flaky_examples.any? - Rails.logger.warn "\nNew flaky examples detected:\n" - Rails.logger.warn Gitlab::Json.pretty_generate(new_flaky_examples.to_h) - - RspecFlaky::Report.new(new_flaky_examples).write(RspecFlaky::Config.new_flaky_examples_report_path) - # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) - end - end - # rubocop:enable Gitlab/RailsLogger - - private - - def init_suite_flaky_examples(suite_flaky_examples_json = nil) - if suite_flaky_examples_json - RspecFlaky::Report.load_json(suite_flaky_examples_json).flaky_examples - else - return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) - - RspecFlaky::Report.load(RspecFlaky::Config.suite_flaky_examples_report_path).flaky_examples - end - end - end -end diff --git a/lib/rspec_flaky/report.rb b/lib/rspec_flaky/report.rb deleted file mode 100644 index 73f30362cfe..00000000000 --- a/lib/rspec_flaky/report.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -require 'json' -require 'time' - -require_dependency 'rspec_flaky/config' -require_dependency 'rspec_flaky/flaky_examples_collection' - -module RspecFlaky - # This class is responsible for loading/saving JSON reports, and pruning - # outdated examples. - class Report < SimpleDelegator - OUTDATED_DAYS_THRESHOLD = 7 - - attr_reader :flaky_examples - - def self.load(file_path) - load_json(File.read(file_path)) - end - - def self.load_json(json) - new(RspecFlaky::FlakyExamplesCollection.new(JSON.parse(json))) - end - - def initialize(flaky_examples) - unless flaky_examples.is_a?(RspecFlaky::FlakyExamplesCollection) - raise ArgumentError, "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!" - end - - @flaky_examples = flaky_examples - super(flaky_examples) - end - - def write(file_path) - unless RspecFlaky::Config.generate_report? - puts "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" # rubocop:disable Rails/Output - return - end - - report_path_dir = File.dirname(file_path) - FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) - - File.write(file_path, JSON.pretty_generate(flaky_examples.to_h)) - end - - def prune_outdated(days: OUTDATED_DAYS_THRESHOLD) - outdated_date_threshold = Time.now - (3600 * 24 * days) - updated_hash = flaky_examples.dup - .delete_if do |uid, hash| - hash[:last_flaky_at] && Time.parse(hash[:last_flaky_at]).to_i < outdated_date_threshold.to_i - end - - self.class.new(RspecFlaky::FlakyExamplesCollection.new(updated_hash)) - end - end -end diff --git a/lib/sentry/api_urls.rb b/lib/sentry/api_urls.rb deleted file mode 100644 index 388d0531da1..00000000000 --- a/lib/sentry/api_urls.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Sentry - class ApiUrls - def initialize(url_base) - @uri = URI(url_base).freeze - end - - def issues_url - with_path(File.join(@uri.path, '/issues/')) - end - - def issue_url(issue_id) - with_path("/api/0/issues/#{escape(issue_id)}/") - end - - def projects_url - with_path('/api/0/projects/') - end - - def issue_latest_event_url(issue_id) - with_path("/api/0/issues/#{escape(issue_id)}/events/latest/") - end - - private - - def with_path(new_path) - new_uri = @uri.dup - # Sentry API returns 404 if there are extra slashes in the URL - new_uri.path = new_path.squeeze('/') - - new_uri - end - - def escape(param) - CGI.escape(param.to_s) - end - end -end diff --git a/lib/sentry/pagination_parser.rb b/lib/sentry/pagination_parser.rb deleted file mode 100644 index fa9c1dd8694..00000000000 --- a/lib/sentry/pagination_parser.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Sentry - module PaginationParser - PATTERN = /rel=\"(?<direction>\w+)\";\sresults=\"(?<results>\w+)\";\scursor=\"(?<cursor>.+)\"/.freeze - - def self.parse(headers) - links = headers['link'].to_s.split(',') - - links.map { |link| parse_link(link) }.compact.to_h - end - - def self.parse_link(link) - match = link.match(PATTERN) - - return unless match - return if match['results'] != "true" - - [match['direction'], { 'cursor' => match['cursor'] }] - end - private_class_method :parse_link - end -end diff --git a/lib/spam/concerns/has_spam_action_response_fields.rb b/lib/spam/concerns/has_spam_action_response_fields.rb new file mode 100644 index 00000000000..d49f5cd6454 --- /dev/null +++ b/lib/spam/concerns/has_spam_action_response_fields.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Spam + module Concerns + # This concern is shared by the controller and GraphQL layer to handle + # addition of spam/CAPTCHA related fields in the response. + module HasSpamActionResponseFields + extend ActiveSupport::Concern + + # spam_action_response_fields(spammable) -> hash + # + # Takes a Spammable as an argument and returns response fields necessary to display a CAPTCHA on + # the client. + def spam_action_response_fields(spammable) + { + spam: spammable.spam?, + # NOTE: These fields are intentionally named with 'captcha' instead of 'recaptcha', so + # that they can be applied to future alternative CAPTCHA implementations other than + # reCAPTCHA (such as FriendlyCaptcha) without having to change the response field name + # in the API. + needs_captcha_response: spammable.render_recaptcha?, + spam_log_id: spammable.spam_log&.id, + captcha_site_key: Gitlab::CurrentSettings.recaptcha_site_key + } + end + + # with_spam_action_response_fields(spammable) { {other_fields: true} } -> hash + # + # Takes a Spammable and a block as arguments. + # + # The block passed should be a hash, which the spam_action_fields will be merged into. + def with_spam_action_response_fields(spammable) + yield.merge(spam_action_response_fields(spammable)) + end + end + end +end diff --git a/lib/system_check/app/git_user_default_ssh_config_check.rb b/lib/system_check/app/git_user_default_ssh_config_check.rb index a331f88873b..ea6bc9c4f01 100644 --- a/lib/system_check/app/git_user_default_ssh_config_check.rb +++ b/lib/system_check/app/git_user_default_ssh_config_check.rb @@ -31,7 +31,7 @@ module SystemCheck end try_fixing_it("mkdir #{backup_dir}", *instructions) - for_more_information('doc/ssh/README.md in section "SSH on the GitLab server"') + for_more_information('doc/ssh/README.md in section "Overriding SSH settings on the GitLab server"') fix_and_rerun end diff --git a/lib/system_check/sidekiq_check.rb b/lib/system_check/sidekiq_check.rb index 4f1533a5b7d..7ac1bd58ede 100644 --- a/lib/system_check/sidekiq_check.rb +++ b/lib/system_check/sidekiq_check.rb @@ -15,7 +15,7 @@ module SystemCheck def check_sidekiq_running $stdout.print "Running? ... " - if sidekiq_process_count > 0 + if sidekiq_worker_process_count > 0 $stdout.puts "yes".color(:green) else $stdout.puts "no".color(:red) @@ -31,15 +31,16 @@ module SystemCheck end def only_one_sidekiq_running - process_count = sidekiq_process_count - return if process_count == 0 + worker_count = sidekiq_worker_process_count + cluster_count = sidekiq_cluster_process_count + return if worker_count == 0 - $stdout.print 'Number of Sidekiq processes ... ' + $stdout.print 'Number of Sidekiq processes (cluster/worker) ... ' - if process_count == 1 - $stdout.puts '1'.color(:green) + if (cluster_count == 1 && worker_count > 0) || (cluster_count == 0 && worker_count == 1) + $stdout.puts "#{cluster_count}/#{worker_count}".color(:green) else - $stdout.puts "#{process_count}".color(:red) + $stdout.puts "#{cluster_count}/#{worker_count}".color(:red) try_fixing_it( 'sudo service gitlab stop', "sudo pkill -u #{gitlab_user} -f sidekiq", @@ -50,9 +51,14 @@ module SystemCheck end end - def sidekiq_process_count + def sidekiq_worker_process_count ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) - ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count + ps_ux.lines.grep(/sidekiq \d+\.\d+\.\d+/).count + end + + def sidekiq_cluster_process_count + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) + ps_ux.lines.grep(/sidekiq-cluster/).count end end end diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake index ad63de66c81..529f17d4d27 100644 --- a/lib/tasks/eslint.rake +++ b/lib/tasks/eslint.rake @@ -3,7 +3,7 @@ unless Rails.env.production? desc "GitLab | Run ESLint" task eslint: ['yarn:check'] do - unless system('yarn run eslint') + unless system('yarn run lint:eslint:all') abort('rake eslint failed') end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index e4eb4604138..77377a7e0fd 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -6,6 +6,7 @@ require 'graphql/rake_task' namespace :gitlab do OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference") + TEMP_SCHEMA_DIR = Rails.root.join('tmp/tests/graphql') TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/' # Make all feature flags enabled so that all feature flag @@ -27,7 +28,7 @@ namespace :gitlab do GraphQL::RakeTask.new( schema_name: 'GitlabSchema', dependencies: [:environment, :enable_feature_flags], - directory: OUTPUT_DIR, + directory: TEMP_SCHEMA_DIR, idl_outfile: "gitlab_schema.graphql", json_outfile: "gitlab_schema.json" ) @@ -130,18 +131,8 @@ namespace :gitlab do end end - desc 'GitLab | GraphQL | Check if GraphQL schemas are up to date' - task check_schema: [:environment, :enable_feature_flags] do - idl_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.graphql')) - json_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.json')) - - if idl_doc == GitlabSchema.to_definition && json_doc == GitlabSchema.to_json - puts "GraphQL schema is up to date" - else - format_output('GraphQL schema is outdated! Please update it by running `bundle exec rake gitlab:graphql:schema:dump`.') - abort - end - end + desc 'GitLab | GraphQL | Update GraphQL docs and schema' + task update_all: [:compile_docs, 'schema:dump'] end end diff --git a/lib/tasks/gitlab/packages/composer.rake b/lib/tasks/gitlab/packages/composer.rake new file mode 100644 index 00000000000..c9bccfe9384 --- /dev/null +++ b/lib/tasks/gitlab/packages/composer.rake @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'logger' + +desc "GitLab | Packages | Build composer cache" +namespace :gitlab do + namespace :packages do + task build_composer_cache: :environment do + logger = Logger.new(STDOUT) + logger.info('Starting to build composer cache files') + + ::Packages::Package.composer.find_in_batches do |packages| + packages.group_by { |pkg| [pkg.project_id, pkg.name] }.each do |(project_id, name), packages| + logger.info("Building cache for #{project_id} -> #{name}") + Gitlab::Composer::Cache.new(project: packages.first.project, name: name).execute + end + end + end + end +end diff --git a/lib/tasks/gitlab/packages/events.rake b/lib/tasks/gitlab/packages/events.rake index 4bf4ce430f1..4a6a014acc5 100644 --- a/lib/tasks/gitlab/packages/events.rake +++ b/lib/tasks/gitlab/packages/events.rake @@ -51,8 +51,7 @@ namespace :gitlab do "name" => event_name, "category" => "#{originator_type}_packages", "aggregation" => "weekly", - "redis_slot" => "package", - "feature_flag" => "collect_package_events_redis" + "redis_slot" => "package" } end diff --git a/lib/tasks/gitlab_danger.rake b/lib/tasks/gitlab_danger.rake index 5df4a8ce4f1..ff9464a588a 100644 --- a/lib/tasks/gitlab_danger.rake +++ b/lib/tasks/gitlab_danger.rake @@ -2,16 +2,16 @@ desc 'Run local Danger rules' task :danger_local do - require_relative '../../tooling/gitlab_danger' + require_relative '../../tooling/danger/project_helper' require 'gitlab/popen' - puts("#{GitlabDanger.local_warning_message}\n") + puts("#{Tooling::Danger::ProjectHelper.local_warning_message}\n") # _status will _always_ be 0, regardless of failure or success :( output, _status = Gitlab::Popen.popen(%w{danger dry_run}) if output.empty? - puts(GitlabDanger.success_message) + puts(Tooling::Danger::ProjectHelper.success_message) else puts(output) exit(1) diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 5d60bc41f21..976ec089011 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -33,7 +33,6 @@ unless Rails.env.production? tasks = %w[ config_lint lint:haml - scss_lint gettext:lint lint:static_verification gitlab:sidekiq:all_queues_yml:check diff --git a/lib/tasks/scss-lint.rake b/lib/tasks/scss-lint.rake deleted file mode 100644 index 8a4809f80fd..00000000000 --- a/lib/tasks/scss-lint.rake +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -unless Rails.env.production? - require 'scss_lint/rake_task' - - SCSSLint::RakeTask.new do |t| - t.config = '.scss-lint.yml' - # See https://github.com/brigade/scss-lint/issues/726 - # Hack, otherwise linter won't respect scss_files option in config file. - t.files = [] - end -end |