diff options
Diffstat (limited to 'lib')
404 files changed, 5641 insertions, 3383 deletions
diff --git a/lib/api/admin/dictionary.rb b/lib/api/admin/dictionary.rb index 038c122c021..b013d584c1c 100644 --- a/lib/api/admin/dictionary.rb +++ b/lib/api/admin/dictionary.rb @@ -31,30 +31,12 @@ module API desc: 'The table name' end get do - not_found!('Table not found') unless File.exist?(safe_file_path!) + table_dictionary = ::Gitlab::Database::Dictionary.entry(params[:table_name]) + not_found!('Table not found') unless table_dictionary present table_dictionary, with: Entities::Dictionary::Table end end - - helpers do - def table_name - params[:table_name] - end - - def table_dictionary - YAML.load_file(safe_file_path!).with_indifferent_access - end - - def safe_file_path! - dir = Gitlab::Database::GitlabSchema.dictionary_paths.first.to_s - path = Rails.root.join(dir, "#{table_name}.yml").to_s - - Gitlab::PathTraversal.check_allowed_absolute_path_and_path_traversal!(path, [dir]) - - path - end - end end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index 43a21c11dbc..97e09795f49 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -331,6 +331,7 @@ module API mount ::API::SystemHooks mount ::API::Tags mount ::API::Terraform::Modules::V1::Packages + mount ::API::Terraform::Modules::V1::ProjectPackages mount ::API::Terraform::State mount ::API::Terraform::StateVersion mount ::API::Topics diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 25ac1780a36..585e9f962a3 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -93,7 +93,7 @@ module API requires :token, type: String, desc: %q(The runner's authentication token) requires :system_id, type: String, desc: %q(The runner's system identifier.) end - delete '/managers', urgency: :low, feature_category: :runner_fleet do + delete '/managers', urgency: :low, feature_category: :fleet_visibility do authenticate_runner!(ensure_runner_manager: false) destroy_conditionally!(current_runner) do diff --git a/lib/api/concerns/milestones/group_project_params.rb b/lib/api/concerns/milestones/group_project_params.rb new file mode 100644 index 00000000000..72d07d7dcdb --- /dev/null +++ b/lib/api/concerns/milestones/group_project_params.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This DRYs up some methods used by both the GraphQL and REST milestone APIs +module API + module Concerns + module Milestones + module GroupProjectParams + extend ActiveSupport::Concern + + private + + def project_finder_params(parent, params) + return { project_ids: parent.id } unless params[:include_ancestors].present? && parent.group.present? + + { + group_ids: parent.group.self_and_ancestors.select(:id), + project_ids: parent.id + } + end + + def group_finder_params(parent, params) + include_ancestors = params[:include_ancestors].present? + include_descendants = params[:include_descendants].present? + return { group_ids: parent.id } unless include_ancestors || include_descendants + + group_ids = if include_ancestors && include_descendants + parent.self_and_hierarchy + elsif include_ancestors + parent.self_and_ancestors + else + parent.self_and_descendants + end + + if include_descendants + project_ids = group_projects(parent).with_issues_or_mrs_available_for_user(current_user) + end + + { + group_ids: group_ids.public_or_visible_to_user(current_user).select(:id), + project_ids: project_ids + } + end + + def group_projects(parent) + GroupProjectsFinder.new( + group: parent, + current_user: current_user, + options: { include_subgroups: true } + ).execute + end + end + end + end +end diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 19d63a39242..69d23c408ab 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -238,7 +238,7 @@ module API not_found!('Packages') if available_packages.empty? - if endpoint_scope == :project && Feature.enabled?(:npm_metadata_cache, project) + if endpoint_scope == :project if metadata_cache&.file&.exists? metadata_cache.touch_last_downloaded_at present_carrierwave_file!(metadata_cache.file) diff --git a/lib/api/concerns/packages/nuget/public_endpoints.rb b/lib/api/concerns/packages/nuget/public_endpoints.rb index b0c9177f452..a710e4acea8 100644 --- a/lib/api/concerns/packages/nuget/public_endpoints.rb +++ b/lib/api/concerns/packages/nuget/public_endpoints.rb @@ -14,6 +14,8 @@ module API module PublicEndpoints extend ActiveSupport::Concern + SHA256_REGEX = /SHA256:([a-f0-9]{64})/i + included do # https://docs.microsoft.com/en-us/nuget/api/service-index desc 'The NuGet V3 Feed Service Index' do @@ -43,6 +45,57 @@ module API ] tags %w[nuget_packages] end + + namespace :symbolfiles do + after_validation do + forbidden! unless symbol_server_enabled? + end + + desc 'The NuGet Symbol File Download Endpoint' do + detail 'This feature was introduced in GitLab 16.7' + success code: 200 + failure [ + { code: 400, message: 'Bad Request' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + headers Symbolchecksum: { + type: String, + desc: 'The SHA256 checksums of the symbol file', + required: true + } + tags %w[nuget_packages] + end + params do + requires :file_name, allow_blank: false, type: String, desc: 'The symbol file name', + regexp: API::NO_SLASH_URL_PART_REGEX, documentation: { example: 'mynugetpkg.pdb' } + requires :signature, allow_blank: false, type: String, desc: 'The symbol file signature', + regexp: API::NO_SLASH_URL_PART_REGEX, + documentation: { example: 'k813f89485474661234z7109cve5709eFFFFFFFF' } + requires :same_file_name, same_as: :file_name + end + get '*file_name/*signature/*same_file_name', format: false, urgency: :low do + bad_request!('Missing checksum header') if headers['Symbolchecksum'].blank? + + project_or_group_without_auth + + # upcase the age part of the signature in case we received it in lowercase: + # https://github.com/dotnet/symstore/blob/main/docs/specs/SSQP_Key_Conventions.md#key-formatting-basic-rules + signature = declared_params[:signature].sub(/.{8}\z/, &:upcase) + checksums = headers['Symbolchecksum'].scan(SHA256_REGEX).flatten + + symbol = ::Packages::Nuget::Symbol + .with_signature(signature) + .with_file_name(declared_params[:file_name]) + .with_file_sha256(checksums) + .first + + not_found!('Symbol') unless symbol + + present_carrierwave_file!(symbol.file) + end + end + namespace '/v2' do get format: :xml, urgency: :low do env['api.format'] = :xml diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index 8161c2b850f..1468164081c 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -139,6 +139,8 @@ module API authorize!(:create_deployment, user_project) authorize!(:create_environment, user_project) + render_api_error!({ ref: ["The branch or tag does not exist"] }, 400) unless user_project.commit(declared_params[:ref]) + environment = user_project .environments .find_or_create_by_name(params[:environment]) diff --git a/lib/api/entities/application_setting.rb b/lib/api/entities/application_setting.rb index 91dae5ab825..de5bd4e8d97 100644 --- a/lib/api/entities/application_setting.rb +++ b/lib/api/entities/application_setting.rb @@ -8,6 +8,7 @@ module API attributes.delete(:performance_bar_allowed_group_path) attributes.delete(:performance_bar_enabled) attributes.delete(:allow_local_requests_from_hooks_and_services) + attributes.delete(:repository_storages_weighted) # let's not expose the secret key in a response attributes.delete(:asset_proxy_secret_key) @@ -49,6 +50,7 @@ module API expose(:housekeeping_full_repack_period) { |settings, _options| settings.housekeeping_optimize_repository_period } expose(:housekeeping_gc_period) { |settings, _options| settings.housekeeping_optimize_repository_period } expose(:housekeeping_incremental_repack_period) { |settings, _options| settings.housekeeping_optimize_repository_period } + expose(:repository_storages_weighted) { |settings, _options| settings.repository_storages_with_default_weight } end end end diff --git a/lib/api/entities/basic_repository_storage_move.rb b/lib/api/entities/basic_repository_storage_move.rb index 83b4f428a56..9124b8b428f 100644 --- a/lib/api/entities/basic_repository_storage_move.rb +++ b/lib/api/entities/basic_repository_storage_move.rb @@ -8,6 +8,7 @@ module API expose :human_state_name, as: :state, documentation: { type: 'string', example: 'scheduled' } expose :source_storage_name, documentation: { type: 'string', example: 'default' } expose :destination_storage_name, documentation: { type: 'string', example: 'storage1' } + expose :error_message, documentation: { type: 'string', example: 'Failed to move repository' } end end end diff --git a/lib/api/entities/ci/job_request/image.rb b/lib/api/entities/ci/job_request/image.rb index 92d68269265..08b0075a657 100644 --- a/lib/api/entities/ci/job_request/image.rb +++ b/lib/api/entities/ci/job_request/image.rb @@ -8,6 +8,7 @@ module API expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port + expose :executor_opts expose :pull_policy end end diff --git a/lib/api/entities/ci/job_request/service.rb b/lib/api/entities/ci/job_request/service.rb index 128591058fe..ae726a6b888 100644 --- a/lib/api/entities/ci/job_request/service.rb +++ b/lib/api/entities/ci/job_request/service.rb @@ -8,6 +8,7 @@ module API expose :name, :entrypoint expose :ports, using: Entities::Ci::JobRequest::Port + expose :executor_opts expose :pull_policy expose :alias, :command expose :variables diff --git a/lib/api/entities/commit.rb b/lib/api/entities/commit.rb index ab1f51289d7..99ae4b66f67 100644 --- a/lib/api/entities/commit.rb +++ b/lib/api/entities/commit.rb @@ -17,6 +17,10 @@ module API expose :committer_email, documentation: { type: 'string', example: 'jack@example.com' } expose :committed_date, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } expose :trailers, documentation: { type: 'object', example: '{ "Merged-By": "Jane Doe janedoe@gitlab.com" }' } + expose :extended_trailers, documentation: { + type: 'object', + example: '{ "Signed-off-by": ["John Doe <johndoe@gitlab.com>", "Jane Doe <janedoe@gitlab.com>"] }' + } expose :web_url, documentation: { diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb index cadd45cb0eb..359dbaf8697 100644 --- a/lib/api/entities/container_registry.rb +++ b/lib/api/entities/container_registry.rb @@ -24,6 +24,7 @@ module API expose :delete_api_path, if: ->(object, options) { Ability.allowed?(options[:user], :admin_container_image, object) }, documentation: { type: 'string', example: 'delete/api/path' } expose :size, if: -> (_, options) { options[:size] }, documentation: { type: 'integer', example: 12345 } + expose :status, documentation: { type: 'string', example: 'delete_scheduled' } private diff --git a/lib/api/entities/hook.rb b/lib/api/entities/hook.rb index e24e201ac57..d92331f7dea 100644 --- a/lib/api/entities/hook.rb +++ b/lib/api/entities/hook.rb @@ -14,7 +14,9 @@ module API expose :alert_status, documentation: { type: 'symbol', example: :executable } expose :disabled_until, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } - expose :url_variables, documentation: { type: 'Hash', example: { "token" => "secr3t" }, is_array: true } + expose :url_variables, + if: ->(_, options) { options[:with_url_variables] != false }, + documentation: { type: 'Hash', example: { "token" => "secr3t" }, is_array: true } def url_variables object.url_variables.keys.map { { key: _1 } } diff --git a/lib/api/entities/ml/mlflow/list_registered_models.rb b/lib/api/entities/ml/mlflow/list_registered_models.rb new file mode 100644 index 00000000000..05ef98c4ac6 --- /dev/null +++ b/lib/api/entities/ml/mlflow/list_registered_models.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class ListRegisteredModels < Grape::Entity + expose :registered_models, with: RegisteredModel, as: :registered_models + expose :next_page_token + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/model_version.rb b/lib/api/entities/ml/mlflow/model_version.rb new file mode 100644 index 00000000000..10fdf3822a5 --- /dev/null +++ b/lib/api/entities/ml/mlflow/model_version.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module API + module Entities + module Ml + module Mlflow + class ModelVersion < Grape::Entity + include ::API::Helpers::RelatedResourcesHelpers + + expose :name + expose :version + expose :creation_timestamp, documentation: { type: Integer } + expose :last_updated_timestamp, documentation: { type: Integer } + expose :user_id + expose :current_stage + expose :description + expose :source + expose :run_id + expose :status + expose :status_message + expose :metadata + expose :run_link + expose :aliases, documentation: { is_array: true, type: String } + + private + + def name + object.name + end + + def creation_timestamp + object.created_at.to_i + end + + def last_updated_timestamp + object.updated_at.to_i + end + + def user_id + nil + end + + def current_stage + "development" + end + + def description + object.description.to_s + end + + def source + expose_url(Gitlab::Routing.url_helpers.project_ml_model_version_path( + object.model.project, + object.model, + object + )) + end + + def run_id + object.candidate.eid + end + + def status + "READY" + end + + def status_message + "" + end + + def metadata + [] + end + + def run_link + "" + end + + def aliases + [] + end + end + end + end + end +end diff --git a/lib/api/entities/ml/mlflow/model_versions/responses/get.rb b/lib/api/entities/ml/mlflow/model_versions/responses/get.rb deleted file mode 100644 index 14baae03644..00000000000 --- a/lib/api/entities/ml/mlflow/model_versions/responses/get.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module Ml - module Mlflow - module ModelVersions - module Responses - class Get < Grape::Entity - expose :model_version, with: Types::ModelVersion - end - end - end - end - end - end -end diff --git a/lib/api/entities/ml/mlflow/model_versions/types/model_version.rb b/lib/api/entities/ml/mlflow/model_versions/types/model_version.rb deleted file mode 100644 index 407158521f7..00000000000 --- a/lib/api/entities/ml/mlflow/model_versions/types/model_version.rb +++ /dev/null @@ -1,85 +0,0 @@ -# frozen_string_literal: true - -module API - module Entities - module Ml - module Mlflow - module ModelVersions - module Types - class ModelVersion < Grape::Entity - expose :name - expose :version - expose :creation_timestamp, documentation: { type: Integer } - expose :last_updated_timestamp, documentation: { type: Integer } - expose :user_id - expose :current_stage - expose :description - expose :source - expose :run_id - expose :status - expose :status_message - expose :metadata - expose :run_link - expose :aliases, documentation: { is_array: true, type: String } - - private - - def name - object.model.name - end - - def creation_timestamp - object.created_at.to_i - end - - def last_updated_timestamp - object.updated_at.to_i - end - - def user_id - nil - end - - def current_stage - "development" - end - - def description - "" - end - - def source - model_name = object.model.name - "api/v4/projects/(id)/packages/ml_models/#{model_name}/model_version/" - end - - def run_id - "" - end - - def status - "READY" - end - - def status_message - "" - end - - def metadata - [] - end - - def run_link - "" - end - - def aliases - [] - end - end - end - end - end - end - end -end diff --git a/lib/api/entities/ml/mlflow/registered_model.rb b/lib/api/entities/ml/mlflow/registered_model.rb index 1ff983e1611..bb547f9c46c 100644 --- a/lib/api/entities/ml/mlflow/registered_model.rb +++ b/lib/api/entities/ml/mlflow/registered_model.rb @@ -6,11 +6,25 @@ module API module Mlflow class RegisteredModel < Grape::Entity expose :name - expose :created_at, as: :creation_timestamp - expose :updated_at, as: :last_updated_timestamp + expose :creation_timestamp, documentation: { type: Integer } + expose :last_updated_timestamp, documentation: { type: Integer } expose :description expose(:user_id) { |model| model.user_id.to_s } expose :metadata, as: :tags, using: KeyValue + + private + + def creation_timestamp + object.created_at.to_i + end + + def last_updated_timestamp + object.updated_at.to_i + end + + def description + object.description.to_s + end end end end diff --git a/lib/api/entities/ml/mlflow/run_info.rb b/lib/api/entities/ml/mlflow/run_info.rb index 6133b3a9d4b..04c6069617c 100644 --- a/lib/api/entities/ml/mlflow/run_info.rb +++ b/lib/api/entities/ml/mlflow/run_info.rb @@ -5,6 +5,8 @@ module API module Ml module Mlflow class RunInfo < Grape::Entity + include ::API::Helpers::RelatedResourcesHelpers + expose :run_id expose :run_id, as: :run_uuid expose(:experiment_id) { |candidate| candidate.experiment.iid.to_s } @@ -12,7 +14,7 @@ module API expose :end_time, expose_nil: false expose :name, as: :run_name, expose_nil: false expose(:status) { |candidate| candidate.status.to_s.upcase } - expose(:artifact_uri) { |candidate, options| "#{options[:packages_url]}#{candidate.artifact_root}" } + expose :artifact_uri expose(:lifecycle_stage) { |candidate| 'active' } expose(:user_id) { |candidate| candidate.user_id.to_s } @@ -21,6 +23,34 @@ module API def run_id object.eid.to_s end + + def artifact_uri + expose_url(model_version_uri || generic_package_uri) + end + + # Example: http://127.0.0.1:3000/api/v4/projects/20/packages/ml_models/my-model-name-4/3.0.0 + def model_version_uri + return unless object.model_version_id + + model_version = object.model_version + + path = api_v4_projects_packages_ml_models_model_version_path( + id: object.project.id, model_name: model_version.model.name, model_version: '', file_name: '' + ) + + path.sub('/model_version', "/#{model_version.version}") + end + + # Example: http://127.0.0.1:3000/api/v4/projects/20/packages/generic/ml_experiment_1/1/ + # Note: legacy format + def generic_package_uri + path = api_v4_projects_packages_generic_package_version_path( + id: object.project.id, package_name: '', file_name: '' + ) + path = path.delete_suffix('/package_version') + + [path, object.artifact_root].join('') + end end end end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 12e022bfb20..0012fcf957b 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -41,6 +41,7 @@ module API end end + expose :code_suggestions, documentation: { type: 'boolean' } expose :packages_enabled, documentation: { type: 'boolean' } expose :empty_repo?, as: :empty_repo, documentation: { type: 'boolean' } expose :archived?, as: :archived, documentation: { type: 'boolean' } @@ -85,6 +86,7 @@ module API expose(:infrastructure_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :infrastructure) } expose(:monitor_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :monitor) } expose(:model_experiments_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :model_experiments) } + expose(:model_registry_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :model_registry) } expose(:emails_disabled, documentation: { type: 'boolean' }) { |project, options| project.emails_disabled? } expose :emails_enabled, documentation: { type: 'boolean' } diff --git a/lib/api/entities/user_preferences.rb b/lib/api/entities/user_preferences.rb index e66e83549b2..e04ddd52f29 100644 --- a/lib/api/entities/user_preferences.rb +++ b/lib/api/entities/user_preferences.rb @@ -3,7 +3,10 @@ module API module Entities class UserPreferences < Grape::Entity - expose :id, :user_id, :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt + expose :id, :user_id, :view_diffs_file_by_file, + :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt end end end + +API::Entities::UserPreferences.prepend_mod_with('API::Entities::UserPreferences', with_descendants: true) diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb index 53fef7a46e2..25c1edb4526 100644 --- a/lib/api/entities/user_with_admin.rb +++ b/lib/api/entities/user_with_admin.rb @@ -7,6 +7,7 @@ module API expose :note expose :namespace_id expose :created_by, with: UserBasic + expose :email_reset_offered_at end end end diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb index 0096e466bef..55c5ddfe557 100644 --- a/lib/api/group_milestones.rb +++ b/lib/api/group_milestones.rb @@ -19,6 +19,8 @@ module API end params do use :list_params + optional :include_descendants, type: Grape::API::Boolean, + desc: 'Include milestones from all subgroups and subprojects' end get ":id/milestones" do list_milestones_for(user_group) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index bb94d5d14d0..f5dcbc07704 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -146,7 +146,7 @@ module API if id.is_a?(Integer) || id =~ INTEGER_ID_REGEX projects.find_by(id: id) elsif id.include?("/") - projects.find_by_full_path(id, follow_redirects: Feature.enabled?(:api_redirect_moved_projects)) + projects.find_by_full_path(id, follow_redirects: true) end end # rubocop: enable CodeReuse/ActiveRecord @@ -378,6 +378,10 @@ module API authorize! :admin_group, user_group end + def authorize_admin_member_role! + authorize! :admin_member_role, user_group + end + def authorize_read_builds! authorize! :read_build, user_project end @@ -905,7 +909,6 @@ module API end def project_moved?(id, project) - return false unless Feature.enabled?(:api_redirect_moved_projects) return false unless id.is_a?(String) && id.include?('/') return false if project.blank? || project.full_path.casecmp?(id) return false unless params[:id] == id diff --git a/lib/api/helpers/import_github_helpers.rb b/lib/api/helpers/import_github_helpers.rb index 1634e064d73..19567e04d87 100644 --- a/lib/api/helpers/import_github_helpers.rb +++ b/lib/api/helpers/import_github_helpers.rb @@ -9,8 +9,7 @@ module API def access_params { - github_access_token: params[:personal_access_token], - additional_access_tokens: params[:additional_access_tokens] + github_access_token: params[:personal_access_token] } end diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index a08337a86ac..dd3009ff1d7 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -126,66 +126,9 @@ module API def self.integrations { - 'apple-app-store' => [ - { - required: true, - name: :app_store_issuer_id, - type: String, - desc: 'The Apple App Store Connect Issuer ID' - }, - { - required: true, - name: :app_store_key_id, - type: String, - desc: 'The Apple App Store Connect Key ID' - }, - { - required: true, - name: :app_store_private_key, - type: String, - desc: 'The Apple App Store Connect Private Key' - }, - { - required: true, - name: :app_store_private_key_file_name, - type: String, - desc: 'The Apple App Store Connect Private Key File Name' - }, - { - required: false, - name: :app_store_protected_refs, - type: ::Grape::API::Boolean, - desc: 'Only enable for protected refs' - } - ], - 'asana' => [ - { - required: true, - name: :api_key, - type: String, - desc: 'User API token' - }, - { - required: false, - name: :restrict_to_branch, - type: String, - desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches' - } - ], - 'assembla' => [ - { - required: true, - name: :token, - type: String, - desc: 'The authentication token' - }, - { - required: false, - name: :subdomain, - type: String, - desc: 'Subdomain setting' - } - ], + 'apple-app-store' => ::Integrations::AppleAppStore.api_fields, + 'asana' => ::Integrations::Asana.api_fields, + 'assembla' => ::Integrations::Assembla.api_fields, 'bamboo' => [ { required: true, @@ -620,14 +563,6 @@ module API desc: 'The Mattermost token' } ], - 'shimo' => [ - { - required: true, - name: :external_wiki_url, - type: String, - desc: 'Shimo workspace URL' - } - ], 'slack-slash-commands' => [ { required: true, @@ -995,7 +930,6 @@ module API ::Integrations::Pumble, ::Integrations::Pushover, ::Integrations::Redmine, - ::Integrations::Shimo, ::Integrations::Slack, ::Integrations::SlackSlashCommands, ::Integrations::SquashTm, diff --git a/lib/api/helpers/kubernetes/agent_helpers.rb b/lib/api/helpers/kubernetes/agent_helpers.rb index aa4f4310e1d..eca26c023cf 100644 --- a/lib/api/helpers/kubernetes/agent_helpers.rb +++ b/lib/api/helpers/kubernetes/agent_helpers.rb @@ -21,19 +21,13 @@ module API strong_memoize_attr :agent def gitaly_info(project) - gitaly_features = Feature::Gitaly.server_feature_flags - - Gitlab::GitalyClient.connection_data(project.repository_storage).merge(features: gitaly_features) + Gitlab::GitalyClient.connection_data(project.repository_storage) end def gitaly_repository(project) project.repository.gitaly_repository.to_h end - def check_feature_enabled - not_found!('Internal API not found') unless Feature.enabled?(:kubernetes_agent_internal_api, type: :ops) - end - def check_agent_token unauthorized! unless agent_token @@ -47,9 +41,9 @@ module API def increment_unique_events events = params[:unique_counters]&.slice( :agent_users_using_ci_tunnel, - :k8s_api_proxy_requests_unique_users_via_ci_access, :k8s_api_proxy_requests_unique_agents_via_ci_access, - :k8s_api_proxy_requests_unique_users_via_user_access, :k8s_api_proxy_requests_unique_agents_via_user_access, - :k8s_api_proxy_requests_unique_users_via_pat_access, :k8s_api_proxy_requests_unique_agents_via_pat_access, + :k8s_api_proxy_requests_unique_agents_via_ci_access, + :k8s_api_proxy_requests_unique_agents_via_user_access, + :k8s_api_proxy_requests_unique_agents_via_pat_access, :flux_git_push_notified_unique_projects ) @@ -58,6 +52,41 @@ module API end end + def track_events + event_lists = params[:events]&.slice( + :k8s_api_proxy_requests_unique_users_via_ci_access, + :k8s_api_proxy_requests_unique_users_via_user_access, + :k8s_api_proxy_requests_unique_users_via_pat_access + ) + return if event_lists.blank? + + users, projects = load_users_and_projects(event_lists) + event_lists.each do |event_name, events| + track_events_for(event_name, events, users, projects) + end + end + + def track_unique_user_events + events = params[:unique_counters]&.slice( + :k8s_api_proxy_requests_unique_users_via_ci_access, + :k8s_api_proxy_requests_unique_users_via_user_access, + :k8s_api_proxy_requests_unique_users_via_pat_access + ) + return if events.blank? + + unique_user_ids = events.values.flatten.uniq + users = User.id_in(unique_user_ids).index_by(&:id) + + events.each do |event, user_ids| + user_ids.each do |user_id| + user = users[user_id] + next if user.nil? + + Gitlab::InternalEvents.track_event(event, user: user) + end + end + end + def increment_count_events events = params[:counters]&.slice( :gitops_sync, :k8s_api_proxy_request, :flux_git_push_notifications_total, @@ -113,6 +142,29 @@ module API PersonalAccessToken.find_by_token(params[:access_key]) end strong_memoize_attr :access_token + + private + + def load_users_and_projects(event_lists) + all_events = event_lists.values.flatten + unique_user_ids = all_events.pluck('user_id').compact.uniq # rubocop:disable CodeReuse/ActiveRecord -- this pluck isn't from ActiveRecord, it's from ActiveSupport + unique_project_ids = all_events.pluck('project_id').compact.uniq # rubocop:disable CodeReuse/ActiveRecord -- this pluck isn't from ActiveRecord, it's from ActiveSupport + users = User.id_in(unique_user_ids).index_by(&:id) + projects = Project.id_in(unique_project_ids).index_by(&:id) + [users, projects] + end + + def track_events_for(event_name, events, users, projects) + events.each do |event| + next if event.blank? + + user = users[event[:user_id]] + project = projects[event[:project_id]] + next if user.nil? || project.nil? + + Gitlab::InternalEvents.track_event(event_name, user: user, project: project) + end + end end end end diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb index 3873fe98a5f..7f695dfde64 100644 --- a/lib/api/helpers/packages/conan/api_helpers.rb +++ b/lib/api/helpers/packages/conan/api_helpers.rb @@ -182,7 +182,7 @@ module API end def track_push_package_event - if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate + if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY track_package_event('push_package', :conan, category: 'API::ConanPackages', project: project, namespace: project.namespace) end end @@ -196,7 +196,7 @@ module API end def create_package_file_with_type(file_type, current_package) - unless params[:file].size == 0 # rubocop: disable Style/ZeroLengthPredicate + unless params[:file].empty_size? # conan sends two upload requests, the first has no file, so we skip record creation if file.size == 0 ::Packages::Conan::CreatePackageFileService.new( current_package, @@ -212,7 +212,7 @@ module API current_package = find_or_create_package - track_push_package_event + track_push_package_event unless params[:file].empty_size? create_package_file_with_type(file_type, current_package) rescue ObjectStorage::RemoteStoreError => e diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index c91eef0c4b0..0ab3b64a2c6 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -86,8 +86,6 @@ module API strong_memoize_attr :project_id_or_nil def enqueue_sync_metadata_cache_worker(project, package_name) - return unless Feature.enabled?(:npm_metadata_cache, project) - ::Packages::Npm::CreateMetadataCacheWorker.perform_async(project.id, package_name) end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 23e83d9d54f..5eda670b832 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -40,6 +40,7 @@ module API optional :infrastructure_access_level, type: String, values: %w[disabled private enabled], desc: 'Infrastructure access level. One of `disabled`, `private` or `enabled`' optional :monitor_access_level, type: String, values: %w[disabled private enabled], desc: 'Monitor access level. One of `disabled`, `private` or `enabled`' optional :model_experiments_access_level, type: String, values: %w[disabled private enabled], desc: 'Model experiments access level. One of `disabled`, `private` or `enabled`' + optional :model_registry_access_level, type: String, values: %w[disabled private enabled], desc: 'Model registry access level. One of `disabled`, `private` or `enabled`' optional :emails_disabled, type: Boolean, desc: 'Deprecated: Use emails_enabled instead.' optional :emails_enabled, type: Boolean, desc: 'Enable email notifications' @@ -197,6 +198,7 @@ module API :infrastructure_access_level, :monitor_access_level, :model_experiments_access_level, + :model_registry_access_level, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/helpers/user_preferences_helpers.rb b/lib/api/helpers/user_preferences_helpers.rb new file mode 100644 index 00000000000..846ad354156 --- /dev/null +++ b/lib/api/helpers/user_preferences_helpers.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module API + module Helpers + module UserPreferencesHelpers + extend ActiveSupport::Concern + extend Grape::API::Helpers + + def update_user_namespace_settings(attrs) + # This method will be redefined in EE. + attrs + end + end + end +end + +API::Helpers::UserPreferencesHelpers.prepend_mod_with('API::Helpers::UserPreferencesHelpers') diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index 29dfa7c9f29..95b830e182c 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -33,11 +33,6 @@ module API optional :optional_stages, type: Hash, desc: 'Optional stages of import to be performed' optional :timeout_strategy, type: String, values: ::ProjectImportData::TIMEOUT_STRATEGIES, desc: 'Strategy for behavior on timeouts' - optional :additional_access_tokens, - type: Array[String], - coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, - desc: 'Additional list of personal access tokens', - documentation: { example: 'foo,bar' } end post 'import/github' do result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index b8a2fde4e36..d3a4d94f8ca 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -5,7 +5,6 @@ module API module Internal class Kubernetes < ::API::Base before do - check_feature_enabled authenticate_gitlab_kas_request! end @@ -141,12 +140,40 @@ module API post '/', feature_category: :deployment_management do increment_count_events increment_unique_events + track_unique_user_events no_content! rescue ArgumentError => e bad_request!(e.message) end end + + namespace 'kubernetes/agent_events' do + desc 'POST agent events' do + detail 'Updates agent events' + end + params do + optional :events, type: Hash, desc: 'Array of events' do + optional :k8s_api_proxy_requests_unique_users_via_ci_access, type: Array, desc: 'An array of events that have interacted with the CI tunnel via `ci_access`' do + optional :user_id, type: Integer, desc: 'User ID' + optional :project_id, type: Integer, desc: 'Project ID' + end + optional :k8s_api_proxy_requests_unique_users_via_user_access, type: Array, desc: 'An array of events that have interacted with the CI tunnel via `ci_access`' do + optional :user_id, type: Integer, desc: 'User ID' + optional :project_id, type: Integer, desc: 'Project ID' + end + optional :k8s_api_proxy_requests_unique_users_via_pat_access, type: Array, desc: 'An array of events that have interacted with the CI tunnel via `ci_access`' do + optional :user_id, type: Integer, desc: 'User ID' + optional :project_id, type: Integer, desc: 'Project ID' + end + end + end + post '/', feature_category: :deployment_management do + track_events + + no_content! + end + end end end end diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index fb71cb0e791..3ef30a5fcd3 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -6,6 +6,8 @@ module API included do helpers do + include ::API::Concerns::Milestones::GroupProjectParams + params :optional_params do optional :description, type: String, desc: 'The description of the milestone' optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' @@ -18,10 +20,11 @@ module API optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IIDs of the milestones' optional :title, type: String, desc: 'The title of the milestones' optional :search, type: String, desc: 'The search criteria for the title or description of the milestone' - optional :include_parent_milestones, type: Grape::API::Boolean, default: false, - desc: 'Include group milestones from parent and its ancestors' + optional :include_parent_milestones, type: Grape::API::Boolean, desc: 'Deprecated: see `include_ancestors`' + optional :include_ancestors, type: Grape::API::Boolean, desc: 'Include milestones from all parent groups' optional :updated_before, type: DateTime, desc: 'Return milestones updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :updated_after, type: DateTime, desc: 'Return milestones updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' + mutually_exclusive :include_ancestors, :include_parent_milestones use :pagination end @@ -35,6 +38,13 @@ module API end def list_milestones_for(parent) + if params.include?(:include_parent_milestones) + params[:include_ancestors] = params.delete(:include_parent_milestones) + else + # include_ancestors should default to false + params[:include_ancestors] ||= false + end + milestones = MilestonesFinder.new( params.merge(parent_finder_params(parent)) ).execute @@ -82,12 +92,10 @@ module API end def parent_finder_params(parent) - include_parent = params[:include_parent_milestones].present? - if parent.is_a?(Project) - { project_ids: parent.id, group_ids: (include_parent ? project_group_ids(parent) : nil) } + project_finder_params(parent, params) else - { group_ids: (include_parent ? group_and_ancestor_ids(parent) : parent.id) } + group_finder_params(parent, params) end end @@ -108,21 +116,6 @@ module API [MergeRequestsFinder, Entities::MergeRequestBasic] end end - - def project_group_ids(parent) - group = parent.group - return unless group.present? - - group.self_and_ancestors.select(:id) - end - - def group_and_ancestor_ids(group) - return unless group.present? - - group.self_and_ancestors - .public_or_visible_to_user(current_user) - .select(:id) - end end end end diff --git a/lib/api/ml/mlflow/api_helpers.rb b/lib/api/ml/mlflow/api_helpers.rb index aefa156717c..66d79753110 100644 --- a/lib/api/ml/mlflow/api_helpers.rb +++ b/lib/api/ml/mlflow/api_helpers.rb @@ -4,6 +4,8 @@ module API module Ml module Mlflow module ApiHelpers + OUTER_QUOTES_REGEXP = /^("|')|("|')?$/ + def check_api_read! not_found! unless can?(current_user, :read_model_experiments, user_project) end @@ -16,6 +18,10 @@ module API not_found! unless can?(current_user, :read_model_registry, user_project) end + def check_api_model_registry_write! + unauthorized! unless can?(current_user, :write_model_registry, user_project) + end + def resource_not_found! render_structured_api_error!({ error_code: 'RESOURCE_DOES_NOT_EXIST' }, 404) end @@ -28,6 +34,10 @@ module API render_structured_api_error!({ error_code: 'INVALID_PARAMETER_VALUE', message: message }, 400) end + def update_failed! + render_structured_api_error!({ error_code: 'UPDATE_FAILED' }, 400) + end + def experiment_repository ::Ml::ExperimentTracking::ExperimentRepository.new(user_project, current_user) end @@ -75,6 +85,34 @@ module API } end + def model_order_params(params) + if params[:order_by].blank? + order_by = 'name' + sort = 'asc' + else + order_by, sort = params[:order_by].downcase.split(' ') + order_by = 'updated_at' if order_by == 'last_updated_timestamp' + sort ||= 'asc' + end + + { + order_by: order_by, + sort: sort + } + end + + def model_filter_params(params) + return {} if params[:filter].blank? + + param, filter = params[:filter].split('=') + + return {} unless param == 'name' + + filter.gsub!(OUTER_QUOTES_REGEXP, '') unless filter.blank? + + { name: filter } + end + def find_experiment!(iid, name) experiment_repository.by_iid_or_name(iid: iid, name: name) || resource_not_found! end @@ -87,13 +125,12 @@ module API ::Ml::FindModelService.new(project, name).execute || resource_not_found! end - def packages_url - path = api_v4_projects_packages_generic_package_version_path( - id: user_project.id, package_name: '', file_name: '' - ) - path = path.delete_suffix('/package_version') + def find_model_version(project, name, version) + ::Ml::ModelVersions::GetModelVersionService.new(project, name, version).execute || resource_not_found! + end - expose_url(path) + def model + @model ||= find_model(user_project, params[:name]) end end end diff --git a/lib/api/ml/mlflow/model_versions.rb b/lib/api/ml/mlflow/model_versions.rb index 989b79e5774..4b211cf540c 100644 --- a/lib/api/ml/mlflow/model_versions.rb +++ b/lib/api/ml/mlflow/model_versions.rb @@ -6,9 +6,42 @@ module API class ModelVersions < ::API::Base feature_category :mlops - resource :model_versions do + before do + check_api_read! + check_api_model_registry_read! + check_api_write! if route.settings.dig(:api, :write) + check_api_model_registry_write! if route.settings.dig(:model_registry, :write) + end + + resource 'model-versions' do + desc 'Creates a Model Version.' do + success Entities::Ml::Mlflow::ModelVersion + detail 'MLFlow Model Versions map to GitLab Model Versions. https://mlflow.org/docs/2.6.0/rest-api.html#create-modelversion' + end + route_setting :api, write: true + route_setting :model_registry, write: true + params do + # The name param is actually required, however it is listed as optional here + # we can send a custom error response required by MLFlow + optional :name, type: String, + desc: 'Register model under this name This field is required.' + optional :description, type: String, + desc: 'Optional description for model version.' + end + post 'create', urgency: :low do + present ::Ml::CreateModelVersionService.new( + model, + { + model_name: params[:name], + description: params[:description] + } + ).execute, + with: Entities::Ml::Mlflow::ModelVersion, + root: :model_version + end + desc 'Fetch model version by name and version' do - success Entities::Ml::Mlflow::ModelVersions::Responses::Get + success Entities::Ml::Mlflow::ModelVersion detail 'https://mlflow.org/docs/2.6.0/rest-api.html#get-modelversion' end params do @@ -16,14 +49,31 @@ module API requires :version, type: String, desc: 'Model version number' end get 'get', urgency: :low do - check_api_model_registry_read! - resource_not_found! unless params[:name] && params[:version] - model_version = ::Ml::ModelVersions::GetModelVersionService.new( - user_project, params[:name], params[:version] + present find_model_version(user_project, params[:name], params[:version]), + with: Entities::Ml::Mlflow::ModelVersion, root: :model_version + end + + desc 'Updates a Model Version.' do + success Entities::Ml::Mlflow::ModelVersion + detail 'https://mlflow.org/docs/2.6.0/rest-api.html#update-modelversion' + end + route_setting :api, write: true + route_setting :model_registry, write: true + params do + # These params are actually required, however it is listed as optional here + # we can send a custom error response required by MLFlow + optional :name, type: String, desc: 'Model version name' + optional :version, type: String, desc: 'Model version number' + optional :description, type: String, desc: 'Model version description' + end + patch 'update', urgency: :low do + invalid_parameter! unless params[:name] && params[:version] && params[:description] + result = ::Ml::ModelVersions::UpdateModelVersionService.new( + user_project, params[:name], params[:version], params[:description] ).execute - resource_not_found! unless model_version - response = { model_version: model_version } - present response, with: Entities::Ml::Mlflow::ModelVersions::Responses::Get + update_failed! unless result.success? + + present result.payload, with: Entities::Ml::Mlflow::ModelVersion, root: :model_version end end end diff --git a/lib/api/ml/mlflow/registered_models.rb b/lib/api/ml/mlflow/registered_models.rb index 18b705ad214..a68a2767a74 100644 --- a/lib/api/ml/mlflow/registered_models.rb +++ b/lib/api/ml/mlflow/registered_models.rb @@ -11,8 +11,9 @@ module API before do check_api_read! - check_api_write! unless request.get? || request.head? check_api_model_registry_read! + check_api_write! if route.settings.dig(:api, :write) + check_api_model_registry_write! if route.settings.dig(:model_registry, :write) end resource 'registered-models' do @@ -20,6 +21,8 @@ module API success Entities::Ml::Mlflow::RegisteredModel detail 'MLFlow Registered Models map to GitLab Models. https://mlflow.org/docs/2.6.0/rest-api.html#create-registeredmodel' end + route_setting :api, write: true + route_setting :model_registry, write: true params do requires :name, type: String, desc: 'Register models under this name.' @@ -60,6 +63,8 @@ module API success Entities::Ml::Mlflow::RegisteredModel detail 'https://mlflow.org/docs/2.6.0/rest-api.html#update-registeredmodel' end + route_setting :api, write: true + route_setting :model_registry, write: true params do # The name param is actually required, however it is listed as optional here # we can send a custom error response required by MLFlow @@ -74,7 +79,7 @@ module API end desc 'Fetch the latest Model Version for the given Registered Model Name' do - success Entities::Ml::Mlflow::ModelVersions::Types::ModelVersion + success Entities::Ml::Mlflow::ModelVersion detail 'https://mlflow.org/docs/2.6.0/rest-api.html#get-latest-modelversions' end params do @@ -86,9 +91,78 @@ module API post 'get-latest-versions', urgency: :low do model = find_model(user_project, params[:name]) - present [model.latest_version], with: Entities::Ml::Mlflow::ModelVersions::Types::ModelVersion, + present [model.latest_version], with: Entities::Ml::Mlflow::ModelVersion, root: :model_versions end + + desc 'Delete a Registered Model by Name' do + success Entities::Ml::Mlflow::RegisteredModel + detail 'https://mlflow.org/docs/2.6.0/rest-api.html#delete-registeredmodel' + end + route_setting :api, write: true + route_setting :model_registry, write: true + params do + # The name param is actually required, however it is listed as optional here + # we can send a custom error response required by MLFlow + optional :name, type: String, + desc: 'Registered model unique name identifier, in reference to the project' + end + delete 'delete', urgency: :low do + resource_not_found! unless params[:name] + + model = ::Ml::FindModelService.new(user_project, params[:name]).execute + + resource_not_found! unless model + + if ::Ml::DestroyModelService.new(model, current_user).execute + present({}) + else + render_api_error!('Model could not be deleted', 400) + end + end + + desc 'Search Registered Models within a project' do + success Entities::Ml::Mlflow::RegisteredModel + detail 'https://mlflow.org/docs/2.6.0/rest-api.html#search-registeredmodels' + end + params do + optional :filter, + type: String, + desc: "Filter to search models. must be in the format `name='value'`. Only filtering by name is supported" + optional :max_results, + type: Integer, + desc: 'Maximum number of models desired. Default is 200. Max threshold is 1000.', + default: 200 + optional :order_by, + type: String, + desc: 'Order criteria. Can be by name or last_updated_timestamp, with optional DESC or ASC (default)' \ + 'Valid examples: `name`, `name DESC`, `last_updated_timestamp DESC`' \ + 'Sorting by model metadata is not supported.', + default: 'name ASC' + optional :page_token, + type: String, + desc: 'Token for pagination' + end + get 'search', urgency: :low do + max_results = [params[:max_results], 1000].min + + finder_params = model_order_params(params) + filter_params = model_filter_params(params) + + if !params[:filter].nil? && !filter_params.key?(:name) + invalid_parameter!("Invalid attribute key specified. Valid keys are '{'name'}'") + end + + finder = ::Projects::Ml::ModelFinder.new(user_project, finder_params.merge(filter_params)) + paginator = finder.execute.keyset_paginate(cursor: params[:page_token], per_page: max_results) + + result = { + registered_models: paginator.records, + next_page_token: paginator.cursor_for_next_page + } + + present result, with: Entities::Ml::Mlflow::ListRegisteredModels + end end end end diff --git a/lib/api/ml/mlflow/runs.rb b/lib/api/ml/mlflow/runs.rb index 6716db21407..d03bf1fc576 100644 --- a/lib/api/ml/mlflow/runs.rb +++ b/lib/api/ml/mlflow/runs.rb @@ -31,7 +31,7 @@ module API end post 'create', urgency: :low do present candidate_repository.create!(experiment, params[:start_time], params[:tags], params[:run_name]), - with: Entities::Ml::Mlflow::GetRun, packages_url: packages_url + with: Entities::Ml::Mlflow::GetRun end desc 'Gets an MLFlow Run, which maps to GitLab Candidates' do @@ -43,7 +43,7 @@ module API optional :run_uuid, type: String, desc: 'This parameter is ignored' end get 'get', urgency: :low do - present candidate, with: Entities::Ml::Mlflow::GetRun, packages_url: packages_url + present candidate, with: Entities::Ml::Mlflow::GetRun end desc 'Searches runs/candidates within a project' do @@ -83,7 +83,7 @@ module API next_page_token: paginator.cursor_for_next_page } - present result, with: Entities::Ml::Mlflow::SearchRuns, packages_url: packages_url + present result, with: Entities::Ml::Mlflow::SearchRuns end desc 'Updates a Run.' do @@ -101,7 +101,7 @@ module API post 'update', urgency: :low do candidate_repository.update(candidate, params[:status], params[:end_time]) - present candidate, with: Entities::Ml::Mlflow::UpdateRun, packages_url: packages_url + present candidate, with: Entities::Ml::Mlflow::UpdateRun end desc 'Logs a metric to a run.' do diff --git a/lib/api/ml_model_packages.rb b/lib/api/ml_model_packages.rb index 8a7a8fc9525..85c8146dda8 100644 --- a/lib/api/ml_model_packages.rb +++ b/lib/api/ml_model_packages.rb @@ -23,8 +23,8 @@ module API end authenticate_with do |accept| - accept.token_types(:personal_access_token, :deploy_token, :job_token) - .sent_through(:http_token) + accept.token_types(:personal_access_token, :job_token) + .sent_through(:http_bearer_token) end helpers do @@ -38,6 +38,15 @@ module API def max_file_size_exceeded? project.actual_limits.exceeded?(:ml_model_max_file_size, params[:file].size) end + + def find_model_version! + ::Ml::ModelVersion.by_project_id_name_and_version(project.id, params[:model_name], params[:model_version]) || + not_found! + end + + def model_version + @model_version ||= find_model_version! + end end params do @@ -88,10 +97,12 @@ module API end put do authorize_upload!(project) + not_found! unless can?(current_user, :write_model_registry, project) bad_request!('File is too large') if max_file_size_exceeded? create_package_file_params = declared(params).merge( + model_version: model_version, build: current_authenticated_job, package_name: params[:model_name], package_version: params[:model_version] @@ -123,9 +134,7 @@ module API get do authorize_read_package!(project) - package = ::Packages::MlModel::PackageFinder.new(project) - .execute!(params[:model_name], params[:model_version]) - package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute! + package_file = ::Packages::PackageFileFinder.new(model_version.package, params[:file_name]).execute! present_package_file!(package_file) end diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index 7a6872ee82f..394f8911e9e 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -38,6 +38,10 @@ module API end strong_memoize_attr :project_or_group_without_auth + def symbol_server_enabled? + project_or_group_without_auth.package_settings.nuget_symbol_server_enabled + end + def require_authenticated! unauthorized! unless current_user end diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index b061876b997..e25b47397a7 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -46,6 +46,10 @@ module API end strong_memoize_attr :project_or_group_without_auth + def symbol_server_enabled? + project_or_group_without_auth.namespace.package_settings.nuget_symbol_server_enabled + end + def snowplow_gitlab_standard_context { project: project_or_group, namespace: project_or_group.namespace } end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index c9cba397f5c..011d5e69f00 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -57,7 +57,7 @@ module API use :pagination end get ":id/hooks" do - present paginate(user_project.hooks), with: Entities::ProjectHook + present paginate(user_project.hooks), with: Entities::ProjectHook, with_url_variables: false end desc 'Get project hook' do diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb index 7f531525870..1dc21982134 100644 --- a/lib/api/project_packages.rb +++ b/lib/api/project_packages.rb @@ -134,8 +134,6 @@ module API destroy_conditionally!(package) do |package| ::Packages::MarkPackageForDestructionService.new(container: package, current_user: current_user).execute - - enqueue_sync_metadata_cache_worker(user_project, package.name) if package.npm? end end end diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index 49e2e4d8a91..a59d1af6250 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -8,7 +8,7 @@ module API # The regex is needed to ensure a period (e.g. agpl-3.0) # isn't confused with a format type. We also need to allow encoded # values (e.g. C%2B%2B for C++), so allow % and + as well. - TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: /[\w%.+-]+/) + TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: /[\w()%.+-]+/) before { authenticate_non_get! } diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3b80fd125ca..de1ed75ca70 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -755,7 +755,7 @@ module API end params do requires :group_id, type: Integer, desc: 'The ID of a group', documentation: { example: 1 } - requires :group_access, type: Integer, values: Gitlab::Access.values, as: :link_group_access, desc: 'The group access level' + requires :group_access, type: Integer, values: Gitlab::Access.all_values, as: :link_group_access, desc: 'The group access level' optional :expires_at, type: Date, desc: 'Share expiration date' end post ":id/share", feature_category: :groups_and_projects do @@ -798,8 +798,7 @@ module API result = ::Projects::GroupLinks::DestroyService.new(user_project, current_user).execute(link) if result.error? - status = :not_found if result.reason == :not_found - render_api_error!(result.message, status) + render_api_error!(result.message, result.reason) end end end diff --git a/lib/api/search.rb b/lib/api/search.rb index 5f78979ec8a..d5be2c12da0 100644 --- a/lib/api/search.rb +++ b/lib/api/search.rb @@ -119,7 +119,7 @@ module API end def search_type(additional_params = {}) - 'basic' + search_service(additional_params).search_type end def search_scope diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 7ad4ecd88b1..9de9f19ccd7 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -212,6 +212,7 @@ module API optional :pipeline_limit_per_project_user_sha, type: Integer, desc: "Maximum number of pipeline creation requests allowed per minute per user and commit. Set to 0 for unlimited requests per minute." optional :jira_connect_application_key, type: String, desc: "Application ID of the OAuth application that should be used to authenticate with the GitLab for Jira Cloud app" optional :jira_connect_proxy_url, type: String, desc: "URL of the GitLab instance that should be used as a proxy for the GitLab for Jira Cloud app" + optional :bulk_import_concurrent_pipeline_batch_limit, type: Integer, desc: 'Maximum simultaneous Direct Transfer pipeline batches to process' optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer' optional :bulk_import_max_download_file, type: Integer, desc: 'Maximum download file size in MB when importing from source GitLab instances by direct transfer' optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token' @@ -226,6 +227,7 @@ module API end optional :namespace_aggregation_schedule_lease_duration_in_seconds, type: Integer, desc: 'Maximum duration (in seconds) between refreshes of namespace statistics (Default: 300)' optional :project_jobs_api_rate_limit, type: Integer, desc: 'Maximum authenticated requests to /project/:id/jobs per minute' + optional :security_txt_content, type: String, desc: 'Public security contact information made available at https://gitlab.example.com/.well-known/security.txt' Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index 8f264097867..9e82a849c98 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -226,82 +226,6 @@ module API end end end - - params do - requires :id, type: String, desc: 'The ID or full path of a project' - includes :module_name - includes :module_version - end - - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - namespace ':id/packages/terraform/modules/:module_name/:module_system/*module_version/file' do - authenticate_with do |accept| - accept.token_types(:deploy_token).sent_through(:http_deploy_token_header) - accept.token_types(:job_token).sent_through(:http_job_token_header) - accept.token_types(:personal_access_token).sent_through(:http_private_token_header) - end - - desc 'Workhorse authorize Terraform Module package file' do - detail 'This feature was introduced in GitLab 13.11' - success code: 200 - failure [ - { code: 403, message: 'Forbidden' } - ] - tags %w[terraform_registry] - end - - put 'authorize' do - authorize_workhorse!( - subject: authorized_user_project, - maximum_size: authorized_user_project.actual_limits.terraform_module_max_file_size - ) - end - - desc 'Upload Terraform Module package file' do - detail 'This feature was introduced in GitLab 13.11' - success code: 201 - failure [ - { code: 400, message: 'Invalid file' }, - { code: 401, message: 'Unauthorized' }, - { code: 403, message: 'Forbidden' }, - { code: 404, message: 'Not found' } - ] - consumes %w[multipart/form-data] - tags %w[terraform_registry] - end - - params do - requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)', documentation: { type: 'file' } - end - - put do - authorize_upload!(authorized_user_project) - bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:terraform_module_max_file_size, params[:file].size) - - create_package_file_params = { - module_name: params['module_name'], - module_system: params['module_system'], - module_version: params['module_version'], - file: params['file'], - build: current_authenticated_job - } - - result = ::Packages::TerraformModule::CreatePackageService - .new(authorized_user_project, current_user, create_package_file_params) - .execute - - render_api_error!(result[:message], result[:http_status]) if result[:status] == :error - - track_package_event('push_package', :terraform_module, project: authorized_user_project, namespace: authorized_user_project.namespace) - - created! - rescue ObjectStorage::RemoteStoreError => e - Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id }) - - forbidden! - end - end - end end end end diff --git a/lib/api/terraform/modules/v1/project_packages.rb b/lib/api/terraform/modules/v1/project_packages.rb new file mode 100644 index 00000000000..07dfddefefc --- /dev/null +++ b/lib/api/terraform/modules/v1/project_packages.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module API + module Terraform + module Modules + module V1 + class ProjectPackages < ::API::Base + include ::API::Helpers::Authentication + helpers ::API::Helpers::PackagesHelpers + helpers ::API::Helpers::Packages::BasicAuthHelpers + + feature_category :package_registry + urgency :low + + after_validation do + require_packages_enabled! + end + + params do + requires :id, type: String, desc: 'The ID or full path of a project' + requires :module_name, type: String, desc: "", regexp: API::NO_SLASH_URL_PART_REGEX + requires :module_system, type: String, regexp: API::NO_SLASH_URL_PART_REGEX + requires :module_version, type: String, desc: 'Module version', regexp: Gitlab::Regex.semver_regex + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + namespace ':id/packages/terraform/modules/:module_name/:module_system/*module_version/file' do + authenticate_with do |accept| + accept.token_types(:deploy_token).sent_through(:http_deploy_token_header) + accept.token_types(:job_token).sent_through(:http_job_token_header) + accept.token_types(:personal_access_token).sent_through(:http_private_token_header) + end + + desc 'Workhorse authorize Terraform Module package file' do + detail 'This feature was introduced in GitLab 13.11' + success code: 200 + failure [ + { code: 403, message: 'Forbidden' } + ] + tags %w[terraform_registry] + end + + put 'authorize' do + authorize_workhorse!( + subject: authorized_user_project, + maximum_size: authorized_user_project.actual_limits.terraform_module_max_file_size + ) + end + + desc 'Upload Terraform Module package file' do + detail 'This feature was introduced in GitLab 13.11' + success code: 201 + failure [ + { code: 400, message: 'Invalid file' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not found' } + ] + consumes %w[multipart/form-data] + tags %w[terraform_registry] + end + + params do + requires :file, type: ::API::Validations::Types::WorkhorseFile, + desc: 'The package file to be published (generated by Multipart middleware)', + documentation: { type: 'file' } + end + + put do + authorize_upload!(authorized_user_project) + + bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?( + :terraform_module_max_file_size, params[:file].size) + + create_package_file_params = { + module_name: params['module_name'], + module_system: params['module_system'], + module_version: params['module_version'], + file: params['file'], + build: current_authenticated_job + } + + result = ::Packages::TerraformModule::CreatePackageService + .new(authorized_user_project, current_user, create_package_file_params) + .execute + + render_api_error!(result[:message], result[:http_status]) if result[:status] == :error + + track_package_event('push_package', :terraform_module, project: authorized_user_project, + namespace: authorized_user_project.namespace) + + created! + rescue ObjectStorage::RemoteStoreError => e + Gitlab::ErrorTracking.track_exception( + e, + extra: { file_name: params[:file_name], project_id: authorized_user_project.id } + ) + + forbidden! + end + end + end + end + end + end + end +end diff --git a/lib/api/user_runners.rb b/lib/api/user_runners.rb index edbd0214bb8..381a1a5aab4 100644 --- a/lib/api/user_runners.rb +++ b/lib/api/user_runners.rb @@ -45,7 +45,7 @@ module API optional :maximum_timeout, type: Integer, desc: 'Maximum timeout that limits the amount of time (in seconds) that runners can run jobs' end - post 'runners', urgency: :low, feature_category: :runner_fleet do + post 'runners', urgency: :low, feature_category: :fleet_visibility do attributes = attributes_for_keys( %i[runner_type group_id project_id description maintenance_note paused locked run_untagged tag_list access_level maximum_timeout] diff --git a/lib/api/users.rb b/lib/api/users.rb index 888623429a2..38fa247055e 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1077,6 +1077,8 @@ module API end end + helpers Helpers::UserPreferencesHelpers + desc "Get the currently authenticated user's SSH keys" do success Entities::SSHKey end @@ -1267,7 +1269,9 @@ module API optional :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page' optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs' optional :pass_user_identities_to_ci_jwt, type: Boolean, desc: 'Flag indicating the user passes their external identities to a CI job as part of a JSON web token.' - at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt + optional :code_suggestions, type: Boolean, desc: 'Flag indicating the user allows code suggestions.' \ + 'Argument is experimental and can be removed in the future without notice.' + at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs, :pass_user_identities_to_ci_jwt, :code_suggestions end put "preferences", feature_category: :user_profile, urgency: :high do authenticate! @@ -1276,6 +1280,10 @@ module API attrs = declared_params(include_missing: false) + attrs = update_user_namespace_settings(attrs) + + render_api_error!('400 Bad Request', 400) unless attrs + service = ::UserPreferences::UpdateService.new(current_user, attrs).execute if service.success? present preferences, with: Entities::UserPreferences diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 58a8c19c1ce..a0eaccb1ca4 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -24,44 +24,37 @@ module Backup end override :dump - def dump(destination_dir, backup_id) + def dump(destination_dir, _) FileUtils.mkdir_p(destination_dir) - each_database(destination_dir) do |database_name, current_db| - model = current_db[:model] - snapshot_id = current_db[:snapshot_id] + each_database(destination_dir) do |backup_connection| + pg_env = backup_connection.database_configuration.pg_env_variables + active_record_config = backup_connection.database_configuration.activerecord_variables + pg_database_name = active_record_config[:database] - pg_env = model.config[:pg_env] - connection = model.connection - active_record_config = model.config[:activerecord] - pg_database = active_record_config[:database] + dump_file_name = file_name(destination_dir, backup_connection.connection_name) + FileUtils.rm_f(dump_file_name) - db_file_name = file_name(destination_dir, database_name) - FileUtils.rm_f(db_file_name) - - progress.print "Dumping PostgreSQL database #{pg_database} ... " + progress.print "Dumping PostgreSQL database #{pg_database_name} ... " - pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. - pgsql_args << '--if-exists' - pgsql_args << "--snapshot=#{snapshot_id}" if snapshot_id + schemas = [] if Gitlab.config.backup.pg_schema - pgsql_args << '-n' - pgsql_args << Gitlab.config.backup.pg_schema - - Gitlab::Database::EXTRA_SCHEMAS.each do |schema| - pgsql_args << '-n' - pgsql_args << schema.to_s - end + schemas << Gitlab.config.backup.pg_schema + schemas.push(*Gitlab::Database::EXTRA_SCHEMAS.map(&:to_s)) end - success = with_transient_pg_env(pg_env) do - Backup::Dump::Postgres.new.dump(pg_database, db_file_name, pgsql_args) - end + pg_dump = ::Gitlab::Backup::Cli::Utils::PgDump.new( + database_name: pg_database_name, + snapshot_id: backup_connection.snapshot_id, + schemas: schemas, + env: pg_env) + + success = Backup::Dump::Postgres.new.dump(dump_file_name, pg_dump) - connection.rollback_transaction if snapshot_id + backup_connection.release_snapshot! if backup_connection.snapshot_id - raise DatabaseBackupError.new(active_record_config, db_file_name) unless success + raise DatabaseBackupError.new(active_record_config, dump_file_name) unless success report_success(success) progress.flush @@ -76,10 +69,10 @@ module Backup override :restore def restore(destination_dir, backup_id) - base_models_for_backup.each do |database_name, _base_model| - backup_model = Backup::DatabaseModel.new(database_name) + base_models_for_backup.each do |database_name, _| + backup_connection = Backup::DatabaseConnection.new(database_name) - config = backup_model.config[:activerecord] + config = backup_connection.database_configuration.activerecord_variables db_file_name = file_name(destination_dir, database_name) database = config[:database] @@ -100,10 +93,10 @@ module Backup # hanging out from a failed upgrade drop_tables(database_name) - pg_env = backup_model.config[:pg_env] + pg_env = backup_connection.database_configuration.pg_env_variables success = with_transient_pg_env(pg_env) do decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%w[gzip -cd], out: decompress_wr, in: db_file_name) + decompress_pid = spawn(decompress_cmd, out: decompress_wr, in: db_file_name) decompress_wr.close status, @errors = @@ -235,6 +228,7 @@ module Backup puts_time 'done'.color(:green) end + # @deprecated This will be removed when restore operation is refactored to use extended_env directly def with_transient_pg_env(extended_env) ENV.merge!(extended_env) result = yield @@ -248,32 +242,36 @@ module Backup end def each_database(destination_dir, &block) - databases = {} + databases = [] + + # each connection will loop through all database connections defined in `database.yml` + # and reject the ones that are shared, so we don't get duplicates + # + # we consider a connection to be shared when it has `database_tasks: false` ::Gitlab::Database::EachDatabase.each_connection( only: base_models_for_backup.keys, include_shared: false - ) do |_connection, name| - next if databases[name] - - backup_model = Backup::DatabaseModel.new(name) - - databases[name] = { - model: backup_model - } + ) do |_, database_connection_name| + backup_connection = Backup::DatabaseConnection.new(database_connection_name) + databases << backup_connection - next unless Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES - - connection = backup_model.connection + next unless multiple_databases? begin - Gitlab::Database::TransactionTimeoutSettings.new(connection).disable_timeouts - connection.begin_transaction(isolation: :repeatable_read) - databases[name][:snapshot_id] = connection.select_value("SELECT pg_export_snapshot()") + # Trigger a transaction snapshot export that will be used by pg_dump later on + backup_connection.export_snapshot! rescue ActiveRecord::ConnectionNotEstablished - raise Backup::DatabaseBackupError.new(backup_model.config[:activerecord], file_name(destination_dir, name)) + raise Backup::DatabaseBackupError.new( + backup_connection.database_configuration.activerecord_variables, + file_name(destination_dir, database_connection_name) + ) end end databases.each(&block) end + + def multiple_databases? + Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES + end end end diff --git a/lib/backup/database_configuration.rb b/lib/backup/database_configuration.rb new file mode 100644 index 00000000000..1a6a476f9c1 --- /dev/null +++ b/lib/backup/database_configuration.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module Backup + class DatabaseConfiguration + # Connection name is the key used in `config/database.yml` for multi-database connection configuration + # + # @return [String] + attr_reader :connection_name + + # ActiveRecord base model that is configured to connect to the database identified by connection_name key + # + # @return [ActiveRecord::Base] + attr_reader :source_model + + # Initializes configuration + # + # @param [String] connection_name the key from `database.yml` for multi-database connection configuration + def initialize(connection_name) + @connection_name = connection_name + @source_model = Gitlab::Database.database_base_models_with_gitlab_shared[connection_name] || + Gitlab::Database.database_base_models_with_gitlab_shared['main'] + @activerecord_database_config = ActiveRecord::Base.configurations.find_db_config(connection_name) + end + + # ENV variables that can override each database configuration + # These are used along with OVERRIDE_PREFIX and database name + # @see #process_config_overrides! + SUPPORTED_OVERRIDES = { + username: 'PGUSER', + host: 'PGHOST', + port: 'PGPORT', + password: 'PGPASSWORD', + # SSL + sslmode: 'PGSSLMODE', + sslkey: 'PGSSLKEY', + sslcert: 'PGSSLCERT', + sslrootcert: 'PGSSLROOTCERT', + sslcrl: 'PGSSLCRL', + sslcompression: 'PGSSLCOMPRESSION' + }.freeze + + # Prefixes used for ENV variables overriding database configuration + OVERRIDE_PREFIXES = %w[GITLAB_BACKUP_ GITLAB_OVERRIDE_].freeze + + # Return the HashConfig for the database + # + # @return [ActiveRecord::DatabaseConfigurations::HashConfig] + def activerecord_configuration + ActiveRecord::DatabaseConfigurations::HashConfig.new( + @activerecord_database_config.env_name, + connection_name, + activerecord_variables + ) + end + + # Return postgres ENV variable values for current database with overrided values + # + # @return[Hash<String,String>] hash of postgres ENV variables + def pg_env_variables + process_config_overrides! unless @pg_env_variables + + @pg_env_variables + end + + # Return activerecord configuration values for current database with overrided values + # + # @return[Hash<String,String>] activerecord database.yml configuration compatible values + def activerecord_variables + process_config_overrides! unless @activerecord_variables + + @activerecord_variables + end + + private + + def process_config_overrides! + @activerecord_variables = original_activerecord_config + @pg_env_variables = {} + + SUPPORTED_OVERRIDES.each do |config_key, env_variable_name| + # This enables the use of different PostgreSQL settings in + # case PgBouncer is used. PgBouncer clears the search path, + # which wreaks havoc on Rails if connections are reused. + OVERRIDE_PREFIXES.each do |override_prefix| + override_all = "#{override_prefix}#{env_variable_name}" + override_db = "#{override_prefix}#{connection_name.upcase}_#{env_variable_name}" + val = ENV[override_db].presence || + ENV[override_all].presence || + @activerecord_variables[config_key].to_s.presence + + next unless val + + @pg_env_variables[env_variable_name] = val + @activerecord_variables[config_key] = val + end + end + end + + # Return the database configuration from rails config/database.yml file + # in the format expected by ActiveRecord::DatabaseConfigurations::HashConfig + # + # @return [Hash] configuration hash + def original_activerecord_config + @activerecord_database_config.configuration_hash.dup + end + end +end diff --git a/lib/backup/database_connection.rb b/lib/backup/database_connection.rb new file mode 100644 index 00000000000..f3f0a5dfcb5 --- /dev/null +++ b/lib/backup/database_connection.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Backup + class DatabaseConnection + attr_reader :database_configuration, :snapshot_id + + delegate :connection_name, to: :database_configuration + delegate :connection, to: :@backup_model + + # Initializes a database connection + # + # @param [String] connection_name the key from `database.yml` for multi-database connection configuration + def initialize(connection_name) + @database_configuration = Backup::DatabaseConfiguration.new(connection_name) + @backup_model = backup_model + @snapshot_id = nil + + configure_backup_model + end + + # Start a new transaction and run pg_export_snapshot() + # Returns the snapshot identifier + # + # @return [String] snapshot identifier + def export_snapshot! + Gitlab::Database::TransactionTimeoutSettings.new(connection).disable_timeouts + + connection.begin_transaction(isolation: :repeatable_read) + @snapshot_id = connection.select_value("SELECT pg_export_snapshot()") + end + + # Rollback the transaction to release the effects of pg_export_snapshot() + def release_snapshot! + return unless snapshot_id + + connection.rollback_transaction + @snapshot_id = nil + end + + private + + delegate :activerecord_configuration, to: :database_configuration, private: true + + def configure_backup_model + @backup_model.establish_connection(activerecord_configuration) + + Gitlab::Database::LoadBalancing::Setup.new(@backup_model).setup + end + + # Creates a disposable model to be used to host the Backup connection only + def backup_model + klass_name = connection_name.camelize + + return "#{self.class.name}::#{klass_name}".constantize if self.class.const_defined?(klass_name.to_sym, false) + + self.class.const_set(klass_name, Class.new(ApplicationRecord)) + end + end +end diff --git a/lib/backup/database_model.rb b/lib/backup/database_model.rb index b2202ad7794..228a7fa5383 100644 --- a/lib/backup/database_model.rb +++ b/lib/backup/database_model.rb @@ -16,7 +16,7 @@ module Backup sslcompression: 'PGSSLCOMPRESSION' }.freeze - OVERRIDE_PREFIX = "GITLAB_BACKUP_" + OVERRIDE_PREFIXES = %w[GITLAB_BACKUP_ GITLAB_OVERRIDE_].freeze attr_reader :config @@ -31,7 +31,8 @@ module Backup private def configure_model(name) - source_model = Gitlab::Database.database_base_models_with_gitlab_shared[name] + source_model = Gitlab::Database.database_base_models_with_gitlab_shared[name] || + Gitlab::Database.database_base_models_with_gitlab_shared['main'] @model = backup_model_for(name) @@ -67,14 +68,16 @@ module Backup # This enables the use of different PostgreSQL settings in # case PgBouncer is used. PgBouncer clears the search path, # which wreaks havoc on Rails if connections are reused. - override_all = "#{OVERRIDE_PREFIX}#{arg}" - override_db = "#{OVERRIDE_PREFIX}#{name.upcase}_#{arg}" - val = ENV[override_db].presence || ENV[override_all].presence || config[opt].to_s.presence + OVERRIDE_PREFIXES.each do |override_prefix| + override_all = "#{override_prefix}#{arg}" + override_db = "#{override_prefix}#{name.upcase}_#{arg}" + val = ENV[override_db].presence || ENV[override_all].presence || config[opt].to_s.presence - next unless val + next unless val - db_config[:pg_env][arg] = val - db_config[:activerecord][opt] = val + db_config[:pg_env][arg] = val + db_config[:activerecord][opt] = val + end end db_config diff --git a/lib/backup/dump/postgres.rb b/lib/backup/dump/postgres.rb index 1a5128b5a6b..80a49971140 100644 --- a/lib/backup/dump/postgres.rb +++ b/lib/backup/dump/postgres.rb @@ -4,14 +4,21 @@ module Backup class Postgres include Backup::Helper + # Owner can read/write, group no permission, others no permission FILE_PERMISSION = 0o600 - def dump(database_name, output_file, pgsql_args) + # Triggers PgDump and outputs to the provided file path + # + # @param [String] output_file_path full path to the output destination + # @param [Gitlab::Backup::Cli::Utils::PgDump] pg_dump + # @return [Boolean] whether pg_dump finished with success + def dump(output_file_path, pg_dump) compress_rd, compress_wr = IO.pipe - compress_pid = spawn(gzip_cmd, in: compress_rd, out: [output_file, 'w', FILE_PERMISSION]) + + compress_pid = spawn(compress_cmd, in: compress_rd, out: [output_file_path, 'w', FILE_PERMISSION]) compress_rd.close - dump_pid = Process.spawn('pg_dump', *pgsql_args, database_name, out: compress_wr) + dump_pid = pg_dump.spawn(output: compress_wr) compress_wr.close [compress_pid, dump_pid].all? do |pid| diff --git a/lib/backup/files.rb b/lib/backup/files.rb index b8ff7fff591..e3a8290e2e3 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -40,14 +40,14 @@ module Backup end tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{backup_files_realpath} -cf - .]].flatten - status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600]) + status_list, output = run_pipeline!([tar_cmd, compress_cmd], out: [backup_tarball, 'w', 0600]) FileUtils.rm_rf(backup_files_realpath) else tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{app_files_realpath} -cf - .]].flatten - status_list, output = run_pipeline!([tar_cmd, gzip_cmd], out: [backup_tarball, 'w', 0600]) + status_list, output = run_pipeline!([tar_cmd, compress_cmd], out: [backup_tarball, 'w', 0600]) end - unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output) + unless pipeline_succeeded?(tar_status: status_list[0], compress_status: status_list[1], output: output) raise_custom_error(backup_tarball) end end @@ -56,9 +56,9 @@ module Backup def restore(backup_tarball, backup_id) backup_existing_files_dir(backup_tarball) - cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]] + cmd_list = [decompress_cmd, %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]] status_list, output = run_pipeline!(cmd_list, in: backup_tarball) - unless pipeline_succeeded?(gzip_status: status_list[0], tar_status: status_list[1], output: output) + unless pipeline_succeeded?(compress_status: status_list[0], tar_status: status_list[1], output: output) raise Backup::Error, "Restore operation failed: #{output}" end end @@ -108,8 +108,8 @@ module Backup noncritical_warnings.map { |w| warning =~ w }.any? end - def pipeline_succeeded?(tar_status:, gzip_status:, output:) - return false unless gzip_status&.success? + def pipeline_succeeded?(tar_status:, compress_status:, output:) + return false unless compress_status&.success? tar_status&.success? || tar_ignore_non_success?(tar_status.exitstatus, output) end diff --git a/lib/backup/helper.rb b/lib/backup/helper.rb index 2c2e35add0e..3af786654be 100644 --- a/lib/backup/helper.rb +++ b/lib/backup/helper.rb @@ -2,6 +2,8 @@ module Backup module Helper + include ::Gitlab::Utils::StrongMemoize + def access_denied_error(path) message = <<~EOS @@ -30,12 +32,27 @@ module Backup raise message end - def gzip_cmd - @gzip_cmd ||= if ENV['GZIP_RSYNCABLE'] == 'yes' - "gzip --rsyncable -c -1" - else - "gzip -c -1" - end + def compress_cmd + if ENV['COMPRESS_CMD'].present? + puts "Using custom COMPRESS_CMD '#{ENV['COMPRESS_CMD']}'" + puts "Ignoring GZIP_RSYNCABLE" if ENV['GZIP_RSYNCABLE'] == 'yes' + ENV['COMPRESS_CMD'] + elsif ENV['GZIP_RSYNCABLE'] == 'yes' + "gzip --rsyncable -c -1" + else + "gzip -c -1" + end + end + strong_memoize_attr :compress_cmd + + def decompress_cmd + if ENV['DECOMPRESS_CMD'].present? + puts "Using custom DECOMPRESS_CMD '#{ENV['DECOMPRESS_CMD']}'" + ENV['DECOMPRESS_CMD'] + else + "gzip -cd" + end end + strong_memoize_attr :decompress_cmd end end diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 46825dbd203..c3154ccfbb5 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -38,7 +38,6 @@ module Backup ensure strategy.finish! - cleanup_snippets_without_repositories restore_object_pools end @@ -133,24 +132,6 @@ module Backup pool.schedule end end - - # Snippets without a repository should be removed because they failed to import - # due to having invalid repositories - def cleanup_snippets_without_repositories - invalid_snippets = [] - - snippet_relation.find_each(batch_size: 1000).each do |snippet| - response = Snippets::RepositoryValidationService.new(nil, snippet).execute - next if response.success? - - snippet.repository.remove - progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}") - - invalid_snippets << snippet.id - end - - Snippet.id_in(invalid_snippets).delete_all - end end end diff --git a/lib/banzai/filter/absolute_link_filter.rb b/lib/banzai/filter/absolute_link_filter.rb index cc7bf3ed556..7e3024c521c 100644 --- a/lib/banzai/filter/absolute_link_filter.rb +++ b/lib/banzai/filter/absolute_link_filter.rb @@ -6,13 +6,13 @@ module Banzai module Filter # HTML filter that converts relative urls into absolute ones. class AbsoluteLinkFilter < HTML::Pipeline::Filter - CSS = 'a.gfm' + CSS = 'a.gfm' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze def call - return doc unless context[:only_path] == false + return doc if skip? - doc.xpath(XPATH).each do |el| + doc.xpath(self.class::XPATH).each do |el| process_link_attr el.attribute('href') end @@ -21,17 +21,21 @@ module Banzai protected + def skip? + context[:only_path] != false + end + def process_link_attr(html_attr) return if html_attr.blank? return if html_attr.value.start_with?('//') uri = URI(html_attr.value) - html_attr.value = absolute_link_attr(uri) if uri.relative? + html_attr.value = convert_link_href(uri) if uri.relative? rescue URI::Error # noop end - def absolute_link_attr(uri) + def convert_link_href(uri) # Here we really want to expand relative path to absolute path URI.join(Gitlab.config.gitlab.url, uri).to_s end diff --git a/lib/banzai/filter/custom_emoji_filter.rb b/lib/banzai/filter/custom_emoji_filter.rb index dddaaebc9de..4dd6bada306 100644 --- a/lib/banzai/filter/custom_emoji_filter.rb +++ b/lib/banzai/filter/custom_emoji_filter.rb @@ -48,10 +48,9 @@ module Banzai private def has_custom_emoji? - strong_memoize(:has_custom_emoji) do - CustomEmoji.for_resource(resource_parent).any? - end + all_custom_emoji&.any? end + strong_memoize_attr :has_custom_emoji? def resource_parent context[:project] || context[:group] @@ -62,9 +61,12 @@ module Banzai end def all_custom_emoji - @all_custom_emoji ||= - CustomEmoji.for_resource(resource_parent).by_name(custom_emoji_candidates).index_by(&:name) + Groups::CustomEmojiFinder.new(resource_parent, { include_ancestor_groups: true }) + .execute + .by_name(custom_emoji_candidates) + .index_by(&:name) end + strong_memoize_attr :all_custom_emoji end end end diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index a546a72da5d..e6a0cdfe020 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -23,6 +23,26 @@ module Banzai raise NameError, "`#{engine_from_context}` is unknown markdown engine" end + # Parses string representing a sourcepos in format + # "start_row:start_column-end_row:end_column" into 0-based + # attributes. For example, "1:10-14:1" becomes + # { + # start: { row: 0, col: 9 }, + # end: { row: 13, col: 0 } + # } + def parse_sourcepos(sourcepos) + start_pos, end_pos = sourcepos&.split('-') + start_row, start_col = start_pos&.split(':') + end_row, end_col = end_pos&.split(':') + + return unless start_row && start_col && end_row && end_col + + { + start: { row: [1, start_row.to_i].max - 1, col: [1, start_col.to_i].max - 1 }, + end: { row: [1, end_row.to_i].max - 1, col: [1, end_col.to_i].max - 1 } + } + end + private def engine(engine_from_context) diff --git a/lib/banzai/filter/quick_action_filter.rb b/lib/banzai/filter/quick_action_filter.rb new file mode 100644 index 00000000000..29f0be68e27 --- /dev/null +++ b/lib/banzai/filter/quick_action_filter.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Banzai + module Filter + # Filter which looks for possible paragraphs with quick action lines, and allows + # another processor to do final determination. Paragraph source position + # is returned in `result[:quick_action_paragraphs]`. + class QuickActionFilter < HTML::Pipeline::Filter + def call + result[:quick_action_paragraphs] = [] + + doc.children.xpath('self::p').each do |node| + next unless node.attributes['data-sourcepos'] + + sourcepos = ::Banzai::Filter::MarkdownFilter.parse_sourcepos(node.attributes['data-sourcepos'].value) + + node.children.xpath('self::text()').each do |text_node| + next unless %r{^/}.match?(text_node.content) + + result[:quick_action_paragraphs] << + { start_line: sourcepos[:start][:row], end_line: sourcepos[:end][:row] } + break + end + end + + doc + end + end + end +end diff --git a/lib/banzai/pipeline/quick_action_pipeline.rb b/lib/banzai/pipeline/quick_action_pipeline.rb new file mode 100644 index 00000000000..1290a8ee2dd --- /dev/null +++ b/lib/banzai/pipeline/quick_action_pipeline.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + # Pipeline for detecting possible paragraphs with quick actions, + # leveraging the markdown processor + class QuickActionPipeline < BasePipeline + def self.filters + FilterArray[ + Filter::NormalizeSourceFilter, + Filter::TruncateSourceFilter, + Filter::FrontMatterFilter, + Filter::BlockquoteFenceFilter, + Filter::MarkdownFilter, + Filter::QuickActionFilter + ] + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index 64550a0525c..f28b2a0a899 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -2,6 +2,8 @@ module Bitbucket class Connection + include Bitbucket::ExponentialBackoff + DEFAULT_API_VERSION = '2.0' DEFAULT_BASE_URI = 'https://api.bitbucket.org/' DEFAULT_QUERY = {}.freeze @@ -22,7 +24,14 @@ module Bitbucket def get(path, extra_query = {}) refresh! if expired? - response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response = if Feature.enabled?(:bitbucket_importer_exponential_backoff) + retry_with_exponential_backoff do + connection.get(build_url(path), params: @default_query.merge(extra_query)) + end + else + connection.get(build_url(path), params: @default_query.merge(extra_query)) + end + response.parsed end @@ -44,6 +53,10 @@ module Bitbucket @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) end + def logger + Gitlab::BitbucketImport::Logger + end + def connection @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) end diff --git a/lib/bitbucket/exponential_backoff.rb b/lib/bitbucket/exponential_backoff.rb new file mode 100644 index 00000000000..702010409de --- /dev/null +++ b/lib/bitbucket/exponential_backoff.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Bitbucket + module ExponentialBackoff + extend ActiveSupport::Concern + + INITIAL_DELAY = 1.second + EXPONENTIAL_BASE = 2 + MAX_RETRIES = 3 + + RateLimitError = Class.new(StandardError) + + def retry_with_exponential_backoff(&block) + run_retry_with_exponential_backoff(&block) + end + + private + + def run_retry_with_exponential_backoff + retries = 0 + delay = INITIAL_DELAY + + loop do + return yield + rescue OAuth2::Error => e + retries, delay = handle_error(retries, delay, e.message) + + next + end + end + + def handle_error(retries, delay, error) + retries += 1 + + raise RateLimitError, "Maximum number of retries (#{MAX_RETRIES}) exceeded. #{error}" if retries >= MAX_RETRIES + + delay *= EXPONENTIAL_BASE * (1 + Random.rand) + + logger.info(message: "Retrying in #{delay} seconds due to #{error}") + sleep delay + + [retries, delay] + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index ab8f5ba17fe..3505e34e0ba 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -76,6 +76,7 @@ module Bitbucket merge_commit_sha: merge_commit_sha, target_branch_name: target_branch_name, target_branch_sha: target_branch_sha, + source_and_target_project_different: source_and_target_project_different, reviewers: reviewers } end @@ -89,6 +90,18 @@ module Bitbucket def target_branch raw['destination'] end + + def source_repo_uuid + source_branch&.dig('repository', 'uuid') + end + + def target_repo_uuid + target_branch&.dig('repository', 'uuid') + end + + def source_and_target_project_different + source_repo_uuid != target_repo_uuid + end end end end diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb index 8e84afe51d7..432928b0591 100644 --- a/lib/bitbucket_server/client.rb +++ b/lib/bitbucket_server/client.rb @@ -29,6 +29,11 @@ module BitbucketServer get_collection(path, :repo, page_offset: page_offset, limit: limit) end + def users(project_key, page_offset: 0, limit: nil) + path = "/projects/#{project_key}/permissions/users" + get_collection(path, :user, page_offset: page_offset, limit: limit) + end + def create_branch(project_key, repo, branch_name, sha) payload = { name: branch_name, diff --git a/lib/bitbucket_server/connection.rb b/lib/bitbucket_server/connection.rb index 845acf034a5..668a4e79da0 100644 --- a/lib/bitbucket_server/connection.rb +++ b/lib/bitbucket_server/connection.rb @@ -3,6 +3,7 @@ module BitbucketServer class Connection include ActionView::Helpers::SanitizeHelper + include BitbucketServer::RetryWithDelay DEFAULT_API_VERSION = '1.0' SEPARATOR = '/' @@ -31,10 +32,13 @@ module BitbucketServer end def get(path, extra_query = {}) - response = Gitlab::HTTP.get(build_url(path), - basic_auth: auth, - headers: accept_headers, - query: extra_query) + response = if Feature.enabled?(:bitbucket_server_importer_exponential_backoff) + retry_with_delay do + Gitlab::HTTP.get(build_url(path), basic_auth: auth, headers: accept_headers, query: extra_query) + end + else + Gitlab::HTTP.get(build_url(path), basic_auth: auth, headers: accept_headers, query: extra_query) + end check_errors!(response) @@ -44,10 +48,13 @@ module BitbucketServer end def post(path, body) - response = Gitlab::HTTP.post(build_url(path), - basic_auth: auth, - headers: post_headers, - body: body) + response = if Feature.enabled?(:bitbucket_server_importer_exponential_backoff) + retry_with_delay do + Gitlab::HTTP.post(build_url(path), basic_auth: auth, headers: post_headers, body: body) + end + else + Gitlab::HTTP.post(build_url(path), basic_auth: auth, headers: post_headers, body: body) + end check_errors!(response) @@ -63,10 +70,13 @@ module BitbucketServer def delete(resource, path, body) url = delete_url(resource, path) - response = Gitlab::HTTP.delete(url, - basic_auth: auth, - headers: post_headers, - body: body) + response = if Feature.enabled?(:bitbucket_server_importer_exponential_backoff) + retry_with_delay do + Gitlab::HTTP.delete(url, basic_auth: auth, headers: post_headers, body: body) + end + else + Gitlab::HTTP.delete(url, basic_auth: auth, headers: post_headers, body: body) + end check_errors!(response) @@ -121,5 +131,9 @@ module BitbucketServer build_url(path) end end + + def logger + Gitlab::BitbucketServerImport::Logger + end end end diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb index 8a494379864..cb2c0a84e2d 100644 --- a/lib/bitbucket_server/paginator.rb +++ b/lib/bitbucket_server/paginator.rb @@ -2,6 +2,7 @@ module BitbucketServer class Paginator + # Should be kept in-sync with `BITBUCKET_SERVER_PAGE_LENGTH` in app/assets/javascripts/import_entities/constants.js PAGE_LENGTH = 25 attr_reader :page_offset diff --git a/lib/bitbucket_server/representation/activity.rb b/lib/bitbucket_server/representation/activity.rb index 08bf30a5d1e..0be7425f2cb 100644 --- a/lib/bitbucket_server/representation/activity.rb +++ b/lib/bitbucket_server/representation/activity.rb @@ -3,6 +3,10 @@ module BitbucketServer module Representation class Activity < Representation::Base + def id + raw['id'] + end + def comment? action == 'COMMENTED' end @@ -45,6 +49,18 @@ module BitbucketServer commit['id'] end + def approved_event? + action == 'APPROVED' + end + + def approver_username + raw.dig('user', 'slug') + end + + def approver_email + raw.dig('user', 'emailAddress') + end + def created_at self.class.convert_timestamp(created_date) end diff --git a/lib/bitbucket_server/representation/user.rb b/lib/bitbucket_server/representation/user.rb new file mode 100644 index 00000000000..433baec1c42 --- /dev/null +++ b/lib/bitbucket_server/representation/user.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module BitbucketServer + module Representation + class User < Representation::Base + def email + user['emailAddress'] + end + + def username + user['slug'] + end + + private + + def user + raw['user'] + end + end + end +end diff --git a/lib/bitbucket_server/retry_with_delay.rb b/lib/bitbucket_server/retry_with_delay.rb new file mode 100644 index 00000000000..8a8c0e2dc14 --- /dev/null +++ b/lib/bitbucket_server/retry_with_delay.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module BitbucketServer + module RetryWithDelay + extend ActiveSupport::Concern + + MAXIMUM_DELAY = 20 + + def retry_with_delay(&block) + run_retry_with_delay(&block) + end + + private + + def run_retry_with_delay + response = yield + + if response.code == 429 && response.headers.has_key?('retry-after') + retry_after = response.headers['retry-after'].to_i + + if retry_after <= MAXIMUM_DELAY + logger.info(message: "Retrying in #{retry_after} seconds due to 429 Too Many Requests") + sleep retry_after + + response = yield + end + end + + response + end + end +end diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 6c2aa41c346..e22e37d66af 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -81,7 +81,7 @@ module BulkImports return true if response['scopes']&.include?('api') - raise ::BulkImports::Error.scope_validation_failure + raise ::BulkImports::Error.scope_or_url_validation_failure end def validate_instance_version! @@ -110,7 +110,7 @@ module BulkImports rescue BulkImports::NetworkError => e case e&.response&.code when 401, 403 - raise ::BulkImports::Error.scope_validation_failure + raise ::BulkImports::Error.scope_or_url_validation_failure when 404 raise ::BulkImports::Error.invalid_url else diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 723359aa438..2ab7a0e12d7 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -8,6 +8,10 @@ module BulkImports false end + def self.abort_on_failure? + false + end + def initialize(context) @context = context @entity = @context.entity @@ -24,13 +28,8 @@ module BulkImports end logger.info( - bulk_import_id: entity.bulk_import_id, - bulk_import_entity_id: entity.id, - bulk_import_entity_type: entity.source_type, - source_full_path: entity.source_full_path, pipeline_class: self.class.name, - message: "Entity #{entity.status_name}", - source_version: entity.bulk_import.source_version_info.to_s + message: "Entity #{entity.status_name}" ) ::BulkImports::FinishProjectImportWorker.perform_async(entity.project_id) if entity.project? @@ -41,7 +40,7 @@ module BulkImports attr_reader :context, :entity, :trackers def logger - @logger ||= Logger.build + @logger ||= Logger.build.with_entity(entity) end def all_other_trackers_failed? diff --git a/lib/bulk_imports/error.rb b/lib/bulk_imports/error.rb index c40b4bc7f34..57a55abafa6 100644 --- a/lib/bulk_imports/error.rb +++ b/lib/bulk_imports/error.rb @@ -3,35 +3,37 @@ module BulkImports class Error < StandardError def self.unsupported_gitlab_version - self.new("Unsupported GitLab version. Minimum supported version is #{BulkImport::MIN_MAJOR_VERSION}.") + self.new(format(s_("BulkImport|Unsupported GitLab version. Minimum supported version is '%{version}'."), + version: BulkImport::MIN_MAJOR_VERSION)) end - def self.scope_validation_failure - self.new("Personal access token does not have the required " \ - "'api' scope or is no longer valid.") + def self.scope_or_url_validation_failure + self.new(s_("BulkImport|Check that the source instance base URL and the personal access " \ + "token meet the necessary requirements.")) end def self.invalid_url - self.new("Invalid source URL. Enter only the base URL of the source GitLab instance.") + self.new(s_("BulkImport|Invalid source URL. Enter only the base URL of the source GitLab instance.")) end def self.destination_namespace_validation_failure(destination_namespace) - self.new("Import failed. Destination '#{destination_namespace}' is invalid, or you don't have permission.") + self.new(format(s_("BulkImport|Import failed. Destination '%{destination}' is invalid, " \ + "or you don't have permission."), destination: destination_namespace)) end def self.destination_slug_validation_failure - self.new("Import failed. Destination URL " \ - "#{Gitlab::Regex.oci_repository_path_regex_message}") + self.new(format(s_("BulkImport|Import failed. Destination URL %{url}"), + url: Gitlab::Regex.oci_repository_path_regex_message)) end def self.destination_full_path_validation_failure(full_path) - self.new("Import failed. '#{full_path}' already exists. Change the destination and try again.") + self.new(format(s_("BulkImport|Import failed. '%{path}' already exists. Change the destination and try again."), + path: full_path)) end def self.setting_not_enabled - self.new("Group import disabled on source or destination instance. " \ - "Ask an administrator to enable it on both instances and try again." - ) + self.new(s_("BulkImport|Group import disabled on source or destination instance. " \ + "Ask an administrator to enable it on both instances and try again.")) end end end diff --git a/lib/bulk_imports/logger.rb b/lib/bulk_imports/logger.rb index be15c050770..3b62d0ffdf3 100644 --- a/lib/bulk_imports/logger.rb +++ b/lib/bulk_imports/logger.rb @@ -4,8 +4,55 @@ module BulkImports class Logger < ::Gitlab::Import::Logger IMPORTER_NAME = 'gitlab_migration' + # Extract key information from a provided entity and include it in log + # entries created from this logger instance. + # @param entity [BulkImports::Entity] + def with_entity(entity) + @entity = entity + self + end + + # Extract key information from a provided tracker and its entity and include + # it in log entries created from this logger instance. + # @param tracker [BulkImports::Tracker] + def with_tracker(tracker) + with_entity(tracker.entity) + @tracker = tracker + self + end + + def entity_attributes + return {} unless entity + + { + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_id: entity.id, + bulk_import_entity_type: entity.source_type, + source_full_path: entity.source_full_path, + source_version: entity.source_version.to_s + } + end + + def tracker_attributes + return {} unless tracker + + { + tracker_id: tracker.id, + pipeline_class: tracker.pipeline_name, + tracker_state: tracker.human_status_name + } + end + def default_attributes - super.merge(importer: IMPORTER_NAME) + super.merge( + { importer: IMPORTER_NAME }, + entity_attributes, + tracker_attributes + ) end + + private + + attr_reader :entity, :tracker end end diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb index b21889adcb3..b49733962f4 100644 --- a/lib/bulk_imports/network_error.rb +++ b/lib/bulk_imports/network_error.rb @@ -18,7 +18,7 @@ module BulkImports Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH ].freeze - RETRIABLE_HTTP_CODES = [429].freeze + RETRIABLE_HTTP_CODES = [429, 500, 502, 503, 504].freeze DEFAULT_RETRY_DELAY_SECONDS = 30 @@ -57,7 +57,7 @@ module BulkImports end def retriable_http_code? - RETRIABLE_HTTP_CODES.include?(response&.code) + RETRIABLE_HTTP_CODES.include?(response&.code.to_i) end def increment(object) diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index e2a14c35e79..7b5e1e68459 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -16,6 +16,8 @@ module BulkImports if extracted_data extracted_data.each_with_index do |entry, index| + refresh_entity_and_import if index % 1000 == 0 + raw_entry = entry.dup next if already_processed?(raw_entry, index) @@ -164,12 +166,8 @@ module BulkImports def log_params(extra) defaults = { bulk_import_id: context.bulk_import_id, - bulk_import_entity_id: context.entity.id, - bulk_import_entity_type: context.entity.source_type, - source_full_path: context.entity.source_full_path, pipeline_class: pipeline, - context_extra: context.extra, - source_version: context.entity.bulk_import.source_version_info.to_s + context_extra: context.extra } defaults @@ -178,7 +176,7 @@ module BulkImports end def logger - @logger ||= Logger.build + @logger ||= Logger.build.with_entity(context.entity) end def log_exception(exception, payload) @@ -193,6 +191,11 @@ module BulkImports payload.stringify_keys.merge(context) end + + def refresh_entity_and_import + context.entity.touch + context.bulk_import.touch + end end end end diff --git a/lib/bulk_imports/projects/pipelines/references_pipeline.rb b/lib/bulk_imports/projects/pipelines/references_pipeline.rb index e2032569ab5..216a8d84b0c 100644 --- a/lib/bulk_imports/projects/pipelines/references_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/references_pipeline.rb @@ -7,123 +7,49 @@ module BulkImports include Pipeline BATCH_SIZE = 100 + DELAY = 1.second - def extract(_context) - data = Enumerator.new do |enum| - add_matching_objects(portable.issues, enum) - add_matching_objects(portable.merge_requests, enum) - add_notes(portable.issues, enum) - add_notes(portable.merge_requests, enum) - end - - BulkImports::Pipeline::ExtractedData.new(data: data) - end + def extract(context) + @tracker_id = context.tracker.id + @counter = 0 - def transform(_context, object) - body = object_body(object).dup + enqueue_ref_workers_for_issues_and_issue_notes + enqueue_ref_workers_for_merge_requests_and_merge_request_notes - body.gsub!(username_regex(mapped_usernames), mapped_usernames) - - matching_urls(object).each do |old_url, new_url| - body.gsub!(old_url, new_url) if body.include?(old_url) - end - - object.assign_attributes(body_field(object) => body) - - object + nil end - def load(_context, object) - object.save! if object_body_changed?(object) - end + attr_reader :tracker_id private - def mapped_usernames - @mapped_usernames ||= ::BulkImports::UsersMapper.new(context: context) - .map_usernames.transform_keys { |key| "@#{key}" } - .transform_values { |value| "@#{value}" } - end - - def username_regex(mapped_usernames) - @username_regex ||= Regexp.new(mapped_usernames.keys.sort_by(&:length) - .reverse.map { |x| Regexp.escape(x) }.join('|')) - end + def enqueue_ref_workers_for_issues_and_issue_notes + portable.issues.select(:id).each_batch(of: BATCH_SIZE, column: :iid) do |batch| + BulkImports::TransformReferencesWorker.perform_in(delay, batch.map(&:id), Issue.to_s, tracker_id) - def add_matching_objects(collection, enum) - collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |object| - enum << object if object_has_reference?(object) || object_has_username?(object) - end - end - end - - def add_notes(collection, enum) - collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |object| - object.notes.each_batch(of: BATCH_SIZE) do |notes_batch| - notes_batch.each do |note| - note.refresh_markdown_cache! - enum << note if object_has_reference?(note) || object_has_username?(note) - end + batch.each do |issue| + issue.notes.select(:id).each_batch(of: BATCH_SIZE) do |notes_batch| + BulkImports::TransformReferencesWorker.perform_in(delay, notes_batch.map(&:id), Note.to_s, tracker_id) end end end end - def object_has_reference?(object) - object_body(object)&.include?(source_full_path) - end - - def object_has_username?(object) - return false unless object_body(object) - - mapped_usernames.keys.any? { |old_username| object_body(object).include?(old_username) } - end - - def object_body(object) - call_object_method(object) - end - - def object_body_changed?(object) - call_object_method(object, suffix: '_changed?') - end - - def call_object_method(object, suffix: nil) - method = body_field(object) - method = "#{method}#{suffix}" if suffix.present? - - object.public_send(method) # rubocop:disable GitlabSecurity/PublicSend - end - - def body_field(object) - object.is_a?(Note) ? 'note' : 'description' - end - - def matching_urls(object) - URI.extract(object_body(object), %w[http https]).each_with_object([]) do |url, array| - parsed_url = URI.parse(url) - - next unless source_host == parsed_url.host - next unless parsed_url.path&.start_with?("/#{source_full_path}") + def enqueue_ref_workers_for_merge_requests_and_merge_request_notes + portable.merge_requests.select(:id).each_batch(of: BATCH_SIZE, column: :iid) do |batch| + BulkImports::TransformReferencesWorker.perform_in(delay, batch.map(&:id), MergeRequest.to_s, tracker_id) - array << [url, new_url(parsed_url)] + batch.each do |merge_request| + merge_request.notes.select(:id).each_batch(of: BATCH_SIZE) do |notes_batch| + BulkImports::TransformReferencesWorker.perform_in(delay, notes_batch.map(&:id), Note.to_s, tracker_id) + end + end end end - def new_url(parsed_old_url) - parsed_old_url.host = ::Gitlab.config.gitlab.host - parsed_old_url.port = ::Gitlab.config.gitlab.port - parsed_old_url.scheme = ::Gitlab.config.gitlab.https ? 'https' : 'http' - parsed_old_url.to_s.gsub!(source_full_path, portable.full_path) - end - - def source_host - @source_host ||= URI.parse(context.configuration.url).host - end - - def source_full_path - context.entity.source_full_path + def delay + @counter += 1 + @counter * DELAY end end end diff --git a/lib/click_house/connection.rb b/lib/click_house/connection.rb new file mode 100644 index 00000000000..79551326d2d --- /dev/null +++ b/lib/click_house/connection.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module ClickHouse + class Connection + def initialize(database, configuration = ClickHouse::Client.configuration) + @database = database + @configuration = configuration + end + + def select(query) + ClickHouse::Client.select(query, database, configuration) + end + + def execute(query) + ClickHouse::Client.execute(query, database, configuration) + end + + def table_exists?(table_name) + raw_query = <<~SQL.squish + SELECT 1 FROM system.tables + WHERE name = {table_name: String} AND database = {database_name: String} + SQL + + database_name = configuration.databases[database]&.database + placeholders = { table_name: table_name, database_name: database_name } + + query = ClickHouse::Client::Query.new(raw_query: raw_query, placeholders: placeholders) + + select(query).any? + end + + private + + attr_reader :database, :configuration + end +end diff --git a/lib/click_house/iterator.rb b/lib/click_house/iterator.rb new file mode 100644 index 00000000000..4bfbc624dc7 --- /dev/null +++ b/lib/click_house/iterator.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module ClickHouse + # This class implements a batch iterator which can be used for ClickHouse database tables. + # The batching logic uses fixed id ranges because that's the only way to efficiently batch + # over the data. This is similar to the implementation of the Gitlab::Database::BatchCount + # utility class. + # + # Usage: + # + # connection = ClickHouse::Connection.new(:main) + # builder = ClickHouse::QueryBuilder.new('event_authors') + # iterator = ClickHouse::Iterator.new(query_builder: builder, connection: connection) + # iterator.each_batch(column: :author_id, of: 100000) do |scope| + # puts scope.to_sql + # puts ClickHouse::Client.select(scope.to_sql, :main) + # end + # + # If your database table structure is optimized for a specific filter, you could scan smaller + # part of the table by adding more condition to the query builder. Example: + # + # builder = ClickHouse::QueryBuilder.new('event_authors').where(type: 'some_type') + class Iterator + # rubocop: disable CodeReuse/ActiveRecord -- this is a ClickHouse query builder class usin Arel + def initialize(query_builder:, connection:) + @query_builder = query_builder + @connection = connection + end + + def each_batch(column: :id, of: 10_000) + min_max_query = query_builder.select( + table[column].minimum.as('min'), + table[column].maximum.as('max') + ) + + row = connection.select(min_max_query.to_sql).first + return if row.nil? + + min_value = row['min'] + max_value = row['max'] + return if max_value == 0 + + loop do + break if min_value > max_value + + yield query_builder + .where(table[column].gteq(min_value)) + .where(table[column].lt(min_value + of)) + + min_value += of + end + end + + private + + delegate :table, to: :query_builder + + attr_reader :query_builder, :connection + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/lib/click_house/migration.rb b/lib/click_house/migration.rb index 410a7ec86bc..0c8110f83d5 100644 --- a/lib/click_house/migration.rb +++ b/lib/click_house/migration.rb @@ -5,41 +5,27 @@ module ClickHouse cattr_accessor :verbose, :client_configuration attr_accessor :name, :version - class << self - attr_accessor :delegate - end - - def initialize(name = self.class.name, version = nil) + def initialize(connection, name = self.class.name, version = nil) + @connection = connection @name = name @version = version end - self.client_configuration = ClickHouse::Client.configuration self.verbose = true - # instantiate the delegate object after initialize is defined - self.delegate = new MIGRATION_FILENAME_REGEXP = /\A([0-9]+)_([_a-z0-9]*)\.?([_a-z0-9]*)?\.rb\z/ - def database - self.class.constants.include?(:SCHEMA) ? self.class.const_get(:SCHEMA, false) : :main - end - def execute(query) - ClickHouse::Client.execute(query, database, self.class.client_configuration) + connection.execute(query) end def up - self.class.delegate = self - return unless self.class.respond_to?(:up) self.class.up end def down - self.class.delegate = self - return unless self.class.respond_to?(:down) self.class.down @@ -68,6 +54,8 @@ module ClickHouse private + attr_reader :connection + def exec_migration(direction) # noinspection RubyCaseWithoutElseBlockInspection case direction diff --git a/lib/click_house/migration_support/errors.rb b/lib/click_house/migration_support/errors.rb new file mode 100644 index 00000000000..f8c6e5a94e0 --- /dev/null +++ b/lib/click_house/migration_support/errors.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module ClickHouse + module MigrationSupport + module Errors + class Base < StandardError + def initialize(message = nil) + message = "\n\n#{message}\n\n" if message + super + end + end + + class IllegalMigrationNameError < Base + def initialize(name = nil) + if name + super("Illegal name for migration file: #{name}\n\t(only lower case letters, numbers, and '_' allowed).") + else + super('Illegal name for migration.') + end + end + end + + IrreversibleMigration = Class.new(Base) + LockError = Class.new(Base) + + class DuplicateMigrationVersionError < Base + def initialize(version = nil) + if version + super("Multiple migrations have the version number #{version}.") + else + super('Duplicate migration version error.') + end + end + end + + class DuplicateMigrationNameError < Base + def initialize(name = nil) + if name + super("Multiple migrations have the name #{name}.") + else + super('Duplicate migration name.') + end + end + end + + class UnknownMigrationVersionError < Base + def initialize(version = nil) + if version + super("No migration with version number #{version}.") + else + super('Unknown migration version.') + end + end + end + end + end +end diff --git a/lib/click_house/migration_support/exclusive_lock.rb b/lib/click_house/migration_support/exclusive_lock.rb new file mode 100644 index 00000000000..d75a75a1920 --- /dev/null +++ b/lib/click_house/migration_support/exclusive_lock.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module ClickHouse + module MigrationSupport + class ExclusiveLock + MIGRATION_LEASE_KEY = 'click_house:migrations' + MIGRATION_RETRY_DELAY = ->(num) { 0.2.seconds * (num**2) } + MIGRATION_LOCK_DURATION = 1.hour + + ACTIVE_WORKERS_REDIS_KEY = 'click_house:workers:active_workers' + DEFAULT_CLICKHOUSE_WORKER_TTL = 30.minutes + WORKERS_WAIT_SLEEP = 5.seconds + + class << self + include ::Gitlab::ExclusiveLeaseHelpers + + def register_running_worker(worker_class, worker_id) + ttl = worker_class.click_house_worker_attrs[:migration_lock_ttl].from_now.utc + + Gitlab::Redis::SharedState.with do |redis| + redis.zadd(ACTIVE_WORKERS_REDIS_KEY, ttl.to_i, worker_id, gt: true) + + yield + ensure + redis.zrem(ACTIVE_WORKERS_REDIS_KEY, worker_id) + end + end + + def execute_migration + in_lock(MIGRATION_LEASE_KEY, ttl: MIGRATION_LOCK_DURATION, retries: 5, sleep_sec: MIGRATION_RETRY_DELAY) do + wait_until_workers_inactive(DEFAULT_CLICKHOUSE_WORKER_TTL.from_now) + + yield + end + rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError => e + raise ClickHouse::MigrationSupport::Errors::LockError, e.message + end + + def pause_workers? + Gitlab::ExclusiveLease.new(MIGRATION_LEASE_KEY, timeout: 0).exists? + end + + def active_sidekiq_workers? + Gitlab::Redis::SharedState.with do |redis| + min = Time.now.utc.to_i + + # expire keys in the past + redis.zremrangebyscore(ACTIVE_WORKERS_REDIS_KEY, 0, "(#{min}") + # Return if any workers are registered with a future expiry date + redis.zrange(ACTIVE_WORKERS_REDIS_KEY, min, '+inf', by_score: true, limit: [0, 1]).any? + end + end + + def wait_until_workers_inactive(worker_wait_ttl) + # Wait until the collection in ClickHouseWorker::CLICKHOUSE_ACTIVE_WORKERS_KEY is empty, + # before continuing migration. + workers_active = true + + loop do + return if Feature.disabled?(:wait_for_clickhouse_workers_during_migration) + + workers_active = active_sidekiq_workers? + break unless workers_active + break if Time.current >= worker_wait_ttl + + sleep WORKERS_WAIT_SLEEP.to_i + end + + return unless workers_active + + raise ClickHouse::MigrationSupport::Errors::LockError, 'Timed out waiting for active workers' + end + end + + private_class_method :wait_until_workers_inactive + end + end +end diff --git a/lib/click_house/migration_support/migration_context.rb b/lib/click_house/migration_support/migration_context.rb index 6e4dd2a97c2..8c264b3ba07 100644 --- a/lib/click_house/migration_support/migration_context.rb +++ b/lib/click_house/migration_support/migration_context.rb @@ -10,33 +10,35 @@ module ClickHouse # sufficient. Multiple database applications need a +SchemaMigration+ # per primary database. class MigrationContext - attr_reader :migrations_paths, :schema_migration - - def initialize(migrations_paths, schema_migration) + def initialize(connection, migrations_paths, schema_migration) + @connection = connection @migrations_paths = migrations_paths @schema_migration = schema_migration end - def up(target_version = nil, &block) + def up(target_version = nil, step = nil, &block) selected_migrations = block ? migrations.select(&block) : migrations - migrate(:up, selected_migrations, target_version) + migrate(:up, selected_migrations, target_version, step) end - def down(target_version = nil, &block) + def down(target_version = nil, step = 1, &block) selected_migrations = block ? migrations.select(&block) : migrations - migrate(:down, selected_migrations, target_version) + migrate(:down, selected_migrations, target_version, step) end private - def migrate(direction, selected_migrations, target_version = nil) + attr_reader :migrations_paths, :schema_migration, :connection + + def migrate(direction, selected_migrations, target_version = nil, step = nil) ClickHouse::MigrationSupport::Migrator.new( direction, selected_migrations, schema_migration, - target_version + target_version, + step ).migrate end @@ -44,12 +46,12 @@ module ClickHouse migrations = migration_files.map do |file| version, name, scope = parse_migration_filename(file) - raise ClickHouse::MigrationSupport::IllegalMigrationNameError, file unless version + raise ClickHouse::MigrationSupport::Errors::IllegalMigrationNameError, file unless version version = version.to_i name = name.camelize - MigrationProxy.new(name, version, file, scope) + MigrationProxy.new(connection, name, version, file, scope) end migrations.sort_by(&:version) @@ -67,9 +69,16 @@ module ClickHouse # MigrationProxy is used to defer loading of the actual migration classes # until they are needed - MigrationProxy = Struct.new(:name, :version, :filename, :scope) do - def initialize(name, version, filename, scope) - super + class MigrationProxy + attr_reader :name, :version, :filename, :scope + + def initialize(connection, name, version, filename, scope) + @connection = connection + @name = name + @version = version + @filename = filename + @scope = scope + @migration = nil end @@ -87,7 +96,7 @@ module ClickHouse def load_migration require(File.expand_path(filename)) - name.constantize.new(name, version) + name.constantize.new(@connection, name, version) end end end diff --git a/lib/click_house/migration_support/migration_error.rb b/lib/click_house/migration_support/migration_error.rb deleted file mode 100644 index 0638d487e37..00000000000 --- a/lib/click_house/migration_support/migration_error.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module ClickHouse - module MigrationSupport - class MigrationError < StandardError - def initialize(message = nil) - message = "\n\n#{message}\n\n" if message - super - end - end - - class IllegalMigrationNameError < MigrationError - def initialize(name = nil) - if name - super("Illegal name for migration file: #{name}\n\t(only lower case letters, numbers, and '_' allowed).") - else - super('Illegal name for migration.') - end - end - end - - IrreversibleMigration = Class.new(MigrationError) - - class DuplicateMigrationVersionError < MigrationError - def initialize(version = nil) - if version - super("Multiple migrations have the version number #{version}.") - else - super('Duplicate migration version error.') - end - end - end - - class DuplicateMigrationNameError < MigrationError - def initialize(name = nil) - if name - super("Multiple migrations have the name #{name}.") - else - super('Duplicate migration name.') - end - end - end - - class UnknownMigrationVersionError < MigrationError - def initialize(version = nil) - if version - super("No migration with version number #{version}.") - else - super('Unknown migration version.') - end - end - end - end -end diff --git a/lib/click_house/migration_support/migrator.rb b/lib/click_house/migration_support/migrator.rb index 5c67b3a5ff1..d7eb9d705f4 100644 --- a/lib/click_house/migration_support/migrator.rb +++ b/lib/click_house/migration_support/migrator.rb @@ -3,31 +3,28 @@ module ClickHouse module MigrationSupport class Migrator - class << self - attr_accessor :migrations_paths - end - attr_accessor :logger - self.migrations_paths = ["db/click_house/migrate"] + def self.migrations_paths(database_name) + File.join("db/click_house/migrate", database_name.to_s) + end - def initialize(direction, migrations, schema_migration, target_version = nil, logger = Gitlab::AppLogger) + def initialize( + direction, migrations, schema_migration, target_version = nil, step = nil, + logger = Gitlab::AppLogger + ) @direction = direction @target_version = target_version - @migrated_versions = {} + @step = step @migrations = migrations @schema_migration = schema_migration @logger = logger validate(@migrations) - - migrations.map(&:database).uniq.each do |database| - @schema_migration.create_table(database) - end end def current_version - @migrated_versions.values.flatten.max || 0 + migrated.max || 0 end def current_migration @@ -35,64 +32,50 @@ module ClickHouse end alias_method :current, :current_migration - def run - run_without_lock - end - def migrate - migrate_without_lock + ClickHouse::MigrationSupport::ExclusiveLock.execute_migration do + migrate_without_lock + end end def runnable runnable = migrations[start..finish] if up? - runnable.reject { |m| ran?(m) } + runnable = runnable.reject { |m| ran?(m) } else # skip the last migration if we're headed down, but not ALL the way down runnable.pop if target - runnable.find_all { |m| ran?(m) } + runnable = runnable.find_all { |m| ran?(m) } end + + runnable = runnable.take(@step) if @step && !@target_version + runnable end def migrations down? ? @migrations.reverse : @migrations.sort_by(&:version) end - def pending_migrations(database) - already_migrated = migrated(database) - - migrations.reject { |m| already_migrated.include?(m.version) } - end - - def migrated(database) - @migrated_versions[database] || load_migrated(database) + def migrated + @migrated_versions || load_migrated end - def load_migrated(database) - @migrated_versions[database] = Set.new(@schema_migration.all_versions(database).map(&:to_i)) + def load_migrated + @migrated_versions = Set.new(@schema_migration.all_versions.map(&:to_i)) end private - # Used for running a specific migration. - def run_without_lock - migration = migrations.detect { |m| m.version == @target_version } - - raise ClickHouse::MigrationSupport::UnknownMigrationVersionError, @target_version if migration.nil? - - execute_migration(migration) - end - # Used for running multiple migrations up to or down to a certain value. def migrate_without_lock - raise ClickHouse::MigrationSupport::UnknownMigrationVersionError, @target_version if invalid_target? + raise ClickHouse::MigrationSupport::Errors::UnknownMigrationVersionError, @target_version if invalid_target? runnable.each(&method(:execute_migration)) # rubocop: disable Performance/MethodObjectAsBlock -- Execute through proxy end def ran?(migration) - migrated(migration.database).include?(migration.version.to_i) + migrated.include?(migration.version.to_i) end # Return true if a valid version is not provided. @@ -104,15 +87,13 @@ module ClickHouse end def execute_migration(migration) - database = migration.database - - return if down? && migrated(database).exclude?(migration.version.to_i) - return if up? && migrated(database).include?(migration.version.to_i) + return if down? && migrated.exclude?(migration.version.to_i) + return if up? && migrated.include?(migration.version.to_i) logger.info "Migrating to #{migration.name} (#{migration.version})" if logger migration.migrate(@direction) - record_version_state_after_migrating(database, migration.version) + record_version_state_after_migrating(migration.version) rescue StandardError => e msg = "An error has occurred, all later migrations canceled:\n\n#{e}" raise StandardError, msg, e.backtrace @@ -132,19 +113,19 @@ module ClickHouse def validate(migrations) name, = migrations.group_by(&:name).find { |_, v| v.length > 1 } - raise ClickHouse::MigrationSupport::DuplicateMigrationNameError, name if name + raise ClickHouse::MigrationSupport::Errors::DuplicateMigrationNameError, name if name version, = migrations.group_by(&:version).find { |_, v| v.length > 1 } - raise ClickHouse::MigrationSupport::DuplicateMigrationVersionError, version if version + raise ClickHouse::MigrationSupport::Errors::DuplicateMigrationVersionError, version if version end - def record_version_state_after_migrating(database, version) + def record_version_state_after_migrating(version) if down? - migrated(database).delete(version) - @schema_migration.create!(database, version: version.to_s, active: 0) + migrated.delete(version) + @schema_migration.create!(version: version.to_s, active: 0) else - migrated(database) << version - @schema_migration.create!(database, version: version.to_s) + migrated << version + @schema_migration.create!(version: version.to_s) end end diff --git a/lib/click_house/migration_support/schema_migration.rb b/lib/click_house/migration_support/schema_migration.rb index e82debbad0d..334389ad444 100644 --- a/lib/click_house/migration_support/schema_migration.rb +++ b/lib/click_house/migration_support/schema_migration.rb @@ -3,69 +3,49 @@ module ClickHouse module MigrationSupport class SchemaMigration - class_attribute :table_name_prefix, instance_writer: false, default: '' - class_attribute :table_name_suffix, instance_writer: false, default: '' - class_attribute :schema_migrations_table_name, instance_accessor: false, default: 'schema_migrations' + def initialize(connection, table_name: 'schema_migrations') + @connection = connection + @table_name = table_name + end - class << self - TABLE_EXISTS_QUERY = <<~SQL.squish - SELECT 1 FROM system.tables - WHERE name = {table_name: String} AND database = {database_name: String} + def ensure_table + return if connection.table_exists?(table_name) + + query = <<~SQL + CREATE TABLE #{table_name} ( + version LowCardinality(String), + active UInt8 NOT NULL DEFAULT 1, + applied_at DateTime64(6, 'UTC') NOT NULL DEFAULT now64() + ) + ENGINE = ReplacingMergeTree(applied_at) + PRIMARY KEY(version) + ORDER BY (version) SQL - def primary_key - 'version' - end - - def table_name - "#{table_name_prefix}#{schema_migrations_table_name}#{table_name_suffix}" - end - - def table_exists?(database, configuration = ClickHouse::Migration.client_configuration) - database_name = configuration.databases[database]&.database - return false unless database_name - - placeholders = { table_name: table_name, database_name: database_name } - query = ClickHouse::Client::Query.new(raw_query: TABLE_EXISTS_QUERY, placeholders: placeholders) - - ClickHouse::Client.select(query, database, configuration).any? - end - - def create_table(database, configuration = ClickHouse::Migration.client_configuration) - return if table_exists?(database, configuration) + connection.execute(query) + end - query = <<~SQL - CREATE TABLE #{table_name} ( - version LowCardinality(String), - active UInt8 NOT NULL DEFAULT 1, - applied_at DateTime64(6, 'UTC') NOT NULL DEFAULT now64() - ) - ENGINE = ReplacingMergeTree(applied_at) - PRIMARY KEY(version) - ORDER BY (version) - SQL + def all_versions + query = <<~SQL + SELECT version FROM #{table_name} FINAL + WHERE active = 1 + ORDER BY (version) + SQL - ClickHouse::Client.execute(query, database, configuration) - end + connection.select(query).pluck('version') + end - def all_versions(database) - query = <<~SQL - SELECT version FROM #{table_name} FINAL - WHERE active = 1 - ORDER BY (version) - SQL + def create!(**args) + insert_sql = <<~SQL + INSERT INTO #{table_name} (#{args.keys.join(',')}) VALUES (#{args.values.join(',')}) + SQL - ClickHouse::Client.select(query, database, ClickHouse::Migration.client_configuration).pluck('version') - end + connection.execute(insert_sql) + end - def create!(database, **args) - insert_sql = <<~SQL - INSERT INTO #{table_name} (#{args.keys.join(',')}) VALUES (#{args.values.join(',')}) - SQL + private - ClickHouse::Client.execute(insert_sql, database, ClickHouse::Migration.client_configuration) - end - end + attr_reader :connection, :table_name end end end diff --git a/lib/click_house/migration_support/sidekiq_middleware.rb b/lib/click_house/migration_support/sidekiq_middleware.rb new file mode 100644 index 00000000000..e4e6c453e8d --- /dev/null +++ b/lib/click_house/migration_support/sidekiq_middleware.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module ClickHouse + module MigrationSupport + class SidekiqMiddleware + def call(worker, job, queue) + return yield unless register_worker?(worker.class) + + ::ClickHouse::MigrationSupport::ExclusiveLock.register_running_worker(worker.class, worker_id(job, queue)) do + yield + end + end + + private + + def worker_id(job, queue) + [queue, job['jid']].join(':') + end + + def register_worker?(worker_class) + worker_class.respond_to?(:click_house_migration_lock) && worker_class.register_click_house_worker? + end + end + end +end diff --git a/lib/click_house/query_builder.rb b/lib/click_house/query_builder.rb index dc139663e7c..c7f73f7ccef 100644 --- a/lib/click_house/query_builder.rb +++ b/lib/click_house/query_builder.rb @@ -61,9 +61,21 @@ module ClickHouse end end - new_projections = existing_fields + fields.map(&:to_s) + new_projections = (existing_fields + fields).map do |field| + if field.is_a?(Symbol) + field.to_s + else + field + end + end - new_instance.manager.projections = new_projections.uniq.map { |field| new_instance.table[field] } + new_instance.manager.projections = new_projections.uniq.map do |field| + if field.is_a?(Arel::Expressions) + field + else + new_instance.table[field.to_s] + end + end new_instance end diff --git a/lib/extracts_ref/ref_extractor.rb b/lib/extracts_ref/ref_extractor.rb index ac9b0ebb7af..e716c64e63a 100644 --- a/lib/extracts_ref/ref_extractor.rb +++ b/lib/extracts_ref/ref_extractor.rb @@ -15,7 +15,7 @@ module ExtractsRef class << self def ref_type(type) - return unless REF_TYPES.include?(type&.downcase) + return unless REF_TYPES.include?(type.to_s.downcase) type.downcase end diff --git a/lib/feature.rb b/lib/feature.rb index 7df692ec552..4b3ebab6dce 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -4,6 +4,33 @@ require 'flipper/adapters/active_record' require 'flipper/adapters/active_support_cache_store' module Feature + module BypassLoadBalancer + FLAG = 'FEATURE_FLAGS_BYPASS_LOAD_BALANCER' + class FlipperRecord < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord -- This class perfectly replaces + # Flipper::Adapters::ActiveRecord::Model, which inherits ActiveRecord::Base + include DatabaseReflection + self.abstract_class = true + + # Bypass the load balancer by restoring the default behavior of `connection` + # before the load balancer patches ActiveRecord::Base + def self.connection + retrieve_connection + end + end + + class FlipperFeature < FlipperRecord + self.table_name = 'features' + end + + class FlipperGate < FlipperRecord + self.table_name = 'feature_gates' + end + + def self.enabled? + Gitlab::Utils.to_boolean(ENV[FLAG], default: false) + end + end + # Classes to override flipper table names class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature include DatabaseReflection @@ -33,14 +60,19 @@ module Feature # Generates the same flipper_id when in a request # If not in a request, it generates a unique flipper_id every time class FlipperRequest - def id + def flipper_id Gitlab::SafeRequestStore.fetch("flipper_request_id") do - SecureRandom.uuid + "FlipperRequest:#{SecureRandom.uuid}".freeze end end + end - def flipper_id - "FlipperRequest:#{id}" + # Generates a unique flipper_id for the current GitLab instance. + class FlipperGitlabInstance + attr_reader :flipper_id + + def initialize + @flipper_id = "FlipperGitlabInstance:#{::Gitlab.config.gitlab.host}".freeze end end @@ -65,7 +97,8 @@ module Feature end def persisted_names - return [] unless ApplicationRecord.database.exists? + model = BypassLoadBalancer.enabled? ? BypassLoadBalancer::FlipperRecord : ApplicationRecord + return [] unless model.database.exists? # This loads names of all stored feature flags # and returns a stable Set in the following order: @@ -89,14 +122,9 @@ module Feature # 2. The `default_enabled_if_undefined:` is tech debt related to Gitaly flags # and should not be used outside of Gitaly's `lib/feature/gitaly.rb` def enabled?(key, thing = nil, type: :development, default_enabled_if_undefined: nil) - if check_feature_flags_definition? - if thing && !thing.respond_to?(:flipper_id) && !thing.is_a?(Flipper::Types::Group) - raise InvalidFeatureFlagError, - "The thing '#{thing.class.name}' for feature flag '#{key}' needs to include `FeatureGate` or implement `flipper_id`" - end + thing = sanitized_thing(thing) - Feature::Definition.valid_usage!(key, type: type) - end + check_feature_flags_definition!(key, thing, type) default_enabled = Feature::Definition.default_enabled?(key, default_enabled_if_undefined: default_enabled_if_undefined) feature_value = current_feature_value(key, thing, default_enabled: default_enabled) @@ -111,11 +139,15 @@ module Feature end def disabled?(key, thing = nil, type: :development, default_enabled_if_undefined: nil) + thing = sanitized_thing(thing) + # we need to make different method calls to make it easy to mock / define expectations in test mode thing.nil? ? !enabled?(key, type: type, default_enabled_if_undefined: default_enabled_if_undefined) : !enabled?(key, thing, type: type, default_enabled_if_undefined: default_enabled_if_undefined) end def enable(key, thing = true) + thing = sanitized_thing(thing) + log(key: key, action: __method__, thing: thing) return_value = with_feature(key) { _1.enable(thing) } @@ -129,12 +161,16 @@ module Feature end def disable(key, thing = false) + thing = sanitized_thing(thing) + log(key: key, action: __method__, thing: thing) with_feature(key) { _1.disable(thing) } end def opted_out?(key, thing) + thing = sanitized_thing(thing) + return false unless thing.respond_to?(:flipper_id) # Ignore Feature::Types::Group return false unless persisted_name?(key) @@ -144,6 +180,8 @@ module Feature end def opt_out(key, thing) + thing = sanitized_thing(thing) + return unless thing.respond_to?(:flipper_id) # Ignore Feature::Types::Group log(key: key, action: __method__, thing: thing) @@ -153,6 +191,8 @@ module Feature end def remove_opt_out(key, thing) + thing = sanitized_thing(thing) + return unless thing.respond_to?(:flipper_id) # Ignore Feature::Types::Group return unless persisted_name?(key) @@ -228,6 +268,10 @@ module Feature end end + def gitlab_instance + @flipper_gitlab_instance ||= FlipperGitlabInstance.new + end + def logger @logger ||= Feature::Logger.build end @@ -246,6 +290,17 @@ module Feature private + def sanitized_thing(thing) + case thing + when :instance + gitlab_instance + when :request, :current_request + current_request + else + thing + end + end + # Compute if thing is enabled, taking opt-out overrides into account # Evaluate if `default enabled: false` or the feature has been persisted. # `persisted_name?` can potentially generate DB queries and also checks for inclusion @@ -279,7 +334,9 @@ module Feature def unsafe_get(key) # During setup the database does not exist yet. So we haven't stored a value # for the feature yet and return the default. - return unless ApplicationRecord.database.exists? + + model = BypassLoadBalancer.enabled? ? BypassLoadBalancer::FlipperRecord : ApplicationRecord + return unless model.database.exists? flag_stack = ::Thread.current[:feature_flag_recursion_check] || [] Thread.current[:feature_flag_recursion_check] = flag_stack @@ -313,10 +370,15 @@ module Feature end def build_flipper_instance(memoize: false) - active_record_adapter = Flipper::Adapters::ActiveRecord.new( - feature_class: FlipperFeature, - gate_class: FlipperGate) - + active_record_adapter = if BypassLoadBalancer.enabled? + Flipper::Adapters::ActiveRecord.new( + feature_class: BypassLoadBalancer::FlipperFeature, + gate_class: BypassLoadBalancer::FlipperGate) + else + Flipper::Adapters::ActiveRecord.new( + feature_class: FlipperFeature, + gate_class: FlipperGate) + end # Redis L2 cache redis_cache_adapter = ActiveSupportCacheStoreAdapter.new( @@ -343,6 +405,17 @@ module Feature Gitlab.dev_or_test_env? end + def check_feature_flags_definition!(key, thing, type) + return unless check_feature_flags_definition? + + if thing && !thing.respond_to?(:flipper_id) && !thing.is_a?(Flipper::Types::Group) + raise InvalidFeatureFlagError, + "The thing '#{thing.class.name}' for feature flag '#{key}' needs to include `FeatureGate` or implement `flipper_id`" + end + + Feature::Definition.valid_usage!(key, type: type) + end + def l1_cache_backend Gitlab::ProcessMemoryCache.cache_backend end diff --git a/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template b/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template index e73bdda64eb..8264a7486af 100644 --- a/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template +++ b/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template @@ -3,8 +3,8 @@ migration_job_name: <%= class_name %> description: # Please capture what <%= class_name %> does feature_category: <%= feature_category %> introduced_by_url: # URL of the MR (or issue/commit) that introduced the migration -milestone: <%= current_milestone %> +milestone: '<%= current_milestone %>' queued_migration_version: <%= migration_number %> # Replace with the approximate date you think it's best to ensure the completion of this BBM. finalize_after: # yyyy-mm-dd -finalized_by: # version of the migration that ensured this bbm +finalized_by: # version of the migration that finalized this BBM diff --git a/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template index df4c5382749..37d67194c59 100644 --- a/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template +++ b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template @@ -19,7 +19,6 @@ class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Data :<%= table_name %>, :<%= column_name %>, job_interval: DELAY_INTERVAL, - queued_migration_version: '<%= migration_number %>', batch_size: BATCH_SIZE, sub_batch_size: SUB_BATCH_SIZE ) diff --git a/lib/generators/gitlab/analytics/group_fetcher.rb b/lib/generators/gitlab/analytics/group_fetcher.rb new file mode 100644 index 00000000000..4a60d8f75bd --- /dev/null +++ b/lib/generators/gitlab/analytics/group_fetcher.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + class GroupFetcher + class << self + def group_unknown?(group) + return false if groups.empty? + + !groups.has_key?(group) + end + + def stage_text(group) + groups[group]&.fetch(:stage) || '' + end + + def section_text(group) + groups.dig(group, :section) || '' + end + + private + + # Output looks like { "import_and_integrate" => { stage: "manage", section: "dev" } ... } + # Returns {} if stages.yml cannot be fetched and parsed + def groups + return @groups if @groups + + response = Gitlab::HTTP.get('https://gitlab.com/gitlab-com/www-gitlab-com/raw/master/data/stages.yml') + raise "Unable to load stages.yml" unless response.success? + + data = YAML.safe_load(response.body) + + groups_data = {} + + data['stages'].each do |stage_name, stage_data| + stage_data['groups'].each_key do |group_name| + groups_data[group_name] = { stage: stage_name, section: stage_data['section'] } + end + end + + @groups = groups_data.sort_by { |key, _value| key }.to_h + rescue StandardError + @groups = {} + end + end + end + end +end diff --git a/lib/generators/gitlab/analytics/internal_events_generator.rb b/lib/generators/gitlab/analytics/internal_events_generator.rb index e0add9ca41d..2e3d273b68d 100644 --- a/lib/generators/gitlab/analytics/internal_events_generator.rb +++ b/lib/generators/gitlab/analytics/internal_events_generator.rb @@ -10,22 +10,10 @@ module Gitlab '7d' => 'counts_7d', '28d' => 'counts_28d' }.freeze + TIME_FRAMES_DEFAULT = TIME_FRAME_DIRS.keys.freeze + TIME_FRAMES_ALLOWED_FOR_UNIQUE = (TIME_FRAMES_DEFAULT - ['all']).freeze - TIME_FRAMES_DEFAULT = TIME_FRAME_DIRS.keys.tap do |time_frame_defaults| - time_frame_defaults.class_eval do - def to_s - join(", ") - end - end - end.freeze - - ALLOWED_TIERS = %w[free premium ultimate].dup.tap do |tiers_default| - tiers_default.class_eval do - def to_s - join(", ") - end - end - end.freeze + ALLOWED_TIERS = %w[free premium ultimate].freeze NEGATIVE_ANSWERS = %w[no n No NO N].freeze POSITIVE_ANSWERS = %w[yes y Yes YES Y].freeze @@ -50,7 +38,6 @@ module Gitlab hide: true class_option :time_frames, optional: true, - default: TIME_FRAMES_DEFAULT, type: :array, banner: TIME_FRAMES_DEFAULT, desc: "Indicates the metrics time frames. Please select one or more from: #{TIME_FRAMES_DEFAULT}" @@ -63,15 +50,8 @@ module Gitlab class_option :group, type: :string, optional: false, - desc: 'Name of group that added this metric' - class_option :stage, - type: :string, - optional: false, - desc: 'Name of stage that added this metric' - class_option :section, - type: :string, - optional: false, - desc: 'Name of section that added this metric' + desc: "Name of group that added this metric. " \ + "See https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml" class_option :mr, type: :string, optional: false, @@ -141,8 +121,8 @@ module Gitlab options[:event] end - def unique(time_frame) - return if time_frame == 'all' + def unique + return unless with_unique? "\n unique: #{options.fetch(:unique)}" end @@ -178,16 +158,14 @@ module Gitlab Gitlab::VERSION.match('(\d+\.\d+)').captures.first end - def class_name(time_frame) - time_frame == 'all' ? 'TotalCountMetric' : 'RedisHLLMetric' + def class_name + with_unique? ? 'RedisHLLMetric' : 'TotalCountMetric' end def key_path(time_frame) - if time_frame == 'all' - "count_total_#{event}" - else - "count_distinct_#{options[:unique].sub('.', '_')}_from_#{event}_#{time_frame}" - end + return "count_distinct_#{options[:unique].sub('.', '_')}_from_#{event}_#{time_frame}" if with_unique? + + "count_total_#{event}_#{time_frame}" end def metric_file_path(time_frame) @@ -199,23 +177,27 @@ module Gitlab def validate! validate_tiers! - %i[event mr section stage group].each do |option| + %i[event mr group].each do |option| raise "The option: --#{option} is missing" unless options.key? option end + raise "Unknown group" if GroupFetcher.group_unknown?(options[:group]) + time_frames.each do |time_frame| validate_time_frame!(time_frame) - raise "The option: --unique is missing" if time_frame != 'all' && !options.key?('unique') - validate_key_path!(time_frame) end end def validate_time_frame!(time_frame) - return if TIME_FRAME_DIRS.key?(time_frame) + unless TIME_FRAME_DIRS.key?(time_frame) + raise "Invalid time frame: #{time_frame}, allowed options are: #{TIME_FRAMES_DEFAULT}" + end + + invalid_time_frame = with_unique? && TIME_FRAMES_ALLOWED_FOR_UNIQUE.exclude?(time_frame) - raise "Invalid time frame: #{time_frame}, allowed options are: #{TIME_FRAMES_DEFAULT}" + raise "Invalid time frame: #{time_frame} for a metric using `unique`" if invalid_time_frame end def validate_tiers! @@ -252,12 +234,20 @@ module Gitlab end end + def with_unique? + options.key?(:unique) + end + def free? options[:tiers].include? "free" end def time_frames - options[:time_frames] + @time_frames ||= options[:time_frames] || default_time_frames + end + + def default_time_frames + with_unique? ? TIME_FRAMES_ALLOWED_FOR_UNIQUE : TIME_FRAMES_DEFAULT end def directory diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb index 81f02c004af..f6d0f8b04b3 100644 --- a/lib/gitlab/access/branch_protection.rb +++ b/lib/gitlab/access/branch_protection.rb @@ -74,7 +74,11 @@ module Gitlab end def protection_partial - protection_none.merge(allow_force_push: false) + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } end def protected_fully @@ -89,15 +93,15 @@ module Gitlab { allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], - allow_force_push: true + allow_force_push: false } end def protected_after_initial_push { allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], - allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], - allow_force_push: true, + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false, developer_can_initial_push: true } end diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb index 2143497f084..6a1529ade92 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb @@ -14,12 +14,14 @@ module Gitlab Issue => { serializer_class: AnalyticsIssueSerializer, includes_for_query: { project: { namespace: [:route] }, author: [] }, - columns_for_select: %I[title iid id created_at author_id project_id] + columns_for_select: %I[title iid id created_at author_id project_id], + finder_class: IssuesFinder }, MergeRequest => { serializer_class: AnalyticsMergeRequestSerializer, includes_for_query: { target_project: [:namespace], author: [] }, - columns_for_select: %I[title iid id created_at author_id state_id target_project_id] + columns_for_select: %I[title iid id created_at author_id state_id target_project_id], + finder_class: MergeRequestsFinder } }.freeze @@ -80,14 +82,17 @@ module Gitlab def load_issuables(stage_event_records) stage_event_records_by_issuable_id = stage_event_records.index_by(&:issuable_id) - issuable_model = stage_event_model.issuable_model - issuables_by_id = issuable_model.id_in(stage_event_records_by_issuable_id.keys).index_by(&:id) + issuables_by_id = finder.execute.id_in(stage_event_records_by_issuable_id.keys).index_by(&:id) stage_event_records_by_issuable_id.map do |issuable_id, record| [issuables_by_id[issuable_id], record] if issuables_by_id[issuable_id] end.compact end + def finder + MAPPINGS.fetch(subject_class).fetch(:finder_class).new(params[:current_user]) + end + def serializer MAPPINGS.fetch(subject_class).fetch(:serializer_class).new end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index 0c4a0afa1d5..4a444b06500 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -119,7 +119,9 @@ module Gitlab attrs[:namespace] = namespace_attributes attrs[:enable_tasks_by_type_chart] = 'false' attrs[:enable_customizable_stages] = 'false' + attrs[:can_edit] = 'false' attrs[:enable_projects_filter] = 'false' + attrs[:enable_vsd_link] = 'false' attrs[:default_stages] = Gitlab::Analytics::CycleAnalytics::DefaultStages.all.map do |stage_params| ::Analytics::CycleAnalytics::StagePresenter.new(stage_params) end.to_json @@ -151,8 +153,8 @@ module Gitlab helpers = ActionController::Base.helpers {}.tap do |paths| - paths[:empty_state_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") - paths[:no_data_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") + paths[:empty_state_svg_path] = helpers.image_path("illustrations/empty-state/empty-dashboard-md.svg") + paths[:no_data_svg_path] = helpers.image_path("illustrations/empty-state/empty-dashboard-md.svg") paths[:no_access_svg_path] = helpers.image_path("illustrations/analytics/no-access.svg") if project diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 67fc2ae2fcc..e46bbc2cfda 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -26,7 +26,8 @@ module Gitlab :artifacts_dependencies_size, :artifacts_dependencies_count, :root_caller_id, - :merge_action_status + :merge_action_status, + :bulk_import_entity_id ].freeze private_constant :KNOWN_KEYS @@ -45,7 +46,8 @@ module Gitlab Attribute.new(:artifacts_dependencies_size, Integer), Attribute.new(:artifacts_dependencies_count, Integer), Attribute.new(:root_caller_id, String), - Attribute.new(:merge_action_status, String) + Attribute.new(:merge_action_status, String), + Attribute.new(:bulk_import_entity_id, Integer) ].freeze private_constant :APPLICATION_ATTRIBUTES @@ -95,6 +97,7 @@ module Gitlab # rubocop: disable Metrics/CyclomaticComplexity # rubocop: disable Metrics/PerceivedComplexity + # rubocop: disable Metrics/AbcSize def to_lazy_hash {}.tap do |hash| assign_hash_if_value(hash, :caller_id) @@ -106,6 +109,7 @@ module Gitlab assign_hash_if_value(hash, :artifacts_dependencies_size) assign_hash_if_value(hash, :artifacts_dependencies_count) assign_hash_if_value(hash, :merge_action_status) + assign_hash_if_value(hash, :bulk_import_entity_id) hash[:user] = -> { username } if include_user? hash[:user_id] = -> { user_id } if include_user? @@ -115,10 +119,12 @@ module Gitlab hash[:pipeline_id] = -> { job&.pipeline_id } if set_values.include?(:job) hash[:job_id] = -> { job&.id } if set_values.include?(:job) hash[:artifact_size] = -> { artifact&.size } if set_values.include?(:artifact) + hash[:bulk_import_entity_id] = -> { bulk_import_entity_id } if set_values.include?(:bulk_import_entity_id) end end # rubocop: enable Metrics/CyclomaticComplexity # rubocop: enable Metrics/PerceivedComplexity + # rubocop: enable Metrics/AbcSize def use Labkit::Context.with_context(to_lazy_hash) { yield } diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 469927b8a53..3d2f13af9dc 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -52,8 +52,9 @@ module Gitlab project_testing_integration: { threshold: 5, interval: 1.minute }, email_verification: { threshold: 10, interval: 10.minutes }, email_verification_code_send: { threshold: 10, interval: 1.hour }, - phone_verification_send_code: { threshold: 10, interval: 1.hour }, - phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, + phone_verification_challenge: { threshold: 3, interval: 1.day }, + phone_verification_send_code: { threshold: 5, interval: 1.day }, + phone_verification_verify_code: { threshold: 5, interval: 1.day }, namespace_exists: { threshold: 20, interval: 1.minute }, update_namespace_name: { threshold: -> { application_settings.update_namespace_name_rate_limit }, interval: 1.hour }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 578cfb52714..8e894be4fc4 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -436,8 +436,7 @@ module Gitlab end def unavailable_scopes_for_resource(resource) - unavailable_observability_scopes_for_resource(resource) + - unavailable_ai_features_scopes_for_resource(resource) + unavailable_observability_scopes_for_resource(resource) end def unavailable_observability_scopes_for_resource(resource) @@ -447,10 +446,6 @@ module Gitlab OBSERVABILITY_SCOPES end - def unavailable_ai_features_scopes_for_resource(_resource) - AI_FEATURES_SCOPES - end - def non_admin_available_scopes API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + AI_FEATURES_SCOPES end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 7524d8b9f85..235c472d292 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -4,10 +4,39 @@ module Gitlab module Auth module Saml class Config + DEFAULT_NICKNAME_ATTRS = %w[username nickname].freeze + DEFAULT_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name + http://schemas.microsoft.com/ws/2008/06/identity/claims/name + ].freeze + DEFAULT_EMAIL_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress + http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress + ].freeze + DEFAULT_FIRST_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname + http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname + ].freeze + DEFAULT_LAST_NAME_ATTRS = %w[ + http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname + http://schemas.microsoft.com/ws/2008/06/identity/claims/surname + ].freeze + class << self def enabled? ::AuthHelper.saml_providers.any? end + + def default_attribute_statements + defaults = OmniAuth::Strategies::SAML.default_options[:attribute_statements].to_hash.deep_symbolize_keys + defaults[:nickname] = DEFAULT_NICKNAME_ATTRS.dup + defaults[:name].concat(DEFAULT_NAME_ATTRS) + defaults[:email].concat(DEFAULT_EMAIL_ATTRS) + defaults[:first_name].concat(DEFAULT_FIRST_NAME_ATTRS) + defaults[:last_name].concat(DEFAULT_LAST_NAME_ATTRS) + + defaults + end end DEFAULT_PROVIDER_NAME = 'saml' diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb index 74f7fdfc180..341edbed9c2 100644 --- a/lib/gitlab/auth/unique_ips_limiter.rb +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -30,13 +30,11 @@ module Gitlab key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" Gitlab::Redis::SharedState.with do |redis| - unique_ips_count = nil redis.multi do |r| r.zadd(key, time, ip) r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) - unique_ips_count = r.zcard(key) - end - unique_ips_count.value + r.zcard(key) + end.last end end end diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml index 9424686340f..e9d54ca3359 100644 --- a/lib/gitlab/background_migration/.rubocop.yml +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -40,12 +40,6 @@ Metrics/BlockLength: Long blocks can be hard to read. Consider splitting the code into separate methods. -Style/Documentation: - Enabled: true - Details: > - Adding documentation makes it easier to figure out what a migration is - supposed to do. - Style/FrozenStringLiteralComment: Enabled: true Details: >- diff --git a/lib/gitlab/background_migration/backfill_branch_protection_namespace_setting.rb b/lib/gitlab/background_migration/backfill_branch_protection_namespace_setting.rb new file mode 100644 index 00000000000..c063a990188 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_branch_protection_namespace_setting.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is used to update the default_branch_protection_defaults column + # for user namespaces of the namespace_settings table. + class BackfillBranchProtectionNamespaceSetting < BatchedMigrationJob + operation_name :set_default_branch_protection_defaults + feature_category :source_code_management + + # Migration only version of `namespaces` table + class Namespace < ::ApplicationRecord + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + has_one :namespace_setting, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::NamespaceSetting' + end + + # Migration only version of `namespace_settings` table + class NamespaceSetting < ::ApplicationRecord + self.table_name = 'namespace_settings' + belongs_to :namespace, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::Namespace' + end + + # Migration only version of Gitlab::Access:BranchProtection application code. + class BranchProtection + attr_reader :level + + def initialize(level) + @level = level + end + + PROTECTION_NONE = 0 + PROTECTION_DEV_CAN_PUSH = 1 + PROTECTION_FULL = 2 + PROTECTION_DEV_CAN_MERGE = 3 + PROTECTION_DEV_CAN_INITIAL_PUSH = 4 + + DEVELOPER = 30 + MAINTAINER = 40 + + def to_hash + case level + when PROTECTION_NONE + self.class.protection_none + when PROTECTION_DEV_CAN_PUSH + self.class.protection_partial + when PROTECTION_FULL + self.class.protected_fully + when PROTECTION_DEV_CAN_MERGE + self.class.protected_against_developer_pushes + when PROTECTION_DEV_CAN_INITIAL_PUSH + self.class.protected_after_initial_push + end + end + + class << self + def protection_none + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true + } + end + + def protection_partial + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } + end + + def protected_fully + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } + end + + def protected_against_developer_pushes + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: false + } + end + + def protected_after_initial_push + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false, + developer_can_initial_push: true + } + end + end + end + + def perform + each_sub_batch do |sub_batch| + update_default_protection_branch_defaults(sub_batch) + end + end + + private + + def update_default_protection_branch_defaults(batch) + namespace_settings = NamespaceSetting.where(namespace_id: batch.pluck(:namespace_id)).includes(:namespace) + + values_list = namespace_settings.map do |namespace_setting| + level = namespace_setting.namespace.default_branch_protection.to_i + value = BranchProtection.new(level).to_hash.to_json + "(#{namespace_setting.namespace_id}, '#{value}'::jsonb)" + end.join(", ") + + sql = <<~SQL + WITH new_values (namespace_id, default_branch_protection_defaults) AS ( + VALUES + #{values_list} + ) + UPDATE namespace_settings + SET default_branch_protection_defaults = new_values.default_branch_protection_defaults + FROM new_values + WHERE namespace_settings.namespace_id = new_values.namespace_id; + SQL + + connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb index 83acd8a27f7..84b0f5c97df 100644 --- a/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb @@ -20,6 +20,8 @@ module Gitlab def perform each_sub_batch do |sub_batch| + reset_has_remediations_attribute(sub_batch) + update_query = update_query_for(sub_batch) connection.execute(update_query) @@ -28,6 +30,10 @@ module Gitlab private + def reset_has_remediations_attribute(sub_batch) + sub_batch.update_all(has_remediations: false) + end + def update_query_for(sub_batch) subquery = sub_batch.joins(" INNER JOIN vulnerability_occurrences ON diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb index 8c151bc36ac..e230fe46466 100644 --- a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -15,7 +15,7 @@ module Gitlab def perform each_sub_batch do |sub_batch| update_search_data(sub_batch) - rescue ActiveRecord::StatementInvalid => e + rescue ActiveRecord::StatementInvalid => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') update_search_data_individually(sub_batch) @@ -44,7 +44,7 @@ module Gitlab relation.pluck(:id).each do |issue_id| update_search_data(relation.klass.where(id: issue_id)) sleep(pause_ms * 0.001) - rescue ActiveRecord::StatementInvalid => e + rescue ActiveRecord::StatementInvalid => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') logger.error( diff --git a/lib/gitlab/background_migration/backfill_merge_request_diffs_project_id.rb b/lib/gitlab/background_migration/backfill_merge_request_diffs_project_id.rb new file mode 100644 index 00000000000..881716b5cc0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_merge_request_diffs_project_id.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration populates the new `merge_request_diffs.project_id` column from joining with `merge_requests` table + class BackfillMergeRequestDiffsProjectId < BatchedMigrationJob + operation_name :update_all + scope_to ->(relation) { relation.where(project_id: nil) } + + feature_category :code_review_workflow + + def perform + each_sub_batch do |sub_batch| + ApplicationRecord.connection.execute <<-SQL + UPDATE merge_request_diffs + SET project_id = merge_requests.target_project_id + FROM merge_requests + WHERE merge_requests.id = merge_request_diffs.merge_request_id + AND merge_request_diffs.id IN (#{sub_batch.select(:id).to_sql}) + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_vs_code_settings_uuid.rb b/lib/gitlab/background_migration/backfill_vs_code_settings_uuid.rb new file mode 100644 index 00000000000..2bb0e0b6d98 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_vs_code_settings_uuid.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillVsCodeSettingsUuid < BatchedMigrationJob + operation_name :backfill_vs_code_settings_uuid + scope_to ->(relation) { relation.where(uuid: nil) } + feature_category :web_ide + + def perform + each_sub_batch do |sub_batch| + vs_code_settings = sub_batch.map do |vs_code_setting| + vs_code_setting.attributes.merge(uuid: SecureRandom.uuid) + end + + VsCode::Settings::VsCodeSetting.upsert_all(vs_code_settings) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb index 44bda3fe2b6..618944e1653 100644 --- a/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb +++ b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb @@ -7,7 +7,7 @@ module Gitlab # This combination fails validation and doesn't make sense: # we always allow descendants to disable shared runners class FixAllowDescendantsOverrideDisabledSharedRunners < BatchedMigrationJob - feature_category :runner_fleet + feature_category :fleet_visibility operation_name :fix_allow_descendants_override_disabled_shared_runners def perform diff --git a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb index 0b79bc143db..4f1f70f3337 100644 --- a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb +++ b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb @@ -36,7 +36,7 @@ module Gitlab def migrate_remediations(findings, existing_links) findings.each do |finding| create_links(build_links_from(finding, existing_links)) - rescue ActiveRecord::StatementInvalid => e + rescue ActiveRecord::StatementInvalid => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 logger.error( message: e.message, class: self.class.name, @@ -76,7 +76,7 @@ module Gitlab return [] if parsed_links.blank? parsed_links.select { |link| link.try(:[], 'url').present? }.uniq - rescue JSON::ParserError => e + rescue JSON::ParserError => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 logger.warn( message: e.message, class: self.class.name diff --git a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb index 9eadef96db6..c6a41bc1c65 100644 --- a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb +++ b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb @@ -74,7 +74,7 @@ module Gitlab create_finding_remediations(finding.id, result_ids) end - rescue StandardError => e + rescue StandardError => e # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 logger.error( message: e.message, class: self.class.name, diff --git a/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb index ee0f73cc3de..c310c10d7fa 100644 --- a/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb +++ b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb @@ -58,7 +58,7 @@ module Gitlab def populate_for(vulnerability) log_warning(vulnerability) unless vulnerability.copy_dismissal_information - rescue StandardError => error + rescue StandardError => error # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 log_error(error, vulnerability) end diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb index 56506814dc0..a83c4625cb4 100644 --- a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -117,7 +117,7 @@ module Gitlab return DUMMY_TAGS unless response response['tags'] || [] - rescue StandardError + rescue StandardError # rubocop:todo BackgroundMigration/AvoidSilentRescueExceptions -- https://gitlab.com/gitlab-org/gitlab/-/issues/431592 DUMMY_TAGS end end diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index c8520993b8e..91994c2fa95 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,8 +3,7 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab - # rubocop:disable Rails/ApplicationController - class BaseDoorkeeperController < ActionController::Base + class BaseDoorkeeperController < BaseActionController include Gitlab::Allowable include EnforcesTwoFactorAuthentication include SessionsHelper @@ -13,5 +12,4 @@ module Gitlab helper_method :can? end - # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb deleted file mode 100644 index 9f87bb2347c..00000000000 --- a/lib/gitlab/bitbucket_import/importer.rb +++ /dev/null @@ -1,339 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BitbucketImport - class Importer - LABELS = [{ title: 'bug', color: '#FF0000' }, - { title: 'enhancement', color: '#428BCA' }, - { title: 'proposal', color: '#69D100' }, - { title: 'task', color: '#7F8C8D' }].freeze - - attr_reader :project, :client, :errors, :users - - ALREADY_IMPORTED_CACHE_KEY = 'bitbucket_cloud-importer/already-imported/%{project}/%{collection}' - - def initialize(project) - @project = project - @client = Bitbucket::Client.new(project.import_data.credentials) - @formatter = Gitlab::ImportFormatter.new - @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) - @labels = {} - @errors = [] - @users = {} - end - - def execute - import_wiki - import_issues - import_pull_requests - handle_errors - metrics.track_finished_import - - true - end - - def create_labels - LABELS.each do |label_params| - label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) - if label.valid? - @labels[label_params[:title]] = label - else - raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\"" - end - end - end - - def import_pull_request_comments(pull_request, merge_request) - comments = client.pull_request_comments(repo, pull_request.iid) - - inline_comments, pr_comments = comments.partition(&:inline?) - - import_inline_comments(inline_comments, pull_request, merge_request) - import_standalone_pr_comments(pr_comments, merge_request) - end - - private - - def already_imported?(collection, iid) - Gitlab::Cache::Import::Caching.set_includes?(cache_key(collection), iid) - end - - def mark_as_imported(collection, iid) - Gitlab::Cache::Import::Caching.set_add(cache_key(collection), iid) - end - - def cache_key(collection) - format(ALREADY_IMPORTED_CACHE_KEY, project: project.id, collection: collection) - end - - def handle_errors - return unless errors.any? - - project.import_state.update_column(:last_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def store_pull_request_error(pull_request, ex) - backtrace = Gitlab::BacktraceCleaner.clean_backtrace(ex.backtrace) - error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw&.to_json } - - Gitlab::ErrorTracking.log_exception(ex, error) - - # Omit the details from the database to avoid blowing up usage in the error column - error.delete(:trace) - error.delete(:raw_response) - - errors << error - end - - def gitlab_user_id(project, username) - find_user_id(username) || project.creator_id - end - - # rubocop: disable CodeReuse/ActiveRecord - def find_user_id(username) - return unless username - - return users[username] if users.key?(username) - - users[username] = User.by_provider_and_extern_uid(:bitbucket, username).select(:id).first&.id - end - # rubocop: enable CodeReuse/ActiveRecord - - def allocate_issues_internal_id!(project, client) - last_bitbucket_issue = client.last_issue(repo) - - return unless last_bitbucket_issue - - Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid) - end - - def repo - @repo ||= client.repo(project.import_source) - end - - def import_wiki - return if project.wiki.repository_exists? - - wiki = WikiFormatter.new(project) - - project.wiki.repository.import_repository(wiki.import_url) - rescue StandardError => e - errors << { type: :wiki, errors: e.message } - end - - def import_issues - return unless repo.issues_enabled? - - create_labels - - issue_type_id = ::WorkItems::Type.default_issue_type.id - - client.issues(repo).each_with_index do |issue, index| - next if already_imported?(:issues, issue.iid) - - # If a user creates an issue while the import is in progress, this can lead to an import failure. - # The workaround is to allocate IIDs before starting the importer. - allocate_issues_internal_id!(project, client) if index == 0 - - import_issue(issue, issue_type_id) - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def import_issue(issue, issue_type_id) - description = '' - description += @formatter.author_line(issue.author) unless find_user_id(issue.author) - description += issue.description - - label_name = issue.kind - milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil - - gitlab_issue = project.issues.create!( - iid: issue.iid, - title: issue.title, - description: description, - state_id: Issue.available_states[issue.state], - author_id: gitlab_user_id(project, issue.author), - namespace_id: project.project_namespace_id, - milestone: milestone, - work_item_type_id: issue_type_id, - created_at: issue.created_at, - updated_at: issue.updated_at - ) - - mark_as_imported(:issues, issue.iid) - - metrics.issues_counter.increment - - gitlab_issue.labels << @labels[label_name] - - import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? - rescue StandardError => e - errors << { type: :issue, iid: issue.iid, errors: e.message } - end - # rubocop: enable CodeReuse/ActiveRecord - - def import_issue_comments(issue, gitlab_issue) - client.issue_comments(repo, issue.iid).each do |comment| - # The note can be blank for issue service messages like "Changed title: ..." - # We would like to import those comments as well but there is no any - # specific parameter that would allow to process them, it's just an empty comment. - # To prevent our importer from just crashing or from creating useless empty comments - # we do this check. - next unless comment.note.present? - - note = '' - note += @formatter.author_line(comment.author) unless find_user_id(comment.author) - note += @ref_converter.convert_note(comment.note.to_s) - - begin - gitlab_issue.notes.create!( - project: project, - note: note, - author_id: gitlab_user_id(project, comment.author), - created_at: comment.created_at, - updated_at: comment.updated_at - ) - rescue StandardError => e - errors << { type: :issue_comment, iid: issue.iid, errors: e.message } - end - end - end - - def import_pull_requests - pull_requests = client.pull_requests(repo) - - pull_requests.each do |pull_request| - next if already_imported?(:pull_requests, pull_request.iid) - - import_pull_request(pull_request) - end - end - - def import_pull_request(pull_request) - description = '' - description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) - description += pull_request.description - - source_branch_sha = pull_request.source_branch_sha - target_branch_sha = pull_request.target_branch_sha - - source_sha_from_commit_sha = project.repository.commit(source_branch_sha)&.sha - source_sha_from_merge_sha = project.repository.commit(pull_request.merge_commit_sha)&.sha - - source_branch_sha = source_sha_from_commit_sha || source_sha_from_merge_sha || source_branch_sha - target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha - - merge_request = project.merge_requests.create!( - iid: pull_request.iid, - title: pull_request.title, - description: description, - source_project: project, - source_branch: pull_request.source_branch_name, - source_branch_sha: source_branch_sha, - target_project: project, - target_branch: pull_request.target_branch_name, - target_branch_sha: target_branch_sha, - state: pull_request.state, - author_id: gitlab_user_id(project, pull_request.author), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - ) - - mark_as_imported(:pull_requests, pull_request.iid) - - metrics.merge_requests_counter.increment - - import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? - rescue StandardError => e - store_pull_request_error(pull_request, e) - end - - def import_inline_comments(inline_comments, pull_request, merge_request) - position_map = {} - discussion_map = {} - - children, parents = inline_comments.partition(&:has_parent?) - - # The Bitbucket API returns threaded replies as parent-child - # relationships. We assume that the child can appear in any order in - # the JSON. - parents.each do |comment| - position_map[comment.iid] = build_position(merge_request, comment) - end - - children.each do |comment| - position_map[comment.iid] = position_map.fetch(comment.parent_id, nil) - end - - inline_comments.each do |comment| - attributes = pull_request_comment_attributes(comment) - attributes[:discussion_id] = discussion_map[comment.parent_id] if comment.has_parent? - - attributes.merge!( - position: position_map[comment.iid], - type: 'DiffNote') - - note = merge_request.notes.create!(attributes) - - # We can't store a discussion ID until a note is created, so if - # replies are created before the parent the discussion ID won't be - # linked properly. - discussion_map[comment.iid] = note.discussion_id - rescue StandardError => e - errors << { type: :pull_request, iid: comment.iid, errors: e.message } - end - end - - def build_position(merge_request, pr_comment) - params = { - diff_refs: merge_request.diff_refs, - old_path: pr_comment.file_path, - new_path: pr_comment.file_path, - old_line: pr_comment.old_pos, - new_line: pr_comment.new_pos - } - - Gitlab::Diff::Position.new(params) - end - - def import_standalone_pr_comments(pr_comments, merge_request) - pr_comments.each do |comment| - merge_request.notes.create!(pull_request_comment_attributes(comment)) - rescue StandardError => e - errors << { type: :pull_request, iid: comment.iid, errors: e.message } - end - end - - def pull_request_comment_attributes(comment) - { - project: project, - author_id: gitlab_user_id(project, comment.author), - note: comment_note(comment), - created_at: comment.created_at, - updated_at: comment.updated_at - } - end - - def comment_note(comment) - author = @formatter.author_line(comment.author) unless find_user_id(comment.author) - author.to_s + @ref_converter.convert_note(comment.note.to_s) - end - - def log_base_data - { - class: self.class.name, - project_id: project.id, - project_path: project.full_path - } - end - - def metrics - @metrics ||= Gitlab::Import::Metrics.new(:bitbucket_importer, @project) - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb index 8ab82ddb0be..678cb4e129d 100644 --- a/lib/gitlab/bitbucket_import/importers/issues_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb @@ -33,6 +33,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb index 03dcc645f07..ecc41cc5436 100644 --- a/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb @@ -22,6 +22,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb index f7b1753a9f9..37bbc1d0c78 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -15,6 +15,8 @@ module Gitlab end def execute + return if skip + log_info(import_stage: 'import_pull_request', message: 'starting', iid: object[:iid]) description = '' @@ -58,6 +60,15 @@ module Gitlab attr_reader :object, :project, :formatter, :user_finder + def skip + return false unless object[:source_and_target_project_different] + + message = 'skipping because source and target projects are different' + log_info(import_stage: 'import_pull_request', message: message, iid: object[:iid]) + + true + end + def author_line return '' if find_user_id diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb index 1c7ce7f2f3a..eedb89c2d49 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb @@ -26,6 +26,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb index a1b0c2a5afe..1dc3c6fbfc1 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb @@ -22,6 +22,7 @@ module Gitlab job_waiter rescue StandardError => e track_import_failure!(project, exception: e) + job_waiter end private diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb index 9be7ed99436..cc950bbe13d 100644 --- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -6,6 +6,11 @@ module Gitlab class RepositoryImporter include Loggable + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + def initialize(project) @project = project end @@ -62,8 +67,9 @@ module Gitlab end def create_labels - importer = Gitlab::BitbucketImport::Importer.new(project) - importer.create_labels + LABELS.each do |label_params| + ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) + end end def wiki diff --git a/lib/gitlab/bitbucket_import/ref_converter.rb b/lib/gitlab/bitbucket_import/ref_converter.rb index 1159159a76d..1763bd26d61 100644 --- a/lib/gitlab/bitbucket_import/ref_converter.rb +++ b/lib/gitlab/bitbucket_import/ref_converter.rb @@ -4,7 +4,7 @@ module Gitlab module BitbucketImport class RefConverter REPO_MATCHER = 'https://bitbucket.org/%s' - PR_NOTE_ISSUE_NAME_REGEX = '(?<=/)[^/\)]+(?=\)[^/]*$)' + PR_NOTE_ISSUE_NAME_REGEX = "(issues\/.*\/(.*)\\))" UNWANTED_NOTE_REF_HTML = "{: data-inline-card='' }" attr_reader :project @@ -24,7 +24,7 @@ module Gitlab if note.match?('issues') note.gsub!('issues', '-/issues') - note.gsub!(issue_name(note), '') + note.gsub!("/#{issue_name(note)}", '') if issue_name(note) else note.gsub!('pull-requests', '-/merge_requests') note.gsub!('src', '-/blob') @@ -41,7 +41,11 @@ module Gitlab end def issue_name(note) - note.match(PR_NOTE_ISSUE_NAME_REGEX)[0] + match_data = note.match(PR_NOTE_ISSUE_NAME_REGEX) + + return unless match_data + + match_data[2] end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb index 69de47e2006..d58f7cec8ff 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_notes_importer.rb @@ -4,21 +4,19 @@ module Gitlab module BitbucketServerImport module Importers class PullRequestNotesImporter + include ::Gitlab::Import::MergeRequestHelpers include Loggable def initialize(project, hash) @project = project - @formatter = Gitlab::ImportFormatter.new - @client = BitbucketServer::Client.new(project.import_data.credentials) - @project_key = project.import_data.data['project_key'] - @repository_slug = project.import_data.data['repo_slug'] @user_finder = UserFinder.new(project) - - # TODO: Convert object into a object instead of using it as a hash + @formatter = Gitlab::ImportFormatter.new @object = hash.with_indifferent_access end def execute + return unless import_data_valid? + log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid]) merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord @@ -35,6 +33,9 @@ module Gitlab import_inline_comments(inline_comments.map(&:comment), merge_request) import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) + + approved_events = other_activities.select(&:approved_event?) + approved_events.each { |event| import_approved_event(merge_request, event) } end log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid]) @@ -42,7 +43,11 @@ module Gitlab private - attr_reader :object, :project, :formatter, :client, :project_key, :repository_slug, :user_finder + attr_reader :object, :project, :formatter, :user_finder + + def import_data_valid? + project.import_data&.credentials && project.import_data&.data + end # rubocop: disable CodeReuse/ActiveRecord def import_merge_event(merge_request, merge_event) @@ -60,6 +65,32 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def import_approved_event(merge_request, approved_event) + log_info( + import_stage: 'import_approved_event', + message: 'starting', + iid: merge_request.iid, + event_id: approved_event.id + ) + + user_id = user_finder.find_user_id(by: :username, value: approved_event.approver_username) || + user_finder.find_user_id(by: :email, value: approved_event.approver_email) + + return unless user_id + + submitted_at = approved_event.created_at || merge_request.updated_at + + create_approval!(project.id, merge_request.id, user_id, submitted_at) + create_reviewer!(merge_request.id, user_id, submitted_at) + + log_info( + import_stage: 'import_approved_event', + message: 'finished', + iid: merge_request.iid, + event_id: approved_event.id + ) + end + def import_inline_comments(inline_comments, merge_request) log_info(import_stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) @@ -177,6 +208,18 @@ module Gitlab updated_at: comment.updated_at } end + + def client + BitbucketServer::Client.new(project.import_data.credentials) + end + + def project_key + project.import_data.data['project_key'] + end + + def repository_slug + project.import_data.data['repo_slug'] + end end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb index ae73681f7f8..61c31fb9644 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb @@ -6,16 +6,20 @@ module Gitlab class PullRequestsImporter include ParallelScheduling + # Reduce fetch limit (from 100) to avoid Gitlab::Git::ResourceExhaustedError + PULL_REQUESTS_BATCH_SIZE = 50 + def execute page = 1 loop do log_info( - import_stage: 'import_pull_requests', message: "importing page #{page} using batch-size #{BATCH_SIZE}" + import_stage: 'import_pull_requests', + message: "importing page #{page} using batch-size #{PULL_REQUESTS_BATCH_SIZE}" ) pull_requests = client.pull_requests( - project_key, repository_slug, page_offset: page, limit: BATCH_SIZE + project_key, repository_slug, page_offset: page, limit: PULL_REQUESTS_BATCH_SIZE ).to_a break if pull_requests.empty? @@ -24,7 +28,15 @@ module Gitlab next if already_processed?(pull_request) next unless pull_request.merged? || pull_request.closed? - [pull_request.source_branch_sha, pull_request.target_branch_sha] + [].tap do |commits| + source_sha = pull_request.source_branch_sha + target_sha = pull_request.target_branch_sha + + existing_commits = repo.commits_by(oids: [source_sha, target_sha]).map(&:sha) + + commits << source_branch_commit(source_sha, pull_request) unless existing_commits.include?(source_sha) + commits << target_branch_commit(target_sha) unless existing_commits.include?(target_sha) + end end.flatten # Bitbucket Server keeps tracks of references for open pull requests in @@ -78,6 +90,22 @@ module Gitlab def id_for_already_processed_cache(object) object.iid end + + def repo + @repo ||= project.repository + end + + def ref_path(pull_request) + "refs/#{Repository::REF_MERGE_REQUEST}/#{pull_request.iid}/head" + end + + def source_branch_commit(source_branch_sha, pull_request) + [source_branch_sha, ':', ref_path(pull_request)].join + end + + def target_branch_commit(target_branch_sha) + [target_branch_sha, ':refs/keep-around/', target_branch_sha].join + end end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/users_importer.rb b/lib/gitlab/bitbucket_server_import/importers/users_importer.rb new file mode 100644 index 00000000000..f8d0521afb2 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/importers/users_importer.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module Importers + class UsersImporter + include Loggable + include UserCaching + + BATCH_SIZE = 100 + + def initialize(project) + @project = project + @project_id = project.id + end + + attr_reader :project, :project_id + + def execute + log_info(import_stage: 'import_users', message: 'starting') + + page = 1 + + loop do + log_info( + import_stage: 'import_users', + message: "importing page #{page} using batch size #{BATCH_SIZE}" + ) + + users = client.users(project_key, page_offset: page, limit: BATCH_SIZE).to_a + + break if users.empty? + + cache_users(users) + + page += 1 + end + + log_info(import_stage: 'import_users', message: 'finished') + end + + private + + def cache_users(users) + users_hash = users.each_with_object({}) do |user, hash| + cache_key = source_user_cache_key(project_id, user.username) + hash[cache_key] = user.email + end + + ::Gitlab::Cache::Import::Caching.write_multiple(users_hash) + end + + def client + @client ||= BitbucketServer::Client.new(project.import_data.credentials) + end + + def project_key + project.import_data.data['project_key'] + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/user_caching.rb b/lib/gitlab/bitbucket_server_import/user_caching.rb new file mode 100644 index 00000000000..0f0169122c5 --- /dev/null +++ b/lib/gitlab/bitbucket_server_import/user_caching.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketServerImport + module UserCaching + SOURCE_USER_CACHE_KEY = 'bitbucket_server/project/%s/source/username/%s' + + def source_user_cache_key(project_id, username) + format(SOURCE_USER_CACHE_KEY, project_id, username) + end + end + end +end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 8f2df29c320..e81a90831f7 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -138,7 +138,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.sismember(key, value) end end @@ -244,7 +244,14 @@ module Gitlab end def self.with_redis(&block) - Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + block_result = Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord -- This is not AR + cache_identity = Gitlab::Redis::Cache.with(&:inspect) # rubocop:disable CodeReuse/ActiveRecord -- This is not AR + + Gitlab::Redis::SharedState.with do |redis| + yield redis unless cache_identity == redis.default_store.inspect + end + + block_result end def self.validate_redis_value!(value) diff --git a/lib/gitlab/checks/global_file_size_check.rb b/lib/gitlab/checks/global_file_size_check.rb index ff24467e9cc..5dc41b2a4cc 100644 --- a/lib/gitlab/checks/global_file_size_check.rb +++ b/lib/gitlab/checks/global_file_size_check.rb @@ -3,6 +3,8 @@ module Gitlab module Checks class GlobalFileSizeCheck < BaseBulkChecker + include ActionView::Helpers::NumberHelper + LOG_MESSAGE = 'Checking for blobs over the file size limit' def validate! @@ -17,31 +19,24 @@ module Gitlab ).find if oversized_blobs.present? - - blob_details = {} - blob_id_size_msg = "" - oversized_blobs.each do |blob| - blob_details[blob.id] = { "size" => blob.size } - - # blob size is in byte, divide it by "/ 1024.0 / 1024.0" to get MiB - blob_id_size_msg += "- #{blob.id} (#{(blob.size / 1024.0 / 1024.0).round(2)} MiB) \n" - end + blob_id_size_msg = oversized_blobs.map do |blob| + "- #{blob.id} (#{number_to_human_size(blob.size)})" + end.join("\n") oversize_err_msg = <<~OVERSIZE_ERR_MSG - You are attempting to check in one or more blobs which exceed the #{file_size_limit}MiB limit: - - #{blob_id_size_msg} - To resolve this error, you must either reduce the size of the above blobs, or utilize LFS. - You may use "git ls-tree -r HEAD | grep $BLOB_ID" to see the file path. - Please refer to #{Rails.application.routes.url_helpers.help_page_url('user/free_push_limit')} and - #{Rails.application.routes.url_helpers.help_page_url('administration/settings/account_and_limit_settings')} - for further information. + You are attempting to check in one or more blobs which exceed the #{file_size_limit}MiB limit: + + #{blob_id_size_msg} + To resolve this error, you must either reduce the size of the above blobs, or utilize LFS. + You may use "git ls-tree -r HEAD | grep $BLOB_ID" to see the file path. + Please refer to #{Rails.application.routes.url_helpers.help_page_url('user/free_push_limit')} and + #{Rails.application.routes.url_helpers.help_page_url('administration/settings/account_and_limit_settings')} + for further information. OVERSIZE_ERR_MSG Gitlab::AppJsonLogger.info( message: 'Found blob over global limit', - blob_sizes: oversized_blobs.map(&:size), - blob_details: blob_details + blob_details: oversized_blobs.map { |blob| { "id" => blob.id, "size" => blob.size } } ) raise ::Gitlab::GitAccess::ForbiddenError, oversize_err_msg if enforce_global_file_size_limit? diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index cdc648bf005..cb0e60a096a 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -6,8 +6,8 @@ module Gitlab ERROR_MESSAGES = { change_existing_tags: 'You are not allowed to change existing tags on this project.', update_protected_tag: 'Protected tags cannot be updated.', - delete_protected_tag: 'You are not allowed to delete protected tags from this project. '\ - 'Only a project maintainer or owner can delete a protected tag.', + delete_protected_tag: 'You are not allowed to delete protected tags from this project. ' \ + 'Only a project maintainer or owner can delete a protected tag.', delete_protected_tag_non_web: 'You can only delete protected tags using the web interface.', create_protected_tag: 'You are not allowed to create this tag as it is protected.', default_branch_collision: 'You cannot use default branch name to create a tag', @@ -27,70 +27,86 @@ module Gitlab def validate! return unless tag_name - logger.log_timed(LOG_MESSAGES[:tag_checks]) do - if tag_exists? && user_access.cannot_do_action?(:admin_tag) - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags] - end - end - - default_branch_collision_check + logger.log_timed(LOG_MESSAGES[:tag_checks]) { tag_checks } + logger.log_timed(LOG_MESSAGES[:default_branch_collision_check]) { default_branch_collision_check } prohibited_tag_checks - protected_tag_checks + logger.log_timed(LOG_MESSAGES[:protected_tag_checks]) { protected_tag_checks } end private - def prohibited_tag_checks - return if deletion? + def tag_checks + return unless tag_exists? && user_access.cannot_do_action?(:admin_tag) - unless Gitlab::GitRefValidator.validate(tag_name) - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] - end + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags] + end - if tag_name.start_with?("refs/tags/") # rubocop: disable Style/GuardClause - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] - end + def default_branch_collision_check + return unless creation? && tag_name == project.default_branch - # rubocop: disable Style/GuardClause - # rubocop: disable Style/SoleNestedConditional - if Feature.enabled?(:prohibited_tag_name_encoding_check, project) - unless Gitlab::EncodingHelper.force_encode_utf8(tag_name).valid_encoding? - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name_encoding] - end - end - # rubocop: enable Style/SoleNestedConditional - # rubocop: enable Style/GuardClause + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:default_branch_collision] + end + + def prohibited_tag_checks + return if deletion? + + # Incorrectly encoded tags names may raise during other checks so we + # need to validate the encoding first + validate_encoding! + validate_valid_tag_name! + validate_tag_name_not_fully_qualified! validate_tag_name_not_sha_like! end def protected_tag_checks - logger.log_timed(LOG_MESSAGES[__method__]) do - return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks + return unless ProtectedTag.protected?(project, tag_name) - raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update? + validate_protected_tag_update! + validate_protected_tag_deletion! + validate_protected_tag_creation! + end - if deletion? - unless user_access.user.can?(:maintainer_access, project) - raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) - end + def validate_encoding! + return unless Feature.enabled?(:prohibited_tag_name_encoding_check, project) + return if Gitlab::EncodingHelper.force_encode_utf8(tag_name).valid_encoding? - unless updated_from_web? - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag_non_web] - end - end + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name_encoding] + end - unless user_access.can_create_tag?(tag_name) - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] - end - end + def validate_valid_tag_name! + return if Gitlab::GitRefValidator.validate(tag_name) + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] end - def default_branch_collision_check - logger.log_timed(LOG_MESSAGES[:default_branch_collision_check]) do - if creation? && tag_name == project.default_branch - raise GitAccess::ForbiddenError, ERROR_MESSAGES[:default_branch_collision] - end + def validate_tag_name_not_fully_qualified! + return unless tag_name.start_with?("refs/tags/") + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] + end + + def validate_protected_tag_update! + return unless update? + + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) + end + + def validate_protected_tag_deletion! + return unless deletion? + + unless user_access.user.can?(:maintainer_access, project) + raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) end + + return if updated_from_web? + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag_non_web] + end + + def validate_protected_tag_creation! + return if user_access.can_create_tag?(tag_name) + + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag] end def validate_tag_name_not_sha_like! diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index 84f8eae8deb..660d7701a8f 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -4,7 +4,7 @@ module Gitlab module Ci module Build class Image - attr_reader :alias, :command, :entrypoint, :name, :ports, :variables, :pull_policy + attr_reader :alias, :command, :entrypoint, :name, :ports, :variables, :executor_opts, :pull_policy class << self def from_image(job) @@ -28,6 +28,7 @@ module Gitlab when String @name = image @ports = [] + @executor_opts = {} when Hash @alias = image[:alias] @command = image[:command] @@ -35,6 +36,7 @@ module Gitlab @name = image[:name] @ports = build_ports(image).select(&:valid?) @variables = build_variables(image) + @executor_opts = image.fetch(:executor_opts, {}) @pull_policy = image[:pull_policy] end end diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index 50731d54fc0..607eff902ea 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -45,14 +45,31 @@ module Gitlab private - attr_reader :version + attr_reader :version, :component_name - # Given a path like "my-org/sub-group/the-project/path/to/component" - # find the project "my-org/sub-group/the-project" by looking at all possible paths. def find_project_by_component_path(path) + if Feature.enabled?(:ci_redirect_component_project, Feature.current_request) + project_full_path = extract_project_path(path) + + Project.find_by_full_path(project_full_path, follow_redirects: true).tap do |project| + next unless project + + @component_name = extract_component_name(project_full_path) + end + else + legacy_finder(path).tap do |project| + next unless project + + @component_name = extract_component_name(project.full_path) + end + end + end + + def legacy_finder(path) return if path.start_with?('/') # exit early if path starts with `/` or it will loop forever. possible_paths = [path] + index = nil loop_until(limit: 20) do @@ -68,17 +85,32 @@ module Gitlab ::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord end + # Given a path like "my-org/sub-group/the-project/the-component" + # we expect that the last `/` is the separator between the project full path and the + # component name. + def extract_project_path(path) + return if path.start_with?('/') # invalid project full path. + + index = path.rindex('/') # find index of last `/` in the path + return unless index + + path[0..index - 1] + end + def instance_path @full_path.delete_prefix(host) end - def component_name - instance_path.delete_prefix(project.full_path).delete_prefix('/') + def extract_component_name(project_path) + instance_path.delete_prefix(project_path).delete_prefix('/') end - strong_memoize_attr :component_name def latest_version_sha - project.releases.latest&.sha + if project.catalog_resource + project.catalog_resource.versions.latest&.sha + else + project.releases.latest&.sha + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 73d329930a5..16e4e473928 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -19,13 +19,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :source_ref_path, :source, :logger + attr_reader :root, :context, :source_ref_path, :source, :logger, :inject_edge_stages # rubocop: disable Metrics/ParameterLists - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil) + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil, inject_edge_stages: true) @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path @project = project + @inject_edge_stages = inject_edge_stages @context = self.logger.instrument(:config_build_context, once: true) do pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) @@ -99,6 +100,10 @@ module Gitlab root.workflow_entry.name end + def workflow_auto_cancel + root.workflow_entry.auto_cancel_value + end + def normalized_jobs @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end @@ -145,6 +150,8 @@ module Gitlab Config::Yaml::Tags::Resolver.new(initial_config).to_hash end + return initial_config unless inject_edge_stages + logger.instrument(:config_stages_inject, once: true) do Config::EdgeStagesInjector.new(initial_config).to_hash end diff --git a/lib/gitlab/ci/config/entry/auto_cancel.rb b/lib/gitlab/ci/config/entry/auto_cancel.rb new file mode 100644 index 00000000000..2c51ab82214 --- /dev/null +++ b/lib/gitlab/ci/config/entry/auto_cancel.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class AutoCancel < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_KEYS = %i[on_new_commit on_job_failure].freeze + ALLOWED_ON_NEW_COMMIT_OPTIONS = ::Ci::PipelineMetadata.auto_cancel_on_new_commits.keys.freeze + ALLOWED_ON_JOB_FAILURE_OPTIONS = ::Ci::PipelineMetadata.auto_cancel_on_job_failures.keys.freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + validates :on_new_commit, allow_nil: true, type: String, inclusion: { + in: ALLOWED_ON_NEW_COMMIT_OPTIONS, + message: format(_("must be one of: %{values}"), values: ALLOWED_ON_NEW_COMMIT_OPTIONS.join(', ')) + } + validates :on_job_failure, allow_nil: true, type: String, inclusion: { + in: ALLOWED_ON_JOB_FAILURE_OPTIONS, + message: format(_("must be one of: %{values}"), values: ALLOWED_ON_JOB_FAILURE_OPTIONS.join(', ')) + } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 84e31ca1fc6..58ab488d833 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -13,21 +13,6 @@ module Gitlab validations do validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS end - - def value - if string? - { name: @config } - elsif hash? - { - name: @config[:name], - entrypoint: @config[:entrypoint], - ports: (ports_value if ports_defined?), - pull_policy: pull_policy_value - }.compact - else - {} - end - end end end end diff --git a/lib/gitlab/ci/config/entry/imageable.rb b/lib/gitlab/ci/config/entry/imageable.rb index 1aecfee9ab9..53b810b3037 100644 --- a/lib/gitlab/ci/config/entry/imageable.rb +++ b/lib/gitlab/ci/config/entry/imageable.rb @@ -12,7 +12,9 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable - IMAGEABLE_ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze + EXECUTOR_OPTS_KEYS = %i[docker].freeze + + IMAGEABLE_ALLOWED_KEYS = EXECUTOR_OPTS_KEYS + %i[name entrypoint ports pull_policy].freeze included do include ::Gitlab::Config::Entry::Validatable @@ -23,9 +25,15 @@ module Gitlab validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true + validates :executor_opts, json_schema: { + base_directory: "lib/gitlab/ci/config/entry/schemas/imageable", + detail_errors: true, + filename: "executor_opts", + hash_conversion: true + }, allow_nil: true end - attributes :ports, :pull_policy + attributes :docker, :ports, :pull_policy entry :ports, Entry::Ports, description: 'Ports used to expose the image/service' @@ -49,6 +57,28 @@ module Gitlab def skip_config_hash_validation? true end + + def executor_opts + return unless config.is_a?(Hash) + + config.slice(*EXECUTOR_OPTS_KEYS).compact.presence + end + + def value + if string? + { name: config } + elsif hash? + { + name: config[:name], + entrypoint: config[:entrypoint], + executor_opts: executor_opts, + ports: (ports_value if ports_defined?), + pull_policy: pull_policy_value + }.compact + else + {} + end + end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 5fcafcba829..7ea4b460640 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -13,7 +13,7 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks - coverage retry parallel interruptible timeout + coverage retry parallel timeout release id_tokens publish pages].freeze validations do @@ -83,10 +83,6 @@ module Gitlab description: 'Services that will be used to execute this job.', inherit: true - entry :interruptible, ::Gitlab::Config::Entry::Boolean, - description: 'Set jobs interruptible value.', - inherit: true - entry :timeout, Entry::Timeout, description: 'Timeout duration of this job.', inherit: true @@ -139,7 +135,7 @@ module Gitlab attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, :release, + :timeout, :release, :allow_failure, :publish, :pages def self.matching?(name, config) @@ -169,7 +165,6 @@ module Gitlab coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, parallel: has_parallel? ? parallel_value : nil, - interruptible: interruptible_defined? ? interruptible_value : nil, timeout: parsed_timeout, artifacts: artifacts_value, release: release_value, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index d0e9a9afc51..0b322fd433c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,8 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs resource_group environment].freeze + inherit allow_failure when needs resource_group environment + interruptible].freeze MAX_NESTING_LEVEL = 10 included do @@ -74,6 +75,10 @@ module Gitlab description: 'Environment configuration for this job.', inherit: false + entry :interruptible, ::Gitlab::Config::Entry::Boolean, + description: 'Set jobs interruptible value.', + inherit: true + attributes :extends, :rules, :resource_group end @@ -133,7 +138,8 @@ module Gitlab except: except_value, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, - resource_group: resource_group }.compact + resource_group: resource_group, + interruptible: interruptible_defined? ? interruptible_value : nil }.compact end def root_variables_inheritance diff --git a/lib/gitlab/ci/config/entry/release.rb b/lib/gitlab/ci/config/entry/release.rb index 2be0eae120b..113e6fefd6a 100644 --- a/lib/gitlab/ci/config/entry/release.rb +++ b/lib/gitlab/ci/config/entry/release.rb @@ -16,12 +16,6 @@ module Gitlab attributes %i[tag_name tag_message name ref milestones assets].freeze attr_reader :released_at - # Attributable description conflicts with - # ::Gitlab::Config::Entry::Node.description - def has_description? - true - end - def description config[:description] end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 3c180674f2a..16755ac320c 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -17,7 +17,7 @@ module Gitlab dast performance browser_performance load_performance license_scanning metrics lsif dotenv terraform accessibility coverage_fuzzing api_fuzzing cluster_image_scanning - requirements requirements_v2 coverage_report cyclonedx annotations].freeze + requirements requirements_v2 coverage_report cyclonedx annotations repository_xray].freeze attributes ALLOWED_KEYS @@ -51,6 +51,7 @@ module Gitlab validates :requirements_v2, array_of_strings_or_string: true validates :cyclonedx, array_of_strings_or_string: true validates :annotations, array_of_strings_or_string: true + validates :repository_xray, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb index e9cbcb31e21..f225ed4caf4 100644 --- a/lib/gitlab/ci/config/entry/retry.rb +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -35,8 +35,8 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[max when].freeze - attributes :max, :when + ALLOWED_KEYS = %i[max when exit_codes].freeze + attributes ALLOWED_KEYS validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -53,6 +53,7 @@ module Gitlab validates :when, inclusion: { in: FullRetry.possible_retry_when_values }, if: -> (config) { config.when.is_a?(String) } + validates :exit_codes, array_of_integers_or_integer: true end end @@ -62,9 +63,14 @@ module Gitlab def value super.tap do |config| - # make sure that `when` is an array, because we allow it to - # be passed as a String in config for simplicity + # make sure that `when` and `exit_codes` are arrays, because we allow them to + # be passed as a String/Integer in config for simplicity config[:when] = Array.wrap(config[:when]) if config[:when] + if config[:exit_codes] && Feature.enabled?(:ci_retry_on_exit_codes, Feature.current_request) + config[:exit_codes] = Array.wrap(config[:exit_codes]) + else + config.delete(:exit_codes) + end end end diff --git a/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json b/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json new file mode 100644 index 00000000000..a31374650e6 --- /dev/null +++ b/lib/gitlab/ci/config/entry/schemas/imageable/executor_opts.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Describe `image:` and `service:` options like `docker:`", + "type": "object", + "properties": { + "docker": { + "type": "object", + "properties": { + "platform": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 4b3a9990df4..94fd28badb7 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -34,14 +34,14 @@ module Gitlab end def value - if string? - { name: @config } - elsif hash? - @config.merge( - pull_policy: pull_policy_value + if hash? + super.merge( + command: @config[:command], + alias: @config[:alias], + variables: (variables_value if variables_defined?) ).compact else - {} + super end end end diff --git a/lib/gitlab/ci/config/entry/workflow.rb b/lib/gitlab/ci/config/entry/workflow.rb index 691d9e2d48b..5b81c74fe4d 100644 --- a/lib/gitlab/ci/config/entry/workflow.rb +++ b/lib/gitlab/ci/config/entry/workflow.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[rules name].freeze + ALLOWED_KEYS = %i[rules name auto_cancel].freeze attributes :name @@ -23,6 +23,9 @@ module Gitlab description: 'List of evaluable Rules to determine Pipeline status.', metadata: { allowed_when: %w[always never] } + entry :auto_cancel, Entry::AutoCancel, + description: 'Auto-cancel configuration for this pipeline.' + def has_rules? @config.try(:key?, :rules) end diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index bc8cebb8c3e..fc90b497f85 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -14,9 +14,20 @@ module Gitlab super end + def preload_content + fetch_async_content + end + def content - strong_memoize(:content) { fetch_remote_content } + fetch_with_error_handling do + if fetch_async_content + fetch_async_content.value + else + fetch_sync_content + end + end end + strong_memoize_attr :content def metadata super.merge( @@ -42,11 +53,23 @@ module Gitlab private - def fetch_remote_content + def fetch_async_content + return if ::Feature.disabled?(:ci_parallel_remote_includes, context.project) + + # It starts fetching the remote content in a separate thread and returns a promise immediately. + Gitlab::HTTP.get(location, async: true).execute + end + strong_memoize_attr :fetch_async_content + + def fetch_sync_content + context.logger.instrument(:config_file_fetch_remote_content) do + Gitlab::HTTP.get(location) + end + end + + def fetch_with_error_handling begin - response = context.logger.instrument(:config_file_fetch_remote_content) do - Gitlab::HTTP.get(location) - end + response = yield rescue SocketError errors.push("Remote file `#{masked_location}` could not be fetched because of a socket error!") rescue Timeout::Error diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 0e296aa0b5b..3bb0df88803 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -25,7 +25,7 @@ module Gitlab file.preload_context if file.valid? end - # We do not combine the loops because we need to load the context of all files via `BatchLoader`. + # We do not combine the loops because we need to preload the context of all files via `BatchLoader`. files.each do |file| # rubocop:disable Style/CombinableLoops verify_execution_time! @@ -33,7 +33,8 @@ module Gitlab file.preload_content if file.valid? end - # We do not combine the loops because we need to load the content of all files via `BatchLoader`. + # We do not combine the loops because we need to preload the content of all files via `BatchLoader` + # or `Concurrent::Promise`. files.each do |file| # rubocop:disable Style/CombinableLoops verify_execution_time! diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb index 987268b0525..e506645df11 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb @@ -10,9 +10,8 @@ module Gitlab class BaseInput ArgumentNotValidError = Class.new(StandardError) - # Checks whether the class matches the type in the specification def self.matches?(spec) - raise NotImplementedError + spec.is_a?(Hash) && spec[:type] == type_name end # Human readable type used in error messages diff --git a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb index 4c34f7e7fdd..51845a2fea8 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb @@ -8,10 +8,6 @@ module Gitlab class BooleanInput < BaseInput extend ::Gitlab::Utils::Override - def self.matches?(spec) - spec.is_a?(Hash) && spec[:type] == type_name - end - def self.type_name 'boolean' end diff --git a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb index 59bc057749a..bb023a8a85b 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb @@ -8,10 +8,6 @@ module Gitlab class NumberInput < BaseInput extend ::Gitlab::Utils::Override - def self.matches?(spec) - spec.is_a?(Hash) && spec[:type] == type_name - end - def self.type_name 'number' end diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb index 01b9d34a883..3c4868b299c 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb @@ -17,7 +17,7 @@ module Gitlab # inputs: # foo: # ``` - spec.nil? || (spec.is_a?(Hash) && [nil, type_name].include?(spec[:type])) + spec.nil? || super || (spec.is_a?(Hash) && !spec.key?(:type)) end def self.type_name diff --git a/lib/gitlab/ci/config/interpolation/text_interpolator.rb b/lib/gitlab/ci/config/interpolation/text_interpolator.rb new file mode 100644 index 00000000000..5c4953f8bbe --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/text_interpolator.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + ## + # Performs CI config file interpolation and either returns the interpolated result or interpolation errors. + # + class TextInterpolator + attr_reader :errors + + def initialize(config, input_args, variables) + @config = config + @input_args = input_args.to_h + @variables = variables + @errors = [] + @interpolated = false + end + + def valid? + errors.none? + end + + def to_result + @result + end + + def error_message + # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown + # interpolation key: `abc`"] ? + # + # We are joining them together into a single one, because only one error can be surfaced when an external + # file gets included and is invalid. The limit to three error messages combined is more than required. + # + errors.first(3).join(', ') + end + + def interpolate! + return errors.push(config.error) unless config.valid? + + if inputs_without_header? + return errors.push( + _('Given inputs not defined in the `spec` section of the included configuration file')) + end + + return @result ||= config.content unless config.has_header? + + return errors.concat(header.errors) unless header.valid? + return errors.concat(inputs.errors) unless inputs.valid? + return errors.concat(context.errors) unless context.valid? + return errors.concat(template.errors) unless template.valid? + + @interpolated = true + + @result ||= template.interpolated + end + + def interpolated? + @interpolated + end + + private + + attr_reader :config, :input_args, :variables + + def inputs_without_header? + input_args.any? && !config.has_header? + end + + def header + @header ||= Header::Root.new(config.header).tap do |header| + header.key = 'header' + + header.compose! + end + end + + def content + @content ||= config.content + end + + def spec + @spec ||= header.inputs_value + end + + def inputs + @inputs ||= Inputs.new(spec, input_args) + end + + def context + @context ||= Context.new({ inputs: inputs.to_hash }, variables: variables) + end + + def template + @template ||= TextTemplate.new(content, context) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/text_template.rb b/lib/gitlab/ci/config/interpolation/text_template.rb new file mode 100644 index 00000000000..e1f5d368e88 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/text_template.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + class TextTemplate + MAX_BLOCKS = 10_000 + + def initialize(content, ctx) + @content = content + @ctx = Interpolation::Context.fabricate(ctx) + @errors = [] + @blocks = {} + + interpolate! if valid? + end + + def valid? + errors.none? + end + + def errors + @errors + ctx.errors + blocks.values.flat_map(&:errors) + end + + def interpolated + @result if valid? + end + + private + + attr_reader :blocks, :content, :ctx + + def interpolate! + return @errors.push('config too large') if content.bytesize > max_total_yaml_size_bytes + + @result = Interpolation::Block.match(content) do |matched, data| + block = (blocks[matched] ||= Interpolation::Block.new(matched, data, ctx)) + + break @errors.push('too many interpolation blocks') if blocks.count > MAX_BLOCKS + break unless block.valid? + + if block.value.is_a?(String) + block.value + else + block.value.to_json + end + end + end + + def max_total_yaml_size_bytes + Gitlab::CurrentSettings.current_application_settings.ci_max_total_yaml_size_bytes + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index 1e5200e8682..79c1c14dc4e 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -5,8 +5,6 @@ module Gitlab module Parsers module Sbom class Cyclonedx - SUPPORTED_SPEC_VERSIONS = %w[1.4].freeze - def parse!(blob, sbom_report) @report = sbom_report @data = Gitlab::Json.parse(blob) @@ -27,18 +25,7 @@ module Gitlab end def valid? - valid_schema? && supported_spec_version? - end - - def supported_spec_version? - return true if SUPPORTED_SPEC_VERSIONS.include?(data['specVersion']) - - report.add_error( - "Unsupported CycloneDX spec version. Must be one of: %{versions}" \ - % { versions: SUPPORTED_SPEC_VERSIONS.join(', ') } - ) - - false + valid_schema? end def valid_schema? diff --git a/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb index 9d56e001c2f..a8d3ef1d6b5 100644 --- a/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb +++ b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb @@ -6,7 +6,9 @@ module Gitlab module Sbom module Validators class CyclonedxSchemaValidator - SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'cyclonedx_report.json').freeze + SUPPORTED_SPEC_VERSIONS = %w[1.4 1.5].freeze + + SCHEMA_BASE_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'cyclonedx').freeze def initialize(report_data) @report_data = report_data @@ -17,13 +19,30 @@ module Gitlab end def errors - @errors ||= pretty_errors + @errors ||= validate! end private + def validate! + if spec_version_valid? + pretty_errors + else + [format("Unsupported CycloneDX spec version. Must be one of: %{versions}", + versions: SUPPORTED_SPEC_VERSIONS.join(', '))] + end + end + + def spec_version_valid? + SUPPORTED_SPEC_VERSIONS.include?(spec_version) + end + + def spec_version + @report_data['specVersion'] + end + def raw_errors - JSONSchemer.schema(SCHEMA_PATH).validate(@report_data) + JSONSchemer.schema(SCHEMA_BASE_PATH.join("bom-#{spec_version}.schema.json")).validate(@report_data) end def pretty_errors diff --git a/lib/gitlab/ci/pipeline/chain/assign_partition.rb b/lib/gitlab/ci/pipeline/chain/assign_partition.rb index 4b8efe13d44..0740226ac9b 100644 --- a/lib/gitlab/ci/pipeline/chain/assign_partition.rb +++ b/lib/gitlab/ci/pipeline/chain/assign_partition.rb @@ -21,7 +21,7 @@ module Gitlab if @command.creates_child_pipeline? @command.parent_pipeline_partition_id else - ::Ci::Pipeline.current_partition_value + ::Ci::Pipeline.current_partition_value(project) end end end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index dcaaefee98f..14ec86c5d62 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -6,7 +6,11 @@ module Gitlab module Chain class CancelPendingPipelines < Chain::Base def perform! - ::Ci::CancelRedundantPipelinesWorker.perform_async(pipeline.id) + if pipeline.schedule? + ::Ci::LowUrgencyCancelRedundantPipelinesWorker.perform_async(pipeline.id) + else + ::Ci::CancelRedundantPipelinesWorker.perform_async(pipeline.id) + end end def break? diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index d4c4f94c7d3..d1153a0990e 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -34,7 +34,7 @@ module Gitlab pipeline .stages .flat_map(&:statuses) - .select { |status| status.respond_to?(:tag_list) } + .select { |status| status.respond_to?(:tag_list=) } end end end diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index cceaa52de16..ab37eb93f18 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -11,7 +11,16 @@ module Gitlab def perform! @command.workflow_rules_result = workflow_rules_result - error('Pipeline filtered out by workflow rules.') unless workflow_passed? + return if workflow_passed? + + if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project) + drop_reason = :filtered_by_workflow_rules + end + + error( + 'Pipeline filtered out by workflow rules.', + drop_reason: drop_reason + ) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 343a189f773..0e55928ff80 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -35,7 +35,7 @@ module Gitlab def drop_pipeline!(drop_reason) return if pipeline.readonly? - if drop_reason && command.save_incompleted + if Enums::Ci::Pipeline.persistable_failure_reason?(drop_reason) && command.save_incompleted # Project iid must be called outside a transaction, so we ensure it is set here # otherwise it may be set within the state transition transaction of the drop! call # which it will lock the InternalId row for the whole transaction @@ -44,6 +44,8 @@ module Gitlab pipeline.drop!(drop_reason) else command.increment_pipeline_failure_reason_counter(drop_reason) + + pipeline.set_failed(drop_reason) if Feature.enabled?(:always_set_pipeline_failure_reason, command.project) end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index c59ef2ba6a4..f73addcd098 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -18,8 +18,15 @@ module Gitlab pipeline.stages = @command.pipeline_seed.stages if stage_names.empty? - return error('Pipeline will not run for the selected trigger. ' \ - 'The rules configuration prevented any jobs from being added to the pipeline.') + if Feature.enabled?(:always_set_pipeline_failure_reason, @command.project) + drop_reason = :filtered_by_rules + end + + return error( + 'Pipeline will not run for the selected trigger. ' \ + 'The rules configuration prevented any jobs from being added to the pipeline.', + drop_reason: drop_reason + ) end if pipeline.invalid? diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb index e7a9009f8f4..3ac910da752 100644 --- a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -9,6 +9,8 @@ module Gitlab def perform! set_pipeline_name + set_auto_cancel + return if pipeline.pipeline_metadata.nil? || pipeline.pipeline_metadata.valid? message = pipeline.pipeline_metadata.errors.full_messages.join(', ') @@ -29,13 +31,45 @@ module Gitlab return if name.blank? - pipeline.build_pipeline_metadata(project: pipeline.project, name: name.strip) + assign_to_metadata(name: name.strip) + end + + def set_auto_cancel + auto_cancel = @command.yaml_processor_result.workflow_auto_cancel + + return if auto_cancel.blank? + + set_auto_cancel_on_new_commit(auto_cancel) + set_auto_cancel_on_job_failure(auto_cancel) + end + + def set_auto_cancel_on_new_commit(auto_cancel) + auto_cancel_on_new_commit = auto_cancel[:on_new_commit] + + return if auto_cancel_on_new_commit.blank? + + assign_to_metadata(auto_cancel_on_new_commit: auto_cancel_on_new_commit) + end + + def set_auto_cancel_on_job_failure(auto_cancel) + return if Feature.disabled?(:auto_cancel_pipeline_on_job_failure, pipeline.project) + + auto_cancel_on_job_failure = auto_cancel[:on_job_failure] + + return if auto_cancel_on_job_failure.blank? + + assign_to_metadata(auto_cancel_on_job_failure: auto_cancel_on_job_failure) end def global_context Gitlab::Ci::Build::Context::Global.new( pipeline, yaml_variables: @command.pipeline_seed.root_variables) end + + def assign_to_metadata(attributes) + metadata = pipeline.pipeline_metadata || pipeline.build_pipeline_metadata(project: pipeline.project) + metadata.assign_attributes(attributes) + end end end end diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index b7af6ea17c3..7d284b5babf 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -5,28 +5,14 @@ module Gitlab module Reports module Sbom class Source + include SourceHelper + attr_reader :source_type, :data def initialize(type:, data:) @source_type = type @data = data end - - def source_file_path - data.dig('source_file', 'path') - end - - def input_file_path - data.dig('input_file', 'path') - end - - def packager - data.dig('package_manager', 'name') - end - - def language - data.dig('language', 'name') - end end end end diff --git a/lib/gitlab/ci/reports/sbom/source_helper.rb b/lib/gitlab/ci/reports/sbom/source_helper.rb new file mode 100644 index 00000000000..49b606f658b --- /dev/null +++ b/lib/gitlab/ci/reports/sbom/source_helper.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Sbom + module SourceHelper + def source_file_path + data.dig('source_file', 'path') + end + + def input_file_path + data.dig('input_file', 'path') + end + + def packager + data.dig('package_manager', 'name') + end + + def language + data.dig('language', 'name') + end + + def image_name + data.dig('image', 'name') + end + + def image_tag + data.dig('image', 'tag') + end + + def operating_system_name + data.dig('operating_system', 'name') + end + + def operating_system_version + data.dig('operating_system', 'version') + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml b/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml new file mode 100644 index 00000000000..c8b3aa1d705 --- /dev/null +++ b/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml @@ -0,0 +1,88 @@ +# This template is provided and maintained by Diffblue. +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# This template is designed to be used with the Cover Pipeline for GitLab integration from Diffblue. +# It will download the latest version of Diffblue Cover, build the associated project, and +# automatically write Java unit tests for the project. +# Note that additional config is required: +# https://docs.diffblue.com/features/cover-pipeline/cover-pipeline-for-gitlab +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Diffblue-Cover.gitlab-ci.yml + +variables: + # Configure the following via the Diffblue Cover integration config for your project, or by + # using CI/CD masked Variables. + # For details, see https://docs.diffblue.com/features/cover-pipeline/cover-pipeline-for-gitlab + + # Diffblue Cover license key: DIFFBLUE_LICENSE_KEY + # Refer to your welcome email or you can obtain a free trial key from + # https://www.diffblue.com/try-cover/gitlab + + # GitLab access token: DIFFBLUE_ACCESS_TOKEN, DIFFBLUE_ACCESS_TOKEN_NAME + # The access token should have a role of Developer or better and should have + # api and write_repository permissions. + + # Diffblue Cover requires a minimum of 4GB of memory. + JVM_ARGS: -Xmx4g + +stages: + - build + +diffblue-cover: + stage: build + + # Select the Cover CLI docker image to use with your CI tool. + # Tag variations are produced for each supported JDK version. + # Go to https://hub.docker.com/r/diffblue/cover-cli for details. + # Note: To use the latest version of Diffblue Cover, use one of the latest-jdk<nn> tags. + # To use a specific release version, use one of the yyyy.mm.dd-jdk<nn> tags. + image: diffblue/cover-cli:latest-jdk17 + + # Diffblue Cover currently only supports running on merge_request_events. + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + + # Diffblue Cover log files are saved to a .diffblue/ directory in the pipeline artifacts, + # and are available for download once the pipeline completes. + artifacts: + paths: + - "**/.diffblue/" + + script: + + # Diffblue Cover requires the project to be built before creating any tests. + # Either specify the build command here (one of the following), or provide + # prebuilt artifacts via a job dependency. + + # Maven project example (comment out the Gradle version if used): + - mvn test-compile --batch-mode --no-transfer-progress + + # Gradle project example (comment out the Maven version if used): + # - gradle testClasses + + # Diffblue Cover commands and options to run. + # dcover – the core Diffblue Cover command + # ci – enable the GitLab CI/CD integration via environment variables + # activate - activate the license key + # validate - remove non-compiling and failing tests + # create - create new tests for your project + # --maven – use the maven build tool + # For detailed information on Cover CLI commands and options, see + # https://docs.diffblue.com/features/cover-cli/commands-and-arguments + - dcover + ci + activate + validate --maven + create --maven + + # Diffblue Cover will also respond to specific project labels: + # Diffblue Cover: Baseline + # Used to mark a merge request as requiring a full suite of tests to be written. + # This overrides the default behaviour where Cover will only write tests related + # to the code changes already in the merge request. This is useful when running Diffblue + # Cover for the first time on a project and when new product enhancements are released. + # Diffblue Cover: Skip + # Used to mark a merge request as requiring no tests to be written. diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 6898923bc53..111df0af67a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.51.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 6898923bc53..111df0af67a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.51.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7d923245d79..a5cddf5d2d7 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 0f8d5bf6d8f..0a899f3bb74 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e29d18ea45a..87a7f79c0ce 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.71.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 488b035d189..c698bd49140 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/accessibility_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/accessibility_testing.html stages: - build - test diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index c279af6acfc..a1c6437bf84 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -34,6 +34,25 @@ module Gitlab end end + def unprotected_scoped_variables(job, expose_project_variables:, expose_group_variables:, environment:, dependencies:) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + variables.concat(predefined_variables(job, environment)) + variables.concat(project.predefined_variables) + variables.concat(pipeline_variables_builder.predefined_variables) + variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner + variables.concat(kubernetes_variables(environment: environment, job: job)) + variables.concat(job.yaml_variables) + variables.concat(user_variables(job.user)) + variables.concat(job.dependency_variables) if dependencies + variables.concat(secret_instance_variables) + variables.concat(secret_group_variables(environment: environment, include_protected_vars: expose_group_variables)) + variables.concat(secret_project_variables(environment: environment, include_protected_vars: expose_project_variables)) + variables.concat(pipeline.variables) + variables.concat(pipeline_schedule_variables) + variables.concat(release_variables) + end + end + def config_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless project @@ -91,21 +110,21 @@ module Gitlab end end - def secret_group_variables(environment:) - strong_memoize_with(:secret_group_variables, environment) do + def secret_group_variables(environment:, include_protected_vars: protected_ref?) + strong_memoize_with(:secret_group_variables, environment, include_protected_vars) do group_variables_builder .secret_variables( environment: environment, - protected_ref: protected_ref?) + protected_ref: include_protected_vars) end end - def secret_project_variables(environment:) - strong_memoize_with(:secret_project_variables, environment) do + def secret_project_variables(environment:, include_protected_vars: protected_ref?) + strong_memoize_with(:secret_project_variables, environment, include_protected_vars) do project_variables_builder .secret_variables( environment: environment, - protected_ref: protected_ref?) + protected_ref: include_protected_vars) end end @@ -183,3 +202,5 @@ module Gitlab end end end + +Gitlab::Ci::Variables::Builder.prepend_mod_with('Gitlab::Ci::Variables::Builder') diff --git a/lib/gitlab/ci/variables/downstream/generator.rb b/lib/gitlab/ci/variables/downstream/generator.rb index 350d29958cf..e1fd8200dd6 100644 --- a/lib/gitlab/ci/variables/downstream/generator.rb +++ b/lib/gitlab/ci/variables/downstream/generator.rb @@ -5,8 +5,6 @@ module Gitlab module Variables module Downstream class Generator - include Gitlab::Utils::StrongMemoize - Context = Struct.new(:all_bridge_variables, :expand_file_refs, keyword_init: true) def initialize(bridge) @@ -33,6 +31,7 @@ module Gitlab # The order of this list refers to the priority of the variables # The variables added later takes priority. downstream_yaml_variables + + downstream_pipeline_dotenv_variables + downstream_pipeline_variables + downstream_pipeline_schedule_variables end @@ -57,6 +56,13 @@ module Gitlab build_downstream_variables_from(pipeline_schedule_variables) end + def downstream_pipeline_dotenv_variables + return [] unless bridge.forward_pipeline_variables? + + pipeline_dotenv_variables = bridge.dependency_variables.to_a + build_downstream_variables_from(pipeline_dotenv_variables) + end + def build_downstream_variables_from(variables) Gitlab::Ci::Variables::Collection.fabricate(variables).flat_map do |item| if item.raw? diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 2435d128bf2..5933b537098 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -8,7 +8,8 @@ module Gitlab class Result attr_reader :errors, :warnings, :root_variables, :root_variables_with_prefill_data, - :stages, :jobs, :workflow_rules, :workflow_name + :stages, :jobs, + :workflow_rules, :workflow_name, :workflow_auto_cancel def initialize(ci_config: nil, errors: [], warnings: []) @ci_config = ci_config @@ -71,6 +72,7 @@ module Gitlab @workflow_rules = @ci_config.workflow_rules @workflow_name = @ci_config.workflow_name&.strip + @workflow_auto_cancel = @ci_config.workflow_auto_cancel end def stage_builds_attributes(stage) diff --git a/lib/gitlab/circuit_breaker.rb b/lib/gitlab/circuit_breaker.rb new file mode 100644 index 00000000000..2b3a6187f27 --- /dev/null +++ b/lib/gitlab/circuit_breaker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# A configurable circuit breaker to protect the application from external service failures. +# The circuit measures the amount of failures and if the threshold is exceeded, stops sending requests. +module Gitlab + module CircuitBreaker + InternalServerError = Class.new(StandardError) + + DEFAULT_ERROR_THRESHOLD = 50 + DEFAULT_VOLUME_THRESHOLD = 10 + + class << self + include ::Gitlab::Utils::StrongMemoize + + # @param [String] unique name for the circuit + # @param options [Hash] an options hash setting optional values per circuit + def run_with_circuit(service_name, options = {}, &block) + circuit(service_name, options).run(exception: false, &block) + end + + private + + def circuit(service_name, options) + strong_memoize_with(:circuit, service_name, options) do + circuit_options = { + exceptions: [InternalServerError], + error_threshold: DEFAULT_ERROR_THRESHOLD, + volume_threshold: DEFAULT_VOLUME_THRESHOLD + }.merge(options) + + Circuitbox.circuit(service_name, circuit_options) + end + end + end + end +end diff --git a/lib/gitlab/circuit_breaker/notifier.rb b/lib/gitlab/circuit_breaker/notifier.rb new file mode 100644 index 00000000000..b555158ee48 --- /dev/null +++ b/lib/gitlab/circuit_breaker/notifier.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module CircuitBreaker + class Notifier + CircuitBreakerError = Class.new(RuntimeError) + + def notify(service_name, event) + return unless event == 'failure' + + exception = CircuitBreakerError.new("Service #{service_name}: #{event}") + exception.set_backtrace(Gitlab::BacktraceCleaner.clean_backtrace(caller)) + + Gitlab::ErrorTracking.track_exception(exception) + end + + def notify_warning(_service_name, _message) + # no-op + end + + def notify_run(_service_name, &_block) + # This gets called by Circuitbox::CircuitBreaker#run to actually execute + # the block passed. + yield + end + end + end +end diff --git a/lib/gitlab/circuit_breaker/store.rb b/lib/gitlab/circuit_breaker/store.rb new file mode 100644 index 00000000000..0ba4f08d5e1 --- /dev/null +++ b/lib/gitlab/circuit_breaker/store.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module CircuitBreaker + class Store + def key?(key) + with { |redis| redis.exists?(key) } + end + + def store(key, value, opts = {}) + with do |redis| + redis.set(key, value, ex: opts[:expires]) + value + end + end + + def increment(key, amount = 1, opts = {}) + expires = opts[:expires] + + with do |redis| + redis.multi do |multi| + multi.incrby(key, amount) + multi.expire(key, expires) if expires + end + end + end + + def load(key, _opts = {}) + with { |redis| redis.get(key) } + end + + def values_at(*keys, **_opts) + keys.map! { |key| load(key) } + end + + def delete(key) + with { |redis| redis.del(key) } + end + + private + + def with(&block) + Gitlab::Redis::RateLimiting.with(&block) + rescue ::Redis::BaseConnectionError + # Do not raise an error if we cannot connect to Redis. If + # Redis::RateLimiting is unavailable it should not take the site down. + nil + end + end + end +end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 6feaab2a791..918f723cd60 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -122,7 +122,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def project_id - @project_id ||= Project.where_full_path_in([full_path]).pluck(:id) + @project_id ||= Project.where_full_path_in([full_path], use_includes: false).pluck(:id) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index e293e5653c7..3df0ec76839 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -12,11 +12,11 @@ module Gitlab end def self.frame_src - "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com https://www.googletagmanager.com/ns.html" + "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://www.googletagmanager.com/ns.html" end def self.script_src - "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com" + "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net" end def self.style_src diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 2068a9ae7d5..a0bb37fb097 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -3,6 +3,7 @@ module Gitlab class ContributionsCalendar include TimeZoneHelper + include ::Gitlab::Utils::StrongMemoize attr_reader :contributor attr_reader :current_user @@ -16,93 +17,89 @@ module Gitlab .execute(current_user, ignore_visibility: @contributor.include_private_contributions?) end - # rubocop: disable CodeReuse/ActiveRecord def activity_dates - return {} if @projects.empty? - return @activity_dates if @activity_dates.present? + return {} if projects.empty? start_time = @contributor_time_instance.years_ago(1).beginning_of_day end_time = @contributor_time_instance.end_of_day date_interval = "INTERVAL '#{@contributor_time_instance.utc_offset} seconds'" - # Can't use Event.contributions here because we need to check 3 different - # project_features for the (currently) 3 different contribution types - repo_events = events_created_between(start_time, end_time, :repository) - .where(action: :pushed) - issue_events = events_created_between(start_time, end_time, :issues) - .where(action: [:created, :closed], target_type: %w[Issue WorkItem]) - mr_events = events_created_between(start_time, end_time, :merge_requests) - .where(action: [:merged, :created, :closed], target_type: "MergeRequest") - note_events = events_created_between(start_time, end_time, :merge_requests) - .where(action: :commented) - - events = Event - .select("date(created_at + #{date_interval}) AS date", 'COUNT(*) AS num_events') - .from_union([repo_events, issue_events, mr_events, note_events], remove_duplicates: false) - .group(:date) - .map(&:attributes) - - @activity_dates = events.each_with_object(Hash.new { |h, k| h[k] = 0 }) do |event, activities| - activities[event["date"]] += event["num_events"] - end + contributions_between(start_time, end_time).count_by_dates(date_interval) end - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def events_by_date(date) return Event.none unless can_read_cross_project? date_in_time_zone = date.in_time_zone(@contributor_time_instance.time_zone) - Event.contributions.where(author_id: contributor.id) - .where(created_at: date_in_time_zone.beginning_of_day..date_in_time_zone.end_of_day) - .where(project_id: projects) - .with_associations + contributions_between(date_in_time_zone.beginning_of_day, date_in_time_zone.end_of_day).with_associations end - # rubocop: enable CodeReuse/ActiveRecord - def starting_year - @contributor_time_instance.years_ago(1).year - end + private - def starting_month - @contributor_time_instance.month + def contributions_between(start_time, end_time) + # Can't use Event.contributions here because we need to check 3 different + # project_features for the (currently) 4 different contribution types + repo_events = + project_events_created_between(start_time, end_time, features: :repository) + .for_action(:pushed) + + issue_events = + project_events_created_between(start_time, end_time, features: :issues) + .for_issue + .for_action(%i[created closed]) + + mr_events = + project_events_created_between(start_time, end_time, features: :merge_requests) + .for_merge_request + .for_action(%i[merged created closed approved]) + + note_events = + project_events_created_between(start_time, end_time, features: %i[issues merge_requests]) + .for_action(:commented) + + Event.from_union([repo_events, issue_events, mr_events, note_events], remove_duplicates: false) end - private - def can_read_cross_project? Ability.allowed?(current_user, :read_cross_project) end - # rubocop: disable CodeReuse/ActiveRecord - def events_created_between(start_time, end_time, feature) + # rubocop: disable CodeReuse/ActiveRecord -- no need to move this to ActiveRecord model + def project_events_created_between(start_time, end_time, features:) + Array(features).reduce(Event.none) do |events, feature| + events.or(contribution_events(start_time, end_time).where(project_id: authed_projects(feature))) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def authed_projects(feature) + strong_memoize("#{feature}_projects") do + # no need to check features access of current user, if the contributor opted-in + # to show all private events anyway - otherwise they would get filtered out again + next contributed_project_ids if contributor.include_private_contributions? + + # rubocop: disable CodeReuse/ActiveRecord -- no need to move this to ActiveRecord model + ProjectFeature + .with_feature_available_for_user(feature, current_user) + .where(project_id: contributed_project_ids) + .pluck(:project_id) + # rubocop: enable CodeReuse/ActiveRecord + end + end + + # rubocop: disable CodeReuse/ActiveRecord -- no need to move this to ActiveRecord model + def contributed_project_ids # re-running the contributed projects query in each union is expensive, so # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short @contributed_project_ids ||= projects.distinct.pluck(:id) - - # no need to check feature access of current user, if the contributor opted-in - # to show all private events anyway - otherwise they would get filtered out again - authed_projects = if @contributor.include_private_contributions? - @contributed_project_ids - else - ProjectFeature - .with_feature_available_for_user(feature, current_user) - .where(project_id: @contributed_project_ids) - .reorder(nil) - .select(:project_id) - end - - Event.reorder(nil) - .select(:created_at) - .where( - author_id: contributor.id, - created_at: start_time..end_time, - events: { project_id: authed_projects } - ) end # rubocop: enable CodeReuse/ActiveRecord + + def contribution_events(start_time, end_time) + contributor.events.created_between(start_time, end_time) + end end end diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb index 258ada864c8..9d704f5613c 100644 --- a/lib/gitlab/counters/buffered_counter.rb +++ b/lib/gitlab/counters/buffered_counter.rb @@ -248,7 +248,7 @@ module Gitlab end def redis_state(&block) - Gitlab::Redis::SharedState.with(&block) + Gitlab::Redis::BufferedCounter.with(&block) end def with_exclusive_lease(&block) diff --git a/lib/gitlab/database/background_migration/batched_background_migration_dictionary.rb b/lib/gitlab/database/background_migration/batched_background_migration_dictionary.rb new file mode 100644 index 00000000000..a6efa09afda --- /dev/null +++ b/lib/gitlab/database/background_migration/batched_background_migration_dictionary.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchedBackgroundMigrationDictionary + def self.entry(migration_job_name) + entries_by_migration_job_name[migration_job_name] + end + + private_class_method def self.entries_by_migration_job_name + @entries_by_migration_job_name ||= Dir.glob(dict_path).to_h do |file_path| + entry = Entry.new(file_path) + [entry.migration_job_name, entry] + end + end + + private_class_method def self.dict_path + Rails.root.join('db/docs/batched_background_migrations/*.yml') + end + + class Entry + def initialize(file_path) + @file_path = file_path + @data = YAML.load_file(file_path) + end + + def migration_job_name + data['migration_job_name'] + end + + def finalized_by + data['finalized_by'] + end + + private + + attr_reader :file_path, :data + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 83beee091f1..d0655fa4564 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -264,6 +264,13 @@ module Gitlab 100 * migrated_tuple_count / total_tuple_count end + def finalize_command + <<~SCRIPT.delete("\n").squeeze(' ').strip + sudo gitlab-rake gitlab:background_migrations:finalize + [#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}'] + SCRIPT + end + private def validate_batched_jobs_status diff --git a/lib/gitlab/database/decomposition/migrate.rb b/lib/gitlab/database/decomposition/migrate.rb new file mode 100644 index 00000000000..b6ca5adf857 --- /dev/null +++ b/lib/gitlab/database/decomposition/migrate.rb @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Decomposition + MigrateError = Class.new(RuntimeError) + + class Migrate + TABLE_SIZE_QUERY = <<-SQL + select sum(pg_table_size(concat(table_schema,'.',table_name))) as total + from information_schema.tables + where table_catalog = :table_catalog and table_type = 'BASE TABLE' + SQL + + TABLE_COUNT_QUERY = <<-SQL + select count(*) as total + from information_schema.tables + where table_catalog = :table_catalog and table_type = 'BASE TABLE' + and table_schema not in ('information_schema', 'pg_catalog') + SQL + + DISKSPACE_HEADROOM_FACTOR = 1.25 + + attr_reader :backup_location + + def initialize(backup_base_location: nil) + random_post_fix = SecureRandom.alphanumeric(10) + @backup_base_location = backup_base_location || Gitlab.config.backup.path + @backup_location = File.join(@backup_base_location, "migration_#{random_post_fix}") + end + + def process! + return unless can_migrate? + + dump_main_db + import_dump_to_ci_db + + FileUtils.remove_entry_secure(@backup_location, true) + end + + private + + def valid_backup_location? + FileUtils.mkdir_p(@backup_base_location) + + true + rescue StandardError => e + raise MigrateError, "Failed to create directory #{@backup_base_location}: #{e.message}" + end + + def main_table_sizes + ApplicationRecord.connection.execute( + ApplicationRecord.sanitize_sql([ + TABLE_SIZE_QUERY, + { table_catalog: main_config.dig(:activerecord, :database) } + ]) + ).first["total"].to_f + end + + def diskspace_free + Sys::Filesystem.stat( + File.expand_path("#{@backup_location}/../") + ).bytes_free + end + + def required_diskspace_available? + needed = main_table_sizes * DISKSPACE_HEADROOM_FACTOR + available = diskspace_free + + if needed > available + raise MigrateError, + "Not enough diskspace available on #{@backup_location}: " \ + "Available: #{ActiveSupport::NumberHelper.number_to_human_size(available)}, " \ + "Needed: #{ActiveSupport::NumberHelper.number_to_human_size(needed)}" + end + + true + end + + def single_database_setup? + if Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES + raise MigrateError, "GitLab is already configured to run on multiple databases" + end + + true + end + + def ci_database_connect_ok? + _, status = with_transient_pg_env(ci_config[:pg_env]) do + psql_args = ["--dbname=#{ci_database_name}", "-tAc", "select 1"] + + Open3.capture2e('psql', *psql_args) + end + + unless status.success? + raise MigrateError, + "Can't connect to database '#{ci_database_name} on host '#{ci_config[:pg_env]['PGHOST']}'. " \ + "Ensure the database has been created." + end + + true + end + + def ci_database_empty? + sql = ApplicationRecord.sanitize_sql([ + TABLE_COUNT_QUERY, + { table_catalog: ci_database_name } + ]) + + output, status = with_transient_pg_env(ci_config[:pg_env]) do + psql_args = ["--dbname=#{ci_database_name}", "-tAc", sql] + + Open3.capture2e('psql', *psql_args) + end + + unless status.success? && output.chomp.to_i == 0 + raise MigrateError, + "Database '#{ci_database_name}' is not empty" + end + + true + end + + def background_migrations_done? + unfinished_count = Gitlab::Database::BackgroundMigration::BatchedMigration.without_status(:finished).count + if unfinished_count > 0 + raise MigrateError, + "Found #{unfinished_count} unfinished Background Migration(s). Please wait until they are finished." + end + + true + end + + def can_migrate? + valid_backup_location? && + single_database_setup? && + ci_database_connect_ok? && + ci_database_empty? && + required_diskspace_available? && + background_migrations_done? + end + + def with_transient_pg_env(extended_env) + ENV.merge!(extended_env) + result = yield + ENV.reject! { |k, _| extended_env.key?(k) } + + result + end + + def import_dump_to_ci_db + with_transient_pg_env(ci_config[:pg_env]) do + restore_args = ["--jobs=4", "--dbname=#{ci_database_name}"] + + Open3.capture2e('pg_restore', *restore_args, @backup_location) + end + end + + def dump_main_db + with_transient_pg_env(main_config[:pg_env]) do + args = ['--format=d', '--jobs=4', "--file=#{@backup_location}"] + + Open3.capture2e('pg_dump', *args, main_config.dig(:activerecord, :database)) + end + end + + def main_config + @main_config ||= ::Backup::DatabaseModel.new('main').config + end + + def ci_config + @ci_config ||= ::Backup::DatabaseModel.new('ci').config + end + + def ci_database_name + @ci_database_name ||= "#{main_config.dig(:activerecord, :database)}_ci" + end + end + end + end +end diff --git a/lib/gitlab/database/dictionary.rb b/lib/gitlab/database/dictionary.rb index 7b0c8560a26..4ef392a4e44 100644 --- a/lib/gitlab/database/dictionary.rb +++ b/lib/gitlab/database/dictionary.rb @@ -3,57 +3,99 @@ module Gitlab module Database class Dictionary - def initialize(file_path) - @file_path = file_path - @data = YAML.load_file(file_path) + def self.entries(scope = '') + @entries ||= {} + @entries[scope] ||= Dir.glob(dictionary_path_globs(scope)).map do |file_path| + dictionary = Entry.new(file_path) + dictionary.validate! + dictionary + end end - def name_and_schema - [key_name, gitlab_schema.to_sym] + def self.entry(name, scope = '') + entries(scope).find do |entry| + entry.key_name == name + end end - def table_name - data['table_name'] + private_class_method def self.dictionary_path_globs(scope) + dictionary_paths.map { |path| Rails.root.join(path, scope, '*.yml') } end - def view_name - data['view_name'] + private_class_method def self.dictionary_paths + ::Gitlab::Database.all_database_connections + .values.map(&:db_docs_dir).uniq end - def milestone - data['milestone'] - end + class Entry + def initialize(file_path) + @file_path = file_path + @data = YAML.load_file(file_path) + end - def gitlab_schema - data['gitlab_schema'] - end + def name_and_schema + [key_name, gitlab_schema.to_sym] + end - def schema?(schema_name) - gitlab_schema == schema_name.to_s - end + def table_name + data['table_name'] + end - def key_name - table_name || view_name - end + def feature_categories + data['feature_categories'] + end - def validate! - return true unless gitlab_schema.nil? + def view_name + data['view_name'] + end - raise( - GitlabSchema::UnknownSchemaError, - "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ - "See #{help_page_url}" - ) - end + def milestone + data['milestone'] + end + + def gitlab_schema + data['gitlab_schema'] + end + + def sharding_key + data['sharding_key'] + end + + def desired_sharding_key + data['desired_sharding_key'] + end + + def classes + data['classes'] + end + + def schema?(schema_name) + gitlab_schema == schema_name.to_s + end + + def key_name + table_name || view_name + end + + def validate! + return true unless gitlab_schema.nil? + + raise( + GitlabSchema::UnknownSchemaError, + "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ + "See #{help_page_url}" + ) + end - private + private - attr_reader :file_path, :data + attr_reader :file_path, :data - def help_page_url - # rubocop:disable Gitlab/DocUrl -- link directly to docs.gitlab.com, always - 'https://docs.gitlab.com/ee/development/database/database_dictionary.html' - # rubocop:enable Gitlab/DocUrl + def help_page_url + # rubocop:disable Gitlab/DocUrl -- link directly to docs.gitlab.com, always + 'https://docs.gitlab.com/ee/development/database/database_dictionary.html' + # rubocop:enable Gitlab/DocUrl + end end end end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index ecb45622061..e6f7dbec69c 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -88,6 +88,10 @@ module Gitlab # rubocop:enable Gitlab/DocUrl end + def self.cell_local?(schema) + Gitlab::Database.all_gitlab_schemas[schema.to_s].cell_local + end + def self.cross_joins_allowed?(table_schemas, all_tables) return true unless table_schemas.many? @@ -121,15 +125,6 @@ module Gitlab end end - def self.dictionary_paths - Gitlab::Database.all_database_connections - .values.map(&:db_docs_dir).uniq - end - - def self.dictionary_path_globs(scope) - self.dictionary_paths.map { |path| Rails.root.join(path, scope, '*.yml') } - end - def self.views_and_tables_to_schema @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end @@ -139,32 +134,24 @@ module Gitlab end def self.deleted_tables_to_schema - @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').map(&:name_and_schema).to_h + @deleted_tables_to_schema ||= ::Gitlab::Database::Dictionary.entries('deleted_tables').map(&:name_and_schema).to_h end def self.deleted_views_to_schema - @deleted_views_to_schema ||= self.build_dictionary('deleted_views').map(&:name_and_schema).to_h + @deleted_views_to_schema ||= ::Gitlab::Database::Dictionary.entries('deleted_views').map(&:name_and_schema).to_h end def self.tables_to_schema - @tables_to_schema ||= self.build_dictionary('').map(&:name_and_schema).to_h + @tables_to_schema ||= ::Gitlab::Database::Dictionary.entries.map(&:name_and_schema).to_h end def self.views_to_schema - @views_to_schema ||= self.build_dictionary('views').map(&:name_and_schema).to_h + @views_to_schema ||= ::Gitlab::Database::Dictionary.entries('views').map(&:name_and_schema).to_h end def self.schema_names @schema_names ||= self.views_and_tables_to_schema.values.to_set end - - def self.build_dictionary(scope) - Dir.glob(dictionary_path_globs(scope)).map do |file_path| - dictionary = Dictionary.new(file_path) - dictionary.validate! - dictionary - end - end end end end diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb index 20d2b31a65c..b7ec3dfc893 100644 --- a/lib/gitlab/database/gitlab_schema_info.rb +++ b/lib/gitlab/database/gitlab_schema_info.rb @@ -14,6 +14,7 @@ module Gitlab :allow_cross_transactions, :allow_cross_foreign_keys, :file_path, + :cell_local, keyword_init: true ) do def initialize(*) diff --git a/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb index 6bf2bbf0c70..f3aa03657c7 100644 --- a/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb +++ b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb @@ -26,6 +26,10 @@ module Gitlab attr_reader :tables def enabled? + if tables.include?('ci_builds') && Feature.enabled?(:skip_autovacuum_health_check_for_ci_builds, type: :ops) + return false + end + Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 9495648d069..55a27f89b36 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -51,6 +51,8 @@ module Gitlab # If no secondaries were available this method will use the primary # instead. def read(&block) + raise_if_concurrent_ruby! + service_discovery&.log_refresh_thread_interruption conflict_retried = 0 @@ -111,6 +113,8 @@ module Gitlab # Yields a connection that can be used for both reads and writes. def read_write + raise_if_concurrent_ruby! + service_discovery&.log_refresh_thread_interruption connection = nil @@ -372,6 +376,12 @@ module Gitlab row = ar_connection.select_all(sql).first row['location'] if row end + + def raise_if_concurrent_ruby! + Gitlab::Utils.raise_if_concurrent_ruby!(:db) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end end end end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 3d4ac113bf6..39706582e3c 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -38,10 +38,6 @@ module Gitlab # batch_class_name - The name of the class that will be called to find the range of each next batch # batch_size - The maximum number of rows per job # sub_batch_size - The maximum number of rows processed per "iteration" within the job - # queued_migration_version - Version of the migration that queues the BBM, this is used to establish dependecies - # - # queued_migration_version is made optional temporarily to allow prior migrations to not fail, - # https://gitlab.com/gitlab-org/gitlab/-/issues/426417 will make it mandatory. # # *Returns the created BatchedMigration record* # @@ -67,7 +63,6 @@ module Gitlab batch_column_name, *job_arguments, job_interval:, - queued_migration_version: nil, batch_min_value: BATCH_MIN_VALUE, batch_max_value: nil, batch_class_name: BATCH_CLASS_NAME, @@ -80,6 +75,8 @@ module Gitlab Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! gitlab_schema ||= gitlab_schema_from_context + # Version of the migration that queued the BBM, this is used to establish dependencies + queued_migration_version = version Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information @@ -120,7 +117,7 @@ module Gitlab "(given #{job_arguments.count}, expected #{migration.job_class.job_arguments_count})" end - assign_attribtues_safely( + assign_attributes_safely( migration, max_batch_size, batch_table_name, @@ -231,7 +228,7 @@ module Gitlab "\n\n" \ "Finalize it manually by running the following command in a `bash` or `sh` shell:" \ "\n\n" \ - "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ + "\t#{migration.finalize_command}" \ "\n\n" \ "For more information, check the documentation" \ "\n\n" \ @@ -246,7 +243,7 @@ module Gitlab # about columns introduced later on because this model is not # isolated in migrations, which is why we need to check for existence # of these columns first. - def assign_attribtues_safely(migration, max_batch_size, batch_table_name, gitlab_schema, queued_migration_version) + def assign_attributes_safely(migration, max_batch_size, batch_table_name, gitlab_schema, queued_migration_version) # We keep track of the estimated number of tuples in 'total_tuple_count' to reason later # about the overall progress of a migration. safe_attributes_value = { diff --git a/lib/gitlab/database/migrations/pg_backend_pid.rb b/lib/gitlab/database/migrations/pg_backend_pid.rb index b59eb55cc6e..52f309e4058 100644 --- a/lib/gitlab/database/migrations/pg_backend_pid.rb +++ b/lib/gitlab/database/migrations/pg_backend_pid.rb @@ -13,7 +13,7 @@ module Gitlab Gitlab::Database::Migrations::PgBackendPid.say(conn) yield(conn) - + ensure Gitlab::Database::Migrations::PgBackendPid.say(conn) end end diff --git a/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb index 69a69091b5c..de6319582cb 100644 --- a/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb @@ -12,6 +12,13 @@ module Gitlab partition_for(active_partition.value + 1) end + def missing_partitions + partitions = [] + partitions << initial_partition if no_partitions_exist? + partitions << next_partition if next_partition_if.call(active_partition) + partitions + end + def validate_and_fix; end def after_adding_partitions; end @@ -20,6 +27,10 @@ module Gitlab [] end + def active_partition + super || initial_partition + end + private def ensure_partitioning_column_ignored_or_readonly!; end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 1c775482e7e..f52785c1e56 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -23,7 +23,7 @@ module Gitlab # Indexes with reindexing support scope :reindexing_support, -> do - where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) + where(exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) .not_match("#{Gitlab::Database::Reindexing::ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") end diff --git a/lib/gitlab/database/postgres_sequence.rb b/lib/gitlab/database/postgres_sequence.rb new file mode 100644 index 00000000000..bf394d80e12 --- /dev/null +++ b/lib/gitlab/database/postgres_sequence.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Backed by the postgres_sequences view + class PostgresSequence < SharedModel + self.primary_key = :seq_name + + scope :by_table_name, ->(table_name) { where(table_name: table_name) } + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb index 583aceba098..847f7064ad4 100644 --- a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb @@ -6,7 +6,7 @@ module Gitlab class PreventSetOperatorMismatch < Base SetOperatorStarError = Class.new(QueryAnalyzerError) - DETECT_REGEX = /.*SELECT.+(UNION|EXCEPT|INTERSECT)/i + DETECT_REGEX = /.*SELECT.+\b(UNION|EXCEPT|INTERSECT)\b/i class << self def enabled? @@ -36,9 +36,8 @@ module Gitlab node.stmt.select_stmt end - # This not entirely correct and will run true on `SELECT union_station, ...` def requires_detection?(sql) - sql.match DETECT_REGEX + DETECT_REGEX.match?(sql) end end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index de7be6efd72..6ddd8a208bc 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -9,7 +9,8 @@ module Gitlab delegate :new_file?, :deleted_file?, :renamed_file?, :unidiff, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, - :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false + :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, + :generated?, to: :diff, prefix: false # Finding a viewer for a diff file happens based only on extension and whether the # diff file blobs are binary or text, which means 1 diff file should only be matched by 1 viewer, diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 63a437b021d..dc5f4e1b324 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -185,18 +185,15 @@ module Gitlab def read_cache return {} unless file_paths.any? - results = [] cache_key = key # Moving out redis calls for feature flags out of redis.pipelined - with_redis do |redis| + results, _ = with_redis do |redis| redis.pipelined do |pipeline| - results = pipeline.hmget(cache_key, file_paths) + pipeline.hmget(cache_key, file_paths) pipeline.expire(key, EXPIRATION) end end - results = results.value - record_hit_ratio(results) results.map! do |result| diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 7daa1bb96a1..817956831e3 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -33,6 +33,8 @@ module Gitlab record: create_note, invalid_exception: InvalidNoteError, record_name: 'comment') + + reopen_issue_on_external_participant_note end def metrics_event @@ -71,6 +73,35 @@ module Gitlab raise UserNotFoundError unless from_address && author.verified_email?(from_address) end + + def reopen_issue_on_external_participant_note + return unless noteable.respond_to?(:closed?) + return unless noteable.closed? + return unless author == Users::Internal.support_bot + return unless project.service_desk_setting&.reopen_issue_on_external_participant_note? + + ::Notes::CreateService.new( + project, + Users::Internal.support_bot, + noteable: noteable, + note: build_reopen_message, + confidential: true + ).execute + end + + def build_reopen_message + translated_text = s_( + "ServiceDesk|This issue has been reopened because it received a new comment from an external participant." + ) + + "#{assignees_references} :wave: #{translated_text}\n/reopen".lstrip + end + + def assignees_references + return unless noteable.assignees.any? + + noteable.assignees.map(&:to_reference).join(' ') + end end end end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index e3249b143c8..b507af3024e 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -74,7 +74,6 @@ module Gitlab attr_reader :project_id, :project_path, :service_desk_key def contains_custom_email_address_verification_subaddress? - return false unless Feature.enabled?(:service_desk_custom_email, project) return false unless to_address.present? # Verification email only has one recipient @@ -230,6 +229,9 @@ module Gitlab def add_email_participants return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) + # Migrate this to ::IssueEmailParticipants::CreateService once the + # feature flag issue_email_participants has been enabled globally + # or removed: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137147#note_1652104416 @issue.issue_email_participants.create(email: from_address) add_external_participants_from_cc @@ -239,11 +241,11 @@ module Gitlab return if project.service_desk_setting.nil? return unless project.service_desk_setting.add_external_participants_from_cc? - cc_addresses.each do |email| - next if service_desk_addresses.include?(email) - - @issue.issue_email_participants.create!(email: email) - end + ::IssueEmailParticipants::CreateService.new( + target: @issue, + current_user: Users::Internal.support_bot, + emails: cc_addresses.excluding(service_desk_addresses) + ).execute end def service_desk_addresses diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d5877234c3a..e36b07da801 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -85,7 +85,13 @@ module Gitlab def mail_key strong_memoize(:mail_key) do - find_first_key_from(to) || key_from_additional_headers + find_most_concrete_key_from(to) || key_from_additional_headers + end + end + + def find_most_concrete_key_from(items) + find_first_key_from(items) do |email| + Gitlab::Email::ServiceDesk::CustomEmail.key_from_reply_address(email) || email_class.key_from_address(email) end end @@ -93,7 +99,8 @@ module Gitlab items.each do |item| email = item.is_a?(Mail::Field) ? item.value : item - key = email_class.key_from_address(email) + key = block_given? ? yield(email) : email_class.key_from_address(email) + return key if key end nil diff --git a/lib/gitlab/email/service_desk/custom_email.rb b/lib/gitlab/email/service_desk/custom_email.rb index 30ae435a6ec..1828f71984b 100644 --- a/lib/gitlab/email/service_desk/custom_email.rb +++ b/lib/gitlab/email/service_desk/custom_email.rb @@ -7,6 +7,9 @@ module Gitlab # support all features and methods of ingestable email addresses like # incoming_email and service_desk_email. module CustomEmail + REPLY_ADDRESS_KEY_REGEXP = /\+([0-9a-f]{32})@/ + EMAIL_REGEXP = /\A[\w\-._]+@[\w\-.]+\.{1}[a-zA-Z]{2,}\z/ + class << self def reply_address(issue, reply_key) return if reply_key.nil? @@ -18,6 +21,29 @@ module Gitlab # We don't have a placeholder. custom_email.sub('@', "+#{reply_key}@") end + + def key_from_reply_address(email) + match_data = REPLY_ADDRESS_KEY_REGEXP.match(email) + return unless match_data + + key = match_data[1] + + settings = find_service_desk_setting_from_reply_address(email, key) + # We intentionally don't check whether custom email is enabled + # so we don't lose emails that are addressed to a disabled custom email address + return unless settings + + key + end + + private + + def find_service_desk_setting_from_reply_address(email, key) + potential_custom_email = email.sub("+#{key}", '') + return unless EMAIL_REGEXP.match?(potential_custom_email) + + ServiceDeskSetting.find_by_custom_email(potential_custom_email) + end end end end diff --git a/lib/gitlab/encrypted_command_base.rb b/lib/gitlab/encrypted_command_base.rb index 679d9d8e31a..5e483fa2b15 100644 --- a/lib/gitlab/encrypted_command_base.rb +++ b/lib/gitlab/encrypted_command_base.rb @@ -41,7 +41,11 @@ module Gitlab encrypted.change do |contents| contents = encrypted_file_template unless File.exist?(encrypted.content_path) File.write(temp_file.path, contents) - system(ENV['EDITOR'], temp_file.path) + + edit_success = system(*editor_args, temp_file.path) + + raise "Unable to run $EDITOR: #{editor_args}" unless edit_success + changes = File.read(temp_file.path) contents_changed = contents != changes validate_contents(changes) @@ -99,6 +103,10 @@ module Gitlab def encrypted_file_template raise NotImplementedError end + + def editor_args + ENV['EDITOR']&.split + end end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 2b00fe48951..239aee97378 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -70,8 +70,8 @@ module Gitlab # returns a Hash, then the return value of that method will be merged into # `extra`. Exceptions can use this mechanism to provide structured data # to sentry in addition to their message and back-trace. - def track_and_raise_exception(exception, extra = {}) - process_exception(exception, extra: extra) + def track_and_raise_exception(exception, extra = {}, tags = {}) + process_exception(exception, extra: extra, tags: tags) raise exception end @@ -90,8 +90,8 @@ module Gitlab # # Provide an issue URL for follow up. # as `issue_url: 'http://gitlab.com/gitlab-org/gitlab/issues/111'` - def track_and_raise_for_dev_exception(exception, extra = {}) - process_exception(exception, extra: extra) + def track_and_raise_for_dev_exception(exception, extra = {}, tags = {}) + process_exception(exception, extra: extra, tags: tags) raise exception if should_raise_for_dev? end @@ -102,8 +102,8 @@ module Gitlab # returns a Hash, then the return value of that method will be merged into # `extra`. Exceptions can use this mechanism to provide structured data # to sentry in addition to their message and back-trace. - def track_exception(exception, extra = {}) - process_exception(exception, extra: extra) + def track_exception(exception, extra = {}, tags = {}) + process_exception(exception, extra: extra, tags: tags) end # This should be used when you only want to log the exception, @@ -157,8 +157,8 @@ module Gitlab end end - def process_exception(exception, extra:, trackers: default_trackers) - context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra) + def process_exception(exception, extra:, tags: {}, trackers: default_trackers) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(exception, extra, tags) trackers.each do |tracker| tracker.capture_exception(exception, **context_payload) diff --git a/lib/gitlab/error_tracking/context_payload_generator.rb b/lib/gitlab/error_tracking/context_payload_generator.rb index 3d0a707608f..23dd2e33a58 100644 --- a/lib/gitlab/error_tracking/context_payload_generator.rb +++ b/lib/gitlab/error_tracking/context_payload_generator.rb @@ -3,14 +3,14 @@ module Gitlab module ErrorTracking class ContextPayloadGenerator - def self.generate(exception, extra = {}) - new.generate(exception, extra) + def self.generate(exception, extra = {}, tags = {}) + new.generate(exception, extra, tags) end - def generate(exception, extra = {}) + def generate(exception, extra = {}, tags = {}) { extra: extra_payload(exception, extra), - tags: tags_payload, + tags: tags_payload(tags), user: user_payload } end @@ -31,12 +31,14 @@ module Gitlab filter.filter(parameters) end - def tags_payload - extra_tags_from_env.merge!( - program: Gitlab.process_name, - locale: I18n.locale, - feature_category: current_context['meta.feature_category'], - Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id + def tags_payload(tags) + tags.merge( + extra_tags_from_env.merge!( + program: Gitlab.process_name, + locale: I18n.locale, + feature_category: current_context['meta.feature_category'], + Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id + ) ) end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index 1e397b52ddf..b422fd061ff 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -17,6 +17,10 @@ module Gitlab instance.publish(event) end + def self.publish_group(events) + instance.publish_group(events) + end + def self.instance @instance ||= Store.new { |store| configure!(store) } end @@ -40,7 +44,9 @@ module Gitlab store.subscribe ::MergeRequests::CreateApprovalNoteWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::ResolveTodosAfterApprovalWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::ExecuteApprovalHooksWorker, to: ::MergeRequests::ApprovedEvent - store.subscribe ::MergeRequests::SetReviewerReviewedWorker, to: ::MergeRequests::ApprovedEvent + store.subscribe ::MergeRequests::SetReviewerReviewedWorker, + to: ::MergeRequests::ApprovedEvent, + if: -> (event) { ::Feature.disabled?(:mr_request_changes, User.find_by_id(event.data[:current_user_id])) } store.subscribe ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker, to: ::Packages::PackageCreatedEvent, if: -> (event) { ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker.handles_event?(event) } diff --git a/lib/gitlab/event_store/event.rb b/lib/gitlab/event_store/event.rb index ee0c329b8e8..ba82ae6dd6a 100644 --- a/lib/gitlab/event_store/event.rb +++ b/lib/gitlab/event_store/event.rb @@ -29,8 +29,13 @@ module Gitlab class Event attr_reader :data + class << self + attr_accessor :json_schema_valid + end + def initialize(data:) - validate_schema!(data) + validate_schema! + validate_data!(data) @data = data end @@ -40,7 +45,17 @@ module Gitlab private - def validate_schema!(data) + def validate_schema! + if self.class.json_schema_valid.nil? + self.class.json_schema_valid = JSONSchemer.schema(self.class.json_schema).valid?(schema) + end + + return if self.class.json_schema_valid == true + + raise Gitlab::EventStore::InvalidEvent, "Schema for event #{self.class} is invalid" + end + + def validate_data!(data) unless data.is_a?(Hash) raise Gitlab::EventStore::InvalidEvent, "Event data must be a Hash" end @@ -49,6 +64,10 @@ module Gitlab raise Gitlab::EventStore::InvalidEvent, "Data for event #{self.class} does not match the defined schema: #{schema}" end end + + def self.json_schema + @json_schema ||= Gitlab::Json.parse(File.read(File.join(__dir__, 'json_schema_draft07.json'))) + end end end end diff --git a/lib/gitlab/event_store/json_schema_draft07.json b/lib/gitlab/event_store/json_schema_draft07.json new file mode 100644 index 00000000000..aea0a29c4dc --- /dev/null +++ b/lib/gitlab/event_store/json_schema_draft07.json @@ -0,0 +1,250 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#" + } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { + "$ref": "#/definitions/nonNegativeInteger" + }, + { + "default": 0 + } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true, + "default": [ + + ] + } + }, + "type": [ + "object", + "boolean" + ], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minLength": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { + "$ref": "#" + }, + "items": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/schemaArray" + } + ], + "default": true + }, + "maxItems": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minItems": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { + "$ref": "#" + }, + "maxProperties": { + "$ref": "#/definitions/nonNegativeInteger" + }, + "minProperties": { + "$ref": "#/definitions/nonNegativeIntegerDefault0" + }, + "required": { + "$ref": "#/definitions/stringArray" + }, + "additionalProperties": { + "$ref": "#" + }, + "definitions": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "default": { + } + }, + "properties": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "default": { + } + }, + "patternProperties": { + "type": "object", + "additionalProperties": { + "$ref": "#" + }, + "propertyNames": { + "format": "regex" + }, + "default": { + } + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "$ref": "#" + }, + { + "$ref": "#/definitions/stringArray" + } + ] + } + }, + "propertyNames": { + "$ref": "#" + }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { + "$ref": "#/definitions/simpleTypes" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/simpleTypes" + }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { + "type": "string" + }, + "contentMediaType": { + "type": "string" + }, + "contentEncoding": { + "type": "string" + }, + "if": { + "$ref": "#" + }, + "then": { + "$ref": "#" + }, + "else": { + "$ref": "#" + }, + "allOf": { + "$ref": "#/definitions/schemaArray" + }, + "anyOf": { + "$ref": "#/definitions/schemaArray" + }, + "oneOf": { + "$ref": "#/definitions/schemaArray" + }, + "not": { + "$ref": "#" + } + }, + "default": true +} diff --git a/lib/gitlab/event_store/store.rb b/lib/gitlab/event_store/store.rb index 318745cc192..c558362122b 100644 --- a/lib/gitlab/event_store/store.rb +++ b/lib/gitlab/event_store/store.rb @@ -15,12 +15,12 @@ module Gitlab lock! end - def subscribe(worker, to:, if: nil, delay: nil) + def subscribe(worker, to:, if: nil, delay: nil, group_size: nil) condition = binding.local_variable_get('if') Array(to).each do |event| validate_subscription!(worker, event) - subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition, delay) + subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition, delay, group_size) end end @@ -34,6 +34,18 @@ module Gitlab end end + def publish_group(events) + event_class = events.first.class + + unless events.all? { |e| e.class < Event && e.instance_of?(event_class) } + raise InvalidEvent, "Not all events being published are valid" + end + + subscriptions.fetch(event_class, []).each do |subscription| + subscription.consume_events(events) + end + end + private def lock! diff --git a/lib/gitlab/event_store/subscriber.rb b/lib/gitlab/event_store/subscriber.rb index da95d3cfcfa..81770624cd9 100644 --- a/lib/gitlab/event_store/subscriber.rb +++ b/lib/gitlab/event_store/subscriber.rb @@ -29,16 +29,22 @@ module Gitlab def perform(event_type, data) raise InvalidEvent, event_type unless self.class.const_defined?(event_type) - event = event_type.constantize.new( - data: data.with_indifferent_access - ) + event_type_class = event_type.constantize - handle_event(event) + Array.wrap(data).each do |single_event_data| + handle_event(construct_event(event_type_class, single_event_data)) + end end def handle_event(event) raise NotImplementedError, 'you must implement this methods in order to handle events' end + + private + + def construct_event(event_type, event_data) + event_type.new(data: event_data.with_indifferent_access) + end end end end diff --git a/lib/gitlab/event_store/subscription.rb b/lib/gitlab/event_store/subscription.rb index 81a65f9a8ff..f39bbc2aaf0 100644 --- a/lib/gitlab/event_store/subscription.rb +++ b/lib/gitlab/event_store/subscription.rb @@ -3,12 +3,17 @@ module Gitlab module EventStore class Subscription - attr_reader :worker, :condition, :delay + DEFAULT_GROUP_SIZE = 10 + SCHEDULING_BATCH_SIZE = 100 + SCHEDULING_BATCH_DELAY = 10.seconds - def initialize(worker, condition, delay) + attr_reader :worker, :condition, :delay, :group_size + + def initialize(worker, condition, delay, group_size) @worker = worker @condition = condition @delay = delay + @group_size = group_size || DEFAULT_GROUP_SIZE end def consume_event(event) @@ -29,6 +34,30 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_class: event.class.name, event_data: event.data) end + def consume_events(events) + event_class = events.first.class + unless events.all? { |e| e.class < Event && e.instance_of?(event_class) } + raise InvalidEvent, "Events being published are not an instance of Gitlab::EventStore::Event" + end + + matched_events = events.select { |event| condition_met?(event) } + worker_args = events_worker_args(event_class, matched_events) + + # rubocop:disable Scalability/BulkPerformWithContext -- Context info is already available in `ApplicationContext` here. + if worker_args.size > SCHEDULING_BATCH_SIZE + # To reduce the number of concurrent jobs, we batch the group of events and add delay between each batch. + # We add a delay of 1s as bulk_perform_in does not support 0s delay. + worker.bulk_perform_in(delay || 1.second, worker_args, batch_size: SCHEDULING_BATCH_SIZE, batch_delay: SCHEDULING_BATCH_DELAY) + elsif delay + worker.bulk_perform_in(delay, worker_args) + else + worker.bulk_perform_async(worker_args) + end + # rubocop:enable Scalability/BulkPerformWithContext + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_class: event_class, events: events.map(&:data)) + end + private def condition_met?(event) @@ -36,6 +65,13 @@ module Gitlab condition.call(event) end + + def events_worker_args(event_class, events) + events + .map { |event| event.data.deep_stringify_keys } + .each_slice(group_size) + .map { |events_data_group| [event_class.name, events_data_group] } + end end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index e887e455792..0b18a337707 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -31,7 +31,7 @@ module Gitlab EOS def self.get_uuid(key) - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.get(redis_shared_state_key(key)) || false end end @@ -61,7 +61,7 @@ module Gitlab def self.cancel(key, uuid) return unless key.present? - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid]) end end @@ -79,7 +79,7 @@ module Gitlab # Removes any existing exclusive_lease from redis # Don't run this in a live system without making sure no one is using the leases def self.reset_all!(scope = '*') - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.scan_each(match: redis_shared_state_key(scope)).each do |key| redis.del(key) end @@ -96,7 +96,7 @@ module Gitlab # false if the lease is already taken. def try_obtain # Performing a single SET is atomic - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid end end @@ -109,7 +109,7 @@ module Gitlab # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end @@ -117,7 +117,7 @@ module Gitlab # Returns true if the key for this lease is set. def exists? - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -126,7 +126,7 @@ module Gitlab # # This method will return `nil` if no TTL could be obtained. def ttl - Gitlab::Redis::ClusterSharedState.with do |redis| + Gitlab::Redis::SharedState.with do |redis| ttl = redis.ttl(@redis_shared_state_key) ttl if ttl > 0 diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 4ff61aa3551..0f2a1b9fb1d 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -13,7 +13,7 @@ module Gitlab # no inclusions, etc.) def enabled? return false unless feature_flag_defined? - return false unless Gitlab.com? + return false unless available? return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops) feature_flag_instance.state != :off @@ -57,8 +57,12 @@ module Gitlab private + def available? + ApplicationExperiment.available? + end + def feature_flag_instance - ::Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet + ::Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet -- We are using at a lower layer here in experiment framework end def feature_flag_defined? diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index ef8f2d4d61b..b586c4b5892 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -22,6 +22,7 @@ module Gitlab # Configuration files gitignore: '.gitignore', gitlab_ci: ::Ci::Pipeline::DEFAULT_CONFIG_PATH, + jenkinsfile: 'jenkinsfile', route_map: '.gitlab/route-map.yml', # Dependency files diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 37f593ed551..8cbd1a4ce72 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -12,6 +12,13 @@ module Gitlab TAG_REF_PREFIX = "refs/tags/" BRANCH_REF_PREFIX = "refs/heads/" + # NOTE: We don't use linguist anymore, but we'd still want to support it + # to be backward/GitHub compatible. Using `gitlab-*` prefixed overrides + # going forward would give us a better control and flexibility. + ATTRIBUTE_OVERRIDES = { + generated: %w[gitlab-generated linguist-generated] + }.freeze + CommandError = Class.new(BaseError) CommitError = Class.new(BaseError) OSError = Class.new(BaseError) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 3744c81f51d..aa59caa4268 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -149,7 +149,7 @@ module Gitlab return if @data == '' # don't mess with submodule blobs # Even if we return early, recalculate whether this blob is binary in - # case a blob was initialized as text but the full data isn't + # case a blob was initialized as text but the full data isn'tspec/requests/api/graphql/mutations/branch_rules/update_spec.rb: @binary = nil return if @loaded_all_data diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 1086ea45a7a..d899ed3ba25 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -28,7 +28,8 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email, :trailers, :referenced_by + :committed_date, :committer_name, :committer_email, + :trailers, :extended_trailers, :referenced_by ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -432,9 +433,17 @@ module Gitlab @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) @trailers = commit.trailers.to_h { |t| [t.key, t.value] } + @extended_trailers = parse_commit_trailers(commit.trailers) @referenced_by = Array(commit.referenced_by) end + # Turn the commit trailers into a hash of key: [value, value] arrays + def parse_commit_trailers(trailers) + trailers.each_with_object({}) do |trailer, hash| + (hash[trailer.key] ||= []) << trailer.value + end + end + # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone # offset in author.timezone. If the latter isn't present, assume UTC. def init_date_from_gitaly(author) diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb index ab5245ba7cb..c6d678c9432 100644 --- a/lib/gitlab/git/compare.rb +++ b/lib/gitlab/git/compare.rb @@ -42,6 +42,16 @@ module Gitlab options[:straight] = @straight Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths) end + + def generated_files + return Set.new unless @base && @head + + changed_paths = @repository + .find_changed_paths([Gitlab::Git::DiffTree.new(@base.id, @head.id)]) + .map(&:path) + + @repository.detect_generated_files(@base.id, changed_paths) + end end end end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 743bac62764..e753d356bc6 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -10,7 +10,7 @@ module Gitlab attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff # Stats properties - attr_accessor :new_file, :renamed_file, :deleted_file + attr_accessor :new_file, :renamed_file, :deleted_file, :generated alias_method :new_file?, :new_file alias_method :deleted_file?, :deleted_file @@ -20,6 +20,7 @@ module Gitlab attr_writer :too_large alias_method :expanded?, :expanded + alias_method :generated?, :generated # The default maximum content size to display a diff patch. # @@ -31,7 +32,18 @@ module Gitlab # persisting limits over that. MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes - SERIALIZE_KEYS = %i[diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large].freeze + SERIALIZE_KEYS = %i[ + diff + new_path + old_path + a_mode + b_mode + new_file + renamed_file + deleted_file + too_large + generated + ].freeze BINARY_NOTICE_PATTERN = %r{Binary files (.*) and (.*) differ} @@ -79,9 +91,12 @@ module Gitlab # If false, patch raw data will not be included in the diff after # `max_files`, `max_lines` or any of the limits in `limits` are # exceeded + # :generated_files :: + # If the list of generated files is given, those files will be marked + # as generated. def filter_diff_options(options, default_options = {}) allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, - :limits, :expanded, :collect_all_paths] + :limits, :expanded, :collect_all_paths, :generated_files] if default_options actual_defaults = default_options.dup @@ -144,8 +159,9 @@ module Gitlab text.start_with?(BINARY_NOTICE_PATTERN) end end - def initialize(raw_diff, expanded: true, replace_invalid_utf8_chars: true) + def initialize(raw_diff, expanded: true, replace_invalid_utf8_chars: true, generated: nil) @expanded = expanded + @generated = generated case raw_diff when Hash @@ -255,6 +271,10 @@ module Gitlab private + def collapse_generated_file? + generated? && !expanded + end + def encode_diff_to_utf8(replace_invalid_utf8_chars) return unless replace_invalid_utf8_chars && diff_should_be_converted? @@ -300,7 +320,7 @@ module Gitlab ::Gitlab::Metrics.add_event(:patch_hard_limit_bytes_hit) too_large! - elsif collapsed? + elsif collapsed? || collapse_generated_file? collapse! end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index c021268a62a..e8b6e5fc181 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -35,6 +35,8 @@ module Gitlab def initialize(iterator, options = {}) @iterator = iterator + @generated_files = options.fetch(:generated_files, nil) + @collapse_generated = options.fetch(:collapse_generated, false) @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) @@ -164,7 +166,10 @@ module Gitlab i = @array.length @iterator.each do |raw| - diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?) + options = { expanded: expand_diff? } + options[:generated] = @generated_files.include?(raw.from_path) if @generated_files + + diff = Gitlab::Git::Diff.new(raw, **options) if raw.overflow_marker @overflow = true @@ -193,7 +198,10 @@ module Gitlab break end - diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?) + # Discard generated field if it is already set when FF is disabled + raw_data = @collapse_generated ? raw : raw.except(:generated) + + diff = Gitlab::Git::Diff.new(raw_data, expanded: expand_diff?) if !expand_diff? && over_safe_limits?(i) && diff.line_count > 0 diff.collapse! diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index db6e6b4d00b..312e05b5f54 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -46,7 +46,7 @@ module Gitlab attr_reader :storage, :gl_repository, :gl_project_path, :container - delegate :list_all_blobs, to: :gitaly_blob_client + delegate :list_all_blobs, :list_blobs, to: :gitaly_blob_client # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -84,13 +84,6 @@ module Gitlab [self.class, storage, relative_path].hash end - # This method will be removed when Gitaly reaches v1.1. - def path - File.join( - Gitlab.config.repositories.storages[@storage].legacy_disk_path, @relative_path - ) - end - # Default branch in the repository def root_ref(head_only: false) wrapped_gitaly_errors do @@ -102,9 +95,9 @@ module Gitlab gitaly_repository_client.exists? end - def create_repository(default_branch = nil) + def create_repository(default_branch = nil, object_format: nil) wrapped_gitaly_errors do - gitaly_repository_client.create_repository(default_branch) + gitaly_repository_client.create_repository(default_branch, object_format: object_format) rescue GRPC::AlreadyExists => e raise RepositoryExists, e.message end @@ -1214,9 +1207,26 @@ module Gitlab gitaly_repository_client .get_file_attributes(revision, file_paths, attributes) .attribute_infos + .map(&:to_h) end end + def object_format + wrapped_gitaly_errors do + gitaly_repository_client.object_format.format + end + end + + # rubocop: disable CodeReuse/ActiveRecord -- not an active record operation + def detect_generated_files(revision, paths) + return Set.new if paths.blank? + + get_file_attributes(revision, paths, Gitlab::Git::ATTRIBUTE_OVERRIDES[:generated]) + .pluck(:path) + .to_set + end + # rubocop: enable CodeReuse/ActiveRecord + private def repository_info_size_megabytes diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index da38c11ebca..6dee9a404f4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -283,6 +283,7 @@ module Gitlab def self.execute(storage, service, rpc, request, remote_storage:, timeout:) enforce_gitaly_request_limits(:call) Gitlab::RequestContext.instance.ensure_deadline_not_exceeded! + raise_if_concurrent_ruby! kwargs = request_kwargs(storage, timeout: timeout.to_f, remote_storage: remote_storage) kwargs = yield(kwargs) if block_given? @@ -547,43 +548,10 @@ module Gitlab end end - def self.storage_metadata_file_path(storage) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join( - Gitlab.config.repositories.storages[storage].legacy_disk_path, GITALY_METADATA_FILENAME - ) - end - end - - def self.can_use_disk?(storage) - cached_value = MUTEX.synchronize do - @can_use_disk ||= {} - @can_use_disk[storage] - end - - return cached_value unless cached_value.nil? - - gitaly_filesystem_id = filesystem_id(storage) - direct_filesystem_id = filesystem_id_from_disk(storage) - - MUTEX.synchronize do - @can_use_disk[storage] = gitaly_filesystem_id.present? && - gitaly_filesystem_id == direct_filesystem_id - end - end - def self.filesystem_id(storage) Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id end - def self.filesystem_id_from_disk(storage) - metadata_file = File.read(storage_metadata_file_path(storage)) - metadata_hash = Gitlab::Json.parse(metadata_file) - metadata_hash['gitaly_filesystem_id'] - rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError - nil - end - def self.filesystem_disk_available(storage) Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available end @@ -669,5 +637,12 @@ module Gitlab Thread.current[:gitaly_feature_flag_actors] ||= {} end end + + def self.raise_if_concurrent_ruby! + Gitlab::Utils.raise_if_concurrent_ruby!(:gitaly) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + private_class_method :raise_if_concurrent_ruby! end end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index ffe65307c80..831c5ca1305 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -30,8 +30,8 @@ module Gitlab end def conflicts? - skip_content = Feature.enabled?(:skip_conflict_files_in_gitaly, type: :experiment) - list_conflict_files(skip_content: skip_content).any? + list_conflict_files(skip_content: true).any? + rescue GRPC::FailedPrecondition, GRPC::Unknown # The server raises FailedPrecondition when it encounters # ConflictSideMissing, which means a conflict exists but its `theirs` or diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 905588c2afc..882982b3cde 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -288,8 +288,6 @@ module Gitlab def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: []) request_enum = QueueEnumerator.new - rebase_sha = nil - response_enum = gitaly_client_call( @repository.storage, :operation_service, @@ -316,16 +314,14 @@ module Gitlab ) ) - perform_next_gitaly_rebase_request(response_enum) do |response| - rebase_sha = response.rebase_sha - end + response = response_enum.next + rebase_sha = response.rebase_sha yield rebase_sha # Second request confirms with gitaly to finalize the rebase request_enum.push(Gitaly::UserRebaseConfirmableRequest.new(apply: true)) - - perform_next_gitaly_rebase_request(response_enum) + response_enum.next rebase_sha rescue GRPC::BadStatus => e @@ -528,20 +524,6 @@ module Gitlab private - def perform_next_gitaly_rebase_request(response_enum) - response = response_enum.next - - if response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError, response.pre_receive_error - elsif response.git_error.present? - raise Gitlab::Git::Repository::GitError, response.git_error - end - - yield response if block_given? - - response - end - def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run:) request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 457380615f7..60d14d18f62 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -114,8 +114,8 @@ module Gitlab end # rubocop: enable Metrics/ParameterLists - def create_repository(default_branch = nil) - request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch)) + def create_repository(default_branch = nil, object_format: nil) + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch), object_format: gitaly_object_format(object_format)) gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end @@ -363,6 +363,12 @@ module Gitlab gitaly_client_call(@repository.storage, :repository_service, :get_file_attributes, request, timeout: GitalyClient.fast_timeout) end + def object_format + request = Gitaly::ObjectFormatRequest.new(repository: @gitaly_repo) + + gitaly_client_call(@storage, :repository_service, :object_format, request, timeout: GitalyClient.fast_timeout) + end + private def search_results_from_response(gitaly_response, options = {}) @@ -449,6 +455,15 @@ module Gitlab entry end + + def gitaly_object_format(format) + case format + when Repository::FORMAT_SHA1 + Gitaly::ObjectFormat::OBJECT_FORMAT_SHA1 + when Repository::FORMAT_SHA256 + Gitaly::ObjectFormat::OBJECT_FORMAT_SHA256 + end + end end end end diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index adf0c811274..253d7c4a93e 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -12,7 +12,7 @@ module Gitlab InvalidConfigurationError = Class.new(StandardError) INVALID_STORAGE_MESSAGE = <<~MSG - Storage is invalid because it has no `path` key. + Storage is invalid because it has no `gitaly_address` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. If you're using the GitLab Development Kit, you can update your configuration running `gdk reconfigure`. @@ -38,13 +38,15 @@ module Gitlab def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) - raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') + + @hash = ActiveSupport::HashWithIndifferentAccess.new(storage) + + raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless @hash.has_key?('gitaly_address') # Support a nil 'path' field because some of the circuit breaker tests use it. - @legacy_disk_path = File.expand_path(storage['path'], Rails.root) if storage['path'] + @legacy_disk_path = File.expand_path(@hash['path'], Rails.root) if @hash['path'] && @hash['path'] != Deprecated - storage['path'] = Deprecated - @hash = ActiveSupport::HashWithIndifferentAccess.new(storage) + @hash['path'] = Deprecated end def gitaly_address diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index d48b25842b3..31fe2461e86 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -8,18 +8,12 @@ module Gitlab def self.new_client_for(project, token: nil, host: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) - token_pool = project.import_data&.credentials&.dig(:additional_access_tokens) - options = { + Client.new( + token_to_use, host: host.presence || self.formatted_import_url(project), per_page: self.per_page(project), parallel: parallel - } - - if token_pool - ClientPool.new(token_pool: token_pool.append(token_to_use), **options) - else - Client.new(token_to_use, **options) - end + ) end # Returns the ID of the ghost user. diff --git a/lib/gitlab/github_import/client_pool.rb b/lib/gitlab/github_import/client_pool.rb deleted file mode 100644 index e8414942d1b..00000000000 --- a/lib/gitlab/github_import/client_pool.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - class ClientPool - delegate_missing_to :best_client - - def initialize(token_pool:, per_page:, parallel:, host: nil) - @token_pool = token_pool - @host = host - @per_page = per_page - @parallel = parallel - end - - # Returns the client with the most remaining requests, or the client with - # the closest rate limit reset time, if all clients are rate limited. - def best_client - clients_with_requests_remaining = clients.select(&:requests_remaining?) - - return clients_with_requests_remaining.max_by(&:remaining_requests) if clients_with_requests_remaining.any? - - clients.min_by(&:rate_limit_resets_in) - end - - private - - def clients - @clients ||= @token_pool.map do |token| - Client.new( - token, - host: @host, - per_page: @per_page, - parallel: @parallel - ) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/collaborator_importer.rb b/lib/gitlab/github_import/importer/collaborator_importer.rb index 9a90ea5a4ed..a5e3373bacb 100644 --- a/lib/gitlab/github_import/importer/collaborator_importer.rb +++ b/lib/gitlab/github_import/importer/collaborator_importer.rb @@ -53,6 +53,7 @@ module Gitlab def create_membership!(user_id, access_level) ::ProjectMember.create!( + importing: true, source: project, access_level: access_level, user_id: user_id, diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index bcf9cd94ad9..23e3f4f4dfa 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -18,6 +18,7 @@ module Gitlab def create_note(issue_event, note_body, author_id) Note.create!( + importing: true, system: true, noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), diff --git a/lib/gitlab/github_import/importer/events/changed_milestone.rb b/lib/gitlab/github_import/importer/events/changed_milestone.rb index 39b92d88b58..f002cfb6478 100644 --- a/lib/gitlab/github_import/importer/events/changed_milestone.rb +++ b/lib/gitlab/github_import/importer/events/changed_milestone.rb @@ -17,10 +17,14 @@ module Gitlab private def create_event(issue_event) + milestone = project.milestones.find_by_title(issue_event.milestone_title) + return unless milestone + attrs = { + importing: true, user_id: author_id(issue_event), created_at: issue_event.created_at, - milestone_id: project.milestones.find_by_title(issue_event.milestone_title)&.id, + milestone_id: milestone.id, action: action(issue_event.event), state: DEFAULT_STATE }.merge(resource_event_belongs_to(issue_event)) diff --git a/lib/gitlab/github_import/importer/events/changed_reviewer.rb b/lib/gitlab/github_import/importer/events/changed_reviewer.rb index 17b1fa4ab45..eb142478b16 100644 --- a/lib/gitlab/github_import/importer/events/changed_reviewer.rb +++ b/lib/gitlab/github_import/importer/events/changed_reviewer.rb @@ -18,6 +18,7 @@ module Gitlab def create_note(issue_event, note_body, review_requester_id) Note.create!( + importing: true, system: true, noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb index 58d9dbf826c..6058ccda1b5 100644 --- a/lib/gitlab/github_import/importer/events/closed.rb +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -26,6 +26,7 @@ module Gitlab def create_state_event(issue_event) attrs = { + importing: true, user_id: author_id(issue_event), source_commit: issue_event.commit_id, state: 'closed', diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index 4fe371e5900..9a67fa1c6fe 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -32,6 +32,7 @@ module Gitlab def create_note(issue_event, note_body, user_id) Note.create!( + importing: true, system: true, noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), diff --git a/lib/gitlab/github_import/importer/events/merged.rb b/lib/gitlab/github_import/importer/events/merged.rb new file mode 100644 index 00000000000..6189fa8f429 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/merged.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Merged < BaseImporter + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: author_id(issue_event), + action: 'merged', + target_type: issuable_type(issue_event), + target_id: issuable_db_id(issue_event), + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + attrs = { + importing: true, + user_id: author_id(issue_event), + source_commit: issue_event.commit_id, + state: 'merged', + close_after_error_tracking_resolve: false, + close_auto_resolve_prometheus_alert: false, + created_at: issue_event.created_at + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb index fb9e08116ba..5d306f9dce7 100644 --- a/lib/gitlab/github_import/importer/events/renamed.rb +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -13,6 +13,7 @@ module Gitlab def note_params(issue_event) { + importing: true, noteable_id: issuable_db_id(issue_event), noteable_type: issuable_type(issue_event), project_id: project.id, diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index 80749aae93c..d20482eca6f 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -6,6 +6,22 @@ module Gitlab class IssueEventImporter attr_reader :issue_event, :project, :client + SUPPORTED_EVENTS = %w[ + assigned + closed + cross-referenced + demilestoned + labeled + merged + milestoned + renamed + reopened + review_request_removed + review_requested + unassigned + unlabeled + ].freeze + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. @@ -47,6 +63,8 @@ module Gitlab Gitlab::GithubImport::Importer::Events::ChangedAssignee when 'review_requested', 'review_request_removed' Gitlab::GithubImport::Importer::Events::ChangedReviewer + when 'merged' + Gitlab::GithubImport::Importer::Events::Merged end end end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb index b250a42a53c..6df130eb6e8 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb @@ -5,6 +5,8 @@ module Gitlab module Importer module PullRequests class ReviewImporter + include ::Gitlab::Import::MergeRequestHelpers + # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview` # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` @@ -83,52 +85,11 @@ module Gitlab def add_approval!(user_id) return unless review.review_type == 'APPROVED' - approval_attribues = { - merge_request_id: merge_request.id, - user_id: user_id, - created_at: submitted_at, - updated_at: submitted_at - } - - result = ::Approval.insert( - approval_attribues, - returning: [:id], - unique_by: [:user_id, :merge_request_id] - ) - - add_approval_system_note!(user_id) if result.rows.present? + create_approval!(project.id, merge_request.id, user_id, submitted_at) end def add_reviewer!(user_id) - return if review_re_requested?(user_id) - - ::MergeRequestReviewer.create!( - merge_request_id: merge_request.id, - user_id: user_id, - state: ::MergeRequestReviewer.states['reviewed'], - created_at: submitted_at - ) - rescue ActiveRecord::RecordNotUnique - # multiple reviews from single person could make a SQL concurrency issue here - nil - end - - # rubocop:disable CodeReuse/ActiveRecord - def review_re_requested?(user_id) - # records that were imported on previous stage with "unreviewed" status - MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists? - end - # rubocop:enable CodeReuse/ActiveRecord - - def add_approval_system_note!(user_id) - attributes = note_attributes( - user_id, - 'approved this merge request', - system: true, - system_note_metadata: SystemNoteMetadata.new(action: 'approved') - ) - - Note.create!(attributes) + create_reviewer!(merge_request.id, user_id, submitted_at) end def submitted_at diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index e0a7e6479f5..d7fa098a775 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -29,7 +29,8 @@ module Gitlab associated = associated.to_h compose_associated_id!(parent_record, associated) - return if already_imported?(associated) + + return if already_imported?(associated) || importer_class::SUPPORTED_EVENTS.exclude?(associated[:event]) Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index 0780ba6119f..5ce50e5b4e7 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -26,8 +26,6 @@ module Gitlab def database_id val = Gitlab::Cache::Import::Caching.read_integer(cache_key, timeout: timeout) - return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) - return if val == CACHE_OBJECT_NOT_FOUND return val if val.present? diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb index 077a27df16c..50cad1aae19 100644 --- a/lib/gitlab/github_import/job_delay_calculator.rb +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -7,7 +7,9 @@ module Gitlab module JobDelayCalculator # Default batch settings for parallel import (can be redefined in Importer/Worker classes) def parallel_import_batch - { size: 1000, delay: 1.minute } + batch_size = Feature.enabled?(:github_import_increased_concurrent_workers, project.creator) ? 5000 : 1000 + + { size: batch_size, delay: 1.minute } end private diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index d0bbd2bc7cf..87d3195eb93 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -19,8 +19,6 @@ module Gitlab cache_key = cache_key_for(name) val = Gitlab::Cache::Import::Caching.read_integer(cache_key) - return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) - return if val == CACHE_OBJECT_NOT_FOUND return val if val.present? diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index dcb679fda6d..fd60fa86e82 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -24,8 +24,6 @@ module Gitlab val = Gitlab::Cache::Import::Caching.read_integer(cache_key) - return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) - return if val == CACHE_OBJECT_NOT_FOUND return val if val.present? diff --git a/lib/gitlab/github_import/representation/collaborator.rb b/lib/gitlab/github_import/representation/collaborator.rb index fb58a572151..3e3706f05b5 100644 --- a/lib/gitlab/github_import/representation/collaborator.rb +++ b/lib/gitlab/github_import/representation/collaborator.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class Collaborator - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :login, :role_name diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index e8e515d1f87..f678fe38688 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -4,8 +4,7 @@ module Gitlab module GithubImport module Representation class DiffNote - include ToHash - include ExposeAttribute + include Representable NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i @@ -134,9 +133,6 @@ module Gitlab private - # Required by ExposeAttribute - attr_reader :attributes - def diff_line_params if addition? { new_line: end_line, old_line: nil } diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index 95a7c5ebf4b..8c072c0ed06 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class Issue - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :iid, :title, :description, :milestone_number, :created_at, :updated_at, :state, :assignees, diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 068d5cf9482..30608112f85 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class IssueEvent - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, :milestone_title, :issue, :source, :assignee, :review_requester, diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index 716e77bf401..153a1680577 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class LfsObject - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :oid, :link, :size diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 76adbb651af..308cab08dea 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class Note - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :note_id diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb index 70dd242303a..43e18a923d6 100644 --- a/lib/gitlab/github_import/representation/note_text.rb +++ b/lib/gitlab/github_import/representation/note_text.rb @@ -8,14 +8,11 @@ module Gitlab module GithubImport module Representation class NoteText - include ToHash - include ExposeAttribute + include Representable MODELS_ALLOWLIST = [::Release, ::Note, ::Issue, ::MergeRequest].freeze ModelNotSupported = Class.new(StandardError) - attr_reader :attributes - expose_attribute :record_db_id, :record_type, :text, :iid, :tag, :noteable_type # Builds a note text representation from DB record of Note or Release. diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index eb9dd3bc247..0b755f0c79d 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class ProtectedBranch - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, :required_pull_request_reviews, :require_code_owner_reviews, :allowed_to_push_users diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index f26fa953773..370d3b541f0 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class PullRequest - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :iid, :title, :description, :source_branch, :source_branch_sha, :target_branch, :target_branch_sha, diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 0c6e281cd6d..86e32bbab7b 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class PullRequestReview - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :merge_request_iid, :review_id diff --git a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb index a6ec1d3178b..a3ca5cb644d 100644 --- a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb +++ b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb @@ -5,10 +5,7 @@ module Gitlab module Representation module PullRequests class ReviewRequests - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :merge_request_id, :merge_request_iid, :users diff --git a/lib/gitlab/github_import/representation/representable.rb b/lib/gitlab/github_import/representation/representable.rb new file mode 100644 index 00000000000..49095d4c819 --- /dev/null +++ b/lib/gitlab/github_import/representation/representable.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module Representable + extend ActiveSupport::Concern + + included do + include ToHash + include ExposeAttribute + + def github_identifiers + error = NotImplementedError.new('Subclasses must implement #github_identifiers') + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + + {} + end + + private + + attr_reader :attributes + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index 02cbe037384..6f172d8fb91 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -4,10 +4,7 @@ module Gitlab module GithubImport module Representation class User - include ToHash - include ExposeAttribute - - attr_reader :attributes + include Representable expose_attribute :id, :login diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index a4170f4147f..3947ae3c63d 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -57,16 +57,13 @@ module Gitlab user_settings = user_settings.to_h.with_indifferent_access optional_stages = fetch_stages_from_params(user_settings[:optional_stages]) - credentials = project.import_data&.credentials&.merge( - additional_access_tokens: user_settings[:additional_access_tokens] - ) import_data = project.build_or_assign_import_data( data: { optional_stages: optional_stages, timeout_strategy: user_settings[:timeout_strategy] }, - credentials: credentials + credentials: project.import_data&.credentials ) import_data.save! diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 59813e4f5a0..caf7cfb3f76 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -56,7 +56,6 @@ module Gitlab gon.dot_com = Gitlab.com? gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX gon.pat_prefix = Gitlab::CurrentSettings.current_application_settings.personal_access_token_prefix - gon.use_new_navigation = NavHelper.show_super_sidebar?(current_user) gon.keyboard_shortcuts_enabled = current_user ? current_user.keyboard_shortcuts_enabled : true gon.diagramsnet_url = Gitlab::CurrentSettings.diagramsnet_url if Gitlab::CurrentSettings.diagramsnet_enabled @@ -77,9 +76,12 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) + push_frontend_feature_flag(:key_contacts_management, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:custom_emoji) + push_frontend_feature_flag(:encoding_logs_tree) + push_frontend_feature_flag(:group_user_saml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb index a99b8c81930..7de4956a668 100644 --- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb +++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb @@ -19,9 +19,7 @@ module Gitlab scope = args[:key] # this logic cannot be placed in the NamespaceResolver due to N+1 scope = scope.without_project_namespaces if scope == Namespace - # `with_route` avoids an N+1 calculating full_path - scope = scope.where_full_path_in(full_paths).with_route - .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") + scope = scope.where_full_path_in(full_paths) scope.each do |model_instance| loader.call(model_instance.full_path.downcase, model_instance) diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 6fe7a0030f0..b112740c4ad 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -13,7 +13,6 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) - groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/427108") members = GroupMember.where(group: groups).non_invite users = super diff --git a/lib/gitlab/hook_data/project_builder.rb b/lib/gitlab/hook_data/project_builder.rb index aec842e061f..56b8b842a78 100644 --- a/lib/gitlab/hook_data/project_builder.rb +++ b/lib/gitlab/hook_data/project_builder.rb @@ -33,7 +33,6 @@ module Gitlab private def project_data - owners = project.owners.compact # When this is removed, also remove the `deprecated_owner` method # See https://gitlab.com/gitlab-org/gitlab/-/issues/350603 owner = project.deprecated_owner @@ -45,13 +44,27 @@ module Gitlab project_id: project.id, owner_name: owner.try(:name), owner_email: user_email(owner), - owners: owners.map do |owner| - owner_data(owner) - end, + owners: owners_data, project_visibility: project.visibility.downcase } end + def owners_data + # Extracted code from ProjectTeam#owners, but works without creating cross joins queries + # Can be consolidate again once https://gitlab.com/gitlab-org/gitlab/-/issues/432606 is addressed + if project.group + project.group.all_owner_members.select(:id, :user_id) + .preload_user.find_each.map { |member| owner_data(member.user) } + else + data = [] + project.project_authorizations.owners.preload_user.each_batch(column: :user_id) do |relation| + data.concat(relation.map { |member| owner_data(member.user) }) + end + data |= Array.wrap(owner_data(project.owner)) if project.owner + data + end + end + def owner_data(user) { name: user.name, email: user_email(user) } end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 96e3d90c139..02afdedb4be 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,8 +44,8 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 29, - 'de' => 97, + 'da_DK' => 28, + 'de' => 95, 'en' => 100, 'eo' => 0, 'es' => 28, @@ -56,18 +56,18 @@ module Gitlab 'it' => 1, 'ja' => 98, 'ko' => 23, - 'nb_NO' => 21, + 'nb_NO' => 20, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 57, - 'ro_RO' => 76, + 'pt_BR' => 60, + 'ro_RO' => 74, 'ru' => 21, - 'si_LK' => 12, + 'si_LK' => 11, 'tr_TR' => 8, - 'uk' => 52, + 'uk' => 51, 'zh_CN' => 99, 'zh_HK' => 1, - 'zh_TW' => 100 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 9fd393c61a0..bcaae530927 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -69,6 +69,52 @@ module Gitlab rows = reviewers.map { |reviewer_id| { merge_request_id: merge_request.id, user_id: reviewer_id } } MergeRequestReviewer.insert_all(rows) end + + def create_approval!(project_id, merge_request_id, user_id, submitted_at) + approval_attributes = { + merge_request_id: merge_request_id, + user_id: user_id, + created_at: submitted_at, + updated_at: submitted_at + } + + result = ::Approval.insert( + approval_attributes, + returning: [:id], + unique_by: [:user_id, :merge_request_id] + ) + + add_approval_system_note!(project_id, merge_request_id, user_id, submitted_at) if result.rows.present? + end + + def add_approval_system_note!(project_id, merge_request_id, user_id, submitted_at) + attributes = { + importing: true, + noteable_id: merge_request_id, + noteable_type: 'MergeRequest', + project_id: project_id, + author_id: user_id, + note: 'approved this merge request', + system: true, + system_note_metadata: SystemNoteMetadata.new(action: 'approved'), + created_at: submitted_at, + updated_at: submitted_at + } + + Note.create!(attributes) + end + + def create_reviewer!(merge_request_id, user_id, submitted_at) + ::MergeRequestReviewer.create!( + merge_request_id: merge_request_id, + user_id: user_id, + state: ::MergeRequestReviewer.states['reviewed'], + created_at: submitted_at + ) + rescue ActiveRecord::RecordNotUnique + # multiple reviews from single person could make a SQL concurrency issue here + nil + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 6f3601e9a21..e38930ed548 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -318,6 +318,7 @@ included_attributes: - :releases_access_level - :infrastructure_access_level - :model_experiments_access_level + - :model_registry_access_level prometheus_metrics: - :created_at - :updated_at @@ -738,6 +739,7 @@ included_attributes: - :releases_access_level - :infrastructure_access_level - :model_experiments_access_level + - :model_registry_access_level - :auto_devops_deploy_strategy - :auto_devops_enabled - :container_registry_enabled diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index fec8b3a7708..6e507142e88 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -11,7 +11,7 @@ module Gitlab IMPORT_TABLE = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), - ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::ParallelImporter), ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::ParallelImporter), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repository by URL', nil), @@ -44,15 +44,7 @@ module Gitlab end def import_table - bitbucket_parallel_enabled = Feature.enabled?(:bitbucket_parallel_importer) - - return IMPORT_TABLE unless bitbucket_parallel_enabled - - import_table = IMPORT_TABLE.deep_dup - - import_table[1].importer = Gitlab::BitbucketImport::ParallelImporter if bitbucket_parallel_enabled - - import_table + IMPORT_TABLE end end end diff --git a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb index 3fdb34d42b7..560c113fb5f 100644 --- a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb +++ b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb @@ -36,7 +36,7 @@ module Gitlab def mark_notified Gitlab::Redis::SharedState.with do |redis| - redis.hset(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}", Date.current) + redis.hset(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}", Date.current.to_s) end end @@ -47,8 +47,9 @@ module Gitlab end def scheduled_deletion_date - if notification_date.present? - (notification_date.to_date + grace_period_after_notification).to_s + notif_date = notification_date + if notif_date.present? + (notif_date.to_date + grace_period_after_notification).to_s else grace_period_after_notification.from_now.to_date.to_s end diff --git a/lib/gitlab/instrumentation/connection_pool.rb b/lib/gitlab/instrumentation/connection_pool.rb new file mode 100644 index 00000000000..76e6af34054 --- /dev/null +++ b/lib/gitlab/instrumentation/connection_pool.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + # rubocop:disable Gitlab/ModuleWithInstanceVariables -- this module patches ConnectionPool to instrument it + module ConnectionPool + def initialize(options = {}, &block) + @name = options.fetch(:name, 'unknown') + + super + end + + def checkout(options = {}) + conn = super + + connection_class = conn.class.to_s + track_available_connections(connection_class) + track_pool_size(connection_class) + + conn + end + + def track_pool_size(connection_class) + # this means that the size metric for this pool key has been sent + return if @size_gauge + + @size_gauge ||= ::Gitlab::Metrics.gauge(:gitlab_connection_pool_size, 'Size of connection pool', {}, :all) + @size_gauge.set({ pool_name: @name, pool_key: @key, connection_class: connection_class }, @size) + end + + def track_available_connections(connection_class) + @available_gauge ||= ::Gitlab::Metrics.gauge(:gitlab_connection_pool_available_count, + 'Number of available connections in the pool', {}, :all) + + @available_gauge.set({ pool_name: @name, pool_key: @key, connection_class: connection_class }, available) + end + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end +end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 88991495a10..1e117172c3a 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -128,6 +128,11 @@ module Gitlab @exception_counter.increment({ storage: storage_key, exception: ex.class.to_s }) end + def instance_count_connection_exception(ex) + @connection_exception_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_connection_exceptions_total, 'Client side Redis connection exception count, per Redis server, per exception class') + @connection_exception_counter.increment({ storage: storage_key, exception: ex.class.to_s }) + end + def instance_count_cluster_redirection(ex) # This metric is meant to give a client side view of how often are commands # redirected to the right node, especially during resharding.. diff --git a/lib/gitlab/instrumentation/redis_helper.rb b/lib/gitlab/instrumentation/redis_helper.rb new file mode 100644 index 00000000000..ba1c8132250 --- /dev/null +++ b/lib/gitlab/instrumentation/redis_helper.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + module RedisHelper + APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax command xread xreadgroup].freeze + + def instrument_call(commands, instrumentation_class, pipelined = false) + start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined + instrumentation_class.instance_count_request(commands.size) + instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined + + if !instrumentation_class.redis_cluster_validate!(commands) && ::RequestStore.active? + instrumentation_class.increment_cross_slot_request_count + end + + yield + rescue ::Redis::BaseError => ex + if ex.message.start_with?('MOVED', 'ASK') + instrumentation_class.instance_count_cluster_redirection(ex) + else + instrumentation_class.instance_count_exception(ex) + end + + instrumentation_class.log_exception(ex) + raise ex + ensure + duration = Gitlab::Metrics::System.monotonic_time - start + + unless exclude_from_apdex?(commands) + commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } + end + + if ::RequestStore.active? + # These metrics measure total Redis usage per Rails request / job. + instrumentation_class.increment_request_count(commands.size) + instrumentation_class.add_duration(duration) + instrumentation_class.add_call_details(duration, commands) + end + end + + def measure_write_size(command, instrumentation_class) + size = 0 + + # Mimic what happens in + # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8. + # This count is an approximation that omits the Redis protocol overhead + # of type prefixes, length prefixes and line endings. + command.each do |x| + size += if x.is_a? Array + x.inject(0) { |sum, y| sum + y.to_s.bytesize } + else + x.to_s.bytesize + end + end + + instrumentation_class.increment_write_bytes(size) + end + + def measure_read_size(result, instrumentation_class) + # The Connection::Ruby#read class can return one of four types of results from read: + # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406 + # + # 1. Error (exception, will not reach this line) + # 2. Status (string) + # 3. Integer (will be converted to string by to_s.bytesize and thrown away) + # 4. "Binary" string (i.e. may contain zero byte) + # 5. Array of binary string + + if result.is_a? Array + # Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here. + result.each { |x| measure_read_size(x, instrumentation_class) } + else + # This count is an approximation that omits the Redis protocol overhead + # of type prefixes, length prefixes and line endings. + instrumentation_class.increment_read_bytes(result.to_s.bytesize) + end + end + + def exclude_from_apdex?(commands) + commands.any? { |command| APDEX_EXCLUDE.include?(command.first.to_s.downcase) } + end + end + end +end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 5934204bd0f..9c89af6a0dc 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -3,103 +3,45 @@ module Gitlab module Instrumentation module RedisInterceptor - APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax command xread xreadgroup].freeze + include RedisHelper def call(command) - instrument_call([command]) do + instrument_call([command], instrumentation_class) do super end end def call_pipeline(pipeline) - instrument_call(pipeline.commands, true) do + instrument_call(pipeline.commands, instrumentation_class, true) do super end end def write(command) - measure_write_size(command) if ::RequestStore.active? + measure_write_size(command, instrumentation_class) if ::RequestStore.active? super end def read result = super - measure_read_size(result) if ::RequestStore.active? + measure_read_size(result, instrumentation_class) if ::RequestStore.active? result end - private - - def instrument_call(commands, pipelined = false) - start = ::Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined - instrumentation_class.instance_count_request(commands.size) - instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined - - if !instrumentation_class.redis_cluster_validate!(commands) && ::RequestStore.active? - instrumentation_class.increment_cross_slot_request_count + def ensure_connected + super do + instrument_reconnection_errors do + yield + end end + end + def instrument_reconnection_errors yield - rescue ::Redis::BaseError => ex - if ex.message.start_with?('MOVED', 'ASK') - instrumentation_class.instance_count_cluster_redirection(ex) - else - instrumentation_class.instance_count_exception(ex) - end + rescue ::Redis::BaseConnectionError => ex + instrumentation_class.instance_count_connection_exception(ex) - instrumentation_class.log_exception(ex) raise ex - ensure - duration = ::Gitlab::Metrics::System.monotonic_time - start - - unless exclude_from_apdex?(commands) - commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } - end - - if ::RequestStore.active? - # These metrics measure total Redis usage per Rails request / job. - instrumentation_class.increment_request_count(commands.size) - instrumentation_class.add_duration(duration) - instrumentation_class.add_call_details(duration, commands) - end - end - - def measure_write_size(command) - size = 0 - - # Mimic what happens in - # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8. - # This count is an approximation that omits the Redis protocol overhead - # of type prefixes, length prefixes and line endings. - command.each do |x| - size += if x.is_a? Array - x.inject(0) { |sum, y| sum + y.to_s.bytesize } - else - x.to_s.bytesize - end - end - - instrumentation_class.increment_write_bytes(size) - end - - def measure_read_size(result) - # The Connection::Ruby#read class can return one of four types of results from read: - # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406 - # - # 1. Error (exception, will not reach this line) - # 2. Status (string) - # 3. Integer (will be converted to string by to_s.bytesize and thrown away) - # 4. "Binary" string (i.e. may contain zero byte) - # 5. Array of binary string - - if result.is_a? Array - # Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here. - result.each { |x| measure_read_size(x) } - else - # This count is an approximation that omits the Redis protocol overhead - # of type prefixes, length prefixes and line endings. - instrumentation_class.increment_read_bytes(result.to_s.bytesize) - end end # That's required so it knows which GitLab Redis instance @@ -108,10 +50,6 @@ module Gitlab def instrumentation_class @options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables end - - def exclude_from_apdex?(commands) - commands.any? { |command| APDEX_EXCLUDE.include?(command.first.to_s.downcase) } - end end end end diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb index e2e4ea75dbf..eb2ba3449fb 100644 --- a/lib/gitlab/internal_events.rb +++ b/lib/gitlab/internal_events.rb @@ -4,30 +4,61 @@ module Gitlab module InternalEvents UnknownEventError = Class.new(StandardError) InvalidPropertyError = Class.new(StandardError) - InvalidMethodError = Class.new(StandardError) + InvalidPropertyTypeError = Class.new(StandardError) class << self include Gitlab::Tracking::Helpers + include Gitlab::Utils::StrongMemoize def track_event(event_name, send_snowplow_event: true, **kwargs) raise UnknownEventError, "Unknown event: #{event_name}" unless EventDefinitions.known_event?(event_name) + validate_property!(kwargs, :user, User) + validate_property!(kwargs, :namespace, Namespaces::UserNamespace, Group) + validate_property!(kwargs, :project, Project) + + project = kwargs[:project] + kwargs[:namespace] ||= project.namespace if project + increase_total_counter(event_name) + increase_weekly_total_counter(event_name) update_unique_counter(event_name, kwargs) trigger_snowplow_event(event_name, kwargs) if send_snowplow_event + + if Feature.enabled?(:internal_events_for_product_analytics) + send_application_instrumentation_event(event_name, kwargs) + end rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs) + extra = {} + kwargs.each_key do |k| + extra[k] = kwargs[k].is_a?(::ApplicationRecord) ? kwargs[k].try(:id) : kwargs[k] + end + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: extra) nil end private + def validate_property!(kwargs, property_name, *class_names) + return unless kwargs.has_key?(property_name) + return if kwargs[property_name].nil? + return if class_names.include?(kwargs[property_name].class) + + raise InvalidPropertyTypeError, "#{property_name} should be an instance of #{class_names.join(', ')}" + end + def increase_total_counter(event_name) redis_counter_key = Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name) Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end + def increase_weekly_total_counter(event_name) + redis_counter_key = + Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name, Date.today) + Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } + end + def update_unique_counter(event_name, kwargs) unique_property = EventDefinitions.unique_property(event_name) return unless unique_property @@ -35,11 +66,9 @@ module Gitlab unique_method = :id unless kwargs.has_key?(unique_property) - raise InvalidPropertyError, "#{event_name} should be triggered with a named parameter '#{unique_property}'." - end - - unless kwargs[unique_property].respond_to?(unique_method) - raise InvalidMethodError, "'#{unique_property}' should have a '#{unique_method}' method." + message = "#{event_name} should be triggered with a named parameter '#{unique_property}'." + Gitlab::AppJsonLogger.warn(message: message) + return end unique_value = kwargs[unique_property].public_send(unique_method) # rubocop:disable GitlabSecurity/PublicSend @@ -75,6 +104,25 @@ module Gitlab Gitlab::ErrorTracking .track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: event_name) end + + def send_application_instrumentation_event(event_name, kwargs) + return if gitlab_sdk_client.nil? + + user = kwargs[:user] + + gitlab_sdk_client.identify(user&.id) + gitlab_sdk_client.track(event_name, { project_id: kwargs[:project]&.id, namespace_id: kwargs[:namespace]&.id }) + end + + def gitlab_sdk_client + app_id = ENV['GITLAB_ANALYTICS_ID'] + host = ENV['GITLAB_ANALYTICS_URL'] + + return unless app_id.present? && host.present? + + GitlabSDK::Client.new(app_id: app_id, host: host) + end + strong_memoize_attr :gitlab_sdk_client end end end diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index f11dd520d2d..13909ca2ce3 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -124,7 +124,7 @@ module Gitlab def cache_issues_count? @store_in_redis_cache && - finder.instance_of?(IssuesFinder) && + finder.class <= IssuesFinder && parent_group.present? && !params_include_filters? end @@ -134,7 +134,7 @@ module Gitlab end def redis_cache_key - ['group', parent_group&.id, 'issues'] + ['group', parent_group&.id, finder.klass.model_name.plural] end def cache_options @@ -143,8 +143,8 @@ module Gitlab def params_include_filters? non_filtering_params = %i[ - scope state sort group_id include_subgroups - attempt_group_search_optimizations non_archived issue_types + scope state sort group_id include_subgroups namespace_id + attempt_group_search_optimizations non_archived issue_types lookahead ] finder.params.except(*non_filtering_params).values.any? diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index fe244bd88a0..464c049ee52 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -19,13 +19,13 @@ module Gitlab raise ConfigurationError, 'KAS internal URL is not configured' unless Gitlab::Kas.internal_url.present? end - def get_connected_agents(project:) - request = Gitlab::Agent::AgentTracker::Rpc::GetConnectedAgentsRequest.new(project_id: project.id) + def get_connected_agents_by_agent_ids(agent_ids:) + request = Gitlab::Agent::AgentTracker::Rpc::GetConnectedAgentsByAgentIdsRequest.new(agent_ids: agent_ids) stub_for(:agent_tracker) - .get_connected_agents(request, metadata: metadata) - .agents - .to_a + .get_connected_agents_by_agent_ids(request, metadata: metadata) + .agents + .to_a end def list_agent_config_files(project:) diff --git a/lib/gitlab/markdown_cache/redis/extension.rb b/lib/gitlab/markdown_cache/redis/extension.rb index add71fa120e..19c14faa3d6 100644 --- a/lib/gitlab/markdown_cache/redis/extension.rb +++ b/lib/gitlab/markdown_cache/redis/extension.rb @@ -27,7 +27,7 @@ module Gitlab fields = Gitlab::MarkdownCache::Redis::Store.bulk_read(objects) objects.each do |object| - fields[object.cache_key].value.each do |field_name, value| + fields[object.cache_key].each do |field_name, value| object.write_markdown_field(field_name, value) end end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index f742cb82b8d..af9098c3300 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -9,16 +9,21 @@ module Gitlab def self.bulk_read(subjects) results = {} - Gitlab::Redis::Cache.with do |r| + data = Gitlab::Redis::Cache.with do |r| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| subjects.each do |subject| - results[subject.cache_key] = new(subject).read(pipeline) + new(subject).read(pipeline) end end end end + # enumerate data + data.each_with_index do |elem, idx| + results[subjects[idx].cache_key] = elem + end + results end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index cc335c00e26..ae567cb7d0e 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -69,10 +69,6 @@ module Gitlab end def handler - # This allows us to keep the watchdog running but turn it into "friendly mode" where - # all that happens is we collect logs and Prometheus events for fragmentation violations. - return Handlers::NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) - configuration.handler end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 80ce155321b..92a8a2b95c4 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -1,172 +1,11 @@ # frozen_string_literal: true +require 'gitlab/utils/system' + module Gitlab module Metrics - # Module for gathering system/process statistics such as the memory usage. - # - # This module relies on the /proc filesystem being available. If /proc is - # not available the methods of this module will be stubbed. module System - extend self - - PROC_STAT_PATH = '/proc/self/stat' - PROC_STATUS_PATH = '/proc/%s/status' - PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' - PROC_LIMITS_PATH = '/proc/self/limits' - PROC_FD_GLOB = '/proc/self/fd/*' - PROC_MEM_INFO = '/proc/meminfo' - - PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/ - PSS_PATTERN = /^Pss:\s+(?<value>\d+)/ - RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/ - RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/ - RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/ - MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/ - MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/ - - def summary - proportional_mem = memory_usage_uss_pss - { - version: RUBY_DESCRIPTION, - gc_stat: GC.stat, - memory_rss: memory_usage_rss[:total], - memory_uss: proportional_mem[:uss], - memory_pss: proportional_mem[:pss], - time_cputime: cpu_time, - time_realtime: real_time, - time_monotonic: monotonic_time - } - end - - # Returns the given process' RSS (resident set size) in bytes. - def memory_usage_rss(pid: 'self') - results = { total: 0, anon: 0, file: 0 } - - safe_yield_procfile(PROC_STATUS_PATH % pid) do |io| - io.each_line do |line| - if (value = parse_metric_value(line, RSS_TOTAL_PATTERN)) > 0 - results[:total] = value.kilobytes - elsif (value = parse_metric_value(line, RSS_ANON_PATTERN)) > 0 - results[:anon] = value.kilobytes - elsif (value = parse_metric_value(line, RSS_FILE_PATTERN)) > 0 - results[:file] = value.kilobytes - end - end - end - - results - end - - # Returns the given process' USS/PSS (unique/proportional set size) in bytes. - def memory_usage_uss_pss(pid: 'self') - sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) - .transform_values(&:kilobytes) - end - - def memory_total - sum_matches(PROC_MEM_INFO, memory_total: MEM_TOTAL_PATTERN)[:memory_total].kilobytes - end - - def file_descriptor_count - Dir.glob(PROC_FD_GLOB).length - end - - def max_open_file_descriptors - sum_matches(PROC_LIMITS_PATH, max_fds: MAX_OPEN_FILES_PATTERN)[:max_fds] - end - - def cpu_time - Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second) - end - - # Returns the current real time in a given precision. - # - # Returns the time as a Float for precision = :float_second. - def real_time(precision = :float_second) - Process.clock_gettime(Process::CLOCK_REALTIME, precision) - end - - # Returns the current monotonic clock time as seconds with microseconds precision. - # - # Returns the time as a Float. - def monotonic_time - Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - end - - def thread_cpu_time - # Not all OS kernels are supporting `Process::CLOCK_THREAD_CPUTIME_ID` - # Refer: https://gitlab.com/gitlab-org/gitlab/issues/30567#note_221765627 - return unless defined?(Process::CLOCK_THREAD_CPUTIME_ID) - - Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) - end - - def thread_cpu_duration(start_time) - end_time = thread_cpu_time - return unless start_time && end_time - - end_time - start_time - end - - # Returns the total time the current process has been running in seconds. - def process_runtime_elapsed_seconds - # Entry 22 (1-indexed) contains the process `starttime`, see: - # https://man7.org/linux/man-pages/man5/proc.5.html - # - # This value is a fixed timestamp in clock ticks. - # To obtain an elapsed time in seconds, we divide by the number - # of ticks per second and subtract from the system uptime. - start_time_ticks = proc_stat_entries[21].to_f - clock_ticks_per_second = Etc.sysconf(Etc::SC_CLK_TCK) - uptime - (start_time_ticks / clock_ticks_per_second) - end - - private - - # Given a path to a file in /proc and a hash of (metric, pattern) pairs, - # sums up all values found for those patterns under the respective metric. - def sum_matches(proc_file, **patterns) - results = patterns.transform_values { 0 } - - safe_yield_procfile(proc_file) do |io| - io.each_line do |line| - patterns.each do |metric, pattern| - results[metric] += parse_metric_value(line, pattern) - end - end - end - - results - end - - def parse_metric_value(line, pattern) - match = line.match(pattern) - return 0 unless match - - match.named_captures.fetch('value', 0).to_i - end - - def proc_stat_entries - safe_yield_procfile(PROC_STAT_PATH) do |io| - io.read.split(' ') - end || [] - end - - def safe_yield_procfile(path, &block) - File.open(path, &block) - rescue Errno::ENOENT - # This means the procfile we're reading from did not exist; - # most likely we're on Darwin. - end - - # Equivalent to reading /proc/uptime on Linux 2.6+. - # - # Returns 0 if not supported, e.g. on Darwin. - def uptime - Process.clock_gettime(Process::CLOCK_BOOTTIME) - rescue NameError - 0 - end + extend Gitlab::Utils::System end end end diff --git a/lib/gitlab/middleware/path_traversal_check.rb b/lib/gitlab/middleware/path_traversal_check.rb index 6fef247b708..d1260c81925 100644 --- a/lib/gitlab/middleware/path_traversal_check.rb +++ b/lib/gitlab/middleware/path_traversal_check.rb @@ -32,20 +32,24 @@ module Gitlab end def call(env) - if Feature.enabled?(:check_path_traversal_middleware, Feature.current_request) - log_params = {} + return @app.call(env) unless Feature.enabled?(:check_path_traversal_middleware, Feature.current_request) - execution_time = measure_execution_time do - request = ::Rack::Request.new(env.dup) - check(request, log_params) unless excluded?(request) - end + log_params = {} - log_params[:duration_ms] = execution_time.round(5) if execution_time + execution_time = measure_execution_time do + request = ::Rack::Request.new(env.dup) + check(request, log_params) unless excluded?(request) + end + log_params[:duration_ms] = execution_time.round(5) if execution_time + + result = @app.call(env) - log(log_params) unless log_params.empty? + unless log_params.empty? + log_params[:status] = result.first + log(log_params) end - @app.call(env) + result end private diff --git a/lib/gitlab/nav/top_nav_menu_builder.rb b/lib/gitlab/nav/top_nav_menu_builder.rb deleted file mode 100644 index dca3432a6a1..00000000000 --- a/lib/gitlab/nav/top_nav_menu_builder.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Nav - class TopNavMenuBuilder - def initialize - @primary = [] - @secondary = [] - @last_header_added = nil - end - - def add_primary_menu_item(header: nil, **args) - if header && (header != @last_header_added) - add_menu_header(dest: @primary, title: header) - @last_header_added = header - end - - add_menu_item(dest: @primary, **args) - end - - def add_secondary_menu_item(**args) - add_menu_item(dest: @secondary, **args) - end - - def build - { - primary: @primary, - secondary: @secondary - } - end - - private - - def add_menu_item(dest:, **args) - item = ::Gitlab::Nav::TopNavMenuItem.build(**args) - - dest.push(item) - end - - def add_menu_header(dest:, **args) - header = ::Gitlab::Nav::TopNavMenuHeader.build(**args) - - dest.push(header) - end - end - end -end diff --git a/lib/gitlab/nav/top_nav_menu_header.rb b/lib/gitlab/nav/top_nav_menu_header.rb deleted file mode 100644 index 520091dbd97..00000000000 --- a/lib/gitlab/nav/top_nav_menu_header.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Nav - class TopNavMenuHeader - def self.build(title:) - { - type: :header, - title: title - } - end - end - end -end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index e7790fd77d0..f6fea97dae9 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -21,7 +21,7 @@ module Gitlab href: href, view: view.to_s, css_class: css_class, - data: data || { testid: 'menu_item_link', qa_title: title }, + data: data || { testid: 'menu-item-link', qa_title: title }, partial: partial, component: component } diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb deleted file mode 100644 index 10b841f777e..00000000000 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Nav - class TopNavViewModelBuilder - def initialize - @menu_builder = ::Gitlab::Nav::TopNavMenuBuilder.new - @views = {} - @shortcuts = [] - end - - # Using delegate hides the stacktrace for some errors, so we choose to be explicit. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62047#note_579031091 - def add_primary_menu_item(...) - @menu_builder.add_primary_menu_item(...) - end - - def add_secondary_menu_item(...) - @menu_builder.add_secondary_menu_item(...) - end - - def add_shortcut(**args) - item = ::Gitlab::Nav::TopNavMenuItem.build(**args) - - @shortcuts.push(item) - end - - def add_primary_menu_item_with_shortcut(shortcut_class:, shortcut_href: nil, **args) - add_primary_menu_item(**args) - add_shortcut( - id: "#{args.fetch(:id)}-shortcut", - title: args.fetch(:title), - href: shortcut_href || args.fetch(:href), - css_class: shortcut_class - ) - end - - def add_view(name, props) - @views[name] = props - end - - def build - menu = @menu_builder.build - - menu.merge({ - views: @views, - shortcuts: @shortcuts, - menuTooltip: _('Main menu') - }.compact) - end - end - end -end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 81ad7a7f9e1..1835aef755f 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -29,6 +29,8 @@ module Gitlab { authorize_params: { gl_auth_type: 'login' } } + when ->(provider_name) { AuthHelper.saml_providers.include?(provider_name.to_sym) } + { attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements } else {} end @@ -61,7 +63,7 @@ module Gitlab provider_arguments.concat arguments provider_arguments << defaults unless defaults.empty? when Hash, GitlabSettings::Options - hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) + hash_arguments = merge_hash_defaults_and_args(defaults, arguments) normalized = normalize_hash_arguments(hash_arguments) # A Hash from the configuration will be passed as is. @@ -80,6 +82,15 @@ module Gitlab provider_arguments end + def merge_hash_defaults_and_args(defaults, arguments) + return arguments.to_hash if defaults.empty? + + revert_merging = Gitlab::Utils.to_boolean(ENV['REVERT_OMNIAUTH_DEFAULT_MERGING']) + return arguments.to_hash.deep_symbolize_keys.deep_merge(defaults) if revert_merging + + defaults.deep_merge(arguments.deep_symbolize_keys) + end + def normalize_hash_arguments(args) args.deep_symbolize_keys! diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb index bf6ac3a056d..a572a59b2f5 100644 --- a/lib/gitlab/pages/deployment_update.rb +++ b/lib/gitlab/pages/deployment_update.rb @@ -92,6 +92,7 @@ module Gitlab # If a newer pipeline already build a PagesDeployment def validate_outdated_sha return if latest? + return if latest_pipeline_id.blank? return if latest_pipeline_id <= build.pipeline_id errors.add(:base, 'build SHA is outdated for this ref') diff --git a/lib/gitlab/pages/url_builder.rb b/lib/gitlab/pages/url_builder.rb index 5a28a5ffd23..f01ec54b853 100644 --- a/lib/gitlab/pages/url_builder.rb +++ b/lib/gitlab/pages/url_builder.rb @@ -14,6 +14,7 @@ module Gitlab end def pages_url(with_unique_domain: false) + return namespace_in_path_url(with_unique_domain && unique_domain_enabled?) if config.namespace_in_path return unique_url if with_unique_domain && unique_domain_enabled? project_path_url = "#{config.protocol}://#{project_path}".downcase @@ -29,6 +30,7 @@ module Gitlab def unique_host return unless unique_domain_enabled? + return if config.namespace_in_path URI(unique_url).host end @@ -40,9 +42,11 @@ module Gitlab def artifact_url(artifact, job) return unless artifact_url_available?(artifact, job) + host_url = config.namespace_in_path ? "#{pages_base_url}/#{project_namespace}" : namespace_url + format( ARTIFACT_URL, - host: namespace_url, + host: host_url, project_path: project_path, job_id: job.id, artifact_path: artifact.path) @@ -67,6 +71,21 @@ module Gitlab @unique_url ||= url_for(project.project_setting.pages_unique_domain) end + def pages_base_url + @pages_url ||= URI(config.url) + .tap { |url| url.port = config.port } + .to_s + .downcase + end + + def namespace_in_path_url(with_unique_domain) + if with_unique_domain + "#{pages_base_url}/#{project.project_setting.pages_unique_domain}".downcase + else + "#{pages_base_url}/#{project_namespace}/#{project_path}".downcase + end + end + def url_for(subdomain) URI(config.url) .tap { |url| url.port = config.port } diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 9e8c0c530a9..b5cc127d232 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -3,20 +3,6 @@ module Gitlab module Pagination module CursorBasedKeyset - SUPPORTED_MULTI_ORDERING = { - Group => { name: [:asc] }, - AuditEvent => { id: [:desc] }, - User => { - id: [:asc, :desc], - name: [:asc, :desc], - username: [:asc, :desc], - created_at: [:asc, :desc], - updated_at: [:asc, :desc] - }, - ::Ci::Build => { id: [:desc] }, - ::Packages::BuildInfo => { id: [:desc] } - }.freeze - # Relation types that are enforced in this list # enforce the use of keyset pagination, thus erroring out requests # made with offset pagination above a certain limit. @@ -26,7 +12,7 @@ module Gitlab ENFORCED_TYPES = [Group].freeze def self.available_for_type?(relation) - SUPPORTED_MULTI_ORDERING.key?(relation.klass) + relation.klass.respond_to?(:supported_keyset_orderings) end def self.available?(cursor_based_request_context, relation) @@ -44,7 +30,7 @@ module Gitlab order_by_from_request = cursor_based_request_context.order sort_from_request = cursor_based_request_context.sort - SUPPORTED_MULTI_ORDERING[relation.klass][order_by_from_request]&.include?(sort_from_request) + !!relation.klass.supported_keyset_orderings[order_by_from_request]&.include?(sort_from_request) end private_class_method :order_satisfied? end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 82d6fc64d89..a1c340baf23 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -64,7 +64,7 @@ module Gitlab def paginate_via_gitaly(finder) finder.execute(gitaly_pagination: true).tap do |records| - apply_headers(records) + apply_headers(records, finder.next_cursor) end end @@ -82,20 +82,18 @@ module Gitlab end end - def apply_headers(records) + def apply_headers(records, next_cursor) if records.count == params[:per_page] Gitlab::Pagination::Keyset::HeaderBuilder .new(request_context) .add_next_page_header( - query_params_for(records.last) + query_params_for(next_cursor) ) end end - def query_params_for(record) - # NOTE: page_token is name for now, but it could be dynamic if we have other gitaly finders - # that is based on something other than name - { page_token: record.name } + def query_params_for(next_cursor) + { page_token: next_cursor } end end end diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb index 8f1fbf53161..3f962c47ae9 100644 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -11,7 +11,7 @@ if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.12') raise 'New version of sidekiq detected, please remove or update this patch' end -if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.8.0') +if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.12.0') raise 'New version of sidekiq-cron detected, please remove or update this patch' end diff --git a/lib/gitlab/patch/sidekiq_scheduled_enq.rb b/lib/gitlab/patch/sidekiq_scheduled_enq.rb deleted file mode 100644 index b5a40c19923..00000000000 --- a/lib/gitlab/patch/sidekiq_scheduled_enq.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2286 -# Using a dual-namespace poller eliminates the need for script based migration of -# schedule-related sets in Sidekiq. -module Gitlab - module Patch - module SidekiqScheduledEnq - # The patched enqueue_jobs will poll non-namespaced scheduled sets before doing the same for - # namespaced sets via super and vice-versa depending on how Sidekiq.redis was configured - def enqueue_jobs(sorted_sets = Sidekiq::Scheduled::SETS) - # checks the other namespace - if Gitlab::Utils.to_boolean(ENV['SIDEKIQ_ENABLE_DUAL_NAMESPACE_POLLING'], default: true) - # Refer to https://github.com/sidekiq/sidekiq/blob/v6.5.7/lib/sidekiq/scheduled.rb#L25 - # this portion swaps out Sidekiq.redis for Gitlab::Redis::Queues - Gitlab::Redis::Queues.with do |conn| # rubocop:disable Cop/RedisQueueUsage - sorted_sets.each do |sorted_set| - # adds namespace since `super` polls with a non-namespaced Sidekiq.redis - sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage - - while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s])) # rubocop:disable Gitlab/ModuleWithInstanceVariables, Lint/AssignmentInCondition - Sidekiq::Client.push(Sidekiq.load_json(job)) # rubocop:disable Cop/SidekiqApiUsage - Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" } - end - end - end - end - - super - end - end - end -end diff --git a/lib/gitlab/puma/error_handler.rb b/lib/gitlab/puma/error_handler.rb index 4efc4866431..9eabe0731e2 100644 --- a/lib/gitlab/puma/error_handler.rb +++ b/lib/gitlab/puma/error_handler.rb @@ -18,10 +18,11 @@ module Gitlab # https://github.com/puma/puma/pull/3094 status_code ||= 500 - if Raven.configuration.capture_allowed? - Raven.capture_exception(ex, tags: { handler: 'puma_low_level' }, - extra: { puma_env: env, status_code: status_code }) - end + Gitlab::ErrorTracking.track_exception( + ex, + { puma_env: env, status_code: status_code }, + { handler: 'puma_low_level' } + ) # note the below is just a Rack response [status_code, {}, message] diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 5cf79db83af..c6a7a39a943 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -9,19 +9,6 @@ module Gitlab # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels]) # ``` class Extractor - CODE_REGEX = %r{ - (?<code> - # Code blocks: - # ``` - # Anything, including `/cmd arg` which are ignored by this filter - # ``` - - ^``` - .+? - \n```$ - ) - }mix - INLINE_CODE_REGEX = %r{ (?<inline_code> # Inline code on separate rows: @@ -46,22 +33,7 @@ module Gitlab ) }mix - QUOTE_BLOCK_REGEX = %r{ - (?<html> - # Quote block: - # >>> - # Anything, including `/cmd arg` which are ignored by this filter - # >>> - - ^>>> - .+? - \n>>>$ - ) - }mix - - EXCLUSION_REGEX = %r{ - #{CODE_REGEX} | #{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX} | #{QUOTE_BLOCK_REGEX} - }mix + EXCLUSION_REGEX = %r{#{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX}}mix attr_reader :command_definitions, :keep_actions @@ -119,15 +91,40 @@ module Gitlab content = content.dup content.delete!("\r") - content.gsub!(commands_regex(names: names, sub_names: sub_names)) do - command, output = if $~[:substitution] - process_substitutions($~) - else - process_commands($~, redact) - end + # use a markdown based pipeline to grab possible paragraphs that might + # contain quick actions. This ensures they are not in HTML blocks, quote blocks, + # or code blocks. + pipeline = Banzai::Pipeline::QuickActionPipeline.html_pipeline + possible_paragraphs = pipeline.call(content, {}, {})[:quick_action_paragraphs] + + if possible_paragraphs.present? + content_lines = content.lines + + # Each paragraph that possibly contains quick actions must be searched. In order + # to use the `sourcepos` information, we need to convert into individual lines, + # and then replace the specific lines. + possible_paragraphs.each do |possible| + endpos = possible[:end_line] + endpos += 1 if content_lines[endpos + 1] == "\n" + + paragraph = content_lines[possible[:start_line]..endpos].join + + paragraph.gsub!(commands_regex(names: names, sub_names: sub_names)) do + command, output = if $~[:substitution] + process_substitutions($~) + else + process_commands($~, redact) + end + + commands << command + output + end + + content_lines.fill('', possible[:start_line]..endpos) + content_lines[possible[:start_line]] = paragraph + end - commands << command - output + content = content_lines.join end [content.rstrip, commands.reject(&:empty?)] @@ -181,7 +178,7 @@ module Gitlab #{EXCLUSION_REGEX} | (?: - # Command not in a blockquote, blockcode, or HTML tag: + # Command such as: # /close ^\/ @@ -194,7 +191,8 @@ module Gitlab ) | (?: - # Substitution not in a blockquote, blockcode, or HTML tag: + # Substitution such as: + # /shrug ^\/ (?<substitution>#{Regexp.new(Regexp.union(sub_names).source, Regexp::IGNORECASE)}) diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 57ed6c5c35e..2f7fa89019e 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -213,7 +213,7 @@ module Gitlab match = emoji_param.match(Banzai::Filter::EmojiFilter.emoji_pattern) match[1] if match end - command :award do |name| + command :award, :react do |name| if name && quick_action_target.user_can_award?(current_user) @updates[:emoji_award] = name end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index ae79db723f2..c79432f36cc 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -226,28 +226,18 @@ module Gitlab params 'email1@example.com email2@example.com (up to 6 emails)' types Issue condition do - Feature.enabled?(:issue_email_participants, parent) && + quick_action_target.persisted? && + Feature.enabled?(:issue_email_participants, parent) && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end command :invite_email do |emails = ""| - MAX_NUMBER_OF_EMAILS = 6 - - existing_emails = quick_action_target.email_participants_emails_downcase - emails_to_add = emails.split(' ').index_by { |email| [email.downcase, email] }.except(*existing_emails).each_value.first(MAX_NUMBER_OF_EMAILS) - added_emails = [] - - emails_to_add.each do |email| - new_participant = quick_action_target.issue_email_participants.create(email: email) - added_emails << email if new_participant.persisted? - end + response = ::IssueEmailParticipants::CreateService.new( + target: quick_action_target, + current_user: current_user, + emails: emails.split(' ') + ).execute - if added_emails.any? - message = _("added %{emails}") % { emails: added_emails.to_sentence } - SystemNoteService.add_email_participants(quick_action_target, quick_action_target.project, current_user, message) - @execution_message[:invite_email] = message.upcase_first << "." - else - @execution_message[:invite_email] = _("No email participants were added. Either none were provided, or they already exist.") - end + @execution_message[:invite_email] = response.message end desc { _('Promote issue to incident') } diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 72bec159226..fe18bc8e133 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -161,10 +161,25 @@ module Gitlab condition do quick_action_target.persisted? end - command :submit_review do + command :submit_review do |state = "reviewed"| next if params[:review_id] result = DraftNotes::PublishService.new(quick_action_target, current_user).execute + + if Feature.enabled?(:mr_request_changes, current_user) + reviewer_state = state.strip.presence + + if reviewer_state === 'approve' + ::MergeRequests::ApprovalService + .new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target) + elsif MergeRequestReviewer.states.key?(reviewer_state) + ::MergeRequests::UpdateReviewerStateService + .new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target, reviewer_state) + end + end + @execution_message[:submit_review] = if result[:status] == :success _('Submitted the current review.') else diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9f7599d2500..a29c37411c3 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -8,6 +8,7 @@ module Gitlab # This will make sure the connection pool is initialized on application boot in # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. ALL_CLASSES = [ + Gitlab::Redis::BufferedCounter, Gitlab::Redis::Cache, Gitlab::Redis::ClusterSharedState, Gitlab::Redis::DbLoadBalancing, diff --git a/lib/gitlab/redis/buffered_counter.rb b/lib/gitlab/redis/buffered_counter.rb new file mode 100644 index 00000000000..21fc4ba8034 --- /dev/null +++ b/lib/gitlab/redis/buffered_counter.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class BufferedCounter < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + SharedState + end + end + end + end +end diff --git a/lib/gitlab/redis/db_load_balancing.rb b/lib/gitlab/redis/db_load_balancing.rb index 01276445611..f6769a39397 100644 --- a/lib/gitlab/redis/db_load_balancing.rb +++ b/lib/gitlab/redis/db_load_balancing.rb @@ -8,15 +8,6 @@ module Gitlab def config_fallback SharedState end - - private - - def redis - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, store_name) - end end end end diff --git a/lib/gitlab/redis/sidekiq_status.rb b/lib/gitlab/redis/sidekiq_status.rb deleted file mode 100644 index 9b8bbf5a0ad..00000000000 --- a/lib/gitlab/redis/sidekiq_status.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - # Pseudo-store to transition `Gitlab::SidekiqStatus` from - # using `Sidekiq.redis` to using the `SharedState` redis store. - class SidekiqStatus < ::Gitlab::Redis::Wrapper - class << self - def store_name - 'SharedState' - end - - private - - def redis - primary_store = ::Redis.new(Gitlab::Redis::SharedState.params) - secondary_store = ::Redis.new(Gitlab::Redis::Queues.params) # rubocop:disable Cop/RedisQueueUsage - - MultiStore.new(primary_store, secondary_store, name.demodulize) - end - end - end - end -end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index c1f346ec7e4..bb231eec226 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -30,7 +30,7 @@ module Gitlab end def pool - @pool ||= ConnectionPool.new(size: pool_size) { redis } + @pool ||= ConnectionPool.new(size: pool_size, name: store_name.underscore) { redis } end def pool_size @@ -83,8 +83,6 @@ module Gitlab "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end - private - def redis ::Redis.new(params) end diff --git a/lib/gitlab/registration_features/password_complexity.rb b/lib/gitlab/registration_features/password_complexity.rb deleted file mode 100644 index 6d165a7a665..00000000000 --- a/lib/gitlab/registration_features/password_complexity.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RegistrationFeatures - class PasswordComplexity - def self.feature_available? - ::License.feature_available?(:password_complexity) || - ::GitlabSubscriptions::Features.usage_ping_feature?(:password_complexity) - end - end - end -end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index d5e80053772..3a389d3363f 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,8 +6,7 @@ module Gitlab module RequestForgeryProtection - # rubocop:disable Rails/ApplicationController - class Controller < ActionController::Base + class Controller < BaseActionController protect_from_forgery with: :exception, prepend: true def initialize @@ -40,6 +39,5 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end - # rubocop:enable Rails/ApplicationController end end diff --git a/lib/gitlab/seeders/ci/catalog/resource_seeder.rb b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb index 2971dabe044..c29075cff32 100644 --- a/lib/gitlab/seeders/ci/catalog/resource_seeder.rb +++ b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb @@ -5,25 +5,23 @@ module Gitlab module Ci module Catalog class ResourceSeeder - # This is currently disabled until it gets fixed: https://gitlab.com/gitlab-org/gitlab/-/issues/429649 # Initializes the class # # @param [String] Path of the group to find # @param [Integer] Number of resources to create - def initialize(group_path:, seed_count:) + # @param[Boolean] If the created resources should be published or not, defaults to false + def initialize(group_path:, seed_count:, publish:) @group = Group.find_by_full_path(group_path) @seed_count = seed_count + @publish = publish @current_user = @group&.first_owner end def seed - if @group.nil? - warn 'ERROR: Group was not found.' - return - end + return warn 'ERROR: Group was not found.' if @group.nil? @seed_count.times do |i| - create_ci_catalog_resource(i) + seed_catalog_resource(i) end end @@ -59,9 +57,16 @@ module Gitlab stage: $[[ inputs.stage ]] YAML + project.repository.create_dir( + @current_user, + 'templates', + message: 'Add template dir', + branch_name: project.default_branch_or_main + ) + project.repository.create_file( @current_user, - 'template.yml', + 'templates/component.yml', template_content, message: 'Add template.yml', branch_name: project.default_branch_or_main @@ -78,21 +83,22 @@ module Gitlab ) end - def create_ci_catalog(project) + def create_catalog_resource(project) result = ::Ci::Catalog::Resources::CreateService.new(project, @current_user).execute if result.success? result.payload else - warn "Project '#{project.name}' could not be converted to a Catalog resource" + warn "Catalog resource could not be created for Project '#{project.name}': #{result.errors.join}" nil end end - def create_ci_catalog_resource(index) + def seed_catalog_resource(index) name = "ci_seed_resource_#{index}" + existing_project = Project.find_by_name(name) - if Project.find_by_name(name).present? - warn "Project '#{name}' already exists!" + if existing_project.present? && existing_project.group.path == @group.path + warn "Project '#{@group.path}/#{name}' already exists!" return end @@ -103,9 +109,12 @@ module Gitlab create_readme(project, index) create_template_yml(project) - return unless create_ci_catalog(project) + new_catalog_resource = create_catalog_resource(project) + return unless new_catalog_resource + + warn "Project '#{@group.path}/#{name}' was saved successfully!" - warn "Project '#{name}' was saved successfully!" + new_catalog_resource.publish! if @publish end end end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index e1c155a4848..96bda86ab08 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -37,6 +37,7 @@ module Gitlab chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server chain.add ::Gitlab::SidekiqMiddleware::PauseControl::Server + chain.add ::ClickHouse::MigrationSupport::SidekiqMiddleware # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 10a69acc037..883e1ba0558 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -20,8 +20,7 @@ module Gitlab class DuplicateJob include Gitlab::Utils::StrongMemoize - DEFAULT_DUPLICATE_KEY_TTL = 6.hours - SHORT_DUPLICATE_KEY_TTL = 10.minutes + DEFAULT_DUPLICATE_KEY_TTL = 10.minutes DEFAULT_STRATEGY = :until_executing STRATEGY_NONE = :none @@ -75,7 +74,8 @@ module Gitlab argv = [] job_wal_locations.each do |connection_name, location| - argv += [connection_name, pg_wal_lsn_diff(connection_name), location] + diff = pg_wal_lsn_diff(connection_name) + argv += [connection_name, diff || '', location] end with_redis { |r| r.eval(UPDATE_WAL_COOKIE_SCRIPT, keys: [cookie_key], argv: argv) } @@ -174,7 +174,7 @@ module Gitlab end def duplicate_key_ttl - options[:ttl] || default_duplicate_key_ttl + options[:ttl] || DEFAULT_DUPLICATE_KEY_TTL end private @@ -183,12 +183,6 @@ module Gitlab attr_reader :queue_name, :job attr_writer :existing_jid - def default_duplicate_key_ttl - return SHORT_DUPLICATE_KEY_TTL if Feature.enabled?(:reduce_duplicate_job_key_ttl) - - DEFAULT_DUPLICATE_KEY_TTL - end - def worker_klass @worker_klass ||= worker_class_name.to_s.safe_constantize end diff --git a/lib/gitlab/sidekiq_middleware/pause_control.rb b/lib/gitlab/sidekiq_middleware/pause_control.rb index 2f0fd0cc799..8f4da7267d7 100644 --- a/lib/gitlab/sidekiq_middleware/pause_control.rb +++ b/lib/gitlab/sidekiq_middleware/pause_control.rb @@ -8,6 +8,7 @@ module Gitlab UnknownStrategyError = Class.new(StandardError) STRATEGIES = { + click_house_migration: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::ClickHouseMigration, zoekt: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::Zoekt, none: ::Gitlab::SidekiqMiddleware::PauseControl::Strategies::None }.freeze diff --git a/lib/gitlab/sidekiq_middleware/pause_control/server.rb b/lib/gitlab/sidekiq_middleware/pause_control/server.rb index cfa02b3ec3a..7beb5f9ca5b 100644 --- a/lib/gitlab/sidekiq_middleware/pause_control/server.rb +++ b/lib/gitlab/sidekiq_middleware/pause_control/server.rb @@ -4,8 +4,8 @@ module Gitlab module SidekiqMiddleware module PauseControl class Server - def call(worker_class, job, _queue, &block) - ::Gitlab::SidekiqMiddleware::PauseControl::StrategyHandler.new(worker_class, job).perform(&block) + def call(worker, job, _queue, &block) + ::Gitlab::SidekiqMiddleware::PauseControl::StrategyHandler.new(worker, job).perform(&block) end end end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/strategies/click_house_migration.rb b/lib/gitlab/sidekiq_middleware/pause_control/strategies/click_house_migration.rb new file mode 100644 index 00000000000..adeb0524567 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/pause_control/strategies/click_house_migration.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + module PauseControl + module Strategies + class ClickHouseMigration < Base + override :should_pause? + def should_pause? + return false unless Feature.enabled?(:pause_clickhouse_workers_during_migration) + + ClickHouse::MigrationSupport::ExclusiveLock.pause_workers? + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb b/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb index dc6aff92f50..97080dc91fc 100644 --- a/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb +++ b/lib/gitlab/sidekiq_middleware/pause_control/workers_map.rb @@ -17,7 +17,8 @@ module Gitlab def strategy_for(worker:) return unless @workers - @workers.find { |_, v| v.include?(worker) }&.first + worker_class = worker.is_a?(Class) ? worker : worker.class + @workers.find { |_, v| v.include?(worker_class) }&.first end end end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index ae4aca7ff92..496ed9de828 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -50,6 +50,16 @@ module Gitlab end end + # Refreshes the timeout on the key if it exists + # + # jid = The Sidekiq job ID + # expire - The expiration time of the Redis key. + def self.expire(jid, expire = DEFAULT_EXPIRATION) + with_redis do |redis| + redis.expire(key_for(jid), expire) + end + end + # Returns true if all the given job have been completed. # # job_ids - The Sidekiq job IDs to check. @@ -132,7 +142,8 @@ module Gitlab Feature.enabled?(:use_primary_store_as_default_for_sidekiq_status) # TODO: Swap for Gitlab::Redis::SharedState after store transition # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/923 - Gitlab::Redis::SidekiqStatus.with { |redis| yield redis } + # For now, we use SharedState to reduce amount of spawned connection to Redis Cluster during initialisation + Gitlab::Redis::SharedState.with { |redis| yield redis } else # Keep the old behavior intact if neither feature flag is turned on Sidekiq.redis { |redis| yield redis } # rubocop:disable Cop/SidekiqRedisCall diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 78c0f04e07e..038808667f4 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -149,12 +149,6 @@ module Gitlab end end - def repository_storage_paths_args - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path } - end - end - def user_home Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 3bbcd59f45e..0b606b712c7 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -6,10 +6,16 @@ module Gitlab module Tracking class << self + delegate :flush, to: :tracker + def enabled? tracker.enabled? end + def micro_verification_enabled? + Gitlab::Utils.to_boolean(ENV['VERIFY_TRACKING'], default: false) + end + def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists action = action.to_s @@ -66,7 +72,7 @@ module Gitlab end def snowplow_micro_enabled? - Rails.env.development? && Gitlab.config.snowplow_micro.enabled + (Rails.env.development? || micro_verification_enabled?) && Gitlab.config.snowplow_micro.enabled rescue GitlabSettings::MissingSetting false end diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb index e15c03b6808..1fc4b4e6d9c 100644 --- a/lib/gitlab/tracking/destinations/snowplow_micro.rb +++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb @@ -7,6 +7,8 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize extend ::Gitlab::Utils::Override + delegate :flush, to: :tracker + DEFAULT_URI = 'http://localhost:9090' override :options diff --git a/lib/gitlab/tracking/event_definition.rb b/lib/gitlab/tracking/event_definition.rb index 928eb6338f6..9d197de454e 100644 --- a/lib/gitlab/tracking/event_definition.rb +++ b/lib/gitlab/tracking/event_definition.rb @@ -17,9 +17,7 @@ module Gitlab end def definitions - paths.each_with_object({}) do |glob_path, definitions| - load_all_from_path!(definitions, glob_path) - end + paths.flat_map { |glob_path| load_all_from_path(glob_path) } end private @@ -34,11 +32,8 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Tracking::InvalidEventError.new(e.message)) end - def load_all_from_path!(definitions, glob_path) - Dir.glob(glob_path).each do |path| - definition = load_from_file(path) - definitions[definition.path] = definition - end + def load_all_from_path(glob_path) + Dir.glob(glob_path).map { |path| load_from_file(path) } end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 8f2dfce67bb..8164cc4524a 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -11,6 +11,7 @@ require 'ipaddress' module Gitlab class UrlBlocker + GETADDRINFO_TIMEOUT_SECONDS = 15 DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT = proc { deny_all_requests_except_allowed_app_setting }.freeze # Result stores the validation result: @@ -181,12 +182,16 @@ module Gitlab # # @return [Array<Addrinfo>] def get_address_info(uri) - Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr| - addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + Timeout.timeout(GETADDRINFO_TIMEOUT_SECONDS) do + Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end end - rescue ArgumentError => error + rescue Timeout::Error => e + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, e.message + rescue ArgumentError => e # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. - raise unless error.message.include?('hostname too long') + raise unless e.message.include?('hostname too long') raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Host is too long (maximum is 1024 characters)" end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index d7e983d126a..1efd8ded77c 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -37,11 +37,8 @@ module Gitlab ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value) end - def instrumentation_class - "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" - end - def instrumentation_object + instrumentation_class = "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" @instrumentation_object ||= instrumentation_class.constantize.new(definition.attributes) end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 941c2f793c4..5eddf8da7dd 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -25,6 +25,14 @@ module Gitlab events_from_new_structure || events_from_old_structure || {} end + def instrumentation_class + if internal_events? + events.each_value.first.nil? ? "TotalCountMetric" : "RedisHLLMetric" + else + attributes[:instrumentation_class] + end + end + def to_context return unless %w[redis redis_hll].include?(data_source) @@ -77,6 +85,10 @@ module Gitlab VALID_SERVICE_PING_STATUSES.include?(attributes[:status]) end + def internal_events? + data_source == 'internal_events' + end + alias_method :to_dictionary, :to_h class << self @@ -97,7 +109,9 @@ module Gitlab end def with_instrumentation_class - all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? } + all.select do |definition| + (definition.internal_events? || definition.attributes[:instrumentation_class].present?) && definition.available? + end end def context_for(key_path) diff --git a/lib/gitlab/usage/metrics/instrumentations/bulk_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/bulk_imports_users_metric.rb new file mode 100644 index 00000000000..453e9a13765 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/bulk_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class BulkImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::BulkImport + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_service_desk_custom_email_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_service_desk_custom_email_enabled_metric.rb new file mode 100644 index 00000000000..85f59f36941 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_service_desk_custom_email_enabled_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountServiceDeskCustomEmailEnabledMetric < DatabaseMetric + operation :count + + relation do + ServiceDeskSetting.where(custom_email_enabled: true) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/csv_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/csv_imports_users_metric.rb new file mode 100644 index 00000000000..16f498fbc5a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/csv_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CsvImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::Issues::CsvImport + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 774f65da3bf..0a47045aab5 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -32,9 +32,9 @@ module Gitlab super(metric_definition.reverse_merge(time_frame: 'none')) end - def value(...) + def value alt_usage_data(fallback: self.class.fallback) do - self.class.metric_value.call(...) + instance_eval(&self.class.metric_value) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_config_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_config_metric.rb new file mode 100644 index 00000000000..daeef06e6c5 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_config_metric.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabConfigMetric < GenericMetric + value do + method_name_array = config_hash_to_method_array(options[:config]) + + method_name_array.inject(Gitlab.config, :public_send) + end + + private + + def config_hash_to_method_array(object) + object.each_with_object([]) do |(key, value), result| + result.append(key) + + if value.is_a?(Hash) + result.concat(config_hash_to_method_array(value)) + else + result.append(value) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_settings_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_settings_metric.rb new file mode 100644 index 00000000000..6a36b69e287 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_settings_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabSettingsMetric < GenericMetric + value do + # rubocop:disable GitlabSecurity/PublicSend -- this is on static data and not a user-controlled input + Gitlab::CurrentSettings.public_send(options[:setting_method]) + # rubocop:enable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/group_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/group_imports_users_metric.rb new file mode 100644 index 00000000000..a1207300c2a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/group_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GroupImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::GroupImportState + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb deleted file mode 100644 index b1a2de29fd7..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class InProductMarketingEmailCtaClickedMetric < DatabaseMetric - operation :count - - def initialize(metric_definition) - super - - unless track.in?(allowed_track) - raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" - end - - return if series.in?(allowed_series) - - raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" - end - - relation { Users::InProductMarketingEmail } - - private - - def relation - scope = super.where.not(cta_clicked_at: nil) - scope = scope.where(series: series) - scope.where(track: track) - end - - def track - options[:track] - end - - def series - options[:series] - end - - def allowed_track - Users::InProductMarketingEmail::ACTIVE_TRACKS.keys - end - - def allowed_series - @allowed_series ||= begin - series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) - 0.upto(series_amount - 1).to_a - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb deleted file mode 100644 index 50dec606d9b..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class InProductMarketingEmailSentMetric < DatabaseMetric - operation :count - - def initialize(metric_definition) - super - - unless track.in?(allowed_track) - raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" - end - - return if series.in?(allowed_series) - - raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" - end - - relation { Users::InProductMarketingEmail } - - private - - def relation - scope = super - scope = scope.where(series: series) - scope.where(track: track) - end - - def track - options[:track] - end - - def series - options[:series] - end - - def allowed_track - Users::InProductMarketingEmail::ACTIVE_TRACKS.keys - end - - def allowed_series - @allowed_series ||= begin - series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) - 0.upto(series_amount - 1).to_a - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb index 2ce7e95ce77..b5c3420b5fe 100644 --- a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb @@ -18,26 +18,24 @@ module Gitlab end end - class << self - private + private - def database - database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] - Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) - end + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) + end - def structure_sql - stucture_sql_path = Rails.root.join('db/structure.sql') - Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) - end + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) + end - def validators - [ - Gitlab::Schema::Validation::Validators::MissingIndexes, - Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes, - Gitlab::Schema::Validation::Validators::ExtraIndexes - ] - end + def validators + [ + Gitlab::Schema::Validation::Validators::MissingIndexes, + Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes, + Gitlab::Schema::Validation::Validators::ExtraIndexes + ] end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/jira_imports_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/jira_imports_users_metric.rb new file mode 100644 index 00000000000..239df5605ae --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/jira_imports_users_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class JiraImportsUsersMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + ::JiraImportState + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/omniauth_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/omniauth_enabled_metric.rb new file mode 100644 index 00000000000..d6496da569a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/omniauth_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class OmniauthEnabledMetric < GenericMetric + value do + Gitlab::Auth.omniauth_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/project_imports_creators_metric.rb b/lib/gitlab/usage/metrics/instrumentations/project_imports_creators_metric.rb new file mode 100644 index 00000000000..f34bd6dbfe3 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/project_imports_creators_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ProjectImportsCreatorsMetric < DatabaseMetric + operation :distinct_count, column: :creator_id + + relation do + ::Project.where.not(import_type: nil) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_enabled_metric.rb new file mode 100644 index 00000000000..d5d07637a9c --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class PrometheusEnabledMetric < GenericMetric + value do + Gitlab::Prometheus::Internal.prometheus_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb index ab1298b63c3..cc6be7fb349 100644 --- a/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb @@ -18,7 +18,7 @@ module Gitlab # end def value with_prometheus_client(verify: false, fallback: FALLBACK) do |client| - super(client) + self.class.metric_value.call(client) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_metrics_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_metrics_enabled_metric.rb new file mode 100644 index 00000000000..76f9c7d2588 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_metrics_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class PrometheusMetricsEnabledMetric < GenericMetric + value do + Gitlab::Metrics.prometheus_metrics_enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/reply_by_email_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/reply_by_email_enabled_metric.rb new file mode 100644 index 00000000000..24502147352 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/reply_by_email_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ReplyByEmailEnabledMetric < GenericMetric + value do + Gitlab::Email::IncomingEmail.enabled? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb index a481f7a5682..737cecccec3 100644 --- a/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb @@ -21,22 +21,20 @@ module Gitlab end end - class << self - private + private - def validators - Gitlab::Schema::Validation::Validators::Base.all_validators - end + def validators + Gitlab::Schema::Validation::Validators::Base.all_validators + end - def database - database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] - Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) - end + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) + end - def structure_sql - stucture_sql_path = Rails.root.join('db/structure.sql') - Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) - end + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb index d07438f4bf7..ce7b2feb745 100644 --- a/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb @@ -14,20 +14,56 @@ module Gitlab # class TotalCountMetric < BaseMetric include Gitlab::UsageDataCounters::RedisCounter + extend Gitlab::Usage::TimeSeriesStorable KEY_PREFIX = "{event_counters}_" - def self.redis_key(event_name) - KEY_PREFIX + event_name + def self.redis_key(event_name, date = nil) + base_key = KEY_PREFIX + event_name + return base_key unless date + + apply_time_aggregation(base_key, date) end def value - events.sum do |event| + return total_value if time_frame == 'all' + + period_value + end + + private + + def total_value + event_names.sum do |event_name| redis_usage_data do - total_count(self.class.redis_key(event[:name])) + total_count(self.class.redis_key(event_name)) end end end + + def period_value + keys = self.class.keys_for_aggregation(events: event_names, **time_constraint) + keys.sum do |key| + redis_usage_data do + total_count(key) + end + end + end + + def time_constraint + case time_frame + when '28d' + monthly_time_range + when '7d' + weekly_time_range + else + raise "Unknown time frame: #{time_frame} for #{self.class} :: #{events}" + end + end + + def event_names + events.pluck(:name) + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/unique_users_all_imports_metric.rb b/lib/gitlab/usage/metrics/instrumentations/unique_users_all_imports_metric.rb new file mode 100644 index 00000000000..931859bf7fa --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/unique_users_all_imports_metric.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class UniqueUsersAllImportsMetric < NumbersMetric + IMPORTS_METRICS = [ + ProjectImportsCreatorsMetric, + BulkImportsUsersMetric, + JiraImportsUsersMetric, + CsvImportsUsersMetric, + GroupImportsUsersMetric + ].freeze + + operation :add + + data do |time_frame| + IMPORTS_METRICS.map { |metric| metric.new(time_frame: time_frame).value } + end + + # overwriting instrumentation to generate the appropriate sql query + def instrumentation + metric_queries = IMPORTS_METRICS.map do |metric| + "(#{metric.new(time_frame: time_frame).instrumentation})" + end.join(' + ') + + "SELECT #{metric_queries}" + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 5f819f060e4..e36bf9ff6ad 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -157,28 +157,7 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def features_usage_data - features_usage_data_ce - end - - def features_usage_data_ce - { - instance_auto_devops_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.auto_devops_enabled? }, - container_registry_enabled: alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled }, - dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled, - gitlab_shared_runners_enabled: alt_usage_data(fallback: nil) { Gitlab.config.gitlab_ci.shared_runners_enabled }, - gravatar_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gravatar_enabled? }, - ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled }, - mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled }, - omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? }, - prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, - prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, - reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::IncomingEmail.enabled? }, - signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, - grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, - gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } - } - end + def features_usage_data = {} def components_usage_data { @@ -365,7 +344,6 @@ module Gitlab users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), - unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab)) }, @@ -417,7 +395,6 @@ module Gitlab service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), service_desk_issues: count(::Issue.service_desk.where(time_period)), projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).where(time_period), :creator_id), - projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_cloud.where(time_period), :creator_id), projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira).with_jira_dvcs_server.where(time_period), :creator_id) } end @@ -565,18 +542,6 @@ module Gitlab end # rubocop:disable CodeReuse/ActiveRecord - def unique_users_all_imports(time_period) - project_imports = distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id) - bulk_imports = distinct_count(::BulkImport.where(time_period), :user_id) - jira_issue_imports = distinct_count(::JiraImportState.where(time_period), :user_id) - csv_issue_imports = distinct_count(::Issues::CsvImport.where(time_period), :user_id) - group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) - - add(project_imports, bulk_imports, jira_issue_imports, csv_issue_imports, group_imports) - end - # rubocop:enable CodeReuse/ActiveRecord - - # rubocop:disable CodeReuse/ActiveRecord def distinct_count_user_auth_by_provider(time_period) counts = auth_providers_except_ldap.index_with do |provider| distinct_count( diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 185b49d4a68..b0444066722 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -66,7 +66,7 @@ module Gitlab rescue StandardError => e # Ignore any exceptions unless is dev or test env - # The application flow should not be blocked by erros in tracking + # The application flow should not be blocked by errors in tracking Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 534a08cad9a..8310c464a59 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -75,16 +75,6 @@ module Gitlab } end - # rubocop: disable CodeReuse/ActiveRecord - def sent_in_product_marketing_email_count(sent_emails, track, series) - count(Users::InProductMarketingEmail.where(track: track, series: series)) - end - - def clicked_in_product_marketing_email_count(clicked_emails, track, series) - count(Users::InProductMarketingEmail.where(track: track, series: series).where.not(cta_clicked_at: nil)) - end - # rubocop: enable CodeReuse/ActiveRecord - def stage_manage_events(time_period) # rubocop: disable CodeReuse/ActiveRecord # rubocop: disable UsageData/LargeTable diff --git a/lib/gitlab/web_ide/default_oauth_application.rb b/lib/gitlab/web_ide/default_oauth_application.rb new file mode 100644 index 00000000000..01b7637c1c0 --- /dev/null +++ b/lib/gitlab/web_ide/default_oauth_application.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module WebIde + module DefaultOauthApplication + class << self + def feature_enabled?(current_user) + Feature.enabled?(:vscode_web_ide, current_user) && Feature.enabled?(:web_ide_oauth, current_user) + end + + def oauth_application + application_settings.web_ide_oauth_application + end + + def oauth_callback_url + Gitlab::Routing.url_helpers.ide_oauth_redirect_url + end + + def ensure_oauth_application! + return if oauth_application + + should_expire_cache = false + + application_settings.transaction do + # note: This should run very rarely and should be safe for us to do a lock + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/132496#note_1587293087 + application_settings.lock! + + # note: `lock!`` breaks applicaiton_settings cache and will trigger another query. + # We need to double check here so that requests previously waiting on the lock can + # now just skip. + next if oauth_application + + application = Doorkeeper::Application.new( + name: 'GitLab Web IDE', + redirect_uri: oauth_callback_url, + scopes: ['api'], + trusted: true, + confidential: false) + application.save! + application_settings.update!(web_ide_oauth_application: application) + should_expire_cache = true + end + + # note: This needs to happen outside the transaction, but only if we actually changed something + ::Gitlab::CurrentSettings.expire_current_application_settings if should_expire_cache + end + + private + + def application_settings + ::Gitlab::CurrentSettings.current_application_settings + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 057e89a2a97..715638ba0d9 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -157,7 +157,15 @@ module Gitlab ] end - def send_url(url, allow_redirects: false, method: 'GET', body: nil, headers: nil) + # response_statuses can be set for 'error' and 'timeout'. They are optional. + # Their values must be a symbol accepted by Rack::Utils::SYMBOL_TO_STATUS_CODE. + # Example: response_statuses : { error: :internal_server_error, timeout: :bad_request } + # timeouts can be given for the opening the connection and reading the response headers. + # Their values must be given in seconds. + # Example: timeouts: { open: 5, read: 5 } + def send_url( + url, allow_redirects: false, method: 'GET', body: nil, headers: nil, timeouts: {}, response_statuses: {} + ) params = { 'URL' => url, 'AllowRedirects' => allow_redirects, @@ -166,9 +174,24 @@ module Gitlab 'Method' => method }.compact + if timeouts.present? + params['DialTimeout'] = "#{timeouts[:open]}s" if timeouts[:open] + params['ResponseHeaderTimeout'] = "#{timeouts[:read]}s" if timeouts[:read] + end + + if response_statuses.present? + if response_statuses[:error] + params['ErrorResponseStatus'] = Rack::Utils::SYMBOL_TO_STATUS_CODE[response_statuses[:error]] + end + + if response_statuses[:timeout] + params['TimeoutResponseStatus'] = Rack::Utils::SYMBOL_TO_STATUS_CODE[response_statuses[:timeout]] + end + end + [ SEND_DATA_HEADER, - "send-url:#{encode(params)}" + "send-url:#{encode(params.compact)}" ] end diff --git a/lib/initializer_connections.rb b/lib/initializer_connections.rb index ae2809b7604..c39340b1d3c 100644 --- a/lib/initializer_connections.rb +++ b/lib/initializer_connections.rb @@ -11,15 +11,17 @@ module InitializerConnections def self.raise_if_new_database_connection return yield if Gitlab::Utils.to_boolean(ENV['SKIP_RAISE_ON_INITIALIZE_CONNECTIONS']) - previous_connection_counts = ActiveRecord::Base.connection_handler.connection_pool_list.to_h do |pool| - [pool.db_config.name, pool.connections.size] - end + previous_connection_counts = + ActiveRecord::Base.connection_handler.connection_pool_list(ApplicationRecord.current_role).to_h do |pool| + [pool.db_config.name, pool.connections.size] + end yield - new_connection_counts = ActiveRecord::Base.connection_handler.connection_pool_list.to_h do |pool| - [pool.db_config.name, pool.connections.size] - end + new_connection_counts = + ActiveRecord::Base.connection_handler.connection_pool_list(ApplicationRecord.current_role).to_h do |pool| + [pool.db_config.name, pool.connections.size] + end raise_database_connection_made_error unless previous_connection_counts == new_connection_counts end diff --git a/lib/integrations/google_cloud_platform/artifact_registry/client.rb b/lib/integrations/google_cloud_platform/artifact_registry/client.rb new file mode 100644 index 00000000000..ae41aa2614e --- /dev/null +++ b/lib/integrations/google_cloud_platform/artifact_registry/client.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Integrations + module GoogleCloudPlatform + module ArtifactRegistry + class Client < Integrations::GoogleCloudPlatform::BaseClient + PAGE_SIZE = 10 + + def initialize(project:, user:, gcp_project_id:, gcp_location:, gcp_repository:, gcp_wlif:) + super(project: project, user: user) + @gcp_project_id = gcp_project_id + @gcp_location = gcp_location + @gcp_repository = gcp_repository + @gcp_wlif = gcp_wlif + end + + def list_docker_images(page_token: nil) + response = ::Gitlab::HTTP.get( + list_docker_images_url, + headers: headers, + query: query_params(page_token: page_token), + format: :plain # disable httparty json parsing + ) + + if response.success? + ::Gitlab::Json.parse(response.body, symbolize_keys: true) + else + {} + end + end + + private + + def list_docker_images_url + "#{GLGO_BASE_URL}/gcp/ar/" \ + "projects/#{@gcp_project_id}/" \ + "locations/#{@gcp_location}/" \ + "repositories/#{@gcp_repository}/docker" + end + + def query_params(page_token: nil) + { + page_token: page_token, + page_size: PAGE_SIZE + }.compact + end + + def headers + jwt = encoded_jwt(wlif: @gcp_wlif) + { + 'Authorization' => "Bearer #{jwt}" + } + end + end + end + end +end diff --git a/lib/integrations/google_cloud_platform/base_client.rb b/lib/integrations/google_cloud_platform/base_client.rb new file mode 100644 index 00000000000..56c05e7987b --- /dev/null +++ b/lib/integrations/google_cloud_platform/base_client.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Integrations + module GoogleCloudPlatform + class BaseClient + GLGO_BASE_URL = if Gitlab.staging? + 'https://glgo.staging.runway.gitlab.net' + else + 'http://glgo.runway.gitlab.net/' + end + + def initialize(project:, user:) + @project = project + @user = user + end + + private + + def encoded_jwt(wlif:) + jwt = ::Integrations::GoogleCloudPlatform::Jwt.new( + project: @project, + user: @user, + claims: { + audience: GLGO_BASE_URL, + wlif: wlif + } + ) + jwt.encoded + end + end + end +end diff --git a/lib/integrations/google_cloud_platform/jwt.rb b/lib/integrations/google_cloud_platform/jwt.rb new file mode 100644 index 00000000000..26343a3a9db --- /dev/null +++ b/lib/integrations/google_cloud_platform/jwt.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Integrations + module GoogleCloudPlatform + class Jwt < ::JSONWebToken::RSAToken + extend ::Gitlab::Utils::Override + + JWT_OPTIONS_ERROR = 'This jwt needs jwt claims audience and wlif to be set.' + + NoSigningKeyError = Class.new(StandardError) + + def initialize(project:, user:, claims:) + super + + raise ArgumentError, JWT_OPTIONS_ERROR if claims[:audience].blank? || claims[:wlif].blank? + + @claims = claims + @project = project + @user = user + end + + def encoded + @custom_payload.merge!(custom_claims) + + super + end + + private + + override :subject + def subject + "project_#{@project.id}_user_#{@user.id}" + end + + override :key_data + def key_data + @key_data ||= begin + # TODO Feels strange to use the CI signing key but do + # we have a different signing key? + key_data = Gitlab::CurrentSettings.ci_jwt_signing_key + + raise NoSigningKeyError unless key_data + + key_data + end + end + + def custom_claims + { + namespace_id: namespace.id.to_s, + namespace_path: namespace.full_path, + root_namespace_path: root_namespace.full_path, + root_namespace_id: root_namespace.id.to_s, + project_id: @project.id.to_s, + project_path: @project.full_path, + user_id: @user&.id.to_s, + user_login: @user&.username, + user_email: @user&.email, + wlif: @claims[:wlif] + } + end + + def namespace + @project.namespace + end + + def root_namespace + @project.root_namespace + end + + override :issuer + def issuer + Feature.enabled?(:oidc_issuer_url) ? Gitlab.config.gitlab.url : Settings.gitlab.base_url + end + + override :audience + def audience + @claims[:audience] + end + + override :kid + def kid + rsa_key = OpenSSL::PKey::RSA.new(key_data) + rsa_key.public_key.to_jwk[:kid] + end + end + end +end diff --git a/lib/organization/current_organization.rb b/lib/organization/current_organization.rb new file mode 100644 index 00000000000..d1f50aba7a1 --- /dev/null +++ b/lib/organization/current_organization.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Organization + module CurrentOrganization + CURRENT_ORGANIZATION_THREAD_VAR = :current_organization + + def current_organization + Thread.current[CURRENT_ORGANIZATION_THREAD_VAR] + end + + def current_organization=(organization) + Thread.current[CURRENT_ORGANIZATION_THREAD_VAR] = organization + end + + def with_current_organization(organization, &_blk) + previous_organization = current_organization + self.current_organization = organization + yield + ensure + self.current_organization = previous_organization + end + end +end diff --git a/lib/product_analytics/event_params.rb b/lib/product_analytics/event_params.rb deleted file mode 100644 index 6cb3d462384..00000000000 --- a/lib/product_analytics/event_params.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module ProductAnalytics - # Converts params from Snowplow tracker to one compatible with - # GitLab ProductAnalyticsEvent model. The field naming corresponds - # with snowplow event model. Only project_id is GitLab specific. - # - # For information on what each field is you can check next resources: - # * Snowplow tracker protocol: https://github.com/snowplow/snowplow/wiki/snowplow-tracker-protocol - # * Canonical event model: https://github.com/snowplow/snowplow/wiki/canonical-event-model - class EventParams - def self.parse_event_params(params) - { - project_id: params['aid'], - platform: params['p'], - collector_tstamp: Time.zone.now, - event_id: params['eid'], - v_tracker: params['tv'], - v_collector: Gitlab::VERSION, - v_etl: Gitlab::VERSION, - os_timezone: params['tz'], - name_tracker: params['tna'], - br_lang: params['lang'], - doc_charset: params['cs'], - br_features_pdf: Gitlab::Utils.to_boolean(params['f_pdf']), - br_features_flash: Gitlab::Utils.to_boolean(params['f_fla']), - br_features_java: Gitlab::Utils.to_boolean(params['f_java']), - br_features_director: Gitlab::Utils.to_boolean(params['f_dir']), - br_features_quicktime: Gitlab::Utils.to_boolean(params['f_qt']), - br_features_realplayer: Gitlab::Utils.to_boolean(params['f_realp']), - br_features_windowsmedia: Gitlab::Utils.to_boolean(params['f_wma']), - br_features_gears: Gitlab::Utils.to_boolean(params['f_gears']), - br_features_silverlight: Gitlab::Utils.to_boolean(params['f_ag']), - br_colordepth: params['cd'], - br_cookies: Gitlab::Utils.to_boolean(params['cookie']), - dvce_created_tstamp: params['dtm'], - br_viewheight: params['vp'], - domain_sessionidx: params['vid'], - domain_sessionid: params['sid'], - domain_userid: params['duid'], - user_fingerprint: params['fp'], - page_referrer: params['refr'], - page_url: params['url'], - se_category: params['se_ca'], - se_action: params['se_ac'], - se_label: params['se_la'], - se_property: params['se_pr'], - se_value: params['se_va'] - } - end - - def self.has_required_params?(params) - params['aid'].present? && params['eid'].present? - end - end -end diff --git a/lib/sidebars/admin/panel.rb b/lib/sidebars/admin/panel.rb index 7f7d7ec843c..e0f9bc70762 100644 --- a/lib/sidebars/admin/panel.rb +++ b/lib/sidebars/admin/panel.rb @@ -9,11 +9,6 @@ module Sidebars add_menus end - override :render_raw_scope_menu_partial - def render_raw_scope_menu_partial - "shared/nav/admin_scope_header" - end - override :aria_label def aria_label s_("Admin|Admin Area") diff --git a/lib/sidebars/concerns/container_with_html_options.rb b/lib/sidebars/concerns/container_with_html_options.rb index 796b7cbe275..55bea2658e5 100644 --- a/lib/sidebars/concerns/container_with_html_options.rb +++ b/lib/sidebars/concerns/container_with_html_options.rb @@ -20,50 +20,10 @@ module Sidebars {} end - # The attributes returned from this method - # will be applied to helper methods like - # `link_to` or the div containing the container - # when it is collapsed. - def collapsed_container_html_options - { - aria: { label: title } - }.merge(extra_collapsed_container_html_options) - end - - # Classes should mostly override this method - # and not `collapsed_container_html_options`. - def extra_collapsed_container_html_options - {} - end - - # Attributes to pass to the html_options attribute - # in the helper method that sets the active class - # on each element. - def nav_link_html_options - { - data: { - track_label: self.class.name.demodulize.underscore - } - }.deep_merge(extra_nav_link_html_options) - end - - # Classes should mostly override this method - # and not `nav_link_html_options`. - def extra_nav_link_html_options - {} - end - def title raise NotImplementedError end - # The attributes returned from this method - # will be applied right next to the title, - # for example in the span that renders the title. - def title_html_options - {} - end - def link raise NotImplementedError end diff --git a/lib/sidebars/concerns/has_hint.rb b/lib/sidebars/concerns/has_hint.rb deleted file mode 100644 index dc4f765e974..00000000000 --- a/lib/sidebars/concerns/has_hint.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -# This module has the necessary methods to store -# hints for menus. Hints are elements displayed -# when the user hover the menu item. -module Sidebars - module Concerns - module HasHint - def show_hint? - false - end - - def hint_html_options - {} - end - end - end -end diff --git a/lib/sidebars/concerns/has_icon.rb b/lib/sidebars/concerns/has_icon.rb index afff466239d..6f797f5a1ff 100644 --- a/lib/sidebars/concerns/has_icon.rb +++ b/lib/sidebars/concerns/has_icon.rb @@ -17,10 +17,6 @@ module Sidebars nil end - def image_html_options - {} - end - def icon_or_image? sprite_icon || image_path end diff --git a/lib/sidebars/explore/menus/catalog_menu.rb b/lib/sidebars/explore/menus/catalog_menu.rb index 2d8e8bba08b..61578147326 100644 --- a/lib/sidebars/explore/menus/catalog_menu.rb +++ b/lib/sidebars/explore/menus/catalog_menu.rb @@ -21,7 +21,7 @@ module Sidebars override :render? def render? - Feature.enabled?(:global_ci_catalog, current_user) + true end override :active_routes diff --git a/lib/sidebars/explore/panel.rb b/lib/sidebars/explore/panel.rb index 3559f7d9627..a07072b5241 100644 --- a/lib/sidebars/explore/panel.rb +++ b/lib/sidebars/explore/panel.rb @@ -13,11 +13,6 @@ module Sidebars _('Explore') end - override :render_raw_scope_menu_partial - def render_raw_scope_menu_partial - "shared/nav/explore_scope_header" - end - override :super_sidebar_context_header def super_sidebar_context_header aria_label diff --git a/lib/sidebars/groups/menus/scope_menu.rb b/lib/sidebars/groups/menus/scope_menu.rb index 5f6663e9919..ca20a86a918 100644 --- a/lib/sidebars/groups/menus/scope_menu.rb +++ b/lib/sidebars/groups/menus/scope_menu.rb @@ -19,15 +19,6 @@ module Sidebars { path: %w[groups#show groups#details groups#new projects#new] } end - override :extra_nav_link_html_options - def extra_nav_link_html_options - { - class: 'context-header has-tooltip', - title: context.group.name, - data: { container: 'body', placement: 'right' } - } - end - override :render? def render? true diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb index b8f345c1ed5..ece6460bb89 100644 --- a/lib/sidebars/groups/menus/settings_menu.rb +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -40,13 +40,6 @@ module Sidebars 'settings' end - override :extra_nav_link_html_options - def extra_nav_link_html_options - { - class: 'shortcuts-settings' - } - end - override :pick_into_super_sidebar? def pick_into_super_sidebar? true diff --git a/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb b/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb index a053288ccea..0724b6d00c3 100644 --- a/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb +++ b/lib/sidebars/groups/super_sidebar_menus/analyze_menu.rb @@ -18,7 +18,6 @@ module Sidebars def configure_menu_items [ :analytics_dashboards, - :dashboards_analytics, :cycle_analytics, :ci_cd_analytics, :contribution_analytics, diff --git a/lib/sidebars/menu_item.rb b/lib/sidebars/menu_item.rb index bd538d2fa90..16f924602d5 100644 --- a/lib/sidebars/menu_item.rb +++ b/lib/sidebars/menu_item.rb @@ -4,11 +4,11 @@ module Sidebars class MenuItem include ::Sidebars::Concerns::LinkWithHtmlOptions - attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :hint_html_options, :has_pill, :pill_count, :super_sidebar_parent, :avatar, :entity_id + attr_reader :title, :link, :active_routes, :item_id, :container_html_options, :sprite_icon, :sprite_icon_html_options, :has_pill, :pill_count, :super_sidebar_parent, :avatar, :entity_id alias_method :has_pill?, :has_pill # rubocop: disable Metrics/ParameterLists - def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, hint_html_options: {}, has_pill: false, pill_count: nil, super_sidebar_parent: nil, avatar: nil, entity_id: nil) + def initialize(title:, link:, active_routes:, item_id: nil, container_html_options: {}, sprite_icon: nil, sprite_icon_html_options: {}, has_pill: false, pill_count: nil, super_sidebar_parent: nil, avatar: nil, entity_id: nil) @title = title @link = link @active_routes = active_routes @@ -18,17 +18,12 @@ module Sidebars @sprite_icon_html_options = sprite_icon_html_options @avatar = avatar @entity_id = entity_id - @hint_html_options = hint_html_options @has_pill = has_pill @pill_count = pill_count @super_sidebar_parent = super_sidebar_parent end # rubocop: enable Metrics/ParameterLists - def show_hint? - hint_html_options.present? - end - def render? true end @@ -49,16 +44,6 @@ module Sidebars # https://gitlab.com/gitlab-org/gitlab/-/issues/391864 # # container_html_options - # hint_html_options - # nav_link_html_options - } - end - - def nav_link_html_options - { - data: { - track_label: item_id - } } end end diff --git a/lib/sidebars/organizations/menus/manage_menu.rb b/lib/sidebars/organizations/menus/manage_menu.rb index 7c342002c31..cdbc5d16f1b 100644 --- a/lib/sidebars/organizations/menus/manage_menu.rb +++ b/lib/sidebars/organizations/menus/manage_menu.rb @@ -21,6 +21,13 @@ module Sidebars override :configure_menu_items def configure_menu_items + groups_and_projects_menu_item + users_menu_item + end + + private + + def groups_and_projects_menu_item add_item( ::Sidebars::MenuItem.new( title: _('Groups and projects'), @@ -30,6 +37,11 @@ module Sidebars item_id: :organization_groups_and_projects ) ) + end + + def users_menu_item + return unless can?(context.current_user, :read_organization_user, context.container) + add_item( ::Sidebars::MenuItem.new( title: _('Users'), diff --git a/lib/sidebars/organizations/menus/scope_menu.rb b/lib/sidebars/organizations/menus/scope_menu.rb index ba46dd7911f..a535be21280 100644 --- a/lib/sidebars/organizations/menus/scope_menu.rb +++ b/lib/sidebars/organizations/menus/scope_menu.rb @@ -24,13 +24,6 @@ module Sidebars true end - override :extra_nav_link_html_options - def extra_nav_link_html_options - { - class: 'context-header' - } - end - override :serialize_as_menu_item_args def serialize_as_menu_item_args super.merge({ diff --git a/lib/sidebars/panel.rb b/lib/sidebars/panel.rb index 518b13f0fae..5f8f95d6699 100644 --- a/lib/sidebars/panel.rb +++ b/lib/sidebars/panel.rb @@ -74,22 +74,6 @@ module Sidebars context.container end - # Auxiliar method that helps with the migration from - # regular views to the new logic - def render_raw_scope_menu_partial - # No-op - end - - # Auxiliar method that helps with the migration from - # regular views to the new logic. - # - # Any menu inside this partial will be added after - # all the menus added in the `configure_menus` - # method. - def render_raw_menus_partial - # No-op - end - private override :index_of diff --git a/lib/sidebars/projects/menus/ci_cd_menu.rb b/lib/sidebars/projects/menus/ci_cd_menu.rb index c77e8e996b0..65e8573753e 100644 --- a/lib/sidebars/projects/menus/ci_cd_menu.rb +++ b/lib/sidebars/projects/menus/ci_cd_menu.rb @@ -27,13 +27,6 @@ module Sidebars _('CI/CD') end - override :title_html_options - def title_html_options - { - id: 'js-onboarding-pipelines-link' - } - end - override :sprite_icon def sprite_icon 'rocket' diff --git a/lib/sidebars/projects/menus/confluence_menu.rb b/lib/sidebars/projects/menus/confluence_menu.rb index 43ef7ac73c4..be80d2dfee3 100644 --- a/lib/sidebars/projects/menus/confluence_menu.rb +++ b/lib/sidebars/projects/menus/confluence_menu.rb @@ -26,13 +26,6 @@ module Sidebars 'confluence.svg' end - override :image_html_options - def image_html_options - { - alt: title - } - end - override :render? def render? context.project.has_confluence? diff --git a/lib/sidebars/projects/menus/external_issue_tracker_menu.rb b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb index f088ccce9f5..860fab4296d 100644 --- a/lib/sidebars/projects/menus/external_issue_tracker_menu.rb +++ b/lib/sidebars/projects/menus/external_issue_tracker_menu.rb @@ -18,26 +18,11 @@ module Sidebars } end - override :extra_collapsed_container_html_options - def extra_collapsed_container_html_options - { - target: '_blank', - rel: 'noopener noreferrer' - } - end - override :title def title external_issue_tracker.title end - override :title_html_options - def title_html_options - { - id: 'js-onboarding-issues-link' - } - end - override :sprite_icon def sprite_icon 'external-link' diff --git a/lib/sidebars/projects/menus/external_wiki_menu.rb b/lib/sidebars/projects/menus/external_wiki_menu.rb index 1af9abc33ff..c273fb7698a 100644 --- a/lib/sidebars/projects/menus/external_wiki_menu.rb +++ b/lib/sidebars/projects/menus/external_wiki_menu.rb @@ -18,14 +18,6 @@ module Sidebars } end - override :extra_collapsed_container_html_options - def extra_collapsed_container_html_options - { - target: '_blank', - rel: 'noopener noreferrer' - } - end - override :title def title s_('ExternalWikiService|External wiki') diff --git a/lib/sidebars/projects/menus/infrastructure_menu.rb b/lib/sidebars/projects/menus/infrastructure_menu.rb index d3c9f3a6466..40a8b70b624 100644 --- a/lib/sidebars/projects/menus/infrastructure_menu.rb +++ b/lib/sidebars/projects/menus/infrastructure_menu.rb @@ -55,24 +55,10 @@ module Sidebars super_sidebar_parent: Sidebars::Projects::SuperSidebarMenus::OperationsMenu, active_routes: { controller: [:cluster_agents, :clusters] }, container_html_options: { class: 'shortcuts-kubernetes' }, - hint_html_options: kubernetes_hint_html_options, item_id: :kubernetes ) end - def kubernetes_hint_html_options - return {} unless context.show_cluster_hint - - { disabled: true, - data: { trigger: 'manual', - container: 'body', - placement: 'right', - highlight: Users::CalloutsHelper::GKE_CLUSTER_INTEGRATION, - highlight_priority: Users::Callout.feature_names[:GKE_CLUSTER_INTEGRATION], - dismiss_endpoint: callouts_path, - auto_devops_help_path: help_page_path('topics/autodevops/index') } } - end - def terraform_states_menu_item unless can?(context.current_user, :read_terraform_state, context.project) return ::Sidebars::NilMenuItem.new(item_id: :terraform_states) diff --git a/lib/sidebars/projects/menus/issues_menu.rb b/lib/sidebars/projects/menus/issues_menu.rb index e599b764ed9..9791a88cf9f 100644 --- a/lib/sidebars/projects/menus/issues_menu.rb +++ b/lib/sidebars/projects/menus/issues_menu.rb @@ -30,13 +30,6 @@ module Sidebars _('Issues') end - override :title_html_options - def title_html_options - { - id: 'js-onboarding-issues-link' - } - end - override :sprite_icon def sprite_icon 'issues' diff --git a/lib/sidebars/projects/menus/merge_requests_menu.rb b/lib/sidebars/projects/menus/merge_requests_menu.rb index ae4fd6b02e7..eb6827f90e7 100644 --- a/lib/sidebars/projects/menus/merge_requests_menu.rb +++ b/lib/sidebars/projects/menus/merge_requests_menu.rb @@ -21,13 +21,6 @@ module Sidebars _('Merge requests') end - override :title_html_options - def title_html_options - { - id: 'js-onboarding-mr-link' - } - end - override :sprite_icon def sprite_icon 'git-merge' diff --git a/lib/sidebars/projects/menus/project_information_menu.rb b/lib/sidebars/projects/menus/project_information_menu.rb index 6ab7e00dad3..ce6e5f3b8d3 100644 --- a/lib/sidebars/projects/menus/project_information_menu.rb +++ b/lib/sidebars/projects/menus/project_information_menu.rb @@ -18,11 +18,6 @@ module Sidebars { class: 'shortcuts-project-information' } end - override :extra_nav_link_html_options - def extra_nav_link_html_options - { class: 'home' } - end - override :title def title _('Project information') @@ -75,10 +70,7 @@ module Sidebars link: project_project_members_path(context.project), super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::ManageMenu, active_routes: { controller: :project_members }, - item_id: :members, - container_html_options: { - id: 'js-onboarding-members-link' - } + item_id: :members ) end end diff --git a/lib/sidebars/projects/menus/repository_menu.rb b/lib/sidebars/projects/menus/repository_menu.rb index 259e3285338..2bf4d782fda 100644 --- a/lib/sidebars/projects/menus/repository_menu.rb +++ b/lib/sidebars/projects/menus/repository_menu.rb @@ -32,13 +32,6 @@ module Sidebars _('Repository') end - override :title_html_options - def title_html_options - { - id: 'js-onboarding-repo-link' - } - end - override :sprite_icon def sprite_icon 'doc-text' @@ -71,7 +64,7 @@ module Sidebars super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu, active_routes: { controller: %w[commit commits] }, item_id: :commits, - container_html_options: { id: 'js-onboarding-commits-link', class: 'shortcuts-commits' } + container_html_options: { class: 'shortcuts-commits' } ) end @@ -81,8 +74,7 @@ module Sidebars link: project_branches_path(context.project), super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::CodeMenu, active_routes: { controller: :branches }, - item_id: :branches, - container_html_options: { id: 'js-onboarding-branches-link' } + item_id: :branches ) end @@ -102,7 +94,7 @@ module Sidebars link = project_graph_path(context.project, context.current_ref, ref_type: ref_type_from_context(context)) ::Sidebars::MenuItem.new( - title: _('Contributor statistics'), + title: _('Contributor analytics'), link: link, super_sidebar_parent: ::Sidebars::Projects::SuperSidebarMenus::AnalyzeMenu, active_routes: { path: 'graphs#show' }, diff --git a/lib/sidebars/projects/menus/scope_menu.rb b/lib/sidebars/projects/menus/scope_menu.rb index d03abfdfb7e..5bbbf3696e8 100644 --- a/lib/sidebars/projects/menus/scope_menu.rb +++ b/lib/sidebars/projects/menus/scope_menu.rb @@ -26,15 +26,6 @@ module Sidebars } end - override :extra_nav_link_html_options - def extra_nav_link_html_options - { - class: 'context-header has-tooltip', - title: context.project.name, - data: { container: 'body', placement: 'right' } - } - end - override :render? def render? true diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 077eebf58b9..5aaccb2644f 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -19,13 +19,6 @@ module Sidebars _('Settings') end - override :title_html_options - def title_html_options - { - id: 'js-onboarding-settings-link' - } - end - override :sprite_icon def sprite_icon 'settings' diff --git a/lib/sidebars/projects/menus/shimo_menu.rb b/lib/sidebars/projects/menus/shimo_menu.rb deleted file mode 100644 index c93c4f6a0a4..00000000000 --- a/lib/sidebars/projects/menus/shimo_menu.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Sidebars - module Projects - module Menus - class ShimoMenu < ::Sidebars::Menu - override :link - def link - project_integrations_shimo_path(context.project) - end - - override :title - def title - s_('Shimo|Shimo') - end - - override :image_path - def image_path - 'logos/shimo.svg' - end - - override :image_html_options - def image_html_options - { - size: 16 - } - end - - override :render? - def render? - context.project.has_shimo? - end - - override :active_routes - def active_routes - { controller: :shimo } - end - end - end - end -end diff --git a/lib/sidebars/projects/menus/zentao_menu.rb b/lib/sidebars/projects/menus/zentao_menu.rb index 1b5ba900a86..c6edcd77cba 100644 --- a/lib/sidebars/projects/menus/zentao_menu.rb +++ b/lib/sidebars/projects/menus/zentao_menu.rb @@ -14,26 +14,11 @@ module Sidebars s_('ZentaoIntegration|ZenTao') end - override :title_html_options - def title_html_options - { - id: 'js-onboarding-settings-link' - } - end - override :sprite_icon def sprite_icon 'external-link' end - # Hardcode sizes so image doesn't flash before CSS loads https://gitlab.com/gitlab-org/gitlab/-/issues/321022 - override :image_html_options - def image_html_options - { - size: 16 - } - end - override :render? def render? return false if zentao_integration.blank? diff --git a/lib/sidebars/projects/panel.rb b/lib/sidebars/projects/panel.rb index 5d8bc18ac88..738a3fba9c6 100644 --- a/lib/sidebars/projects/panel.rb +++ b/lib/sidebars/projects/panel.rb @@ -42,9 +42,9 @@ module Sidebars end def third_party_wiki_menu - wiki_menu_list = [::Sidebars::Projects::Menus::ConfluenceMenu, ::Sidebars::Projects::Menus::ShimoMenu] + return unless ::Sidebars::Projects::Menus::ConfluenceMenu.new(context).render? - wiki_menu_list.find { |wiki_menu| wiki_menu.new(context).render? } + ::Sidebars::Projects::Menus::ConfluenceMenu end end end diff --git a/lib/sidebars/user_settings/menus/access_tokens_menu.rb b/lib/sidebars/user_settings/menus/access_tokens_menu.rb index ed39b5d6720..cb3f319ddde 100644 --- a/lib/sidebars/user_settings/menus/access_tokens_menu.rb +++ b/lib/sidebars/user_settings/menus/access_tokens_menu.rb @@ -6,7 +6,7 @@ module Sidebars class AccessTokensMenu < ::Sidebars::Menu override :link def link - profile_personal_access_tokens_path + user_settings_personal_access_tokens_path end override :title diff --git a/lib/sidebars/user_settings/menus/active_sessions_menu.rb b/lib/sidebars/user_settings/menus/active_sessions_menu.rb index f806c04e77c..3ddf8700556 100644 --- a/lib/sidebars/user_settings/menus/active_sessions_menu.rb +++ b/lib/sidebars/user_settings/menus/active_sessions_menu.rb @@ -8,7 +8,7 @@ module Sidebars override :link def link - profile_active_sessions_path + user_settings_active_sessions_path end override :title diff --git a/lib/sidebars/user_settings/menus/applications_menu.rb b/lib/sidebars/user_settings/menus/applications_menu.rb index c71f9a9660b..5e83c6a1355 100644 --- a/lib/sidebars/user_settings/menus/applications_menu.rb +++ b/lib/sidebars/user_settings/menus/applications_menu.rb @@ -8,7 +8,7 @@ module Sidebars override :link def link - applications_profile_path + user_settings_applications_path end override :title diff --git a/lib/sidebars/user_settings/menus/authentication_log_menu.rb b/lib/sidebars/user_settings/menus/authentication_log_menu.rb index c5a27acf1fd..fc4b0bba9c3 100644 --- a/lib/sidebars/user_settings/menus/authentication_log_menu.rb +++ b/lib/sidebars/user_settings/menus/authentication_log_menu.rb @@ -8,7 +8,7 @@ module Sidebars override :link def link - audit_log_profile_path + user_settings_authentication_log_path end override :title @@ -23,7 +23,7 @@ module Sidebars override :active_routes def active_routes - { path: 'profiles#audit_log' } + { path: 'user_settings#authentication_log' } end end end diff --git a/lib/sidebars/user_settings/menus/password_menu.rb b/lib/sidebars/user_settings/menus/password_menu.rb index e518e1f8bf7..d38d7e94746 100644 --- a/lib/sidebars/user_settings/menus/password_menu.rb +++ b/lib/sidebars/user_settings/menus/password_menu.rb @@ -6,7 +6,7 @@ module Sidebars class PasswordMenu < ::Sidebars::Menu override :link def link - edit_profile_password_path + edit_user_settings_password_path end override :title diff --git a/lib/sidebars/user_settings/panel.rb b/lib/sidebars/user_settings/panel.rb index b61cb3ed144..5be27936d99 100644 --- a/lib/sidebars/user_settings/panel.rb +++ b/lib/sidebars/user_settings/panel.rb @@ -13,11 +13,6 @@ module Sidebars _('User settings') end - override :render_raw_scope_menu_partial - def render_raw_scope_menu_partial - "shared/nav/user_settings_scope_header" - end - override :super_sidebar_context_header def super_sidebar_context_header aria_label diff --git a/lib/sidebars/your_work/panel.rb b/lib/sidebars/your_work/panel.rb index 6316023a8cb..d1297fb439d 100644 --- a/lib/sidebars/your_work/panel.rb +++ b/lib/sidebars/your_work/panel.rb @@ -13,11 +13,6 @@ module Sidebars _('Your work') end - override :render_raw_scope_menu_partial - def render_raw_scope_menu_partial - "shared/nav/your_work_scope_header" - end - override :super_sidebar_context_header def super_sidebar_context_header aria_label diff --git a/lib/support/systemd/gitlab-puma.service b/lib/support/systemd/gitlab-puma.service index c0affa92ddf..7978dc898fc 100644 --- a/lib/support/systemd/gitlab-puma.service +++ b/lib/support/systemd/gitlab-puma.service @@ -4,7 +4,7 @@ Conflicts=gitlab.service ReloadPropagatedFrom=gitlab.target PartOf=gitlab.target After=network.target -StartLimitIntervalSec=100s +StartLimitIntervalSec=11min [Service] Type=notify @@ -15,7 +15,7 @@ ExecStart=/usr/local/bin/bundle exec puma --config /home/git/gitlab/config/puma. ExecReload=/usr/bin/kill -USR2 $MAINPID PIDFile=/home/git/gitlab/tmp/pids/puma.pid # puma can be slow to start -TimeoutStartSec=120 +TimeoutStartSec=2min WatchdogSec=10 Restart=on-failure RestartSec=1 diff --git a/lib/support/systemd/gitlab-sidekiq.service b/lib/support/systemd/gitlab-sidekiq.service index d8585a59085..0f475ba6b0d 100644 --- a/lib/support/systemd/gitlab-sidekiq.service +++ b/lib/support/systemd/gitlab-sidekiq.service @@ -4,6 +4,7 @@ ReloadPropagatedFrom=gitlab.target PartOf=gitlab.target After=network.target JoinsNamespaceOf=gitlab-puma.service +StartLimitIntervalSec=11min [Service] Type=notify @@ -14,6 +15,8 @@ Environment=SIDEKIQ_QUEUES=* ExecStart=/home/git/gitlab/bin/sidekiq-cluster $SIDEKIQ_QUEUES -P /home/git/gitlab/tmp/pids/sidekiq.pid NotifyAccess=all PIDFile=/home/git/gitlab/tmp/pids/sidekiq.pid +# sidekiq can be slow to start +TimeoutStartSec=2min Restart=on-failure RestartSec=1 SyslogIdentifier=gitlab-sidekiq diff --git a/lib/system_check/orphans/namespace_check.rb b/lib/system_check/orphans/namespace_check.rb deleted file mode 100644 index 53b2d8fd5b3..00000000000 --- a/lib/system_check/orphans/namespace_check.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module SystemCheck - module Orphans - class NamespaceCheck < SystemCheck::BaseCheck - set_name 'Orphaned namespaces:' - - def multi_check - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{repository_storage.legacy_disk_path})".color(:yellow) - toplevel_namespace_dirs = disk_namespaces(repository_storage.legacy_disk_path) - - orphans = (toplevel_namespace_dirs - existing_namespaces) - print_orphans(orphans, storage_name) - end - end - - clear_namespaces! # releases memory when check finishes - end - - private - - def print_orphans(orphans, storage_name) - if orphans.empty? - $stdout.puts "* No orphaned namespaces for #{storage_name} storage".color(:green) - return - end - - orphans.each do |orphan| - $stdout.puts " - #{orphan}".color(:red) - end - end - - def disk_namespaces(storage_path) - fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| - namespace = File.basename(namespace_path) - next if namespace.eql?('@hashed') - - result << namespace - end - end - - def fetch_disk_namespaces(storage_path) - Dir.glob(File.join(storage_path, '*')) - end - - def existing_namespaces - @namespaces ||= Namespace.where(parent: nil).all.pluck(:path) - end - - def clear_namespaces! - @namespaces = nil - end - end - end -end diff --git a/lib/system_check/orphans/repository_check.rb b/lib/system_check/orphans/repository_check.rb deleted file mode 100644 index 8f15872de22..00000000000 --- a/lib/system_check/orphans/repository_check.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module SystemCheck - module Orphans - class RepositoryCheck < SystemCheck::BaseCheck - set_name 'Orphaned repositories:' - - def multi_check - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab.config.repositories.storages.each do |storage_name, repository_storage| - storage_path = repository_storage.legacy_disk_path - - $stdout.puts - $stdout.puts "* Storage: #{storage_name} (#{storage_path})".color(:yellow) - - repositories = disk_repositories(storage_path) - orphans = (repositories - fetch_repositories(storage_name)) - - print_orphans(orphans, storage_name) - end - end - end - - private - - def print_orphans(orphans, storage_name) - if orphans.empty? - $stdout.puts "* No orphaned repositories for #{storage_name} storage".color(:green) - return - end - - orphans.each do |orphan| - $stdout.puts " - #{orphan}".color(:red) - end - end - - def disk_repositories(storage_path) - fetch_disk_namespaces(storage_path).each_with_object([]) do |namespace_path, result| - namespace = File.basename(namespace_path) - next if namespace.eql?('@hashed') - - fetch_disk_repositories(namespace_path).each do |repo| - result << "#{namespace}/#{File.basename(repo)}" - end - end - end - - def fetch_repositories(storage_name) - sql = " - SELECT - CONCAT(n.path, '/', p.path, '.git') repo, - CONCAT(n.path, '/', p.path, '.wiki.git') wiki - FROM projects p - JOIN namespaces n - ON (p.namespace_id = n.id AND - n.parent_id IS NULL) - WHERE (p.repository_storage LIKE ?) - " - - query = ::Project.sanitize_sql_array([sql, storage_name]) - ::Project.connection.select_all(query).rows.try(:flatten!) || [] - end - - def fetch_disk_namespaces(storage_path) - Dir.glob(File.join(storage_path, '*')) - end - - def fetch_disk_repositories(namespace_path) - Dir.glob(File.join(namespace_path, '*')) - end - end - end -end diff --git a/lib/system_check/rake_task/orphans/namespace_task.rb b/lib/system_check/rake_task/orphans/namespace_task.rb deleted file mode 100644 index 2822da45bc1..00000000000 --- a/lib/system_check/rake_task/orphans/namespace_task.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module SystemCheck - module RakeTask - module Orphans - # Used by gitlab:orphans:check_namespaces rake task - class NamespaceTask - extend RakeTaskHelpers - - def self.name - 'Orphans' - end - - def self.checks - [SystemCheck::Orphans::NamespaceCheck] - end - end - end - end -end diff --git a/lib/system_check/rake_task/orphans/repository_task.rb b/lib/system_check/rake_task/orphans/repository_task.rb deleted file mode 100644 index f14b3af22e8..00000000000 --- a/lib/system_check/rake_task/orphans/repository_task.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module SystemCheck - module RakeTask - module Orphans - # Used by gitlab:orphans:check_repositories rake task - class RepositoryTask - extend RakeTaskHelpers - - def self.name - 'Orphans' - end - - def self.checks - [SystemCheck::Orphans::RepositoryCheck] - end - end - end - end -end diff --git a/lib/system_check/rake_task/orphans_task.rb b/lib/system_check/rake_task/orphans_task.rb deleted file mode 100644 index 31f8ede25e0..00000000000 --- a/lib/system_check/rake_task/orphans_task.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module SystemCheck - module RakeTask - # Used by gitlab:orphans:check rake task - class OrphansTask - extend RakeTaskHelpers - - def self.name - 'Orphans' - end - - def self.checks - [ - SystemCheck::Orphans::NamespaceCheck, - SystemCheck::Orphans::RepositoryCheck - ] - end - end - end -end diff --git a/lib/tasks/cleanup.rake b/lib/tasks/cleanup.rake deleted file mode 100644 index 31695d3da79..00000000000 --- a/lib/tasks/cleanup.rake +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - namespace :cleanup do - desc "GitLab | Cleanup | Delete moved repositories" - task moved: :gitlab_environment do - warn_user_is_not_gitlab - remove_flag = ENV['REMOVE'] - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_root = repository_storage.legacy_disk_path.chomp('/') - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W[find #{repo_root} -mindepth 1 -maxdepth 2 -name *+moved*.git]) do |find| - find.each_line do |path| - path.chomp! - - if remove_flag - if FileUtils.rm_rf(path) - puts "Removed...#{path}".color(:green) - else - puts "Cannot remove #{path}".color(:red) - end - else - puts "Can be removed: #{path}".color(:green) - end - end - end - end - - unless remove_flag - puts "To cleanup these repositories run this command with REMOVE=true".color(:yellow) - end - end - end -end diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 1a659a930ab..2d649a061b5 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -1,8 +1,7 @@ # frozen_string_literal: true -require "gettext_i18n_rails/tasks" - namespace :gettext do + desc 'Compile po files to json, for usage in the frontend' task :compile do # See: https://gitlab.com/gitlab-org/gitlab-foss/issues/33014#note_31218998 FileUtils.touch(pot_file_path) @@ -71,6 +70,7 @@ namespace :gettext do end end + desc 'Check whether gitlab.pot needs updates, used during CI' task updated_check: [:regenerate] do pot_diff = `git diff -- #{pot_file_path} | grep -E '^(\\+|-)msgid'`.strip diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index d4e38100609..68a14e58fdb 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -48,21 +48,4 @@ namespace :gitlab do SystemCheck::RakeTask::LdapTask.run! end end - - namespace :orphans do - desc 'Gitlab | Orphans | Check for orphaned namespaces and repositories' - task check: :gitlab_environment do - SystemCheck::RakeTask::OrphansTask.run! - end - - desc 'GitLab | Orphans | Check for orphaned namespaces in the repositories path' - task check_namespaces: :gitlab_environment do - SystemCheck::RakeTask::Orphans::NamespaceTask.run! - end - - desc 'GitLab | Orphans | Check for orphaned repositories in the repositories path' - task check_repositories: :gitlab_environment do - SystemCheck::RakeTask::Orphans::RepositoryTask.run! - end - end end diff --git a/lib/tasks/gitlab/click_house/migration.rake b/lib/tasks/gitlab/click_house/migration.rake index ddac81ec98f..a8ef5024599 100644 --- a/lib/tasks/gitlab/click_house/migration.rake +++ b/lib/tasks/gitlab/click_house/migration.rake @@ -1,21 +1,39 @@ # frozen_string_literal: true +click_house_database_names = %i[main] + namespace :gitlab do namespace :clickhouse do - task :prepare_schema_migration_table, [:database] => :environment do |_t, args| - require_relative '../../../../lib/click_house/migration_support/schema_migration' + namespace :migrate do + click_house_database_names.each do |database| + desc "GitLab | ClickHouse | Migrate the #{database} database (options: VERSION=x, VERBOSE=false, SCOPE=y)" + task database, [:skip_unless_configured] => :environment do |_t, args| + if args[:skip_unless_configured] && !::ClickHouse::Client.database_configured?(database) + puts "The '#{database}' ClickHouse database is not configured, skipping migrations" + next + end - ClickHouse::MigrationSupport::SchemaMigration.create_table(args.database&.to_sym || :main) + migrate(:up, database) + end + end end - desc 'GitLab | ClickHouse | Migrate' - task migrate: [:prepare_schema_migration_table] do - migrate(:up) + namespace :rollback do + click_house_database_names.each do |database| + desc "GitLab | ClickHouse | Rolls the #{database} database back to the previous version " \ + "(specify steps w/ STEP=n)" + task database => :environment do + migrate(:down, database) + end + end end - desc 'GitLab | ClickHouse | Rollback' - task rollback: [:prepare_schema_migration_table] do - migrate(:down) + desc 'GitLab | ClickHouse | Migrate the databases (options: VERSION=x, VERBOSE=false, SCOPE=y)' + task :migrate, [:skip_unless_configured] => :environment do |_t, args| + click_house_database_names.each do |database| + puts "Running gitlab:clickhouse:migrate:#{database} rake task" + Rake::Task["gitlab:clickhouse:migrate:#{database}"].invoke(args[:skip_unless_configured]) + end end private @@ -34,7 +52,7 @@ namespace :gitlab do ENV['VERSION'].to_i if ENV['VERSION'] && !ENV['VERSION'].empty? end - def migrate(direction) + def migrate(direction, database) require_relative '../../../../lib/click_house/migration_support/schema_migration' require_relative '../../../../lib/click_house/migration_support/migration_context' require_relative '../../../../lib/click_house/migration_support/migrator' @@ -42,13 +60,22 @@ namespace :gitlab do check_target_version scope = ENV['SCOPE'] - verbose_was = ClickHouse::Migration.verbose + step = ENV['STEP'] ? Integer(ENV['STEP']) : nil + step = 1 if step.nil? && direction == :down + raise ArgumentError, 'STEP should be a positive number' if step.present? && step < 1 + + verbose_was = ::ClickHouse::Migration.verbose ClickHouse::Migration.verbose = ENV['VERBOSE'] ? ENV['VERBOSE'] != 'false' : true - migrations_paths = ClickHouse::MigrationSupport::Migrator.migrations_paths - schema_migration = ClickHouse::MigrationSupport::SchemaMigration - migration_context = ClickHouse::MigrationSupport::MigrationContext.new(migrations_paths, schema_migration) - migrations_ran = migration_context.public_send(direction, target_version) do |migration| + migrations_paths = ::ClickHouse::MigrationSupport::Migrator.migrations_paths(database) + connection = ::ClickHouse::Connection.new(database) + schema_migration = ClickHouse::MigrationSupport::SchemaMigration.new(connection) + schema_migration.ensure_table + + migration_context = ClickHouse::MigrationSupport::MigrationContext.new(connection, migrations_paths, + schema_migration) + + migrations_ran = migration_context.public_send(direction, target_version, step) do |migration| scope.blank? || scope == migration.scope end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index d89ab548419..dcb86de7eff 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -5,6 +5,7 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml def each_database(databases, include_geo: false) ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| next if database == 'embedding' + next if database == 'jh' next if !include_geo && database == 'geo' yield database @@ -90,12 +91,17 @@ namespace :gitlab do desc 'GitLab | DB | Configures the database by running migrate, or by loading the schema and seeding if needed' task configure: :environment do + configure_pg_databases + configure_clickhouse_databases + end + + def configure_pg_databases databases_with_tasks = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env) databases_loaded = [] if databases_with_tasks.size == 1 - next unless databases_with_tasks.first.name == 'main' + return unless databases_with_tasks.first.name == 'main' connection = Gitlab::Database.database_base_models['main'].connection databases_loaded << configure_database(connection) @@ -107,10 +113,16 @@ namespace :gitlab do end end - if databases_loaded.present? && databases_loaded.all? - Rake::Task["gitlab:db:lock_writes"].invoke - Rake::Task['db:seed_fu'].invoke - end + return unless databases_loaded.present? && databases_loaded.all? + + Rake::Task["gitlab:db:lock_writes"].invoke + Rake::Task['db:seed_fu'].invoke + end + + def configure_clickhouse_databases + return unless Feature.enabled?(:run_clickhouse_migrations_automatically, type: :ops) + + Rake::Task['gitlab:clickhouse:migrate'].invoke(true) end def configure_database(connection, database_name: nil) @@ -223,12 +235,12 @@ namespace :gitlab do # :nocov: end - # During testing, db:test:load restores the database schema from scratch + # During testing, db:test:load_schema restores the database schema from scratch # which does not include dynamic partitions. We cannot rely on application # initializers here as the application can continue to run while # a rake task reloads the database schema. - Rake::Task['db:test:load'].enhance do - # Due to bug in `db:test:load` if many DBs are used + Rake::Task['db:test:load_schema'].enhance do + # Due to bug in `db:test:load_schema` if many DBs are used # the `ActiveRecord::Base.connection` might be switched to another one # This is due to `if should_reconnect`: # https://github.com/rails/rails/blob/a81aeb63a007ede2fe606c50539417dada9030c7/activerecord/lib/active_record/railties/databases.rake#L622 diff --git a/lib/tasks/gitlab/db/decomposition/migrate.rake b/lib/tasks/gitlab/db/decomposition/migrate.rake new file mode 100644 index 00000000000..cdbd3709c71 --- /dev/null +++ b/lib/tasks/gitlab/db/decomposition/migrate.rake @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + namespace :decomposition do + desc 'Migrate single database to two database setup' + task migrate: :environment do + Gitlab::Database::Decomposition::Migrate.new(backup_base_location: ENV['BACKUP_BASE_LOCATION']).process! + + puts "Database migration finished!" + rescue Gitlab::Database::Decomposition::MigrateError => e + puts e.message + exit 1 + end + end + end +end diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index f42d30e9817..3b6b58bd8f5 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -87,7 +87,11 @@ namespace :gitlab do # Skip if databases are yet to be provisioned next unless connection[:identifier] && shared_connection[:identifier] - unless connection[:identifier] == shared_connection[:identifier] + connection_identifier, shared_connection_identifier = [ + connection[:identifier], shared_connection[:identifier] + ].map { |identifier| identifier.slice("system_identifier", "current_database") } + + unless connection_identifier == shared_connection_identifier warnings << "- The '#{connection[:name]}' since it is using 'database_tasks: false' " \ "should share database with '#{share_with}:'." end diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c0929466b7c..5a83fd5e2fd 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -2,7 +2,7 @@ namespace :gitlab do task list_repos: :environment do - warn "The Rake task gitlab:list_repos is deprecated in 16.4 and will be removed in 17.0: " \ + warn "The Rake task gitlab:list_repos is deprecated in 16.7 and will be removed in 17.0: " \ "https://gitlab.com/gitlab-org/gitlab/-/issues/384361" scope = Project diff --git a/lib/tasks/gitlab/seed/ci_catalog_resources.rake b/lib/tasks/gitlab/seed/ci_catalog_resources.rake index 1db995aa801..3a7451beea4 100644 --- a/lib/tasks/gitlab/seed/ci_catalog_resources.rake +++ b/lib/tasks/gitlab/seed/ci_catalog_resources.rake @@ -1,26 +1,27 @@ # frozen_string_literal: true -# This task should be enabled when the seeder gets fixed: -# https://gitlab.com/gitlab-org/gitlab/-/issues/429649 -# # Seed CI/CD catalog resources # # @param group_path - Group name under which to create the projects -# @param seed_count - Total number of Catalog resources to create (default: 30) +# @param seed_count - Total number of Catalog resources to create +# @param publish - Whether or not created resources should be published in the catalog. Defaults to true. # -# @example -# bundle exec rake "gitlab:seed:ci_catalog_resources[root, 50]" +# @example to create published resources +# bundle exec rake "gitlab:seed:ci_catalog_resources[Twitter, 50]" +# @example to create draft resources +# bundle exec rake "gitlab:seed:ci_catalog_resources[Flightjs, 2, false]" # -# namespace :gitlab do -# namespace :seed do -# desc 'Seed CI Catalog resources' -# task :ci_catalog_resources, -# [:group_path, :seed_count] => :gitlab_environment do |_t, args| -# Gitlab::Seeders::Ci::Catalog::ResourceSeeder.new( -# group_path: args.group_path, -# seed_count: args.seed_count&.to_i -# ).seed -# puts "Task finished!" -# end -# end -# end +namespace :gitlab do + namespace :seed do + desc 'Seed CI Catalog resources' + task :ci_catalog_resources, + [:group_path, :seed_count, :publish] => :gitlab_environment do |_t, args| + Gitlab::Seeders::Ci::Catalog::ResourceSeeder.new( + group_path: args.group_path, + seed_count: args.seed_count.to_i, + publish: Gitlab::Utils.to_boolean(args.publish, default: true) + ).seed + puts "Task finished!" + end + end +end diff --git a/lib/tasks/gitlab/seed/group_seed.rake b/lib/tasks/gitlab/seed/group_seed.rake index 4f5df7841e2..cc9180d56a3 100644 --- a/lib/tasks/gitlab/seed/group_seed.rake +++ b/lib/tasks/gitlab/seed/group_seed.rake @@ -120,13 +120,17 @@ class GroupSeeder end def create_user + # rubocop:disable Style/SymbolProc -- Incorrect rubocop advice. User.create!( username: FFaker::Internet.user_name, name: FFaker::Name.name, email: FFaker::Internet.email, confirmed_at: DateTime.now, password: Devise.friendly_token - ) + ) do |user| + user.assign_personal_namespace + end + # rubocop:enable Style/SymbolProc end def create_member(user_id, group_id) diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 23a518564e1..abeb5bbdf29 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -30,7 +30,7 @@ namespace :gitlab do File.open("config.yml", "w+") { |f| f.puts config.to_yaml } [ - %w[bin/install] + repository_storage_paths_args, + %w[bin/install], %w[make build] ].each do |cmd| unless Kernel.system(*cmd) @@ -46,21 +46,6 @@ namespace :gitlab do task setup: :gitlab_environment do setup_gitlab_shell end - - desc "GitLab | Shell | Build missing projects" - task build_missing_projects: :gitlab_environment do - Project.find_each(batch_size: 1000) do |project| - path_to_repo = project.repository.path_to_repo - if File.exist?(path_to_repo) - print '-' - elsif Gitlab::Shell.new.create_repository(project.repository_storage, - project.disk_path) - print '.' - else - print 'F' - end - end - end end def setup_gitlab_shell diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index de1401feb8a..9def51c36a6 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -23,7 +23,7 @@ namespace :tw do # CodeOwnerRule.new('Acquisition', ''), CodeOwnerRule.new('AI Framework', '@sselhorn'), CodeOwnerRule.new('AI Model Validation', '@sselhorn'), - CodeOwnerRule.new('Analytics Instrumentation', '@lciutacu'), + # CodeOwnerRule.new('Analytics Instrumentation', ''), CodeOwnerRule.new('Anti-Abuse', '@phillipwells'), CodeOwnerRule.new('Cloud Connector', '@jglassman1'), CodeOwnerRule.new('Authentication', '@jglassman1'), @@ -35,7 +35,7 @@ namespace :tw do CodeOwnerRule.new('Composition Analysis', '@rdickenson'), CodeOwnerRule.new('Container Registry', '@marcel.amirault'), CodeOwnerRule.new('Contributor Experience', '@eread'), - CodeOwnerRule.new('Database', '@aqualls'), + # CodeOwnerRule.new('Database', ''), CodeOwnerRule.new('DataOps', '@sselhorn'), # CodeOwnerRule.new('Delivery', ''), CodeOwnerRule.new('Distribution', '@axil'), @@ -61,8 +61,8 @@ namespace :tw do CodeOwnerRule.new('Optimize', '@lciutacu'), CodeOwnerRule.new('Organization', '@lciutacu'), CodeOwnerRule.new('Package Registry', '@phillipwells'), - CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault'), - CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault'), + CodeOwnerRule.new('Pipeline Authoring', '@marcel.amirault @lyspin'), + CodeOwnerRule.new('Pipeline Execution', '@marcel.amirault @lyspin'), CodeOwnerRule.new('Pipeline Security', '@marcel.amirault'), CodeOwnerRule.new('Product Analytics', '@lciutacu'), CodeOwnerRule.new('Product Planning', '@msedlakjakubowski'), diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index 0ad982dc127..fb20557bb6a 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -105,6 +105,10 @@ class UploadedFile @tempfile&.close end + def empty_size? + size == 0 + end + alias_method :local_path, :path def method_missing(method_name, *args, &block) #:nodoc: diff --git a/lib/users/internal.rb b/lib/users/internal.rb index 30ef20dbd7b..4b6df4ed928 100644 --- a/lib/users/internal.rb +++ b/lib/users/internal.rb @@ -138,6 +138,7 @@ module Users email: email, &creation_block ) + user.assign_personal_namespace Users::UpdateService.new(user, user: user).execute(validate: false) user diff --git a/lib/vite_gdk.rb b/lib/vite_gdk.rb new file mode 100644 index 00000000000..f50c6cab515 --- /dev/null +++ b/lib/vite_gdk.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module ViteGdk + def self.load_gdk_vite_config + # can't use Rails.env.production? here because this file is required outside of Gitlab app instance + return if ENV['RAILS_ENV'] == 'production' + + return unless File.exist?(vite_gdk_config_path) + + config = YAML.safe_load_file(vite_gdk_config_path) + enabled = config.fetch('enabled', false) + # ViteRuby doesn't like if env vars aren't strings + ViteRuby.env['VITE_ENABLED'] = enabled.to_s + + return unless enabled + + ViteRuby.configure( + host: config.fetch('host', 'localhost'), + port: Integer(config.fetch('port', 3038)) + ) + end + + def self.vite_gdk_config_path + File.join(__dir__, '../config/vite.gdk.json') + end +end |